🐛 fix(sync/mysql): 修复批量导入占位符超限

- 识别 MySQL Prepared Statement 占位符超限错误
- 对被服务端拒绝的批量 INSERT 自动递归拆批重试
- 增加 24 列 1000 行数据同步回归测试
This commit is contained in:
Syngnat
2026-07-30 17:41:01 +08:00
parent d23e82d77b
commit b293bf0da6
2 changed files with 60 additions and 0 deletions

View File

@@ -163,6 +163,13 @@ func execParameterizedInsertBatch(config parameterizedInsertConfig, rows []prepa
)
res, err := config.Exec(query, args...)
if err != nil {
if len(rows) > 1 && isPreparedStatementPlaceholderLimitError(err) {
middle := len(rows) / 2
if err := execParameterizedInsertBatch(config, rows[:middle]); err != nil {
return err
}
return execParameterizedInsertBatch(config, rows[middle:])
}
return localizedDatabaseRuntimeError("db.backend.error.batch_insert_failed", map[string]any{"detail": err.Error()})
}
if config.RequireAffected {
@@ -173,6 +180,13 @@ func execParameterizedInsertBatch(config parameterizedInsertConfig, rows []prepa
return nil
}
func isPreparedStatementPlaceholderLimitError(err error) bool {
if err == nil {
return false
}
return strings.Contains(strings.ToLower(err.Error()), "prepared statement contains too many placeholders")
}
func requireInsertAffected(result sql.Result) error {
if result == nil {
return nil

View File

@@ -160,6 +160,52 @@ func TestExecParameterizedInsertBatchesSplitsByArgumentLimit(t *testing.T) {
}
}
func TestExecParameterizedInsertBatchesRetriesMySQLPlaceholderLimitWithSmallerBatches(t *testing.T) {
t.Parallel()
const (
columnCount = 24
rowCount = 1000
serverPlaceholder = 12000
)
rows := make([]map[string]interface{}, 0, rowCount)
for rowIndex := 0; rowIndex < rowCount; rowIndex++ {
row := make(map[string]interface{}, columnCount)
for columnIndex := 0; columnIndex < columnCount; columnIndex++ {
row[fmt.Sprintf("column_%02d", columnIndex)] = rowIndex*columnCount + columnIndex
}
rows = append(rows, row)
}
attemptedArgCounts := make([]int, 0, 3)
succeededRows := 0
err := execParameterizedInsertBatches(parameterizedInsertConfig{
Table: "`events`",
Rows: rows,
QuoteColumn: func(column string) string { return "`" + column + "`" },
Placeholder: func(int) string { return "?" },
Exec: func(_ string, values ...interface{}) (sql.Result, error) {
attemptedArgCounts = append(attemptedArgCounts, len(values))
if len(values) > serverPlaceholder {
return nil, errors.New("Error 1390 (HY000): Prepared statement contains too many placeholders")
}
succeededRows += len(values) / columnCount
return driver.RowsAffected(len(values) / columnCount), nil
},
MaxRows: 1000,
MaxArgs: 60000,
})
if err != nil {
t.Fatalf("execParameterizedInsertBatches() error = %v", err)
}
if got, want := fmt.Sprint(attemptedArgCounts), "[24000 12000 12000]"; got != want {
t.Fatalf("attempted arg counts = %s, want %s", got, want)
}
if succeededRows != rowCount {
t.Fatalf("succeeded rows = %d, want %d", succeededRows, rowCount)
}
}
func TestExecParameterizedInsertBatchesOmitsColumnsPerRow(t *testing.T) {
t.Parallel()