mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-08-11 01:03:51 +08:00
🐛 fix(sync/mysql): 修复批量导入占位符超限
- 识别 MySQL Prepared Statement 占位符超限错误 - 对被服务端拒绝的批量 INSERT 自动递归拆批重试 - 增加 24 列 1000 行数据同步回归测试
This commit is contained in:
@@ -163,6 +163,13 @@ func execParameterizedInsertBatch(config parameterizedInsertConfig, rows []prepa
|
||||
)
|
||||
res, err := config.Exec(query, args...)
|
||||
if err != nil {
|
||||
if len(rows) > 1 && isPreparedStatementPlaceholderLimitError(err) {
|
||||
middle := len(rows) / 2
|
||||
if err := execParameterizedInsertBatch(config, rows[:middle]); err != nil {
|
||||
return err
|
||||
}
|
||||
return execParameterizedInsertBatch(config, rows[middle:])
|
||||
}
|
||||
return localizedDatabaseRuntimeError("db.backend.error.batch_insert_failed", map[string]any{"detail": err.Error()})
|
||||
}
|
||||
if config.RequireAffected {
|
||||
@@ -173,6 +180,13 @@ func execParameterizedInsertBatch(config parameterizedInsertConfig, rows []prepa
|
||||
return nil
|
||||
}
|
||||
|
||||
func isPreparedStatementPlaceholderLimitError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(strings.ToLower(err.Error()), "prepared statement contains too many placeholders")
|
||||
}
|
||||
|
||||
func requireInsertAffected(result sql.Result) error {
|
||||
if result == nil {
|
||||
return nil
|
||||
|
||||
@@ -160,6 +160,52 @@ func TestExecParameterizedInsertBatchesSplitsByArgumentLimit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecParameterizedInsertBatchesRetriesMySQLPlaceholderLimitWithSmallerBatches(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const (
|
||||
columnCount = 24
|
||||
rowCount = 1000
|
||||
serverPlaceholder = 12000
|
||||
)
|
||||
rows := make([]map[string]interface{}, 0, rowCount)
|
||||
for rowIndex := 0; rowIndex < rowCount; rowIndex++ {
|
||||
row := make(map[string]interface{}, columnCount)
|
||||
for columnIndex := 0; columnIndex < columnCount; columnIndex++ {
|
||||
row[fmt.Sprintf("column_%02d", columnIndex)] = rowIndex*columnCount + columnIndex
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
|
||||
attemptedArgCounts := make([]int, 0, 3)
|
||||
succeededRows := 0
|
||||
err := execParameterizedInsertBatches(parameterizedInsertConfig{
|
||||
Table: "`events`",
|
||||
Rows: rows,
|
||||
QuoteColumn: func(column string) string { return "`" + column + "`" },
|
||||
Placeholder: func(int) string { return "?" },
|
||||
Exec: func(_ string, values ...interface{}) (sql.Result, error) {
|
||||
attemptedArgCounts = append(attemptedArgCounts, len(values))
|
||||
if len(values) > serverPlaceholder {
|
||||
return nil, errors.New("Error 1390 (HY000): Prepared statement contains too many placeholders")
|
||||
}
|
||||
succeededRows += len(values) / columnCount
|
||||
return driver.RowsAffected(len(values) / columnCount), nil
|
||||
},
|
||||
MaxRows: 1000,
|
||||
MaxArgs: 60000,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("execParameterizedInsertBatches() error = %v", err)
|
||||
}
|
||||
if got, want := fmt.Sprint(attemptedArgCounts), "[24000 12000 12000]"; got != want {
|
||||
t.Fatalf("attempted arg counts = %s, want %s", got, want)
|
||||
}
|
||||
if succeededRows != rowCount {
|
||||
t.Fatalf("succeeded rows = %d, want %d", succeededRows, rowCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecParameterizedInsertBatchesOmitsColumnsPerRow(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user