diff --git a/internal/db/batch_insert.go b/internal/db/batch_insert.go index d2e7efc6..aa62c5ec 100644 --- a/internal/db/batch_insert.go +++ b/internal/db/batch_insert.go @@ -163,6 +163,13 @@ func execParameterizedInsertBatch(config parameterizedInsertConfig, rows []prepa ) res, err := config.Exec(query, args...) if err != nil { + if len(rows) > 1 && isPreparedStatementPlaceholderLimitError(err) { + middle := len(rows) / 2 + if err := execParameterizedInsertBatch(config, rows[:middle]); err != nil { + return err + } + return execParameterizedInsertBatch(config, rows[middle:]) + } return localizedDatabaseRuntimeError("db.backend.error.batch_insert_failed", map[string]any{"detail": err.Error()}) } if config.RequireAffected { @@ -173,6 +180,13 @@ func execParameterizedInsertBatch(config parameterizedInsertConfig, rows []prepa return nil } +func isPreparedStatementPlaceholderLimitError(err error) bool { + if err == nil { + return false + } + return strings.Contains(strings.ToLower(err.Error()), "prepared statement contains too many placeholders") +} + func requireInsertAffected(result sql.Result) error { if result == nil { return nil diff --git a/internal/db/batch_insert_test.go b/internal/db/batch_insert_test.go index 83c1c12b..9c2a4c37 100644 --- a/internal/db/batch_insert_test.go +++ b/internal/db/batch_insert_test.go @@ -160,6 +160,52 @@ func TestExecParameterizedInsertBatchesSplitsByArgumentLimit(t *testing.T) { } } +func TestExecParameterizedInsertBatchesRetriesMySQLPlaceholderLimitWithSmallerBatches(t *testing.T) { + t.Parallel() + + const ( + columnCount = 24 + rowCount = 1000 + serverPlaceholder = 12000 + ) + rows := make([]map[string]interface{}, 0, rowCount) + for rowIndex := 0; rowIndex < rowCount; rowIndex++ { + row := make(map[string]interface{}, columnCount) + for columnIndex := 0; columnIndex < columnCount; columnIndex++ { + row[fmt.Sprintf("column_%02d", columnIndex)] = rowIndex*columnCount + columnIndex + } + rows = append(rows, row) + } + + attemptedArgCounts := make([]int, 0, 3) + succeededRows := 0 + err := execParameterizedInsertBatches(parameterizedInsertConfig{ + Table: "`events`", + Rows: rows, + QuoteColumn: func(column string) string { return "`" + column + "`" }, + Placeholder: func(int) string { return "?" }, + Exec: func(_ string, values ...interface{}) (sql.Result, error) { + attemptedArgCounts = append(attemptedArgCounts, len(values)) + if len(values) > serverPlaceholder { + return nil, errors.New("Error 1390 (HY000): Prepared statement contains too many placeholders") + } + succeededRows += len(values) / columnCount + return driver.RowsAffected(len(values) / columnCount), nil + }, + MaxRows: 1000, + MaxArgs: 60000, + }) + if err != nil { + t.Fatalf("execParameterizedInsertBatches() error = %v", err) + } + if got, want := fmt.Sprint(attemptedArgCounts), "[24000 12000 12000]"; got != want { + t.Fatalf("attempted arg counts = %s, want %s", got, want) + } + if succeededRows != rowCount { + t.Fatalf("succeeded rows = %d, want %d", succeededRows, rowCount) + } +} + func TestExecParameterizedInsertBatchesOmitsColumnsPerRow(t *testing.T) { t.Parallel()