️ perf(sync): 优化大表同步分页与批量写入

- 同步分析和预览改为分页扫描差异,避免一次性加载源表和目标表

- 直接导入与源查询同步支持分页读取和分批提交,降低低内存机器 OOM 风险

- 各数据库 ApplyChanges 统一使用参数化批量 INSERT,减少大表同步 SQL 超时

- MySQL 批量写入按行数和参数数量拆分,兼容超宽表场景

- 补充批量插入、分页差异和源查询同步回归测试
This commit is contained in:
Syngnat
2026-05-26 08:27:15 +08:00
parent aa2177d35a
commit 5ab50db51c
27 changed files with 2846 additions and 319 deletions

View File

@@ -406,19 +406,24 @@ func (t *TDengineDB) ApplyChanges(tableName string, changes connection.ChangeSet
}
qualifiedTable := quoteTDengineTable("", tableName)
for _, row := range changes.Inserts {
query, err := buildTDengineInsertSQL(qualifiedTable, row)
if err != nil {
return err
}
if query == "" {
continue
}
if _, err := t.conn.Exec(query); err != nil {
return fmt.Errorf("插入失败:%v; sql=%s", err, query)
}
return execTDengineInsertBatches(t.conn, qualifiedTable, changes.Inserts)
}
func execTDengineInsertBatches(conn *sql.DB, qualifiedTable string, rows []map[string]interface{}) error {
if conn == nil {
return fmt.Errorf("连接未打开")
}
return nil
return execLiteralInsertBatches(literalInsertConfig{
Table: qualifiedTable,
Rows: rows,
QuoteColumn: func(column string) string {
return fmt.Sprintf("`%s`", escapeBacktickIdent(column))
},
Literal: tdengineLiteral,
Exec: func(query string) (sql.Result, error) {
return conn.Exec(query)
},
})
}
func buildTDengineInsertSQL(qualifiedTable string, row map[string]interface{}) (string, error) {