mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-06-08 15:39:51 +08:00
⚡️ perf(sync): 优化大表同步分页与批量写入
- 同步分析和预览改为分页扫描差异,避免一次性加载源表和目标表 - 直接导入与源查询同步支持分页读取和分批提交,降低低内存机器 OOM 风险 - 各数据库 ApplyChanges 统一使用参数化批量 INSERT,减少大表同步 SQL 超时 - MySQL 批量写入按行数和参数数量拆分,兼容超宽表场景 - 补充批量插入、分页差异和源查询同步回归测试
This commit is contained in:
@@ -406,19 +406,24 @@ func (t *TDengineDB) ApplyChanges(tableName string, changes connection.ChangeSet
|
||||
}
|
||||
|
||||
qualifiedTable := quoteTDengineTable("", tableName)
|
||||
for _, row := range changes.Inserts {
|
||||
query, err := buildTDengineInsertSQL(qualifiedTable, row)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if query == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := t.conn.Exec(query); err != nil {
|
||||
return fmt.Errorf("插入失败:%v; sql=%s", err, query)
|
||||
}
|
||||
return execTDengineInsertBatches(t.conn, qualifiedTable, changes.Inserts)
|
||||
}
|
||||
|
||||
func execTDengineInsertBatches(conn *sql.DB, qualifiedTable string, rows []map[string]interface{}) error {
|
||||
if conn == nil {
|
||||
return fmt.Errorf("连接未打开")
|
||||
}
|
||||
return nil
|
||||
return execLiteralInsertBatches(literalInsertConfig{
|
||||
Table: qualifiedTable,
|
||||
Rows: rows,
|
||||
QuoteColumn: func(column string) string {
|
||||
return fmt.Sprintf("`%s`", escapeBacktickIdent(column))
|
||||
},
|
||||
Literal: tdengineLiteral,
|
||||
Exec: func(query string) (sql.Result, error) {
|
||||
return conn.Exec(query)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func buildTDengineInsertSQL(qualifiedTable string, row map[string]interface{}) (string, error) {
|
||||
|
||||
Reference in New Issue
Block a user