mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-05-31 13:39:48 +08:00
⚡️ perf(sync): 优化大表同步分页与批量写入
- 同步分析和预览改为分页扫描差异,避免一次性加载源表和目标表 - 直接导入与源查询同步支持分页读取和分批提交,降低低内存机器 OOM 风险 - 各数据库 ApplyChanges 统一使用参数化批量 INSERT,减少大表同步 SQL 超时 - MySQL 批量写入按行数和参数数量拆分,兼容超宽表场景 - 补充批量插入、分页差异和源查询同步回归测试
This commit is contained in:
@@ -864,28 +864,18 @@ func (p *PostgresDB) ApplyChanges(tableName string, changes connection.ChangeSet
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Inserts
|
||||
for _, row := range changes.Inserts {
|
||||
var cols []string
|
||||
var placeholders []string
|
||||
var args []interface{}
|
||||
idx := 0
|
||||
|
||||
for k, v := range row {
|
||||
idx++
|
||||
cols = append(cols, quoteIdent(k))
|
||||
placeholders = append(placeholders, fmt.Sprintf("$%d", idx))
|
||||
args = append(args, v)
|
||||
}
|
||||
|
||||
if len(cols) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", qualifiedTable, strings.Join(cols, ", "), strings.Join(placeholders, ", "))
|
||||
if _, err := tx.Exec(query, args...); err != nil {
|
||||
return fmt.Errorf("插入失败:%v", err)
|
||||
}
|
||||
if err := execParameterizedInsertBatches(parameterizedInsertConfig{
|
||||
Table: qualifiedTable,
|
||||
Rows: changes.Inserts,
|
||||
QuoteColumn: quoteIdent,
|
||||
Placeholder: func(idx int) string {
|
||||
return fmt.Sprintf("$%d", idx)
|
||||
},
|
||||
Exec: func(query string, args ...interface{}) (sql.Result, error) {
|
||||
return tx.Exec(query, args...)
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
|
||||
Reference in New Issue
Block a user