️ perf(sync): 优化大表同步分页与批量写入

- 同步分析和预览改为分页扫描差异,避免一次性加载源表和目标表

- 直接导入与源查询同步支持分页读取和分批提交,降低低内存机器 OOM 风险

- 各数据库 ApplyChanges 统一使用参数化批量 INSERT,减少大表同步 SQL 超时

- MySQL 批量写入按行数和参数数量拆分,兼容超宽表场景

- 补充批量插入、分页差异和源查询同步回归测试
This commit is contained in:
Syngnat
2026-05-26 08:27:15 +08:00
parent aa2177d35a
commit 5ab50db51c
27 changed files with 2846 additions and 319 deletions

View File

@@ -1314,15 +1314,37 @@ func (m *MongoDB) ApplyChanges(tableName string, changes connection.ChangeSet) e
}
}
// Process inserts
for _, row := range changes.Inserts {
doc := copyMongoChangeDocument(row)
if len(doc) > 0 {
if _, err := collection.InsertOne(ctx, doc); err != nil {
return fmt.Errorf("插入失败:%v", err)
}
}
if err := insertMongoDocuments(ctx, collection, changes.Inserts); err != nil {
return err
}
return nil
}
func insertMongoDocuments(ctx context.Context, collection *mongo.Collection, rows []map[string]interface{}) error {
if len(rows) == 0 {
return nil
}
docs := make([]interface{}, 0, len(rows))
for _, row := range rows {
doc := copyMongoChangeDocument(row)
if len(doc) > 0 {
docs = append(docs, doc)
}
}
if len(docs) == 0 {
return nil
}
for start := 0; start < len(docs); start += defaultBatchInsertRows {
end := start + defaultBatchInsertRows
if end > len(docs) {
end = len(docs)
}
if _, err := collection.InsertMany(ctx, docs[start:end]); err != nil {
return fmt.Errorf("插入失败:%v", err)
}
}
return nil
}