mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-07-12 16:02:48 +08:00
⚡️ perf(sync): 优化大表同步分页与批量写入
- 同步分析和预览改为分页扫描差异,避免一次性加载源表和目标表 - 直接导入与源查询同步支持分页读取和分批提交,降低低内存机器 OOM 风险 - 各数据库 ApplyChanges 统一使用参数化批量 INSERT,减少大表同步 SQL 超时 - MySQL 批量写入按行数和参数数量拆分,兼容超宽表场景 - 补充批量插入、分页差异和源查询同步回归测试
This commit is contained in:
@@ -1314,15 +1314,37 @@ func (m *MongoDB) ApplyChanges(tableName string, changes connection.ChangeSet) e
|
||||
}
|
||||
}
|
||||
|
||||
// Process inserts
|
||||
for _, row := range changes.Inserts {
|
||||
doc := copyMongoChangeDocument(row)
|
||||
if len(doc) > 0 {
|
||||
if _, err := collection.InsertOne(ctx, doc); err != nil {
|
||||
return fmt.Errorf("插入失败:%v", err)
|
||||
}
|
||||
}
|
||||
if err := insertMongoDocuments(ctx, collection, changes.Inserts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func insertMongoDocuments(ctx context.Context, collection *mongo.Collection, rows []map[string]interface{}) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
docs := make([]interface{}, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
doc := copyMongoChangeDocument(row)
|
||||
if len(doc) > 0 {
|
||||
docs = append(docs, doc)
|
||||
}
|
||||
}
|
||||
if len(docs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for start := 0; start < len(docs); start += defaultBatchInsertRows {
|
||||
end := start + defaultBatchInsertRows
|
||||
if end > len(docs) {
|
||||
end = len(docs)
|
||||
}
|
||||
if _, err := collection.InsertMany(ctx, docs[start:end]); err != nil {
|
||||
return fmt.Errorf("插入失败:%v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user