mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-08-03 20:38:16 +08:00
⚡️ perf(sync): 优化大表同步分页与批量写入
- 同步分析和预览改为分页扫描差异,避免一次性加载源表和目标表 - 直接导入与源查询同步支持分页读取和分批提交,降低低内存机器 OOM 风险 - 各数据库 ApplyChanges 统一使用参数化批量 INSERT,减少大表同步 SQL 超时 - MySQL 批量写入按行数和参数数量拆分,兼容超宽表场景 - 补充批量插入、分页差异和源查询同步回归测试
This commit is contained in:
@@ -101,7 +101,7 @@ func (s *SyncEngine) Analyze(config SyncConfig) SyncAnalyzeResult {
|
||||
HasSchema: syncSchema,
|
||||
}
|
||||
|
||||
plan, cols, _, err := buildSchemaMigrationPlan(config, tableName, sourceDB, targetDB)
|
||||
plan, cols, targetCols, err := buildSchemaMigrationPlan(config, tableName, sourceDB, targetDB)
|
||||
if err != nil {
|
||||
summary.Message = err.Error()
|
||||
result.Tables = append(result.Tables, summary)
|
||||
@@ -140,16 +140,27 @@ func (s *SyncEngine) Analyze(config SyncConfig) SyncAnalyzeResult {
|
||||
}
|
||||
}
|
||||
|
||||
sourceRows, _, err := sourceDB.Query(fmt.Sprintf("SELECT * FROM %s", quoteQualifiedIdentByType(config.SourceConfig.Type, plan.SourceQueryTable)))
|
||||
sourceType := resolveMigrationDBType(config.SourceConfig)
|
||||
targetType := resolveMigrationDBType(config.TargetConfig)
|
||||
sourceCount, counted, err := countTableRowsForSync(sourceDB, sourceType, plan.SourceQueryTable)
|
||||
if err != nil {
|
||||
summary.Message = "读取源表失败: " + err.Error()
|
||||
result.Tables = append(result.Tables, summary)
|
||||
return
|
||||
}
|
||||
if !counted {
|
||||
sourceRows, _, err := sourceDB.Query(fmt.Sprintf("SELECT * FROM %s", quoteQualifiedIdentByType(sourceType, plan.SourceQueryTable)))
|
||||
if err != nil {
|
||||
summary.Message = "读取源表失败: " + err.Error()
|
||||
result.Tables = append(result.Tables, summary)
|
||||
return
|
||||
}
|
||||
sourceCount = len(sourceRows)
|
||||
}
|
||||
|
||||
if !plan.TargetTableExists && plan.AutoCreate {
|
||||
summary.CanSync = true
|
||||
summary.Inserts = len(sourceRows)
|
||||
summary.Inserts = sourceCount
|
||||
summary.Message = firstNonEmpty(plan.PlannedAction, "目标表不存在,执行时将自动建表并导入全部源数据")
|
||||
result.Tables = append(result.Tables, summary)
|
||||
return
|
||||
@@ -157,7 +168,7 @@ func (s *SyncEngine) Analyze(config SyncConfig) SyncAnalyzeResult {
|
||||
|
||||
if tableMode != "insert_update" {
|
||||
summary.CanSync = true
|
||||
summary.Inserts = len(sourceRows)
|
||||
summary.Inserts = sourceCount
|
||||
summary.Message = firstNonEmpty(plan.PlannedAction, "当前模式无需差异对比,将按源表数据执行导入")
|
||||
result.Tables = append(result.Tables, summary)
|
||||
return
|
||||
@@ -175,6 +186,32 @@ func (s *SyncEngine) Analyze(config SyncConfig) SyncAnalyzeResult {
|
||||
}
|
||||
summary.PKColumn = pkCols[0]
|
||||
|
||||
targetColSet := buildTargetColumnSet(targetCols)
|
||||
handled, counts, scanErr := scanTableDiffInPages(sourceDB, targetDB, sourceType, targetType, plan, cols, targetCols, summary.PKColumn, targetColSet, true, nil)
|
||||
if handled {
|
||||
if scanErr != nil {
|
||||
summary.Message = scanErr.Error()
|
||||
result.Tables = append(result.Tables, summary)
|
||||
return
|
||||
}
|
||||
summary.CanSync = true
|
||||
summary.Inserts = counts.Inserts
|
||||
summary.Updates = counts.Updates
|
||||
summary.Deletes = counts.Deletes
|
||||
summary.Same = counts.Same
|
||||
if strings.TrimSpace(summary.Message) == "" {
|
||||
summary.Message = firstNonEmpty(plan.PlannedAction, "差异分析完成")
|
||||
}
|
||||
result.Tables = append(result.Tables, summary)
|
||||
return
|
||||
}
|
||||
|
||||
sourceRows, _, err := sourceDB.Query(fmt.Sprintf("SELECT * FROM %s", quoteQualifiedIdentByType(sourceType, plan.SourceQueryTable)))
|
||||
if err != nil {
|
||||
summary.Message = "读取源表失败: " + err.Error()
|
||||
result.Tables = append(result.Tables, summary)
|
||||
return
|
||||
}
|
||||
targetRows, _, err := targetDB.Query(fmt.Sprintf("SELECT * FROM %s", quoteQualifiedIdentByType(config.TargetConfig.Type, plan.TargetQueryTable)))
|
||||
if err != nil {
|
||||
summary.Message = "读取目标表失败: " + err.Error()
|
||||
|
||||
587
internal/sync/diff_paging.go
Normal file
587
internal/sync/diff_paging.go
Normal file
@@ -0,0 +1,587 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"GoNavi-Wails/internal/connection"
|
||||
"GoNavi-Wails/internal/db"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type pagedDiffCounts struct {
|
||||
Inserts int
|
||||
Updates int
|
||||
Deletes int
|
||||
Same int
|
||||
}
|
||||
|
||||
type pagedUpdateDiff struct {
|
||||
UpdateRow connection.UpdateRow
|
||||
Source map[string]interface{}
|
||||
Target map[string]interface{}
|
||||
ChangedColumns []string
|
||||
}
|
||||
|
||||
type pagedDiffPage struct {
|
||||
Inserts []map[string]interface{}
|
||||
Updates []pagedUpdateDiff
|
||||
Deletes []map[string]interface{}
|
||||
Same int
|
||||
}
|
||||
|
||||
func (s *SyncEngine) tryApplyDiffInPages(config SyncConfig, res *SyncResult, tableIndex, totalTables int, tableName string, sourceDB db.Database, targetDB db.Database, plan SchemaMigrationPlan, sourceCols, targetCols []connection.ColumnDefinition, opts TableOptions, sourceType, targetType, applyTableName, pkCol string) (bool, pagedDiffCounts, error) {
|
||||
if normalizeSyncMode(config.Mode) != "insert_update" || !plan.TargetTableExists {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
if !supportsPagedDiffSelect(sourceType) || !supportsPagedDiffSelect(targetType) {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
if opts.Delete && (!supportsPagedDiffKeysetSelect(targetType) || !supportsPagedDiffPKLookup(sourceType)) {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
|
||||
applier, ok := targetDB.(db.BatchApplier)
|
||||
if !ok {
|
||||
return true, pagedDiffCounts{}, fmt.Errorf("目标驱动不支持应用数据变更 (ApplyChanges)")
|
||||
}
|
||||
|
||||
targetColSet, err := s.prepareDirectImportTargetColumnSet(config, res, targetDB, plan, sourceType, targetType, sourceCols, targetCols)
|
||||
if err != nil {
|
||||
return true, pagedDiffCounts{}, err
|
||||
}
|
||||
|
||||
s.appendLog(config.JobID, res, "info", fmt.Sprintf(" -> 启用分页差异同步:按主键 %s 每批读取 %d 行", pkCol, defaultSyncReadPageSize))
|
||||
s.progress(config.JobID, tableIndex, totalTables, tableName, "分页对比数据")
|
||||
|
||||
applied := pagedDiffCounts{}
|
||||
handled, _, err := scanTableDiffInPages(sourceDB, targetDB, sourceType, targetType, plan, sourceCols, targetCols, pkCol, targetColSet, opts.Delete, func(page pagedDiffPage) error {
|
||||
changeSet := connection.ChangeSet{
|
||||
Inserts: filterRowsByPKSelection(pkCol, page.Inserts, opts.Insert, opts.SelectedInsertPKs),
|
||||
Updates: filterPagedUpdatesByPKSelection(pkCol, page.Updates, opts.Update, opts.SelectedUpdatePKs),
|
||||
Deletes: filterRowsByPKSelection(pkCol, page.Deletes, opts.Delete, opts.SelectedDeletePKs),
|
||||
}
|
||||
if len(targetColSet) > 0 {
|
||||
changeSet.Inserts = filterInsertRows(changeSet.Inserts, targetColSet)
|
||||
changeSet.Updates = filterUpdateRows(changeSet.Updates, targetColSet)
|
||||
}
|
||||
if len(changeSet.Inserts) == 0 && len(changeSet.Updates) == 0 && len(changeSet.Deletes) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := s.applyChangesInBatches(config.JobID, res, applyTableName, applier, changeSet); err != nil {
|
||||
return err
|
||||
}
|
||||
applied.Inserts += len(changeSet.Inserts)
|
||||
applied.Updates += len(changeSet.Updates)
|
||||
applied.Deletes += len(changeSet.Deletes)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return true, applied, err
|
||||
}
|
||||
return handled, applied, nil
|
||||
}
|
||||
|
||||
func scanTableDiffInPages(sourceDB db.Database, targetDB db.Database, sourceType, targetType string, plan SchemaMigrationPlan, sourceCols, targetCols []connection.ColumnDefinition, pkCol string, targetColSet map[string]struct{}, includeDeletes bool, consume func(page pagedDiffPage) error) (bool, pagedDiffCounts, error) {
|
||||
if !supportsPagedDiffSelect(sourceType) || !supportsPagedDiffPKLookup(targetType) {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
if includeDeletes && (!supportsPagedDiffKeysetSelect(targetType) || !supportsPagedDiffPKLookup(sourceType)) {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
|
||||
sourceReadCols := diffReadableColumns(sourceCols, targetColSet, pkCol)
|
||||
if len(sourceReadCols) == 0 {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
targetLookupCols := diffLookupColumns(sourceReadCols, targetCols, targetColSet, pkCol)
|
||||
if len(targetLookupCols) == 0 {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
|
||||
totals := pagedDiffCounts{}
|
||||
for offset := 0; ; offset += defaultSyncReadPageSize {
|
||||
query := buildPagedSourceTableQuery(sourceType, plan.SourceQueryTable, sourceReadCols, pkCol, defaultSyncReadPageSize, offset)
|
||||
if strings.TrimSpace(query) == "" {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
sourceRows, _, err := sourceDB.Query(query)
|
||||
if err != nil {
|
||||
return true, totals, fmt.Errorf("分页读取源表失败(offset=%d): %w", offset, err)
|
||||
}
|
||||
if len(sourceRows) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
pkValues := collectPKValues(sourceRows, pkCol)
|
||||
targetRows := make([]map[string]interface{}, 0)
|
||||
if len(pkValues) > 0 {
|
||||
targetQuery := buildPKInSelectQuery(targetType, plan.TargetQueryTable, targetLookupCols, pkCol, pkValues)
|
||||
if strings.TrimSpace(targetQuery) == "" {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
targetRows, _, err = targetDB.Query(targetQuery)
|
||||
if err != nil {
|
||||
return true, totals, fmt.Errorf("按主键读取目标表失败(offset=%d): %w", offset, err)
|
||||
}
|
||||
}
|
||||
|
||||
page := diffSourcePageByPK(pkCol, sourceRows, targetRows)
|
||||
totals.Inserts += len(page.Inserts)
|
||||
totals.Updates += len(page.Updates)
|
||||
totals.Same += page.Same
|
||||
if consume != nil {
|
||||
if err := consume(page); err != nil {
|
||||
return true, totals, err
|
||||
}
|
||||
}
|
||||
if len(sourceRows) < defaultSyncReadPageSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if includeDeletes {
|
||||
lastPK, hasLastPK := interface{}(nil), false
|
||||
targetPKCols := []connection.ColumnDefinition{{Name: pkCol}}
|
||||
for {
|
||||
query := buildKeysetPagedTableQuery(targetType, plan.TargetQueryTable, targetPKCols, pkCol, lastPK, hasLastPK, defaultSyncReadPageSize)
|
||||
if strings.TrimSpace(query) == "" {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
targetRows, _, err := targetDB.Query(query)
|
||||
if err != nil {
|
||||
return true, totals, fmt.Errorf("分页读取目标主键失败: %w", err)
|
||||
}
|
||||
if len(targetRows) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
nextLastPK, ok := lastValidPKValue(targetRows, pkCol)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
lastPK, hasLastPK = nextLastPK, true
|
||||
|
||||
pkValues := collectPKValues(targetRows, pkCol)
|
||||
sourcePKRows := make([]map[string]interface{}, 0)
|
||||
if len(pkValues) > 0 {
|
||||
sourceQuery := buildPKInSelectQuery(sourceType, plan.SourceQueryTable, targetPKCols, pkCol, pkValues)
|
||||
if strings.TrimSpace(sourceQuery) == "" {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
sourcePKRows, _, err = sourceDB.Query(sourceQuery)
|
||||
if err != nil {
|
||||
return true, totals, fmt.Errorf("按主键反查源表失败: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
sourcePKSet := buildPKSet(sourcePKRows, pkCol)
|
||||
deletes := make([]map[string]interface{}, 0)
|
||||
for _, row := range targetRows {
|
||||
pkKey, ok := pkValueKey(row[pkCol])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, exists := sourcePKSet[pkKey]; exists {
|
||||
continue
|
||||
}
|
||||
deletes = append(deletes, map[string]interface{}{pkCol: row[pkCol]})
|
||||
}
|
||||
if len(deletes) > 0 {
|
||||
totals.Deletes += len(deletes)
|
||||
if consume != nil {
|
||||
if err := consume(pagedDiffPage{Deletes: deletes}); err != nil {
|
||||
return true, totals, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(targetRows) < defaultSyncReadPageSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true, totals, nil
|
||||
}
|
||||
|
||||
func diffSourcePageByPK(pkCol string, sourceRows, targetRows []map[string]interface{}) pagedDiffPage {
|
||||
targetMap := make(map[string]map[string]interface{}, len(targetRows))
|
||||
for _, row := range targetRows {
|
||||
pkKey, ok := pkValueKey(row[pkCol])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
targetMap[pkKey] = row
|
||||
}
|
||||
|
||||
page := pagedDiffPage{
|
||||
Inserts: make([]map[string]interface{}, 0),
|
||||
Updates: make([]pagedUpdateDiff, 0),
|
||||
}
|
||||
for _, sourceRow := range sourceRows {
|
||||
pkKey, ok := pkValueKey(sourceRow[pkCol])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
targetRow, exists := targetMap[pkKey]
|
||||
if !exists {
|
||||
page.Inserts = append(page.Inserts, sourceRow)
|
||||
continue
|
||||
}
|
||||
|
||||
changes := make(map[string]interface{})
|
||||
changedColumns := make([]string, 0)
|
||||
for key, value := range sourceRow {
|
||||
if fmt.Sprintf("%v", value) == fmt.Sprintf("%v", targetRow[key]) {
|
||||
continue
|
||||
}
|
||||
changes[key] = value
|
||||
changedColumns = append(changedColumns, key)
|
||||
}
|
||||
if len(changes) == 0 {
|
||||
page.Same++
|
||||
continue
|
||||
}
|
||||
sort.Strings(changedColumns)
|
||||
page.Updates = append(page.Updates, pagedUpdateDiff{
|
||||
UpdateRow: connection.UpdateRow{
|
||||
Keys: map[string]interface{}{pkCol: sourceRow[pkCol]},
|
||||
Values: changes,
|
||||
},
|
||||
Source: sourceRow,
|
||||
Target: targetRow,
|
||||
ChangedColumns: changedColumns,
|
||||
})
|
||||
}
|
||||
return page
|
||||
}
|
||||
|
||||
func filterPagedUpdatesByPKSelection(pkCol string, updates []pagedUpdateDiff, enabled bool, selectedPKs []string) []connection.UpdateRow {
|
||||
if !enabled {
|
||||
return nil
|
||||
}
|
||||
if len(updates) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]connection.UpdateRow, 0, len(updates))
|
||||
for _, update := range updates {
|
||||
out = append(out, update.UpdateRow)
|
||||
}
|
||||
return filterUpdatesByPKSelection(pkCol, out, true, selectedPKs)
|
||||
}
|
||||
|
||||
func diffReadableColumns(sourceCols []connection.ColumnDefinition, allowedLower map[string]struct{}, pkCol string) []connection.ColumnDefinition {
|
||||
out := make([]connection.ColumnDefinition, 0, len(sourceCols))
|
||||
seen := map[string]struct{}{}
|
||||
add := func(col connection.ColumnDefinition) {
|
||||
name := strings.TrimSpace(col.Name)
|
||||
lower := strings.ToLower(name)
|
||||
if name == "" {
|
||||
return
|
||||
}
|
||||
if _, ok := seen[lower]; ok {
|
||||
return
|
||||
}
|
||||
seen[lower] = struct{}{}
|
||||
out = append(out, col)
|
||||
}
|
||||
for _, col := range sourceCols {
|
||||
name := strings.TrimSpace(col.Name)
|
||||
lower := strings.ToLower(name)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
if strings.EqualFold(name, pkCol) {
|
||||
add(col)
|
||||
continue
|
||||
}
|
||||
if len(allowedLower) > 0 {
|
||||
if _, ok := allowedLower[lower]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
add(col)
|
||||
}
|
||||
if _, ok := seen[strings.ToLower(strings.TrimSpace(pkCol))]; !ok && strings.TrimSpace(pkCol) != "" {
|
||||
add(connection.ColumnDefinition{Name: pkCol})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func diffLookupColumns(sourceReadCols, targetCols []connection.ColumnDefinition, allowedLower map[string]struct{}, pkCol string) []connection.ColumnDefinition {
|
||||
targetByLower := make(map[string]connection.ColumnDefinition, len(targetCols))
|
||||
for _, col := range targetCols {
|
||||
name := strings.TrimSpace(col.Name)
|
||||
if name != "" {
|
||||
targetByLower[strings.ToLower(name)] = col
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]connection.ColumnDefinition, 0, len(sourceReadCols))
|
||||
seen := map[string]struct{}{}
|
||||
for _, sourceCol := range sourceReadCols {
|
||||
name := strings.TrimSpace(sourceCol.Name)
|
||||
lower := strings.ToLower(name)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[lower]; ok {
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(name, pkCol) && len(allowedLower) > 0 {
|
||||
if _, ok := allowedLower[lower]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if targetCol, ok := targetByLower[lower]; ok {
|
||||
out = append(out, targetCol)
|
||||
} else {
|
||||
out = append(out, connection.ColumnDefinition{Name: name})
|
||||
}
|
||||
seen[lower] = struct{}{}
|
||||
}
|
||||
if _, ok := seen[strings.ToLower(strings.TrimSpace(pkCol))]; !ok && strings.TrimSpace(pkCol) != "" {
|
||||
out = append(out, connection.ColumnDefinition{Name: pkCol})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func collectPKValues(rows []map[string]interface{}, pkCol string) []interface{} {
|
||||
values := make([]interface{}, 0, len(rows))
|
||||
seen := make(map[string]struct{}, len(rows))
|
||||
for _, row := range rows {
|
||||
key, ok := pkValueKey(row[pkCol])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[key]; exists {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
values = append(values, row[pkCol])
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func buildPKSet(rows []map[string]interface{}, pkCol string) map[string]struct{} {
|
||||
set := make(map[string]struct{}, len(rows))
|
||||
for _, row := range rows {
|
||||
key, ok := pkValueKey(row[pkCol])
|
||||
if ok {
|
||||
set[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
func lastValidPKValue(rows []map[string]interface{}, pkCol string) (interface{}, bool) {
|
||||
for i := len(rows) - 1; i >= 0; i-- {
|
||||
if _, ok := pkValueKey(rows[i][pkCol]); ok {
|
||||
return rows[i][pkCol], true
|
||||
}
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func pkValueKey(value interface{}) (string, bool) {
|
||||
if value == nil {
|
||||
return "", false
|
||||
}
|
||||
key := strings.TrimSpace(fmt.Sprintf("%v", value))
|
||||
if key == "" || key == "<nil>" {
|
||||
return "", false
|
||||
}
|
||||
return key, true
|
||||
}
|
||||
|
||||
func buildPKInSelectQuery(dbType, queryTable string, cols []connection.ColumnDefinition, pkCol string, pkValues []interface{}) string {
|
||||
if len(pkValues) == 0 {
|
||||
return ""
|
||||
}
|
||||
selectList := buildColumnSelectListForSync(dbType, cols)
|
||||
if strings.TrimSpace(selectList) == "" {
|
||||
selectList = "*"
|
||||
}
|
||||
literals := make([]string, 0, len(pkValues))
|
||||
for _, value := range pkValues {
|
||||
literal, ok := formatSyncSQLLiteral(value)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
literals = append(literals, literal)
|
||||
}
|
||||
if len(literals) == 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("SELECT %s FROM %s WHERE %s IN (%s)",
|
||||
selectList,
|
||||
quoteQualifiedIdentByType(dbType, queryTable),
|
||||
quoteIdentByType(dbType, pkCol),
|
||||
strings.Join(literals, ", "))
|
||||
}
|
||||
|
||||
func buildKeysetPagedTableQuery(dbType, queryTable string, cols []connection.ColumnDefinition, orderCol string, lastValue interface{}, hasLastValue bool, limit int) string {
|
||||
selectList := buildColumnSelectListForSync(dbType, cols)
|
||||
if strings.TrimSpace(selectList) == "" {
|
||||
selectList = "*"
|
||||
}
|
||||
safeLimit := limit
|
||||
if safeLimit <= 0 {
|
||||
safeLimit = defaultSyncReadPageSize
|
||||
}
|
||||
where := ""
|
||||
if hasLastValue {
|
||||
literal, ok := formatSyncSQLLiteral(lastValue)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
where = fmt.Sprintf(" WHERE %s > %s", quoteIdentByType(dbType, orderCol), literal)
|
||||
}
|
||||
orderBy := fmt.Sprintf(" ORDER BY %s ASC", quoteIdentByType(dbType, orderCol))
|
||||
if normalizeMigrationDBType(dbType) == "sqlserver" {
|
||||
return fmt.Sprintf("SELECT TOP (%d) %s FROM %s%s%s", safeLimit, selectList, quoteQualifiedIdentByType(dbType, queryTable), where, orderBy)
|
||||
}
|
||||
return fmt.Sprintf("SELECT %s FROM %s%s%s LIMIT %d", selectList, quoteQualifiedIdentByType(dbType, queryTable), where, orderBy, safeLimit)
|
||||
}
|
||||
|
||||
func countTableRowsForSync(database db.Database, dbType, queryTable string) (int, bool, error) {
|
||||
query := fmt.Sprintf("SELECT COUNT(*) AS __gonavi_count__ FROM %s", quoteQualifiedIdentByType(dbType, queryTable))
|
||||
rows, _, err := database.Query(query)
|
||||
if err != nil {
|
||||
return 0, true, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return 0, false, nil
|
||||
}
|
||||
for _, value := range rows[0] {
|
||||
count, ok := intFromSyncValue(value)
|
||||
if ok {
|
||||
return count, true, nil
|
||||
}
|
||||
}
|
||||
return 0, false, nil
|
||||
}
|
||||
|
||||
func intFromSyncValue(value interface{}) (int, bool) {
|
||||
if value == nil {
|
||||
return 0, false
|
||||
}
|
||||
switch v := value.(type) {
|
||||
case int:
|
||||
return v, true
|
||||
case int8:
|
||||
return int(v), true
|
||||
case int16:
|
||||
return int(v), true
|
||||
case int32:
|
||||
return int(v), true
|
||||
case int64:
|
||||
return int(v), true
|
||||
case uint:
|
||||
return int(v), true
|
||||
case uint8:
|
||||
return int(v), true
|
||||
case uint16:
|
||||
return int(v), true
|
||||
case uint32:
|
||||
return int(v), true
|
||||
case uint64:
|
||||
return int(v), true
|
||||
case float32:
|
||||
return int(v), true
|
||||
case float64:
|
||||
return int(v), true
|
||||
case []byte:
|
||||
i, err := strconv.Atoi(strings.TrimSpace(string(v)))
|
||||
return i, err == nil
|
||||
case string:
|
||||
i, err := strconv.Atoi(strings.TrimSpace(v))
|
||||
return i, err == nil
|
||||
default:
|
||||
rv := reflect.ValueOf(value)
|
||||
switch rv.Kind() {
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
i := rv.Int()
|
||||
if i > int64(^uint(0)>>1) {
|
||||
return 0, false
|
||||
}
|
||||
return int(i), true
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
u := rv.Uint()
|
||||
if u > uint64(^uint(0)>>1) {
|
||||
return 0, false
|
||||
}
|
||||
return int(u), true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func buildColumnSelectListForSync(dbType string, cols []connection.ColumnDefinition) string {
|
||||
quoted := make([]string, 0, len(cols))
|
||||
seen := map[string]struct{}{}
|
||||
for _, col := range cols {
|
||||
name := strings.TrimSpace(col.Name)
|
||||
lower := strings.ToLower(name)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[lower]; ok {
|
||||
continue
|
||||
}
|
||||
seen[lower] = struct{}{}
|
||||
quoted = append(quoted, quoteIdentByType(dbType, name))
|
||||
}
|
||||
return strings.Join(quoted, ", ")
|
||||
}
|
||||
|
||||
func formatSyncSQLLiteral(value interface{}) (string, bool) {
|
||||
if value == nil {
|
||||
return "", false
|
||||
}
|
||||
switch v := value.(type) {
|
||||
case time.Time:
|
||||
return quoteSyncSQLString(v.Format("2006-01-02 15:04:05.999999999")), true
|
||||
case []byte:
|
||||
return quoteSyncSQLString(string(v)), true
|
||||
case string:
|
||||
if strings.TrimSpace(v) == "" {
|
||||
return "", false
|
||||
}
|
||||
return quoteSyncSQLString(v), true
|
||||
case bool:
|
||||
if v {
|
||||
return "1", true
|
||||
}
|
||||
return "0", true
|
||||
default:
|
||||
text := strings.TrimSpace(fmt.Sprintf("%v", value))
|
||||
if text == "" || text == "<nil>" {
|
||||
return "", false
|
||||
}
|
||||
switch value.(type) {
|
||||
case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64:
|
||||
return text, true
|
||||
default:
|
||||
return quoteSyncSQLString(text), true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func quoteSyncSQLString(value string) string {
|
||||
return "'" + strings.ReplaceAll(value, "'", "''") + "'"
|
||||
}
|
||||
|
||||
func supportsPagedDiffSelect(dbType string) bool {
|
||||
return supportsDirectImportPagination(dbType)
|
||||
}
|
||||
|
||||
func supportsPagedDiffPKLookup(dbType string) bool {
|
||||
return supportsDirectImportPagination(dbType)
|
||||
}
|
||||
|
||||
func supportsPagedDiffKeysetSelect(dbType string) bool {
|
||||
return supportsDirectImportPagination(dbType)
|
||||
}
|
||||
304
internal/sync/direct_import_paging.go
Normal file
304
internal/sync/direct_import_paging.go
Normal file
@@ -0,0 +1,304 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"GoNavi-Wails/internal/connection"
|
||||
"GoNavi-Wails/internal/db"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const defaultSyncReadPageSize = defaultSyncApplyBatchSize
|
||||
|
||||
func (s *SyncEngine) tryApplyDirectImportInPages(config SyncConfig, res *SyncResult, tableIndex, totalTables int, tableName string, sourceDB db.Database, targetDB db.Database, plan SchemaMigrationPlan, sourceCols, targetCols []connection.ColumnDefinition, opts TableOptions, sourceType, targetType, applyTableName string) (bool, int, error) {
|
||||
tableMode := normalizeSyncMode(config.Mode)
|
||||
if tableMode == "insert_update" && plan.TargetTableExists {
|
||||
return false, 0, nil
|
||||
}
|
||||
if tableMode == "full_overwrite" && plan.TargetTableExists && isSamePhysicalSyncTable(config, plan, sourceType, targetType) {
|
||||
return false, 0, nil
|
||||
}
|
||||
if !opts.Insert {
|
||||
return false, 0, nil
|
||||
}
|
||||
|
||||
pkCol, ok := directImportPaginationPK(sourceType, sourceCols)
|
||||
if !ok && !supportsDirectImportPagination(sourceType) {
|
||||
return false, 0, nil
|
||||
}
|
||||
if !ok && len(opts.SelectedInsertPKs) > 0 {
|
||||
return false, 0, nil
|
||||
}
|
||||
|
||||
firstPageQuery := buildPagedSourceTableQuery(sourceType, plan.SourceQueryTable, sourceCols, pkCol, defaultSyncReadPageSize, 0)
|
||||
if strings.TrimSpace(firstPageQuery) == "" {
|
||||
return false, 0, nil
|
||||
}
|
||||
|
||||
applier, ok := targetDB.(db.BatchApplier)
|
||||
if !ok {
|
||||
return true, 0, fmt.Errorf("目标驱动不支持应用数据变更 (ApplyChanges)")
|
||||
}
|
||||
|
||||
if strings.TrimSpace(pkCol) != "" {
|
||||
s.appendLog(config.JobID, res, "info", fmt.Sprintf(" -> 启用分页流式导入:按主键 %s 每批读取 %d 行", pkCol, defaultSyncReadPageSize))
|
||||
} else {
|
||||
s.appendLog(config.JobID, res, "info", fmt.Sprintf(" -> 启用分页流式导入:每批读取 %d 行", defaultSyncReadPageSize))
|
||||
}
|
||||
s.progress(config.JobID, tableIndex, totalTables, tableName, "分页读取源表数据")
|
||||
firstRows, _, err := sourceDB.Query(firstPageQuery)
|
||||
if err != nil {
|
||||
return true, 0, fmt.Errorf("分页读取源表失败: %w", err)
|
||||
}
|
||||
|
||||
if tableMode == "full_overwrite" && plan.TargetTableExists {
|
||||
s.appendLog(config.JobID, res, "warn", fmt.Sprintf(" -> 全量覆盖模式:即将清空目标表 %s", tableName))
|
||||
s.progress(config.JobID, tableIndex, totalTables, tableName, "清空目标表")
|
||||
clearSQL := buildClearTargetTableSQL(targetType, plan.TargetQueryTable)
|
||||
if _, err := targetDB.Exec(clearSQL); err != nil {
|
||||
return true, 0, fmt.Errorf("清空目标表失败: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
targetColSet, err := s.prepareDirectImportTargetColumnSet(config, res, targetDB, plan, sourceType, targetType, sourceCols, targetCols)
|
||||
if err != nil {
|
||||
return true, 0, err
|
||||
}
|
||||
|
||||
inserted, err := s.applyDirectImportPage(config.JobID, res, applyTableName, applier, targetColSet, pkCol, opts, firstRows)
|
||||
if err != nil {
|
||||
return true, inserted, err
|
||||
}
|
||||
if len(firstRows) < defaultSyncReadPageSize {
|
||||
return true, inserted, nil
|
||||
}
|
||||
|
||||
for offset := defaultSyncReadPageSize; ; offset += defaultSyncReadPageSize {
|
||||
s.progress(config.JobID, tableIndex, totalTables, tableName, fmt.Sprintf("分页读取源表数据(%d+)", offset))
|
||||
query := buildPagedSourceTableQuery(sourceType, plan.SourceQueryTable, sourceCols, pkCol, defaultSyncReadPageSize, offset)
|
||||
rows, _, err := sourceDB.Query(query)
|
||||
if err != nil {
|
||||
return true, inserted, fmt.Errorf("分页读取源表失败(offset=%d): %w", offset, err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return true, inserted, nil
|
||||
}
|
||||
applied, err := s.applyDirectImportPage(config.JobID, res, applyTableName, applier, targetColSet, pkCol, opts, rows)
|
||||
inserted += applied
|
||||
if err != nil {
|
||||
return true, inserted, err
|
||||
}
|
||||
if len(rows) < defaultSyncReadPageSize {
|
||||
return true, inserted, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *SyncEngine) prepareDirectImportTargetColumnSet(config SyncConfig, res *SyncResult, targetDB db.Database, plan SchemaMigrationPlan, sourceType, targetType string, sourceCols, targetCols []connection.ColumnDefinition) (map[string]struct{}, error) {
|
||||
targetColsResolved := targetCols
|
||||
if len(targetColsResolved) == 0 {
|
||||
cols, err := targetDB.GetColumns(plan.TargetSchema, plan.TargetTable)
|
||||
if err != nil {
|
||||
s.appendLog(config.JobID, res, "warn", fmt.Sprintf(" -> 获取目标表字段失败,已跳过字段一致性检查: %v", err))
|
||||
return nil, nil
|
||||
}
|
||||
targetColsResolved = cols
|
||||
}
|
||||
if len(targetColsResolved) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
targetColSet := buildTargetColumnSet(targetColsResolved)
|
||||
missing := missingSourceColumns(sourceCols, targetColSet)
|
||||
if len(missing) == 0 {
|
||||
return targetColSet, nil
|
||||
}
|
||||
|
||||
if config.AutoAddColumns && supportsAutoAddColumnsForPair(sourceType, targetType) {
|
||||
s.appendLog(config.JobID, res, "warn", fmt.Sprintf(" -> 目标表缺少字段 %d 个,开始自动补齐: %s", len(missing), strings.Join(missing, ", ")))
|
||||
added := 0
|
||||
sourceColsByLower := make(map[string]connection.ColumnDefinition, len(sourceCols))
|
||||
for _, col := range sourceCols {
|
||||
key := strings.ToLower(strings.TrimSpace(col.Name))
|
||||
if key != "" {
|
||||
sourceColsByLower[key] = col
|
||||
}
|
||||
}
|
||||
for _, colName := range missing {
|
||||
srcCol, ok := sourceColsByLower[strings.ToLower(strings.TrimSpace(colName))]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
alterSQL, err := buildAddColumnSQLForPair(sourceType, targetType, plan.TargetQueryTable, srcCol)
|
||||
if err != nil {
|
||||
s.appendLog(config.JobID, res, "error", fmt.Sprintf(" -> 自动补字段失败:字段=%s 错误=%v", colName, err))
|
||||
continue
|
||||
}
|
||||
if _, err := targetDB.Exec(alterSQL); err != nil {
|
||||
s.appendLog(config.JobID, res, "error", fmt.Sprintf(" -> 自动补字段失败:字段=%s 错误=%v", colName, err))
|
||||
continue
|
||||
}
|
||||
added++
|
||||
targetColSet[strings.ToLower(strings.TrimSpace(colName))] = struct{}{}
|
||||
}
|
||||
s.appendLog(config.JobID, res, "info", fmt.Sprintf(" -> 自动补字段完成:成功=%d 失败=%d", added, len(missing)-added))
|
||||
return targetColSet, nil
|
||||
}
|
||||
|
||||
s.appendLog(config.JobID, res, "warn", fmt.Sprintf(" -> 目标表缺少字段 %d 个(未开启自动补齐),将自动忽略:%s", len(missing), strings.Join(missing, ", ")))
|
||||
return targetColSet, nil
|
||||
}
|
||||
|
||||
func missingSourceColumns(sourceCols []connection.ColumnDefinition, targetColSet map[string]struct{}) []string {
|
||||
missing := make([]string, 0)
|
||||
seen := make(map[string]struct{}, len(sourceCols))
|
||||
for _, col := range sourceCols {
|
||||
name := strings.TrimSpace(col.Name)
|
||||
lower := strings.ToLower(name)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[lower]; ok {
|
||||
continue
|
||||
}
|
||||
seen[lower] = struct{}{}
|
||||
if _, ok := targetColSet[lower]; !ok {
|
||||
missing = append(missing, name)
|
||||
}
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
func (s *SyncEngine) applyDirectImportPage(jobID string, res *SyncResult, tableName string, applier db.BatchApplier, targetColSet map[string]struct{}, pkCol string, opts TableOptions, rows []map[string]interface{}) (int, error) {
|
||||
if len(rows) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
rows = filterRowsByPKSelection(pkCol, rows, opts.Insert, opts.SelectedInsertPKs)
|
||||
if len(rows) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if len(targetColSet) > 0 {
|
||||
rows = filterInsertRows(rows, targetColSet)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
changeSet := connection.ChangeSet{Inserts: rows}
|
||||
if err := s.applyChangesInBatches(jobID, res, tableName, applier, changeSet); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(rows), nil
|
||||
}
|
||||
|
||||
func directImportPaginationPK(sourceType string, sourceCols []connection.ColumnDefinition) (string, bool) {
|
||||
if !supportsDirectImportPagination(sourceType) {
|
||||
return "", false
|
||||
}
|
||||
pkCols := make([]string, 0, 2)
|
||||
for _, col := range sourceCols {
|
||||
if col.Key == "PRI" || col.Key == "PK" {
|
||||
pkCols = append(pkCols, col.Name)
|
||||
}
|
||||
}
|
||||
if len(pkCols) != 1 || strings.TrimSpace(pkCols[0]) == "" {
|
||||
return "", false
|
||||
}
|
||||
return pkCols[0], true
|
||||
}
|
||||
|
||||
func supportsDirectImportPagination(dbType string) bool {
|
||||
switch normalizeMigrationDBType(dbType) {
|
||||
case "mysql", "mariadb", "postgres", "kingbase", "highgo", "vastbase", "opengauss", "sqlserver", "sqlite", "duckdb", "clickhouse", "tdengine", "starrocks", "diros":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func buildPagedSourceTableQuery(dbType, queryTable string, cols []connection.ColumnDefinition, orderCol string, limit, offset int) string {
|
||||
selectList := buildSourceColumnSelectList(dbType, cols)
|
||||
if strings.TrimSpace(selectList) == "" {
|
||||
return ""
|
||||
}
|
||||
pageSelectList := selectList
|
||||
if normalizeMigrationDBType(dbType) == "sqlserver" {
|
||||
pageSelectList = buildSQLServerPageSelectList(cols)
|
||||
}
|
||||
baseSQL := fmt.Sprintf("SELECT %s FROM %s", selectList, quoteQualifiedIdentByType(dbType, queryTable))
|
||||
orderBy := ""
|
||||
if strings.TrimSpace(orderCol) != "" {
|
||||
orderBy = fmt.Sprintf(" ORDER BY %s ASC", quoteIdentByType(dbType, orderCol))
|
||||
}
|
||||
return buildPaginatedSelectSQLForSync(dbType, baseSQL, pageSelectList, orderBy, limit, offset)
|
||||
}
|
||||
|
||||
func buildSourceColumnSelectList(dbType string, cols []connection.ColumnDefinition) string {
|
||||
quoted := make([]string, 0, len(cols))
|
||||
for _, col := range cols {
|
||||
name := strings.TrimSpace(col.Name)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
quoted = append(quoted, quoteIdentByType(dbType, name))
|
||||
}
|
||||
return strings.Join(quoted, ", ")
|
||||
}
|
||||
|
||||
func buildSQLServerPageSelectList(cols []connection.ColumnDefinition) string {
|
||||
quoted := make([]string, 0, len(cols))
|
||||
for _, col := range cols {
|
||||
name := strings.TrimSpace(col.Name)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
quoted = append(quoted, fmt.Sprintf("[__gonavi_page_result__].%s", quoteIdentByType("sqlserver", name)))
|
||||
}
|
||||
return strings.Join(quoted, ", ")
|
||||
}
|
||||
|
||||
func buildPaginatedSelectSQLForSync(dbType, baseSQL, selectList, orderBySQL string, limit, offset int) string {
|
||||
safeLimit := limit
|
||||
if safeLimit <= 0 {
|
||||
safeLimit = defaultSyncReadPageSize
|
||||
}
|
||||
safeOffset := offset
|
||||
if safeOffset < 0 {
|
||||
safeOffset = 0
|
||||
}
|
||||
base := strings.TrimSpace(baseSQL)
|
||||
orderBy := strings.TrimSpace(orderBySQL)
|
||||
|
||||
switch normalizeMigrationDBType(dbType) {
|
||||
case "sqlserver":
|
||||
upperBound := safeOffset + safeLimit
|
||||
if orderBy == "" {
|
||||
orderBy = "ORDER BY (SELECT NULL)"
|
||||
}
|
||||
return fmt.Sprintf("SELECT %s FROM (SELECT [__gonavi_page__].*, ROW_NUMBER() OVER (%s) AS [__gonavi_rn__] FROM (%s) AS [__gonavi_page__]) AS [__gonavi_page_result__] WHERE [__gonavi_rn__] > %d AND [__gonavi_rn__] <= %d ORDER BY [__gonavi_rn__]", selectList, orderBy, base, safeOffset, upperBound)
|
||||
default:
|
||||
return fmt.Sprintf("%s %s LIMIT %d OFFSET %d", base, orderBy, safeLimit, safeOffset)
|
||||
}
|
||||
}
|
||||
|
||||
func buildClearTargetTableSQL(targetType, targetQueryTable string) string {
|
||||
quotedTable := quoteQualifiedIdentByType(targetType, targetQueryTable)
|
||||
if normalizeMigrationDBType(targetType) == "mysql" {
|
||||
return fmt.Sprintf("TRUNCATE TABLE %s", quotedTable)
|
||||
}
|
||||
return fmt.Sprintf("DELETE FROM %s", quotedTable)
|
||||
}
|
||||
|
||||
func isSamePhysicalSyncTable(config SyncConfig, plan SchemaMigrationPlan, sourceType, targetType string) bool {
|
||||
if normalizeMigrationDBType(sourceType) != normalizeMigrationDBType(targetType) {
|
||||
return false
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(plan.SourceQueryTable), strings.TrimSpace(plan.TargetQueryTable)) {
|
||||
return false
|
||||
}
|
||||
source := config.SourceConfig
|
||||
target := config.TargetConfig
|
||||
return strings.EqualFold(strings.TrimSpace(source.Host), strings.TrimSpace(target.Host)) &&
|
||||
source.Port == target.Port &&
|
||||
strings.EqualFold(strings.TrimSpace(source.Database), strings.TrimSpace(target.Database)) &&
|
||||
strings.EqualFold(strings.TrimSpace(source.Driver), strings.TrimSpace(target.Driver)) &&
|
||||
strings.EqualFold(strings.TrimSpace(source.DSN), strings.TrimSpace(target.DSN))
|
||||
}
|
||||
@@ -104,31 +104,8 @@ func (s *SyncEngine) Preview(config SyncConfig, tableName string, limit int) (Ta
|
||||
}
|
||||
pkCol := pkCols[0]
|
||||
|
||||
sourceRows, _, err := sourceDB.Query(fmt.Sprintf("SELECT * FROM %s", quoteQualifiedIdentByType(resolveMigrationDBType(config.SourceConfig), plan.SourceQueryTable)))
|
||||
if err != nil {
|
||||
return TableDiffPreview{}, fmt.Errorf("读取源表失败: %w", err)
|
||||
}
|
||||
|
||||
targetRows := make([]map[string]interface{}, 0)
|
||||
if plan.TargetTableExists {
|
||||
targetRows, _, err = targetDB.Query(fmt.Sprintf("SELECT * FROM %s", quoteQualifiedIdentByType(resolveMigrationDBType(config.TargetConfig), plan.TargetQueryTable)))
|
||||
if err != nil {
|
||||
return TableDiffPreview{}, fmt.Errorf("读取目标表失败: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
targetMap := make(map[string]map[string]interface{}, len(targetRows))
|
||||
for _, row := range targetRows {
|
||||
if row[pkCol] == nil {
|
||||
continue
|
||||
}
|
||||
pkVal := strings.TrimSpace(fmt.Sprintf("%v", row[pkCol]))
|
||||
if pkVal == "" || pkVal == "<nil>" {
|
||||
continue
|
||||
}
|
||||
targetMap[pkVal] = row
|
||||
}
|
||||
|
||||
sourceType := resolveMigrationDBType(config.SourceConfig)
|
||||
targetType := resolveMigrationDBType(config.TargetConfig)
|
||||
out := TableDiffPreview{
|
||||
Table: tableName,
|
||||
PKColumn: pkCol,
|
||||
@@ -152,6 +129,119 @@ func (s *SyncEngine) Preview(config SyncConfig, tableName string, limit int) (Ta
|
||||
out.ColumnTypes[name] = typ
|
||||
}
|
||||
|
||||
tableMode := normalizeSyncMode(config.Mode)
|
||||
targetColSet := map[string]struct{}{}
|
||||
if plan.TargetTableExists {
|
||||
targetCols, err := targetDB.GetColumns(plan.TargetSchema, plan.TargetTable)
|
||||
if err == nil {
|
||||
targetColSet = buildTargetColumnSet(targetCols)
|
||||
}
|
||||
}
|
||||
|
||||
if !plan.TargetTableExists || tableMode != "insert_update" {
|
||||
sourceCount, counted, err := countTableRowsForSync(sourceDB, sourceType, plan.SourceQueryTable)
|
||||
if err != nil {
|
||||
return TableDiffPreview{}, fmt.Errorf("读取源表数量失败: %w", err)
|
||||
}
|
||||
query := buildPagedSourceTableQuery(sourceType, plan.SourceQueryTable, cols, pkCol, limit, 0)
|
||||
if strings.TrimSpace(query) == "" {
|
||||
return TableDiffPreview{}, fmt.Errorf("当前数据源不支持分页预览")
|
||||
}
|
||||
sourceRows, _, err := sourceDB.Query(query)
|
||||
if err != nil {
|
||||
return TableDiffPreview{}, fmt.Errorf("读取源表失败: %w", err)
|
||||
}
|
||||
if !counted {
|
||||
sourceCount = len(sourceRows)
|
||||
}
|
||||
out.TotalInserts = sourceCount
|
||||
for _, row := range sourceRows {
|
||||
if len(out.Inserts) >= limit {
|
||||
break
|
||||
}
|
||||
pkVal := strings.TrimSpace(fmt.Sprintf("%v", row[pkCol]))
|
||||
if pkVal == "" || pkVal == "<nil>" {
|
||||
continue
|
||||
}
|
||||
out.Inserts = append(out.Inserts, PreviewRow{PK: pkVal, Row: row})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
handled, _, err := scanTableDiffInPages(sourceDB, targetDB, sourceType, targetType, plan, cols, nil, pkCol, targetColSet, true, func(page pagedDiffPage) error {
|
||||
out.TotalInserts += len(page.Inserts)
|
||||
out.TotalUpdates += len(page.Updates)
|
||||
out.TotalDeletes += len(page.Deletes)
|
||||
|
||||
for _, row := range page.Inserts {
|
||||
if len(out.Inserts) >= limit {
|
||||
break
|
||||
}
|
||||
pkVal := strings.TrimSpace(fmt.Sprintf("%v", row[pkCol]))
|
||||
if pkVal == "" || pkVal == "<nil>" {
|
||||
continue
|
||||
}
|
||||
out.Inserts = append(out.Inserts, PreviewRow{PK: pkVal, Row: row})
|
||||
}
|
||||
for _, update := range page.Updates {
|
||||
if len(out.Updates) >= limit {
|
||||
break
|
||||
}
|
||||
pkVal := strings.TrimSpace(fmt.Sprintf("%v", update.UpdateRow.Keys[pkCol]))
|
||||
if pkVal == "" || pkVal == "<nil>" {
|
||||
continue
|
||||
}
|
||||
out.Updates = append(out.Updates, PreviewUpdateRow{
|
||||
PK: pkVal,
|
||||
ChangedColumns: append([]string(nil), update.ChangedColumns...),
|
||||
Source: update.Source,
|
||||
Target: update.Target,
|
||||
})
|
||||
}
|
||||
for _, row := range page.Deletes {
|
||||
if len(out.Deletes) >= limit {
|
||||
break
|
||||
}
|
||||
pkVal := strings.TrimSpace(fmt.Sprintf("%v", row[pkCol]))
|
||||
if pkVal == "" || pkVal == "<nil>" {
|
||||
continue
|
||||
}
|
||||
out.Deletes = append(out.Deletes, PreviewRow{PK: pkVal, Row: row})
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if handled {
|
||||
if err != nil {
|
||||
return TableDiffPreview{}, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
sourceRows, _, err := sourceDB.Query(fmt.Sprintf("SELECT * FROM %s", quoteQualifiedIdentByType(sourceType, plan.SourceQueryTable)))
|
||||
if err != nil {
|
||||
return TableDiffPreview{}, fmt.Errorf("读取源表失败: %w", err)
|
||||
}
|
||||
|
||||
targetRows := make([]map[string]interface{}, 0)
|
||||
if plan.TargetTableExists {
|
||||
targetRows, _, err = targetDB.Query(fmt.Sprintf("SELECT * FROM %s", quoteQualifiedIdentByType(targetType, plan.TargetQueryTable)))
|
||||
if err != nil {
|
||||
return TableDiffPreview{}, fmt.Errorf("读取目标表失败: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
targetMap := make(map[string]map[string]interface{}, len(targetRows))
|
||||
for _, row := range targetRows {
|
||||
if row[pkCol] == nil {
|
||||
continue
|
||||
}
|
||||
pkVal := strings.TrimSpace(fmt.Sprintf("%v", row[pkCol]))
|
||||
if pkVal == "" || pkVal == "<nil>" {
|
||||
continue
|
||||
}
|
||||
targetMap[pkVal] = row
|
||||
}
|
||||
|
||||
sourcePKSet := make(map[string]struct{}, len(sourceRows))
|
||||
for _, sRow := range sourceRows {
|
||||
if sRow[pkCol] == nil {
|
||||
|
||||
@@ -14,12 +14,14 @@ type fakeMigrationDB struct {
|
||||
tables map[string][]string
|
||||
queryData map[string][]map[string]interface{}
|
||||
queryCols map[string][]string
|
||||
queryLog []string
|
||||
}
|
||||
|
||||
func (f *fakeMigrationDB) Connect(config connection.ConnectionConfig) error { return nil }
|
||||
func (f *fakeMigrationDB) Close() error { return nil }
|
||||
func (f *fakeMigrationDB) Ping() error { return nil }
|
||||
func (f *fakeMigrationDB) Query(query string) ([]map[string]interface{}, []string, error) {
|
||||
f.queryLog = append(f.queryLog, query)
|
||||
if rows, ok := f.queryData[query]; ok {
|
||||
return rows, f.queryCols[query], nil
|
||||
}
|
||||
|
||||
286
internal/sync/source_query_paging.go
Normal file
286
internal/sync/source_query_paging.go
Normal file
@@ -0,0 +1,286 @@
|
||||
package sync
|
||||
|
||||
import (
|
||||
"GoNavi-Wails/internal/connection"
|
||||
"GoNavi-Wails/internal/db"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func (s *SyncEngine) tryApplySourceQueryInPages(config SyncConfig, res *SyncResult, tableName string, sourceDB db.Database, targetDB db.Database, ctx sourceQuerySyncContext, opts TableOptions, tableMode string, applyTableName string) (bool, pagedDiffCounts, error) {
|
||||
sourceType := resolveMigrationDBType(config.SourceConfig)
|
||||
if !supportsPagedSourceQuery(sourceType) || !supportsPagedDiffPKLookup(ctx.TargetType) {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
if strings.TrimSpace(buildSourceQueryPageSQL(sourceType, config.SourceQuery, ctx.PKColumn, defaultSyncReadPageSize, 0)) == "" {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
|
||||
applier, ok := targetDB.(db.BatchApplier)
|
||||
if !ok {
|
||||
return true, pagedDiffCounts{}, fmt.Errorf("目标驱动不支持应用数据变更 (ApplyChanges)")
|
||||
}
|
||||
targetColSet := buildTargetColumnSet(ctx.TargetCols)
|
||||
counts := pagedDiffCounts{}
|
||||
|
||||
if tableMode == "insert_update" {
|
||||
includeDeletes := opts.Delete
|
||||
handled, _, err := scanSourceQueryDiffInPages(sourceDB, targetDB, sourceType, ctx.TargetType, strings.TrimSpace(config.SourceQuery), ctx.TargetQueryTable, ctx.TargetCols, ctx.PKColumn, includeDeletes, func(page pagedDiffPage) error {
|
||||
changeSet := connection.ChangeSet{
|
||||
Inserts: filterRowsByPKSelection(ctx.PKColumn, page.Inserts, opts.Insert, opts.SelectedInsertPKs),
|
||||
Updates: filterPagedUpdatesByPKSelection(ctx.PKColumn, page.Updates, opts.Update, opts.SelectedUpdatePKs),
|
||||
Deletes: filterRowsByPKSelection(ctx.PKColumn, page.Deletes, opts.Delete, opts.SelectedDeletePKs),
|
||||
}
|
||||
changeSet.Inserts = filterInsertRows(changeSet.Inserts, targetColSet)
|
||||
changeSet.Updates = filterUpdateRows(changeSet.Updates, targetColSet)
|
||||
if len(changeSet.Inserts) == 0 && len(changeSet.Updates) == 0 && len(changeSet.Deletes) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := s.applyChangesInBatches(config.JobID, res, applyTableName, applier, changeSet); err != nil {
|
||||
return err
|
||||
}
|
||||
counts.Inserts += len(changeSet.Inserts)
|
||||
counts.Updates += len(changeSet.Updates)
|
||||
counts.Deletes += len(changeSet.Deletes)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return true, counts, err
|
||||
}
|
||||
return handled, counts, nil
|
||||
}
|
||||
|
||||
if tableMode == "full_overwrite" {
|
||||
clearSQL := buildClearTargetTableSQL(ctx.TargetType, ctx.TargetQueryTable)
|
||||
if _, err := targetDB.Exec(clearSQL); err != nil {
|
||||
return true, counts, fmt.Errorf("清空目标表失败: %w", err)
|
||||
}
|
||||
}
|
||||
if !opts.Insert {
|
||||
return true, counts, nil
|
||||
}
|
||||
|
||||
for offset := 0; ; offset += defaultSyncReadPageSize {
|
||||
query := buildSourceQueryPageSQL(sourceType, config.SourceQuery, ctx.PKColumn, defaultSyncReadPageSize, offset)
|
||||
rows, _, err := sourceDB.Query(query)
|
||||
if err != nil {
|
||||
return true, counts, fmt.Errorf("分页读取源查询失败(offset=%d): %w", offset, err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return true, counts, nil
|
||||
}
|
||||
pageSize := len(rows)
|
||||
insertRows := filterRowsByPKSelection(ctx.PKColumn, rows, opts.Insert, opts.SelectedInsertPKs)
|
||||
insertRows = filterInsertRows(insertRows, targetColSet)
|
||||
if len(insertRows) > 0 {
|
||||
if err := s.applyChangesInBatches(config.JobID, res, applyTableName, applier, connection.ChangeSet{Inserts: insertRows}); err != nil {
|
||||
return true, counts, err
|
||||
}
|
||||
counts.Inserts += len(insertRows)
|
||||
}
|
||||
if pageSize < defaultSyncReadPageSize {
|
||||
return true, counts, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func scanSourceQueryDiffInPages(sourceDB db.Database, targetDB db.Database, sourceType, targetType, sourceQuery, targetQueryTable string, targetCols []connection.ColumnDefinition, pkCol string, includeDeletes bool, consume func(page pagedDiffPage) error) (bool, pagedDiffCounts, error) {
|
||||
if !supportsPagedSourceQuery(sourceType) || !supportsPagedDiffPKLookup(targetType) {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
if includeDeletes && (!supportsPagedDiffKeysetSelect(targetType) || !supportsPagedSourceQueryPKLookup(sourceType)) {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
|
||||
sourcePageQuery := buildSourceQueryPageSQL(sourceType, sourceQuery, pkCol, defaultSyncReadPageSize, 0)
|
||||
if strings.TrimSpace(sourcePageQuery) == "" {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
targetLookupCols := diffLookupColumns(targetCols, targetCols, buildTargetColumnSet(targetCols), pkCol)
|
||||
if len(targetLookupCols) == 0 {
|
||||
targetLookupCols = []connection.ColumnDefinition{{Name: pkCol}}
|
||||
}
|
||||
|
||||
totals := pagedDiffCounts{}
|
||||
for offset := 0; ; offset += defaultSyncReadPageSize {
|
||||
query := buildSourceQueryPageSQL(sourceType, sourceQuery, pkCol, defaultSyncReadPageSize, offset)
|
||||
sourceRows, _, err := sourceDB.Query(query)
|
||||
if err != nil {
|
||||
return true, totals, fmt.Errorf("分页读取源查询失败(offset=%d): %w", offset, err)
|
||||
}
|
||||
if len(sourceRows) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
pkValues := collectPKValues(sourceRows, pkCol)
|
||||
targetRows := make([]map[string]interface{}, 0)
|
||||
if len(pkValues) > 0 {
|
||||
targetQuery := buildPKInSelectQuery(targetType, targetQueryTable, targetLookupCols, pkCol, pkValues)
|
||||
if strings.TrimSpace(targetQuery) == "" {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
targetRows, _, err = targetDB.Query(targetQuery)
|
||||
if err != nil {
|
||||
return true, totals, fmt.Errorf("按主键读取目标表失败(offset=%d): %w", offset, err)
|
||||
}
|
||||
}
|
||||
|
||||
page := diffSourcePageByPK(pkCol, sourceRows, targetRows)
|
||||
totals.Inserts += len(page.Inserts)
|
||||
totals.Updates += len(page.Updates)
|
||||
totals.Same += page.Same
|
||||
if consume != nil {
|
||||
if err := consume(page); err != nil {
|
||||
return true, totals, err
|
||||
}
|
||||
}
|
||||
if len(sourceRows) < defaultSyncReadPageSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if includeDeletes {
|
||||
lastPK, hasLastPK := interface{}(nil), false
|
||||
targetPKCols := []connection.ColumnDefinition{{Name: pkCol}}
|
||||
for {
|
||||
query := buildKeysetPagedTableQuery(targetType, targetQueryTable, targetPKCols, pkCol, lastPK, hasLastPK, defaultSyncReadPageSize)
|
||||
targetRows, _, err := targetDB.Query(query)
|
||||
if err != nil {
|
||||
return true, totals, fmt.Errorf("分页读取目标主键失败: %w", err)
|
||||
}
|
||||
if len(targetRows) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
nextLastPK, ok := lastValidPKValue(targetRows, pkCol)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
lastPK, hasLastPK = nextLastPK, true
|
||||
|
||||
pkValues := collectPKValues(targetRows, pkCol)
|
||||
sourcePKRows := make([]map[string]interface{}, 0)
|
||||
if len(pkValues) > 0 {
|
||||
sourceQuery := buildSourceQueryPKInSelectSQL(sourceType, sourceQuery, []connection.ColumnDefinition{{Name: pkCol}}, pkCol, pkValues)
|
||||
if strings.TrimSpace(sourceQuery) == "" {
|
||||
return false, pagedDiffCounts{}, nil
|
||||
}
|
||||
sourcePKRows, _, err = sourceDB.Query(sourceQuery)
|
||||
if err != nil {
|
||||
return true, totals, fmt.Errorf("按主键反查源查询失败: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
sourcePKSet := buildPKSet(sourcePKRows, pkCol)
|
||||
deletes := make([]map[string]interface{}, 0)
|
||||
for _, row := range targetRows {
|
||||
pkKey, ok := pkValueKey(row[pkCol])
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, exists := sourcePKSet[pkKey]; exists {
|
||||
continue
|
||||
}
|
||||
deletes = append(deletes, map[string]interface{}{pkCol: row[pkCol]})
|
||||
}
|
||||
if len(deletes) > 0 {
|
||||
totals.Deletes += len(deletes)
|
||||
if consume != nil {
|
||||
if err := consume(pagedDiffPage{Deletes: deletes}); err != nil {
|
||||
return true, totals, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(targetRows) < defaultSyncReadPageSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return true, totals, nil
|
||||
}
|
||||
|
||||
func buildSourceQueryPageSQL(dbType, sourceQuery, orderCol string, limit, offset int) string {
|
||||
subquery, ok := normalizeSourceQueryForPaging(sourceQuery)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
baseSQL := fmt.Sprintf("SELECT * FROM (%s) AS __gonavi_source_query__", subquery)
|
||||
orderBy := ""
|
||||
if strings.TrimSpace(orderCol) != "" {
|
||||
orderBy = fmt.Sprintf(" ORDER BY %s ASC", quoteIdentByType(dbType, orderCol))
|
||||
}
|
||||
return buildPaginatedSelectSQLForSync(dbType, baseSQL, "*", orderBy, limit, offset)
|
||||
}
|
||||
|
||||
func buildSourceQueryPKInSelectSQL(dbType, sourceQuery string, cols []connection.ColumnDefinition, pkCol string, pkValues []interface{}) string {
|
||||
subquery, ok := normalizeSourceQueryForPaging(sourceQuery)
|
||||
if !ok || len(pkValues) == 0 {
|
||||
return ""
|
||||
}
|
||||
selectList := buildColumnSelectListForSync(dbType, cols)
|
||||
if strings.TrimSpace(selectList) == "" {
|
||||
selectList = "*"
|
||||
}
|
||||
literals := make([]string, 0, len(pkValues))
|
||||
for _, value := range pkValues {
|
||||
literal, ok := formatSyncSQLLiteral(value)
|
||||
if ok {
|
||||
literals = append(literals, literal)
|
||||
}
|
||||
}
|
||||
if len(literals) == 0 {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("SELECT %s FROM (%s) AS __gonavi_source_query__ WHERE %s IN (%s)",
|
||||
selectList,
|
||||
subquery,
|
||||
quoteIdentByType(dbType, pkCol),
|
||||
strings.Join(literals, ", "))
|
||||
}
|
||||
|
||||
func countSourceQueryRowsForSync(database db.Database, dbType, sourceQuery string) (int, bool, error) {
|
||||
subquery, ok := normalizeSourceQueryForPaging(sourceQuery)
|
||||
if !ok {
|
||||
return 0, false, nil
|
||||
}
|
||||
query := fmt.Sprintf("SELECT COUNT(*) AS __gonavi_count__ FROM (%s) AS __gonavi_source_query__", subquery)
|
||||
rows, _, err := database.Query(query)
|
||||
if err != nil {
|
||||
return 0, true, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return 0, false, nil
|
||||
}
|
||||
for _, value := range rows[0] {
|
||||
count, ok := intFromSyncValue(value)
|
||||
if ok {
|
||||
return count, true, nil
|
||||
}
|
||||
}
|
||||
return 0, false, nil
|
||||
}
|
||||
|
||||
func normalizeSourceQueryForPaging(query string) (string, bool) {
|
||||
trimmed := strings.TrimSpace(query)
|
||||
if trimmed == "" {
|
||||
return "", false
|
||||
}
|
||||
trimmed = strings.TrimSuffix(trimmed, ";")
|
||||
trimmed = strings.TrimSpace(trimmed)
|
||||
lower := strings.ToLower(trimmed)
|
||||
if !(strings.HasPrefix(lower, "select ") || strings.HasPrefix(lower, "with ")) {
|
||||
return "", false
|
||||
}
|
||||
if strings.Contains(trimmed, ";") {
|
||||
return "", false
|
||||
}
|
||||
return trimmed, true
|
||||
}
|
||||
|
||||
func supportsPagedSourceQuery(dbType string) bool {
|
||||
return supportsDirectImportPagination(dbType)
|
||||
}
|
||||
|
||||
func supportsPagedSourceQueryPKLookup(dbType string) bool {
|
||||
return supportsDirectImportPagination(dbType)
|
||||
}
|
||||
@@ -68,7 +68,7 @@ func resolveSinglePKColumn(cols []connection.ColumnDefinition) (string, error) {
|
||||
return pkCols[0], nil
|
||||
}
|
||||
|
||||
func loadSourceQuerySyncContext(config SyncConfig, sourceDB db.Database, targetDB db.Database, needTargetRows bool, requirePK bool) (sourceQuerySyncContext, error) {
|
||||
func loadSourceQuerySyncContext(config SyncConfig, sourceDB db.Database, targetDB db.Database, needSourceRows bool, needTargetRows bool, requirePK bool) (sourceQuerySyncContext, error) {
|
||||
tableName, err := validateSourceQuerySyncConfig(config)
|
||||
if err != nil {
|
||||
return sourceQuerySyncContext{}, err
|
||||
@@ -83,11 +83,6 @@ func loadSourceQuerySyncContext(config SyncConfig, sourceDB db.Database, targetD
|
||||
return sourceQuerySyncContext{}, fmt.Errorf("目标表 %s 不存在或未读取到字段定义", tableName)
|
||||
}
|
||||
|
||||
sourceRows, _, err := sourceDB.Query(strings.TrimSpace(config.SourceQuery))
|
||||
if err != nil {
|
||||
return sourceQuerySyncContext{}, fmt.Errorf("执行源查询失败: %w", err)
|
||||
}
|
||||
|
||||
ctx := sourceQuerySyncContext{
|
||||
TableName: tableName,
|
||||
TargetSchema: targetSchema,
|
||||
@@ -95,10 +90,18 @@ func loadSourceQuerySyncContext(config SyncConfig, sourceDB db.Database, targetD
|
||||
TargetQueryTable: targetQueryTable,
|
||||
TargetType: targetType,
|
||||
TargetCols: targetCols,
|
||||
SourceRows: sourceRows,
|
||||
SourceRows: make([]map[string]interface{}, 0),
|
||||
TargetRows: make([]map[string]interface{}, 0),
|
||||
}
|
||||
|
||||
if needSourceRows {
|
||||
sourceRows, _, err := sourceDB.Query(strings.TrimSpace(config.SourceQuery))
|
||||
if err != nil {
|
||||
return sourceQuerySyncContext{}, fmt.Errorf("执行源查询失败: %w", err)
|
||||
}
|
||||
ctx.SourceRows = sourceRows
|
||||
}
|
||||
|
||||
if requirePK {
|
||||
pkColumn, err := resolveSinglePKColumn(targetCols)
|
||||
if err != nil {
|
||||
@@ -226,7 +229,40 @@ func (s *SyncEngine) analyzeSourceQuery(config SyncConfig) SyncAnalyzeResult {
|
||||
Table: tableName,
|
||||
CanSync: false,
|
||||
}
|
||||
ctx, err := loadSourceQuerySyncContext(config, sourceDB, targetDB, true, true)
|
||||
ctx, err := loadSourceQuerySyncContext(config, sourceDB, targetDB, false, false, true)
|
||||
if err != nil {
|
||||
summary.Message = err.Error()
|
||||
result.Tables = append(result.Tables, summary)
|
||||
result.Message = "已完成 1 个目标表的差异分析"
|
||||
s.progress(config.JobID, totalTables, totalTables, tableName, "差异分析完成")
|
||||
return result
|
||||
}
|
||||
|
||||
sourceType := resolveMigrationDBType(config.SourceConfig)
|
||||
handled, counts, scanErr := scanSourceQueryDiffInPages(sourceDB, targetDB, sourceType, ctx.TargetType, strings.TrimSpace(config.SourceQuery), ctx.TargetQueryTable, ctx.TargetCols, ctx.PKColumn, true, nil)
|
||||
if handled {
|
||||
if scanErr != nil {
|
||||
summary.Message = scanErr.Error()
|
||||
result.Tables = append(result.Tables, summary)
|
||||
result.Message = "已完成 1 个目标表的差异分析"
|
||||
s.progress(config.JobID, totalTables, totalTables, tableName, "差异分析完成")
|
||||
return result
|
||||
}
|
||||
summary.CanSync = true
|
||||
summary.PKColumn = ctx.PKColumn
|
||||
summary.Inserts = counts.Inserts
|
||||
summary.Updates = counts.Updates
|
||||
summary.Deletes = counts.Deletes
|
||||
summary.Same = counts.Same
|
||||
summary.TargetTableExists = true
|
||||
summary.Message = "SQL 结果集差异分析完成"
|
||||
result.Tables = append(result.Tables, summary)
|
||||
result.Message = "已完成 1 个目标表的差异分析"
|
||||
s.progress(config.JobID, totalTables, totalTables, tableName, "差异分析完成")
|
||||
return result
|
||||
}
|
||||
|
||||
ctx, err = loadSourceQuerySyncContext(config, sourceDB, targetDB, true, true, true)
|
||||
if err != nil {
|
||||
summary.Message = err.Error()
|
||||
result.Tables = append(result.Tables, summary)
|
||||
@@ -270,13 +306,83 @@ func (s *SyncEngine) previewSourceQuery(config SyncConfig, limit int) (TableDiff
|
||||
}
|
||||
defer targetDB.Close()
|
||||
|
||||
ctx, err := loadSourceQuerySyncContext(config, sourceDB, targetDB, true, true)
|
||||
ctx, err := loadSourceQuerySyncContext(config, sourceDB, targetDB, false, false, true)
|
||||
if err != nil {
|
||||
return TableDiffPreview{}, err
|
||||
}
|
||||
|
||||
sourceType := resolveMigrationDBType(config.SourceConfig)
|
||||
out := TableDiffPreview{
|
||||
Table: ctx.TableName,
|
||||
PKColumn: ctx.PKColumn,
|
||||
ColumnTypes: make(map[string]string, len(ctx.TargetCols)),
|
||||
SchemaSummary: "SQL 结果集同步预览",
|
||||
Inserts: make([]PreviewRow, 0, limit),
|
||||
Updates: make([]PreviewUpdateRow, 0, limit),
|
||||
Deletes: make([]PreviewRow, 0, limit),
|
||||
}
|
||||
for _, col := range ctx.TargetCols {
|
||||
name := strings.ToLower(strings.TrimSpace(col.Name))
|
||||
typ := strings.TrimSpace(col.Type)
|
||||
if name == "" || typ == "" {
|
||||
continue
|
||||
}
|
||||
out.ColumnTypes[name] = typ
|
||||
}
|
||||
|
||||
handled, _, scanErr := scanSourceQueryDiffInPages(sourceDB, targetDB, sourceType, ctx.TargetType, strings.TrimSpace(config.SourceQuery), ctx.TargetQueryTable, ctx.TargetCols, ctx.PKColumn, true, func(page pagedDiffPage) error {
|
||||
out.TotalInserts += len(page.Inserts)
|
||||
out.TotalUpdates += len(page.Updates)
|
||||
out.TotalDeletes += len(page.Deletes)
|
||||
for _, row := range page.Inserts {
|
||||
if len(out.Inserts) >= limit {
|
||||
break
|
||||
}
|
||||
pk := strings.TrimSpace(fmt.Sprintf("%v", row[ctx.PKColumn]))
|
||||
if pk != "" && pk != "<nil>" {
|
||||
out.Inserts = append(out.Inserts, PreviewRow{PK: pk, Row: row})
|
||||
}
|
||||
}
|
||||
for _, update := range page.Updates {
|
||||
if len(out.Updates) >= limit {
|
||||
break
|
||||
}
|
||||
pk := strings.TrimSpace(fmt.Sprintf("%v", update.UpdateRow.Keys[ctx.PKColumn]))
|
||||
if pk == "" || pk == "<nil>" {
|
||||
continue
|
||||
}
|
||||
out.Updates = append(out.Updates, PreviewUpdateRow{
|
||||
PK: pk,
|
||||
ChangedColumns: append([]string(nil), update.ChangedColumns...),
|
||||
Source: update.Source,
|
||||
Target: update.Target,
|
||||
})
|
||||
}
|
||||
for _, row := range page.Deletes {
|
||||
if len(out.Deletes) >= limit {
|
||||
break
|
||||
}
|
||||
pk := strings.TrimSpace(fmt.Sprintf("%v", row[ctx.PKColumn]))
|
||||
if pk != "" && pk != "<nil>" {
|
||||
out.Deletes = append(out.Deletes, PreviewRow{PK: pk, Row: row})
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if handled {
|
||||
if scanErr != nil {
|
||||
return TableDiffPreview{}, scanErr
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
ctx, err = loadSourceQuerySyncContext(config, sourceDB, targetDB, true, true, true)
|
||||
if err != nil {
|
||||
return TableDiffPreview{}, err
|
||||
}
|
||||
|
||||
inserts, updates, deletes, _ := diffRowsByPK(ctx.PKColumn, ctx.SourceRows, ctx.TargetRows)
|
||||
out := TableDiffPreview{
|
||||
out = TableDiffPreview{
|
||||
Table: ctx.TableName,
|
||||
PKColumn: ctx.PKColumn,
|
||||
ColumnTypes: make(map[string]string, len(ctx.TargetCols)),
|
||||
@@ -389,7 +495,7 @@ func (s *SyncEngine) runSourceQuerySync(config SyncConfig) SyncResult {
|
||||
|
||||
needTargetRows := tableMode == "insert_update"
|
||||
requirePK := tableMode == "insert_update"
|
||||
ctx, err := loadSourceQuerySyncContext(config, sourceDB, targetDB, needTargetRows, requirePK)
|
||||
ctx, err := loadSourceQuerySyncContext(config, sourceDB, targetDB, false, false, requirePK)
|
||||
if err != nil {
|
||||
return s.fail(config.JobID, totalTables, result, err.Error())
|
||||
}
|
||||
@@ -397,6 +503,33 @@ func (s *SyncEngine) runSourceQuerySync(config SyncConfig) SyncResult {
|
||||
inserts := make([]map[string]interface{}, 0)
|
||||
updates := make([]connection.UpdateRow, 0)
|
||||
deletes := make([]map[string]interface{}, 0)
|
||||
applyTableName := ctx.TargetTable
|
||||
switch ctx.TargetType {
|
||||
case "postgres", "kingbase", "highgo", "vastbase", "opengauss", "sqlserver":
|
||||
applyTableName = ctx.TargetQueryTable
|
||||
}
|
||||
|
||||
if handled, counts, err := s.tryApplySourceQueryInPages(config, &result, tableName, sourceDB, targetDB, ctx, opts, tableMode, applyTableName); handled {
|
||||
if err != nil {
|
||||
return s.fail(config.JobID, totalTables, result, "分页同步 SQL 结果集失败: "+err.Error())
|
||||
}
|
||||
result.TablesSynced++
|
||||
result.RowsInserted += counts.Inserts
|
||||
result.RowsUpdated += counts.Updates
|
||||
result.RowsDeleted += counts.Deletes
|
||||
if counts.Inserts == 0 && counts.Updates == 0 && counts.Deletes == 0 {
|
||||
s.appendLog(config.JobID, &result, "info", "SQL 结果集与目标表一致,无需应用变更")
|
||||
} else {
|
||||
s.appendLog(config.JobID, &result, "info", fmt.Sprintf("SQL 结果集分页同步完成:插入=%d 更新=%d 删除=%d", counts.Inserts, counts.Updates, counts.Deletes))
|
||||
}
|
||||
s.progress(config.JobID, totalTables, totalTables, tableName, "同步完成")
|
||||
return result
|
||||
}
|
||||
|
||||
ctx, err = loadSourceQuerySyncContext(config, sourceDB, targetDB, true, needTargetRows, requirePK)
|
||||
if err != nil {
|
||||
return s.fail(config.JobID, totalTables, result, err.Error())
|
||||
}
|
||||
if tableMode == "insert_update" {
|
||||
inserts, updates, deletes, _ = diffRowsByPK(ctx.PKColumn, ctx.SourceRows, ctx.TargetRows)
|
||||
inserts = filterRowsByPKSelection(ctx.PKColumn, inserts, opts.Insert, opts.SelectedInsertPKs)
|
||||
@@ -431,16 +564,11 @@ func (s *SyncEngine) runSourceQuerySync(config SyncConfig) SyncResult {
|
||||
return result
|
||||
}
|
||||
|
||||
applyTableName := ctx.TargetTable
|
||||
switch ctx.TargetType {
|
||||
case "postgres", "kingbase", "highgo", "vastbase", "opengauss", "sqlserver":
|
||||
applyTableName = ctx.TargetQueryTable
|
||||
}
|
||||
applier, ok := targetDB.(db.BatchApplier)
|
||||
if !ok {
|
||||
return s.fail(config.JobID, totalTables, result, "目标驱动不支持应用数据变更 (ApplyChanges)")
|
||||
}
|
||||
if err := applier.ApplyChanges(applyTableName, changeSet); err != nil {
|
||||
if err := s.applyChangesInBatches(config.JobID, &result, applyTableName, applier, changeSet); err != nil {
|
||||
return s.fail(config.JobID, totalTables, result, "应用 SQL 结果集变更失败: "+err.Error())
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"GoNavi-Wails/internal/connection"
|
||||
"GoNavi-Wails/internal/db"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -11,11 +12,15 @@ type fakeQuerySyncTargetDB struct {
|
||||
fakeMigrationDB
|
||||
appliedTable string
|
||||
appliedChanges connection.ChangeSet
|
||||
appliedBatches []connection.ChangeSet
|
||||
}
|
||||
|
||||
func (f *fakeQuerySyncTargetDB) ApplyChanges(tableName string, changes connection.ChangeSet) error {
|
||||
f.appliedTable = tableName
|
||||
f.appliedChanges = changes
|
||||
f.appliedChanges.Inserts = append(f.appliedChanges.Inserts, changes.Inserts...)
|
||||
f.appliedChanges.Updates = append(f.appliedChanges.Updates, changes.Updates...)
|
||||
f.appliedChanges.Deletes = append(f.appliedChanges.Deletes, changes.Deletes...)
|
||||
f.appliedBatches = append(f.appliedBatches, changes)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -30,10 +35,13 @@ func TestAnalyze_SourceQueryUsesQueryResultAsSourceDataset(t *testing.T) {
|
||||
},
|
||||
},
|
||||
queryData: map[string][]map[string]interface{}{
|
||||
"SELECT id, name FROM active_users": {
|
||||
"SELECT * FROM (SELECT id, name FROM active_users) AS __gonavi_source_query__ ORDER BY `id` ASC LIMIT 1000 OFFSET 0": {
|
||||
{"id": 1, "name": "Alice New"},
|
||||
{"id": 2, "name": "Bob"},
|
||||
},
|
||||
"SELECT `id` FROM (SELECT id, name FROM active_users) AS __gonavi_source_query__ WHERE `id` IN (1, 3)": {
|
||||
{"id": 1},
|
||||
},
|
||||
},
|
||||
}
|
||||
targetDB := &fakeQuerySyncTargetDB{
|
||||
@@ -45,8 +53,11 @@ func TestAnalyze_SourceQueryUsesQueryResultAsSourceDataset(t *testing.T) {
|
||||
},
|
||||
},
|
||||
queryData: map[string][]map[string]interface{}{
|
||||
"SELECT * FROM `app`.`users`": {
|
||||
"SELECT `id`, `name` FROM `app`.`users` WHERE `id` IN (1, 2)": {
|
||||
{"id": 1, "name": "Alice Old"},
|
||||
},
|
||||
"SELECT `id` FROM `app`.`users` ORDER BY `id` ASC LIMIT 1000": {
|
||||
{"id": 1},
|
||||
{"id": 3, "name": "Carol"},
|
||||
},
|
||||
},
|
||||
@@ -101,10 +112,13 @@ func TestRunSync_SourceQueryAppliesDiffAgainstTargetTable(t *testing.T) {
|
||||
},
|
||||
},
|
||||
queryData: map[string][]map[string]interface{}{
|
||||
"SELECT id, name FROM active_users": {
|
||||
"SELECT * FROM (SELECT id, name FROM active_users) AS __gonavi_source_query__ ORDER BY `id` ASC LIMIT 1000 OFFSET 0": {
|
||||
{"id": 1, "name": "Alice New"},
|
||||
{"id": 2, "name": "Bob"},
|
||||
},
|
||||
"SELECT `id` FROM (SELECT id, name FROM active_users) AS __gonavi_source_query__ WHERE `id` IN (1, 3)": {
|
||||
{"id": 1},
|
||||
},
|
||||
},
|
||||
}
|
||||
targetDB := &fakeQuerySyncTargetDB{
|
||||
@@ -116,8 +130,11 @@ func TestRunSync_SourceQueryAppliesDiffAgainstTargetTable(t *testing.T) {
|
||||
},
|
||||
},
|
||||
queryData: map[string][]map[string]interface{}{
|
||||
"SELECT * FROM `app`.`users`": {
|
||||
"SELECT `id`, `name` FROM `app`.`users` WHERE `id` IN (1, 2)": {
|
||||
{"id": 1, "name": "Alice Old"},
|
||||
},
|
||||
"SELECT `id` FROM `app`.`users` ORDER BY `id` ASC LIMIT 1000": {
|
||||
{"id": 1},
|
||||
{"id": 3, "name": "Carol"},
|
||||
},
|
||||
},
|
||||
@@ -175,3 +192,300 @@ func TestRunSync_SourceQueryAppliesDiffAgainstTargetTable(t *testing.T) {
|
||||
t.Fatalf("unexpected deletes: got=%v want=%v", targetDB.appliedChanges.Deletes, wantDeletes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSync_SourceQueryInsertUpdateUsesPagedQueries(t *testing.T) {
|
||||
columns := []connection.ColumnDefinition{
|
||||
{Name: "id", Type: "bigint", Nullable: "NO", Key: "PRI"},
|
||||
{Name: "name", Type: "varchar(64)", Nullable: "YES"},
|
||||
}
|
||||
sourceDB := &fakeMigrationDB{
|
||||
queryData: map[string][]map[string]interface{}{
|
||||
"SELECT * FROM (SELECT id, name FROM active_users) AS __gonavi_source_query__ ORDER BY `id` ASC LIMIT 1000 OFFSET 0": {
|
||||
{"id": 1, "name": "Alice New"},
|
||||
{"id": 2, "name": "Bob"},
|
||||
},
|
||||
"SELECT `id` FROM (SELECT id, name FROM active_users) AS __gonavi_source_query__ WHERE `id` IN (1, 3)": {
|
||||
{"id": 1},
|
||||
},
|
||||
},
|
||||
}
|
||||
targetDB := &fakeQuerySyncTargetDB{
|
||||
fakeMigrationDB: fakeMigrationDB{
|
||||
columns: map[string][]connection.ColumnDefinition{
|
||||
"app.users": columns,
|
||||
},
|
||||
queryData: map[string][]map[string]interface{}{
|
||||
"SELECT `id`, `name` FROM `app`.`users` WHERE `id` IN (1, 2)": {
|
||||
{"id": 1, "name": "Alice Old"},
|
||||
},
|
||||
"SELECT `id` FROM `app`.`users` ORDER BY `id` ASC LIMIT 1000": {
|
||||
{"id": 1},
|
||||
{"id": 3},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
oldFactory := newSyncDatabase
|
||||
defer func() { newSyncDatabase = oldFactory }()
|
||||
callCount := 0
|
||||
newSyncDatabase = func(dbType string) (db.Database, error) {
|
||||
callCount++
|
||||
if callCount == 1 {
|
||||
return sourceDB, nil
|
||||
}
|
||||
return targetDB, nil
|
||||
}
|
||||
|
||||
engine := NewSyncEngine(Reporter{})
|
||||
result := engine.RunSync(SyncConfig{
|
||||
SourceConfig: connection.ConnectionConfig{Type: "mysql", Database: "app"},
|
||||
TargetConfig: connection.ConnectionConfig{Type: "mysql", Database: "app"},
|
||||
Tables: []string{"users"},
|
||||
Mode: "insert_update",
|
||||
SourceQuery: "SELECT id, name FROM active_users",
|
||||
TableOptions: map[string]TableOptions{
|
||||
"users": {Insert: true, Update: true, Delete: true},
|
||||
},
|
||||
})
|
||||
|
||||
if !result.Success {
|
||||
t.Fatalf("RunSync 返回失败: %+v", result)
|
||||
}
|
||||
if result.RowsInserted != 1 || result.RowsUpdated != 1 || result.RowsDeleted != 1 {
|
||||
t.Fatalf("unexpected sync result: %+v", result)
|
||||
}
|
||||
for _, query := range sourceDB.queryLog {
|
||||
if query == "SELECT id, name FROM active_users" {
|
||||
t.Fatalf("SQL 结果集分页同步不应全量执行原始查询,实际查询=%s", query)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSync_BatchesLargeTableChanges(t *testing.T) {
|
||||
sourceRows := make([]map[string]interface{}, 2501)
|
||||
for i := range sourceRows {
|
||||
sourceRows[i] = map[string]interface{}{
|
||||
"id": i + 1,
|
||||
"name": "event",
|
||||
}
|
||||
}
|
||||
|
||||
columns := []connection.ColumnDefinition{
|
||||
{Name: "id", Type: "bigint", Nullable: "NO", Key: "PRI"},
|
||||
{Name: "name", Type: "varchar(64)", Nullable: "YES"},
|
||||
}
|
||||
sourceDB := &fakeMigrationDB{
|
||||
columns: map[string][]connection.ColumnDefinition{
|
||||
"app.events": columns,
|
||||
},
|
||||
queryData: map[string][]map[string]interface{}{
|
||||
"SELECT `id`, `name` FROM `app`.`events` ORDER BY `id` ASC LIMIT 1000 OFFSET 0": sourceRows[:1000],
|
||||
"SELECT `id`, `name` FROM `app`.`events` ORDER BY `id` ASC LIMIT 1000 OFFSET 1000": sourceRows[1000:2000],
|
||||
"SELECT `id`, `name` FROM `app`.`events` ORDER BY `id` ASC LIMIT 1000 OFFSET 2000": sourceRows[2000:],
|
||||
},
|
||||
}
|
||||
targetDB := &fakeQuerySyncTargetDB{
|
||||
fakeMigrationDB: fakeMigrationDB{
|
||||
columns: map[string][]connection.ColumnDefinition{
|
||||
"app.events": columns,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
oldFactory := newSyncDatabase
|
||||
defer func() { newSyncDatabase = oldFactory }()
|
||||
callCount := 0
|
||||
newSyncDatabase = func(dbType string) (db.Database, error) {
|
||||
callCount++
|
||||
if callCount == 1 {
|
||||
return sourceDB, nil
|
||||
}
|
||||
return targetDB, nil
|
||||
}
|
||||
|
||||
engine := NewSyncEngine(Reporter{})
|
||||
result := engine.RunSync(SyncConfig{
|
||||
SourceConfig: connection.ConnectionConfig{Type: "mysql", Database: "app"},
|
||||
TargetConfig: connection.ConnectionConfig{Type: "mysql", Database: "app"},
|
||||
Tables: []string{"events"},
|
||||
Mode: "insert_only",
|
||||
})
|
||||
|
||||
if !result.Success {
|
||||
t.Fatalf("RunSync 返回失败: %+v", result)
|
||||
}
|
||||
if result.RowsInserted != len(sourceRows) {
|
||||
t.Fatalf("RowsInserted=%d, want %d", result.RowsInserted, len(sourceRows))
|
||||
}
|
||||
for _, query := range sourceDB.queryLog {
|
||||
if strings.HasPrefix(query, "SELECT * FROM") {
|
||||
t.Fatalf("期望分页流式导入不再全量读取源表,实际查询=%s", query)
|
||||
}
|
||||
}
|
||||
if len(targetDB.appliedBatches) != 3 {
|
||||
t.Fatalf("期望大表拆成 3 批提交,实际 %d 批", len(targetDB.appliedBatches))
|
||||
}
|
||||
wantBatchSizes := []int{1000, 1000, 501}
|
||||
for idx, want := range wantBatchSizes {
|
||||
if got := len(targetDB.appliedBatches[idx].Inserts); got != want {
|
||||
t.Fatalf("batch %d inserts=%d, want %d", idx+1, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSync_DirectImportPagingKeepsSelectedPKFilter(t *testing.T) {
|
||||
sourceRows := []map[string]interface{}{
|
||||
{"id": 1, "name": "event-1"},
|
||||
{"id": 2, "name": "event-2"},
|
||||
{"id": 3, "name": "event-3"},
|
||||
}
|
||||
columns := []connection.ColumnDefinition{
|
||||
{Name: "id", Type: "bigint", Nullable: "NO", Key: "PRI"},
|
||||
{Name: "name", Type: "varchar(64)", Nullable: "YES"},
|
||||
}
|
||||
sourceDB := &fakeMigrationDB{
|
||||
columns: map[string][]connection.ColumnDefinition{
|
||||
"app.events": columns,
|
||||
},
|
||||
queryData: map[string][]map[string]interface{}{
|
||||
"SELECT `id`, `name` FROM `app`.`events` ORDER BY `id` ASC LIMIT 1000 OFFSET 0": sourceRows,
|
||||
},
|
||||
}
|
||||
targetDB := &fakeQuerySyncTargetDB{
|
||||
fakeMigrationDB: fakeMigrationDB{
|
||||
columns: map[string][]connection.ColumnDefinition{
|
||||
"app.events": columns,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
oldFactory := newSyncDatabase
|
||||
defer func() { newSyncDatabase = oldFactory }()
|
||||
callCount := 0
|
||||
newSyncDatabase = func(dbType string) (db.Database, error) {
|
||||
callCount++
|
||||
if callCount == 1 {
|
||||
return sourceDB, nil
|
||||
}
|
||||
return targetDB, nil
|
||||
}
|
||||
|
||||
engine := NewSyncEngine(Reporter{})
|
||||
result := engine.RunSync(SyncConfig{
|
||||
SourceConfig: connection.ConnectionConfig{Type: "mysql", Database: "app"},
|
||||
TargetConfig: connection.ConnectionConfig{Type: "mysql", Database: "app"},
|
||||
Tables: []string{"events"},
|
||||
Mode: "insert_only",
|
||||
TableOptions: map[string]TableOptions{
|
||||
"events": {
|
||||
Insert: true,
|
||||
SelectedInsertPKs: []string{"2"},
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
if !result.Success {
|
||||
t.Fatalf("RunSync 返回失败: %+v", result)
|
||||
}
|
||||
if result.RowsInserted != 1 {
|
||||
t.Fatalf("RowsInserted=%d, want 1", result.RowsInserted)
|
||||
}
|
||||
if len(targetDB.appliedBatches) != 1 || len(targetDB.appliedBatches[0].Inserts) != 1 {
|
||||
t.Fatalf("expected one selected insert batch, got %+v", targetDB.appliedBatches)
|
||||
}
|
||||
if got := targetDB.appliedBatches[0].Inserts[0]["id"]; got != 2 {
|
||||
t.Fatalf("selected insert id=%v, want 2", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunSync_InsertUpdateDiffUsesPagedPKLookups(t *testing.T) {
|
||||
sourceRows := []map[string]interface{}{
|
||||
{"id": 1, "name": "one-new"},
|
||||
{"id": 2, "name": "two"},
|
||||
{"id": 3, "name": "three"},
|
||||
}
|
||||
columns := []connection.ColumnDefinition{
|
||||
{Name: "id", Type: "bigint", Nullable: "NO", Key: "PRI"},
|
||||
{Name: "name", Type: "varchar(64)", Nullable: "YES"},
|
||||
}
|
||||
sourceDB := &fakeMigrationDB{
|
||||
columns: map[string][]connection.ColumnDefinition{
|
||||
"app.events": columns,
|
||||
},
|
||||
queryData: map[string][]map[string]interface{}{
|
||||
"SELECT `id`, `name` FROM `app`.`events` ORDER BY `id` ASC LIMIT 1000 OFFSET 0": sourceRows,
|
||||
"SELECT `id` FROM `app`.`events` WHERE `id` IN (1, 4)": {
|
||||
{"id": 1},
|
||||
},
|
||||
},
|
||||
}
|
||||
targetDB := &fakeQuerySyncTargetDB{
|
||||
fakeMigrationDB: fakeMigrationDB{
|
||||
columns: map[string][]connection.ColumnDefinition{
|
||||
"app.events": columns,
|
||||
},
|
||||
queryData: map[string][]map[string]interface{}{
|
||||
"SELECT `id`, `name` FROM `app`.`events` WHERE `id` IN (1, 2, 3)": {
|
||||
{"id": 1, "name": "one-old"},
|
||||
{"id": 2, "name": "two"},
|
||||
},
|
||||
"SELECT `id` FROM `app`.`events` ORDER BY `id` ASC LIMIT 1000": {
|
||||
{"id": 1},
|
||||
{"id": 4},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
oldFactory := newSyncDatabase
|
||||
defer func() { newSyncDatabase = oldFactory }()
|
||||
callCount := 0
|
||||
newSyncDatabase = func(dbType string) (db.Database, error) {
|
||||
callCount++
|
||||
if callCount == 1 {
|
||||
return sourceDB, nil
|
||||
}
|
||||
return targetDB, nil
|
||||
}
|
||||
|
||||
engine := NewSyncEngine(Reporter{})
|
||||
result := engine.RunSync(SyncConfig{
|
||||
SourceConfig: connection.ConnectionConfig{Type: "mysql", Database: "app"},
|
||||
TargetConfig: connection.ConnectionConfig{Type: "mysql", Database: "app"},
|
||||
Tables: []string{"events"},
|
||||
Mode: "insert_update",
|
||||
TableOptions: map[string]TableOptions{
|
||||
"events": {Insert: true, Update: true, Delete: true},
|
||||
},
|
||||
})
|
||||
|
||||
if !result.Success {
|
||||
t.Fatalf("RunSync 返回失败: %+v", result)
|
||||
}
|
||||
if result.RowsInserted != 1 || result.RowsUpdated != 1 || result.RowsDeleted != 1 {
|
||||
t.Fatalf("unexpected sync result: %+v", result)
|
||||
}
|
||||
if len(targetDB.appliedBatches) != 2 {
|
||||
t.Fatalf("expected source diff batch and delete batch, got %d", len(targetDB.appliedBatches))
|
||||
}
|
||||
firstBatch := targetDB.appliedBatches[0]
|
||||
if !reflect.DeepEqual(firstBatch.Inserts, []map[string]interface{}{{"id": 3, "name": "three"}}) {
|
||||
t.Fatalf("unexpected inserts: %+v", firstBatch.Inserts)
|
||||
}
|
||||
wantUpdates := []connection.UpdateRow{{
|
||||
Keys: map[string]interface{}{"id": 1},
|
||||
Values: map[string]interface{}{"name": "one-new"},
|
||||
}}
|
||||
if !reflect.DeepEqual(firstBatch.Updates, wantUpdates) {
|
||||
t.Fatalf("unexpected updates: %+v", firstBatch.Updates)
|
||||
}
|
||||
if !reflect.DeepEqual(targetDB.appliedBatches[1].Deletes, []map[string]interface{}{{"id": 4}}) {
|
||||
t.Fatalf("unexpected deletes: %+v", targetDB.appliedBatches[1].Deletes)
|
||||
}
|
||||
for _, query := range append(sourceDB.queryLog, targetDB.queryLog...) {
|
||||
if strings.HasPrefix(query, "SELECT * FROM") {
|
||||
t.Fatalf("分页差异同步不应全量读取表,实际查询=%s", query)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package sync
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"GoNavi-Wails/internal/connection"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestQuoteQualifiedIdentByType_KingbaseLeavesLowercaseQualifiedTableUnquoted(t *testing.T) {
|
||||
t.Parallel()
|
||||
@@ -58,3 +61,93 @@ func TestNormalizeMigrationDBType_KingbaseAliases(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPagedSourceTableQuery_MySQLUsesStablePKPagination(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
query := buildPagedSourceTableQuery("mysql", "app.events", []connection.ColumnDefinition{
|
||||
{Name: "id"},
|
||||
{Name: "name"},
|
||||
}, "id", 1000, 2000)
|
||||
|
||||
want := "SELECT `id`, `name` FROM `app`.`events` ORDER BY `id` ASC LIMIT 1000 OFFSET 2000"
|
||||
if query != want {
|
||||
t.Fatalf("unexpected paged query:\n got: %s\nwant: %s", query, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPagedSourceTableQuery_SQLServerUsesOuterAliasColumns(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
query := buildPagedSourceTableQuery("sqlserver", "dbo.events", []connection.ColumnDefinition{
|
||||
{Name: "id"},
|
||||
{Name: "name"},
|
||||
}, "id", 1000, 2000)
|
||||
|
||||
want := "SELECT [__gonavi_page_result__].[id], [__gonavi_page_result__].[name] FROM (SELECT [__gonavi_page__].*, ROW_NUMBER() OVER (ORDER BY [id] ASC) AS [__gonavi_rn__] FROM (SELECT [id], [name] FROM [dbo].[events]) AS [__gonavi_page__]) AS [__gonavi_page_result__] WHERE [__gonavi_rn__] > 2000 AND [__gonavi_rn__] <= 3000 ORDER BY [__gonavi_rn__]"
|
||||
if query != want {
|
||||
t.Fatalf("unexpected paged query:\n got: %s\nwant: %s", query, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsSamePhysicalSyncTableDetectsFullOverwriteSelfTarget(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
cfg := SyncConfig{
|
||||
SourceConfig: connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306, Database: "app"},
|
||||
TargetConfig: connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306, Database: "app"},
|
||||
}
|
||||
plan := SchemaMigrationPlan{SourceQueryTable: "app.events", TargetQueryTable: "app.events"}
|
||||
if !isSamePhysicalSyncTable(cfg, plan, "mysql", "mysql") {
|
||||
t.Fatal("expected identical connection/table to be detected")
|
||||
}
|
||||
|
||||
cfg.TargetConfig.Database = "archive"
|
||||
if isSamePhysicalSyncTable(cfg, plan, "mysql", "mysql") {
|
||||
t.Fatal("different database should not be treated as same physical table")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildPKInSelectQueryEscapesStringLiterals(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
query := buildPKInSelectQuery("mysql", "app.users", []connection.ColumnDefinition{
|
||||
{Name: "id"},
|
||||
{Name: "name"},
|
||||
}, "id", []interface{}{"a'1", "b2"})
|
||||
|
||||
want := "SELECT `id`, `name` FROM `app`.`users` WHERE `id` IN ('a''1', 'b2')"
|
||||
if query != want {
|
||||
t.Fatalf("unexpected PK IN query:\n got: %s\nwant: %s", query, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildKeysetPagedTableQueryUsesLastPK(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
query := buildKeysetPagedTableQuery("mysql", "app.users", []connection.ColumnDefinition{{Name: "id"}}, "id", 100, true, 50)
|
||||
|
||||
want := "SELECT `id` FROM `app`.`users` WHERE `id` > 100 ORDER BY `id` ASC LIMIT 50"
|
||||
if query != want {
|
||||
t.Fatalf("unexpected keyset query:\n got: %s\nwant: %s", query, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSourceQueryPageSQLWrapsSelect(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
query := buildSourceQueryPageSQL("mysql", "SELECT id, name FROM active_users;", "id", 1000, 2000)
|
||||
|
||||
want := "SELECT * FROM (SELECT id, name FROM active_users) AS __gonavi_source_query__ ORDER BY `id` ASC LIMIT 1000 OFFSET 2000"
|
||||
if query != want {
|
||||
t.Fatalf("unexpected source query page SQL:\n got: %s\nwant: %s", query, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSourceQueryForPagingRejectsMultiStatement(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if _, ok := normalizeSourceQueryForPaging("SELECT * FROM users; DELETE FROM users"); ok {
|
||||
t.Fatal("expected multi-statement source query to be rejected for pagination")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,14 @@ import (
|
||||
"GoNavi-Wails/internal/db"
|
||||
"GoNavi-Wails/internal/logger"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const defaultSyncApplyBatchSize = 1000
|
||||
|
||||
// SyncConfig defines the parameters for a synchronization task
|
||||
type SyncConfig struct {
|
||||
SourceConfig connection.ConnectionConfig `json:"sourceConfig"`
|
||||
@@ -251,6 +254,54 @@ func (s *SyncEngine) RunSync(config SyncConfig) SyncResult {
|
||||
return
|
||||
}
|
||||
|
||||
if handled, inserted, err := s.tryApplyDirectImportInPages(config, &result, i, totalTables, tableName, sourceDB, targetDB, plan, cols, targetCols, opts, sourceType, targetType, applyTableName); handled {
|
||||
if err != nil {
|
||||
logger.Error(err, "分页流式导入失败:表=%s", tableName)
|
||||
s.appendLog(config.JobID, &result, "error", fmt.Sprintf(" -> 分页流式导入失败: %v", err))
|
||||
return
|
||||
}
|
||||
result.RowsInserted += inserted
|
||||
if inserted > 0 {
|
||||
s.appendLog(config.JobID, &result, "info", fmt.Sprintf(" -> 分页流式导入完成:插入=%d 行", inserted))
|
||||
} else {
|
||||
s.appendLog(config.JobID, &result, "info", " -> 源表无可导入数据")
|
||||
}
|
||||
if len(plan.PostDataSQL) > 0 {
|
||||
s.progress(config.JobID, i, totalTables, tableName, "创建索引")
|
||||
if err := executeSQLStatements(targetDB.Exec, plan.PostDataSQL); err != nil {
|
||||
s.appendLog(config.JobID, &result, "error", fmt.Sprintf("创建索引失败:表=%s 错误=%v", tableName, err))
|
||||
return
|
||||
}
|
||||
}
|
||||
result.TablesSynced++
|
||||
return
|
||||
}
|
||||
|
||||
if handled, counts, err := s.tryApplyDiffInPages(config, &result, i, totalTables, tableName, sourceDB, targetDB, plan, cols, targetCols, opts, sourceType, targetType, applyTableName, pkCol); handled {
|
||||
if err != nil {
|
||||
logger.Error(err, "分页差异同步失败:表=%s", tableName)
|
||||
s.appendLog(config.JobID, &result, "error", fmt.Sprintf(" -> 分页差异同步失败: %v", err))
|
||||
return
|
||||
}
|
||||
result.RowsInserted += counts.Inserts
|
||||
result.RowsUpdated += counts.Updates
|
||||
result.RowsDeleted += counts.Deletes
|
||||
if counts.Inserts > 0 || counts.Updates > 0 || counts.Deletes > 0 {
|
||||
s.appendLog(config.JobID, &result, "info", fmt.Sprintf(" -> 分页差异同步完成:插入=%d 更新=%d 删除=%d", counts.Inserts, counts.Updates, counts.Deletes))
|
||||
} else {
|
||||
s.appendLog(config.JobID, &result, "info", " -> 数据一致,无需变更.")
|
||||
}
|
||||
if len(plan.PostDataSQL) > 0 {
|
||||
s.progress(config.JobID, i, totalTables, tableName, "创建索引")
|
||||
if err := executeSQLStatements(targetDB.Exec, plan.PostDataSQL); err != nil {
|
||||
s.appendLog(config.JobID, &result, "error", fmt.Sprintf("创建索引失败:表=%s 错误=%v", tableName, err))
|
||||
return
|
||||
}
|
||||
}
|
||||
result.TablesSynced++
|
||||
return
|
||||
}
|
||||
|
||||
s.progress(config.JobID, i, totalTables, tableName, "读取源表数据")
|
||||
sourceRows, _, err := sourceDB.Query(fmt.Sprintf("SELECT * FROM %s", quoteQualifiedIdentByType(sourceType, sourceQueryTable)))
|
||||
if err != nil {
|
||||
@@ -401,7 +452,7 @@ func (s *SyncEngine) RunSync(config SyncConfig) SyncResult {
|
||||
if len(changeSet.Inserts) > 0 || len(changeSet.Updates) > 0 || len(changeSet.Deletes) > 0 {
|
||||
s.appendLog(config.JobID, &result, "info", fmt.Sprintf(" -> 需插入: %d 行, 需更新: %d 行, 需删除: %d 行", len(changeSet.Inserts), len(changeSet.Updates), len(changeSet.Deletes)))
|
||||
if applier, ok := targetDB.(db.BatchApplier); ok {
|
||||
if err := applier.ApplyChanges(applyTableName, changeSet); err != nil {
|
||||
if err := s.applyChangesInBatches(config.JobID, &result, applyTableName, applier, changeSet); err != nil {
|
||||
s.appendLog(config.JobID, &result, "error", fmt.Sprintf(" -> 应用变更失败: %v", err))
|
||||
return
|
||||
}
|
||||
@@ -497,6 +548,75 @@ func (s *SyncEngine) fail(jobID string, totalTables int, res SyncResult, msg str
|
||||
return res
|
||||
}
|
||||
|
||||
func (s *SyncEngine) applyChangesInBatches(jobID string, res *SyncResult, tableName string, applier db.BatchApplier, changes connection.ChangeSet) error {
|
||||
batches := splitChangeSetBatches(changes, defaultSyncApplyBatchSize)
|
||||
if len(batches) == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(batches) > 1 {
|
||||
s.appendLog(jobID, res, "info", fmt.Sprintf(" -> 大批量变更将拆分为 %d 批提交(每批最多 %d 行)", len(batches), defaultSyncApplyBatchSize))
|
||||
}
|
||||
for idx, batch := range batches {
|
||||
if len(batches) > 1 {
|
||||
s.appendLog(jobID, res, "info", fmt.Sprintf(" -> 提交批次 %d/%d:插入=%d 更新=%d 删除=%d",
|
||||
idx+1, len(batches), len(batch.Inserts), len(batch.Updates), len(batch.Deletes)))
|
||||
}
|
||||
if err := applier.ApplyChanges(tableName, batch); err != nil {
|
||||
if len(batches) > 1 {
|
||||
return fmt.Errorf("批次 %d/%d 失败: %w", idx+1, len(batches), err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func splitChangeSetBatches(changes connection.ChangeSet, batchSize int) []connection.ChangeSet {
|
||||
if batchSize <= 0 {
|
||||
batchSize = defaultSyncApplyBatchSize
|
||||
}
|
||||
total := len(changes.Deletes) + len(changes.Updates) + len(changes.Inserts)
|
||||
if total == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
batches := make([]connection.ChangeSet, 0, int(math.Ceil(float64(total)/float64(batchSize))))
|
||||
current := connection.ChangeSet{LocatorStrategy: changes.LocatorStrategy}
|
||||
currentSize := 0
|
||||
flush := func() {
|
||||
if currentSize == 0 {
|
||||
return
|
||||
}
|
||||
batches = append(batches, current)
|
||||
current = connection.ChangeSet{LocatorStrategy: changes.LocatorStrategy}
|
||||
currentSize = 0
|
||||
}
|
||||
|
||||
for _, row := range changes.Deletes {
|
||||
if currentSize >= batchSize {
|
||||
flush()
|
||||
}
|
||||
current.Deletes = append(current.Deletes, row)
|
||||
currentSize++
|
||||
}
|
||||
for _, row := range changes.Updates {
|
||||
if currentSize >= batchSize {
|
||||
flush()
|
||||
}
|
||||
current.Updates = append(current.Updates, row)
|
||||
currentSize++
|
||||
}
|
||||
for _, row := range changes.Inserts {
|
||||
if currentSize >= batchSize {
|
||||
flush()
|
||||
}
|
||||
current.Inserts = append(current.Inserts, row)
|
||||
currentSize++
|
||||
}
|
||||
flush()
|
||||
return batches
|
||||
}
|
||||
|
||||
func (s *SyncEngine) execDDLStatements(jobID string, res *SyncResult, database db.Database, tableName string, stage string, statements []string) error {
|
||||
for _, statement := range statements {
|
||||
sqlText := strings.TrimSpace(statement)
|
||||
|
||||
Reference in New Issue
Block a user