From 384aea132c791361e845ec94ecacc3ae1640902f Mon Sep 17 00:00:00 2001 From: Syngnat Date: Fri, 17 Apr 2026 12:35:23 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(sync):=20=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=E4=BB=85=E5=90=8C=E6=AD=A5=E7=BB=93=E6=9E=84=E6=9C=AA=E7=94=9F?= =?UTF-8?q?=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 让已存在目标表场景复用通用补字段逻辑生成结构变更 SQL - 为分析与预览结果补充结构差异计数与结构 SQL 明细 - 补充结构同步回归测试并更新 backlog 记录 Fixes #342 --- .../2026-04-11-issue-backlog-tracking.md | 7 ++ frontend/src/components/DataSyncModal.tsx | 94 +++++++++++++++++-- internal/sync/analyze.go | 8 +- internal/sync/preview.go | 55 +++++++---- internal/sync/schema_migration.go | 42 +++++++-- internal/sync/schema_migration_test.go | 48 ++++++++++ 6 files changed, 221 insertions(+), 33 deletions(-) diff --git a/docs/issues/2026-04-11-issue-backlog-tracking.md b/docs/issues/2026-04-11-issue-backlog-tracking.md index c388736..3d9a258 100644 --- a/docs/issues/2026-04-11-issue-backlog-tracking.md +++ b/docs/issues/2026-04-11-issue-backlog-tracking.md @@ -33,6 +33,7 @@ | #333 | AI 功能添加供应商测试正常，但问答显示失败 | Fixed | Pending | | #337 | 自动更新无效 | Fixed | Pending | | #338 | 连接clickhouse不能通过8132端口 | Fixed | Pending | +| #342 | 数据同步功能不能用，mysql数据库8.4版本选了结构同步，最后没同步成功 | Fixed | Pending | | #351 | 为什么没有截断和清空表的功能呀？ | Fixed | Pending | ## Notes @@ -109,6 +110,12 @@ - 处理：将 `8132` 纳入 ClickHouse HTTP 端口识别，并同步更新自动切换日志和错误提示中的端口说明，避免排障信息继续误导。 - 验证：补充 `internal/db/clickhouse_impl_test.go` 回归测试，覆盖 `8132`、`8123`、`8443` 的 HTTP 判定以及 `9000/9440` 的 native 判定，并执行 `go test -tags gonavi_clickhouse_driver ./internal/db -run 'TestClickHouse(PingValidatesQueryPath|GetDatabasesFallsBackToCurrentDatabase|DetectClickHouseProtocolTreatsHTTPPortsAsHTTP)' -count=1`。 +### #342 + +- 根因：结构同步现有执行链路统一依赖 `buildSchemaMigrationPlan(...).PreDataSQL`。但 legacy planner 在“目标表已存在”分支里只给 `MySQL -> Kingbase` 生成补字段 SQL，`MySQL -> MySQL` 即使目标表缺列也只记 warning，不会产生任何可执行结构变更；同时前端 schema 模式的预览入口完全按数据差异计数启用，导致结构同步场景无法点开预览。 +- 处理：将 existing-target 分支的自动补字段逻辑改为复用通用 `buildAddColumnSQLForPair`，让 `MySQL -> MySQL` 也能生成并执行缺失字段补齐 SQL；同时为 analyze/preview 响应补充 `schemaDiffCount`、`schemaStatements`、`schemaSummary` 和 warning 信息，前端 schema 模式下可直接查看结构变更语句与风险提示，SQL 预览也会包含结构语句。 +- 验证：新增 `internal/sync/schema_migration_test.go` 回归测试，覆盖 `MySQL -> MySQL` 已存在目标表时生成补字段 SQL，并执行 `go test ./internal/sync -count=1` 与 `frontend` 下 `npm run build`。 + ### #330 - 根因：查询结果表格已经支持拖拽调整列宽，但 resize handle 没有提供双击自适应逻辑，导致用户只能靠手工拖拽慢慢试宽度。 diff --git a/frontend/src/components/DataSyncModal.tsx b/frontend/src/components/DataSyncModal.tsx index 720eb3e..9c01443 100644 --- a/frontend/src/components/DataSyncModal.tsx +++ b/frontend/src/components/DataSyncModal.tsx @@ -24,6 +24,7 @@ type TableDiffSummary = { updates?: number; deletes?: number; same?: number; + schemaDiffCount?: number; message?: string; targetTableExists?: boolean; plannedAction?: string; @@ -123,6 +124,15 @@ const buildSqlPreview = ( ? previewData.columnTypes as Record : {}; const statements: string[] = []; + const schemaStatements = Array.isArray(previewData.schemaStatements) + ? previewData.schemaStatements + .map((item: any) => String(item || '').trim()) + .filter((item: string) => item.length > 0) + : []; + + schemaStatements.forEach((statement: string) => { + statements.push(statement.endsWith(';') ? statement : `${statement};`); + }); const insertRows = Array.isArray(previewData.inserts) ? previewData.inserts : []; const updateRows = Array.isArray(previewData.updates) ? previewData.updates : []; @@ -478,7 +488,7 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open, sourceConfig: normalizeConnConfig(sConn, sourceDb), targetConfig: normalizeConnConfig(tConn, targetDb), tables: selectedTables, - content: "data", + content: syncContent, mode: "insert_update", autoAddColumns, targetTableStrategy, @@ -595,6 +605,18 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open, const ops = tableOptions[previewTable] || { insert: true, update: true, delete: false }; return buildSqlPreview(previewData, previewTable, targetType, ops); }, [previewData, previewTable, targetConnId, connections, tableOptions]); + const previewHasSchemaStatements = useMemo( + () => Array.isArray(previewData?.schemaStatements) && previewData.schemaStatements.length > 0, + [previewData], + ); + const previewSchemaWarnings = useMemo( + () => Array.isArray(previewData?.schemaWarnings) ? previewData.schemaWarnings as string[] : [], + [previewData], + ); + const previewHasDataDiff = useMemo( + () => Number(previewData?.totalInserts || 0) + Number(previewData?.totalUpdates || 0) + Number(previewData?.totalDeletes || 0) > 0, + [previewData], + ); const analysisWarnings = useMemo(() => { const items: string[] = []; @@ -1060,8 +1082,9 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open, render: (_: any, r: any) => { const can = !!r.canSync; const hasDiff = Number(r.inserts || 0) + Number(r.updates || 0) + Number(r.deletes || 0) > 0; + const hasSchemaDiff = Number(r.schemaDiffCount || 0) > 0; return ( - ); @@ -1168,12 +1191,59 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open, + {previewSchemaWarnings.length > 0 && ( + + {previewSchemaWarnings.slice(0, 8).map((item) =>
  • {item}
    • )} + {previewSchemaWarnings.length > 8 &&
  • 还有 {previewSchemaWarnings.length - 8} 项未展开
    • } + + } + /> + )} + + {previewData.schemaSummary || '以下为本次结构同步计划执行的语句。'} + + 
    +                                        {Array.isArray(previewData.schemaStatements) && previewData.schemaStatements.length > 0
+                                            ? previewData.schemaStatements.join('\n')
+                                            : '-- 当前表结构无可执行变更'}
+
    + + ) + }] : []), + ...(previewHasDataDiff ? [{ key: 'insert', label: `插入(${previewData.totalInserts || 0})`, children: ( @@ -1273,7 +1343,7 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open, /> ) - }, + }] : []), { key: 'sql', label: `SQL(${previewSql.statementCount})`, @@ -1282,10 +1352,18 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open,
    - 共 {previewSql.statementCount} 条语句（预览数据最多 200 条/类型） + + {previewHasDataDiff + ? `共 ${previewSql.statementCount} 条语句（预览数据最多 200 条/类型）` + : `共 ${previewSql.statementCount} 条结构变更语句`} +
    ) diff --git a/internal/sync/analyze.go b/internal/sync/analyze.go index e1a4af1..ac89fb9 100644 --- a/internal/sync/analyze.go +++ b/internal/sync/analyze.go @@ -14,6 +14,7 @@ type TableDiffSummary struct { Updates int `json:"updates"` Deletes int `json:"deletes"` Same int `json:"same"` + SchemaDiffCount int `json:"schemaDiffCount,omitempty"` Message string `json:"message,omitempty"` HasSchema bool `json:"hasSchema,omitempty"` TargetTableExists bool `json:"targetTableExists,omitempty"` @@ -109,6 +110,7 @@ func (s *SyncEngine) Analyze(config SyncConfig) SyncAnalyzeResult { summary.UnsupportedObjects = append(summary.UnsupportedObjects, plan.UnsupportedObjects...) summary.IndexesToCreate = plan.IndexesToCreate summary.IndexesSkipped = plan.IndexesSkipped + summary.SchemaDiffCount = len(plan.PreDataSQL) + len(plan.PostDataSQL) if !plan.TargetTableExists && !plan.AutoCreate { summary.Message = firstNonEmpty(plan.PlannedAction, "目标表不存在，无法执行同步") @@ -118,7 +120,11 @@ func (s *SyncEngine) Analyze(config SyncConfig) SyncAnalyzeResult { if !syncData { summary.CanSync = true - summary.Message = firstNonEmpty(plan.PlannedAction, "仅同步结构，未执行数据差异分析") + if summary.SchemaDiffCount > 0 { + summary.Message = firstNonEmpty(plan.PlannedAction, fmt.Sprintf("检测到 %d 条结构变更", summary.SchemaDiffCount)) + } else { + summary.Message = firstNonEmpty(plan.PlannedAction, "仅同步结构，未执行数据差异分析") + } result.Tables = append(result.Tables, summary) return } diff --git a/internal/sync/preview.go b/internal/sync/preview.go index 592d0de..3ba4045 100644 --- a/internal/sync/preview.go +++ b/internal/sync/preview.go @@ -19,15 +19,18 @@ type PreviewUpdateRow struct { } type TableDiffPreview struct { - Table string `json:"table"` - PKColumn string `json:"pkColumn"` - ColumnTypes map[string]string `json:"columnTypes,omitempty"` - TotalInserts int `json:"totalInserts"` - TotalUpdates int `json:"totalUpdates"` - TotalDeletes int `json:"totalDeletes"` - Inserts []PreviewRow `json:"inserts"` - Updates []PreviewUpdateRow `json:"updates"` - Deletes []PreviewRow `json:"deletes"` + Table string `json:"table"` + PKColumn string `json:"pkColumn"` + ColumnTypes map[string]string `json:"columnTypes,omitempty"` + SchemaSummary string `json:"schemaSummary,omitempty"` + SchemaWarnings []string `json:"schemaWarnings,omitempty"` + SchemaStatements []string `json:"schemaStatements,omitempty"` + TotalInserts int `json:"totalInserts"` + TotalUpdates int `json:"totalUpdates"` + TotalDeletes int `json:"totalDeletes"` + Inserts []PreviewRow `json:"inserts"` + Updates []PreviewUpdateRow `json:"updates"` + Deletes []PreviewRow `json:"deletes"` } func (s *SyncEngine) Preview(config SyncConfig, tableName string, limit int) (TableDiffPreview, error) { @@ -70,6 +73,19 @@ func (s *SyncEngine) Preview(config SyncConfig, tableName string, limit int) (Ta if !plan.TargetTableExists && !plan.AutoCreate { return TableDiffPreview{}, errors.New(firstNonEmpty(plan.PlannedAction, "目标表不存在，无法预览差异")) } + schemaStatements := make([]string, 0, len(plan.PreDataSQL)+len(plan.PostDataSQL)) + schemaStatements = append(schemaStatements, plan.PreDataSQL...) + schemaStatements = append(schemaStatements, plan.PostDataSQL...) + + contentRaw := strings.ToLower(strings.TrimSpace(config.Content)) + if contentRaw == "schema" { + return TableDiffPreview{ + Table: tableName, + SchemaSummary: firstNonEmpty(plan.PlannedAction, "仅同步结构"), + SchemaWarnings: append([]string(nil), plan.Warnings...), + SchemaStatements: append([]string(nil), schemaStatements...), + }, nil + } pkCols := make([]string, 0, 2) for _, c := range cols { @@ -111,15 +127,18 @@ func (s *SyncEngine) Preview(config SyncConfig, tableName string, limit int) (Ta } out := TableDiffPreview{ - Table: tableName, - PKColumn: pkCol, - ColumnTypes: make(map[string]string, len(cols)), - TotalInserts: 0, - TotalUpdates: 0, - TotalDeletes: 0, - Inserts: make([]PreviewRow, 0), - Updates: make([]PreviewUpdateRow, 0), - Deletes: make([]PreviewRow, 0), + Table: tableName, + PKColumn: pkCol, + ColumnTypes: make(map[string]string, len(cols)), + SchemaSummary: firstNonEmpty(plan.PlannedAction, "结构预览"), + SchemaWarnings: append([]string(nil), plan.Warnings...), + SchemaStatements: append([]string(nil), schemaStatements...), + TotalInserts: 0, + TotalUpdates: 0, + TotalDeletes: 0, + Inserts: make([]PreviewRow, 0), + Updates: make([]PreviewUpdateRow, 0), + Deletes: make([]PreviewRow, 0), } for _, col := range cols { name := strings.ToLower(strings.TrimSpace(col.Name)) diff --git a/internal/sync/schema_migration.go b/internal/sync/schema_migration.go index ad6cdc6..0506a59 100644 --- a/internal/sync/schema_migration.go +++ b/internal/sync/schema_migration.go @@ -127,12 +127,42 @@ func buildSchemaMigrationPlanLegacy(config SyncConfig, tableName string, sourceD if len(missing) > 0 { plan.Warnings = append(plan.Warnings, fmt.Sprintf("目标表缺失字段 %d 个：%s", len(missing), strings.Join(missing, ", "))) } - if config.AutoAddColumns && isMySQLLikeSourceType(sourceType) && normalizeMigrationDBType(targetType) == "kingbase" { - addSQL, addWarnings := buildMySQLToKingbaseAddColumnSQL(plan.TargetQueryTable, sourceCols, targetCols) - plan.PreDataSQL = append(plan.PreDataSQL, addSQL...) - plan.Warnings = append(plan.Warnings, addWarnings...) - if len(addSQL) > 0 { - plan.PlannedAction = fmt.Sprintf("补齐缺失字段(%d)后导入", len(addSQL)) + if len(missing) == 0 { + plan.PlannedAction = "表结构已一致" + } else if config.AutoAddColumns && supportsAutoAddColumnsForPair(sourceType, targetType) { + targetSet := make(map[string]struct{}, len(targetCols)) + for _, col := range targetCols { + key := strings.ToLower(strings.TrimSpace(col.Name)) + if key == "" { + continue + } + targetSet[key] = struct{}{} + } + for _, col := range sourceCols { + key := strings.ToLower(strings.TrimSpace(col.Name)) + if key == "" { + continue + } + if _, ok := targetSet[key]; ok { + continue + } + addSQL, err := buildAddColumnSQLForPair(sourceType, targetType, plan.TargetQueryTable, col) + if err != nil { + plan.Warnings = append(plan.Warnings, fmt.Sprintf("字段 %s 自动补齐 SQL 生成失败：%v", col.Name, err)) + continue + } + plan.PreDataSQL = append(plan.PreDataSQL, addSQL) + } + if len(plan.PreDataSQL) > 0 { + plan.PlannedAction = fmt.Sprintf("补齐缺失字段(%d)后导入", len(plan.PreDataSQL)) + } else { + plan.PlannedAction = fmt.Sprintf("目标表缺失字段(%d)，但未生成可执行补齐 SQL", len(missing)) + } + } else { + if config.AutoAddColumns { + plan.PlannedAction = fmt.Sprintf("目标表缺失字段(%d)，当前库对暂不支持自动补齐", len(missing)) + } else { + plan.PlannedAction = fmt.Sprintf("目标表缺失字段(%d)，未开启自动补齐", len(missing)) } } if strategy != "existing_only" { diff --git a/internal/sync/schema_migration_test.go b/internal/sync/schema_migration_test.go index c946fbe..a8cdbd0 100644 --- a/internal/sync/schema_migration_test.go +++ b/internal/sync/schema_migration_test.go @@ -266,6 +266,54 @@ func TestBuildPGLikeToMySQLPlan_AutoCreateWhenTargetMissing(t *testing.T) { } } +func TestBuildSchemaMigrationPlan_MySQLToMySQLAddsMissingColumnsForExistingTarget(t *testing.T) { + t.Parallel() + + sourceDB := &fakeMigrationDB{ + columns: map[string][]connection.ColumnDefinition{ + "shop.users": { + {Name: "id", Type: "bigint", Nullable: "NO", Key: "PRI"}, + {Name: "name", Type: "varchar(128)", Nullable: "YES"}, + {Name: "aaaa", Type: "varchar(255)", Nullable: "YES"}, + }, + }, + } + targetDB := &fakeMigrationDB{ + columns: map[string][]connection.ColumnDefinition{ + "app.users": { + {Name: "id", Type: "bigint", Nullable: "NO", Key: "PRI"}, + {Name: "name", Type: "varchar(128)", Nullable: "YES"}, + }, + }, + } + cfg := SyncConfig{ + SourceConfig: connection.ConnectionConfig{Type: "mysql", Database: "shop"}, + TargetConfig: connection.ConnectionConfig{Type: "mysql", Database: "app"}, + TargetTableStrategy: "existing_only", + AutoAddColumns: true, + } + + plan, sourceCols, targetCols, err := buildSchemaMigrationPlan(cfg, "users", sourceDB, targetDB) + if err != nil { + t.Fatalf("buildSchemaMigrationPlan returned error: %v", err) + } + if len(sourceCols) != 3 || len(targetCols) != 2 { + t.Fatalf("unexpected source/target columns: %d / %d", len(sourceCols), len(targetCols)) + } + if !plan.TargetTableExists { + t.Fatalf("expected target table to exist") + } + if len(plan.PreDataSQL) != 1 { + t.Fatalf("expected one pre-data SQL statement, got=%v", plan.PreDataSQL) + } + if !strings.Contains(plan.PreDataSQL[0], "ALTER TABLE `app`.`users` ADD COLUMN `aaaa` varchar(255) NULL") { + t.Fatalf("unexpected add-column SQL: %v", plan.PreDataSQL) + } + if !strings.Contains(plan.PlannedAction, "补齐缺失字段(1)") { + t.Fatalf("unexpected planned action: %s", plan.PlannedAction) + } +} + func TestBuildMySQLToPGLikeCreateTablePlan_GeneratesPostgresDDL(t *testing.T) { t.Parallel()