🐛 fix(sync): 修正仅同步结构未生效

- 让已存在目标表场景复用通用补字段逻辑生成结构变更 SQL
- 为分析与预览结果补充结构差异计数与结构 SQL 明细
- 补充结构同步回归测试并更新 backlog 记录

Fixes #342
This commit is contained in:
Syngnat
2026-04-17 12:35:23 +08:00
parent 890478eb7b
commit 384aea132c
6 changed files with 221 additions and 33 deletions

View File

@@ -33,6 +33,7 @@
| #333 | AI 功能添加供应商测试正常,但问答显示失败 | Fixed | Pending |
| #337 | 自动更新无效 | Fixed | Pending |
| #338 | 连接clickhouse不能通过8132端口 | Fixed | Pending |
| #342 | 数据同步功能不能用mysql数据库8.4版本选了结构同步,最后没同步成功 | Fixed | Pending |
| #351 | 为什么没有截断和清空表的功能呀? | Fixed | Pending |
## Notes
@@ -109,6 +110,12 @@
- 处理:将 `8132` 纳入 ClickHouse HTTP 端口识别,并同步更新自动切换日志和错误提示中的端口说明,避免排障信息继续误导。
- 验证:补充 `internal/db/clickhouse_impl_test.go` 回归测试,覆盖 `8132``8123``8443` 的 HTTP 判定以及 `9000/9440` 的 native 判定,并执行 `go test -tags gonavi_clickhouse_driver ./internal/db -run 'TestClickHouse(PingValidatesQueryPath|GetDatabasesFallsBackToCurrentDatabase|DetectClickHouseProtocolTreatsHTTPPortsAsHTTP)' -count=1`
### #342
- 根因:结构同步现有执行链路统一依赖 `buildSchemaMigrationPlan(...).PreDataSQL`。但 legacy planner 在“目标表已存在”分支里只给 `MySQL -> Kingbase` 生成补字段 SQL`MySQL -> MySQL` 即使目标表缺列也只记 warning不会产生任何可执行结构变更同时前端 schema 模式的预览入口完全按数据差异计数启用,导致结构同步场景无法点开预览。
- 处理:将 existing-target 分支的自动补字段逻辑改为复用通用 `buildAddColumnSQLForPair`,让 `MySQL -> MySQL` 也能生成并执行缺失字段补齐 SQL同时为 analyze/preview 响应补充 `schemaDiffCount``schemaStatements``schemaSummary` 和 warning 信息,前端 schema 模式下可直接查看结构变更语句与风险提示SQL 预览也会包含结构语句。
- 验证:新增 `internal/sync/schema_migration_test.go` 回归测试,覆盖 `MySQL -> MySQL` 已存在目标表时生成补字段 SQL并执行 `go test ./internal/sync -count=1``frontend``npm run build`
### #330
- 根因:查询结果表格已经支持拖拽调整列宽,但 resize handle 没有提供双击自适应逻辑,导致用户只能靠手工拖拽慢慢试宽度。

View File

@@ -24,6 +24,7 @@ type TableDiffSummary = {
updates?: number;
deletes?: number;
same?: number;
schemaDiffCount?: number;
message?: string;
targetTableExists?: boolean;
plannedAction?: string;
@@ -123,6 +124,15 @@ const buildSqlPreview = (
? previewData.columnTypes as Record<string, string>
: {};
const statements: string[] = [];
const schemaStatements = Array.isArray(previewData.schemaStatements)
? previewData.schemaStatements
.map((item: any) => String(item || '').trim())
.filter((item: string) => item.length > 0)
: [];
schemaStatements.forEach((statement: string) => {
statements.push(statement.endsWith(';') ? statement : `${statement};`);
});
const insertRows = Array.isArray(previewData.inserts) ? previewData.inserts : [];
const updateRows = Array.isArray(previewData.updates) ? previewData.updates : [];
@@ -478,7 +488,7 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open,
sourceConfig: normalizeConnConfig(sConn, sourceDb),
targetConfig: normalizeConnConfig(tConn, targetDb),
tables: selectedTables,
content: "data",
content: syncContent,
mode: "insert_update",
autoAddColumns,
targetTableStrategy,
@@ -595,6 +605,18 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open,
const ops = tableOptions[previewTable] || { insert: true, update: true, delete: false };
return buildSqlPreview(previewData, previewTable, targetType, ops);
}, [previewData, previewTable, targetConnId, connections, tableOptions]);
const previewHasSchemaStatements = useMemo(
() => Array.isArray(previewData?.schemaStatements) && previewData.schemaStatements.length > 0,
[previewData],
);
const previewSchemaWarnings = useMemo(
() => Array.isArray(previewData?.schemaWarnings) ? previewData.schemaWarnings as string[] : [],
[previewData],
);
const previewHasDataDiff = useMemo(
() => Number(previewData?.totalInserts || 0) + Number(previewData?.totalUpdates || 0) + Number(previewData?.totalDeletes || 0) > 0,
[previewData],
);
const analysisWarnings = useMemo(() => {
const items: string[] = [];
@@ -1060,8 +1082,9 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open,
render: (_: any, r: any) => {
const can = !!r.canSync;
const hasDiff = Number(r.inserts || 0) + Number(r.updates || 0) + Number(r.deletes || 0) > 0;
const hasSchemaDiff = Number(r.schemaDiffCount || 0) > 0;
return (
<Button size="small" disabled={!can || !hasDiff || analyzing} onClick={() => openPreview(r.table)}>
<Button size="small" disabled={!can || !(hasDiff || hasSchemaDiff) || analyzing} onClick={() => openPreview(r.table)}>
</Button>
);
@@ -1168,12 +1191,59 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open,
<Alert
type="info"
showIcon
message={`插入 ${previewData.totalInserts || 0},更新 ${previewData.totalUpdates || 0},删除 ${previewData.totalDeletes || 0}(预览最多展示 200 条/类型)`}
message={
previewHasDataDiff
? `插入 ${previewData.totalInserts || 0},更新 ${previewData.totalUpdates || 0},删除 ${previewData.totalDeletes || 0}(预览最多展示 200 条/类型)`
: (previewData.schemaSummary || `检测到 ${previewSql.statementCount} 条结构变更语句`)
}
/>
{previewSchemaWarnings.length > 0 && (
<Alert
style={{ marginTop: 12 }}
type="warning"
showIcon
message="结构预览包含风险或降级项"
description={
<ul style={{ margin: 0, paddingLeft: 18 }}>
{previewSchemaWarnings.slice(0, 8).map((item) => <li key={item}>{item}</li>)}
{previewSchemaWarnings.length > 8 && <li> {previewSchemaWarnings.length - 8} </li>}
</ul>
}
/>
)}
<Divider />
<Tabs
items={[
{
...(previewHasSchemaStatements ? [{
key: 'schema',
label: `结构(${Array.isArray(previewData.schemaStatements) ? previewData.schemaStatements.length : 0})`,
children: (
<div>
<Text type="secondary">
{previewData.schemaSummary || '以下为本次结构同步计划执行的语句。'}
</Text>
<pre
style={{
marginTop: 8,
marginBottom: 0,
padding: 10,
border: '1px solid #f0f0f0',
borderRadius: 6,
background: '#fafafa',
maxHeight: 420,
overflow: 'auto',
whiteSpace: 'pre-wrap',
wordBreak: 'break-word'
}}
>
{Array.isArray(previewData.schemaStatements) && previewData.schemaStatements.length > 0
? previewData.schemaStatements.join('\n')
: '-- 当前表结构无可执行变更'}
</pre>
</div>
)
}] : []),
...(previewHasDataDiff ? [{
key: 'insert',
label: `插入(${previewData.totalInserts || 0})`,
children: (
@@ -1273,7 +1343,7 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open,
/>
</div>
)
},
}] : []),
{
key: 'sql',
label: `SQL(${previewSql.statementCount})`,
@@ -1282,10 +1352,18 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open,
<Alert
type="info"
showIcon
message="SQL 预览会按当前勾选的插入/更新/删除与行选择范围生成,用于审核确认。"
message={
previewHasDataDiff
? "SQL 预览会按当前勾选的插入/更新/删除与行选择范围生成,用于审核确认。"
: "SQL 预览展示将执行的结构变更语句,用于审核确认。"
}
/>
<div style={{ marginTop: 8, marginBottom: 8, display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
<Text type="secondary"> {previewSql.statementCount} 200 /</Text>
<Text type="secondary">
{previewHasDataDiff
? `${previewSql.statementCount} 条语句(预览数据最多 200 条/类型)`
: `${previewSql.statementCount} 条结构变更语句`}
</Text>
<Button
size="small"
disabled={!previewSql.sqlText}
@@ -1314,7 +1392,7 @@ const DataSyncModal: React.FC<{ open: boolean; onClose: () => void }> = ({ open,
wordBreak: 'break-word'
}}
>
{previewSql.sqlText || '-- 当前勾选范围下无 SQL 可预览'}
{previewSql.sqlText || (previewHasDataDiff ? '-- 当前勾选范围下无 SQL 可预览' : '-- 当前表结构无可执行变更')}
</pre>
</div>
)

View File

@@ -14,6 +14,7 @@ type TableDiffSummary struct {
Updates int `json:"updates"`
Deletes int `json:"deletes"`
Same int `json:"same"`
SchemaDiffCount int `json:"schemaDiffCount,omitempty"`
Message string `json:"message,omitempty"`
HasSchema bool `json:"hasSchema,omitempty"`
TargetTableExists bool `json:"targetTableExists,omitempty"`
@@ -109,6 +110,7 @@ func (s *SyncEngine) Analyze(config SyncConfig) SyncAnalyzeResult {
summary.UnsupportedObjects = append(summary.UnsupportedObjects, plan.UnsupportedObjects...)
summary.IndexesToCreate = plan.IndexesToCreate
summary.IndexesSkipped = plan.IndexesSkipped
summary.SchemaDiffCount = len(plan.PreDataSQL) + len(plan.PostDataSQL)
if !plan.TargetTableExists && !plan.AutoCreate {
summary.Message = firstNonEmpty(plan.PlannedAction, "目标表不存在,无法执行同步")
@@ -118,7 +120,11 @@ func (s *SyncEngine) Analyze(config SyncConfig) SyncAnalyzeResult {
if !syncData {
summary.CanSync = true
summary.Message = firstNonEmpty(plan.PlannedAction, "仅同步结构,未执行数据差异分析")
if summary.SchemaDiffCount > 0 {
summary.Message = firstNonEmpty(plan.PlannedAction, fmt.Sprintf("检测到 %d 条结构变更", summary.SchemaDiffCount))
} else {
summary.Message = firstNonEmpty(plan.PlannedAction, "仅同步结构,未执行数据差异分析")
}
result.Tables = append(result.Tables, summary)
return
}

View File

@@ -19,15 +19,18 @@ type PreviewUpdateRow struct {
}
type TableDiffPreview struct {
Table string `json:"table"`
PKColumn string `json:"pkColumn"`
ColumnTypes map[string]string `json:"columnTypes,omitempty"`
TotalInserts int `json:"totalInserts"`
TotalUpdates int `json:"totalUpdates"`
TotalDeletes int `json:"totalDeletes"`
Inserts []PreviewRow `json:"inserts"`
Updates []PreviewUpdateRow `json:"updates"`
Deletes []PreviewRow `json:"deletes"`
Table string `json:"table"`
PKColumn string `json:"pkColumn"`
ColumnTypes map[string]string `json:"columnTypes,omitempty"`
SchemaSummary string `json:"schemaSummary,omitempty"`
SchemaWarnings []string `json:"schemaWarnings,omitempty"`
SchemaStatements []string `json:"schemaStatements,omitempty"`
TotalInserts int `json:"totalInserts"`
TotalUpdates int `json:"totalUpdates"`
TotalDeletes int `json:"totalDeletes"`
Inserts []PreviewRow `json:"inserts"`
Updates []PreviewUpdateRow `json:"updates"`
Deletes []PreviewRow `json:"deletes"`
}
func (s *SyncEngine) Preview(config SyncConfig, tableName string, limit int) (TableDiffPreview, error) {
@@ -70,6 +73,19 @@ func (s *SyncEngine) Preview(config SyncConfig, tableName string, limit int) (Ta
if !plan.TargetTableExists && !plan.AutoCreate {
return TableDiffPreview{}, errors.New(firstNonEmpty(plan.PlannedAction, "目标表不存在,无法预览差异"))
}
schemaStatements := make([]string, 0, len(plan.PreDataSQL)+len(plan.PostDataSQL))
schemaStatements = append(schemaStatements, plan.PreDataSQL...)
schemaStatements = append(schemaStatements, plan.PostDataSQL...)
contentRaw := strings.ToLower(strings.TrimSpace(config.Content))
if contentRaw == "schema" {
return TableDiffPreview{
Table: tableName,
SchemaSummary: firstNonEmpty(plan.PlannedAction, "仅同步结构"),
SchemaWarnings: append([]string(nil), plan.Warnings...),
SchemaStatements: append([]string(nil), schemaStatements...),
}, nil
}
pkCols := make([]string, 0, 2)
for _, c := range cols {
@@ -111,15 +127,18 @@ func (s *SyncEngine) Preview(config SyncConfig, tableName string, limit int) (Ta
}
out := TableDiffPreview{
Table: tableName,
PKColumn: pkCol,
ColumnTypes: make(map[string]string, len(cols)),
TotalInserts: 0,
TotalUpdates: 0,
TotalDeletes: 0,
Inserts: make([]PreviewRow, 0),
Updates: make([]PreviewUpdateRow, 0),
Deletes: make([]PreviewRow, 0),
Table: tableName,
PKColumn: pkCol,
ColumnTypes: make(map[string]string, len(cols)),
SchemaSummary: firstNonEmpty(plan.PlannedAction, "结构预览"),
SchemaWarnings: append([]string(nil), plan.Warnings...),
SchemaStatements: append([]string(nil), schemaStatements...),
TotalInserts: 0,
TotalUpdates: 0,
TotalDeletes: 0,
Inserts: make([]PreviewRow, 0),
Updates: make([]PreviewUpdateRow, 0),
Deletes: make([]PreviewRow, 0),
}
for _, col := range cols {
name := strings.ToLower(strings.TrimSpace(col.Name))

View File

@@ -127,12 +127,42 @@ func buildSchemaMigrationPlanLegacy(config SyncConfig, tableName string, sourceD
if len(missing) > 0 {
plan.Warnings = append(plan.Warnings, fmt.Sprintf("目标表缺失字段 %d 个:%s", len(missing), strings.Join(missing, ", ")))
}
if config.AutoAddColumns && isMySQLLikeSourceType(sourceType) && normalizeMigrationDBType(targetType) == "kingbase" {
addSQL, addWarnings := buildMySQLToKingbaseAddColumnSQL(plan.TargetQueryTable, sourceCols, targetCols)
plan.PreDataSQL = append(plan.PreDataSQL, addSQL...)
plan.Warnings = append(plan.Warnings, addWarnings...)
if len(addSQL) > 0 {
plan.PlannedAction = fmt.Sprintf("补齐缺失字段(%d)后导入", len(addSQL))
if len(missing) == 0 {
plan.PlannedAction = "表结构已一致"
} else if config.AutoAddColumns && supportsAutoAddColumnsForPair(sourceType, targetType) {
targetSet := make(map[string]struct{}, len(targetCols))
for _, col := range targetCols {
key := strings.ToLower(strings.TrimSpace(col.Name))
if key == "" {
continue
}
targetSet[key] = struct{}{}
}
for _, col := range sourceCols {
key := strings.ToLower(strings.TrimSpace(col.Name))
if key == "" {
continue
}
if _, ok := targetSet[key]; ok {
continue
}
addSQL, err := buildAddColumnSQLForPair(sourceType, targetType, plan.TargetQueryTable, col)
if err != nil {
plan.Warnings = append(plan.Warnings, fmt.Sprintf("字段 %s 自动补齐 SQL 生成失败:%v", col.Name, err))
continue
}
plan.PreDataSQL = append(plan.PreDataSQL, addSQL)
}
if len(plan.PreDataSQL) > 0 {
plan.PlannedAction = fmt.Sprintf("补齐缺失字段(%d)后导入", len(plan.PreDataSQL))
} else {
plan.PlannedAction = fmt.Sprintf("目标表缺失字段(%d),但未生成可执行补齐 SQL", len(missing))
}
} else {
if config.AutoAddColumns {
plan.PlannedAction = fmt.Sprintf("目标表缺失字段(%d),当前库对暂不支持自动补齐", len(missing))
} else {
plan.PlannedAction = fmt.Sprintf("目标表缺失字段(%d),未开启自动补齐", len(missing))
}
}
if strategy != "existing_only" {

View File

@@ -266,6 +266,54 @@ func TestBuildPGLikeToMySQLPlan_AutoCreateWhenTargetMissing(t *testing.T) {
}
}
func TestBuildSchemaMigrationPlan_MySQLToMySQLAddsMissingColumnsForExistingTarget(t *testing.T) {
t.Parallel()
sourceDB := &fakeMigrationDB{
columns: map[string][]connection.ColumnDefinition{
"shop.users": {
{Name: "id", Type: "bigint", Nullable: "NO", Key: "PRI"},
{Name: "name", Type: "varchar(128)", Nullable: "YES"},
{Name: "aaaa", Type: "varchar(255)", Nullable: "YES"},
},
},
}
targetDB := &fakeMigrationDB{
columns: map[string][]connection.ColumnDefinition{
"app.users": {
{Name: "id", Type: "bigint", Nullable: "NO", Key: "PRI"},
{Name: "name", Type: "varchar(128)", Nullable: "YES"},
},
},
}
cfg := SyncConfig{
SourceConfig: connection.ConnectionConfig{Type: "mysql", Database: "shop"},
TargetConfig: connection.ConnectionConfig{Type: "mysql", Database: "app"},
TargetTableStrategy: "existing_only",
AutoAddColumns: true,
}
plan, sourceCols, targetCols, err := buildSchemaMigrationPlan(cfg, "users", sourceDB, targetDB)
if err != nil {
t.Fatalf("buildSchemaMigrationPlan returned error: %v", err)
}
if len(sourceCols) != 3 || len(targetCols) != 2 {
t.Fatalf("unexpected source/target columns: %d / %d", len(sourceCols), len(targetCols))
}
if !plan.TargetTableExists {
t.Fatalf("expected target table to exist")
}
if len(plan.PreDataSQL) != 1 {
t.Fatalf("expected one pre-data SQL statement, got=%v", plan.PreDataSQL)
}
if !strings.Contains(plan.PreDataSQL[0], "ALTER TABLE `app`.`users` ADD COLUMN `aaaa` varchar(255) NULL") {
t.Fatalf("unexpected add-column SQL: %v", plan.PreDataSQL)
}
if !strings.Contains(plan.PlannedAction, "补齐缺失字段(1)") {
t.Fatalf("unexpected planned action: %s", plan.PlannedAction)
}
}
func TestBuildMySQLToPGLikeCreateTablePlan_GeneratesPostgresDDL(t *testing.T) {
t.Parallel()