🐛 fix(sync): 明确未迁移索引并提供补救 DDL (#885)

Fixes #885
This commit is contained in:
mango
2026-08-09 15:48:34 +08:00
parent f887deb44c
commit bb882faac7
17 changed files with 1053 additions and 237 deletions

View File

@@ -2,14 +2,64 @@ import React from 'react';
import TestRenderer, { act } from 'react-test-renderer';
import { afterEach, describe, expect, it, vi } from 'vitest';
const runtimeApi = vi.hoisted(() => ({
ClipboardSetText: vi.fn(() => Promise.resolve(false)),
}));
vi.mock('../../../wailsjs/runtime/runtime', () => runtimeApi);
import { DataSyncPreflightPanel } from './DataSyncPreflightPanel';
Object.assign(globalThis, {
navigator: {
clipboard: {
writeText: vi.fn(() => Promise.resolve()),
},
},
});
import {
createDataSyncWorkbenchTranslate,
dataSyncValidationIssueText,
} from './text';
const unmigratedIndexSnapshot = () => ({
taskId: 'task-1',
taskRevision: 3,
status: 'warning' as const,
issues: [
{
id: 'unmigrated_index:map-1:0',
code: 'unmigrated_index' as const,
severity: 'warning' as const,
stage: 'mappings' as const,
mappingId: 'map-1',
message: 'review remediation',
detail: {
unmigratedIndex: {
name: 'idx_name_prefix',
columns: [{ name: 'name', prefixLength: 12 }],
unique: false,
indexType: 'BTREE',
reason: 'review remediation',
remediationStatements: [
'CREATE INDEX idx_name_prefix ON public.users (left(name, 12))',
],
},
},
},
],
definitionHash: 'hash-1',
approvalRequired: false,
approvalSatisfied: false,
checkedAt: '2030-08-08T00:00:00.000Z',
});
afterEach(() => {
vi.useRealTimers();
runtimeApi.ClipboardSetText.mockReset();
runtimeApi.ClipboardSetText.mockResolvedValue(false);
(navigator.clipboard.writeText as ReturnType<typeof vi.fn>).mockReset();
(navigator.clipboard.writeText as ReturnType<typeof vi.fn>).mockResolvedValue(undefined);
});
describe('DataSyncPreflightPanel production approval', () => {
@@ -73,6 +123,64 @@ describe('DataSyncPreflightPanel production approval', () => {
);
});
it('renders unmigrated indexes and copies remediation DDL', async () => {
const writeText = navigator.clipboard.writeText as ReturnType<typeof vi.fn>;
const renderer = TestRenderer.create(
<DataSyncPreflightPanel
snapshot={unmigratedIndexSnapshot()}
currentRevision={3}
stale={false}
running={false}
t={createDataSyncWorkbenchTranslate('en-US')}
onLocateIssue={() => undefined}
/>,
);
expect(JSON.stringify(renderer.toJSON())).toContain('Unmigrated indexes (1)');
expect(JSON.stringify(renderer.toJSON())).toContain('Columns: name(12)');
const copyButton = renderer.root
.findAllByType('button')
.find((button) => button.children.includes('Copy remediation DDL'))!;
await act(async () => {
await copyButton.props.onClick();
});
expect(runtimeApi.ClipboardSetText).toHaveBeenCalledWith(
'-- idx_name_prefix: review remediation\n\nCREATE INDEX idx_name_prefix ON public.users (left(name, 12));',
);
expect(writeText).toHaveBeenCalledWith(
'-- idx_name_prefix: review remediation\n\nCREATE INDEX idx_name_prefix ON public.users (left(name, 12));',
);
});
it('shows a visible error when both clipboard paths fail', async () => {
runtimeApi.ClipboardSetText.mockRejectedValueOnce(new Error('runtime unavailable'));
(navigator.clipboard.writeText as ReturnType<typeof vi.fn>).mockRejectedValueOnce(
new Error('clipboard denied'),
);
const renderer = TestRenderer.create(
<DataSyncPreflightPanel
snapshot={unmigratedIndexSnapshot()}
currentRevision={3}
stale={false}
running={false}
t={createDataSyncWorkbenchTranslate('en-US')}
onLocateIssue={() => undefined}
/>,
);
const copyButton = renderer.root
.findAllByType('button')
.find((button) => button.children.includes('Copy remediation DDL'))!;
await act(async () => {
await copyButton.props.onClick();
});
expect(JSON.stringify(renderer.toJSON())).toContain(
'Copy failed. Copy the DDL manually from the list below.',
);
});
it('uses the backend notBefore window and never creates a frontend-only approval', () => {
vi.useFakeTimers();
const now = Date.parse('2030-08-08T00:00:00.000Z');

View File

@@ -1,5 +1,7 @@
import React, { useEffect, useMemo, useState } from 'react';
import { ClipboardSetText } from '../../../wailsjs/runtime/runtime';
import type {
DataSyncPreflightSnapshot,
DataSyncApprovalChallenge,
@@ -7,6 +9,11 @@ import type {
DataSyncTaskStage,
DataSyncValidationIssue,
} from './model';
import {
buildDataSyncPreflightRemediationSQL,
collectDataSyncPreflightIndexes,
formatDataSyncPreflightIndexColumns,
} from './dataSyncPreflightIndexes';
import {
dataSyncValidationIssueText,
type DataSyncWorkbenchTranslate,
@@ -69,6 +76,7 @@ export const DataSyncPreflightPanel: React.FC<{
onApprove,
}) => {
const [clock, setClock] = useState(Date.now());
const [copyError, setCopyError] = useState(false);
const effectiveIssues = stale ? [] : snapshot?.issues || [];
const approvalRequired = Boolean(snapshot && snapshot.approvalRequired !== false);
const approvalCurrent = Boolean(
@@ -93,6 +101,29 @@ export const DataSyncPreflightPanel: React.FC<{
: stale
? 'stale'
: snapshot?.status || 'stale';
const unmigratedIndexes = useMemo(
() => collectDataSyncPreflightIndexes(effectiveIssues),
[effectiveIssues],
);
const remediationSQL = useMemo(
() => buildDataSyncPreflightRemediationSQL(unmigratedIndexes),
[unmigratedIndexes],
);
const copyRemediationSQL = async () => {
if (!remediationSQL) return;
setCopyError(false);
try {
if (await ClipboardSetText(remediationSQL)) return;
} catch {
// Fall back to the browser clipboard when the Wails runtime is unavailable.
}
try {
await navigator.clipboard.writeText(remediationSQL);
} catch {
setCopyError(true);
}
};
useEffect(() => {
setClock(Date.now());
@@ -183,25 +214,69 @@ export const DataSyncPreflightPanel: React.FC<{
<p>{running ? t('preflight.running') : t('preflight.empty')}</p>
</div>
) : (
<ol className="gn-data-sync-issue-list">
{effectiveIssues.map((issue) => (
<li key={issue.id} data-severity={issue.severity}>
<div>
<span className="gn-data-sync-issue-list__severity">
{t(`preflight.severity.${issue.severity}`)}
</span>
<p title={issue.message || undefined}>{issueText(issue, t)}</p>
<>
<ol className="gn-data-sync-issue-list">
{effectiveIssues.map((issue) => (
<li key={issue.id} data-severity={issue.severity}>
<div>
<span className="gn-data-sync-issue-list__severity">
{t(`preflight.severity.${issue.severity}`)}
</span>
<p title={issue.message || undefined}>{issueText(issue, t)}</p>
</div>
<button
type="button"
className="gn-data-sync-link-button"
onClick={() => onLocateIssue(issue.stage, issue.mappingId)}
>
{t('preflight.open_issue')}
</button>
</li>
))}
</ol>
{unmigratedIndexes.length > 0 ? (
<section className="gn-data-sync-index-remediation">
<div className="gn-data-sync-index-remediation__header">
<strong>
{t('preflight.index_remediation.title', {
count: unmigratedIndexes.length,
})}
</strong>
<button
type="button"
className="gn-data-sync-button"
disabled={!remediationSQL}
onClick={() => void copyRemediationSQL()}
>
{t('preflight.index_remediation.copy')}
</button>
</div>
<button
type="button"
className="gn-data-sync-link-button"
onClick={() => onLocateIssue(issue.stage, issue.mappingId)}
>
{t('preflight.open_issue')}
</button>
</li>
))}
</ol>
{copyError ? (
<p className="gn-data-sync-index-remediation__error" role="alert">
{t('preflight.index_remediation.copy_failed')}
</p>
) : null}
<ol className="gn-data-sync-summary-list">
{unmigratedIndexes.map((index, itemIndex) => {
const remediationStatements = index.remediationStatements || [];
return (
<li key={`${index.name}:${index.mappingId || ''}:${itemIndex}`}>
<strong>{index.name}</strong>
<span>{index.indexType || 'BTREE'}</span>
<p>{t('preflight.index_remediation.columns', {
columns: formatDataSyncPreflightIndexColumns(index.columns),
})}</p>
<p>{index.reason}</p>
{remediationStatements.length > 0 ? (
<pre>{remediationStatements.join('\n')}</pre>
) : null}
</li>
);
})}
</ol>
</section>
) : null}
</>
)}
</aside>
);

View File

@@ -1673,6 +1673,44 @@ textarea.gn-data-sync-control {
.gn-data-sync-preflight-checklist p { margin: 0; }
.gn-data-sync-index-remediation {
margin-top: 14px;
padding-top: 12px;
border-top: 0.5px solid var(--gn-br-2, rgba(15, 23, 42, 0.12));
}
.gn-data-sync-index-remediation__header {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
margin-bottom: 10px;
}
.gn-data-sync-index-remediation__error {
margin: 0 0 8px;
color: var(--gn-danger, #dc2626);
font-size: var(--gn-font-size-xs, 11px);
}
.gn-data-sync-index-remediation .gn-data-sync-summary-list li {
padding: 10px 0;
border-bottom: 0.5px solid var(--gn-br-1, rgba(15, 23, 42, 0.08));
}
.gn-data-sync-index-remediation .gn-data-sync-summary-list p {
margin: 4px 0 0;
}
.gn-data-sync-index-remediation pre {
margin: 8px 0 0;
padding: 8px;
overflow: auto;
white-space: pre-wrap;
border-radius: 10px;
background: var(--gn-bg-soft, rgba(15, 23, 42, 0.04));
}
.gn-data-sync-action-bar {
display: flex;
min-height: 52px;

View File

@@ -0,0 +1,68 @@
import { describe, expect, it } from 'vitest';
import {
buildDataSyncPreflightRemediationSQL,
collectDataSyncPreflightIndexes,
formatDataSyncPreflightIndexColumns,
} from './dataSyncPreflightIndexes';
describe('data sync preflight indexes', () => {
it('collects only structured unmigrated index warnings and preserves prefixes', () => {
const indexes = collectDataSyncPreflightIndexes([
{
id: 'unmigrated_index:map-1:0',
code: 'unmigrated_index',
severity: 'warning',
stage: 'mappings',
mappingId: 'map-1',
message: 'review',
detail: {
unmigratedIndex: {
name: 'idx_lookup',
columns: [{ name: 'name', prefixLength: 12 }, { name: 'email' }],
unique: false,
indexType: 'BTREE',
reason: 'review',
remediationStatements: [],
},
},
},
]);
expect(indexes).toHaveLength(1);
expect(formatDataSyncPreflightIndexColumns(indexes[0].columns)).toBe('name(12), email');
});
it('keeps MongoDB remediation commands as valid JSON', () => {
const command = '{"createIndexes":"articles","indexes":[{"name":"idx_body","key":{"body":"text"},"unique":false}]}';
const text = buildDataSyncPreflightRemediationSQL([
{
name: 'idx_body',
columns: [{ name: 'body' }],
unique: false,
indexType: 'FULLTEXT',
reason: 'review text index semantics',
remediationStatements: [command],
},
]);
expect(text).toContain(`\n\n${command}`);
expect(text).not.toContain(`${command};`);
});
it('sanitizes multiline metadata before building copyable SQL', () => {
const sql = buildDataSyncPreflightRemediationSQL([
{
name: 'idx\nunsafe',
columns: [],
unique: false,
indexType: 'FULLTEXT',
reason: 'line one\nline two',
remediationStatements: ['CREATE INDEX idx ON target (body)'],
},
]);
expect(sql).toBe('-- idx unsafe: line one line two\n\nCREATE INDEX idx ON target (body);');
expect(sql).not.toContain('\n--');
});
});

View File

@@ -0,0 +1,55 @@
import type { DataSyncUnmigratedIndex, DataSyncValidationIssue } from './model';
export type DataSyncPreflightIndexItem = DataSyncUnmigratedIndex & {
mappingId?: string;
};
const singleLine = (value: string): string =>
String(value || '').replace(/[\r\n]+/g, ' ').replace(/\s+/g, ' ').trim();
export const collectDataSyncPreflightIndexes = (
issues: DataSyncValidationIssue[],
): DataSyncPreflightIndexItem[] =>
issues.flatMap((issue) => {
const index = issue.detail?.unmigratedIndex;
return issue.code === 'unmigrated_index' && index
? [{ ...index, columns: Array.isArray(index.columns) ? index.columns : [], mappingId: issue.mappingId }]
: [];
});
const copyableStatement = (value: string): string => {
const statement = value.trim();
try {
const parsed = JSON.parse(statement);
if (parsed && typeof parsed === 'object') return statement;
} catch {
// SQL statements are not JSON and use a trailing semicolon below.
}
return statement.endsWith(';') ? statement : `${statement};`;
};
export const buildDataSyncPreflightRemediationSQL = (
indexes: DataSyncPreflightIndexItem[],
): string =>
indexes
.flatMap((index) => {
const statements = (Array.isArray(index.remediationStatements) ? index.remediationStatements : [])
.map((statement) => statement.trim())
.filter(Boolean)
.map(copyableStatement);
return statements.length > 0
? [`-- ${singleLine(index.name)}: ${singleLine(index.reason)}`, ...statements]
: [];
})
.join('\n\n');
export const formatDataSyncPreflightIndexColumns = (
columns: DataSyncUnmigratedIndex['columns'],
): string =>
(Array.isArray(columns) ? columns : [])
.map((column) =>
column.prefixLength && column.prefixLength > 0
? `${column.name}(${column.prefixLength})`
: column.name,
)
.join(', ');

View File

@@ -226,8 +226,25 @@ export type DataSyncValidationCode =
| 'cdc_checkpoint_required'
| 'cdc_checkpoint_incompatible'
| 'compare_route_unsupported'
| 'index_inspection_failed'
| 'unmigrated_index'
| 'capability_unverified';
export type DataSyncIndexColumn = {
name: string;
prefixLength?: number;
};
export type DataSyncUnmigratedIndex = {
name: string;
columns: DataSyncIndexColumn[];
unique: boolean;
indexType: string;
reasonCode?: string;
reason: string;
remediationStatements?: string[];
};
export type DataSyncValidationIssue = {
id: string;
severity: DataSyncValidationSeverity;
@@ -235,6 +252,9 @@ export type DataSyncValidationIssue = {
stage: DataSyncTaskStage;
mappingId?: string;
message?: string;
detail?: {
unmigratedIndex?: DataSyncUnmigratedIndex;
};
};
export type DataSyncPreflightStatus =

View File

@@ -258,6 +258,10 @@ const zhCN = {
'preflight.severity.blocker': '阻断',
'preflight.severity.warning': '警告',
'preflight.severity.info': '提示',
'preflight.index_remediation.title': '未迁移索引({count}',
'preflight.index_remediation.columns': '列:{columns}',
'preflight.index_remediation.copy': '复制补救 DDL',
'preflight.index_remediation.copy_failed': '复制失败,请从下方清单手动复制 DDL。',
'runs.title': '运行记录',
'runs.subtitle': '运行实例由 gateway 提供;后续接入持久化历史与 checkpoint。',
'runs.empty_title': '还没有运行记录',
@@ -398,6 +402,8 @@ const zhCN = {
'validation.cdc_checkpoint_required': '选择 checkpoint 起点前必须存在兼容的持久化 checkpoint。',
'validation.cdc_checkpoint_incompatible': '已存在的 CDC checkpoint 与当前任务定义不兼容,请在暂停任务后显式重置。',
'validation.compare_route_unsupported': '当前源端与目标端组合不支持执行数据比对。',
'validation.index_inspection_failed': '无法读取源端索引元数据,不能确认所有索引都已迁移。',
'validation.unmigrated_index': '有索引无法自动迁移,请在执行前审核候选补救 DDL。',
'validation.capability_unverified': '当前使用静态 gateway运行前仍需后端确认数据源组合能力。',
'validation.unknown': '预检发现未识别的配置问题。',
} as const;
@@ -672,6 +678,10 @@ const enUS: Record<DataSyncWorkbenchTextKey, string> = {
'preflight.severity.blocker': 'Blocker',
'preflight.severity.warning': 'Warning',
'preflight.severity.info': 'Info',
'preflight.index_remediation.title': 'Unmigrated indexes ({count})',
'preflight.index_remediation.columns': 'Columns: {columns}',
'preflight.index_remediation.copy': 'Copy remediation DDL',
'preflight.index_remediation.copy_failed': 'Copy failed. Copy the DDL manually from the list below.',
'runs.title': 'Run history',
'runs.subtitle': 'Run instances come from the gateway; persisted history and checkpoints plug in later.',
'runs.empty_title': 'No runs yet',
@@ -812,6 +822,8 @@ const enUS: Record<DataSyncWorkbenchTextKey, string> = {
'validation.cdc_checkpoint_required': 'Checkpoint start requires a compatible durable checkpoint for this task.',
'validation.cdc_checkpoint_incompatible': 'The stored CDC checkpoint is incompatible with this task. Pause the task and reset it explicitly.',
'validation.compare_route_unsupported': 'The selected source-target route cannot execute compare tasks.',
'validation.index_inspection_failed': 'Source index metadata could not be read, so complete index migration cannot be verified.',
'validation.unmigrated_index': 'An index could not be migrated automatically. Review the remediation DDL before execution.',
'validation.capability_unverified': 'This task uses the static gateway. The backend must still verify the source-target capability before execution.',
'validation.unknown': 'Preflight found an unrecognized configuration issue.',
};

View File

@@ -322,6 +322,23 @@ describe('data sync Wails DTO boundary', () => {
stage: 'endpoints',
message: 'unsupported route',
},
{
code: 'unmigrated_index',
severity: 'warning',
stage: 'mappings',
message: 'index requires review',
detail: {
unmigratedIndex: {
name: 'idx_name_prefix',
columns: [{ name: 'name', prefixLength: 12 }],
unique: false,
indexType: 'BTREE',
reasonCode: 'prefix_index_requires_review',
reason: 'index requires review',
remediationStatements: ['CREATE INDEX idx_name_prefix ON public.users (left(name, 12))'],
},
},
},
],
checkedAt: Date.parse('2026-08-08T00:02:00.000Z'),
},
@@ -332,7 +349,18 @@ describe('data sync Wails DTO boundary', () => {
expect(blocked.snapshot).toMatchObject({
status: 'blocked',
approvalSatisfied: false,
issues: [{ message: 'unsupported route' }],
issues: [
{ message: 'unsupported route' },
{
detail: {
unmigratedIndex: {
name: 'idx_name_prefix',
reasonCode: 'prefix_index_requires_review',
remediationStatements: ['CREATE INDEX idx_name_prefix ON public.users (left(name, 12))'],
},
},
},
],
});
const earlyBlocked = decodeDataSyncPreflightQuery(
{

View File

@@ -7,6 +7,7 @@ import {
type DataSyncErrorRow,
type DataSyncFieldMetadata,
type DataSyncObjectMetadata,
type DataSyncIndexColumn,
type DataSyncPreflightSnapshot,
type DataSyncRouteCapability,
type DataSyncRunRecord,
@@ -15,6 +16,7 @@ import {
type DataSyncTableMapping,
type DataSyncTaskDefinition,
type DataSyncTaskLifecycle,
type DataSyncUnmigratedIndex,
type DataSyncValidationCode,
type DataSyncValidationIssue,
} from './model';
@@ -883,6 +885,14 @@ export const decodeDataSyncPreflight = (
),
mappingId: optionalString(issue.mappingId, 'issue.mappingId') || undefined,
message: optionalString(issue.message, 'issue.message') || undefined,
detail: isRecord(issue.detail) && issue.detail.unmigratedIndex
? {
unmigratedIndex: decodeUnmigratedIndex(
issue.detail.unmigratedIndex,
`DataSyncJobPreflight.data.issues[${index}].detail.unmigratedIndex`,
),
}
: undefined,
};
});
let capability: DataSyncRouteCapability;
@@ -1082,6 +1092,34 @@ const previewJSON = (value: unknown, limit = 480): string => {
return text.length > limit ? `${text.slice(0, limit)}` : text;
};
const decodeIndexColumns = (value: unknown, path: string): DataSyncIndexColumn[] =>
array(value, path).map((item, index) => {
const column = record(item, `${path}[${index}]`);
return {
name: string(column.name, `${path}[${index}].name`, false),
prefixLength:
optionalNumber(column.prefixLength, `${path}[${index}].prefixLength`) || undefined,
};
});
const decodeUnmigratedIndex = (value: unknown, path: string): DataSyncUnmigratedIndex => {
const index = record(value, path);
return {
name: string(index.name, `${path}.name`, false),
columns: decodeIndexColumns(index.columns || [], `${path}.columns`),
unique: boolean(index.unique, `${path}.unique`),
indexType: optionalString(index.indexType, `${path}.indexType`) || '',
reasonCode: optionalString(index.reasonCode, `${path}.reasonCode`) || undefined,
reason: string(index.reason, `${path}.reason`, false),
remediationStatements: array(
index.remediationStatements || [],
`${path}.remediationStatements`,
).map((statement, indexOffset) =>
string(statement, `${path}.remediationStatements[${indexOffset}]`, false),
),
};
};
export const decodeErrorRow = (value: unknown): DataSyncErrorRow => {
const row = record(value, 'errorRow');
const source = optionalString(row.sourceTable, 'errorRow.sourceTable');