From 390a2491050688a45d043b629ca8b953602ab672 Mon Sep 17 00:00:00 2001 From: Syngnat Date: Sat, 8 Aug 2026 20:51:23 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(data-import):=20=E9=87=8D?= =?UTF-8?q?=E6=9E=84=E5=A4=A7=E6=96=87=E4=BB=B6=E5=AF=BC=E5=85=A5=E4=B8=8E?= =?UTF-8?q?=E5=AE=89=E5=85=A8=E4=BB=BB=E5=8A=A1=E8=BF=90=E8=A1=8C=E6=97=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - SQL、CSV、JSON 与 XLSX 改为资源受限的流式解析，并支持压缩、编码和方言预检 - 增加遇错停止/继续策略、持久任务、错误行导出、源文件身份与安全取消 - 为数据库驱动补齐上下文事务和写入结果未知语义，避免失败重放与脏会话复用 --- internal/app/app.go | 51 + internal/app/data_import_capability.go | 214 ++ internal/app/data_import_capability_test.go | 455 +++++ internal/app/import_conflict_policy_test.go | 222 +++ internal/app/import_error_artifact.go | 173 ++ internal/app/import_error_artifact_session.go | 63 + .../app/import_error_artifact_session_test.go | 33 + internal/app/import_error_artifact_test.go | 106 + internal/app/import_job_identity.go | 100 + internal/app/import_job_identity_test.go | 50 + internal/app/import_job_lifecycle.go | 294 +++ internal/app/import_job_lifecycle_test.go | 290 +++ internal/app/import_lexical_limit_reader.go | 622 ++++++ .../app/import_lexical_limit_reader_test.go | 249 +++ internal/app/import_pipeline.go | 1027 +++++++++- internal/app/import_pipeline_parser_test.go | 716 +++++++ internal/app/import_source_identity.go | 97 + internal/app/import_source_identity_test.go | 55 + internal/app/import_task_runtime.go | 160 ++ internal/app/import_task_runtime_test.go | 154 ++ internal/app/import_text_source.go | 221 ++ internal/app/import_unknown_outcome_test.go | 100 + internal/app/methods_db.go | 2 +- .../app/methods_db_metadata_retry_test.go | 12 +- internal/app/methods_file.go | 1774 +++++++++++++++-- internal/app/methods_file_export_test.go | 7 + internal/app/methods_file_i18n_test.go | 4 + .../app/methods_file_import_managed_test.go | 299 +++ .../methods_file_import_registration_test.go | 77 + internal/app/methods_file_import_test.go | 415 +++- .../app/methods_file_sql_execution_test.go | 1320 +++++++++++- internal/app/methods_file_sql_managed_test.go | 308 +++ internal/app/methods_file_sql_source_test.go | 275 +++ internal/app/methods_import_artifacts.go | 75 + internal/app/methods_import_jobs.go | 121 ++ internal/app/methods_import_jobs_test.go | 169 ++ .../app/running_query_registration_test.go | 48 + internal/app/sql_file_redaction_test.go | 38 + internal/app/sql_import_compat_test.go | 139 ++ internal/app/sql_import_preflight.go | 371 ++++ internal/app/sql_import_preflight_test.go | 134 ++ internal/app/sql_import_source.go | 206 ++ internal/app/sql_import_source_test.go | 240 +++ internal/app/sql_split.go | 84 +- internal/app/sql_split_stream.go | 479 ++++- internal/app/sql_split_stream_test.go | 65 + internal/app/sql_split_test.go | 44 + internal/app/xlsx_import_stream.go | 780 ++++++-- internal/app/xlsx_import_stream_test.go | 403 +++- internal/db/dameng_impl.go | 21 +- internal/db/dameng_transaction_test.go | 65 +- internal/db/database.go | 16 + internal/db/mariadb_batch_capability_test.go | 22 + internal/db/mariadb_impl.go | 47 +- internal/db/mariadb_write_outcome_test.go | 45 + internal/db/mysql_connection_params_test.go | 9 + internal/db/mysql_impl.go | 68 +- internal/db/oceanbase_impl.go | 49 +- internal/db/oceanbase_impl_test.go | 131 +- internal/db/optional_driver_agent_impl.go | 33 +- internal/db/oracle_applychanges_test.go | 160 +- internal/db/oracle_impl.go | 30 +- .../db/postgres_applychanges_context_test.go | 94 + internal/db/postgres_impl.go | 23 +- internal/db/sqlite_impl.go | 23 +- internal/db/sqlserver_impl.go | 21 +- internal/db/sqlserver_impl_test.go | 102 +- internal/db/write_outcome.go | 103 + internal/db/write_outcome_transaction_test.go | 213 ++ internal/importjob/model.go | 61 + internal/importjob/store.go | 238 +++ internal/importjob/store_test.go | 176 ++ internal/importjob/validation.go | 24 + internal/webserver/server.go | 6 + internal/webserver/server_test.go | 3 +- shared/i18n/de-DE.json | 128 +- shared/i18n/en-US.json | 128 +- shared/i18n/ja-JP.json | 128 +- shared/i18n/ru-RU.json | 128 +- shared/i18n/zh-CN.json | 128 +- shared/i18n/zh-TW.json | 128 +- 81 files changed, 15267 insertions(+), 625 deletions(-) create mode 100644 internal/app/data_import_capability.go create mode 100644 internal/app/data_import_capability_test.go create mode 100644 internal/app/import_conflict_policy_test.go create mode 100644 internal/app/import_error_artifact.go create mode 100644 internal/app/import_error_artifact_session.go create mode 100644 internal/app/import_error_artifact_session_test.go create mode 100644 internal/app/import_error_artifact_test.go create mode 100644 internal/app/import_job_identity.go create mode 100644 internal/app/import_job_identity_test.go create mode 100644 internal/app/import_job_lifecycle.go create mode 100644 internal/app/import_job_lifecycle_test.go create mode 100644 internal/app/import_lexical_limit_reader.go create mode 100644 internal/app/import_lexical_limit_reader_test.go create mode 100644 internal/app/import_pipeline_parser_test.go create mode 100644 internal/app/import_source_identity.go create mode 100644 internal/app/import_source_identity_test.go create mode 100644 internal/app/import_task_runtime.go create mode 100644 internal/app/import_task_runtime_test.go create mode 100644 internal/app/import_text_source.go create mode 100644 internal/app/import_unknown_outcome_test.go create mode 100644 internal/app/methods_file_import_managed_test.go create mode 100644 internal/app/methods_file_import_registration_test.go create mode 100644 internal/app/methods_file_sql_managed_test.go create mode 100644 internal/app/methods_file_sql_source_test.go create mode 100644 internal/app/methods_import_artifacts.go create mode 100644 internal/app/methods_import_jobs.go create mode 100644 internal/app/methods_import_jobs_test.go create mode 100644 internal/app/running_query_registration_test.go create mode 100644 internal/app/sql_file_redaction_test.go create mode 100644 internal/app/sql_import_compat_test.go create mode 100644 internal/app/sql_import_preflight.go create mode 100644 internal/app/sql_import_preflight_test.go create mode 100644 internal/app/sql_import_source.go create mode 100644 internal/app/sql_import_source_test.go create mode 100644 internal/db/mariadb_batch_capability_test.go create mode 100644 internal/db/mariadb_write_outcome_test.go create mode 100644 internal/db/postgres_applychanges_context_test.go create mode 100644 internal/db/write_outcome.go create mode 100644 internal/db/write_outcome_transaction_test.go create mode 100644 internal/importjob/model.go create mode 100644 internal/importjob/store.go create mode 100644 internal/importjob/store_test.go create mode 100644 internal/importjob/validation.go diff --git a/internal/app/app.go b/internal/app/app.go index 0f9f3d9c..9131e914 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -18,6 +18,7 @@ import ( "GoNavi-Wails/internal/appdata" "GoNavi-Wails/internal/connection" "GoNavi-Wails/internal/db" + "GoNavi-Wails/internal/importjob" "GoNavi-Wails/internal/jvm" "GoNavi-Wails/internal/logger" nacosbackend "GoNavi-Wails/internal/nacos" @@ -179,6 +180,14 @@ type App struct { applicationQuitPromptInFlight bool queryMu sync.RWMutex nextQueryRegistrationID uint64 + importArtifactMu sync.Mutex + importErrorArtifacts *importErrorArtifactStore + importJobMu sync.Mutex + importJobStore *importjob.Store + importTaskMu sync.Mutex + importTasks map[string]importTaskRegistration + importTasksWG sync.WaitGroup + importTasksClosing bool dataRootApplyMu sync.Mutex configDir string secretStore secretstore.SecretStore @@ -259,6 +268,7 @@ func NewAppWithSecretStore(store secretstore.SecretStore) *App { connectFailures: make(map[string]cachedConnectFailure), dbConnectFlights: make(map[uint64]*databaseConnectFlight), runningQueries: make(map[string]queryContext), + importTasks: make(map[string]importTaskRegistration), sqlTransactions: make(map[string]*managedSQLTransaction), configDir: resolveAppConfigDir(), secretStore: store, @@ -403,6 +413,9 @@ func (a *App) startup(ctx context.Context) { if err := migrateDailySecretsIfNeeded(a); err != nil { logger.Warnf("迁移日常密文失败：%v", err) } + if err := a.recoverImportJobsOnStartup(); err != nil { + logger.Warnf("恢复导入任务状态失败：%v", err) + } a.loadPersistedGlobalProxy() if err := migrateLegacyWebKitStorageIfNeeded(a); err != nil { logger.Warnf("迁移旧 WebKit 连接存储失败：%v", err) @@ -486,6 +499,9 @@ func (a *App) Shutdown() { logger.Infof("应用开始关闭，准备释放资源") a.shutdownCloudBackup() a.shutdownDataSyncJobs() + if !a.cancelAndWaitImportTasks(5 * time.Second) { + logger.Warnf("导入任务未能在关闭超时内全部退出；将继续释放数据库资源") + } a.beginDatabaseShutdown() a.stopConnectionKeepAliveLoop() closeJVMMonitoringSessions() @@ -1806,6 +1822,41 @@ func (a *App) registerRunningQuery(queryID string, cancel context.CancelFunc, re } } +// registerExclusiveRunningQuery registers a long-running task only when the +// caller-provided ID is not already owned by another task. Import jobs use this +// stricter contract because replacing an owner would make cancellation target +// the wrong operation and let an older cleanup remove the newer task. +func (a *App) registerExclusiveRunningQuery(queryID string, cancel context.CancelFunc, retainUntilDone bool) (func(), bool) { + a.queryMu.Lock() + if a.runningQueries == nil { + a.runningQueries = make(map[string]queryContext) + } + if _, exists := a.runningQueries[queryID]; exists { + a.queryMu.Unlock() + return func() {}, false + } + a.nextQueryRegistrationID++ + if a.nextQueryRegistrationID == 0 { + a.nextQueryRegistrationID++ + } + registrationID := a.nextQueryRegistrationID + a.runningQueries[queryID] = queryContext{ + cancel: cancel, + started: time.Now(), + retainUntilDone: retainUntilDone, + registrationID: registrationID, + } + a.queryMu.Unlock() + + return func() { + a.queryMu.Lock() + if current, exists := a.runningQueries[queryID]; exists && current.registrationID == registrationID { + delete(a.runningQueries, queryID) + } + a.queryMu.Unlock() + }, true +} + // CancelQuery cancels a running query by its ID func (a *App) CancelQuery(queryID string) connection.QueryResult { a.queryMu.Lock() diff --git a/internal/app/data_import_capability.go b/internal/app/data_import_capability.go new file mode 100644 index 00000000..884b26be --- /dev/null +++ b/internal/app/data_import_capability.go @@ -0,0 +1,214 @@ +package app + +import ( + "GoNavi-Wails/internal/connection" + "GoNavi-Wails/internal/db" +) + +const ( + DataImportReasonDatabaseUnsupported = "database_type_unsupported" + DataImportReasonPinnedSessionUnavailable = "pinned_session_unavailable" + DataImportReasonRestricted = "data_import_restricted" + DataImportReasonRuntimeUnavailable = "database_runtime_unavailable" + DataImportReasonSQLFileRestricted = "sql_file_import_restricted" + DataImportReasonTableRuntimeUnavailable = "table_import_runtime_unavailable" +) + +// DataImportCapability is the backend-authoritative contract for import entry points. +type DataImportCapability struct { + DatabaseType string `json:"databaseType"` + TableImport DataImportModeCapability `json:"tableImport"` + SQLFileImport DataImportModeCapability `json:"sqlFileImport"` +} + +// DataImportModeCapability describes one import mode without requiring the +// frontend to infer support from a database type. +type DataImportModeCapability struct { + Supported bool `json:"supported"` + Reason string `json:"reason"` + RequiresPinnedSession bool `json:"requiresPinnedSession"` + SupportsTransactionalBatch bool `json:"supportsTransactionalBatch"` + SupportsContinue bool `json:"supportsContinue"` + SupportedFormats []string `json:"supportedFormats"` + SupportedEncodings []string `json:"supportedEncodings"` + SupportedCompressions []string `json:"supportedCompressions"` + SupportedClientDirectives []string `json:"supportedClientDirectives"` + SupportedConflictPolicies []string `json:"supportedConflictPolicies"` +} + +// DataImportCapability returns the effective import contract for the live +// driver instance. Connection failures are represented as a fail-closed DTO so +// callers never need database-type heuristics as a fallback. +func (a *App) DataImportCapability(config connection.ConnectionConfig) DataImportCapability { + if dataImportCapabilityRestricted(config) { + return ResolveDataImportCapability(config, nil) + } + runtime, err := a.getDatabase(config) + if err == nil { + return ResolveDataImportCapability(config, runtime) + } + + capability := ResolveDataImportCapability(config, nil) + if capability.TableImport.Reason == DataImportReasonTableRuntimeUnavailable { + capability.TableImport.Reason = DataImportReasonRuntimeUnavailable + } + if capability.SQLFileImport.Reason == DataImportReasonPinnedSessionUnavailable { + capability.SQLFileImport.Reason = DataImportReasonRuntimeUnavailable + } + return capability +} + +// ResolveDataImportCapability derives import support from both the normalized +// dialect and runtime interfaces. A dialect name alone never proves that SQL +// file execution can preserve session state. +func ResolveDataImportCapability(config connection.ConnectionConfig, runtime db.Database) DataImportCapability { + dbType := normalizeDataImportDatabaseType(config) + capability := DataImportCapability{ + DatabaseType: dbType, + TableImport: DataImportModeCapability{ + Reason: DataImportReasonDatabaseUnsupported, + SupportedFormats: []string{}, + SupportedEncodings: []string{}, + SupportedCompressions: []string{}, + SupportedClientDirectives: []string{}, + SupportedConflictPolicies: []string{}, + }, + SQLFileImport: DataImportModeCapability{ + Reason: DataImportReasonDatabaseUnsupported, + RequiresPinnedSession: true, + SupportedFormats: []string{}, + SupportedEncodings: []string{}, + SupportedCompressions: []string{}, + SupportedClientDirectives: []string{}, + SupportedConflictPolicies: []string{}, + }, + } + if dataImportCapabilityRestricted(config) { + capability.TableImport.Reason = DataImportReasonRestricted + capability.SQLFileImport.Reason = DataImportReasonRestricted + return capability + } + if dbType == "dameng" || dbType == "tdengine" || dbType == "clickhouse" { + capability.TableImport.Reason = DataImportReasonTableRuntimeUnavailable + if _, ok := runtime.(db.BatchApplierContext); ok { + capability.TableImport = DataImportModeCapability{ + Supported: true, + SupportsTransactionalBatch: dbType == "dameng", + SupportsContinue: true, + SupportedFormats: []string{"csv", "json", "xlsx"}, + SupportedEncodings: []string{"auto", "utf-8", "utf-16le", "utf-16be", "gb18030"}, + SupportedCompressions: []string{}, + SupportedClientDirectives: []string{}, + SupportedConflictPolicies: dataImportTableConflictPolicies(dbType), + } + } + if sqlFileImportCapabilityRestricted(config) { + capability.SQLFileImport.Reason = DataImportReasonSQLFileRestricted + } else { + capability.SQLFileImport.Reason = DataImportReasonPinnedSessionUnavailable + } + return capability + } + + if !isDataImportMySQLFamilyDialect(dbType) && dbType != "postgres" && dbType != "sqlite" && dbType != "oracle" && dbType != "sqlserver" { + return capability + } + capability.TableImport.Reason = DataImportReasonTableRuntimeUnavailable + if _, ok := runtime.(db.BatchApplierContext); ok { + capability.TableImport = DataImportModeCapability{ + Supported: true, + SupportsTransactionalBatch: dataImportTableSupportsTransactionalBatch(dbType), + SupportsContinue: true, + SupportedFormats: []string{"csv", "json", "xlsx"}, + SupportedEncodings: []string{"auto", "utf-8", "utf-16le", "utf-16be", "gb18030"}, + SupportedCompressions: []string{}, + SupportedClientDirectives: []string{}, + SupportedConflictPolicies: dataImportTableConflictPolicies(dbType), + } + } + if sqlFileImportCapabilityRestricted(config) { + capability.SQLFileImport.Reason = DataImportReasonSQLFileRestricted + return capability + } + if _, ok := runtime.(db.SessionExecerProvider); !ok { + capability.SQLFileImport.Reason = DataImportReasonPinnedSessionUnavailable + return capability + } + _, supportsBatch := runtime.(db.BatchWriteExecer) + if conditional, ok := runtime.(db.BatchWriteCapability); ok { + supportsBatch = supportsBatch && conditional.SupportsBatchWrites() + } + clientDirectives := []string{} + if isDataImportMySQLFamilyDialect(dbType) { + clientDirectives = []string{"delimiter"} + } else if dbType == "oracle" { + clientDirectives = []string{"sqlplus-slash"} + } else if dbType == "sqlserver" { + clientDirectives = []string{"go"} + } + capability.SQLFileImport = DataImportModeCapability{ + Supported: true, + RequiresPinnedSession: true, + SupportsTransactionalBatch: supportsBatch && dataImportTableSupportsTransactionalBatch(dbType), + SupportsContinue: true, + SupportedFormats: []string{"sql"}, + SupportedEncodings: []string{"auto", "utf-8", "utf-16le", "utf-16be"}, + SupportedCompressions: []string{"gzip"}, + SupportedClientDirectives: clientDirectives, + SupportedConflictPolicies: []string{}, + } + return capability +} + +func dataImportTableSupportsTransactionalBatch(dbType string) bool { + switch dbType { + case "mysql", "mariadb": + return false + default: + return true + } +} + +func dataImportTableConflictPolicies(dbType string) []string { + if dbType == "postgres" || dbType == "sqlite" { + return []string{"stop", "skip_duplicates", "upsert"} + } + if isDataImportMySQLFamilyDialect(dbType) { + return []string{"stop", "skip_duplicates"} + } + return []string{"stop"} +} + +func isDataImportMySQLFamilyDialect(dbType string) bool { + switch dbType { + case "mysql", "mariadb", "oceanbase": + return true + default: + return false + } +} + +func isDataImportSQLDialectSupported(config connection.ConnectionConfig) bool { + switch normalizeDataImportDatabaseType(config) { + case "mysql", "mariadb", "oceanbase", "postgres", "sqlite", "oracle", "sqlserver": + return true + default: + return false + } +} + +func sqlFileImportCapabilityRestricted(config connection.ConnectionConfig) bool { + return config.Protection.RestrictScriptExecution || config.Protection.RestrictStructureEdit +} + +func dataImportCapabilityRestricted(config connection.ConnectionConfig) bool { + protection := config.Protection + if protection.RestrictDataEdit || protection.RestrictStructureEdit || protection.RestrictScriptExecution || protection.RestrictDataImport { + return protection.RestrictDataImport + } + return config.ReadOnly +} + +func normalizeDataImportDatabaseType(config connection.ConnectionConfig) string { + return resolveDDLDBType(config) +} diff --git a/internal/app/data_import_capability_test.go b/internal/app/data_import_capability_test.go new file mode 100644 index 00000000..b20e5576 --- /dev/null +++ b/internal/app/data_import_capability_test.go @@ -0,0 +1,455 @@ +package app + +import ( + "context" + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + "GoNavi-Wails/internal/connection" + "GoNavi-Wails/internal/db" +) + +type fullyCapableImportDatabase struct { + db.Database +} + +type pinnedImportDatabase struct { + db.Database +} + +type tableImportDatabase struct { + db.Database +} + +type baseImportDatabase struct { + db.Database +} + +type legacyBatchImportDatabase struct { + db.Database +} + +func (*fullyCapableImportDatabase) ApplyChanges(string, connection.ChangeSet) error { + return nil +} + +func (*fullyCapableImportDatabase) ApplyChangesContext(context.Context, string, connection.ChangeSet) error { + return nil +} + +func (*fullyCapableImportDatabase) OpenSessionExecer(context.Context) (db.StatementExecer, error) { + return nil, nil +} + +func (*fullyCapableImportDatabase) ExecBatchContext(context.Context, string) (int64, error) { + return 0, nil +} + +func (*fullyCapableImportDatabase) SupportsBatchWrites() bool { + return true +} + +func (*pinnedImportDatabase) ApplyChanges(string, connection.ChangeSet) error { + return nil +} + +func (*pinnedImportDatabase) ApplyChangesContext(context.Context, string, connection.ChangeSet) error { + return nil +} + +func (*pinnedImportDatabase) OpenSessionExecer(context.Context) (db.StatementExecer, error) { + return nil, nil +} + +func (*tableImportDatabase) ApplyChanges(string, connection.ChangeSet) error { + return nil +} + +func (*tableImportDatabase) ApplyChangesContext(context.Context, string, connection.ChangeSet) error { + return nil +} + +func (*legacyBatchImportDatabase) ApplyChanges(string, connection.ChangeSet) error { + return nil +} + +func TestResolveDataImportCapabilityRejectsLegacyNonCancellableBatchRuntime(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: "mysql"}, + &legacyBatchImportDatabase{}, + ) + + if got.TableImport.Supported { + t.Fatalf("legacy non-cancellable table runtime must fail closed: %#v", got.TableImport) + } + if got.TableImport.Reason != DataImportReasonTableRuntimeUnavailable { + t.Fatalf("reason = %q, want %q", got.TableImport.Reason, DataImportReasonTableRuntimeUnavailable) + } +} + +func TestResolveDataImportCapabilityMySQL(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: "mysql"}, + &fullyCapableImportDatabase{}, + ) + + want := DataImportCapability{ + DatabaseType: "mysql", + TableImport: DataImportModeCapability{ + Supported: true, + RequiresPinnedSession: false, + SupportsTransactionalBatch: false, + SupportsContinue: true, + SupportedFormats: []string{"csv", "json", "xlsx"}, + SupportedEncodings: []string{"auto", "utf-8", "utf-16le", "utf-16be", "gb18030"}, + SupportedCompressions: []string{}, + SupportedClientDirectives: []string{}, + SupportedConflictPolicies: []string{"stop", "skip_duplicates"}, + }, + SQLFileImport: DataImportModeCapability{ + Supported: true, + RequiresPinnedSession: true, + SupportsTransactionalBatch: false, + SupportsContinue: true, + SupportedFormats: []string{"sql"}, + SupportedEncodings: []string{"auto", "utf-8", "utf-16le", "utf-16be"}, + SupportedCompressions: []string{"gzip"}, + SupportedClientDirectives: []string{"delimiter"}, + SupportedConflictPolicies: []string{}, + }, + } + + if !reflect.DeepEqual(got, want) { + t.Fatalf("capability = %#v, want %#v", got, want) + } +} + +func TestResolveDataImportCapabilityMySQLCompatibleDialects(t *testing.T) { + testCases := []struct { + name string + config connection.ConnectionConfig + wantDBType string + }{ + {name: "MariaDB", config: connection.ConnectionConfig{Type: "mariadb"}, wantDBType: "mariadb"}, + {name: "OceanBase MySQL", config: connection.ConnectionConfig{Type: "oceanbase"}, wantDBType: "oceanbase"}, + {name: "custom GreatDB", config: connection.ConnectionConfig{Type: "custom", Driver: "greatdb"}, wantDBType: "mysql"}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + got := ResolveDataImportCapability(testCase.config, &fullyCapableImportDatabase{}) + if got.DatabaseType != testCase.wantDBType { + t.Fatalf("database type = %q, want %q", got.DatabaseType, testCase.wantDBType) + } + if !got.TableImport.Supported || !got.SQLFileImport.Supported { + t.Fatalf("MySQL-compatible import capability = %#v", got) + } + if !reflect.DeepEqual(got.TableImport.SupportedConflictPolicies, []string{"stop", "skip_duplicates"}) { + t.Fatalf("conflict policies = %#v", got.TableImport.SupportedConflictPolicies) + } + if !reflect.DeepEqual(got.SQLFileImport.SupportedClientDirectives, []string{"delimiter"}) { + t.Fatalf("client directives = %#v", got.SQLFileImport.SupportedClientDirectives) + } + }) + } +} + +func TestResolveDataImportCapabilityOceanBaseOracleUsesOracleDialect(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: "oceanbase", OceanBaseProtocol: "oracle"}, + &fullyCapableImportDatabase{}, + ) + + if got.DatabaseType != "oracle" || !got.SQLFileImport.Supported { + t.Fatalf("OceanBase Oracle capability = %#v", got) + } + if !reflect.DeepEqual(got.SQLFileImport.SupportedClientDirectives, []string{"sqlplus-slash"}) { + t.Fatalf("client directives = %#v", got.SQLFileImport.SupportedClientDirectives) + } +} + +func TestResolveDataImportCapabilityPostgres(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: "postgresql"}, + &fullyCapableImportDatabase{}, + ) + + if !got.TableImport.Supported || !got.TableImport.SupportsTransactionalBatch { + t.Fatalf("table import capability = %#v", got.TableImport) + } + if !got.SQLFileImport.Supported || !got.SQLFileImport.RequiresPinnedSession { + t.Fatalf("SQL file import capability = %#v", got.SQLFileImport) + } + if !got.SQLFileImport.SupportsTransactionalBatch || !got.SQLFileImport.SupportsContinue { + t.Fatalf("SQL file execution semantics = %#v", got.SQLFileImport) + } + if got.DatabaseType != "postgres" { + t.Fatalf("database type = %q, want postgres", got.DatabaseType) + } + if len(got.SQLFileImport.SupportedClientDirectives) != 0 { + t.Fatalf("client directives = %#v, want none", got.SQLFileImport.SupportedClientDirectives) + } +} + +func TestResolveDataImportCapabilitySQLite(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: "sqlite"}, + &fullyCapableImportDatabase{}, + ) + + if !got.TableImport.Supported || !got.SQLFileImport.Supported { + t.Fatalf("SQLite import capability = %#v", got) + } + if !got.TableImport.SupportsTransactionalBatch || !got.SQLFileImport.SupportsTransactionalBatch { + t.Fatalf("SQLite transactional capability = %#v", got) + } + if !got.TableImport.SupportsContinue || !got.SQLFileImport.SupportsContinue { + t.Fatalf("SQLite continue capability = %#v", got) + } +} + +func TestResolveDataImportCapabilityReportsConflictPoliciesByMode(t *testing.T) { + testCases := []struct { + dbType string + want []string + }{ + {dbType: "mysql", want: []string{"stop", "skip_duplicates"}}, + {dbType: "postgres", want: []string{"stop", "skip_duplicates", "upsert"}}, + {dbType: "sqlite", want: []string{"stop", "skip_duplicates", "upsert"}}, + {dbType: "oracle", want: []string{"stop"}}, + {dbType: "sqlserver", want: []string{"stop"}}, + } + + for _, testCase := range testCases { + t.Run(testCase.dbType, func(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: testCase.dbType}, + &fullyCapableImportDatabase{}, + ) + + if !reflect.DeepEqual(got.TableImport.SupportedConflictPolicies, testCase.want) { + t.Fatalf("table conflict policies = %#v, want %#v", got.TableImport.SupportedConflictPolicies, testCase.want) + } + if !reflect.DeepEqual(got.SQLFileImport.SupportedConflictPolicies, []string{}) { + t.Fatalf("SQL conflict policies = %#v, want none", got.SQLFileImport.SupportedConflictPolicies) + } + }) + } +} + +func TestResolveDataImportCapabilityOracle(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: "oracle"}, + &pinnedImportDatabase{}, + ) + + if !got.TableImport.Supported || !got.TableImport.SupportsTransactionalBatch { + t.Fatalf("Oracle table import capability = %#v", got.TableImport) + } + if !got.SQLFileImport.Supported || !got.SQLFileImport.RequiresPinnedSession { + t.Fatalf("Oracle SQL file import capability = %#v", got.SQLFileImport) + } + if got.SQLFileImport.SupportsTransactionalBatch { + t.Fatalf("Oracle SQL file import must not claim batch support: %#v", got.SQLFileImport) + } + if !reflect.DeepEqual(got.SQLFileImport.SupportedClientDirectives, []string{"sqlplus-slash"}) { + t.Fatalf("client directives = %#v", got.SQLFileImport.SupportedClientDirectives) + } +} + +func TestResolveDataImportCapabilityDamengRejectsStatefulSQLWithoutPinnedSession(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: "dameng"}, + &tableImportDatabase{}, + ) + + if !got.TableImport.Supported || !got.TableImport.SupportsTransactionalBatch { + t.Fatalf("Dameng table import capability = %#v", got.TableImport) + } + if !got.TableImport.SupportsContinue { + t.Fatalf("Dameng table import must support continue mode: %#v", got.TableImport) + } + if !reflect.DeepEqual(got.TableImport.SupportedFormats, []string{"csv", "json", "xlsx"}) { + t.Fatalf("Dameng table formats = %#v", got.TableImport.SupportedFormats) + } + if !reflect.DeepEqual(got.TableImport.SupportedEncodings, []string{"auto", "utf-8", "utf-16le", "utf-16be", "gb18030"}) { + t.Fatalf("Dameng table encodings = %#v", got.TableImport.SupportedEncodings) + } + if !reflect.DeepEqual(got.TableImport.SupportedConflictPolicies, []string{"stop"}) { + t.Fatalf("Dameng conflict policies = %#v", got.TableImport.SupportedConflictPolicies) + } + if got.SQLFileImport.Supported { + t.Fatalf("Dameng SQL file import must fail closed: %#v", got.SQLFileImport) + } + if got.SQLFileImport.Reason != DataImportReasonPinnedSessionUnavailable { + t.Fatalf("reason = %q, want %q", got.SQLFileImport.Reason, DataImportReasonPinnedSessionUnavailable) + } +} + +func TestResolveDataImportCapabilityNonTransactionalDriversRejectSQLWithoutPinnedSession(t *testing.T) { + for _, dbType := range []string{"tdengine", "clickhouse"} { + t.Run(dbType, func(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: dbType}, + &tableImportDatabase{}, + ) + + if !got.TableImport.Supported || !got.TableImport.SupportsContinue { + t.Fatalf("table import capability = %#v", got.TableImport) + } + if got.TableImport.SupportsTransactionalBatch { + t.Fatalf("table import must not claim transactions: %#v", got.TableImport) + } + if !reflect.DeepEqual(got.TableImport.SupportedConflictPolicies, []string{"stop"}) { + t.Fatalf("table conflict policies = %#v", got.TableImport.SupportedConflictPolicies) + } + if got.SQLFileImport.Supported || got.SQLFileImport.Reason != DataImportReasonPinnedSessionUnavailable { + t.Fatalf("SQL file import capability = %#v", got.SQLFileImport) + } + }) + } +} + +func TestResolveDataImportCapabilityFailsClosedWhenPinnedSessionIsMissing(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: "mysql"}, + &tableImportDatabase{}, + ) + + if !got.TableImport.Supported { + t.Fatalf("table import capability = %#v", got.TableImport) + } + if got.SQLFileImport.Supported { + t.Fatalf("SQL file import must fail closed: %#v", got.SQLFileImport) + } + if got.SQLFileImport.Reason != DataImportReasonPinnedSessionUnavailable { + t.Fatalf("reason = %q, want %q", got.SQLFileImport.Reason, DataImportReasonPinnedSessionUnavailable) + } +} + +func TestResolveDataImportCapabilityRejectsUnknownDatabaseTypes(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: "future-db"}, + &fullyCapableImportDatabase{}, + ) + + if got.TableImport.Supported || got.SQLFileImport.Supported { + t.Fatalf("unknown database capability = %#v", got) + } + if got.TableImport.Reason != DataImportReasonDatabaseUnsupported { + t.Fatalf("table reason = %q", got.TableImport.Reason) + } + if got.SQLFileImport.Reason != DataImportReasonDatabaseUnsupported { + t.Fatalf("SQL reason = %q", got.SQLFileImport.Reason) + } + if !reflect.DeepEqual(got.TableImport.SupportedConflictPolicies, []string{}) || + !reflect.DeepEqual(got.SQLFileImport.SupportedConflictPolicies, []string{}) { + t.Fatalf("unsupported conflict policies must be empty: %#v", got) + } +} + +func TestResolveDataImportCapabilityHonorsDataImportProtection(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{ + Type: "mysql", + Protection: connection.ConnectionProtectionConfig{ + RestrictDataImport: true, + }, + }, + &fullyCapableImportDatabase{}, + ) + + if got.TableImport.Supported || got.SQLFileImport.Supported { + t.Fatalf("protected connection capability = %#v", got) + } + if got.TableImport.Reason != DataImportReasonRestricted || got.SQLFileImport.Reason != DataImportReasonRestricted { + t.Fatalf("protected connection reasons = %q, %q", got.TableImport.Reason, got.SQLFileImport.Reason) + } +} + +func TestResolveDataImportCapabilityExplainsMissingTableRuntime(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: "postgres"}, + &baseImportDatabase{}, + ) + + if got.TableImport.Supported { + t.Fatalf("table import must fail closed: %#v", got.TableImport) + } + if got.TableImport.Reason != DataImportReasonTableRuntimeUnavailable { + t.Fatalf("reason = %q, want %q", got.TableImport.Reason, DataImportReasonTableRuntimeUnavailable) + } +} + +func TestAppDataImportCapabilityUsesTheEffectiveRuntime(t *testing.T) { + application := NewApp() + config := connection.ConnectionConfig{Type: "mysql", Host: "db.local", Port: 3306} + application.dbCache[getCacheKey(config)] = cachedDatabase{ + inst: &fullyCapableImportDatabase{}, + lastPing: time.Now(), + config: normalizeCacheKeyConfig(config), + } + + got := application.DataImportCapability(config) + + if !got.TableImport.Supported || !got.SQLFileImport.Supported { + t.Fatalf("capability = %#v", got) + } +} + +func TestDataImportCapabilityJSONAlwaysIncludesReason(t *testing.T) { + capability := ResolveDataImportCapability( + connection.ConnectionConfig{Type: "mysql"}, + &fullyCapableImportDatabase{}, + ) + + raw, err := json.Marshal(capability) + if err != nil { + t.Fatalf("marshal capability: %v", err) + } + if count := strings.Count(string(raw), `"reason":""`); count != 2 { + t.Fatalf("reason field count = %d, JSON = %s", count, raw) + } + if count := strings.Count(string(raw), `"supportedConflictPolicies":`); count != 2 { + t.Fatalf("conflict policy field count = %d, JSON = %s", count, raw) + } + if strings.Contains(string(raw), `"supportedConflictPolicies":null`) { + t.Fatalf("conflict policy arrays must not be null, JSON = %s", raw) + } +} + +func TestResolveDataImportCapabilityRestrictsOnlySQLFileForScriptProtection(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{ + Type: "postgres", + Protection: connection.ConnectionProtectionConfig{ + RestrictScriptExecution: true, + }, + }, + &fullyCapableImportDatabase{}, + ) + + if !got.TableImport.Supported { + t.Fatalf("table import should remain available: %#v", got.TableImport) + } + if got.SQLFileImport.Supported || got.SQLFileImport.Reason != DataImportReasonSQLFileRestricted { + t.Fatalf("SQL file import capability = %#v", got.SQLFileImport) + } +} + +func TestResolveDataImportCapabilitySQLServerReportsGoDirective(t *testing.T) { + got := ResolveDataImportCapability( + connection.ConnectionConfig{Type: "mssql"}, + &fullyCapableImportDatabase{}, + ) + + if got.DatabaseType != "sqlserver" || !got.SQLFileImport.Supported { + t.Fatalf("SQL Server capability = %#v", got) + } + if !reflect.DeepEqual(got.SQLFileImport.SupportedClientDirectives, []string{"go"}) { + t.Fatalf("client directives = %#v", got.SQLFileImport.SupportedClientDirectives) + } +} diff --git a/internal/app/import_conflict_policy_test.go b/internal/app/import_conflict_policy_test.go new file mode 100644 index 00000000..c5de0cfb --- /dev/null +++ b/internal/app/import_conflict_policy_test.go @@ -0,0 +1,222 @@ +package app + +import ( + "errors" + "strings" + "testing" + + "GoNavi-Wails/internal/connection" + "GoNavi-Wails/internal/db" + + mysqlDriver "github.com/go-sql-driver/mysql" +) + +type conflictPolicyImportDB struct { + db.Database + affected int64 + err error + queries []string +} + +func (database *conflictPolicyImportDB) Exec(query string) (int64, error) { + database.queries = append(database.queries, query) + return database.affected, database.err +} + +func TestBuildImportInsertQueryWithConflictPostgresUpsert(t *testing.T) { + query, err := buildImportInsertQueryWithConflict( + "postgres", + "public.users", + []string{"id", "name"}, + map[string]interface{}{"id": 1, "name": "alice"}, + newImportColumnTypeLookup(nil), + importConflictPolicyUpsert, + []string{"id"}, + ) + if err != nil { + t.Fatal(err) + } + want := `INSERT INTO "public"."users" ("id", "name") VALUES (1, 'alice') ON CONFLICT ("id") DO UPDATE SET "name"=EXCLUDED."name"` + if query != want { + t.Fatalf("query = %q, want %q", query, want) + } +} + +func TestBuildImportInsertQueryWithConflictSQLiteSkipsDuplicates(t *testing.T) { + query, err := buildImportInsertQueryWithConflict( + "sqlite", + "users", + []string{"id"}, + map[string]interface{}{"id": 1}, + newImportColumnTypeLookup(nil), + importConflictPolicySkipDuplicates, + nil, + ) + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(query, " ON CONFLICT DO NOTHING") { + t.Fatalf("unexpected skip query: %q", query) + } +} + +func TestBuildImportInsertQueryWithConflictPreservesFractionalSeconds(t *testing.T) { + query, err := buildImportInsertQueryWithConflict( + "postgres", + "events", + []string{"id", "created_at", "local_time", "zoned_at"}, + map[string]interface{}{ + "id": 1, + "created_at": "2026-08-08 12:34:56.123456", + "local_time": "12:34:56.120000", + "zoned_at": "2026-08-08T12:34:56.123456+08:00", + }, + newImportColumnTypeLookup([]connection.ColumnDefinition{ + {Name: "id", Type: "bigint"}, + {Name: "created_at", Type: "timestamp(6)"}, + {Name: "local_time", Type: "time(6)"}, + {Name: "zoned_at", Type: "timestamp(6) with time zone"}, + }), + importConflictPolicyUpsert, + []string{"id"}, + ) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `'2026-08-08 12:34:56.123456'`, + `'12:34:56.120000'`, + `'2026-08-08 12:34:56.123456+08:00'`, + } { + if !strings.Contains(query, want) { + t.Fatalf("query lost temporal precision %s: %s", want, query) + } + } +} + +func TestBuildImportInsertQueryWithConflictMarshalsNestedJSONValues(t *testing.T) { + query, err := buildImportInsertQueryWithConflict( + "postgres", + "events", + []string{"id", "metadata", "tags"}, + map[string]interface{}{ + "id": 1, + "metadata": map[string]interface{}{ + "label": "O'Reilly", + "nested": map[string]interface{}{"a": 1}, + }, + "tags": []interface{}{"alpha", map[string]interface{}{"enabled": true}}, + }, + newImportColumnTypeLookup([]connection.ColumnDefinition{ + {Name: "id", Type: "bigint"}, + {Name: "metadata", Type: "jsonb"}, + {Name: "tags", Type: "jsonb"}, + }), + importConflictPolicyUpsert, + []string{"id"}, + ) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + `'{"label":"O''Reilly","nested":{"a":1}}'`, + `'["alpha",{"enabled":true}]'`, + } { + if !strings.Contains(query, want) { + t.Fatalf("query did not contain safely quoted JSON %s: %s", want, query) + } + } + if strings.Contains(query, "map[") { + t.Fatalf("nested JSON was formatted with fmt.Sprint: %s", query) + } +} + +func TestImportDatabaseRowWriterClassifiesMySQLDuplicateAsSkipped(t *testing.T) { + database := &conflictPolicyImportDB{err: &mysqlDriver.MySQLError{Number: 1062, Message: "Duplicate entry 'private' for key 'PRIMARY'"}} + writer := newImportDatabaseRowWriterWithOptions(database, "mysql", "users", newImportColumnTypeLookup(nil), ImportFileOptions{ + ConflictPolicy: importConflictPolicySkipDuplicates, + }) + writer.SetColumns([]string{"id"}) + outcome, err := writer.ApplyOneWithOutcome(map[string]interface{}{"id": 1}) + if err != nil { + t.Fatalf("duplicate should be skipped: %v", err) + } + if outcome != importRowApplySkipped { + t.Fatalf("outcome = %q, want skipped", outcome) + } + if writer.BatchEnabled() { + t.Fatal("conflict policies must not use ambiguous BatchApplier writes") + } +} + +func TestImportDatabaseRowWriterDoesNotHideNonDuplicateMySQLError(t *testing.T) { + database := &conflictPolicyImportDB{err: &mysqlDriver.MySQLError{Number: 1048, Message: "Column cannot be null"}} + writer := newImportDatabaseRowWriterWithOptions(database, "mysql", "users", newImportColumnTypeLookup(nil), ImportFileOptions{ + ConflictPolicy: importConflictPolicySkipDuplicates, + }) + writer.SetColumns([]string{"id"}) + if _, err := writer.ApplyOneWithOutcome(map[string]interface{}{"id": nil}); err == nil { + t.Fatal("non-duplicate database error must not be skipped") + } +} + +func TestImportBatchConsumerReportsSkippedRowsSeparately(t *testing.T) { + database := &conflictPolicyImportDB{affected: 0} + writer := newImportDatabaseRowWriterWithOptions(database, "sqlite", "users", newImportColumnTypeLookup(nil), ImportFileOptions{ + ConflictPolicy: importConflictPolicySkipDuplicates, + }) + consumer := newImportBatchConsumer(writer, 1000, 1, true, true, nil) + if err := consumer.SetColumns([]string{"id"}); err != nil { + t.Fatal(err) + } + if err := consumer.ConsumeRow(map[string]interface{}{"id": 1}); err != nil { + t.Fatal(err) + } + if err := consumer.Flush(); err != nil { + t.Fatal(err) + } + result := consumer.Result() + if result.Success != 0 || result.Skipped != 1 || result.Failed != 0 || result.Total != 1 { + t.Fatalf("unexpected result: %#v", result) + } +} + +func TestValidateImportConflictPolicyRejectsUnsupportedDialectAndMissingKeys(t *testing.T) { + if err := validateImportConflictPolicyForDB("oracle", ImportFileOptions{ConflictPolicy: importConflictPolicyUpsert, ConflictKeyColumns: []string{"id"}}); err == nil { + t.Fatal("Oracle upsert must fail closed until a safe implementation exists") + } + if err := validateImportConflictPolicyForDB("postgres", ImportFileOptions{ConflictPolicy: importConflictPolicyUpsert}); err == nil { + t.Fatal("upsert without conflict keys must fail") + } + if err := validateImportConflictPolicyForDB("mysql", ImportFileOptions{ConflictPolicy: importConflictPolicySkipDuplicates}); err != nil { + t.Fatalf("MySQL skip_duplicates should be supported: %v", err) + } + if err := validateImportConflictPolicyForDB("mysql", ImportFileOptions{ + ConflictPolicy: importConflictPolicyUpsert, + ConflictKeyColumns: []string{"id"}, + }); err == nil { + t.Fatal("MySQL upsert must fail closed because ON DUPLICATE KEY cannot target the selected conflict key") + } +} + +func TestImportDatabaseRowWriterRejectsMissingConflictKeyBeforeWriting(t *testing.T) { + database := &conflictPolicyImportDB{} + writer := newImportDatabaseRowWriterWithOptions(database, "postgres", "users", newImportColumnTypeLookup(nil), ImportFileOptions{ + ConflictPolicy: importConflictPolicyUpsert, + ConflictKeyColumns: []string{"id"}, + }) + consumer := newImportBatchConsumer(writer, 1000, 1, true, false, nil) + err := consumer.SetColumns([]string{"name"}) + if err == nil || !strings.Contains(err.Error(), "id") { + t.Fatalf("missing key error = %v", err) + } + if len(database.queries) != 0 { + t.Fatalf("database was written before key validation: %v", database.queries) + } +} + +func TestMySQLDuplicateClassifierDoesNotMatchPlainTextErrors(t *testing.T) { + if isMySQLDuplicateKeyError(errors.New("duplicate entry")) { + t.Fatal("untyped error text must not be trusted as a duplicate-key code") + } +} diff --git a/internal/app/import_error_artifact.go b/internal/app/import_error_artifact.go new file mode 100644 index 00000000..9b04843f --- /dev/null +++ b/internal/app/import_error_artifact.go @@ -0,0 +1,173 @@ +package app + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "GoNavi-Wails/internal/sqlaudit" + + "github.com/google/uuid" +) + +// ImportRowError is the durable, exportable representation of a rejected +// source row. Values are retained only in the user-local artifact; driver +// messages are sanitized before persistence. +type ImportRowError struct { + SourceRow int64 `json:"sourceRow"` + Line int64 `json:"line,omitempty"` + Byte int64 `json:"byteOffset,omitempty"` + Column string `json:"column,omitempty"` + Category string `json:"category"` + Code string `json:"code,omitempty"` + Message string `json:"message"` + Retryable bool `json:"retryable,omitempty"` + Values map[string]interface{} `json:"values,omitempty"` +} + +type ImportErrorArtifact struct { + ID string `json:"id"` + Count int64 `json:"count"` + CreatedAt int64 `json:"createdAt"` +} + +type importErrorArtifactStore struct { + root string +} + +type importErrorArtifactWriter struct { + store *importErrorArtifactStore + id string + path string + file *os.File + buffered *bufio.Writer + encoder *json.Encoder + count int64 + finished bool + createdAt int64 +} + +func newImportErrorArtifactStore(root string) (*importErrorArtifactStore, error) { + root = strings.TrimSpace(root) + if root == "" { + return nil, errors.New("import error artifact directory is empty") + } + absRoot, err := filepath.Abs(root) + if err != nil { + return nil, err + } + if err := os.MkdirAll(absRoot, 0o700); err != nil { + return nil, err + } + return &importErrorArtifactStore{root: absRoot}, nil +} + +func (s *importErrorArtifactStore) Begin(_ string) (*importErrorArtifactWriter, error) { + if s == nil || strings.TrimSpace(s.root) == "" { + return nil, errors.New("import error artifact store is unavailable") + } + id := uuid.NewString() + path := filepath.Join(s.root, id+".jsonl") + f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return nil, err + } + buffered := bufio.NewWriterSize(f, 64*1024) + return &importErrorArtifactWriter{ + store: s, + id: id, + path: path, + file: f, + buffered: buffered, + encoder: json.NewEncoder(buffered), + createdAt: time.Now().UnixMilli(), + }, nil +} + +func (w *importErrorArtifactWriter) Append(row ImportRowError) error { + if w == nil || w.finished || w.encoder == nil { + return errors.New("import error artifact writer is closed") + } + row.Category = strings.ToLower(strings.TrimSpace(row.Category)) + row.Code = strings.TrimSpace(row.Code) + row.Column = strings.TrimSpace(row.Column) + row.Message = sqlaudit.RedactError(row.Message) + if err := w.encoder.Encode(row); err != nil { + return err + } + w.count++ + return nil +} + +func (w *importErrorArtifactWriter) Finish() (ImportErrorArtifact, error) { + if w == nil || w.finished { + return ImportErrorArtifact{}, errors.New("import error artifact writer is closed") + } + w.finished = true + if err := w.buffered.Flush(); err != nil { + _ = w.file.Close() + _ = os.Remove(w.path) + return ImportErrorArtifact{}, err + } + if err := w.file.Sync(); err != nil { + _ = w.file.Close() + _ = os.Remove(w.path) + return ImportErrorArtifact{}, err + } + if err := w.file.Close(); err != nil { + _ = os.Remove(w.path) + return ImportErrorArtifact{}, err + } + return ImportErrorArtifact{ID: w.id, Count: w.count, CreatedAt: w.createdAt}, nil +} + +func (w *importErrorArtifactWriter) Abort() { + if w == nil || w.finished { + return + } + w.finished = true + if w.file != nil { + _ = w.file.Close() + } + if w.path != "" { + _ = os.Remove(w.path) + } +} + +func (s *importErrorArtifactStore) Open(id string) (*os.File, error) { + parsed, err := uuid.Parse(strings.TrimSpace(id)) + if err != nil || parsed.String() != strings.ToLower(strings.TrimSpace(id)) { + return nil, os.ErrNotExist + } + path := filepath.Join(s.root, parsed.String()+".jsonl") + f, err := os.Open(path) + if err != nil { + return nil, err + } + info, err := f.Stat() + if err != nil { + _ = f.Close() + return nil, err + } + if !info.Mode().IsRegular() { + _ = f.Close() + return nil, fmt.Errorf("import error artifact is not a regular file") + } + return f, nil +} + +func (s *importErrorArtifactStore) Delete(id string) error { + parsed, err := uuid.Parse(strings.TrimSpace(id)) + if err != nil || parsed.String() != strings.ToLower(strings.TrimSpace(id)) { + return os.ErrNotExist + } + if err := os.Remove(filepath.Join(s.root, parsed.String()+".jsonl")); err != nil { + return err + } + return nil +} diff --git a/internal/app/import_error_artifact_session.go b/internal/app/import_error_artifact_session.go new file mode 100644 index 00000000..67cc54c0 --- /dev/null +++ b/internal/app/import_error_artifact_session.go @@ -0,0 +1,63 @@ +package app + +import ( + "fmt" + "strings" +) + +type managedImportErrorArtifact struct { + writer *importErrorArtifactWriter + err error +} + +func (a *App) beginManagedImportErrorArtifact(jobID string) (*managedImportErrorArtifact, error) { + store, err := a.ensureImportErrorArtifactStore() + if err != nil { + return nil, err + } + writer, err := store.Begin(strings.TrimSpace(jobID)) + if err != nil { + return nil, err + } + return &managedImportErrorArtifact{writer: writer}, nil +} + +func (artifact *managedImportErrorArtifact) append(row ImportRowError) error { + if artifact == nil || artifact.writer == nil || artifact.err != nil { + if artifact == nil || artifact.err == nil { + return nil + } + return artifact.err + } + artifact.err = artifact.writer.Append(row) + return artifact.err +} + +func (artifact *managedImportErrorArtifact) finish(result *importExecutionResult) error { + if artifact == nil || artifact.writer == nil { + return nil + } + if artifact.err != nil { + artifact.writer.Abort() + return fmt.Errorf("persist rejected import row: %w", artifact.err) + } + if artifact.writer.count == 0 { + artifact.writer.Abort() + return nil + } + finished, err := artifact.writer.Finish() + if err != nil { + return err + } + if result != nil { + result.ErrorArtifactID = finished.ID + result.ErrorArtifactCount = finished.Count + } + return nil +} + +func (artifact *managedImportErrorArtifact) abort() { + if artifact != nil && artifact.writer != nil { + artifact.writer.Abort() + } +} diff --git a/internal/app/import_error_artifact_session_test.go b/internal/app/import_error_artifact_session_test.go new file mode 100644 index 00000000..3dcfc3ba --- /dev/null +++ b/internal/app/import_error_artifact_session_test.go @@ -0,0 +1,33 @@ +package app + +import "testing" + +func TestManagedImportErrorArtifactPublishesOnlyNonEmptyArtifacts(t *testing.T) { + app := NewApp() + app.configDir = t.TempDir() + + empty, err := app.beginManagedImportErrorArtifact("import-empty") + if err != nil { + t.Fatal(err) + } + emptyResult := importExecutionResult{} + if err := empty.finish(&emptyResult); err != nil { + t.Fatal(err) + } + if emptyResult.ErrorArtifactID != "" { + t.Fatalf("empty artifact was published: %#v", emptyResult) + } + + failed, err := app.beginManagedImportErrorArtifact("import-failed") + if err != nil { + t.Fatal(err) + } + failed.append(ImportRowError{SourceRow: 2, Category: "database", Message: "duplicate key"}) + failedResult := importExecutionResult{Failed: 1} + if err := failed.finish(&failedResult); err != nil { + t.Fatal(err) + } + if failedResult.ErrorArtifactID == "" || failedResult.ErrorArtifactCount != 1 { + t.Fatalf("rejected row artifact missing: %#v", failedResult) + } +} diff --git a/internal/app/import_error_artifact_test.go b/internal/app/import_error_artifact_test.go new file mode 100644 index 00000000..6287bce8 --- /dev/null +++ b/internal/app/import_error_artifact_test.go @@ -0,0 +1,106 @@ +package app + +import ( + "bufio" + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/wailsapp/wails/v2/pkg/runtime" +) + +func TestImportErrorArtifactStoresAllRowsBehindOpaqueID(t *testing.T) { + store, err := newImportErrorArtifactStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + w, err := store.Begin("import-job-1") + if err != nil { + t.Fatal(err) + } + for _, row := range []ImportRowError{ + {SourceRow: 2, Category: "constraint", Code: "duplicate", Message: "duplicate key value is (alice@example.com)", Values: map[string]interface{}{"id": 1}}, + {SourceRow: 9, Category: "validation", Code: "invalid_date", Message: "bad date", Values: map[string]interface{}{"id": 8}}, + } { + if err := w.Append(row); err != nil { + t.Fatal(err) + } + } + artifact, err := w.Finish() + if err != nil { + t.Fatal(err) + } + if artifact.ID == "" || artifact.Count != 2 { + t.Fatalf("unexpected artifact: %+v", artifact) + } + + f, err := store.Open(artifact.ID) + if err != nil { + t.Fatal(err) + } + defer f.Close() + scanner := bufio.NewScanner(f) + var rows []ImportRowError + for scanner.Scan() { + var row ImportRowError + if err := json.Unmarshal(scanner.Bytes(), &row); err != nil { + t.Fatal(err) + } + rows = append(rows, row) + } + if err := scanner.Err(); err != nil { + t.Fatal(err) + } + if len(rows) != 2 || rows[0].SourceRow != 2 || rows[1].SourceRow != 9 { + t.Fatalf("unexpected stored rows: %#v", rows) + } + if rows[0].Message == "duplicate key value is (alice@example.com)" { + t.Fatalf("stored driver message was not sanitized: %q", rows[0].Message) + } + + if _, err := store.Open("../../outside"); !os.IsNotExist(err) { + t.Fatalf("opaque artifact lookup should reject unknown IDs, got %v", err) + } +} + +func TestExportImportErrorRowsCopiesManagedArtifactAtomically(t *testing.T) { + app := NewApp() + app.ctx = context.Background() + app.configDir = t.TempDir() + store, err := app.ensureImportErrorArtifactStore() + if err != nil { + t.Fatal(err) + } + w, err := store.Begin("import-job-export") + if err != nil { + t.Fatal(err) + } + if err := w.Append(ImportRowError{SourceRow: 3, Category: "constraint", Message: "duplicate"}); err != nil { + t.Fatal(err) + } + artifact, err := w.Finish() + if err != nil { + t.Fatal(err) + } + + destination := filepath.Join(t.TempDir(), "failed-rows.jsonl") + app.saveFileDialog = func(context.Context, runtime.SaveDialogOptions) (string, error) { + return destination, nil + } + result := app.ExportImportErrorRows(artifact.ID) + if !result.Success { + t.Fatalf("export failed: %#v", result) + } + content, err := os.ReadFile(destination) + if err != nil { + t.Fatal(err) + } + if len(content) == 0 { + t.Fatal("exported artifact is empty") + } + if _, err := os.Stat(destination + ".part"); !os.IsNotExist(err) { + t.Fatalf("temporary export file leaked: %v", err) + } +} diff --git a/internal/app/import_job_identity.go b/internal/app/import_job_identity.go new file mode 100644 index 00000000..653e031c --- /dev/null +++ b/internal/app/import_job_identity.go @@ -0,0 +1,100 @@ +package app + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "strings" + + "GoNavi-Wails/internal/connection" +) + +func hashImportJobContract(value any) string { + payload, err := json.Marshal(value) + if err != nil { + return "" + } + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]) +} + +func buildImportTargetFingerprint(config connection.ConnectionConfig, dbName, tableName string) string { + return hashImportJobContract(struct { + Type string `json:"type"` + Host string `json:"host"` + Hosts []string `json:"hosts,omitempty"` + Port int `json:"port"` + User string `json:"user"` + Database string `json:"database"` + DBName string `json:"dbName"` + TableName string `json:"tableName,omitempty"` + }{ + Type: strings.ToLower(strings.TrimSpace(config.Type)), + Host: strings.ToLower(strings.TrimSpace(config.Host)), + Hosts: append([]string(nil), config.Hosts...), + Port: config.Port, + User: strings.TrimSpace(config.User), + Database: strings.TrimSpace(config.Database), + DBName: strings.TrimSpace(dbName), + TableName: strings.TrimSpace(tableName), + }) +} + +func buildImportFileOptionsHash(options ImportFileOptions) string { + continueOnError := true + if options.ContinueOnError != nil { + continueOnError = *options.ContinueOnError + } + encoding := strings.ToLower(strings.TrimSpace(options.Encoding)) + if encoding == "" { + encoding = importTextEncodingAuto + } + delimiter := strings.ToLower(strings.TrimSpace(options.Delimiter)) + if delimiter == "" { + delimiter = importDelimiterAuto + } + headerRow := options.HeaderRow + if headerRow == 0 { + headerRow = 1 + } + conflictPolicy := strings.ToLower(strings.TrimSpace(options.ConflictPolicy)) + if conflictPolicy == "" { + conflictPolicy = importConflictPolicyStop + } + return hashImportJobContract(struct { + ColumnMappings map[string]string `json:"columnMappings,omitempty"` + ContinueOnError bool `json:"continueOnError"` + Encoding string `json:"encoding,omitempty"` + Delimiter string `json:"delimiter,omitempty"` + HeaderRow int `json:"headerRow,omitempty"` + NullToken *string `json:"nullToken,omitempty"` + EmptyStringAsNull bool `json:"emptyStringAsNull,omitempty"` + SheetName string `json:"sheetName,omitempty"` + ConflictPolicy string `json:"conflictPolicy,omitempty"` + ConflictKeyColumns []string `json:"conflictKeyColumns,omitempty"` + }{ + ColumnMappings: options.ColumnMappings, + ContinueOnError: continueOnError, + Encoding: encoding, + Delimiter: delimiter, + HeaderRow: headerRow, + NullToken: options.NullToken, + EmptyStringAsNull: options.EmptyStringAsNull, + SheetName: options.SheetName, + ConflictPolicy: conflictPolicy, + ConflictKeyColumns: append([]string(nil), options.ConflictKeyColumns...), + }) +} + +func buildSQLImportOptionsHash(continueOnError bool, maxStatementBytes int64) string { + if maxStatementBytes <= 0 { + maxStatementBytes = DefaultSQLImportMaxStatementBytes + } + return hashImportJobContract(struct { + ContinueOnError bool `json:"continueOnError"` + MaxStatementBytes int64 `json:"maxStatementBytes"` + }{ + ContinueOnError: continueOnError, + MaxStatementBytes: maxStatementBytes, + }) +} diff --git a/internal/app/import_job_identity_test.go b/internal/app/import_job_identity_test.go new file mode 100644 index 00000000..017cb1f0 --- /dev/null +++ b/internal/app/import_job_identity_test.go @@ -0,0 +1,50 @@ +package app + +import ( + "testing" + + "GoNavi-Wails/internal/connection" +) + +func TestImportTargetFingerprintDoesNotDependOnCredentials(t *testing.T) { + first := connection.ConnectionConfig{Type: "mysql", Host: "db.example", Port: 3306, User: "operator", Password: "secret-a"} + second := first + second.Password = "secret-b" + if got, want := buildImportTargetFingerprint(first, "app", "users"), buildImportTargetFingerprint(second, "app", "users"); got != want { + t.Fatalf("credential rotation changed target fingerprint: %q != %q", got, want) + } + if buildImportTargetFingerprint(first, "app", "users") == buildImportTargetFingerprint(first, "app", "orders") { + t.Fatal("different target tables must not share a fingerprint") + } +} + +func TestImportOptionsHashIgnoresRuntimeJobID(t *testing.T) { + stop := false + first := ImportFileOptions{JobID: "job-a", ContinueOnError: &stop, ColumnMappings: map[string]string{"id": "id"}} + second := first + second.JobID = "job-b" + if got, want := buildImportFileOptionsHash(first), buildImportFileOptionsHash(second); got != want { + t.Fatalf("job id changed semantic options hash: %q != %q", got, want) + } +} + +func TestImportOptionsHashCanonicalizesEquivalentDefaults(t *testing.T) { + implicit := ImportFileOptions{} + explicit := ImportFileOptions{ + Encoding: "auto", + Delimiter: "auto", + HeaderRow: 1, + ConflictPolicy: "stop", + } + if got, want := buildImportFileOptionsHash(implicit), buildImportFileOptionsHash(explicit); got != want { + t.Fatalf("equivalent defaults changed semantic options hash: %q != %q", got, want) + } +} + +func TestImportOptionsHashPreservesExactSheetName(t *testing.T) { + plain := ImportFileOptions{SheetName: "Sheet"} + spaced := ImportFileOptions{SheetName: " Sheet "} + if buildImportFileOptionsHash(plain) == buildImportFileOptionsHash(spaced) { + t.Fatal("distinct workbook sheet names must not share an options hash") + } +} diff --git a/internal/app/import_job_lifecycle.go b/internal/app/import_job_lifecycle.go new file mode 100644 index 00000000..bd808f8e --- /dev/null +++ b/internal/app/import_job_lifecycle.go @@ -0,0 +1,294 @@ +package app + +import ( + "errors" + "strings" + "sync" + "time" + + "GoNavi-Wails/internal/connection" + "GoNavi-Wails/internal/importjob" +) + +const importJobProgressPersistInterval = 500 * time.Millisecond + +type managedImportJobStart struct { + ID string + Kind importjob.Kind + SourcePath string + SourceIdentityToken string + SourceContentSHA256 string + SourceBytesTotal int64 + ByteProgressKind string + TargetFingerprint string + ConnectionID string + DatabaseName string + TableName string + OptionsHash string +} + +type managedImportJobProgress struct { + Stage string + Current int64 + Total int64 + Succeeded int64 + Skipped int64 + Failed int64 + BytesRead int64 + SourceBytesTotal int64 + ByteProgressKind string + SourceContentSHA256 string + Checkpoint importjob.Checkpoint + OutcomeUnknown bool + ForcePersist bool +} + +type managedImportJobFinish struct { + Status importjob.Status + Message string + OutcomeUnknown bool + ErrorArtifactID string +} + +type managedImportJob struct { + mu sync.Mutex + store *importjob.Store + job importjob.Job + lastPersisted time.Time +} + +func isImportJobTerminalStatus(status importjob.Status) bool { + switch status { + case importjob.StatusCompleted, importjob.StatusPartial, importjob.StatusFailed, + importjob.StatusCancelled, importjob.StatusUnknown, importjob.StatusInterrupted: + return true + default: + return false + } +} + +func (a *App) beginManagedImportJob(start managedImportJobStart) (*managedImportJob, error) { + store, err := a.ensureImportJobStore() + if err != nil { + return nil, err + } + job := importjob.Job{ + ID: strings.TrimSpace(start.ID), + Kind: start.Kind, + Status: importjob.StatusPreparing, + Stage: "preparing", + SourcePath: strings.TrimSpace(start.SourcePath), + SourceIdentityToken: strings.TrimSpace(start.SourceIdentityToken), + SourceContentSHA256: strings.TrimSpace(start.SourceContentSHA256), + SourceBytesTotal: max(0, start.SourceBytesTotal), + ByteProgressKind: strings.TrimSpace(start.ByteProgressKind), + TargetFingerprint: strings.TrimSpace(start.TargetFingerprint), + ConnectionID: strings.TrimSpace(start.ConnectionID), + DatabaseName: strings.TrimSpace(start.DatabaseName), + TableName: strings.TrimSpace(start.TableName), + OptionsHash: strings.TrimSpace(start.OptionsHash), + Checkpoint: importjob.Checkpoint{}, + } + created, err := store.Put(job) + if err != nil { + return nil, err + } + managed := &managedImportJob{store: store, job: created, lastPersisted: time.Now()} + if _, err := a.bindImportTaskLifecycle(created.ID, created.Kind, managed); err != nil { + _ = managed.finish(managedImportJobFinish{Status: importjob.StatusFailed, Message: err.Error()}) + return nil, err + } + return managed, nil +} + +func (managed *managedImportJob) requestStop() error { + if managed == nil || managed.store == nil { + return errors.New("import job lifecycle is unavailable") + } + managed.mu.Lock() + defer managed.mu.Unlock() + if isImportJobTerminalStatus(managed.job.Status) || managed.job.Status == importjob.StatusStopping { + return nil + } + managed.job.Status = importjob.StatusStopping + managed.job.Stage = string(importjob.StatusStopping) + managed.job.Resumable = false + updated, err := managed.store.Put(managed.job) + if err != nil { + return err + } + managed.job = updated + managed.lastPersisted = time.Now() + return nil +} + +func (managed *managedImportJob) update(progress managedImportJobProgress) error { + if managed == nil || managed.store == nil { + return errors.New("import job lifecycle is unavailable") + } + managed.mu.Lock() + defer managed.mu.Unlock() + + if isImportJobTerminalStatus(managed.job.Status) { + return nil + } + stopping := managed.job.Status == importjob.StatusStopping + if !stopping { + managed.job.Status = importjob.StatusRunning + } + if stage := strings.TrimSpace(progress.Stage); stage != "" && !stopping { + managed.job.Stage = stage + } + managed.job.Current = max(0, progress.Current) + managed.job.Total = max(0, progress.Total) + managed.job.Succeeded = max(0, progress.Succeeded) + managed.job.Skipped = max(0, progress.Skipped) + managed.job.Failed = max(0, progress.Failed) + managed.job.BytesRead = max(0, progress.BytesRead) + if progress.SourceBytesTotal > 0 { + managed.job.SourceBytesTotal = progress.SourceBytesTotal + } + if kind := strings.TrimSpace(progress.ByteProgressKind); kind != "" { + managed.job.ByteProgressKind = kind + } + if digest := strings.TrimSpace(progress.SourceContentSHA256); digest != "" { + managed.job.SourceContentSHA256 = digest + } + managed.job.OutcomeUnknown = managed.job.OutcomeUnknown || progress.OutcomeUnknown + managed.job.Checkpoint = progress.Checkpoint + managed.job.Resumable = false + + if !progress.ForcePersist && time.Since(managed.lastPersisted) < importJobProgressPersistInterval { + return nil + } + updated, err := managed.store.Put(managed.job) + if err != nil { + return err + } + managed.job = updated + managed.lastPersisted = time.Now() + return nil +} + +func (managed *managedImportJob) finish(finish managedImportJobFinish) error { + if managed == nil || managed.store == nil { + return errors.New("import job lifecycle is unavailable") + } + managed.mu.Lock() + defer managed.mu.Unlock() + if isImportJobTerminalStatus(managed.job.Status) { + return nil + } + + outcomeUnknown := managed.job.OutcomeUnknown || finish.OutcomeUnknown + if outcomeUnknown { + managed.job.Status = importjob.StatusUnknown + } else { + switch finish.Status { + case importjob.StatusCompleted, importjob.StatusPartial, importjob.StatusFailed, + importjob.StatusCancelled, importjob.StatusUnknown, importjob.StatusInterrupted: + managed.job.Status = finish.Status + default: + managed.job.Status = importjob.StatusFailed + } + } + managed.job.Stage = string(managed.job.Status) + managed.job.Message = strings.TrimSpace(finish.Message) + managed.job.OutcomeUnknown = outcomeUnknown + managed.job.ErrorArtifactID = strings.TrimSpace(finish.ErrorArtifactID) + managed.job.Resumable = false + updated, err := managed.store.Put(managed.job) + if err != nil { + return err + } + managed.job = updated + managed.lastPersisted = time.Now() + return nil +} + +func (managed *managedImportJob) snapshot() importjob.Job { + if managed == nil { + return importjob.Job{} + } + managed.mu.Lock() + defer managed.mu.Unlock() + return managed.job +} + +func managedImportJobFinishFromResult(result connection.QueryResult) managedImportJobFinish { + finish := managedImportJobFinish{ + Status: importjob.StatusFailed, + Message: strings.TrimSpace(result.Message), + } + payload, _ := result.Data.(map[string]interface{}) + if payload != nil { + finish.OutcomeUnknown, _ = payload["outcomeUnknown"].(bool) + finish.ErrorArtifactID, _ = payload["errorArtifactId"].(string) + if finish.OutcomeUnknown { + finish.Status = importjob.StatusUnknown + return finish + } + if cancelled, _ := payload["cancelled"].(bool); cancelled { + finish.Status = importjob.StatusCancelled + return finish + } + succeeded := importJobPayloadInt64(payload, "success") + if executed := importJobPayloadInt64(payload, "executed"); executed > succeeded { + succeeded = executed + } + failed := importJobPayloadInt64(payload, "failed") + switch outcome, _ := payload["outcome"].(string); strings.ToLower(strings.TrimSpace(outcome)) { + case "completed": + if failed > 0 { + finish.Status = importjob.StatusPartial + } else { + finish.Status = importjob.StatusCompleted + } + return finish + case "partial": + finish.Status = importjob.StatusPartial + return finish + case "cancelled": + finish.Status = importjob.StatusCancelled + return finish + case "failed", "stopped": + if succeeded > 0 { + finish.Status = importjob.StatusPartial + } else { + finish.Status = importjob.StatusFailed + } + return finish + } + if !result.Success && succeeded > 0 { + finish.Status = importjob.StatusPartial + return finish + } + } + if failed := importJobPayloadInt64(payload, "failed"); failed > 0 { + if result.Success || importJobPayloadInt64(payload, "success") > 0 || importJobPayloadInt64(payload, "executed") > 0 { + finish.Status = importjob.StatusPartial + } + return finish + } + if !result.Success { + return finish + } + finish.Status = importjob.StatusCompleted + return finish +} + +func importJobPayloadInt64(payload map[string]interface{}, key string) int64 { + if payload == nil { + return 0 + } + switch value := payload[key].(type) { + case int: + return int64(value) + case int64: + return value + case float64: + return int64(value) + default: + return 0 + } +} diff --git a/internal/app/import_job_lifecycle_test.go b/internal/app/import_job_lifecycle_test.go new file mode 100644 index 00000000..c767d618 --- /dev/null +++ b/internal/app/import_job_lifecycle_test.go @@ -0,0 +1,290 @@ +package app + +import ( + "fmt" + "sync" + "sync/atomic" + "testing" + + "GoNavi-Wails/internal/connection" + "GoNavi-Wails/internal/importjob" +) + +func TestManagedImportJobPersistsProgressAndTerminalState(t *testing.T) { + app := NewApp() + app.configDir = t.TempDir() + + lifecycle, err := app.beginManagedImportJob(managedImportJobStart{ + ID: "import-job-progress", + Kind: importjob.KindTable, + SourcePath: "D:/imports/users.csv", + SourceIdentityToken: "source-v1", + TargetFingerprint: "target-v1", + OptionsHash: "options-v1", + }) + if err != nil { + t.Fatal(err) + } + if err := lifecycle.update(managedImportJobProgress{ + Stage: "writing", + Current: 1000, + Total: 5000, + Succeeded: 998, + Failed: 2, + BytesRead: 65536, + Checkpoint: importjob.Checkpoint{Safe: true, SourceRow: 1000, ByteOffset: 65536}, + ForcePersist: true, + }); err != nil { + t.Fatal(err) + } + if err := lifecycle.finish(managedImportJobFinish{ + Status: importjob.StatusPartial, + Message: "completed with rejected rows", + ErrorArtifactID: "artifact-v1", + }); err != nil { + t.Fatal(err) + } + + stored, err := lifecycle.store.Get("import-job-progress") + if err != nil { + t.Fatal(err) + } + if stored.Status != importjob.StatusPartial || stored.Current != 1000 || stored.Succeeded != 998 || stored.Failed != 2 { + t.Fatalf("unexpected terminal job: %#v", stored) + } + if stored.Checkpoint.SourceRow != 1000 || stored.ErrorArtifactID != "artifact-v1" || stored.Resumable { + t.Fatalf("unexpected checkpoint/artifact state: %#v", stored) + } +} + +func TestManagedImportJobCancelProgressFinishRaceEndsTerminalWithoutRevisionConflict(t *testing.T) { + app := NewApp() + app.configDir = t.TempDir() + var cancelCalls atomic.Int32 + cleanup, registered := app.registerImportTask("import-job-race", func() { + cancelCalls.Add(1) + }, importjob.KindTable) + if !registered { + t.Fatal("import task registration failed") + } + defer cleanup() + lifecycle, err := app.beginManagedImportJob(managedImportJobStart{ + ID: "import-job-race", + Kind: importjob.KindTable, + }) + if err != nil { + t.Fatal(err) + } + + start := make(chan struct{}) + errs := make(chan error, 64) + var workers sync.WaitGroup + workers.Add(3) + go func() { + defer workers.Done() + <-start + for i := int64(1); i <= 20; i++ { + if err := lifecycle.update(managedImportJobProgress{ + Stage: "writing", + Current: i, + Succeeded: i, + ForcePersist: true, + }); err != nil { + errs <- fmt.Errorf("progress %d: %w", i, err) + return + } + } + }() + go func() { + defer workers.Done() + <-start + for i := 0; i < 3; i++ { + if result := app.CancelImportJob("import-job-race"); !result.Success { + errs <- fmt.Errorf("cancel %d failed: %s", i, result.Message) + return + } + } + }() + go func() { + defer workers.Done() + <-start + if err := lifecycle.finish(managedImportJobFinish{Status: importjob.StatusCancelled}); err != nil { + errs <- fmt.Errorf("finish: %w", err) + } + }() + close(start) + workers.Wait() + close(errs) + for err := range errs { + t.Error(err) + } + if t.Failed() { + return + } + if got := cancelCalls.Load(); got != 1 { + t.Fatalf("cancel callback calls = %d, want 1", got) + } + stored, err := lifecycle.store.Get("import-job-race") + if err != nil { + t.Fatal(err) + } + if stored.Status != importjob.StatusCancelled { + t.Fatalf("status = %q, want %q; job=%#v", stored.Status, importjob.StatusCancelled, stored) + } +} + +func TestManagedImportJobRejectsReusedJobID(t *testing.T) { + app := NewApp() + app.configDir = t.TempDir() + start := managedImportJobStart{ + ID: "import-job-duplicate", + Kind: importjob.KindSQL, + SourceIdentityToken: "source-v1", + TargetFingerprint: "target-v1", + OptionsHash: "options-v1", + } + if _, err := app.beginManagedImportJob(start); err != nil { + t.Fatal(err) + } + if _, err := app.beginManagedImportJob(start); err == nil { + t.Fatal("expected duplicate durable job id to be rejected") + } +} + +func TestManagedImportJobKeepsStoppingAndTerminalStatesMonotonic(t *testing.T) { + app := NewApp() + app.configDir = t.TempDir() + lifecycle, err := app.beginManagedImportJob(managedImportJobStart{ + ID: "import-job-monotonic", + Kind: importjob.KindTable, + }) + if err != nil { + t.Fatal(err) + } + if err := lifecycle.requestStop(); err != nil { + t.Fatal(err) + } + if err := lifecycle.update(managedImportJobProgress{ + Stage: "writing", + Current: 20, + Succeeded: 19, + Failed: 1, + ForcePersist: true, + }); err != nil { + t.Fatal(err) + } + stopping, err := lifecycle.store.Get("import-job-monotonic") + if err != nil { + t.Fatal(err) + } + if stopping.Status != importjob.StatusStopping { + t.Fatalf("progress regressed stopping job to %q", stopping.Status) + } + if stopping.Current != 20 || stopping.Succeeded != 19 || stopping.Failed != 1 { + t.Fatalf("stopping progress was not retained: %#v", stopping) + } + + if err := lifecycle.finish(managedImportJobFinish{Status: importjob.StatusCancelled}); err != nil { + t.Fatal(err) + } + terminal, err := lifecycle.store.Get("import-job-monotonic") + if err != nil { + t.Fatal(err) + } + if terminal.Status != importjob.StatusCancelled { + t.Fatalf("status = %q, want %q", terminal.Status, importjob.StatusCancelled) + } + if err := lifecycle.update(managedImportJobProgress{ + Stage: "writing", + Current: 999, + Succeeded: 999, + ForcePersist: true, + }); err != nil { + t.Fatal(err) + } + afterLateProgress, err := lifecycle.store.Get("import-job-monotonic") + if err != nil { + t.Fatal(err) + } + if afterLateProgress.Revision != terminal.Revision || afterLateProgress.Current != terminal.Current || afterLateProgress.Status != terminal.Status { + t.Fatalf("late progress changed terminal job: before=%#v after=%#v", terminal, afterLateProgress) + } + if err := lifecycle.finish(managedImportJobFinish{Status: importjob.StatusCompleted}); err != nil { + t.Fatal(err) + } + afterLateFinish, err := lifecycle.store.Get("import-job-monotonic") + if err != nil { + t.Fatal(err) + } + if afterLateFinish.Revision != terminal.Revision || afterLateFinish.Status != importjob.StatusCancelled { + t.Fatalf("late finish changed terminal job: before=%#v after=%#v", terminal, afterLateFinish) + } +} + +func TestManagedImportJobDoesNotClearAnUnknownOutcomeWithLaterProgress(t *testing.T) { + app := NewApp() + app.configDir = t.TempDir() + lifecycle, err := app.beginManagedImportJob(managedImportJobStart{ + ID: "import-job-outcome-monotonic", + Kind: importjob.KindSQL, + }) + if err != nil { + t.Fatal(err) + } + if err := lifecycle.update(managedImportJobProgress{OutcomeUnknown: true, ForcePersist: true}); err != nil { + t.Fatal(err) + } + if err := lifecycle.update(managedImportJobProgress{OutcomeUnknown: false, ForcePersist: true}); err != nil { + t.Fatal(err) + } + stored, err := lifecycle.store.Get("import-job-outcome-monotonic") + if err != nil { + t.Fatal(err) + } + if !stored.OutcomeUnknown { + t.Fatalf("late progress cleared an unknown outcome: %#v", stored) + } + if err := lifecycle.finish(managedImportJobFinish{Status: importjob.StatusCompleted}); err != nil { + t.Fatal(err) + } + terminal, err := lifecycle.store.Get("import-job-outcome-monotonic") + if err != nil { + t.Fatal(err) + } + if terminal.Status != importjob.StatusUnknown || !terminal.OutcomeUnknown { + t.Fatalf("terminal update cleared an unknown outcome: %#v", terminal) + } +} + +func TestManagedImportJobFinishClassifiesCommittedPrefixAsPartial(t *testing.T) { + tests := []struct { + name string + data map[string]interface{} + }{ + {name: "table parser fails after writes", data: map[string]interface{}{"success": 10, "failed": 0}}, + {name: "SQL preflight rejects after statements", data: map[string]interface{}{ + "executed": 10, "failed": 0, "outcome": "failed", "previousStatementsMayHaveCompleted": true, + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + finish := managedImportJobFinishFromResult(connection.QueryResult{Success: false, Data: test.data}) + if finish.Status != importjob.StatusPartial || finish.OutcomeUnknown { + t.Fatalf("finish = %#v, want known partial", finish) + } + }) + } +} + +func TestManagedImportJobFinishClassifiesAmbiguousCancellationAsUnknown(t *testing.T) { + finish := managedImportJobFinishFromResult(connection.QueryResult{ + Success: false, + Data: map[string]interface{}{ + "cancelled": true, + "outcomeUnknown": true, + }, + }) + if finish.Status != importjob.StatusUnknown || !finish.OutcomeUnknown { + t.Fatalf("ambiguous cancellation finish = %#v, want unknown", finish) + } +} diff --git a/internal/app/import_lexical_limit_reader.go b/internal/app/import_lexical_limit_reader.go new file mode 100644 index 00000000..28c9a17a --- /dev/null +++ b/internal/app/import_lexical_limit_reader.go @@ -0,0 +1,622 @@ +package app + +import ( + "bufio" + "bytes" + "encoding/csv" + "encoding/json" + "fmt" + "io" +) + +type ImportFileLimitKind string + +const ( + ImportFileCellByteLimit ImportFileLimitKind = "cell_byte_limit" + ImportFileRowByteLimit ImportFileLimitKind = "row_byte_limit" +) + +// ImportFileLimitError reports a bounded parsing failure without retaining or +// exposing the rejected source value. +type ImportFileLimitError struct { + Format string + Kind ImportFileLimitKind + Row int + Cell int + Column string + Limit int +} + +func (err *ImportFileLimitError) Error() string { + if err == nil { + return "import file safety limit exceeded" + } + if err.Kind == ImportFileRowByteLimit { + return fmt.Sprintf("%s row %d exceeds %d-byte limit", err.Format, err.Row, err.Limit) + } + if err.Column != "" { + return fmt.Sprintf("%s row %d column %q exceeds %d-byte cell limit", err.Format, err.Row, err.Column, err.Limit) + } + if err.Cell > 0 { + return fmt.Sprintf("%s row %d cell %d exceeds %d-byte limit", err.Format, err.Row, err.Cell, err.Limit) + } + return fmt.Sprintf("%s row %d string exceeds %d-byte cell limit", err.Format, err.Row, err.Limit) +} + +type importLexicalLimits struct { + maxCellBytes int + maxRowBytes int +} + +func normalizeImportLexicalLimits(limits importLexicalLimits) importLexicalLimits { + if limits.maxCellBytes <= 0 { + limits.maxCellBytes = maxImportCellBytes + } + if limits.maxRowBytes <= 0 { + limits.maxRowBytes = maxImportRowBytes + } + return limits +} + +type importCSVLexicalLimitReader struct { + reader io.Reader + delimiter byte + limits importLexicalLimits + row int + cell int + fieldRaw int + fieldDecoded int + rowRaw int + rowDecoded int + atFieldStart bool + inQuotes bool + afterQuote bool + pendingCR bool + terminalErr error +} + +func newImportCSVLexicalLimitReader(source io.Reader, delimiter rune, limits importLexicalLimits) io.Reader { + return &importCSVLexicalLimitReader{ + reader: source, + delimiter: byte(delimiter), + limits: normalizeImportLexicalLimits(limits), + row: 1, + cell: 1, + atFieldStart: true, + } +} + +func (reader *importCSVLexicalLimitReader) Read(buffer []byte) (int, error) { + if reader.terminalErr != nil { + return 0, reader.terminalErr + } + n, readErr := reader.reader.Read(buffer) + for index := 0; index < n; index++ { + value := buffer[index] + if reader.pendingCR { + reader.pendingCR = false + if value == '\n' { + if err := reader.consumeNewline(2); err != nil { + reader.terminalErr = err + return index, err + } + continue + } + if err := reader.consumeDataByte('\r', 1, 1); err != nil { + reader.terminalErr = err + return index, err + } + } + if value == '\r' { + reader.pendingCR = true + continue + } + if value == '\n' { + if err := reader.consumeNewline(1); err != nil { + reader.terminalErr = err + return index, err + } + continue + } + if err := reader.consumeDataByte(value, 1, 1); err != nil { + reader.terminalErr = err + return index, err + } + } + if readErr != nil && reader.pendingCR { + reader.pendingCR = false + if err := reader.consumeDataByte('\r', 1, 1); err != nil { + reader.terminalErr = err + return n, err + } + } + return n, readErr +} + +func (reader *importCSVLexicalLimitReader) consumeDataByte(value byte, rawBytes int, decodedBytes int) error { + if err := reader.addRowRawBytes(rawBytes); err != nil { + return err + } + if reader.inQuotes { + if reader.afterQuote { + reader.afterQuote = false + switch value { + case '"': + return reader.addFieldBytes(2, 1) + case reader.delimiter: + reader.inQuotes = false + reader.finishField() + return nil + default: + // Keep malformed CSV for encoding/csv to diagnose. Counting the + // byte still bounds the parser's allocation before that happens. + reader.inQuotes = false + return reader.addFieldBytes(rawBytes, decodedBytes) + } + } + if value == '"' { + reader.afterQuote = true + return nil + } + return reader.addFieldBytes(rawBytes, decodedBytes) + } + + if reader.atFieldStart && value == '"' { + reader.atFieldStart = false + reader.inQuotes = true + return nil + } + if value == reader.delimiter { + reader.finishField() + return nil + } + reader.atFieldStart = false + return reader.addFieldBytes(rawBytes, decodedBytes) +} + +func (reader *importCSVLexicalLimitReader) consumeNewline(rawBytes int) error { + if reader.inQuotes && !reader.afterQuote { + if err := reader.addRowRawBytes(rawBytes); err != nil { + return err + } + return reader.addFieldBytes(rawBytes, 1) + } + if reader.inQuotes && reader.afterQuote { + reader.inQuotes = false + reader.afterQuote = false + } + reader.finishRecord() + return nil +} + +func (reader *importCSVLexicalLimitReader) addFieldBytes(rawBytes int, decodedBytes int) error { + if reader.fieldRaw > reader.limits.maxCellBytes-rawBytes || reader.fieldDecoded > reader.limits.maxCellBytes-decodedBytes { + return &ImportFileLimitError{ + Format: "CSV", + Kind: ImportFileCellByteLimit, + Row: reader.row, + Cell: reader.cell, + Limit: reader.limits.maxCellBytes, + } + } + if reader.rowDecoded > reader.limits.maxRowBytes-decodedBytes { + return &ImportFileLimitError{ + Format: "CSV", + Kind: ImportFileRowByteLimit, + Row: reader.row, + Limit: reader.limits.maxRowBytes, + } + } + reader.fieldRaw += rawBytes + reader.fieldDecoded += decodedBytes + reader.rowDecoded += decodedBytes + return nil +} + +func (reader *importCSVLexicalLimitReader) addRowRawBytes(rawBytes int) error { + if reader.rowRaw > reader.limits.maxRowBytes-rawBytes { + return &ImportFileLimitError{ + Format: "CSV", + Kind: ImportFileRowByteLimit, + Row: reader.row, + Limit: reader.limits.maxRowBytes, + } + } + reader.rowRaw += rawBytes + return nil +} + +func (reader *importCSVLexicalLimitReader) finishField() { + reader.cell++ + reader.fieldRaw = 0 + reader.fieldDecoded = 0 + reader.atFieldStart = true +} + +func (reader *importCSVLexicalLimitReader) finishRecord() { + reader.row++ + reader.cell = 1 + reader.fieldRaw = 0 + reader.fieldDecoded = 0 + reader.rowRaw = 0 + reader.rowDecoded = 0 + reader.atFieldStart = true +} + +func newImportCSVReaderWithLimits(source io.Reader, delimiterName string, limits importLexicalLimits) (*csv.Reader, error) { + delimiter, explicit, err := resolveImportDelimiter(delimiterName) + if err != nil { + return nil, err + } + parseSource := source + if !explicit { + prefix, readErr := io.ReadAll(io.LimitReader(source, importDelimiterProbeSize+1)) + if readErr != nil { + return nil, fmt.Errorf("CSV delimiter probe failed: %w", readErr) + } + delimiter, err = detectImportCSVDelimiter(prefix) + if err != nil { + return nil, err + } + parseSource = io.MultiReader(bytes.NewReader(prefix), source) + } + limited := newImportCSVLexicalLimitReader(parseSource, delimiter, limits) + reader := csv.NewReader(bufio.NewReader(limited)) + reader.Comma = delimiter + return reader, nil +} + +type importJSONLexicalLimitReader struct { + reader io.Reader + limits importLexicalLimits + row int + rootStarted bool + rootClosed bool + depth int + inElement bool + elementRaw int + inString bool + escaped bool + unicodeDigits int + stringRaw int + stringDecoded int + stringColumn string + stringIsTopKey bool + topObjectDepth int + topExpectKey bool + topColumn string + topKeyBuffer []byte + topKeyUsable bool + topValuePending bool + topValueMode importJSONValueLimitMode + topValueRaw int + terminalErr error +} + +type importJSONValueLimitMode uint8 + +const ( + importJSONValueLimitNone importJSONValueLimitMode = iota + importJSONValueLimitString + importJSONValueLimitPrimitive + importJSONValueLimitComposite +) + +func newImportJSONLexicalLimitReader(source io.Reader, limits importLexicalLimits) io.Reader { + return &importJSONLexicalLimitReader{ + reader: source, + limits: normalizeImportLexicalLimits(limits), + } +} + +func (reader *importJSONLexicalLimitReader) Read(buffer []byte) (int, error) { + if reader.terminalErr != nil { + return 0, reader.terminalErr + } + n, readErr := reader.reader.Read(buffer) + for index := 0; index < n; index++ { + if err := reader.consumeByte(buffer[index]); err != nil { + reader.terminalErr = err + return index, err + } + } + return n, readErr +} + +func (reader *importJSONLexicalLimitReader) consumeByte(value byte) error { + if reader.inString { + if reader.inElement { + if err := reader.addElementRawByte(); err != nil { + return err + } + } + if reader.topValueMode == importJSONValueLimitComposite { + if err := reader.addTopValueRawByte(); err != nil { + return err + } + } + return reader.consumeStringByte(value) + } + + if !reader.rootStarted { + if isImportJSONWhitespace(value) { + return nil + } + if value == '[' { + reader.rootStarted = true + reader.depth = 1 + } + return nil + } + if reader.rootClosed { + return nil + } + + if reader.depth == 1 && !reader.inElement { + if isImportJSONWhitespace(value) || value == ',' { + return nil + } + if value == ']' { + reader.rootClosed = true + reader.depth = 0 + return nil + } + reader.row++ + reader.inElement = true + reader.elementRaw = 0 + } + if reader.depth == 1 && reader.inElement && (value == ',' || value == ']') { + reader.finishElement() + if value == ']' { + reader.rootClosed = true + reader.depth = 0 + } + return nil + } + if err := reader.consumeTopValueByte(value); err != nil { + return err + } + if reader.inElement { + if err := reader.addElementRawByte(); err != nil { + return err + } + } + + if value == '"' { + reader.startString() + return nil + } + + switch value { + case '{': + reader.depth++ + if reader.topObjectDepth == 0 && reader.depth == 2 { + reader.topObjectDepth = reader.depth + reader.topExpectKey = true + } + case '[': + reader.depth++ + case '}': + if reader.topObjectDepth == reader.depth { + reader.topObjectDepth = 0 + reader.topExpectKey = false + reader.topColumn = "" + } + if reader.depth > 1 { + reader.depth-- + } + case ']': + if reader.depth > 1 { + reader.depth-- + } + case ',': + if reader.topObjectDepth > 0 && reader.depth == reader.topObjectDepth { + reader.topExpectKey = true + reader.topColumn = "" + } + case ':': + if reader.topObjectDepth > 0 && reader.depth == reader.topObjectDepth && !reader.topExpectKey { + reader.topValuePending = true + reader.topValueMode = importJSONValueLimitNone + reader.topValueRaw = 0 + } + } + return nil +} + +func (reader *importJSONLexicalLimitReader) startString() { + reader.inString = true + reader.escaped = false + reader.unicodeDigits = 0 + reader.stringRaw = 0 + reader.stringDecoded = 0 + reader.stringIsTopKey = reader.topObjectDepth > 0 && reader.depth == reader.topObjectDepth && reader.topExpectKey + reader.stringColumn = "" + reader.topKeyBuffer = reader.topKeyBuffer[:0] + reader.topKeyUsable = reader.stringIsTopKey + if !reader.stringIsTopKey { + reader.stringColumn = reader.topColumn + } +} + +func (reader *importJSONLexicalLimitReader) consumeStringByte(value byte) error { + if reader.unicodeDigits > 0 { + if err := reader.addStringBytes(1, 0); err != nil { + return err + } + reader.unicodeDigits-- + if reader.unicodeDigits == 0 { + // A single JSON unicode escape decodes to at most three UTF-8 + // bytes. Surrogate pairs are conservatively counted as six. + if err := reader.addStringBytes(0, 3); err != nil { + return err + } + } + return nil + } + if reader.escaped { + reader.escaped = false + if err := reader.addStringBytes(1, 0); err != nil { + return err + } + if value == 'u' { + reader.unicodeDigits = 4 + reader.topKeyUsable = false + return nil + } + if reader.stringIsTopKey { + mapped, ok := importJSONSimpleEscape(value) + if !ok || len(reader.topKeyBuffer) >= 128 { + reader.topKeyUsable = false + } else { + reader.topKeyBuffer = append(reader.topKeyBuffer, mapped) + } + } + return reader.addStringBytes(0, 1) + } + if value == '\\' { + reader.escaped = true + return reader.addStringBytes(1, 0) + } + if value == '"' { + reader.inString = false + if reader.stringIsTopKey { + reader.topExpectKey = false + if reader.topKeyUsable { + reader.topColumn = string(reader.topKeyBuffer) + } else { + reader.topColumn = "" + } + } + return nil + } + if reader.stringIsTopKey { + if len(reader.topKeyBuffer) >= 128 { + reader.topKeyUsable = false + } else { + reader.topKeyBuffer = append(reader.topKeyBuffer, value) + } + } + return reader.addStringBytes(1, 1) +} + +func (reader *importJSONLexicalLimitReader) addStringBytes(rawBytes int, decodedBytes int) error { + if reader.stringRaw > reader.limits.maxCellBytes-rawBytes || reader.stringDecoded > reader.limits.maxCellBytes-decodedBytes { + return &ImportFileLimitError{ + Format: "JSON", + Kind: ImportFileCellByteLimit, + Row: reader.row, + Column: reader.stringColumn, + Limit: reader.limits.maxCellBytes, + } + } + reader.stringRaw += rawBytes + reader.stringDecoded += decodedBytes + return nil +} + +func (reader *importJSONLexicalLimitReader) addElementRawByte() error { + if reader.elementRaw >= reader.limits.maxRowBytes { + return &ImportFileLimitError{ + Format: "JSON", + Kind: ImportFileRowByteLimit, + Row: reader.row, + Limit: reader.limits.maxRowBytes, + } + } + reader.elementRaw++ + return nil +} + +func (reader *importJSONLexicalLimitReader) consumeTopValueByte(value byte) error { + if reader.topObjectDepth == 0 { + return nil + } + if reader.depth == reader.topObjectDepth && (value == ',' || value == '}') { + reader.finishTopValue() + return nil + } + if reader.topValuePending { + if isImportJSONWhitespace(value) { + return nil + } + reader.topValuePending = false + switch value { + case '"': + reader.topValueMode = importJSONValueLimitString + return nil + case '{', '[': + reader.topValueMode = importJSONValueLimitComposite + default: + reader.topValueMode = importJSONValueLimitPrimitive + } + } + switch reader.topValueMode { + case importJSONValueLimitComposite: + return reader.addTopValueRawByte() + case importJSONValueLimitPrimitive: + if !isImportJSONWhitespace(value) { + return reader.addTopValueRawByte() + } + } + return nil +} + +func (reader *importJSONLexicalLimitReader) addTopValueRawByte() error { + if reader.topValueRaw >= reader.limits.maxCellBytes { + return &ImportFileLimitError{ + Format: "JSON", + Kind: ImportFileCellByteLimit, + Row: reader.row, + Column: reader.topColumn, + Limit: reader.limits.maxCellBytes, + } + } + reader.topValueRaw++ + return nil +} + +func (reader *importJSONLexicalLimitReader) finishTopValue() { + reader.topValuePending = false + reader.topValueMode = importJSONValueLimitNone + reader.topValueRaw = 0 +} + +func (reader *importJSONLexicalLimitReader) finishElement() { + reader.inElement = false + reader.elementRaw = 0 + reader.topObjectDepth = 0 + reader.topExpectKey = false + reader.topColumn = "" + reader.finishTopValue() +} + +func importJSONSimpleEscape(value byte) (byte, bool) { + switch value { + case '"', '\\', '/': + return value, true + case 'b': + return '\b', true + case 'f': + return '\f', true + case 'n': + return '\n', true + case 'r': + return '\r', true + case 't': + return '\t', true + default: + return 0, false + } +} + +func isImportJSONWhitespace(value byte) bool { + return value == ' ' || value == '\t' || value == '\r' || value == '\n' +} + +func newImportJSONDecoderWithLimits(source io.Reader, limits importLexicalLimits) *json.Decoder { + limited := newImportJSONLexicalLimitReader(source, limits) + decoder := json.NewDecoder(bufio.NewReader(limited)) + decoder.UseNumber() + return decoder +} diff --git a/internal/app/import_lexical_limit_reader_test.go b/internal/app/import_lexical_limit_reader_test.go new file mode 100644 index 00000000..6feb782b --- /dev/null +++ b/internal/app/import_lexical_limit_reader_test.go @@ -0,0 +1,249 @@ +package app + +import ( + "errors" + "io" + "strings" + "testing" +) + +type importLexicalTestChunkReader struct { + source io.Reader + chunk int +} + +func (reader *importLexicalTestChunkReader) Read(buffer []byte) (int, error) { + if len(buffer) > reader.chunk { + buffer = buffer[:reader.chunk] + } + return reader.source.Read(buffer) +} + +func TestCSVReaderRejectsOversizedLogicalCellBeforeDecode(t *testing.T) { + reader, err := newImportCSVReaderWithLimits( + strings.NewReader("id,payload\n1,123456789\n"), + importDelimiterComma, + importLexicalLimits{maxCellBytes: 8, maxRowBytes: 64}, + ) + if err != nil { + t.Fatalf("create CSV reader: %v", err) + } + reader.FieldsPerRecord = -1 + + if header, err := reader.Read(); err != nil || len(header) != 2 { + t.Fatalf("read header = %#v, %v", header, err) + } + _, err = reader.Read() + var limitErr *ImportFileLimitError + if !errors.As(err, &limitErr) { + t.Fatalf("read oversized row error = %v, want ImportFileLimitError", err) + } + if limitErr.Kind != ImportFileCellByteLimit || limitErr.Format != "CSV" || limitErr.Row != 2 || limitErr.Cell != 2 || limitErr.Limit != 8 { + t.Fatalf("limit error = %#v", limitErr) + } + if strings.Contains(err.Error(), "123456789") { + t.Fatalf("limit error leaked cell content: %v", err) + } +} + +func TestJSONDecoderRejectsOversizedStringBeforeDecode(t *testing.T) { + decoder := newImportJSONDecoderWithLimits( + strings.NewReader(`[{"id":1,"payload":"123456789"}]`), + importLexicalLimits{maxCellBytes: 8, maxRowBytes: 128}, + ) + if _, err := decoder.Token(); err != nil { + t.Fatalf("read root token: %v", err) + } + var row map[string]interface{} + err := decoder.Decode(&row) + var limitErr *ImportFileLimitError + if !errors.As(err, &limitErr) { + t.Fatalf("decode oversized string error = %v, want ImportFileLimitError", err) + } + if limitErr.Kind != ImportFileCellByteLimit || limitErr.Format != "JSON" || limitErr.Row != 1 || limitErr.Column != "payload" || limitErr.Limit != 8 { + t.Fatalf("limit error = %#v", limitErr) + } + if strings.Contains(err.Error(), "123456789") { + t.Fatalf("limit error leaked JSON value: %v", err) + } +} + +func TestJSONDecoderReportsLaterElementRowAndColumn(t *testing.T) { + decoder := newImportJSONDecoderWithLimits( + strings.NewReader(`[{"a":"ok"},{"b":"12345"}]`), + importLexicalLimits{maxCellBytes: 4, maxRowBytes: 64}, + ) + if _, err := decoder.Token(); err != nil { + t.Fatalf("read root token: %v", err) + } + var first map[string]interface{} + if err := decoder.Decode(&first); err != nil { + t.Fatalf("decode first row: %v", err) + } + var second map[string]interface{} + err := decoder.Decode(&second) + var limitErr *ImportFileLimitError + if !errors.As(err, &limitErr) || limitErr.Row != 2 || limitErr.Column != "b" { + t.Fatalf("second-row limit error = %#v (%v)", limitErr, err) + } +} + +func TestJSONDecoderTracksEscapedStringsAcrossNestedValues(t *testing.T) { + decoder := newImportJSONDecoderWithLimits( + strings.NewReader(`[{"m":{"n":"a\"b","u":"\u4F60"}}]`), + importLexicalLimits{maxCellBytes: 64, maxRowBytes: 128}, + ) + if _, err := decoder.Token(); err != nil { + t.Fatalf("read root token: %v", err) + } + var row map[string]interface{} + if err := decoder.Decode(&row); err != nil { + t.Fatalf("decode nested row: %v", err) + } + nested, ok := row["m"].(map[string]interface{}) + if !ok || nested["n"] != `a"b` || nested["u"] != "你" { + t.Fatalf("decoded nested row = %#v", row) + } +} + +func TestJSONDecoderTracksUnicodeEscapeAcrossReadBoundaries(t *testing.T) { + source := &importLexicalTestChunkReader{ + source: strings.NewReader(`[{"u":"\u4F60"}]`), + chunk: 1, + } + decoder := newImportJSONDecoderWithLimits( + source, + importLexicalLimits{maxCellBytes: 6, maxRowBytes: 64}, + ) + if _, err := decoder.Token(); err != nil { + t.Fatalf("read root token: %v", err) + } + var row map[string]interface{} + if err := decoder.Decode(&row); err != nil { + t.Fatalf("decode unicode row: %v", err) + } + if row["u"] != "你" { + t.Fatalf("decoded row = %#v", row) + } +} + +func TestJSONDecoderRejectsOversizedTopLevelArrayElementBeforeDecode(t *testing.T) { + decoder := newImportJSONDecoderWithLimits( + strings.NewReader(`[{"a":[1,2,3]}]`), + importLexicalLimits{maxCellBytes: 64, maxRowBytes: 12}, + ) + if _, err := decoder.Token(); err != nil { + t.Fatalf("read root token: %v", err) + } + var row map[string]interface{} + err := decoder.Decode(&row) + var limitErr *ImportFileLimitError + if !errors.As(err, &limitErr) || limitErr.Kind != ImportFileRowByteLimit || limitErr.Row != 1 || limitErr.Limit != 12 { + t.Fatalf("row limit error = %#v (%v)", limitErr, err) + } +} + +func TestJSONDecoderRejectsOversizedCompositeColumnBeforeDecode(t *testing.T) { + decoder := newImportJSONDecoderWithLimits( + strings.NewReader(`[{"payload":{"a":"123","b":"456"}}]`), + importLexicalLimits{maxCellBytes: 16, maxRowBytes: 128}, + ) + if _, err := decoder.Token(); err != nil { + t.Fatalf("read root token: %v", err) + } + var row map[string]interface{} + err := decoder.Decode(&row) + var limitErr *ImportFileLimitError + if !errors.As(err, &limitErr) || limitErr.Kind != ImportFileCellByteLimit || limitErr.Row != 1 || limitErr.Column != "payload" || limitErr.Limit != 16 { + t.Fatalf("composite cell limit error = %#v (%v)", limitErr, err) + } +} + +func TestCSVReaderCountsQuotedEscapesAndCRLFAsLogicalFieldContent(t *testing.T) { + reader, err := newImportCSVReaderWithLimits( + strings.NewReader("id;data\r\n1;\"a\"\"b\"\r\n2;\"a\r\nb\"\r\n"), + importDelimiterSemicolon, + importLexicalLimits{maxCellBytes: 4, maxRowBytes: 16}, + ) + if err != nil { + t.Fatalf("create CSV reader: %v", err) + } + reader.FieldsPerRecord = -1 + + want := [][]string{{"id", "data"}, {"1", `a"b`}, {"2", "a\nb"}} + for index, wantRecord := range want { + got, readErr := reader.Read() + if readErr != nil { + t.Fatalf("read record %d: %v", index+1, readErr) + } + if len(got) != len(wantRecord) || got[0] != wantRecord[0] || got[1] != wantRecord[1] { + t.Fatalf("record %d = %#v, want %#v", index+1, got, wantRecord) + } + } +} + +func TestCSVReaderRejectsCombinedLogicalRowBeforeDecode(t *testing.T) { + reader, err := newImportCSVReaderWithLimits( + strings.NewReader("a,b\n12,345\n"), + importDelimiterComma, + importLexicalLimits{maxCellBytes: 8, maxRowBytes: 4}, + ) + if err != nil { + t.Fatalf("create CSV reader: %v", err) + } + reader.FieldsPerRecord = -1 + if _, err := reader.Read(); err != nil { + t.Fatalf("read header: %v", err) + } + _, err = reader.Read() + var limitErr *ImportFileLimitError + if !errors.As(err, &limitErr) || limitErr.Kind != ImportFileRowByteLimit || limitErr.Row != 2 || limitErr.Limit != 4 { + t.Fatalf("row limit error = %#v (%v)", limitErr, err) + } +} + +func TestCSVReaderBoundsRawRecordSyntaxBeforeFieldSliceGrowth(t *testing.T) { + reader, err := newImportCSVReaderWithLimits( + strings.NewReader("a\n,,,,,\n"), + importDelimiterComma, + importLexicalLimits{maxCellBytes: 8, maxRowBytes: 4}, + ) + if err != nil { + t.Fatalf("create CSV reader: %v", err) + } + reader.FieldsPerRecord = -1 + if _, err := reader.Read(); err != nil { + t.Fatalf("read header: %v", err) + } + _, err = reader.Read() + var limitErr *ImportFileLimitError + if !errors.As(err, &limitErr) || limitErr.Kind != ImportFileRowByteLimit || limitErr.Row != 2 { + t.Fatalf("raw row limit error = %#v (%v)", limitErr, err) + } +} + +func TestCSVReaderTracksCRLFAndEscapedQuoteAcrossReadBoundaries(t *testing.T) { + source := &importLexicalTestChunkReader{ + source: strings.NewReader("a;b\r\n1;\"x\"\"y\"\r\n"), + chunk: 1, + } + reader, err := newImportCSVReaderWithLimits( + source, + importDelimiterSemicolon, + importLexicalLimits{maxCellBytes: 4, maxRowBytes: 16}, + ) + if err != nil { + t.Fatalf("create CSV reader: %v", err) + } + reader.FieldsPerRecord = -1 + if _, err := reader.Read(); err != nil { + t.Fatalf("read header: %v", err) + } + row, err := reader.Read() + if err != nil { + t.Fatalf("read row: %v", err) + } + if len(row) != 2 || row[0] != "1" || row[1] != `x"y` { + t.Fatalf("row = %#v", row) + } +} diff --git a/internal/app/import_pipeline.go b/internal/app/import_pipeline.go index b744b72e..1771d7fd 100644 --- a/internal/app/import_pipeline.go +++ b/internal/app/import_pipeline.go @@ -1,35 +1,89 @@ package app import ( - "bufio" + "bytes" "context" "encoding/csv" "encoding/json" "errors" "fmt" "io" - "os" "sort" "strings" + "time" + "unicode/utf8" "GoNavi-Wails/internal/connection" "GoNavi-Wails/internal/db" + "GoNavi-Wails/internal/sqlaudit" + + mysqlDriver "github.com/go-sql-driver/mysql" ) const ( defaultImportPreviewLimit = 5 defaultImportApplyBatchSize = 1000 + maxImportErrorDetails = 20 + maxImportCellBytes = 16 * 1024 * 1024 + maxImportRowBytes = 64 * 1024 * 1024 + maxImportBatchBytes = 64 * 1024 * 1024 + importProgressRowInterval = 100 + importProgressTimeInterval = 250 * time.Millisecond ) +var errImportStoppedOnError = errors.New("import stopped on error") +var errImportPreviewLimitReached = errors.New("import preview limit reached") + +type importStoppedOnError struct { + detail string + cause error +} + +func (e *importStoppedOnError) Error() string { + if e == nil { + return errImportStoppedOnError.Error() + } + return e.detail +} + +func (e *importStoppedOnError) Unwrap() error { + if e != nil && e.cause != nil { + return errors.Join(errImportStoppedOnError, e.cause) + } + return errImportStoppedOnError +} + type importFileConsumer interface { SetColumns(columns []string) error ConsumeRow(row map[string]interface{}) error } +type importSourceProgressConsumer interface { + SetImportSourceProgress(bytesRead int64, totalBytes int64, stage string) +} + +type importByteCountingReader struct { + reader io.Reader + bytesRead int64 +} + +func (r *importByteCountingReader) Read(buffer []byte) (int, error) { + read, err := r.reader.Read(buffer) + r.bytesRead += int64(read) + return read, err +} + +func reportImportSourceProgress(consumer importFileConsumer, bytesRead int64, totalBytes int64) { + if progressConsumer, ok := consumer.(importSourceProgressConsumer); ok { + progressConsumer.SetImportSourceProgress(bytesRead, totalBytes, "parse") + } +} + type importPreviewData struct { - Columns []string - TotalRows int - PreviewRows []map[string]interface{} + Columns []string + TotalRows int + TotalRowsKnown bool + PreviewRows []map[string]interface{} } // ImportFileOptions controls how a selected import file is applied to the target table. @@ -37,8 +91,26 @@ type importPreviewData struct { // directly as database column names. A non-nil map enables explicit source-to-target // mapping; entries with an empty target are skipped. type ImportFileOptions struct { - ColumnMappings map[string]string `json:"columnMappings,omitempty"` - JobID string `json:"jobId,omitempty"` + ColumnMappings map[string]string `json:"columnMappings,omitempty"` + JobID string `json:"jobId,omitempty"` + ContinueOnError *bool `json:"continueOnError,omitempty"` + Encoding string `json:"encoding,omitempty"` + Delimiter string `json:"delimiter,omitempty"` + HeaderRow int `json:"headerRow,omitempty"` + NullToken *string `json:"nullToken,omitempty"` + EmptyStringAsNull bool `json:"emptyStringAsNull,omitempty"` + SheetName string `json:"sheetName,omitempty"` + SourceIdentityToken string `json:"sourceIdentityToken,omitempty"` + ConflictPolicy string `json:"conflictPolicy,omitempty"` + ConflictKeyColumns []string `json:"conflictKeyColumns,omitempty"` + ResumeJobID string `json:"resumeJobId,omitempty"` +} + +func resolveImportContinueOnError(options ImportFileOptions) bool { + // Keep the public compatibility entrypoint's historical continue behavior + // when the field is omitted. The workbench always sends an explicit value + // and defaults to fail-fast. + return options.ContinueOnError == nil || *options.ContinueOnError } type importProgressState struct { @@ -46,15 +118,24 @@ type importProgressState struct { Current int `json:"current"` Total int `json:"total,omitempty"` Success int `json:"success"` + Skipped int `json:"skipped,omitempty"` Errors int `json:"errors"` TotalRowsKnown bool `json:"totalRowsKnown,omitempty"` + BytesRead int64 `json:"bytesRead,omitempty"` + TotalBytes int64 `json:"totalBytes,omitempty"` + Stage string `json:"stage,omitempty"` } type importExecutionResult struct { - Success int - Failed int - Total int - ErrorLogs []string + Success int + Skipped int + Failed int + Total int + ErrorLogs []string + ErrorArtifactID string + ErrorArtifactCount int64 + StoppedOnError bool + OutcomeUnknown bool } type importPreviewCollector struct { @@ -81,6 +162,9 @@ func (c *importPreviewCollector) ConsumeRow(row map[string]interface{}) error { if len(c.previewRows) < c.previewLimit { c.previewRows = append(c.previewRows, cloneImportRow(row)) } + if len(c.previewRows) >= c.previewLimit { + return errImportPreviewLimitReached + } return nil } @@ -236,6 +320,15 @@ func (c *importColumnMappingConsumer) ConsumeRow(row map[string]interface{}) err return c.downstream.ConsumeRow(mappedRow) } +func (c *importColumnMappingConsumer) SetImportSourceProgress(bytesRead int64, totalBytes int64, stage string) { + if c == nil { + return + } + if progressConsumer, ok := c.downstream.(importSourceProgressConsumer); ok { + progressConsumer.SetImportSourceProgress(bytesRead, totalBytes, stage) + } +} + type importRowWriter interface { SetColumns(columns []string) ApplyBatch(rows []map[string]interface{}) error @@ -243,6 +336,36 @@ type importRowWriter interface { BatchEnabled() bool } +// importBatchContextWriter is the optional cancellation-aware batch extension. +// Keep it separate from the single-row extension so a writer can support one +// operation safely without having to implement an unrelated method. +type importBatchContextWriter interface { + ApplyBatchContext(ctx context.Context, rows []map[string]interface{}) error +} + +type importRowContextWriter interface { + ApplyOneContext(ctx context.Context, row map[string]interface{}) error +} + +type importRowApplyOutcome string + +const ( + importRowApplySucceeded importRowApplyOutcome = "succeeded" + importRowApplySkipped importRowApplyOutcome = "skipped" +) + +type importRowOutcomeWriter interface { + ApplyOneWithOutcome(row map[string]interface{}) (importRowApplyOutcome, error) +} + +type importRowContextOutcomeWriter interface { + ApplyOneWithOutcomeContext(ctx context.Context, row map[string]interface{}) (importRowApplyOutcome, error) +} + +type importRowColumnValidator interface { + ValidateColumns(columns []string) error +} + type importColumnTypeLookup struct { byExactName map[string]string byFoldedName map[string][]string @@ -279,20 +402,28 @@ func (l importColumnTypeLookup) Resolve(columnName string) string { } type importDatabaseRowWriter struct { - dbInst db.Database - applier db.BatchApplier - dbType string - tableName string - columns []string - columnTypes importColumnTypeLookup + dbInst db.Database + applier db.BatchApplier + dbType string + tableName string + columns []string + columnTypes importColumnTypeLookup + conflictPolicy string + conflictKeyColumns []string } func newImportDatabaseRowWriter(dbInst db.Database, dbType, tableName string, columnTypes importColumnTypeLookup) *importDatabaseRowWriter { + return newImportDatabaseRowWriterWithOptions(dbInst, dbType, tableName, columnTypes, ImportFileOptions{}) +} + +func newImportDatabaseRowWriterWithOptions(dbInst db.Database, dbType, tableName string, columnTypes importColumnTypeLookup, options ImportFileOptions) *importDatabaseRowWriter { writer := &importDatabaseRowWriter{ - dbInst: dbInst, - dbType: dbType, - tableName: tableName, - columnTypes: columnTypes, + dbInst: dbInst, + dbType: dbType, + tableName: tableName, + columnTypes: columnTypes, + conflictPolicy: normalizeImportConflictPolicy(options.ConflictPolicy), + conflictKeyColumns: append([]string(nil), options.ConflictKeyColumns...), } if applier, ok := dbInst.(db.BatchApplier); ok { writer.applier = applier @@ -305,54 +436,174 @@ func (w *importDatabaseRowWriter) SetColumns(columns []string) { } func (w *importDatabaseRowWriter) BatchEnabled() bool { - return w.applier != nil + return w.applier != nil && w.conflictPolicy == importConflictPolicyStop } func (w *importDatabaseRowWriter) ApplyBatch(rows []map[string]interface{}) error { + return w.ApplyBatchContext(context.Background(), rows) +} + +func (w *importDatabaseRowWriter) ApplyBatchContext(ctx context.Context, rows []map[string]interface{}) error { if w.applier == nil { return fmt.Errorf("当前数据库类型不支持批量提交") } - return w.applier.ApplyChanges(w.tableName, connection.ChangeSet{Inserts: cloneImportRows(rows)}) + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + changes := connection.ChangeSet{Inserts: cloneImportRows(rows)} + if contextApplier, ok := w.applier.(db.BatchApplierContext); ok { + return contextApplier.ApplyChangesContext(ctx, w.tableName, changes) + } + return w.applier.ApplyChanges(w.tableName, changes) } func (w *importDatabaseRowWriter) ApplyOne(row map[string]interface{}) error { - if w.applier != nil { - return w.applier.ApplyChanges(w.tableName, connection.ChangeSet{Inserts: []map[string]interface{}{cloneImportRow(row)}}) - } - query, err := buildImportInsertQuery(w.dbType, w.tableName, w.columns, row, w.columnTypes) - if err != nil { - return err - } - _, err = w.dbInst.Exec(query) + _, err := w.ApplyOneWithOutcomeContext(context.Background(), row) return err } -type importBatchConsumer struct { - writer importRowWriter - ctx context.Context - jobID string - batchSize int - totalRows int - totalRowsKnown bool - report func(importProgressState) - batch []map[string]interface{} - batchStartRow int - currentRow int - successCount int - errorLogs []string +func (w *importDatabaseRowWriter) ApplyOneContext(ctx context.Context, row map[string]interface{}) error { + _, err := w.ApplyOneWithOutcomeContext(ctx, row) + return err } -func newImportBatchConsumer(writer importRowWriter, batchSize int, totalRows int, totalRowsKnown bool, report func(importProgressState)) *importBatchConsumer { +func (w *importDatabaseRowWriter) ApplyOneWithOutcome(row map[string]interface{}) (importRowApplyOutcome, error) { + return w.ApplyOneWithOutcomeContext(context.Background(), row) +} + +func (w *importDatabaseRowWriter) ApplyOneWithOutcomeContext(ctx context.Context, row map[string]interface{}) (importRowApplyOutcome, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return importRowApplySucceeded, err + } + if w.applier != nil && w.conflictPolicy == importConflictPolicyStop { + changes := connection.ChangeSet{Inserts: []map[string]interface{}{cloneImportRow(row)}} + var err error + if contextApplier, ok := w.applier.(db.BatchApplierContext); ok { + err = contextApplier.ApplyChangesContext(ctx, w.tableName, changes) + } else { + err = w.applier.ApplyChanges(w.tableName, changes) + } + return importRowApplySucceeded, err + } + query, err := buildImportInsertQueryWithConflict( + w.dbType, + w.tableName, + w.columns, + row, + w.columnTypes, + w.conflictPolicy, + w.conflictKeyColumns, + ) + if err != nil { + return importRowApplySucceeded, err + } + var affected int64 + if contextExecer, ok := w.dbInst.(interface { + ExecContext(context.Context, string) (int64, error) + }); ok { + affected, err = contextExecer.ExecContext(ctx, query) + } else { + affected, err = w.dbInst.Exec(query) + } + if err != nil { + if w.conflictPolicy == importConflictPolicySkipDuplicates && isMySQLConflictDialect(w.dbType) && isMySQLDuplicateKeyError(err) { + return importRowApplySkipped, nil + } + if db.IsAmbiguousWriteResponse(err) || ctx.Err() != nil { + return importRowApplySucceeded, db.MarkWriteOutcomeUnknown(err) + } + return importRowApplySucceeded, err + } + if w.conflictPolicy == importConflictPolicySkipDuplicates && (isPostgresConflictDialect(w.dbType) || resolveDDLDBType(connection.ConnectionConfig{Type: w.dbType}) == "sqlite") && affected == 0 { + return importRowApplySkipped, nil + } + return importRowApplySucceeded, nil +} + +func (w *importDatabaseRowWriter) ValidateColumns(columns []string) error { + if w == nil || w.conflictPolicy != importConflictPolicyUpsert { + return nil + } + available := make(map[string]int, len(columns)) + for _, column := range columns { + available[normalizeColumnName(column)]++ + } + for _, key := range w.conflictKeyColumns { + matches := available[normalizeColumnName(key)] + if matches == 0 { + return fmt.Errorf("import conflict key column %q is not present in the selected import columns", key) + } + if matches > 1 { + return fmt.Errorf("import conflict key column %q is ambiguous in the selected import columns", key) + } + } + return nil +} + +func isMySQLDuplicateKeyError(err error) bool { + var mysqlError *mysqlDriver.MySQLError + return errors.As(err, &mysqlError) && (mysqlError.Number == 1062 || mysqlError.Number == 1586) +} + +type importBatchConsumer struct { + writer importRowWriter + ctx context.Context + jobID string + batchSize int + totalRows int + totalRowsKnown bool + continueOnError bool + onRowError func(ImportRowError) error + report func(importProgressState) + bytesRead int64 + totalBytes int64 + sourceStage string + batch []map[string]interface{} + batchBytes int + batchStartRow int + currentRow int + successCount int + skippedCount int + failedCount int + errorLogs []string + stoppedOnError bool + outcomeUnknown bool + lastProgressRow int + lastProgressAt time.Time +} + +func (c *importBatchConsumer) SetRowErrorHandler(handler func(ImportRowError) error) { + c.onRowError = handler +} + +func (c *importBatchConsumer) SetImportSourceProgress(bytesRead int64, totalBytes int64, stage string) { + if bytesRead >= 0 { + c.bytesRead = bytesRead + } + if totalBytes >= 0 { + c.totalBytes = totalBytes + } + c.sourceStage = strings.TrimSpace(stage) +} + +func newImportBatchConsumer(writer importRowWriter, batchSize int, totalRows int, totalRowsKnown bool, continueOnError bool, report func(importProgressState)) *importBatchConsumer { if batchSize <= 0 { batchSize = defaultImportApplyBatchSize } return &importBatchConsumer{ - writer: writer, - ctx: context.Background(), - batchSize: batchSize, - totalRows: totalRows, - totalRowsKnown: totalRowsKnown, - report: report, + writer: writer, + ctx: context.Background(), + batchSize: batchSize, + totalRows: totalRows, + totalRowsKnown: totalRowsKnown, + continueOnError: continueOnError, + report: report, } } @@ -376,6 +627,9 @@ func (c *importBatchConsumer) SetColumns(columns []string) error { } if c.writer != nil { c.writer.SetColumns(columns) + if validator, ok := c.writer.(importRowColumnValidator); ok { + return validator.ValidateColumns(columns) + } } return nil } @@ -384,12 +638,22 @@ func (c *importBatchConsumer) ConsumeRow(row map[string]interface{}) error { if err := c.contextError(); err != nil { return err } + rowBytes, err := validateImportMapRowBytes("Import", c.currentRow+1, row) + if err != nil { + return err + } + if len(c.batch) > 0 && c.batchBytes > maxImportBatchBytes-rowBytes { + if err := c.flush(); err != nil { + return err + } + } c.currentRow++ if len(c.batch) == 0 { c.batchStartRow = c.currentRow } c.batch = append(c.batch, cloneImportRow(row)) - if len(c.batch) >= c.batchSize { + c.batchBytes += rowBytes + if len(c.batch) >= c.batchSize || c.batchBytes >= maxImportBatchBytes { return c.flush() } return nil @@ -401,10 +665,20 @@ func (c *importBatchConsumer) Flush() error { func (c *importBatchConsumer) Result() importExecutionResult { return importExecutionResult{ - Success: c.successCount, - Failed: len(c.errorLogs), - Total: c.currentRow, - ErrorLogs: append([]string(nil), c.errorLogs...), + Success: c.successCount, + Skipped: c.skippedCount, + Failed: c.failedCount, + Total: c.currentRow, + ErrorLogs: append([]string(nil), c.errorLogs...), + StoppedOnError: c.stoppedOnError, + OutcomeUnknown: c.outcomeUnknown, + } +} + +func (c *importBatchConsumer) recordError(detail string) { + c.failedCount++ + if len(c.errorLogs) < maxImportErrorDetails { + c.errorLogs = append(c.errorLogs, detail) } } @@ -418,21 +692,35 @@ func (c *importBatchConsumer) flush() error { rows := c.batch startRow := c.batchStartRow c.batch = nil + c.batchBytes = 0 c.batchStartRow = 0 - if c.writer != nil && c.writer.BatchEnabled() { - batchErr := c.writer.ApplyBatch(rows) + if c.writer != nil && c.writer.BatchEnabled() && !c.continueOnError { + var batchErr error + if contextWriter, ok := c.writer.(importBatchContextWriter); ok { + batchErr = contextWriter.ApplyBatchContext(c.ctx, rows) + } else { + batchErr = c.writer.ApplyBatch(rows) + } if batchErr == nil { c.successCount += len(rows) - c.emitProgress(startRow + len(rows) - 1) + c.emitProgress(startRow+len(rows)-1, true) return c.contextError() } if errors.Is(batchErr, context.Canceled) { + c.outcomeUnknown = true return batchErr } if err := c.contextError(); err != nil { + c.outcomeUnknown = true return err } + detail := fmt.Sprintf("Rows %d-%d: %s", startRow, startRow+len(rows)-1, sqlaudit.RedactError(batchErr.Error())) + c.recordError(detail) + c.stoppedOnError = true + c.outcomeUnknown = true + c.emitProgress(startRow+len(rows)-1, true) + return &importStoppedOnError{detail: detail} } for idx, row := range rows { @@ -440,44 +728,113 @@ func (c *importBatchConsumer) flush() error { return err } if c.writer != nil { - if err := c.writer.ApplyOne(row); err != nil { + outcome := importRowApplySucceeded + var err error + if contextOutcomeWriter, ok := c.writer.(importRowContextOutcomeWriter); ok { + outcome, err = contextOutcomeWriter.ApplyOneWithOutcomeContext(c.ctx, row) + } else if outcomeWriter, ok := c.writer.(importRowOutcomeWriter); ok { + outcome, err = outcomeWriter.ApplyOneWithOutcome(row) + } else if contextWriter, ok := c.writer.(importRowContextWriter); ok { + err = contextWriter.ApplyOneContext(c.ctx, row) + } else { + err = c.writer.ApplyOne(row) + } + if err != nil { + if db.IsWriteOutcomeUnknown(err) { + sourceRow := startRow + idx + sanitizedMessage := sqlaudit.RedactError(err.Error()) + detail := fmt.Sprintf("Row %d: %s", sourceRow, sanitizedMessage) + c.recordError(detail) + c.stoppedOnError = true + c.outcomeUnknown = true + // Only rows through the uncertain write were submitted. Rows later in + // this parser buffer must not inflate the processed/unknown total. + c.currentRow = sourceRow + c.emitProgress(sourceRow, true) + return &importStoppedOnError{detail: detail, cause: err} + } if errors.Is(err, context.Canceled) { + c.outcomeUnknown = true return err } if contextErr := c.contextError(); contextErr != nil { + c.outcomeUnknown = true return contextErr } - c.errorLogs = append(c.errorLogs, fmt.Sprintf("Row %d: %s", startRow+idx, err.Error())) + sourceRow := startRow + idx + sanitizedMessage := sqlaudit.RedactError(err.Error()) + detail := fmt.Sprintf("Row %d: %s", sourceRow, sanitizedMessage) + c.recordError(detail) + if c.onRowError != nil { + if persistErr := c.onRowError(ImportRowError{ + SourceRow: int64(sourceRow), + Category: "database", + Message: sanitizedMessage, + Values: cloneImportRow(row), + }); persistErr != nil { + c.stoppedOnError = true + c.emitProgress(startRow+idx, true) + return persistErr + } + } + if !c.continueOnError { + c.stoppedOnError = true + c.emitProgress(startRow+idx, true) + return &importStoppedOnError{detail: detail} + } + } else if outcome == importRowApplySkipped { + c.skippedCount++ } else { c.successCount++ } } c.emitProgress(startRow + idx) if err := c.contextError(); err != nil { + c.emitProgress(startRow+idx, true) return err } } + c.emitProgress(startRow+len(rows)-1, true) return nil } -func (c *importBatchConsumer) emitProgress(current int) { +func (c *importBatchConsumer) emitProgress(current int, force ...bool) { if c.report == nil { return } + forced := len(force) > 0 && force[0] + if !forced && current > 10 && current-c.lastProgressRow < importProgressRowInterval && + !c.lastProgressAt.IsZero() && time.Since(c.lastProgressAt) < importProgressTimeInterval { + return + } + c.lastProgressRow = current + c.lastProgressAt = time.Now() c.report(importProgressState{ JobID: c.jobID, Current: current, Total: c.totalRows, Success: c.successCount, - Errors: len(c.errorLogs), + Skipped: c.skippedCount, + Errors: c.failedCount, TotalRowsKnown: c.totalRowsKnown, + BytesRead: c.bytesRead, + TotalBytes: c.totalBytes, + Stage: "write", }) } func buildImportPreview(filePath string, previewLimit int) (importPreviewData, error) { + return buildImportPreviewWithOptions(filePath, previewLimit, ImportFileOptions{}) +} + +func buildImportPreviewWithOptions(filePath string, previewLimit int, options ImportFileOptions) (importPreviewData, error) { collector := newImportPreviewCollector(previewLimit) - if err := streamImportFile(filePath, collector); err != nil { + if err := streamImportFileWithOptions(filePath, collector, options); err != nil && !errors.Is(err, errImportPreviewLimitReached) { return importPreviewData{}, err + } else if err == nil { + collectorResult := collector.Result() + collectorResult.TotalRowsKnown = true + return collectorResult, nil } return collector.Result(), nil } @@ -491,29 +848,132 @@ func parseImportFile(filePath string) ([]map[string]interface{}, []string, error } func streamImportFile(filePath string, consumer importFileConsumer) error { + return streamImportFileWithOptions(filePath, consumer, ImportFileOptions{}) +} + +func streamImportFileWithOptions(filePath string, consumer importFileConsumer, options ImportFileOptions) error { + if consumer == nil { + return fmt.Errorf("import file consumer is required") + } + if err := validateImportFileOptions(options); err != nil { + return err + } lower := strings.ToLower(filePath) switch { case strings.HasSuffix(lower, ".json"): - return streamJSONImportFile(filePath, consumer) + return streamJSONImportFileWithOptions(filePath, consumer, options) case strings.HasSuffix(lower, ".csv"): - return streamCSVImportFile(filePath, consumer) + return streamCSVImportFileWithOptions(filePath, consumer, options) case strings.HasSuffix(lower, ".xlsx"): - return streamXLSXImportFile(filePath, consumer) + return streamXLSXImportFileWithOptions(filePath, consumer, options) case strings.HasSuffix(lower, ".xls"): - return streamLegacyExcelImportFile(filePath, consumer) + return fmt.Errorf("legacy binary .xls workbooks are not supported; convert the file to .xlsx or CSV") default: return fmt.Errorf("Unsupported file format") } } +const ( + importConflictPolicyStop = "stop" + importConflictPolicySkipDuplicates = "skip_duplicates" + importConflictPolicyUpsert = "upsert" + maxImportNullTokenRunes = 64 + maxImportSheetNameRunes = 255 +) + +func normalizeImportConflictPolicy(value string) string { + policy := strings.ToLower(strings.TrimSpace(value)) + if policy == "" { + return importConflictPolicyStop + } + return policy +} + +func isMySQLConflictDialect(dbType string) bool { + switch resolveDDLDBType(connection.ConnectionConfig{Type: dbType}) { + case "mysql", "mariadb", "oceanbase": + return true + default: + return false + } +} + +func isPostgresConflictDialect(dbType string) bool { + return resolveDDLDBType(connection.ConnectionConfig{Type: dbType}) == "postgres" +} + +func validateImportConflictPolicyForDB(dbType string, options ImportFileOptions) error { + policy := normalizeImportConflictPolicy(options.ConflictPolicy) + if policy == importConflictPolicyStop { + return nil + } + if !isMySQLConflictDialect(dbType) && !isPostgresConflictDialect(dbType) && resolveDDLDBType(connection.ConnectionConfig{Type: dbType}) != "sqlite" { + return fmt.Errorf("import conflict policy %q is not supported for database type %q", policy, dbType) + } + if policy == importConflictPolicyUpsert && isMySQLConflictDialect(dbType) { + return fmt.Errorf("import upsert cannot safely target selected conflict keys for database type %q", dbType) + } + if policy == importConflictPolicyUpsert && len(options.ConflictKeyColumns) == 0 { + return fmt.Errorf("import upsert requires at least one conflict key column") + } + return nil +} + +func validateImportFileOptions(options ImportFileOptions) error { + if _, err := normalizeImportTextEncoding(options.Encoding); err != nil { + return fmt.Errorf("invalid import encoding: %w", err) + } + if _, _, err := resolveImportDelimiter(options.Delimiter); err != nil { + return fmt.Errorf("invalid import delimiter: %w", err) + } + if _, err := resolveImportHeaderRow(options.HeaderRow); err != nil { + return err + } + switch normalizeImportConflictPolicy(options.ConflictPolicy) { + case importConflictPolicyStop, importConflictPolicySkipDuplicates, importConflictPolicyUpsert: + default: + return fmt.Errorf("unsupported import conflictPolicy %q", options.ConflictPolicy) + } + if options.NullToken != nil { + if !utf8.ValidString(*options.NullToken) { + return fmt.Errorf("import nullToken must be valid UTF-8") + } + if utf8.RuneCountInString(*options.NullToken) > maxImportNullTokenRunes { + return fmt.Errorf("import nullToken exceeds %d-character limit", maxImportNullTokenRunes) + } + } + if !utf8.ValidString(options.SheetName) { + return fmt.Errorf("import sheetName must be valid UTF-8") + } + if utf8.RuneCountInString(options.SheetName) > maxImportSheetNameRunes { + return fmt.Errorf("import sheetName exceeds %d-character limit", maxImportSheetNameRunes) + } + seenConflictKeys := make(map[string]struct{}, len(options.ConflictKeyColumns)) + for _, column := range options.ConflictKeyColumns { + if strings.TrimSpace(column) == "" { + return fmt.Errorf("import conflictKeyColumns must not contain empty names") + } + normalizedColumn := normalizeColumnName(column) + if _, duplicate := seenConflictKeys[normalizedColumn]; duplicate { + return fmt.Errorf("import conflictKeyColumns contains duplicate column %q", column) + } + seenConflictKeys[normalizedColumn] = struct{}{} + } + return nil +} + func streamJSONImportFile(filePath string, consumer importFileConsumer) error { - f, err := os.Open(filePath) + return streamJSONImportFileWithOptions(filePath, consumer, ImportFileOptions{}) +} + +func streamJSONImportFileWithOptions(filePath string, consumer importFileConsumer, options ImportFileOptions) error { + source, err := openImportTextSource(filePath, options.Encoding) if err != nil { return err } - defer f.Close() + defer source.Close() - decoder := json.NewDecoder(bufio.NewReader(f)) + decoder := newImportJSONDecoderWithLimits(source, importLexicalLimits{}) token, err := decoder.Token() if err != nil { return fmt.Errorf("JSON Parse Error: %w", err) @@ -524,52 +984,116 @@ func streamJSONImportFile(filePath string, consumer importFileConsumer) error { } var columns []string + var columnSet map[string]struct{} + rowNumber := 0 for decoder.More() { + rowNumber++ var raw map[string]interface{} if err := decoder.Decode(&raw); err != nil { return fmt.Errorf("JSON Parse Error: %w", err) } + if _, err := validateImportMapRowBytes("JSON", rowNumber, raw); err != nil { + return err + } if columns == nil { columns = importJSONColumns(raw) + columnSet = make(map[string]struct{}, len(columns)) + for _, column := range columns { + columnSet[column] = struct{}{} + } if err := consumer.SetColumns(columns); err != nil { return err } + } else { + unknown := make([]string, 0) + for key := range raw { + if _, ok := columnSet[key]; !ok { + unknown = append(unknown, key) + } + } + if len(unknown) > 0 { + sort.Strings(unknown) + return fmt.Errorf("JSON Structure Drift at row %d: unknown fields %q", rowNumber, unknown) + } } - if err := consumer.ConsumeRow(normalizeImportMapRow(columns, raw)); err != nil { + reportImportSourceProgress(consumer, source.RawBytesRead(), source.TotalBytes()) + if err := consumer.ConsumeRow(normalizeImportMapRowWithOptions(columns, raw, options)); err != nil { return err } } - if _, err := decoder.Token(); err != nil { + closing, err := decoder.Token() + if err != nil { return fmt.Errorf("JSON Parse Error: %w", err) } + closingDelim, ok := closing.(json.Delim) + if !ok || closingDelim != ']' { + return fmt.Errorf("JSON Parse Error: root array is not closed") + } + var trailing interface{} + if err := decoder.Decode(&trailing); err == nil { + return fmt.Errorf("JSON Parse Error: trailing content after root array") + } else if !errors.Is(err, io.EOF) { + return fmt.Errorf("JSON Parse Error: trailing content after root array: %w", err) + } + reportImportSourceProgress(consumer, source.RawBytesRead(), source.TotalBytes()) return nil } func streamCSVImportFile(filePath string, consumer importFileConsumer) error { - f, err := os.Open(filePath) + return streamCSVImportFileWithOptions(filePath, consumer, ImportFileOptions{}) +} + +func streamCSVImportFileWithOptions(filePath string, consumer importFileConsumer, options ImportFileOptions) error { + source, err := openImportTextSource(filePath, options.Encoding) if err != nil { return err } - defer f.Close() + defer source.Close() - reader := csv.NewReader(bufio.NewReader(f)) - reader.ReuseRecord = true - - header, err := reader.Read() + reader, err := newImportCSVReader(source, options.Delimiter) if err != nil { - if err == io.EOF { - return fmt.Errorf("CSV empty or missing header") + return err + } + reader.ReuseRecord = true + reader.FieldsPerRecord = -1 + + headerRow, err := resolveImportHeaderRow(options.HeaderRow) + if err != nil { + return err + } + var header []string + for sourceRow := 1; sourceRow <= headerRow; sourceRow++ { + record, err := reader.Read() + if err != nil { + if err == io.EOF { + return fmt.Errorf("CSV header row %d is missing", headerRow) + } + return fmt.Errorf("CSV Parse Error: %w", err) } - return fmt.Errorf("CSV Parse Error: %w", err) + if err := validateImportStringCells("CSV", sourceRow, record); err != nil { + return err + } + if sourceRow == headerRow { + header = cloneImportColumns(record) + } + } + if len(header) > 0 { + header[0] = strings.TrimPrefix(header[0], "\uFEFF") } columns := cloneImportColumns(header) if !hasImportUsableColumns(columns) { return fmt.Errorf("CSV empty or missing header") } + if err := validateImportUniqueColumns("CSV", columns); err != nil { + return err + } if err := consumer.SetColumns(columns); err != nil { return err } + reader.FieldsPerRecord = len(columns) + reportImportSourceProgress(consumer, source.RawBytesRead(), source.TotalBytes()) + rowNumber := headerRow for { record, err := reader.Read() if err != nil { @@ -578,19 +1102,246 @@ func streamCSVImportFile(filePath string, consumer importFileConsumer) error { } return fmt.Errorf("CSV Parse Error: %w", err) } - if err := consumer.ConsumeRow(buildImportRowFromValues(columns, record)); err != nil { + rowNumber++ + if err := validateImportStringCells("CSV", rowNumber, record); err != nil { + return err + } + reportImportSourceProgress(consumer, source.RawBytesRead(), source.TotalBytes()) + if err := consumer.ConsumeRow(buildImportRowFromValuesWithOptions(columns, record, options)); err != nil { return err } } } +const maxImportHeaderRow = 1_000_000 + +func resolveImportHeaderRow(value int) (int, error) { + if value == 0 { + return 1, nil + } + if value < 1 || value > maxImportHeaderRow { + return 0, fmt.Errorf("import headerRow must be between 1 and %d", maxImportHeaderRow) + } + return value, nil +} + +const ( + importDelimiterAuto = "auto" + importDelimiterComma = "comma" + importDelimiterTab = "tab" + importDelimiterSemicolon = "semicolon" + importDelimiterPipe = "pipe" + importDelimiterProbeSize = 256 * 1024 +) + +var importDelimiterCandidates = []rune{',', '\t', ';', '|'} + +func newImportCSVReader(source io.Reader, delimiterName string) (*csv.Reader, error) { + return newImportCSVReaderWithLimits(source, delimiterName, importLexicalLimits{}) +} + +func resolveImportDelimiter(value string) (delimiter rune, explicit bool, err error) { + if value == "" || value == importDelimiterAuto { + return 0, false, nil + } + switch value { + case importDelimiterComma: + return ',', true, nil + case importDelimiterTab: + return '\t', true, nil + case importDelimiterSemicolon: + return ';', true, nil + case importDelimiterPipe: + return '|', true, nil + default: + return 0, false, fmt.Errorf("unsupported import delimiter %q", value) + } +} + +type importDelimiterProbeScore struct { + delimiter rune + records int + consistentRows int + fieldCount int +} + +func detectImportCSVDelimiter(prefix []byte) (rune, error) { + best := importDelimiterProbeScore{delimiter: ','} + tied := false + for _, delimiter := range importDelimiterCandidates { + score := scoreImportCSVDelimiter(prefix, delimiter) + if compareImportDelimiterScores(score, best) > 0 { + best = score + tied = false + } else if delimiter != best.delimiter && compareImportDelimiterScores(score, best) == 0 && score.consistentRows > 0 { + tied = true + } + } + if tied { + return 0, fmt.Errorf("CSV delimiter probe is ambiguous; specify delimiter explicitly") + } + if best.consistentRows == 0 { + // Preserve single-column CSV compatibility when no supported delimiter + // appears outside quoted fields. + return ',', nil + } + return best.delimiter, nil +} + +func scoreImportCSVDelimiter(prefix []byte, delimiter rune) importDelimiterProbeScore { + reader := csv.NewReader(bytes.NewReader(prefix)) + reader.Comma = delimiter + reader.FieldsPerRecord = -1 + widthCounts := make(map[int]int) + score := importDelimiterProbeScore{delimiter: delimiter} + for score.records < 32 { + record, err := reader.Read() + if err != nil { + break + } + score.records++ + if len(record) > 1 { + widthCounts[len(record)]++ + } + } + for width, count := range widthCounts { + if count > score.consistentRows || (count == score.consistentRows && width < score.fieldCount) { + score.consistentRows = count + score.fieldCount = width + } + } + return score +} + +func compareImportDelimiterScores(left, right importDelimiterProbeScore) int { + if left.consistentRows != right.consistentRows { + return left.consistentRows - right.consistentRows + } + if left.records != right.records { + return left.records - right.records + } + return 0 +} + +func validateImportStringCells(format string, rowNumber int, values []string) error { + totalBytes := 0 + for idx, value := range values { + if len(value) > maxImportCellBytes { + return &ImportFileLimitError{ + Format: format, + Kind: ImportFileCellByteLimit, + Row: rowNumber, + Cell: idx + 1, + Limit: maxImportCellBytes, + } + } + if totalBytes > maxImportRowBytes-len(value) { + return &ImportFileLimitError{ + Format: format, + Kind: ImportFileRowByteLimit, + Row: rowNumber, + Limit: maxImportRowBytes, + } + } + totalBytes += len(value) + } + return nil +} + +func validateImportMapRowBytes(format string, rowNumber int, row map[string]interface{}) (int, error) { + totalBytes := 0 + for column, value := range row { + valueBytes := importValueByteSize(value) + if valueBytes > maxImportCellBytes { + return 0, &ImportFileLimitError{ + Format: format, + Kind: ImportFileCellByteLimit, + Row: rowNumber, + Column: column, + Limit: maxImportCellBytes, + } + } + if totalBytes > maxImportRowBytes-valueBytes { + return 0, &ImportFileLimitError{ + Format: format, + Kind: ImportFileRowByteLimit, + Row: rowNumber, + Limit: maxImportRowBytes, + } + } + totalBytes += valueBytes + } + return totalBytes, nil +} + +func importValueByteSize(value interface{}) int { + switch typed := value.(type) { + case nil: + return 0 + case string: + return len(typed) + case []byte: + return len(typed) + case json.Number: + return len(typed.String()) + case bool: + if typed { + return len("true") + } + return len("false") + case []interface{}: + total := 0 + for _, item := range typed { + total += importValueByteSize(item) + } + return total + case map[string]interface{}: + total := 0 + for key, item := range typed { + total += len(key) + importValueByteSize(item) + } + return total + default: + return len(fmt.Sprintf("%v", typed)) + } +} + func buildImportInsertQuery(dbType, tableName string, columns []string, row map[string]interface{}, columnTypes importColumnTypeLookup) (string, error) { + return buildImportInsertQueryWithConflict( + dbType, + tableName, + columns, + row, + columnTypes, + importConflictPolicyStop, + nil, + ) +} + +func buildImportInsertQueryWithConflict( + dbType string, + tableName string, + columns []string, + row map[string]interface{}, + columnTypes importColumnTypeLookup, + conflictPolicy string, + conflictKeyColumns []string, +) (string, error) { + conflictPolicy = normalizeImportConflictPolicy(conflictPolicy) + if err := validateImportConflictPolicyForDB(dbType, ImportFileOptions{ + ConflictPolicy: conflictPolicy, + ConflictKeyColumns: conflictKeyColumns, + }); err != nil { + return "", err + } quotedCols := make([]string, 0, len(columns)) values := make([]string, 0, len(columns)) + usableColumns := make([]string, 0, len(columns)) for _, column := range columns { if strings.TrimSpace(column) == "" { continue } + usableColumns = append(usableColumns, column) quotedCols = append(quotedCols, quoteIdentByType(dbType, column)) colType := columnTypes.Resolve(column) values = append(values, formatImportSQLValue(dbType, colType, row[column])) @@ -598,10 +1349,65 @@ func buildImportInsertQuery(dbType, tableName string, columns []string, row map[ if len(quotedCols) == 0 { return "", fmt.Errorf("导入文件缺少有效列头") } - return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", + query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", quoteQualifiedIdentByType(dbType, tableName), strings.Join(quotedCols, ", "), - strings.Join(values, ", ")), nil + strings.Join(values, ", ")) + if conflictPolicy == importConflictPolicyStop { + return query, nil + } + if conflictPolicy == importConflictPolicySkipDuplicates { + if isMySQLConflictDialect(dbType) { + // MySQL duplicates are classified by their typed error code after a + // normal INSERT. INSERT IGNORE would also hide truncation and NOT NULL + // errors, so it is deliberately not used here. + return query, nil + } + return query + " ON CONFLICT DO NOTHING", nil + } + + keySet := make(map[string]struct{}, len(conflictKeyColumns)) + availableColumns := make(map[string][]string, len(usableColumns)) + for _, column := range usableColumns { + normalizedColumn := normalizeColumnName(column) + availableColumns[normalizedColumn] = append(availableColumns[normalizedColumn], column) + } + quotedKeys := make([]string, 0, len(conflictKeyColumns)) + for _, key := range conflictKeyColumns { + normalizedKey := normalizeColumnName(key) + matches := availableColumns[normalizedKey] + if len(matches) == 0 { + return "", fmt.Errorf("import conflict key column %q is not present in the selected import columns", key) + } + if len(matches) > 1 { + return "", fmt.Errorf("import conflict key column %q is ambiguous in the selected import columns", key) + } + keySet[normalizedKey] = struct{}{} + quotedKeys = append(quotedKeys, quoteIdentByType(dbType, matches[0])) + } + assignments := make([]string, 0, len(usableColumns)) + for _, column := range usableColumns { + if _, key := keySet[normalizeColumnName(column)]; key { + continue + } + quoted := quoteIdentByType(dbType, column) + if isMySQLConflictDialect(dbType) { + assignments = append(assignments, quoted+"=VALUES("+quoted+")") + } else { + assignments = append(assignments, quoted+"=EXCLUDED."+quoted) + } + } + if isMySQLConflictDialect(dbType) { + if len(assignments) == 0 { + quoted := quotedKeys[0] + assignments = append(assignments, quoted+"=VALUES("+quoted+")") + } + return query + " ON DUPLICATE KEY UPDATE " + strings.Join(assignments, ", "), nil + } + if len(assignments) == 0 { + return query + " ON CONFLICT (" + strings.Join(quotedKeys, ", ") + ") DO NOTHING", nil + } + return query + " ON CONFLICT (" + strings.Join(quotedKeys, ", ") + ") DO UPDATE SET " + strings.Join(assignments, ", "), nil } func importJSONColumns(row map[string]interface{}) []string { @@ -629,7 +1435,26 @@ func hasImportUsableColumns(columns []string) bool { return false } +func validateImportUniqueColumns(format string, columns []string) error { + seen := make(map[string]string, len(columns)) + for _, column := range columns { + normalized := normalizeColumnName(column) + if normalized == "" { + continue + } + if previous, exists := seen[normalized]; exists { + return fmt.Errorf("%s duplicate header columns %q and %q", format, previous, column) + } + seen[normalized] = column + } + return nil +} + func buildImportRowFromValues(columns []string, values []string) map[string]interface{} { + return buildImportRowFromValuesWithOptions(columns, values, ImportFileOptions{}) +} + +func buildImportRowFromValuesWithOptions(columns []string, values []string, options ImportFileOptions) map[string]interface{} { row := make(map[string]interface{}, len(columns)) for idx, column := range columns { if strings.TrimSpace(column) == "" { @@ -639,20 +1464,40 @@ func buildImportRowFromValues(columns []string, values []string) map[string]inte row[column] = nil continue } - if values[idx] == "NULL" { - row[column] = nil - continue - } - row[column] = values[idx] + row[column] = normalizeImportStringValue(values[idx], options) } return row } +func normalizeImportStringValue(value string, options ImportFileOptions) interface{} { + if options.NullToken != nil { + if value == *options.NullToken { + return nil + } + } else if value == "NULL" { + // Preserve the legacy import wrapper's historical NULL convention when + // no explicit token was supplied. + return nil + } + if options.EmptyStringAsNull && value == "" { + return nil + } + return value +} + func normalizeImportMapRow(columns []string, raw map[string]interface{}) map[string]interface{} { + return normalizeImportMapRowWithOptions(columns, raw, ImportFileOptions{}) +} + +func normalizeImportMapRowWithOptions(columns []string, raw map[string]interface{}, options ImportFileOptions) map[string]interface{} { row := make(map[string]interface{}, len(columns)) for _, column := range columns { if value, ok := raw[column]; ok { - row[column] = value + if text, isText := value.(string); isText { + row[column] = normalizeImportStringValue(text, options) + } else { + row[column] = value + } continue } row[column] = nil diff --git a/internal/app/import_pipeline_parser_test.go b/internal/app/import_pipeline_parser_test.go new file mode 100644 index 00000000..37384b91 --- /dev/null +++ b/internal/app/import_pipeline_parser_test.go @@ -0,0 +1,716 @@ +package app + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" +) + +type importBatchSizeRecorder struct { + batchSizes []int + batchErr error + singleErr error +} + +type importSourceProgressSnapshot struct { + bytesRead int64 + totalBytes int64 + stage string +} + +type importSourceProgressRecorder struct { + importCollectConsumer + progress []importSourceProgressSnapshot +} + +func (c *importSourceProgressRecorder) SetImportSourceProgress(bytesRead int64, totalBytes int64, stage string) { + c.progress = append(c.progress, importSourceProgressSnapshot{ + bytesRead: bytesRead, + totalBytes: totalBytes, + stage: stage, + }) +} + +func (w *importBatchSizeRecorder) SetColumns([]string) {} + +func (w *importBatchSizeRecorder) ApplyBatch(rows []map[string]interface{}) error { + w.batchSizes = append(w.batchSizes, len(rows)) + return w.batchErr +} + +func (w *importBatchSizeRecorder) ApplyOne(map[string]interface{}) error { return w.singleErr } + +func (w *importBatchSizeRecorder) BatchEnabled() bool { return true } + +func TestBuildImportPreviewStopsAtLimitAndMarksTotalUnknown(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.csv") + content := "id,name\n" + + "1,user_1\n" + + "2,user_2\n" + + "3,user_3\n" + + "4,user_4\n" + + "5,user_5\n" + + "6,\"unterminated\n" + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write csv: %v", err) + } + + preview, err := buildImportPreview(path, 5) + if err != nil { + t.Fatalf("preview should stop before malformed tail: %v", err) + } + if preview.TotalRows != 5 || len(preview.PreviewRows) != 5 { + t.Fatalf("preview rows = total %d, retained %d; want 5 and 5", preview.TotalRows, len(preview.PreviewRows)) + } + + known := reflect.ValueOf(preview).FieldByName("TotalRowsKnown") + if !known.IsValid() { + t.Fatal("preview result must expose TotalRowsKnown") + } + if known.Bool() { + t.Fatal("short-circuited preview must mark total rows unknown") + } +} + +func TestBuildImportPreviewWithOptionsUsesConfiguredParser(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.csv") + content := "exported by GoNavi\nid;name\n1;alice\n" + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write CSV: %v", err) + } + preview, err := buildImportPreviewWithOptions( + path, + 5, + ImportFileOptions{Delimiter: "semicolon", HeaderRow: 2}, + ) + if err != nil { + t.Fatalf("build configured preview: %v", err) + } + if !reflect.DeepEqual(preview.Columns, []string{"id", "name"}) { + t.Fatalf("columns = %#v", preview.Columns) + } + if !reflect.DeepEqual(preview.PreviewRows, []map[string]interface{}{{"id": "1", "name": "alice"}}) { + t.Fatalf("preview rows = %#v", preview.PreviewRows) + } +} + +func TestTextImportStreamsReportSourceByteProgress(t *testing.T) { + tests := []struct { + name string + ext string + content string + }{ + {name: "CSV", ext: ".csv", content: "id,name\n1,alice\n2,bob\n"}, + {name: "JSON", ext: ".json", content: `[{"id":1},{"id":2}]`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows"+test.ext) + if err := os.WriteFile(path, []byte(test.content), 0o600); err != nil { + t.Fatalf("write import file: %v", err) + } + consumer := &importSourceProgressRecorder{} + if err := streamImportFile(path, consumer); err != nil { + t.Fatalf("stream import file: %v", err) + } + if len(consumer.progress) == 0 { + t.Fatal("source byte progress was not reported") + } + last := consumer.progress[len(consumer.progress)-1] + if last.bytesRead != int64(len(test.content)) || last.totalBytes != int64(len(test.content)) || last.stage != "parse" { + t.Fatalf("last source progress = %#v, want full parse bytes", last) + } + }) + } +} + +func TestCSVImportStripsUTF8BOMFromFirstHeader(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.csv") + if err := os.WriteFile(path, []byte("\xEF\xBB\xBFid,name\n1,alice\n"), 0o600); err != nil { + t.Fatalf("write csv: %v", err) + } + + preview, err := buildImportPreview(path, 5) + if err != nil { + t.Fatalf("build preview: %v", err) + } + if !reflect.DeepEqual(preview.Columns, []string{"id", "name"}) { + t.Fatalf("columns = %#v, want BOM-free header", preview.Columns) + } + if got := preview.PreviewRows[0]["id"]; got != "1" { + t.Fatalf("id = %#v, want 1", got) + } +} + +func TestCSVImportRejectsDuplicateHeadersBeforeRowsAreMapped(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.csv") + if err := os.WriteFile(path, []byte("id, ID \n1,2\n"), 0o600); err != nil { + t.Fatalf("write csv: %v", err) + } + + err := streamImportFile(path, &importCollectConsumer{}) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "duplicate") { + t.Fatalf("duplicate header error = %v", err) + } +} + +func TestCSVImportDecodesUTF16LEBOMWithAutoEncoding(t *testing.T) { + raw, _, err := transform.Bytes( + unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder(), + []byte("id,name\n1,张三\n"), + ) + if err != nil { + t.Fatalf("encode UTF-16LE fixture: %v", err) + } + path := filepath.Join(t.TempDir(), "rows.csv") + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatalf("write CSV: %v", err) + } + + consumer := &importSourceProgressRecorder{} + if err := streamImportFileWithOptions(path, consumer, ImportFileOptions{Encoding: "auto"}); err != nil { + t.Fatalf("stream UTF-16LE CSV: %v", err) + } + if !reflect.DeepEqual(consumer.columns, []string{"id", "name"}) { + t.Fatalf("columns = %#v", consumer.columns) + } + if got := consumer.rows[0]["name"]; got != "张三" { + t.Fatalf("name = %#v, want 张三", got) + } + last := consumer.progress[len(consumer.progress)-1] + if last.bytesRead != int64(len(raw)) || last.totalBytes != int64(len(raw)) { + t.Fatalf("raw progress = %#v, want %d bytes", last, len(raw)) + } +} + +func TestCSVImportDecodesExplicitUTF16BEWithoutBOM(t *testing.T) { + raw, _, err := transform.Bytes( + unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM).NewEncoder(), + []byte("id,name\n1,李四\n"), + ) + if err != nil { + t.Fatalf("encode UTF-16BE fixture: %v", err) + } + path := filepath.Join(t.TempDir(), "rows.csv") + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatalf("write CSV: %v", err) + } + + consumer := &importCollectConsumer{} + if err := streamImportFileWithOptions(path, consumer, ImportFileOptions{Encoding: "utf-16be"}); err != nil { + t.Fatalf("stream UTF-16BE CSV: %v", err) + } + if got := consumer.rows[0]["name"]; got != "李四" { + t.Fatalf("name = %#v, want 李四", got) + } +} + +func TestCSVImportAutoEncodingFallsBackToGB18030(t *testing.T) { + raw, _, err := transform.Bytes( + simplifiedchinese.GB18030.NewEncoder(), + []byte("id,name\n1,张三\n"), + ) + if err != nil { + t.Fatalf("encode GB18030 fixture: %v", err) + } + path := filepath.Join(t.TempDir(), "rows.csv") + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatalf("write CSV: %v", err) + } + + consumer := &importCollectConsumer{} + if err := streamImportFileWithOptions(path, consumer, ImportFileOptions{Encoding: "auto"}); err != nil { + t.Fatalf("stream GB18030 CSV: %v", err) + } + if got := consumer.rows[0]["name"]; got != "张三" { + t.Fatalf("name = %#v, want 张三", got) + } +} + +func TestOpenImportTextSourceBoundsAutomaticEncodingDetection(t *testing.T) { + const detectionSampleBytes = 1 << 20 + path := filepath.Join(t.TempDir(), "large.csv") + content := append(bytes.Repeat([]byte("a"), detectionSampleBytes+1), 0xff) + if err := os.WriteFile(path, content, 0o600); err != nil { + t.Fatalf("write CSV: %v", err) + } + + source, err := openImportTextSource(path, "auto") + if err != nil { + t.Fatalf("open import text source: %v", err) + } + defer source.Close() + if source.encoding != importTextEncodingUTF8 { + t.Fatalf("encoding = %q, want bounded UTF-8 detection", source.encoding) + } +} + +func TestCSVImportAutoDetectsTabWithoutCountingQuotedComma(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.csv") + content := "id\tdescription\n1\t\"alpha,beta\"\n" + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write CSV: %v", err) + } + + consumer := &importCollectConsumer{} + if err := streamImportFileWithOptions(path, consumer, ImportFileOptions{Delimiter: "auto"}); err != nil { + t.Fatalf("stream tab-delimited CSV: %v", err) + } + if !reflect.DeepEqual(consumer.columns, []string{"id", "description"}) { + t.Fatalf("columns = %#v", consumer.columns) + } + if got := consumer.rows[0]["description"]; got != "alpha,beta" { + t.Fatalf("description = %#v, want quoted comma preserved", got) + } +} + +func TestCSVImportAutoDelimiterFailsClosedWhenAmbiguous(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.csv") + content := "left|right,third\n1|2,3\n" + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write CSV: %v", err) + } + err := streamImportFileWithOptions(path, &importCollectConsumer{}, ImportFileOptions{Delimiter: "auto"}) + if err == nil || !strings.Contains(strings.ToLower(err.Error()), "ambiguous") { + t.Fatalf("ambiguous delimiter error = %v", err) + } +} + +func TestCSVImportUsesConfiguredHeaderRow(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.csv") + content := "exported by GoNavi\nid;name\n1;alice\n" + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write CSV: %v", err) + } + + consumer := &importCollectConsumer{} + options := ImportFileOptions{Delimiter: "semicolon", HeaderRow: 2} + if err := streamImportFileWithOptions(path, consumer, options); err != nil { + t.Fatalf("stream CSV with second header row: %v", err) + } + if !reflect.DeepEqual(consumer.columns, []string{"id", "name"}) { + t.Fatalf("columns = %#v", consumer.columns) + } + if !reflect.DeepEqual(consumer.rows, []map[string]interface{}{{"id": "1", "name": "alice"}}) { + t.Fatalf("rows = %#v", consumer.rows) + } +} + +func TestCSVImportNormalizesConfiguredNullTokenAndEmptyString(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.csv") + if err := os.WriteFile(path, []byte("id,marker,note\n1,\\N,\n"), 0o600); err != nil { + t.Fatalf("write CSV: %v", err) + } + nullToken := "\\N" + consumer := &importCollectConsumer{} + options := ImportFileOptions{NullToken: &nullToken, EmptyStringAsNull: true} + if err := streamImportFileWithOptions(path, consumer, options); err != nil { + t.Fatalf("stream CSV with null options: %v", err) + } + row := consumer.rows[0] + if row["marker"] != nil || row["note"] != nil { + t.Fatalf("normalized row = %#v, want marker and note null", row) + } +} + +func TestImportFileOptionsRejectUnknownConflictPolicy(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.csv") + if err := os.WriteFile(path, []byte("id\n1\n"), 0o600); err != nil { + t.Fatalf("write CSV: %v", err) + } + err := streamImportFileWithOptions( + path, + &importCollectConsumer{}, + ImportFileOptions{ConflictPolicy: "overwrite_everything"}, + ) + if err == nil || !strings.Contains(err.Error(), "conflictPolicy") { + t.Fatalf("unknown conflict policy error = %v", err) + } +} + +func TestImportFileOptionsRejectUnknownParserEnums(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.csv") + if err := os.WriteFile(path, []byte("id\n1\n"), 0o600); err != nil { + t.Fatalf("write CSV: %v", err) + } + tests := []struct { + name string + options ImportFileOptions + }{ + {name: "encoding", options: ImportFileOptions{Encoding: "windows-1252"}}, + {name: "delimiter", options: ImportFileOptions{Delimiter: "colon"}}, + {name: "header row", options: ImportFileOptions{HeaderRow: maxImportHeaderRow + 1}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := streamImportFileWithOptions(path, &importCollectConsumer{}, test.options); err == nil { + t.Fatalf("options %#v were accepted", test.options) + } + }) + } +} + +func TestImportFileOptionsExposeStableJSONNames(t *testing.T) { + nullToken := "\\N" + encoded, err := json.Marshal(ImportFileOptions{ + Encoding: "gb18030", + Delimiter: "pipe", + HeaderRow: 2, + NullToken: &nullToken, + EmptyStringAsNull: true, + SheetName: "Data", + SourceIdentityToken: "source-v1", + ConflictPolicy: "upsert", + ConflictKeyColumns: []string{"id"}, + ResumeJobID: "job-v1", + }) + if err != nil { + t.Fatalf("marshal options: %v", err) + } + var payload map[string]interface{} + if err := json.Unmarshal(encoded, &payload); err != nil { + t.Fatalf("unmarshal options payload: %v", err) + } + for _, key := range []string{ + "encoding", "delimiter", "headerRow", "nullToken", "emptyStringAsNull", + "sheetName", "sourceIdentityToken", "conflictPolicy", "conflictKeyColumns", "resumeJobId", + } { + if _, ok := payload[key]; !ok { + t.Fatalf("options JSON %s missing key %q", encoded, key) + } + } +} + +func TestImportFileRejectsLegacyBinaryXLSWithClearError(t *testing.T) { + path := filepath.Join(t.TempDir(), "legacy.xls") + if err := os.WriteFile(path, []byte("legacy binary workbook"), 0o600); err != nil { + t.Fatalf("write XLS fixture: %v", err) + } + err := streamImportFile(path, &importCollectConsumer{}) + if err == nil || !strings.Contains(err.Error(), ".xls") || !strings.Contains(strings.ToLower(err.Error()), "not supported") { + t.Fatalf("legacy XLS error = %v", err) + } +} + +func TestCSVImportRejectsCellOverByteLimit(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.csv") + content := "id,payload\n1," + strings.Repeat("x", 16*1024*1024+1) + "\n" + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write csv: %v", err) + } + + _, err := buildImportPreview(path, 5) + if err == nil { + t.Fatal("oversized CSV cell must be rejected") + } + if !strings.Contains(err.Error(), "cell 2 exceeds") { + t.Fatalf("unexpected oversized-cell error: %v", err) + } +} + +func TestImportStringRowRejectsCombinedBytesOverLimit(t *testing.T) { + value := strings.Repeat("x", 13*1024*1024) + err := validateImportStringCells("CSV", 1, []string{value, value, value, value, value}) + if err == nil { + t.Fatal("row whose cells exceed the combined byte limit must be rejected") + } + if !strings.Contains(err.Error(), "row 1 exceeds") { + t.Fatalf("unexpected oversized-row error: %v", err) + } +} + +func TestImportBatchFlushesAtByteLimitBeforeRowLimit(t *testing.T) { + writer := &importBatchSizeRecorder{} + consumer := newImportBatchConsumer(writer, 1000, 7, true, false, nil) + value := strings.Repeat("x", 10*1024*1024) + for row := 1; row <= 7; row++ { + if err := consumer.ConsumeRow(map[string]interface{}{"payload": value}); err != nil { + t.Fatalf("consume row %d: %v", row, err) + } + } + if err := consumer.Flush(); err != nil { + t.Fatalf("flush: %v", err) + } + if !reflect.DeepEqual(writer.batchSizes, []int{6, 1}) { + t.Fatalf("batch sizes = %#v, want byte-bounded batches [6 1]", writer.batchSizes) + } +} + +func TestImportBatchConsumerReportsStructuredSanitizedRowError(t *testing.T) { + writer := &importBatchSizeRecorder{ + singleErr: errors.New("duplicate key value is (alice@example.com); password=secret-token-123"), + } + consumer := newImportBatchConsumer(writer, 1, 1, true, true, nil) + var reported []ImportRowError + consumer.SetRowErrorHandler(func(rowError ImportRowError) error { + reported = append(reported, rowError) + return nil + }) + row := map[string]interface{}{"id": 7, "name": "alice"} + + if err := consumer.ConsumeRow(row); err != nil { + t.Fatalf("consume row: %v", err) + } + row["name"] = "mutated" + if len(reported) != 1 { + t.Fatalf("reported row errors = %d, want 1", len(reported)) + } + got := reported[0] + if got.SourceRow != 1 || got.Category == "" { + t.Fatalf("unexpected row error identity: %#v", got) + } + if strings.Contains(got.Message, "alice@example.com") || strings.Contains(got.Message, "secret-token-123") { + t.Fatalf("row error message was not sanitized: %q", got.Message) + } + if !reflect.DeepEqual(got.Values, map[string]interface{}{"id": 7, "name": "alice"}) { + t.Fatalf("row error values = %#v, want cloned source values", got.Values) + } +} + +func TestImportBatchConsumerDoesNotInventRowErrorForUnknownBatchOutcome(t *testing.T) { + writer := &importBatchSizeRecorder{batchErr: errors.New("password=secret-token-123")} + consumer := newImportBatchConsumer(writer, 1, 1, true, false, nil) + reportedRows := 0 + consumer.SetRowErrorHandler(func(ImportRowError) error { + reportedRows++ + return nil + }) + + err := consumer.ConsumeRow(map[string]interface{}{"id": 7}) + if !errors.Is(err, errImportStoppedOnError) { + t.Fatalf("consume row error = %v, want errImportStoppedOnError", err) + } + if reportedRows != 0 { + t.Fatalf("unknown batch outcome reported %d concrete row errors", reportedRows) + } + result := consumer.Result() + if !result.OutcomeUnknown || len(result.ErrorLogs) != 1 || !strings.Contains(result.ErrorLogs[0], "Rows 1-1") { + t.Fatalf("unexpected unknown batch result: %#v", result) + } + if strings.Contains(result.ErrorLogs[0], "secret-token-123") { + t.Fatalf("batch range error was not sanitized: %q", result.ErrorLogs[0]) + } +} + +func TestImportBatchConsumerStopsWhenRejectedRowPersistenceFails(t *testing.T) { + writer := &importBatchSizeRecorder{singleErr: errors.New("duplicate key")} + consumer := newImportBatchConsumer(writer, 2, 2, true, true, nil) + persistErr := errors.New("rejected-row artifact is full") + callbackCalls := 0 + consumer.SetRowErrorHandler(func(ImportRowError) error { + callbackCalls++ + return persistErr + }) + + if err := consumer.ConsumeRow(map[string]interface{}{"id": 1}); err != nil { + t.Fatalf("buffer first row: %v", err) + } + err := consumer.ConsumeRow(map[string]interface{}{"id": 2}) + if !errors.Is(err, persistErr) { + t.Fatalf("consume row error = %v, want persistence failure", err) + } + if callbackCalls != 1 { + t.Fatalf("row error callbacks = %d, want 1", callbackCalls) + } + result := consumer.Result() + if !result.StoppedOnError || result.Failed != 1 || result.Success != 0 { + t.Fatalf("unexpected persistence-stop result: %#v", result) + } +} + +func TestImportBatchConsumerThrottlesLargeSequentialProgress(t *testing.T) { + writer := &importBatchSizeRecorder{} + progress := make([]importProgressState, 0) + consumer := newImportBatchConsumer(writer, 1000, 1000, true, true, func(state importProgressState) { + progress = append(progress, state) + }) + for row := 1; row <= 1000; row++ { + if err := consumer.ConsumeRow(map[string]interface{}{"id": row}); err != nil { + t.Fatalf("consume row %d: %v", row, err) + } + } + if err := consumer.Flush(); err != nil { + t.Fatal(err) + } + if len(progress) >= 100 { + t.Fatalf("progress events = %d, want bounded reporting", len(progress)) + } + if len(progress) == 0 || progress[len(progress)-1].Current != 1000 { + t.Fatalf("final progress = %#v, want row 1000", progress) + } +} + +func TestImportProgressStateExposesByteAndStageSeams(t *testing.T) { + encoded, err := json.Marshal(importProgressState{ + BytesRead: 12, + TotalBytes: 24, + Stage: "write", + }) + if err != nil { + t.Fatalf("marshal progress state: %v", err) + } + text := string(encoded) + for _, want := range []string{`"bytesRead":12`, `"totalBytes":24`, `"stage":"write"`} { + if !strings.Contains(text, want) { + t.Fatalf("progress payload %s missing %s", text, want) + } + } +} + +func TestImportBatchConsumerProgressUsesWriteStage(t *testing.T) { + writer := &importBatchSizeRecorder{} + var progress []importProgressState + consumer := newImportBatchConsumer(writer, 1, 1, true, false, func(state importProgressState) { + progress = append(progress, state) + }) + + if err := consumer.ConsumeRow(map[string]interface{}{"id": 1}); err != nil { + t.Fatalf("consume row: %v", err) + } + if len(progress) != 1 || progress[0].Stage != "write" { + t.Fatalf("progress = %#v, want one write-stage event", progress) + } +} + +func TestImportBatchConsumerProgressCarriesSourceBytes(t *testing.T) { + writer := &importBatchSizeRecorder{} + var progress []importProgressState + consumer := newImportBatchConsumer(writer, 1, 1, true, false, func(state importProgressState) { + progress = append(progress, state) + }) + consumer.SetImportSourceProgress(12, 24, "parse") + + if err := consumer.ConsumeRow(map[string]interface{}{"id": 1}); err != nil { + t.Fatalf("consume row: %v", err) + } + if len(progress) != 1 || progress[0].BytesRead != 12 || progress[0].TotalBytes != 24 { + t.Fatalf("progress = %#v, want source byte counters", progress) + } +} + +func TestImportColumnMappingConsumerForwardsSourceProgress(t *testing.T) { + downstream := &importSourceProgressRecorder{} + consumer := &importColumnMappingConsumer{downstream: downstream} + reportImportSourceProgress(consumer, 12, 24) + + if len(downstream.progress) != 1 || downstream.progress[0].bytesRead != 12 || downstream.progress[0].totalBytes != 24 { + t.Fatalf("forwarded progress = %#v, want source byte counters", downstream.progress) + } +} + +func TestJSONImportRejectsFieldsIntroducedAfterFirstRow(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.json") + content := `[{"id":1},{"id":2,"name":"alice"}]` + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write json: %v", err) + } + + _, err := buildImportPreview(path, 5) + if err == nil { + t.Fatal("later JSON fields must not be silently dropped") + } + if !strings.Contains(err.Error(), "row 2") || !strings.Contains(err.Error(), `"name"`) { + t.Fatalf("structure drift error lacks row/field context: %v", err) + } +} + +func TestJSONImportPreservesLargeIntegerPrecision(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.json") + if err := os.WriteFile(path, []byte(`[{"id":9007199254740993}]`), 0o600); err != nil { + t.Fatalf("write json: %v", err) + } + + preview, err := buildImportPreview(path, 5) + if err != nil { + t.Fatalf("build preview: %v", err) + } + value, ok := preview.PreviewRows[0]["id"].(json.Number) + if !ok { + t.Fatalf("id type = %T, want json.Number", preview.PreviewRows[0]["id"]) + } + if value.String() != "9007199254740993" { + t.Fatalf("id = %q, want exact integer", value.String()) + } +} + +func TestJSONImportUsesStreamingUTF16TextSource(t *testing.T) { + raw, _, err := transform.Bytes( + unicode.UTF16(unicode.LittleEndian, unicode.UseBOM).NewEncoder(), + []byte(`[{"id":1,"name":"张三"}]`), + ) + if err != nil { + t.Fatalf("encode UTF-16 JSON fixture: %v", err) + } + path := filepath.Join(t.TempDir(), "rows.json") + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Fatalf("write JSON: %v", err) + } + consumer := &importSourceProgressRecorder{} + if err := streamImportFileWithOptions(path, consumer, ImportFileOptions{Encoding: "auto"}); err != nil { + t.Fatalf("stream UTF-16 JSON: %v", err) + } + if got := consumer.rows[0]["name"]; got != "张三" { + t.Fatalf("name = %#v, want 张三", got) + } + last := consumer.progress[len(consumer.progress)-1] + if last.bytesRead != int64(len(raw)) || last.totalBytes != int64(len(raw)) { + t.Fatalf("raw progress = %#v, want %d bytes", last, len(raw)) + } +} + +func TestJSONImportNormalizesConfiguredStringNulls(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.json") + if err := os.WriteFile(path, []byte(`[{"id":1,"marker":"\\N","note":""}]`), 0o600); err != nil { + t.Fatalf("write JSON: %v", err) + } + nullToken := "\\N" + consumer := &importCollectConsumer{} + options := ImportFileOptions{NullToken: &nullToken, EmptyStringAsNull: true} + if err := streamImportFileWithOptions(path, consumer, options); err != nil { + t.Fatalf("stream JSON with null options: %v", err) + } + row := consumer.rows[0] + if row["marker"] != nil || row["note"] != nil { + t.Fatalf("normalized row = %#v, want marker and note null", row) + } +} + +func TestJSONImportRejectsTrailingContentAfterArray(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.json") + if err := os.WriteFile(path, []byte(`[{"id":1}] {"id":2}`), 0o600); err != nil { + t.Fatalf("write json: %v", err) + } + + _, err := buildImportPreview(path, 5) + if err == nil { + t.Fatal("JSON content after the root array must be rejected") + } + if !strings.Contains(err.Error(), "trailing content") { + t.Fatalf("unexpected trailing-content error: %v", err) + } +} + +func TestJSONImportRejectsCellOverByteLimitDuringPreview(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.json") + content := `[{"payload":"` + strings.Repeat("x", 16*1024*1024+1) + `"}]` + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write json: %v", err) + } + + _, err := buildImportPreview(path, 5) + if err == nil { + t.Fatal("oversized JSON cell must be rejected during preview") + } + if !strings.Contains(err.Error(), `column "payload" exceeds`) { + t.Fatalf("unexpected oversized JSON cell error: %v", err) + } +} diff --git a/internal/app/import_source_identity.go b/internal/app/import_source_identity.go new file mode 100644 index 00000000..6d636455 --- /dev/null +++ b/internal/app/import_source_identity.go @@ -0,0 +1,97 @@ +package app + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "path/filepath" +) + +const importSourceIdentitySampleBytes int64 = 64 * 1024 + +// ImportSourceIdentity binds a preview and any later checkpoint to the exact +// local source selected by the user without forcing a multi-gigabyte preview +// to hash the whole file. +type ImportSourceIdentity struct { + Size int64 `json:"size"` + ModifiedUnixNano int64 `json:"modifiedUnixNano"` + QuickSHA256 string `json:"quickSha256"` + Token string `json:"token"` +} + +func captureImportSourceIdentity(filePath string) (ImportSourceIdentity, error) { + absPath, err := filepath.Abs(filePath) + if err != nil { + return ImportSourceIdentity{}, err + } + absPath = filepath.Clean(absPath) + f, err := os.Open(absPath) + if err != nil { + return ImportSourceIdentity{}, err + } + defer f.Close() + info, err := f.Stat() + if err != nil { + return ImportSourceIdentity{}, err + } + if !info.Mode().IsRegular() { + return ImportSourceIdentity{}, fmt.Errorf("import source is not a regular file") + } + + quickHash, err := hashImportSourceSamples(f, info.Size()) + if err != nil { + return ImportSourceIdentity{}, err + } + modified := info.ModTime().UnixNano() + tokenHasher := sha256.New() + _, _ = fmt.Fprintf(tokenHasher, "%s\x00%d\x00%d\x00%s", absPath, info.Size(), modified, quickHash) + return ImportSourceIdentity{ + Size: info.Size(), + ModifiedUnixNano: modified, + QuickSHA256: quickHash, + Token: hex.EncodeToString(tokenHasher.Sum(nil)), + }, nil +} + +func hashImportSourceSamples(f *os.File, size int64) (string, error) { + hasher := sha256.New() + if size <= importSourceIdentitySampleBytes*2 { + if _, err := f.Seek(0, io.SeekStart); err != nil { + return "", err + } + if _, err := io.Copy(hasher, io.LimitReader(f, size)); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil + } + + if _, err := f.Seek(0, io.SeekStart); err != nil { + return "", err + } + if _, err := io.Copy(hasher, io.LimitReader(f, importSourceIdentitySampleBytes)); err != nil { + return "", err + } + if _, err := f.Seek(size-importSourceIdentitySampleBytes, io.SeekStart); err != nil { + return "", err + } + if _, err := io.Copy(hasher, io.LimitReader(f, importSourceIdentitySampleBytes)); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func validateImportSourceIdentity(filePath string, expected ImportSourceIdentity) error { + if expected.Token == "" { + return nil + } + actual, err := captureImportSourceIdentity(filePath) + if err != nil { + return err + } + if actual.Token != expected.Token { + return fmt.Errorf("import source changed after preview") + } + return nil +} diff --git a/internal/app/import_source_identity_test.go b/internal/app/import_source_identity_test.go new file mode 100644 index 00000000..a83c9cdb --- /dev/null +++ b/internal/app/import_source_identity_test.go @@ -0,0 +1,55 @@ +package app + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestCaptureImportSourceIdentityDetectsSameSizeReplacement(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "rows.csv") + original := []byte("id,name\n1,alice\n") + if err := os.WriteFile(path, original, 0o600); err != nil { + t.Fatal(err) + } + + identity, err := captureImportSourceIdentity(path) + if err != nil { + t.Fatal(err) + } + if identity.Size != int64(len(original)) || identity.Token == "" { + t.Fatalf("unexpected identity: %+v", identity) + } + + replacement := []byte("id,name\n2,bobby\n") + if len(replacement) != len(original) { + t.Fatalf("test fixture must preserve file size: %d != %d", len(replacement), len(original)) + } + if err := os.WriteFile(path, replacement, 0o600); err != nil { + t.Fatal(err) + } + modifiedAt := time.Unix(0, identity.ModifiedUnixNano) + if err := os.Chtimes(path, modifiedAt, modifiedAt); err != nil { + t.Fatal(err) + } + + if err := validateImportSourceIdentity(path, identity); err == nil { + t.Fatal("same-size replacement with restored timestamp must be rejected") + } +} + +func TestValidateImportSourceIdentityAcceptsUnchangedFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "rows.json") + if err := os.WriteFile(path, []byte(`[{"id":1}]`), 0o600); err != nil { + t.Fatal(err) + } + identity, err := captureImportSourceIdentity(path) + if err != nil { + t.Fatal(err) + } + if err := validateImportSourceIdentity(path, identity); err != nil { + t.Fatalf("unchanged source should validate: %v", err) + } +} diff --git a/internal/app/import_task_runtime.go b/internal/app/import_task_runtime.go new file mode 100644 index 00000000..fb8e56fb --- /dev/null +++ b/internal/app/import_task_runtime.go @@ -0,0 +1,160 @@ +package app + +import ( + "context" + "errors" + "strings" + "sync" + "time" + + "GoNavi-Wails/internal/importjob" + + "github.com/google/uuid" +) + +var errImportTaskNotFound = errors.New("import task not found") + +type importTaskRegistration struct { + token string + kind importjob.Kind + cancel context.CancelFunc + lifecycle *managedImportJob + stopRequested bool + cancelDispatched bool +} + +func (a *App) registerImportTask(jobID string, cancel context.CancelFunc, kinds ...importjob.Kind) (func(), bool) { + kind := importjob.Kind("") + if len(kinds) > 0 { + kind = kinds[0] + } + a.importTaskMu.Lock() + closing := a.importTasksClosing + a.importTaskMu.Unlock() + if closing { + return func() {}, false + } + cleanupQuery, registered := a.registerExclusiveRunningQuery(jobID, cancel, true) + if !registered { + return func() {}, false + } + token := uuid.NewString() + a.importTaskMu.Lock() + if a.importTasksClosing { + a.importTaskMu.Unlock() + cleanupQuery() + return func() {}, false + } + if a.importTasks == nil { + a.importTasks = make(map[string]importTaskRegistration) + } + a.importTasks[jobID] = importTaskRegistration{token: token, kind: kind, cancel: cancel} + a.importTasksWG.Add(1) + a.importTaskMu.Unlock() + + var once sync.Once + return func() { + once.Do(func() { + cleanupQuery() + a.importTaskMu.Lock() + if current, exists := a.importTasks[jobID]; exists && current.token == token { + delete(a.importTasks, jobID) + } + a.importTaskMu.Unlock() + a.importTasksWG.Done() + }) + }, true +} + +// bindImportTaskLifecycle attaches durable state after task registration. A +// cancel request that arrives while the durable job is still being created is +// remembered and persisted as stopping immediately after the bind completes. +func (a *App) bindImportTaskLifecycle(jobID string, kind importjob.Kind, lifecycle *managedImportJob) (bool, error) { + jobID = strings.TrimSpace(jobID) + if lifecycle == nil { + return false, errors.New("import job lifecycle is unavailable") + } + a.importTaskMu.Lock() + task, exists := a.importTasks[jobID] + if !exists { + a.importTaskMu.Unlock() + return false, nil + } + if task.kind != "" && kind != "" && task.kind != kind { + a.importTaskMu.Unlock() + return false, errors.New("import task kind does not match durable job kind") + } + if kind != "" { + task.kind = kind + } + task.lifecycle = lifecycle + stopRequested := task.stopRequested + a.importTasks[jobID] = task + a.importTaskMu.Unlock() + + if stopRequested { + if err := lifecycle.requestStop(); err != nil { + return true, err + } + } + return true, nil +} + +// requestImportTaskCancellation only resolves registrations owned by the +// import runtime. It intentionally never falls back to runningQueries, where +// ordinary query executions are tracked too. +func (a *App) requestImportTaskCancellation(jobID string, expectedKind importjob.Kind) error { + jobID = strings.TrimSpace(jobID) + a.importTaskMu.Lock() + task, exists := a.importTasks[jobID] + if !exists || (expectedKind != "" && task.kind != expectedKind) { + a.importTaskMu.Unlock() + return errImportTaskNotFound + } + task.stopRequested = true + var cancel context.CancelFunc + if !task.cancelDispatched { + task.cancelDispatched = true + cancel = task.cancel + } + lifecycle := task.lifecycle + a.importTasks[jobID] = task + a.importTaskMu.Unlock() + + var stopErr error + if lifecycle != nil { + stopErr = lifecycle.requestStop() + } + if cancel != nil { + cancel() + } + return stopErr +} + +func (a *App) cancelAndWaitImportTasks(timeout time.Duration) bool { + a.importTaskMu.Lock() + a.importTasksClosing = true + jobIDs := make([]string, 0, len(a.importTasks)) + for jobID := range a.importTasks { + jobIDs = append(jobIDs, jobID) + } + a.importTaskMu.Unlock() + for _, jobID := range jobIDs { + _ = a.requestImportTaskCancellation(jobID, "") + } + done := make(chan struct{}) + go func() { + a.importTasksWG.Wait() + close(done) + }() + if timeout <= 0 { + <-done + return true + } + select { + case <-done: + return true + case <-time.After(timeout): + return false + } +} diff --git a/internal/app/import_task_runtime_test.go b/internal/app/import_task_runtime_test.go new file mode 100644 index 00000000..1c2fe8e9 --- /dev/null +++ b/internal/app/import_task_runtime_test.go @@ -0,0 +1,154 @@ +package app + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "GoNavi-Wails/internal/importjob" +) + +func TestCancelAndWaitImportTasksKeepsRegistrationUntilCleanup(t *testing.T) { + app := NewApp() + ctx, cancel := context.WithCancel(context.Background()) + cleanup, registered := app.registerImportTask("shutdown-import", cancel) + if !registered { + t.Fatal("import task registration failed") + } + done := make(chan struct{}) + go func() { + <-ctx.Done() + cleanup() + close(done) + }() + + if !app.cancelAndWaitImportTasks(time.Second) { + t.Fatal("import task did not unwind before shutdown deadline") + } + select { + case <-done: + default: + t.Fatal("shutdown returned before task cleanup") + } + app.queryMu.RLock() + _, retained := app.runningQueries["shutdown-import"] + app.queryMu.RUnlock() + if retained { + t.Fatal("completed import registration leaked") + } +} + +func TestCancelAndWaitImportTasksRejectsRegistrationsAfterShutdownStarts(t *testing.T) { + app := NewApp() + if !app.cancelAndWaitImportTasks(0) { + t.Fatal("empty import runtime did not finish shutdown") + } + cleanup, registered := app.registerImportTask("late-shutdown-import", func() {}) + defer cleanup() + if registered { + t.Fatal("import task registered after import runtime shutdown started") + } +} + +func TestCancelImportJobBeforeLifecycleBindingPersistsStoppingWhenBound(t *testing.T) { + app := NewApp() + app.configDir = t.TempDir() + var cancelCalls atomic.Int32 + cleanup, registered := app.registerImportTask("late-bound-import", func() { + cancelCalls.Add(1) + }, importjob.KindTable) + if !registered { + t.Fatal("import task registration failed") + } + defer cleanup() + + first := app.CancelImportJob("late-bound-import") + second := app.CancelImportJob("late-bound-import") + if !first.Success || !second.Success { + t.Fatalf("repeated cancellation should be idempotent: first=%#v second=%#v", first, second) + } + if got := cancelCalls.Load(); got != 1 { + t.Fatalf("cancel callback calls = %d, want 1", got) + } + + lifecycle, err := app.beginManagedImportJob(managedImportJobStart{ + ID: "late-bound-import", + Kind: importjob.KindTable, + }) + if err != nil { + t.Fatal(err) + } + stored, err := lifecycle.store.Get("late-bound-import") + if err != nil { + t.Fatal(err) + } + if stored.Status != importjob.StatusStopping { + t.Fatalf("status = %q, want %q", stored.Status, importjob.StatusStopping) + } + if result := app.CancelImportJob("late-bound-import"); !result.Success { + t.Fatalf("cancelling an already-stopping import should succeed: %#v", result) + } + afterRepeatedCancel, err := lifecycle.store.Get("late-bound-import") + if err != nil { + t.Fatal(err) + } + if afterRepeatedCancel.Revision != stored.Revision { + t.Fatalf("repeated cancellation rewrote stopping job: before=%d after=%d", stored.Revision, afterRepeatedCancel.Revision) + } + if got := cancelCalls.Load(); got != 1 { + t.Fatalf("cancel callback calls after lifecycle bind = %d, want 1", got) + } +} + +func TestCancelImportJobDoesNotCancelOrdinaryRunningQuery(t *testing.T) { + app := NewApp() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cleanup, registered := app.registerExclusiveRunningQuery("ordinary-query", cancel, true) + if !registered { + t.Fatal("query registration failed") + } + defer cleanup() + + result := app.CancelImportJob("ordinary-query") + if result.Success { + t.Fatalf("ordinary query cancellation unexpectedly succeeded: %#v", result) + } + select { + case <-ctx.Done(): + t.Fatal("import cancellation touched an ordinary running query") + default: + } +} + +func TestCancelImportTaskByKindRejectsDifferentImportKind(t *testing.T) { + app := NewApp() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cleanup, registered := app.registerImportTask("table-import-kind", cancel, importjob.KindTable) + if !registered { + t.Fatal("import task registration failed") + } + defer cleanup() + + wrongKind := app.cancelImportTaskByKind("table-import-kind", importjob.KindSQL) + if wrongKind.Success { + t.Fatalf("wrong-kind cancellation unexpectedly succeeded: %#v", wrongKind) + } + select { + case <-ctx.Done(): + t.Fatal("wrong-kind cancellation reached the task") + default: + } + + matchingKind := app.cancelImportTaskByKind("table-import-kind", importjob.KindTable) + if !matchingKind.Success { + t.Fatalf("matching-kind cancellation failed: %#v", matchingKind) + } + select { + case <-ctx.Done(): + default: + t.Fatal("matching-kind cancellation did not reach the task") + } +} diff --git a/internal/app/import_text_source.go b/internal/app/import_text_source.go new file mode 100644 index 00000000..d504b2da --- /dev/null +++ b/internal/app/import_text_source.go @@ -0,0 +1,221 @@ +package app + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "os" + "unicode/utf8" + + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" +) + +const ( + importTextEncodingAuto = "auto" + importTextEncodingUTF8 = "utf-8" + importTextEncodingUTF16LE = "utf-16le" + importTextEncodingUTF16BE = "utf-16be" + importTextEncodingGB18030 = "gb18030" + + // Automatic detection must stay bounded: multi-gigabyte imports should not + // read the complete source once merely to decide between UTF-8 and GB18030. + importTextEncodingDetectionSampleBytes = int64(1 << 20) +) + +var ( + importUTF8BOM = []byte{0xef, 0xbb, 0xbf} + importUTF16LEBOM = []byte{0xff, 0xfe} + importUTF16BEBOM = []byte{0xfe, 0xff} +) + +// importTextSource exposes decoded UTF-8 while retaining progress in original +// on-disk bytes. It never buffers the complete decoded import payload. +type importTextSource struct { + io.Reader + file *os.File + rawCounter *importByteCountingReader + totalBytes int64 + encoding string +} + +func (source *importTextSource) RawBytesRead() int64 { + if source == nil || source.rawCounter == nil { + return 0 + } + return source.rawCounter.bytesRead +} + +func (source *importTextSource) TotalBytes() int64 { + if source == nil { + return 0 + } + return source.totalBytes +} + +func (source *importTextSource) Close() error { + if source == nil || source.file == nil { + return nil + } + return source.file.Close() +} + +func normalizeImportTextEncoding(value string) (string, error) { + if value == "" { + return importTextEncodingAuto, nil + } + switch value { + case importTextEncodingAuto, + importTextEncodingUTF8, + importTextEncodingUTF16LE, + importTextEncodingUTF16BE, + importTextEncodingGB18030: + return value, nil + default: + return "", fmt.Errorf("unsupported import text encoding %q", value) + } +} + +func openImportTextSource(filePath string, requestedEncoding string) (*importTextSource, error) { + requestedEncoding, err := normalizeImportTextEncoding(requestedEncoding) + if err != nil { + return nil, err + } + + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + closeOnError := true + defer func() { + if closeOnError { + _ = file.Close() + } + }() + + info, err := file.Stat() + if err != nil { + return nil, err + } + totalBytes := info.Size() + if totalBytes < 0 { + totalBytes = 0 + } + + detectedEncoding, bomBytes, err := detectImportTextBOM(file) + if err != nil { + return nil, err + } + selectedEncoding := requestedEncoding + if requestedEncoding == importTextEncodingAuto { + if detectedEncoding != "" { + selectedEncoding = detectedEncoding + } else { + validUTF8, err := importFilePrefixIsValidUTF8(file, totalBytes) + if err != nil { + return nil, err + } + if validUTF8 { + selectedEncoding = importTextEncodingUTF8 + } else { + selectedEncoding = importTextEncodingGB18030 + } + } + } else if detectedEncoding != "" && detectedEncoding != requestedEncoding { + return nil, fmt.Errorf( + "import text encoding %q conflicts with %s BOM", + requestedEncoding, + detectedEncoding, + ) + } + + if _, err := file.Seek(0, io.SeekStart); err != nil { + return nil, err + } + rawCounter := &importByteCountingReader{reader: file} + buffered := bufio.NewReader(rawCounter) + if bomBytes > 0 { + if _, err := buffered.Discard(bomBytes); err != nil { + return nil, fmt.Errorf("read import text BOM: %w", err) + } + } + + var decoded io.Reader = buffered + switch selectedEncoding { + case importTextEncodingUTF8: + decoded = transform.NewReader(buffered, encoding.UTF8Validator) + case importTextEncodingUTF16LE: + decoded = transform.NewReader(buffered, unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewDecoder()) + case importTextEncodingUTF16BE: + decoded = transform.NewReader(buffered, unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM).NewDecoder()) + case importTextEncodingGB18030: + decoded = transform.NewReader(buffered, simplifiedchinese.GB18030.NewDecoder()) + default: + return nil, fmt.Errorf("unsupported import text encoding %q", selectedEncoding) + } + + closeOnError = false + return &importTextSource{ + Reader: decoded, + file: file, + rawCounter: rawCounter, + totalBytes: totalBytes, + encoding: selectedEncoding, + }, nil +} + +func detectImportTextBOM(file *os.File) (encodingName string, bomBytes int, err error) { + prefix := make([]byte, len(importUTF8BOM)) + read, readErr := file.ReadAt(prefix, 0) + if readErr != nil && !errors.Is(readErr, io.EOF) { + return "", 0, readErr + } + prefix = prefix[:read] + switch { + case bytes.HasPrefix(prefix, importUTF8BOM): + return importTextEncodingUTF8, len(importUTF8BOM), nil + case bytes.HasPrefix(prefix, importUTF16LEBOM): + return importTextEncodingUTF16LE, len(importUTF16LEBOM), nil + case bytes.HasPrefix(prefix, importUTF16BEBOM): + return importTextEncodingUTF16BE, len(importUTF16BEBOM), nil + default: + return "", 0, nil + } +} + +func importFilePrefixIsValidUTF8(file *os.File, totalBytes int64) (bool, error) { + sampleBytes := totalBytes + if sampleBytes > importTextEncodingDetectionSampleBytes { + sampleBytes = importTextEncodingDetectionSampleBytes + } + if sampleBytes <= 0 { + return true, nil + } + + sample := make([]byte, sampleBytes) + n, err := file.ReadAt(sample, 0) + if err != nil && !errors.Is(err, io.EOF) { + return false, err + } + sample = sample[:n] + for len(sample) > 0 { + if sample[0] < utf8.RuneSelf { + sample = sample[1:] + continue + } + if !utf8.FullRune(sample) { + // A bounded prefix can end in the middle of an otherwise valid rune. + return int64(n) < totalBytes, nil + } + r, size := utf8.DecodeRune(sample) + if r == utf8.RuneError && size == 1 { + return false, nil + } + sample = sample[size:] + } + return true, nil +} diff --git a/internal/app/import_unknown_outcome_test.go b/internal/app/import_unknown_outcome_test.go new file mode 100644 index 00000000..fe84f947 --- /dev/null +++ b/internal/app/import_unknown_outcome_test.go @@ -0,0 +1,100 @@ +package app + +import ( + "context" + "database/sql/driver" + "errors" + "testing" + + "GoNavi-Wails/internal/db" +) + +type unknownAutocommitImportDB struct { + db.Database + calls int + err error +} + +func (database *unknownAutocommitImportDB) ExecContext(context.Context, string) (int64, error) { + database.calls++ + return 0, database.err +} + +type unknownOutcomeImportWriter struct { + calls int +} + +func (writer *unknownOutcomeImportWriter) SetColumns([]string) {} + +func (writer *unknownOutcomeImportWriter) ApplyBatch([]map[string]interface{}) error { + return errors.New("batch path must not be used") +} + +func (writer *unknownOutcomeImportWriter) ApplyOne(map[string]interface{}) error { + writer.calls++ + return db.MarkWriteOutcomeUnknown(errors.New("commit response lost")) +} + +func (writer *unknownOutcomeImportWriter) BatchEnabled() bool { return false } + +func TestImportBatchConsumerStopsContinueModeOnUnknownWriteOutcome(t *testing.T) { + writer := &unknownOutcomeImportWriter{} + consumer := newImportBatchConsumer(writer, 10, 2, true, true, nil) + + if err := consumer.ConsumeRow(map[string]interface{}{"id": 1}); err != nil { + t.Fatal(err) + } + if err := consumer.ConsumeRow(map[string]interface{}{"id": 2}); err != nil { + t.Fatal(err) + } + err := consumer.Flush() + if !errors.Is(err, errImportStoppedOnError) { + t.Fatalf("unknown write outcome must stop the import, got %v", err) + } + if !db.IsWriteOutcomeUnknown(err) { + t.Fatalf("consumer must preserve the typed unknown-outcome cause, got %T: %v", err, err) + } + + result := consumer.Result() + if !result.OutcomeUnknown || !result.StoppedOnError { + t.Fatalf("unknown write outcome was not preserved: %#v", result) + } + if result.Success != 0 || result.Failed != 1 || writer.calls != 1 { + t.Fatalf("continue mode wrote past the uncertain row: result=%#v calls=%d", result, writer.calls) + } +} + +func TestImportBatchConsumerStopsContinueModeOnAmbiguousAutocommitResponse(t *testing.T) { + for name, writeErr := range map[string]error{ + "transport": driver.ErrBadConn, + "cancellation": context.Canceled, + } { + t.Run(name, func(t *testing.T) { + database := &unknownAutocommitImportDB{err: writeErr} + writer := newImportDatabaseRowWriterWithOptions(database, "postgres", "users", newImportColumnTypeLookup(nil), ImportFileOptions{ + ConflictPolicy: importConflictPolicySkipDuplicates, + }) + consumer := newImportBatchConsumer(writer, 10, 2, true, true, nil) + if err := consumer.SetColumns([]string{"id"}); err != nil { + t.Fatal(err) + } + for _, id := range []int{1, 2} { + if err := consumer.ConsumeRow(map[string]interface{}{"id": id}); err != nil { + t.Fatal(err) + } + } + + err := consumer.Flush() + if !errors.Is(err, errImportStoppedOnError) { + t.Fatalf("ambiguous autocommit response must stop the import, got %v", err) + } + if name == "cancellation" && !errors.Is(err, context.Canceled) { + t.Fatalf("consumer lost the cancellation cause while marking the outcome unknown: %v", err) + } + result := consumer.Result() + if !result.OutcomeUnknown || !result.StoppedOnError || result.Failed != 1 || database.calls != 1 { + t.Fatalf("import continued after an ambiguous autocommit response: result=%#v calls=%d", result, database.calls) + } + }) + } +} diff --git a/internal/app/methods_db.go b/internal/app/methods_db.go index 7bb28268..1e864527 100644 --- a/internal/app/methods_db.go +++ b/internal/app/methods_db.go @@ -1520,7 +1520,7 @@ func (a *App) dbQueryMulti( if shouldTryQueryResultFirst(runConfig.Type, stmt) { containsQueryFirstWrite = true } - if isPLSQLBlockStatement(stmt) { + if isPLSQLBlockStatementForDialect(resolvedDBType, stmt) { containsPLSQLBlock = true } } diff --git a/internal/app/methods_db_metadata_retry_test.go b/internal/app/methods_db_metadata_retry_test.go index 03693649..28bbc446 100644 --- a/internal/app/methods_db_metadata_retry_test.go +++ b/internal/app/methods_db_metadata_retry_test.go @@ -1,6 +1,7 @@ package app import ( + "context" "errors" "fmt" "path/filepath" @@ -88,7 +89,16 @@ func (f *fakeMetadataRetryDB) Query(query string) ([]map[string]interface{}, []s return f.queryRows, f.queryFields, nil } func (f *fakeMetadataRetryDB) Exec(query string) (int64, error) { return 0, nil } -func (f *fakeMetadataRetryDB) GetDatabases() ([]string, error) { return nil, nil } +func (f *fakeMetadataRetryDB) ApplyChanges(string, connection.ChangeSet) error { + return nil +} +func (f *fakeMetadataRetryDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) error { + if err := ctx.Err(); err != nil { + return err + } + return f.ApplyChanges(tableName, changes) +} +func (f *fakeMetadataRetryDB) GetDatabases() ([]string, error) { return nil, nil } func (f *fakeMetadataRetryDB) GetTables(dbName string) ([]string, error) { f.tableCalls++ f.tableSchema = dbName diff --git a/internal/app/methods_file.go b/internal/app/methods_file.go index 03d698d4..af445f36 100644 --- a/internal/app/methods_file.go +++ b/internal/app/methods_file.go @@ -2,6 +2,7 @@ package app import ( "bufio" + "bytes" "context" "crypto/sha256" "encoding/csv" @@ -24,10 +25,13 @@ import ( "GoNavi-Wails/internal/connection" "GoNavi-Wails/internal/db" + "GoNavi-Wails/internal/importjob" "GoNavi-Wails/internal/logger" + "GoNavi-Wails/internal/sqlaudit" "GoNavi-Wails/internal/uievents" "GoNavi-Wails/internal/utils" + "github.com/google/uuid" "github.com/wailsapp/wails/v2/pkg/runtime" ) @@ -40,6 +44,9 @@ const sqlFileBatchMaxStatements = 1000 const sqlFileBatchMaxBytes = 4 * 1024 * 1024 const sqlFileProgressStatementInterval = 100 const sqlFileProgressTimeInterval = time.Second +const sqlFileSessionCleanupTimeout = 5 * time.Second +const sqlFileMaxErrorDetails = 20 +const sqlFileBatchIsolationSequentialThreshold = 16 const exportProgressEvent = "export:progress" const exportProgressRowInterval int64 = 1000 const exportProgressTimeInterval = 500 * time.Millisecond @@ -50,6 +57,8 @@ const maxAppLogTailLineLimit = 200 const appLogTailReadWindowBytes int64 = 256 * 1024 var mysqlCreateViewPrefixPattern = regexp.MustCompile(`(?is)^\s*create\s+(?:algorithm\s*=\s*\w+\s+)?(?:definer\s*=\s*(?:` + "`[^`]+`" + `|\S+)\s*@\s*(?:` + "`[^`]+`" + `|\S+)\s+)?(?:sql\s+security\s+(?:definer|invoker)\s+)?view\s+`) +var sqlFileMySQLAutocommitAssignmentPattern = regexp.MustCompile(`(?is)(?:^|,)\s*(?:(?:session|local)\s+)?(?:@@\s*(?:session\s*\.\s*)?)?autocommit\s*(?::=|=)\s*([^,;\s]+)`) +var jsonNumberSQLLiteralPattern = regexp.MustCompile(`^-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?$`) type saveFileDialogFunc func(context.Context, runtime.SaveDialogOptions) (string, error) @@ -71,17 +80,21 @@ type sqlFileExecutionProgress struct { } type sqlFileExecutionOptions struct { - DBType string - BatchMaxStatements int - BatchMaxBytes int - Text fileBackendTextFunc - OnProgress func(sqlFileExecutionProgress) + DBType string + BatchMaxStatements int + BatchMaxBytes int + MaxStatementBytes int64 + ContinueOnError bool + PreflightEachStatement bool + Text fileBackendTextFunc + OnProgress func(sqlFileExecutionProgress) } type sqlFileExecutionResult struct { - Executed int - Failed int - Errors []string + Executed int + Failed int + Errors []string + OutcomeUnknown bool } type sqlFilePendingStatement struct { @@ -89,6 +102,80 @@ type sqlFilePendingStatement struct { SQL string } +var errSQLFileStoppedOnError = errors.New("sql file execution stopped on error") + +type sqlFileCancelledError struct{} + +func (sqlFileCancelledError) Error() string { return "已取消" } + +func (sqlFileCancelledError) Unwrap() error { return context.Canceled } + +var errSQLFileCancelled error = sqlFileCancelledError{} + +type sqlFileStoppedOnError struct { + detail string +} + +type sqlFilePreflightRejectedError struct { + reason SQLImportPreflightReason + executed int + failed int + possibleSideEffects bool + outcomeUnknown bool +} + +func (err *sqlFilePreflightRejectedError) Error() string { + if err == nil { + return "" + } + reason := string(err.reason.Code) + if err.reason.Directive != "" { + reason += ": " + err.reason.Directive + } + if !err.possibleSideEffects && err.executed == 0 && err.failed == 0 { + return fmt.Sprintf("SQL import preflight rejected unsupported client script (%s); no database statement was executed", reason) + } + return fmt.Sprintf("SQL import preflight rejected unsupported client script (%s); %d preceding statement(s) may already have completed", reason, err.executed+err.failed) +} + +func buildSQLFilePreflightFailurePayload(err *sqlFilePreflightRejectedError) map[string]interface{} { + executed := 0 + failed := 0 + reason := "" + directive := "" + if err != nil { + executed = err.executed + failed = err.failed + reason = string(err.reason.Code) + directive = err.reason.Directive + } + payload := buildSQLFileExecutionPayload(executed, failed, "failed") + payload["preflightRejected"] = true + payload["preflightReason"] = reason + payload["preflightDirective"] = directive + payload["previousStatementsMayHaveCompleted"] = err != nil && (err.possibleSideEffects || executed > 0 || failed > 0) + payload["outcomeUnknown"] = err != nil && err.outcomeUnknown + return payload +} + +func isSQLFilePreExecutionValidationError(err error) bool { + var preflightErr *sqlFilePreflightRejectedError + var statementLimitErr *SQLStatementTooLargeError + var sourceLimitErr *SQLImportSourceLimitError + return errors.As(err, &preflightErr) || errors.As(err, &statementLimitErr) || errors.As(err, &sourceLimitErr) +} + +func (e *sqlFileStoppedOnError) Error() string { + if e == nil { + return "" + } + return e.detail +} + +func (e *sqlFileStoppedOnError) Unwrap() error { + return errSQLFileStoppedOnError +} + type sqlFileStatementExecer interface { Exec(query string) (int64, error) } @@ -829,6 +916,19 @@ func selectSQLFileForExecutionByPathWithText(filePath string, text fileBackendTe } } +func sqlFileExecutionDialogFilters(text fileBackendTextFunc) []runtime.FileFilter { + return []runtime.FileFilter{ + { + DisplayName: fileBackendText(text, "file.backend.filter.sql_files", nil), + Pattern: "*.sql;*.sql.gz", + }, + { + DisplayName: fileBackendText(text, "file.backend.filter.all_files_pattern", nil), + Pattern: "*.*", + }, + } +} + func readSQLFileWithMetadataByPath(filePath string) connection.QueryResult { return readSQLFileWithMetadataByPathWithText(filePath, nil) } @@ -1094,17 +1194,8 @@ func (a *App) OpenSQLFile() connection.QueryResult { func (a *App) SelectSQLFileForExecution() connection.QueryResult { selection, err := runtime.OpenFileDialog(a.ctx, runtime.OpenDialogOptions{ - Title: a.appText("file.backend.dialog.select_sql_file", nil), - Filters: []runtime.FileFilter{ - { - DisplayName: a.appText("file.backend.filter.sql_files", nil), - Pattern: "*.sql", - }, - { - DisplayName: a.appText("file.backend.filter.all_files_pattern", nil), - Pattern: "*.*", - }, - }, + Title: a.appText("file.backend.dialog.select_sql_file", nil), + Filters: sqlFileExecutionDialogFilters(a.appText), }) if err != nil { @@ -1358,6 +1449,9 @@ func normalizeSQLFileExecutionOptions(options sqlFileExecutionOptions) sqlFileEx if options.BatchMaxBytes <= 0 { options.BatchMaxBytes = sqlFileBatchMaxBytes } + if options.MaxStatementBytes <= 0 { + options.MaxStatementBytes = DefaultSQLImportMaxStatementBytes + } return options } @@ -1388,13 +1482,24 @@ func joinSQLFileBatchStatements(batch []sqlFilePendingStatement) string { } func sqlFileStatementSnippet(stmt string, maxLen int) string { - snippet := strings.TrimSpace(stmt) + snippet := strings.TrimSpace(sqlaudit.RedactSQL(stmt)) if maxLen > 0 && len(snippet) > maxLen { return snippet[:maxLen] + "..." } return snippet } +func sanitizeSQLFileExecutionError(message string) string { + return sqlaudit.RedactError(message) +} + +func sanitizeSQLFileExecutionErr(err error) string { + if err == nil { + return "" + } + return sanitizeSQLFileExecutionError(err.Error()) +} + func execSQLFileStatement(ctx context.Context, execer sqlFileStatementExecer, stmt string) (int64, error) { if ctxErr := ctx.Err(); ctxErr != nil { return 0, ctxErr @@ -1405,11 +1510,25 @@ func execSQLFileStatement(ctx context.Context, execer sqlFileStatementExecer, st return execer.Exec(stmt) } +func rollbackSQLFileTransaction(execer sqlFileStatementExecer, rollbackSQL string) error { + cleanupCtx, cancel := context.WithTimeout(context.Background(), sqlFileSessionCleanupTimeout) + defer cancel() + if _, err := execSQLFileStatement(cleanupCtx, execer, rollbackSQL); err != nil { + if discarder, ok := execer.(db.StatementExecerDiscarter); ok { + if discardErr := discarder.Discard(); discardErr != nil { + return fmt.Errorf("%w; discard contaminated session: %v", err, discardErr) + } + } + return err + } + return nil +} + func isSQLFileBatchableWriteStatement(dbType string, stmt string) bool { if isReadOnlySQLQuery(dbType, stmt) { return false } - if isPLSQLBlockStatement(stmt) { + if isPLSQLBlockStatementForDialect(dbType, stmt) { return false } if shouldTryQueryResultFirst(dbType, stmt) { @@ -1431,54 +1550,401 @@ func sqlFileBatchTransactionSQL(dbType string) (beginSQL string, commitSQL strin } } -func updateSQLFileTransactionState(inTransaction bool, stmt string) bool { - switch leadingSQLKeyword(stmt) { +func updateSQLFileTransactionState(dbType string, inTransaction bool, stmt string) bool { + depth := 0 + if inTransaction { + depth = 1 + } + return updateSQLFileTransactionDepth(dbType, depth, stmt) > 0 +} + +func updateSQLFileTransactionDepth(dbType string, depth int, stmt string) int { + keyword, keywordEnd := nextSQLKeyword(stmt, 0) + switch keyword { case "begin": - return true - case "start": - return strings.Contains(strings.ToLower(stmt), "transaction") - case "commit": - return false - case "rollback": - lower := strings.ToLower(stmt) - if strings.Contains(lower, " rollback to ") || strings.Contains(lower, "rollback to ") { - return inTransaction + if sqlBeginStartsTransactionForDialect(dbType, stmt, keywordEnd) { + if normalizeSQLClassifierDBType(dbType) == "sqlserver" { + return depth + 1 + } + return 1 + } + return depth + case "start": + if second, _ := nextSQLKeyword(stmt, keywordEnd); second == "transaction" { + return 1 + } + return depth + case "commit": + if sqlFileTransactionCommandUsesChain(stmt, keywordEnd) { + return 1 + } + if normalizeSQLClassifierDBType(dbType) == "sqlserver" && depth > 0 { + return depth - 1 + } + return 0 + case "rollback": + if sqlFileRollbackTargetsSavepoint(stmt, keywordEnd) { + return depth + } + if sqlFileSQLServerRollbackHasNamedTarget(dbType, stmt, keywordEnd) { + if depth > 0 { + return depth + } + // SQL Server uses the same syntax for transaction names and savepoints. + // A successful named rollback without tracked depth therefore leaves the + // session state uncertain; keep cleanup active rather than reusing it. + return 1 + } + if sqlFileTransactionCommandUsesChain(stmt, keywordEnd) { + return 1 + } + return 0 + case "end": + if !sqlFileStatementIsTransactionEndAlias(dbType, stmt, keywordEnd) { + return depth + } + if sqlFileTransactionCommandUsesChain(stmt, keywordEnd) { + return 1 + } + return 0 + case "abort": + switch normalizeSQLClassifierDBType(dbType) { + case "postgres", "kingbase", "highgo", "vastbase", "opengauss", "gaussdb", "duckdb": + if sqlFileTransactionCommandUsesChain(stmt, keywordEnd) { + return 1 + } + return 0 + default: + return depth } - return false default: - return inTransaction + return depth + } +} + +type sqlFileSQLServerTransactionTracker struct { + depth int + outerTransactionName string + savepointNames map[string]struct{} +} + +func (tracker sqlFileSQLServerTransactionTracker) reset() sqlFileSQLServerTransactionTracker { + return sqlFileSQLServerTransactionTracker{} +} + +func updateSQLFileSQLServerTransactionTracker(tracker sqlFileSQLServerTransactionTracker, stmt string) sqlFileSQLServerTransactionTracker { + keyword, keywordEnd := nextSQLKeyword(stmt, 0) + switch keyword { + case "begin": + if !sqlBeginStartsTransactionForDialect("sqlserver", stmt, keywordEnd) { + return tracker + } + if tracker.depth == 0 { + tracker.outerTransactionName, _ = sqlFileSQLServerBeginTransactionName(stmt, keywordEnd) + tracker.savepointNames = nil + } + tracker.depth++ + return tracker + case "save": + if tracker.depth == 0 { + return tracker + } + if name, ok := sqlFileSQLServerTransactionNameAfterCommand(stmt, keywordEnd); ok { + if tracker.savepointNames == nil { + tracker.savepointNames = make(map[string]struct{}) + } + tracker.savepointNames[name] = struct{}{} + } + return tracker + case "commit": + if tracker.depth > 0 { + tracker.depth-- + } + if tracker.depth == 0 { + return tracker.reset() + } + return tracker + case "rollback": + name, named := sqlFileSQLServerTransactionNameAfterCommand(stmt, keywordEnd) + if !named { + return tracker.reset() + } + if tracker.outerTransactionName != "" && name == tracker.outerTransactionName { + return tracker.reset() + } + if _, isSavepoint := tracker.savepointNames[name]; isSavepoint { + return tracker + } + // SQL Server uses identical syntax for outer transaction names and + // savepoints. An unrecognised successful target is therefore uncertain; + // retain the depth so EOF cleanup discards rather than reuses the session. + if tracker.depth == 0 { + tracker.depth = 1 + } + return tracker + default: + return tracker + } +} + +func sqlFileSQLServerBeginTransactionName(stmt string, keywordEnd int) (string, bool) { + token, tokenEnd := nextSQLKeyword(stmt, keywordEnd) + if token == "distributed" { + token, tokenEnd = nextSQLKeyword(stmt, tokenEnd) + } + if token != "transaction" && token != "tran" { + return "", false + } + name, ok := sqlFileSQLServerIdentifierAt(stmt, tokenEnd) + if !ok || strings.EqualFold(name, "with") { + return "", false + } + return name, true +} + +func sqlFileSQLServerTransactionNameAfterCommand(stmt string, keywordEnd int) (string, bool) { + token, tokenEnd := nextSQLKeyword(stmt, keywordEnd) + if token != "transaction" && token != "tran" { + return "", false + } + return sqlFileSQLServerIdentifierAt(stmt, tokenEnd) +} + +func sqlFileSQLServerIdentifierAt(stmt string, start int) (string, bool) { + start = skipSQLTrivia(stmt, start) + if start >= len(stmt) { + return "", false + } + if stmt[start] == '@' { + end := start + 1 + for end < len(stmt) && isSQLKeywordByte(stmt[end]) { + end++ + } + if end == start+1 { + return "", false + } + return stmt[start:end], true + } + end, ok := skipSQLIdentifierToken(stmt, start) + if !ok { + return "", false + } + name := strings.TrimSpace(stmt[start:end]) + if len(name) >= 2 { + switch { + case name[0] == '[' && name[len(name)-1] == ']': + name = strings.ReplaceAll(name[1:len(name)-1], "]]", "]") + case (name[0] == '"' && name[len(name)-1] == '"') || (name[0] == '`' && name[len(name)-1] == '`'): + quote := string(name[0]) + name = strings.ReplaceAll(name[1:len(name)-1], quote+quote, quote) + } + } + if name == "" { + return "", false + } + // SQL Server transaction and savepoint names are case-sensitive even on + // case-insensitive servers, so preserve their spelling for matching. + return name, true +} + +func isSQLFileMySQLCompatibleDialect(dbType string) bool { + switch normalizeSQLClassifierDBType(dbType) { + case "mysql", "mariadb", "oceanbase", "diros", "starrocks", "sphinx": + return true + default: + return false + } +} + +func sqlFileMySQLAutocommitAssignment(dbType string, stmt string) (disabled bool, known bool, assigned bool) { + if !isSQLFileMySQLCompatibleDialect(dbType) { + return false, false, false + } + start := skipSQLTrivia(stmt, 0) + if start >= len(stmt) { + return false, false, false + } + keyword, keywordEnd := nextSQLKeyword(stmt, start) + if keyword != "set" { + return false, false, false + } + matches := sqlFileMySQLAutocommitAssignmentPattern.FindAllStringSubmatch(stmt[keywordEnd:], -1) + if len(matches) == 0 || len(matches[len(matches)-1]) != 2 { + return false, false, false + } + switch strings.ToLower(strings.TrimSpace(matches[len(matches)-1][1])) { + case "0", "off", "false": + return true, true, true + case "1", "on", "true": + return false, true, true + default: + return false, false, true + } +} + +func sqlFileMySQLImplicitCommitBeforeStatement(dbType string, stmt string) bool { + if !isSQLFileMySQLCompatibleDialect(dbType) { + return false + } + keyword, keywordEnd := nextSQLKeyword(stmt, 0) + switch keyword { + case "create", "drop": + return !sqlFileMySQLTemporaryTableDDL(stmt, keywordEnd) + case "alter", "analyze", "cache", "check", "flush", "grant", "install", "optimize", "rename", "repair", "revoke", "truncate", "uninstall": + return true + case "reset": + second, _ := nextSQLKeyword(stmt, keywordEnd) + return second != "persist" + case "set": + second, _ := nextSQLKeyword(stmt, keywordEnd) + return second == "password" + case "begin": + return sqlBeginStartsTransactionForDialect(dbType, stmt, keywordEnd) + case "start": + second, _ := nextSQLKeyword(stmt, keywordEnd) + return second == "transaction" || second == "replica" || second == "slave" + case "stop": + second, _ := nextSQLKeyword(stmt, keywordEnd) + return second == "replica" || second == "slave" + case "lock": + second, _ := nextSQLKeyword(stmt, keywordEnd) + return second == "tables" + default: + return false + } +} + +func sqlFileMySQLTableLockCommand(dbType string, stmt string) (locks bool, unlocks bool) { + if !isSQLFileMySQLCompatibleDialect(dbType) { + return false, false + } + keyword, keywordEnd := nextSQLKeyword(stmt, 0) + second, _ := nextSQLKeyword(stmt, keywordEnd) + if second != "tables" { + return false, false + } + return keyword == "lock", keyword == "unlock" +} + +func sqlFileMySQLTemporaryTableDDL(stmt string, keywordEnd int) bool { + token, tokenEnd := nextSQLKeyword(stmt, keywordEnd) + if token == "or" { + if next, nextEnd := nextSQLKeyword(stmt, tokenEnd); next == "replace" { + token, _ = nextSQLKeyword(stmt, nextEnd) + } + } + return token == "temporary" +} + +func sqlFileTransactionCommandUsesChain(stmt string, keywordEnd int) bool { + token, tokenEnd := nextSQLKeyword(stmt, keywordEnd) + if token == "work" || token == "transaction" { + token, tokenEnd = nextSQLKeyword(stmt, tokenEnd) + } + if token != "and" { + return false + } + token, tokenEnd = nextSQLKeyword(stmt, tokenEnd) + if token == "no" { + return false + } + return token == "chain" +} + +func sqlFileRollbackTargetsSavepoint(stmt string, keywordEnd int) bool { + token, tokenEnd := nextSQLKeyword(stmt, keywordEnd) + if token == "work" || token == "transaction" { + token, _ = nextSQLKeyword(stmt, tokenEnd) + } + return token == "to" +} + +func sqlFileSQLServerRollbackHasNamedTarget(dbType string, stmt string, keywordEnd int) bool { + if normalizeSQLClassifierDBType(dbType) != "sqlserver" { + return false + } + token, tokenEnd := nextSQLKeyword(stmt, keywordEnd) + if token != "transaction" && token != "tran" { + return false + } + return skipSQLTrivia(stmt, tokenEnd) < len(stmt) +} + +func sqlFileStatementIsTransactionEndAlias(dbType string, stmt string, keywordEnd int) bool { + next, _ := nextSQLKeyword(stmt, keywordEnd) + switch normalizeSQLClassifierDBType(dbType) { + case "sqlite": + return next == "" || next == "transaction" + case "postgres", "kingbase", "highgo", "vastbase", "opengauss", "gaussdb": + return next == "" || next == "work" || next == "transaction" || next == "and" + default: + return false + } +} + +func sqlFileStatementFinishesTransaction(stmt string) bool { + keyword, keywordEnd := nextSQLKeyword(stmt, 0) + switch keyword { + case "commit": + return true + case "rollback": + return !sqlFileRollbackTargetsSavepoint(stmt, keywordEnd) + default: + return false } } func executeSQLFileBatch(ctx context.Context, execer sqlFileStatementExecer, batcher sqlFileBatchStatementExecer, dbType string, batchSQL string, useTransaction bool, text fileBackendTextFunc) (bool, error) { + canFallback, _, err := executeSQLFileBatchWithOutcome(ctx, execer, batcher, dbType, batchSQL, useTransaction, text) + return canFallback, err +} + +func executeSQLFileBatchWithOutcome(ctx context.Context, execer sqlFileStatementExecer, batcher sqlFileBatchStatementExecer, dbType string, batchSQL string, useTransaction bool, text fileBackendTextFunc) (canFallback bool, outcomeUnknown bool, err error) { if !useTransaction { - _, err := batcher.ExecBatchContext(ctx, batchSQL) - return false, err + _, err = batcher.ExecBatchContext(ctx, batchSQL) + return false, false, err } beginSQL, commitSQL, rollbackSQL, ok := sqlFileBatchTransactionSQL(dbType) if !ok { - _, err := batcher.ExecBatchContext(ctx, batchSQL) - return false, err + _, err = batcher.ExecBatchContext(ctx, batchSQL) + return false, false, err } if _, err := execSQLFileStatement(ctx, execer, beginSQL); err != nil { - return true, err - } - if _, err := batcher.ExecBatchContext(ctx, batchSQL); err != nil { - if _, rollbackErr := execSQLFileStatement(ctx, execer, rollbackSQL); rollbackErr != nil { - return false, errors.New(fileBackendText(text, "file.backend.error.sql_file_batch_rollback_failed", map[string]any{ - "detail": err.Error(), - "rollbackDetail": rollbackErr.Error(), + if rollbackErr := rollbackSQLFileTransaction(execer, rollbackSQL); rollbackErr != nil { + return false, true, errors.New(fileBackendText(text, "file.backend.error.sql_file_batch_rollback_failed", map[string]any{ + "detail": sanitizeSQLFileExecutionErr(err), + "rollbackDetail": sanitizeSQLFileExecutionErr(rollbackErr), })) } - return true, err + return false, false, err + } + if _, err := batcher.ExecBatchContext(ctx, batchSQL); err != nil { + if rollbackErr := rollbackSQLFileTransaction(execer, rollbackSQL); rollbackErr != nil { + return false, true, errors.New(fileBackendText(text, "file.backend.error.sql_file_batch_rollback_failed", map[string]any{ + "detail": sanitizeSQLFileExecutionErr(err), + "rollbackDetail": sanitizeSQLFileExecutionErr(rollbackErr), + })) + } + // MySQL-family tables can use non-transactional engines. A successful + // ROLLBACK therefore cannot prove that a partially executed batch left no + // writes behind. Stop and surface the uncertainty instead of inviting a + // blind replay. + return true, isSQLFileMySQLCompatibleDialect(dbType), err } if _, err := execSQLFileStatement(ctx, execer, commitSQL); err != nil { - _, _ = execSQLFileStatement(ctx, execer, rollbackSQL) - return false, err + if rollbackErr := rollbackSQLFileTransaction(execer, rollbackSQL); rollbackErr != nil { + return false, true, errors.New(fileBackendText(text, "file.backend.error.sql_file_batch_rollback_failed", map[string]any{ + "detail": sanitizeSQLFileExecutionErr(err), + "rollbackDetail": sanitizeSQLFileExecutionErr(rollbackErr), + })) + } + // Once COMMIT has been dispatched, an error does not prove whether the + // server committed before the connection/context failure was observed. + return false, true, err } - return false, nil + return false, false, nil } func executeSQLFileStream(ctx context.Context, dbInst db.Database, reader io.Reader, options sqlFileExecutionOptions, bytesRead func() int64) (sqlFileExecutionResult, error) { @@ -1487,16 +1953,28 @@ func executeSQLFileStream(ctx context.Context, dbInst db.Database, reader io.Rea var batch []sqlFilePendingStatement var batchBytes int var lastProgressAt time.Time - var inUserTransaction bool + var userTransactionDepth int + var sqlServerTransaction sqlFileSQLServerTransactionTracker + var mysqlAutocommitDisabled bool + var mysqlAutocommitTransactionActive bool + var mysqlAutocommitStateUnknown bool + var mysqlTablesLocked bool var useTransactionalBatch bool + safeSequentialContinue := options.ContinueOnError && isSQLFileMySQLCompatibleDialect(options.DBType) + var hasPinnedSession bool execer := sqlFileStatementExecer(dbInst) batcher, supportsBatch := dbInst.(sqlFileBatchStatementExecer) + if capability, ok := dbInst.(db.BatchWriteCapability); ok && !capability.SupportsBatchWrites() { + supportsBatch = false + batcher = nil + } if provider, ok := dbInst.(db.SessionExecerProvider); ok { sessionExecer, err := provider.OpenSessionExecer(ctx) if err != nil { return result, err } defer sessionExecer.Close() + hasPinnedSession = true execer = sessionExecer if supportsBatch { var ok bool @@ -1505,6 +1983,24 @@ func executeSQLFileStream(ctx context.Context, dbInst db.Database, reader io.Rea } useTransactionalBatch = supportsBatch } + defer func() { + if userTransactionDepth > 0 || mysqlAutocommitTransactionActive { + _, _, rollbackSQL, ok := sqlFileBatchTransactionSQL(options.DBType) + if !ok { + rollbackSQL = "ROLLBACK" + } + if err := rollbackSQLFileTransaction(execer, rollbackSQL); err != nil { + logger.Warnf("ExecuteSQLFile 未结束事务清理失败，连接已尝试淘汰：type=%s err=%s", options.DBType, sanitizeSQLFileExecutionErr(err)) + } + } + if hasPinnedSession { + if discarder, ok := execer.(db.StatementExecerDiscarter); ok { + if err := discarder.Discard(); err != nil { + logger.Warnf("ExecuteSQLFile 淘汰专用会话失败：type=%s err=%s", options.DBType, sanitizeSQLFileExecutionErr(err)) + } + } + } + }() readBytes := func() int64 { if bytesRead == nil { @@ -1539,67 +2035,177 @@ func executeSQLFileStream(ctx context.Context, dbInst db.Database, reader io.Rea } return !lastProgressAt.IsZero() && time.Since(lastProgressAt) >= sqlFileProgressTimeInterval } + appendErrorDetail := func(detail string) { + if len(result.Errors) < sqlFileMaxErrorDetails { + result.Errors = append(result.Errors, detail) + } + } - recordError := func(index int, stmt string, err error) { + recordError := func(index int, stmt string, err error) string { result.Failed++ errLog := fileBackendText(options.Text, "file.backend.message.statement_failed", map[string]any{ "index": index + 1, - "detail": err.Error(), + "detail": sanitizeSQLFileExecutionError(err.Error()), "sql": sqlFileStatementSnippet(stmt, 200), }) - result.Errors = append(result.Errors, errLog) - logger.Warnf("ExecuteSQLFile %s", errLog) + appendErrorDetail(errLog) + if result.Failed <= sqlFileMaxErrorDetails || result.Failed%1000 == 0 { + logger.Warnf("ExecuteSQLFile %s", errLog) + } + return errLog } - executeSingle := func(item sqlFilePendingStatement) error { - if _, err := execSQLFileStatement(ctx, execer, item.SQL); err != nil { - if ctx.Err() != nil { - return fmt.Errorf("已取消") - } - recordError(item.Index, item.SQL, err) - } else { - result.Executed++ + executeSingle := func(item sqlFilePendingStatement) (bool, error) { + if sqlFileMySQLImplicitCommitBeforeStatement(options.DBType, item.SQL) { + // MySQL-family engines commit the current transaction before attempting + // these statements. This state transition happens even when the DDL or + // administrative statement itself subsequently fails. + userTransactionDepth = 0 + mysqlAutocommitTransactionActive = false } + if ctx.Err() != nil { + return false, errSQLFileCancelled + } + if _, err := execSQLFileStatement(ctx, execer, item.SQL); err != nil { + if sqlFileStatementFinishesTransaction(item.SQL) { + // A user-authored COMMIT/ROLLBACK may have reached the server even + // when its result (including cancellation) was not observed. + result.OutcomeUnknown = true + } + if ctx.Err() != nil { + result.OutcomeUnknown = true + return false, errSQLFileCancelled + } + errLog := recordError(item.Index, item.SQL, err) + if !options.ContinueOnError { + if shouldEmitProgress() { + emitProgress(sqlFileStatementSnippet(item.SQL, 100)) + } + return false, &sqlFileStoppedOnError{detail: errLog} + } + if shouldEmitProgress() { + emitProgress(sqlFileStatementSnippet(item.SQL, 100)) + } + return false, nil + } + result.Executed++ if shouldEmitProgress() { emitProgress(sqlFileStatementSnippet(item.SQL, 100)) } - return nil + return true, nil } executeBatchSequentially := func(items []sqlFilePendingStatement) error { for _, item := range items { - if err := executeSingle(item); err != nil { + if _, err := executeSingle(item); err != nil { return err } } return nil } + var executeIsolationBatch func([]sqlFilePendingStatement) error + var isolateFailedBatch func([]sqlFilePendingStatement, error) error + isolateFailedBatch = func(items []sqlFilePendingStatement, observedErr error) error { + if len(items) == 0 { + return nil + } + if ctx.Err() != nil { + return errSQLFileCancelled + } + if len(items) == 1 { + recordError(items[0].Index, items[0].SQL, observedErr) + emitProgress(sqlFileStatementSnippet(items[0].SQL, 100)) + logger.Warnf("ExecuteSQLFile 已定位失败语句，未重复执行：第 %d 条: %s", items[0].Index+1, sanitizeSQLFileExecutionErr(observedErr)) + return nil + } + if len(items) <= sqlFileBatchIsolationSequentialThreshold { + logger.Warnf("ExecuteSQLFile 失败子批已缩小到 %d 条，将逐条定位：第 %d 条起", len(items), items[0].Index+1) + return executeBatchSequentially(items) + } + + middle := len(items) / 2 + if err := executeIsolationBatch(items[:middle]); err != nil { + return err + } + return executeIsolationBatch(items[middle:]) + } + executeIsolationBatch = func(items []sqlFilePendingStatement) error { + if ctx.Err() != nil { + return errSQLFileCancelled + } + batchSQL := joinSQLFileBatchStatements(items) + canFallback, outcomeUnknown, err := executeSQLFileBatchWithOutcome(ctx, execer, batcher, options.DBType, batchSQL, useTransactionalBatch, options.Text) + if outcomeUnknown { + result.OutcomeUnknown = true + } + if err == nil { + result.Executed += len(items) + if shouldEmitProgress() { + emitProgress(sqlFileStatementSnippet(items[len(items)-1].SQL, 100)) + } + return nil + } + if ctx.Err() != nil { + if !useTransactionalBatch || !canFallback { + result.OutcomeUnknown = true + } + return errSQLFileCancelled + } + if !canFallback { + return errors.New(fileBackendText(options.Text, "file.backend.error.sql_file_batch_execution_failed", map[string]any{ + "index": items[0].Index + 1, + "detail": sanitizeSQLFileExecutionErr(err), + })) + } + return isolateFailedBatch(items, err) + } + flushBatch := func() error { if len(batch) == 0 { return nil } select { case <-ctx.Done(): - return fmt.Errorf("已取消") + return errSQLFileCancelled default: } startIndex := batch[0].Index batchSQL := joinSQLFileBatchStatements(batch) - canFallback, err := executeSQLFileBatch(ctx, execer, batcher, options.DBType, batchSQL, useTransactionalBatch, options.Text) + canFallback, outcomeUnknown, err := executeSQLFileBatchWithOutcome(ctx, execer, batcher, options.DBType, batchSQL, useTransactionalBatch, options.Text) + if outcomeUnknown { + result.OutcomeUnknown = true + } if err != nil { - logger.Warnf("ExecuteSQLFile 批量执行 %d 条语句失败，将降级逐条执行：第 %d 条起: %v", len(batch), startIndex+1, err) + if ctx.Err() != nil { + if !useTransactionalBatch || !canFallback { + result.OutcomeUnknown = true + } + return errSQLFileCancelled + } pending := append([]sqlFilePendingStatement(nil), batch...) batch = batch[:0] batchBytes = 0 if !canFallback { return errors.New(fileBackendText(options.Text, "file.backend.error.sql_file_batch_execution_failed", map[string]any{ "index": startIndex + 1, - "detail": err.Error(), + "detail": sanitizeSQLFileExecutionErr(err), })) } - return executeBatchSequentially(pending) + if !options.ContinueOnError { + errLog := fileBackendText(options.Text, "file.backend.error.sql_file_batch_execution_failed", map[string]any{ + "index": startIndex + 1, + "detail": sanitizeSQLFileExecutionErr(err), + }) + result.Failed++ + appendErrorDetail(errLog) + logger.Warnf("ExecuteSQLFile 批量执行失败并已停止，未逐条重放：第 %d 条起，共 %d 条: %s", startIndex+1, len(pending), sanitizeSQLFileExecutionErr(err)) + emitProgress(sqlFileStatementSnippet(pending[0].SQL, 100)) + return &sqlFileStoppedOnError{detail: errLog} + } + logger.Warnf("ExecuteSQLFile 批量执行 %d 条语句失败，将自适应拆分定位错误：第 %d 条起: %s", len(pending), startIndex+1, sanitizeSQLFileExecutionErr(err)) + return isolateFailedBatch(pending, err) } result.Executed += len(batch) if shouldEmitProgress() { @@ -1610,10 +2216,13 @@ func executeSQLFileStream(ctx context.Context, dbInst db.Database, reader io.Rea return nil } - _, streamErr := streamSQLFile(reader, func(index int, stmt string) error { + _, streamErr := StreamSQLFileWithOptions(reader, SQLStreamOptions{ + DBType: options.DBType, + MaxStatementBytes: options.MaxStatementBytes, + }, func(index int, stmt string) error { select { case <-ctx.Done(): - return fmt.Errorf("已取消") + return errSQLFileCancelled default: } @@ -1621,8 +2230,20 @@ func executeSQLFileStream(ctx context.Context, dbInst db.Database, reader io.Rea if stmt == "" { return nil } + if options.PreflightEachStatement { + preflightResult := PreflightSQLStatement(stmt, options.DBType, index) + if !preflightResult.Safe && preflightResult.Reason != nil { + return &sqlFilePreflightRejectedError{ + reason: *preflightResult.Reason, + executed: result.Executed, + failed: result.Failed, + possibleSideEffects: result.Executed > 0 || result.Failed > 0, + outcomeUnknown: result.Failed > 0, + } + } + } - if supportsBatch && !inUserTransaction && isSQLFileBatchableWriteStatement(options.DBType, stmt) { + if supportsBatch && !safeSequentialContinue && userTransactionDepth == 0 && !mysqlAutocommitDisabled && !mysqlTablesLocked && isSQLFileBatchableWriteStatement(options.DBType, stmt) { stmtBytes := len(stmt) if len(batch) > 0 && (len(batch) >= options.BatchMaxStatements || batchBytes+2+stmtBytes > options.BatchMaxBytes) { if err := flushBatch(); err != nil { @@ -1633,16 +2254,31 @@ func executeSQLFileStream(ctx context.Context, dbInst db.Database, reader io.Rea if err := flushBatch(); err != nil { return err } - canFallback, err := executeSQLFileBatch(ctx, execer, batcher, options.DBType, stmt, useTransactionalBatch, options.Text) + canFallback, outcomeUnknown, err := executeSQLFileBatchWithOutcome(ctx, execer, batcher, options.DBType, stmt, useTransactionalBatch, options.Text) + if outcomeUnknown { + result.OutcomeUnknown = true + } if err != nil { - logger.Warnf("ExecuteSQLFile 超大语句批量执行失败，将降级单条执行：第 %d 条: %v", index+1, err) + if ctx.Err() != nil { + return errSQLFileCancelled + } if !canFallback { return errors.New(fileBackendText(options.Text, "file.backend.error.sql_file_statement_execution_failed", map[string]any{ "index": index + 1, - "detail": err.Error(), + "detail": sanitizeSQLFileExecutionErr(err), })) } - return executeSingle(sqlFilePendingStatement{Index: index, SQL: stmt}) + // This batch contains exactly one oversized statement. The failed + // transactional attempt already executed that statement and rolled it + // back, so calling executeSingle here would repeat the same SQL for no + // diagnostic value and may duplicate writes on non-transactional tables. + errLog := recordError(index, stmt, err) + emitProgress(sqlFileStatementSnippet(stmt, 100)) + if !options.ContinueOnError { + return &sqlFileStoppedOnError{detail: errLog} + } + logger.Warnf("ExecuteSQLFile 超大语句执行失败，已记录并继续，未重复执行：第 %d 条: %s", index+1, sanitizeSQLFileExecutionErr(err)) + return nil } result.Executed++ if shouldEmitProgress() { @@ -1662,10 +2298,53 @@ func executeSQLFileStream(ctx context.Context, dbInst db.Database, reader io.Rea if err := flushBatch(); err != nil { return err } - if err := executeSingle(sqlFilePendingStatement{Index: index, SQL: stmt}); err != nil { + succeeded, err := executeSingle(sqlFilePendingStatement{Index: index, SQL: stmt}) + if err != nil { return err } - inUserTransaction = updateSQLFileTransactionState(inUserTransaction, stmt) + if succeeded { + if normalizeSQLClassifierDBType(options.DBType) == "sqlserver" { + sqlServerTransaction = updateSQLFileSQLServerTransactionTracker(sqlServerTransaction, stmt) + userTransactionDepth = sqlServerTransaction.depth + } else { + userTransactionDepth = updateSQLFileTransactionDepth(options.DBType, userTransactionDepth, stmt) + } + if disabled, known, assigned := sqlFileMySQLAutocommitAssignment(options.DBType, stmt); assigned { + wasKnownDisabled := mysqlAutocommitDisabled && !mysqlAutocommitStateUnknown + mysqlAutocommitStateUnknown = !known + if known { + mysqlAutocommitDisabled = disabled + } else { + // A server-side variable can restore autocommit to either value. + // Disable batching conservatively and discard this session at EOF. + mysqlAutocommitDisabled = true + } + if known && !disabled { + mysqlAutocommitTransactionActive = false + if wasKnownDisabled { + // In the MySQL family, changing autocommit from 0 to 1 + // commits an active explicit transaction as well. + userTransactionDepth = 0 + } + } + } else if mysqlAutocommitDisabled { + if mysqlAutocommitTransactionActive { + mysqlAutocommitTransactionActive = updateSQLFileTransactionState(options.DBType, true, stmt) + } + if isBatchableWriteSQLStatement(options.DBType, stmt) { + mysqlAutocommitTransactionActive = true + } + } + if locksTables, unlocksTables := sqlFileMySQLTableLockCommand(options.DBType, stmt); locksTables { + mysqlTablesLocked = true + } else if unlocksTables && mysqlTablesLocked { + // UNLOCK TABLES commits only when this session actually acquired + // table locks. The tracked LOCK makes that conditional transition known. + userTransactionDepth = 0 + mysqlAutocommitTransactionActive = false + mysqlTablesLocked = false + } + } return nil }) if streamErr != nil { @@ -1674,12 +2353,173 @@ func executeSQLFileStream(ctx context.Context, dbInst db.Database, reader io.Rea if err := flushBatch(); err != nil { return result, err } + if userTransactionDepth > 0 || mysqlAutocommitTransactionActive { + detail := fileBackendText(options.Text, "file.backend.error.sql_file_unclosed_transaction", nil) + result.Failed++ + appendErrorDetail(detail) + return result, &sqlFileStoppedOnError{detail: detail} + } return result, nil } // ExecuteSQLFile 在后端流式读取并执行大 SQL 文件，通过事件推送进度。 // 前端通过 EventsOn("sqlfile:progress", ...) 监听进度。 const sqlFileExecutionPreambleBytes = 64 * 1024 +const sqlFileFullPreflightMaxRawBytes int64 = 64 << 20 + +type preparedSQLFileExecutionSource struct { + source *SQLImportSource + reader io.Reader + preamble []byte + rawSize int64 +} + +type sqlImportContextReader struct { + ctx context.Context + reader io.Reader + beforeRead func(context.Context) +} + +type sqlFileRawProgressObserver struct { + bytesRead int64 + lastReported int64 + lastReportAt time.Time + report func(int64) error +} + +func (observer *sqlFileRawProgressObserver) Write(buffer []byte) (int, error) { + observer.bytesRead += int64(len(buffer)) + shouldReport := observer.lastReportAt.IsZero() || + observer.bytesRead-observer.lastReported >= 1<<20 || + time.Since(observer.lastReportAt) >= 250*time.Millisecond + if shouldReport && observer.report != nil { + observer.lastReported = observer.bytesRead + observer.lastReportAt = time.Now() + if err := observer.report(observer.bytesRead); err != nil { + return len(buffer), err + } + } + return len(buffer), nil +} + +func (reader *sqlImportContextReader) Read(buffer []byte) (int, error) { + if reader == nil || reader.reader == nil { + return 0, io.EOF + } + if reader.ctx != nil { + if err := reader.ctx.Err(); err != nil { + return 0, err + } + } + if reader.beforeRead != nil { + reader.beforeRead(reader.ctx) + if reader.ctx != nil { + if err := reader.ctx.Err(); err != nil { + return 0, err + } + } + } + read, err := reader.reader.Read(buffer) + if reader.ctx != nil { + if contextErr := reader.ctx.Err(); contextErr != nil { + return read, contextErr + } + } + return read, err +} + +var sqlFilePreflightReadHook func(context.Context) + +func (prepared *preparedSQLFileExecutionSource) Close() error { + if prepared == nil || prepared.source == nil { + return nil + } + return prepared.source.Close() +} + +func readSQLFileExecutionPreambleStream(reader io.Reader) ([]byte, io.Reader, error) { + buffer := make([]byte, sqlFileExecutionPreambleBytes) + read, err := io.ReadFull(reader, buffer) + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { + return nil, nil, err + } + preamble := buffer[:read] + return preamble, io.MultiReader(bytes.NewReader(preamble), reader), nil +} + +func shouldFullyPreflightSQLFile(rawSize int64) bool { + return rawSize >= 0 && rawSize <= sqlFileFullPreflightMaxRawBytes +} + +func prepareSQLFileExecutionSource(filePath, dbType string, maxStatementBytes int64, rawObserver io.Writer) (*preparedSQLFileExecutionSource, error) { + return prepareSQLFileExecutionSourceWithContext(context.Background(), filePath, dbType, maxStatementBytes, rawObserver, nil) +} + +func prepareSQLFileExecutionSourceWithContext(ctx context.Context, filePath, dbType string, maxStatementBytes int64, rawObserver io.Writer, preflightRawObserver io.Writer) (*preparedSQLFileExecutionSource, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + info, err := os.Stat(filePath) + if err != nil { + return nil, err + } + if info.IsDir() { + return nil, fmt.Errorf("SQL import source is a directory") + } + fullPreflight := shouldFullyPreflightSQLFile(info.Size()) + var preamble []byte + if fullPreflight { + preflightSource, err := OpenSQLImportSource(filePath, SQLImportSourceOptions{RawObserver: preflightRawObserver}) + if err != nil { + return nil, err + } + preflightPreamble, preflightReader, readErr := readSQLFileExecutionPreambleStream(&sqlImportContextReader{ + ctx: ctx, + reader: preflightSource, + beforeRead: sqlFilePreflightReadHook, + }) + if readErr == nil { + var preflightResult SQLImportPreflightResult + preflightResult, readErr = PreflightSQLImportWithOptions(preflightReader, SQLStreamOptions{ + DBType: dbType, + MaxStatementBytes: maxStatementBytes, + }) + if readErr == nil && !preflightResult.Safe && preflightResult.Reason != nil { + readErr = &sqlFilePreflightRejectedError{reason: *preflightResult.Reason} + } + } + closeErr := preflightSource.Close() + if readErr != nil { + return nil, readErr + } + if closeErr != nil { + return nil, closeErr + } + preamble = preflightPreamble + } + + executionSource, err := OpenSQLImportSource(filePath, SQLImportSourceOptions{RawObserver: rawObserver}) + if err != nil { + return nil, err + } + executionReader := io.Reader(&sqlImportContextReader{ctx: ctx, reader: executionSource}) + if !fullPreflight { + preamble, executionReader, err = readSQLFileExecutionPreambleStream(executionReader) + if err != nil { + _ = executionSource.Close() + return nil, err + } + } + return &preparedSQLFileExecutionSource{ + source: executionSource, + reader: executionReader, + preamble: preamble, + rawSize: info.Size(), + }, nil +} func readSQLFileExecutionPreamble(reader io.ReadSeeker) ([]byte, error) { buffer := make([]byte, sqlFileExecutionPreambleBytes) @@ -1767,9 +2607,24 @@ func resolveSQLFileExecutionRunConfig(config connection.ConnectionConfig, dbName return runConfig } +func buildSQLFileExecutionPayload(executed, failed int, outcome string) map[string]interface{} { + outcome = strings.ToLower(strings.TrimSpace(outcome)) + completed := outcome == "completed" || outcome == "partial" + stoppedOnError := outcome == "stopped" + cancelled := outcome == "cancelled" + return map[string]interface{}{ + "executed": executed, + "failed": failed, + "completed": completed, + "stoppedOnError": stoppedOnError, + "cancelled": cancelled, + "outcome": outcome, + } +} + // ImportDatabaseSQL restores a database from a SQL file while honoring the // connection protections that apply to destructive import workflows. -func (a *App) ImportDatabaseSQL(config connection.ConnectionConfig, dbName string, filePath string, jobID string) connection.QueryResult { +func (a *App) ImportDatabaseSQL(config connection.ConnectionConfig, dbName string, filePath string, jobID string, continueOnError bool) connection.QueryResult { for _, protection := range []connectionProtectionKey{ connectionProtectionDataImport, connectionProtectionStructureEdit, @@ -1784,10 +2639,39 @@ func (a *App) ImportDatabaseSQL(config connection.ConnectionConfig, dbName strin return connection.QueryResult{Success: false, Message: err.Error()} } } - return a.ExecuteSQLFile(config, dbName, filePath, jobID) + if !isDataImportSQLDialectSupported(config) { + return connection.QueryResult{Success: false, Message: a.appText("data_import.capability.reason.database_type_unsupported", nil)} + } + return a.executeSQLFileWithStatementLimitPolicy(config, dbName, filePath, jobID, continueOnError, DefaultSQLImportMaxStatementBytes, true) } -func (a *App) ExecuteSQLFile(config connection.ConnectionConfig, dbName string, filePath string, jobID string) (result connection.QueryResult) { +func (a *App) ExecuteSQLFile(config connection.ConnectionConfig, dbName string, filePath string, jobID string) connection.QueryResult { + // The generic SQL-file runner retains its established compatibility + // behavior. Database restore calls ImportDatabaseSQL and chooses the policy + // explicitly, defaulting to fail-fast in the UI. + if err := ensureConnectionAllowsActionWithText( + config, + connectionProtectionScriptExecution, + "connection.backend.action.import_data", + a.appText, + ); err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + return a.executeSQLFile(config, dbName, filePath, jobID, true) +} + +func (a *App) executeSQLFile(config connection.ConnectionConfig, dbName string, filePath string, jobID string, continueOnError bool) (result connection.QueryResult) { + return a.executeSQLFileWithStatementLimit(config, dbName, filePath, jobID, continueOnError, DefaultSQLImportMaxStatementBytes) +} + +func (a *App) executeSQLFileWithStatementLimit(config connection.ConnectionConfig, dbName string, filePath string, jobID string, continueOnError bool, maxStatementBytes int64) (result connection.QueryResult) { + return a.executeSQLFileWithStatementLimitPolicy(config, dbName, filePath, jobID, continueOnError, maxStatementBytes, false) +} + +func (a *App) executeSQLFileWithStatementLimitPolicy(config connection.ConnectionConfig, dbName string, filePath string, jobID string, continueOnError bool, maxStatementBytes int64, requirePinnedSession bool) (result connection.QueryResult) { + if maxStatementBytes <= 0 { + maxStatementBytes = DefaultSQLImportMaxStatementBytes + } auditSQL := "EXECUTE SQL FILE" auditStatementCount := 0 auditSafeError := "SQL file task failed before an execution summary was available" @@ -1799,21 +2683,137 @@ func (a *App) ExecuteSQLFile(config connection.ConnectionConfig, dbName string, return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.file_path_empty", nil)} } if strings.TrimSpace(jobID) == "" { - jobID = fmt.Sprintf("sqlfile-%d", time.Now().UnixMilli()) + jobID = "sqlfile-" + uuid.NewString() } - logger.Warnf("ExecuteSQLFile 开始：file=%s db=%s jobID=%s", filePath, dbName, jobID) - - // 打开文件 - f, err := os.Open(filePath) + sourceIdentity, err := captureImportSourceIdentity(filePath) if err != nil { return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.open_file_failed", map[string]any{"detail": err.Error()})} } - defer f.Close() - preamble, err := readSQLFileExecutionPreamble(f) + logger.Warnf("ExecuteSQLFile 开始：source=%s size=%d db=%s jobID=%s", sourceIdentity.Token, sourceIdentity.Size, dbName, jobID) + + ctx, cancel := context.WithCancel(context.Background()) + cleanupRegistration, registered := a.registerImportTask(jobID, cancel, importjob.KindSQL) + if !registered { + cancel() + return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.import_job_already_running", nil)} + } + defer cancel() + defer cleanupRegistration() + + managedJob, err := a.beginManagedImportJob(managedImportJobStart{ + ID: jobID, + Kind: importjob.KindSQL, + SourcePath: filePath, + SourceIdentityToken: sourceIdentity.Token, + SourceBytesTotal: sourceIdentity.Size, + ByteProgressKind: "rawSource", + TargetFingerprint: buildImportTargetFingerprint(config, dbName, ""), + ConnectionID: config.ID, + DatabaseName: dbName, + OptionsHash: buildSQLImportOptionsHash(continueOnError, maxStatementBytes), + }) if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + defer func() { + if finishErr := managedJob.finish(managedImportJobFinishFromResult(result)); finishErr != nil && result.Success { + result = connection.QueryResult{Success: false, Message: finishErr.Error(), Data: result.Data} + } + }() + mayHaveDatabaseSideEffects := false + defer func() { + if identityErr := validateImportSourceIdentity(filePath, sourceIdentity); identityErr != nil { + payload, _ := result.Data.(map[string]interface{}) + if payload == nil { + payload = map[string]interface{}{} + } + payload["sourceChanged"] = true + payload["outcomeUnknown"] = mayHaveDatabaseSideEffects + result = connection.QueryResult{ + Success: false, + Message: a.appText("file.backend.error.import_source_changed", nil), + Data: payload, + } + } + }() + + var jobPersistErr error + preflightObserver := &sqlFileRawProgressObserver{report: func(bytesRead int64) error { + uievents.Emit(a.ctx, "sqlfile:progress", map[string]interface{}{ + "jobId": jobID, + "status": "running", + "stage": "preflight", + "executed": 0, + "failed": 0, + "total": 0, + "percent": resolveSQLFileExecutionProgressPercent("running", bytesRead, sourceIdentity.Size), + "bytesRead": bytesRead, + "totalBytes": sourceIdentity.Size, + "byteProgressKind": "rawSource", + "decodedBytes": nil, + "decodedTotalBytes": nil, + "currentSQL": "", + "error": "", + }) + if managedJob == nil || jobPersistErr != nil { + return jobPersistErr + } + jobPersistErr = managedJob.update(managedImportJobProgress{ + Stage: "preflight", + BytesRead: bytesRead, + SourceBytesTotal: sourceIdentity.Size, + ByteProgressKind: "rawSource", + Checkpoint: importjob.Checkpoint{Safe: false, ByteOffset: bytesRead}, + }) + if jobPersistErr != nil { + cancel() + } + return jobPersistErr + }} + fileDigest := sha256.New() + preparedSource, err := prepareSQLFileExecutionSourceWithContext(ctx, filePath, resolveDDLDBType(config), maxStatementBytes, fileDigest, preflightObserver) + if err != nil { + if jobPersistErr != nil { + return connection.QueryResult{ + Success: false, + Data: buildSQLFileExecutionPayload(0, 0, "failed"), + Message: a.appText("file.backend.error.import_job_persist", map[string]any{"detail": jobPersistErr.Error()}), + } + } + if errors.Is(err, context.Canceled) { + return connection.QueryResult{ + Success: false, + Data: buildSQLFileExecutionPayload(0, 0, "cancelled"), + Message: a.appText("file.backend.message.execution_cancelled", map[string]any{"executed": 0, "failed": 0, "duration": 0}), + } + } + if isSQLFilePreExecutionValidationError(err) { + var preflightErr *sqlFilePreflightRejectedError + data := buildSQLFileExecutionPayload(0, 0, "failed") + if errors.As(err, &preflightErr) { + data = buildSQLFilePreflightFailurePayload(preflightErr) + } + return connection.QueryResult{ + Success: false, + Data: data, + Message: a.appText("file.backend.error.sql_file_execution_failed_summary", map[string]any{"detail": err.Error(), "count": 0}), + } + } return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.open_file_failed", map[string]any{"detail": err.Error()})} } + defer preparedSource.Close() + if err := validateImportSourceIdentity(filePath, sourceIdentity); err != nil { + return connection.QueryResult{ + Success: false, + Data: map[string]interface{}{ + "sourceChanged": true, + "outcomeUnknown": false, + }, + Message: a.appText("file.backend.error.import_source_changed", nil), + } + } + preamble := preparedSource.preamble backupPreamble := goNaviMySQLDatabaseBackupPreamble{} isGoNaviMySQLDatabaseBackup := false if strings.EqualFold(strings.TrimSpace(config.Type), "mysql") { @@ -1822,39 +2822,53 @@ func (a *App) ExecuteSQLFile(config connection.ConnectionConfig, dbName string, // GoNavi 的 MySQL 整库备份会在脚本中创建并 USE 源库，因此不能先连接到该库。 runConfig := resolveSQLFileExecutionRunConfig(config, dbName, preamble) + dbInst, err := a.getDatabase(runConfig) if err != nil { - logger.Error(err, "ExecuteSQLFile 获取连接失败：%s", formatConnSummary(runConfig)) - return connection.QueryResult{Success: false, Message: err.Error()} + if errors.Is(err, context.Canceled) || errors.Is(ctx.Err(), context.Canceled) { + return connection.QueryResult{ + Success: false, + Data: buildSQLFileExecutionPayload(0, 0, "cancelled"), + Message: a.appText("file.backend.message.execution_cancelled", map[string]any{"executed": 0, "failed": 0, "duration": 0}), + } + } + logger.Errorf("ExecuteSQLFile 获取连接失败：%s err=%s", formatConnSummary(runConfig), sanitizeSQLFileExecutionErr(err)) + return connection.QueryResult{Success: false, Message: sanitizeSQLFileExecutionErr(err)} + } + if err := ctx.Err(); err != nil { + return connection.QueryResult{ + Success: false, + Data: buildSQLFileExecutionPayload(0, 0, "cancelled"), + Message: a.appText("file.backend.message.execution_cancelled", map[string]any{"executed": 0, "failed": 0, "duration": 0}), + } + } + if requirePinnedSession { + if _, ok := dbInst.(db.SessionExecerProvider); !ok { + return connection.QueryResult{ + Success: false, + Data: buildSQLFileExecutionPayload(0, 0, "failed"), + Message: a.appText("data_import.capability.reason.pinned_session_unavailable", nil), + } + } } - // 获取文件大小用于计算进度 - var totalSize int64 - totalSizeKnown := false - if fi, statErr := f.Stat(); statErr == nil { - totalSize = fi.Size() - totalSizeKnown = true - } - - // 设置取消上下文 - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - a.queryMu.Lock() - a.runningQueries[jobID] = queryContext{ - cancel: cancel, - started: time.Now(), - } - a.queryMu.Unlock() - defer func() { - a.queryMu.Lock() - delete(a.runningQueries, jobID) - a.queryMu.Unlock() - }() + totalSize := preparedSource.rawSize + totalSizeKnown := true if bootstrapSQL := buildGoNaviMySQLDatabaseBackupBootstrapSQL(backupPreamble); isGoNaviMySQLDatabaseBackup && bootstrapSQL != "" { + mayHaveDatabaseSideEffects = true if _, err := execSQLFileStatement(ctx, dbInst, bootstrapSQL); err != nil { - return connection.QueryResult{Success: false, Message: err.Error()} + if errors.Is(err, context.Canceled) || errors.Is(ctx.Err(), context.Canceled) { + return connection.QueryResult{ + Success: false, + Data: buildSQLFileExecutionPayload(0, 0, "cancelled"), + Message: a.appText("file.backend.message.execution_cancelled", map[string]any{"executed": 0, "failed": 0, "duration": 0}), + } + } + data := buildSQLFileExecutionPayload(0, 1, "failed") + data["outcomeUnknown"] = true + data["bootstrapAttempted"] = true + return connection.QueryResult{Success: false, Data: data, Message: sanitizeSQLFileExecutionErr(err)} } } @@ -1862,29 +2876,56 @@ func (a *App) ExecuteSQLFile(config connection.ConnectionConfig, dbName string, emitProgress := func(status string, executed, failed, total int, bytesRead int64, currentSQL string, errMsg string) { percent := resolveSQLFileExecutionProgressPercent(status, bytesRead, totalSize) uievents.Emit(a.ctx, "sqlfile:progress", map[string]interface{}{ - "jobId": jobID, - "status": status, - "executed": executed, - "failed": failed, - "total": total, - "percent": percent, - "bytesRead": bytesRead, - "totalBytes": totalSize, - "currentSQL": currentSQL, - "error": errMsg, + "jobId": jobID, + "status": status, + "stage": "write", + "executed": executed, + "failed": failed, + "total": total, + "percent": percent, + "bytesRead": bytesRead, + "totalBytes": totalSize, + "byteProgressKind": "rawSource", + "decodedBytes": nil, + "decodedTotalBytes": nil, + "currentSQL": currentSQL, + "error": errMsg, }) + if managedJob != nil && jobPersistErr == nil { + jobPersistErr = managedJob.update(managedImportJobProgress{ + Stage: "write", + Current: int64(total), + Total: int64(total), + Succeeded: int64(executed), + Failed: int64(failed), + BytesRead: bytesRead, + SourceBytesTotal: totalSize, + ByteProgressKind: "rawSource", + Checkpoint: importjob.Checkpoint{ + Safe: false, + StatementIndex: int64(total), + ByteOffset: bytesRead, + }, + ForcePersist: status != "running", + }) + if jobPersistErr != nil { + cancel() + } + } } emitProgress("running", 0, 0, 0, 0, "", "") - // 使用 countingReader 追踪已读取字节数 - fileDigest := sha256.New() - cr := &countingReader{r: io.TeeReader(f, fileDigest)} - startTime := time.Now() - execResult, streamErr := executeSQLFileStream(ctx, dbInst, cr, sqlFileExecutionOptions{ - DBType: resolveDDLDBType(runConfig), - Text: a.appText, + execResult, streamErr := executeSQLFileStream(ctx, dbInst, preparedSource.reader, sqlFileExecutionOptions{ + DBType: resolveDDLDBType(runConfig), + MaxStatementBytes: maxStatementBytes, + ContinueOnError: continueOnError, + // Keep the callback guard even after a full small-file preflight so a + // source replacement between the two opens cannot send client commands + // to the database. + PreflightEachStatement: true, + Text: a.appText, OnProgress: func(progress sqlFileExecutionProgress) { emitProgress( progress.Status, @@ -1897,7 +2938,7 @@ func (a *App) ExecuteSQLFile(config connection.ConnectionConfig, dbName string, ) }, }, func() int64 { - return cr.n + return preparedSource.source.RawBytesRead() }) duration := time.Since(startTime) @@ -1907,15 +2948,51 @@ func (a *App) ExecuteSQLFile(config connection.ConnectionConfig, dbName string, auditStatementCount = executedCount + failedCount auditSQL = fmt.Sprintf("EXECUTE SQL FILE EXECUTED_%d FAILED_%d", executedCount, failedCount) auditSafeError = fmt.Sprintf("SQL file task failed after executing %d statement(s); %d statement(s) failed", executedCount, failedCount) - if totalSizeKnown && cr.n == totalSize { - auditSQL += " SHA256_" + hex.EncodeToString(fileDigest.Sum(nil)) + rawBytesRead := preparedSource.source.RawBytesRead() + mayHaveDatabaseSideEffects = mayHaveDatabaseSideEffects || executedCount > 0 || failedCount > 0 + contentSHA256 := "" + if totalSizeKnown && rawBytesRead == totalSize { + contentSHA256 = hex.EncodeToString(fileDigest.Sum(nil)) + auditSQL += " SHA256_" + contentSHA256 } - - if streamErr != nil && streamErr.Error() == "已取消" { - emitProgress("cancelled", executedCount, failedCount, executedCount+failedCount, cr.n, "", a.appText("file.backend.message.user_cancelled", nil)) - logger.Warnf("ExecuteSQLFile 已取消：executed=%d failed=%d duration=%v", executedCount, failedCount, duration) + if managedJob != nil && jobPersistErr == nil { + jobPersistErr = managedJob.update(managedImportJobProgress{ + Stage: "write", + Current: int64(executedCount + failedCount), + Total: int64(executedCount + failedCount), + Succeeded: int64(executedCount), + Failed: int64(failedCount), + BytesRead: rawBytesRead, + SourceBytesTotal: totalSize, + ByteProgressKind: "rawSource", + SourceContentSHA256: contentSHA256, + Checkpoint: importjob.Checkpoint{ + Safe: false, + StatementIndex: int64(executedCount + failedCount), + ByteOffset: rawBytesRead, + }, + OutcomeUnknown: execResult.OutcomeUnknown, + ForcePersist: true, + }) + } + if jobPersistErr != nil { + data := buildSQLFileExecutionPayload(executedCount, failedCount, "failed") + data["outcomeUnknown"] = mayHaveDatabaseSideEffects return connection.QueryResult{ Success: false, + Data: data, + Message: a.appText("file.backend.error.import_job_persist", map[string]any{"detail": jobPersistErr.Error()}), + } + } + + if errors.Is(streamErr, context.Canceled) || errors.Is(ctx.Err(), context.Canceled) { + emitProgress("cancelled", executedCount, failedCount, executedCount+failedCount, rawBytesRead, "", a.appText("file.backend.message.user_cancelled", nil)) + logger.Warnf("ExecuteSQLFile 已取消：executed=%d failed=%d duration=%v", executedCount, failedCount, duration) + data := buildSQLFileExecutionPayload(executedCount, failedCount, "cancelled") + data["outcomeUnknown"] = execResult.OutcomeUnknown + return connection.QueryResult{ + Success: false, + Data: data, Message: a.appText("file.backend.message.execution_cancelled", map[string]any{ "executed": executedCount, "failed": failedCount, @@ -1923,13 +3000,41 @@ func (a *App) ExecuteSQLFile(config connection.ConnectionConfig, dbName string, }), } } + safeStreamError := sanitizeSQLFileExecutionErr(streamErr) - if streamErr != nil { - emitProgress("error", executedCount, failedCount, executedCount+failedCount, cr.n, "", streamErr.Error()) + if errors.Is(streamErr, errSQLFileStoppedOnError) { + emitProgress("error", executedCount, failedCount, executedCount+failedCount, rawBytesRead, "", safeStreamError) + data := buildSQLFileExecutionPayload(executedCount, failedCount, "stopped") + if execResult.OutcomeUnknown { + data["outcomeUnknown"] = true + } return connection.QueryResult{ Success: false, - Message: a.appText("file.backend.error.read_file_error_summary", map[string]any{ - "detail": streamErr.Error(), + Data: data, + Message: a.appText("file.backend.error.sql_file_stopped_on_error_summary", map[string]any{ + "detail": safeStreamError, + "success": executedCount, + "failed": failedCount, + }), + } + } + + if streamErr != nil { + emitProgress("error", executedCount, failedCount, executedCount+failedCount, rawBytesRead, "", safeStreamError) + data := buildSQLFileExecutionPayload(executedCount, failedCount, "failed") + var preflightErr *sqlFilePreflightRejectedError + if errors.As(streamErr, &preflightErr) { + preflightErr.possibleSideEffects = preflightErr.possibleSideEffects || mayHaveDatabaseSideEffects + preflightErr.outcomeUnknown = preflightErr.outcomeUnknown || failedCount > 0 + data = buildSQLFilePreflightFailurePayload(preflightErr) + } else if execResult.OutcomeUnknown || failedCount > 0 { + data["outcomeUnknown"] = true + } + return connection.QueryResult{ + Success: false, + Data: data, + Message: a.appText("file.backend.error.sql_file_execution_failed_summary", map[string]any{ + "detail": safeStreamError, "count": executedCount, }), } @@ -1943,43 +3048,33 @@ func (a *App) ExecuteSQLFile(config connection.ConnectionConfig, dbName string, "duration": duration.Round(time.Millisecond), }) if len(errorLogs) > 0 { - maxShow := 20 - if len(errorLogs) < maxShow { - maxShow = len(errorLogs) - } + maxShow := len(errorLogs) summary += "\n\n" + a.appText("file.backend.message.execution_error_detail_header", map[string]any{"count": maxShow}) + "\n" + strings.Join(errorLogs[:maxShow], "\n") - if len(errorLogs) > maxShow { - summary += "\n" + a.appText("file.backend.message.execution_more_errors", map[string]any{"count": len(errorLogs) - maxShow}) + if omitted := failedCount - maxShow; omitted > 0 { + summary += "\n" + a.appText("file.backend.message.execution_more_errors", map[string]any{"count": omitted}) } } logger.Warnf("ExecuteSQLFile 完成：executed=%d failed=%d duration=%v", executedCount, failedCount, duration) - return connection.QueryResult{Success: failedCount == 0, Message: summary} + data := buildSQLFileExecutionPayload(executedCount, failedCount, func() string { + if failedCount > 0 { + return "partial" + } + return "completed" + }()) + if execResult.OutcomeUnknown { + data["outcomeUnknown"] = true + } + return connection.QueryResult{ + Success: failedCount == 0, + Data: data, + Message: summary, + } } // CancelSQLFileExecution 取消正在执行的 SQL 文件任务。 func (a *App) CancelSQLFileExecution(jobID string) connection.QueryResult { - a.queryMu.Lock() - defer a.queryMu.Unlock() - - if ctx, exists := a.runningQueries[jobID]; exists { - ctx.cancel() - delete(a.runningQueries, jobID) - return connection.QueryResult{Success: true, Message: a.appText("file.backend.message.cancel_requested", nil)} - } - return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.task_not_found", nil)} -} - -// countingReader 包装 io.Reader，追踪已读取的字节数。 -type countingReader struct { - r io.Reader - n int64 -} - -func (cr *countingReader) Read(p []byte) (int, error) { - n, err := cr.r.Read(p) - cr.n += int64(n) - return n, err + return a.cancelImportTaskByKind(jobID, importjob.KindSQL) } func readImportedConnectionConfigFile(path string) (string, error) { @@ -2254,20 +3349,36 @@ func (a *App) SelectDatabaseFile(currentPath string, driverType string) connecti // PreviewImportFile 解析导入文件，返回字段列表、总行数、前 5 行预览数据 func (a *App) PreviewImportFile(filePath string) connection.QueryResult { + return a.PreviewImportFileWithOptions(filePath, ImportFileOptions{}) +} + +// PreviewImportFileWithOptions previews a file with the same parser settings +// that will be used by ImportDataWithProgressOptions. +func (a *App) PreviewImportFileWithOptions(filePath string, options ImportFileOptions) connection.QueryResult { if strings.TrimSpace(filePath) == "" { return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.import_file_empty", nil)} } + if err := validateImportFileOptions(options); err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + sourceIdentity, err := captureImportSourceIdentity(filePath) + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } - preview, err := buildImportPreview(filePath, defaultImportPreviewLimit) + preview, err := buildImportPreviewWithOptions(filePath, defaultImportPreviewLimit, options) if err != nil { return connection.QueryResult{Success: false, Message: err.Error()} } result := map[string]interface{}{ - "columns": preview.Columns, - "totalRows": preview.TotalRows, - "previewRows": preview.PreviewRows, - "filePath": filePath, + "columns": preview.Columns, + "totalRows": preview.TotalRows, + "totalRowsKnown": preview.TotalRowsKnown, + "previewRows": preview.PreviewRows, + "filePath": filePath, + "fileSize": sourceIdentity.Size, + "sourceIdentity": sourceIdentity, } return connection.QueryResult{Success: true, Data: result} @@ -2282,7 +3393,7 @@ func (a *App) ImportData(config connection.ConnectionConfig, dbName, tableName s Filters: []runtime.FileFilter{ { DisplayName: a.appText("file.backend.filter.data_files", nil), - Pattern: "*.csv;*.json;*.xlsx;*.xls", + Pattern: "*.csv;*.json;*.xlsx", }, }, }) @@ -2454,6 +3565,30 @@ func normalizeExportTemporalText(text string) string { return text } +func importTemporalFractionDigits(raw string) int { + text := strings.TrimSpace(raw) + for index := 0; index+8 < len(text); index++ { + if !isDigit(text[index]) || !isDigit(text[index+1]) || text[index+2] != ':' || + !isDigit(text[index+3]) || !isDigit(text[index+4]) || text[index+5] != ':' || + !isDigit(text[index+6]) || !isDigit(text[index+7]) || text[index+8] != '.' { + continue + } + digits := 0 + for cursor := index + 9; cursor < len(text) && isDigit(text[cursor]) && digits < 9; cursor++ { + digits++ + } + return digits + } + return 0 +} + +func importTemporalLayout(base string, fractionDigits int) string { + if fractionDigits <= 0 { + return base + } + return base + "." + strings.Repeat("0", fractionDigits) +} + func normalizeImportTemporalValue(dbType, columnType, raw string) string { text := strings.TrimSpace(raw) if text == "" { @@ -2474,16 +3609,17 @@ func normalizeImportTemporalValue(dbType, columnType, raw string) string { return text } + fractionDigits := importTemporalFractionDigits(text) if isTimeOnlyColumnType(columnType) { - return parsed.Format("15:04:05") + return parsed.Format(importTemporalLayout("15:04:05", fractionDigits)) } if isDateOnlyColumnType(dbType, columnType) { return parsed.Format("2006-01-02") } if isTimezoneAwareColumnType(columnType) { - return parsed.Format("2006-01-02 15:04:05-07:00") + return parsed.Format(importTemporalLayout("2006-01-02 15:04:05", fractionDigits) + "-07:00") } - return parsed.Format("2006-01-02 15:04:05") + return parsed.Format(importTemporalLayout("2006-01-02 15:04:05", fractionDigits)) } func isPgLikeBooleanDBType(dbType string) bool { @@ -2602,6 +3738,9 @@ func formatImportSQLValue(dbType, columnType string, value interface{}) string { if value == nil { return "NULL" } + if literal, ok := formatImportCompositeJSONSQLValue(dbType, value); ok { + return literal + } if isPgLikeBooleanDBType(dbType) && isBooleanColumnType(columnType) { if literal, ok := formatPostgresBooleanSQLValue(value); ok { @@ -2613,10 +3752,33 @@ func formatImportSQLValue(dbType, columnType string, value interface{}) string { normalized := normalizeImportTemporalValue(dbType, columnType, fmt.Sprintf("%v", value)) return "'" + escapeSQLStringLiteralBody(dbType, normalized) + "'" } + if text, ok := value.(string); ok { + return "'" + escapeSQLStringLiteralBody(dbType, text) + "'" + } return formatSQLValue(dbType, value) } +func formatImportCompositeJSONSQLValue(dbType string, value interface{}) (string, bool) { + if _, rawBytes := value.([]byte); rawBytes { + return "", false + } + valueType := reflect.TypeOf(value) + if valueType == nil { + return "", false + } + switch valueType.Kind() { + case reflect.Map, reflect.Slice, reflect.Array: + encoded, err := json.Marshal(value) + if err != nil { + return "NULL", true + } + return "'" + escapeSQLStringLiteralBody(dbType, string(encoded)) + "'", true + default: + return "", false + } +} + // ImportDataWithProgress 执行导入并发送进度事件 func (a *App) ImportDataWithProgress(config connection.ConnectionConfig, dbName, tableName, filePath string) (result connection.QueryResult) { return a.ImportDataWithProgressOptions(config, dbName, tableName, filePath, ImportFileOptions{}) @@ -2624,24 +3786,33 @@ func (a *App) ImportDataWithProgress(config connection.ConnectionConfig, dbName, func buildImportExecutionPayload(resultData importExecutionResult, summary string, cancelled bool) map[string]interface{} { total := resultData.Total - if cancelled { - // Rows that were parsed into an uncommitted buffer are not processed rows. - total = resultData.Success + resultData.Failed + if cancelled || (resultData.StoppedOnError && !resultData.OutcomeUnknown) { + // Rows parsed into a buffer but never attempted are not processed rows. + // A failed batch remains unknown because the batch API may have written a + // subset before returning its error. + total = resultData.Success + resultData.Skipped + resultData.Failed } return map[string]interface{}{ - "success": resultData.Success, - "failed": resultData.Failed, - "total": total, - "affectedRows": int64(resultData.Success), - "errorLogs": resultData.ErrorLogs, - "errorSummary": summary, - "cancelled": cancelled, + "success": resultData.Success, + "skipped": resultData.Skipped, + "failed": resultData.Failed, + "total": total, + "affectedRows": int64(resultData.Success), + "errorLogs": resultData.ErrorLogs, + "errorLogsOmitted": max(0, resultData.Failed-len(resultData.ErrorLogs)), + "errorArtifactId": resultData.ErrorArtifactID, + "errorArtifactCount": resultData.ErrorArtifactCount, + "errorSummary": summary, + "cancelled": cancelled, + "stoppedOnError": resultData.StoppedOnError, + "outcomeUnknown": resultData.OutcomeUnknown, } } func (a *App) cancelledImportResult(resultData importExecutionResult) connection.QueryResult { summary := a.appText("file.backend.message.import_cancelled", map[string]any{ "imported": resultData.Success, + "skipped": resultData.Skipped, "failed": resultData.Failed, }) return connection.QueryResult{ @@ -2651,6 +3822,20 @@ func (a *App) cancelledImportResult(resultData importExecutionResult) connection } } +func (a *App) stoppedImportResult(resultData importExecutionResult, detail string) connection.QueryResult { + summary := a.appText("file.backend.error.import_stopped_on_error", map[string]any{ + "imported": resultData.Success, + "skipped": resultData.Skipped, + "failed": resultData.Failed, + "detail": detail, + }) + return connection.QueryResult{ + Success: false, + Data: buildImportExecutionPayload(resultData, summary, false), + Message: summary, + } +} + // ImportDataWithProgressOptions executes a streamed import with optional source-header // to database-column mappings. ImportDataWithProgress remains the compatibility entrypoint. func (a *App) ImportDataWithProgressOptions(config connection.ConnectionConfig, dbName, tableName, filePath string, options ImportFileOptions) (result connection.QueryResult) { @@ -2659,7 +3844,6 @@ func (a *App) ImportDataWithProgressOptions(config connection.ConnectionConfig, } dbType := resolveDDLDBType(config) schemaName, pureTableName := normalizeSchemaAndTable(config, dbName, tableName) - metadataSchemaName, metadataTableName := normalizeMetadataSchemaAndTable(config, dbName, tableName) auditTarget := strings.TrimSpace(tableName) if pureTableName != "" { auditTarget = quoteTableIdentByType(dbType, schemaName, pureTableName) @@ -2675,37 +3859,78 @@ func (a *App) ImportDataWithProgressOptions(config connection.ConnectionConfig, if err := ensureConnectionAllowsDataImport(config, "connection.backend.action.import_data"); err != nil { return connection.QueryResult{Success: false, Message: err.Error()} } + if err := validateImportFileOptions(options); err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + if err := validateImportConflictPolicyForDB(dbType, options); err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + if strings.TrimSpace(options.ResumeJobID) != "" { + return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.import_resume_unavailable", nil)} + } + sourceIdentity, err := captureImportSourceIdentity(filePath) + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + if expected := strings.TrimSpace(options.SourceIdentityToken); expected != "" && expected != sourceIdentity.Token { + return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.import_source_changed", nil)} + } + metadataSchemaName, metadataTableName := normalizeMetadataSchemaAndTable(config, dbName, tableName) - importCtx := context.Background() + importCtx, importCancel := context.WithCancel(context.Background()) + defer importCancel() jobID := strings.TrimSpace(options.JobID) + var managedJob *managedImportJob + var managedArtifact *managedImportErrorArtifact + mayHaveDatabaseSideEffects := false if jobID != "" { - registeredAt := time.Now() - ctx, cancel := context.WithCancel(context.Background()) - importCtx = ctx - a.queryMu.Lock() - if a.runningQueries == nil { - a.runningQueries = make(map[string]queryContext) - } - if _, exists := a.runningQueries[jobID]; exists { - a.queryMu.Unlock() - cancel() + cleanupRegistration, registered := a.registerImportTask(jobID, importCancel, importjob.KindTable) + if !registered { return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.import_job_already_running", nil)} } - a.runningQueries[jobID] = queryContext{ - cancel: cancel, - started: registeredAt, - retainUntilDone: true, + defer cleanupRegistration() + managedJob, err = a.beginManagedImportJob(managedImportJobStart{ + ID: jobID, + Kind: importjob.KindTable, + SourcePath: filePath, + SourceIdentityToken: sourceIdentity.Token, + SourceBytesTotal: sourceIdentity.Size, + ByteProgressKind: "rawSource", + TargetFingerprint: buildImportTargetFingerprint(config, dbName, tableName), + ConnectionID: config.ID, + DatabaseName: dbName, + TableName: tableName, + OptionsHash: buildImportFileOptionsHash(options), + }) + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} } - a.queryMu.Unlock() - defer cancel() defer func() { - a.queryMu.Lock() - if running, exists := a.runningQueries[jobID]; exists && running.started.Equal(registeredAt) { - delete(a.runningQueries, jobID) + if finishErr := managedJob.finish(managedImportJobFinishFromResult(result)); finishErr != nil && result.Success { + result = connection.QueryResult{Success: false, Message: finishErr.Error(), Data: result.Data} } - a.queryMu.Unlock() }() + managedArtifact, err = a.beginManagedImportErrorArtifact(jobID) + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + defer managedArtifact.abort() } + defer func() { + if identityErr := validateImportSourceIdentity(filePath, sourceIdentity); identityErr != nil { + payload, _ := result.Data.(map[string]interface{}) + if payload == nil { + payload = map[string]interface{}{} + } + payload["sourceChanged"] = true + payload["outcomeUnknown"] = mayHaveDatabaseSideEffects + result = connection.QueryResult{ + Success: false, + Message: a.appText("file.backend.error.import_source_changed", nil), + Data: payload, + } + } + }() if err := importCtx.Err(); err != nil { return a.cancelledImportResult(importExecutionResult{}) } @@ -2720,6 +3945,17 @@ func (a *App) ImportDataWithProgressOptions(config connection.ConnectionConfig, if err := importCtx.Err(); err != nil { return a.cancelledImportResult(importExecutionResult{}) } + tableCapability := ResolveDataImportCapability(runConfig, dbInst).TableImport + if !tableCapability.Supported { + reason := strings.TrimSpace(tableCapability.Reason) + if reason == "" { + reason = DataImportReasonTableRuntimeUnavailable + } + return connection.QueryResult{ + Success: false, + Message: a.appText("data_import.capability.reason."+reason, nil), + } + } targetColumns, colErr := getColumnsWithMetadataFallback(dbInst, config, metadataSchemaName, metadataTableName, a.appText) if errors.Is(importCtx.Err(), context.Canceled) { @@ -2729,36 +3965,131 @@ func (a *App) ImportDataWithProgressOptions(config connection.ConnectionConfig, return connection.QueryResult{Success: false, Message: colErr.Error()} } - writer := newImportDatabaseRowWriter(dbInst, dbType, tableName, newImportColumnTypeLookup(targetColumns)) - batchConsumer := newImportBatchConsumer(writer, defaultImportApplyBatchSize, 0, false, func(state importProgressState) { + writer := newImportDatabaseRowWriterWithOptions(dbInst, dbType, tableName, newImportColumnTypeLookup(targetColumns), options) + continueOnError := resolveImportContinueOnError(options) + var jobPersistErr error + batchConsumer := newImportBatchConsumer(writer, defaultImportApplyBatchSize, 0, false, resolveImportContinueOnError(options), func(state importProgressState) { + if state.Success+state.Skipped+state.Errors > 0 { + mayHaveDatabaseSideEffects = true + } uievents.Emit(a.ctx, "import:progress", state) + if managedJob == nil || jobPersistErr != nil { + return + } + jobPersistErr = managedJob.update(managedImportJobProgress{ + Stage: state.Stage, + Current: int64(state.Current), + Total: int64(state.Total), + Succeeded: int64(state.Success), + Skipped: int64(state.Skipped), + Failed: int64(state.Errors), + BytesRead: state.BytesRead, + SourceBytesTotal: state.TotalBytes, + ByteProgressKind: "rawSource", + Checkpoint: importjob.Checkpoint{ + Safe: false, + SourceRow: int64(state.Current), + ByteOffset: state.BytesRead, + }, + ForcePersist: !continueOnError, + }) + if jobPersistErr != nil { + importCancel() + } }) batchConsumer.SetContext(importCtx) batchConsumer.jobID = jobID + if managedArtifact != nil { + batchConsumer.SetRowErrorHandler(managedArtifact.append) + } consumer, err := newImportColumnMappingConsumer(batchConsumer, options.ColumnMappings, targetColumns) if err != nil { return connection.QueryResult{Success: false, Message: err.Error()} } - if err := streamImportFile(filePath, consumer); err != nil { + finishArtifact := func(resultData *importExecutionResult) error { + if managedArtifact == nil { + return nil + } + return managedArtifact.finish(resultData) + } + if err := streamImportFileWithOptions(filePath, consumer, options); err != nil { resultData := batchConsumer.Result() + if jobPersistErr != nil { + if artifactErr := finishArtifact(&resultData); artifactErr != nil { + jobPersistErr = errors.Join(jobPersistErr, artifactErr) + } + message := a.appText("file.backend.error.import_job_persist", map[string]any{"detail": jobPersistErr.Error()}) + return connection.QueryResult{ + Success: false, + Data: buildImportExecutionPayload(resultData, message, false), + Message: message, + } + } if errors.Is(err, context.Canceled) { + if artifactErr := finishArtifact(&resultData); artifactErr != nil { + return connection.QueryResult{Success: false, Data: buildImportExecutionPayload(resultData, artifactErr.Error(), false), Message: artifactErr.Error()} + } maybeReleaseFileTransferMemory("import-cancelled", int64(resultData.Success+resultData.Failed), filePath) return a.cancelledImportResult(resultData) } + if !errors.Is(err, errImportStoppedOnError) && managedArtifact != nil { + managedArtifact.append(ImportRowError{ + SourceRow: int64(resultData.Total + 1), + Category: "parse", + Message: err.Error(), + }) + } + if artifactErr := finishArtifact(&resultData); artifactErr != nil { + return connection.QueryResult{Success: false, Data: buildImportExecutionPayload(resultData, artifactErr.Error(), false), Message: artifactErr.Error()} + } maybeReleaseFileTransferMemory("import-stream-error", int64(resultData.Total), filePath) - return connection.QueryResult{Success: false, Message: err.Error()} + if errors.Is(err, errImportStoppedOnError) { + return a.stoppedImportResult(resultData, err.Error()) + } + return connection.QueryResult{ + Success: false, + Data: buildImportExecutionPayload(resultData, err.Error(), false), + Message: err.Error(), + } } if err := batchConsumer.Flush(); err != nil { resultData := batchConsumer.Result() + if jobPersistErr != nil { + if artifactErr := finishArtifact(&resultData); artifactErr != nil { + jobPersistErr = errors.Join(jobPersistErr, artifactErr) + } + message := a.appText("file.backend.error.import_job_persist", map[string]any{"detail": jobPersistErr.Error()}) + return connection.QueryResult{ + Success: false, + Data: buildImportExecutionPayload(resultData, message, false), + Message: message, + } + } if errors.Is(err, context.Canceled) { + if artifactErr := finishArtifact(&resultData); artifactErr != nil { + return connection.QueryResult{Success: false, Data: buildImportExecutionPayload(resultData, artifactErr.Error(), false), Message: artifactErr.Error()} + } maybeReleaseFileTransferMemory("import-cancelled", int64(resultData.Success+resultData.Failed), filePath) return a.cancelledImportResult(resultData) } + if artifactErr := finishArtifact(&resultData); artifactErr != nil { + return connection.QueryResult{Success: false, Data: buildImportExecutionPayload(resultData, artifactErr.Error(), false), Message: artifactErr.Error()} + } maybeReleaseFileTransferMemory("import-flush-error", int64(resultData.Total), filePath) - return connection.QueryResult{Success: false, Message: err.Error()} + if errors.Is(err, errImportStoppedOnError) { + return a.stoppedImportResult(resultData, err.Error()) + } + return connection.QueryResult{ + Success: false, + Data: buildImportExecutionPayload(resultData, err.Error(), false), + Message: err.Error(), + } } resultData := batchConsumer.Result() + if artifactErr := finishArtifact(&resultData); artifactErr != nil { + return connection.QueryResult{Success: false, Data: buildImportExecutionPayload(resultData, artifactErr.Error(), false), Message: artifactErr.Error()} + } if resultData.Total == 0 { maybeReleaseFileTransferMemory("import-empty", 0, filePath) return connection.QueryResult{Success: true, Message: a.appText("file.backend.message.import_no_data", nil)} @@ -2766,6 +4097,7 @@ func (a *App) ImportDataWithProgressOptions(config connection.ConnectionConfig, summary := a.appText("file.backend.message.import_summary", map[string]any{ "imported": resultData.Success, + "skipped": resultData.Skipped, "failed": resultData.Failed, }) resultPayload := buildImportExecutionPayload(resultData, summary, false) @@ -4772,6 +6104,16 @@ func formatSQLValue(dbType string, v interface{}) string { return "NULL" } return strconv.FormatFloat(val, 'f', -1, 64) + case json.Number: + literal := strings.TrimSpace(val.String()) + if !jsonNumberSQLLiteralPattern.MatchString(literal) { + return "NULL" + } + // JSON numbers may exceed float64 while remaining valid database numeric + // literals. The strict token pattern above prevents SQL injection; let the + // target column/driver decide its actual numeric range instead of silently + // replacing a valid value with NULL. + return literal case time.Time: return "'" + val.Format("2006-01-02 15:04:05") + "'" case string: diff --git a/internal/app/methods_file_export_test.go b/internal/app/methods_file_export_test.go index c393d30e..fac9cf77 100644 --- a/internal/app/methods_file_export_test.go +++ b/internal/app/methods_file_export_test.go @@ -1546,6 +1546,13 @@ func TestFormatImportSQLValue_LeavesTextLiteralUntouched(t *testing.T) { } } +func TestFormatImportSQLValue_MySQLHexLookingTextRemainsText(t *testing.T) { + got := formatImportSQLValue("mysql", "varchar(32)", "0xDEADBEEF") + if got != "'0xDEADBEEF'" { + t.Fatalf("hex-looking text must remain quoted during import, got %q", got) + } +} + func TestFormatImportSQLValue_PostgresBooleanColumnUsesBooleanLiteral(t *testing.T) { cases := []struct { name string diff --git a/internal/app/methods_file_i18n_test.go b/internal/app/methods_file_i18n_test.go index c6cfa9e8..72aca108 100644 --- a/internal/app/methods_file_i18n_test.go +++ b/internal/app/methods_file_i18n_test.go @@ -134,6 +134,7 @@ func TestImportDataBackendCatalogKeysExist(t *testing.T) { "file.backend.error.import_file_empty", "file.backend.error.import_job_already_running", "file.backend.error.import_json_parse_failed", + "file.backend.error.import_stopped_on_error", "file.backend.error.import_unsupported_format", "file.backend.filter.data_files", "file.backend.message.import_cancelled", @@ -261,7 +262,10 @@ func TestExecuteSQLFileMessageCatalogKeysExist(t *testing.T) { "file.backend.error.read_file_error_summary", "file.backend.error.sql_file_batch_execution_failed", "file.backend.error.sql_file_batch_rollback_failed", + "file.backend.error.sql_file_execution_failed_summary", "file.backend.error.sql_file_statement_execution_failed", + "file.backend.error.sql_file_stopped_on_error_summary", + "file.backend.error.sql_file_unclosed_transaction", "file.backend.error.task_not_found", "file.backend.message.cancel_requested", "file.backend.message.execution_cancelled", diff --git a/internal/app/methods_file_import_managed_test.go b/internal/app/methods_file_import_managed_test.go new file mode 100644 index 00000000..7588a441 --- /dev/null +++ b/internal/app/methods_file_import_managed_test.go @@ -0,0 +1,299 @@ +package app + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "GoNavi-Wails/internal/connection" + "GoNavi-Wails/internal/db" + "GoNavi-Wails/internal/importjob" + "GoNavi-Wails/internal/secretstore" + "GoNavi-Wails/internal/uievents" +) + +type rowErrorImportTestDB struct { + fakeMetadataRetryDB + execCalls int + failAt int + afterExec func(int) +} + +func (database *rowErrorImportTestDB) Exec(string) (int64, error) { + database.execCalls++ + if database.afterExec != nil { + database.afterExec(database.execCalls) + } + if database.execCalls == database.failAt { + return 0, errors.New("duplicate key token=private-value") + } + return 1, nil +} + +func (database *rowErrorImportTestDB) ApplyChanges(_ string, changes connection.ChangeSet) error { + for range changes.Inserts { + if _, err := database.Exec(""); err != nil { + return err + } + } + return nil +} + +func (database *rowErrorImportTestDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) error { + if err := ctx.Err(); err != nil { + return err + } + return database.ApplyChanges(tableName, changes) +} + +func installImportTestDatabase(t *testing.T, database db.Database) { + t.Helper() + originalNewDatabaseFunc := newDatabaseFunc + originalResolveDialConfigWithProxyFunc := resolveDialConfigWithProxyFunc + newDatabaseFunc = func(string) (db.Database, error) { return database, nil } + resolveDialConfigWithProxyFunc = func(config connection.ConnectionConfig) (connection.ConnectionConfig, error) { + return config, nil + } + t.Cleanup(func() { + newDatabaseFunc = originalNewDatabaseFunc + resolveDialConfigWithProxyFunc = originalResolveDialConfigWithProxyFunc + }) +} + +func newManagedImportTestApp(t *testing.T) *App { + t.Helper() + app := NewAppWithSecretStore(secretstore.NewUnavailableStore("test")) + app.configDir = t.TempDir() + app.ctx = uievents.WithEmitter(context.Background(), noopImportEventEmitter{}) + return app +} + +func TestImportDataWithProgressOptionsPersistsCompletedManagedJob(t *testing.T) { + database := &rowErrorImportTestDB{fakeMetadataRetryDB: fakeMetadataRetryDB{ + columns: []connection.ColumnDefinition{{Name: "id", Type: "bigint"}}, + }} + installImportTestDatabase(t, database) + + path := filepath.Join(t.TempDir(), "users.csv") + if err := os.WriteFile(path, []byte("id\n1\n2\n"), 0o600); err != nil { + t.Fatal(err) + } + app := newManagedImportTestApp(t) + stopOnError := false + result := app.ImportDataWithProgressOptions( + connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306, Database: "app"}, + "app", + "users", + path, + ImportFileOptions{JobID: "table-import-complete", ContinueOnError: &stopOnError}, + ) + if !result.Success { + t.Fatalf("import failed: %#v", result) + } + + store, err := app.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + job, err := store.Get("table-import-complete") + if err != nil { + t.Fatal(err) + } + if job.Status != importjob.StatusCompleted || job.Current != 2 || job.Succeeded != 2 || job.Failed != 0 { + t.Fatalf("unexpected managed job: %#v", job) + } + if job.SourceIdentityToken == "" || job.TargetFingerprint == "" || job.OptionsHash == "" { + t.Fatalf("managed job identity is incomplete: %#v", job) + } +} + +func TestImportDataWithProgressOptionsPublishesRejectedRowsAndPartialJob(t *testing.T) { + database := &rowErrorImportTestDB{ + fakeMetadataRetryDB: fakeMetadataRetryDB{columns: []connection.ColumnDefinition{{Name: "id", Type: "bigint"}}}, + failAt: 2, + } + installImportTestDatabase(t, database) + + path := filepath.Join(t.TempDir(), "users.csv") + if err := os.WriteFile(path, []byte("id\n1\n2\n3\n"), 0o600); err != nil { + t.Fatal(err) + } + app := newManagedImportTestApp(t) + continueOnError := true + result := app.ImportDataWithProgressOptions( + connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306, Database: "app"}, + "app", + "users", + path, + ImportFileOptions{JobID: "table-import-partial", ContinueOnError: &continueOnError}, + ) + if !result.Success { + t.Fatalf("continue import failed: %#v", result) + } + payload, ok := result.Data.(map[string]interface{}) + if !ok { + t.Fatalf("unexpected payload type: %T", result.Data) + } + artifactID, _ := payload["errorArtifactId"].(string) + if artifactID == "" || payload["errorArtifactCount"] != int64(1) { + t.Fatalf("missing rejected-row artifact: %#v", payload) + } + + store, err := app.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + job, err := store.Get("table-import-partial") + if err != nil { + t.Fatal(err) + } + if job.Status != importjob.StatusPartial || job.Succeeded != 2 || job.Failed != 1 || job.ErrorArtifactID != artifactID { + t.Fatalf("unexpected partial job: %#v", job) + } + + artifactStore, err := app.ensureImportErrorArtifactStore() + if err != nil { + t.Fatal(err) + } + artifact, err := artifactStore.Open(artifactID) + if err != nil { + t.Fatal(err) + } + defer artifact.Close() + var rejected ImportRowError + if err := json.NewDecoder(artifact).Decode(&rejected); err != nil { + t.Fatal(err) + } + if rejected.SourceRow != 2 || rejected.Category != "database" || rejected.Values["id"] != "2" { + t.Fatalf("unexpected rejected row: %#v", rejected) + } + if rejected.Message == "" || rejected.Message == "duplicate key token=private-value" { + t.Fatalf("database error was not sanitized: %q", rejected.Message) + } +} + +func TestImportDataWithProgressOptionsRejectsStaleSourceBeforeDatabaseAccess(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.csv") + if err := os.WriteFile(path, []byte("id\n1\n"), 0o600); err != nil { + t.Fatal(err) + } + identity, err := captureImportSourceIdentity(path) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("id\n1\n2\n"), 0o600); err != nil { + t.Fatal(err) + } + + originalNewDatabaseFunc := newDatabaseFunc + t.Cleanup(func() { newDatabaseFunc = originalNewDatabaseFunc }) + opened := false + newDatabaseFunc = func(string) (db.Database, error) { + opened = true + return &rowErrorImportTestDB{}, nil + } + result := NewApp().ImportDataWithProgressOptions( + connection.ConnectionConfig{Type: "mysql"}, "app", "users", path, + ImportFileOptions{SourceIdentityToken: identity.Token}, + ) + if result.Success { + t.Fatalf("stale source should fail: %#v", result) + } + if opened { + t.Fatal("stale source must be rejected before opening the database") + } +} + +func TestImportDataWithProgressOptionsMarksOutcomeUnknownWhenSourceChangesDuringRun(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.csv") + if err := os.WriteFile(path, []byte("id\n1\n2\n"), 0o600); err != nil { + t.Fatal(err) + } + database := &rowErrorImportTestDB{ + fakeMetadataRetryDB: fakeMetadataRetryDB{columns: []connection.ColumnDefinition{{Name: "id", Type: "bigint"}}}, + afterExec: func(call int) { + if call == 1 { + file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + t.Errorf("open source for mutation: %v", err) + return + } + if _, err := file.WriteString("3\n"); err != nil { + t.Errorf("mutate source: %v", err) + } + if err := file.Close(); err != nil { + t.Errorf("close mutated source: %v", err) + } + } + }, + } + installImportTestDatabase(t, database) + app := newManagedImportTestApp(t) + continueOnError := true + result := app.ImportDataWithProgressOptions( + connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306, Database: "app"}, + "app", "users", path, + ImportFileOptions{JobID: "table-import-mutated", ContinueOnError: &continueOnError}, + ) + if result.Success { + t.Fatalf("mutated source must not be reported as a certain success: %#v", result) + } + payload, ok := result.Data.(map[string]interface{}) + if !ok || payload["sourceChanged"] != true || payload["outcomeUnknown"] != true { + t.Fatalf("unexpected source-change payload: %#v", result.Data) + } + store, err := app.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + job, err := store.Get("table-import-mutated") + if err != nil { + t.Fatal(err) + } + if job.Status != importjob.StatusUnknown || !job.OutcomeUnknown { + t.Fatalf("unexpected source-change job: %#v", job) + } +} + +func TestCompatibilityTableImportAlsoDetectsSourceChangesDuringRun(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.csv") + if err := os.WriteFile(path, []byte("id\n1\n2\n"), 0o600); err != nil { + t.Fatal(err) + } + database := &rowErrorImportTestDB{ + fakeMetadataRetryDB: fakeMetadataRetryDB{columns: []connection.ColumnDefinition{{Name: "id", Type: "bigint"}}}, + afterExec: func(call int) { + if call != 1 { + return + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + t.Errorf("open source for mutation: %v", err) + return + } + if _, err := file.WriteString("3\n"); err != nil { + t.Errorf("mutate source: %v", err) + } + if err := file.Close(); err != nil { + t.Errorf("close mutated source: %v", err) + } + }, + } + installImportTestDatabase(t, database) + continueOnError := true + result := NewApp().ImportDataWithProgressOptions( + connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306, Database: "app"}, + "app", "users", path, + ImportFileOptions{ContinueOnError: &continueOnError}, + ) + if result.Success { + t.Fatalf("mutated source must not be reported as a certain success: %#v", result) + } + payload, ok := result.Data.(map[string]interface{}) + if !ok || payload["sourceChanged"] != true || payload["outcomeUnknown"] != true { + t.Fatalf("unexpected source-change payload: %#v", result.Data) + } +} diff --git a/internal/app/methods_file_import_registration_test.go b/internal/app/methods_file_import_registration_test.go new file mode 100644 index 00000000..042f546c --- /dev/null +++ b/internal/app/methods_file_import_registration_test.go @@ -0,0 +1,77 @@ +package app + +import ( + "context" + "os" + "path/filepath" + "testing" + + "GoNavi-Wails/internal/connection" + "GoNavi-Wails/internal/db" + "GoNavi-Wails/internal/importjob" +) + +func TestImportDatabaseSQLRejectsDuplicateJobBeforeOpeningDatabase(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "database.sql") + if err := os.WriteFile(filePath, []byte("SELECT 1;"), 0o600); err != nil { + t.Fatal(err) + } + + app := NewApp() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cleanup, registered := app.registerExclusiveRunningQuery("duplicate-sql-import", cancel, true) + if !registered { + t.Fatal("fixture registration failed") + } + defer cleanup() + + originalNewDatabaseFunc := newDatabaseFunc + t.Cleanup(func() { newDatabaseFunc = originalNewDatabaseFunc }) + opened := false + newDatabaseFunc = func(string) (db.Database, error) { + opened = true + return &fakeSQLFileBatchDB{}, nil + } + + result := app.ImportDatabaseSQL(connection.ConnectionConfig{Type: "mysql"}, "app", filePath, "duplicate-sql-import", false) + if result.Success { + t.Fatalf("duplicate SQL import should fail: %#v", result) + } + if opened { + t.Fatal("duplicate SQL import must be rejected before opening a database") + } + select { + case <-ctx.Done(): + t.Fatal("duplicate import must not cancel the existing owner") + default: + } +} + +func TestCancelSQLFileExecutionRetainsRegistrationUntilOwnerCleanup(t *testing.T) { + app := NewApp() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cleanup, registered := app.registerImportTask("sql-import-cancel", cancel, importjob.KindSQL) + if !registered { + t.Fatal("fixture registration failed") + } + defer cleanup() + + first := app.CancelSQLFileExecution("sql-import-cancel") + second := app.CancelSQLFileExecution("sql-import-cancel") + if !first.Success || !second.Success { + t.Fatalf("repeated cancellation should remain idempotent until owner exits: first=%#v second=%#v", first, second) + } + select { + case <-ctx.Done(): + default: + t.Fatal("cancel request did not reach task context") + } + app.queryMu.RLock() + _, retained := app.runningQueries["sql-import-cancel"] + app.queryMu.RUnlock() + if !retained { + t.Fatal("cancel request must not remove a task that is still unwinding") + } +} diff --git a/internal/app/methods_file_import_test.go b/internal/app/methods_file_import_test.go index c9b872bc..74e8fd75 100644 --- a/internal/app/methods_file_import_test.go +++ b/internal/app/methods_file_import_test.go @@ -2,6 +2,7 @@ package app import ( "context" + "encoding/json" "errors" "fmt" "os" @@ -9,6 +10,7 @@ import ( "reflect" "strings" "testing" + "time" "GoNavi-Wails/internal/connection" "GoNavi-Wails/internal/db" @@ -63,8 +65,11 @@ func TestBuildImportPreviewCSVStreamKeepsFirstFiveRows(t *testing.T) { if !reflect.DeepEqual(preview.Columns, []string{"id", "name"}) { t.Fatalf("unexpected columns: %#v", preview.Columns) } - if preview.TotalRows != 7 { - t.Fatalf("expected 7 rows, got %d", preview.TotalRows) + if preview.TotalRows != 5 { + t.Fatalf("expected preview to stop after 5 rows, got %d", preview.TotalRows) + } + if preview.TotalRowsKnown { + t.Fatal("short-circuited preview must report an unknown total row count") } if len(preview.PreviewRows) != 5 { t.Fatalf("expected 5 preview rows, got %d", len(preview.PreviewRows)) @@ -77,6 +82,73 @@ func TestBuildImportPreviewCSVStreamKeepsFirstFiveRows(t *testing.T) { } } +func TestPreviewImportFileReportsBoundedTotalAndStableSourceIdentity(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.csv") + if err := os.WriteFile(path, []byte("id,name\n1,a\n2,b\n3,c\n4,d\n5,e\n6,f\n"), 0o600); err != nil { + t.Fatal(err) + } + + result := NewApp().PreviewImportFile(path) + if !result.Success { + t.Fatalf("preview failed: %#v", result) + } + payload, ok := result.Data.(map[string]interface{}) + if !ok { + t.Fatalf("unexpected payload type: %T", result.Data) + } + if known, _ := payload["totalRowsKnown"].(bool); known { + t.Fatalf("bounded preview incorrectly reported a known total: %#v", payload) + } + if size, _ := payload["fileSize"].(int64); size <= 0 { + t.Fatalf("missing file size: %#v", payload) + } + identity, ok := payload["sourceIdentity"].(ImportSourceIdentity) + if !ok || identity.Token == "" { + t.Fatalf("missing source identity: %T %#v", payload["sourceIdentity"], payload["sourceIdentity"]) + } +} + +func TestPreviewImportFileWithOptionsUsesTheSameParserSettingsAsImport(t *testing.T) { + path := filepath.Join(t.TempDir(), "users.csv") + if err := os.WriteFile(path, []byte("ignored\nid;name\n1;alice\n"), 0o600); err != nil { + t.Fatal(err) + } + + result := NewApp().PreviewImportFileWithOptions(path, ImportFileOptions{ + Delimiter: "semicolon", + HeaderRow: 2, + }) + if !result.Success { + t.Fatalf("preview failed: %#v", result) + } + payload, ok := result.Data.(map[string]interface{}) + if !ok { + t.Fatalf("unexpected payload type: %T", result.Data) + } + if !reflect.DeepEqual(payload["columns"], []string{"id", "name"}) { + t.Fatalf("unexpected columns: %#v", payload["columns"]) + } + rows, ok := payload["previewRows"].([]map[string]interface{}) + if !ok || len(rows) != 1 || rows[0]["name"] != "alice" { + t.Fatalf("unexpected preview rows: %#v", payload["previewRows"]) + } +} + +func TestFormatSQLValuePreservesValidatedJSONNumbersWithoutQuoting(t *testing.T) { + if got := formatSQLValue("mysql", json.Number("9007199254740993")); got != "9007199254740993" { + t.Fatalf("large JSON integer = %q", got) + } + if got := formatSQLValue("postgres", json.Number("-1.25e+4")); got != "-1.25e+4" { + t.Fatalf("JSON decimal = %q", got) + } + if got := formatSQLValue("postgres", json.Number("1e1000")); got != "1e1000" { + t.Fatalf("database-sized JSON decimal must not be silently converted to NULL: %q", got) + } + if got := formatSQLValue("mysql", json.Number("0); DROP TABLE users;--")); got != "NULL" { + t.Fatalf("invalid JSON number must not become SQL: %q", got) + } +} + func TestBuildImportRowFromValuesPreservesPositionsWhenHeaderContainsBlankColumns(t *testing.T) { row := buildImportRowFromValues([]string{"id", "", "name"}, []string{"1", "ignored", "alice"}) if got := row["id"]; got != "1" { @@ -180,6 +252,7 @@ func TestBuildImportPreviewXLSXStreamSupportsSharedStrings(t *testing.T) { type fakeImportRowWriter struct { columns []string + disableBatch bool batchCalls int singleCalls int batchSizes []int @@ -189,6 +262,17 @@ type fakeImportRowWriter struct { afterSingleCall func() } +type contextBlockingImportRowWriter struct { + fakeImportRowWriter + started chan struct{} +} + +func (w *contextBlockingImportRowWriter) ApplyBatchContext(ctx context.Context, _ []map[string]interface{}) error { + close(w.started) + <-ctx.Done() + return ctx.Err() +} + type noopImportEventEmitter struct{} func (noopImportEventEmitter) Emit(string, ...any) {} @@ -199,6 +283,25 @@ type cancellableImportTestDB struct { afterFirstExec func() } +type failingBatchImportTestDB struct { + fakeMetadataRetryDB + batchCalls int + execCalls int +} + +type unsupportedTableImportRuntimeDB struct { + db.Database + connectCalls int +} + +func (d *unsupportedTableImportRuntimeDB) Connect(connection.ConnectionConfig) error { + d.connectCalls++ + return nil +} + +func (*unsupportedTableImportRuntimeDB) Close() error { return nil } +func (*unsupportedTableImportRuntimeDB) Ping() error { return nil } + func (d *cancellableImportTestDB) Exec(string) (int64, error) { d.execCalls++ if d.execCalls == 1 && d.afterFirstExec != nil { @@ -207,6 +310,39 @@ func (d *cancellableImportTestDB) Exec(string) (int64, error) { return 1, nil } +func (d *cancellableImportTestDB) ApplyChanges(_ string, changes connection.ChangeSet) error { + for range changes.Inserts { + if _, err := d.Exec(""); err != nil { + return err + } + } + return nil +} + +func (d *cancellableImportTestDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) error { + if err := ctx.Err(); err != nil { + return err + } + return d.ApplyChanges(tableName, changes) +} + +func (d *failingBatchImportTestDB) ApplyChanges(string, connection.ChangeSet) error { + d.batchCalls++ + return fmt.Errorf("batch rejected") +} + +func (d *failingBatchImportTestDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) error { + if err := ctx.Err(); err != nil { + return err + } + return d.ApplyChanges(tableName, changes) +} + +func (d *failingBatchImportTestDB) Exec(string) (int64, error) { + d.execCalls++ + return 1, nil +} + func (w *fakeImportRowWriter) SetColumns(columns []string) { w.columns = append([]string(nil), columns...) } @@ -230,7 +366,7 @@ func (w *fakeImportRowWriter) ApplyOne(row map[string]interface{}) error { } func (w *fakeImportRowWriter) BatchEnabled() bool { - return true + return !w.disableBatch } func TestImportColumnMappingConsumerStreamsMappedColumnsAndRows(t *testing.T) { @@ -240,7 +376,7 @@ func TestImportColumnMappingConsumerStreamsMappedColumnsAndRows(t *testing.T) { } writer := &fakeImportRowWriter{} - batchConsumer := newImportBatchConsumer(writer, 1000, 0, false, nil) + batchConsumer := newImportBatchConsumer(writer, 1000, 0, false, false, nil) consumer, err := newImportColumnMappingConsumer(batchConsumer, map[string]string{ "User ID": "ID", "Display Name": "display_name", @@ -422,6 +558,77 @@ func TestImportDataWithProgressOptionsRejectsEmptyFilePathBeforeDatabaseAccess(t } } +func TestBuildImportExecutionPayloadUsesAttemptedRowsForKnownStop(t *testing.T) { + payload := buildImportExecutionPayload(importExecutionResult{ + Success: 1, + Failed: 1, + Total: 1000, + StoppedOnError: true, + }, "stopped", false) + if payload["total"] != 2 { + t.Fatalf("known stopped total = %v, want 2 attempted rows", payload["total"]) + } + + unknownPayload := buildImportExecutionPayload(importExecutionResult{ + Failed: 1, + Total: 1000, + StoppedOnError: true, + OutcomeUnknown: true, + }, "stopped", false) + if unknownPayload["total"] != 1000 { + t.Fatalf("unknown batch total = %v, want 1000 submitted rows", unknownPayload["total"]) + } +} + +func TestImportDataWithProgressOptionsStopModeReturnsPartialResultWithoutReplay(t *testing.T) { + originalNewDatabaseFunc := newDatabaseFunc + originalResolveDialConfigWithProxyFunc := resolveDialConfigWithProxyFunc + t.Cleanup(func() { + newDatabaseFunc = originalNewDatabaseFunc + resolveDialConfigWithProxyFunc = originalResolveDialConfigWithProxyFunc + }) + + database := &failingBatchImportTestDB{fakeMetadataRetryDB: fakeMetadataRetryDB{ + columns: []connection.ColumnDefinition{{Name: "id", Type: "bigint"}}, + }} + newDatabaseFunc = func(string) (db.Database, error) { return database, nil } + resolveDialConfigWithProxyFunc = func(config connection.ConnectionConfig) (connection.ConnectionConfig, error) { + return config, nil + } + + path := filepath.Join(t.TempDir(), "users.csv") + if err := os.WriteFile(path, []byte("id\n1\n2\n3\n"), 0o600); err != nil { + t.Fatalf("write csv: %v", err) + } + continueOnError := false + app := NewAppWithSecretStore(secretstore.NewUnavailableStore("test")) + app.configDir = t.TempDir() + app.ctx = uievents.WithEmitter(context.Background(), noopImportEventEmitter{}) + result := app.ImportDataWithProgressOptions( + connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306, Database: "app"}, + "app", + "users", + path, + ImportFileOptions{ + ColumnMappings: map[string]string{"id": "id"}, + ContinueOnError: &continueOnError, + }, + ) + if result.Success { + t.Fatal("stop mode batch failure must not be reported as success") + } + payload, ok := result.Data.(map[string]interface{}) + if !ok { + t.Fatalf("result data type = %T, want map[string]interface{}", result.Data) + } + if payload["stoppedOnError"] != true || payload["outcomeUnknown"] != true || payload["success"] != 0 || payload["failed"] != 1 || payload["total"] != 3 { + t.Fatalf("unexpected stopped payload: %#v", payload) + } + if database.batchCalls != 1 || database.execCalls != 0 { + t.Fatalf("failed batch was replayed: batchCalls=%d execCalls=%d", database.batchCalls, database.execCalls) + } +} + func TestImportDataWithProgressOptionsStopsByJobIDAndReturnsCommittedPartialResult(t *testing.T) { originalNewDatabaseFunc := newDatabaseFunc originalResolveDialConfigWithProxyFunc := resolveDialConfigWithProxyFunc @@ -446,10 +653,11 @@ func TestImportDataWithProgressOptionsStopsByJobIDAndReturnsCommittedPartialResu app := NewAppWithSecretStore(secretstore.NewUnavailableStore("test")) app.ctx = uievents.WithEmitter(context.Background(), noopImportEventEmitter{}) const jobID = "import-cancel-test" + app.configDir = t.TempDir() database.afterFirstExec = func() { - cancelResult := app.CancelQuery(jobID) + cancelResult := app.CancelImportJob(jobID) if !cancelResult.Success { - t.Errorf("CancelQuery returned failure: %s", cancelResult.Message) + t.Errorf("CancelImportJob returned failure: %s", cancelResult.Message) } app.queryMu.Lock() _, retainedWhileStopping := app.runningQueries[jobID] @@ -544,9 +752,44 @@ func TestImportDataWithProgressOptionsUsesOracleColumnMetadataFallback(t *testin } } +func TestImportDataWithProgressOptionsRejectsUnsupportedTableRuntimeBeforeMetadata(t *testing.T) { + originalNewDatabaseFunc := newDatabaseFunc + originalResolveDialConfigWithProxyFunc := resolveDialConfigWithProxyFunc + t.Cleanup(func() { + newDatabaseFunc = originalNewDatabaseFunc + resolveDialConfigWithProxyFunc = originalResolveDialConfigWithProxyFunc + }) + + database := &unsupportedTableImportRuntimeDB{} + newDatabaseFunc = func(string) (db.Database, error) { return database, nil } + resolveDialConfigWithProxyFunc = func(config connection.ConnectionConfig) (connection.ConnectionConfig, error) { + return config, nil + } + path := filepath.Join(t.TempDir(), "users.csv") + if err := os.WriteFile(path, []byte("id\n1\n"), 0o600); err != nil { + t.Fatal(err) + } + + app := NewAppWithSecretStore(secretstore.NewUnavailableStore("test")) + app.ctx = uievents.WithEmitter(context.Background(), noopImportEventEmitter{}) + result := app.ImportDataWithProgressOptions( + connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306}, + "app", + "users", + path, + ImportFileOptions{}, + ) + if result.Success { + t.Fatalf("unsupported table runtime unexpectedly imported: %#v", result) + } + if database.connectCalls != 1 { + t.Fatalf("database connect calls = %d, want 1 capability probe", database.connectCalls) + } +} + func TestImportBatchConsumerUsesBatchWriterInConfiguredBatches(t *testing.T) { writer := &fakeImportRowWriter{} - consumer := newImportBatchConsumer(writer, 1000, 1201, true, nil) + consumer := newImportBatchConsumer(writer, 1000, 1201, true, false, nil) if err := consumer.SetColumns([]string{"id"}); err != nil { t.Fatalf("SetColumns returned error: %v", err) } @@ -574,14 +817,80 @@ func TestImportBatchConsumerUsesBatchWriterInConfiguredBatches(t *testing.T) { } } -func TestImportBatchConsumerFallsBackToSingleRowsWhenBatchFails(t *testing.T) { +func TestImportBatchConsumerCancelsInFlightContextBatchAndMarksOutcomeUnknown(t *testing.T) { + writer := &contextBlockingImportRowWriter{started: make(chan struct{})} + consumer := newImportBatchConsumer(writer, 2, 2, true, false, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + consumer.SetContext(ctx) + if err := consumer.SetColumns([]string{"id"}); err != nil { + t.Fatal(err) + } + if err := consumer.ConsumeRow(map[string]interface{}{"id": 1}); err != nil { + t.Fatal(err) + } + + done := make(chan error, 1) + go func() { + done <- consumer.ConsumeRow(map[string]interface{}{"id": 2}) + }() + select { + case <-writer.started: + case <-time.After(time.Second): + t.Fatal("context-aware batch write did not start") + } + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("ConsumeRow error = %v, want context.Canceled", err) + } + case <-time.After(time.Second): + t.Fatal("in-flight batch write did not stop after cancellation") + } + + result := consumer.Result() + if !result.OutcomeUnknown || result.Success != 0 || result.Total != 2 { + t.Fatalf("unexpected cancellation result: %#v", result) + } +} + +func TestImportBatchConsumerStopModeDoesNotReplayFailedBatch(t *testing.T) { + writer := &fakeImportRowWriter{batchErr: fmt.Errorf("batch failed")} + consumer := newImportBatchConsumer(writer, 1000, 3, true, false, nil) + if err := consumer.SetColumns([]string{"id"}); err != nil { + t.Fatalf("SetColumns returned error: %v", err) + } + for i := 1; i <= 3; i++ { + if err := consumer.ConsumeRow(map[string]interface{}{"id": i}); err != nil { + t.Fatalf("ConsumeRow(%d) returned error before flush: %v", i, err) + } + } + + err := consumer.Flush() + if !errors.Is(err, errImportStoppedOnError) { + t.Fatalf("Flush error = %v, want errImportStoppedOnError", err) + } + if writer.batchCalls != 1 { + t.Fatalf("batch calls = %d, want 1", writer.batchCalls) + } + if writer.singleCalls != 0 { + t.Fatalf("failed batch must not be replayed row by row, single calls=%d", writer.singleCalls) + } + result := consumer.Result() + if result.Success != 0 || result.Failed != 1 || !result.StoppedOnError || !result.OutcomeUnknown { + t.Fatalf("unexpected stopped result: %#v", result) + } +} + +func TestImportBatchConsumerContinueModeExecutesEachRowOnce(t *testing.T) { writer := &fakeImportRowWriter{ batchErr: fmt.Errorf("batch failed"), singleErrByRowID: map[interface{}]error{ 2: fmt.Errorf("duplicate key"), }, } - consumer := newImportBatchConsumer(writer, 1000, 3, true, nil) + consumer := newImportBatchConsumer(writer, 1000, 3, true, true, nil) if err := consumer.SetColumns([]string{"id"}); err != nil { t.Fatalf("SetColumns returned error: %v", err) } @@ -598,8 +907,8 @@ func TestImportBatchConsumerFallsBackToSingleRowsWhenBatchFails(t *testing.T) { if result.Success != 2 || result.Failed != 1 || result.Total != 3 { t.Fatalf("unexpected result: %#v", result) } - if writer.batchCalls != 1 { - t.Fatalf("expected 1 batch call, got %d", writer.batchCalls) + if writer.batchCalls != 0 { + t.Fatalf("continue mode must not attempt an ambiguous batch, got %d calls", writer.batchCalls) } if writer.singleCalls != 3 { t.Fatalf("expected 3 single-row fallback calls, got %d", writer.singleCalls) @@ -609,13 +918,89 @@ func TestImportBatchConsumerFallsBackToSingleRowsWhenBatchFails(t *testing.T) { } } +func TestImportBatchConsumerStopModeStopsAtFirstSingleRowError(t *testing.T) { + writer := &fakeImportRowWriter{ + disableBatch: true, + singleErrByRowID: map[interface{}]error{ + 2: fmt.Errorf("duplicate key"), + }, + } + consumer := newImportBatchConsumer(writer, 1000, 3, true, false, nil) + for i := 1; i <= 3; i++ { + if err := consumer.ConsumeRow(map[string]interface{}{"id": i}); err != nil { + t.Fatalf("ConsumeRow(%d) returned error before flush: %v", i, err) + } + } + + err := consumer.Flush() + if !errors.Is(err, errImportStoppedOnError) { + t.Fatalf("Flush error = %v, want errImportStoppedOnError", err) + } + if writer.singleCalls != 2 { + t.Fatalf("single calls = %d, want stop after row 2", writer.singleCalls) + } + result := consumer.Result() + if result.Success != 1 || result.Failed != 1 || !result.StoppedOnError || result.OutcomeUnknown { + t.Fatalf("unexpected stopped result: %#v", result) + } +} + +func TestImportBatchConsumerCapsErrorDetailsWithoutLosingFailureCount(t *testing.T) { + rowCount := maxImportErrorDetails + 7 + singleErrors := make(map[interface{}]error, rowCount) + for i := 1; i <= rowCount; i++ { + singleErrors[i] = fmt.Errorf("duplicate key %d", i) + } + writer := &fakeImportRowWriter{ + disableBatch: true, + singleErrByRowID: singleErrors, + } + consumer := newImportBatchConsumer(writer, rowCount, rowCount, true, true, nil) + for i := 1; i <= rowCount; i++ { + if err := consumer.ConsumeRow(map[string]interface{}{"id": i}); err != nil { + t.Fatalf("ConsumeRow(%d) returned error: %v", i, err) + } + } + if err := consumer.Flush(); err != nil { + t.Fatalf("Flush returned error: %v", err) + } + result := consumer.Result() + if result.Failed != rowCount { + t.Fatalf("failed count = %d, want %d", result.Failed, rowCount) + } + if len(result.ErrorLogs) != maxImportErrorDetails { + t.Fatalf("error detail count = %d, want cap %d", len(result.ErrorLogs), maxImportErrorDetails) + } +} + +func TestResolveImportContinueOnErrorPreservesLegacyNilPolicy(t *testing.T) { + continueValue := true + stopValue := false + tests := []struct { + name string + options ImportFileOptions + want bool + }{ + {name: "legacy omitted policy continues safely", options: ImportFileOptions{}, want: true}, + {name: "explicit continue", options: ImportFileOptions{ContinueOnError: &continueValue}, want: true}, + {name: "explicit stop", options: ImportFileOptions{ContinueOnError: &stopValue}, want: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := resolveImportContinueOnError(test.options); got != test.want { + t.Fatalf("resolveImportContinueOnError() = %v, want %v", got, test.want) + } + }) + } +} + func TestImportBatchConsumerStopsSingleRowFallbackAfterCancellation(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) writer := &fakeImportRowWriter{ batchErr: fmt.Errorf("batch failed"), afterSingleCall: cancel, } - consumer := newImportBatchConsumer(writer, 1000, 3, true, nil) + consumer := newImportBatchConsumer(writer, 1000, 3, true, true, nil) consumer.SetContext(ctx) for i := 1; i <= 3; i++ { @@ -642,14 +1027,14 @@ func TestImportBatchConsumerDoesNotCountCancellationAsRowFailure(t *testing.T) { batchErr: fmt.Errorf("batch failed"), singleErrByRowID: map[interface{}]error{1: context.Canceled}, } - consumer := newImportBatchConsumer(writer, 1, 1, true, nil) + consumer := newImportBatchConsumer(writer, 1, 1, true, true, nil) err := consumer.ConsumeRow(map[string]interface{}{"id": 1}) if !errors.Is(err, context.Canceled) { t.Fatalf("ConsumeRow error = %v, want context.Canceled", err) } result := consumer.Result() - if result.Success != 0 || result.Failed != 0 { + if result.Success != 0 || result.Failed != 0 || !result.OutcomeUnknown { t.Fatalf("unexpected cancelled result: %#v", result) } } @@ -657,7 +1042,7 @@ func TestImportBatchConsumerDoesNotCountCancellationAsRowFailure(t *testing.T) { func TestImportBatchConsumerProgressIncludesJobID(t *testing.T) { writer := &fakeImportRowWriter{} var progress []importProgressState - consumer := newImportBatchConsumer(writer, 1, 1, true, func(state importProgressState) { + consumer := newImportBatchConsumer(writer, 1, 1, true, false, func(state importProgressState) { progress = append(progress, state) }) consumer.jobID = "import-job-1" diff --git a/internal/app/methods_file_sql_execution_test.go b/internal/app/methods_file_sql_execution_test.go index 81735fbc..5219741d 100644 --- a/internal/app/methods_file_sql_execution_test.go +++ b/internal/app/methods_file_sql_execution_test.go @@ -43,7 +43,11 @@ type fakeSQLFileBatchDB struct { batchQueries []string execQueries []string failBatch bool + failBatchSQL string + batchError error failExecSQL string + execError func(string) error + onBatch func() session *fakeSQLFileSessionDB } @@ -66,6 +70,11 @@ func (f *fakeSQLFileBatchDB) Query(query string) ([]map[string]interface{}, []st func (f *fakeSQLFileBatchDB) Exec(query string) (int64, error) { f.execCalls++ f.execQueries = append(f.execQueries, query) + if f.execError != nil { + if err := f.execError(query); err != nil { + return 0, err + } + } if f.failExecSQL != "" && strings.Contains(query, f.failExecSQL) { return 0, errors.New("exec failed") } @@ -75,12 +84,44 @@ func (f *fakeSQLFileBatchDB) Exec(query string) (int64, error) { func (f *fakeSQLFileBatchDB) ExecBatchContext(ctx context.Context, query string) (int64, error) { f.batchCalls++ f.batchQueries = append(f.batchQueries, query) - if f.failBatch { + if f.onBatch != nil { + f.onBatch() + } + if f.failBatch || (f.failBatchSQL != "" && strings.Contains(query, f.failBatchSQL)) { + if f.batchError != nil { + return 0, f.batchError + } return 0, errors.New("batch failed") } return int64(strings.Count(query, "INSERT")), nil } +func TestExecuteSQLFileStreamRedactsBatchExecutionErrors(t *testing.T) { + const secret = "password=super-secret-token" + fakeDB := &fakeSQLFileBatchDB{ + failBatch: true, + batchError: errors.New("duplicate key value is (alice@example.com); " + secret), + } + input := "INSERT INTO demo(email) VALUES ('alice@example.com');" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "postgres", + ContinueOnError: false, + Text: func(key string, params map[string]any) string { + return fmt.Sprintf("%s: %v", key, params["detail"]) + }, + }, nil) + if err == nil { + t.Fatal("failed batch must stop SQL file execution") + } + combined := err.Error() + " " + strings.Join(result.Errors, " ") + for _, sensitive := range []string{secret, "super-secret-token", "alice@example.com"} { + if strings.Contains(combined, sensitive) { + t.Fatalf("batch error leaked %q: %s", sensitive, combined) + } + } +} + func (f *fakeSQLFileBatchDB) GetDatabases() ([]string, error) { return nil, nil } @@ -121,8 +162,32 @@ func (f *fakeSQLFileBatchDB) OpenSessionExecer(ctx context.Context) (db.Statemen } type fakeSQLFileSessionDB struct { - parent *fakeSQLFileBatchDB - closed bool + parent *fakeSQLFileBatchDB + closed bool + discarded bool +} + +type fakeSQLFileBatchCapabilityDB struct { + *fakeSQLFileBatchDB + batchWritesEnabled bool +} + +type fakeSQLFileUnpinnedDB struct { + db.Database + execCalls int +} + +func (*fakeSQLFileUnpinnedDB) Connect(connection.ConnectionConfig) error { return nil } +func (*fakeSQLFileUnpinnedDB) Close() error { return nil } +func (*fakeSQLFileUnpinnedDB) Ping() error { return nil } + +func (database *fakeSQLFileUnpinnedDB) Exec(string) (int64, error) { + database.execCalls++ + return 1, nil +} + +func (f *fakeSQLFileBatchCapabilityDB) SupportsBatchWrites() bool { + return f != nil && f.batchWritesEnabled } func (s *fakeSQLFileSessionDB) Exec(query string) (int64, error) { @@ -142,6 +207,11 @@ func (s *fakeSQLFileSessionDB) Close() error { return nil } +func (s *fakeSQLFileSessionDB) Discard() error { + s.discarded = true + return nil +} + func TestExecuteSQLFileStreamBatchesWriteStatements(t *testing.T) { fakeDB := &fakeSQLFileBatchDB{} input := strings.Join([]string{ @@ -178,6 +248,69 @@ func TestExecuteSQLFileStreamBatchesWriteStatements(t *testing.T) { } } +func TestExecuteSQLFileStreamMarksAutomaticBatchTransactionFinishFailureUnknown(t *testing.T) { + tests := []struct { + name string + failBatch bool + finishSQL string + }{ + {name: "commit fails", finishSQL: "COMMIT"}, + {name: "rollback fails", failBatch: true, finishSQL: "ROLLBACK"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{ + failBatch: test.failBatch, + failExecSQL: test.finishSQL, + } + input := "INSERT INTO demo(id) VALUES (1);\nINSERT INTO demo(id) VALUES (2);" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + BatchMaxStatements: 100, + BatchMaxBytes: 1024, + ContinueOnError: false, + }, nil) + if err == nil { + t.Fatal("failed transaction finish must stop SQL file execution") + } + if !result.OutcomeUnknown { + t.Fatalf("failed %s after dispatch must retain an unknown commit outcome: %#v", test.finishSQL, result) + } + }) + } +} + +func TestExecuteSQLFileStreamSkipsBatchAttemptWhenRuntimeCapabilityIsDisabled(t *testing.T) { + baseDB := &fakeSQLFileBatchDB{} + fakeDB := &fakeSQLFileBatchCapabilityDB{ + fakeSQLFileBatchDB: baseDB, + batchWritesEnabled: false, + } + input := strings.Join([]string{ + "INSERT INTO demo(id) VALUES (1);", + "INSERT INTO demo(id) VALUES (2);", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: true, + }, nil) + if err != nil { + t.Fatalf("executeSQLFileStream returned error: %v", err) + } + if result.Executed != 2 || result.Failed != 0 { + t.Fatalf("expected both statements to execute sequentially, got %#v", result) + } + if baseDB.batchCalls != 0 { + t.Fatalf("disabled runtime capability still attempted %d batches", baseDB.batchCalls) + } + if baseDB.execCalls != 2 { + t.Fatalf("expected two direct statement calls without failed batch preflight, got %d: %#v", baseDB.execCalls, baseDB.execQueries) + } +} + func TestExecuteSQLFileStreamFlushesBatchBeforeReadStatement(t *testing.T) { fakeDB := &fakeSQLFileBatchDB{} input := strings.Join([]string{ @@ -209,7 +342,75 @@ func TestExecuteSQLFileStreamFlushesBatchBeforeReadStatement(t *testing.T) { } } -func TestExecuteSQLFileStreamFallsBackToSequentialWhenBatchFails(t *testing.T) { +func TestExecuteSQLFileStreamUsesSafeSequentialExecutionForMySQLFamilyContinueOnError(t *testing.T) { + for _, dbType := range []string{"mysql", "mariadb"} { + t.Run(dbType, func(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{failBatch: true, failExecSQL: "VALUES (2)"} + input := strings.Join([]string{ + "INSERT INTO demo(id) VALUES (1);", + "INSERT INTO demo(id) VALUES (2);", + "INSERT INTO demo(id) VALUES (3);", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: dbType, + BatchMaxStatements: 100, + BatchMaxBytes: 1024, + ContinueOnError: true, + }, nil) + if err != nil { + t.Fatalf("executeSQLFileStream returned error: %v", err) + } + if result.Executed != 2 || result.Failed != 1 { + t.Fatalf("expected 2 executed and 1 failed, got %#v", result) + } + if fakeDB.batchCalls != 0 { + t.Fatalf("%s continue mode must not batch before knowing whether writes are transactional, got %d calls", dbType, fakeDB.batchCalls) + } + if fakeDB.execCalls != 3 { + t.Fatalf("expected exactly 3 sequential statement calls, got %d", fakeDB.execCalls) + } + if fakeDB.execQueries[0] != "INSERT INTO demo(id) VALUES (1)" || fakeDB.execQueries[2] != "INSERT INTO demo(id) VALUES (3)" { + t.Fatalf("unexpected sequential execution order: %#v", fakeDB.execQueries) + } + if len(result.Errors) != 1 || result.Errors[0] != "file.backend.message.statement_failed" { + t.Fatalf("expected per-statement error for second statement, got %#v", result.Errors) + } + }) + } +} + +func TestExecuteSQLFileStreamAdaptivelyNarrowsLargeFailedBatchInContinueMode(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{ + failBatchSQL: "VALUES (33)", + failExecSQL: "VALUES (33)", + } + statements := make([]string, 64) + for index := range statements { + statements[index] = fmt.Sprintf("INSERT INTO demo(id) VALUES (%d);", index+1) + } + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(strings.Join(statements, "\n")), sqlFileExecutionOptions{ + DBType: "postgres", + BatchMaxStatements: 100, + BatchMaxBytes: 64 * 1024, + ContinueOnError: true, + }, nil) + if err != nil { + t.Fatalf("executeSQLFileStream returned error: %v", err) + } + if result.Executed != 63 || result.Failed != 1 { + t.Fatalf("expected 63 executed and 1 failed, got %#v", result) + } + if fakeDB.batchCalls != 5 { + t.Fatalf("expected five adaptive batch attempts, got %d", fakeDB.batchCalls) + } + if fakeDB.execCalls >= 40 { + t.Fatalf("adaptive isolation regressed toward whole-batch sequential replay: execCalls=%d", fakeDB.execCalls) + } +} + +func TestExecuteSQLFileStreamStopsAfterFailedBatchWithoutSequentialReplay(t *testing.T) { fakeDB := &fakeSQLFileBatchDB{failBatch: true, failExecSQL: "VALUES (2)"} input := strings.Join([]string{ "INSERT INTO demo(id) VALUES (1);", @@ -221,24 +422,952 @@ func TestExecuteSQLFileStreamFallsBackToSequentialWhenBatchFails(t *testing.T) { DBType: "mysql", BatchMaxStatements: 100, BatchMaxBytes: 1024, + ContinueOnError: false, }, nil) - if err != nil { - t.Fatalf("executeSQLFileStream returned error: %v", err) + if err == nil { + t.Fatal("expected failed batch to stop SQL file execution") } - if result.Executed != 2 || result.Failed != 1 { - t.Fatalf("expected 2 executed and 1 failed, got %#v", result) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("expected stop-on-error sentinel, got %v", err) + } + if result.Executed != 0 || result.Failed != 1 { + t.Fatalf("expected 0 executed and 1 observed failure, got %#v", result) + } + if !result.OutcomeUnknown { + t.Fatalf("MySQL-family batch rollback cannot prove non-transactional tables were restored: %#v", result) } if fakeDB.batchCalls != 1 { t.Fatalf("expected one failed batch attempt, got %d", fakeDB.batchCalls) } - if fakeDB.execCalls != 5 { - t.Fatalf("expected transaction wrapper plus 3 sequential exec calls, got %d", fakeDB.execCalls) + if fakeDB.execCalls != 2 { + t.Fatalf("expected only transaction begin and rollback, got %d calls: %#v", fakeDB.execCalls, fakeDB.execQueries) } if fakeDB.execQueries[0] != "START TRANSACTION" || fakeDB.execQueries[1] != "ROLLBACK" { - t.Fatalf("expected failed batch to roll back before sequential fallback, got %#v", fakeDB.execQueries) + t.Fatalf("expected failed batch to roll back without replay, got %#v", fakeDB.execQueries) } - if len(result.Errors) != 1 || result.Errors[0] != "file.backend.message.statement_failed" { - t.Fatalf("expected per-statement error for second statement, got %#v", result.Errors) +} + +func TestExecuteSQLFileStreamDoesNotReplayWhenAutomaticBatchBeginFails(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{failExecSQL: "BEGIN"} + input := strings.Join([]string{ + "INSERT INTO demo(id) VALUES (1);", + "INSERT INTO demo(id) VALUES (2);", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "postgres", + ContinueOnError: true, + }, nil) + if err == nil { + t.Fatal("expected failed automatic batch transaction to stop execution") + } + if result.Executed != 0 || result.Failed != 0 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if fakeDB.batchCalls != 0 { + t.Fatalf("batch SQL must not run after START TRANSACTION fails, got %d calls", fakeDB.batchCalls) + } + wantQueries := []string{"BEGIN", "ROLLBACK"} + if fmt.Sprint(fakeDB.execQueries) != fmt.Sprint(wantQueries) { + t.Fatalf("failed batch BEGIN was replayed or left dirty: got %#v want %#v", fakeDB.execQueries, wantQueries) + } +} + +func TestExecuteSQLFileStreamTreatsCancelledBatchAsCancellationWithoutReplay(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + fakeDB := &fakeSQLFileBatchDB{ + failBatch: true, + onBatch: cancel, + } + input := strings.Join([]string{ + "INSERT INTO demo(id) VALUES (1);", + "INSERT INTO demo(id) VALUES (2);", + }, "\n") + + result, err := executeSQLFileStream(ctx, fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "postgres", + BatchMaxStatements: 100, + BatchMaxBytes: 1024, + ContinueOnError: true, + }, nil) + if err == nil || err.Error() != "已取消" { + t.Fatalf("expected cancellation, got %v", err) + } + if result.Executed != 0 || result.Failed != 0 { + t.Fatalf("cancellation must not be counted as a SQL failure, got %#v", result) + } + if fakeDB.batchCalls != 1 { + t.Fatalf("expected one interrupted batch attempt, got %d", fakeDB.batchCalls) + } + if fakeDB.execCalls != 2 { + t.Fatalf("expected only transaction begin and rollback, got %d calls: %#v", fakeDB.execCalls, fakeDB.execQueries) + } +} + +func TestExecuteSQLFileStreamMarksInFlightStatementCancellationUnknown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + fakeDB := &fakeSQLFileBatchDB{execError: func(query string) error { + if strings.Contains(query, "INSERT INTO demo") { + cancel() + return context.Canceled + } + return nil + }} + + result, err := executeSQLFileStream(ctx, fakeDB, strings.NewReader("INSERT INTO demo(id) VALUES (1);"), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: true, + }, nil) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected cancellation, got %v", err) + } + if !result.OutcomeUnknown || result.Executed != 0 || result.Failed != 0 { + t.Fatalf("in-flight cancellation must retain unknown commit outcome: %#v", result) + } +} + +func TestExecuteSQLFileStreamDiscardsSuccessfulSessionToPreventStateLeak(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader("USE tenant_b;"), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if err != nil || result.Executed != 1 || result.Failed != 0 { + t.Fatalf("successful session-scoped statement failed: result=%#v err=%v", result, err) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("successful SQL-file session must be discarded before returning to the pool: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamRollsBackOpenUserTransactionAfterStatementError(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{failExecSQL: "INSERT INTO broken"} + input := strings.Join([]string{ + "START TRANSACTION;", + "INSERT INTO broken(id) VALUES (1);", + "INSERT INTO demo(id) VALUES (2);", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("expected stop-on-error sentinel, got %v", err) + } + if result.Executed != 1 || result.Failed != 1 { + t.Fatalf("unexpected execution counters: %#v", result) + } + wantQueries := []string{"START TRANSACTION", "INSERT INTO broken(id) VALUES (1)", "ROLLBACK"} + if fmt.Sprint(fakeDB.execQueries) != fmt.Sprint(wantQueries) { + t.Fatalf("open transaction was not rolled back: got %#v want %#v", fakeDB.execQueries, wantQueries) + } + if fakeDB.session == nil || !fakeDB.session.closed || !fakeDB.session.discarded { + t.Fatalf("an interrupted import session must be discarded after rollback: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamDiscardsSessionAfterErrorWithoutTrackedTransaction(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{failExecSQL: "CREATE TABLE broken"} + + _, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader("CREATE TABLE broken(id INT);"), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("expected stop-on-error sentinel, got %v", err) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("aborted SQL-file sessions may retain autocommit or other session state and must be discarded: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamDiscardsSessionWhenOpenTransactionRollbackFails(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{ + execError: func(query string) error { + if strings.Contains(query, "INSERT INTO broken") || query == "ROLLBACK" { + return errors.New("forced execution failure") + } + return nil + }, + } + input := "START TRANSACTION;\nINSERT INTO broken(id) VALUES (1);" + + _, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("expected stop-on-error sentinel, got %v", err) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("rollback failure must discard then close the session: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamRejectsUnclosedUserTransactionAtEndOfFile(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := "START TRANSACTION;\nINSERT INTO demo(id) VALUES (1);" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: true, + }, nil) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("expected unclosed transaction to fail the import, got %v", err) + } + if result.Executed != 2 || result.Failed != 1 { + t.Fatalf("unexpected execution counters: %#v", result) + } + wantQueries := []string{"START TRANSACTION", "INSERT INTO demo(id) VALUES (1)", "ROLLBACK"} + if fmt.Sprint(fakeDB.execQueries) != fmt.Sprint(wantQueries) { + t.Fatalf("unclosed transaction was not rolled back: got %#v want %#v", fakeDB.execQueries, wantQueries) + } +} + +func TestExecuteSQLFileStreamDoesNotTreatOracleAnonymousBlockAsOpenTransaction(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + block := strings.Join([]string{ + "BEGIN", + " NULL;", + "END;", + }, "\n") + input := block + "\n/\nSELECT 1 FROM dual;" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "oracle", + ContinueOnError: false, + }, nil) + if err != nil { + t.Fatalf("Oracle anonymous block must not leave a synthetic transaction open: %v", err) + } + if result.Executed != 2 || result.Failed != 0 { + t.Fatalf("unexpected execution counters: %#v", result) + } + wantQueries := []string{block, "SELECT 1 FROM dual"} + if fmt.Sprint(fakeDB.execQueries) != fmt.Sprint(wantQueries) { + t.Fatalf("anonymous block execution changed unexpectedly: got %#v want %#v", fakeDB.execQueries, wantQueries) + } + if fakeDB.session == nil || !fakeDB.session.closed || !fakeDB.session.discarded { + t.Fatalf("SQL-file session must be discarded even after a successful anonymous block: %#v", fakeDB.session) + } +} + +func TestUpdateSQLFileTransactionStateDistinguishesBlocksFromTransactions(t *testing.T) { + tests := []struct { + name string + dbType string + inTransaction bool + stmt string + want bool + }{ + {name: "mysql bare begin", dbType: "mysql", stmt: "BEGIN", want: true}, + {name: "mysql begin work", dbType: "mysql", stmt: "BEGIN WORK", want: true}, + {name: "mariadb anonymous block", dbType: "mariadb", stmt: "BEGIN NOT ATOMIC\n SET @value = 1;\nEND", want: false}, + {name: "postgres begin work", dbType: "postgres", stmt: "BEGIN WORK", want: true}, + {name: "postgres deferrable", dbType: "postgres", stmt: "BEGIN DEFERRABLE", want: true}, + {name: "postgres not deferrable", dbType: "postgres", stmt: "BEGIN NOT DEFERRABLE", want: true}, + {name: "postgres family oracle compatible block", dbType: "kingbase", stmt: "BEGIN\n NULL;\nEND", want: false}, + {name: "oracle anonymous block", dbType: "oracle", stmt: "BEGIN\n NULL;\nEND", want: false}, + {name: "oracle block preserves active transaction", dbType: "oracle", inTransaction: true, stmt: "BEGIN\n NULL;\nEND", want: true}, + {name: "dameng anonymous block", dbType: "dameng", stmt: "BEGIN\n NULL;\nEND", want: false}, + {name: "sqlserver control block", dbType: "sqlserver", stmt: "BEGIN\n PRINT 'done';\nEND", want: false}, + {name: "sqlserver try block", dbType: "sqlserver", stmt: "BEGIN TRY\n SELECT 1;\nEND TRY", want: false}, + {name: "sqlserver dialog", dbType: "sqlserver", stmt: "BEGIN DIALOG CONVERSATION @handle", want: false}, + {name: "sqlserver transaction", dbType: "sqlserver", stmt: "BEGIN TRANSACTION", want: true}, + {name: "sqlserver tran alias", dbType: "sqlserver", stmt: "BEGIN TRAN", want: true}, + {name: "sqlserver distributed transaction", dbType: "sqlserver", stmt: "BEGIN DISTRIBUTED TRANSACTION", want: true}, + {name: "sqlite deferred", dbType: "sqlite", stmt: "BEGIN DEFERRED", want: true}, + {name: "sqlite immediate", dbType: "sqlite", stmt: "BEGIN IMMEDIATE", want: true}, + {name: "sqlite exclusive", dbType: "sqlite", stmt: "BEGIN EXCLUSIVE TRANSACTION", want: true}, + {name: "unknown ansi atomic block", dbType: "custom", stmt: "BEGIN ATOMIC\n VALUES 1;\nEND", want: false}, + {name: "leading comment before transaction", dbType: "postgres", stmt: "-- restore transaction\nBEGIN TRANSACTION", want: true}, + {name: "leading hash comment before mysql transaction", dbType: "mysql", stmt: "# restore transaction\nBEGIN", want: true}, + {name: "unrelated start preserves active transaction", dbType: "mysql", inTransaction: true, stmt: "START REPLICA", want: true}, + {name: "rollback to savepoint", dbType: "postgres", inTransaction: true, stmt: "ROLLBACK WORK TO SAVEPOINT before_import", want: true}, + {name: "rollback to savepoint with comment", dbType: "sqlite", inTransaction: true, stmt: "ROLLBACK /* keep outer transaction */ TRANSACTION TO before_import", want: true}, + {name: "commit and chain", dbType: "mysql", inTransaction: true, stmt: "COMMIT WORK AND CHAIN", want: true}, + {name: "commit and no chain", dbType: "mysql", inTransaction: true, stmt: "COMMIT AND NO CHAIN", want: false}, + {name: "rollback and chain", dbType: "postgres", inTransaction: true, stmt: "ROLLBACK AND CHAIN", want: true}, + {name: "postgres end transaction", dbType: "postgres", inTransaction: true, stmt: "END TRANSACTION", want: false}, + {name: "sqlite end transaction", dbType: "sqlite", inTransaction: true, stmt: "END TRANSACTION", want: false}, + {name: "postgres abort", dbType: "postgres", inTransaction: true, stmt: "ABORT", want: false}, + {name: "duckdb abort", dbType: "duckdb", inTransaction: true, stmt: "ABORT", want: false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := updateSQLFileTransactionState(test.dbType, test.inTransaction, test.stmt); got != test.want { + t.Fatalf("transaction state = %v, want %v", got, test.want) + } + }) + } +} + +func TestExecuteSQLFileStreamHandlesSQLServerBlocksAndTransactions(t *testing.T) { + t.Run("control block", func(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + block := "BEGIN\n PRINT 'done';\nEND" + input := block + ";\nSELECT 1;" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "sqlserver", + ContinueOnError: false, + }, nil) + if err != nil { + t.Fatalf("SQL Server control block must not leave a synthetic transaction open: %v", err) + } + if result.Executed != 2 || result.Failed != 0 { + t.Fatalf("unexpected execution counters: %#v", result) + } + wantQueries := []string{block + ";", "SELECT 1"} + if fmt.Sprint(fakeDB.execQueries) != fmt.Sprint(wantQueries) { + t.Fatalf("control block execution changed unexpectedly: got %#v want %#v", fakeDB.execQueries, wantQueries) + } + }) + + t.Run("explicit transaction", func(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := "BEGIN TRAN;\nUPDATE demo SET value = 2;\nCOMMIT;" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "sqlserver", + ContinueOnError: false, + }, nil) + if err != nil { + t.Fatalf("SQL Server explicit transaction should complete normally: %v", err) + } + if result.Executed != 3 || result.Failed != 0 { + t.Fatalf("unexpected execution counters: %#v", result) + } + wantQueries := []string{"BEGIN TRAN", "UPDATE demo SET value = 2", "COMMIT"} + if fmt.Sprint(fakeDB.execQueries) != fmt.Sprint(wantQueries) { + t.Fatalf("explicit transaction split changed unexpectedly: got %#v want %#v", fakeDB.execQueries, wantQueries) + } + }) +} + +func TestExecuteSQLFileStreamDoesNotReuseSQLServerSessionWithNestedTransactionOpen(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := strings.Join([]string{ + "BEGIN TRAN;", + "BEGIN TRAN;", + "INSERT INTO demo(id) VALUES (1);", + "COMMIT;", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "sqlserver", + ContinueOnError: false, + }, nil) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("nested SQL Server transaction left open at EOF must fail, got %v", err) + } + if result.Executed != 4 || result.Failed != 1 { + t.Fatalf("unexpected execution counters: %#v", result) + } + wantQueries := []string{ + "BEGIN TRAN", + "BEGIN TRAN", + "INSERT INTO demo(id) VALUES (1)", + "COMMIT", + "ROLLBACK TRANSACTION", + } + if fmt.Sprint(fakeDB.execQueries) != fmt.Sprint(wantQueries) { + t.Fatalf("remaining nested transaction was not rolled back: got %#v want %#v", fakeDB.execQueries, wantQueries) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("nested transaction session must be discarded: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamDoesNotTreatSQLServerNamedRollbackAsTransactionEnd(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := strings.Join([]string{ + "BEGIN TRAN;", + "SAVE TRANSACTION before_import;", + "ROLLBACK TRANSACTION before_import;", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "sqlserver", + ContinueOnError: false, + }, nil) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("named SQL Server rollback has ambiguous savepoint semantics and must keep cleanup active, got %v", err) + } + if result.Executed != 3 || result.Failed != 1 { + t.Fatalf("unexpected execution counters: %#v", result) + } + wantQueries := []string{ + "BEGIN TRAN", + "SAVE TRANSACTION before_import", + "ROLLBACK TRANSACTION before_import", + "ROLLBACK TRANSACTION", + } + if fmt.Sprint(fakeDB.execQueries) != fmt.Sprint(wantQueries) { + t.Fatalf("named rollback session was not cleaned conservatively: got %#v want %#v", fakeDB.execQueries, wantQueries) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("named rollback session must not be reused: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamClosesSQLServerNamedOuterTransactionRollback(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := strings.Join([]string{ + "BEGIN TRANSACTION import_work;", + "INSERT INTO demo(id) VALUES (1);", + "ROLLBACK TRANSACTION import_work;", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "sqlserver", + ContinueOnError: false, + }, nil) + if err != nil { + t.Fatalf("rollback to the tracked outer transaction name must close it: %v", err) + } + if result.Executed != 3 || result.Failed != 0 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("dedicated SQL-file session must be discarded after named rollback: %#v", fakeDB.session) + } + if len(fakeDB.execQueries) != 3 { + t.Fatalf("named outer rollback must not trigger an extra cleanup rollback: %#v", fakeDB.execQueries) + } +} + +func TestExecuteSQLFileStreamKeepsCaseDistinctSQLServerSavepointTransactionOpen(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := strings.Join([]string{ + "BEGIN TRANSACTION ImportWork;", + "SAVE TRANSACTION importwork;", + "ROLLBACK TRANSACTION importwork;", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "sqlserver", + ContinueOnError: false, + }, nil) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("case-distinct savepoint rollback must leave the outer transaction open: %v", err) + } + if result.Executed != 3 || result.Failed != 1 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if fakeDB.execQueries[len(fakeDB.execQueries)-1] != "ROLLBACK TRANSACTION" { + t.Fatalf("outer transaction was not cleaned up: %#v", fakeDB.execQueries) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("savepoint rollback session must be discarded: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamClosesSQLServerTransactionAfterNamedRollbackAndCommit(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := strings.Join([]string{ + "BEGIN TRAN;", + "SAVE TRANSACTION before_import;", + "ROLLBACK TRANSACTION before_import;", + "COMMIT;", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "sqlserver", + ContinueOnError: false, + }, nil) + if err != nil { + t.Fatalf("final COMMIT should close the transaction retained after named rollback: %v", err) + } + if result.Executed != 4 || result.Failed != 0 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("dedicated SQL-file session must be discarded after commit: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamKeepsTransactionOpenWhenFinishStatementFailsInContinueMode(t *testing.T) { + tests := []struct { + name string + finishSQL string + }{ + {name: "commit fails", finishSQL: "COMMIT"}, + {name: "rollback fails", finishSQL: "ROLLBACK"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{failExecSQL: test.finishSQL} + input := strings.Join([]string{ + "START TRANSACTION;", + "INSERT INTO demo(id) VALUES (1);", + test.finishSQL + ";", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: true, + }, nil) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("failed transaction finish must leave cleanup active, got %v", err) + } + if result.Executed != 2 || result.Failed != 2 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if !result.OutcomeUnknown { + t.Fatalf("failed user %s after dispatch must retain an unknown commit outcome: %#v", test.finishSQL, result) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("unexpected cleanup state: %#v", fakeDB.session) + } + if fakeDB.execQueries[len(fakeDB.execQueries)-1] != "ROLLBACK" { + t.Fatalf("expected final cleanup rollback, got %#v", fakeDB.execQueries) + } + }) + } +} + +func TestExecuteSQLFileStreamMarksCancelledUserTransactionFinishUnknown(t *testing.T) { + for _, finishSQL := range []string{"COMMIT", "ROLLBACK"} { + t.Run(strings.ToLower(finishSQL), func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + fakeDB := &fakeSQLFileBatchDB{execError: func(query string) error { + if query == finishSQL { + cancel() + return context.Canceled + } + return nil + }} + input := "START TRANSACTION;\nINSERT INTO demo(id) VALUES (1);\n" + finishSQL + ";" + + result, err := executeSQLFileStream(ctx, fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled %s returned %v", finishSQL, err) + } + if !result.OutcomeUnknown || result.Executed != 2 || result.Failed != 0 { + t.Fatalf("cancelled %s after dispatch must retain an unknown outcome: %#v", finishSQL, result) + } + }) + } +} + +func TestExecuteSQLFileStreamDoesNotOpenTransactionWhenStartFailsInContinueMode(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{failExecSQL: "START TRANSACTION"} + input := "START TRANSACTION;\nCREATE TABLE demo(id INT);" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: true, + }, nil) + if err != nil { + t.Fatalf("failed START must not create a synthetic unclosed transaction: %v", err) + } + if result.Executed != 1 || result.Failed != 1 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if len(fakeDB.execQueries) != 2 { + t.Fatalf("failed START unexpectedly triggered cleanup: %#v", fakeDB.execQueries) + } +} + +func TestExecuteSQLFileStreamPreservesMySQLAutocommitOffRollbackSemantics(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := strings.Join([]string{ + "SET autocommit=0;", + "INSERT INTO demo(id) VALUES (1);", + "ROLLBACK;", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if err != nil { + t.Fatalf("autocommit-controlled rollback should complete normally: %v", err) + } + if result.Executed != 3 || result.Failed != 0 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if fakeDB.batchCalls != 0 { + t.Fatalf("autocommit=0 DML must not be wrapped in an auto-committed batch: %d batch calls", fakeDB.batchCalls) + } + wantQueries := []string{ + "SET autocommit=0", + "INSERT INTO demo(id) VALUES (1)", + "ROLLBACK", + } + if fmt.Sprint(fakeDB.execQueries) != fmt.Sprint(wantQueries) { + t.Fatalf("autocommit-controlled transaction semantics changed: got %#v want %#v", fakeDB.execQueries, wantQueries) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("session left with autocommit=0 must be discarded after import: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamPreservesMariaDBAutocommitOffRollbackSemantics(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := "SET autocommit=0;\nINSERT INTO demo(id) VALUES (1);\nROLLBACK;" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mariadb", + ContinueOnError: false, + }, nil) + if err != nil || result.Executed != 3 || result.Failed != 0 { + t.Fatalf("MariaDB autocommit-controlled rollback failed: result=%#v err=%v", result, err) + } + if fakeDB.batchCalls != 0 { + t.Fatalf("MariaDB autocommit=0 DML must not be auto-committed in a batch: %d calls", fakeDB.batchCalls) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("MariaDB session left with autocommit=0 must be discarded: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamRollsBackUnfinishedMySQLAutocommitOffWorkAtEOF(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := strings.Join([]string{ + "SET autocommit=0;", + "INSERT INTO demo(id) VALUES (1);", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: true, + }, nil) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("unfinished autocommit=0 work must fail at EOF, got %v", err) + } + if result.Executed != 2 || result.Failed != 1 { + t.Fatalf("unexpected execution counters: %#v", result) + } + wantQueries := []string{ + "SET autocommit=0", + "INSERT INTO demo(id) VALUES (1)", + "ROLLBACK", + } + if fmt.Sprint(fakeDB.execQueries) != fmt.Sprint(wantQueries) { + t.Fatalf("unfinished autocommit=0 work was not rolled back: got %#v want %#v", fakeDB.execQueries, wantQueries) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("unfinished autocommit=0 session must be discarded: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamRecognizesMySQLDumpCompositeAutocommitOff(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := strings.Join([]string{ + "SET @OLD_AUTOCOMMIT=@@AUTOCOMMIT, AUTOCOMMIT=0;", + "INSERT INTO demo(id) VALUES (1);", + "ROLLBACK;", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if err != nil { + t.Fatalf("dump-style autocommit-controlled rollback should complete normally: %v", err) + } + if result.Executed != 3 || result.Failed != 0 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if fakeDB.batchCalls != 0 { + t.Fatalf("composite AUTOCOMMIT=0 must disable automatic batching: %d batch calls", fakeDB.batchCalls) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("session left with dump-controlled autocommit must be discarded: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamDiscardsSessionAfterMySQLAutocommitVariableRestore(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader("SET AUTOCOMMIT=@OLD_AUTOCOMMIT;"), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if err != nil { + t.Fatalf("variable-based autocommit restore should execute normally: %v", err) + } + if result.Executed != 1 || result.Failed != 0 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("unknown restored autocommit state must not return to the pool: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamRecognizesMySQLAutocommitEnableImplicitCommit(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := strings.Join([]string{ + "SET AUTOCOMMIT=0;", + "START TRANSACTION;", + "INSERT INTO demo(id) VALUES (1);", + "SET AUTOCOMMIT=1;", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if err != nil { + t.Fatalf("enabling autocommit after an explicit transaction must commit it: %v", err) + } + if result.Executed != 4 || result.Failed != 0 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if fakeDB.batchCalls != 0 { + t.Fatalf("autocommit-controlled DML must remain sequential: %d batch calls", fakeDB.batchCalls) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("dedicated SQL-file session must be discarded after restoring autocommit: %#v", fakeDB.session) + } + if strings.Contains(fmt.Sprint(fakeDB.execQueries), "ROLLBACK") { + t.Fatalf("SET AUTOCOMMIT=1 already committed the transaction: %#v", fakeDB.execQueries) + } +} + +func TestExecuteSQLFileStreamRecognizesMySQLFamilyDDLImplicitCommit(t *testing.T) { + for _, dbType := range []string{"mysql", "mariadb"} { + t.Run(dbType, func(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := strings.Join([]string{ + "START TRANSACTION;", + "INSERT INTO demo(id) VALUES (1);", + "CREATE TABLE demo_copy(id INT);", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: dbType, + ContinueOnError: false, + }, nil) + if err != nil { + t.Fatalf("DDL implicit commit must close the tracked transaction: %v", err) + } + if result.Executed != 3 || result.Failed != 0 { + t.Fatalf("unexpected execution counters: %#v", result) + } + wantQueries := []string{ + "START TRANSACTION", + "INSERT INTO demo(id) VALUES (1)", + "CREATE TABLE demo_copy(id INT)", + } + if fmt.Sprint(fakeDB.execQueries) != fmt.Sprint(wantQueries) { + t.Fatalf("successful DDL triggered a synthetic EOF rollback: got %#v want %#v", fakeDB.execQueries, wantQueries) + } + }) + } +} + +func TestExecuteSQLFileStreamRecognizesMySQLDDLPreCommitWhenDDLAttemptFails(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{failExecSQL: "CREATE TABLE broken"} + input := strings.Join([]string{ + "START TRANSACTION;", + "INSERT INTO demo(id) VALUES (1);", + "CREATE TABLE broken(id INT);", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: true, + }, nil) + if err != nil { + t.Fatalf("failed DDL must not leave a synthetic transaction open after its pre-commit: %v", err) + } + if result.Executed != 2 || result.Failed != 1 { + t.Fatalf("DDL failure must be counted exactly once: %#v", result) + } + wantQueries := []string{ + "START TRANSACTION", + "INSERT INTO demo(id) VALUES (1)", + "CREATE TABLE broken(id INT)", + } + if fmt.Sprint(fakeDB.execQueries) != fmt.Sprint(wantQueries) { + t.Fatalf("failed DDL triggered an invalid EOF rollback: got %#v want %#v", fakeDB.execQueries, wantQueries) + } +} + +func TestExecuteSQLFileStreamDoesNotTreatMySQLTemporaryTableDDLAsImplicitCommit(t *testing.T) { + for _, ddl := range []string{ + "CREATE TEMPORARY TABLE temp_import(id INT)", + "DROP TEMPORARY TABLE temp_import", + } { + t.Run(strings.Fields(ddl)[0], func(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := "START TRANSACTION;\n" + ddl + ";" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("temporary-table DDL must leave the explicit transaction open, got %v", err) + } + if result.Executed != 2 || result.Failed != 1 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if fakeDB.execQueries[len(fakeDB.execQueries)-1] != "ROLLBACK" { + t.Fatalf("temporary-table transaction was not rolled back: %#v", fakeDB.execQueries) + } + }) + } +} + +func TestSQLFileMySQLImplicitCommitClassificationAvoidsConditionalFalsePositives(t *testing.T) { + tests := []struct { + name string + stmt string + want bool + }{ + {name: "set password", stmt: "SET PASSWORD FOR 'app'@'%' = 'secret'", want: true}, + {name: "reset replica", stmt: "RESET REPLICA ALL", want: true}, + {name: "reset persist exception", stmt: "RESET PERSIST IF EXISTS max_connections", want: false}, + {name: "lock tables", stmt: "LOCK TABLES demo WRITE", want: true}, + {name: "lock instance is not table lock", stmt: "LOCK INSTANCE FOR BACKUP", want: false}, + {name: "conditional unlock tables", stmt: "UNLOCK TABLES", want: false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := sqlFileMySQLImplicitCommitBeforeStatement("mysql", test.stmt); got != test.want { + t.Fatalf("sqlFileMySQLImplicitCommitBeforeStatement(%q) = %v, want %v", test.stmt, got, test.want) + } + }) + } +} + +func TestExecuteSQLFileStreamDoesNotAssumeUnmatchedMySQLUnlockCommits(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := "START TRANSACTION;\nINSERT INTO demo(id) VALUES (1);\nUNLOCK TABLES;" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("UNLOCK TABLES without a tracked table lock must not clear the transaction: %v", err) + } + if result.Executed != 3 || result.Failed != 1 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if fakeDB.execQueries[len(fakeDB.execQueries)-1] != "ROLLBACK" { + t.Fatalf("uncommitted work must be rolled back: %#v", fakeDB.execQueries) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("uncertain transaction session must be discarded: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamPreservesMySQLTableLocksUntilUnlock(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := "LOCK TABLES demo WRITE;\nINSERT INTO demo(id) VALUES (1);\nUNLOCK TABLES;" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if err != nil || result.Executed != 3 || result.Failed != 0 { + t.Fatalf("tracked LOCK/UNLOCK TABLES sequence failed: result=%#v err=%v", result, err) + } + if fakeDB.batchCalls != 0 { + t.Fatalf("automatic transaction batching would release LOCK TABLES: %d calls", fakeDB.batchCalls) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("dedicated SQL-file session must be discarded after unlocking tables: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamDiscardsMySQLSessionWithTableLocksAtEOF(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader("LOCK TABLES demo WRITE;"), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if err != nil || result.Executed != 1 || result.Failed != 0 { + t.Fatalf("LOCK TABLES execution failed: result=%#v err=%v", result, err) + } + if fakeDB.session == nil || !fakeDB.session.discarded || !fakeDB.session.closed { + t.Fatalf("session retaining table locks must not return to the pool: %#v", fakeDB.session) + } +} + +func TestExecuteSQLFileStreamStopsAfterSingleStatementError(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{failExecSQL: "CREATE TABLE broken"} + input := strings.Join([]string{ + "CREATE TABLE broken(id INT);", + "INSERT INTO demo(id) VALUES (2);", + }, "\n") + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: false, + }, nil) + if !errors.Is(err, errSQLFileStoppedOnError) { + t.Fatalf("expected stop-on-error sentinel, got %v", err) + } + if result.Executed != 0 || result.Failed != 1 { + t.Fatalf("expected the first failed statement to stop execution, got %#v", result) + } + if fakeDB.batchCalls != 0 { + t.Fatalf("expected no later write batch, got %d batch calls", fakeDB.batchCalls) + } + if len(fakeDB.execQueries) != 1 || fakeDB.execQueries[0] != "CREATE TABLE broken(id INT)" { + t.Fatalf("expected only the failing statement to run, got %#v", fakeDB.execQueries) + } +} + +func TestExecuteSQLFileStreamCapsRetainedErrorDetailsInContinueMode(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{failExecSQL: "CREATE TABLE broken_"} + statements := make([]string, 25) + for index := range statements { + statements[index] = fmt.Sprintf("CREATE TABLE broken_%d(id INT);", index) + } + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(strings.Join(statements, "\n")), sqlFileExecutionOptions{ + DBType: "mysql", + ContinueOnError: true, + }, nil) + if err != nil { + t.Fatalf("executeSQLFileStream returned error: %v", err) + } + if result.Executed != 0 || result.Failed != 25 { + t.Fatalf("unexpected execution counters: %#v", result) + } + if len(result.Errors) != sqlFileMaxErrorDetails { + t.Fatalf("retained %d error details, want cap %d", len(result.Errors), sqlFileMaxErrorDetails) + } +} + +func TestExecuteSQLFileStreamDoesNotRetryFailedOversizedStatement(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{failBatch: true} + largeValue := strings.Repeat("x", 256) + input := fmt.Sprintf("INSERT INTO demo(value) VALUES ('%s');", largeValue) + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "postgres", + BatchMaxStatements: 100, + BatchMaxBytes: 64, + ContinueOnError: true, + }, nil) + if err != nil { + t.Fatalf("executeSQLFileStream returned error: %v", err) + } + if result.Executed != 0 || result.Failed != 1 { + t.Fatalf("expected the oversized statement failure to be recorded once, got %#v", result) + } + if fakeDB.batchCalls != 1 { + t.Fatalf("expected one oversized statement attempt, got %d", fakeDB.batchCalls) + } + if len(fakeDB.execQueries) != 2 || fakeDB.execQueries[0] != "BEGIN" || fakeDB.execQueries[1] != "ROLLBACK" { + t.Fatalf("expected no second execution of the oversized statement, got %#v", fakeDB.execQueries) } } @@ -253,6 +1382,7 @@ func TestExecuteSQLFileStreamUsesLocalizedStatementFailure(t *testing.T) { DBType: "mysql", BatchMaxStatements: 100, BatchMaxBytes: 1024, + ContinueOnError: true, Text: func(key string, params map[string]any) string { if key != "file.backend.message.statement_failed" { t.Fatalf("unexpected i18n key %q", key) @@ -266,8 +1396,11 @@ func TestExecuteSQLFileStreamUsesLocalizedStatementFailure(t *testing.T) { if len(result.Errors) != 1 { t.Fatalf("expected one localized statement error, got %#v", result.Errors) } - if !strings.Contains(result.Errors[0], "localized statement 2 failed") || !strings.Contains(result.Errors[0], "VALUES (2)") { - t.Fatalf("expected localized per-statement error with raw SQL snippet, got %#v", result.Errors) + if !strings.Contains(result.Errors[0], "localized statement 2 failed") || !strings.Contains(result.Errors[0], "VALUES (?)") { + t.Fatalf("expected localized per-statement error with redacted SQL snippet, got %#v", result.Errors) + } + if strings.Contains(result.Errors[0], "VALUES (2)") { + t.Fatalf("expected statement failure to omit SQL literal values, got %#v", result.Errors) } } @@ -767,10 +1900,11 @@ func TestImportDatabaseSQLHonorsConnectionProtections(t *testing.T) { } app := NewApp() + app.configDir = t.TempDir() result := app.ImportDatabaseSQL(connection.ConnectionConfig{ Type: "mysql", Protection: test.protection, - }, "app", test.filePath, "database-import-protection-test") + }, "app", test.filePath, "database-import-protection-test", false) if test.wantBlock { if result.Success { @@ -801,3 +1935,157 @@ func TestImportDatabaseSQLHonorsConnectionProtections(t *testing.T) { }) } } + +func TestImportDatabaseSQLFailsClosedWithoutPinnedSession(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "database.sql") + if err := os.WriteFile(filePath, []byte("CREATE TABLE demo(id INT);"), 0o600); err != nil { + t.Fatal(err) + } + originalNewDatabaseFunc := newDatabaseFunc + t.Cleanup(func() { newDatabaseFunc = originalNewDatabaseFunc }) + database := &fakeSQLFileUnpinnedDB{} + newDatabaseFunc = func(string) (db.Database, error) { return database, nil } + + app := NewApp() + app.configDir = t.TempDir() + result := app.ImportDatabaseSQL( + connection.ConnectionConfig{Type: "mysql"}, + "app", + filePath, + "database-import-unpinned-test", + false, + ) + if result.Success || result.Message != app.appText("data_import.capability.reason.pinned_session_unavailable", nil) { + t.Fatalf("unexpected unpinned result: %#v", result) + } + if database.execCalls != 0 { + t.Fatalf("unpinned import executed %d statement(s)", database.execCalls) + } +} + +func TestImportDatabaseSQLRejectsUnsupportedDialectBeforeFileAccess(t *testing.T) { + originalNewDatabaseFunc := newDatabaseFunc + t.Cleanup(func() { newDatabaseFunc = originalNewDatabaseFunc }) + opened := false + newDatabaseFunc = func(string) (db.Database, error) { + opened = true + return &fakeSQLFileUnpinnedDB{}, nil + } + + app := NewApp() + app.configDir = t.TempDir() + result := app.ImportDatabaseSQL( + connection.ConnectionConfig{Type: "future-db"}, + "app", + filepath.Join(t.TempDir(), "missing.sql"), + "database-import-unsupported-test", + false, + ) + if result.Success || result.Message != app.appText("data_import.capability.reason.database_type_unsupported", nil) { + t.Fatalf("unexpected unsupported-dialect result: %#v", result) + } + if opened { + t.Fatal("unsupported database import opened a database") + } +} + +func TestExecuteSQLFileHonorsScriptExecutionProtection(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "script.sql") + if err := os.WriteFile(filePath, []byte("DROP TABLE users;"), 0o600); err != nil { + t.Fatal(err) + } + originalNewDatabaseFunc := newDatabaseFunc + t.Cleanup(func() { newDatabaseFunc = originalNewDatabaseFunc }) + opened := false + newDatabaseFunc = func(string) (db.Database, error) { + opened = true + return &fakeSQLFileBatchDB{}, nil + } + + app := NewApp() + app.configDir = t.TempDir() + result := app.ExecuteSQLFile(connection.ConnectionConfig{ + Type: "mysql", + Protection: connection.ConnectionProtectionConfig{ + RestrictScriptExecution: true, + }, + }, "app", filePath, "protected-script") + if result.Success { + t.Fatalf("protected SQL file unexpectedly succeeded: %#v", result) + } + if opened { + t.Fatal("protected SQL file opened a database") + } +} + +func TestImportDatabaseSQLStopPolicyDoesNotReplayFailedBatch(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "database.sql") + input := strings.Join([]string{ + "INSERT INTO demo(id) VALUES (1);", + "INSERT INTO demo(id) VALUES (2);", + "INSERT INTO demo(id) VALUES (3);", + }, "\n") + if err := os.WriteFile(filePath, []byte(input), 0o600); err != nil { + t.Fatalf("write SQL import fixture: %v", err) + } + + originalNewDatabaseFunc := newDatabaseFunc + t.Cleanup(func() { newDatabaseFunc = originalNewDatabaseFunc }) + fakeDB := &fakeSQLFileBatchDB{failBatch: true, failExecSQL: "VALUES (2)"} + newDatabaseFunc = func(string) (db.Database, error) { + return fakeDB, nil + } + + app := NewApp() + app.configDir = t.TempDir() + result := app.ImportDatabaseSQL(connection.ConnectionConfig{Type: "mysql"}, "app", filePath, "database-import-stop-policy-test", false) + if result.Success { + t.Fatalf("ImportDatabaseSQL unexpectedly succeeded: %#v", result) + } + payload, ok := result.Data.(map[string]interface{}) + if !ok { + t.Fatalf("result data type = %T, want map[string]interface{}", result.Data) + } + if payload["completed"] != false || payload["stoppedOnError"] != true { + t.Fatalf("unexpected stop-on-error payload: %#v", payload) + } + if fakeDB.batchCalls != 1 || fakeDB.execCalls != 2 { + t.Fatalf("failed database import replayed its batch: batchCalls=%d execCalls=%d queries=%#v", fakeDB.batchCalls, fakeDB.execCalls, fakeDB.execQueries) + } +} + +func TestImportDatabaseSQLContinuePolicyCompletesWithRecordedErrors(t *testing.T) { + filePath := filepath.Join(t.TempDir(), "database.sql") + input := strings.Join([]string{ + "INSERT INTO demo(id) VALUES (1);", + "INSERT INTO demo(id) VALUES (2);", + "INSERT INTO demo(id) VALUES (3);", + }, "\n") + if err := os.WriteFile(filePath, []byte(input), 0o600); err != nil { + t.Fatalf("write SQL import fixture: %v", err) + } + + originalNewDatabaseFunc := newDatabaseFunc + t.Cleanup(func() { newDatabaseFunc = originalNewDatabaseFunc }) + fakeDB := &fakeSQLFileBatchDB{failBatch: true, failExecSQL: "VALUES (2)"} + newDatabaseFunc = func(string) (db.Database, error) { + return fakeDB, nil + } + + app := NewApp() + app.configDir = t.TempDir() + result := app.ImportDatabaseSQL(connection.ConnectionConfig{Type: "mysql"}, "app", filePath, "database-import-continue-policy-test", true) + if result.Success { + t.Fatalf("backend result with statement errors should remain unsuccessful: %#v", result) + } + payload, ok := result.Data.(map[string]interface{}) + if !ok { + t.Fatalf("result data type = %T, want map[string]interface{}", result.Data) + } + if payload["completed"] != true || payload["stoppedOnError"] != false || payload["failed"] != 1 { + t.Fatalf("unexpected completed-with-errors payload: %#v", payload) + } + if fakeDB.batchCalls != 0 || fakeDB.execCalls != 3 { + t.Fatalf("MySQL continue policy must execute safely without a replayable batch: batchCalls=%d execCalls=%d queries=%#v", fakeDB.batchCalls, fakeDB.execCalls, fakeDB.execQueries) + } +} diff --git a/internal/app/methods_file_sql_managed_test.go b/internal/app/methods_file_sql_managed_test.go new file mode 100644 index 00000000..69fcc3e7 --- /dev/null +++ b/internal/app/methods_file_sql_managed_test.go @@ -0,0 +1,308 @@ +package app + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "GoNavi-Wails/internal/connection" + "GoNavi-Wails/internal/db" + "GoNavi-Wails/internal/importjob" + "GoNavi-Wails/internal/uievents" +) + +func installSQLImportTestDatabase(t *testing.T, database db.Database) { + t.Helper() + originalNewDatabaseFunc := newDatabaseFunc + originalResolveDialConfigWithProxyFunc := resolveDialConfigWithProxyFunc + newDatabaseFunc = func(string) (db.Database, error) { return database, nil } + resolveDialConfigWithProxyFunc = func(config connection.ConnectionConfig) (connection.ConnectionConfig, error) { + return config, nil + } + t.Cleanup(func() { + newDatabaseFunc = originalNewDatabaseFunc + resolveDialConfigWithProxyFunc = originalResolveDialConfigWithProxyFunc + }) +} + +func newManagedSQLImportTestApp(t *testing.T) *App { + t.Helper() + app := NewApp() + app.configDir = t.TempDir() + app.ctx = uievents.WithEmitter(context.Background(), noopImportEventEmitter{}) + return app +} + +func TestImportDatabaseSQLPersistsCompletedManagedJobAndSourceDigest(t *testing.T) { + database := &fakeSQLFileBatchDB{} + installSQLImportTestDatabase(t, database) + path := filepath.Join(t.TempDir(), "database.sql") + if err := os.WriteFile(path, []byte("INSERT INTO users(id) VALUES (1);"), 0o600); err != nil { + t.Fatal(err) + } + app := newManagedSQLImportTestApp(t) + result := app.ImportDatabaseSQL( + connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306}, + "app", path, "sql-import-complete", false, + ) + if !result.Success { + t.Fatalf("SQL import failed: %#v", result) + } + store, err := app.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + job, err := store.Get("sql-import-complete") + if err != nil { + t.Fatal(err) + } + if job.Kind != importjob.KindSQL || job.Status != importjob.StatusCompleted || job.Succeeded != 1 || job.Failed != 0 { + t.Fatalf("unexpected SQL job: %#v", job) + } + if job.SourceIdentityToken == "" || job.SourceContentSHA256 == "" || job.SourceBytesTotal <= 0 || job.BytesRead != job.SourceBytesTotal || job.ByteProgressKind != "rawSource" { + t.Fatalf("SQL source identity/progress was not persisted: %#v", job) + } +} + +func TestImportDatabaseSQLPersistsPartialManagedJob(t *testing.T) { + database := &fakeSQLFileBatchDB{failExecSQL: "broken_proc"} + installSQLImportTestDatabase(t, database) + path := filepath.Join(t.TempDir(), "database.sql") + if err := os.WriteFile(path, []byte("CALL broken_proc();\nINSERT INTO users(id) VALUES (1);"), 0o600); err != nil { + t.Fatal(err) + } + app := newManagedSQLImportTestApp(t) + result := app.ImportDatabaseSQL( + connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306}, + "app", path, "sql-import-partial", true, + ) + if result.Success { + t.Fatalf("partial SQL import must not report full success: %#v", result) + } + store, err := app.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + job, err := store.Get("sql-import-partial") + if err != nil { + t.Fatal(err) + } + if job.Status != importjob.StatusPartial || job.Succeeded != 1 || job.Failed != 1 || job.OutcomeUnknown { + t.Fatalf("unexpected partial SQL job: %#v", job) + } +} + +func TestImportDatabaseSQLPersistsUnknownJobWhenAutomaticCommitFails(t *testing.T) { + database := &fakeSQLFileBatchDB{failExecSQL: "COMMIT"} + installSQLImportTestDatabase(t, database) + path := filepath.Join(t.TempDir(), "database.sql") + if err := os.WriteFile(path, []byte("INSERT INTO users(id) VALUES (1);"), 0o600); err != nil { + t.Fatal(err) + } + app := newManagedSQLImportTestApp(t) + result := app.ImportDatabaseSQL( + connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306}, + "app", path, "sql-import-commit-unknown", false, + ) + if result.Success { + t.Fatalf("failed automatic commit must not report success: %#v", result) + } + payload, _ := result.Data.(map[string]interface{}) + if payload["outcomeUnknown"] != true { + t.Fatalf("failed automatic commit must report an unknown outcome: %#v", result) + } + store, err := app.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + job, err := store.Get("sql-import-commit-unknown") + if err != nil { + t.Fatal(err) + } + if job.Status != importjob.StatusUnknown || !job.OutcomeUnknown { + t.Fatalf("managed job must preserve the unknown commit outcome: %#v", job) + } +} + +func TestSQLImportOptionsHashSeparatesErrorPolicies(t *testing.T) { + if buildSQLImportOptionsHash(false, DefaultSQLImportMaxStatementBytes) == buildSQLImportOptionsHash(true, DefaultSQLImportMaxStatementBytes) { + t.Fatal("different SQL error policies must not share an options hash") + } + if buildSQLImportOptionsHash(false, 0) != buildSQLImportOptionsHash(false, DefaultSQLImportMaxStatementBytes) { + t.Fatal("default SQL statement limit should have a canonical options hash") + } +} + +func TestImportDatabaseSQLMarksOutcomeUnknownWhenSourceChangesDuringExecution(t *testing.T) { + path := filepath.Join(t.TempDir(), "database.sql") + if err := os.WriteFile(path, []byte("INSERT INTO users(id) VALUES (1);\nINSERT INTO users(id) VALUES (2);"), 0o600); err != nil { + t.Fatal(err) + } + mutated := false + database := &fakeSQLFileBatchDB{execError: func(string) error { + if mutated { + return nil + } + mutated = true + file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + t.Errorf("open SQL source for mutation: %v", err) + return nil + } + if _, err := file.WriteString("\nSELECT 3;"); err != nil { + t.Errorf("mutate SQL source: %v", err) + } + if err := file.Close(); err != nil { + t.Errorf("close mutated SQL source: %v", err) + } + return nil + }} + installSQLImportTestDatabase(t, database) + app := newManagedSQLImportTestApp(t) + result := app.ImportDatabaseSQL( + connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306}, + "app", path, "sql-import-mutated", true, + ) + if result.Success { + t.Fatalf("mutated SQL source must not report a certain success: %#v", result) + } + payload, ok := result.Data.(map[string]interface{}) + if !ok || payload["sourceChanged"] != true || payload["outcomeUnknown"] != true { + t.Fatalf("unexpected source-change payload: %#v", result.Data) + } + store, err := app.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + job, err := store.Get("sql-import-mutated") + if err != nil { + t.Fatal(err) + } + if job.Status != importjob.StatusUnknown || !job.OutcomeUnknown { + t.Fatalf("unexpected source-change job: %#v", job) + } +} + +func TestImportDatabaseSQLCanCancelFullPreflightBeforeDatabaseOpen(t *testing.T) { + path := filepath.Join(t.TempDir(), "database.sql") + if err := os.WriteFile(path, []byte("INSERT INTO users(id) VALUES (1);"), 0o600); err != nil { + t.Fatal(err) + } + originalHook := sqlFilePreflightReadHook + t.Cleanup(func() { sqlFilePreflightReadHook = originalHook }) + entered := make(chan struct{}) + var once sync.Once + sqlFilePreflightReadHook = func(ctx context.Context) { + once.Do(func() { close(entered) }) + <-ctx.Done() + } + + originalNewDatabaseFunc := newDatabaseFunc + t.Cleanup(func() { newDatabaseFunc = originalNewDatabaseFunc }) + opened := false + newDatabaseFunc = func(string) (db.Database, error) { + opened = true + return &fakeSQLFileBatchDB{}, nil + } + app := newManagedSQLImportTestApp(t) + resultCh := make(chan connection.QueryResult, 1) + go func() { + resultCh <- app.ImportDatabaseSQL( + connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306}, + "app", path, "sql-import-preflight-cancel", false, + ) + }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("SQL preflight did not start") + } + if cancelResult := app.CancelSQLFileExecution("sql-import-preflight-cancel"); !cancelResult.Success { + t.Fatalf("cancel failed: %#v", cancelResult) + } + var result connection.QueryResult + select { + case result = <-resultCh: + case <-time.After(5 * time.Second): + t.Fatal("cancelled SQL preflight did not unwind") + } + if result.Success { + t.Fatalf("cancelled preflight reported success: %#v", result) + } + payload, _ := result.Data.(map[string]interface{}) + if payload["cancelled"] != true || opened { + t.Fatalf("result=%#v opened=%v, want cancellation before database open", result, opened) + } + store, err := app.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + job, err := store.Get("sql-import-preflight-cancel") + if err != nil { + t.Fatal(err) + } + if job.Status != importjob.StatusCancelled { + t.Fatalf("status = %q, want cancelled", job.Status) + } +} + +func TestImportDatabaseSQLMarksInFlightCancellationOutcomeUnknown(t *testing.T) { + path := filepath.Join(t.TempDir(), "database.sql") + if err := os.WriteFile(path, []byte("INSERT INTO users(id) VALUES (1);"), 0o600); err != nil { + t.Fatal(err) + } + entered := make(chan struct{}) + release := make(chan struct{}) + var once sync.Once + database := &fakeSQLFileBatchDB{execError: func(query string) error { + if strings.Contains(query, "INSERT INTO users") { + once.Do(func() { close(entered) }) + <-release + return context.Canceled + } + return nil + }} + installSQLImportTestDatabase(t, database) + app := newManagedSQLImportTestApp(t) + resultCh := make(chan connection.QueryResult, 1) + go func() { + resultCh <- app.ImportDatabaseSQL( + connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306}, + "app", path, "sql-import-write-cancel", true, + ) + }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("SQL write did not start") + } + if cancelResult := app.CancelSQLFileExecution("sql-import-write-cancel"); !cancelResult.Success { + t.Fatalf("cancel failed: %#v", cancelResult) + } + close(release) + var result connection.QueryResult + select { + case result = <-resultCh: + case <-time.After(5 * time.Second): + t.Fatal("cancelled SQL write did not unwind") + } + payload, _ := result.Data.(map[string]interface{}) + if result.Success || payload["cancelled"] != true || payload["outcomeUnknown"] != true { + t.Fatalf("in-flight cancellation must report an unknown outcome: %#v", result) + } + store, err := app.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + job, err := store.Get("sql-import-write-cancel") + if err != nil { + t.Fatal(err) + } + if job.Status != importjob.StatusUnknown || !job.OutcomeUnknown { + t.Fatalf("managed job must preserve the unknown cancellation outcome: %#v", job) + } +} diff --git a/internal/app/methods_file_sql_source_test.go b/internal/app/methods_file_sql_source_test.go new file mode 100644 index 00000000..79125069 --- /dev/null +++ b/internal/app/methods_file_sql_source_test.go @@ -0,0 +1,275 @@ +package app + +import ( + "context" + "encoding/binary" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "GoNavi-Wails/internal/connection" + "GoNavi-Wails/internal/db" + "GoNavi-Wails/internal/uievents" +) + +func TestExecuteSQLFileStreamRejectsOversizedStatementBeforeExec(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader("ABCDEFGHIJK;"), sqlFileExecutionOptions{ + DBType: "postgres", + MaxStatementBytes: 4, + }, nil) + var limitErr *SQLStatementTooLargeError + if !errors.As(err, &limitErr) { + t.Fatalf("execute error = %v, want SQLStatementTooLargeError", err) + } + if result.Executed != 0 || fakeDB.execCalls != 0 || fakeDB.batchCalls != 0 { + t.Fatalf("result = %#v, exec=%d batch=%d; want no SQL side effect", result, fakeDB.execCalls, fakeDB.batchCalls) + } +} + +func TestExecuteSQLFileStreamPreflightsStatementBeforeExec(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader("\\connect reporting\nSELECT 1;"), sqlFileExecutionOptions{ + DBType: "postgres", + PreflightEachStatement: true, + }, nil) + var preflightErr *sqlFilePreflightRejectedError + if !errors.As(err, &preflightErr) { + t.Fatalf("execute error = %v, want preflight rejection", err) + } + if result.Executed != 0 || fakeDB.execCalls != 0 || fakeDB.batchCalls != 0 { + t.Fatalf("result = %#v, exec=%d batch=%d; want no SQL side effect", result, fakeDB.execCalls, fakeDB.batchCalls) + } +} + +func TestSQLFileExecutionDialogFiltersIncludeGzipSQL(t *testing.T) { + filters := sqlFileExecutionDialogFilters(nil) + if len(filters) == 0 || filters[0].Pattern != "*.sql;*.sql.gz" { + t.Fatalf("filters = %#v, want SQL and gzip SQL pattern", filters) + } +} + +func TestShouldFullyPreflightSQLFileSkipsLargeSources(t *testing.T) { + if !shouldFullyPreflightSQLFile(sqlFileFullPreflightMaxRawBytes) { + t.Fatal("source at threshold should receive a full preflight") + } + if shouldFullyPreflightSQLFile(sqlFileFullPreflightMaxRawBytes + 1) { + t.Fatal("source above threshold should use per-statement preflight") + } +} + +func TestExecuteSQLFileStreamReportsPossiblePriorEffectsOnLatePreflightRejection(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{} + input := "CREATE TABLE completed_first(id int);\n\\connect reporting\nSELECT 1;" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "postgres", + PreflightEachStatement: true, + }, nil) + var preflightErr *sqlFilePreflightRejectedError + if !errors.As(err, &preflightErr) { + t.Fatalf("execute error = %v, want preflight rejection", err) + } + if result.Executed != 1 || preflightErr.executed != 1 || !strings.Contains(err.Error(), "may already have completed") { + t.Fatalf("result = %#v, error = %v; want explicit possible prior effects", result, err) + } + for _, query := range fakeDB.execQueries { + if strings.Contains(query, "\\connect") { + t.Fatalf("unsafe client command reached database: %q", query) + } + } +} + +func TestExecuteSQLFileStreamTreatsFailedPriorStatementAsUnknownOnLatePreflightRejection(t *testing.T) { + fakeDB := &fakeSQLFileBatchDB{failExecSQL: "broken_proc"} + input := "CALL broken_proc();\n\\connect reporting\nSELECT 1;" + + result, err := executeSQLFileStream(context.Background(), fakeDB, strings.NewReader(input), sqlFileExecutionOptions{ + DBType: "postgres", + ContinueOnError: true, + PreflightEachStatement: true, + }, nil) + var preflightErr *sqlFilePreflightRejectedError + if !errors.As(err, &preflightErr) { + t.Fatalf("execute error = %v, want preflight rejection", err) + } + if result.Executed != 0 || result.Failed != 1 || !preflightErr.possibleSideEffects || !preflightErr.outcomeUnknown { + t.Fatalf("result = %#v, error = %#v", result, preflightErr) + } + payload := buildSQLFilePreflightFailurePayload(preflightErr) + if payload["previousStatementsMayHaveCompleted"] != true || payload["outcomeUnknown"] != true { + t.Fatalf("unexpected preflight payload: %#v", payload) + } +} + +func TestExecuteSQLFileDecodesUTF16LEEndToEnd(t *testing.T) { + path := filepath.Join(t.TempDir(), "utf16.sql") + if err := os.WriteFile(path, encodeUTF16SQL("CREATE TABLE utf16_demo(id int);", binary.LittleEndian), 0o600); err != nil { + t.Fatalf("write UTF-16 source: %v", err) + } + fakeDB := &fakeSQLFileBatchDB{} + installSQLFileSourceDatabaseFactory(t, func(string) (db.Database, error) { return fakeDB, nil }) + + result := newSQLFileSourceTestApp(t).executeSQLFile(sqlFileSourceTestConfig(), "demo", path, "utf16-job", false) + if !result.Success { + t.Fatalf("execute UTF-16 SQL file: %#v", result) + } + if !containsSQLFileQuery(fakeDB.execQueries, "CREATE TABLE utf16_demo(id int)") { + t.Fatalf("executed queries = %#v, want decoded UTF-16 statement", fakeDB.execQueries) + } +} + +func TestExecuteSQLFileStreamsGzipEndToEnd(t *testing.T) { + path := filepath.Join(t.TempDir(), "dump.sql.gz") + writeGzipSQL(t, path, []byte("CREATE TABLE gzip_demo(id int);")) + fakeDB := &fakeSQLFileBatchDB{} + installSQLFileSourceDatabaseFactory(t, func(string) (db.Database, error) { return fakeDB, nil }) + + result := newSQLFileSourceTestApp(t).executeSQLFile(sqlFileSourceTestConfig(), "demo", path, "gzip-job", false) + if !result.Success { + t.Fatalf("execute gzip SQL file: %#v", result) + } + if !containsSQLFileQuery(fakeDB.execQueries, "CREATE TABLE gzip_demo(id int)") { + t.Fatalf("executed queries = %#v, want decompressed statement", fakeDB.execQueries) + } +} + +func TestExecuteSQLFileGzipProgressUsesRawCompressedBytes(t *testing.T) { + path := filepath.Join(t.TempDir(), "progress.sql.gz") + payload := "/*" + strings.Repeat("0123456789abcdef", 64<<10) + "*/\nCREATE TABLE progress_demo(id int);" + writeGzipSQL(t, path, []byte(payload)) + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat gzip source: %v", err) + } + fakeDB := &fakeSQLFileBatchDB{} + installSQLFileSourceDatabaseFactory(t, func(string) (db.Database, error) { return fakeDB, nil }) + recorder := &sqlFileProgressRecorder{} + app := newSQLFileSourceTestApp(t) + app.ctx = uievents.WithEmitter(context.Background(), recorder) + + result := app.executeSQLFile(sqlFileSourceTestConfig(), "demo", path, "gzip-progress-job", false) + if !result.Success { + t.Fatalf("execute gzip SQL file: %#v", result) + } + if len(recorder.events) == 0 { + t.Fatal("expected SQL file progress events") + } + for _, event := range recorder.events { + bytesRead, _ := event["bytesRead"].(int64) + if bytesRead > info.Size() { + t.Fatalf("progress bytes = %d, compressed size = %d; decoded bytes were used as numerator", bytesRead, info.Size()) + } + if event["byteProgressKind"] != "rawSource" || event["decodedBytes"] != nil || event["decodedTotalBytes"] != nil { + t.Fatalf("progress metadata = %#v, want explicit raw-source progress and unknown decoded totals", event) + } + } +} + +func TestExecuteSQLFileSmallSourcePreflightRunsBeforeDatabaseOpen(t *testing.T) { + path := filepath.Join(t.TempDir(), "unsafe.sql") + if err := os.WriteFile(path, []byte("\\connect reporting\nSELECT 1;"), 0o600); err != nil { + t.Fatalf("write unsafe source: %v", err) + } + factoryCalls := 0 + installSQLFileSourceDatabaseFactory(t, func(string) (db.Database, error) { + factoryCalls++ + return &fakeSQLFileBatchDB{}, nil + }) + + result := newSQLFileSourceTestApp(t).executeSQLFile(sqlFileSourceTestConfig(), "demo", path, "unsafe-job", false) + if result.Success || factoryCalls != 0 { + t.Fatalf("result = %#v, factory calls = %d; want rejection before database open", result, factoryCalls) + } + payload, ok := result.Data.(map[string]interface{}) + if !ok || payload["preflightRejected"] != true || payload["previousStatementsMayHaveCompleted"] != false { + t.Fatalf("payload = %#v, want structured no-side-effect preflight rejection", result.Data) + } +} + +func TestExecuteSQLFileSmallSourceStatementLimitRunsBeforeDatabaseOpen(t *testing.T) { + path := filepath.Join(t.TempDir(), "oversized.sql") + if err := os.WriteFile(path, []byte("ABCDEFGHIJK;"), 0o600); err != nil { + t.Fatalf("write oversized source: %v", err) + } + factoryCalls := 0 + installSQLFileSourceDatabaseFactory(t, func(string) (db.Database, error) { + factoryCalls++ + return &fakeSQLFileBatchDB{}, nil + }) + + result := newSQLFileSourceTestApp(t).executeSQLFileWithStatementLimit(sqlFileSourceTestConfig(), "demo", path, "oversized-job", false, 4) + if result.Success || factoryCalls != 0 { + t.Fatalf("result = %#v, factory calls = %d; want statement limit before database open", result, factoryCalls) + } + if !strings.Contains(result.Message, "source byte") { + t.Fatalf("message = %q, want structured statement position", result.Message) + } +} + +func installSQLFileSourceDatabaseFactory(t *testing.T, factory func(string) (db.Database, error)) { + t.Helper() + originalNewDatabaseFunc := newDatabaseFunc + originalResolveDialConfigWithProxyFunc := resolveDialConfigWithProxyFunc + originalDriverRuntimeSupportStatusFunc := driverRuntimeSupportStatusFunc + originalVerifyDriverAgentRevisionFunc := verifyDriverAgentRevisionFunc + t.Cleanup(func() { + newDatabaseFunc = originalNewDatabaseFunc + resolveDialConfigWithProxyFunc = originalResolveDialConfigWithProxyFunc + driverRuntimeSupportStatusFunc = originalDriverRuntimeSupportStatusFunc + verifyDriverAgentRevisionFunc = originalVerifyDriverAgentRevisionFunc + }) + newDatabaseFunc = factory + resolveDialConfigWithProxyFunc = func(config connection.ConnectionConfig) (connection.ConnectionConfig, error) { + return config, nil + } + driverRuntimeSupportStatusFunc = func(string) (bool, string) { return true, "" } + verifyDriverAgentRevisionFunc = func(connection.ConnectionConfig) error { return nil } +} + +func newSQLFileSourceTestApp(t *testing.T) *App { + t.Helper() + app := NewApp() + app.ctx = nil + app.configDir = t.TempDir() + app.startedAt = time.Now().Add(-startupConnectRetryWindow - time.Second) + return app +} + +func sqlFileSourceTestConfig() connection.ConnectionConfig { + return connection.ConnectionConfig{ + Type: "postgres", + Host: "127.0.0.1", + Port: 5432, + User: "tester", + Database: "demo", + } +} + +func containsSQLFileQuery(queries []string, expected string) bool { + for _, query := range queries { + if strings.TrimSpace(query) == expected { + return true + } + } + return false +} + +type sqlFileProgressRecorder struct { + events []map[string]interface{} +} + +func (recorder *sqlFileProgressRecorder) Emit(name string, args ...any) { + if name != "sqlfile:progress" || len(args) == 0 { + return + } + payload, ok := args[0].(map[string]interface{}) + if ok { + recorder.events = append(recorder.events, payload) + } +} diff --git a/internal/app/methods_import_artifacts.go b/internal/app/methods_import_artifacts.go new file mode 100644 index 00000000..1bbf537f --- /dev/null +++ b/internal/app/methods_import_artifacts.go @@ -0,0 +1,75 @@ +package app + +import ( + "io" + "path/filepath" + "strings" + + "GoNavi-Wails/internal/connection" + + "github.com/wailsapp/wails/v2/pkg/runtime" +) + +func (a *App) ensureImportErrorArtifactStore() (*importErrorArtifactStore, error) { + a.importArtifactMu.Lock() + defer a.importArtifactMu.Unlock() + if a.importErrorArtifacts != nil { + return a.importErrorArtifacts, nil + } + configDir := strings.TrimSpace(a.configDir) + if configDir == "" { + configDir = resolveAppConfigDir() + } + store, err := newImportErrorArtifactStore(filepath.Join(configDir, "import-artifacts")) + if err != nil { + return nil, err + } + a.importErrorArtifacts = store + return store, nil +} + +// ExportImportErrorRows copies a managed rejected-row artifact to a path the +// desktop user explicitly selected. The opaque ID prevents arbitrary local +// files from being read through the RPC boundary. +func (a *App) ExportImportErrorRows(artifactID string) connection.QueryResult { + store, err := a.ensureImportErrorArtifactStore() + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + source, err := store.Open(artifactID) + if err != nil { + return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.import_error_artifact_not_found", nil)} + } + defer source.Close() + + targetPath, err := a.showSaveFileDialog(runtime.SaveDialogOptions{ + Title: a.appText("file.backend.dialog.export_import_errors", nil), + DefaultFilename: "gonavi-import-errors.jsonl", + Filters: []runtime.FileFilter{{ + DisplayName: "JSON Lines (*.jsonl)", + Pattern: "*.jsonl", + }}, + }) + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + if strings.TrimSpace(targetPath) == "" { + return connection.QueryResult{Success: false, Message: a.appText("file.backend.message.user_cancelled", nil)} + } + target, err := createAtomicExportTarget(targetPath) + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + defer target.abort() + if _, err := io.Copy(target.file, source); err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + if err := target.commit(); err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + return connection.QueryResult{ + Success: true, + Data: map[string]interface{}{"filePath": targetPath}, + Message: a.appText("file.backend.message.import_errors_exported", nil), + } +} diff --git a/internal/app/methods_import_jobs.go b/internal/app/methods_import_jobs.go new file mode 100644 index 00000000..b1b8d2db --- /dev/null +++ b/internal/app/methods_import_jobs.go @@ -0,0 +1,121 @@ +package app + +import ( + "errors" + "os" + "path/filepath" + "strings" + + "GoNavi-Wails/internal/connection" + "GoNavi-Wails/internal/importjob" + "GoNavi-Wails/internal/logger" +) + +func (a *App) ensureImportJobStore() (*importjob.Store, error) { + a.importJobMu.Lock() + defer a.importJobMu.Unlock() + if a.importJobStore != nil { + return a.importJobStore, nil + } + configDir := strings.TrimSpace(a.configDir) + if configDir == "" { + configDir = resolveAppConfigDir() + } + store, err := importjob.Open(filepath.Join(configDir, "import-jobs")) + if err != nil { + return nil, err + } + a.importJobStore = store + return store, nil +} + +func (a *App) recoverImportJobsOnStartup() error { + store, err := a.ensureImportJobStore() + if err != nil { + return err + } + _, err = store.RecoverInterrupted() + var warning *importjob.CorruptJobFilesWarning + if errors.As(err, &warning) { + logger.Warnf("已跳过损坏的导入任务元数据文件：数量=%d", warning.Count) + return nil + } + return err +} + +func (a *App) ListImportJobs() connection.QueryResult { + store, err := a.ensureImportJobStore() + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + jobs, err := store.List() + if err != nil { + var warning *importjob.CorruptJobFilesWarning + if errors.As(err, &warning) { + logger.Warnf("已跳过损坏的导入任务元数据文件：数量=%d", warning.Count) + return connection.QueryResult{Success: true, Message: warning.Error(), Data: jobs} + } + return connection.QueryResult{Success: false, Message: err.Error()} + } + return connection.QueryResult{Success: true, Data: jobs} +} + +func (a *App) GetImportJob(jobID string) connection.QueryResult { + store, err := a.ensureImportJobStore() + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + job, err := store.Get(jobID) + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + return connection.QueryResult{Success: true, Data: job} +} + +// CancelImportJob requests cancellation for a table or SQL import task. The +// registration remains owned by the running task until it has fully unwound, +// so repeated requests are idempotent during shutdown. +func (a *App) CancelImportJob(jobID string) connection.QueryResult { + return a.cancelImportTaskByKind(jobID, "") +} + +func (a *App) cancelImportTaskByKind(jobID string, kind importjob.Kind) connection.QueryResult { + if err := a.requestImportTaskCancellation(jobID, kind); err != nil { + if errors.Is(err, errImportTaskNotFound) { + return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.task_not_found", nil)} + } + return connection.QueryResult{Success: false, Message: err.Error()} + } + return connection.QueryResult{Success: true, Message: a.appText("file.backend.message.cancel_requested", nil)} +} + +func (a *App) DeleteImportJob(jobID string) connection.QueryResult { + store, err := a.ensureImportJobStore() + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + job, err := store.Get(jobID) + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + switch job.Status { + case importjob.StatusPreparing, importjob.StatusRunning, importjob.StatusStopping: + return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.import_job_running", nil)} + } + if err := store.Delete(jobID); err != nil { + if errors.Is(err, importjob.ErrNotFound) { + return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.import_job_not_found", nil)} + } + return connection.QueryResult{Success: false, Message: err.Error()} + } + if artifactID := strings.TrimSpace(job.ErrorArtifactID); artifactID != "" { + artifactStore, err := a.ensureImportErrorArtifactStore() + if err != nil { + return connection.QueryResult{Success: false, Message: err.Error()} + } + if err := artifactStore.Delete(artifactID); err != nil && !errors.Is(err, os.ErrNotExist) { + return connection.QueryResult{Success: false, Message: err.Error()} + } + } + return connection.QueryResult{Success: true} +} diff --git a/internal/app/methods_import_jobs_test.go b/internal/app/methods_import_jobs_test.go new file mode 100644 index 00000000..26009b88 --- /dev/null +++ b/internal/app/methods_import_jobs_test.go @@ -0,0 +1,169 @@ +package app + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "GoNavi-Wails/internal/importjob" +) + +func TestListImportJobsSkipsCorruptMetadataAndKeepsJobArrayContract(t *testing.T) { + app := NewApp() + app.configDir = t.TempDir() + store, err := app.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + valid, err := store.Put(importjob.Job{ID: "import-valid", Kind: importjob.KindSQL, Status: importjob.StatusCompleted}) + if err != nil { + t.Fatal(err) + } + corruptContents := `{"sourcePath":"C:\\private\\customer.sql"` + if err := os.WriteFile(filepath.Join(app.configDir, "import-jobs", "import-corrupt.json"), []byte(corruptContents), 0o600); err != nil { + t.Fatal(err) + } + + result := app.ListImportJobs() + if !result.Success { + t.Fatalf("list import jobs failed: %#v", result) + } + jobs, ok := result.Data.([]importjob.Job) + if !ok { + t.Fatalf("data type = %T, want []importjob.Job", result.Data) + } + if len(jobs) != 1 || jobs[0].ID != valid.ID { + t.Fatalf("jobs = %#v, want only %q", jobs, valid.ID) + } + if !strings.Contains(result.Message, "1") { + t.Fatalf("warning = %q, want skipped file count", result.Message) + } + if strings.Contains(result.Message, app.configDir) || strings.Contains(result.Message, corruptContents) { + t.Fatalf("warning leaked path or contents: %q", result.Message) + } +} + +func TestRecoverImportJobsOnStartupSkipsCorruptMetadataAndRecoversValidJobs(t *testing.T) { + app := NewApp() + app.configDir = t.TempDir() + store, err := app.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + running, err := store.Put(importjob.Job{ + ID: "import-running-valid", + Kind: importjob.KindTable, + Status: importjob.StatusRunning, + SourceIdentityToken: "source-v1", + TargetFingerprint: "target-v1", + OptionsHash: "options-v1", + Checkpoint: importjob.Checkpoint{Safe: true, SourceRow: 1000}, + }) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(app.configDir, "import-jobs", "import-truncated.json"), []byte(`{"id":`), 0o600); err != nil { + t.Fatal(err) + } + + if err := app.recoverImportJobsOnStartup(); err != nil { + t.Fatalf("startup recovery failed because one metadata file was corrupt: %v", err) + } + persisted, err := store.Get(running.ID) + if err != nil { + t.Fatal(err) + } + if persisted.Status != importjob.StatusInterrupted || !persisted.Resumable { + t.Fatalf("valid job was not recovered: %#v", persisted) + } +} + +func TestImportJobsRecoverAcrossApplicationRestart(t *testing.T) { + configDir := t.TempDir() + first := NewApp() + first.configDir = configDir + store, err := first.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + if _, err := store.Put(importjob.Job{ + ID: "import-persisted-job", + Kind: importjob.KindTable, + Status: importjob.StatusRunning, + SourceIdentityToken: "source-v1", + TargetFingerprint: "target-v1", + OptionsHash: "options-v1", + Checkpoint: importjob.Checkpoint{Safe: true, SourceRow: 1000}, + }); err != nil { + t.Fatal(err) + } + + second := NewApp() + second.configDir = configDir + if err := second.recoverImportJobsOnStartup(); err != nil { + t.Fatal(err) + } + result := second.ListImportJobs() + if !result.Success { + t.Fatalf("list import jobs failed: %#v", result) + } + jobs, ok := result.Data.([]importjob.Job) + if !ok || len(jobs) != 1 { + t.Fatalf("unexpected jobs payload: %T %#v", result.Data, result.Data) + } + if jobs[0].Status != importjob.StatusInterrupted || !jobs[0].Resumable { + t.Fatalf("job was not recovered safely: %#v", jobs[0]) + } +} + +func TestDeleteImportJobRejectsRunningAndDeletesTerminalHistory(t *testing.T) { + app := NewApp() + app.configDir = t.TempDir() + store, err := app.ensureImportJobStore() + if err != nil { + t.Fatal(err) + } + running, err := store.Put(importjob.Job{ID: "import-running", Kind: importjob.KindTable, Status: importjob.StatusRunning}) + if err != nil { + t.Fatal(err) + } + artifactStore, err := app.ensureImportErrorArtifactStore() + if err != nil { + t.Fatal(err) + } + artifactWriter, err := artifactStore.Begin("import-terminal") + if err != nil { + t.Fatal(err) + } + if err := artifactWriter.Append(ImportRowError{SourceRow: 2, Category: "database", Message: "duplicate"}); err != nil { + t.Fatal(err) + } + artifact, err := artifactWriter.Finish() + if err != nil { + t.Fatal(err) + } + terminal, err := store.Put(importjob.Job{ + ID: "import-terminal", + Kind: importjob.KindSQL, + Status: importjob.StatusCompleted, + ErrorArtifactID: artifact.ID, + }) + if err != nil { + t.Fatal(err) + } + + if result := app.DeleteImportJob(running.ID); result.Success { + t.Fatalf("running job deletion unexpectedly succeeded: %#v", result) + } + if result := app.DeleteImportJob(terminal.ID); !result.Success { + t.Fatalf("terminal job deletion failed: %#v", result) + } + if _, err := store.Get(terminal.ID); err == nil { + t.Fatal("terminal job still exists after deletion") + } + if file, err := artifactStore.Open(artifact.ID); err == nil { + file.Close() + t.Fatal("terminal job artifact still exists after deletion") + } +} diff --git a/internal/app/running_query_registration_test.go b/internal/app/running_query_registration_test.go new file mode 100644 index 00000000..53531061 --- /dev/null +++ b/internal/app/running_query_registration_test.go @@ -0,0 +1,48 @@ +package app + +import ( + "context" + "testing" +) + +func TestRegisterExclusiveRunningQueryRejectsDuplicateWithoutReplacingOwner(t *testing.T) { + app := NewApp() + queryID := "exclusive-import-job" + + firstCtx, firstCancel := context.WithCancel(context.Background()) + defer firstCancel() + cleanupFirst, registered := app.registerExclusiveRunningQuery(queryID, firstCancel, true) + if !registered { + t.Fatal("first exclusive registration should succeed") + } + defer cleanupFirst() + + secondCtx, secondCancel := context.WithCancel(context.Background()) + defer secondCancel() + cleanupSecond, registered := app.registerExclusiveRunningQuery(queryID, secondCancel, true) + if registered { + cleanupSecond() + t.Fatal("duplicate exclusive registration should be rejected") + } + + if result := app.CancelQuery(queryID); !result.Success { + t.Fatalf("registered owner should remain cancellable: %s", result.Message) + } + select { + case <-firstCtx.Done(): + default: + t.Fatal("cancellation should reach the original registration") + } + select { + case <-secondCtx.Done(): + t.Fatal("rejected duplicate must not replace or receive cancellation") + default: + } + + app.queryMu.RLock() + _, retained := app.runningQueries[queryID] + app.queryMu.RUnlock() + if !retained { + t.Fatal("retainUntilDone registration must remain until owner cleanup") + } +} diff --git a/internal/app/sql_file_redaction_test.go b/internal/app/sql_file_redaction_test.go new file mode 100644 index 00000000..a5925268 --- /dev/null +++ b/internal/app/sql_file_redaction_test.go @@ -0,0 +1,38 @@ +package app + +import ( + "strings" + "testing" +) + +func TestSQLFileStatementSnippetRedactsDataLiterals(t *testing.T) { + got := sqlFileStatementSnippet("INSERT INTO users(email, token) VALUES ('alice@example.com', 'secret-token-123')", 200) + for _, secret := range []string{"alice@example.com", "secret-token-123"} { + if strings.Contains(got, secret) { + t.Fatalf("statement snippet leaked %q: %s", secret, got) + } + } + if !strings.Contains(got, "?") { + t.Fatalf("redacted snippet should retain diagnostic structure: %s", got) + } +} + +func TestSQLFileErrorDetailRedactsDriverValues(t *testing.T) { + got := sanitizeSQLFileExecutionError("duplicate key value is (alice@example.com); password=secret-token-123") + for _, secret := range []string{"alice@example.com", "secret-token-123"} { + if strings.Contains(got, secret) { + t.Fatalf("driver error leaked %q: %s", secret, got) + } + } +} + +func TestSQLFileExecutionPayloadCarriesStructuredOutcome(t *testing.T) { + cancelled := buildSQLFileExecutionPayload(12, 1, "cancelled") + if cancelled["outcome"] != "cancelled" || cancelled["cancelled"] != true || cancelled["completed"] != false { + t.Fatalf("unexpected cancelled payload: %#v", cancelled) + } + partial := buildSQLFileExecutionPayload(12, 1, "partial") + if partial["outcome"] != "partial" || partial["completed"] != true || partial["stoppedOnError"] != false { + t.Fatalf("unexpected partial payload: %#v", partial) + } +} diff --git a/internal/app/sql_import_compat_test.go b/internal/app/sql_import_compat_test.go new file mode 100644 index 00000000..9ad996f4 --- /dev/null +++ b/internal/app/sql_import_compat_test.go @@ -0,0 +1,139 @@ +package app + +import ( + "errors" + "reflect" + "strings" + "testing" +) + +func TestStreamSQLFileWithOptionsRejectsOversizedStatementWithPosition(t *testing.T) { + var statements []string + _, err := StreamSQLFileWithOptions( + strings.NewReader("A;ABCDEFGHIJK;"), + SQLStreamOptions{DBType: "postgres", MaxStatementBytes: 4}, + func(_ int, statement string) error { + statements = append(statements, statement) + return nil + }, + ) + var limitErr *SQLStatementTooLargeError + if !errors.As(err, &limitErr) { + t.Fatalf("stream error = %v, want SQLStatementTooLargeError", err) + } + if limitErr.StatementIndex != 1 || limitErr.SourceByte != 7 || limitErr.MaxBytes != 4 { + t.Fatalf("limit error = %#v, want statement 1 at source byte 7", limitErr) + } + if !reflect.DeepEqual(statements, []string{"A"}) { + t.Fatalf("statements before limit = %#v, want first completed statement", statements) + } +} + +func TestStreamSQLFileForDialectUsesMySQLDashCommentRule(t *testing.T) { + var statements []string + _, err := streamSQLFileForDialect(strings.NewReader("SELECT 3--2;\nSELECT 4;"), "mysql", func(_ int, stmt string) error { + statements = append(statements, stmt) + return nil + }) + if err != nil { + t.Fatalf("stream SQL file: %v", err) + } + + want := []string{"SELECT 3--2", "SELECT 4"} + if !reflect.DeepEqual(statements, want) { + t.Fatalf("statements = %#v, want %#v", statements, want) + } +} + +func TestStreamSQLFileForDialectConsumesMySQLDelimiterDirectives(t *testing.T) { + input := strings.Join([]string{ + "DELIMITER $$", + "CREATE PROCEDURE rebuild_demo()", + "BEGIN", + " INSERT INTO demo(id) VALUES (1);", + " INSERT INTO demo(id) VALUES (2);", + "END$$", + "DELIMITER ;", + "INSERT INTO demo(id) VALUES (3);", + }, "\n") + + var statements []string + _, err := streamSQLFileForDialect(strings.NewReader(input), "mysql", func(_ int, stmt string) error { + statements = append(statements, stmt) + return nil + }) + if err != nil { + t.Fatalf("stream SQL file: %v", err) + } + + want := []string{ + "CREATE PROCEDURE rebuild_demo()\nBEGIN\n INSERT INTO demo(id) VALUES (1);\n INSERT INTO demo(id) VALUES (2);\nEND", + "INSERT INTO demo(id) VALUES (3)", + } + if !reflect.DeepEqual(statements, want) { + t.Fatalf("statements = %#v, want %#v", statements, want) + } +} + +func TestStreamSQLFileForDialectConsumesMySQLDelimiterAfterHeaderComment(t *testing.T) { + input := "-- generated dump\nDELIMITER $$\nCREATE PROCEDURE p() BEGIN SELECT 1; END$$\n" + + var statements []string + _, err := streamSQLFileForDialect(strings.NewReader(input), "mysql", func(_ int, stmt string) error { + statements = append(statements, stmt) + return nil + }) + if err != nil { + t.Fatalf("stream SQL file: %v", err) + } + + want := []string{"CREATE PROCEDURE p() BEGIN SELECT 1; END"} + if !reflect.DeepEqual(statements, want) { + t.Fatalf("statements = %#v, want delimiter directive after header comment %#v", statements, want) + } +} + +func TestStreamSQLFileForDialectConsumesSQLServerGoBatches(t *testing.T) { + input := strings.Join([]string{ + "CREATE TABLE demo(id int)", + "GO", + "INSERT INTO demo(id) VALUES (1);", + "GO 2", + }, "\n") + + var statements []string + _, err := streamSQLFileForDialect(strings.NewReader(input), "sqlserver", func(_ int, stmt string) error { + statements = append(statements, stmt) + return nil + }) + if err != nil { + t.Fatalf("stream SQL file: %v", err) + } + + want := []string{ + "CREATE TABLE demo(id int)", + "INSERT INTO demo(id) VALUES (1);", + "INSERT INTO demo(id) VALUES (1);", + } + if !reflect.DeepEqual(statements, want) { + t.Fatalf("statements = %#v, want %#v", statements, want) + } +} + +func TestStreamSQLFileForDialectKeepsSQLServerBatchScope(t *testing.T) { + input := "DECLARE @value int = 1;\nSELECT @value;\nGO\n" + + var statements []string + _, err := streamSQLFileForDialect(strings.NewReader(input), "sqlserver", func(_ int, stmt string) error { + statements = append(statements, stmt) + return nil + }) + if err != nil { + t.Fatalf("stream SQL file: %v", err) + } + + want := []string{"DECLARE @value int = 1;\nSELECT @value;"} + if !reflect.DeepEqual(statements, want) { + t.Fatalf("statements = %#v, want one SQL Server batch %#v", statements, want) + } +} diff --git a/internal/app/sql_import_preflight.go b/internal/app/sql_import_preflight.go new file mode 100644 index 00000000..e423efc2 --- /dev/null +++ b/internal/app/sql_import_preflight.go @@ -0,0 +1,371 @@ +package app + +import ( + "errors" + "io" + "strings" +) + +type SQLImportPreflightReasonCode string + +const ( + SQLImportPreflightPostgresCopyFromStdin SQLImportPreflightReasonCode = "postgres_copy_from_stdin" + SQLImportPreflightPsqlMetaCommand SQLImportPreflightReasonCode = "psql_meta_command" + SQLImportPreflightSQLCmdCommand SQLImportPreflightReasonCode = "sqlcmd_command" + SQLImportPreflightMySQLClientCommand SQLImportPreflightReasonCode = "mysql_client_command" + SQLImportPreflightSQLPlusCommand SQLImportPreflightReasonCode = "sqlplus_command" + SQLImportPreflightSQLiteClientCommand SQLImportPreflightReasonCode = "sqlite_client_command" +) + +type SQLImportPreflightReason struct { + Code SQLImportPreflightReasonCode `json:"code"` + DBType string `json:"dbType"` + StatementIndex int `json:"statementIndex"` + SourceByte int64 `json:"sourceByte,omitempty"` + Directive string `json:"directive,omitempty"` +} + +type SQLImportPreflightResult struct { + Safe bool `json:"safe"` + Reason *SQLImportPreflightReason `json:"reason,omitempty"` +} + +var errSQLImportPreflightRejected = errors.New("SQL import preflight rejected the source") + +func PreflightSQLImport(reader io.Reader, dbType string) (SQLImportPreflightResult, error) { + return PreflightSQLImportWithOptions(reader, SQLStreamOptions{ + DBType: dbType, + MaxStatementBytes: DefaultSQLImportMaxStatementBytes, + }) +} + +func PreflightSQLImportWithOptions(reader io.Reader, options SQLStreamOptions) (SQLImportPreflightResult, error) { + result := SQLImportPreflightResult{Safe: true} + normalizedType := normalizeExplainLexicalDBType(options.DBType) + maxStatementBytes := options.MaxStatementBytes + if maxStatementBytes <= 0 { + maxStatementBytes = DefaultSQLImportMaxStatementBytes + } + _, err := StreamSQLFileWithOptions(reader, SQLStreamOptions{ + DBType: normalizedType, + MaxStatementBytes: maxStatementBytes, + }, func(index int, stmt string) error { + statementResult := PreflightSQLStatement(stmt, normalizedType, index) + if !statementResult.Safe { + result = statementResult + return errSQLImportPreflightRejected + } + return nil + }) + if errors.Is(err, errSQLImportPreflightRejected) { + return result, nil + } + if err != nil { + return SQLImportPreflightResult{}, err + } + return result, nil +} + +// PreflightSQLStatement classifies one already-split statement without +// allocating another streaming splitter. statementIndex is zero-based. +func PreflightSQLStatement(stmt, dbType string, statementIndex int) SQLImportPreflightResult { + normalizedType := normalizeExplainLexicalDBType(dbType) + if isSQLImportPostgresDialect(normalizedType) && isPostgresCopyFromStdinStatement(stmt) { + return SQLImportPreflightResult{ + Safe: false, + Reason: &SQLImportPreflightReason{ + Code: SQLImportPreflightPostgresCopyFromStdin, + DBType: normalizedType, + StatementIndex: statementIndex, + Directive: "COPY FROM STDIN", + }, + } + } + if code, directive := findSQLImportClientCommand(stmt, normalizedType); code != "" { + return SQLImportPreflightResult{ + Safe: false, + Reason: &SQLImportPreflightReason{ + Code: code, + DBType: normalizedType, + StatementIndex: statementIndex, + Directive: directive, + }, + } + } + return SQLImportPreflightResult{Safe: true} +} + +func isSQLImportPostgresDialect(dbType string) bool { + switch normalizeExplainLexicalDBType(dbType) { + case "postgres", "kingbase", "highgo", "vastbase", "opengauss", "gaussdb": + return true + default: + return false + } +} + +func isPostgresCopyFromStdinStatement(stmt string) bool { + keyword, position := nextSQLKeyword(stmt, 0) + if keyword != "copy" { + return false + } + parenDepth := 0 + for position < len(stmt) { + position = skipSQLTrivia(stmt, position) + if position >= len(stmt) { + break + } + switch stmt[position] { + case '\'', '"', '`': + position = skipSQLImportQuotedValue(stmt, position, stmt[position]) + continue + case '(': + parenDepth++ + position++ + continue + case ')': + if parenDepth > 0 { + parenDepth-- + } + position++ + continue + case '$': + if tag := parseSQLDollarTagAt(stmt, position); tag != "" { + position += len(tag) + if closeOffset := strings.Index(stmt[position:], tag); closeOffset >= 0 { + position += closeOffset + len(tag) + } else { + return false + } + continue + } + } + if !isSQLIdentifierStart(stmt[position]) { + position++ + continue + } + end := position + 1 + for end < len(stmt) && isSQLIdentifierPart(stmt[end]) { + end++ + } + if parenDepth == 0 && strings.EqualFold(stmt[position:end], "from") { + nextPosition := skipSQLTrivia(stmt, end) + next, _ := nextSQLKeyword(stmt, nextPosition) + return next == "stdin" + } + position = end + } + return false +} + +func skipSQLImportQuotedValue(text string, start int, quote byte) int { + for position := start + 1; position < len(text); position++ { + if text[position] == '\\' { + position++ + continue + } + if text[position] != quote { + continue + } + if position+1 < len(text) && text[position+1] == quote { + position++ + continue + } + return position + 1 + } + return len(text) +} + +func normalizeSQLImportDirective(raw string) string { + return strings.TrimSpace(raw) +} + +func findSQLImportClientCommand(statement, dbType string) (SQLImportPreflightReasonCode, string) { + state := sqlImportPreflightLexicalState{} + for lineStart := 0; lineStart <= len(statement); { + lineEnd := strings.IndexByte(statement[lineStart:], '\n') + if lineEnd < 0 { + lineEnd = len(statement) + } else { + lineEnd += lineStart + } + line := statement[lineStart:lineEnd] + if state.acceptsClientCommandLine() { + trimmed := normalizeSQLImportDirective(line) + switch normalizeExplainLexicalDBType(dbType) { + case "postgres", "kingbase", "highgo", "vastbase", "opengauss", "gaussdb": + if strings.HasPrefix(trimmed, "\\") { + return SQLImportPreflightPsqlMetaCommand, firstSQLImportDirectiveToken(trimmed) + } + case "sqlserver": + if strings.HasPrefix(trimmed, ":") || strings.HasPrefix(trimmed, "!!") { + return SQLImportPreflightSQLCmdCommand, firstSQLImportDirectiveToken(trimmed) + } + case "mysql", "mariadb", "oceanbase", "diros", "starrocks", "goldendb", "sphinx", "tidb": + token := firstSQLImportDirectiveToken(trimmed) + if strings.EqualFold(token, "source") || strings.HasPrefix(token, "\\.") || strings.HasPrefix(token, "\\!") { + return SQLImportPreflightMySQLClientCommand, token + } + case "oracle", "dameng": + token := firstSQLImportDirectiveToken(trimmed) + if isSQLPlusClientDirective(token) { + if strings.HasPrefix(token, "@@") { + return SQLImportPreflightSQLPlusCommand, "@@" + } + if strings.HasPrefix(token, "@") { + return SQLImportPreflightSQLPlusCommand, "@" + } + return SQLImportPreflightSQLPlusCommand, token + } + case "sqlite": + token := firstSQLImportDirectiveToken(trimmed) + if strings.HasPrefix(token, ".") { + return SQLImportPreflightSQLiteClientCommand, token + } + } + } + state.consume(line, dbType) + if state.clientCommand != "" { + return state.clientCommandCode, state.clientCommand + } + if lineEnd >= len(statement) { + break + } + state.consume("\n", dbType) + lineStart = lineEnd + 1 + } + return "", "" +} + +func firstSQLImportDirectiveToken(line string) string { + if end := strings.IndexAny(line, " \t\r\n;"); end >= 0 { + return line[:end] + } + return line +} + +func isSQLPlusClientDirective(token string) bool { + if strings.HasPrefix(token, "@") { + return true + } + switch strings.ToLower(strings.TrimSuffix(token, ";")) { + case "accept", "btitle", "break", "column", "compute", "connect", "define", "disconnect", "exit", "host", "print", "prompt", "quit", "remark", "spool", "start", "ttitle", "undefine", "variable", "whenever": + return true + default: + return false + } +} + +type sqlImportPreflightLexicalState struct { + inSingle bool + inDouble bool + inBacktick bool + inLineComment bool + inBlockComment bool + escaped bool + dollarTag string + clientCommand string + clientCommandCode SQLImportPreflightReasonCode +} + +func (state *sqlImportPreflightLexicalState) acceptsClientCommandLine() bool { + return !state.inSingle && !state.inDouble && !state.inBacktick && !state.inBlockComment && state.dollarTag == "" +} + +func (state *sqlImportPreflightLexicalState) consume(text, dbType string) { + for index := 0; index < len(text); index++ { + ch := text[index] + next := byte(0) + if index+1 < len(text) { + next = text[index+1] + } + if state.inLineComment { + if ch == '\n' { + state.inLineComment = false + } + continue + } + if state.inBlockComment { + if ch == '*' && next == '/' { + state.inBlockComment = false + index++ + } + continue + } + if state.dollarTag != "" { + if strings.HasPrefix(text[index:], state.dollarTag) { + index += len(state.dollarTag) - 1 + state.dollarTag = "" + } + continue + } + if state.escaped { + state.escaped = false + continue + } + if (state.inSingle || state.inDouble) && ch == '\\' { + state.escaped = true + continue + } + if !state.inDouble && !state.inBacktick && ch == '\'' { + if state.inSingle && next == '\'' { + index++ + continue + } + state.inSingle = !state.inSingle + continue + } + if !state.inSingle && !state.inBacktick && ch == '"' { + state.inDouble = !state.inDouble + continue + } + if !state.inSingle && !state.inDouble && ch == '`' { + state.inBacktick = !state.inBacktick + continue + } + if state.inSingle || state.inDouble || state.inBacktick { + continue + } + if ch == '\\' { + switch { + case isSQLImportPostgresDialect(dbType): + state.clientCommandCode = SQLImportPreflightPsqlMetaCommand + case isSQLImportMySQLClientDialect(dbType): + state.clientCommandCode = SQLImportPreflightMySQLClientCommand + default: + continue + } + state.clientCommand = firstSQLImportDirectiveToken(text[index:]) + return + } + if ch == '-' && next == '-' && isSQLDashLineCommentStart(dbType, text, index) { + state.inLineComment = true + index++ + continue + } + if ch == '#' && supportsSQLHashLineComment(dbType) { + state.inLineComment = true + continue + } + if ch == '/' && next == '*' { + state.inBlockComment = true + index++ + continue + } + if ch == '$' && supportsSQLDollarQuote(dbType) { + if tag := parseSQLDollarTagAt(text, index); tag != "" { + state.dollarTag = tag + index += len(tag) - 1 + } + } + } +} + +func isSQLImportMySQLClientDialect(dbType string) bool { + switch normalizeExplainLexicalDBType(dbType) { + case "mysql", "mariadb", "oceanbase", "diros", "starrocks", "goldendb", "sphinx", "tidb": + return true + default: + return false + } +} diff --git a/internal/app/sql_import_preflight_test.go b/internal/app/sql_import_preflight_test.go new file mode 100644 index 00000000..395ab4ef --- /dev/null +++ b/internal/app/sql_import_preflight_test.go @@ -0,0 +1,134 @@ +package app + +import ( + "strings" + "testing" +) + +func TestPreflightSQLImportRejectsPostgresCopyFromStdin(t *testing.T) { + result, err := PreflightSQLImport(strings.NewReader("COPY public.demo (id, name) FROM STDIN;\n1\talpha\n\\.\n"), "postgres") + if err != nil { + t.Fatalf("preflight SQL import: %v", err) + } + if result.Safe || result.Reason == nil { + t.Fatalf("result = %#v, want a structured rejection", result) + } + if result.Reason.Code != SQLImportPreflightPostgresCopyFromStdin { + t.Fatalf("reason code = %q, want %q", result.Reason.Code, SQLImportPreflightPostgresCopyFromStdin) + } +} + +func TestPreflightSQLImportRejectsPsqlMetaCommand(t *testing.T) { + result, err := PreflightSQLImport(strings.NewReader("\\connect reporting\nSELECT 1;\n"), "postgres") + if err != nil { + t.Fatalf("preflight SQL import: %v", err) + } + if result.Safe || result.Reason == nil { + t.Fatalf("result = %#v, want a structured rejection", result) + } + if result.Reason.Code != SQLImportPreflightReasonCode("psql_meta_command") { + t.Fatalf("reason code = %q, want psql_meta_command", result.Reason.Code) + } +} + +func TestPreflightSQLImportRejectsInlinePsqlMetaCommand(t *testing.T) { + result, err := PreflightSQLImport(strings.NewReader("SELECT 1 \\g\n"), "postgres") + if err != nil { + t.Fatalf("preflight SQL import: %v", err) + } + if result.Safe || result.Reason == nil || result.Reason.Code != SQLImportPreflightPsqlMetaCommand { + t.Fatalf("result = %#v, want inline psql meta-command rejection", result) + } +} + +func TestPreflightSQLImportRejectsSQLCmdInclude(t *testing.T) { + result, err := PreflightSQLImport(strings.NewReader(":r child.sql\nSELECT 1;\n"), "sqlserver") + if err != nil { + t.Fatalf("preflight SQL import: %v", err) + } + if result.Safe || result.Reason == nil { + t.Fatalf("result = %#v, want a structured rejection", result) + } + if result.Reason.Code != SQLImportPreflightReasonCode("sqlcmd_command") { + t.Fatalf("reason code = %q, want sqlcmd_command", result.Reason.Code) + } +} + +func TestPreflightSQLImportRejectsMySQLSourceCommand(t *testing.T) { + result, err := PreflightSQLImport(strings.NewReader("SOURCE child.sql;\n"), "mysql") + if err != nil { + t.Fatalf("preflight SQL import: %v", err) + } + if result.Safe || result.Reason == nil { + t.Fatalf("result = %#v, want a structured rejection", result) + } + if result.Reason.Code != SQLImportPreflightReasonCode("mysql_client_command") { + t.Fatalf("reason code = %q, want mysql_client_command", result.Reason.Code) + } +} + +func TestPreflightSQLImportRejectsInlineMySQLClientCommand(t *testing.T) { + result, err := PreflightSQLImport(strings.NewReader("SELECT 1 \\G\n"), "mysql") + if err != nil { + t.Fatalf("preflight SQL import: %v", err) + } + if result.Safe || result.Reason == nil || result.Reason.Code != SQLImportPreflightMySQLClientCommand { + t.Fatalf("result = %#v, want inline mysql client-command rejection", result) + } +} + +func TestPreflightSQLImportRejectsSQLPlusInclude(t *testing.T) { + result, err := PreflightSQLImport(strings.NewReader("@child.sql\nSELECT 1 FROM dual;\n"), "oracle") + if err != nil { + t.Fatalf("preflight SQL import: %v", err) + } + if result.Safe || result.Reason == nil { + t.Fatalf("result = %#v, want a structured rejection", result) + } + if result.Reason.Code != SQLImportPreflightReasonCode("sqlplus_command") { + t.Fatalf("reason code = %q, want sqlplus_command", result.Reason.Code) + } + if result.Reason.Directive != "@" { + t.Fatalf("directive = %q, want redacted include marker", result.Reason.Directive) + } +} + +func TestPreflightSQLImportRejectsSQLiteDotCommand(t *testing.T) { + result, err := PreflightSQLImport(strings.NewReader(".read child.sql\nSELECT 1;\n"), "sqlite") + if err != nil { + t.Fatalf("preflight SQL import: %v", err) + } + if result.Safe || result.Reason == nil || result.Reason.Code != SQLImportPreflightReasonCode("sqlite_client_command") { + t.Fatalf("result = %#v, want sqlite client-command rejection", result) + } +} + +func TestPreflightSQLImportIgnoresClientCommandTextInsidePostgresDollarQuote(t *testing.T) { + result, err := PreflightSQLImport(strings.NewReader("SELECT $$line one\n\\connect not_a_command\nline three$$;\n"), "postgres") + if err != nil { + t.Fatalf("preflight SQL import: %v", err) + } + if !result.Safe || result.Reason != nil { + t.Fatalf("result = %#v, want quoted client-command text to remain safe", result) + } +} + +func TestPreflightSQLImportDoesNotTreatQuotedStdinAsCopyProtocol(t *testing.T) { + result, err := PreflightSQLImport(strings.NewReader("COPY demo FROM 'stdin';\n"), "postgres") + if err != nil { + t.Fatalf("preflight SQL import: %v", err) + } + if !result.Safe || result.Reason != nil { + t.Fatalf("result = %#v, want quoted filename to remain ordinary SQL", result) + } +} + +func TestPreflightSQLImportDoesNotTreatCopyQueryTableAsStdinProtocol(t *testing.T) { + result, err := PreflightSQLImport(strings.NewReader("COPY (SELECT * FROM stdin) TO STDOUT;\n"), "postgres") + if err != nil { + t.Fatalf("preflight SQL import: %v", err) + } + if !result.Safe || result.Reason != nil { + t.Fatalf("result = %#v, want COPY query to remain ordinary SQL", result) + } +} diff --git a/internal/app/sql_import_source.go b/internal/app/sql_import_source.go new file mode 100644 index 00000000..8170b6e7 --- /dev/null +++ b/internal/app/sql_import_source.go @@ -0,0 +1,206 @@ +package app + +import ( + "bufio" + "bytes" + "compress/gzip" + "fmt" + "io" + "os" + "strings" + + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" +) + +// SQLImportSourceOptions defines safety limits applied while decoding an SQL +// import source. Zero values select production-safe defaults. +type SQLImportSourceOptions struct { + MaxDecodedBytes int64 + MaxCompressionRatio float64 + MinCompressedBytesForRatio int64 + // RawObserver receives the original on-disk bytes as they are read. It is + // suitable for streaming hashes and never receives decoded SQL content. + RawObserver io.Writer +} + +type SQLImportSourceLimitKind string + +const ( + SQLImportSourceDecodedByteLimit SQLImportSourceLimitKind = "decoded_byte_limit" + SQLImportSourceCompressionRatio SQLImportSourceLimitKind = "compression_ratio" + defaultSQLImportMaxDecodedBytes = int64(16 << 30) + defaultSQLImportMaxCompressionRatio = 1000.0 + defaultSQLImportMinCompressedBytesForRatio = int64(1) +) + +// SQLImportSourceLimitError reports a streaming safety limit without exposing +// source content. +type SQLImportSourceLimitError struct { + Kind SQLImportSourceLimitKind + Limit int64 + DecodedBytes int64 + CompressedBytes int64 + Ratio float64 + MaxCompressionRatio float64 +} + +func (err *SQLImportSourceLimitError) Error() string { + return fmt.Sprintf("SQL import source exceeded %s safety limit", err.Kind) +} + +type sqlImportDecodedLimitReader struct { + reader io.Reader + maxBytes int64 + decodedBytes int64 + compressed *sqlImportCountingReader + maxCompressionRatio float64 + minCompressedRatioBytes int64 +} + +func (reader *sqlImportDecodedLimitReader) Read(buffer []byte) (int, error) { + remaining := reader.maxBytes - reader.decodedBytes + if remaining < 0 { + remaining = 0 + } + probeSize := int64(len(buffer)) + if probeSize > remaining+1 { + probeSize = remaining + 1 + } + n, readErr := reader.reader.Read(buffer[:int(probeSize)]) + projectedDecodedBytes := reader.decodedBytes + int64(n) + if int64(n) > remaining { + allowed := int(remaining) + reader.decodedBytes += int64(allowed) + return allowed, &SQLImportSourceLimitError{ + Kind: SQLImportSourceDecodedByteLimit, + Limit: reader.maxBytes, + DecodedBytes: reader.decodedBytes, + } + } + if reader.compressed != nil && reader.compressed.bytes >= reader.minCompressedRatioBytes { + ratio := float64(projectedDecodedBytes) / float64(reader.compressed.bytes) + if ratio > reader.maxCompressionRatio { + reader.decodedBytes = projectedDecodedBytes + return 0, &SQLImportSourceLimitError{ + Kind: SQLImportSourceCompressionRatio, + DecodedBytes: projectedDecodedBytes, + CompressedBytes: reader.compressed.bytes, + Ratio: ratio, + MaxCompressionRatio: reader.maxCompressionRatio, + } + } + } + reader.decodedBytes = projectedDecodedBytes + return n, readErr +} + +type sqlImportCountingReader struct { + reader io.Reader + bytes int64 +} + +func (reader *sqlImportCountingReader) Read(buffer []byte) (int, error) { + n, err := reader.reader.Read(buffer) + reader.bytes += int64(n) + return n, err +} + +// SQLImportSource is a streaming, UTF-8 SQL source. Callers must close it. +type SQLImportSource struct { + io.Reader + Encoding string + Compressed bool + rawCounter *sqlImportCountingReader + close func() error +} + +// RawBytesRead returns bytes consumed from the original file. For .sql.gz it +// therefore reports compressed bytes rather than decoded SQL bytes. +func (source *SQLImportSource) RawBytesRead() int64 { + if source == nil || source.rawCounter == nil { + return 0 + } + return source.rawCounter.bytes +} + +func (source *SQLImportSource) Close() error { + if source == nil || source.close == nil { + return nil + } + return source.close() +} + +// OpenSQLImportSource opens path without loading it into memory. +func OpenSQLImportSource(path string, options SQLImportSourceOptions) (*SQLImportSource, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + rawReader := io.Reader(file) + if options.RawObserver != nil { + rawReader = io.TeeReader(rawReader, options.RawObserver) + } + rawCounter := &sqlImportCountingReader{reader: rawReader} + payload := io.Reader(rawCounter) + compressed := strings.HasSuffix(strings.ToLower(path), ".sql.gz") + var compressionCounter *sqlImportCountingReader + closeSource := file.Close + if compressed { + compressionCounter = rawCounter + gzipReader, gzipErr := gzip.NewReader(rawCounter) + if gzipErr != nil { + _ = file.Close() + return nil, gzipErr + } + payload = gzipReader + closeSource = func() error { + gzipCloseErr := gzipReader.Close() + fileCloseErr := file.Close() + if gzipCloseErr != nil { + return gzipCloseErr + } + return fileCloseErr + } + } + buffered := bufio.NewReader(payload) + reader := io.Reader(buffered) + encodingName := "utf-8" + if prefix, _ := buffered.Peek(3); bytes.Equal(prefix, []byte{0xef, 0xbb, 0xbf}) { + _, _ = buffered.Discard(3) + } else if prefix, _ := buffered.Peek(2); bytes.Equal(prefix, []byte{0xff, 0xfe}) { + _, _ = buffered.Discard(2) + reader = transform.NewReader(buffered, unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewDecoder()) + encodingName = "utf-16le" + } else if prefix, _ := buffered.Peek(2); bytes.Equal(prefix, []byte{0xfe, 0xff}) { + _, _ = buffered.Discard(2) + reader = transform.NewReader(buffered, unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM).NewDecoder()) + encodingName = "utf-16be" + } + maxDecodedBytes := options.MaxDecodedBytes + if maxDecodedBytes <= 0 { + maxDecodedBytes = defaultSQLImportMaxDecodedBytes + } + maxCompressionRatio := options.MaxCompressionRatio + if maxCompressionRatio <= 0 { + maxCompressionRatio = defaultSQLImportMaxCompressionRatio + } + minCompressedRatioBytes := options.MinCompressedBytesForRatio + if minCompressedRatioBytes <= 0 { + minCompressedRatioBytes = defaultSQLImportMinCompressedBytesForRatio + } + reader = &sqlImportDecodedLimitReader{ + reader: reader, + maxBytes: maxDecodedBytes, + compressed: compressionCounter, + maxCompressionRatio: maxCompressionRatio, + minCompressedRatioBytes: minCompressedRatioBytes, + } + return &SQLImportSource{ + Reader: reader, + Encoding: encodingName, + Compressed: compressed, + rawCounter: rawCounter, + close: closeSource, + }, nil +} diff --git a/internal/app/sql_import_source_test.go b/internal/app/sql_import_source_test.go new file mode 100644 index 00000000..0fc21178 --- /dev/null +++ b/internal/app/sql_import_source_test.go @@ -0,0 +1,240 @@ +package app + +import ( + "bytes" + "compress/gzip" + "encoding/binary" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "unicode/utf16" +) + +func TestOpenSQLImportSourceStripsUTF8BOM(t *testing.T) { + path := filepath.Join(t.TempDir(), "dump.sql") + if err := os.WriteFile(path, append([]byte{0xef, 0xbb, 0xbf}, []byte("SELECT 1;")...), 0o600); err != nil { + t.Fatalf("write source: %v", err) + } + + source, err := OpenSQLImportSource(path, SQLImportSourceOptions{}) + if err != nil { + t.Fatalf("open SQL import source: %v", err) + } + defer source.Close() + + got, err := io.ReadAll(source) + if err != nil { + t.Fatalf("read SQL import source: %v", err) + } + if string(got) != "SELECT 1;" { + t.Fatalf("decoded source = %q, want BOM-free UTF-8", got) + } +} + +func TestOpenSQLImportSourceDecodesUTF16LE(t *testing.T) { + path := filepath.Join(t.TempDir(), "dump.sql") + if err := os.WriteFile(path, encodeUTF16SQL("SELECT '中文';", binary.LittleEndian), 0o600); err != nil { + t.Fatalf("write source: %v", err) + } + + source, err := OpenSQLImportSource(path, SQLImportSourceOptions{}) + if err != nil { + t.Fatalf("open SQL import source: %v", err) + } + defer source.Close() + + got, err := io.ReadAll(source) + if err != nil { + t.Fatalf("read SQL import source: %v", err) + } + if string(got) != "SELECT '中文';" || source.Encoding != "utf-16le" { + t.Fatalf("decoded source = %q (%s), want UTF-16LE decoded as UTF-8", got, source.Encoding) + } +} + +func TestOpenSQLImportSourceDecodesUTF16BE(t *testing.T) { + path := filepath.Join(t.TempDir(), "dump.sql") + if err := os.WriteFile(path, encodeUTF16SQL("SELECT '中文';", binary.BigEndian), 0o600); err != nil { + t.Fatalf("write source: %v", err) + } + + source, err := OpenSQLImportSource(path, SQLImportSourceOptions{}) + if err != nil { + t.Fatalf("open SQL import source: %v", err) + } + defer source.Close() + + got, err := io.ReadAll(source) + if err != nil { + t.Fatalf("read SQL import source: %v", err) + } + if string(got) != "SELECT '中文';" || source.Encoding != "utf-16be" { + t.Fatalf("decoded source = %q (%s), want UTF-16BE decoded as UTF-8", got, source.Encoding) + } +} + +func TestOpenSQLImportSourceStreamsGzipBeforeEncodingDetection(t *testing.T) { + path := filepath.Join(t.TempDir(), "dump.sql.gz") + writeGzipSQL(t, path, append([]byte{0xef, 0xbb, 0xbf}, []byte("SELECT 1;")...)) + + source, err := OpenSQLImportSource(path, SQLImportSourceOptions{}) + if err != nil { + t.Fatalf("open SQL import source: %v", err) + } + defer source.Close() + + got, err := io.ReadAll(source) + if err != nil { + t.Fatalf("read SQL import source: %v", err) + } + if string(got) != "SELECT 1;" || !source.Compressed { + t.Fatalf("decoded source = %q (compressed=%v), want streamed gzip SQL", got, source.Compressed) + } +} + +func TestOpenSQLImportSourceReportsAndObservesRawCompressedBytes(t *testing.T) { + path := filepath.Join(t.TempDir(), "dump.sql.gz") + writeGzipSQL(t, path, []byte("SELECT 1;")) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read raw source: %v", err) + } + var observed bytes.Buffer + + source, err := OpenSQLImportSource(path, SQLImportSourceOptions{RawObserver: &observed}) + if err != nil { + t.Fatalf("open SQL import source: %v", err) + } + defer source.Close() + if _, err := io.ReadAll(source); err != nil { + t.Fatalf("read SQL import source: %v", err) + } + if source.RawBytesRead() != int64(len(raw)) { + t.Fatalf("raw bytes read = %d, want compressed size %d", source.RawBytesRead(), len(raw)) + } + if !bytes.Equal(observed.Bytes(), raw) { + t.Fatal("raw observer did not receive the original compressed bytes") + } +} + +func TestOpenSQLImportSourceEnforcesDecodedByteLimit(t *testing.T) { + path := filepath.Join(t.TempDir(), "dump.sql") + if err := os.WriteFile(path, []byte("SELECT 1234567890;"), 0o600); err != nil { + t.Fatalf("write source: %v", err) + } + + source, err := OpenSQLImportSource(path, SQLImportSourceOptions{MaxDecodedBytes: 8}) + if err != nil { + t.Fatalf("open SQL import source: %v", err) + } + defer source.Close() + + got, readErr := io.ReadAll(source) + var limitErr *SQLImportSourceLimitError + if !errors.As(readErr, &limitErr) || limitErr.Kind != SQLImportSourceDecodedByteLimit { + t.Fatalf("read error = %v, want decoded-byte limit error", readErr) + } + if len(got) > 8 || limitErr.DecodedBytes != 8 || limitErr.Limit != 8 { + t.Fatalf("read %d bytes, error = %#v; want no bytes beyond limit", len(got), limitErr) + } +} + +func TestOpenSQLImportSourceRejectsExcessiveCompressionRatio(t *testing.T) { + path := filepath.Join(t.TempDir(), "dump.sql.gz") + writeGzipSQL(t, path, []byte(strings.Repeat("INSERT INTO t VALUES (1);\n", 4096))) + + source, err := OpenSQLImportSource(path, SQLImportSourceOptions{ + MaxDecodedBytes: 1 << 20, + MaxCompressionRatio: 2, + MinCompressedBytesForRatio: 1, + }) + if err != nil { + t.Fatalf("open SQL import source: %v", err) + } + defer source.Close() + + _, readErr := io.ReadAll(source) + var limitErr *SQLImportSourceLimitError + if !errors.As(readErr, &limitErr) || limitErr.Kind != SQLImportSourceCompressionRatio { + t.Fatalf("read error = %v, want compression-ratio limit error", readErr) + } + if limitErr.Ratio <= 2 || limitErr.MaxCompressionRatio != 2 || limitErr.CompressedBytes <= 0 { + t.Fatalf("limit error = %#v, want measured ratio above limit", limitErr) + } +} + +func TestOpenSQLImportSourceDefaultCompressionRatioProtectsSmallGzipBomb(t *testing.T) { + path := filepath.Join(t.TempDir(), "dump.sql.gz") + writeGzipSQL(t, path, []byte(strings.Repeat("A", 8<<20))) + + source, err := OpenSQLImportSource(path, SQLImportSourceOptions{}) + if err != nil { + t.Fatalf("open SQL import source: %v", err) + } + defer source.Close() + + _, readErr := io.ReadAll(source) + var limitErr *SQLImportSourceLimitError + if !errors.As(readErr, &limitErr) || limitErr.Kind != SQLImportSourceCompressionRatio { + t.Fatalf("read error = %v, want default compression-ratio protection", readErr) + } +} + +func TestOpenSQLImportSourceReportsDecodedLimitBeforeRatioWhenBothCross(t *testing.T) { + path := filepath.Join(t.TempDir(), "dump.sql.gz") + writeGzipSQL(t, path, []byte(strings.Repeat("A", 4096))) + + source, err := OpenSQLImportSource(path, SQLImportSourceOptions{ + MaxDecodedBytes: 8, + MaxCompressionRatio: 0.01, + MinCompressedBytesForRatio: 1, + }) + if err != nil { + t.Fatalf("open SQL import source: %v", err) + } + defer source.Close() + + _, readErr := io.ReadAll(source) + var limitErr *SQLImportSourceLimitError + if !errors.As(readErr, &limitErr) || limitErr.Kind != SQLImportSourceDecodedByteLimit { + t.Fatalf("read error = %v, want earlier decoded-byte limit", readErr) + } +} + +func writeGzipSQL(t *testing.T, path string, payload []byte) { + t.Helper() + file, err := os.Create(path) + if err != nil { + t.Fatalf("create gzip source: %v", err) + } + writer := gzip.NewWriter(file) + if _, err := writer.Write(payload); err != nil { + _ = file.Close() + t.Fatalf("write gzip source: %v", err) + } + if err := writer.Close(); err != nil { + _ = file.Close() + t.Fatalf("close gzip writer: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("close gzip source: %v", err) + } +} + +func encodeUTF16SQL(value string, order binary.ByteOrder) []byte { + encoded := utf16.Encode([]rune(value)) + result := make([]byte, 2, 2+len(encoded)*2) + if order == binary.LittleEndian { + result[0], result[1] = 0xff, 0xfe + } else { + result[0], result[1] = 0xfe, 0xff + } + for _, codeUnit := range encoded { + result = append(result, 0, 0) + order.PutUint16(result[len(result)-2:], codeUnit) + } + return result +} diff --git a/internal/app/sql_split.go b/internal/app/sql_split.go index 99e5caf1..458ac6b4 100644 --- a/internal/app/sql_split.go +++ b/internal/app/sql_split.go @@ -147,7 +147,7 @@ func splitSQLStatementsForDialect(dbType, sql string) []string { if token == "begin" && plsqlDeclareBeginSkips > 0 { plsqlDeclareBeginSkips-- justClosedPLSQLBlock = false - } else if token == "begin" && shouldEnterPLSQLBlock(text, tokenEnd) { + } else if token == "begin" && shouldEnterPLSQLBlockForDialect(dbType, text, tokenEnd) { plsqlDepth++ justClosedPLSQLBlock = false } else if token == "declare" && shouldEnterPLSQLDeclareBlock(text, tokenEnd) { @@ -494,20 +494,80 @@ func nextSQLSignificantByte(text string, pos int) byte { return text[i] } -func shouldEnterPLSQLBlock(text string, tokenEnd int) bool { - switch nextSQLSignificantByte(text, tokenEnd) { - case 0, ';': - return false - } - switch nextSQLSignificantToken(text, tokenEnd) { - case "transaction", "work", "isolation", "read", "write": +func sqlBeginStartsTransactionForDialect(dbType string, text string, tokenEnd int) bool { + second, secondEnd := nextSQLKeyword(text, tokenEnd) + switch normalizeSQLClassifierDBType(dbType) { + case "sqlserver": + switch second { + case "transaction", "tran": + return true + case "distributed": + third, _ := nextSQLKeyword(text, secondEnd) + return third == "transaction" || third == "tran" + default: + return false + } + case "oracle", "dameng": return false + case "sqlite": + switch second { + case "", "transaction", "deferred", "immediate", "exclusive": + return true + default: + return false + } + case "mysql", "mariadb", "diros", "starrocks", "sphinx", "oceanbase": + return second == "" || second == "work" + case "postgres", "kingbase", "highgo", "vastbase", "opengauss", "gaussdb": + switch second { + case "", "work", "transaction", "isolation", "read", "write", "deferrable": + return true + case "not": + third, _ := nextSQLKeyword(text, secondEnd) + return third == "deferrable" + default: + return false + } + case "duckdb", "iris": + return second == "" || second == "work" || second == "transaction" default: - return true + return false } } +func shouldEnterPLSQLBlockForDialect(dbType string, text string, tokenEnd int) bool { + nextPos := skipSQLTrivia(text, tokenEnd) + if nextPos >= len(text) || text[nextPos] == ';' { + return false + } + if sqlBeginStartsTransactionForDialect(dbType, text, tokenEnd) { + return false + } + + nextToken, _ := nextSQLKeyword(text, tokenEnd) + normalizedType := normalizeSQLClassifierDBType(dbType) + if normalizedType == "sqlserver" && (nextToken == "dialog" || nextToken == "conversation") { + // BEGIN DIALOG CONVERSATION and BEGIN CONVERSATION TIMER are standalone + // Service Broker commands, not BEGIN ... END control-flow blocks. + return false + } + if normalizedType == "" { + // Preserve the historical dialect-neutral behavior for callers that do + // not know their target engine. Ambiguous forms are resolved only on the + // new dialect-aware paths. + switch nextToken { + case "transaction", "work", "isolation", "read", "write": + return false + } + } + return true +} + func isPLSQLBlockStatement(stmt string) bool { + return isPLSQLBlockStatementForDialect("", stmt) +} + +func isPLSQLBlockStatementForDialect(dbType string, stmt string) bool { text := strings.TrimSpace(stmt) if text == "" { return false @@ -515,12 +575,12 @@ func isPLSQLBlockStatement(stmt string) bool { if strings.HasSuffix(text, "/") { text = strings.TrimSpace(strings.TrimSuffix(text, "/")) } - token := nextSQLSignificantToken(text, 0) + token, tokenEnd := nextSQLKeyword(text, 0) if token == "declare" { - return shouldEnterPLSQLDeclareBlock(text, len("declare")) + return shouldEnterPLSQLDeclareBlock(text, tokenEnd) } if token == "begin" { - return shouldEnterPLSQLBlock(text, len("begin")) + return shouldEnterPLSQLBlockForDialect(dbType, text, tokenEnd) } return isCreateRoutineHeaderPrefix(text) } diff --git a/internal/app/sql_split_stream.go b/internal/app/sql_split_stream.go index f0b6caaa..7b2ec67a 100644 --- a/internal/app/sql_split_stream.go +++ b/internal/app/sql_split_stream.go @@ -2,46 +2,178 @@ package app import ( "bufio" + "fmt" "io" + "strconv" "strings" ) +const DefaultSQLImportMaxStatementBytes int64 = 64 << 20 + +type SQLStreamOptions struct { + DBType string + // MaxStatementBytes must be positive to enforce a limit. SQL import + // execution should use DefaultSQLImportMaxStatementBytes unless explicitly + // configured otherwise. + MaxStatementBytes int64 +} + +// SQLStatementTooLargeError identifies the statement and one-based source +// byte at which the configured in-memory statement limit was exceeded. +type SQLStatementTooLargeError struct { + StatementIndex int + SourceByte int64 + MaxBytes int64 +} + +func (err *SQLStatementTooLargeError) Error() string { + return fmt.Sprintf("SQL statement %d exceeded %d bytes at source byte %d", err.StatementIndex, err.MaxBytes, err.SourceByte) +} + +type sqlStatementBuilder struct { + strings.Builder + maxBytes int64 + sourceOffset int64 + statementIndex int + limitErr *SQLStatementTooLargeError +} + +func (builder *sqlStatementBuilder) prepareWrite(sourceOffset int64, statementIndex int) { + builder.sourceOffset = sourceOffset + builder.statementIndex = statementIndex +} + +func (builder *sqlStatementBuilder) WriteByte(value byte) error { + if builder.limitErr != nil { + builder.sourceOffset++ + return nil + } + if builder.maxBytes > 0 && int64(builder.Len()) >= builder.maxBytes { + builder.limitErr = &SQLStatementTooLargeError{ + StatementIndex: builder.statementIndex, + SourceByte: builder.sourceOffset + 1, + MaxBytes: builder.maxBytes, + } + builder.sourceOffset++ + return nil + } + err := builder.Builder.WriteByte(value) + builder.sourceOffset++ + return err +} + +func (builder *sqlStatementBuilder) WriteString(value string) (int, error) { + if builder.limitErr != nil { + builder.sourceOffset += int64(len(value)) + return 0, nil + } + allowed := len(value) + if builder.maxBytes > 0 { + remaining := builder.maxBytes - int64(builder.Len()) + if remaining < int64(allowed) { + if remaining < 0 { + remaining = 0 + } + allowed = int(remaining) + builder.limitErr = &SQLStatementTooLargeError{ + StatementIndex: builder.statementIndex, + SourceByte: builder.sourceOffset + int64(allowed) + 1, + MaxBytes: builder.maxBytes, + } + } + } + written, err := builder.Builder.WriteString(value[:allowed]) + builder.sourceOffset += int64(len(value)) + return written, err +} + +func (builder *sqlStatementBuilder) Reset() { + builder.Builder.Reset() + builder.limitErr = nil +} + // sqlStreamSplitter 是一个流式 SQL 语句拆分器，适用于处理大文件。 // 调用方通过 Feed(chunk) 逐块喂入数据，通过 Flush() 获取最后一条残余语句。 // 内部维护与 splitSQLStatements 完全一致的状态机逻辑。 type sqlStreamSplitter struct { - cur strings.Builder - pending string - inSingle bool - inDouble bool - inBacktick bool - escaped bool - inLineComment bool - inBlockComment bool - dollarTag string - plsqlDepth int - declareSkips int - plsqlCaseDepth int - skipCaseEnd bool - closedPLSQL bool + dbType string + delimiter string + cur sqlStatementBuilder + pending string + inputBytes int64 + statementIndex int + inSingle bool + inDouble bool + inBacktick bool + escaped bool + inLineComment bool + inBlockComment bool + dollarTag string + plsqlDepth int + declareSkips int + plsqlCaseDepth int + skipCaseEnd bool + closedPLSQL bool + sqlServerBatch []string + preserveSQLServerBatch bool + sqlServerGoSeen bool } func (s *sqlStreamSplitter) takeStatement() string { stmt := strings.TrimSpace(s.cur.String()) s.cur.Reset() - if !hasExecutableSQLStatementContent("", stmt) { + if !hasExecutableSQLStatementContent(s.dbType, stmt) { return "" } return stmt } +func (s *sqlStreamSplitter) activeDelimiter() string { + if s.delimiter == "" { + return ";" + } + return s.delimiter +} + +func (s *sqlStreamSplitter) appendCompletedStatement(statements *[]string, stmt string) { + if stmt == "" { + return + } + *statements = append(*statements, stmt) + s.statementIndex++ + if normalizeExplainLexicalDBType(s.dbType) == "sqlserver" { + s.sqlServerBatch = append(s.sqlServerBatch, stmt) + } +} + +func (s *sqlStreamSplitter) finishSQLServerBatch(statements *[]string, repeat int) { + if repeat < 1 { + repeat = 1 + } + batch := append([]string(nil), s.sqlServerBatch...) + for iteration := 1; iteration < repeat; iteration++ { + *statements = append(*statements, batch...) + s.statementIndex += len(batch) + } + s.sqlServerBatch = nil +} + // Feed 将一个 chunk 喂入拆分器，返回在此 chunk 中完成的 SQL 语句列表。 func (s *sqlStreamSplitter) Feed(chunk []byte) []string { var statements []string + textSourceOffset := s.inputBytes + if s.pending != "" { + textSourceOffset -= int64(len(s.pending)) + } + s.inputBytes += int64(len(chunk)) text := s.pending + string(chunk) s.pending = "" for i := 0; i < len(text); i++ { + if s.cur.limitErr != nil { + break + } + s.cur.prepareWrite(textSourceOffset+int64(i), s.statementIndex) ch := text[i] next := byte(0) if i+1 < len(text) { @@ -133,10 +265,55 @@ func (s *sqlStreamSplitter) Feed(chunk []byte) []string { continue } + if isSQLStreamMySQLClientDialect(s.dbType) && s.canConsumeClientDirective() { + delimiter, lineEnd, matched, incomplete := scanSQLStreamDelimiterDirective(text, i, false) + if incomplete { + s.pending = text[i:] + break + } + if matched { + s.cur.Reset() + s.delimiter = delimiter + i = lineEnd - 1 + continue + } + } + + if normalizeExplainLexicalDBType(s.dbType) == "sqlserver" && sqlStreamCurrentLineWhitespaceOnly(&s.cur) { + repeat, lineEnd, matched, incomplete := scanSQLStreamGoDirective(text, i, false) + if incomplete { + s.pending = text[i:] + break + } + if matched { + s.sqlServerGoSeen = true + s.appendCompletedStatement(&statements, s.takeStatement()) + s.finishSQLServerBatch(&statements, repeat) + s.closedPLSQL = false + i = lineEnd - 1 + continue + } + } + + if delimiter := s.activeDelimiter(); delimiter != ";" { + remaining := text[i:] + if strings.HasPrefix(remaining, delimiter) { + stmt := s.takeStatement() + s.appendCompletedStatement(&statements, stmt) + s.closedPLSQL = false + i += len(delimiter) - 1 + continue + } + if len(remaining) < len(delimiter) && strings.HasPrefix(delimiter, remaining) { + s.pending = remaining + break + } + } + if isSQLIdentifierStart(ch) { tokenStart := i tokenEnd := i + 1 - for tokenEnd < len(text) && isSQLIdentifierPart(text[tokenEnd]) { + for tokenEnd < len(text) && isSQLIdentifierPart(text[tokenEnd]) && !s.delimiterStartsAt(text, tokenEnd) { tokenEnd++ } token := strings.ToLower(text[tokenStart:tokenEnd]) @@ -161,7 +338,7 @@ func (s *sqlStreamSplitter) Feed(chunk []byte) []string { if token == "begin" && s.declareSkips > 0 { s.declareSkips-- s.closedPLSQL = false - } else if token == "begin" && shouldEnterPLSQLBlock(text, tokenEnd) { + } else if token == "begin" && shouldEnterPLSQLBlockForDialect(s.dbType, text, tokenEnd) { s.plsqlDepth++ s.closedPLSQL = false } else if token == "declare" && shouldEnterPLSQLDeclareBlock(text, tokenEnd) { @@ -200,12 +377,16 @@ func (s *sqlStreamSplitter) Feed(chunk []byte) []string { s.pending = text[i:] break } - if ch == '-' && next == '-' { + if ch == '-' && next == '-' && isSQLFileMySQLDashCommentDecisionIncomplete(s.dbType, text, i) { + s.pending = text[i:] + break + } + if ch == '-' && next == '-' && isSQLDashLineCommentStart(s.dbType, text, i) { s.inLineComment = true s.cur.WriteByte(ch) continue } - if ch == '#' { + if ch == '#' && supportsSQLHashLineComment(s.dbType) { s.inLineComment = true s.cur.WriteByte(ch) continue @@ -219,9 +400,7 @@ func (s *sqlStreamSplitter) Feed(chunk []byte) []string { break } stmt := s.takeStatement() - if stmt != "" { - statements = append(statements, stmt) - } + s.appendCompletedStatement(&statements, stmt) s.closedPLSQL = false i = lineEnd continue @@ -241,14 +420,14 @@ func (s *sqlStreamSplitter) Feed(chunk []byte) []string { } // Dollar-quoting 开始 - if ch == '$' { - if tag := parseSQLDollarTag(text[i:]); tag != "" { + if ch == '$' && supportsSQLDollarQuote(s.dbType) { + if tag := parseSQLDollarTagAt(text, i); tag != "" { s.dollarTag = tag s.cur.WriteString(tag) i += len(tag) - 1 continue } - if isIncompleteSQLDollarTag(text[i:]) { + if isIncompleteSQLDollarTagAt(text, i) { s.pending = text[i:] break } @@ -256,6 +435,10 @@ func (s *sqlStreamSplitter) Feed(chunk []byte) []string { // 分号分隔 if ch == ';' { + if s.preserveSQLServerBatch { + s.cur.WriteByte(ch) + continue + } if s.plsqlDepth > 0 { s.cur.WriteByte(ch) continue @@ -263,16 +446,12 @@ func (s *sqlStreamSplitter) Feed(chunk []byte) []string { if s.closedPLSQL { s.cur.WriteByte(ch) stmt := s.takeStatement() - if stmt != "" { - statements = append(statements, stmt) - } + s.appendCompletedStatement(&statements, stmt) s.closedPLSQL = false continue } stmt := s.takeStatement() - if stmt != "" { - statements = append(statements, stmt) - } + s.appendCompletedStatement(&statements, stmt) continue } // 全角分号 @@ -281,6 +460,11 @@ func (s *sqlStreamSplitter) Feed(chunk []byte) []string { break } if ch == 0xEF && i+2 < len(text) && text[i+1] == 0xBC && text[i+2] == 0x9B { + if s.preserveSQLServerBatch { + s.cur.WriteString("；") + i += 2 + continue + } if s.plsqlDepth > 0 { s.cur.WriteString("；") i += 2 @@ -289,17 +473,13 @@ func (s *sqlStreamSplitter) Feed(chunk []byte) []string { if s.closedPLSQL { s.cur.WriteString("；") stmt := s.takeStatement() - if stmt != "" { - statements = append(statements, stmt) - } + s.appendCompletedStatement(&statements, stmt) s.closedPLSQL = false i += 2 continue } stmt := s.takeStatement() - if stmt != "" { - statements = append(statements, stmt) - } + s.appendCompletedStatement(&statements, stmt) i += 2 continue } @@ -310,28 +490,180 @@ func (s *sqlStreamSplitter) Feed(chunk []byte) []string { return statements } -// Flush 返回缓冲区中剩余的不完整语句（文件结束时调用）。 -func (s *sqlStreamSplitter) Flush() string { +func (s *sqlStreamSplitter) delimiterStartsAt(text string, start int) bool { + delimiter := s.activeDelimiter() + if delimiter == ";" || start < 0 || start >= len(text) { + return false + } + remaining := text[start:] + return strings.HasPrefix(remaining, delimiter) || (len(remaining) < len(delimiter) && strings.HasPrefix(delimiter, remaining)) +} + +func isSQLFileMySQLDashCommentDecisionIncomplete(dbType, text string, index int) bool { + switch normalizeExplainLexicalDBType(dbType) { + case "mysql", "mariadb", "oceanbase", "diros", "starrocks", "goldendb", "sphinx", "tidb": + return index >= 0 && index+2 >= len(text) + default: + return false + } +} + +func (s *sqlStreamSplitter) flushStatements() []string { + var statements []string if s.pending != "" { + if normalizeExplainLexicalDBType(s.dbType) == "sqlserver" && sqlStreamCurrentLineWhitespaceOnly(&s.cur) { + if repeat, _, matched, _ := scanSQLStreamGoDirective(s.pending, 0, true); matched { + s.pending = "" + s.sqlServerGoSeen = true + s.appendCompletedStatement(&statements, s.takeStatement()) + s.finishSQLServerBatch(&statements, repeat) + s.closedPLSQL = false + return statements + } + } + if isSQLStreamMySQLClientDialect(s.dbType) && s.canConsumeClientDirective() { + if delimiter, _, matched, _ := scanSQLStreamDelimiterDirective(s.pending, 0, true); matched { + s.pending = "" + s.cur.Reset() + s.delimiter = delimiter + return statements + } + } if (s.closedPLSQL || strings.TrimSpace(s.cur.String()) == "") && sqlStreamCurrentLineWhitespaceOnly(&s.cur) { if _, standalone, _ := scanSQLStandaloneSlashLineSuffix(s.pending, 0); standalone { s.pending = "" stmt := s.takeStatement() s.closedPLSQL = false - return stmt + s.appendCompletedStatement(&statements, stmt) + return statements } } + s.cur.prepareWrite(s.inputBytes-int64(len(s.pending)), s.statementIndex) s.cur.WriteString(s.pending) s.pending = "" + if s.cur.limitErr != nil { + return statements + } + } + if s.preserveSQLServerBatch && normalizeExplainLexicalDBType(s.dbType) == "sqlserver" && !s.sqlServerGoSeen { + rawBatch := strings.TrimSpace(s.cur.String()) + s.cur.Reset() + for _, stmt := range splitSQLStatementsForDialect(s.dbType, rawBatch) { + s.appendCompletedStatement(&statements, stmt) + } + return statements } stmt := s.takeStatement() - if stmt == "/" { - return "" + if stmt != "/" { + s.appendCompletedStatement(&statements, stmt) } - return stmt + return statements } -func sqlStreamCurrentLineWhitespaceOnly(builder *strings.Builder) bool { +// Flush 返回缓冲区中剩余的不完整语句（文件结束时调用）。 +// 多语句结果仅用于兼容直接调用；流式入口使用 flushStatements 保留 GO n 的逐条回调语义。 +func (s *sqlStreamSplitter) Flush() string { + return strings.Join(s.flushStatements(), ";\n") +} + +func isSQLStreamMySQLClientDialect(dbType string) bool { + switch normalizeExplainLexicalDBType(dbType) { + case "mysql", "mariadb": + return true + default: + return false + } +} + +func (s *sqlStreamSplitter) canConsumeClientDirective() bool { + return sqlStreamCurrentLineWhitespaceOnly(&s.cur) && !hasExecutableSQLStatementContent(s.dbType, s.cur.String()) +} + +func scanSQLStreamDelimiterDirective(text string, start int, eof bool) (delimiter string, lineEnd int, matched bool, incomplete bool) { + const keyword = "delimiter" + if start < 0 || start >= len(text) { + return "", start, false, false + } + remaining := text[start:] + prefixLength := len(remaining) + if prefixLength > len(keyword) { + prefixLength = len(keyword) + } + if !strings.EqualFold(remaining[:prefixLength], keyword[:prefixLength]) { + return "", start, false, false + } + if len(remaining) < len(keyword) { + return "", start, false, !eof + } + if len(remaining) == len(keyword) { + return "", start, false, !eof + } + if !isSQLHorizontalWhitespace(remaining[len(keyword)]) { + return "", start, false, false + } + + newline := strings.IndexByte(remaining, '\n') + if newline < 0 && !eof { + return "", start, false, true + } + line := remaining + lineEnd = len(text) + if newline >= 0 { + line = remaining[:newline] + lineEnd = start + newline + 1 + } + value := strings.TrimSpace(line[len(keyword):]) + if value == "" || strings.IndexFunc(value, func(r rune) bool { return r == ' ' || r == '\t' || r == '\r' || r == '\n' }) >= 0 { + return "", start, false, false + } + return value, lineEnd, true, false +} + +func scanSQLStreamGoDirective(text string, start int, eof bool) (repeat int, lineEnd int, matched bool, incomplete bool) { + const keyword = "go" + if start < 0 || start >= len(text) { + return 0, start, false, false + } + remaining := text[start:] + prefixLength := len(remaining) + if prefixLength > len(keyword) { + prefixLength = len(keyword) + } + if !strings.EqualFold(remaining[:prefixLength], keyword[:prefixLength]) { + return 0, start, false, false + } + if len(remaining) < len(keyword) { + return 0, start, false, !eof + } + if len(remaining) > len(keyword) && !isSQLHorizontalWhitespace(remaining[len(keyword)]) && remaining[len(keyword)] != '\n' { + return 0, start, false, false + } + + newline := strings.IndexByte(remaining, '\n') + if newline < 0 && !eof { + return 0, start, false, true + } + line := remaining + lineEnd = len(text) + if newline >= 0 { + line = remaining[:newline] + lineEnd = start + newline + 1 + } + remainder := strings.TrimSpace(line[len(keyword):]) + if comment := strings.Index(remainder, "--"); comment >= 0 { + remainder = strings.TrimSpace(remainder[:comment]) + } + if remainder == "" { + return 1, lineEnd, true, false + } + parsed, err := strconv.Atoi(remainder) + if err != nil || parsed <= 0 { + return 0, start, false, false + } + return parsed, lineEnd, true, false +} + +func sqlStreamCurrentLineWhitespaceOnly(builder interface{ String() string }) bool { text := builder.String() for i := len(text) - 1; i >= 0; i-- { if text[i] == '\n' { @@ -360,6 +692,16 @@ func isIncompleteSQLDollarTag(s string) bool { return true } +func isIncompleteSQLDollarTagAt(text string, start int) bool { + if start < 0 || start >= len(text) || text[start] != '$' { + return false + } + if start > 0 && isSQLIdentifierPart(text[start-1]) { + return false + } + return isIncompleteSQLDollarTag(text[start:]) +} + func shouldDeferPLSQLKeywordInStream(text string, tokenStart int, tokenEnd int, token string) bool { switch token { case "begin", "declare", "end", "create", "or", "replace", "editionable", "noneditionable", "procedure", "function", "package", "body", "is", "as": @@ -369,7 +711,7 @@ func shouldDeferPLSQLKeywordInStream(text string, tokenStart int, tokenEnd int, if tokenEnd >= len(text) { return true } - next := skipSQLWhitespaceAndComments(text, tokenEnd) + next := skipSQLTrivia(text, tokenEnd) if next >= len(text) { return true } @@ -378,7 +720,25 @@ func shouldDeferPLSQLKeywordInStream(text string, tokenStart int, tokenEnd int, for nextEnd < len(text) && isSQLIdentifierPart(text[nextEnd]) { nextEnd++ } - return nextEnd >= len(text) + if nextEnd >= len(text) { + return true + } + if token == "begin" { + nextToken := strings.ToLower(text[next:nextEnd]) + if nextToken == "not" || nextToken == "distributed" { + following := skipSQLTrivia(text, nextEnd) + if following >= len(text) { + return true + } + if isSQLIdentifierStart(text[following]) { + followingEnd := following + 1 + for followingEnd < len(text) && isSQLIdentifierPart(text[followingEnd]) { + followingEnd++ + } + return followingEnd >= len(text) + } + } + } } return false } @@ -402,7 +762,17 @@ func shouldDeferPLSQLKeywordPrefixInStream(text string, tokenStart int, tokenEnd // onStatement 返回 error 时停止读取并返回该 error。 // 返回总处理语句数和可能的错误。 func streamSQLFile(reader io.Reader, onStatement func(index int, stmt string) error) (int, error) { - splitter := &sqlStreamSplitter{} + return streamSQLFileForDialect(reader, "", onStatement) +} + +func streamSQLFileForDialect(reader io.Reader, dbType string, onStatement func(index int, stmt string) error) (int, error) { + return streamSQLFileWithOptions(reader, SQLStreamOptions{DBType: dbType}, onStatement) +} + +func streamSQLFileWithOptions(reader io.Reader, options SQLStreamOptions, onStatement func(index int, stmt string) error) (int, error) { + splitter := &sqlStreamSplitter{dbType: normalizeSQLClassifierDBType(options.DBType)} + splitter.preserveSQLServerBatch = normalizeExplainLexicalDBType(splitter.dbType) == "sqlserver" + splitter.cur.maxBytes = options.MaxStatementBytes bufferedReader := bufio.NewReaderSize(reader, 1024*1024) buffer := make([]byte, 1024*1024) @@ -417,6 +787,9 @@ func streamSQLFile(reader io.Reader, onStatement func(index int, stmt string) er } count++ } + if splitter.cur.limitErr != nil { + return count, splitter.cur.limitErr + } } if err == io.EOF { break @@ -429,13 +802,21 @@ func streamSQLFile(reader io.Reader, onStatement func(index int, stmt string) er } } - // 处理文件末尾不以分号结尾的最后一条语句 - if last := splitter.Flush(); last != "" { + // 处理文件末尾不以分号结尾的最后一条语句或 GO n 重复批次。 + for _, last := range splitter.flushStatements() { if err := onStatement(count, last); err != nil { return count, err } count++ } + if splitter.cur.limitErr != nil { + return count, splitter.cur.limitErr + } return count, nil } + +// StreamSQLFileWithOptions streams statements without loading the full source. +func StreamSQLFileWithOptions(reader io.Reader, options SQLStreamOptions, onStatement func(index int, stmt string) error) (int, error) { + return streamSQLFileWithOptions(reader, options, onStatement) +} diff --git a/internal/app/sql_split_stream_test.go b/internal/app/sql_split_stream_test.go index 0209428e..ac1c1aa3 100644 --- a/internal/app/sql_split_stream_test.go +++ b/internal/app/sql_split_stream_test.go @@ -39,3 +39,68 @@ func TestSQLStreamSplitterPreservesExecutableMySQLComment(t *testing.T) { t.Fatalf("expected statements %#v, got %#v", want, got) } } + +func TestSQLStreamSplitterTransactionBeginFormsAcrossChunkBoundaries(t *testing.T) { + tests := []struct { + dbType string + begin string + }{ + {dbType: "sqlserver", begin: "BEGIN TRAN"}, + {dbType: "sqlserver", begin: "BEGIN DISTRIBUTED TRANSACTION"}, + {dbType: "sqlserver", begin: "BEGIN DIALOG CONVERSATION @handle"}, + {dbType: "sqlserver", begin: "BEGIN CONVERSATION TIMER (@handle) TIMEOUT = 30"}, + {dbType: "sqlite", begin: "BEGIN IMMEDIATE"}, + {dbType: "postgres", begin: "BEGIN NOT DEFERRABLE"}, + } + + for _, test := range tests { + t.Run(test.dbType+" "+test.begin, func(t *testing.T) { + input := test.begin + "; UPDATE demo SET value = 2; COMMIT;" + splitter := &sqlStreamSplitter{dbType: test.dbType} + var got []string + for index := range len(input) { + got = append(got, splitter.Feed([]byte(input[index:index+1]))...) + } + if last := splitter.Flush(); last != "" { + got = append(got, last) + } + want := []string{test.begin, "UPDATE demo SET value = 2", "COMMIT"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("one-byte streaming split of %q = %#v, want %#v", input, got, want) + } + }) + } +} + +func TestSQLStreamSplitterKeepsMariaDBNotAtomicBlockAcrossChunkBoundaries(t *testing.T) { + block := "BEGIN NOT ATOMIC\n SET @value = 1;\nEND" + input := block + "; SELECT 1;" + splitter := &sqlStreamSplitter{dbType: "mariadb"} + var got []string + for index := range len(input) { + got = append(got, splitter.Feed([]byte(input[index:index+1]))...) + } + if last := splitter.Flush(); last != "" { + got = append(got, last) + } + want := []string{block + ";", "SELECT 1"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("one-byte streaming split of %q = %#v, want %#v", input, got, want) + } +} + +func TestSQLStreamSplitterUsesDialectForAmbiguousBeginTran(t *testing.T) { + input := "BEGIN\n TRAN;\nEND;\nSELECT 1 FROM dual;" + splitter := &sqlStreamSplitter{dbType: "oracle"} + var got []string + for index := range len(input) { + got = append(got, splitter.Feed([]byte(input[index:index+1]))...) + } + if last := splitter.Flush(); last != "" { + got = append(got, last) + } + want := []string{"BEGIN\n TRAN;\nEND;", "SELECT 1 FROM dual"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("one-byte Oracle BEGIN TRAN procedure block = %#v, want %#v", got, want) + } +} diff --git a/internal/app/sql_split_test.go b/internal/app/sql_split_test.go index d1372222..2a42a3e0 100644 --- a/internal/app/sql_split_test.go +++ b/internal/app/sql_split_test.go @@ -14,6 +14,50 @@ func TestSplitSQLStatements_BasicSplit(t *testing.T) { } } +func TestSplitSQLStatements_TransactionBeginFormsAreNotProceduralBlocks(t *testing.T) { + tests := []struct { + dbType string + begin string + }{ + {dbType: "sqlserver", begin: "BEGIN TRAN"}, + {dbType: "sqlserver", begin: "BEGIN DISTRIBUTED TRANSACTION"}, + {dbType: "sqlserver", begin: "BEGIN DIALOG CONVERSATION @handle"}, + {dbType: "sqlserver", begin: "BEGIN CONVERSATION TIMER (@handle) TIMEOUT = 30"}, + {dbType: "sqlite", begin: "BEGIN IMMEDIATE"}, + {dbType: "postgres", begin: "BEGIN NOT DEFERRABLE"}, + } + + for _, test := range tests { + t.Run(test.dbType+" "+test.begin, func(t *testing.T) { + input := test.begin + "; UPDATE demo SET value = 2; COMMIT;" + got := splitSQLStatementsForDialect(test.dbType, input) + want := []string{test.begin, "UPDATE demo SET value = 2", "COMMIT"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("splitSQLStatementsForDialect(%q, %q) = %#v, want %#v", test.dbType, input, got, want) + } + }) + } +} + +func TestSplitSQLStatements_KeepsMariaDBNotAtomicBlockTogether(t *testing.T) { + block := "BEGIN NOT ATOMIC\n SET @value = 1;\nEND" + input := block + "; SELECT 1;" + got := splitSQLStatementsForDialect("mariadb", input) + want := []string{block + ";", "SELECT 1"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("splitSQLStatements(%q) = %#v, want %#v", input, got, want) + } +} + +func TestSplitSQLStatements_UsesDialectForAmbiguousBeginTran(t *testing.T) { + input := "BEGIN\n TRAN;\nEND;\nSELECT 1 FROM dual;" + got := splitSQLStatementsForDialect("oracle", input) + want := []string{"BEGIN\n TRAN;\nEND;", "SELECT 1 FROM dual"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("Oracle BEGIN TRAN procedure block = %#v, want %#v", got, want) + } +} + func TestSplitSQLStatements_QuotedSemicolon(t *testing.T) { input := `SELECT 'hello;world'; SELECT 2` got := splitSQLStatements(input) diff --git a/internal/app/xlsx_import_stream.go b/internal/app/xlsx_import_stream.go index 17548861..3b463fd6 100644 --- a/internal/app/xlsx_import_stream.go +++ b/internal/app/xlsx_import_stream.go @@ -7,37 +7,71 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path" "strconv" "strings" - - "github.com/xuri/excelize/v2" + "time" ) const ( xlsxWorkbookXMLPath = "xl/workbook.xml" xlsxWorkbookRelsXMLPath = "xl/_rels/workbook.xml.rels" xlsxSharedStringsXML = "xl/sharedStrings.xml" + xlsxStylesXMLPath = "xl/styles.xml" + maxXLSXSharedStrings = 10_000_000 + maxXLSXCellStyles = 65_536 + maxXLSXNumberFormats = 65_536 ) +type xlsxArchiveResourceLimits struct { + MaxEntryUncompressedBytes uint64 + MaxTotalUncompressedBytes uint64 + MaxCompressionRatio uint64 +} + +var defaultXLSXArchiveResourceLimits = xlsxArchiveResourceLimits{ + MaxEntryUncompressedBytes: 16 << 30, + MaxTotalUncompressedBytes: 64 << 30, + MaxCompressionRatio: 1000, +} + type xlsxSharedStringStore struct { - path string - file *os.File - writer *bufio.Writer - offsets []int64 - size int64 + path string + file *os.File + writer *bufio.Writer + offsets []int64 + size int64 + maxCount int + maxValueBytes int +} + +type xlsxSharedStringResolver struct { + reader io.ReadCloser + decoder *xml.Decoder + store *xlsxSharedStringStore + exhausted bool } func newXLSXSharedStringStore() (*xlsxSharedStringStore, error) { + return newXLSXSharedStringStoreWithLimits(maxXLSXSharedStrings, maxImportCellBytes) +} + +func newXLSXSharedStringStoreWithLimits(maxCount int, maxValueBytes int) (*xlsxSharedStringStore, error) { + if maxCount <= 0 || maxValueBytes <= 0 { + return nil, fmt.Errorf("invalid shared string limits") + } file, err := os.CreateTemp("", "gonavi-xlsx-shared-strings-*.bin") if err != nil { return nil, err } return &xlsxSharedStringStore{ - path: file.Name(), - file: file, - writer: bufio.NewWriterSize(file, 1024*256), + path: file.Name(), + file: file, + writer: bufio.NewWriterSize(file, 1024*256), + maxCount: maxCount, + maxValueBytes: maxValueBytes, }, nil } @@ -45,6 +79,12 @@ func (s *xlsxSharedStringStore) Add(value string) error { if s == nil || s.file == nil || s.writer == nil { return fmt.Errorf("shared string store unavailable") } + if len(s.offsets) >= s.maxCount { + return fmt.Errorf("shared string count exceeds %d limit", s.maxCount) + } + if len(value) > s.maxValueBytes { + return fmt.Errorf("shared string exceeds %d-byte cell limit", s.maxValueBytes) + } s.offsets = append(s.offsets, s.size) if err := binary.Write(s.writer, binary.LittleEndian, uint32(len(value))); err != nil { return err @@ -71,6 +111,9 @@ func (s *xlsxSharedStringStore) Get(index int) (string, error) { if err := binary.Read(s.file, binary.LittleEndian, &length); err != nil { return "", err } + if uint64(length) > uint64(s.maxValueBytes) { + return "", fmt.Errorf("shared string exceeds %d-byte cell limit", s.maxValueBytes) + } buf := make([]byte, int(length)) if _, err := io.ReadFull(s.file, buf); err != nil { return "", err @@ -102,19 +145,141 @@ func (s *xlsxSharedStringStore) Close() error { return err } +func (r *xlsxSharedStringResolver) Get(index int) (string, error) { + if r == nil { + return "", nil + } + if index < 0 { + return "", fmt.Errorf("shared string index out of range: %d", index) + } + for len(r.store.offsets) <= index && !r.exhausted { + found, err := r.parseNext() + if err != nil { + return "", err + } + if !found { + break + } + } + return r.store.Get(index) +} + +func (r *xlsxSharedStringResolver) parseNext() (bool, error) { + for { + token, err := r.decoder.Token() + if err != nil { + if err == io.EOF { + r.exhausted = true + return false, nil + } + return false, err + } + start, ok := token.(xml.StartElement) + if !ok || start.Name.Local != "si" { + continue + } + value, err := readXLSXSharedStringItem(r.decoder) + if err != nil { + return false, err + } + if err := r.store.Add(value); err != nil { + return false, err + } + return true, nil + } +} + +func (r *xlsxSharedStringResolver) Close() error { + if r == nil { + return nil + } + var firstErr error + if r.reader != nil { + firstErr = r.reader.Close() + } + if r.store != nil { + if err := r.store.Close(); firstErr == nil { + firstErr = err + } + } + return firstErr +} + func streamXLSXImportFile(filePath string, consumer importFileConsumer) error { + return streamXLSXImportFileWithOptions(filePath, consumer, ImportFileOptions{}) +} + +func streamXLSXImportFileWithOptions(filePath string, consumer importFileConsumer, options ImportFileOptions) error { + return streamXLSXImportFileWithOptionsAndLimits(filePath, consumer, options, defaultXLSXArchiveResourceLimits) +} + +func streamXLSXImportFileWithLimits(filePath string, consumer importFileConsumer, limits xlsxArchiveResourceLimits) error { + return streamXLSXImportFileWithOptionsAndLimits(filePath, consumer, ImportFileOptions{}, limits) +} + +func streamXLSXImportFileWithOptionsAndLimits( + filePath string, + consumer importFileConsumer, + options ImportFileOptions, + limits xlsxArchiveResourceLimits, +) error { + if consumer == nil { + return fmt.Errorf("import file consumer is required") + } + if err := validateImportFileOptions(options); err != nil { + return err + } reader, err := zip.OpenReader(filePath) if err != nil { return fmt.Errorf("Excel Parse Error: %w", err) } defer reader.Close() - + var totalUncompressedBytes uint64 + for _, entry := range reader.File { + if limits.MaxEntryUncompressedBytes > 0 && entry.UncompressedSize64 > limits.MaxEntryUncompressedBytes { + return fmt.Errorf( + "Excel Parse Error: entry %q uncompressed size %d exceeds %d-byte limit", + entry.Name, + entry.UncompressedSize64, + limits.MaxEntryUncompressedBytes, + ) + } + if limits.MaxCompressionRatio > 0 && entry.UncompressedSize64 > 0 { + compressedBytes := entry.CompressedSize64 + ratioExceeded := compressedBytes == 0 + if compressedBytes > 0 { + quotient := entry.UncompressedSize64 / compressedBytes + ratioExceeded = quotient > limits.MaxCompressionRatio || + (quotient == limits.MaxCompressionRatio && entry.UncompressedSize64%compressedBytes > 0) + } + if ratioExceeded { + return fmt.Errorf( + "Excel Parse Error: entry %q compression ratio exceeds %d:1 limit", + entry.Name, + limits.MaxCompressionRatio, + ) + } + } + if limits.MaxTotalUncompressedBytes > 0 && + (totalUncompressedBytes > limits.MaxTotalUncompressedBytes || + entry.UncompressedSize64 > limits.MaxTotalUncompressedBytes-totalUncompressedBytes) { + return fmt.Errorf( + "Excel Parse Error: total uncompressed size exceeds %d-byte limit", + limits.MaxTotalUncompressedBytes, + ) + } + totalUncompressedBytes += entry.UncompressedSize64 + } entryByPath := make(map[string]*zip.File, len(reader.File)) for _, entry := range reader.File { entryByPath[entry.Name] = entry } - sheetPath, err := resolveXLSXFirstSheetPath(entryByPath) + sheetPath, err := resolveXLSXSheetPath(entryByPath, options.SheetName) + if err != nil { + return fmt.Errorf("Excel Parse Error: %w", err) + } + date1904, err := readXLSXWorkbookDate1904(entryByPath[xlsxWorkbookXMLPath]) if err != nil { return fmt.Errorf("Excel Parse Error: %w", err) } @@ -126,69 +291,71 @@ func streamXLSXImportFile(filePath string, consumer importFileConsumer) error { if sharedStrings != nil { defer sharedStrings.Close() } + styles, err := loadXLSXStyles(entryByPath[xlsxStylesXMLPath]) + if err != nil { + return fmt.Errorf("Excel Parse Error: %w", err) + } sheetEntry := entryByPath[sheetPath] if sheetEntry == nil { return fmt.Errorf("Excel Parse Error: worksheet not found: %s", sheetPath) } - if err := streamXLSXSheetRows(sheetEntry, sharedStrings, consumer); err != nil { - return fmt.Errorf("Excel Read Error: %w", err) - } - return nil -} - -func streamLegacyExcelImportFile(filePath string, consumer importFileConsumer) error { - workbook, err := excelize.OpenFile(filePath) - if err != nil { - return fmt.Errorf("Excel Parse Error: %w", err) - } - defer workbook.Close() - - sheetName := workbook.GetSheetName(0) - if sheetName == "" { - return fmt.Errorf("Excel file has no sheets") - } - - rows, err := workbook.Rows(sheetName) - if err != nil { - return fmt.Errorf("Excel Read Error: %w", err) - } - defer rows.Close() - - if !rows.Next() { - if err := rows.Error(); err != nil { - return fmt.Errorf("Excel Read Error: %w", err) - } - return fmt.Errorf("Excel empty or missing header") - } - header, err := rows.Columns() - if err != nil { - return fmt.Errorf("Excel Read Error: %w", err) - } - columns := cloneImportColumns(header) - if !hasImportUsableColumns(columns) { - return fmt.Errorf("Excel empty or missing header") - } - if err := consumer.SetColumns(columns); err != nil { - return err - } - - for rows.Next() { - record, err := rows.Columns() - if err != nil { - return fmt.Errorf("Excel Read Error: %w", err) - } - if err := consumer.ConsumeRow(buildImportRowFromValues(columns, record)); err != nil { - return err - } - } - if err := rows.Error(); err != nil { + if err := streamXLSXSheetRowsWithOptionsAndStyles(sheetEntry, sharedStrings, styles, date1904, consumer, options); err != nil { return fmt.Errorf("Excel Read Error: %w", err) } return nil } func resolveXLSXFirstSheetPath(entryByPath map[string]*zip.File) (string, error) { + return resolveXLSXSheetPath(entryByPath, "") +} + +func readXLSXWorkbookDate1904(entry *zip.File) (bool, error) { + if entry == nil { + return false, fmt.Errorf("workbook.xml missing") + } + reader, err := entry.Open() + if err != nil { + return false, err + } + defer reader.Close() + + decoder := xml.NewDecoder(reader) + for { + token, err := decoder.Token() + if err != nil { + if err == io.EOF { + return false, nil + } + return false, err + } + start, ok := token.(xml.StartElement) + if !ok || start.Name.Local != "workbookPr" { + continue + } + for _, attr := range start.Attr { + if attr.Name.Local != "date1904" { + continue + } + switch strings.ToLower(strings.TrimSpace(attr.Value)) { + case "", "0", "false": + return false, nil + case "1", "true": + return true, nil + default: + return false, fmt.Errorf("invalid workbook date1904 value %q", attr.Value) + } + } + return false, nil + } +} + +type xlsxSheetReference struct { + name string + relID string +} + +func resolveXLSXSheetPath(entryByPath map[string]*zip.File, sheetName string) (string, error) { workbookEntry := entryByPath[xlsxWorkbookXMLPath] if workbookEntry == nil { return "", fmt.Errorf("workbook.xml missing") @@ -199,13 +366,16 @@ func resolveXLSXFirstSheetPath(entryByPath map[string]*zip.File) (string, error) } defer workbookReader.Close() - firstSheetRelID, err := readXLSXFirstSheetRelID(workbookReader) + selectedSheet, found, err := readXLSXSheetReference(workbookReader, sheetName) if err != nil { return "", err } - if firstSheetRelID == "" { + if !found && sheetName == "" { return "", fmt.Errorf("workbook has no sheets") } + if !found { + return "", fmt.Errorf("worksheet %q not found", sheetName) + } relsEntry := entryByPath[xlsxWorkbookRelsXMLPath] if relsEntry == nil { @@ -217,12 +387,12 @@ func resolveXLSXFirstSheetPath(entryByPath map[string]*zip.File) (string, error) } defer relsReader.Close() - target, err := readXLSXWorkbookRelTarget(relsReader, firstSheetRelID) + target, err := readXLSXWorkbookRelTarget(relsReader, selectedSheet.relID) if err != nil { return "", err } if target == "" { - return "", fmt.Errorf("worksheet target missing for relationship %s", firstSheetRelID) + return "", fmt.Errorf("worksheet target missing for relationship %s", selectedSheet.relID) } target = strings.TrimPrefix(strings.TrimSpace(target), "/") if strings.HasPrefix(target, "xl/") { @@ -232,24 +402,42 @@ func resolveXLSXFirstSheetPath(entryByPath map[string]*zip.File) (string, error) } func readXLSXFirstSheetRelID(reader io.Reader) (string, error) { + sheet, found, err := readXLSXSheetReference(reader, "") + if err != nil || !found { + return "", err + } + return sheet.relID, nil +} + +func readXLSXSheetReference(reader io.Reader, sheetName string) (xlsxSheetReference, bool, error) { decoder := xml.NewDecoder(reader) for { token, err := decoder.Token() if err != nil { if err == io.EOF { - return "", nil + return xlsxSheetReference{}, false, nil } - return "", err + return xlsxSheetReference{}, false, err } start, ok := token.(xml.StartElement) if !ok || start.Name.Local != "sheet" { continue } + var sheet xlsxSheetReference for _, attr := range start.Attr { - if attr.Name.Local == "id" { - return strings.TrimSpace(attr.Value), nil + switch attr.Name.Local { + case "name": + sheet.name = attr.Value + case "id": + sheet.relID = strings.TrimSpace(attr.Value) } } + if sheet.relID == "" { + return xlsxSheetReference{}, false, fmt.Errorf("worksheet relationship id missing") + } + if sheetName == "" || sheet.name == sheetName { + return sheet, true, nil + } } } @@ -283,7 +471,52 @@ func readXLSXWorkbookRelTarget(reader io.Reader, relID string) (string, error) { } } -func loadXLSXSharedStrings(entry *zip.File) (*xlsxSharedStringStore, error) { +func loadXLSXSharedStrings(entry *zip.File) (*xlsxSharedStringResolver, error) { + if entry == nil { + return nil, nil + } + reader, err := entry.Open() + if err != nil { + return nil, err + } + + store, err := newXLSXSharedStringStore() + if err != nil { + _ = reader.Close() + return nil, err + } + return &xlsxSharedStringResolver{ + reader: reader, + decoder: xml.NewDecoder(reader), + store: store, + }, nil +} + +type xlsxTemporalKind uint8 + +const ( + xlsxTemporalNone xlsxTemporalKind = iota + xlsxTemporalDate + xlsxTemporalTime + xlsxTemporalDateTime + xlsxTemporalElapsedTime +) + +type xlsxStyleTable struct { + temporalKinds []xlsxTemporalKind +} + +func (s *xlsxStyleTable) temporalKind(styleIndex int) (xlsxTemporalKind, error) { + if s == nil { + return xlsxTemporalNone, nil + } + if styleIndex < 0 || styleIndex >= len(s.temporalKinds) { + return xlsxTemporalNone, fmt.Errorf("cell style index out of range: %d", styleIndex) + } + return s.temporalKinds[styleIndex], nil +} + +func loadXLSXStyles(entry *zip.File) (*xlsxStyleTable, error) { if entry == nil { return nil, nil } @@ -293,43 +526,176 @@ func loadXLSXSharedStrings(entry *zip.File) (*xlsxSharedStringStore, error) { } defer reader.Close() - store, err := newXLSXSharedStringStore() - if err != nil { - return nil, err - } - if err := parseXLSXSharedStrings(reader, store); err != nil { - _ = store.Close() - return nil, err - } - if err := store.flush(); err != nil { - _ = store.Close() - return nil, err - } - return store, nil -} - -func parseXLSXSharedStrings(reader io.Reader, store *xlsxSharedStringStore) error { decoder := xml.NewDecoder(reader) + customFormats := make(map[int]string) + styleFormatIDs := make([]int, 0, 32) + inCellXFs := false + cellXFsDepth := 0 for { token, err := decoder.Token() if err != nil { if err == io.EOF { - return nil + break } - return err + return nil, err } - start, ok := token.(xml.StartElement) - if !ok || start.Name.Local != "si" { - continue - } - value, err := readXLSXSharedStringItem(decoder) - if err != nil { - return err - } - if err := store.Add(value); err != nil { - return err + switch typed := token.(type) { + case xml.StartElement: + if typed.Name.Local == "numFmt" { + numFmtID, formatCode, err := readXLSXNumberFormatAttributes(typed) + if err != nil { + return nil, err + } + if numFmtID >= 0 { + if _, exists := customFormats[numFmtID]; !exists && len(customFormats) >= maxXLSXNumberFormats { + return nil, fmt.Errorf("number format count exceeds %d limit", maxXLSXNumberFormats) + } + customFormats[numFmtID] = formatCode + } + } + if !inCellXFs && typed.Name.Local == "cellXfs" { + inCellXFs = true + cellXFsDepth = 1 + continue + } + if inCellXFs { + if cellXFsDepth == 1 && typed.Name.Local == "xf" { + if len(styleFormatIDs) >= maxXLSXCellStyles { + return nil, fmt.Errorf("cell style count exceeds %d limit", maxXLSXCellStyles) + } + numFmtID, err := readXLSXCellFormatID(typed) + if err != nil { + return nil, err + } + styleFormatIDs = append(styleFormatIDs, numFmtID) + } + cellXFsDepth++ + } + case xml.EndElement: + if !inCellXFs { + continue + } + cellXFsDepth-- + if cellXFsDepth == 0 { + inCellXFs = false + } } } + + table := &xlsxStyleTable{temporalKinds: make([]xlsxTemporalKind, len(styleFormatIDs))} + for index, numFmtID := range styleFormatIDs { + table.temporalKinds[index] = classifyXLSXTemporalFormat(numFmtID, customFormats[numFmtID]) + } + return table, nil +} + +func readXLSXNumberFormatAttributes(start xml.StartElement) (int, string, error) { + numFmtID := -1 + formatCode := "" + for _, attr := range start.Attr { + switch attr.Name.Local { + case "numFmtId": + value, err := strconv.Atoi(strings.TrimSpace(attr.Value)) + if err != nil || value < 0 { + return -1, "", fmt.Errorf("invalid number format id %q", attr.Value) + } + numFmtID = value + case "formatCode": + formatCode = attr.Value + } + } + if numFmtID < 0 { + return -1, "", fmt.Errorf("number format id missing") + } + return numFmtID, formatCode, nil +} + +func readXLSXCellFormatID(start xml.StartElement) (int, error) { + for _, attr := range start.Attr { + if attr.Name.Local != "numFmtId" { + continue + } + value, err := strconv.Atoi(strings.TrimSpace(attr.Value)) + if err != nil || value < 0 { + return 0, fmt.Errorf("invalid cell number format id %q", attr.Value) + } + return value, nil + } + return 0, nil +} + +func classifyXLSXTemporalFormat(numFmtID int, customCode string) xlsxTemporalKind { + switch numFmtID { + case 14, 15, 16, 17, 27, 28, 29, 30, 31, 34, 35, 36, 50, 51, 52, 53, 54, 55, 56, 57, 58: + return xlsxTemporalDate + case 18, 19, 20, 21, 32, 33, 45, 47: + return xlsxTemporalTime + case 22: + return xlsxTemporalDateTime + case 46: + return xlsxTemporalElapsedTime + } + return classifyXLSXCustomTemporalFormat(customCode) +} + +func classifyXLSXCustomTemporalFormat(formatCode string) xlsxTemporalKind { + normalized, elapsed := normalizeXLSXNumberFormatCode(formatCode) + hasDate := strings.ContainsAny(normalized, "yd") + hasTime := strings.ContainsAny(normalized, "hs") + if elapsed { + return xlsxTemporalElapsedTime + } + switch { + case hasDate && hasTime: + return xlsxTemporalDateTime + case hasDate: + return xlsxTemporalDate + case hasTime: + return xlsxTemporalTime + default: + return xlsxTemporalNone + } +} + +func normalizeXLSXNumberFormatCode(formatCode string) (string, bool) { + var builder strings.Builder + inQuote := false + elapsed := false + for index := 0; index < len(formatCode); index++ { + ch := formatCode[index] + if ch == '"' { + inQuote = !inQuote + continue + } + if inQuote { + continue + } + switch ch { + case '\\', '_', '*': + if index+1 < len(formatCode) { + index++ + } + continue + case '[': + end := strings.IndexByte(formatCode[index+1:], ']') + if end < 0 { + continue + } + end += index + 1 + content := strings.ToLower(strings.TrimSpace(formatCode[index+1 : end])) + if content == "h" || content == "hh" || content == "m" || content == "mm" || content == "s" || content == "ss" { + elapsed = true + builder.WriteString(content) + } + index = end + continue + } + if ch >= 'A' && ch <= 'Z' { + ch += 'a' - 'A' + } + builder.WriteByte(ch) + } + return builder.String(), elapsed } func readXLSXSharedStringItem(decoder *xml.Decoder) (string, error) { @@ -347,7 +713,8 @@ func readXLSXSharedStringItem(decoder *xml.Decoder) (string, error) { continue } if typed.Name.Local == "t" { - text, err := readXMLTextNode(decoder, typed.Name.Local) + remaining := maxImportCellBytes - builder.Len() + text, err := readXMLTextNodeLimited(decoder, typed.Name.Local, remaining, "shared string") if err != nil { return "", err } @@ -362,7 +729,27 @@ func readXLSXSharedStringItem(decoder *xml.Decoder) (string, error) { return builder.String(), nil } -func streamXLSXSheetRows(entry *zip.File, sharedStrings *xlsxSharedStringStore, consumer importFileConsumer) error { +func streamXLSXSheetRows(entry *zip.File, sharedStrings *xlsxSharedStringResolver, consumer importFileConsumer) error { + return streamXLSXSheetRowsWithOptions(entry, sharedStrings, consumer, ImportFileOptions{}) +} + +func streamXLSXSheetRowsWithOptions( + entry *zip.File, + sharedStrings *xlsxSharedStringResolver, + consumer importFileConsumer, + options ImportFileOptions, +) error { + return streamXLSXSheetRowsWithOptionsAndStyles(entry, sharedStrings, nil, false, consumer, options) +} + +func streamXLSXSheetRowsWithOptionsAndStyles( + entry *zip.File, + sharedStrings *xlsxSharedStringResolver, + styles *xlsxStyleTable, + date1904 bool, + consumer importFileConsumer, + options ImportFileOptions, +) error { reader, err := entry.Open() if err != nil { return err @@ -371,6 +758,11 @@ func streamXLSXSheetRows(entry *zip.File, sharedStrings *xlsxSharedStringStore, decoder := xml.NewDecoder(reader) var columns []string + headerRow, err := resolveImportHeaderRow(options.HeaderRow) + if err != nil { + return err + } + rowNumber := 0 for { token, err := decoder.Token() if err != nil { @@ -383,31 +775,80 @@ func streamXLSXSheetRows(entry *zip.File, sharedStrings *xlsxSharedStringStore, if !ok || start.Name.Local != "row" { continue } - values, err := readXLSXRow(decoder, sharedStrings) + rowNumber = resolveXLSXRowNumber(start, rowNumber+1) + if rowNumber < headerRow { + if err := decoder.Skip(); err != nil { + return err + } + continue + } + if columns == nil && rowNumber > headerRow { + return fmt.Errorf("Excel header row %d is missing", headerRow) + } + values, err := readXLSXRowWithStyles(decoder, sharedStrings, styles, date1904) if err != nil { return err } + if err := validateImportStringCells("Excel", rowNumber, values); err != nil { + return err + } if columns == nil { columns = cloneImportColumns(values) if !hasImportUsableColumns(columns) { return fmt.Errorf("Excel empty or missing header") } + if err := validateImportUniqueColumns("Excel", columns); err != nil { + return err + } if err := consumer.SetColumns(columns); err != nil { return err } continue } - if err := consumer.ConsumeRow(buildImportRowFromValues(columns, values)); err != nil { + if len(values) > len(columns) { + return fmt.Errorf( + "Excel row %d has %d columns, wider than the %d-column header", + rowNumber, + len(values), + len(columns), + ) + } + if err := consumer.ConsumeRow(buildImportRowFromValuesWithOptions(columns, values, options)); err != nil { return err } } if columns == nil { - return fmt.Errorf("Excel empty or missing header") + return fmt.Errorf("Excel header row %d is missing", headerRow) } return nil } -func readXLSXRow(decoder *xml.Decoder, sharedStrings *xlsxSharedStringStore) ([]string, error) { +const xlsxMaxRows = 1_048_576 + +func resolveXLSXRowNumber(start xml.StartElement, fallback int) int { + for _, attr := range start.Attr { + if attr.Name.Local != "r" { + continue + } + rowNumber, err := strconv.Atoi(strings.TrimSpace(attr.Value)) + if err == nil && rowNumber >= fallback && rowNumber <= xlsxMaxRows { + return rowNumber + } + break + } + return fallback +} + +func readXLSXRow(decoder *xml.Decoder, sharedStrings *xlsxSharedStringResolver) ([]string, error) { + return readXLSXRowWithStyles(decoder, sharedStrings, nil, false) +} + +func readXLSXRowWithStyles( + decoder *xml.Decoder, + sharedStrings *xlsxSharedStringResolver, + styles *xlsxStyleTable, + date1904 bool, +) ([]string, error) { values := make([]string, 0, 16) currentColumn := 0 for { @@ -422,6 +863,7 @@ func readXLSXRow(decoder *xml.Decoder, sharedStrings *xlsxSharedStringStore) ([] } columnIndex := currentColumn + 1 cellType := "" + styleIndex := 0 for _, attr := range typed.Attr { switch attr.Name.Local { case "r": @@ -430,12 +872,22 @@ func readXLSXRow(decoder *xml.Decoder, sharedStrings *xlsxSharedStringStore) ([] } case "t": cellType = strings.TrimSpace(attr.Value) + case "s": + parsed, err := strconv.Atoi(strings.TrimSpace(attr.Value)) + if err != nil || parsed < 0 { + return nil, fmt.Errorf("invalid cell style index %q", attr.Value) + } + styleIndex = parsed } } if columnIndex <= 0 { columnIndex = currentColumn + 1 } - cellValue, err := readXLSXCell(decoder, cellType, sharedStrings) + temporalKind, err := styles.temporalKind(styleIndex) + if err != nil { + return nil, err + } + cellValue, err := readXLSXCellWithTemporalStyle(decoder, cellType, temporalKind, date1904, sharedStrings) if err != nil { return nil, err } @@ -452,7 +904,17 @@ func readXLSXRow(decoder *xml.Decoder, sharedStrings *xlsxSharedStringStore) ([] } } -func readXLSXCell(decoder *xml.Decoder, cellType string, sharedStrings *xlsxSharedStringStore) (string, error) { +func readXLSXCell(decoder *xml.Decoder, cellType string, sharedStrings *xlsxSharedStringResolver) (string, error) { + return readXLSXCellWithTemporalStyle(decoder, cellType, xlsxTemporalNone, false, sharedStrings) +} + +func readXLSXCellWithTemporalStyle( + decoder *xml.Decoder, + cellType string, + temporalKind xlsxTemporalKind, + date1904 bool, + sharedStrings *xlsxSharedStringResolver, +) (string, error) { var rawValue strings.Builder var inlineValue strings.Builder for { @@ -464,13 +926,13 @@ func readXLSXCell(decoder *xml.Decoder, cellType string, sharedStrings *xlsxShar case xml.StartElement: switch typed.Name.Local { case "v": - text, err := readXMLTextNode(decoder, typed.Name.Local) + text, err := readXMLTextNodeLimited(decoder, typed.Name.Local, maxImportCellBytes-rawValue.Len(), "cell") if err != nil { return "", err } rawValue.WriteString(text) case "t": - text, err := readXMLTextNode(decoder, typed.Name.Local) + text, err := readXMLTextNodeLimited(decoder, typed.Name.Local, maxImportCellBytes-inlineValue.Len(), "cell") if err != nil { return "", err } @@ -497,13 +959,102 @@ func readXLSXCell(decoder *xml.Decoder, cellType string, sharedStrings *xlsxShar if inlineValue.Len() > 0 { return inlineValue.String(), nil } - return rawValue.String(), nil + value := rawValue.String() + if temporalKind == xlsxTemporalNone || (cellType != "" && cellType != "n") || strings.TrimSpace(value) == "" { + return value, nil + } + return formatXLSXTemporalSerial(value, temporalKind, date1904) } } } } -func readXMLTextNode(decoder *xml.Decoder, endLocal string) (string, error) { +func formatXLSXTemporalSerial(raw string, kind xlsxTemporalKind, date1904 bool) (string, error) { + serial, err := strconv.ParseFloat(strings.TrimSpace(raw), 64) + if err != nil || math.IsNaN(serial) || math.IsInf(serial, 0) { + return "", fmt.Errorf("invalid Excel date/time serial %q", raw) + } + if kind == xlsxTemporalElapsedTime { + return formatXLSXElapsedTime(serial) + } + + wholeDays := math.Floor(serial) + if wholeDays < -3_000_000 || wholeDays > 3_000_000 { + return "", fmt.Errorf("Excel date/time serial out of range: %q", raw) + } + fraction := serial - wholeDays + nanos := roundXLSXTemporalNanos(fraction * float64(24*time.Hour)) + if nanos >= int64(24*time.Hour) { + wholeDays++ + nanos -= int64(24 * time.Hour) + } + + clock := formatXLSXClockTime(nanos) + if kind == xlsxTemporalTime { + return clock, nil + } + + dateText := "" + if !date1904 && wholeDays == 60 { + // Excel's 1900 date system intentionally preserves Lotus 1-2-3's + // fictitious leap day. Keep the workbook-visible value stable even + // though time.Time cannot represent this date. + dateText = "1900-02-29" + } else { + base := time.Date(1899, time.December, 31, 0, 0, 0, 0, time.UTC) + adjustedDays := wholeDays + if date1904 { + base = time.Date(1904, time.January, 1, 0, 0, 0, 0, time.UTC) + } else if adjustedDays > 60 { + adjustedDays-- + } + date := base.AddDate(0, 0, int(adjustedDays)) + dateText = date.Format("2006-01-02") + } + if kind == xlsxTemporalDate { + return dateText, nil + } + return dateText + " " + clock, nil +} + +func formatXLSXClockTime(nanos int64) string { + if nanos < 0 { + nanos = 0 + } + hours := nanos / int64(time.Hour) + nanos %= int64(time.Hour) + minutes := nanos / int64(time.Minute) + nanos %= int64(time.Minute) + seconds := nanos / int64(time.Second) + nanos %= int64(time.Second) + formatted := fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) + if nanos == 0 { + return formatted + } + return formatted + "." + strings.TrimRight(fmt.Sprintf("%09d", nanos), "0") +} + +func formatXLSXElapsedTime(serial float64) (string, error) { + negative := serial < 0 + if negative { + serial = -serial + } + if serial > float64(math.MaxInt64)/float64(24*time.Hour) { + return "", fmt.Errorf("Excel elapsed time serial out of range") + } + totalNanos := roundXLSXTemporalNanos(serial * float64(24*time.Hour)) + formatted := formatXLSXClockTime(totalNanos) + if negative { + return "-" + formatted, nil + } + return formatted, nil +} + +func roundXLSXTemporalNanos(value float64) int64 { + return int64(math.Round(value/float64(time.Microsecond))) * int64(time.Microsecond) +} + +func readXMLTextNodeLimited(decoder *xml.Decoder, endLocal string, maxBytes int, label string) (string, error) { var builder strings.Builder for { token, err := decoder.Token() @@ -512,6 +1063,9 @@ func readXMLTextNode(decoder *xml.Decoder, endLocal string) (string, error) { } switch typed := token.(type) { case xml.CharData: + if len(typed) > maxBytes-builder.Len() { + return "", fmt.Errorf("%s exceeds %d-byte limit", label, maxBytes) + } builder.Write([]byte(typed)) case xml.EndElement: if typed.Name.Local == endLocal { diff --git a/internal/app/xlsx_import_stream_test.go b/internal/app/xlsx_import_stream_test.go index 218a6ca4..81e07753 100644 --- a/internal/app/xlsx_import_stream_test.go +++ b/internal/app/xlsx_import_stream_test.go @@ -1,6 +1,407 @@ package app -import "testing" +import ( + "archive/zip" + "os" + "path/filepath" + "reflect" + "strconv" + "strings" + "testing" +) + +type xlsxTestEntry struct { + body string + method uint16 +} + +func writeXLSXTestArchive(t *testing.T, entries map[string]xlsxTestEntry) string { + t.Helper() + path := filepath.Join(t.TempDir(), "fixture.xlsx") + file, err := os.Create(path) + if err != nil { + t.Fatalf("create xlsx: %v", err) + } + writer := zip.NewWriter(file) + for name, entry := range entries { + header := &zip.FileHeader{Name: name, Method: entry.method} + part, err := writer.CreateHeader(header) + if err != nil { + t.Fatalf("create xlsx entry %s: %v", name, err) + } + if _, err := part.Write([]byte(entry.body)); err != nil { + t.Fatalf("write xlsx entry %s: %v", name, err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("close xlsx zip: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("close xlsx file: %v", err) + } + return path +} + +func minimalXLSXTestEntries(sharedStrings string, sheet string) map[string]xlsxTestEntry { + return map[string]xlsxTestEntry{ + xlsxWorkbookXMLPath: { + body: ``, + method: zip.Store, + }, + xlsxWorkbookRelsXMLPath: { + body: ``, + method: zip.Store, + }, + "xl/worksheets/sheet1.xml": {body: sheet, method: zip.Store}, + xlsxSharedStringsXML: {body: sharedStrings, method: zip.Store}, + } +} + +func TestXLSXImportRejectsOversizedSharedString(t *testing.T) { + tempRoot := t.TempDir() + t.Setenv("TMP", tempRoot) + t.Setenv("TEMP", tempRoot) + t.Setenv("TMPDIR", tempRoot) + sharedStrings := `payload` + strings.Repeat("x", maxImportCellBytes+1) + `` + sheet := `01` + path := writeXLSXTestArchive(t, minimalXLSXTestEntries(sharedStrings, sheet)) + + _, err := buildImportPreview(path, 5) + if err == nil { + t.Fatal("oversized shared string must be rejected") + } + if !strings.Contains(err.Error(), "shared string") || !strings.Contains(err.Error(), "exceeds") { + t.Fatalf("unexpected shared-string limit error: %v", err) + } + entries, err := os.ReadDir(tempRoot) + if err != nil { + t.Fatalf("read XLSX temp directory: %v", err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), "gonavi-xlsx-shared-strings-") { + t.Fatalf("temporary shared-string file leaked after parse failure: %s", entry.Name()) + } + } +} + +func TestXLSXPreviewDoesNotParseSharedStringsBeyondRowLimit(t *testing.T) { + tempRoot := t.TempDir() + t.Setenv("TMP", tempRoot) + t.Setenv("TEMP", tempRoot) + t.Setenv("TMPDIR", tempRoot) + sharedStrings := `` + + `id12345` + + `malformed tail` + sheet := `` + + `0` + + `1` + + `2` + + `3` + + `4` + + `5` + + `6` + + `` + path := writeXLSXTestArchive(t, minimalXLSXTestEntries(sharedStrings, sheet)) + + preview, err := buildImportPreview(path, 5) + if err != nil { + t.Fatalf("preview parsed shared-string tail beyond row limit: %v", err) + } + if preview.TotalRows != 5 || preview.TotalRowsKnown { + t.Fatalf("unexpected short preview result: %#v", preview) + } + entries, err := os.ReadDir(tempRoot) + if err != nil { + t.Fatalf("read XLSX temp directory: %v", err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), "gonavi-xlsx-shared-strings-") { + t.Fatalf("temporary shared-string file leaked after preview short-circuit: %s", entry.Name()) + } + } +} + +func TestXLSXImportRejectsRowOverCombinedByteLimit(t *testing.T) { + largeValue := strings.Repeat("x", 13*1024*1024) + sharedStrings := `` + + `c1c2c3c4c5` + + `` + largeValue + `` + sheet := `` + + `01234` + + `55555` + + `` + path := writeXLSXTestArchive(t, minimalXLSXTestEntries(sharedStrings, sheet)) + + _, err := buildImportPreview(path, 5) + if err == nil { + t.Fatal("XLSX row over the combined byte limit must be rejected") + } + if !strings.Contains(err.Error(), "row 2") || !strings.Contains(err.Error(), "byte limit") { + t.Fatalf("unexpected XLSX row limit error: %v", err) + } +} + +func TestXLSXSharedStringStoreRejectsCountOverLimitAndCleansTempFile(t *testing.T) { + store, err := newXLSXSharedStringStoreWithLimits(2, maxImportCellBytes) + if err != nil { + t.Fatalf("create shared-string store: %v", err) + } + tempPath := store.path + if err := store.Add("one"); err != nil { + t.Fatalf("add first shared string: %v", err) + } + if err := store.Add("two"); err != nil { + t.Fatalf("add second shared string: %v", err) + } + if err := store.Add("three"); err == nil || !strings.Contains(err.Error(), "count exceeds") { + t.Fatalf("third shared string error = %v, want count limit", err) + } + if err := store.Close(); err != nil { + t.Fatalf("close shared-string store: %v", err) + } + if _, err := os.Stat(tempPath); !os.IsNotExist(err) { + t.Fatalf("temporary shared-string file still exists after close: %v", err) + } +} + +func TestXLSXImportRejectsEntryOverUncompressedByteLimit(t *testing.T) { + entries := minimalXLSXTestEntries( + `id`, + `0`, + ) + path := writeXLSXTestArchive(t, entries) + limits := defaultXLSXArchiveResourceLimits + limits.MaxEntryUncompressedBytes = 32 + + err := streamXLSXImportFileWithLimits(path, newImportPreviewCollector(5), limits) + if err == nil { + t.Fatal("XLSX entry over the uncompressed byte limit must be rejected") + } + if !strings.Contains(err.Error(), "entry") || !strings.Contains(err.Error(), "uncompressed") { + t.Fatalf("unexpected entry limit error: %v", err) + } +} + +func TestXLSXImportRejectsArchiveOverTotalUncompressedByteLimit(t *testing.T) { + entries := minimalXLSXTestEntries( + `id`, + `0`, + ) + path := writeXLSXTestArchive(t, entries) + limits := defaultXLSXArchiveResourceLimits + limits.MaxEntryUncompressedBytes = 1 << 20 + limits.MaxTotalUncompressedBytes = 64 + + err := streamXLSXImportFileWithLimits(path, newImportPreviewCollector(5), limits) + if err == nil { + t.Fatal("XLSX archive over the total uncompressed byte limit must be rejected") + } + if !strings.Contains(err.Error(), "total uncompressed") { + t.Fatalf("unexpected total uncompressed limit error: %v", err) + } +} + +func TestXLSXImportRejectsEntryOverCompressionRatioLimit(t *testing.T) { + entries := minimalXLSXTestEntries( + `id`, + `0`, + ) + entries["xl/media/highly-compressible.bin"] = xlsxTestEntry{ + body: strings.Repeat("x", 4096), + method: zip.Deflate, + } + path := writeXLSXTestArchive(t, entries) + limits := defaultXLSXArchiveResourceLimits + limits.MaxEntryUncompressedBytes = 1 << 20 + limits.MaxTotalUncompressedBytes = 1 << 20 + limits.MaxCompressionRatio = 2 + + err := streamXLSXImportFileWithLimits(path, newImportPreviewCollector(5), limits) + if err == nil { + t.Fatal("XLSX entry over the compression ratio limit must be rejected") + } + if !strings.Contains(err.Error(), "compression ratio") { + t.Fatalf("unexpected compression ratio error: %v", err) + } +} + +func TestXLSXImportSelectsConfiguredSheet(t *testing.T) { + entries := map[string]xlsxTestEntry{ + xlsxWorkbookXMLPath: { + body: `` + + `` + + `` + + ``, + method: zip.Store, + }, + xlsxWorkbookRelsXMLPath: { + body: `` + + `` + + `` + + ``, + method: zip.Store, + }, + "xl/worksheets/sheet1.xml": { + body: inlineStringXLSXSheet([][]string{{"summary"}, {"not data"}}), + method: zip.Store, + }, + "xl/worksheets/sheet2.xml": { + body: inlineStringXLSXSheet([][]string{{"id", "name"}, {"1", "alice"}}), + method: zip.Store, + }, + } + path := writeXLSXTestArchive(t, entries) + consumer := &importCollectConsumer{} + if err := streamImportFileWithOptions(path, consumer, ImportFileOptions{SheetName: "Data"}); err != nil { + t.Fatalf("stream selected worksheet: %v", err) + } + if !reflect.DeepEqual(consumer.columns, []string{"id", "name"}) { + t.Fatalf("columns = %#v", consumer.columns) + } + if !reflect.DeepEqual(consumer.rows, []map[string]interface{}{{"id": "1", "name": "alice"}}) { + t.Fatalf("rows = %#v", consumer.rows) + } +} + +func TestXLSXImportUsesConfiguredHeaderRow(t *testing.T) { + entries := minimalXLSXTestEntries( + ``, + inlineStringXLSXSheet([][]string{{"exported by GoNavi"}, {"id", "name"}, {"1", "alice"}}), + ) + path := writeXLSXTestArchive(t, entries) + consumer := &importCollectConsumer{} + if err := streamImportFileWithOptions(path, consumer, ImportFileOptions{HeaderRow: 2}); err != nil { + t.Fatalf("stream XLSX with second header row: %v", err) + } + if !reflect.DeepEqual(consumer.columns, []string{"id", "name"}) { + t.Fatalf("columns = %#v", consumer.columns) + } + if !reflect.DeepEqual(consumer.rows, []map[string]interface{}{{"id": "1", "name": "alice"}}) { + t.Fatalf("rows = %#v", consumer.rows) + } +} + +func TestXLSXImportRejectsDataRowWiderThanHeader(t *testing.T) { + entries := minimalXLSXTestEntries( + ``, + inlineStringXLSXSheet([][]string{{"id", "name"}, {"1", "alice", "unexpected"}}), + ) + path := writeXLSXTestArchive(t, entries) + consumer := &importCollectConsumer{} + + err := streamImportFile(path, consumer) + if err == nil { + t.Fatal("XLSX data row wider than its header must be rejected") + } + if !strings.Contains(err.Error(), "row 2") || !strings.Contains(err.Error(), "3 columns") || !strings.Contains(err.Error(), "2-column header") { + t.Fatalf("unexpected wide-row error: %v", err) + } + if len(consumer.rows) != 0 { + t.Fatalf("wide row reached import consumer: %#v", consumer.rows) + } +} + +func TestXLSXPreviewAndImportConvertStyledDateAndTimeSerials(t *testing.T) { + entries := minimalXLSXTestEntries( + ``, + ``+ + ``+ + `created_on`+ + `at_time`+ + `created_at`+ + `precise_time`+ + ``+ + ``+ + `45293`+ + `0.5`+ + `45293.75`+ + `45293.04309170139`+ + ``+ + ``, + ) + entries["xl/styles.xml"] = xlsxTestEntry{ + body: `` + + `` + + `` + + `` + + `` + + ``, + method: zip.Store, + } + path := writeXLSXTestArchive(t, entries) + want := map[string]interface{}{ + "created_on": "2024-01-02", + "at_time": "12:00:00", + "created_at": "2024-01-02 18:00:00", + "precise_time": "2024-01-02 01:02:03.123", + } + + preview, err := buildImportPreview(path, 5) + if err != nil { + t.Fatalf("preview styled XLSX values: %v", err) + } + if !reflect.DeepEqual(preview.PreviewRows, []map[string]interface{}{want}) { + t.Fatalf("preview rows = %#v, want %#v", preview.PreviewRows, []map[string]interface{}{want}) + } + + consumer := &importCollectConsumer{} + if err := streamImportFile(path, consumer); err != nil { + t.Fatalf("import styled XLSX values: %v", err) + } + if !reflect.DeepEqual(consumer.rows, preview.PreviewRows) { + t.Fatalf("import rows = %#v, preview rows = %#v", consumer.rows, preview.PreviewRows) + } +} + +func TestXLSXImportUsesWorkbook1904DateSystem(t *testing.T) { + entries := minimalXLSXTestEntries( + ``, + ``+ + `created_at`+ + `1.25`+ + ``, + ) + entries[xlsxWorkbookXMLPath] = xlsxTestEntry{ + body: `` + + ``, + method: zip.Store, + } + entries["xl/styles.xml"] = xlsxTestEntry{ + body: ``, + method: zip.Store, + } + path := writeXLSXTestArchive(t, entries) + consumer := &importCollectConsumer{} + + if err := streamImportFile(path, consumer); err != nil { + t.Fatalf("import XLSX using 1904 date system: %v", err) + } + want := []map[string]interface{}{{"created_at": "1904-01-02 06:00:00"}} + if !reflect.DeepEqual(consumer.rows, want) { + t.Fatalf("rows = %#v, want %#v", consumer.rows, want) + } +} + +func inlineStringXLSXSheet(rows [][]string) string { + var builder strings.Builder + builder.WriteString(``) + for rowIndex, row := range rows { + builder.WriteString(``) + for columnIndex, value := range row { + builder.WriteString(``) + builder.WriteString(value) + builder.WriteString(``) + } + builder.WriteString(``) + } + builder.WriteString(``) + return builder.String() +} // TestXLSXCellRefColumnIndexRejectsOutOfRangeColumns 覆盖 xlsx 单元格 r 属性的列号上限。 // diff --git a/internal/db/dameng_impl.go b/internal/db/dameng_impl.go index 128103ed..b0821cec 100644 --- a/internal/db/dameng_impl.go +++ b/internal/db/dameng_impl.go @@ -403,15 +403,20 @@ func (d *DamengDB) GetTriggers(dbName, tableName string) ([]connection.TriggerDe } func (d *DamengDB) ApplyChanges(tableName string, changes connection.ChangeSet) error { + return d.ApplyChangesContext(context.Background(), tableName, changes) +} + +func (d *DamengDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) (err error) { if d.conn == nil { return fmt.Errorf("连接未打开") } - tx, err := d.conn.Begin() + tx, err := d.conn.BeginTx(ctx, nil) if err != nil { return err } - defer tx.Rollback() + transactionCommitted := false + defer func() { rollbackUnfinishedWriteTransaction(tx, transactionCommitted, &err) }() quoteIdent := func(name string) string { n := strings.TrimSpace(name) @@ -446,7 +451,7 @@ func (d *DamengDB) ApplyChanges(tableName string, changes connection.ChangeSet) continue } query := fmt.Sprintf("DELETE FROM %s WHERE %s", qualifiedTable, strings.Join(wheres, " AND ")) - if _, err := tx.Exec(query, args...); err != nil { + if _, err := tx.ExecContext(ctx, query, args...); err != nil { return fmt.Errorf("删除失败：%v", err) } } @@ -479,7 +484,7 @@ func (d *DamengDB) ApplyChanges(tableName string, changes connection.ChangeSet) } query := fmt.Sprintf("UPDATE %s SET %s WHERE %s", qualifiedTable, strings.Join(sets, ", "), strings.Join(wheres, " AND ")) - if _, err := tx.Exec(query, args...); err != nil { + if _, err := tx.ExecContext(ctx, query, args...); err != nil { return fmt.Errorf("更新失败：%v", err) } } @@ -503,12 +508,16 @@ func (d *DamengDB) ApplyChanges(tableName string, changes connection.ChangeSet) } query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", qualifiedTable, strings.Join(cols, ", "), strings.Join(placeholders, ", ")) - if _, err := tx.Exec(query, args...); err != nil { + if _, err := tx.ExecContext(ctx, query, args...); err != nil { return fmt.Errorf("插入失败：%v", err) } } - return tx.Commit() + if err := commitWriteTransaction(tx); err != nil { + return err + } + transactionCommitted = true + return nil } func (d *DamengDB) GetAllColumns(dbName string) ([]connection.ColumnDefinitionWithTable, error) { diff --git a/internal/db/dameng_transaction_test.go b/internal/db/dameng_transaction_test.go index 449a3262..3ce9063d 100644 --- a/internal/db/dameng_transaction_test.go +++ b/internal/db/dameng_transaction_test.go @@ -8,8 +8,10 @@ import ( "database/sql/driver" "errors" "reflect" + "strings" "sync" "testing" + "time" "GoNavi-Wails/internal/connection" ) @@ -20,6 +22,9 @@ type damengTransactionRecordingState struct { commitCalls int rollbackCalls int execQueries []string + execStarted chan struct{} + execRelease chan struct{} + blockExec bool } type damengTransactionConnector struct { @@ -57,10 +62,29 @@ func (c *damengTransactionConn) Begin() (driver.Tx, error) { return &damengTransactionTx{state: c.state}, nil } -func (c *damengTransactionConn) ExecContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) { +func (c *damengTransactionConn) BeginTx(context.Context, driver.TxOptions) (driver.Tx, error) { + return c.Begin() +} + +func (c *damengTransactionConn) ExecContext(ctx context.Context, query string, _ []driver.NamedValue) (driver.Result, error) { c.state.mu.Lock() c.state.execQueries = append(c.state.execQueries, query) + blockExec := c.state.blockExec + execStarted := c.state.execStarted + execRelease := c.state.execRelease c.state.mu.Unlock() + if blockExec { + select { + case execStarted <- struct{}{}: + default: + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-execRelease: + return nil, errors.New("execution released without cancellation") + } + } return driver.RowsAffected(1), nil } @@ -171,3 +195,42 @@ func TestDamengApplyChangesPreservesSchemaContainingDot(t *testing.T) { t.Fatalf("ApplyChanges queries = %#v, want %#v", execQueries, want) } } + +func TestDamengApplyChangesContextCancelsInFlightInsert(t *testing.T) { + state := &damengTransactionRecordingState{ + execStarted: make(chan struct{}, 1), + execRelease: make(chan struct{}), + blockExec: true, + } + dbConn := sql.OpenDB(&damengTransactionConnector{state: state}) + t.Cleanup(func() { _ = dbConn.Close() }) + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + errCh <- (&DamengDB{conn: dbConn}).ApplyChangesContext(ctx, "APP.ORDERS", connection.ChangeSet{ + Inserts: []map[string]interface{}{{"ID": 42, "STATUS": "pending"}}, + }) + }() + + select { + case <-state.execStarted: + cancel() + case <-time.After(time.Second): + cancel() + close(state.execRelease) + t.Fatal("ApplyChangesContext did not reach the context-aware insert path") + } + + select { + case err := <-errCh: + if err == nil || !strings.Contains(err.Error(), context.Canceled.Error()) { + t.Fatalf("ApplyChangesContext error = %v, want context cancellation", err) + } + case <-time.After(time.Second): + close(state.execRelease) + t.Fatal("ApplyChangesContext did not return after cancellation") + } +} + +var _ BatchApplierContext = (*DamengDB)(nil) diff --git a/internal/db/database.go b/internal/db/database.go index c9ce21e0..7289bbfc 100644 --- a/internal/db/database.go +++ b/internal/db/database.go @@ -256,6 +256,13 @@ type BatchWriteExecer interface { ExecBatchContext(ctx context.Context, query string) (int64, error) } +// BatchWriteCapability lets a driver that conditionally supports the +// multi-statement protocol opt out at runtime. MySQL uses this when the +// connection had to fall back to multiStatements=false. +type BatchWriteCapability interface { + SupportsBatchWrites() bool +} + // StatementExecer is a single-session SQL execution handle. // It is used by long-running import jobs that must preserve session-scoped // settings across multiple statements. @@ -795,6 +802,15 @@ type BatchApplier interface { ApplyChanges(tableName string, changes connection.ChangeSet) error } +// BatchApplierContext is the optional cancellation-aware form of BatchApplier. +// Long-running import and synchronization jobs prefer it so cancellation can +// reach an in-flight driver transaction. BatchApplier remains for backwards +// compatibility with drivers that cannot yet expose context cancellation. +type BatchApplierContext interface { + BatchApplier + ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) error +} + // ChangePreviewer 是可选的变更预览接口。 // 驱动可实现此接口提供自定义 SQL 预览格式；若未实现，调用方回退到 GenerateChangePreview。 type ChangePreviewer interface { diff --git a/internal/db/mariadb_batch_capability_test.go b/internal/db/mariadb_batch_capability_test.go new file mode 100644 index 00000000..5aa87e16 --- /dev/null +++ b/internal/db/mariadb_batch_capability_test.go @@ -0,0 +1,22 @@ +//go:build gonavi_full_drivers || gonavi_mariadb_driver + +package db + +import "testing" + +func TestMariaDBBatchWriteCapabilityReflectsNegotiatedDSN(t *testing.T) { + mariaDB := &MariaDB{} + if _, ok := any(mariaDB).(BatchWriteCapability); !ok { + t.Fatal("MariaDB must expose runtime batch-write capability") + } + + mariaDB.batchWritesEnabled = mysqlDSNSupportsBatchWrites("user:pass@tcp(localhost:3306)/app?multiStatements=true") + if !mariaDB.SupportsBatchWrites() { + t.Fatal("multiStatements=true MariaDB connection should allow batch writes") + } + + mariaDB.batchWritesEnabled = mysqlDSNSupportsBatchWrites("user:pass@tcp(localhost:3306)/app?multiStatements=false") + if mariaDB.SupportsBatchWrites() { + t.Fatal("multiStatements=false MariaDB connection must disable batch writes") + } +} diff --git a/internal/db/mariadb_impl.go b/internal/db/mariadb_impl.go index 1e8d924b..736ddd2d 100644 --- a/internal/db/mariadb_impl.go +++ b/internal/db/mariadb_impl.go @@ -19,10 +19,13 @@ import ( // MariaDB implements Database interface for MariaDB // MariaDB is MySQL-compatible, so we reuse the MySQL driver type MariaDB struct { - conn *sql.DB - pingTimeout time.Duration + conn *sql.DB + pingTimeout time.Duration + batchWritesEnabled bool } +var _ BatchApplierContext = (*MariaDB)(nil) + func (m *MariaDB) getDSN(config connection.ConnectionConfig) (string, error) { database := config.Database protocol := "tcp" @@ -40,6 +43,7 @@ func (m *MariaDB) getDSN(config connection.ConnectionConfig) (string, error) { } func (m *MariaDB) Connect(config connection.ConnectionConfig) error { + m.batchWritesEnabled = false runConfig := applyMySQLURI(config) dsn, err := m.getDSN(runConfig) if err != nil { @@ -58,9 +62,14 @@ func (m *MariaDB) Connect(config connection.ConnectionConfig) error { m.conn = nil return wrapDatabaseConnectionVerifyError(err) } + m.batchWritesEnabled = mysqlDSNSupportsBatchWrites(dsn) return nil } +func (m *MariaDB) SupportsBatchWrites() bool { + return m != nil && m.batchWritesEnabled +} + func (m *MariaDB) Close() error { if m.conn != nil { return m.conn.Close() @@ -357,15 +366,20 @@ func (m *MariaDB) GetTriggers(dbName, tableName string) ([]connection.TriggerDef } func (m *MariaDB) ApplyChanges(tableName string, changes connection.ChangeSet) error { + return m.ApplyChangesContext(context.Background(), tableName, changes) +} + +func (m *MariaDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) (err error) { if m.conn == nil { return fmt.Errorf("连接未打开") } - tx, err := m.conn.Begin() + tx, err := m.conn.BeginTx(ctx, nil) if err != nil { return err } - defer tx.Rollback() + transactionCommitted := false + defer func() { rollbackUnfinishedWriteTransaction(tx, transactionCommitted, &err) }() // 1. Deletes for _, pk := range changes.Deletes { @@ -379,9 +393,9 @@ func (m *MariaDB) ApplyChanges(tableName string, changes connection.ChangeSet) e continue } query := fmt.Sprintf("DELETE FROM `%s` WHERE %s", tableName, strings.Join(wheres, " AND ")) - res, err := tx.Exec(query, args...) + res, err := tx.ExecContext(ctx, query, args...) if err != nil { - return fmt.Errorf("删除失败：%v", err) + return markWriteOutcomeUnknownIfAmbiguous(ctx, fmt.Errorf("删除失败：%w", err)) } // 与 mysql_impl.go:1259 一致：本函数是 MySQL 版的拷贝，但漏掉了影响行数校验。 // 缺少该校验时，无主键表上一次单元格编辑可能静默改写多行； @@ -416,9 +430,9 @@ func (m *MariaDB) ApplyChanges(tableName string, changes connection.ChangeSet) e } query := fmt.Sprintf("UPDATE `%s` SET %s WHERE %s", tableName, strings.Join(sets, ", "), strings.Join(wheres, " AND ")) - res, err := tx.Exec(query, args...) + res, err := tx.ExecContext(ctx, query, args...) if err != nil { - return fmt.Errorf("更新失败：%v", err) + return markWriteOutcomeUnknownIfAmbiguous(ctx, fmt.Errorf("更新失败：%w", err)) } // 与 mysql_impl.go:1293 一致，避免一次编辑静默改写多行或 0 行命中仍提示成功。 if err := requireSingleRowAffected(res, rowMutationActionUpdate); err != nil { @@ -426,6 +440,7 @@ func (m *MariaDB) ApplyChanges(tableName string, changes connection.ChangeSet) e } } + var unknownWriteErr error if err := execParameterizedInsertBatches(parameterizedInsertConfig{ Table: fmt.Sprintf("`%s`", escapeMySQLBacktickIdent(tableName)), Rows: changes.Inserts, @@ -437,15 +452,27 @@ func (m *MariaDB) ApplyChanges(tableName string, changes connection.ChangeSet) e return normalizeMySQLComplexValue(normalizeMySQLDateTimeValue(value)), false }, Exec: func(query string, args ...interface{}) (sql.Result, error) { - return tx.Exec(query, args...) + result, err := tx.ExecContext(ctx, query, args...) + err = markWriteOutcomeUnknownIfAmbiguous(ctx, err) + if IsWriteOutcomeUnknown(err) { + unknownWriteErr = err + } + return result, err }, MaxRows: defaultMySQLInsertBatchSize, MaxArgs: maxMySQLInsertBatchArgs, }); err != nil { + if unknownWriteErr != nil { + return fmt.Errorf("%s: %w", err.Error(), unknownWriteErr) + } return err } - return tx.Commit() + if err := commitWriteTransaction(tx); err != nil { + return err + } + transactionCommitted = true + return nil } func (m *MariaDB) GetAllColumns(dbName string) ([]connection.ColumnDefinitionWithTable, error) { diff --git a/internal/db/mariadb_write_outcome_test.go b/internal/db/mariadb_write_outcome_test.go new file mode 100644 index 00000000..47a1e889 --- /dev/null +++ b/internal/db/mariadb_write_outcome_test.go @@ -0,0 +1,45 @@ +//go:build gonavi_full_drivers || gonavi_mariadb_driver + +package db + +import ( + "context" + "errors" + "io" + "testing" + + "GoNavi-Wails/internal/connection" +) + +func TestMariaDBApplyChangesMarksAmbiguousDMLResponseOutcomeUnknown(t *testing.T) { + for name, testCase := range map[string]struct { + writeErr error + changes connection.ChangeSet + }{ + "delete transport": {writeErr: io.ErrUnexpectedEOF, changes: connection.ChangeSet{Deletes: []map[string]interface{}{{"id": int64(1)}}}}, + "insert transport": {writeErr: io.ErrUnexpectedEOF, changes: connection.ChangeSet{Inserts: []map[string]interface{}{{"id": int64(1)}}}}, + "delete cancellation": {writeErr: context.Canceled, changes: connection.ChangeSet{Deletes: []map[string]interface{}{{"id": int64(1)}}}}, + } { + t.Run(name, func(t *testing.T) { + state := &writeOutcomeTransactionState{execErr: testCase.writeErr} + database := openWriteOutcomeTransactionDB(t, state) + + err := (&MariaDB{conn: database}).ApplyChangesContext(context.Background(), "users", testCase.changes) + if !IsWriteOutcomeUnknown(err) || !errors.Is(err, testCase.writeErr) { + t.Fatalf("ambiguous DML response must mark the outcome unknown for non-transactional tables, got %v", err) + } + }) + } +} + +func TestMariaDBApplyChangesKeepsSemanticDMLRejectionKnown(t *testing.T) { + state := &writeOutcomeTransactionState{execErr: errors.New("constraint rejected")} + database := openWriteOutcomeTransactionDB(t, state) + + err := (&MariaDB{conn: database}).ApplyChangesContext(context.Background(), "users", connection.ChangeSet{ + Deletes: []map[string]interface{}{{"id": int64(1)}}, + }) + if err == nil || IsWriteOutcomeUnknown(err) { + t.Fatalf("semantic DML rejection must remain a known error, got %v", err) + } +} diff --git a/internal/db/mysql_connection_params_test.go b/internal/db/mysql_connection_params_test.go index 4e366005..be36f740 100644 --- a/internal/db/mysql_connection_params_test.go +++ b/internal/db/mysql_connection_params_test.go @@ -382,6 +382,9 @@ func TestBuildMySQLCompatibleConnectPlans_AddsHandshakeFallbackWhenMultiStatemen if got := defaultQuery.Get("charset"); got != "utf8mb4,utf8" { t.Fatalf("default plan should use utf8 fallback charset, got=%q", got) } + if !mysqlDSNSupportsBatchWrites(plans[0].dsn) { + t.Fatal("default multiStatements plan should advertise batch-write support") + } fallbackQuery := parseMySQLDSNQueryForTest(t, plans[1].dsn) if got := fallbackQuery.Get("multiStatements"); got != "false" { @@ -390,6 +393,9 @@ func TestBuildMySQLCompatibleConnectPlans_AddsHandshakeFallbackWhenMultiStatemen if got := fallbackQuery.Get("charset"); got != "utf8mb4,utf8" { t.Fatalf("fallback plan should preserve charset fallback, got=%q", got) } + if mysqlDSNSupportsBatchWrites(plans[1].dsn) { + t.Fatal("multiStatements=false fallback must disable batch-write support") + } } func TestBuildMySQLCompatibleConnectPlans_RespectsExplicitMultiStatementsChoice(t *testing.T) { @@ -412,6 +418,9 @@ func TestBuildMySQLCompatibleConnectPlans_RespectsExplicitMultiStatementsChoice( if got := query.Get("multiStatements"); got != "false" { t.Fatalf("explicit allowMultiQueries=false should be preserved, got=%q", got) } + if mysqlDSNSupportsBatchWrites(plans[0].dsn) { + t.Fatal("explicit multiStatements=false plan must disable batch-write support") + } } func TestMySQLDSN_AsiaShanghaiLocationAcceptedByDriver(t *testing.T) { diff --git a/internal/db/mysql_impl.go b/internal/db/mysql_impl.go index 1ab20b9a..7f8c4002 100644 --- a/internal/db/mysql_impl.go +++ b/internal/db/mysql_impl.go @@ -21,10 +21,13 @@ import ( ) type MySQLDB struct { - conn *sql.DB - pingTimeout time.Duration + conn *sql.DB + pingTimeout time.Duration + batchWritesEnabled bool } +var _ BatchApplierContext = (*MySQLDB)(nil) + const ( defaultMySQLPort = 3306 defaultGoldenDBPort = 1523 @@ -477,6 +480,11 @@ func buildMySQLCompatibleConnectPlans(config connection.ConnectionConfig, protoc }), nil } +func mysqlDSNSupportsBatchWrites(dsn string) bool { + parsed, err := mysql.ParseDSN(dsn) + return err == nil && parsed.MultiStatements +} + func normalizeMySQLRawDSNCompatibilityParams(raw string) string { text := strings.TrimSpace(raw) queryIndex := strings.Index(text, "?") @@ -809,6 +817,7 @@ func resolveMySQLCredential(config connection.ConnectionConfig, addressIndex int } func (m *MySQLDB) Connect(config connection.ConnectionConfig) error { + m.batchWritesEnabled = false runConfig := applyMySQLURI(config) addresses := collectMySQLAddresses(runConfig) if len(addresses) == 0 { @@ -870,6 +879,7 @@ func (m *MySQLDB) Connect(config connection.ConnectionConfig) error { m.conn = db m.pingTimeout = timeout + m.batchWritesEnabled = mysqlDSNSupportsBatchWrites(plan.dsn) return nil } } @@ -880,6 +890,10 @@ func (m *MySQLDB) Connect(config connection.ConnectionConfig) error { return fmt.Errorf("连接建立后验证失败：%s", strings.Join(errorDetails, "；")) } +func (m *MySQLDB) SupportsBatchWrites() bool { + return m != nil && m.batchWritesEnabled +} + func (m *MySQLDB) Close() error { if m.conn != nil { return m.conn.Close() @@ -1228,17 +1242,22 @@ func (m *MySQLDB) GetTriggers(dbName, tableName string) ([]connection.TriggerDef } func (m *MySQLDB) ApplyChanges(tableName string, changes connection.ChangeSet) error { + return m.ApplyChangesContext(context.Background(), tableName, changes) +} + +func (m *MySQLDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) (err error) { if m.conn == nil { return fmt.Errorf("连接未打开") } - columnTypeMap := m.loadColumnTypeMap(tableName) + columnTypeMap := m.loadColumnTypeMapContext(ctx, tableName) - tx, err := m.conn.Begin() + tx, err := m.conn.BeginTx(ctx, nil) if err != nil { return err } - defer tx.Rollback() + transactionCommitted := false + defer func() { rollbackUnfinishedWriteTransaction(tx, transactionCommitted, &err) }() // 1. Deletes for _, pk := range changes.Deletes { @@ -1252,9 +1271,9 @@ func (m *MySQLDB) ApplyChanges(tableName string, changes connection.ChangeSet) e continue } query := fmt.Sprintf("DELETE FROM `%s` WHERE %s", tableName, strings.Join(wheres, " AND ")) - res, err := tx.Exec(query, args...) + res, err := tx.ExecContext(ctx, query, args...) if err != nil { - return fmt.Errorf("删除失败：%v", err) + return markWriteOutcomeUnknownIfAmbiguous(ctx, fmt.Errorf("删除失败：%w", err)) } if err := requireSingleRowAffected(res, rowMutationActionDelete); err != nil { return err @@ -1286,24 +1305,29 @@ func (m *MySQLDB) ApplyChanges(tableName string, changes connection.ChangeSet) e } query := fmt.Sprintf("UPDATE `%s` SET %s WHERE %s", tableName, strings.Join(sets, ", "), strings.Join(wheres, " AND ")) - res, err := tx.Exec(query, args...) + res, err := tx.ExecContext(ctx, query, args...) if err != nil { - return fmt.Errorf("更新失败：%v", err) + return markWriteOutcomeUnknownIfAmbiguous(ctx, fmt.Errorf("更新失败：%w", err)) } if err := requireSingleRowAffected(res, rowMutationActionUpdate); err != nil { return err } } - if err := m.applyInsertChanges(tx, tableName, changes.Inserts, columnTypeMap); err != nil { + if err := m.applyInsertChangesContext(ctx, tx, tableName, changes.Inserts, columnTypeMap); err != nil { return err } - return tx.Commit() + if err := commitWriteTransaction(tx); err != nil { + return err + } + transactionCommitted = true + return nil } -func (m *MySQLDB) applyInsertChanges(tx *sql.Tx, tableName string, rows []map[string]interface{}, columnTypeMap map[string]string) error { - return execParameterizedInsertBatches(parameterizedInsertConfig{ +func (m *MySQLDB) applyInsertChangesContext(ctx context.Context, tx *sql.Tx, tableName string, rows []map[string]interface{}, columnTypeMap map[string]string) error { + var unknownWriteErr error + err := execParameterizedInsertBatches(parameterizedInsertConfig{ Table: fmt.Sprintf("`%s`", escapeMySQLBacktickIdent(tableName)), Rows: rows, QuoteColumn: func(column string) string { @@ -1314,7 +1338,12 @@ func (m *MySQLDB) applyInsertChanges(tx *sql.Tx, tableName string, rows []map[st return normalizeMySQLValueForInsert(column, value, columnTypeMap) }, Exec: func(query string, args ...interface{}) (sql.Result, error) { - return tx.Exec(query, args...) + result, err := tx.ExecContext(ctx, query, args...) + err = markWriteOutcomeUnknownIfAmbiguous(ctx, err) + if IsWriteOutcomeUnknown(err) { + unknownWriteErr = err + } + return result, err }, MaxRows: defaultMySQLInsertBatchSize, MaxArgs: maxMySQLInsertBatchArgs, @@ -1323,6 +1352,10 @@ func (m *MySQLDB) applyInsertChanges(tx *sql.Tx, tableName string, rows []map[st return fmt.Sprintf("INSERT INTO %s () VALUES ()", table) }, }) + if err != nil && unknownWriteErr != nil { + return fmt.Errorf("%s: %w", err.Error(), unknownWriteErr) + } + return err } func escapeMySQLBacktickIdent(ident string) string { @@ -1379,20 +1412,21 @@ func normalizeMySQLDateTimeValue(value interface{}) interface{} { return value } -func (m *MySQLDB) loadColumnTypeMap(tableName string) map[string]string { +func (m *MySQLDB) loadColumnTypeMapContext(ctx context.Context, tableName string) map[string]string { result := map[string]string{} table := strings.TrimSpace(tableName) if table == "" { return result } - columns, err := m.GetColumns("", table) + data, _, err := m.QueryContext(ctx, buildMySQLShowFullColumnsQuery("", table)) if err != nil { logger.Warnf("加载列元数据失败（不影响提交）：表=%s err=%v", table, err) return result } - for _, col := range columns { + for _, row := range data { + col := buildMySQLColumnDefinition(row) name := strings.ToLower(strings.TrimSpace(col.Name)) if name == "" { continue diff --git a/internal/db/oceanbase_impl.go b/internal/db/oceanbase_impl.go index b0016520..2c9bb0ea 100644 --- a/internal/db/oceanbase_impl.go +++ b/internal/db/oceanbase_impl.go @@ -678,6 +678,7 @@ func (o *OceanBaseDB) bindConnectedDatabase(db *sql.DB, timeout time.Duration, p o.oracle = nil o.conn = nil o.pingTimeout = 0 + o.batchWritesEnabled = false if protocol == oceanBaseProtocolOracle { o.oracle = &OracleDB{conn: db, pingTimeout: timeout, scanDialect: oceanBaseOracleScanDialect} o.protocol = oceanBaseProtocolOracle @@ -688,8 +689,16 @@ func (o *OceanBaseDB) bindConnectedDatabase(db *sql.DB, timeout time.Duration, p o.protocol = oceanBaseProtocolMySQL } +func (o *OceanBaseDB) setMySQLBatchWritesFromDSN(dsn string) { + if o == nil { + return + } + o.batchWritesEnabled = mysqlDSNSupportsBatchWrites(dsn) +} + func (o *OceanBaseDB) Connect(config connection.ConnectionConfig) (err error) { _ = o.Close() + o.batchWritesEnabled = false defer func() { if err != nil { _ = o.Close() @@ -782,6 +791,7 @@ func (o *OceanBaseDB) Connect(config connection.ConnectionConfig) (err error) { o.conn = db o.pingTimeout = timeout o.protocol = oceanBaseProtocolMySQL + o.setMySQLBatchWritesFromDSN(dsn) return nil } @@ -995,11 +1005,21 @@ func (o *OceanBaseDB) GetTriggers(dbName, tableName string) ([]connection.Trigge } func (o *OceanBaseDB) ApplyChanges(tableName string, changes connection.ChangeSet) error { + return o.ApplyChangesContext(context.Background(), tableName, changes) +} + +func (o *OceanBaseDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) error { // Oracle 协议走 OBClient 路径时，o.oracle.conn 实际上是 mysql wire 的 *sql.DB， // Oracle 风格 SQL（双引号引用 + ROWID）由 OceanBase 服务端按 Oracle 解析器处理， // 但占位符必须是 mysql 风格的 "?"，不能用 OracleDB.ApplyChanges 的 ":1" Oracle bind 风格。 if o.protocol == oceanBaseProtocolOracle && o.oracle != nil { - return o.applyOracleChangesMySQLWire(tableName, changes) + return o.applyOracleChangesMySQLWireContext(ctx, tableName, changes) + } + if applier, ok := o.activeDatabase().(BatchApplierContext); ok { + return applier.ApplyChangesContext(ctx, tableName, changes) + } + if err := ctx.Err(); err != nil { + return err } if applier, ok := o.activeDatabase().(BatchApplier); ok { return applier.ApplyChanges(tableName, changes) @@ -1064,20 +1084,31 @@ func buildOceanBaseOracleAssignment(columnName string, value interface{}, column // applyOracleChangesMySQLWire 在 OceanBase Oracle 租户的 mysql wire 连接上执行 // DELETE/UPDATE/INSERT，使用 Oracle 风格双引号引用标识符 + mysql wire 风格 "?" 占位符。 func (o *OceanBaseDB) applyOracleChangesMySQLWire(tableName string, changes connection.ChangeSet) error { + return o.applyOracleChangesMySQLWireContext(context.Background(), tableName, changes) +} + +func (o *OceanBaseDB) applyOracleChangesMySQLWireContext(ctx context.Context, tableName string, changes connection.ChangeSet) (err error) { if o.oracle == nil || o.oracle.conn == nil { return fmt.Errorf("连接未打开") } + if err := ctx.Err(); err != nil { + return err + } columnTypeMap, err := o.oracle.loadColumnTypeMap(tableName) if err != nil { return fmt.Errorf("OceanBase Oracle 租户 %w", err) } + if err := ctx.Err(); err != nil { + return err + } - tx, err := o.oracle.conn.Begin() + tx, err := o.oracle.conn.BeginTx(ctx, nil) if err != nil { return err } - defer tx.Rollback() + transactionCommitted := false + defer func() { rollbackUnfinishedWriteTransaction(tx, transactionCommitted, &err) }() quoteIdent := func(name string) string { n := strings.TrimSpace(name) @@ -1126,7 +1157,7 @@ func (o *OceanBaseDB) applyOracleChangesMySQLWire(tableName string, changes conn continue } query := fmt.Sprintf("DELETE FROM %s WHERE %s", qualifiedTable, strings.Join(wheres, " AND ")) - res, err := tx.Exec(query, args...) + res, err := tx.ExecContext(ctx, query, args...) if err != nil { return fmt.Errorf("删除失败：%v", err) } @@ -1157,7 +1188,7 @@ func (o *OceanBaseDB) applyOracleChangesMySQLWire(tableName string, changes conn } query := fmt.Sprintf("UPDATE %s SET %s WHERE %s", qualifiedTable, strings.Join(sets, ", "), strings.Join(wheres, " AND ")) - res, err := tx.Exec(query, args...) + res, err := tx.ExecContext(ctx, query, args...) if err != nil { return fmt.Errorf("更新失败：%v", err) } @@ -1183,7 +1214,7 @@ func (o *OceanBaseDB) applyOracleChangesMySQLWire(tableName string, changes conn } query := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", qualifiedTable, strings.Join(cols, ", "), strings.Join(placeholders, ", ")) - res, err := tx.Exec(query, args...) + res, err := tx.ExecContext(ctx, query, args...) if err != nil { return fmt.Errorf("插入失败：%v", err) } @@ -1192,5 +1223,9 @@ func (o *OceanBaseDB) applyOracleChangesMySQLWire(tableName string, changes conn } } - return tx.Commit() + if err := commitWriteTransaction(tx); err != nil { + return err + } + transactionCommitted = true + return nil } diff --git a/internal/db/oceanbase_impl_test.go b/internal/db/oceanbase_impl_test.go index 25144d1c..a9a3fff0 100644 --- a/internal/db/oceanbase_impl_test.go +++ b/internal/db/oceanbase_impl_test.go @@ -13,8 +13,10 @@ import ( "strings" "testing" "time" + "GoNavi-Wails/internal/connection" "GoNavi-Wails/shared/i18n" + mysqlDriver "github.com/go-sql-driver/mysql" ) @@ -783,6 +785,121 @@ func TestOceanBaseOracleOBClientApplyChangesUsesMySQLWirePlaceholders(t *testing } } +func TestOceanBaseOracleApplyChangesContextCancelsMySQLWireStatement(t *testing.T) { + dbConn, state := openOracleRecordingDB(t) + state.mu.Lock() + state.blockExecUntilCanceled = true + state.execStarted = make(chan struct{}, 1) + state.execRelease = make(chan struct{}) + state.mu.Unlock() + + oceanbaseDB := &OceanBaseDB{} + oceanbaseDB.bindConnectedDatabase(dbConn, 0, oceanBaseProtocolOracle) + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + errCh <- oceanbaseDB.ApplyChangesContext(ctx, "APP.USERS", connection.ChangeSet{ + Updates: []connection.UpdateRow{{ + Keys: map[string]interface{}{"ID": 42}, + Values: map[string]interface{}{"NAME": "cancel-me"}, + }}, + }) + }() + + select { + case <-state.execStarted: + cancel() + case <-time.After(time.Second): + cancel() + close(state.execRelease) + t.Fatal("ApplyChangesContext did not reach the Oracle-protocol MySQL-wire execution path") + } + + select { + case err := <-errCh: + if err == nil || !strings.Contains(err.Error(), context.Canceled.Error()) { + t.Fatalf("ApplyChangesContext error = %v, want context cancellation", err) + } + case <-time.After(time.Second): + close(state.execRelease) + t.Fatal("ApplyChangesContext did not return after cancellation") + } +} + +func TestOceanBaseMySQLApplyChangesContextForwardsCancellation(t *testing.T) { + dbConn, state := openOracleRecordingDB(t) + state.mu.Lock() + state.blockExecUntilCanceled = true + state.execStarted = make(chan struct{}, 1) + state.execRelease = make(chan struct{}) + state.mu.Unlock() + + oceanbaseDB := &OceanBaseDB{} + oceanbaseDB.bindConnectedDatabase(dbConn, 0, oceanBaseProtocolMySQL) + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + errCh <- oceanbaseDB.ApplyChangesContext(ctx, "app.orders", connection.ChangeSet{ + Inserts: []map[string]interface{}{{"id": 42, "status": "pending"}}, + }) + }() + + select { + case <-state.execStarted: + cancel() + case <-time.After(time.Second): + cancel() + close(state.execRelease) + t.Fatal("ApplyChangesContext did not reach the underlying MySQL context applier") + } + + select { + case err := <-errCh: + if err == nil || !strings.Contains(err.Error(), context.Canceled.Error()) { + t.Fatalf("ApplyChangesContext error = %v, want context cancellation", err) + } + case <-time.After(time.Second): + close(state.execRelease) + t.Fatal("ApplyChangesContext did not return after cancellation") + } +} + +func TestOceanBaseForwardsUnknownCommitOutcomeFromMySQLApplier(t *testing.T) { + commitErr := errors.New("commit response lost") + state := &writeOutcomeTransactionState{commitErr: commitErr} + database := openWriteOutcomeTransactionDB(t, state) + oceanbase := &OceanBaseDB{} + oceanbase.bindConnectedDatabase(database, 0, oceanBaseProtocolMySQL) + + err := oceanbase.ApplyChangesContext(context.Background(), "users", connection.ChangeSet{ + Inserts: []map[string]interface{}{{"id": int64(1)}}, + }) + if !IsWriteOutcomeUnknown(err) || !errors.Is(err, commitErr) { + t.Fatalf("OceanBase forwarding must preserve the typed MySQL commit outcome, got %v", err) + } +} + +func TestOceanBaseOracleMySQLWireMarksCommitFailureOutcomeUnknown(t *testing.T) { + dbConn, state := openOracleRecordingDB(t) + commitErr := errors.New("commit response lost") + state.mu.Lock() + state.txCommitErr = commitErr + state.mu.Unlock() + oceanbase := &OceanBaseDB{} + oceanbase.bindConnectedDatabase(dbConn, 0, oceanBaseProtocolOracle) + + err := oceanbase.ApplyChangesContext(context.Background(), "APP.USERS", connection.ChangeSet{ + Inserts: []map[string]interface{}{{"ID": int64(1)}}, + }) + if !IsWriteOutcomeUnknown(err) || !errors.Is(err, commitErr) { + t.Fatalf("OceanBase Oracle MySQL-wire commit failure must mark the outcome unknown, got %v", err) + } +} + +var _ BatchApplierContext = (*OceanBaseDB)(nil) + func TestOceanBaseOracleOBClientApplyChangesFormatsTemporalValuesExplicitly(t *testing.T) { t.Parallel() @@ -920,7 +1037,6 @@ func TestOceanBaseOracleCreateStatementFallbackErrorUsesCurrentLanguage(t *testi } } - // 用户通过 ConnectionParams 设置 connectionAttributes 时，OceanBase MySQL wire 路径必须把 // 这些 attribute 透传到 go-sql-driver/mysql DSN，让 driver 在握手响应里发 CLIENT_CONNECT_ATTRS。 // 这是 OBClient 协议握手探索的入口：高级用户/DBA 可以试错不同 attribute 组合而不需要改 GoNavi 代码。 @@ -953,6 +1069,19 @@ func TestOceanBaseMySQLDSNPassesThroughConnectionAttributes(t *testing.T) { } } +func TestOceanBaseMySQLBatchWriteCapabilityTracksConnectedDSN(t *testing.T) { + oceanBase := &OceanBaseDB{} + oceanBase.setMySQLBatchWritesFromDSN("user:pass@tcp(localhost:2881)/app?multiStatements=true") + if !oceanBase.SupportsBatchWrites() { + t.Fatal("OceanBase MySQL multiStatements=true connection should allow batch writes") + } + + oceanBase.setMySQLBatchWritesFromDSN("user:pass@tcp(localhost:2881)/app?multiStatements=false") + if oceanBase.SupportsBatchWrites() { + t.Fatal("OceanBase MySQL multiStatements=false connection must disable batch writes") + } +} + // 当用户错选 MySQL 协议但租户实际是 Oracle 模式时，OceanBase 服务端返回 Error 1235， // 我们必须在错误消息里明确指引用户切换协议，避免方向摇摆。 func TestFormatOceanBaseMySQLAttemptErrorHintsOracleProtocol(t *testing.T) { diff --git a/internal/db/optional_driver_agent_impl.go b/internal/db/optional_driver_agent_impl.go index c0b9871b..1b8452cb 100644 --- a/internal/db/optional_driver_agent_impl.go +++ b/internal/db/optional_driver_agent_impl.go @@ -230,6 +230,13 @@ func (c *optionalDriverAgentClient) call(req optionalAgentRequest, out interface }) } +func markOptionalAgentApplyChangesTransportUnknown(req optionalAgentRequest, err error) error { + if err != nil && req.Method == optionalAgentMethodApplyChanges { + return MarkWriteOutcomeUnknown(err) + } + return err +} + func (c *optionalDriverAgentClient) callLocked(req optionalAgentRequest, out interface{}, fields *[]string, messages *[]string, rowsAffected *int64) error { if err := c.stoppedError(); err != nil { return fmt.Errorf("%s 驱动代理传输不可用：%w", driverDisplayName(c.driver), err) @@ -246,23 +253,23 @@ func (c *optionalDriverAgentClient) callLocked(req optionalAgentRequest, out int if _, err := c.stdin.Write(payload); err != nil { stderrText := c.stderrText() if stderrText == "" { - return fmt.Errorf("调用 %s 驱动代理失败：%w", driverDisplayName(c.driver), err) + return markOptionalAgentApplyChangesTransportUnknown(req, fmt.Errorf("调用 %s 驱动代理失败：%w", driverDisplayName(c.driver), err)) } - return fmt.Errorf("调用 %s 驱动代理失败：%w（stderr: %s）", driverDisplayName(c.driver), err, stderrText) + return markOptionalAgentApplyChangesTransportUnknown(req, fmt.Errorf("调用 %s 驱动代理失败：%w（stderr: %s）", driverDisplayName(c.driver), err, stderrText)) } line, err := c.reader.ReadBytes('\n') if err != nil { stderrText := c.stderrText() if stderrText == "" { - return fmt.Errorf("读取 %s 驱动代理响应失败：%w", driverDisplayName(c.driver), err) + return markOptionalAgentApplyChangesTransportUnknown(req, fmt.Errorf("读取 %s 驱动代理响应失败：%w", driverDisplayName(c.driver), err)) } - return fmt.Errorf("读取 %s 驱动代理响应失败：%w（stderr: %s）", driverDisplayName(c.driver), err, stderrText) + return markOptionalAgentApplyChangesTransportUnknown(req, fmt.Errorf("读取 %s 驱动代理响应失败：%w（stderr: %s）", driverDisplayName(c.driver), err, stderrText)) } var resp optionalAgentResponse if err := json.Unmarshal(line, &resp); err != nil { - return fmt.Errorf("解析 %s 驱动代理响应失败：%w", driverDisplayName(c.driver), err) + return markOptionalAgentApplyChangesTransportUnknown(req, fmt.Errorf("解析 %s 驱动代理响应失败：%w", driverDisplayName(c.driver), err)) } if !resp.Success { errText := strings.TrimSpace(resp.Error) @@ -1307,6 +1314,16 @@ func (d *OptionalDriverAgentDB) GetTriggers(dbName, tableName string) ([]connect } func (d *OptionalDriverAgentDB) ApplyChanges(tableName string, changes connection.ChangeSet) error { + return d.ApplyChangesContext(context.Background(), tableName, changes) +} + +func (d *OptionalDriverAgentDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } client, err := d.requireClient() if err != nil { return err @@ -1321,11 +1338,15 @@ func (d *OptionalDriverAgentDB) ApplyChanges(tableName string, changes connectio logger.Warnf("Kingbase ApplyChanges 字段名规范化失败：%v", normErr) } } - return client.call(optionalAgentRequest{ + err = client.callContext(ctx, optionalAgentRequest{ Method: optionalAgentMethodApplyChanges, TableName: tableName, Changes: &changes, }, nil, nil, nil, nil) + if err != nil && ctx.Err() != nil { + return MarkWriteOutcomeUnknown(err) + } + return err } func (d *OptionalDriverAgentDB) requireClient() (*optionalDriverAgentClient, error) { diff --git a/internal/db/oracle_applychanges_test.go b/internal/db/oracle_applychanges_test.go index 88dfd31a..0670acd4 100644 --- a/internal/db/oracle_applychanges_test.go +++ b/internal/db/oracle_applychanges_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "database/sql/driver" + "errors" "fmt" "io" "reflect" @@ -28,9 +29,16 @@ type oracleRecordingState struct { mu sync.Mutex execQueries []string execArgs [][]driver.NamedValue + execStarted chan struct{} + execRelease chan struct{} + blockExecUntilCanceled bool queries []string beginCalls int rowsAffected int64 + execErrors map[string]error + closeCalls int + txCommitErr error + txRollbackErr error queryResults map[string]oracleRecordingQueryResult queryError error disableDefaultTabColumns bool @@ -72,6 +80,12 @@ func (s *oracleRecordingState) snapshotBeginCalls() int { return s.beginCalls } +func (s *oracleRecordingState) snapshotCloseCalls() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.closeCalls +} + type oracleRecordingDriver struct{} func (oracleRecordingDriver) Open(name string) (driver.Conn, error) { @@ -92,21 +106,47 @@ func (c *oracleRecordingConn) Prepare(query string) (driver.Stmt, error) { return nil, fmt.Errorf("prepare not supported in oracle recording driver: %s", query) } -func (c *oracleRecordingConn) Close() error { return nil } +func (c *oracleRecordingConn) Close() error { + c.state.mu.Lock() + c.state.closeCalls++ + c.state.mu.Unlock() + return nil +} func (c *oracleRecordingConn) Begin() (driver.Tx, error) { c.state.mu.Lock() c.state.beginCalls++ c.state.mu.Unlock() - return oracleRecordingTx{}, nil + return &oracleRecordingTx{state: c.state}, nil } -func (c *oracleRecordingConn) ExecContext(_ context.Context, query string, args []driver.NamedValue) (driver.Result, error) { +func (c *oracleRecordingConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { c.state.mu.Lock() - defer c.state.mu.Unlock() c.state.execQueries = append(c.state.execQueries, query) c.state.execArgs = append(c.state.execArgs, append([]driver.NamedValue(nil), args...)) - return driver.RowsAffected(c.state.rowsAffected), nil + blockUntilCanceled := c.state.blockExecUntilCanceled && query != "ROLLBACK" + execStarted := c.state.execStarted + execRelease := c.state.execRelease + rowsAffected := c.state.rowsAffected + execErr := c.state.execErrors[query] + c.state.mu.Unlock() + + if blockUntilCanceled { + select { + case execStarted <- struct{}{}: + default: + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-execRelease: + return nil, fmt.Errorf("recording execution released without cancellation") + } + } + if execErr != nil { + return nil, execErr + } + return driver.RowsAffected(rowsAffected), nil } func (c *oracleRecordingConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) { @@ -151,10 +191,21 @@ func cloneOracleRecordingRows(src [][]driver.Value) [][]driver.Value { var _ driver.ExecerContext = (*oracleRecordingConn)(nil) var _ driver.QueryerContext = (*oracleRecordingConn)(nil) -type oracleRecordingTx struct{} +type oracleRecordingTx struct { + state *oracleRecordingState +} -func (oracleRecordingTx) Commit() error { return nil } -func (oracleRecordingTx) Rollback() error { return nil } +func (tx *oracleRecordingTx) Commit() error { + tx.state.mu.Lock() + defer tx.state.mu.Unlock() + return tx.state.txCommitErr +} + +func (tx *oracleRecordingTx) Rollback() error { + tx.state.mu.Lock() + defer tx.state.mu.Unlock() + return tx.state.txRollbackErr +} type oracleRecordingRows struct { columns []string @@ -313,6 +364,52 @@ func TestOracleApplyChangesUsesPinnedSessionTransactionSQL(t *testing.T) { } } +func TestOracleApplyChangesContextCancelsInFlightStatementAndRollsBack(t *testing.T) { + dbConn, state := openOracleRecordingDB(t) + state.mu.Lock() + state.blockExecUntilCanceled = true + state.execStarted = make(chan struct{}, 1) + state.execRelease = make(chan struct{}) + state.mu.Unlock() + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + errCh <- (&OracleDB{conn: dbConn}).ApplyChangesContext(ctx, "APP.USERS", connection.ChangeSet{ + Updates: []connection.UpdateRow{{ + Keys: map[string]interface{}{"ID": 42}, + Values: map[string]interface{}{"NAME": "cancel-me"}, + }}, + }) + }() + + select { + case <-state.execStarted: + cancel() + case <-time.After(time.Second): + cancel() + close(state.execRelease) + t.Fatal("ApplyChangesContext did not reach the context-aware SQL execution path") + } + + select { + case err := <-errCh: + if err == nil || !strings.Contains(err.Error(), context.Canceled.Error()) { + t.Fatalf("ApplyChangesContext error = %v, want context cancellation", err) + } + case <-time.After(time.Second): + close(state.execRelease) + t.Fatal("ApplyChangesContext did not return after cancellation") + } + + wantExecs := []string{`UPDATE "APP"."USERS" SET "NAME" = :1 WHERE "ID" = :2`, "ROLLBACK"} + if got := state.snapshotExecQueries(); !reflect.DeepEqual(got, wantExecs) { + t.Fatalf("expected canceled statement followed by independent rollback, got %#v", got) + } +} + +var _ BatchApplierContext = (*OracleDB)(nil) + func TestOracleApplyChangesRollsBackPinnedSessionOnError(t *testing.T) { t.Parallel() @@ -347,6 +444,53 @@ func TestOracleApplyChangesRollsBackPinnedSessionOnError(t *testing.T) { } } +func TestOracleApplyChangesDiscardsPinnedConnectionWhenRollbackFails(t *testing.T) { + dbConn, state := openOracleRecordingDB(t) + statementSQL := `UPDATE "APP"."USERS" SET "NAME" = :1 WHERE "ID" = :2` + rollbackErr := errors.New("rollback response lost") + state.mu.Lock() + state.execErrors = map[string]error{ + statementSQL: errors.New("known statement rejection"), + "ROLLBACK": rollbackErr, + } + state.mu.Unlock() + + err := (&OracleDB{conn: dbConn}).ApplyChangesContext(context.Background(), "APP.USERS", connection.ChangeSet{ + Updates: []connection.UpdateRow{{ + Keys: map[string]interface{}{"ID": int64(1)}, + Values: map[string]interface{}{"NAME": "alice"}, + }}, + }) + if !IsWriteOutcomeUnknown(err) || !errors.Is(err, rollbackErr) { + t.Fatalf("rollback failure must mark the result unknown and preserve its cause, got %v", err) + } + if got := state.snapshotCloseCalls(); got != 1 { + t.Fatalf("rollback failure must discard the pinned physical connection, close calls = %d", got) + } +} + +func TestOracleApplyChangesMarksManualCommitFailureOutcomeUnknown(t *testing.T) { + dbConn, state := openOracleRecordingDB(t) + commitErr := errors.New("commit response lost") + state.mu.Lock() + state.execErrors = map[string]error{"COMMIT": commitErr} + state.mu.Unlock() + + err := (&OracleDB{conn: dbConn}).ApplyChangesContext(context.Background(), "APP.USERS", connection.ChangeSet{ + Inserts: []map[string]interface{}{{"ID": int64(1)}}, + }) + if !IsWriteOutcomeUnknown(err) || !errors.Is(err, commitErr) { + t.Fatalf("manual COMMIT failure must mark the result unknown and preserve its cause, got %v", err) + } + if got := state.snapshotExecQueries(); !reflect.DeepEqual(got, []string{ + `INSERT INTO "APP"."USERS" ("ID") VALUES (:1)`, + "COMMIT", + "ROLLBACK", + }) { + t.Fatalf("unexpected manual transaction sequence after COMMIT failure: %#v", got) + } +} + func TestOracleApplyChangesReturnsErrorWhenUpdateMatchesNoRows(t *testing.T) { t.Parallel() diff --git a/internal/db/oracle_impl.go b/internal/db/oracle_impl.go index 66101265..7c43cec7 100644 --- a/internal/db/oracle_impl.go +++ b/internal/db/oracle_impl.go @@ -3,6 +3,7 @@ package db import ( "context" "database/sql" + "errors" "fmt" "net" "net/url" @@ -1364,21 +1365,32 @@ func parseOracleTemporalString(raw string) (time.Time, bool) { } func (o *OracleDB) ApplyChanges(tableName string, changes connection.ChangeSet) (err error) { + return o.ApplyChangesContext(context.Background(), tableName, changes) +} + +func (o *OracleDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) (err error) { if o.conn == nil { return fmt.Errorf("连接未打开") } + if err := ctx.Err(); err != nil { + return err + } columnTypeMap, err := o.loadColumnTypeMap(tableName) if err != nil { return err } - - ctx := context.Background() + if err := ctx.Err(); err != nil { + return err + } conn, err := o.conn.Conn(ctx) if err != nil { return err } defer func() { + if conn == nil { + return + } if closeErr := conn.Close(); closeErr != nil && err == nil { err = closeErr } @@ -1389,8 +1401,18 @@ func (o *OracleDB) ApplyChanges(tableName string, changes connection.ChangeSet) if transactionFinished { return } - if _, rollbackErr := conn.ExecContext(context.Background(), "ROLLBACK"); rollbackErr != nil { + rollbackCtx, cancelRollback := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelRollback() + if _, rollbackErr := conn.ExecContext(rollbackCtx, "ROLLBACK"); rollbackErr != nil { logger.Warnf("Oracle 表格编辑事务回滚失败：table=%s err=%v", tableName, rollbackErr) + unknownErr := fmt.Errorf("Oracle 事务回滚失败：%w", rollbackErr) + if err != nil { + unknownErr = errors.Join(err, unknownErr) + } + if discardErr := discardSQLConn(&conn); discardErr != nil { + unknownErr = errors.Join(unknownErr, fmt.Errorf("Oracle 事务连接丢弃失败：%w", discardErr)) + } + err = MarkWriteOutcomeUnknown(unknownErr) } }() @@ -1514,7 +1536,7 @@ func (o *OracleDB) ApplyChanges(tableName string, changes connection.ChangeSet) } if _, err := conn.ExecContext(ctx, "COMMIT"); err != nil { - return fmt.Errorf("事务提交失败：%v", err) + return MarkWriteOutcomeUnknown(fmt.Errorf("事务提交失败：%w", err)) } transactionFinished = true return nil diff --git a/internal/db/postgres_applychanges_context_test.go b/internal/db/postgres_applychanges_context_test.go new file mode 100644 index 00000000..95a83483 --- /dev/null +++ b/internal/db/postgres_applychanges_context_test.go @@ -0,0 +1,94 @@ +package db + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "strings" + "sync" + "testing" + "time" + + "GoNavi-Wails/internal/connection" +) + +const postgresApplyChangesContextDriverName = "postgres_apply_changes_context" + +var ( + registerPostgresApplyChangesContextDriverOnce sync.Once + postgresApplyChangesContextExecStarted = make(chan struct{}, 1) +) + +type postgresApplyChangesContextDriver struct{} + +type postgresApplyChangesContextConn struct{} + +type postgresApplyChangesContextTx struct{} + +func (postgresApplyChangesContextDriver) Open(string) (driver.Conn, error) { + return postgresApplyChangesContextConn{}, nil +} + +func (postgresApplyChangesContextConn) Prepare(string) (driver.Stmt, error) { + return nil, driver.ErrSkip +} + +func (postgresApplyChangesContextConn) Close() error { return nil } + +func (postgresApplyChangesContextConn) Begin() (driver.Tx, error) { + return nil, errors.New("legacy Begin must not be used") +} + +func (postgresApplyChangesContextConn) BeginTx(context.Context, driver.TxOptions) (driver.Tx, error) { + return postgresApplyChangesContextTx{}, nil +} + +func (postgresApplyChangesContextConn) ExecContext(ctx context.Context, _ string, _ []driver.NamedValue) (driver.Result, error) { + select { + case postgresApplyChangesContextExecStarted <- struct{}{}: + default: + } + <-ctx.Done() + return nil, ctx.Err() +} + +func (postgresApplyChangesContextTx) Commit() error { return nil } +func (postgresApplyChangesContextTx) Rollback() error { return nil } + +func TestPostgresApplyChangesContextCancelsInFlightStatement(t *testing.T) { + registerPostgresApplyChangesContextDriverOnce.Do(func() { + sql.Register(postgresApplyChangesContextDriverName, postgresApplyChangesContextDriver{}) + }) + + conn, err := sql.Open(postgresApplyChangesContextDriverName, "") + if err != nil { + t.Fatalf("open context-aware test database: %v", err) + } + t.Cleanup(func() { _ = conn.Close() }) + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + errCh <- (&PostgresDB{conn: conn}).ApplyChangesContext(ctx, "public.orders", connection.ChangeSet{ + Deletes: []map[string]interface{}{{"id": 42}}, + }) + }() + + select { + case <-postgresApplyChangesContextExecStarted: + cancel() + case <-time.After(time.Second): + cancel() + t.Fatal("ApplyChangesContext did not reach the context-aware SQL execution path") + } + + select { + case err := <-errCh: + if err == nil || !strings.Contains(err.Error(), context.Canceled.Error()) { + t.Fatalf("ApplyChangesContext error = %v, want context cancellation", err) + } + case <-time.After(time.Second): + t.Fatal("ApplyChangesContext did not return after cancellation") + } +} diff --git a/internal/db/postgres_impl.go b/internal/db/postgres_impl.go index d1fcf5dd..8db4cf32 100644 --- a/internal/db/postgres_impl.go +++ b/internal/db/postgres_impl.go @@ -25,6 +25,8 @@ type PostgresDB struct { forwarder *ssh.LocalForwarder // Store SSH tunnel forwarder } +var _ BatchApplierContext = (*PostgresDB)(nil) + type postgresSessionExecer struct { *sqlConnStatementExecer } @@ -679,15 +681,20 @@ func (p *PostgresDB) queryUserSchemas() []string { } func (p *PostgresDB) ApplyChanges(tableName string, changes connection.ChangeSet) error { + return p.ApplyChangesContext(context.Background(), tableName, changes) +} + +func (p *PostgresDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) (err error) { if p.conn == nil { return fmt.Errorf("连接未打开") } - tx, err := p.conn.Begin() + tx, err := p.conn.BeginTx(ctx, nil) if err != nil { return err } - defer tx.Rollback() + transactionCommitted := false + defer func() { rollbackUnfinishedWriteTransaction(tx, transactionCommitted, &err) }() quoteIdent := func(name string) string { n := strings.TrimSpace(name) @@ -727,7 +734,7 @@ func (p *PostgresDB) ApplyChanges(tableName string, changes connection.ChangeSet continue } query := fmt.Sprintf("DELETE FROM %s WHERE %s", qualifiedTable, strings.Join(wheres, " AND ")) - res, err := tx.Exec(query, args...) + res, err := tx.ExecContext(ctx, query, args...) if err != nil { return fmt.Errorf("删除失败：%v", err) } @@ -764,7 +771,7 @@ func (p *PostgresDB) ApplyChanges(tableName string, changes connection.ChangeSet } query := fmt.Sprintf("UPDATE %s SET %s WHERE %s", qualifiedTable, strings.Join(sets, ", "), strings.Join(wheres, " AND ")) - res, err := tx.Exec(query, args...) + res, err := tx.ExecContext(ctx, query, args...) if err != nil { return fmt.Errorf("更新失败：%v", err) } @@ -781,7 +788,7 @@ func (p *PostgresDB) ApplyChanges(tableName string, changes connection.ChangeSet return fmt.Sprintf("$%d", idx) }, Exec: func(query string, args ...interface{}) (sql.Result, error) { - return tx.Exec(query, args...) + return tx.ExecContext(ctx, query, args...) }, EmptyInsertSQL: func(table string) string { return fmt.Sprintf("INSERT INTO %s DEFAULT VALUES", table) @@ -790,5 +797,9 @@ func (p *PostgresDB) ApplyChanges(tableName string, changes connection.ChangeSet return err } - return tx.Commit() + if err := commitWriteTransaction(tx); err != nil { + return err + } + transactionCommitted = true + return nil } diff --git a/internal/db/sqlite_impl.go b/internal/db/sqlite_impl.go index 4bb95215..de869567 100644 --- a/internal/db/sqlite_impl.go +++ b/internal/db/sqlite_impl.go @@ -23,6 +23,8 @@ type SQLiteDB struct { pingTimeout time.Duration } +var _ BatchApplierContext = (*SQLiteDB)(nil) + func (s *SQLiteDB) Connect(config connection.ConnectionConfig) error { dsn, err := resolveSQLiteDSN(config) if err != nil { @@ -613,15 +615,20 @@ func (s *SQLiteDB) GetTriggers(dbName, tableName string) ([]connection.TriggerDe } func (s *SQLiteDB) ApplyChanges(tableName string, changes connection.ChangeSet) error { + return s.ApplyChangesContext(context.Background(), tableName, changes) +} + +func (s *SQLiteDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) (err error) { if s.conn == nil { return fmt.Errorf("连接未打开") } - tx, err := s.conn.Begin() + tx, err := s.conn.BeginTx(ctx, nil) if err != nil { return err } - defer tx.Rollback() + transactionCommitted := false + defer func() { rollbackUnfinishedWriteTransaction(tx, transactionCommitted, &err) }() quoteIdent := func(name string) string { n := strings.TrimSpace(name) @@ -659,7 +666,7 @@ func (s *SQLiteDB) ApplyChanges(tableName string, changes connection.ChangeSet) continue } query := fmt.Sprintf("DELETE FROM %s WHERE %s", qualifiedTable, strings.Join(wheres, " AND ")) - if _, err := tx.Exec(query, args...); err != nil { + if _, err := tx.ExecContext(ctx, query, args...); err != nil { return fmt.Errorf("删除失败：%v", err) } } @@ -689,7 +696,7 @@ func (s *SQLiteDB) ApplyChanges(tableName string, changes connection.ChangeSet) } query := fmt.Sprintf("UPDATE %s SET %s WHERE %s", qualifiedTable, strings.Join(sets, ", "), strings.Join(wheres, " AND ")) - if _, err := tx.Exec(query, args...); err != nil { + if _, err := tx.ExecContext(ctx, query, args...); err != nil { return fmt.Errorf("更新失败：%v", err) } } @@ -700,14 +707,18 @@ func (s *SQLiteDB) ApplyChanges(tableName string, changes connection.ChangeSet) QuoteColumn: quoteIdent, Placeholder: func(int) string { return "?" }, Exec: func(query string, args ...interface{}) (sql.Result, error) { - return tx.Exec(query, args...) + return tx.ExecContext(ctx, query, args...) }, MaxArgs: sqliteBatchInsertArgs, }); err != nil { return err } - return tx.Commit() + if err := commitWriteTransaction(tx); err != nil { + return err + } + transactionCommitted = true + return nil } func (s *SQLiteDB) GetAllColumns(dbName string) ([]connection.ColumnDefinitionWithTable, error) { diff --git a/internal/db/sqlserver_impl.go b/internal/db/sqlserver_impl.go index 2e768428..25bf1943 100644 --- a/internal/db/sqlserver_impl.go +++ b/internal/db/sqlserver_impl.go @@ -840,15 +840,20 @@ ORDER BY tr.name`, } func (s *SqlServerDB) ApplyChanges(tableName string, changes connection.ChangeSet) error { + return s.ApplyChangesContext(context.Background(), tableName, changes) +} + +func (s *SqlServerDB) ApplyChangesContext(ctx context.Context, tableName string, changes connection.ChangeSet) (err error) { if s.conn == nil { return fmt.Errorf("连接未打开") } - tx, err := s.conn.Begin() + tx, err := s.conn.BeginTx(ctx, nil) if err != nil { return err } - defer tx.Rollback() + transactionCommitted := false + defer func() { rollbackUnfinishedWriteTransaction(tx, transactionCommitted, &err) }() quoteIdent := func(name string) string { n := strings.TrimSpace(name) @@ -883,7 +888,7 @@ func (s *SqlServerDB) ApplyChanges(tableName string, changes connection.ChangeSe continue } query := fmt.Sprintf("DELETE FROM %s WHERE %s", qualifiedTable, strings.Join(wheres, " AND ")) - if _, err := tx.Exec(query, args...); err != nil { + if _, err := tx.ExecContext(ctx, query, args...); err != nil { return fmt.Errorf("删除失败：%v", err) } } @@ -916,7 +921,7 @@ func (s *SqlServerDB) ApplyChanges(tableName string, changes connection.ChangeSe } query := fmt.Sprintf("UPDATE %s SET %s WHERE %s", qualifiedTable, strings.Join(sets, ", "), strings.Join(wheres, " AND ")) - if _, err := tx.Exec(query, args...); err != nil { + if _, err := tx.ExecContext(ctx, query, args...); err != nil { return fmt.Errorf("更新失败：%v", err) } } @@ -932,12 +937,16 @@ func (s *SqlServerDB) ApplyChanges(tableName string, changes connection.ChangeSe return sql.Named(fmt.Sprintf("p%d", idx), value) }, Exec: func(query string, args ...interface{}) (sql.Result, error) { - return tx.Exec(query, args...) + return tx.ExecContext(ctx, query, args...) }, MaxArgs: sqlServerBatchInsertArgs, }); err != nil { return err } - return tx.Commit() + if err := commitWriteTransaction(tx); err != nil { + return err + } + transactionCommitted = true + return nil } diff --git a/internal/db/sqlserver_impl_test.go b/internal/db/sqlserver_impl_test.go index f36b7209..cd5daad5 100644 --- a/internal/db/sqlserver_impl_test.go +++ b/internal/db/sqlserver_impl_test.go @@ -12,7 +12,11 @@ import ( "strings" "sync" "testing" + "time" + + "GoNavi-Wails/internal/connection" "GoNavi-Wails/shared/i18n" + "github.com/golang-sql/sqlexp" _ "modernc.org/sqlite" ) @@ -115,6 +119,103 @@ type fakeSQLServerExecResult struct { rowErr error } +type sqlServerApplyChangesContextState struct { + execStarted chan struct{} + execRelease chan struct{} +} + +type sqlServerApplyChangesContextConnector struct { + state *sqlServerApplyChangesContextState +} + +type sqlServerApplyChangesContextDriver struct{} + +type sqlServerApplyChangesContextConn struct { + state *sqlServerApplyChangesContextState +} + +type sqlServerApplyChangesContextTx struct{} + +func (c sqlServerApplyChangesContextConnector) Connect(context.Context) (driver.Conn, error) { + return &sqlServerApplyChangesContextConn{state: c.state}, nil +} + +func (sqlServerApplyChangesContextConnector) Driver() driver.Driver { + return sqlServerApplyChangesContextDriver{} +} + +func (sqlServerApplyChangesContextDriver) Open(string) (driver.Conn, error) { + return nil, errors.New("use connector") +} + +func (*sqlServerApplyChangesContextConn) Prepare(string) (driver.Stmt, error) { + return nil, driver.ErrSkip +} + +func (*sqlServerApplyChangesContextConn) Close() error { return nil } + +func (*sqlServerApplyChangesContextConn) Begin() (driver.Tx, error) { + return nil, errors.New("legacy Begin must not be used") +} + +func (*sqlServerApplyChangesContextConn) BeginTx(context.Context, driver.TxOptions) (driver.Tx, error) { + return sqlServerApplyChangesContextTx{}, nil +} + +func (c *sqlServerApplyChangesContextConn) ExecContext(ctx context.Context, _ string, _ []driver.NamedValue) (driver.Result, error) { + select { + case c.state.execStarted <- struct{}{}: + default: + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-c.state.execRelease: + return nil, errors.New("execution released without cancellation") + } +} + +func (sqlServerApplyChangesContextTx) Commit() error { return nil } +func (sqlServerApplyChangesContextTx) Rollback() error { return nil } + +func TestSQLServerApplyChangesContextCancelsInFlightBatchInsert(t *testing.T) { + state := &sqlServerApplyChangesContextState{ + execStarted: make(chan struct{}, 1), + execRelease: make(chan struct{}), + } + dbConn := sql.OpenDB(sqlServerApplyChangesContextConnector{state: state}) + t.Cleanup(func() { _ = dbConn.Close() }) + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { + errCh <- (&SqlServerDB{conn: dbConn}).ApplyChangesContext(ctx, "dbo.orders", connection.ChangeSet{ + Inserts: []map[string]interface{}{{"id": 42, "status": "pending"}}, + }) + }() + + select { + case <-state.execStarted: + cancel() + case <-time.After(time.Second): + cancel() + close(state.execRelease) + t.Fatal("ApplyChangesContext did not reach the context-aware batch insert path") + } + + select { + case err := <-errCh: + if err == nil || !strings.Contains(err.Error(), context.Canceled.Error()) { + t.Fatalf("ApplyChangesContext error = %v, want context cancellation", err) + } + case <-time.After(time.Second): + close(state.execRelease) + t.Fatal("ApplyChangesContext did not return after cancellation") + } +} + +var _ BatchApplierContext = (*SqlServerDB)(nil) + func (r fakeSQLServerExecResult) LastInsertId() (int64, error) { return 0, errors.New("not implemented") } @@ -341,4 +442,3 @@ func TestSQLServerMetadataErrorsUseCurrentLanguage(t *testing.T) { }) } } - diff --git a/internal/db/write_outcome.go b/internal/db/write_outcome.go new file mode 100644 index 00000000..d7c7cfc4 --- /dev/null +++ b/internal/db/write_outcome.go @@ -0,0 +1,103 @@ +package db + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "io" + "net" +) + +// WriteOutcomeUnknownError marks a write whose server-side outcome cannot be +// determined from the response. Callers must not retry or continue past it as +// though the row were known to have been rejected. +type WriteOutcomeUnknownError struct { + cause error +} + +func (err *WriteOutcomeUnknownError) Error() string { + if err == nil || err.cause == nil { + return "write outcome is unknown" + } + return err.cause.Error() +} + +func (err *WriteOutcomeUnknownError) Unwrap() error { + if err == nil { + return nil + } + return err.cause +} + +// MarkWriteOutcomeUnknown preserves the original error while attaching the +// no-retry contract used by import and synchronization callers. +func MarkWriteOutcomeUnknown(err error) error { + if err == nil || IsWriteOutcomeUnknown(err) { + return err + } + return &WriteOutcomeUnknownError{cause: err} +} + +func IsWriteOutcomeUnknown(err error) bool { + var unknown *WriteOutcomeUnknownError + return errors.As(err, &unknown) +} + +// IsAmbiguousWriteResponse reports transport and cancellation failures that +// can occur after an autocommit statement was dispatched but before its server +// response was observed. A database semantic error is deliberately not +// classified as ambiguous because it proves the statement was rejected. +func IsAmbiguousWriteResponse(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.Canceled) || + errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, driver.ErrBadConn) || + errors.Is(err, io.EOF) || + errors.Is(err, io.ErrClosedPipe) || + errors.Is(err, io.ErrUnexpectedEOF) { + return true + } + var networkErr net.Error + return errors.As(err, &networkErr) +} + +func markWriteOutcomeUnknownIfAmbiguous(ctx context.Context, err error) error { + if err == nil || IsWriteOutcomeUnknown(err) { + return err + } + if IsAmbiguousWriteResponse(err) || (ctx != nil && ctx.Err() != nil) { + return MarkWriteOutcomeUnknown(err) + } + return err +} + +// rollbackUnfinishedWriteTransaction keeps a failed rollback from being +// flattened into the preceding statement error. A rollback failure means the +// transaction's final server-side state cannot safely be inferred. +func rollbackUnfinishedWriteTransaction(tx *sql.Tx, committed bool, resultErr *error) { + if tx == nil || committed { + return + } + rollbackErr := tx.Rollback() + if rollbackErr == nil || errors.Is(rollbackErr, sql.ErrTxDone) { + return + } + rollbackErr = fmt.Errorf("事务回滚失败：%w", rollbackErr) + if resultErr != nil && *resultErr != nil { + rollbackErr = errors.Join(*resultErr, rollbackErr) + } + if resultErr != nil { + *resultErr = MarkWriteOutcomeUnknown(rollbackErr) + } +} + +func commitWriteTransaction(tx *sql.Tx) error { + if err := tx.Commit(); err != nil { + return MarkWriteOutcomeUnknown(fmt.Errorf("事务提交失败：%w", err)) + } + return nil +} diff --git a/internal/db/write_outcome_transaction_test.go b/internal/db/write_outcome_transaction_test.go new file mode 100644 index 00000000..2c99eb52 --- /dev/null +++ b/internal/db/write_outcome_transaction_test.go @@ -0,0 +1,213 @@ +package db + +import ( + "bufio" + "bytes" + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "io" + "strings" + "sync" + "testing" + + "GoNavi-Wails/internal/connection" +) + +const writeOutcomeTransactionDriverName = "gonavi_write_outcome_transaction" + +var ( + registerWriteOutcomeTransactionDriverOnce sync.Once + writeOutcomeTransactionDriverMu sync.Mutex + writeOutcomeTransactionDriverSeq int + writeOutcomeTransactionDriverStates = map[string]*writeOutcomeTransactionState{} +) + +type writeOutcomeTransactionState struct { + mu sync.Mutex + execErr error + commitErr error + rollbackErr error + commits int + rollbacks int +} + +type writeOutcomeTransactionDriver struct{} + +func (writeOutcomeTransactionDriver) Open(name string) (driver.Conn, error) { + writeOutcomeTransactionDriverMu.Lock() + state := writeOutcomeTransactionDriverStates[name] + writeOutcomeTransactionDriverMu.Unlock() + if state == nil { + return nil, fmt.Errorf("write outcome transaction state not found: %s", name) + } + return &writeOutcomeTransactionConn{state: state}, nil +} + +type writeOutcomeTransactionConn struct { + state *writeOutcomeTransactionState +} + +func (*writeOutcomeTransactionConn) Prepare(string) (driver.Stmt, error) { + return nil, driver.ErrSkip +} + +func (*writeOutcomeTransactionConn) Close() error { return nil } + +func (conn *writeOutcomeTransactionConn) Begin() (driver.Tx, error) { + return &writeOutcomeTransactionTx{state: conn.state}, nil +} + +func (conn *writeOutcomeTransactionConn) BeginTx(context.Context, driver.TxOptions) (driver.Tx, error) { + return &writeOutcomeTransactionTx{state: conn.state}, nil +} + +func (conn *writeOutcomeTransactionConn) ExecContext(context.Context, string, []driver.NamedValue) (driver.Result, error) { + conn.state.mu.Lock() + defer conn.state.mu.Unlock() + if conn.state.execErr != nil { + return nil, conn.state.execErr + } + return driver.RowsAffected(1), nil +} + +type writeOutcomeTransactionTx struct { + state *writeOutcomeTransactionState +} + +func (tx *writeOutcomeTransactionTx) Commit() error { + tx.state.mu.Lock() + defer tx.state.mu.Unlock() + tx.state.commits++ + return tx.state.commitErr +} + +func (tx *writeOutcomeTransactionTx) Rollback() error { + tx.state.mu.Lock() + defer tx.state.mu.Unlock() + tx.state.rollbacks++ + return tx.state.rollbackErr +} + +func openWriteOutcomeTransactionDB(t *testing.T, state *writeOutcomeTransactionState) *sql.DB { + t.Helper() + registerWriteOutcomeTransactionDriverOnce.Do(func() { + sql.Register(writeOutcomeTransactionDriverName, writeOutcomeTransactionDriver{}) + }) + writeOutcomeTransactionDriverMu.Lock() + writeOutcomeTransactionDriverSeq++ + dsn := fmt.Sprintf("write-outcome-%d", writeOutcomeTransactionDriverSeq) + writeOutcomeTransactionDriverStates[dsn] = state + writeOutcomeTransactionDriverMu.Unlock() + + database, err := sql.Open(writeOutcomeTransactionDriverName, dsn) + if err != nil { + t.Fatalf("open write outcome transaction database: %v", err) + } + t.Cleanup(func() { + _ = database.Close() + writeOutcomeTransactionDriverMu.Lock() + delete(writeOutcomeTransactionDriverStates, dsn) + writeOutcomeTransactionDriverMu.Unlock() + }) + return database +} + +func TestPostgresApplyChangesMarksCommitFailureOutcomeUnknown(t *testing.T) { + commitErr := errors.New("commit response lost") + state := &writeOutcomeTransactionState{commitErr: commitErr} + database := openWriteOutcomeTransactionDB(t, state) + + err := (&PostgresDB{conn: database}).ApplyChangesContext(context.Background(), "public.users", connection.ChangeSet{ + Deletes: []map[string]interface{}{{"id": int64(1)}}, + }) + if !IsWriteOutcomeUnknown(err) || !errors.Is(err, commitErr) { + t.Fatalf("commit error must preserve its cause and mark the outcome unknown, got %v", err) + } +} + +func TestPostgresApplyChangesMarksRollbackFailureOutcomeUnknown(t *testing.T) { + execErr := errors.New("known statement rejection") + rollbackErr := errors.New("rollback response lost") + state := &writeOutcomeTransactionState{execErr: execErr, rollbackErr: rollbackErr} + database := openWriteOutcomeTransactionDB(t, state) + + err := (&PostgresDB{conn: database}).ApplyChangesContext(context.Background(), "public.users", connection.ChangeSet{ + Deletes: []map[string]interface{}{{"id": int64(1)}}, + }) + if !IsWriteOutcomeUnknown(err) || !errors.Is(err, rollbackErr) { + t.Fatalf("rollback failure must preserve its cause and mark the outcome unknown, got %v", err) + } +} + +func TestMySQLApplyChangesMarksAmbiguousDMLResponseOutcomeUnknown(t *testing.T) { + for name, testCase := range map[string]struct { + writeErr error + changes connection.ChangeSet + }{ + "delete transport": {writeErr: io.ErrUnexpectedEOF, changes: connection.ChangeSet{Deletes: []map[string]interface{}{{"id": int64(1)}}}}, + "insert transport": {writeErr: io.ErrUnexpectedEOF, changes: connection.ChangeSet{Inserts: []map[string]interface{}{{"id": int64(1)}}}}, + "delete cancellation": {writeErr: context.Canceled, changes: connection.ChangeSet{Deletes: []map[string]interface{}{{"id": int64(1)}}}}, + } { + t.Run(name, func(t *testing.T) { + state := &writeOutcomeTransactionState{execErr: testCase.writeErr} + database := openWriteOutcomeTransactionDB(t, state) + + err := (&MySQLDB{conn: database}).ApplyChangesContext(context.Background(), "users", testCase.changes) + if !IsWriteOutcomeUnknown(err) || !errors.Is(err, testCase.writeErr) { + t.Fatalf("ambiguous DML response must mark the outcome unknown for non-transactional tables, got %v", err) + } + }) + } +} + +func TestMySQLApplyChangesKeepsSemanticDMLRejectionKnown(t *testing.T) { + state := &writeOutcomeTransactionState{execErr: errors.New("constraint rejected")} + database := openWriteOutcomeTransactionDB(t, state) + + err := (&MySQLDB{conn: database}).ApplyChangesContext(context.Background(), "users", connection.ChangeSet{ + Deletes: []map[string]interface{}{{"id": int64(1)}}, + }) + if err == nil || IsWriteOutcomeUnknown(err) { + t.Fatalf("semantic DML rejection must remain a known error, got %v", err) + } +} + +func TestOptionalDriverAgentApplyChangesMarksLostRPCResponseOutcomeUnknown(t *testing.T) { + var stdin optionalAgentTestWriteCloser + client := &optionalDriverAgentClient{ + stdin: &stdin, + reader: bufio.NewReader(bytes.NewReader(nil)), + driver: "dameng", + } + database := &OptionalDriverAgentDB{driverType: "dameng", client: client} + + err := database.ApplyChangesContext(context.Background(), "public.users", connection.ChangeSet{ + Inserts: []map[string]interface{}{{"id": int64(1)}}, + }) + if !IsWriteOutcomeUnknown(err) || !errors.Is(err, io.EOF) { + t.Fatalf("lost RPC response after dispatch must mark the outcome unknown, got %v", err) + } +} + +func TestOptionalDriverAgentApplyChangesKeepsRemoteRejectionKnown(t *testing.T) { + var stdin optionalAgentTestWriteCloser + client := &optionalDriverAgentClient{ + stdin: &stdin, + reader: bufio.NewReader(strings.NewReader(`{"id":1,"success":false,"error":"constraint rejected"}` + "\n")), + driver: "dameng", + } + database := &OptionalDriverAgentDB{driverType: "dameng", client: client} + + err := database.ApplyChangesContext(context.Background(), "public.users", connection.ChangeSet{ + Inserts: []map[string]interface{}{{"id": int64(1)}}, + }) + if err == nil || IsWriteOutcomeUnknown(err) { + t.Fatalf("explicit remote rejection must remain a known row error, got %v", err) + } +} + +var _ driver.ConnBeginTx = (*writeOutcomeTransactionConn)(nil) +var _ driver.ExecerContext = (*writeOutcomeTransactionConn)(nil) diff --git a/internal/importjob/model.go b/internal/importjob/model.go new file mode 100644 index 00000000..8420b509 --- /dev/null +++ b/internal/importjob/model.go @@ -0,0 +1,61 @@ +package importjob + +type Kind string + +const ( + KindTable Kind = "table" + KindSQL Kind = "sql" +) + +type Status string + +const ( + StatusPreparing Status = "preparing" + StatusRunning Status = "running" + StatusStopping Status = "stopping" + StatusCompleted Status = "completed" + StatusPartial Status = "partial" + StatusFailed Status = "failed" + StatusCancelled Status = "cancelled" + StatusUnknown Status = "unknown" + StatusInterrupted Status = "interrupted" +) + +type Checkpoint struct { + Safe bool `json:"safe"` + SourceRow int64 `json:"sourceRow,omitempty"` + ByteOffset int64 `json:"byteOffset,omitempty"` + StatementIndex int64 `json:"statementIndex,omitempty"` + TransactionStart int64 `json:"transactionStart,omitempty"` +} + +type Job struct { + ID string `json:"id"` + Kind Kind `json:"kind"` + Status Status `json:"status"` + Stage string `json:"stage,omitempty"` + SourcePath string `json:"sourcePath,omitempty"` + SourceIdentityToken string `json:"sourceIdentityToken"` + SourceContentSHA256 string `json:"sourceContentSha256,omitempty"` + TargetFingerprint string `json:"targetFingerprint"` + ConnectionID string `json:"connectionId,omitempty"` + DatabaseName string `json:"databaseName,omitempty"` + TableName string `json:"tableName,omitempty"` + OptionsHash string `json:"optionsHash"` + Current int64 `json:"current,omitempty"` + Total int64 `json:"total,omitempty"` + Succeeded int64 `json:"succeeded,omitempty"` + Skipped int64 `json:"skipped,omitempty"` + Failed int64 `json:"failed,omitempty"` + BytesRead int64 `json:"bytesRead,omitempty"` + SourceBytesTotal int64 `json:"sourceBytesTotal,omitempty"` + ByteProgressKind string `json:"byteProgressKind,omitempty"` + OutcomeUnknown bool `json:"outcomeUnknown,omitempty"` + Resumable bool `json:"resumable,omitempty"` + Checkpoint Checkpoint `json:"checkpoint"` + ErrorArtifactID string `json:"errorArtifactId,omitempty"` + Message string `json:"message,omitempty"` + Revision int64 `json:"revision"` + CreatedAt int64 `json:"createdAt"` + UpdatedAt int64 `json:"updatedAt"` +} diff --git a/internal/importjob/store.go b/internal/importjob/store.go new file mode 100644 index 00000000..201d742a --- /dev/null +++ b/internal/importjob/store.go @@ -0,0 +1,238 @@ +package importjob + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" + + "github.com/google/uuid" +) + +var ( + ErrNotFound = errors.New("import job not found") + ErrRevisionConflict = errors.New("import job revision conflict") + errCorruptMetadata = errors.New("import job metadata is corrupt") + validJobIDPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$`) +) + +// CorruptJobFilesWarning reports that list/recovery skipped unreadable job +// metadata. It deliberately exposes only a count: persisted metadata may +// contain sensitive source or target details, and paths are machine-specific. +type CorruptJobFilesWarning struct { + Count int +} + +func (w *CorruptJobFilesWarning) Error() string { + return fmt.Sprintf("skipped %d corrupt import job metadata file(s)", w.Count) +} + +type Store struct { + root string + mu sync.Mutex +} + +func Open(root string) (*Store, error) { + root = strings.TrimSpace(root) + if root == "" { + return nil, errors.New("import job directory is empty") + } + absRoot, err := filepath.Abs(root) + if err != nil { + return nil, err + } + if err := os.MkdirAll(absRoot, 0o700); err != nil { + return nil, err + } + return &Store{root: absRoot}, nil +} + +func (s *Store) Put(job Job) (Job, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.putLocked(job) +} + +func (s *Store) putLocked(job Job) (Job, error) { + job.ID = strings.TrimSpace(job.ID) + if !validJobIDPattern.MatchString(job.ID) { + return Job{}, errors.New("invalid import job id") + } + if job.Kind != KindTable && job.Kind != KindSQL { + return Job{}, fmt.Errorf("invalid import job kind %q", job.Kind) + } + path := s.jobPath(job.ID) + now := time.Now().UnixMilli() + existing, err := readJob(path) + switch { + case err == nil: + if job.Revision != existing.Revision { + return Job{}, ErrRevisionConflict + } + job.Revision++ + job.CreatedAt = existing.CreatedAt + case errors.Is(err, os.ErrNotExist): + if job.Revision != 0 { + return Job{}, ErrNotFound + } + job.Revision = 1 + job.CreatedAt = now + case err != nil: + return Job{}, err + } + job.UpdatedAt = now + if err := writeJobAtomic(path, job); err != nil { + return Job{}, err + } + return job, nil +} + +func (s *Store) Get(id string) (Job, error) { + s.mu.Lock() + defer s.mu.Unlock() + if !validJobIDPattern.MatchString(strings.TrimSpace(id)) { + return Job{}, ErrNotFound + } + job, err := readJob(s.jobPath(strings.TrimSpace(id))) + if errors.Is(err, os.ErrNotExist) { + return Job{}, ErrNotFound + } + return job, err +} + +func (s *Store) Delete(id string) error { + s.mu.Lock() + defer s.mu.Unlock() + id = strings.TrimSpace(id) + if !validJobIDPattern.MatchString(id) { + return ErrNotFound + } + if err := os.Remove(s.jobPath(id)); err != nil { + if errors.Is(err, os.ErrNotExist) { + return ErrNotFound + } + return err + } + return nil +} + +func (s *Store) List() ([]Job, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.listLocked() +} + +func (s *Store) listLocked() ([]Job, error) { + entries, err := os.ReadDir(s.root) + if err != nil { + return nil, err + } + jobs := make([]Job, 0, len(entries)) + corruptCount := 0 + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + continue + } + job, err := readJob(filepath.Join(s.root, entry.Name())) + if err != nil { + if errors.Is(err, errCorruptMetadata) { + corruptCount++ + continue + } + return nil, err + } + jobs = append(jobs, job) + } + sort.Slice(jobs, func(i, j int) bool { + if jobs[i].UpdatedAt == jobs[j].UpdatedAt { + return jobs[i].ID < jobs[j].ID + } + return jobs[i].UpdatedAt > jobs[j].UpdatedAt + }) + if corruptCount > 0 { + return jobs, &CorruptJobFilesWarning{Count: corruptCount} + } + return jobs, nil +} + +func (s *Store) RecoverInterrupted() ([]Job, error) { + s.mu.Lock() + defer s.mu.Unlock() + jobs, listErr := s.listLocked() + if listErr != nil { + var warning *CorruptJobFilesWarning + if !errors.As(listErr, &warning) { + return nil, listErr + } + } + recovered := make([]Job, 0) + for _, job := range jobs { + if job.Status != StatusPreparing && job.Status != StatusRunning && job.Status != StatusStopping { + continue + } + job.Status = StatusInterrupted + job.Resumable = job.Checkpoint.Safe && !job.OutcomeUnknown && + job.SourceIdentityToken != "" && job.TargetFingerprint != "" && job.OptionsHash != "" + updated, err := s.putLocked(job) + if err != nil { + return nil, err + } + recovered = append(recovered, updated) + } + return recovered, listErr +} + +func (s *Store) jobPath(id string) string { + return filepath.Join(s.root, id+".json") +} + +func readJob(path string) (Job, error) { + data, err := os.ReadFile(path) + if err != nil { + return Job{}, err + } + var job Job + if err := json.Unmarshal(data, &job); err != nil { + return Job{}, fmt.Errorf("%w: %v", errCorruptMetadata, err) + } + return job, nil +} + +func writeJobAtomic(path string, job Job) error { + data, err := json.MarshalIndent(job, "", " ") + if err != nil { + return err + } + tempPath := filepath.Join(filepath.Dir(path), "."+filepath.Base(path)+"."+uuid.NewString()+".tmp") + f, err := os.OpenFile(tempPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return err + } + ok := false + defer func() { + _ = f.Close() + if !ok { + _ = os.Remove(tempPath) + } + }() + if _, err := f.Write(data); err != nil { + return err + } + if err := f.Sync(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + if err := os.Rename(tempPath, path); err != nil { + return err + } + ok = true + return nil +} diff --git a/internal/importjob/store_test.go b/internal/importjob/store_test.go new file mode 100644 index 00000000..6fb25fae --- /dev/null +++ b/internal/importjob/store_test.go @@ -0,0 +1,176 @@ +package importjob + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestStoreListSkipsCorruptMetadataAndReturnsValidJobs(t *testing.T) { + root := t.TempDir() + store, err := Open(root) + if err != nil { + t.Fatal(err) + } + valid, err := store.Put(Job{ID: "import-valid", Kind: KindTable, Status: StatusCompleted}) + if err != nil { + t.Fatal(err) + } + corruptContents := `{"id":"private-payload-that-must-not-be-logged"` + if err := os.WriteFile(filepath.Join(root, "import-corrupt.json"), []byte(corruptContents), 0o600); err != nil { + t.Fatal(err) + } + + jobs, err := store.List() + var warning *CorruptJobFilesWarning + if !errors.As(err, &warning) { + t.Fatalf("error = %v, want CorruptJobFilesWarning", err) + } + if warning.Count != 1 { + t.Fatalf("warning count = %d, want 1", warning.Count) + } + if len(jobs) != 1 || jobs[0].ID != valid.ID { + t.Fatalf("jobs = %#v, want only %q", jobs, valid.ID) + } + if strings.Contains(err.Error(), root) || strings.Contains(err.Error(), corruptContents) { + t.Fatalf("warning leaked path or contents: %q", err.Error()) + } +} + +func TestStoreRecoverInterruptedSkipsCorruptMetadataAndRecoversValidJobs(t *testing.T) { + root := t.TempDir() + store, err := Open(root) + if err != nil { + t.Fatal(err) + } + running, err := store.Put(Job{ + ID: "import-running-valid", + Kind: KindTable, + Status: StatusRunning, + SourceIdentityToken: "source-v1", + TargetFingerprint: "target-v1", + OptionsHash: "options-v1", + Checkpoint: Checkpoint{Safe: true, SourceRow: 1000}, + }) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "import-truncated.json"), []byte(`{"id":`), 0o600); err != nil { + t.Fatal(err) + } + + recovered, err := store.RecoverInterrupted() + var warning *CorruptJobFilesWarning + if !errors.As(err, &warning) || warning.Count != 1 { + t.Fatalf("error = %v, want one-file CorruptJobFilesWarning", err) + } + if len(recovered) != 1 || recovered[0].ID != running.ID || recovered[0].Status != StatusInterrupted { + t.Fatalf("recovered = %#v, want interrupted %q", recovered, running.ID) + } + persisted, err := store.Get(running.ID) + if err != nil { + t.Fatal(err) + } + if persisted.Status != StatusInterrupted || !persisted.Resumable { + t.Fatalf("valid job was not durably recovered: %#v", persisted) + } +} + +func TestStoreRecoversInterruptedJobOnlyFromSafeCheckpoint(t *testing.T) { + root := t.TempDir() + store, err := Open(root) + if err != nil { + t.Fatal(err) + } + job, err := store.Put(Job{ + ID: "import-job-safe", + Kind: KindTable, + Status: StatusRunning, + SourceIdentityToken: "source-v1", + TargetFingerprint: "target-v1", + OptionsHash: "options-v1", + Checkpoint: Checkpoint{ + Safe: true, + SourceRow: 2000, + ByteOffset: 65536, + }, + }) + if err != nil { + t.Fatal(err) + } + if job.Revision != 1 { + t.Fatalf("revision = %d, want 1", job.Revision) + } + + reopened, err := Open(root) + if err != nil { + t.Fatal(err) + } + recovered, err := reopened.RecoverInterrupted() + if err != nil { + t.Fatal(err) + } + if len(recovered) != 1 || recovered[0].Status != StatusInterrupted || !recovered[0].Resumable { + t.Fatalf("unexpected recovered jobs: %#v", recovered) + } + if recovered[0].Checkpoint.SourceRow != 2000 { + t.Fatalf("checkpoint was not preserved: %#v", recovered[0].Checkpoint) + } +} + +func TestValidateResumeRejectsChangedInputsAndUnknownOutcome(t *testing.T) { + base := Job{ + Status: StatusInterrupted, + Resumable: true, + SourceIdentityToken: "source-v1", + TargetFingerprint: "target-v1", + OptionsHash: "options-v1", + Checkpoint: Checkpoint{Safe: true, SourceRow: 1000}, + } + if err := ValidateResume(base, "source-v1", "target-v1", "options-v1"); err != nil { + t.Fatalf("matching inputs should resume: %v", err) + } + for name, mutate := range map[string]func(*Job, *string, *string, *string){ + "source changed": func(_ *Job, source, _, _ *string) { *source = "source-v2" }, + "target changed": func(_ *Job, _, target, _ *string) { *target = "target-v2" }, + "options changed": func(_ *Job, _, _, options *string) { *options = "options-v2" }, + "outcome unknown": func(job *Job, _, _, _ *string) { job.OutcomeUnknown = true }, + "unsafe checkpoint": func(job *Job, _, _, _ *string) { job.Checkpoint.Safe = false }, + } { + t.Run(name, func(t *testing.T) { + job := base + source, target, options := "source-v1", "target-v1", "options-v1" + mutate(&job, &source, &target, &options) + if err := ValidateResume(job, source, target, options); !errors.Is(err, ErrResumeUnsafe) { + t.Fatalf("error = %v, want ErrResumeUnsafe", err) + } + }) + } +} + +func TestStoreDeleteRemovesOnlyTheRequestedJob(t *testing.T) { + store, err := Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{"import-job-a", "import-job-b"} { + if _, err := store.Put(Job{ID: id, Kind: KindTable, Status: StatusCompleted}); err != nil { + t.Fatal(err) + } + } + + if err := store.Delete("import-job-a"); err != nil { + t.Fatal(err) + } + if _, err := store.Get("import-job-a"); !errors.Is(err, ErrNotFound) { + t.Fatalf("deleted job error = %v, want ErrNotFound", err) + } + if _, err := store.Get("import-job-b"); err != nil { + t.Fatalf("unrelated job was removed: %v", err) + } + if err := store.Delete("../outside"); !errors.Is(err, ErrNotFound) { + t.Fatalf("invalid id error = %v, want ErrNotFound", err) + } +} diff --git a/internal/importjob/validation.go b/internal/importjob/validation.go new file mode 100644 index 00000000..42e2cf5a --- /dev/null +++ b/internal/importjob/validation.go @@ -0,0 +1,24 @@ +package importjob + +import ( + "errors" + "strings" +) + +var ErrResumeUnsafe = errors.New("import job cannot be resumed safely") + +func ValidateResume(job Job, sourceIdentityToken, targetFingerprint, optionsHash string) error { + if job.Status != StatusInterrupted || !job.Resumable || !job.Checkpoint.Safe || job.OutcomeUnknown { + return ErrResumeUnsafe + } + if strings.TrimSpace(sourceIdentityToken) == "" || sourceIdentityToken != job.SourceIdentityToken { + return ErrResumeUnsafe + } + if strings.TrimSpace(targetFingerprint) == "" || targetFingerprint != job.TargetFingerprint { + return ErrResumeUnsafe + } + if strings.TrimSpace(optionsHash) == "" || optionsHash != job.OptionsHash { + return ErrResumeUnsafe + } + return nil +} diff --git a/internal/webserver/server.go b/internal/webserver/server.go index cdad5869..3de9d9c3 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -76,8 +76,14 @@ var desktopOnlyAppMethods = map[string]struct{}{ "ImportData": {}, "ImportDatabaseSQL": {}, "PreviewImportFile": {}, + "PreviewImportFileWithOptions": {}, "ImportDataWithProgress": {}, "ImportDataWithProgressOptions": {}, + "ListImportJobs": {}, + "GetImportJob": {}, + "CancelImportJob": {}, + "DeleteImportJob": {}, + "ExportImportErrorRows": {}, "ExportTable": {}, "ExportTableWithOptions": {}, "ExportTablesSQL": {}, diff --git a/internal/webserver/server_test.go b/internal/webserver/server_test.go index 27129f41..bd18e68d 100644 --- a/internal/webserver/server_test.go +++ b/internal/webserver/server_test.go @@ -104,7 +104,8 @@ func TestMethodInvokerRejectsDesktopOnlyAppMethodsBeforeReflection(t *testing.T) for _, method := range []string{ "Shutdown", "ExportSQLAuditFile", "OpenSQLFile", "ExecuteSQLFile", "ReadSQLFile", - "PreviewImportFile", "ImportDatabaseSQL", "ImportDataWithProgress", "ImportDataWithProgressOptions", "GetDataRootDirectoryInfo", + "PreviewImportFile", "PreviewImportFileWithOptions", "ImportDatabaseSQL", "ImportDataWithProgress", "ImportDataWithProgressOptions", + "ListImportJobs", "GetImportJob", "CancelImportJob", "DeleteImportJob", "ExportImportErrorRows", "GetDataRootDirectoryInfo", "ExportDatabaseSQLWithOptions", "ExportSchemaSQLWithOptions", "ApplyDataRootDirectory", "OpenDataRootDirectory", "SelectLogDirectory", "ApplyLogDirectory", "OpenLogDirectory", "SelectSavedQueryDirectory", "ApplySavedQueryDirectory", "OpenSavedQueryDirectory", "RevealSavedQueryInFolder", "SetApplicationBrandIcon", diff --git a/shared/i18n/de-DE.json b/shared/i18n/de-DE.json index ab33ce5e..7f6789f3 100644 --- a/shared/i18n/de-DE.json +++ b/shared/i18n/de-DE.json @@ -4419,17 +4419,101 @@ "data_grid.view.result_view": "Ergebnisansicht", "data_grid.view.table": "Tabelle", "data_grid.view.text": "Text", + "data_import.capability.loading": "Import-Unterstützung für diese Verbindung wird geprüft...", + "data_import.capability.details.formats": "Dateiformate", + "data_import.capability.details.encodings": "Zeichenkodierungen", + "data_import.capability.details.compressions": "Komprimierung", + "data_import.capability.details.directives": "Client-Anweisungen", + "data_import.capability.reason.capability_unavailable": "Informationen zur Import-Unterstützung sind nicht verfügbar. Dateiauswahl und Ausführung sind deaktiviert.", + "data_import.capability.reason.data_import_restricted": "Der Datenimport ist durch die Schutzrichtlinie dieser Verbindung deaktiviert.", + "data_import.capability.reason.database_runtime_unavailable": "Die Datenbanklaufzeit ist nicht verfügbar; die Import-Unterstützung kann nicht geprüft werden.", + "data_import.capability.reason.database_type_unsupported": "Dieser Datenbanktyp wird für den aktuellen Importmodus nicht unterstützt.", + "data_import.capability.reason.pinned_session_unavailable": "Der SQL-Dateiimport ist deaktiviert, weil dieser Treiber keine verifizierte gebundene Sitzung bereitstellen kann.", + "data_import.capability.reason.sql_file_import_restricted": "Der SQL-Dateiimport ist durch den Skript- oder Strukturschutz deaktiviert.", + "data_import.capability.reason.table_import_runtime_unavailable": "Die Laufzeit dieses Treibers stellt keine Schnittstelle für Tabellenimporte bereit.", + "data_import.capability.reason.unsupported": "Der aktuelle Importmodus ist für die ausgewählte Verbindung nicht verfügbar.", + "data_import.capability.rpc_failed": "Die Import-Unterstützung konnte nicht geprüft werden. Prüfen Sie die Verbindung und versuchen Sie es erneut.", + "data_import.history.action.delete": "Eintrag löschen", + "data_import.history.action.details": "Details", + "data_import.history.action.export_errors": "Abgelehnte Zeilen exportieren", + "data_import.history.action.refresh": "Aktualisieren", + "data_import.history.confirm.delete_content": "Diesen abgeschlossenen Importeintrag löschen? Exportieren Sie benötigte abgelehnte Zeilen vorher.", + "data_import.history.confirm.delete_title": "Importeintrag löschen", + "data_import.history.description": "Letzte Tabellen- und SQL-Importe anzeigen. Laufende Aufträge können nicht gelöscht werden.", + "data_import.history.detail.job_id": "Auftrags-ID: {{id}}", + "data_import.history.detail.outcome_unknown": "Der fehlgeschlagene Batch wurde möglicherweise teilweise geschrieben. Prüfen Sie das Ziel vor einem erneuten Import.", + "data_import.history.detail.stage": "Phase: {{stage}}", + "data_import.history.empty": "Kein Importverlauf", + "data_import.history.error.delete_failed": "Importeintrag konnte nicht gelöscht werden", + "data_import.history.error.delete_failed_detail": "Importeintrag konnte nicht gelöscht werden: {{detail}}", + "data_import.history.error.details_failed": "Importdetails konnten nicht geladen werden", + "data_import.history.error.details_failed_detail": "Importdetails konnten nicht geladen werden: {{detail}}", + "data_import.history.error.export_failed": "Abgelehnte Zeilen konnten nicht exportiert werden", + "data_import.history.error.export_failed_detail": "Abgelehnte Zeilen konnten nicht exportiert werden: {{detail}}", + "data_import.history.error.load_failed": "Importverlauf konnte nicht geladen werden", + "data_import.history.error.load_failed_detail": "Importverlauf konnte nicht geladen werden: {{detail}}", + "data_import.history.kind.sql": "SQL-Datei", + "data_import.history.kind.table": "Tabellendaten", + "data_import.history.message.deleted": "Importeintrag gelöscht", + "data_import.history.message.exported": "Abgelehnte Zeilen exportiert", + "data_import.history.progress": "Verarbeitet {{current}} · Erfolgreich {{success}} · Übersprungen {{skipped}} · Fehlgeschlagen {{failed}}", + "data_import.history.status.cancelled": "Abgebrochen", + "data_import.history.status.completed": "Abgeschlossen", + "data_import.history.status.failed": "Fehlgeschlagen", + "data_import.history.status.interrupted": "Unterbrochen", + "data_import.history.status.partial": "Mit Fehlern abgeschlossen", + "data_import.history.status.preparing": "Wird vorbereitet", + "data_import.history.status.running": "Wird ausgeführt", + "data_import.history.status.stopping": "Wird beendet", + "data_import.history.status.unknown": "Ergebnis unbekannt", + "data_import.history.title": "Importverlauf", "data_import.workbench.action.cancel_database_import": "Import abbrechen", "data_import.workbench.action.change_file": "Datei wechseln", "data_import.workbench.action.change_sql_file": "SQL-Datei wechseln", - "data_import.workbench.action.retry_database_import": "Datenbankimport wiederholen", + "data_import.workbench.action.retry_database_import": "Gesamte SQL-Datei erneut ausführen", "data_import.workbench.action.select_file": "Datei auswählen und Vorschau öffnen", "data_import.workbench.action.select_sql_file": "SQL-Datei auswählen", "data_import.workbench.action.start_database_import": "Datenbankimport starten", + "data_import.workbench.advanced.conflict.skip_duplicates": "Zeilen mit doppeltem Schlüssel überspringen", + "data_import.workbench.advanced.conflict.stop": "Bei Konflikt stoppen", + "data_import.workbench.advanced.conflict.upsert": "Einfügen oder aktualisieren", + "data_import.workbench.advanced.conflict_keys": "Konfliktschlüsselspalten", + "data_import.workbench.advanced.conflict_keys_not_mapped": "Konfliktschlüsselspalten müssen in den ausgewählten Spaltenzuordnungen enthalten sein: {{columns}}", + "data_import.workbench.advanced.conflict_keys_placeholder": "Zum Beispiel: id, tenant_id", + "data_import.workbench.advanced.conflict_keys_required": "Geben Sie für Einfügen oder Aktualisieren mindestens eine Konfliktschlüsselspalte an.", + "data_import.workbench.advanced.conflict_unsupported": "Diese Verbindung unterstützt die gespeicherte Richtlinie für doppelte Schlüssel nicht. Wählen Sie vor dem Import eine unterstützte Richtlinie.", + "data_import.workbench.advanced.conflict_policy": "Behandlung doppelter Schlüssel", + "data_import.workbench.advanced.delimiter": "CSV-Trennzeichen", + "data_import.workbench.advanced.delimiter.auto": "Automatisch erkennen", + "data_import.workbench.advanced.delimiter.comma": "Komma (,)", + "data_import.workbench.advanced.delimiter.pipe": "Senkrechter Strich (|)", + "data_import.workbench.advanced.delimiter.semicolon": "Semikolon (;)", + "data_import.workbench.advanced.delimiter.tab": "Tabulator", + "data_import.workbench.advanced.description": "Diese Einstellungen gelten für Vorschau und Import und werden auf diesem Gerät gespeichert.", + "data_import.workbench.advanced.empty_string_as_null": "Leere Zeichenfolgen als NULL behandeln", + "data_import.workbench.advanced.encoding": "Textkodierung", + "data_import.workbench.advanced.encoding.auto": "Automatisch erkennen", + "data_import.workbench.advanced.encoding.gb18030": "GB18030", + "data_import.workbench.advanced.encoding.utf16be": "UTF-16 BE", + "data_import.workbench.advanced.encoding.utf16le": "UTF-16 LE", + "data_import.workbench.advanced.encoding.utf8": "UTF-8", + "data_import.workbench.advanced.header_row": "Kopfzeile", + "data_import.workbench.advanced.null_token": "NULL-Wert (leer zum Deaktivieren)", + "data_import.workbench.advanced.sheet_name": "Arbeitsblattname (leer verwendet das erste Blatt)", + "data_import.workbench.advanced.title": "Erweiterte Optionen für Tabellenimporte", + "data_import.workbench.confirm.rerun_content": "Die gesamte SQL-Datei wird von Anfang an ausgeführt. Bereits bestätigte Anweisungen des vorherigen Laufs werden nicht zurückgesetzt und können erneut ausgeführt werden.", + "data_import.workbench.confirm.rerun_title": "Gesamte SQL-Datei erneut ausführen?", "data_import.workbench.description": "Zieltabelle und Datendatei auswählen, anschließend Vorschau, Spaltenzuordnung und Import durchführen.", "data_import.workbench.description.database": "Verbindung und SQL-Datei auswählen, um eine vollständige Datenbank zu importieren. Für reguläre SQL-Dateien kann optional eine Standarddatenbank angegeben werden.", - "data_import.workbench.helper.file_formats": "Unterstützt CSV-, JSON-, XLSX- und XLS-Dateien.", - "data_import.workbench.helper.sql_file": "Unterstützt .sql-Dateien. Die Dateiauswahl startet den Import nicht automatisch.", + "data_import.workbench.error_policy.continue": "Nach SQL-Fehlern fortfahren", + "data_import.workbench.error_policy.continue_description": "Fehler werden protokolliert, danach wird fortgesetzt. MySQL/MariaDB führt Anweisungen einzeln aus, damit fehlgeschlagene Batches nicht erneut ausgeführt werden; große Dateien sind dadurch langsamer, und bereits bestätigte Anweisungen werden nicht automatisch zurückgesetzt.", + "data_import.workbench.error_policy.continue_table": "Nach Zeilenfehlern fortfahren", + "data_import.workbench.error_policy.continue_table_description": "Schreibt von Anfang an zeilenweise und fährt nach protokollierten Fehlerzeilen fort. Fehlgeschlagene Batches werden weder vorab versucht noch wiederholt; große Dateien sind langsamer, und geschriebene Daten werden nicht automatisch zurückgesetzt.", + "data_import.workbench.error_policy.stop_description": "Empfohlen. Beim ersten SQL-Fehler stoppen, ohne den fehlgeschlagenen Batch zu wiederholen. Nicht transaktionale Tabellen können Teilschreibvorgänge behalten.", + "data_import.workbench.error_policy.stop_table_description": "Empfohlen. Batch-Schreibvorgänge verwenden und beim ersten fehlgeschlagenen Batch ohne Wiederholung stoppen. Wenn die API einen Fehler meldet, kann der Batch teilweise geschrieben worden sein; prüfen Sie die Zieltabelle.", + "data_import.workbench.error_policy.title": "Fehlerbehandlung", + "data_import.workbench.helper.file_formats": "Unterstützt CSV-, JSON- und XLSX-Dateien.", + "data_import.workbench.helper.sql_file": "Unterstützt .sql- und .sql.gz-Dateien. Die Dateiauswahl startet den Import nicht automatisch.", "data_import.workbench.label.connection": "Verbindung", "data_import.workbench.label.database": "Datenbank", "data_import.workbench.label.default_database": "Standarddatenbank (optional)", @@ -4446,7 +4530,9 @@ "data_import.workbench.mode.database": "Datenbank importieren", "data_import.workbench.mode.table": "Tabelle importieren", "data_import.workbench.notice.gonavi_mysql_restore": "MySQL-Sicherungen von GoNavi stellen die in der Datei benannte Datenbank wieder her. Andere SQL-Dateien können die oben optional ausgewählte Standarddatenbank verwenden.", + "data_import.workbench.notice.continue_on_error": "Im Fortsetzungsmodus werden Fehler protokolliert und spätere Anweisungen verarbeitet. MySQL/MariaDB führt Anweisungen einzeln aus und wiederholt keine fehlgeschlagenen Batches, damit nicht transaktionale Tabellen keine doppelten Schreibvorgänge erhalten; große Dateien können deutlich langsamer werden. Bereits bestätigte Anweisungen werden nicht automatisch zurückgesetzt.", "data_import.workbench.notice.partial_execution": "SQL-Anweisungen werden fortlaufend ausgeführt. Anweisungen, die vor einem Fehler oder Abbruch abgeschlossen wurden, werden nicht automatisch zurückgesetzt. Zielverbindung und Sicherung vorher prüfen.", + "data_import.workbench.notice.stop_on_error": "Beim ersten SQL-Fehler wird die Ausführung beendet und der fehlgeschlagene Batch nicht wiederholt. Ein Rollback wird versucht, aber nicht transaktionale Tabellen können frühere Schreibvorgänge behalten. Zuvor bestätigte Anweisungen werden nicht automatisch zurückgesetzt.", "data_import.workbench.placeholder.loading_databases": "Datenbanken werden geladen...", "data_import.workbench.placeholder.loading_tables": "Tabellen werden geladen...", "data_import.workbench.placeholder.select_connection": "Verbindung auswählen", @@ -4455,6 +4541,9 @@ "data_import.workbench.placeholder.select_default_database": "Optional: Standarddatenbank auswählen", "data_import.workbench.placeholder.select_table": "Zieltabelle auswählen", "data_import.workbench.progress.bytes": "Dateifortschritt: {{processed}} / {{total}}", + "data_import.workbench.progress.throughput": "Durchsatz: {{rate}}/s", + "data_import.workbench.progress.eta": "Restzeit: {{duration}}", + "data_import.workbench.progress.skipped": "{{count}} Zeilen übersprungen", "data_import.workbench.progress.statements": "Anweisungsfortschritt: {{executed}} ausgeführt, {{failed}} fehlgeschlagen", "data_import.workbench.section.target": "Importziel", "data_import.workbench.state.awaiting_file_description": "Nach Auswahl des Ziels eine Datendatei auswählen, um den Inhalt zu prüfen und Spalten zuzuordnen.", @@ -4464,6 +4553,7 @@ "data_import.workbench.state.cancelled": "Abgebrochen", "data_import.workbench.state.cancelling": "Wird abgebrochen...", "data_import.workbench.state.completed": "Import abgeschlossen", + "data_import.workbench.state.completed_with_errors": "Import mit Fehlern abgeschlossen", "data_import.workbench.state.failed": "Import fehlgeschlagen", "data_import.workbench.state.ready_sql_description": "Ziel und Risikohinweis prüfen und den Import anschließend manuell starten.", "data_import.workbench.state.ready_sql_title": "SQL-Datei bereit", @@ -5581,6 +5671,7 @@ "driver_manager.version.unlabeled": "Version ohne Kennzeichnung", "file.backend.dialog.export_connections": "Verbindungen exportieren", "file.backend.dialog.export_data": "Daten exportieren", + "file.backend.dialog.export_import_errors": "Abgelehnte Importzeilen exportieren", "file.backend.dialog.export_database_sql": "{{database}} exportieren (SQL)", "file.backend.dialog.export_query_result": "Abfrageergebnis exportieren", "file.backend.dialog.export_table": "{{table}} exportieren", @@ -5627,8 +5718,15 @@ "file.backend.error.import_excel_parse_failed": "Excel konnte nicht geparst werden: {{detail}}", "file.backend.error.import_excel_read_failed": "Excel konnte nicht gelesen werden: {{detail}}", "file.backend.error.import_file_empty": "Dateipfad darf nicht leer sein", + "file.backend.error.import_error_artifact_not_found": "Der Export abgelehnter Zeilen ist nicht verfügbar oder abgelaufen", + "file.backend.error.import_job_not_found": "Importauftrag nicht gefunden", + "file.backend.error.import_job_persist": "Status des Importauftrags konnte nicht gespeichert werden: {{detail}}", + "file.backend.error.import_job_running": "Der Importauftrag läuft noch und kann nicht gelöscht werden", + "file.backend.error.import_resume_unavailable": "Dieser Import kann nicht sicher fortgesetzt werden", + "file.backend.error.import_source_changed": "Die Importquelle wurde nach der Vorschau geändert. Wählen Sie die Datei erneut aus und öffnen Sie die Vorschau.", "file.backend.error.import_json_parse_failed": "JSON konnte nicht geparst werden: {{detail}}", "file.backend.error.import_json_root_not_array": "JSON-Wurzel muss ein Array sein", + "file.backend.error.import_stopped_on_error": "Der Tabellenimport wurde wegen eines Fehlers gestoppt. {{imported}} Zeilen wurden bestätigt importiert und {{failed}} Fehler protokolliert: {{detail}}", "file.backend.error.import_unsupported_format": "Nicht unterstütztes Dateiformat", "file.backend.error.invalid_export_mode": "Ungültiger Exportmodus", "file.backend.error.mysql_workbench_no_connections": "Im XML wurden keine gültigen Verbindungskonfigurationen gefunden", @@ -5656,11 +5754,14 @@ "file.backend.error.selected_path_not_sql_file": "Der ausgewählte Pfad ist keine SQL-Datei", "file.backend.error.sql_file_batch_execution_failed": "Batch-Ausführung ab Anweisung {{index}} fehlgeschlagen: {{detail}}", "file.backend.error.sql_file_batch_rollback_failed": "Batch-Ausführung fehlgeschlagen: {{detail}}; Rollback fehlgeschlagen: {{rollbackDetail}}", + "file.backend.error.sql_file_execution_failed_summary": "Die SQL-Dateiausführung wurde nach {{count}} Anweisungen unterbrochen: {{detail}}", "file.backend.error.sql_file_exists": "SQL-Datei ist bereits vorhanden", "file.backend.error.sql_file_extension_required": "Es werden nur SQL-Dateien unterstützt", "file.backend.error.sql_file_name_no_separator": "SQL-Dateiname darf keine Pfadtrennzeichen enthalten", "file.backend.error.sql_file_name_required": "SQL-Dateiname darf nicht leer sein", "file.backend.error.sql_file_statement_execution_failed": "Anweisung {{index}} fehlgeschlagen: {{detail}}", + "file.backend.error.sql_file_stopped_on_error_summary": "Die SQL-Dateiausführung wurde beendet. Ausgeführte Anweisungen: {{success}}; gefundene Fehler: {{failed}}. {{detail}}", + "file.backend.error.sql_file_unclosed_transaction": "Die SQL-Datei endete vor Abschluss der Transaktion. Der Import wurde beendet und die nicht bestätigte Transaktion wird bereinigt.", "file.backend.error.table_data_batch_limit": "Es können höchstens {{max}} Tabellen gleichzeitig verarbeitet werden; aktuell ausgewählt: {{count}}", "file.backend.error.table_data_clear_failed": "Leeren von {{table}} fehlgeschlagen: {{detail}}", "file.backend.error.table_data_clear_failed_partial": "Leeren von {{table}} fehlgeschlagen: {{detail}}. Warnung: Die ersten {{count}} Tabellen wurden bereits geleert und können nicht wiederhergestellt werden", @@ -5698,6 +5799,7 @@ "file.backend.message.export_completed": "Export abgeschlossen", "file.backend.message.import_no_data": "Keine importierbaren Daten", "file.backend.message.import_row_failed": "Importzeile {{index}} fehlgeschlagen: {{detail}}", + "file.backend.message.import_errors_exported": "Abgelehnte Importzeilen wurden exportiert", "file.backend.message.import_summary": "Importiert: {{imported}}, fehlgeschlagen: {{failed}}", "file.backend.message.statement_failed": "Anweisung {{index}} fehlgeschlagen: {{detail}}\n SQL: {{sql}}", "file.backend.message.table_data_clear_succeeded": "Leeren erfolgreich", @@ -5734,9 +5836,12 @@ "find_in_database.tooltip.collapse_details": "Details ausblenden", "find_in_database.tooltip.view_details": "Details anzeigen", "import_preview.action.start": "Import starten", + "import_preview.action.export_rejected_rows": "Abgelehnte Zeilen exportieren", "import_preview.error.connection_config_not_found": "Verbindungskonfiguration nicht gefunden", "import_preview.error.import_failed": "Import fehlgeschlagen", "import_preview.error.import_failed_detail": "Import fehlgeschlagen: {{detail}}", + "import_preview.error.export_rejected_rows_failed": "Abgelehnte Zeilen konnten nicht exportiert werden", + "import_preview.error.export_rejected_rows_failed_detail": "Abgelehnte Zeilen konnten nicht exportiert werden: {{detail}}", "import_preview.error.preview_failed": "Vorschau fehlgeschlagen", "import_preview.error.preview_failed_detail": "Vorschau fehlgeschlagen: {{detail}}", "import_preview.error.target_columns_failed": "Spalten der Zieltabelle konnten nicht geladen werden", @@ -5754,12 +5859,23 @@ "import_preview.preview.description": "Die ersten 5 Zeilen werden unten angezeigt. Starten Sie den Import nach der Prüfung der Daten.", "import_preview.preview.field_list": "Feldliste:", "import_preview.preview.summary": "{{rows}} Zeilen und {{columns}} Felder", + "import_preview.preview.summary_sample": "{{rows}} Beispielzeilen werden angezeigt; die Gesamtzahl wurde nicht ermittelt. {{columns}} Felder", "import_preview.preview.table_title": "Datenvorschau (erste 5 Zeilen):", "import_preview.progress.error_count": "Fehlgeschlagen {{count}}", "import_preview.progress.processed_rows": "{{current}} / {{total}} Zeilen verarbeitet", + "import_preview.progress.processed_rows_unknown": "{{current}} Zeilen verarbeitet", "import_preview.progress.success_count": "Erfolgreich {{count}}", + "import_preview.stage.prepare": "Import wird vorbereitet", + "import_preview.stage.preflight": "Vorabprüfung wird ausgeführt", + "import_preview.stage.read": "Quelldatei wird gelesen", + "import_preview.stage.parse": "Quelldatei wird analysiert", + "import_preview.stage.write": "Zeilen werden geschrieben", + "import_preview.stage.finalize": "Ergebnis wird abgeschlossen", "import_preview.result.completed": "Import abgeschlossen", + "import_preview.result.batch_outcome_unknown": "Der fehlgeschlagene Batch wurde möglicherweise teilweise geschrieben. Prüfen Sie vor einem erneuten Versuch die Zieltabelle.", "import_preview.result.error_logs": "Fehlerprotokolle:", + "import_preview.result.error_count": "{{count}} Fehler protokolliert", + "import_preview.result.error_logs_omitted": "{{count}} weitere Fehlerdetails werden nicht angezeigt.", "import_preview.result.failed_rows": "{{count}} Zeilen fehlgeschlagen", "import_preview.result.success_rows": "{{count}} Zeilen erfolgreich importiert", "import_preview.status.importing": "Daten werden importiert...", @@ -5770,6 +5886,7 @@ "jvm.backend.connection_error.agent.base_url_invalid.summary": "Agent-Verbindung fehlgeschlagen: Agent Base URL ist ungültig.", "jvm.backend.connection_error.agent.base_url_required": "Agent-Verbindung fehlgeschlagen: Agent Base URL fehlt.", "import_preview.result.stopped": "Import gestoppt (übernommene Daten wurden nicht zurückgerollt)", + "import_preview.result.stopped_on_error": "Import wegen eines Fehlers gestoppt", "jvm.backend.connection_error.agent.connection_refused.help": "Prüfe, ob der Java-Dienst den GoNavi Agent mit `-javaagent` gestartet hat, und prüfe Base URL, Port-Mapping und Firewall.", "jvm.backend.connection_error.agent.connection_refused.summary": "Agent-Verbindung fehlgeschlagen: Der Ziel-Agent-Management-Port lauscht nicht oder die Adresse ist nicht erreichbar.", "jvm.backend.connection_error.agent.forbidden.help": "Prüfe, ob Client-Quelle, Authentifizierung und Agent-Zugriffsrichtlinie GoNavi den Zugriff erlauben.", @@ -7731,12 +7848,13 @@ "sidebar.sql_file_exec.executed_label": "Ausgeführt: ", "sidebar.sql_file_exec.file_size": "Dateigröße: ", "sidebar.sql_file_exec.message.already_running": "Eine externe SQL-Ausführung läuft bereits", - "sidebar.sql_file_exec.rows_separator": " Zeilen | Fehlgeschlagen: ", - "sidebar.sql_file_exec.rows_suffix": " Zeilen", + "sidebar.sql_file_exec.statements_separator": " Anweisungen | Fehlgeschlagen: ", + "sidebar.sql_file_exec.statements_suffix": " Anweisungen", "sidebar.sql_file_exec.status.cancelled": "Abgebrochen", "sidebar.sql_file_exec.status.done": "Abgeschlossen", "sidebar.sql_file_exec.status.error": "Fehler", "sidebar.sql_file_exec.status.running": "Wird ausgeführt", + "sidebar.sql_file_exec.status.stopping": "Wird beendet...", "sidebar.sql_file_exec.status_label": "Status: ", "sidebar.sql_file_exec.title": "Externe SQL-Datei ausführen", "sidebar.sql_file_exec.workbench.action.clear_history": "Verlauf leeren", diff --git a/shared/i18n/en-US.json b/shared/i18n/en-US.json index 4db0697c..aeddba8a 100644 --- a/shared/i18n/en-US.json +++ b/shared/i18n/en-US.json @@ -4419,17 +4419,101 @@ "data_grid.view.result_view": "Result view", "data_grid.view.table": "Table", "data_grid.view.text": "Text", + "data_import.capability.loading": "Checking import support for this connection...", + "data_import.capability.details.formats": "File formats", + "data_import.capability.details.encodings": "Encodings", + "data_import.capability.details.compressions": "Compression", + "data_import.capability.details.directives": "Client directives", + "data_import.capability.reason.capability_unavailable": "Import support information is unavailable. File selection and execution are disabled.", + "data_import.capability.reason.data_import_restricted": "Data import is disabled by this connection's protection policy.", + "data_import.capability.reason.database_runtime_unavailable": "The database runtime is unavailable, so import support cannot be verified.", + "data_import.capability.reason.database_type_unsupported": "This database type is not supported for this import mode.", + "data_import.capability.reason.pinned_session_unavailable": "SQL file import is disabled because this driver cannot provide a verified pinned session.", + "data_import.capability.reason.sql_file_import_restricted": "SQL file import is disabled by script or structure protection.", + "data_import.capability.reason.table_import_runtime_unavailable": "This driver runtime does not provide the table import interface.", + "data_import.capability.reason.unsupported": "This import mode is unavailable for the selected connection.", + "data_import.capability.rpc_failed": "Import support could not be verified. Check the connection and try again.", + "data_import.history.action.delete": "Delete record", + "data_import.history.action.details": "Details", + "data_import.history.action.export_errors": "Export rejected rows", + "data_import.history.action.refresh": "Refresh", + "data_import.history.confirm.delete_content": "Delete this completed import record? Export any rejected rows first if you need them.", + "data_import.history.confirm.delete_title": "Delete import record", + "data_import.history.description": "Review recent table and SQL imports. Running jobs cannot be deleted.", + "data_import.history.detail.job_id": "Job ID: {{id}}", + "data_import.history.detail.outcome_unknown": "The failed batch may have been partially written. Verify the target before importing again.", + "data_import.history.detail.stage": "Stage: {{stage}}", + "data_import.history.empty": "No import history", + "data_import.history.error.delete_failed": "Failed to delete the import record", + "data_import.history.error.delete_failed_detail": "Failed to delete the import record: {{detail}}", + "data_import.history.error.details_failed": "Failed to load import details", + "data_import.history.error.details_failed_detail": "Failed to load import details: {{detail}}", + "data_import.history.error.export_failed": "Failed to export rejected rows", + "data_import.history.error.export_failed_detail": "Failed to export rejected rows: {{detail}}", + "data_import.history.error.load_failed": "Failed to load import history", + "data_import.history.error.load_failed_detail": "Failed to load import history: {{detail}}", + "data_import.history.kind.sql": "SQL file", + "data_import.history.kind.table": "Table data", + "data_import.history.message.deleted": "Import record deleted", + "data_import.history.message.exported": "Rejected rows exported", + "data_import.history.progress": "Processed {{current}} · Succeeded {{success}} · Skipped {{skipped}} · Failed {{failed}}", + "data_import.history.status.cancelled": "Cancelled", + "data_import.history.status.completed": "Completed", + "data_import.history.status.failed": "Failed", + "data_import.history.status.interrupted": "Interrupted", + "data_import.history.status.partial": "Completed with errors", + "data_import.history.status.preparing": "Preparing", + "data_import.history.status.running": "Running", + "data_import.history.status.stopping": "Stopping", + "data_import.history.status.unknown": "Outcome unknown", + "data_import.history.title": "Import history", "data_import.workbench.action.cancel_database_import": "Cancel import", "data_import.workbench.action.change_file": "Change file", "data_import.workbench.action.change_sql_file": "Change SQL file", - "data_import.workbench.action.retry_database_import": "Retry database import", + "data_import.workbench.action.retry_database_import": "Run the entire SQL file again", "data_import.workbench.action.select_file": "Select file and preview", "data_import.workbench.action.select_sql_file": "Select SQL file", "data_import.workbench.action.start_database_import": "Start database import", + "data_import.workbench.advanced.conflict.skip_duplicates": "Skip duplicate-key rows", + "data_import.workbench.advanced.conflict.stop": "Stop on conflict", + "data_import.workbench.advanced.conflict.upsert": "Insert or update", + "data_import.workbench.advanced.conflict_keys": "Conflict key columns", + "data_import.workbench.advanced.conflict_keys_not_mapped": "Conflict key columns must be included in the selected mappings: {{columns}}", + "data_import.workbench.advanced.conflict_keys_placeholder": "For example: id, tenant_id", + "data_import.workbench.advanced.conflict_keys_required": "Select at least one conflict key column for insert-or-update imports.", + "data_import.workbench.advanced.conflict_unsupported": "This connection does not support the saved duplicate-key policy. Select a supported policy before importing.", + "data_import.workbench.advanced.conflict_policy": "Duplicate-key handling", + "data_import.workbench.advanced.delimiter": "CSV delimiter", + "data_import.workbench.advanced.delimiter.auto": "Detect automatically", + "data_import.workbench.advanced.delimiter.comma": "Comma (,)", + "data_import.workbench.advanced.delimiter.pipe": "Pipe (|)", + "data_import.workbench.advanced.delimiter.semicolon": "Semicolon (;)", + "data_import.workbench.advanced.delimiter.tab": "Tab", + "data_import.workbench.advanced.description": "These settings are used for both preview and import and are saved on this device.", + "data_import.workbench.advanced.empty_string_as_null": "Treat empty strings as NULL", + "data_import.workbench.advanced.encoding": "Text encoding", + "data_import.workbench.advanced.encoding.auto": "Detect automatically", + "data_import.workbench.advanced.encoding.gb18030": "GB18030", + "data_import.workbench.advanced.encoding.utf16be": "UTF-16 BE", + "data_import.workbench.advanced.encoding.utf16le": "UTF-16 LE", + "data_import.workbench.advanced.encoding.utf8": "UTF-8", + "data_import.workbench.advanced.header_row": "Header row", + "data_import.workbench.advanced.null_token": "NULL token (blank disables it)", + "data_import.workbench.advanced.sheet_name": "Worksheet name (blank uses the first sheet)", + "data_import.workbench.advanced.title": "Advanced table import options", + "data_import.workbench.confirm.rerun_content": "This starts the entire SQL file from the beginning. Statements committed by the previous run are not rolled back and may be executed again.", + "data_import.workbench.confirm.rerun_title": "Run the entire SQL file again?", "data_import.workbench.description": "Select a target table and data file, then preview, map columns, and import.", "data_import.workbench.description.database": "Select a connection and SQL file to import a complete database. A default database is optional for regular SQL files.", - "data_import.workbench.helper.file_formats": "Supports CSV, JSON, XLSX, and XLS files.", - "data_import.workbench.helper.sql_file": "Supports .sql files. Selecting a file does not start the import automatically.", + "data_import.workbench.error_policy.continue": "Continue after SQL errors", + "data_import.workbench.error_policy.continue_description": "Records errors and continues. MySQL/MariaDB execute statements one at a time to avoid replaying failed batches; large files are slower, and committed statements are not rolled back automatically.", + "data_import.workbench.error_policy.continue_table": "Continue after row errors", + "data_import.workbench.error_policy.continue_table_description": "Writes one row at a time from the start and continues after logging failed rows. Failed batches are never attempted or replayed; large files are slower, and written data is not rolled back automatically.", + "data_import.workbench.error_policy.stop_description": "Recommended. Stop at the first SQL error without replaying the failed batch. Non-transactional tables may still keep partial writes.", + "data_import.workbench.error_policy.stop_table_description": "Recommended. Use batch writes and stop at the first failed batch without replaying it. A batch may have been partially written when its API returns an error; verify the target table.", + "data_import.workbench.error_policy.title": "Error handling", + "data_import.workbench.helper.file_formats": "Supports CSV, JSON, and XLSX files.", + "data_import.workbench.helper.sql_file": "Supports .sql and .sql.gz files. Selecting a file does not start the import automatically.", "data_import.workbench.label.connection": "Connection", "data_import.workbench.label.database": "Database", "data_import.workbench.label.default_database": "Default database (optional)", @@ -4446,7 +4530,9 @@ "data_import.workbench.mode.database": "Import database", "data_import.workbench.mode.table": "Import table", "data_import.workbench.notice.gonavi_mysql_restore": "GoNavi MySQL backups restore the database named in the file. Other SQL files can use the optional default database selected above.", + "data_import.workbench.notice.continue_on_error": "Continue mode records errors and processes later statements. To prevent duplicate writes to non-transactional tables, MySQL/MariaDB execute statements one at a time and never replay failed batches; large files can be noticeably slower. Committed statements are not rolled back automatically.", "data_import.workbench.notice.partial_execution": "SQL statements are executed as a stream. Statements completed before a failure or cancellation are not rolled back automatically. Verify the target connection and backup first.", + "data_import.workbench.notice.stop_on_error": "Execution stops at the first SQL error and does not replay the failed batch. A rollback is attempted, but non-transactional tables may keep earlier writes. Previously committed statements are not rolled back automatically.", "data_import.workbench.placeholder.loading_databases": "Loading databases...", "data_import.workbench.placeholder.loading_tables": "Loading tables...", "data_import.workbench.placeholder.select_connection": "Select a connection", @@ -4455,6 +4541,9 @@ "data_import.workbench.placeholder.select_default_database": "Optional: select a default database", "data_import.workbench.placeholder.select_table": "Select a target table", "data_import.workbench.progress.bytes": "File progress: {{processed}} / {{total}}", + "data_import.workbench.progress.throughput": "Throughput: {{rate}}/s", + "data_import.workbench.progress.eta": "ETA: {{duration}}", + "data_import.workbench.progress.skipped": "Skipped {{count}} rows", "data_import.workbench.progress.statements": "Statement progress: {{executed}} executed, {{failed}} failed", "data_import.workbench.section.target": "Import target", "data_import.workbench.state.awaiting_file_description": "After choosing a target, select a data file to preview its contents and configure column mappings.", @@ -4464,6 +4553,7 @@ "data_import.workbench.state.cancelled": "Cancelled", "data_import.workbench.state.cancelling": "Cancelling...", "data_import.workbench.state.completed": "Import completed", + "data_import.workbench.state.completed_with_errors": "Import completed with errors", "data_import.workbench.state.failed": "Import failed", "data_import.workbench.state.ready_sql_description": "Verify the target and risk notice, then start the import manually.", "data_import.workbench.state.ready_sql_title": "SQL file ready", @@ -5581,6 +5671,7 @@ "driver_manager.version.unlabeled": "Unlabeled version", "file.backend.dialog.export_connections": "Export connections", "file.backend.dialog.export_data": "Export data", + "file.backend.dialog.export_import_errors": "Export rejected import rows", "file.backend.dialog.export_database_sql": "Export {{database}} (SQL)", "file.backend.dialog.export_query_result": "Export query result", "file.backend.dialog.export_table": "Export {{table}}", @@ -5627,8 +5718,15 @@ "file.backend.error.import_excel_parse_failed": "Unable to parse Excel: {{detail}}", "file.backend.error.import_excel_read_failed": "Unable to read Excel: {{detail}}", "file.backend.error.import_file_empty": "File path cannot be empty", + "file.backend.error.import_error_artifact_not_found": "The rejected-row export is unavailable or has expired", + "file.backend.error.import_job_not_found": "Import job not found", + "file.backend.error.import_job_persist": "Failed to save import job state: {{detail}}", + "file.backend.error.import_job_running": "The import job is still running and cannot be deleted", + "file.backend.error.import_resume_unavailable": "This import cannot be resumed safely", + "file.backend.error.import_source_changed": "The import source changed after preview. Select and preview the file again.", "file.backend.error.import_json_parse_failed": "Unable to parse JSON: {{detail}}", "file.backend.error.import_json_root_not_array": "JSON root must be array", + "file.backend.error.import_stopped_on_error": "Table import stopped on error. {{imported}} rows were confirmed imported and {{failed}} error(s) were recorded: {{detail}}", "file.backend.error.import_unsupported_format": "Unsupported file format", "file.backend.error.invalid_export_mode": "Invalid export mode", "file.backend.error.mysql_workbench_no_connections": "No valid connection profiles were found in the XML", @@ -5656,11 +5754,14 @@ "file.backend.error.selected_path_not_sql_file": "Selected path is not a SQL file", "file.backend.error.sql_file_batch_execution_failed": "Batch execution failed starting at statement {{index}}: {{detail}}", "file.backend.error.sql_file_batch_rollback_failed": "Batch execution failed: {{detail}}; rollback failed: {{rollbackDetail}}", + "file.backend.error.sql_file_execution_failed_summary": "SQL file execution was interrupted after {{count}} statements: {{detail}}", "file.backend.error.sql_file_exists": "SQL file already exists", "file.backend.error.sql_file_extension_required": "Only SQL files are supported", "file.backend.error.sql_file_name_no_separator": "SQL file name cannot contain path separators", "file.backend.error.sql_file_name_required": "SQL file name cannot be empty", "file.backend.error.sql_file_statement_execution_failed": "Statement {{index}} failed: {{detail}}", + "file.backend.error.sql_file_stopped_on_error_summary": "SQL file execution stopped. Statements executed: {{success}}; errors found: {{failed}}. {{detail}}", + "file.backend.error.sql_file_unclosed_transaction": "The SQL file ended before the transaction was closed. The import stopped and the uncommitted transaction will be cleaned up.", "file.backend.error.table_data_batch_limit": "You can process at most {{max}} tables at once; currently selected {{count}}", "file.backend.error.table_data_clear_failed": "Failed to clear {{table}}: {{detail}}", "file.backend.error.table_data_clear_failed_partial": "Failed to clear {{table}}: {{detail}}. Warning: the first {{count}} tables have been cleared and cannot be restored", @@ -5698,6 +5799,7 @@ "file.backend.message.export_completed": "Export completed", "file.backend.message.import_no_data": "No data to import", "file.backend.message.import_row_failed": "Import row {{index}} failed: {{detail}}", + "file.backend.message.import_errors_exported": "Rejected import rows exported", "file.backend.message.import_summary": "Imported: {{imported}}, Failed: {{failed}}", "file.backend.message.statement_failed": "Statement {{index}} failed: {{detail}}\n SQL: {{sql}}", "file.backend.message.table_data_clear_succeeded": "Clearing succeeded", @@ -5734,9 +5836,12 @@ "find_in_database.tooltip.collapse_details": "Collapse details", "find_in_database.tooltip.view_details": "View details", "import_preview.action.start": "Start import", + "import_preview.action.export_rejected_rows": "Export rejected rows", "import_preview.error.connection_config_not_found": "Connection configuration not found", "import_preview.error.import_failed": "Import failed", "import_preview.error.import_failed_detail": "Import failed: {{detail}}", + "import_preview.error.export_rejected_rows_failed": "Could not export rejected rows", + "import_preview.error.export_rejected_rows_failed_detail": "Could not export rejected rows: {{detail}}", "import_preview.error.preview_failed": "Preview failed", "import_preview.error.preview_failed_detail": "Preview failed: {{detail}}", "import_preview.error.target_columns_failed": "Could not load target table columns", @@ -5754,12 +5859,23 @@ "import_preview.preview.description": "The first 5 rows are shown below. Start the import after confirming the data.", "import_preview.preview.field_list": "Field list:", "import_preview.preview.summary": "{{rows}} rows and {{columns}} fields", + "import_preview.preview.summary_sample": "Showing {{rows}} sample rows; total row count was not scanned. {{columns}} fields", "import_preview.preview.table_title": "Data preview (first 5 rows):", "import_preview.progress.error_count": "Failed {{count}}", "import_preview.progress.processed_rows": "Processed {{current}} / {{total}} rows", + "import_preview.progress.processed_rows_unknown": "Processed {{current}} rows", "import_preview.progress.success_count": "Success {{count}}", + "import_preview.stage.prepare": "Preparing import", + "import_preview.stage.preflight": "Running preflight checks", + "import_preview.stage.read": "Reading source file", + "import_preview.stage.parse": "Parsing source file", + "import_preview.stage.write": "Writing rows", + "import_preview.stage.finalize": "Finalizing result", "import_preview.result.completed": "Import completed", + "import_preview.result.batch_outcome_unknown": "The failed batch may have been partially written. Verify the target table before retrying.", "import_preview.result.error_logs": "Error logs:", + "import_preview.result.error_count": "Errors recorded: {{count}}", + "import_preview.result.error_logs_omitted": "{{count}} more error details are not shown.", "import_preview.result.failed_rows": "Failed {{count}} rows", "import_preview.result.success_rows": "Successfully imported {{count}} rows", "import_preview.status.importing": "Importing data...", @@ -5770,6 +5886,7 @@ "jvm.backend.connection_error.agent.base_url_invalid.summary": "Agent connection failed: Agent Base URL is invalid.", "jvm.backend.connection_error.agent.base_url_required": "Agent connection failed: Agent Base URL is missing.", "import_preview.result.stopped": "Import stopped (committed data was not rolled back)", + "import_preview.result.stopped_on_error": "Import stopped on error", "jvm.backend.connection_error.agent.connection_refused.help": "Confirm the Java service started GoNavi Agent with `-javaagent`, and check Base URL, port mapping, and firewall rules.", "jvm.backend.connection_error.agent.connection_refused.summary": "Agent connection failed: the target Agent management port is not listening, or the address is unreachable.", "jvm.backend.connection_error.agent.forbidden.help": "Confirm that client source, authentication settings, and Agent access policy allow GoNavi to access it.", @@ -7731,12 +7848,13 @@ "sidebar.sql_file_exec.executed_label": "Executed: ", "sidebar.sql_file_exec.file_size": "File size: ", "sidebar.sql_file_exec.message.already_running": "An external SQL execution task is already running", - "sidebar.sql_file_exec.rows_separator": " rows | Failed: ", - "sidebar.sql_file_exec.rows_suffix": " rows", + "sidebar.sql_file_exec.statements_separator": " statements | Failed: ", + "sidebar.sql_file_exec.statements_suffix": " statements", "sidebar.sql_file_exec.status.cancelled": "Cancelled", "sidebar.sql_file_exec.status.done": "Done", "sidebar.sql_file_exec.status.error": "Error", "sidebar.sql_file_exec.status.running": "Running", + "sidebar.sql_file_exec.status.stopping": "Stopping...", "sidebar.sql_file_exec.status_label": "Status: ", "sidebar.sql_file_exec.title": "Run external SQL file", "sidebar.sql_file_exec.workbench.action.clear_history": "Clear history", diff --git a/shared/i18n/ja-JP.json b/shared/i18n/ja-JP.json index 7cdb14f7..ea758ef6 100644 --- a/shared/i18n/ja-JP.json +++ b/shared/i18n/ja-JP.json @@ -4419,17 +4419,101 @@ "data_grid.view.result_view": "結果ビュー", "data_grid.view.table": "テーブル", "data_grid.view.text": "テキスト", + "data_import.capability.loading": "選択した接続のインポート対応状況を確認しています...", + "data_import.capability.details.formats": "ファイル形式", + "data_import.capability.details.encodings": "文字エンコーディング", + "data_import.capability.details.compressions": "圧縮形式", + "data_import.capability.details.directives": "クライアントディレクティブ", + "data_import.capability.reason.capability_unavailable": "インポート対応情報を取得できないため、ファイル選択と実行を無効にしました。", + "data_import.capability.reason.data_import_restricted": "この接続の保護ポリシーにより、データインポートは無効です。", + "data_import.capability.reason.database_runtime_unavailable": "データベースランタイムを利用できないため、インポート対応状況を確認できません。", + "data_import.capability.reason.database_type_unsupported": "現在のインポートモードでは、このデータベース種別をサポートしていません。", + "data_import.capability.reason.pinned_session_unavailable": "このドライバーでは固定セッションを検証できないため、SQL ファイルのインポートは無効です。", + "data_import.capability.reason.sql_file_import_restricted": "スクリプトまたは構造保護により、SQL ファイルのインポートは無効です。", + "data_import.capability.reason.table_import_runtime_unavailable": "このドライバーランタイムにはテーブルインポート用インターフェースがありません。", + "data_import.capability.reason.unsupported": "選択した接続では、現在のインポートモードを利用できません。", + "data_import.capability.rpc_failed": "インポート対応状況を確認できませんでした。接続を確認して再試行してください。", + "data_import.history.action.delete": "履歴を削除", + "data_import.history.action.details": "詳細", + "data_import.history.action.export_errors": "失敗行をエクスポート", + "data_import.history.action.refresh": "更新", + "data_import.history.confirm.delete_content": "完了したインポート履歴を削除しますか？必要な失敗行は先にエクスポートしてください。", + "data_import.history.confirm.delete_title": "インポート履歴を削除", + "data_import.history.description": "最近のテーブルおよび SQL ファイルのインポートを確認します。実行中のジョブは削除できません。", + "data_import.history.detail.job_id": "ジョブ ID：{{id}}", + "data_import.history.detail.outcome_unknown": "失敗したバッチが一部書き込まれた可能性があります。再インポート前に対象を確認してください。", + "data_import.history.detail.stage": "ステージ：{{stage}}", + "data_import.history.empty": "インポート履歴はありません", + "data_import.history.error.delete_failed": "インポート履歴を削除できませんでした", + "data_import.history.error.delete_failed_detail": "インポート履歴を削除できませんでした：{{detail}}", + "data_import.history.error.details_failed": "インポート詳細を読み込めませんでした", + "data_import.history.error.details_failed_detail": "インポート詳細を読み込めませんでした：{{detail}}", + "data_import.history.error.export_failed": "失敗行をエクスポートできませんでした", + "data_import.history.error.export_failed_detail": "失敗行をエクスポートできませんでした：{{detail}}", + "data_import.history.error.load_failed": "インポート履歴を読み込めませんでした", + "data_import.history.error.load_failed_detail": "インポート履歴を読み込めませんでした：{{detail}}", + "data_import.history.kind.sql": "SQL ファイル", + "data_import.history.kind.table": "テーブルデータ", + "data_import.history.message.deleted": "インポート履歴を削除しました", + "data_import.history.message.exported": "失敗行をエクスポートしました", + "data_import.history.progress": "処理 {{current}} · 成功 {{success}} · スキップ {{skipped}} · 失敗 {{failed}}", + "data_import.history.status.cancelled": "キャンセル済み", + "data_import.history.status.completed": "完了", + "data_import.history.status.failed": "失敗", + "data_import.history.status.interrupted": "中断", + "data_import.history.status.partial": "エラーありで完了", + "data_import.history.status.preparing": "準備中", + "data_import.history.status.running": "実行中", + "data_import.history.status.stopping": "停止中", + "data_import.history.status.unknown": "結果不明", + "data_import.history.title": "インポート履歴", "data_import.workbench.action.cancel_database_import": "インポートをキャンセル", "data_import.workbench.action.change_file": "ファイルを変更", "data_import.workbench.action.change_sql_file": "SQL ファイルを変更", - "data_import.workbench.action.retry_database_import": "データベースのインポートを再実行", + "data_import.workbench.action.retry_database_import": "SQL ファイル全体を再実行", "data_import.workbench.action.select_file": "ファイルを選択してプレビュー", "data_import.workbench.action.select_sql_file": "SQL ファイルを選択", "data_import.workbench.action.start_database_import": "データベースのインポートを開始", + "data_import.workbench.advanced.conflict.skip_duplicates": "重複キー行をスキップ", + "data_import.workbench.advanced.conflict.stop": "競合時に停止", + "data_import.workbench.advanced.conflict.upsert": "挿入または更新", + "data_import.workbench.advanced.conflict_keys": "競合キー列", + "data_import.workbench.advanced.conflict_keys_not_mapped": "競合キー列を選択済みの列マッピングに含めてください：{{columns}}", + "data_import.workbench.advanced.conflict_keys_placeholder": "例：id, tenant_id", + "data_import.workbench.advanced.conflict_keys_required": "挿入または更新では、競合キー列を少なくとも 1 つ指定してください。", + "data_import.workbench.advanced.conflict_unsupported": "この接続では保存済みの重複キーポリシーを利用できません。対応するポリシーを選択してください。", + "data_import.workbench.advanced.conflict_policy": "重複キーの処理", + "data_import.workbench.advanced.delimiter": "CSV 区切り文字", + "data_import.workbench.advanced.delimiter.auto": "自動検出", + "data_import.workbench.advanced.delimiter.comma": "カンマ（,）", + "data_import.workbench.advanced.delimiter.pipe": "パイプ（|）", + "data_import.workbench.advanced.delimiter.semicolon": "セミコロン（;）", + "data_import.workbench.advanced.delimiter.tab": "タブ", + "data_import.workbench.advanced.description": "この設定はプレビューとインポートの両方で使用され、この端末に保存されます。", + "data_import.workbench.advanced.empty_string_as_null": "空文字列を NULL として扱う", + "data_import.workbench.advanced.encoding": "文字エンコーディング", + "data_import.workbench.advanced.encoding.auto": "自動検出", + "data_import.workbench.advanced.encoding.gb18030": "GB18030", + "data_import.workbench.advanced.encoding.utf16be": "UTF-16 BE", + "data_import.workbench.advanced.encoding.utf16le": "UTF-16 LE", + "data_import.workbench.advanced.encoding.utf8": "UTF-8", + "data_import.workbench.advanced.header_row": "ヘッダー行", + "data_import.workbench.advanced.null_token": "NULL トークン（空欄で無効）", + "data_import.workbench.advanced.sheet_name": "ワークシート名（空欄では先頭シート）", + "data_import.workbench.advanced.title": "テーブルインポートの詳細オプション", + "data_import.workbench.confirm.rerun_content": "SQL ファイル全体を先頭から実行します。前回の実行でコミットされた文はロールバックされず、再実行される可能性があります。", + "data_import.workbench.confirm.rerun_title": "SQL ファイル全体を再実行しますか？", "data_import.workbench.description": "対象テーブルとデータファイルを選択し、プレビュー、列マッピング、インポートを実行します。", "data_import.workbench.description.database": "接続と SQL ファイルを選択してデータベース全体をインポートします。通常の SQL ファイルでは既定のデータベースを任意で指定できます。", - "data_import.workbench.helper.file_formats": "CSV、JSON、XLSX、XLS ファイルに対応しています。", - "data_import.workbench.helper.sql_file": ".sql ファイルに対応しています。ファイルを選択してもインポートは自動的に開始されません。", + "data_import.workbench.error_policy.continue": "SQL エラー後も実行を続ける", + "data_import.workbench.error_policy.continue_description": "エラーを記録して続行します。失敗したバッチの再実行を避けるため、MySQL/MariaDB では文を1件ずつ実行します。大きなファイルでは遅くなり、コミット済みの文は自動でロールバックされません。", + "data_import.workbench.error_policy.continue_table": "行エラー後もインポートを続ける", + "data_import.workbench.error_policy.continue_table_description": "最初から1行ずつ書き込み、失敗行を記録して続行します。失敗バッチを先に試行または再実行しないため、大きなファイルでは遅くなり、書き込み済みデータは自動でロールバックされません。", + "data_import.workbench.error_policy.stop_description": "推奨。最初の SQL エラーで停止し、失敗したバッチを再実行しません。非トランザクションテーブルでは部分書き込みが残る可能性があります。", + "data_import.workbench.error_policy.stop_table_description": "推奨。バッチ書き込みを使用し、最初の失敗バッチで再実行せず停止します。API がエラーを返した時点で一部が書き込まれている可能性があるため、対象テーブルを確認してください。", + "data_import.workbench.error_policy.title": "エラー処理", + "data_import.workbench.helper.file_formats": "CSV、JSON、XLSX ファイルに対応しています。", + "data_import.workbench.helper.sql_file": ".sql と .sql.gz ファイルに対応しています。ファイルを選択してもインポートは自動的に開始されません。", "data_import.workbench.label.connection": "接続", "data_import.workbench.label.database": "データベース", "data_import.workbench.label.default_database": "既定のデータベース（任意）", @@ -4446,7 +4530,9 @@ "data_import.workbench.mode.database": "データベースをインポート", "data_import.workbench.mode.table": "テーブルをインポート", "data_import.workbench.notice.gonavi_mysql_restore": "GoNavi で作成した MySQL バックアップは、ファイル内に記録されたデータベース名で復元されます。その他の SQL ファイルでは、上で選択した任意の既定データベースを使用できます。", + "data_import.workbench.notice.continue_on_error": "継続モードではエラーを記録して後続の文を処理します。MySQL/MariaDB では非トランザクションテーブルへの重複書き込みを避けるため、文を1件ずつ実行し、失敗したバッチを再実行しません。大きなファイルでは大幅に遅くなる場合があります。コミット済みの文は自動でロールバックされません。", "data_import.workbench.notice.partial_execution": "SQL ファイルはステートメント単位でストリーミング実行されます。失敗またはキャンセルまでに完了したステートメントは自動的にロールバックされません。対象の接続とバックアップを事前に確認してください。", + "data_import.workbench.notice.stop_on_error": "最初の SQL エラーで停止し、失敗したバッチを再実行しません。ロールバックを試みますが、非トランザクションテーブルでは以前の書き込みが残る場合があります。コミット済みの文は自動的にロールバックされません。", "data_import.workbench.placeholder.loading_databases": "データベースを読み込み中...", "data_import.workbench.placeholder.loading_tables": "テーブルを読み込み中...", "data_import.workbench.placeholder.select_connection": "接続を選択", @@ -4455,6 +4541,9 @@ "data_import.workbench.placeholder.select_default_database": "任意: 既定のデータベースを選択", "data_import.workbench.placeholder.select_table": "対象テーブルを選択", "data_import.workbench.progress.bytes": "ファイルの進捗: {{processed}} / {{total}}", + "data_import.workbench.progress.throughput": "速度: {{rate}}/秒", + "data_import.workbench.progress.eta": "残り時間: {{duration}}", + "data_import.workbench.progress.skipped": "{{count}} 行をスキップ", "data_import.workbench.progress.statements": "ステートメントの進捗: 実行済み {{executed}} 件、失敗 {{failed}} 件", "data_import.workbench.section.target": "インポート先", "data_import.workbench.state.awaiting_file_description": "対象を選択した後、データファイルを選択して内容をプレビューし、列マッピングを設定します。", @@ -4464,6 +4553,7 @@ "data_import.workbench.state.cancelled": "キャンセル済み", "data_import.workbench.state.cancelling": "キャンセル中...", "data_import.workbench.state.completed": "インポート完了", + "data_import.workbench.state.completed_with_errors": "エラーありでインポート完了", "data_import.workbench.state.failed": "インポート失敗", "data_import.workbench.state.ready_sql_description": "対象とリスクに関する注意事項を確認し、手動でインポートを開始してください。", "data_import.workbench.state.ready_sql_title": "SQL ファイルの準備ができました", @@ -5581,6 +5671,7 @@ "driver_manager.version.unlabeled": "バージョン未記載", "file.backend.dialog.export_connections": "接続をエクスポート", "file.backend.dialog.export_data": "データをエクスポート", + "file.backend.dialog.export_import_errors": "インポート失敗行をエクスポート", "file.backend.dialog.export_database_sql": "{{database}} をエクスポート (SQL)", "file.backend.dialog.export_query_result": "クエリ結果をエクスポート", "file.backend.dialog.export_table": "{{table}} をエクスポート", @@ -5627,8 +5718,15 @@ "file.backend.error.import_excel_parse_failed": "Excel を解析できません: {{detail}}", "file.backend.error.import_excel_read_failed": "Excel を読み取れません: {{detail}}", "file.backend.error.import_file_empty": "ファイルパスは空にできません", + "file.backend.error.import_error_artifact_not_found": "失敗行のエクスポートデータが存在しないか、有効期限が切れています", + "file.backend.error.import_job_not_found": "インポートジョブが見つかりません", + "file.backend.error.import_job_persist": "インポートジョブの状態を保存できませんでした：{{detail}}", + "file.backend.error.import_job_running": "インポートジョブは実行中のため削除できません", + "file.backend.error.import_resume_unavailable": "このインポートは安全に再開できません", + "file.backend.error.import_source_changed": "プレビュー後にインポート元が変更されました。ファイルを選択し直してプレビューしてください。", "file.backend.error.import_json_parse_failed": "JSON を解析できません: {{detail}}", "file.backend.error.import_json_root_not_array": "JSON ルートは配列である必要があります", + "file.backend.error.import_stopped_on_error": "テーブルデータのインポートはエラーで停止しました。{{imported}} 行のインポートを確認し、{{failed}} 件のエラーを記録しました: {{detail}}", "file.backend.error.import_unsupported_format": "サポートされていないファイル形式です", "file.backend.error.invalid_export_mode": "無効なエクスポートモードです", "file.backend.error.mysql_workbench_no_connections": "XML 内に有効な接続設定が見つかりません", @@ -5656,11 +5754,14 @@ "file.backend.error.selected_path_not_sql_file": "選択されたパスは SQL ファイルではありません", "file.backend.error.sql_file_batch_execution_failed": "{{index}} 件目からのバッチ文の実行に失敗しました: {{detail}}", "file.backend.error.sql_file_batch_rollback_failed": "バッチ実行に失敗しました: {{detail}}。ロールバックにも失敗しました: {{rollbackDetail}}", + "file.backend.error.sql_file_execution_failed_summary": "SQL ファイルの実行は {{count}} 件の処理後に中断されました: {{detail}}", "file.backend.error.sql_file_exists": "SQL ファイルは既に存在します", "file.backend.error.sql_file_extension_required": "SQL ファイルのみサポートしています", "file.backend.error.sql_file_name_no_separator": "SQL ファイル名にパス区切り文字は使用できません", "file.backend.error.sql_file_name_required": "SQL ファイル名は空にできません", "file.backend.error.sql_file_statement_execution_failed": "{{index}} 件目の文の実行に失敗しました: {{detail}}", + "file.backend.error.sql_file_stopped_on_error_summary": "SQL ファイルの実行を停止しました。実行済み {{success}} 件、検出エラー {{failed}} 件。{{detail}}", + "file.backend.error.sql_file_unclosed_transaction": "トランザクションが終了する前に SQL ファイルの末尾に達しました。インポートを停止し、未コミットのトランザクションをクリーンアップします。", "file.backend.error.table_data_batch_limit": "一度に処理できるテーブルは最大 {{max}} 件です。現在 {{count}} 件が選択されています", "file.backend.error.table_data_clear_failed": "{{table}} のクリアに失敗しました: {{detail}}", "file.backend.error.table_data_clear_failed_partial": "{{table}} のクリアに失敗しました: {{detail}}。警告: 先頭 {{count}} 件のテーブルはすでにクリアされ、復元できません", @@ -5698,6 +5799,7 @@ "file.backend.message.export_completed": "エクスポートが完了しました", "file.backend.message.import_no_data": "インポートできるデータがありません", "file.backend.message.import_row_failed": "{{index}} 行目のインポートに失敗しました: {{detail}}", + "file.backend.message.import_errors_exported": "インポート失敗行をエクスポートしました", "file.backend.message.import_summary": "インポート済み: {{imported}}, 失敗: {{failed}}", "file.backend.message.statement_failed": "{{index}} 件目の文の実行に失敗しました: {{detail}}\n SQL: {{sql}}", "file.backend.message.table_data_clear_succeeded": "クリアが完了しました", @@ -5734,9 +5836,12 @@ "find_in_database.tooltip.collapse_details": "詳細を折りたたむ", "find_in_database.tooltip.view_details": "詳細を表示", "import_preview.action.start": "インポート開始", + "import_preview.action.export_rejected_rows": "失敗行をエクスポート", "import_preview.error.connection_config_not_found": "接続設定が見つかりません", "import_preview.error.import_failed": "インポートに失敗しました", "import_preview.error.import_failed_detail": "インポートに失敗しました: {{detail}}", + "import_preview.error.export_rejected_rows_failed": "失敗行をエクスポートできませんでした", + "import_preview.error.export_rejected_rows_failed_detail": "失敗行をエクスポートできませんでした: {{detail}}", "import_preview.error.preview_failed": "プレビューに失敗しました", "import_preview.error.preview_failed_detail": "プレビューに失敗しました: {{detail}}", "import_preview.error.target_columns_failed": "対象テーブルの列を読み込めませんでした", @@ -5754,12 +5859,23 @@ "import_preview.preview.description": "最初の 5 行を表示しています。内容を確認してからインポートを開始してください。", "import_preview.preview.field_list": "フィールド一覧:", "import_preview.preview.summary": "{{rows}} 行、{{columns}} フィールド", + "import_preview.preview.summary_sample": "{{rows}} 行のサンプルを表示しています。総行数はスキャンしていません。{{columns}} フィールド", "import_preview.preview.table_title": "データプレビュー（最初の 5 行）:", "import_preview.progress.error_count": "失敗 {{count}}", "import_preview.progress.processed_rows": "{{current}} / {{total}} 行を処理済み", + "import_preview.progress.processed_rows_unknown": "{{current}} 行を処理済み", "import_preview.progress.success_count": "成功 {{count}}", + "import_preview.stage.prepare": "インポートを準備中", + "import_preview.stage.preflight": "インポート前チェックを実行中", + "import_preview.stage.read": "ソースファイルを読み込み中", + "import_preview.stage.parse": "ソースファイルを解析中", + "import_preview.stage.write": "行を書き込み中", + "import_preview.stage.finalize": "結果を確定中", "import_preview.result.completed": "インポートが完了しました", + "import_preview.result.batch_outcome_unknown": "失敗したバッチは一部書き込まれている可能性があります。再試行前に対象テーブルを確認してください。", "import_preview.result.error_logs": "エラーログ:", + "import_preview.result.error_count": "{{count}} 件のエラーを記録しました", + "import_preview.result.error_logs_omitted": "ほか {{count}} 件のエラー詳細は表示されていません。", "import_preview.result.failed_rows": "{{count}} 行が失敗しました", "import_preview.result.success_rows": "{{count}} 行を正常にインポートしました", "import_preview.status.importing": "データをインポート中...", @@ -5770,6 +5886,7 @@ "jvm.backend.connection_error.agent.base_url_invalid.summary": "Agent 接続に失敗しました: Agent Base URL の形式が正しくありません。", "jvm.backend.connection_error.agent.base_url_required": "Agent 接続に失敗しました: Agent Base URL が未入力です。", "import_preview.result.stopped": "インポートを停止しました（コミット済みデータはロールバックされません）", + "import_preview.result.stopped_on_error": "エラーによりインポートを停止しました", "jvm.backend.connection_error.agent.connection_refused.help": "Java サービスが `-javaagent` で GoNavi Agent を起動していることを確認し、Base URL、ポートマッピング、ファイアウォールを確認してください。", "jvm.backend.connection_error.agent.connection_refused.summary": "Agent 接続に失敗しました: ターゲット Agent 管理ポートが待ち受けていないか、現在のアドレスに到達できません。", "jvm.backend.connection_error.agent.forbidden.help": "現在のクライアント送信元、認証設定、Agent アクセスポリシーが GoNavi のアクセスを許可しているか確認してください。", @@ -7731,12 +7848,13 @@ "sidebar.sql_file_exec.executed_label": "実行済み: ", "sidebar.sql_file_exec.file_size": "ファイルサイズ: ", "sidebar.sql_file_exec.message.already_running": "外部 SQL の実行タスクはすでに実行中です", - "sidebar.sql_file_exec.rows_separator": " 行 | 失敗: ", - "sidebar.sql_file_exec.rows_suffix": " 行", + "sidebar.sql_file_exec.statements_separator": " ステートメント | 失敗: ", + "sidebar.sql_file_exec.statements_suffix": " ステートメント", "sidebar.sql_file_exec.status.cancelled": "キャンセル済み", "sidebar.sql_file_exec.status.done": "完了", "sidebar.sql_file_exec.status.error": "エラー", "sidebar.sql_file_exec.status.running": "実行中", + "sidebar.sql_file_exec.status.stopping": "停止中...", "sidebar.sql_file_exec.status_label": "ステータス: ", "sidebar.sql_file_exec.title": "外部 SQL ファイルを実行", "sidebar.sql_file_exec.workbench.action.clear_history": "履歴をクリア", diff --git a/shared/i18n/ru-RU.json b/shared/i18n/ru-RU.json index 315d6a02..e6c5291e 100644 --- a/shared/i18n/ru-RU.json +++ b/shared/i18n/ru-RU.json @@ -4419,17 +4419,101 @@ "data_grid.view.result_view": "Просмотр результата", "data_grid.view.table": "Таблица", "data_grid.view.text": "Текст", + "data_import.capability.loading": "Проверяется поддержка импорта для этого подключения...", + "data_import.capability.details.formats": "Форматы файлов", + "data_import.capability.details.encodings": "Кодировки", + "data_import.capability.details.compressions": "Сжатие", + "data_import.capability.details.directives": "Клиентские директивы", + "data_import.capability.reason.capability_unavailable": "Сведения о поддержке импорта недоступны. Выбор файла и запуск отключены.", + "data_import.capability.reason.data_import_restricted": "Импорт данных отключён политикой защиты этого подключения.", + "data_import.capability.reason.database_runtime_unavailable": "Среда выполнения базы данных недоступна, поэтому поддержку импорта проверить нельзя.", + "data_import.capability.reason.database_type_unsupported": "Этот тип базы данных не поддерживается в текущем режиме импорта.", + "data_import.capability.reason.pinned_session_unavailable": "Импорт SQL-файла отключён: драйвер не может предоставить проверенный закреплённый сеанс.", + "data_import.capability.reason.sql_file_import_restricted": "Импорт SQL-файла отключён защитой скриптов или структуры.", + "data_import.capability.reason.table_import_runtime_unavailable": "Среда выполнения этого драйвера не предоставляет интерфейс импорта таблиц.", + "data_import.capability.reason.unsupported": "Текущий режим импорта недоступен для выбранного подключения.", + "data_import.capability.rpc_failed": "Не удалось проверить поддержку импорта. Проверьте подключение и повторите попытку.", + "data_import.history.action.delete": "Удалить запись", + "data_import.history.action.details": "Подробнее", + "data_import.history.action.export_errors": "Экспортировать отклонённые строки", + "data_import.history.action.refresh": "Обновить", + "data_import.history.confirm.delete_content": "Удалить запись завершённого импорта? Сначала экспортируйте нужные отклонённые строки.", + "data_import.history.confirm.delete_title": "Удалить запись импорта", + "data_import.history.description": "Просмотр последних импортов таблиц и SQL-файлов. Выполняемые задания удалить нельзя.", + "data_import.history.detail.job_id": "ID задания: {{id}}", + "data_import.history.detail.outcome_unknown": "Сбойный пакет мог быть записан частично. Перед повторным импортом проверьте целевые данные.", + "data_import.history.detail.stage": "Этап: {{stage}}", + "data_import.history.empty": "История импорта пуста", + "data_import.history.error.delete_failed": "Не удалось удалить запись импорта", + "data_import.history.error.delete_failed_detail": "Не удалось удалить запись импорта: {{detail}}", + "data_import.history.error.details_failed": "Не удалось загрузить сведения об импорте", + "data_import.history.error.details_failed_detail": "Не удалось загрузить сведения об импорте: {{detail}}", + "data_import.history.error.export_failed": "Не удалось экспортировать отклонённые строки", + "data_import.history.error.export_failed_detail": "Не удалось экспортировать отклонённые строки: {{detail}}", + "data_import.history.error.load_failed": "Не удалось загрузить историю импорта", + "data_import.history.error.load_failed_detail": "Не удалось загрузить историю импорта: {{detail}}", + "data_import.history.kind.sql": "SQL-файл", + "data_import.history.kind.table": "Данные таблицы", + "data_import.history.message.deleted": "Запись импорта удалена", + "data_import.history.message.exported": "Отклонённые строки экспортированы", + "data_import.history.progress": "Обработано {{current}} · Успешно {{success}} · Пропущено {{skipped}} · Ошибок {{failed}}", + "data_import.history.status.cancelled": "Отменено", + "data_import.history.status.completed": "Завершено", + "data_import.history.status.failed": "Ошибка", + "data_import.history.status.interrupted": "Прервано", + "data_import.history.status.partial": "Завершено с ошибками", + "data_import.history.status.preparing": "Подготовка", + "data_import.history.status.running": "Выполняется", + "data_import.history.status.stopping": "Останавливается", + "data_import.history.status.unknown": "Результат неизвестен", + "data_import.history.title": "История импорта", "data_import.workbench.action.cancel_database_import": "Отменить импорт", "data_import.workbench.action.change_file": "Сменить файл", "data_import.workbench.action.change_sql_file": "Сменить SQL-файл", - "data_import.workbench.action.retry_database_import": "Повторить импорт базы данных", + "data_import.workbench.action.retry_database_import": "Повторно выполнить весь SQL-файл", "data_import.workbench.action.select_file": "Выбрать файл и открыть предпросмотр", "data_import.workbench.action.select_sql_file": "Выбрать SQL-файл", "data_import.workbench.action.start_database_import": "Начать импорт базы данных", + "data_import.workbench.advanced.conflict.skip_duplicates": "Пропускать строки с повторяющимся ключом", + "data_import.workbench.advanced.conflict.stop": "Остановиться при конфликте", + "data_import.workbench.advanced.conflict.upsert": "Вставить или обновить", + "data_import.workbench.advanced.conflict_keys": "Столбцы ключа конфликта", + "data_import.workbench.advanced.conflict_keys_not_mapped": "Столбцы ключа конфликта должны входить в выбранные сопоставления: {{columns}}", + "data_import.workbench.advanced.conflict_keys_placeholder": "Например: id, tenant_id", + "data_import.workbench.advanced.conflict_keys_required": "Для вставки или обновления укажите хотя бы один столбец ключа конфликта.", + "data_import.workbench.advanced.conflict_unsupported": "Это подключение не поддерживает сохранённую политику повторяющихся ключей. Перед импортом выберите поддерживаемую политику.", + "data_import.workbench.advanced.conflict_policy": "Обработка повторяющихся ключей", + "data_import.workbench.advanced.delimiter": "Разделитель CSV", + "data_import.workbench.advanced.delimiter.auto": "Определять автоматически", + "data_import.workbench.advanced.delimiter.comma": "Запятая (,)", + "data_import.workbench.advanced.delimiter.pipe": "Вертикальная черта (|)", + "data_import.workbench.advanced.delimiter.semicolon": "Точка с запятой (;)", + "data_import.workbench.advanced.delimiter.tab": "Табуляция", + "data_import.workbench.advanced.description": "Эти настройки применяются к предпросмотру и импорту и сохраняются на этом устройстве.", + "data_import.workbench.advanced.empty_string_as_null": "Считать пустые строки значением NULL", + "data_import.workbench.advanced.encoding": "Кодировка текста", + "data_import.workbench.advanced.encoding.auto": "Определять автоматически", + "data_import.workbench.advanced.encoding.gb18030": "GB18030", + "data_import.workbench.advanced.encoding.utf16be": "UTF-16 BE", + "data_import.workbench.advanced.encoding.utf16le": "UTF-16 LE", + "data_import.workbench.advanced.encoding.utf8": "UTF-8", + "data_import.workbench.advanced.header_row": "Строка заголовка", + "data_import.workbench.advanced.null_token": "Маркер NULL (пустое значение отключает его)", + "data_import.workbench.advanced.sheet_name": "Имя листа (пустое значение выбирает первый лист)", + "data_import.workbench.advanced.title": "Расширенные параметры импорта таблицы", + "data_import.workbench.confirm.rerun_content": "Весь SQL-файл будет выполнен с начала. Зафиксированные операторы предыдущего запуска не откатываются и могут выполниться повторно.", + "data_import.workbench.confirm.rerun_title": "Повторно выполнить весь SQL-файл?", "data_import.workbench.description": "Выберите целевую таблицу и файл данных, затем проверьте данные, настройте сопоставление столбцов и запустите импорт.", "data_import.workbench.description.database": "Выберите подключение и SQL-файл для импорта всей базы данных. Для обычных SQL-файлов можно при необходимости указать базу данных по умолчанию.", - "data_import.workbench.helper.file_formats": "Поддерживаются файлы CSV, JSON, XLSX и XLS.", - "data_import.workbench.helper.sql_file": "Поддерживаются файлы .sql. Выбор файла не запускает импорт автоматически.", + "data_import.workbench.error_policy.continue": "Продолжать после ошибок SQL", + "data_import.workbench.error_policy.continue_description": "Ошибки записываются, выполнение продолжается. В MySQL/MariaDB операторы выполняются по одному, чтобы не повторять сбойные пакеты; большие файлы обрабатываются медленнее, а уже зафиксированные операторы автоматически не откатываются.", + "data_import.workbench.error_policy.continue_table": "Продолжать после ошибок строк", + "data_import.workbench.error_policy.continue_table_description": "С самого начала строки записываются по одной, ошибки фиксируются, после чего импорт продолжается. Сбойные пакеты не запускаются и не повторяются; большие файлы обрабатываются медленнее, а записанные данные автоматически не откатываются.", + "data_import.workbench.error_policy.stop_description": "Рекомендуется. Остановиться при первой ошибке SQL без повторного выполнения сбойного пакета. В нетранзакционных таблицах могут сохраниться частичные записи.", + "data_import.workbench.error_policy.stop_table_description": "Рекомендуется. Использовать пакетную запись и остановиться при первой ошибке пакета без его повтора. Если API вернул ошибку, пакет мог быть записан частично; проверьте целевую таблицу.", + "data_import.workbench.error_policy.title": "Обработка ошибок", + "data_import.workbench.helper.file_formats": "Поддерживаются файлы CSV, JSON и XLSX.", + "data_import.workbench.helper.sql_file": "Поддерживаются файлы .sql и .sql.gz. Выбор файла не запускает импорт автоматически.", "data_import.workbench.label.connection": "Подключение", "data_import.workbench.label.database": "База данных", "data_import.workbench.label.default_database": "База данных по умолчанию (необязательно)", @@ -4446,7 +4530,9 @@ "data_import.workbench.mode.database": "Импортировать базу данных", "data_import.workbench.mode.table": "Импортировать таблицу", "data_import.workbench.notice.gonavi_mysql_restore": "Резервные копии MySQL, созданные GoNavi, восстанавливают базу данных с именем, указанным в файле. Для других SQL-файлов можно использовать выбранную выше необязательную базу данных по умолчанию.", + "data_import.workbench.notice.continue_on_error": "В режиме продолжения ошибки записываются, а последующие операторы выполняются. Чтобы избежать повторной записи в нетранзакционные таблицы, MySQL/MariaDB выполняет операторы по одному и не повторяет сбойные пакеты; большие файлы могут обрабатываться заметно медленнее. Уже зафиксированные операторы автоматически не откатываются.", "data_import.workbench.notice.partial_execution": "SQL-инструкции выполняются последовательно в потоковом режиме. Инструкции, завершённые до ошибки или отмены, не откатываются автоматически. Заранее проверьте целевое подключение и резервную копию.", + "data_import.workbench.notice.stop_on_error": "Выполнение останавливается при первой ошибке SQL без повторного выполнения сбойного пакета. Выполняется попытка отката, но в нетранзакционных таблицах могут сохраниться прежние записи. Ранее зафиксированные инструкции автоматически не откатываются.", "data_import.workbench.placeholder.loading_databases": "Загрузка баз данных...", "data_import.workbench.placeholder.loading_tables": "Загрузка таблиц...", "data_import.workbench.placeholder.select_connection": "Выберите подключение", @@ -4455,6 +4541,9 @@ "data_import.workbench.placeholder.select_default_database": "Необязательно: выберите базу данных по умолчанию", "data_import.workbench.placeholder.select_table": "Выберите целевую таблицу", "data_import.workbench.progress.bytes": "Обработка файла: {{processed}} / {{total}}", + "data_import.workbench.progress.throughput": "Скорость: {{rate}}/с", + "data_import.workbench.progress.eta": "Осталось: {{duration}}", + "data_import.workbench.progress.skipped": "Пропущено строк: {{count}}", "data_import.workbench.progress.statements": "Ход выполнения: выполнено {{executed}}, с ошибкой {{failed}}", "data_import.workbench.section.target": "Цель импорта", "data_import.workbench.state.awaiting_file_description": "После выбора цели выберите файл данных, чтобы просмотреть содержимое и настроить сопоставление столбцов.", @@ -4464,6 +4553,7 @@ "data_import.workbench.state.cancelled": "Отменено", "data_import.workbench.state.cancelling": "Отмена...", "data_import.workbench.state.completed": "Импорт завершён", + "data_import.workbench.state.completed_with_errors": "Импорт завершён с ошибками", "data_import.workbench.state.failed": "Ошибка импорта", "data_import.workbench.state.ready_sql_description": "Проверьте цель и предупреждение о рисках, затем запустите импорт вручную.", "data_import.workbench.state.ready_sql_title": "SQL-файл готов", @@ -5581,6 +5671,7 @@ "driver_manager.version.unlabeled": "Версия не указана", "file.backend.dialog.export_connections": "Экспорт подключений", "file.backend.dialog.export_data": "Экспорт данных", + "file.backend.dialog.export_import_errors": "Экспорт отклонённых строк импорта", "file.backend.dialog.export_database_sql": "Экспорт {{database}} (SQL)", "file.backend.dialog.export_query_result": "Экспорт результата запроса", "file.backend.dialog.export_table": "Экспорт {{table}}", @@ -5627,8 +5718,15 @@ "file.backend.error.import_excel_parse_failed": "Не удалось разобрать Excel: {{detail}}", "file.backend.error.import_excel_read_failed": "Не удалось прочитать Excel: {{detail}}", "file.backend.error.import_file_empty": "Путь к файлу не может быть пустым", + "file.backend.error.import_error_artifact_not_found": "Данные для экспорта отклонённых строк недоступны или устарели", + "file.backend.error.import_job_not_found": "Задание импорта не найдено", + "file.backend.error.import_job_persist": "Не удалось сохранить состояние задания импорта: {{detail}}", + "file.backend.error.import_job_running": "Задание импорта ещё выполняется и не может быть удалено", + "file.backend.error.import_resume_unavailable": "Этот импорт нельзя безопасно возобновить", + "file.backend.error.import_source_changed": "Источник импорта изменился после предпросмотра. Выберите файл заново и повторите предпросмотр.", "file.backend.error.import_json_parse_failed": "Не удалось разобрать JSON: {{detail}}", "file.backend.error.import_json_root_not_array": "Корень JSON должен быть массивом", + "file.backend.error.import_stopped_on_error": "Импорт таблицы остановлен из-за ошибки. Подтвержден импорт {{imported}} строк, зарегистрировано ошибок: {{failed}}. {{detail}}", "file.backend.error.import_unsupported_format": "Неподдерживаемый формат файла", "file.backend.error.invalid_export_mode": "Недопустимый режим экспорта", "file.backend.error.mysql_workbench_no_connections": "В XML не найдены допустимые конфигурации подключений", @@ -5656,11 +5754,14 @@ "file.backend.error.selected_path_not_sql_file": "Выбранный путь не является SQL-файлом", "file.backend.error.sql_file_batch_execution_failed": "Ошибка пакетного выполнения, начиная с инструкции {{index}}: {{detail}}", "file.backend.error.sql_file_batch_rollback_failed": "Ошибка пакетного выполнения: {{detail}}; ошибка отката: {{rollbackDetail}}", + "file.backend.error.sql_file_execution_failed_summary": "Выполнение SQL-файла прервано после {{count}} инструкций: {{detail}}", "file.backend.error.sql_file_exists": "SQL-файл уже существует", "file.backend.error.sql_file_extension_required": "Поддерживаются только SQL-файлы", "file.backend.error.sql_file_name_no_separator": "Имя SQL-файла не может содержать разделители пути", "file.backend.error.sql_file_name_required": "Имя SQL-файла не может быть пустым", "file.backend.error.sql_file_statement_execution_failed": "Ошибка выполнения инструкции {{index}}: {{detail}}", + "file.backend.error.sql_file_stopped_on_error_summary": "Выполнение SQL-файла остановлено. Выполнено инструкций: {{success}}, обнаружено ошибок: {{failed}}. {{detail}}", + "file.backend.error.sql_file_unclosed_transaction": "SQL-файл завершился до закрытия транзакции. Импорт остановлен, незавершённая транзакция будет очищена.", "file.backend.error.table_data_batch_limit": "За один раз можно обработать не более {{max}} таблиц; сейчас выбрано {{count}}", "file.backend.error.table_data_clear_failed": "Не удалось очистить {{table}}: {{detail}}", "file.backend.error.table_data_clear_failed_partial": "Не удалось очистить {{table}}: {{detail}}. Предупреждение: первые {{count}} таблиц уже очищены, их нельзя восстановить", @@ -5698,6 +5799,7 @@ "file.backend.message.export_completed": "Экспорт завершен", "file.backend.message.import_no_data": "Нет данных для импорта", "file.backend.message.import_row_failed": "Ошибка импорта строки {{index}}: {{detail}}", + "file.backend.message.import_errors_exported": "Отклонённые строки импорта экспортированы", "file.backend.message.import_summary": "Импортировано: {{imported}}, ошибок: {{failed}}", "file.backend.message.statement_failed": "Ошибка выполнения инструкции {{index}}: {{detail}}\n SQL: {{sql}}", "file.backend.message.table_data_clear_succeeded": "Очистка выполнена успешно", @@ -5734,9 +5836,12 @@ "find_in_database.tooltip.collapse_details": "Свернуть подробности", "find_in_database.tooltip.view_details": "Показать подробности", "import_preview.action.start": "Начать импорт", + "import_preview.action.export_rejected_rows": "Экспортировать отклонённые строки", "import_preview.error.connection_config_not_found": "Конфигурация подключения не найдена", "import_preview.error.import_failed": "Не удалось выполнить импорт", "import_preview.error.import_failed_detail": "Не удалось выполнить импорт: {{detail}}", + "import_preview.error.export_rejected_rows_failed": "Не удалось экспортировать отклонённые строки", + "import_preview.error.export_rejected_rows_failed_detail": "Не удалось экспортировать отклонённые строки: {{detail}}", "import_preview.error.preview_failed": "Не удалось выполнить предпросмотр", "import_preview.error.preview_failed_detail": "Не удалось выполнить предпросмотр: {{detail}}", "import_preview.error.target_columns_failed": "Не удалось загрузить столбцы целевой таблицы", @@ -5754,12 +5859,23 @@ "import_preview.preview.description": "Ниже показаны первые 5 строк. Начните импорт после проверки данных.", "import_preview.preview.field_list": "Список полей:", "import_preview.preview.summary": "{{rows}} строк и {{columns}} полей", + "import_preview.preview.summary_sample": "Показано строк выборки: {{rows}}; общее число строк не сканировалось. Полей: {{columns}}", "import_preview.preview.table_title": "Предпросмотр данных (первые 5 строк):", "import_preview.progress.error_count": "Ошибок {{count}}", "import_preview.progress.processed_rows": "Обработано {{current}} / {{total}} строк", + "import_preview.progress.processed_rows_unknown": "Обработано строк: {{current}}", "import_preview.progress.success_count": "Успешно {{count}}", + "import_preview.stage.prepare": "Подготовка импорта", + "import_preview.stage.preflight": "Предварительная проверка", + "import_preview.stage.read": "Чтение исходного файла", + "import_preview.stage.parse": "Разбор исходного файла", + "import_preview.stage.write": "Запись строк", + "import_preview.stage.finalize": "Завершение результата", "import_preview.result.completed": "Импорт завершен", + "import_preview.result.batch_outcome_unknown": "Сбойный пакет мог быть записан частично. Проверьте целевую таблицу перед повторной попыткой.", "import_preview.result.error_logs": "Журнал ошибок:", + "import_preview.result.error_count": "Зарегистрировано ошибок: {{count}}", + "import_preview.result.error_logs_omitted": "Не показано дополнительных сведений об ошибках: {{count}}.", "import_preview.result.failed_rows": "Строк с ошибками: {{count}}", "import_preview.result.success_rows": "Успешно импортировано строк: {{count}}", "import_preview.status.importing": "Импорт данных...", @@ -5770,6 +5886,7 @@ "jvm.backend.connection_error.agent.base_url_invalid.summary": "Подключение Agent не удалось: Agent Base URL недействителен.", "jvm.backend.connection_error.agent.base_url_required": "Подключение Agent не удалось: Agent Base URL отсутствует.", "import_preview.result.stopped": "Импорт остановлен (зафиксированные данные не были отменены)", + "import_preview.result.stopped_on_error": "Импорт остановлен из-за ошибки", "jvm.backend.connection_error.agent.connection_refused.help": "Убедитесь, что Java service запущен с GoNavi Agent через `-javaagent`, и проверьте Base URL, port mapping и правила firewall.", "jvm.backend.connection_error.agent.connection_refused.summary": "Подключение Agent не удалось: целевой management port Agent не слушается или адрес недоступен.", "jvm.backend.connection_error.agent.forbidden.help": "Убедитесь, что источник клиента, настройки authentication и access policy Agent разрешают GoNavi доступ.", @@ -7731,12 +7848,13 @@ "sidebar.sql_file_exec.executed_label": "Выполнено: ", "sidebar.sql_file_exec.file_size": "Размер файла: ", "sidebar.sql_file_exec.message.already_running": "Задача выполнения внешнего SQL уже запущена", - "sidebar.sql_file_exec.rows_separator": " строк | Ошибок: ", - "sidebar.sql_file_exec.rows_suffix": " строк", + "sidebar.sql_file_exec.statements_separator": " операторов | Ошибок: ", + "sidebar.sql_file_exec.statements_suffix": " операторов", "sidebar.sql_file_exec.status.cancelled": "Отменено", "sidebar.sql_file_exec.status.done": "Готово", "sidebar.sql_file_exec.status.error": "Ошибка", "sidebar.sql_file_exec.status.running": "Выполняется", + "sidebar.sql_file_exec.status.stopping": "Останавливается...", "sidebar.sql_file_exec.status_label": "Статус: ", "sidebar.sql_file_exec.title": "Запустить внешний SQL-файл", "sidebar.sql_file_exec.workbench.action.clear_history": "Очистить историю", diff --git a/shared/i18n/zh-CN.json b/shared/i18n/zh-CN.json index 5b98fd9a..ce2cac3e 100644 --- a/shared/i18n/zh-CN.json +++ b/shared/i18n/zh-CN.json @@ -4419,17 +4419,101 @@ "data_grid.view.result_view": "结果视图", "data_grid.view.table": "表格", "data_grid.view.text": "文本", + "data_import.capability.loading": "正在检查所选连接的导入能力...", + "data_import.capability.details.formats": "文件格式", + "data_import.capability.details.encodings": "字符编码", + "data_import.capability.details.compressions": "压缩格式", + "data_import.capability.details.directives": "客户端指令", + "data_import.capability.reason.capability_unavailable": "导入能力信息不可用，已禁用文件选择和执行。", + "data_import.capability.reason.data_import_restricted": "该连接的保护策略已禁用数据导入。", + "data_import.capability.reason.database_runtime_unavailable": "数据库运行时不可用，无法验证导入能力。", + "data_import.capability.reason.database_type_unsupported": "当前导入模式不支持此数据库类型。", + "data_import.capability.reason.pinned_session_unavailable": "该驱动无法提供已验证的固定会话，已禁用 SQL 文件导入。", + "data_import.capability.reason.sql_file_import_restricted": "脚本或结构保护策略已禁用 SQL 文件导入。", + "data_import.capability.reason.table_import_runtime_unavailable": "该驱动运行时未提供表导入接口。", + "data_import.capability.reason.unsupported": "所选连接不支持当前导入模式。", + "data_import.capability.rpc_failed": "无法验证导入能力，请检查连接后重试。", + "data_import.history.action.delete": "删除记录", + "data_import.history.action.details": "详情", + "data_import.history.action.export_errors": "导出失败行", + "data_import.history.action.refresh": "刷新", + "data_import.history.confirm.delete_content": "确定删除这条已结束的导入记录吗？如需失败行，请先导出。", + "data_import.history.confirm.delete_title": "删除导入记录", + "data_import.history.description": "查看最近的表数据和 SQL 文件导入；运行中的任务不能删除。", + "data_import.history.detail.job_id": "任务 ID：{{id}}", + "data_import.history.detail.outcome_unknown": "失败批次可能已部分写入。再次导入前请先核对目标数据。", + "data_import.history.detail.stage": "阶段：{{stage}}", + "data_import.history.empty": "暂无导入历史", + "data_import.history.error.delete_failed": "删除导入记录失败", + "data_import.history.error.delete_failed_detail": "删除导入记录失败：{{detail}}", + "data_import.history.error.details_failed": "加载导入详情失败", + "data_import.history.error.details_failed_detail": "加载导入详情失败：{{detail}}", + "data_import.history.error.export_failed": "导出失败行失败", + "data_import.history.error.export_failed_detail": "导出失败行失败：{{detail}}", + "data_import.history.error.load_failed": "加载导入历史失败", + "data_import.history.error.load_failed_detail": "加载导入历史失败：{{detail}}", + "data_import.history.kind.sql": "SQL 文件", + "data_import.history.kind.table": "表数据", + "data_import.history.message.deleted": "导入记录已删除", + "data_import.history.message.exported": "失败行已导出", + "data_import.history.progress": "已处理 {{current}} · 成功 {{success}} · 跳过 {{skipped}} · 失败 {{failed}}", + "data_import.history.status.cancelled": "已取消", + "data_import.history.status.completed": "已完成", + "data_import.history.status.failed": "失败", + "data_import.history.status.interrupted": "已中断", + "data_import.history.status.partial": "完成但存在错误", + "data_import.history.status.preparing": "准备中", + "data_import.history.status.running": "运行中", + "data_import.history.status.stopping": "停止中", + "data_import.history.status.unknown": "结果未知", + "data_import.history.title": "导入任务历史", "data_import.workbench.action.cancel_database_import": "取消导入", "data_import.workbench.action.change_file": "更换文件", "data_import.workbench.action.change_sql_file": "更换 SQL 文件", - "data_import.workbench.action.retry_database_import": "重新导入数据库", + "data_import.workbench.action.retry_database_import": "重新执行整个 SQL 文件", "data_import.workbench.action.select_file": "选择文件并预览", "data_import.workbench.action.select_sql_file": "选择 SQL 文件", "data_import.workbench.action.start_database_import": "开始导入数据库", + "data_import.workbench.advanced.conflict.skip_duplicates": "跳过重复键行", + "data_import.workbench.advanced.conflict.stop": "冲突时停止", + "data_import.workbench.advanced.conflict.upsert": "新增或更新", + "data_import.workbench.advanced.conflict_keys": "冲突键字段", + "data_import.workbench.advanced.conflict_keys_not_mapped": "冲突键字段必须包含在已选字段映射中：{{columns}}", + "data_import.workbench.advanced.conflict_keys_placeholder": "例如：id, tenant_id", + "data_import.workbench.advanced.conflict_keys_required": "新增或更新时，请至少选择一个冲突键字段。", + "data_import.workbench.advanced.conflict_unsupported": "当前连接不支持已保存的重复键策略，请先选择受支持的策略。", + "data_import.workbench.advanced.conflict_policy": "重复键处理", + "data_import.workbench.advanced.delimiter": "CSV 分隔符", + "data_import.workbench.advanced.delimiter.auto": "自动检测", + "data_import.workbench.advanced.delimiter.comma": "逗号（,）", + "data_import.workbench.advanced.delimiter.pipe": "竖线（|）", + "data_import.workbench.advanced.delimiter.semicolon": "分号（;）", + "data_import.workbench.advanced.delimiter.tab": "制表符", + "data_import.workbench.advanced.description": "这些设置会同时用于预览和导入，并保存在当前设备。", + "data_import.workbench.advanced.empty_string_as_null": "将空字符串视为 NULL", + "data_import.workbench.advanced.encoding": "文本编码", + "data_import.workbench.advanced.encoding.auto": "自动检测", + "data_import.workbench.advanced.encoding.gb18030": "GB18030", + "data_import.workbench.advanced.encoding.utf16be": "UTF-16 BE", + "data_import.workbench.advanced.encoding.utf16le": "UTF-16 LE", + "data_import.workbench.advanced.encoding.utf8": "UTF-8", + "data_import.workbench.advanced.header_row": "表头所在行", + "data_import.workbench.advanced.null_token": "NULL 标记（留空则禁用）", + "data_import.workbench.advanced.sheet_name": "工作表名称（留空使用第一张表）", + "data_import.workbench.advanced.title": "表导入高级选项", + "data_import.workbench.confirm.rerun_content": "这会从头执行整个 SQL 文件。上次执行已提交的语句不会回滚，可能被再次执行。", + "data_import.workbench.confirm.rerun_title": "重新执行整个 SQL 文件？", "data_import.workbench.description": "先选择目标表和数据文件，再完成预览、字段映射与导入。", "data_import.workbench.description.database": "选择连接和 SQL 文件以导入整个数据库；普通 SQL 文件可按需指定默认数据库。", - "data_import.workbench.helper.file_formats": "支持 CSV、JSON、XLSX 和 XLS 文件。", - "data_import.workbench.helper.sql_file": "支持 .sql 文件；选择文件后不会自动开始导入。", + "data_import.workbench.error_policy.continue": "遇到 SQL 错误后仍继续执行", + "data_import.workbench.error_policy.continue_description": "记录错误后继续执行。为避免失败批次被重放，MySQL/MariaDB 会逐条执行；大文件会更慢，已提交语句不会自动回滚。", + "data_import.workbench.error_policy.continue_table": "遇到行错误后仍继续导入", + "data_import.workbench.error_policy.continue_table_description": "从第一行起逐行写入，失败行记录后继续；不会先尝试失败批次，因此不会重放。大文件会更慢，已写入数据不会自动回滚。", + "data_import.workbench.error_policy.stop_description": "推荐。首个 SQL 错误会立即停止且不逐条重放；非事务表仍可能保留部分写入。", + "data_import.workbench.error_policy.stop_table_description": "推荐。使用批量写入，首个失败批次立即停止且不重放；批接口返回错误时，该批次可能已部分写入，请核对目标表。", + "data_import.workbench.error_policy.title": "错误处理", + "data_import.workbench.helper.file_formats": "支持 CSV、JSON 和 XLSX 文件。", + "data_import.workbench.helper.sql_file": "支持 .sql 和 .sql.gz 文件；选择文件后不会自动开始导入。", "data_import.workbench.label.connection": "连接", "data_import.workbench.label.database": "数据库", "data_import.workbench.label.default_database": "默认数据库（可选）", @@ -4446,7 +4530,9 @@ "data_import.workbench.mode.database": "导入数据库", "data_import.workbench.mode.table": "导入表", "data_import.workbench.notice.gonavi_mysql_restore": "GoNavi 生成的 MySQL 备份将按文件内记录的数据库名称恢复；其他 SQL 文件可使用上方选择的默认数据库。", + "data_import.workbench.notice.continue_on_error": "继续模式会记录错误并处理后续语句。MySQL/MariaDB 为避免非事务表重复写入，将逐条执行且不会重放失败批次；大文件可能明显变慢。已提交语句不会自动回滚。", "data_import.workbench.notice.partial_execution": "SQL 文件按语句流式执行。失败或取消前已成功执行的语句不会自动回滚，请先确认目标连接和备份。", + "data_import.workbench.notice.stop_on_error": "遇到首个 SQL 错误后将停止且不会逐条重放失败批次；系统会尝试回滚，但非事务表此前写入可能保留。此前已提交语句不会自动回滚。", "data_import.workbench.placeholder.loading_databases": "正在加载数据库...", "data_import.workbench.placeholder.loading_tables": "正在加载表...", "data_import.workbench.placeholder.select_connection": "选择连接", @@ -4455,6 +4541,9 @@ "data_import.workbench.placeholder.select_default_database": "可选：选择默认数据库", "data_import.workbench.placeholder.select_table": "选择目标表", "data_import.workbench.progress.bytes": "文件进度：{{processed}} / {{total}}", + "data_import.workbench.progress.throughput": "速度：{{rate}}/秒", + "data_import.workbench.progress.eta": "预计剩余：{{duration}}", + "data_import.workbench.progress.skipped": "已跳过 {{count}} 行", "data_import.workbench.progress.statements": "语句进度：已执行 {{executed}} 条，失败 {{failed}} 条", "data_import.workbench.section.target": "导入目标", "data_import.workbench.state.awaiting_file_description": "目标确定后选择数据文件，随后可预览内容并配置字段映射。", @@ -4464,6 +4553,7 @@ "data_import.workbench.state.cancelled": "已取消", "data_import.workbench.state.cancelling": "正在取消...", "data_import.workbench.state.completed": "导入完成", + "data_import.workbench.state.completed_with_errors": "导入完成（存在错误）", "data_import.workbench.state.failed": "导入失败", "data_import.workbench.state.ready_sql_description": "确认目标和风险提示后，手动开始导入。", "data_import.workbench.state.ready_sql_title": "SQL 文件已就绪", @@ -5581,6 +5671,7 @@ "driver_manager.version.unlabeled": "未标注版本", "file.backend.dialog.export_connections": "导出连接", "file.backend.dialog.export_data": "导出数据", + "file.backend.dialog.export_import_errors": "导出导入失败行", "file.backend.dialog.export_database_sql": "导出 {{database}} (SQL)", "file.backend.dialog.export_query_result": "导出查询结果", "file.backend.dialog.export_table": "导出 {{table}}", @@ -5627,8 +5718,15 @@ "file.backend.error.import_excel_parse_failed": "无法解析 Excel: {{detail}}", "file.backend.error.import_excel_read_failed": "无法读取 Excel: {{detail}}", "file.backend.error.import_file_empty": "文件路径不能为空", + "file.backend.error.import_error_artifact_not_found": "失败行导出文件不存在或已过期", + "file.backend.error.import_job_not_found": "未找到导入任务", + "file.backend.error.import_job_persist": "保存导入任务状态失败：{{detail}}", + "file.backend.error.import_job_running": "导入任务仍在运行，不能删除", + "file.backend.error.import_resume_unavailable": "当前导入无法安全恢复", + "file.backend.error.import_source_changed": "导入源在预览后发生变化，请重新选择并预览文件。", "file.backend.error.import_json_parse_failed": "无法解析 JSON: {{detail}}", "file.backend.error.import_json_root_not_array": "JSON 根必须是数组", + "file.backend.error.import_stopped_on_error": "表数据导入遇错停止。已确认导入 {{imported}} 行，记录 {{failed}} 个错误：{{detail}}", "file.backend.error.import_unsupported_format": "不支持的文件格式", "file.backend.error.invalid_export_mode": "无效的导出模式", "file.backend.error.mysql_workbench_no_connections": "未在 XML 中找到有效的连接配置", @@ -5656,11 +5754,14 @@ "file.backend.error.selected_path_not_sql_file": "所选路径不是 SQL 文件", "file.backend.error.sql_file_batch_execution_failed": "第 {{index}} 条起的批量语句执行失败: {{detail}}", "file.backend.error.sql_file_batch_rollback_failed": "批量执行失败: {{detail}}；回滚失败: {{rollbackDetail}}", + "file.backend.error.sql_file_execution_failed_summary": "SQL 文件执行中断（已执行 {{count}} 条）：{{detail}}", "file.backend.error.sql_file_exists": "SQL 文件已存在", "file.backend.error.sql_file_extension_required": "仅支持 SQL 文件", "file.backend.error.sql_file_name_no_separator": "SQL 文件名不能包含路径分隔符", "file.backend.error.sql_file_name_required": "SQL 文件名不能为空", "file.backend.error.sql_file_statement_execution_failed": "第 {{index}} 条语句执行失败: {{detail}}", + "file.backend.error.sql_file_stopped_on_error_summary": "SQL 文件执行已停止。成功执行 {{success}} 条，发现 {{failed}} 个错误。{{detail}}", + "file.backend.error.sql_file_unclosed_transaction": "SQL 文件在事务结束前已读取完毕；导入已停止，并将清理未提交事务。", "file.backend.error.table_data_batch_limit": "单次最多处理 {{max}} 张表，当前选中 {{count}} 张", "file.backend.error.table_data_clear_failed": "清空 {{table}} 失败: {{detail}}", "file.backend.error.table_data_clear_failed_partial": "清空 {{table}} 失败: {{detail}}（注意：前 {{count}} 张表已清空且无法恢复）", @@ -5698,6 +5799,7 @@ "file.backend.message.export_completed": "导出完成", "file.backend.message.import_no_data": "无可导入数据", "file.backend.message.import_row_failed": "第 {{index}} 行导入失败: {{detail}}", + "file.backend.message.import_errors_exported": "导入失败行已导出", "file.backend.message.import_summary": "已导入: {{imported}}, 失败: {{failed}}", "file.backend.message.statement_failed": "第 {{index}} 条语句执行失败: {{detail}}\n SQL: {{sql}}", "file.backend.message.table_data_clear_succeeded": "清空成功", @@ -5734,9 +5836,12 @@ "find_in_database.tooltip.collapse_details": "收起详情", "find_in_database.tooltip.view_details": "查看详情", "import_preview.action.start": "开始导入", + "import_preview.action.export_rejected_rows": "导出失败行", "import_preview.error.connection_config_not_found": "连接配置未找到", "import_preview.error.import_failed": "导入失败", "import_preview.error.import_failed_detail": "导入失败: {{detail}}", + "import_preview.error.export_rejected_rows_failed": "导出失败行失败", + "import_preview.error.export_rejected_rows_failed_detail": "导出失败行失败: {{detail}}", "import_preview.error.preview_failed": "预览失败", "import_preview.error.preview_failed_detail": "预览失败: {{detail}}", "import_preview.error.target_columns_failed": "无法加载目标表字段", @@ -5754,12 +5859,23 @@ "import_preview.preview.description": "以下是前 5 行预览数据，确认无误后点击“开始导入”", "import_preview.preview.field_list": "字段列表：", "import_preview.preview.summary": "共 {{rows}} 行数据，{{columns}} 个字段", + "import_preview.preview.summary_sample": "当前展示 {{rows}} 行样本，未扫描总行数；共 {{columns}} 个字段", "import_preview.preview.table_title": "数据预览（前 5 行）：", "import_preview.progress.error_count": "失败 {{count}}", "import_preview.progress.processed_rows": "已处理 {{current}} / {{total}} 行", + "import_preview.progress.processed_rows_unknown": "已处理 {{current}} 行", "import_preview.progress.success_count": "成功 {{count}}", + "import_preview.stage.prepare": "准备导入", + "import_preview.stage.preflight": "导入前检查", + "import_preview.stage.read": "读取源文件", + "import_preview.stage.parse": "解析源文件", + "import_preview.stage.write": "写入数据", + "import_preview.stage.finalize": "整理导入结果", "import_preview.result.completed": "导入完成", + "import_preview.result.batch_outcome_unknown": "失败批次的写入结果可能不确定，请核对目标表后再重试。", "import_preview.result.error_logs": "错误日志：", + "import_preview.result.error_count": "记录 {{count}} 个错误", + "import_preview.result.error_logs_omitted": "另有 {{count}} 条错误详情未显示。", "import_preview.result.failed_rows": "失败 {{count}} 行", "import_preview.result.success_rows": "成功导入 {{count}} 行", "import_preview.status.importing": "正在导入数据...", @@ -5770,6 +5886,7 @@ "jvm.backend.connection_error.agent.base_url_invalid.summary": "Agent 连接失败：Agent Base URL 格式不合法。", "jvm.backend.connection_error.agent.base_url_required": "Agent 连接失败：未填写 Agent Base URL。", "import_preview.result.stopped": "导入已停止（已提交的数据不会回滚）", + "import_preview.result.stopped_on_error": "导入因错误停止", "jvm.backend.connection_error.agent.connection_refused.help": "请确认 Java 服务已通过 `-javaagent` 启动 GoNavi Agent，并检查 Base URL、端口映射和防火墙。", "jvm.backend.connection_error.agent.connection_refused.summary": "Agent 连接失败：目标 Agent 管理端口未监听，或当前地址不可达。", "jvm.backend.connection_error.agent.forbidden.help": "请确认当前客户端来源、鉴权配置和 Agent 访问策略允许 GoNavi 访问。", @@ -7731,12 +7848,13 @@ "sidebar.sql_file_exec.executed_label": "已执行：", "sidebar.sql_file_exec.file_size": "文件大小：", "sidebar.sql_file_exec.message.already_running": "当前已有外部 SQL 执行任务在运行", - "sidebar.sql_file_exec.rows_separator": " 行 | 失败：", - "sidebar.sql_file_exec.rows_suffix": " 行", + "sidebar.sql_file_exec.statements_separator": " 条语句 | 失败：", + "sidebar.sql_file_exec.statements_suffix": " 条语句", "sidebar.sql_file_exec.status.cancelled": "已中止", "sidebar.sql_file_exec.status.done": "完成", "sidebar.sql_file_exec.status.error": "错误", "sidebar.sql_file_exec.status.running": "运行中", + "sidebar.sql_file_exec.status.stopping": "正在停止...", "sidebar.sql_file_exec.status_label": "状态：", "sidebar.sql_file_exec.title": "运行外部 SQL 文件", "sidebar.sql_file_exec.workbench.action.clear_history": "清空历史", diff --git a/shared/i18n/zh-TW.json b/shared/i18n/zh-TW.json index 706bbc28..4d64c271 100644 --- a/shared/i18n/zh-TW.json +++ b/shared/i18n/zh-TW.json @@ -4419,17 +4419,101 @@ "data_grid.view.result_view": "結果檢視", "data_grid.view.table": "表格", "data_grid.view.text": "文字", + "data_import.capability.loading": "正在檢查所選連線的匯入能力...", + "data_import.capability.details.formats": "檔案格式", + "data_import.capability.details.encodings": "字元編碼", + "data_import.capability.details.compressions": "壓縮格式", + "data_import.capability.details.directives": "用戶端指令", + "data_import.capability.reason.capability_unavailable": "匯入能力資訊無法使用，已停用檔案選擇與執行。", + "data_import.capability.reason.data_import_restricted": "此連線的保護原則已停用資料匯入。", + "data_import.capability.reason.database_runtime_unavailable": "資料庫執行階段無法使用，無法驗證匯入能力。", + "data_import.capability.reason.database_type_unsupported": "目前匯入模式不支援此資料庫類型。", + "data_import.capability.reason.pinned_session_unavailable": "此驅動程式無法提供已驗證的固定工作階段，已停用 SQL 檔案匯入。", + "data_import.capability.reason.sql_file_import_restricted": "指令碼或結構保護原則已停用 SQL 檔案匯入。", + "data_import.capability.reason.table_import_runtime_unavailable": "此驅動程式執行階段未提供資料表匯入介面。", + "data_import.capability.reason.unsupported": "所選連線不支援目前匯入模式。", + "data_import.capability.rpc_failed": "無法驗證匯入能力，請檢查連線後再試一次。", + "data_import.history.action.delete": "刪除記錄", + "data_import.history.action.details": "詳細資料", + "data_import.history.action.export_errors": "匯出失敗資料列", + "data_import.history.action.refresh": "重新整理", + "data_import.history.confirm.delete_content": "確定要刪除這筆已結束的匯入記錄嗎？如需失敗資料列，請先匯出。", + "data_import.history.confirm.delete_title": "刪除匯入記錄", + "data_import.history.description": "查看最近的資料表與 SQL 檔案匯入；執行中的工作無法刪除。", + "data_import.history.detail.job_id": "工作 ID：{{id}}", + "data_import.history.detail.outcome_unknown": "失敗批次可能已部分寫入。再次匯入前請先核對目標資料。", + "data_import.history.detail.stage": "階段：{{stage}}", + "data_import.history.empty": "尚無匯入歷史", + "data_import.history.error.delete_failed": "刪除匯入記錄失敗", + "data_import.history.error.delete_failed_detail": "刪除匯入記錄失敗：{{detail}}", + "data_import.history.error.details_failed": "載入匯入詳細資料失敗", + "data_import.history.error.details_failed_detail": "載入匯入詳細資料失敗：{{detail}}", + "data_import.history.error.export_failed": "匯出失敗資料列失敗", + "data_import.history.error.export_failed_detail": "匯出失敗資料列失敗：{{detail}}", + "data_import.history.error.load_failed": "載入匯入歷史失敗", + "data_import.history.error.load_failed_detail": "載入匯入歷史失敗：{{detail}}", + "data_import.history.kind.sql": "SQL 檔案", + "data_import.history.kind.table": "資料表資料", + "data_import.history.message.deleted": "匯入記錄已刪除", + "data_import.history.message.exported": "失敗資料列已匯出", + "data_import.history.progress": "已處理 {{current}} · 成功 {{success}} · 略過 {{skipped}} · 失敗 {{failed}}", + "data_import.history.status.cancelled": "已取消", + "data_import.history.status.completed": "已完成", + "data_import.history.status.failed": "失敗", + "data_import.history.status.interrupted": "已中斷", + "data_import.history.status.partial": "完成但有錯誤", + "data_import.history.status.preparing": "準備中", + "data_import.history.status.running": "執行中", + "data_import.history.status.stopping": "停止中", + "data_import.history.status.unknown": "結果未知", + "data_import.history.title": "匯入工作歷史", "data_import.workbench.action.cancel_database_import": "取消匯入", "data_import.workbench.action.change_file": "更換檔案", "data_import.workbench.action.change_sql_file": "更換 SQL 檔案", - "data_import.workbench.action.retry_database_import": "重新匯入資料庫", + "data_import.workbench.action.retry_database_import": "重新執行整個 SQL 檔案", "data_import.workbench.action.select_file": "選擇檔案並預覽", "data_import.workbench.action.select_sql_file": "選擇 SQL 檔案", "data_import.workbench.action.start_database_import": "開始匯入資料庫", + "data_import.workbench.advanced.conflict.skip_duplicates": "略過重複鍵資料列", + "data_import.workbench.advanced.conflict.stop": "衝突時停止", + "data_import.workbench.advanced.conflict.upsert": "新增或更新", + "data_import.workbench.advanced.conflict_keys": "衝突鍵欄位", + "data_import.workbench.advanced.conflict_keys_not_mapped": "衝突鍵欄位必須包含在已選欄位對應中：{{columns}}", + "data_import.workbench.advanced.conflict_keys_placeholder": "例如：id, tenant_id", + "data_import.workbench.advanced.conflict_keys_required": "新增或更新時，請至少選擇一個衝突鍵欄位。", + "data_import.workbench.advanced.conflict_unsupported": "目前連線不支援已儲存的重複鍵策略，請先選擇支援的策略。", + "data_import.workbench.advanced.conflict_policy": "重複鍵處理", + "data_import.workbench.advanced.delimiter": "CSV 分隔符號", + "data_import.workbench.advanced.delimiter.auto": "自動偵測", + "data_import.workbench.advanced.delimiter.comma": "逗號（,）", + "data_import.workbench.advanced.delimiter.pipe": "豎線（|）", + "data_import.workbench.advanced.delimiter.semicolon": "分號（;）", + "data_import.workbench.advanced.delimiter.tab": "定位字元", + "data_import.workbench.advanced.description": "這些設定會同時用於預覽和匯入，並儲存在目前裝置。", + "data_import.workbench.advanced.empty_string_as_null": "將空字串視為 NULL", + "data_import.workbench.advanced.encoding": "文字編碼", + "data_import.workbench.advanced.encoding.auto": "自動偵測", + "data_import.workbench.advanced.encoding.gb18030": "GB18030", + "data_import.workbench.advanced.encoding.utf16be": "UTF-16 BE", + "data_import.workbench.advanced.encoding.utf16le": "UTF-16 LE", + "data_import.workbench.advanced.encoding.utf8": "UTF-8", + "data_import.workbench.advanced.header_row": "標題列所在行", + "data_import.workbench.advanced.null_token": "NULL 標記（留空則停用）", + "data_import.workbench.advanced.sheet_name": "工作表名稱（留空使用第一張表）", + "data_import.workbench.advanced.title": "資料表匯入進階選項", + "data_import.workbench.confirm.rerun_content": "這會從頭執行整個 SQL 檔案。上次執行已提交的陳述式不會回復，可能再次執行。", + "data_import.workbench.confirm.rerun_title": "重新執行整個 SQL 檔案？", "data_import.workbench.description": "先選擇目標資料表和資料檔案，再完成預覽、欄位對應與匯入。", "data_import.workbench.description.database": "選擇連線和 SQL 檔案以匯入整個資料庫；一般 SQL 檔案可視需要指定預設資料庫。", - "data_import.workbench.helper.file_formats": "支援 CSV、JSON、XLSX 和 XLS 檔案。", - "data_import.workbench.helper.sql_file": "支援 .sql 檔案；選擇檔案後不會自動開始匯入。", + "data_import.workbench.error_policy.continue": "遇到 SQL 錯誤後仍繼續執行", + "data_import.workbench.error_policy.continue_description": "記錄錯誤後繼續執行。為避免失敗批次被重放，MySQL/MariaDB 會逐筆執行；大型檔案會較慢，已提交的語句不會自動回滾。", + "data_import.workbench.error_policy.continue_table": "遇到資料列錯誤後仍繼續匯入", + "data_import.workbench.error_policy.continue_table_description": "從第一列起逐列寫入，記錄失敗列後繼續；不會先嘗試失敗批次，因此不會重放。大型檔案會較慢，已寫入資料不會自動回滾。", + "data_import.workbench.error_policy.stop_description": "建議。遇到第一個 SQL 錯誤便立即停止且不逐條重放；非交易式資料表仍可能保留部分寫入。", + "data_import.workbench.error_policy.stop_table_description": "建議。使用批次寫入，第一個失敗批次會立即停止且不重放；批次介面回傳錯誤時，該批次可能已部分寫入，請核對目標資料表。", + "data_import.workbench.error_policy.title": "錯誤處理", + "data_import.workbench.helper.file_formats": "支援 CSV、JSON 和 XLSX 檔案。", + "data_import.workbench.helper.sql_file": "支援 .sql 與 .sql.gz 檔案；選擇檔案後不會自動開始匯入。", "data_import.workbench.label.connection": "連線", "data_import.workbench.label.database": "資料庫", "data_import.workbench.label.default_database": "預設資料庫（選填）", @@ -4446,7 +4530,9 @@ "data_import.workbench.mode.database": "匯入資料庫", "data_import.workbench.mode.table": "匯入資料表", "data_import.workbench.notice.gonavi_mysql_restore": "GoNavi 產生的 MySQL 備份會依檔案中記錄的資料庫名稱還原；其他 SQL 檔案可使用上方選擇的預設資料庫。", + "data_import.workbench.notice.continue_on_error": "繼續模式會記錄錯誤並處理後續語句。MySQL/MariaDB 為避免非交易式資料表重複寫入，將逐筆執行且不會重放失敗批次；大型檔案可能明顯變慢。已提交的語句不會自動回滾。", "data_import.workbench.notice.partial_execution": "SQL 檔案會逐句串流執行。失敗或取消前已成功執行的語句不會自動回復，請先確認目標連線與備份。", + "data_import.workbench.notice.stop_on_error": "遇到第一個 SQL 錯誤後將停止且不會逐條重放失敗批次；系統會嘗試回滾，但非交易式資料表先前的寫入可能保留。先前已提交語句不會自動回滾。", "data_import.workbench.placeholder.loading_databases": "正在載入資料庫...", "data_import.workbench.placeholder.loading_tables": "正在載入資料表...", "data_import.workbench.placeholder.select_connection": "選擇連線", @@ -4455,6 +4541,9 @@ "data_import.workbench.placeholder.select_default_database": "選填：選擇預設資料庫", "data_import.workbench.placeholder.select_table": "選擇目標資料表", "data_import.workbench.progress.bytes": "檔案進度：{{processed}} / {{total}}", + "data_import.workbench.progress.throughput": "速度：{{rate}}/秒", + "data_import.workbench.progress.eta": "預計剩餘：{{duration}}", + "data_import.workbench.progress.skipped": "已略過 {{count}} 列", "data_import.workbench.progress.statements": "語句進度：已執行 {{executed}} 句，失敗 {{failed}} 句", "data_import.workbench.section.target": "匯入目標", "data_import.workbench.state.awaiting_file_description": "目標確定後選擇資料檔案，接著可預覽內容並設定欄位對應。", @@ -4464,6 +4553,7 @@ "data_import.workbench.state.cancelled": "已取消", "data_import.workbench.state.cancelling": "正在取消...", "data_import.workbench.state.completed": "匯入完成", + "data_import.workbench.state.completed_with_errors": "匯入完成（有錯誤）", "data_import.workbench.state.failed": "匯入失敗", "data_import.workbench.state.ready_sql_description": "確認目標和風險提示後，手動開始匯入。", "data_import.workbench.state.ready_sql_title": "SQL 檔案已就緒", @@ -5581,6 +5671,7 @@ "driver_manager.version.unlabeled": "未標註版本", "file.backend.dialog.export_connections": "匯出連線", "file.backend.dialog.export_data": "匯出資料", + "file.backend.dialog.export_import_errors": "匯出匯入失敗資料列", "file.backend.dialog.export_database_sql": "匯出 {{database}} (SQL)", "file.backend.dialog.export_query_result": "匯出查詢結果", "file.backend.dialog.export_table": "匯出 {{table}}", @@ -5627,8 +5718,15 @@ "file.backend.error.import_excel_parse_failed": "無法解析 Excel: {{detail}}", "file.backend.error.import_excel_read_failed": "無法讀取 Excel: {{detail}}", "file.backend.error.import_file_empty": "檔案路徑不能為空", + "file.backend.error.import_error_artifact_not_found": "失敗資料列匯出檔案不存在或已過期", + "file.backend.error.import_job_not_found": "找不到匯入工作", + "file.backend.error.import_job_persist": "儲存匯入工作狀態失敗：{{detail}}", + "file.backend.error.import_job_running": "匯入工作仍在執行，無法刪除", + "file.backend.error.import_resume_unavailable": "目前匯入無法安全恢復", + "file.backend.error.import_source_changed": "匯入來源在預覽後已變更，請重新選擇並預覽檔案。", "file.backend.error.import_json_parse_failed": "無法解析 JSON: {{detail}}", "file.backend.error.import_json_root_not_array": "JSON 根必須是陣列", + "file.backend.error.import_stopped_on_error": "資料表匯入遇錯停止。已確認匯入 {{imported}} 列，記錄 {{failed}} 個錯誤：{{detail}}", "file.backend.error.import_unsupported_format": "不支援的檔案格式", "file.backend.error.invalid_export_mode": "無效的匯出模式", "file.backend.error.mysql_workbench_no_connections": "未在 XML 中找到有效的連線設定", @@ -5656,11 +5754,14 @@ "file.backend.error.selected_path_not_sql_file": "所選路徑不是 SQL 檔案", "file.backend.error.sql_file_batch_execution_failed": "第 {{index}} 條起的批次語句執行失敗: {{detail}}", "file.backend.error.sql_file_batch_rollback_failed": "批次執行失敗: {{detail}}；回滾失敗: {{rollbackDetail}}", + "file.backend.error.sql_file_execution_failed_summary": "SQL 檔案執行中斷（已執行 {{count}} 條）：{{detail}}", "file.backend.error.sql_file_exists": "SQL 檔案已存在", "file.backend.error.sql_file_extension_required": "僅支援 SQL 檔案", "file.backend.error.sql_file_name_no_separator": "SQL 檔案名稱不能包含路徑分隔符", "file.backend.error.sql_file_name_required": "SQL 檔案名稱不能為空", "file.backend.error.sql_file_statement_execution_failed": "第 {{index}} 條語句執行失敗: {{detail}}", + "file.backend.error.sql_file_stopped_on_error_summary": "SQL 檔案執行已停止。成功執行 {{success}} 條，發現 {{failed}} 個錯誤。{{detail}}", + "file.backend.error.sql_file_unclosed_transaction": "SQL 檔案在交易結束前已讀取完畢；匯入已停止，並將清理未提交交易。", "file.backend.error.table_data_batch_limit": "單次最多處理 {{max}} 張表，目前選取 {{count}} 張", "file.backend.error.table_data_clear_failed": "清空 {{table}} 失敗: {{detail}}", "file.backend.error.table_data_clear_failed_partial": "清空 {{table}} 失敗: {{detail}}（注意：前 {{count}} 張表已清空且無法復原）", @@ -5698,6 +5799,7 @@ "file.backend.message.export_completed": "匯出完成", "file.backend.message.import_no_data": "無可匯入資料", "file.backend.message.import_row_failed": "第 {{index}} 列匯入失敗: {{detail}}", + "file.backend.message.import_errors_exported": "匯入失敗資料列已匯出", "file.backend.message.import_summary": "已匯入: {{imported}}, 失敗: {{failed}}", "file.backend.message.statement_failed": "第 {{index}} 條語句執行失敗: {{detail}}\n SQL: {{sql}}", "file.backend.message.table_data_clear_succeeded": "清空成功", @@ -5734,9 +5836,12 @@ "find_in_database.tooltip.collapse_details": "收合詳細資料", "find_in_database.tooltip.view_details": "檢視詳細資料", "import_preview.action.start": "開始匯入", + "import_preview.action.export_rejected_rows": "匯出失敗資料列", "import_preview.error.connection_config_not_found": "找不到連線設定", "import_preview.error.import_failed": "匯入失敗", "import_preview.error.import_failed_detail": "匯入失敗: {{detail}}", + "import_preview.error.export_rejected_rows_failed": "無法匯出失敗資料列", + "import_preview.error.export_rejected_rows_failed_detail": "無法匯出失敗資料列: {{detail}}", "import_preview.error.preview_failed": "預覽失敗", "import_preview.error.preview_failed_detail": "預覽失敗: {{detail}}", "import_preview.error.target_columns_failed": "無法載入目標資料表欄位", @@ -5754,12 +5859,23 @@ "import_preview.preview.description": "以下顯示前 5 列預覽資料，確認無誤後開始匯入。", "import_preview.preview.field_list": "欄位清單：", "import_preview.preview.summary": "共 {{rows}} 列資料，{{columns}} 個欄位", + "import_preview.preview.summary_sample": "目前顯示 {{rows}} 列樣本，未掃描總列數；共 {{columns}} 個欄位", "import_preview.preview.table_title": "資料預覽（前 5 列）：", "import_preview.progress.error_count": "失敗 {{count}}", "import_preview.progress.processed_rows": "已處理 {{current}} / {{total}} 列", + "import_preview.progress.processed_rows_unknown": "已處理 {{current}} 列", "import_preview.progress.success_count": "成功 {{count}}", + "import_preview.stage.prepare": "準備匯入", + "import_preview.stage.preflight": "匯入前檢查", + "import_preview.stage.read": "讀取來源檔案", + "import_preview.stage.parse": "解析來源檔案", + "import_preview.stage.write": "寫入資料", + "import_preview.stage.finalize": "整理匯入結果", "import_preview.result.completed": "匯入完成", + "import_preview.result.batch_outcome_unknown": "失敗批次的寫入結果可能不確定，請核對目標資料表後再重試。", "import_preview.result.error_logs": "錯誤記錄：", + "import_preview.result.error_count": "記錄 {{count}} 個錯誤", + "import_preview.result.error_logs_omitted": "另有 {{count}} 筆錯誤詳細資料未顯示。", "import_preview.result.failed_rows": "失敗 {{count}} 列", "import_preview.result.success_rows": "成功匯入 {{count}} 列", "import_preview.status.importing": "正在匯入資料...", @@ -5770,6 +5886,7 @@ "jvm.backend.connection_error.agent.base_url_invalid.summary": "Agent 連線失敗：Agent Base URL 格式不合法。", "jvm.backend.connection_error.agent.base_url_required": "Agent 連線失敗：未填寫 Agent Base URL。", "import_preview.result.stopped": "匯入已停止（已提交的資料不會回復）", + "import_preview.result.stopped_on_error": "匯入因錯誤停止", "jvm.backend.connection_error.agent.connection_refused.help": "請確認 Java 服務已透過 `-javaagent` 啟動 GoNavi Agent，並檢查 Base URL、連接埠對映與防火牆。", "jvm.backend.connection_error.agent.connection_refused.summary": "Agent 連線失敗：目標 Agent 管理連接埠未監聽，或目前位址無法連線。", "jvm.backend.connection_error.agent.forbidden.help": "請確認目前用戶端來源、鑑權設定和 Agent 存取策略允許 GoNavi 存取。", @@ -7731,12 +7848,13 @@ "sidebar.sql_file_exec.executed_label": "已執行：", "sidebar.sql_file_exec.file_size": "檔案大小：", "sidebar.sql_file_exec.message.already_running": "目前已有外部 SQL 執行任務在進行中", - "sidebar.sql_file_exec.rows_separator": " 列 | 失敗：", - "sidebar.sql_file_exec.rows_suffix": " 列", + "sidebar.sql_file_exec.statements_separator": " 條語句 | 失敗：", + "sidebar.sql_file_exec.statements_suffix": " 條語句", "sidebar.sql_file_exec.status.cancelled": "已終止", "sidebar.sql_file_exec.status.done": "完成", "sidebar.sql_file_exec.status.error": "錯誤", "sidebar.sql_file_exec.status.running": "執行中", + "sidebar.sql_file_exec.status.stopping": "正在停止...", "sidebar.sql_file_exec.status_label": "狀態：", "sidebar.sql_file_exec.title": "執行外部 SQL 檔案", "sidebar.sql_file_exec.workbench.action.clear_history": "清空歷史",