feat(data-export): 增强备份进度与原子写入

- 为整库与 Schema 备份补齐逐对象进度和最终文件路径
- 通过临时文件同步落盘和原子替换保护已有完整备份
- 增加任务进度及失败保留回归测试
This commit is contained in:
Syngnat
2026-07-20 13:43:20 +08:00
parent 89e35b087a
commit 19d962d9ba
2 changed files with 340 additions and 36 deletions

View File

@@ -2779,15 +2779,14 @@ func (a *App) ExportTableWithOptions(config connection.ConnectionConfig, dbName
if format == "sql" {
reporter.Start(a.appText("data_export.progress.stage.exporting_sql_file", nil))
f, err := os.Create(filename)
target, err := createAtomicExportTarget(filename)
if err != nil {
reporter.Error(0, err.Error())
return connection.QueryResult{Success: false, Message: err.Error()}
}
defer f.Close()
defer target.abort()
w := bufio.NewWriterSize(f, 1024*1024)
defer w.Flush()
w := bufio.NewWriterSize(target.file, 1024*1024)
if err := writeSQLHeader(w, runConfig, dbName); err != nil {
reporter.Error(0, err.Error())
@@ -2816,6 +2815,14 @@ func (a *App) ExportTableWithOptions(config connection.ConnectionConfig, dbName
}
reporter.Finalizing(0)
if err := w.Flush(); err != nil {
reporter.Error(0, err.Error())
return connection.QueryResult{Success: false, Message: err.Error()}
}
if err := target.commit(); err != nil {
reporter.Error(0, err.Error())
return connection.QueryResult{Success: false, Message: err.Error()}
}
reporter.Done(0)
maybeReleaseFileTransferMemory("export-table-sql-finished", 0, filename)
return connection.QueryResult{Success: true, Message: a.appText("file.backend.message.export_completed", nil)}
@@ -2919,6 +2926,58 @@ func (a *App) exportTablesSQL(config connection.ConnectionConfig, dbName string,
)
}
type atomicExportTarget struct {
file *os.File
tempPath string
targetPath string
closed bool
committed bool
}
func createAtomicExportTarget(targetPath string) (*atomicExportTarget, error) {
temporary, err := os.CreateTemp(filepath.Dir(targetPath), ".gonavi-export-*.part")
if err != nil {
return nil, err
}
return &atomicExportTarget{
file: temporary,
tempPath: temporary.Name(),
targetPath: targetPath,
}, nil
}
func (target *atomicExportTarget) abort() {
if target == nil {
return
}
if !target.closed {
_ = target.file.Close()
target.closed = true
}
if !target.committed {
_ = os.Remove(target.tempPath)
}
}
func (target *atomicExportTarget) commit() error {
if target == nil || target.file == nil {
return errors.New("invalid atomic export target")
}
if err := target.file.Sync(); err != nil {
return err
}
closeErr := target.file.Close()
target.closed = true
if closeErr != nil {
return closeErr
}
if err := atomicReplaceSQLAuditFile(target.tempPath, target.targetPath); err != nil {
return err
}
target.committed = true
return nil
}
func (a *App) exportTablesSQLToFile(
config connection.ConnectionConfig,
dbName string,
@@ -2945,17 +3004,16 @@ func (a *App) exportTablesSQLToFile(
viewLookup := listViewNameLookup(dbInst, runConfig, dbName)
objects := buildExportObjectOrder(runConfig, dbName, normalizeExportNameList(tableNames), viewLookup, false)
f, err := os.Create(filename)
target, err := createAtomicExportTarget(filename)
if err != nil {
if reporter != nil {
reporter.Error(0, err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
defer f.Close()
defer target.abort()
w := bufio.NewWriterSize(f, 1024*1024)
defer w.Flush()
w := bufio.NewWriterSize(target.file, 1024*1024)
if err := writeSQLHeader(w, runConfig, dbName); err != nil {
if reporter != nil {
@@ -2991,13 +3049,6 @@ func (a *App) exportTablesSQLToFile(
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
if reporter != nil {
reporter.ForceRunning(int64(index+1), a.appText("data_export.progress.stage.exporting_item_with_progress", map[string]any{
"name": objectName,
"current": index + 1,
"total": len(objects),
}))
}
}
if err := writeSQLFooter(w, runConfig); err != nil {
if reporter != nil {
@@ -3008,6 +3059,20 @@ func (a *App) exportTablesSQLToFile(
if reporter != nil {
reporter.Finalizing(int64(len(objects)))
}
if err := w.Flush(); err != nil {
if reporter != nil {
reporter.Error(int64(len(objects)), err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
if err := target.commit(); err != nil {
if reporter != nil {
reporter.Error(int64(len(objects)), err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
if reporter != nil {
reporter.Done(int64(len(objects)))
}
return connection.QueryResult{
@@ -3088,20 +3153,16 @@ func (a *App) ExportDatabasesSQLWithOptions(
}))
}
targetFile := filepath.Join(directory, buildDatabaseExportDefaultFilename(name, includeData))
result := a.exportDatabaseSQLToFile(config, name, includeData, targetFile, options)
innerOptions := options
innerOptions.JobID = ""
result := a.exportDatabaseSQLToFile(config, name, includeData, targetFile, innerOptions)
if !result.Success {
result.Message = fmt.Sprintf("%s: %s", targetFile, result.Message)
if reporter != nil {
reporter.Error(int64(index), result.Message)
}
return result
}
if reporter != nil {
reporter.ForceRunning(int64(index+1), a.appText("data_export.progress.stage.exporting_item_with_progress", map[string]any{
"name": name,
"current": index + 1,
"total": len(normalizedDbNames),
}))
}
}
if reporter != nil {
@@ -3129,30 +3190,50 @@ func (a *App) exportDatabaseSQLToFile(
if safeDbName == "" {
return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.database_name_required", nil)}
}
reporter := newExportProgressReporter(a, options, safeDbName, filename)
if reporter != nil {
reporter.Start(a.appText("data_export.progress.stage.preparing_export", nil))
}
runConfig := normalizeRunConfig(config, dbName)
dbInst, err := a.getDatabase(runConfig)
if err != nil {
if reporter != nil {
reporter.Error(0, err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
tables, err := dbInst.GetTables(dbName)
if err != nil {
if reporter != nil {
reporter.Error(0, err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
viewLookup := listViewNameLookup(dbInst, runConfig, dbName)
objects := buildExportObjectOrder(runConfig, dbName, tables, viewLookup, true)
if reporter != nil {
reporter.totalRows = int64(len(objects))
reporter.totalRowsKnown = true
reporter.ForceRunning(0, a.appText("data_export.progress.stage.exporting_sql_file", nil))
}
f, err := os.Create(filename)
target, err := createAtomicExportTarget(filename)
if err != nil {
if reporter != nil {
reporter.Error(0, err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
defer f.Close()
defer target.abort()
w := bufio.NewWriterSize(f, 1024*1024)
defer w.Flush()
w := bufio.NewWriterSize(target.file, 1024*1024)
if err := writeSQLDatabaseBackupHeader(w, runConfig, dbName); err != nil {
if reporter != nil {
reporter.Error(0, err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
if err := writeSQLDropIfExistsPreamble(
@@ -3164,16 +3245,50 @@ func (a *App) exportDatabaseSQLToFile(
true,
options,
); err != nil {
if reporter != nil {
reporter.Error(0, err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
for _, objectName := range objects {
for index, objectName := range objects {
if reporter != nil {
reporter.ForceRunning(int64(index), a.appText("data_export.progress.stage.exporting_item_with_progress", map[string]any{
"name": objectName,
"current": index + 1,
"total": len(objects),
}))
}
if err := dumpTableSQL(w, dbInst, runConfig, dbName, objectName, true, includeData, viewLookup); err != nil {
if reporter != nil {
reporter.Error(int64(index), err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
}
if err := writeSQLFooter(w, runConfig); err != nil {
if reporter != nil {
reporter.Error(int64(len(objects)), err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
if reporter != nil {
reporter.Finalizing(int64(len(objects)))
}
if err := w.Flush(); err != nil {
if reporter != nil {
reporter.Error(int64(len(objects)), err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
if err := target.commit(); err != nil {
if reporter != nil {
reporter.Error(int64(len(objects)), err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
if reporter != nil {
reporter.Done(int64(len(objects)))
}
return connection.QueryResult{
Success: true,
@@ -3222,15 +3337,25 @@ func (a *App) ExportSchemaSQLWithOptions(
if err != nil {
return connection.QueryResult{Success: false, Message: err.Error()}
}
reporter := newExportProgressReporter(a, options, safeDbName+"."+safeSchemaName, filename)
if reporter != nil {
reporter.Start(a.appText("data_export.progress.stage.preparing_export", nil))
}
runConfig := normalizeRunConfig(config, dbName)
dbInst, err := a.getDatabase(runConfig)
if err != nil {
if reporter != nil {
reporter.Error(0, err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
tables, err := dbInst.GetTables(dbName)
if err != nil {
if reporter != nil {
reporter.Error(0, err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
viewLookup := listViewNameLookup(dbInst, runConfig, dbName)
@@ -3238,19 +3363,33 @@ func (a *App) ExportSchemaSQLWithOptions(
filteredViews := filterExportViewLookupBySchema(runConfig, dbName, viewLookup, safeSchemaName)
objects := buildExportObjectOrder(runConfig, dbName, filteredTables, filteredViews, true)
if len(objects) == 0 {
return connection.QueryResult{Success: false, Message: a.appText("file.backend.error.schema_export_no_objects", map[string]any{"schema": safeSchemaName})}
message := a.appText("file.backend.error.schema_export_no_objects", map[string]any{"schema": safeSchemaName})
if reporter != nil {
reporter.Error(0, message)
}
return connection.QueryResult{Success: false, Message: message}
}
if reporter != nil {
reporter.totalRows = int64(len(objects))
reporter.totalRowsKnown = true
reporter.ForceRunning(0, a.appText("data_export.progress.stage.exporting_sql_file", nil))
}
f, err := os.Create(filename)
target, err := createAtomicExportTarget(filename)
if err != nil {
if reporter != nil {
reporter.Error(0, err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
defer f.Close()
defer target.abort()
w := bufio.NewWriterSize(f, 1024*1024)
defer w.Flush()
w := bufio.NewWriterSize(target.file, 1024*1024)
if err := writeSQLSchemaExportHeader(w, runConfig, dbName, safeSchemaName); err != nil {
if reporter != nil {
reporter.Error(0, err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
if err := writeSQLDropIfExistsPreamble(
@@ -3262,18 +3401,58 @@ func (a *App) ExportSchemaSQLWithOptions(
true,
options,
); err != nil {
if reporter != nil {
reporter.Error(0, err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
for _, objectName := range objects {
for index, objectName := range objects {
if reporter != nil {
reporter.ForceRunning(int64(index), a.appText("data_export.progress.stage.exporting_item_with_progress", map[string]any{
"name": objectName,
"current": index + 1,
"total": len(objects),
}))
}
if err := dumpTableSQL(w, dbInst, runConfig, dbName, objectName, true, includeData, filteredViews); err != nil {
if reporter != nil {
reporter.Error(int64(index), err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
}
if err := writeSQLFooter(w, runConfig); err != nil {
if reporter != nil {
reporter.Error(int64(len(objects)), err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
if reporter != nil {
reporter.Finalizing(int64(len(objects)))
}
if err := w.Flush(); err != nil {
if reporter != nil {
reporter.Error(int64(len(objects)), err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
if err := target.commit(); err != nil {
if reporter != nil {
reporter.Error(int64(len(objects)), err.Error())
}
return connection.QueryResult{Success: false, Message: err.Error()}
}
if reporter != nil {
reporter.Done(int64(len(objects)))
}
return connection.QueryResult{Success: true, Message: a.appText("file.backend.message.export_completed", nil)}
return connection.QueryResult{
Success: true,
Message: a.appText("file.backend.message.export_completed", nil),
Data: map[string]interface{}{
"filePath": filename,
},
}
}
type tableDataClearMode string

View File

@@ -8,12 +8,15 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"GoNavi-Wails/internal/connection"
"GoNavi-Wails/internal/db"
"GoNavi-Wails/internal/uievents"
"GoNavi-Wails/shared/i18n"
"github.com/xuri/excelize/v2"
)
@@ -57,6 +60,21 @@ type fakeSQLDumpExportDB struct {
fakeExportQueryDB
tables []string
createSQL string
createErr error
}
type captureExportProgressEmitter struct {
events []exportProgressPayload
}
func (e *captureExportProgressEmitter) Emit(name string, args ...any) {
if name != exportProgressEvent || len(args) == 0 {
return
}
payload, ok := args[0].(exportProgressPayload)
if ok {
e.events = append(e.events, payload)
}
}
func (f *fakeExportQueryDB) Connect(config connection.ConnectionConfig) error { return nil }
@@ -103,7 +121,7 @@ func (f *fakeSQLDumpExportDB) GetTables(dbName string) ([]string, error) {
}
func (f *fakeSQLDumpExportDB) GetCreateStatement(dbName, tableName string) (string, error) {
return f.createSQL, nil
return f.createSQL, f.createErr
}
func (f *fakeStreamExportDB) Query(query string) ([]map[string]interface{}, []string, error) {
@@ -1780,6 +1798,113 @@ func TestExportDatabaseSQLToFileDefaultOptionsDoNotEmitDrops(t *testing.T) {
}
}
func TestExportDatabaseSQLToFileReportsObjectProgress(t *testing.T) {
originalNewDatabaseFunc := newDatabaseFunc
t.Cleanup(func() {
newDatabaseFunc = originalNewDatabaseFunc
})
fakeDB := &fakeSQLDumpExportDB{
tables: []string{"users", "orders"},
createSQL: "CREATE TABLE `placeholder` (`id` BIGINT)",
}
newDatabaseFunc = func(string) (db.Database, error) {
return fakeDB, nil
}
emitter := &captureExportProgressEmitter{}
app := NewApp()
app.ctx = uievents.WithEmitter(context.Background(), emitter)
filePath := filepath.Join(t.TempDir(), "app_backup.sql")
result := app.exportDatabaseSQLToFile(
connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306},
"app",
false,
filePath,
ExportFileOptions{Format: "sql", JobID: "database-backup-job"},
)
if !result.Success {
t.Fatalf("database export failed: %+v", result)
}
if len(emitter.events) < 4 {
t.Fatalf("expected start/running/finalizing/done events, got %#v", emitter.events)
}
statuses := make([]string, 0, len(emitter.events))
for _, event := range emitter.events {
statuses = append(statuses, event.Status)
if event.JobID != "database-backup-job" {
t.Fatalf("unexpected progress job id: %#v", event)
}
if event.FilePath != filePath {
t.Fatalf("progress must expose selected backup path: %#v", event)
}
}
for _, want := range []string{"start", "running", "finalizing", "done"} {
if !slices.Contains(statuses, want) {
t.Fatalf("missing %q progress status in %v", want, statuses)
}
}
itemEvents := make([]exportProgressPayload, 0, 2)
for _, event := range emitter.events {
if event.Status == "running" && (strings.Contains(event.Stage, "users") || strings.Contains(event.Stage, "orders")) {
itemEvents = append(itemEvents, event)
}
}
if len(itemEvents) != 2 || itemEvents[0].Current != 0 || itemEvents[1].Current != 1 {
t.Fatalf("expected one running event per object with completed-object counts, got %#v", itemEvents)
}
last := emitter.events[len(emitter.events)-1]
if !last.TotalRowsKnown || last.Total != 2 || last.Current != 2 {
t.Fatalf("done progress must report all exported objects: %#v", last)
}
}
func TestExportDatabaseSQLToFilePreservesExistingBackupOnFailure(t *testing.T) {
originalNewDatabaseFunc := newDatabaseFunc
t.Cleanup(func() {
newDatabaseFunc = originalNewDatabaseFunc
})
fakeDB := &fakeSQLDumpExportDB{
tables: []string{"users"},
createErr: fmt.Errorf("forced create statement failure"),
}
newDatabaseFunc = func(string) (db.Database, error) {
return fakeDB, nil
}
directory := t.TempDir()
filePath := filepath.Join(directory, "app_backup.sql")
const previousBackup = "-- previous complete backup\n"
if err := os.WriteFile(filePath, []byte(previousBackup), 0o600); err != nil {
t.Fatalf("write previous backup: %v", err)
}
result := NewApp().exportDatabaseSQLToFile(
connection.ConnectionConfig{Type: "mysql", Host: "127.0.0.1", Port: 3306},
"app",
false,
filePath,
ExportFileOptions{Format: "sql"},
)
if result.Success {
t.Fatalf("expected export failure, got %+v", result)
}
content, err := os.ReadFile(filePath)
if err != nil {
t.Fatalf("read preserved backup: %v", err)
}
if string(content) != previousBackup {
t.Fatalf("failed export must preserve previous backup, got %q", content)
}
temporaryFiles, err := filepath.Glob(filepath.Join(directory, ".gonavi-export-*.part"))
if err != nil {
t.Fatalf("glob temporary export files: %v", err)
}
if len(temporaryFiles) != 0 {
t.Fatalf("failed export must remove temporary files, got %v", temporaryFiles)
}
}
func TestWriteSQLDropIfExistsPreambleReversesCreateOrderAndDistinguishesViews(t *testing.T) {
config := connection.ConnectionConfig{Type: "mysql"}
objects := []string{"accounts", "orders", "active_orders"}