️ perf(import-export): 降低 OceanBase 导出链路内存占用

- 为 optional driver-agent 补齐 streamQuery 分片协议,避免大结果集整批缓冲到内存
- 在 OceanBase 整表导出和查询结果导出前强校验 driver-agent revision,旧版代理直接拦截并提示重装
- 为 driver-agent 增加大查询和流式导出完成后的 GC/FreeOSMemory 回收逻辑
- 补充导出前校验、流式分片消费和 agent 内存回收的定向测试
- 更新 driver-agent revisions 以匹配新的流式导出协议
This commit is contained in:
Syngnat
2026-06-18 11:32:08 +08:00
parent 6bd87fa568
commit c8fe90cbee
8 changed files with 801 additions and 22 deletions

View File

@@ -418,6 +418,7 @@ var (
var optionalDriverSourceBuildTimeout = 8 * time.Minute
var validateOptionalDriverAgentExecutableFunc = db.ValidateOptionalDriverAgentExecutable
var resolveOptionalDriverAgentExecutablePathFunc = db.ResolveOptionalDriverAgentExecutablePath
type driverVersionWarmupState struct {
Running bool

View File

@@ -342,6 +342,23 @@ func tryResolveExportTableTotalRows(dbInst db.Database, config connection.Connec
return resolveExportTotalRowsFromRows(rows)
}
func verifyOptionalDriverAgentReadyForExport(config connection.ConnectionConfig) error {
driverType := normalizeDriverType(config.Type)
if !db.IsOptionalGoDriver(driverType) {
return nil
}
executablePath, err := resolveOptionalDriverAgentExecutablePathFunc("", driverType)
if err != nil {
return err
}
if _, err := verifyInstalledOptionalDriverAgentRevision(driverType, executablePath); err != nil {
displayName := resolveDriverDisplayName(driverDefinition{Type: driverType})
return fmt.Errorf("当前导出依赖最新的 %s driver-agent 流式协议;为避免大结果集回退到高内存缓冲模式,请在驱动管理中重装后重试:%w", displayName, err)
}
return nil
}
var exportFileNameSanitizer = strings.NewReplacer(
"/", "_",
"\\", "_",
@@ -2249,6 +2266,11 @@ func (a *App) ExportTable(config connection.ConnectionConfig, dbName string, tab
func (a *App) ExportTableWithOptions(config connection.ConnectionConfig, dbName string, tableName string, options ExportFileOptions) connection.QueryResult {
options = normalizeExportFileOptions("", options)
format := options.Format
if format != "sql" {
if err := verifyOptionalDriverAgentReadyForExport(config); err != nil {
return connection.QueryResult{Success: false, Message: err.Error()}
}
}
filename, err := runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{
Title: fmt.Sprintf("Export %s", tableName),
DefaultFilename: fmt.Sprintf("%s.%s", tableName, format),
@@ -3656,6 +3678,11 @@ func (a *App) ExportQueryWithOptions(config connection.ConnectionConfig, dbName
}
options = normalizeExportFileOptions("", options)
format := options.Format
if format != "sql" {
if err := verifyOptionalDriverAgentReadyForExport(config); err != nil {
return connection.QueryResult{Success: false, Message: err.Error()}
}
}
filename, err := runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{
Title: "Export Query Result",

View File

@@ -475,6 +475,49 @@ func TestTryResolveExportTableTotalRows_UsesCountQuery(t *testing.T) {
}
}
func TestVerifyOptionalDriverAgentReadyForExport_RejectsStaleAgent(t *testing.T) {
originalProbe := optionalDriverAgentMetadataProbe
originalResolvePath := resolveOptionalDriverAgentExecutablePathFunc
t.Cleanup(func() {
optionalDriverAgentMetadataProbe = originalProbe
resolveOptionalDriverAgentExecutablePathFunc = originalResolvePath
})
resolveOptionalDriverAgentExecutablePathFunc = func(downloadDir string, driverType string) (string, error) {
return "/tmp/oceanbase-driver-agent", nil
}
optionalDriverAgentMetadataProbe = func(driverType string, executablePath string) (db.OptionalDriverAgentMetadata, error) {
return db.OptionalDriverAgentMetadata{
DriverType: driverType,
AgentRevision: "src-stale-agent",
}, nil
}
err := verifyOptionalDriverAgentReadyForExport(connection.ConnectionConfig{Type: "oceanbase"})
if err == nil {
t.Fatal("预期旧版 OceanBase driver-agent 被导出前校验拦截")
}
if !strings.Contains(err.Error(), "流式协议") {
t.Fatalf("错误信息应说明需要流式协议got=%q", err.Error())
}
}
func TestVerifyOptionalDriverAgentReadyForExport_SkipsBuiltInDriver(t *testing.T) {
originalResolvePath := resolveOptionalDriverAgentExecutablePathFunc
t.Cleanup(func() {
resolveOptionalDriverAgentExecutablePathFunc = originalResolvePath
})
resolveOptionalDriverAgentExecutablePathFunc = func(downloadDir string, driverType string) (string, error) {
t.Fatalf("内置驱动导出不应探测 optional driver-agent 路径")
return "", nil
}
if err := verifyOptionalDriverAgentReadyForExport(connection.ConnectionConfig{Type: "mysql"}); err != nil {
t.Fatalf("内置驱动导出不应被 optional driver-agent 校验阻断: %v", err)
}
}
func TestExportQueryResultToFile_UsesStreamQueryPath(t *testing.T) {
f, err := os.CreateTemp("", "gonavi-export-stream-*.csv")
if err != nil {