mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-07-12 16:02:48 +08:00
⚡️ perf(import-export): 降低 OceanBase 导出链路内存占用
- 为 optional driver-agent 补齐 streamQuery 分片协议,避免大结果集整批缓冲到内存 - 在 OceanBase 整表导出和查询结果导出前强校验 driver-agent revision,旧版代理直接拦截并提示重装 - 为 driver-agent 增加大查询和流式导出完成后的 GC/FreeOSMemory 回收逻辑 - 补充导出前校验、流式分片消费和 agent 内存回收的定向测试 - 更新 driver-agent revisions 以匹配新的流式导出协议
This commit is contained in:
@@ -418,6 +418,7 @@ var (
|
||||
var optionalDriverSourceBuildTimeout = 8 * time.Minute
|
||||
|
||||
var validateOptionalDriverAgentExecutableFunc = db.ValidateOptionalDriverAgentExecutable
|
||||
var resolveOptionalDriverAgentExecutablePathFunc = db.ResolveOptionalDriverAgentExecutablePath
|
||||
|
||||
type driverVersionWarmupState struct {
|
||||
Running bool
|
||||
|
||||
@@ -342,6 +342,23 @@ func tryResolveExportTableTotalRows(dbInst db.Database, config connection.Connec
|
||||
return resolveExportTotalRowsFromRows(rows)
|
||||
}
|
||||
|
||||
func verifyOptionalDriverAgentReadyForExport(config connection.ConnectionConfig) error {
|
||||
driverType := normalizeDriverType(config.Type)
|
||||
if !db.IsOptionalGoDriver(driverType) {
|
||||
return nil
|
||||
}
|
||||
|
||||
executablePath, err := resolveOptionalDriverAgentExecutablePathFunc("", driverType)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := verifyInstalledOptionalDriverAgentRevision(driverType, executablePath); err != nil {
|
||||
displayName := resolveDriverDisplayName(driverDefinition{Type: driverType})
|
||||
return fmt.Errorf("当前导出依赖最新的 %s driver-agent 流式协议;为避免大结果集回退到高内存缓冲模式,请在驱动管理中重装后重试:%w", displayName, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var exportFileNameSanitizer = strings.NewReplacer(
|
||||
"/", "_",
|
||||
"\\", "_",
|
||||
@@ -2249,6 +2266,11 @@ func (a *App) ExportTable(config connection.ConnectionConfig, dbName string, tab
|
||||
func (a *App) ExportTableWithOptions(config connection.ConnectionConfig, dbName string, tableName string, options ExportFileOptions) connection.QueryResult {
|
||||
options = normalizeExportFileOptions("", options)
|
||||
format := options.Format
|
||||
if format != "sql" {
|
||||
if err := verifyOptionalDriverAgentReadyForExport(config); err != nil {
|
||||
return connection.QueryResult{Success: false, Message: err.Error()}
|
||||
}
|
||||
}
|
||||
filename, err := runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{
|
||||
Title: fmt.Sprintf("Export %s", tableName),
|
||||
DefaultFilename: fmt.Sprintf("%s.%s", tableName, format),
|
||||
@@ -3656,6 +3678,11 @@ func (a *App) ExportQueryWithOptions(config connection.ConnectionConfig, dbName
|
||||
}
|
||||
options = normalizeExportFileOptions("", options)
|
||||
format := options.Format
|
||||
if format != "sql" {
|
||||
if err := verifyOptionalDriverAgentReadyForExport(config); err != nil {
|
||||
return connection.QueryResult{Success: false, Message: err.Error()}
|
||||
}
|
||||
}
|
||||
|
||||
filename, err := runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{
|
||||
Title: "Export Query Result",
|
||||
|
||||
@@ -475,6 +475,49 @@ func TestTryResolveExportTableTotalRows_UsesCountQuery(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyOptionalDriverAgentReadyForExport_RejectsStaleAgent(t *testing.T) {
|
||||
originalProbe := optionalDriverAgentMetadataProbe
|
||||
originalResolvePath := resolveOptionalDriverAgentExecutablePathFunc
|
||||
t.Cleanup(func() {
|
||||
optionalDriverAgentMetadataProbe = originalProbe
|
||||
resolveOptionalDriverAgentExecutablePathFunc = originalResolvePath
|
||||
})
|
||||
|
||||
resolveOptionalDriverAgentExecutablePathFunc = func(downloadDir string, driverType string) (string, error) {
|
||||
return "/tmp/oceanbase-driver-agent", nil
|
||||
}
|
||||
optionalDriverAgentMetadataProbe = func(driverType string, executablePath string) (db.OptionalDriverAgentMetadata, error) {
|
||||
return db.OptionalDriverAgentMetadata{
|
||||
DriverType: driverType,
|
||||
AgentRevision: "src-stale-agent",
|
||||
}, nil
|
||||
}
|
||||
|
||||
err := verifyOptionalDriverAgentReadyForExport(connection.ConnectionConfig{Type: "oceanbase"})
|
||||
if err == nil {
|
||||
t.Fatal("预期旧版 OceanBase driver-agent 被导出前校验拦截")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "流式协议") {
|
||||
t.Fatalf("错误信息应说明需要流式协议,got=%q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyOptionalDriverAgentReadyForExport_SkipsBuiltInDriver(t *testing.T) {
|
||||
originalResolvePath := resolveOptionalDriverAgentExecutablePathFunc
|
||||
t.Cleanup(func() {
|
||||
resolveOptionalDriverAgentExecutablePathFunc = originalResolvePath
|
||||
})
|
||||
|
||||
resolveOptionalDriverAgentExecutablePathFunc = func(downloadDir string, driverType string) (string, error) {
|
||||
t.Fatalf("内置驱动导出不应探测 optional driver-agent 路径")
|
||||
return "", nil
|
||||
}
|
||||
|
||||
if err := verifyOptionalDriverAgentReadyForExport(connection.ConnectionConfig{Type: "mysql"}); err != nil {
|
||||
t.Fatalf("内置驱动导出不应被 optional driver-agent 校验阻断: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExportQueryResultToFile_UsesStreamQueryPath(t *testing.T) {
|
||||
f, err := os.CreateTemp("", "gonavi-export-stream-*.csv")
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user