️ perf(import-export): 降低 OceanBase 导出链路内存占用

- 为 optional driver-agent 补齐 streamQuery 分片协议,避免大结果集整批缓冲到内存
- 在 OceanBase 整表导出和查询结果导出前强校验 driver-agent revision,旧版代理直接拦截并提示重装
- 为 driver-agent 增加大查询和流式导出完成后的 GC/FreeOSMemory 回收逻辑
- 补充导出前校验、流式分片消费和 agent 内存回收的定向测试
- 更新 driver-agent revisions 以匹配新的流式导出协议
This commit is contained in:
Syngnat
2026-06-18 11:32:08 +08:00
parent 6bd87fa568
commit c8fe90cbee
8 changed files with 801 additions and 22 deletions

View File

@@ -1,6 +1,9 @@
package db
import (
"bufio"
"bytes"
"strings"
"testing"
"GoNavi-Wails/internal/connection"
@@ -65,3 +68,71 @@ func TestNormalizeKingbaseAgentChangeSetByColumns(t *testing.T) {
t.Fatalf("unexpected update value key \"event name\" after normalization")
}
}
type optionalAgentTestWriteCloser struct {
bytes.Buffer
}
func (w *optionalAgentTestWriteCloser) Close() error { return nil }
type optionalAgentTestStreamConsumer struct {
columns []string
rows [][]interface{}
}
func (c *optionalAgentTestStreamConsumer) SetColumns(columns []string) error {
c.columns = append([]string(nil), columns...)
return nil
}
func (c *optionalAgentTestStreamConsumer) ConsumeRow(row map[string]interface{}) error {
values := make([]interface{}, len(c.columns))
for idx, column := range c.columns {
values[idx] = row[column]
}
c.rows = append(c.rows, values)
return nil
}
func (c *optionalAgentTestStreamConsumer) ConsumeRowValues(values []interface{}) error {
c.rows = append(c.rows, append([]interface{}(nil), values...))
return nil
}
func TestOptionalDriverAgentClientCallStreamQueryConsumesChunks(t *testing.T) {
var stdin optionalAgentTestWriteCloser
stdout := strings.Join([]string{
`{"id":1,"success":true,"chunkType":"columns","fields":["id","name"]}`,
`{"id":1,"success":true,"chunkType":"rows","data":[[1,"alice"],[2,"bob"]]}`,
`{"id":1,"success":true,"chunkType":"done"}`,
}, "\n") + "\n"
client := &optionalDriverAgentClient{
stdin: &stdin,
reader: bufio.NewReader(strings.NewReader(stdout)),
driver: "oceanbase",
}
consumer := &optionalAgentTestStreamConsumer{}
if err := client.callStreamQuery(optionalAgentRequest{
Method: optionalAgentMethodStreamQuery,
Query: "SELECT 1",
}, consumer); err != nil {
t.Fatalf("callStreamQuery 返回错误: %v", err)
}
if len(consumer.columns) != 2 || consumer.columns[0] != "id" || consumer.columns[1] != "name" {
t.Fatalf("流式列定义异常: %#v", consumer.columns)
}
if len(consumer.rows) != 2 {
t.Fatalf("流式行数异常: %#v", consumer.rows)
}
if got := consumer.rows[0][1]; got != "alice" {
t.Fatalf("第 1 行数据异常want=%q got=%v", "alice", got)
}
if got := consumer.rows[1][0]; got != int64(2) {
t.Fatalf("第 2 行 ID 异常want=%d got=%v (%T)", 2, got, got)
}
if !strings.Contains(stdin.String(), `"method":"streamQuery"`) {
t.Fatalf("请求未使用 streamQuery 方法: %s", stdin.String())
}
}