mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-08-25 02:00:09 +08:00
⚡️ perf(import-export): 降低 OceanBase 导出链路内存占用
- 为 optional driver-agent 补齐 streamQuery 分片协议,避免大结果集整批缓冲到内存 - 在 OceanBase 整表导出和查询结果导出前强校验 driver-agent revision,旧版代理直接拦截并提示重装 - 为 driver-agent 增加大查询和流式导出完成后的 GC/FreeOSMemory 回收逻辑 - 补充导出前校验、流式分片消费和 agent 内存回收的定向测试 - 更新 driver-agent revisions 以匹配新的流式导出协议
This commit is contained in:
@@ -7,8 +7,11 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"runtime"
|
||||||
|
"runtime/debug"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"GoNavi-Wails/internal/connection"
|
"GoNavi-Wails/internal/connection"
|
||||||
@@ -33,6 +36,7 @@ type agentResponse struct {
|
|||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
Data interface{} `json:"data,omitempty"`
|
Data interface{} `json:"data,omitempty"`
|
||||||
Fields []string `json:"fields,omitempty"`
|
Fields []string `json:"fields,omitempty"`
|
||||||
|
ChunkType string `json:"chunkType,omitempty"`
|
||||||
RowsAffected int64 `json:"rowsAffected,omitempty"`
|
RowsAffected int64 `json:"rowsAffected,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -44,6 +48,7 @@ const (
|
|||||||
agentMethodOpenSession = "openSession"
|
agentMethodOpenSession = "openSession"
|
||||||
agentMethodCloseSession = "closeSession"
|
agentMethodCloseSession = "closeSession"
|
||||||
agentMethodQuery = "query"
|
agentMethodQuery = "query"
|
||||||
|
agentMethodStreamQuery = "streamQuery"
|
||||||
agentMethodExec = "exec"
|
agentMethodExec = "exec"
|
||||||
agentMethodGetDatabases = "getDatabases"
|
agentMethodGetDatabases = "getDatabases"
|
||||||
agentMethodGetTables = "getTables"
|
agentMethodGetTables = "getTables"
|
||||||
@@ -58,9 +63,27 @@ const (
|
|||||||
|
|
||||||
const legacyClickHouseDefaultTimeout = 2 * time.Hour
|
const legacyClickHouseDefaultTimeout = 2 * time.Hour
|
||||||
|
|
||||||
|
const (
|
||||||
|
agentChunkColumns = "columns"
|
||||||
|
agentChunkRows = "rows"
|
||||||
|
agentChunkDone = "done"
|
||||||
|
agentStreamBatchSize = 256
|
||||||
|
agentMemoryTrimRowsThreshold = 100000
|
||||||
|
agentMemoryTrimMinInterval = 3 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
agentDriverType string
|
agentDriverType string
|
||||||
agentDatabaseFactory func() db.Database
|
agentDatabaseFactory func() db.Database
|
||||||
|
agentMemoryTrimRunning atomic.Bool
|
||||||
|
agentMemoryTrimLastAt atomic.Int64
|
||||||
|
runAgentMemoryTrimAsync = func(fn func()) {
|
||||||
|
go fn()
|
||||||
|
}
|
||||||
|
agentMemoryTrimFn = func() {
|
||||||
|
runtime.GC()
|
||||||
|
debug.FreeOSMemory()
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
type agentRuntime struct {
|
type agentRuntime struct {
|
||||||
@@ -99,11 +122,22 @@ func main() {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if strings.TrimSpace(req.Method) == agentMethodStreamQuery {
|
||||||
|
if err := handleStreamRequest(runtimeState, req, writer); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "写入流式响应失败:%v\n", err)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
resp := handleRequest(runtimeState, req)
|
resp := handleRequest(runtimeState, req)
|
||||||
if err := writeResponse(writer, resp); err != nil {
|
if err := writeResponse(writer, resp); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "写入响应失败:%v\n", err)
|
fmt.Fprintf(os.Stderr, "写入响应失败:%v\n", err)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
if strings.TrimSpace(req.Method) == agentMethodQuery {
|
||||||
|
maybeReleaseAgentMemory("query-response", countAgentResponseRows(resp.Data))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
runtimeState.close()
|
runtimeState.close()
|
||||||
@@ -288,6 +322,108 @@ func handleRequest(runtimeState *agentRuntime, req agentRequest) agentResponse {
|
|||||||
return resp
|
return resp
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type agentStreamResponseWriter struct {
|
||||||
|
writer *bufio.Writer
|
||||||
|
requestID int64
|
||||||
|
columns []string
|
||||||
|
rows [][]interface{}
|
||||||
|
rowCount int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func newAgentStreamResponseWriter(writer *bufio.Writer, requestID int64) *agentStreamResponseWriter {
|
||||||
|
return &agentStreamResponseWriter{
|
||||||
|
writer: writer,
|
||||||
|
requestID: requestID,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *agentStreamResponseWriter) SetColumns(columns []string) error {
|
||||||
|
w.columns = append([]string(nil), columns...)
|
||||||
|
return writeResponse(w.writer, agentResponse{
|
||||||
|
ID: w.requestID,
|
||||||
|
Success: true,
|
||||||
|
ChunkType: agentChunkColumns,
|
||||||
|
Fields: w.columns,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *agentStreamResponseWriter) ConsumeRow(row map[string]interface{}) error {
|
||||||
|
if len(w.columns) == 0 {
|
||||||
|
return fmt.Errorf("流式查询缺少列定义")
|
||||||
|
}
|
||||||
|
values := make([]interface{}, len(w.columns))
|
||||||
|
for idx, column := range w.columns {
|
||||||
|
values[idx] = row[column]
|
||||||
|
}
|
||||||
|
return w.ConsumeRowValues(values)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *agentStreamResponseWriter) ConsumeRowValues(values []interface{}) error {
|
||||||
|
row := append([]interface{}(nil), values...)
|
||||||
|
w.rows = append(w.rows, row)
|
||||||
|
w.rowCount++
|
||||||
|
if len(w.rows) < agentStreamBatchSize {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return w.flushRows()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *agentStreamResponseWriter) flushRows() error {
|
||||||
|
if len(w.rows) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
rows := w.rows
|
||||||
|
w.rows = nil
|
||||||
|
return writeResponse(w.writer, agentResponse{
|
||||||
|
ID: w.requestID,
|
||||||
|
Success: true,
|
||||||
|
ChunkType: agentChunkRows,
|
||||||
|
Data: rows,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *agentStreamResponseWriter) finish() error {
|
||||||
|
return w.flushRows()
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleStreamRequest(runtimeState *agentRuntime, req agentRequest, writer *bufio.Writer) error {
|
||||||
|
resp := agentResponse{ID: req.ID, Success: true}
|
||||||
|
if runtimeState.inst == nil {
|
||||||
|
return writeResponse(writer, fail(resp, "connection not open"))
|
||||||
|
}
|
||||||
|
|
||||||
|
streamWriter := newAgentStreamResponseWriter(writer, req.ID)
|
||||||
|
if session, ok, err := runtimeState.session(req.SessionID); err != nil {
|
||||||
|
return writeResponse(writer, fail(resp, err.Error()))
|
||||||
|
} else if ok {
|
||||||
|
if err := streamStatementWithOptionalTimeout(session, req.Query, req.TimeoutMs, streamWriter); err != nil {
|
||||||
|
_ = streamWriter.finish()
|
||||||
|
return writeResponse(writer, fail(resp, err.Error()))
|
||||||
|
}
|
||||||
|
if err := streamWriter.finish(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := writeResponse(writer, agentResponse{ID: req.ID, Success: true, ChunkType: agentChunkDone}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
maybeReleaseAgentMemory("stream-query-session", streamWriter.rowCount)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := streamDatabaseWithOptionalTimeout(runtimeState.inst, req.Query, req.TimeoutMs, streamWriter); err != nil {
|
||||||
|
_ = streamWriter.finish()
|
||||||
|
return writeResponse(writer, fail(resp, err.Error()))
|
||||||
|
}
|
||||||
|
if err := streamWriter.finish(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := writeResponse(writer, agentResponse{ID: req.ID, Success: true, ChunkType: agentChunkDone}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
maybeReleaseAgentMemory("stream-query-db", streamWriter.rowCount)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *agentRuntime) nextID() string {
|
func (r *agentRuntime) nextID() string {
|
||||||
r.ensureSessionMap()
|
r.ensureSessionMap()
|
||||||
r.nextSessionID++
|
r.nextSessionID++
|
||||||
@@ -459,6 +595,82 @@ func queryStatementWithOptionalTimeout(inst db.StatementExecer, query string, ti
|
|||||||
return queryWithOptionalTimeout(queryRunner, query, timeoutMs)
|
return queryWithOptionalTimeout(queryRunner, query, timeoutMs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func streamWithOptionalTimeout(inst db.StreamQueryExecer, query string, timeoutMs int64, consumer db.QueryStreamConsumer) error {
|
||||||
|
effectiveTimeoutMs := timeoutMs
|
||||||
|
if effectiveTimeoutMs <= 0 && strings.EqualFold(strings.TrimSpace(agentDriverType), "clickhouse") {
|
||||||
|
effectiveTimeoutMs = int64(legacyClickHouseDefaultTimeout / time.Millisecond)
|
||||||
|
}
|
||||||
|
if effectiveTimeoutMs <= 0 {
|
||||||
|
return inst.StreamQuery(query, consumer)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(effectiveTimeoutMs)*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
return inst.StreamQueryContext(ctx, query, consumer)
|
||||||
|
}
|
||||||
|
|
||||||
|
func streamBufferedQueryResult(fields []string, data []map[string]interface{}, consumer db.QueryStreamConsumer) error {
|
||||||
|
if err := consumer.SetColumns(fields); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if valueConsumer, ok := consumer.(db.QueryStreamValueConsumer); ok {
|
||||||
|
for _, row := range data {
|
||||||
|
values := make([]interface{}, len(fields))
|
||||||
|
for idx, field := range fields {
|
||||||
|
values[idx] = row[field]
|
||||||
|
}
|
||||||
|
if err := valueConsumer.ConsumeRowValues(values); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, row := range data {
|
||||||
|
if err := consumer.ConsumeRow(row); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func streamStatementWithOptionalTimeout(inst db.StatementExecer, query string, timeoutMs int64, consumer db.QueryStreamConsumer) error {
|
||||||
|
if streamer, ok := inst.(db.StreamQueryExecer); ok {
|
||||||
|
return streamWithOptionalTimeout(streamer, query, timeoutMs, consumer)
|
||||||
|
}
|
||||||
|
data, fields, err := queryStatementWithOptionalTimeout(inst, query, timeoutMs)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return streamBufferedQueryResult(fields, data, consumer)
|
||||||
|
}
|
||||||
|
|
||||||
|
func streamDatabaseWithOptionalTimeout(inst db.Database, query string, timeoutMs int64, consumer db.QueryStreamConsumer) error {
|
||||||
|
if streamer, ok := inst.(db.StreamQueryExecer); ok {
|
||||||
|
return streamWithOptionalTimeout(streamer, query, timeoutMs, consumer)
|
||||||
|
}
|
||||||
|
if provider, ok := inst.(db.SessionExecerProvider); ok {
|
||||||
|
openCtx := context.Background()
|
||||||
|
var cancel context.CancelFunc
|
||||||
|
effectiveTimeoutMs := timeoutMs
|
||||||
|
if effectiveTimeoutMs <= 0 && strings.EqualFold(strings.TrimSpace(agentDriverType), "clickhouse") {
|
||||||
|
effectiveTimeoutMs = int64(legacyClickHouseDefaultTimeout / time.Millisecond)
|
||||||
|
}
|
||||||
|
if effectiveTimeoutMs > 0 {
|
||||||
|
openCtx, cancel = context.WithTimeout(context.Background(), time.Duration(effectiveTimeoutMs)*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
}
|
||||||
|
session, err := provider.OpenSessionExecer(openCtx)
|
||||||
|
if err == nil {
|
||||||
|
defer session.Close()
|
||||||
|
return streamStatementWithOptionalTimeout(session, query, timeoutMs, consumer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
data, fields, err := queryWithOptionalTimeout(inst, query, timeoutMs)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return streamBufferedQueryResult(fields, data, consumer)
|
||||||
|
}
|
||||||
|
|
||||||
func execWithOptionalTimeout(inst agentExecRunner, query string, timeoutMs int64) (int64, error) {
|
func execWithOptionalTimeout(inst agentExecRunner, query string, timeoutMs int64) (int64, error) {
|
||||||
effectiveTimeoutMs := timeoutMs
|
effectiveTimeoutMs := timeoutMs
|
||||||
if effectiveTimeoutMs <= 0 && strings.EqualFold(strings.TrimSpace(agentDriverType), "clickhouse") {
|
if effectiveTimeoutMs <= 0 && strings.EqualFold(strings.TrimSpace(agentDriverType), "clickhouse") {
|
||||||
@@ -478,3 +690,41 @@ func execWithOptionalTimeout(inst agentExecRunner, query string, timeoutMs int64
|
|||||||
func execStatementWithOptionalTimeout(inst db.StatementExecer, query string, timeoutMs int64) (int64, error) {
|
func execStatementWithOptionalTimeout(inst db.StatementExecer, query string, timeoutMs int64) (int64, error) {
|
||||||
return execWithOptionalTimeout(inst, query, timeoutMs)
|
return execWithOptionalTimeout(inst, query, timeoutMs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func countAgentResponseRows(data interface{}) int64 {
|
||||||
|
rows, ok := data.([]map[string]interface{})
|
||||||
|
if !ok {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return int64(len(rows))
|
||||||
|
}
|
||||||
|
|
||||||
|
func maybeReleaseAgentMemory(reason string, rows int64) {
|
||||||
|
if rows < agentMemoryTrimRowsThreshold {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !agentMemoryTrimRunning.CompareAndSwap(false, true) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
runAgentMemoryTrimAsync(func() {
|
||||||
|
defer agentMemoryTrimRunning.Store(false)
|
||||||
|
if delay := nextAgentMemoryTrimDelay(); delay > 0 {
|
||||||
|
time.Sleep(delay)
|
||||||
|
}
|
||||||
|
agentMemoryTrimFn()
|
||||||
|
agentMemoryTrimLastAt.Store(time.Now().UnixNano())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func nextAgentMemoryTrimDelay() time.Duration {
|
||||||
|
lastUnixNano := agentMemoryTrimLastAt.Load()
|
||||||
|
if lastUnixNano <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
elapsed := time.Since(time.Unix(0, lastUnixNano))
|
||||||
|
if elapsed >= agentMemoryTrimMinInterval {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return agentMemoryTrimMinInterval - elapsed
|
||||||
|
}
|
||||||
|
|||||||
@@ -190,6 +190,64 @@ func (f *fakeAgentStatementSession) Close() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type fakeAgentStreamSession struct {
|
||||||
|
closed bool
|
||||||
|
streamCalls int
|
||||||
|
deadlineSet bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeAgentStreamSession) Exec(query string) (int64, error) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeAgentStreamSession) ExecContext(ctx context.Context, query string) (int64, error) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeAgentStreamSession) Close() error {
|
||||||
|
f.closed = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeAgentStreamSession) StreamQuery(query string, consumer db.QueryStreamConsumer) error {
|
||||||
|
return f.StreamQueryContext(context.Background(), query, consumer)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeAgentStreamSession) StreamQueryContext(ctx context.Context, query string, consumer db.QueryStreamConsumer) error {
|
||||||
|
f.streamCalls++
|
||||||
|
if _, ok := ctx.Deadline(); ok {
|
||||||
|
f.deadlineSet = true
|
||||||
|
}
|
||||||
|
if err := consumer.SetColumns([]string{"id", "name"}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if valueConsumer, ok := consumer.(db.QueryStreamValueConsumer); ok {
|
||||||
|
if err := valueConsumer.ConsumeRowValues([]interface{}{1, "alice"}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := valueConsumer.ConsumeRowValues([]interface{}{2, "bob"}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := consumer.ConsumeRow(map[string]interface{}{"id": 1, "name": "alice"}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return consumer.ConsumeRow(map[string]interface{}{"id": 2, "name": "bob"})
|
||||||
|
}
|
||||||
|
|
||||||
|
type fakeAgentSessionStreamDB struct {
|
||||||
|
fakeAgentTimeoutDB
|
||||||
|
session *fakeAgentStreamSession
|
||||||
|
openCalls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeAgentSessionStreamDB) OpenSessionExecer(ctx context.Context) (db.StatementExecer, error) {
|
||||||
|
f.openCalls++
|
||||||
|
f.session = &fakeAgentStreamSession{}
|
||||||
|
return f.session, nil
|
||||||
|
}
|
||||||
|
|
||||||
func TestQueryWithOptionalTimeout_UsesQueryContext(t *testing.T) {
|
func TestQueryWithOptionalTimeout_UsesQueryContext(t *testing.T) {
|
||||||
fake := &fakeAgentTimeoutDB{}
|
fake := &fakeAgentTimeoutDB{}
|
||||||
data, fields, err := queryWithOptionalTimeout(fake, "SELECT 1", int64((2 * time.Second).Milliseconds()))
|
data, fields, err := queryWithOptionalTimeout(fake, "SELECT 1", int64((2 * time.Second).Milliseconds()))
|
||||||
@@ -306,3 +364,135 @@ func TestHandleRequest_UsesPinnedSessionForSessionScopedQueryAndExec(t *testing.
|
|||||||
t.Fatal("expected pinned session to close")
|
t.Fatal("expected pinned session to close")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestHandleStreamRequest_UsesSessionStreamerAndWritesChunks(t *testing.T) {
|
||||||
|
old := agentDriverType
|
||||||
|
originalAsync := runAgentMemoryTrimAsync
|
||||||
|
originalTrim := agentMemoryTrimFn
|
||||||
|
originalLastAt := agentMemoryTrimLastAt.Load()
|
||||||
|
defer func() { agentDriverType = old }()
|
||||||
|
defer func() {
|
||||||
|
runAgentMemoryTrimAsync = originalAsync
|
||||||
|
agentMemoryTrimFn = originalTrim
|
||||||
|
agentMemoryTrimRunning.Store(false)
|
||||||
|
agentMemoryTrimLastAt.Store(originalLastAt)
|
||||||
|
}()
|
||||||
|
agentDriverType = "oceanbase"
|
||||||
|
agentMemoryTrimRunning.Store(false)
|
||||||
|
agentMemoryTrimLastAt.Store(0)
|
||||||
|
|
||||||
|
fake := &fakeAgentSessionStreamDB{}
|
||||||
|
runtimeState := &agentRuntime{
|
||||||
|
inst: fake,
|
||||||
|
sessions: make(map[string]db.StatementExecer),
|
||||||
|
}
|
||||||
|
|
||||||
|
trimmed := 0
|
||||||
|
runAgentMemoryTrimAsync = func(fn func()) {
|
||||||
|
fn()
|
||||||
|
}
|
||||||
|
agentMemoryTrimFn = func() {
|
||||||
|
trimmed++
|
||||||
|
}
|
||||||
|
|
||||||
|
var out bytes.Buffer
|
||||||
|
writer := bufio.NewWriter(&out)
|
||||||
|
if err := handleStreamRequest(runtimeState, agentRequest{
|
||||||
|
ID: 9,
|
||||||
|
Method: agentMethodStreamQuery,
|
||||||
|
Query: "SELECT * FROM person_info",
|
||||||
|
TimeoutMs: int64((2 * time.Second).Milliseconds()),
|
||||||
|
}, writer); err != nil {
|
||||||
|
t.Fatalf("handleStreamRequest 返回错误: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fake.openCalls != 1 {
|
||||||
|
t.Fatalf("expected OpenSessionExecer called once, got %d", fake.openCalls)
|
||||||
|
}
|
||||||
|
if fake.session == nil || fake.session.streamCalls != 1 {
|
||||||
|
t.Fatalf("expected session streamer used once, session=%#v", fake.session)
|
||||||
|
}
|
||||||
|
if !fake.session.deadlineSet {
|
||||||
|
t.Fatal("expected stream query context deadline to be set")
|
||||||
|
}
|
||||||
|
if !fake.session.closed {
|
||||||
|
t.Fatal("expected session to close after streaming")
|
||||||
|
}
|
||||||
|
if fake.queryCalled || fake.queryContextCalled {
|
||||||
|
t.Fatalf("unexpected fallback query path, Query=%v QueryContext=%v", fake.queryCalled, fake.queryContextCalled)
|
||||||
|
}
|
||||||
|
|
||||||
|
lines := strings.Split(strings.TrimSpace(out.String()), "\n")
|
||||||
|
if len(lines) != 3 {
|
||||||
|
t.Fatalf("expected 3 stream responses, got %d: %q", len(lines), out.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
var columnsResp struct {
|
||||||
|
Success bool `json:"success"`
|
||||||
|
ChunkType string `json:"chunkType"`
|
||||||
|
Fields []string `json:"fields"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(lines[0]), &columnsResp); err != nil {
|
||||||
|
t.Fatalf("decode columns response failed: %v", err)
|
||||||
|
}
|
||||||
|
if !columnsResp.Success || columnsResp.ChunkType != agentChunkColumns || len(columnsResp.Fields) != 2 {
|
||||||
|
t.Fatalf("unexpected columns response: %#v", columnsResp)
|
||||||
|
}
|
||||||
|
|
||||||
|
var rowsResp struct {
|
||||||
|
Success bool `json:"success"`
|
||||||
|
ChunkType string `json:"chunkType"`
|
||||||
|
Data [][]interface{} `json:"data"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(lines[1]), &rowsResp); err != nil {
|
||||||
|
t.Fatalf("decode rows response failed: %v", err)
|
||||||
|
}
|
||||||
|
if !rowsResp.Success || rowsResp.ChunkType != agentChunkRows || len(rowsResp.Data) != 2 {
|
||||||
|
t.Fatalf("unexpected rows response: %#v", rowsResp)
|
||||||
|
}
|
||||||
|
if got := rowsResp.Data[1][1]; got != "bob" {
|
||||||
|
t.Fatalf("unexpected streamed row payload: %v", rowsResp.Data)
|
||||||
|
}
|
||||||
|
|
||||||
|
var doneResp struct {
|
||||||
|
Success bool `json:"success"`
|
||||||
|
ChunkType string `json:"chunkType"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(lines[2]), &doneResp); err != nil {
|
||||||
|
t.Fatalf("decode done response failed: %v", err)
|
||||||
|
}
|
||||||
|
if !doneResp.Success || doneResp.ChunkType != agentChunkDone {
|
||||||
|
t.Fatalf("unexpected done response: %#v", doneResp)
|
||||||
|
}
|
||||||
|
if trimmed != 0 {
|
||||||
|
t.Fatalf("小流式任务不应触发内存回收,got=%d", trimmed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMaybeReleaseAgentMemory_TriggersTrimForLargeJobs(t *testing.T) {
|
||||||
|
originalAsync := runAgentMemoryTrimAsync
|
||||||
|
originalTrim := agentMemoryTrimFn
|
||||||
|
originalLastAt := agentMemoryTrimLastAt.Load()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
runAgentMemoryTrimAsync = originalAsync
|
||||||
|
agentMemoryTrimFn = originalTrim
|
||||||
|
agentMemoryTrimRunning.Store(false)
|
||||||
|
agentMemoryTrimLastAt.Store(originalLastAt)
|
||||||
|
})
|
||||||
|
|
||||||
|
agentMemoryTrimRunning.Store(false)
|
||||||
|
agentMemoryTrimLastAt.Store(0)
|
||||||
|
triggered := 0
|
||||||
|
runAgentMemoryTrimAsync = func(fn func()) {
|
||||||
|
fn()
|
||||||
|
}
|
||||||
|
agentMemoryTrimFn = func() {
|
||||||
|
triggered++
|
||||||
|
}
|
||||||
|
|
||||||
|
maybeReleaseAgentMemory("test-large-query", agentMemoryTrimRowsThreshold)
|
||||||
|
|
||||||
|
if triggered != 1 {
|
||||||
|
t.Fatalf("大查询完成后应触发一次内存回收,got=%d", triggered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -418,6 +418,7 @@ var (
|
|||||||
var optionalDriverSourceBuildTimeout = 8 * time.Minute
|
var optionalDriverSourceBuildTimeout = 8 * time.Minute
|
||||||
|
|
||||||
var validateOptionalDriverAgentExecutableFunc = db.ValidateOptionalDriverAgentExecutable
|
var validateOptionalDriverAgentExecutableFunc = db.ValidateOptionalDriverAgentExecutable
|
||||||
|
var resolveOptionalDriverAgentExecutablePathFunc = db.ResolveOptionalDriverAgentExecutablePath
|
||||||
|
|
||||||
type driverVersionWarmupState struct {
|
type driverVersionWarmupState struct {
|
||||||
Running bool
|
Running bool
|
||||||
|
|||||||
@@ -342,6 +342,23 @@ func tryResolveExportTableTotalRows(dbInst db.Database, config connection.Connec
|
|||||||
return resolveExportTotalRowsFromRows(rows)
|
return resolveExportTotalRowsFromRows(rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func verifyOptionalDriverAgentReadyForExport(config connection.ConnectionConfig) error {
|
||||||
|
driverType := normalizeDriverType(config.Type)
|
||||||
|
if !db.IsOptionalGoDriver(driverType) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
executablePath, err := resolveOptionalDriverAgentExecutablePathFunc("", driverType)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := verifyInstalledOptionalDriverAgentRevision(driverType, executablePath); err != nil {
|
||||||
|
displayName := resolveDriverDisplayName(driverDefinition{Type: driverType})
|
||||||
|
return fmt.Errorf("当前导出依赖最新的 %s driver-agent 流式协议;为避免大结果集回退到高内存缓冲模式,请在驱动管理中重装后重试:%w", displayName, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
var exportFileNameSanitizer = strings.NewReplacer(
|
var exportFileNameSanitizer = strings.NewReplacer(
|
||||||
"/", "_",
|
"/", "_",
|
||||||
"\\", "_",
|
"\\", "_",
|
||||||
@@ -2249,6 +2266,11 @@ func (a *App) ExportTable(config connection.ConnectionConfig, dbName string, tab
|
|||||||
func (a *App) ExportTableWithOptions(config connection.ConnectionConfig, dbName string, tableName string, options ExportFileOptions) connection.QueryResult {
|
func (a *App) ExportTableWithOptions(config connection.ConnectionConfig, dbName string, tableName string, options ExportFileOptions) connection.QueryResult {
|
||||||
options = normalizeExportFileOptions("", options)
|
options = normalizeExportFileOptions("", options)
|
||||||
format := options.Format
|
format := options.Format
|
||||||
|
if format != "sql" {
|
||||||
|
if err := verifyOptionalDriverAgentReadyForExport(config); err != nil {
|
||||||
|
return connection.QueryResult{Success: false, Message: err.Error()}
|
||||||
|
}
|
||||||
|
}
|
||||||
filename, err := runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{
|
filename, err := runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{
|
||||||
Title: fmt.Sprintf("Export %s", tableName),
|
Title: fmt.Sprintf("Export %s", tableName),
|
||||||
DefaultFilename: fmt.Sprintf("%s.%s", tableName, format),
|
DefaultFilename: fmt.Sprintf("%s.%s", tableName, format),
|
||||||
@@ -3656,6 +3678,11 @@ func (a *App) ExportQueryWithOptions(config connection.ConnectionConfig, dbName
|
|||||||
}
|
}
|
||||||
options = normalizeExportFileOptions("", options)
|
options = normalizeExportFileOptions("", options)
|
||||||
format := options.Format
|
format := options.Format
|
||||||
|
if format != "sql" {
|
||||||
|
if err := verifyOptionalDriverAgentReadyForExport(config); err != nil {
|
||||||
|
return connection.QueryResult{Success: false, Message: err.Error()}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
filename, err := runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{
|
filename, err := runtime.SaveFileDialog(a.ctx, runtime.SaveDialogOptions{
|
||||||
Title: "Export Query Result",
|
Title: "Export Query Result",
|
||||||
|
|||||||
@@ -475,6 +475,49 @@ func TestTryResolveExportTableTotalRows_UsesCountQuery(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestVerifyOptionalDriverAgentReadyForExport_RejectsStaleAgent(t *testing.T) {
|
||||||
|
originalProbe := optionalDriverAgentMetadataProbe
|
||||||
|
originalResolvePath := resolveOptionalDriverAgentExecutablePathFunc
|
||||||
|
t.Cleanup(func() {
|
||||||
|
optionalDriverAgentMetadataProbe = originalProbe
|
||||||
|
resolveOptionalDriverAgentExecutablePathFunc = originalResolvePath
|
||||||
|
})
|
||||||
|
|
||||||
|
resolveOptionalDriverAgentExecutablePathFunc = func(downloadDir string, driverType string) (string, error) {
|
||||||
|
return "/tmp/oceanbase-driver-agent", nil
|
||||||
|
}
|
||||||
|
optionalDriverAgentMetadataProbe = func(driverType string, executablePath string) (db.OptionalDriverAgentMetadata, error) {
|
||||||
|
return db.OptionalDriverAgentMetadata{
|
||||||
|
DriverType: driverType,
|
||||||
|
AgentRevision: "src-stale-agent",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
err := verifyOptionalDriverAgentReadyForExport(connection.ConnectionConfig{Type: "oceanbase"})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("预期旧版 OceanBase driver-agent 被导出前校验拦截")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "流式协议") {
|
||||||
|
t.Fatalf("错误信息应说明需要流式协议,got=%q", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestVerifyOptionalDriverAgentReadyForExport_SkipsBuiltInDriver(t *testing.T) {
|
||||||
|
originalResolvePath := resolveOptionalDriverAgentExecutablePathFunc
|
||||||
|
t.Cleanup(func() {
|
||||||
|
resolveOptionalDriverAgentExecutablePathFunc = originalResolvePath
|
||||||
|
})
|
||||||
|
|
||||||
|
resolveOptionalDriverAgentExecutablePathFunc = func(downloadDir string, driverType string) (string, error) {
|
||||||
|
t.Fatalf("内置驱动导出不应探测 optional driver-agent 路径")
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := verifyOptionalDriverAgentReadyForExport(connection.ConnectionConfig{Type: "mysql"}); err != nil {
|
||||||
|
t.Fatalf("内置驱动导出不应被 optional driver-agent 校验阻断: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExportQueryResultToFile_UsesStreamQueryPath(t *testing.T) {
|
func TestExportQueryResultToFile_UsesStreamQueryPath(t *testing.T) {
|
||||||
f, err := os.CreateTemp("", "gonavi-export-stream-*.csv")
|
f, err := os.CreateTemp("", "gonavi-export-stream-*.csv")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -4,25 +4,25 @@ package db
|
|||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
optionalDriverAgentRevisions = map[string]string{
|
optionalDriverAgentRevisions = map[string]string{
|
||||||
"mariadb": "src-0a4176f4b5743323",
|
"mariadb": "src-b23e2ce1581a5064",
|
||||||
"oceanbase": "src-7cb0f2c4dc0510a5",
|
"oceanbase": "src-5067dbdf0ca7b9c4",
|
||||||
"diros": "src-cc11b882e28fa5d4",
|
"diros": "src-db43faca6bf15d9b",
|
||||||
"starrocks": "src-83a6d81c91c7f5c8",
|
"starrocks": "src-01e9f06c0fab09d5",
|
||||||
"sphinx": "src-a70c2cd4d223dac2",
|
"sphinx": "src-38ee5cae952cc809",
|
||||||
"sqlserver": "src-6d5cf334034bce41",
|
"sqlserver": "src-7a87f6deb816f110",
|
||||||
"sqlite": "src-762863d48f653b89",
|
"sqlite": "src-d3d439cd788880e2",
|
||||||
"duckdb": "src-df5d60ebb175bbbc",
|
"duckdb": "src-b11506b8706bfb73",
|
||||||
"dameng": "src-596bebeaa016fc74",
|
"dameng": "src-1638124bfd7fce09",
|
||||||
"kingbase": "src-2e5a1337b0405c57",
|
"kingbase": "src-fb3a404cf4eb1bd9",
|
||||||
"highgo": "src-5a29a1d3685eb6b4",
|
"highgo": "src-72fe51afa884f6bc",
|
||||||
"vastbase": "src-e3cfef65512feb23",
|
"vastbase": "src-3d48607603bfd8b7",
|
||||||
"opengauss": "src-58227ba3bc1ec894",
|
"opengauss": "src-709acf442f016e30",
|
||||||
"gaussdb": "src-1458564993a9d455",
|
"gaussdb": "src-f6beccc924d71031",
|
||||||
"iris": "src-1b072c57af08bec4",
|
"iris": "src-9ebf5b970a73b341",
|
||||||
"mongodb": "src-57fdd8bfebdcd46e",
|
"mongodb": "src-367d11cd04e982c1",
|
||||||
"tdengine": "src-939715f94df1ec9c",
|
"tdengine": "src-3c13c42f18ba01e1",
|
||||||
"iotdb": "src-473c39891f926db2",
|
"iotdb": "src-5ba9da13c6a272f9",
|
||||||
"clickhouse": "src-482d62ed565b3e69",
|
"clickhouse": "src-99c8babfefdf142c",
|
||||||
"elasticsearch": "src-2fb00b94d7067c56",
|
"elasticsearch": "src-36b2e2b5f49db9d1",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package db
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
@@ -28,6 +29,7 @@ const (
|
|||||||
optionalAgentMethodOpenSession = "openSession"
|
optionalAgentMethodOpenSession = "openSession"
|
||||||
optionalAgentMethodCloseSession = "closeSession"
|
optionalAgentMethodCloseSession = "closeSession"
|
||||||
optionalAgentMethodQuery = "query"
|
optionalAgentMethodQuery = "query"
|
||||||
|
optionalAgentMethodStreamQuery = "streamQuery"
|
||||||
optionalAgentMethodExec = "exec"
|
optionalAgentMethodExec = "exec"
|
||||||
optionalAgentMethodGetDatabases = "getDatabases"
|
optionalAgentMethodGetDatabases = "getDatabases"
|
||||||
optionalAgentMethodGetTables = "getTables"
|
optionalAgentMethodGetTables = "getTables"
|
||||||
@@ -42,6 +44,12 @@ const (
|
|||||||
optionalAgentMetadataProbeTimeout = 5 * time.Second
|
optionalAgentMetadataProbeTimeout = 5 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
optionalAgentChunkColumns = "columns"
|
||||||
|
optionalAgentChunkRows = "rows"
|
||||||
|
optionalAgentChunkDone = "done"
|
||||||
|
)
|
||||||
|
|
||||||
type optionalAgentRequest struct {
|
type optionalAgentRequest struct {
|
||||||
ID int64 `json:"id"`
|
ID int64 `json:"id"`
|
||||||
Method string `json:"method"`
|
Method string `json:"method"`
|
||||||
@@ -60,6 +68,7 @@ type optionalAgentResponse struct {
|
|||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
Data json.RawMessage `json:"data,omitempty"`
|
Data json.RawMessage `json:"data,omitempty"`
|
||||||
Fields []string `json:"fields,omitempty"`
|
Fields []string `json:"fields,omitempty"`
|
||||||
|
ChunkType string `json:"chunkType,omitempty"`
|
||||||
RowsAffected int64 `json:"rowsAffected,omitempty"`
|
RowsAffected int64 `json:"rowsAffected,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -269,6 +278,114 @@ func (c *optionalDriverAgentClient) callWithTimeout(req optionalAgentRequest, ou
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *optionalDriverAgentClient) callStreamQuery(req optionalAgentRequest, consumer QueryStreamConsumer) error {
|
||||||
|
if consumer == nil {
|
||||||
|
return fmt.Errorf("query stream consumer required")
|
||||||
|
}
|
||||||
|
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
c.nextID++
|
||||||
|
req.ID = c.nextID
|
||||||
|
|
||||||
|
payload, err := json.Marshal(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
payload = append(payload, '\n')
|
||||||
|
if _, err := c.stdin.Write(payload); err != nil {
|
||||||
|
stderrText := c.stderrText()
|
||||||
|
if stderrText == "" {
|
||||||
|
return fmt.Errorf("调用 %s 驱动代理失败:%w", driverDisplayName(c.driver), err)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("调用 %s 驱动代理失败:%w(stderr: %s)", driverDisplayName(c.driver), err, stderrText)
|
||||||
|
}
|
||||||
|
|
||||||
|
var columns []string
|
||||||
|
valueConsumer, useValueConsumer := consumer.(QueryStreamValueConsumer)
|
||||||
|
|
||||||
|
for {
|
||||||
|
line, err := c.reader.ReadBytes('\n')
|
||||||
|
if err != nil {
|
||||||
|
stderrText := c.stderrText()
|
||||||
|
if stderrText == "" {
|
||||||
|
return fmt.Errorf("读取 %s 驱动代理响应失败:%w", driverDisplayName(c.driver), err)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("读取 %s 驱动代理响应失败:%w(stderr: %s)", driverDisplayName(c.driver), err, stderrText)
|
||||||
|
}
|
||||||
|
|
||||||
|
var resp optionalAgentResponse
|
||||||
|
if err := json.Unmarshal(line, &resp); err != nil {
|
||||||
|
return fmt.Errorf("解析 %s 驱动代理响应失败:%w", driverDisplayName(c.driver), err)
|
||||||
|
}
|
||||||
|
if !resp.Success {
|
||||||
|
errText := strings.TrimSpace(resp.Error)
|
||||||
|
if errText == "" {
|
||||||
|
errText = fmt.Sprintf("%s 驱动代理返回失败", driverDisplayName(c.driver))
|
||||||
|
}
|
||||||
|
return errors.New(errText)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch resp.ChunkType {
|
||||||
|
case optionalAgentChunkColumns:
|
||||||
|
columns = append(columns[:0], resp.Fields...)
|
||||||
|
if err := consumer.SetColumns(columns); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
case optionalAgentChunkRows:
|
||||||
|
if len(columns) == 0 {
|
||||||
|
return fmt.Errorf("%s 驱动代理流式响应缺少列信息", driverDisplayName(c.driver))
|
||||||
|
}
|
||||||
|
rows, err := decodeOptionalAgentRowValueBatch(resp.Data)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("解析 %s 驱动代理流式数据失败:%w", driverDisplayName(c.driver), err)
|
||||||
|
}
|
||||||
|
for _, row := range rows {
|
||||||
|
if useValueConsumer {
|
||||||
|
if err := valueConsumer.ConsumeRowValues(row); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
entry := make(map[string]interface{}, len(columns))
|
||||||
|
for i, column := range columns {
|
||||||
|
if i < len(row) {
|
||||||
|
entry[column] = row[i]
|
||||||
|
} else {
|
||||||
|
entry[column] = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := consumer.ConsumeRow(entry); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case optionalAgentChunkDone:
|
||||||
|
return nil
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%s 驱动代理返回未知流式分片类型:%s", driverDisplayName(c.driver), strings.TrimSpace(resp.ChunkType))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeOptionalAgentRowValueBatch(data []byte) ([][]interface{}, error) {
|
||||||
|
if len(data) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||||
|
decoder.UseNumber()
|
||||||
|
var rows [][]interface{}
|
||||||
|
if err := decoder.Decode(&rows); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for rowIdx := range rows {
|
||||||
|
for colIdx := range rows[rowIdx] {
|
||||||
|
rows[rowIdx][colIdx] = normalizeQueryValue(rows[rowIdx][colIdx])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rows, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (c *optionalDriverAgentClient) forceTerminate() {
|
func (c *optionalDriverAgentClient) forceTerminate() {
|
||||||
if c.stdin != nil {
|
if c.stdin != nil {
|
||||||
_ = c.stdin.Close()
|
_ = c.stdin.Close()
|
||||||
@@ -397,6 +514,42 @@ func (d *OptionalDriverAgentDB) Query(query string) ([]map[string]interface{}, [
|
|||||||
return data, fields, nil
|
return data, fields, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (d *OptionalDriverAgentDB) StreamQuery(query string, consumer QueryStreamConsumer) error {
|
||||||
|
return d.StreamQueryContext(context.Background(), query, consumer)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *OptionalDriverAgentDB) StreamQueryContext(ctx context.Context, query string, consumer QueryStreamConsumer) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
client, err := d.requireClient()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err = client.callStreamQuery(optionalAgentRequest{
|
||||||
|
Method: optionalAgentMethodStreamQuery,
|
||||||
|
Query: query,
|
||||||
|
TimeoutMs: timeoutMsFromContext(ctx),
|
||||||
|
}, consumer)
|
||||||
|
if isOptionalAgentStreamUnsupportedError(err) {
|
||||||
|
logger.Warnf("%s 驱动代理暂不支持流式查询,回退到缓冲模式:err=%v", driverDisplayName(d.driverType), err)
|
||||||
|
data, columns, queryErr := d.QueryContext(ctx, query)
|
||||||
|
if queryErr != nil {
|
||||||
|
return queryErr
|
||||||
|
}
|
||||||
|
if err := consumer.SetColumns(columns); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, row := range data {
|
||||||
|
if err := consumer.ConsumeRow(row); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
func (d *OptionalDriverAgentDB) ExecContext(ctx context.Context, query string) (int64, error) {
|
func (d *OptionalDriverAgentDB) ExecContext(ctx context.Context, query string) (int64, error) {
|
||||||
if err := ctx.Err(); err != nil {
|
if err := ctx.Err(); err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
@@ -458,6 +611,39 @@ func (s *optionalDriverAgentSession) Query(query string) ([]map[string]interface
|
|||||||
return s.QueryContext(context.Background(), query)
|
return s.QueryContext(context.Background(), query)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *optionalDriverAgentSession) StreamQuery(query string, consumer QueryStreamConsumer) error {
|
||||||
|
return s.StreamQueryContext(context.Background(), query, consumer)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *optionalDriverAgentSession) StreamQueryContext(ctx context.Context, query string, consumer QueryStreamConsumer) error {
|
||||||
|
if err := s.ensureOpen(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err := s.client.callStreamQuery(optionalAgentRequest{
|
||||||
|
Method: optionalAgentMethodStreamQuery,
|
||||||
|
SessionID: s.sessionID,
|
||||||
|
Query: query,
|
||||||
|
TimeoutMs: timeoutMsFromContext(ctx),
|
||||||
|
}, consumer)
|
||||||
|
if isOptionalAgentStreamUnsupportedError(err) {
|
||||||
|
logger.Warnf("%s 驱动代理事务会话暂不支持流式查询,回退到缓冲模式:err=%v", driverDisplayName(s.driver), err)
|
||||||
|
data, columns, queryErr := s.QueryContext(ctx, query)
|
||||||
|
if queryErr != nil {
|
||||||
|
return queryErr
|
||||||
|
}
|
||||||
|
if err := consumer.SetColumns(columns); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, row := range data {
|
||||||
|
if err := consumer.ConsumeRow(row); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
func (s *optionalDriverAgentSession) QueryContext(ctx context.Context, query string) ([]map[string]interface{}, []string, error) {
|
func (s *optionalDriverAgentSession) QueryContext(ctx context.Context, query string) ([]map[string]interface{}, []string, error) {
|
||||||
if err := s.ensureOpen(); err != nil {
|
if err := s.ensureOpen(); err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
@@ -525,6 +711,17 @@ func (s *optionalDriverAgentSession) ensureOpen() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isOptionalAgentStreamUnsupportedError(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
text := strings.TrimSpace(err.Error())
|
||||||
|
if text == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return strings.Contains(text, "不支持的方法") || strings.Contains(text, "不支持流式查询")
|
||||||
|
}
|
||||||
|
|
||||||
func (d *OptionalDriverAgentDB) GetDatabases() ([]string, error) {
|
func (d *OptionalDriverAgentDB) GetDatabases() ([]string, error) {
|
||||||
client, err := d.requireClient()
|
client, err := d.requireClient()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
package db
|
package db
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
|
"bytes"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"GoNavi-Wails/internal/connection"
|
"GoNavi-Wails/internal/connection"
|
||||||
@@ -65,3 +68,71 @@ func TestNormalizeKingbaseAgentChangeSetByColumns(t *testing.T) {
|
|||||||
t.Fatalf("unexpected update value key \"event name\" after normalization")
|
t.Fatalf("unexpected update value key \"event name\" after normalization")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type optionalAgentTestWriteCloser struct {
|
||||||
|
bytes.Buffer
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *optionalAgentTestWriteCloser) Close() error { return nil }
|
||||||
|
|
||||||
|
type optionalAgentTestStreamConsumer struct {
|
||||||
|
columns []string
|
||||||
|
rows [][]interface{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *optionalAgentTestStreamConsumer) SetColumns(columns []string) error {
|
||||||
|
c.columns = append([]string(nil), columns...)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *optionalAgentTestStreamConsumer) ConsumeRow(row map[string]interface{}) error {
|
||||||
|
values := make([]interface{}, len(c.columns))
|
||||||
|
for idx, column := range c.columns {
|
||||||
|
values[idx] = row[column]
|
||||||
|
}
|
||||||
|
c.rows = append(c.rows, values)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *optionalAgentTestStreamConsumer) ConsumeRowValues(values []interface{}) error {
|
||||||
|
c.rows = append(c.rows, append([]interface{}(nil), values...))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOptionalDriverAgentClientCallStreamQueryConsumesChunks(t *testing.T) {
|
||||||
|
var stdin optionalAgentTestWriteCloser
|
||||||
|
stdout := strings.Join([]string{
|
||||||
|
`{"id":1,"success":true,"chunkType":"columns","fields":["id","name"]}`,
|
||||||
|
`{"id":1,"success":true,"chunkType":"rows","data":[[1,"alice"],[2,"bob"]]}`,
|
||||||
|
`{"id":1,"success":true,"chunkType":"done"}`,
|
||||||
|
}, "\n") + "\n"
|
||||||
|
|
||||||
|
client := &optionalDriverAgentClient{
|
||||||
|
stdin: &stdin,
|
||||||
|
reader: bufio.NewReader(strings.NewReader(stdout)),
|
||||||
|
driver: "oceanbase",
|
||||||
|
}
|
||||||
|
consumer := &optionalAgentTestStreamConsumer{}
|
||||||
|
if err := client.callStreamQuery(optionalAgentRequest{
|
||||||
|
Method: optionalAgentMethodStreamQuery,
|
||||||
|
Query: "SELECT 1",
|
||||||
|
}, consumer); err != nil {
|
||||||
|
t.Fatalf("callStreamQuery 返回错误: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(consumer.columns) != 2 || consumer.columns[0] != "id" || consumer.columns[1] != "name" {
|
||||||
|
t.Fatalf("流式列定义异常: %#v", consumer.columns)
|
||||||
|
}
|
||||||
|
if len(consumer.rows) != 2 {
|
||||||
|
t.Fatalf("流式行数异常: %#v", consumer.rows)
|
||||||
|
}
|
||||||
|
if got := consumer.rows[0][1]; got != "alice" {
|
||||||
|
t.Fatalf("第 1 行数据异常,want=%q got=%v", "alice", got)
|
||||||
|
}
|
||||||
|
if got := consumer.rows[1][0]; got != int64(2) {
|
||||||
|
t.Fatalf("第 2 行 ID 异常,want=%d got=%v (%T)", 2, got, got)
|
||||||
|
}
|
||||||
|
if !strings.Contains(stdin.String(), `"method":"streamQuery"`) {
|
||||||
|
t.Fatalf("请求未使用 streamQuery 方法: %s", stdin.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user