🐛 fix(oracle): 修复大对象查询导致客户端崩溃

- 限制交互查询中 BLOB、CLOB 和 LONG 字段的预览大小
- 将默认预取行数降至 25,控制大对象批量加载内存
- 保持流式导出完整内容并补充回归测试
This commit is contained in:
Syngnat
2026-07-28 15:28:23 +08:00
parent b806f17cbb
commit ce9f5cffb9
4 changed files with 297 additions and 6 deletions

View File

@@ -25,8 +25,8 @@ func TestOracleGetDSNIncludesQueryPerformanceOptions(t *testing.T) {
t.Fatalf("解析 Oracle DSN 失败: %v", err)
}
query := parsed.Query()
if got := query.Get("PREFETCH_ROWS"); got != "10000" {
t.Fatalf("PREFETCH_ROWS = %q, want 10000", got)
if got := query.Get("PREFETCH_ROWS"); got != "25" {
t.Fatalf("PREFETCH_ROWS = %q, want 25", got)
}
if got := query.Get("LOB FETCH"); got != "POST" {
t.Fatalf("LOB FETCH = %q, want POST", got)

View File

@@ -29,6 +29,8 @@ type OracleDB struct {
var _ SessionExecerProvider = (*OracleDB)(nil)
var _ TransactionExecerProvider = (*OracleDB)(nil)
const oracleDefaultPrefetchRows = 25
var (
oracleTriggerCreatePattern = regexp.MustCompile(`(?is)^\s*CREATE\s+(?:OR\s+REPLACE\s+)?TRIGGER\b`)
oracleTriggerTimingPattern = regexp.MustCompile(`(?is)^\s*(?:BEFORE|AFTER|INSTEAD\s+OF)\b`)
@@ -58,8 +60,9 @@ func (o *OracleDB) getDSN(config connection.ConnectionConfig) string {
q.Set("SSL", "TRUE")
q.Set("SSL VERIFY", "FALSE")
}
// 提高 prefetch 行数,减少大结果集的网络往返次数(默认仅 25 行/次)
q.Set("PREFETCH_ROWS", "10000")
// Keep fetch batches bounded. go-ora materializes every LOB in a fetched batch,
// so a large prefetch value can retain many BLOBs at once before rows are scanned.
q.Set("PREFETCH_ROWS", strconv.Itoa(oracleDefaultPrefetchRows))
// LOB 数据延迟加载,避免大 LOB 列影响普通查询性能
q.Set("LOB FETCH", "POST")
timeoutSeconds := strconv.Itoa(getConnectTimeoutSeconds(config))

View File

@@ -4,6 +4,8 @@ import (
"database/sql"
"fmt"
"runtime"
"strings"
"unicode/utf8"
"GoNavi-Wails/internal/connection"
)
@@ -18,6 +20,10 @@ import (
// 同时保证单次 GC 之间累积的临时对象不超过几百 MB避免 GC 间隙堆膨胀。
const streamRowsPeriodicGCInterval = 50000
// interactiveOracleLargeObjectPreviewBytes bounds Oracle large objects before
// they cross the Wails bridge. The streaming export path stays unbounded.
const interactiveOracleLargeObjectPreviewBytes = 4 * 1024
func scanRows(rows *sql.Rows) ([]map[string]interface{}, []string, error) {
return scanRowsForDialect(rows, "")
}
@@ -51,7 +57,7 @@ func scanRowsForDialect(rows *sql.Rows, dialect string) ([]map[string]interface{
resultData := make([]map[string]interface{}, 0)
for rows.Next() {
entry, err := scanner.scanCurrentRow(rows)
entry, err := scanner.scanCurrentPreviewRow(rows)
if err != nil {
continue
}
@@ -145,15 +151,105 @@ func newQueryRowScanner(columns []string, colTypes []*sql.ColumnType, dialect st
}
func (s *queryRowScanner) scanCurrentRowValues(rows *sql.Rows) ([]interface{}, error) {
return s.scanCurrentRowValuesWithPreview(rows, false)
}
func (s *queryRowScanner) scanCurrentRowValuesWithPreview(rows *sql.Rows, boundOracleLargeObjects bool) ([]interface{}, error) {
if err := rows.Scan(s.valuePtrs...); err != nil {
return nil, err
}
for i := range s.columns {
s.normalized[i] = normalizeQueryValueWithDBTypeAndDialect(s.values[i], s.dbTypeNames[i], s.dialect)
if boundOracleLargeObjects {
s.normalized[i] = normalizeInteractiveQueryValue(s.values[i], s.dbTypeNames[i], s.dialect)
} else {
s.normalized[i] = normalizeQueryValueWithDBTypeAndDialect(s.values[i], s.dbTypeNames[i], s.dialect)
}
}
return s.normalized, nil
}
func normalizeInteractiveQueryValue(value interface{}, databaseTypeName, dialect string) interface{} {
switch typedValue := value.(type) {
case []byte:
if len(typedValue) > interactiveOracleLargeObjectPreviewBytes && isOracleBinaryLargeObjectType(databaseTypeName) {
preview := normalizeQueryValueWithDBTypeAndDialect(
typedValue[:interactiveOracleLargeObjectPreviewBytes],
databaseTypeName,
dialect,
)
previewText, ok := preview.(string)
if !ok {
previewText = fmt.Sprint(preview)
}
return fmt.Sprintf(
"[BLOB preview: %d/%d bytes] %s",
interactiveOracleLargeObjectPreviewBytes,
len(typedValue),
previewText,
)
}
case string:
if len(typedValue) > interactiveOracleLargeObjectPreviewBytes && isOracleTextLargeObjectType(databaseTypeName) {
preview := truncateUTF8Prefix(typedValue, interactiveOracleLargeObjectPreviewBytes)
return fmt.Sprintf(
"[CLOB preview: %d/%d bytes] %s",
len(preview),
len(typedValue),
preview,
)
}
}
return normalizeQueryValueWithDBTypeAndDialect(value, databaseTypeName, dialect)
}
func isOracleBinaryLargeObjectType(databaseTypeName string) bool {
typeName := strings.ToUpper(strings.ReplaceAll(strings.TrimSpace(databaseTypeName), " ", ""))
switch typeName {
case "OCIBLOBLOCATOR", "LONGRAW", "LONGVARRAW":
return true
default:
return false
}
}
func isOracleTextLargeObjectType(databaseTypeName string) bool {
typeName := strings.ToUpper(strings.ReplaceAll(strings.TrimSpace(databaseTypeName), " ", ""))
switch typeName {
case "OCICLOBLOCATOR", "LONG", "LONGVARCHAR":
return true
default:
return false
}
}
func truncateUTF8Prefix(value string, maxBytes int) string {
if maxBytes <= 0 {
return ""
}
if len(value) <= maxBytes {
return value
}
end := maxBytes
for end > 0 && !utf8.RuneStart(value[end]) {
end--
}
return value[:end]
}
func (s *queryRowScanner) scanCurrentPreviewRow(rows *sql.Rows) (map[string]interface{}, error) {
normalized, err := s.scanCurrentRowValuesWithPreview(rows, true)
if err != nil {
return nil, err
}
entry := make(map[string]interface{}, len(s.columns))
for i, col := range s.columns {
entry[col] = normalized[i]
}
return entry, nil
}
func (s *queryRowScanner) scanCurrentRow(rows *sql.Rows) (map[string]interface{}, error) {
normalized, err := s.scanCurrentRowValues(rows)
if err != nil {

View File

@@ -1,17 +1,22 @@
package db
import (
"bytes"
"context"
"database/sql"
"database/sql/driver"
"fmt"
"io"
"reflect"
"strings"
"sync"
"testing"
"time"
"unicode/utf8"
)
const scanRowsDuplicateDriverName = "gonavi-scan-rows-duplicate"
const scanRowsOracleBlobTestBytes = 16*1024 + 17
var registerScanRowsDuplicateDriverOnce sync.Once
@@ -28,6 +33,24 @@ func (scanRowsDuplicateConn) Close() error { return
func (scanRowsDuplicateConn) Begin() (driver.Tx, error) { return nil, driver.ErrSkip }
func (scanRowsDuplicateConn) QueryContext(_ context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
if query == "SELECT blob_columns" {
return &scanRowsDuplicateRows{
columns: []string{"payload"},
columnTypes: []string{"OCIBlobLocator"},
rows: [][]driver.Value{
{bytes.Repeat([]byte{0xff}, scanRowsOracleBlobTestBytes)},
},
}, nil
}
if query == "SELECT clob_columns" {
return &scanRowsDuplicateRows{
columns: []string{"content"},
columnTypes: []string{"OCIClobLocator"},
rows: [][]driver.Value{
{strings.Repeat("数", 6*1024)},
},
}, nil
}
if query == "SELECT date_columns" {
return &scanRowsDuplicateRows{
columns: []string{"ship_date", "created_at"},
@@ -120,6 +143,175 @@ func (scanRowsDuplicateConn) QueryContext(_ context.Context, query string, args
}, nil
}
type scanRowsValueConsumer struct {
columns []string
rows [][]interface{}
}
func (c *scanRowsValueConsumer) SetColumns(columns []string) error {
c.columns = append([]string(nil), columns...)
return nil
}
func (c *scanRowsValueConsumer) ConsumeRow(row map[string]interface{}) error {
values := make([]interface{}, len(c.columns))
for index, column := range c.columns {
values[index] = row[column]
}
c.rows = append(c.rows, values)
return nil
}
func (c *scanRowsValueConsumer) ConsumeRowValues(values []interface{}) error {
c.rows = append(c.rows, append([]interface{}(nil), values...))
return nil
}
func TestScanRowsBoundsOracleBlobPreview(t *testing.T) {
t.Parallel()
registerScanRowsDuplicateDriverOnce.Do(func() {
sql.Register(scanRowsDuplicateDriverName, scanRowsDuplicateDriver{})
})
dbConn, err := sql.Open(scanRowsDuplicateDriverName, "")
if err != nil {
t.Fatalf("open blob scan rows db failed: %v", err)
}
defer dbConn.Close()
rows, err := dbConn.QueryContext(context.Background(), "SELECT blob_columns")
if err != nil {
t.Fatalf("query blob scan rows db failed: %v", err)
}
defer rows.Close()
// Production OracleDB leaves scanDialect empty; go-ora's column type is the
// reliable signal for applying the interactive BLOB guard.
data, columns, err := scanRowsForDialect(rows, "")
if err != nil {
t.Fatalf("scanRowsForDialect returned error: %v", err)
}
if !reflect.DeepEqual(columns, []string{"payload"}) || len(data) != 1 {
t.Fatalf("unexpected blob result: columns=%v rows=%d", columns, len(data))
}
preview, ok := data[0]["payload"].(string)
if !ok {
t.Fatalf("Oracle BLOB preview type = %T, want string", data[0]["payload"])
}
wantPrefix := fmt.Sprintf("[BLOB preview: 4096/%d bytes] 0x", scanRowsOracleBlobTestBytes)
if !strings.HasPrefix(preview, wantPrefix+strings.Repeat("ff", 4*1024)) {
t.Fatalf("Oracle BLOB preview is missing bounded data or visible metadata: length=%d", len(preview))
}
fullHexLength := len("0x") + scanRowsOracleBlobTestBytes*2
if len(preview) >= fullHexLength {
t.Fatalf("Oracle BLOB preview length = %d, want less than full hex length %d", len(preview), fullHexLength)
}
}
func TestStreamRowsKeepsCompleteOracleBlobValue(t *testing.T) {
t.Parallel()
registerScanRowsDuplicateDriverOnce.Do(func() {
sql.Register(scanRowsDuplicateDriverName, scanRowsDuplicateDriver{})
})
dbConn, err := sql.Open(scanRowsDuplicateDriverName, "")
if err != nil {
t.Fatalf("open streaming blob rows db failed: %v", err)
}
defer dbConn.Close()
rows, err := dbConn.QueryContext(context.Background(), "SELECT blob_columns")
if err != nil {
t.Fatalf("query streaming blob rows db failed: %v", err)
}
defer rows.Close()
consumer := &scanRowsValueConsumer{}
if err := streamRowsForDialect(rows, "", consumer); err != nil {
t.Fatalf("streamRowsForDialect returned error: %v", err)
}
if len(consumer.rows) != 1 || len(consumer.rows[0]) != 1 {
t.Fatalf("unexpected streamed blob rows: %#v", consumer.rows)
}
want := "0x" + strings.Repeat("ff", scanRowsOracleBlobTestBytes)
if got := consumer.rows[0][0]; got != want {
t.Fatalf("streamed Oracle BLOB was truncated: got length=%d want length=%d", len(got.(string)), len(want))
}
}
func TestScanRowsBoundsOracleClobPreviewAtUTF8Boundary(t *testing.T) {
t.Parallel()
registerScanRowsDuplicateDriverOnce.Do(func() {
sql.Register(scanRowsDuplicateDriverName, scanRowsDuplicateDriver{})
})
dbConn, err := sql.Open(scanRowsDuplicateDriverName, "")
if err != nil {
t.Fatalf("open clob scan rows db failed: %v", err)
}
defer dbConn.Close()
rows, err := dbConn.QueryContext(context.Background(), "SELECT clob_columns")
if err != nil {
t.Fatalf("query clob scan rows db failed: %v", err)
}
defer rows.Close()
data, _, err := scanRowsForDialect(rows, "")
if err != nil {
t.Fatalf("scanRowsForDialect returned error: %v", err)
}
preview, ok := data[0]["content"].(string)
if !ok {
t.Fatalf("Oracle CLOB preview type = %T, want string", data[0]["content"])
}
wantContentPrefix := strings.Repeat("数", (4*1024)/len("数"))
wantPrefix := fmt.Sprintf(
"[CLOB preview: %d/%d bytes] ",
len(wantContentPrefix),
len(strings.Repeat("数", 6*1024)),
)
if !strings.HasPrefix(preview, wantPrefix+wantContentPrefix) {
t.Fatalf("Oracle CLOB preview does not preserve the UTF-8 prefix: length=%d", len(preview))
}
if !utf8.ValidString(preview) {
t.Fatalf("Oracle CLOB preview split a UTF-8 character: %q", preview[:min(len(preview), 32)])
}
}
func TestStreamRowsKeepsCompleteOracleClobValue(t *testing.T) {
t.Parallel()
registerScanRowsDuplicateDriverOnce.Do(func() {
sql.Register(scanRowsDuplicateDriverName, scanRowsDuplicateDriver{})
})
dbConn, err := sql.Open(scanRowsDuplicateDriverName, "")
if err != nil {
t.Fatalf("open streaming clob rows db failed: %v", err)
}
defer dbConn.Close()
rows, err := dbConn.QueryContext(context.Background(), "SELECT clob_columns")
if err != nil {
t.Fatalf("query streaming clob rows db failed: %v", err)
}
defer rows.Close()
consumer := &scanRowsValueConsumer{}
if err := streamRowsForDialect(rows, "", consumer); err != nil {
t.Fatalf("streamRowsForDialect returned error: %v", err)
}
want := strings.Repeat("数", 6*1024)
if len(consumer.rows) != 1 || len(consumer.rows[0]) != 1 || consumer.rows[0][0] != want {
t.Fatalf("streamed Oracle CLOB was truncated: rows=%d", len(consumer.rows))
}
}
var _ driver.QueryerContext = (*scanRowsDuplicateConn)(nil)
type scanRowsDuplicateRows struct {