mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-08-22 08:53:46 +08:00
🐛 fix(oracle): 修复大对象查询导致客户端崩溃
- 限制交互查询中 BLOB、CLOB 和 LONG 字段的预览大小 - 将默认预取行数降至 25,控制大对象批量加载内存 - 保持流式导出完整内容并补充回归测试
This commit is contained in:
@@ -25,8 +25,8 @@ func TestOracleGetDSNIncludesQueryPerformanceOptions(t *testing.T) {
|
||||
t.Fatalf("解析 Oracle DSN 失败: %v", err)
|
||||
}
|
||||
query := parsed.Query()
|
||||
if got := query.Get("PREFETCH_ROWS"); got != "10000" {
|
||||
t.Fatalf("PREFETCH_ROWS = %q, want 10000", got)
|
||||
if got := query.Get("PREFETCH_ROWS"); got != "25" {
|
||||
t.Fatalf("PREFETCH_ROWS = %q, want 25", got)
|
||||
}
|
||||
if got := query.Get("LOB FETCH"); got != "POST" {
|
||||
t.Fatalf("LOB FETCH = %q, want POST", got)
|
||||
|
||||
@@ -29,6 +29,8 @@ type OracleDB struct {
|
||||
var _ SessionExecerProvider = (*OracleDB)(nil)
|
||||
var _ TransactionExecerProvider = (*OracleDB)(nil)
|
||||
|
||||
const oracleDefaultPrefetchRows = 25
|
||||
|
||||
var (
|
||||
oracleTriggerCreatePattern = regexp.MustCompile(`(?is)^\s*CREATE\s+(?:OR\s+REPLACE\s+)?TRIGGER\b`)
|
||||
oracleTriggerTimingPattern = regexp.MustCompile(`(?is)^\s*(?:BEFORE|AFTER|INSTEAD\s+OF)\b`)
|
||||
@@ -58,8 +60,9 @@ func (o *OracleDB) getDSN(config connection.ConnectionConfig) string {
|
||||
q.Set("SSL", "TRUE")
|
||||
q.Set("SSL VERIFY", "FALSE")
|
||||
}
|
||||
// 提高 prefetch 行数,减少大结果集的网络往返次数(默认仅 25 行/次)
|
||||
q.Set("PREFETCH_ROWS", "10000")
|
||||
// Keep fetch batches bounded. go-ora materializes every LOB in a fetched batch,
|
||||
// so a large prefetch value can retain many BLOBs at once before rows are scanned.
|
||||
q.Set("PREFETCH_ROWS", strconv.Itoa(oracleDefaultPrefetchRows))
|
||||
// LOB 数据延迟加载,避免大 LOB 列影响普通查询性能
|
||||
q.Set("LOB FETCH", "POST")
|
||||
timeoutSeconds := strconv.Itoa(getConnectTimeoutSeconds(config))
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"GoNavi-Wails/internal/connection"
|
||||
)
|
||||
@@ -18,6 +20,10 @@ import (
|
||||
// 同时保证单次 GC 之间累积的临时对象不超过几百 MB,避免 GC 间隙堆膨胀。
|
||||
const streamRowsPeriodicGCInterval = 50000
|
||||
|
||||
// interactiveOracleLargeObjectPreviewBytes bounds Oracle large objects before
|
||||
// they cross the Wails bridge. The streaming export path stays unbounded.
|
||||
const interactiveOracleLargeObjectPreviewBytes = 4 * 1024
|
||||
|
||||
func scanRows(rows *sql.Rows) ([]map[string]interface{}, []string, error) {
|
||||
return scanRowsForDialect(rows, "")
|
||||
}
|
||||
@@ -51,7 +57,7 @@ func scanRowsForDialect(rows *sql.Rows, dialect string) ([]map[string]interface{
|
||||
resultData := make([]map[string]interface{}, 0)
|
||||
|
||||
for rows.Next() {
|
||||
entry, err := scanner.scanCurrentRow(rows)
|
||||
entry, err := scanner.scanCurrentPreviewRow(rows)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -145,15 +151,105 @@ func newQueryRowScanner(columns []string, colTypes []*sql.ColumnType, dialect st
|
||||
}
|
||||
|
||||
func (s *queryRowScanner) scanCurrentRowValues(rows *sql.Rows) ([]interface{}, error) {
|
||||
return s.scanCurrentRowValuesWithPreview(rows, false)
|
||||
}
|
||||
|
||||
func (s *queryRowScanner) scanCurrentRowValuesWithPreview(rows *sql.Rows, boundOracleLargeObjects bool) ([]interface{}, error) {
|
||||
if err := rows.Scan(s.valuePtrs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := range s.columns {
|
||||
s.normalized[i] = normalizeQueryValueWithDBTypeAndDialect(s.values[i], s.dbTypeNames[i], s.dialect)
|
||||
if boundOracleLargeObjects {
|
||||
s.normalized[i] = normalizeInteractiveQueryValue(s.values[i], s.dbTypeNames[i], s.dialect)
|
||||
} else {
|
||||
s.normalized[i] = normalizeQueryValueWithDBTypeAndDialect(s.values[i], s.dbTypeNames[i], s.dialect)
|
||||
}
|
||||
}
|
||||
return s.normalized, nil
|
||||
}
|
||||
|
||||
func normalizeInteractiveQueryValue(value interface{}, databaseTypeName, dialect string) interface{} {
|
||||
switch typedValue := value.(type) {
|
||||
case []byte:
|
||||
if len(typedValue) > interactiveOracleLargeObjectPreviewBytes && isOracleBinaryLargeObjectType(databaseTypeName) {
|
||||
preview := normalizeQueryValueWithDBTypeAndDialect(
|
||||
typedValue[:interactiveOracleLargeObjectPreviewBytes],
|
||||
databaseTypeName,
|
||||
dialect,
|
||||
)
|
||||
previewText, ok := preview.(string)
|
||||
if !ok {
|
||||
previewText = fmt.Sprint(preview)
|
||||
}
|
||||
return fmt.Sprintf(
|
||||
"[BLOB preview: %d/%d bytes] %s",
|
||||
interactiveOracleLargeObjectPreviewBytes,
|
||||
len(typedValue),
|
||||
previewText,
|
||||
)
|
||||
}
|
||||
case string:
|
||||
if len(typedValue) > interactiveOracleLargeObjectPreviewBytes && isOracleTextLargeObjectType(databaseTypeName) {
|
||||
preview := truncateUTF8Prefix(typedValue, interactiveOracleLargeObjectPreviewBytes)
|
||||
return fmt.Sprintf(
|
||||
"[CLOB preview: %d/%d bytes] %s",
|
||||
len(preview),
|
||||
len(typedValue),
|
||||
preview,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return normalizeQueryValueWithDBTypeAndDialect(value, databaseTypeName, dialect)
|
||||
}
|
||||
|
||||
func isOracleBinaryLargeObjectType(databaseTypeName string) bool {
|
||||
typeName := strings.ToUpper(strings.ReplaceAll(strings.TrimSpace(databaseTypeName), " ", ""))
|
||||
switch typeName {
|
||||
case "OCIBLOBLOCATOR", "LONGRAW", "LONGVARRAW":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func isOracleTextLargeObjectType(databaseTypeName string) bool {
|
||||
typeName := strings.ToUpper(strings.ReplaceAll(strings.TrimSpace(databaseTypeName), " ", ""))
|
||||
switch typeName {
|
||||
case "OCICLOBLOCATOR", "LONG", "LONGVARCHAR":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func truncateUTF8Prefix(value string, maxBytes int) string {
|
||||
if maxBytes <= 0 {
|
||||
return ""
|
||||
}
|
||||
if len(value) <= maxBytes {
|
||||
return value
|
||||
}
|
||||
|
||||
end := maxBytes
|
||||
for end > 0 && !utf8.RuneStart(value[end]) {
|
||||
end--
|
||||
}
|
||||
return value[:end]
|
||||
}
|
||||
|
||||
func (s *queryRowScanner) scanCurrentPreviewRow(rows *sql.Rows) (map[string]interface{}, error) {
|
||||
normalized, err := s.scanCurrentRowValuesWithPreview(rows, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entry := make(map[string]interface{}, len(s.columns))
|
||||
for i, col := range s.columns {
|
||||
entry[col] = normalized[i]
|
||||
}
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (s *queryRowScanner) scanCurrentRow(rows *sql.Rows) (map[string]interface{}, error) {
|
||||
normalized, err := s.scanCurrentRowValues(rows)
|
||||
if err != nil {
|
||||
|
||||
@@ -1,17 +1,22 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const scanRowsDuplicateDriverName = "gonavi-scan-rows-duplicate"
|
||||
const scanRowsOracleBlobTestBytes = 16*1024 + 17
|
||||
|
||||
var registerScanRowsDuplicateDriverOnce sync.Once
|
||||
|
||||
@@ -28,6 +33,24 @@ func (scanRowsDuplicateConn) Close() error { return
|
||||
func (scanRowsDuplicateConn) Begin() (driver.Tx, error) { return nil, driver.ErrSkip }
|
||||
|
||||
func (scanRowsDuplicateConn) QueryContext(_ context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
|
||||
if query == "SELECT blob_columns" {
|
||||
return &scanRowsDuplicateRows{
|
||||
columns: []string{"payload"},
|
||||
columnTypes: []string{"OCIBlobLocator"},
|
||||
rows: [][]driver.Value{
|
||||
{bytes.Repeat([]byte{0xff}, scanRowsOracleBlobTestBytes)},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
if query == "SELECT clob_columns" {
|
||||
return &scanRowsDuplicateRows{
|
||||
columns: []string{"content"},
|
||||
columnTypes: []string{"OCIClobLocator"},
|
||||
rows: [][]driver.Value{
|
||||
{strings.Repeat("数", 6*1024)},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
if query == "SELECT date_columns" {
|
||||
return &scanRowsDuplicateRows{
|
||||
columns: []string{"ship_date", "created_at"},
|
||||
@@ -120,6 +143,175 @@ func (scanRowsDuplicateConn) QueryContext(_ context.Context, query string, args
|
||||
}, nil
|
||||
}
|
||||
|
||||
type scanRowsValueConsumer struct {
|
||||
columns []string
|
||||
rows [][]interface{}
|
||||
}
|
||||
|
||||
func (c *scanRowsValueConsumer) SetColumns(columns []string) error {
|
||||
c.columns = append([]string(nil), columns...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *scanRowsValueConsumer) ConsumeRow(row map[string]interface{}) error {
|
||||
values := make([]interface{}, len(c.columns))
|
||||
for index, column := range c.columns {
|
||||
values[index] = row[column]
|
||||
}
|
||||
c.rows = append(c.rows, values)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *scanRowsValueConsumer) ConsumeRowValues(values []interface{}) error {
|
||||
c.rows = append(c.rows, append([]interface{}(nil), values...))
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestScanRowsBoundsOracleBlobPreview(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
registerScanRowsDuplicateDriverOnce.Do(func() {
|
||||
sql.Register(scanRowsDuplicateDriverName, scanRowsDuplicateDriver{})
|
||||
})
|
||||
|
||||
dbConn, err := sql.Open(scanRowsDuplicateDriverName, "")
|
||||
if err != nil {
|
||||
t.Fatalf("open blob scan rows db failed: %v", err)
|
||||
}
|
||||
defer dbConn.Close()
|
||||
|
||||
rows, err := dbConn.QueryContext(context.Background(), "SELECT blob_columns")
|
||||
if err != nil {
|
||||
t.Fatalf("query blob scan rows db failed: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// Production OracleDB leaves scanDialect empty; go-ora's column type is the
|
||||
// reliable signal for applying the interactive BLOB guard.
|
||||
data, columns, err := scanRowsForDialect(rows, "")
|
||||
if err != nil {
|
||||
t.Fatalf("scanRowsForDialect returned error: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(columns, []string{"payload"}) || len(data) != 1 {
|
||||
t.Fatalf("unexpected blob result: columns=%v rows=%d", columns, len(data))
|
||||
}
|
||||
|
||||
preview, ok := data[0]["payload"].(string)
|
||||
if !ok {
|
||||
t.Fatalf("Oracle BLOB preview type = %T, want string", data[0]["payload"])
|
||||
}
|
||||
wantPrefix := fmt.Sprintf("[BLOB preview: 4096/%d bytes] 0x", scanRowsOracleBlobTestBytes)
|
||||
if !strings.HasPrefix(preview, wantPrefix+strings.Repeat("ff", 4*1024)) {
|
||||
t.Fatalf("Oracle BLOB preview is missing bounded data or visible metadata: length=%d", len(preview))
|
||||
}
|
||||
fullHexLength := len("0x") + scanRowsOracleBlobTestBytes*2
|
||||
if len(preview) >= fullHexLength {
|
||||
t.Fatalf("Oracle BLOB preview length = %d, want less than full hex length %d", len(preview), fullHexLength)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamRowsKeepsCompleteOracleBlobValue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
registerScanRowsDuplicateDriverOnce.Do(func() {
|
||||
sql.Register(scanRowsDuplicateDriverName, scanRowsDuplicateDriver{})
|
||||
})
|
||||
|
||||
dbConn, err := sql.Open(scanRowsDuplicateDriverName, "")
|
||||
if err != nil {
|
||||
t.Fatalf("open streaming blob rows db failed: %v", err)
|
||||
}
|
||||
defer dbConn.Close()
|
||||
|
||||
rows, err := dbConn.QueryContext(context.Background(), "SELECT blob_columns")
|
||||
if err != nil {
|
||||
t.Fatalf("query streaming blob rows db failed: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
consumer := &scanRowsValueConsumer{}
|
||||
if err := streamRowsForDialect(rows, "", consumer); err != nil {
|
||||
t.Fatalf("streamRowsForDialect returned error: %v", err)
|
||||
}
|
||||
if len(consumer.rows) != 1 || len(consumer.rows[0]) != 1 {
|
||||
t.Fatalf("unexpected streamed blob rows: %#v", consumer.rows)
|
||||
}
|
||||
want := "0x" + strings.Repeat("ff", scanRowsOracleBlobTestBytes)
|
||||
if got := consumer.rows[0][0]; got != want {
|
||||
t.Fatalf("streamed Oracle BLOB was truncated: got length=%d want length=%d", len(got.(string)), len(want))
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanRowsBoundsOracleClobPreviewAtUTF8Boundary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
registerScanRowsDuplicateDriverOnce.Do(func() {
|
||||
sql.Register(scanRowsDuplicateDriverName, scanRowsDuplicateDriver{})
|
||||
})
|
||||
|
||||
dbConn, err := sql.Open(scanRowsDuplicateDriverName, "")
|
||||
if err != nil {
|
||||
t.Fatalf("open clob scan rows db failed: %v", err)
|
||||
}
|
||||
defer dbConn.Close()
|
||||
|
||||
rows, err := dbConn.QueryContext(context.Background(), "SELECT clob_columns")
|
||||
if err != nil {
|
||||
t.Fatalf("query clob scan rows db failed: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
data, _, err := scanRowsForDialect(rows, "")
|
||||
if err != nil {
|
||||
t.Fatalf("scanRowsForDialect returned error: %v", err)
|
||||
}
|
||||
preview, ok := data[0]["content"].(string)
|
||||
if !ok {
|
||||
t.Fatalf("Oracle CLOB preview type = %T, want string", data[0]["content"])
|
||||
}
|
||||
wantContentPrefix := strings.Repeat("数", (4*1024)/len("数"))
|
||||
wantPrefix := fmt.Sprintf(
|
||||
"[CLOB preview: %d/%d bytes] ",
|
||||
len(wantContentPrefix),
|
||||
len(strings.Repeat("数", 6*1024)),
|
||||
)
|
||||
if !strings.HasPrefix(preview, wantPrefix+wantContentPrefix) {
|
||||
t.Fatalf("Oracle CLOB preview does not preserve the UTF-8 prefix: length=%d", len(preview))
|
||||
}
|
||||
if !utf8.ValidString(preview) {
|
||||
t.Fatalf("Oracle CLOB preview split a UTF-8 character: %q", preview[:min(len(preview), 32)])
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamRowsKeepsCompleteOracleClobValue(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
registerScanRowsDuplicateDriverOnce.Do(func() {
|
||||
sql.Register(scanRowsDuplicateDriverName, scanRowsDuplicateDriver{})
|
||||
})
|
||||
|
||||
dbConn, err := sql.Open(scanRowsDuplicateDriverName, "")
|
||||
if err != nil {
|
||||
t.Fatalf("open streaming clob rows db failed: %v", err)
|
||||
}
|
||||
defer dbConn.Close()
|
||||
|
||||
rows, err := dbConn.QueryContext(context.Background(), "SELECT clob_columns")
|
||||
if err != nil {
|
||||
t.Fatalf("query streaming clob rows db failed: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
consumer := &scanRowsValueConsumer{}
|
||||
if err := streamRowsForDialect(rows, "", consumer); err != nil {
|
||||
t.Fatalf("streamRowsForDialect returned error: %v", err)
|
||||
}
|
||||
want := strings.Repeat("数", 6*1024)
|
||||
if len(consumer.rows) != 1 || len(consumer.rows[0]) != 1 || consumer.rows[0][0] != want {
|
||||
t.Fatalf("streamed Oracle CLOB was truncated: rows=%d", len(consumer.rows))
|
||||
}
|
||||
}
|
||||
|
||||
var _ driver.QueryerContext = (*scanRowsDuplicateConn)(nil)
|
||||
|
||||
type scanRowsDuplicateRows struct {
|
||||
|
||||
Reference in New Issue
Block a user