mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-08-12 01:24:12 +08:00
356 lines
9.7 KiB
Go
356 lines
9.7 KiB
Go
package db
|
||
|
||
import (
|
||
"database/sql"
|
||
"fmt"
|
||
"runtime"
|
||
"strings"
|
||
"unicode/utf8"
|
||
|
||
"GoNavi-Wails/internal/connection"
|
||
)
|
||
|
||
// streamRowsPeriodicGCInterval 控制 streamRowsForDialect 每处理多少行主动触发一次 runtime.GC。
|
||
//
|
||
// 背景：大结果集（88W+ 行）流式扫描时，每行 scanner 会分配 []interface{} 和 map[string]interface{}，
|
||
// Go 默认 GOGC=100 下堆翻倍才触发 GC，瞬时峰值可达数据总量 5-8 倍。
|
||
// 这里周期性主动 GC，让内存在扫描过程中及时回收，避免 RSS 单调爬升。
|
||
//
|
||
// 取值 50000：每 5W 行触发一次 GC，对 88W 行导出场景约触发 18 次，CPU 开销可忽略；
|
||
// 同时保证单次 GC 之间累积的临时对象不超过几百 MB，避免 GC 间隙堆膨胀。
|
||
const streamRowsPeriodicGCInterval = 50000
|
||
|
||
// interactiveOracleLargeObjectPreviewBytes bounds Oracle large objects before
|
||
// they cross the Wails bridge. The streaming export path stays unbounded.
|
||
const interactiveOracleLargeObjectPreviewBytes = 4 * 1024
|
||
|
||
func scanRows(rows *sql.Rows) ([]map[string]interface{}, []string, error) {
|
||
return scanRowsForDialect(rows, "")
|
||
}
|
||
|
||
func streamRows(rows *sql.Rows, consumer QueryStreamConsumer) error {
|
||
return streamRowsForDialect(rows, "", consumer)
|
||
}
|
||
|
||
type queryRowScanner struct {
|
||
columns []string
|
||
dbTypeNames []string
|
||
dialect string
|
||
values []interface{}
|
||
normalized []interface{}
|
||
valuePtrs []interface{}
|
||
}
|
||
|
||
func scanRowsForDialect(rows *sql.Rows, dialect string) ([]map[string]interface{}, []string, error) {
|
||
return scanRowsForDialectWithPreview(rows, dialect, true)
|
||
}
|
||
|
||
func scanRowsUnboundedForDialect(rows *sql.Rows, dialect string) ([]map[string]interface{}, []string, error) {
|
||
return scanRowsForDialectWithPreview(rows, dialect, false)
|
||
}
|
||
|
||
func scanRowsForDialectWithPreview(rows *sql.Rows, dialect string, boundOracleLargeObjects bool) ([]map[string]interface{}, []string, error) {
|
||
columns, err := rows.Columns()
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
columns = ensureUniqueQueryColumnNames(columns)
|
||
|
||
colTypes, err := rows.ColumnTypes()
|
||
if err != nil || len(colTypes) != len(columns) {
|
||
colTypes = nil
|
||
}
|
||
|
||
scanner := newQueryRowScanner(columns, colTypes, dialect)
|
||
resultData := make([]map[string]interface{}, 0)
|
||
|
||
for rows.Next() {
|
||
var entry map[string]interface{}
|
||
if boundOracleLargeObjects {
|
||
entry, err = scanner.scanCurrentPreviewRow(rows)
|
||
} else {
|
||
entry, err = scanner.scanCurrentRow(rows)
|
||
}
|
||
if err != nil {
|
||
continue
|
||
}
|
||
resultData = append(resultData, entry)
|
||
}
|
||
|
||
if err := rows.Err(); err != nil {
|
||
return resultData, columns, err
|
||
}
|
||
return resultData, columns, nil
|
||
}
|
||
|
||
func streamRowsForDialect(rows *sql.Rows, dialect string, consumer QueryStreamConsumer) error {
|
||
if consumer == nil {
|
||
return fmt.Errorf("query stream consumer required")
|
||
}
|
||
|
||
columns, err := rows.Columns()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
columns = ensureUniqueQueryColumnNames(columns)
|
||
|
||
colTypes, err := rows.ColumnTypes()
|
||
if err != nil || len(colTypes) != len(columns) {
|
||
colTypes = nil
|
||
}
|
||
|
||
scanner := newQueryRowScanner(columns, colTypes, dialect)
|
||
if err := consumer.SetColumns(columns); err != nil {
|
||
return err
|
||
}
|
||
valueConsumer, useValueConsumer := consumer.(QueryStreamValueConsumer)
|
||
|
||
// processedRows 用于周期性触发 GC，见 streamRowsPeriodicGCInterval 注释。
|
||
// 注意：此路径同时被 driver-agent 进程（OceanBase 等 optional driver）和
|
||
// 主进程的 in-process 流式查询调用，所以一处加 GC 即可覆盖两端。
|
||
var processedRows int64
|
||
|
||
for rows.Next() {
|
||
if useValueConsumer {
|
||
values, err := scanner.scanCurrentRowValues(rows)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
if err := valueConsumer.ConsumeRowValues(values); err != nil {
|
||
return err
|
||
}
|
||
} else {
|
||
entry, err := scanner.scanCurrentRow(rows)
|
||
if err != nil {
|
||
continue
|
||
}
|
||
if err := consumer.ConsumeRow(entry); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
|
||
processedRows++
|
||
if processedRows%streamRowsPeriodicGCInterval == 0 {
|
||
runtime.GC()
|
||
// 自适应抬升 driver-agent 进程的内存 soft limit。
|
||
// 主进程未启用 soft limit（未调 InitMemorySoftLimit），此调用是 no-op。
|
||
MaybeGrowMemoryLimit()
|
||
}
|
||
}
|
||
|
||
return rows.Err()
|
||
}
|
||
|
||
func newQueryRowScanner(columns []string, colTypes []*sql.ColumnType, dialect string) *queryRowScanner {
|
||
values := make([]interface{}, len(columns))
|
||
valuePtrs := make([]interface{}, len(columns))
|
||
for i := range columns {
|
||
valuePtrs[i] = &values[i]
|
||
}
|
||
dbTypeNames := make([]string, len(columns))
|
||
for i := range columns {
|
||
if colTypes != nil && i < len(colTypes) && colTypes[i] != nil {
|
||
dbTypeNames[i] = colTypes[i].DatabaseTypeName()
|
||
}
|
||
}
|
||
return &queryRowScanner{
|
||
columns: columns,
|
||
dbTypeNames: dbTypeNames,
|
||
dialect: dialect,
|
||
values: values,
|
||
normalized: make([]interface{}, len(columns)),
|
||
valuePtrs: valuePtrs,
|
||
}
|
||
}
|
||
|
||
func (s *queryRowScanner) scanCurrentRowValues(rows *sql.Rows) ([]interface{}, error) {
|
||
return s.scanCurrentRowValuesWithPreview(rows, false)
|
||
}
|
||
|
||
func (s *queryRowScanner) scanCurrentRowValuesWithPreview(rows *sql.Rows, boundOracleLargeObjects bool) ([]interface{}, error) {
|
||
if err := rows.Scan(s.valuePtrs...); err != nil {
|
||
return nil, err
|
||
}
|
||
for i := range s.columns {
|
||
if boundOracleLargeObjects {
|
||
s.normalized[i] = normalizeInteractiveQueryValue(s.values[i], s.dbTypeNames[i], s.dialect)
|
||
} else {
|
||
s.normalized[i] = normalizeQueryValueWithDBTypeAndDialect(s.values[i], s.dbTypeNames[i], s.dialect)
|
||
}
|
||
}
|
||
return s.normalized, nil
|
||
}
|
||
|
||
func normalizeInteractiveQueryValue(value interface{}, databaseTypeName, dialect string) interface{} {
|
||
switch typedValue := value.(type) {
|
||
case []byte:
|
||
if len(typedValue) > interactiveOracleLargeObjectPreviewBytes && isOracleBinaryLargeObjectType(databaseTypeName) {
|
||
preview := normalizeQueryValueWithDBTypeAndDialect(
|
||
typedValue[:interactiveOracleLargeObjectPreviewBytes],
|
||
databaseTypeName,
|
||
dialect,
|
||
)
|
||
previewText, ok := preview.(string)
|
||
if !ok {
|
||
previewText = fmt.Sprint(preview)
|
||
}
|
||
return fmt.Sprintf(
|
||
"[BLOB preview: %d/%d bytes] %s",
|
||
interactiveOracleLargeObjectPreviewBytes,
|
||
len(typedValue),
|
||
previewText,
|
||
)
|
||
}
|
||
case string:
|
||
if len(typedValue) > interactiveOracleLargeObjectPreviewBytes && isOracleTextLargeObjectType(databaseTypeName) {
|
||
preview := truncateUTF8Prefix(typedValue, interactiveOracleLargeObjectPreviewBytes)
|
||
return fmt.Sprintf(
|
||
"[CLOB preview: %d/%d bytes] %s",
|
||
len(preview),
|
||
len(typedValue),
|
||
preview,
|
||
)
|
||
}
|
||
}
|
||
|
||
return normalizeQueryValueWithDBTypeAndDialect(value, databaseTypeName, dialect)
|
||
}
|
||
|
||
func isOracleBinaryLargeObjectType(databaseTypeName string) bool {
|
||
typeName := strings.ToUpper(strings.ReplaceAll(strings.TrimSpace(databaseTypeName), " ", ""))
|
||
switch typeName {
|
||
case "OCIBLOBLOCATOR", "LONGRAW", "LONGVARRAW":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func isOracleTextLargeObjectType(databaseTypeName string) bool {
|
||
typeName := strings.ToUpper(strings.ReplaceAll(strings.TrimSpace(databaseTypeName), " ", ""))
|
||
switch typeName {
|
||
case "OCICLOBLOCATOR", "LONG", "LONGVARCHAR":
|
||
return true
|
||
default:
|
||
return false
|
||
}
|
||
}
|
||
|
||
func truncateUTF8Prefix(value string, maxBytes int) string {
|
||
if maxBytes <= 0 {
|
||
return ""
|
||
}
|
||
if len(value) <= maxBytes {
|
||
return value
|
||
}
|
||
|
||
end := maxBytes
|
||
for end > 0 && !utf8.RuneStart(value[end]) {
|
||
end--
|
||
}
|
||
return value[:end]
|
||
}
|
||
|
||
func (s *queryRowScanner) scanCurrentPreviewRow(rows *sql.Rows) (map[string]interface{}, error) {
|
||
normalized, err := s.scanCurrentRowValuesWithPreview(rows, true)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
entry := make(map[string]interface{}, len(s.columns))
|
||
for i, col := range s.columns {
|
||
entry[col] = normalized[i]
|
||
}
|
||
return entry, nil
|
||
}
|
||
|
||
func (s *queryRowScanner) scanCurrentRow(rows *sql.Rows) (map[string]interface{}, error) {
|
||
normalized, err := s.scanCurrentRowValues(rows)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
entry := make(map[string]interface{}, len(s.columns))
|
||
for i, col := range s.columns {
|
||
entry[col] = normalized[i]
|
||
}
|
||
return entry, nil
|
||
}
|
||
|
||
func ensureUniqueQueryColumnNames(columns []string) []string {
|
||
if len(columns) == 0 {
|
||
return columns
|
||
}
|
||
|
||
uniqueColumns := make([]string, len(columns))
|
||
taken := make(map[string]struct{}, len(columns))
|
||
nextSuffix := make(map[string]int, len(columns))
|
||
|
||
for idx, column := range columns {
|
||
base := column
|
||
if base == "" {
|
||
base = fmt.Sprintf("column_%d", idx+1)
|
||
}
|
||
|
||
candidate := base
|
||
if _, exists := taken[candidate]; exists {
|
||
suffix := nextSuffix[base]
|
||
if suffix < 2 {
|
||
suffix = 2
|
||
}
|
||
for {
|
||
candidate = fmt.Sprintf("%s_%d", base, suffix)
|
||
if _, exists := taken[candidate]; !exists {
|
||
break
|
||
}
|
||
suffix++
|
||
}
|
||
nextSuffix[base] = suffix + 1
|
||
} else {
|
||
nextSuffix[base] = 2
|
||
}
|
||
|
||
uniqueColumns[idx] = candidate
|
||
taken[candidate] = struct{}{}
|
||
}
|
||
|
||
return uniqueColumns
|
||
}
|
||
|
||
// scanMultiRows 遍历 sql.Rows 中的所有结果集，将每个结果集作为 ResultSetData 返回。
|
||
// 利用 rows.NextResultSet() 支持一次 query 返回多个结果集的场景。
|
||
func scanMultiRows(rows *sql.Rows) ([]connection.ResultSetData, error) {
|
||
return scanMultiRowsForDialect(rows, "")
|
||
}
|
||
|
||
func scanMultiRowsForDialect(rows *sql.Rows, dialect string) ([]connection.ResultSetData, error) {
|
||
var results []connection.ResultSetData
|
||
for {
|
||
data, cols, err := scanRowsForDialect(rows, dialect)
|
||
if err != nil {
|
||
return results, err
|
||
}
|
||
if data == nil {
|
||
data = make([]map[string]interface{}, 0)
|
||
}
|
||
if cols == nil {
|
||
cols = []string{}
|
||
}
|
||
results = append(results, connection.ResultSetData{
|
||
Rows: data,
|
||
Columns: cols,
|
||
})
|
||
if !rows.NextResultSet() {
|
||
break
|
||
}
|
||
}
|
||
if len(results) == 0 {
|
||
results = []connection.ResultSetData{{
|
||
Rows: make([]map[string]interface{}, 0),
|
||
Columns: []string{},
|
||
}}
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return results, err
|
||
}
|
||
return results, nil
|
||
}
|