mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-08-12 01:24:12 +08:00
- 新增 ExportFileOptions 统一承载导出格式、进度任务和 XLSX sheet 行数上限 - 查询导出改为流式写入文件,避免一次性缓存整批结果导致高内存占用 - 增加值数组快速路径并复用扫描与写入缓冲,减少逐行 map 分配开销 - 为 ClickHouse、自定义驱动、达梦、SQLServer 和 TDengine 补齐 StreamQuery 支持 - 导出时间字符串仅在形似时间时再解析,避免普通文本被误判改写 - 补充 XLSX 分 sheet、流式导出和基准测试覆盖
223 lines
5.2 KiB
Go
223 lines
5.2 KiB
Go
package db
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
|
|
"GoNavi-Wails/internal/connection"
|
|
)
|
|
|
|
func scanRows(rows *sql.Rows) ([]map[string]interface{}, []string, error) {
|
|
return scanRowsForDialect(rows, "")
|
|
}
|
|
|
|
func streamRows(rows *sql.Rows, consumer QueryStreamConsumer) error {
|
|
return streamRowsForDialect(rows, "", consumer)
|
|
}
|
|
|
|
type queryRowScanner struct {
|
|
columns []string
|
|
dbTypeNames []string
|
|
dialect string
|
|
values []interface{}
|
|
normalized []interface{}
|
|
valuePtrs []interface{}
|
|
}
|
|
|
|
func scanRowsForDialect(rows *sql.Rows, dialect string) ([]map[string]interface{}, []string, error) {
|
|
columns, err := rows.Columns()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
columns = ensureUniqueQueryColumnNames(columns)
|
|
|
|
colTypes, err := rows.ColumnTypes()
|
|
if err != nil || len(colTypes) != len(columns) {
|
|
colTypes = nil
|
|
}
|
|
|
|
scanner := newQueryRowScanner(columns, colTypes, dialect)
|
|
resultData := make([]map[string]interface{}, 0)
|
|
|
|
for rows.Next() {
|
|
entry, err := scanner.scanCurrentRow(rows)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
resultData = append(resultData, entry)
|
|
}
|
|
|
|
if err := rows.Err(); err != nil {
|
|
return resultData, columns, err
|
|
}
|
|
return resultData, columns, nil
|
|
}
|
|
|
|
func streamRowsForDialect(rows *sql.Rows, dialect string, consumer QueryStreamConsumer) error {
|
|
if consumer == nil {
|
|
return fmt.Errorf("query stream consumer required")
|
|
}
|
|
|
|
columns, err := rows.Columns()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
columns = ensureUniqueQueryColumnNames(columns)
|
|
|
|
colTypes, err := rows.ColumnTypes()
|
|
if err != nil || len(colTypes) != len(columns) {
|
|
colTypes = nil
|
|
}
|
|
|
|
scanner := newQueryRowScanner(columns, colTypes, dialect)
|
|
if err := consumer.SetColumns(columns); err != nil {
|
|
return err
|
|
}
|
|
valueConsumer, useValueConsumer := consumer.(QueryStreamValueConsumer)
|
|
|
|
for rows.Next() {
|
|
if useValueConsumer {
|
|
values, err := scanner.scanCurrentRowValues(rows)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if err := valueConsumer.ConsumeRowValues(values); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
entry, err := scanner.scanCurrentRow(rows)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if err := consumer.ConsumeRow(entry); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return rows.Err()
|
|
}
|
|
|
|
func newQueryRowScanner(columns []string, colTypes []*sql.ColumnType, dialect string) *queryRowScanner {
|
|
values := make([]interface{}, len(columns))
|
|
valuePtrs := make([]interface{}, len(columns))
|
|
for i := range columns {
|
|
valuePtrs[i] = &values[i]
|
|
}
|
|
dbTypeNames := make([]string, len(columns))
|
|
for i := range columns {
|
|
if colTypes != nil && i < len(colTypes) && colTypes[i] != nil {
|
|
dbTypeNames[i] = colTypes[i].DatabaseTypeName()
|
|
}
|
|
}
|
|
return &queryRowScanner{
|
|
columns: columns,
|
|
dbTypeNames: dbTypeNames,
|
|
dialect: dialect,
|
|
values: values,
|
|
normalized: make([]interface{}, len(columns)),
|
|
valuePtrs: valuePtrs,
|
|
}
|
|
}
|
|
|
|
func (s *queryRowScanner) scanCurrentRowValues(rows *sql.Rows) ([]interface{}, error) {
|
|
if err := rows.Scan(s.valuePtrs...); err != nil {
|
|
return nil, err
|
|
}
|
|
for i := range s.columns {
|
|
s.normalized[i] = normalizeQueryValueWithDBTypeAndDialect(s.values[i], s.dbTypeNames[i], s.dialect)
|
|
}
|
|
return s.normalized, nil
|
|
}
|
|
|
|
func (s *queryRowScanner) scanCurrentRow(rows *sql.Rows) (map[string]interface{}, error) {
|
|
normalized, err := s.scanCurrentRowValues(rows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
entry := make(map[string]interface{}, len(s.columns))
|
|
for i, col := range s.columns {
|
|
entry[col] = normalized[i]
|
|
}
|
|
return entry, nil
|
|
}
|
|
|
|
func ensureUniqueQueryColumnNames(columns []string) []string {
|
|
if len(columns) == 0 {
|
|
return columns
|
|
}
|
|
|
|
uniqueColumns := make([]string, len(columns))
|
|
taken := make(map[string]struct{}, len(columns))
|
|
nextSuffix := make(map[string]int, len(columns))
|
|
|
|
for idx, column := range columns {
|
|
base := column
|
|
if base == "" {
|
|
base = fmt.Sprintf("column_%d", idx+1)
|
|
}
|
|
|
|
candidate := base
|
|
if _, exists := taken[candidate]; exists {
|
|
suffix := nextSuffix[base]
|
|
if suffix < 2 {
|
|
suffix = 2
|
|
}
|
|
for {
|
|
candidate = fmt.Sprintf("%s_%d", base, suffix)
|
|
if _, exists := taken[candidate]; !exists {
|
|
break
|
|
}
|
|
suffix++
|
|
}
|
|
nextSuffix[base] = suffix + 1
|
|
} else {
|
|
nextSuffix[base] = 2
|
|
}
|
|
|
|
uniqueColumns[idx] = candidate
|
|
taken[candidate] = struct{}{}
|
|
}
|
|
|
|
return uniqueColumns
|
|
}
|
|
|
|
// scanMultiRows 遍历 sql.Rows 中的所有结果集,将每个结果集作为 ResultSetData 返回。
|
|
// 利用 rows.NextResultSet() 支持一次 query 返回多个结果集的场景。
|
|
func scanMultiRows(rows *sql.Rows) ([]connection.ResultSetData, error) {
|
|
return scanMultiRowsForDialect(rows, "")
|
|
}
|
|
|
|
func scanMultiRowsForDialect(rows *sql.Rows, dialect string) ([]connection.ResultSetData, error) {
|
|
var results []connection.ResultSetData
|
|
for {
|
|
data, cols, err := scanRowsForDialect(rows, dialect)
|
|
if err != nil {
|
|
return results, err
|
|
}
|
|
if data == nil {
|
|
data = make([]map[string]interface{}, 0)
|
|
}
|
|
if cols == nil {
|
|
cols = []string{}
|
|
}
|
|
results = append(results, connection.ResultSetData{
|
|
Rows: data,
|
|
Columns: cols,
|
|
})
|
|
if !rows.NextResultSet() {
|
|
break
|
|
}
|
|
}
|
|
if len(results) == 0 {
|
|
results = []connection.ResultSetData{{
|
|
Rows: make([]map[string]interface{}, 0),
|
|
Columns: []string{},
|
|
}}
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return results, err
|
|
}
|
|
return results, nil
|
|
}
|