mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-06-01 12:10:05 +08:00
🐛 fix(data-viewer): 修复ClickHouse尾部分页异常并增强DuckDB复杂类型兼容
- DataViewer 新增 ClickHouse 反向分页策略,修复最后页与倒数页查询失败 - DuckDB 查询失败时按列类型生成安全 SELECT,复杂类型转 VARCHAR 重试 - 分页状态统一使用 currentPage 回填,避免页码与总数推导不一致 - 增强查询异常日志与重试路径,降低大表场景卡顿与误报
This commit is contained in:
54
.github/workflows/release.yml
vendored
54
.github/workflows/release.yml
vendored
@@ -131,6 +131,24 @@ jobs:
|
||||
- name: Install Wails
|
||||
run: go install -v github.com/wailsapp/wails/v2/cmd/wails@latest
|
||||
|
||||
- name: Prepare MinGW For DuckDB (Windows)
|
||||
if: ${{ matrix.build_optional_agents && contains(matrix.platform, 'windows') }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
$mingwBin = "C:\msys64\mingw64\bin"
|
||||
if (!(Test-Path $mingwBin)) {
|
||||
choco install mingw --yes --no-progress
|
||||
$mingwBin = "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
|
||||
}
|
||||
if (!(Test-Path $mingwBin)) {
|
||||
Write-Error "❌ 未找到 MinGW GCC 路径:$mingwBin"
|
||||
exit 1
|
||||
}
|
||||
"$mingwBin" | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8
|
||||
"CC=$mingwBin\gcc.exe" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
"CXX=$mingwBin\g++.exe" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
|
||||
Write-Host "✅ 已配置 DuckDB cgo 编译器: $mingwBin"
|
||||
|
||||
- name: Build
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -166,20 +184,12 @@ jobs:
|
||||
OUTPUT_PATH="${OUTDIR}/${OUTPUT}"
|
||||
echo "🔧 构建 ${OUTPUT_PATH} (tag=${TAG})"
|
||||
if [ "$DRIVER" = "duckdb" ]; then
|
||||
set +e
|
||||
CGO_ENABLED=1 GOOS="$GOOS" GOARCH="$GOARCH" go build \
|
||||
-tags "${TAG}" \
|
||||
-trimpath \
|
||||
-ldflags "-s -w" \
|
||||
-o "${OUTPUT_PATH}" \
|
||||
./cmd/optional-driver-agent
|
||||
DUCKDB_RC=$?
|
||||
set -e
|
||||
if [ "${DUCKDB_RC}" -ne 0 ]; then
|
||||
echo "⚠️ DuckDB 代理构建失败(平台 ${GOOS}/${GOARCH}),跳过该资产,不阻断发布"
|
||||
rm -f "${OUTPUT_PATH}"
|
||||
continue
|
||||
fi
|
||||
else
|
||||
CGO_ENABLED=0 GOOS="$GOOS" GOARCH="$GOARCH" go build \
|
||||
-tags "${TAG}" \
|
||||
@@ -369,6 +379,34 @@ jobs:
|
||||
- name: List Assets
|
||||
run: ls -R release-assets
|
||||
|
||||
- name: Verify DuckDB Driver Assets
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cd release-assets
|
||||
|
||||
REQUIRED_FILES=(
|
||||
"drivers/Windows/duckdb-driver-agent-windows-amd64.exe"
|
||||
"drivers/MacOS/duckdb-driver-agent-darwin-amd64"
|
||||
"drivers/MacOS/duckdb-driver-agent-darwin-arm64"
|
||||
"drivers/Linux/duckdb-driver-agent-linux-amd64"
|
||||
)
|
||||
|
||||
missing=0
|
||||
for file in "${REQUIRED_FILES[@]}"; do
|
||||
if [ ! -f "$file" ]; then
|
||||
echo "❌ 缺少 DuckDB 驱动资产:$file"
|
||||
missing=1
|
||||
else
|
||||
echo "✅ 已找到 DuckDB 驱动资产:$file"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$missing" -ne 0 ]; then
|
||||
echo "❌ DuckDB 驱动资产不完整,终止发布"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Package Driver Agents Bundle
|
||||
shell: bash
|
||||
run: |
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
"GoNavi-Wails/internal/connection"
|
||||
@@ -218,7 +219,11 @@ func handleRequest(inst *db.Database, req agentRequest) agentResponse {
|
||||
}
|
||||
|
||||
func writeResponse(writer *bufio.Writer, resp agentResponse) error {
|
||||
payload, err := json.Marshal(resp)
|
||||
// 对响应数据做统一 JSON 安全归一化:
|
||||
// 将 map[any]any(如 duckdb.Map)递归转换为 map[string]any,避免序列化失败导致代理进程退出。
|
||||
safeResp := resp
|
||||
safeResp.Data = normalizeAgentResponseData(resp.Data)
|
||||
payload, err := json.Marshal(safeResp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -234,3 +239,51 @@ func fail(resp agentResponse, errText string) agentResponse {
|
||||
resp.Error = strings.TrimSpace(errText)
|
||||
return resp
|
||||
}
|
||||
|
||||
func normalizeAgentResponseData(v interface{}) interface{} {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
rv := reflect.ValueOf(v)
|
||||
switch rv.Kind() {
|
||||
case reflect.Pointer, reflect.Interface:
|
||||
if rv.IsNil() {
|
||||
return nil
|
||||
}
|
||||
return normalizeAgentResponseData(rv.Elem().Interface())
|
||||
case reflect.Map:
|
||||
if rv.IsNil() {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]interface{}, rv.Len())
|
||||
iter := rv.MapRange()
|
||||
for iter.Next() {
|
||||
out[fmt.Sprint(iter.Key().Interface())] = normalizeAgentResponseData(iter.Value().Interface())
|
||||
}
|
||||
return out
|
||||
case reflect.Slice:
|
||||
if rv.IsNil() {
|
||||
return nil
|
||||
}
|
||||
// 保持 []byte 原样,避免改变现有二进制列的 JSON 编码行为(base64)。
|
||||
if rv.Type().Elem().Kind() == reflect.Uint8 {
|
||||
return v
|
||||
}
|
||||
size := rv.Len()
|
||||
items := make([]interface{}, size)
|
||||
for i := 0; i < size; i++ {
|
||||
items[i] = normalizeAgentResponseData(rv.Index(i).Interface())
|
||||
}
|
||||
return items
|
||||
case reflect.Array:
|
||||
size := rv.Len()
|
||||
items := make([]interface{}, size)
|
||||
for i := 0; i < size; i++ {
|
||||
items[i] = normalizeAgentResponseData(rv.Index(i).Interface())
|
||||
}
|
||||
return items
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
62
cmd/optional-driver-agent/main_test.go
Normal file
62
cmd/optional-driver-agent/main_test.go
Normal file
@@ -0,0 +1,62 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type duckMapLike map[any]any
|
||||
|
||||
func TestWriteResponse_NormalizesMapAnyAny(t *testing.T) {
|
||||
resp := agentResponse{
|
||||
ID: 1,
|
||||
Success: true,
|
||||
Data: []map[string]interface{}{
|
||||
{
|
||||
"id": int64(7),
|
||||
"meta": duckMapLike{"k": "v", 2: "two"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var out bytes.Buffer
|
||||
writer := bufio.NewWriter(&out)
|
||||
if err := writeResponse(writer, resp); err != nil {
|
||||
t.Fatalf("writeResponse 返回错误: %v", err)
|
||||
}
|
||||
|
||||
var decoded struct {
|
||||
Data []map[string]interface{} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(bytes.TrimSpace(out.Bytes()), &decoded); err != nil {
|
||||
t.Fatalf("解码响应失败: %v", err)
|
||||
}
|
||||
|
||||
if len(decoded.Data) != 1 {
|
||||
t.Fatalf("期望 1 行数据,实际 %d", len(decoded.Data))
|
||||
}
|
||||
meta, ok := decoded.Data[0]["meta"].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("meta 字段类型异常: %T", decoded.Data[0]["meta"])
|
||||
}
|
||||
if meta["k"] != "v" {
|
||||
t.Fatalf("字符串 key 转换异常: %v", meta["k"])
|
||||
}
|
||||
if meta["2"] != "two" {
|
||||
t.Fatalf("数字 key 未字符串化: %v", meta["2"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeAgentResponseData_KeepByteSlice(t *testing.T) {
|
||||
raw := []byte{0x61, 0x62, 0x63}
|
||||
normalized := normalizeAgentResponseData(raw)
|
||||
out, ok := normalized.([]byte)
|
||||
if !ok {
|
||||
t.Fatalf("期望 []byte,实际 %T", normalized)
|
||||
}
|
||||
if !bytes.Equal(out, raw) {
|
||||
t.Fatalf("[]byte 内容被意外改写: %v", out)
|
||||
}
|
||||
}
|
||||
@@ -33,7 +33,7 @@
|
||||
},
|
||||
"duckdb": {
|
||||
"engine": "go",
|
||||
"version": "2.5.5",
|
||||
"version": "2.5.6",
|
||||
"checksumPolicy": "off",
|
||||
"downloadUrl": "builtin://activate/duckdb"
|
||||
},
|
||||
|
||||
@@ -3,7 +3,7 @@ import { Modal, Form, Input, InputNumber, Button, message, Checkbox, Divider, Se
|
||||
import { DatabaseOutlined, ConsoleSqlOutlined, FileTextOutlined, CloudServerOutlined, AppstoreAddOutlined, CloudOutlined, CheckCircleFilled, CloseCircleFilled } from '@ant-design/icons';
|
||||
import { useStore } from '../store';
|
||||
import { normalizeOpacityForPlatform } from '../utils/appearance';
|
||||
import { DBGetDatabases, GetDriverStatusList, MongoDiscoverMembers, TestConnection, RedisConnect, SelectSSHKeyFile } from '../../wailsjs/go/app/App';
|
||||
import { DBGetDatabases, GetDriverStatusList, MongoDiscoverMembers, TestConnection, RedisConnect, SelectDatabaseFile, SelectSSHKeyFile } from '../../wailsjs/go/app/App';
|
||||
import { ConnectionConfig, MongoMemberInfo, SavedConnection } from '../types';
|
||||
|
||||
const { Meta } = Card;
|
||||
@@ -80,6 +80,7 @@ const ConnectionModal: React.FC<{
|
||||
const [typeSelectWarning, setTypeSelectWarning] = useState<{ driverName: string; reason: string } | null>(null);
|
||||
const [driverStatusMap, setDriverStatusMap] = useState<Record<string, DriverStatusSnapshot>>({});
|
||||
const [driverStatusLoaded, setDriverStatusLoaded] = useState(false);
|
||||
const [selectingDbFile, setSelectingDbFile] = useState(false);
|
||||
const [selectingSSHKey, setSelectingSSHKey] = useState(false);
|
||||
const testInFlightRef = useRef(false);
|
||||
const testTimerRef = useRef<number | null>(null);
|
||||
@@ -665,6 +666,30 @@ const ConnectionModal: React.FC<{
|
||||
}
|
||||
};
|
||||
|
||||
const handleSelectDatabaseFile = async () => {
|
||||
if (selectingDbFile) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
setSelectingDbFile(true);
|
||||
const currentPath = String(form.getFieldValue('host') || '').trim();
|
||||
const res = await SelectDatabaseFile(currentPath, dbType);
|
||||
if (res?.success) {
|
||||
const data = res.data || {};
|
||||
const selectedPath = typeof data === 'string' ? data : String(data.path || '').trim();
|
||||
if (selectedPath) {
|
||||
form.setFieldValue('host', normalizeFileDbPath(selectedPath));
|
||||
}
|
||||
} else if (res?.message !== 'Cancelled') {
|
||||
message.error(`选择数据库文件失败: ${res?.message || '未知错误'}`);
|
||||
}
|
||||
} catch (e: any) {
|
||||
message.error(`选择数据库文件失败: ${e?.message || String(e)}`);
|
||||
} finally {
|
||||
setSelectingDbFile(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (open) {
|
||||
setTestResult(null); // Reset test result
|
||||
@@ -1392,6 +1417,13 @@ const ConnectionModal: React.FC<{
|
||||
onDoubleClick={requestTest}
|
||||
/>
|
||||
</Form.Item>
|
||||
{isFileDb && (
|
||||
<Form.Item label=" " style={{ width: 120 }}>
|
||||
<Button style={{ width: '100%' }} onClick={handleSelectDatabaseFile} loading={selectingDbFile}>
|
||||
浏览...
|
||||
</Button>
|
||||
</Form.Item>
|
||||
)}
|
||||
{!isFileDb && (
|
||||
<Form.Item
|
||||
name="port"
|
||||
|
||||
@@ -2,9 +2,9 @@ import React, { useEffect, useState, useCallback, useRef } from 'react';
|
||||
import { message } from 'antd';
|
||||
import { TabData, ColumnDefinition } from '../types';
|
||||
import { useStore } from '../store';
|
||||
import { DBQuery, DBGetColumns, DBQueryIsolated } from '../../wailsjs/go/app/App';
|
||||
import { DBQuery, DBGetColumns } from '../../wailsjs/go/app/App';
|
||||
import DataGrid, { GONAVI_ROW_KEY } from './DataGrid';
|
||||
import { buildOrderBySQL, buildWhereSQL, quoteQualifiedIdent, withSortBufferTuningSQL, type FilterCondition } from '../utils/sql';
|
||||
import { buildOrderBySQL, buildWhereSQL, quoteIdentPart, quoteQualifiedIdent, withSortBufferTuningSQL, type FilterCondition } from '../utils/sql';
|
||||
|
||||
type ViewerPaginationState = {
|
||||
current: number;
|
||||
@@ -108,6 +108,33 @@ const resolveDuckDBSchemaAndTable = (dbName: string, tableName: string) => {
|
||||
|
||||
const escapeSQLLiteral = (value: string): string => String(value || '').replace(/'/g, "''");
|
||||
|
||||
const isDuckDBUnsupportedTypeError = (msg: string): boolean => /unsupported\s*type:\s*duckdb\./i.test(String(msg || ''));
|
||||
|
||||
const isDuckDBComplexColumnType = (columnType?: string): boolean => {
|
||||
const raw = String(columnType || '').trim().toLowerCase();
|
||||
if (!raw) return false;
|
||||
return raw.includes('map') || raw.includes('struct') || raw.includes('union') || raw.includes('array') || raw.includes('list');
|
||||
};
|
||||
|
||||
const reverseOrderBySQL = (orderBySQL: string): string => {
|
||||
const raw = String(orderBySQL || '').trim();
|
||||
if (!raw) return '';
|
||||
const body = raw.replace(/^order\s+by\s+/i, '').trim();
|
||||
if (!body) return '';
|
||||
|
||||
const parts = body
|
||||
.split(',')
|
||||
.map((part) => part.trim())
|
||||
.filter(Boolean)
|
||||
.map((part) => {
|
||||
if (/\s+asc$/i.test(part)) return part.replace(/\s+asc$/i, ' DESC');
|
||||
if (/\s+desc$/i.test(part)) return part.replace(/\s+desc$/i, ' ASC');
|
||||
return `${part} DESC`;
|
||||
});
|
||||
if (parts.length === 0) return '';
|
||||
return ` ORDER BY ${parts.join(', ')}`;
|
||||
};
|
||||
|
||||
const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
|
||||
const [data, setData] = useState<any[]>([]);
|
||||
const [columnNames, setColumnNames] = useState<string[]>([]);
|
||||
@@ -144,19 +171,17 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
|
||||
|
||||
const [showFilter, setShowFilter] = useState(false);
|
||||
const [filterConditions, setFilterConditions] = useState<FilterCondition[]>([]);
|
||||
const duckdbSafeSelectCacheRef = useRef<Record<string, string>>({});
|
||||
const currentConnType = (connections.find(c => c.id === tab.connectionId)?.config?.type || '').toLowerCase();
|
||||
const forceReadOnly = currentConnType === 'tdengine' || currentConnType === 'clickhouse';
|
||||
|
||||
const runIsolatedQuery = useCallback(async (queryConfig: any, dbName: string, sql: string) => {
|
||||
return DBQueryIsolated(queryConfig as any, dbName, sql);
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
setPkColumns([]);
|
||||
pkKeyRef.current = '';
|
||||
countKeyRef.current = '';
|
||||
duckdbApproxKeyRef.current = '';
|
||||
manualCountKeyRef.current = '';
|
||||
duckdbSafeSelectCacheRef.current = {};
|
||||
latestConfigRef.current = null;
|
||||
latestDbTypeRef.current = '';
|
||||
latestDbNameRef.current = '';
|
||||
@@ -194,7 +219,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
|
||||
const countConfig: any = { ...(config as any), timeout: 120 };
|
||||
|
||||
try {
|
||||
const resCount = await runIsolatedQuery(countConfig, dbName, countSql);
|
||||
const resCount = await DBQuery(countConfig as any, dbName, countSql);
|
||||
const countDuration = Date.now() - countStart;
|
||||
addSqlLog({
|
||||
id: `log-${Date.now()}-duckdb-manual-count`,
|
||||
@@ -240,7 +265,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
|
||||
setPagination(prev => ({ ...prev, totalCountLoading: false }));
|
||||
message.error(`统计总数失败: ${String(e?.message || e)}`);
|
||||
}
|
||||
}, [addSqlLog, runIsolatedQuery]);
|
||||
}, [addSqlLog]);
|
||||
|
||||
const handleDuckDBCancelManualCount = useCallback(() => {
|
||||
manualCountSeqRef.current++;
|
||||
@@ -277,35 +302,112 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
|
||||
|
||||
const countSql = `SELECT COUNT(*) as total FROM ${quoteQualifiedIdent(dbType, tableName)} ${whereSQL}`;
|
||||
|
||||
let sql = `SELECT * FROM ${quoteQualifiedIdent(dbType, tableName)} ${whereSQL}`;
|
||||
sql += buildOrderBySQL(dbType, sortInfo, pkColumns);
|
||||
const offset = (page - 1) * size;
|
||||
// 大表性能:打开表不阻塞在 COUNT(*),先通过多取 1 条判断是否还有下一页;总数在后台统计并异步回填。
|
||||
sql += ` LIMIT ${size + 1} OFFSET ${offset}`;
|
||||
const baseSql = `SELECT * FROM ${quoteQualifiedIdent(dbType, tableName)} ${whereSQL}`;
|
||||
const orderBySQL = buildOrderBySQL(dbType, sortInfo, pkColumns);
|
||||
let sql = `${baseSql}${orderBySQL}`;
|
||||
const totalRows = Number(pagination.total);
|
||||
const hasFiniteTotal = Number.isFinite(totalRows) && totalRows >= 0;
|
||||
const totalKnown = pagination.totalKnown && hasFiniteTotal;
|
||||
const totalPages = hasFiniteTotal ? Math.max(1, Math.ceil(totalRows / size)) : 0;
|
||||
const currentPage = totalPages > 0 ? Math.min(Math.max(1, page), totalPages) : Math.max(1, page);
|
||||
const offset = (currentPage - 1) * size;
|
||||
const isClickHouse = dbTypeLower === 'clickhouse';
|
||||
const reverseOrderSQL = isClickHouse ? reverseOrderBySQL(orderBySQL) : '';
|
||||
let useClickHouseReversePagination = false;
|
||||
let clickHouseReverseLimit = 0;
|
||||
let clickHouseReverseHasMore = false;
|
||||
// ClickHouse 深分页在超大 OFFSET 下容易超时。对于总数已知且存在 ORDER BY 的场景,
|
||||
// 当“尾部偏移”小于“头部偏移”时,改为反向 ORDER BY + 小 OFFSET,并在前端翻转结果。
|
||||
if (isClickHouse && totalKnown && offset > 0 && reverseOrderSQL) {
|
||||
const pageRowCount = Math.max(0, Math.min(size, totalRows - offset));
|
||||
if (pageRowCount > 0) {
|
||||
const tailOffset = Math.max(0, totalRows - (offset + pageRowCount));
|
||||
if (tailOffset < offset) {
|
||||
sql = `${baseSql}${reverseOrderSQL} LIMIT ${pageRowCount} OFFSET ${tailOffset}`;
|
||||
useClickHouseReversePagination = true;
|
||||
clickHouseReverseLimit = pageRowCount;
|
||||
clickHouseReverseHasMore = currentPage < totalPages;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!useClickHouseReversePagination) {
|
||||
// 大表性能:打开表不阻塞在 COUNT(*),先通过多取 1 条判断是否还有下一页;总数在后台统计并异步回填。
|
||||
sql += ` LIMIT ${size + 1} OFFSET ${offset}`;
|
||||
}
|
||||
|
||||
const requestStartTime = Date.now();
|
||||
let executedSql = sql;
|
||||
try {
|
||||
const executeDataQuery = async (querySql: string, attemptLabel: string) => {
|
||||
const startTime = Date.now();
|
||||
const result = await DBQuery(config as any, dbName, querySql);
|
||||
addSqlLog({
|
||||
id: `log-${Date.now()}-data`,
|
||||
timestamp: Date.now(),
|
||||
sql: querySql,
|
||||
status: result.success ? 'success' : 'error',
|
||||
duration: Date.now() - startTime,
|
||||
message: result.success ? '' : `${attemptLabel}: ${result.message}`,
|
||||
affectedRows: Array.isArray(result.data) ? result.data.length : undefined,
|
||||
dbName
|
||||
});
|
||||
return result;
|
||||
try {
|
||||
const result = await DBQuery(config as any, dbName, querySql);
|
||||
addSqlLog({
|
||||
id: `log-${Date.now()}-data`,
|
||||
timestamp: Date.now(),
|
||||
sql: querySql,
|
||||
status: result.success ? 'success' : 'error',
|
||||
duration: Date.now() - startTime,
|
||||
message: result.success ? '' : `${attemptLabel}: ${result.message}`,
|
||||
affectedRows: Array.isArray(result.data) ? result.data.length : undefined,
|
||||
dbName
|
||||
});
|
||||
return result;
|
||||
} catch (e: any) {
|
||||
const errMessage = String(e?.message || e || 'query failed');
|
||||
addSqlLog({
|
||||
id: `log-${Date.now()}-data`,
|
||||
timestamp: Date.now(),
|
||||
sql: querySql,
|
||||
status: 'error',
|
||||
duration: Date.now() - startTime,
|
||||
message: `${attemptLabel}: ${errMessage}`,
|
||||
dbName
|
||||
});
|
||||
return { success: false, message: errMessage, data: [], fields: [] };
|
||||
}
|
||||
};
|
||||
|
||||
const hasSort = !!sortInfo?.columnKey && (sortInfo?.order === 'ascend' || sortInfo?.order === 'descend');
|
||||
const isSortMemoryErr = (msg: string) => /error\s*1038|out of sort memory/i.test(String(msg || ''));
|
||||
let resData = await executeDataQuery(sql, '主查询');
|
||||
|
||||
if (!resData.success && dbTypeLower === 'duckdb' && isDuckDBUnsupportedTypeError(String(resData.message || ''))) {
|
||||
const cacheKey = `${tab.connectionId}|${dbName}|${tableName}`;
|
||||
let safeSelect = duckdbSafeSelectCacheRef.current[cacheKey] || '';
|
||||
if (!safeSelect) {
|
||||
try {
|
||||
const resCols = await DBGetColumns(config as any, dbName, tableName);
|
||||
if (resCols?.success && Array.isArray(resCols.data)) {
|
||||
const columnDefs = resCols.data as ColumnDefinition[];
|
||||
const selectParts = columnDefs.map((col) => {
|
||||
const colName = String(col?.name || '').trim();
|
||||
if (!colName) return '';
|
||||
const quotedCol = quoteIdentPart(dbType, colName);
|
||||
if (isDuckDBComplexColumnType(col?.type)) {
|
||||
return `CAST(${quotedCol} AS VARCHAR) AS ${quotedCol}`;
|
||||
}
|
||||
return quotedCol;
|
||||
}).filter(Boolean);
|
||||
if (selectParts.length > 0) {
|
||||
safeSelect = selectParts.join(', ');
|
||||
duckdbSafeSelectCacheRef.current[cacheKey] = safeSelect;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore and keep original error path
|
||||
}
|
||||
}
|
||||
|
||||
if (safeSelect) {
|
||||
let fallbackSql = `SELECT ${safeSelect} FROM ${quoteQualifiedIdent(dbType, tableName)} ${whereSQL}`;
|
||||
fallbackSql += buildOrderBySQL(dbType, sortInfo, pkColumns);
|
||||
fallbackSql += ` LIMIT ${size + 1} OFFSET ${offset}`;
|
||||
executedSql = fallbackSql;
|
||||
resData = await executeDataQuery(fallbackSql, '复杂类型降级重试');
|
||||
}
|
||||
}
|
||||
|
||||
if (!resData.success && isMySQLFamily && hasSort && isSortMemoryErr(resData.message)) {
|
||||
const retrySql32MB = withSortBufferTuningSQL(dbType, sql, 32 * 1024 * 1024);
|
||||
if (retrySql32MB !== sql) {
|
||||
@@ -348,7 +450,12 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
|
||||
let resultData = resData.data as any[];
|
||||
if (!Array.isArray(resultData)) resultData = [];
|
||||
|
||||
const hasMore = resultData.length > size;
|
||||
if (useClickHouseReversePagination) {
|
||||
// 反向查询后恢复为原排序方向,保证用户看到的仍是“最后一页正序数据”。
|
||||
resultData = resultData.slice(0, clickHouseReverseLimit).reverse();
|
||||
}
|
||||
|
||||
const hasMore = useClickHouseReversePagination ? clickHouseReverseHasMore : resultData.length > size;
|
||||
if (hasMore) resultData = resultData.slice(0, size);
|
||||
|
||||
let fieldNames = resData.fields || [];
|
||||
@@ -363,7 +470,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
|
||||
setData(resultData);
|
||||
const countKey = `${tab.connectionId}|${dbName}|${tableName}|${whereSQL}`;
|
||||
const derivedTotalKnown = !hasMore;
|
||||
const derivedTotal = derivedTotalKnown ? offset + resultData.length : page * size + 1;
|
||||
const derivedTotal = derivedTotalKnown ? offset + resultData.length : currentPage * size + 1;
|
||||
const isDuckDB = dbTypeLower === 'duckdb';
|
||||
const minExpectedTotal = hasMore ? offset + resultData.length + 1 : offset + resultData.length;
|
||||
if (derivedTotalKnown) countKeyRef.current = countKey;
|
||||
@@ -377,7 +484,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
|
||||
if (derivedTotalKnown) {
|
||||
return {
|
||||
...prev,
|
||||
current: page,
|
||||
current: currentPage,
|
||||
pageSize: size,
|
||||
total: derivedTotal,
|
||||
totalKnown: true,
|
||||
@@ -388,19 +495,19 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
|
||||
}
|
||||
if (prev.totalKnown && countKeyRef.current === countKey) {
|
||||
if (!isDuckDB) {
|
||||
return { ...prev, current: page, pageSize: size };
|
||||
return { ...prev, current: currentPage, pageSize: size };
|
||||
}
|
||||
// 当当前页存在“下一页”信号时,已知总数至少应大于当前页末尾。
|
||||
// 若旧总数不满足该条件(例如历史统计值为 0),降级为未知总数并回退到 derivedTotal。
|
||||
if (Number.isFinite(prev.total) && prev.total >= minExpectedTotal) {
|
||||
return { ...prev, current: page, pageSize: size };
|
||||
return { ...prev, current: currentPage, pageSize: size };
|
||||
}
|
||||
}
|
||||
const keepManualCounting = prev.totalCountLoading && manualCountKeyRef.current === countKey;
|
||||
if (isDuckDB && prev.totalApprox && duckdbApproxKeyRef.current === countKey && Number.isFinite(prev.total) && prev.total >= minExpectedTotal) {
|
||||
return {
|
||||
...prev,
|
||||
current: page,
|
||||
current: currentPage,
|
||||
pageSize: size,
|
||||
totalKnown: false,
|
||||
totalApprox: true,
|
||||
@@ -410,7 +517,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
|
||||
}
|
||||
return {
|
||||
...prev,
|
||||
current: page,
|
||||
current: currentPage,
|
||||
pageSize: size,
|
||||
total: derivedTotal,
|
||||
totalKnown: false,
|
||||
@@ -489,7 +596,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
|
||||
(async () => {
|
||||
for (const approxSql of approxSqlCandidates) {
|
||||
try {
|
||||
const approxRes = await runIsolatedQuery(approxConfig, dbName, approxSql);
|
||||
const approxRes = await DBQuery(approxConfig as any, dbName, approxSql);
|
||||
if (duckdbApproxSeqRef.current !== approxSeq) return;
|
||||
if (countKeyRef.current !== countKey) return;
|
||||
if (!approxRes?.success || !Array.isArray(approxRes.data) || approxRes.data.length === 0) continue;
|
||||
@@ -534,7 +641,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
|
||||
});
|
||||
}
|
||||
if (fetchSeqRef.current === seq) setLoading(false);
|
||||
}, [connections, tab, sortInfo, filterConditions, pkColumns, runIsolatedQuery]);
|
||||
}, [connections, tab, sortInfo, filterConditions, pkColumns, pagination.total, pagination.totalKnown]);
|
||||
// 依赖 pkColumns:在无手动排序时可回退到主键稳定排序。
|
||||
// 主键信息只会在首次加载后更新一次,避免循环查询。
|
||||
|
||||
|
||||
2
frontend/wailsjs/go/app/App.d.ts
vendored
2
frontend/wailsjs/go/app/App.d.ts
vendored
@@ -164,6 +164,8 @@ export function ResolveDriverPackageDownloadURL(arg1:string,arg2:string):Promise
|
||||
|
||||
export function ResolveDriverRepositoryURL(arg1:string):Promise<connection.QueryResult>;
|
||||
|
||||
export function SelectDatabaseFile(arg1:string,arg2:string):Promise<connection.QueryResult>;
|
||||
|
||||
export function SelectDriverDownloadDirectory(arg1:string):Promise<connection.QueryResult>;
|
||||
|
||||
export function SelectDriverPackageDirectory(arg1:string):Promise<connection.QueryResult>;
|
||||
|
||||
@@ -322,6 +322,10 @@ export function ResolveDriverRepositoryURL(arg1) {
|
||||
return window['go']['app']['App']['ResolveDriverRepositoryURL'](arg1);
|
||||
}
|
||||
|
||||
export function SelectDatabaseFile(arg1, arg2) {
|
||||
return window['go']['app']['App']['SelectDatabaseFile'](arg1, arg2);
|
||||
}
|
||||
|
||||
export function SelectDriverDownloadDirectory(arg1) {
|
||||
return window['go']['app']['App']['SelectDriverDownloadDirectory'](arg1);
|
||||
}
|
||||
|
||||
@@ -74,16 +74,67 @@ func (a *App) Shutdown(ctx context.Context) {
|
||||
logger.Close()
|
||||
}
|
||||
|
||||
// Helper: Generate a unique key for the connection config
|
||||
func getCacheKey(config connection.ConnectionConfig) string {
|
||||
if !config.UseSSH {
|
||||
config.SSH = connection.SSHConfig{}
|
||||
func normalizeCacheKeyConfig(config connection.ConnectionConfig) connection.ConnectionConfig {
|
||||
normalized := config
|
||||
normalized.Type = strings.ToLower(strings.TrimSpace(normalized.Type))
|
||||
// timeout 仅用于 Query/Ping 控制,不应作为物理连接复用键的一部分。
|
||||
normalized.Timeout = 0
|
||||
normalized.SavePassword = false
|
||||
|
||||
if !normalized.UseSSH {
|
||||
normalized.SSH = connection.SSHConfig{}
|
||||
}
|
||||
if !config.UseProxy {
|
||||
config.Proxy = connection.ProxyConfig{}
|
||||
if !normalized.UseProxy {
|
||||
normalized.Proxy = connection.ProxyConfig{}
|
||||
}
|
||||
|
||||
b, _ := json.Marshal(config)
|
||||
if isFileDatabaseType(normalized.Type) {
|
||||
dsn := strings.TrimSpace(normalized.Host)
|
||||
if dsn == "" {
|
||||
dsn = strings.TrimSpace(normalized.Database)
|
||||
}
|
||||
if dsn == "" {
|
||||
dsn = ":memory:"
|
||||
}
|
||||
|
||||
// DuckDB/SQLite 仅基于文件来源识别连接,其他网络字段不参与键计算。
|
||||
normalized.Host = dsn
|
||||
normalized.Database = ""
|
||||
normalized.Port = 0
|
||||
normalized.User = ""
|
||||
normalized.Password = ""
|
||||
normalized.URI = ""
|
||||
normalized.Hosts = nil
|
||||
normalized.Topology = ""
|
||||
normalized.MySQLReplicaUser = ""
|
||||
normalized.MySQLReplicaPassword = ""
|
||||
normalized.ReplicaSet = ""
|
||||
normalized.AuthSource = ""
|
||||
normalized.ReadPreference = ""
|
||||
normalized.MongoSRV = false
|
||||
normalized.MongoAuthMechanism = ""
|
||||
normalized.MongoReplicaUser = ""
|
||||
normalized.MongoReplicaPassword = ""
|
||||
}
|
||||
|
||||
return normalized
|
||||
}
|
||||
|
||||
func resolveFileDatabaseDSN(config connection.ConnectionConfig) string {
|
||||
dsn := strings.TrimSpace(config.Host)
|
||||
if dsn == "" {
|
||||
dsn = strings.TrimSpace(config.Database)
|
||||
}
|
||||
if dsn == "" {
|
||||
dsn = ":memory:"
|
||||
}
|
||||
return dsn
|
||||
}
|
||||
|
||||
// Helper: Generate a unique key for the connection config
|
||||
func getCacheKey(config connection.ConnectionConfig) string {
|
||||
normalized := normalizeCacheKeyConfig(config)
|
||||
b, _ := json.Marshal(normalized)
|
||||
sum := sha256.Sum256(b)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -235,12 +286,19 @@ func (a *App) openDatabaseIsolated(config connection.ConnectionConfig) (db.Datab
|
||||
|
||||
func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing bool) (db.Database, error) {
|
||||
effectiveConfig := applyGlobalProxyToConnection(config)
|
||||
isFileDB := isFileDatabaseType(effectiveConfig.Type)
|
||||
|
||||
key := getCacheKey(effectiveConfig)
|
||||
shortKey := key
|
||||
if len(shortKey) > 12 {
|
||||
shortKey = shortKey[:12]
|
||||
}
|
||||
if isFileDB {
|
||||
rawDSN := resolveFileDatabaseDSN(effectiveConfig)
|
||||
normalizedDSN := resolveFileDatabaseDSN(normalizeCacheKeyConfig(effectiveConfig))
|
||||
logger.Infof("文件库连接缓存探测:类型=%s 原始DSN=%s 归一化DSN=%s timeout=%ds forcePing=%t 缓存Key=%s",
|
||||
strings.TrimSpace(effectiveConfig.Type), rawDSN, normalizedDSN, effectiveConfig.Timeout, forcePing, shortKey)
|
||||
}
|
||||
|
||||
if supported, reason := db.DriverRuntimeSupportStatus(effectiveConfig.Type); !supported {
|
||||
if strings.TrimSpace(reason) == "" {
|
||||
@@ -260,6 +318,9 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
|
||||
entry, ok := a.dbCache[key]
|
||||
a.mu.RUnlock()
|
||||
if ok {
|
||||
if isFileDB {
|
||||
logger.Infof("命中文件库连接缓存:类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
|
||||
}
|
||||
needPing := forcePing
|
||||
if !needPing {
|
||||
lastPing := entry.lastPing
|
||||
@@ -269,6 +330,9 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
|
||||
}
|
||||
|
||||
if !needPing {
|
||||
if isFileDB {
|
||||
logger.Infof("复用文件库连接缓存(免 Ping):类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
|
||||
}
|
||||
return entry.inst, nil
|
||||
}
|
||||
|
||||
@@ -280,6 +344,9 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
|
||||
a.dbCache[key] = cur
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if isFileDB {
|
||||
logger.Infof("复用文件库连接缓存(Ping 成功):类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
|
||||
}
|
||||
return entry.inst, nil
|
||||
} else {
|
||||
logger.Error(err, "缓存连接不可用,准备重建:%s 缓存Key=%s", formatConnSummary(effectiveConfig), shortKey)
|
||||
@@ -294,6 +361,12 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
|
||||
delete(a.dbCache, key)
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if isFileDB {
|
||||
logger.Infof("文件库缓存连接已剔除,准备新建连接:类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
|
||||
}
|
||||
}
|
||||
if isFileDB {
|
||||
logger.Infof("未命中文件库连接缓存,开始创建连接:类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
|
||||
}
|
||||
|
||||
logger.Infof("获取数据库连接:%s 缓存Key=%s", formatConnSummary(effectiveConfig), shortKey)
|
||||
@@ -324,6 +397,9 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
|
||||
a.mu.Unlock()
|
||||
// Prefer existing cached connection to avoid cache racing duplicates.
|
||||
_ = dbInst.Close()
|
||||
if isFileDB {
|
||||
logger.Infof("并发创建命中已存在文件库连接,关闭新建连接并复用缓存:类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
|
||||
}
|
||||
return existing.inst, nil
|
||||
}
|
||||
a.dbCache[key] = cachedDatabase{inst: dbInst, lastPing: now}
|
||||
|
||||
63
internal/app/app_cache_key_test.go
Normal file
63
internal/app/app_cache_key_test.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"GoNavi-Wails/internal/connection"
|
||||
)
|
||||
|
||||
func TestGetCacheKey_IgnoreTimeout(t *testing.T) {
|
||||
base := connection.ConnectionConfig{
|
||||
Type: "duckdb",
|
||||
Host: `C:\data\songs.duckdb`,
|
||||
Timeout: 30,
|
||||
UseProxy: false,
|
||||
UseSSH: false,
|
||||
}
|
||||
modified := base
|
||||
modified.Timeout = 120
|
||||
|
||||
left := getCacheKey(base)
|
||||
right := getCacheKey(modified)
|
||||
if left != right {
|
||||
t.Fatalf("expected same cache key when only timeout differs, got %s vs %s", left, right)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetCacheKey_DuckDBHostAndDatabaseEquivalent(t *testing.T) {
|
||||
withHost := connection.ConnectionConfig{
|
||||
Type: "duckdb",
|
||||
Host: `D:\music\songs.duckdb`,
|
||||
}
|
||||
withDatabase := connection.ConnectionConfig{
|
||||
Type: "duckdb",
|
||||
Database: `D:\music\songs.duckdb`,
|
||||
}
|
||||
|
||||
left := getCacheKey(withHost)
|
||||
right := getCacheKey(withDatabase)
|
||||
if left != right {
|
||||
t.Fatalf("expected same cache key for duckdb host/database path, got %s vs %s", left, right)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetCacheKey_KeepDatabaseIsolation(t *testing.T) {
|
||||
a := connection.ConnectionConfig{
|
||||
Type: "mysql",
|
||||
Host: "127.0.0.1",
|
||||
Port: 3306,
|
||||
User: "root",
|
||||
Password: "root",
|
||||
Database: "db_a",
|
||||
Timeout: 30,
|
||||
}
|
||||
b := a
|
||||
b.Database = "db_b"
|
||||
b.Timeout = 5
|
||||
|
||||
left := getCacheKey(a)
|
||||
right := getCacheKey(b)
|
||||
if left == right {
|
||||
t.Fatalf("expected different cache key for different database targets")
|
||||
}
|
||||
}
|
||||
@@ -218,7 +218,7 @@ const builtinDriverManifestJSON = `{
|
||||
"sphinx": { "engine": "go", "version": "1.9.3", "checksumPolicy": "off", "downloadUrl": "builtin://activate/sphinx" },
|
||||
"sqlserver": { "engine": "go", "version": "1.9.6", "checksumPolicy": "off", "downloadUrl": "builtin://activate/sqlserver" },
|
||||
"sqlite": { "engine": "go", "version": "1.44.3", "checksumPolicy": "off", "downloadUrl": "builtin://activate/sqlite" },
|
||||
"duckdb": { "engine": "go", "version": "2.5.5", "checksumPolicy": "off", "downloadUrl": "builtin://activate/duckdb" },
|
||||
"duckdb": { "engine": "go", "version": "2.5.6", "checksumPolicy": "off", "downloadUrl": "builtin://activate/duckdb" },
|
||||
"dameng": { "engine": "go", "version": "1.8.22", "checksumPolicy": "off", "downloadUrl": "builtin://activate/dameng" },
|
||||
"kingbase": { "engine": "go", "version": "0.0.0-20201021123113-29bd62a876c3", "checksumPolicy": "off", "downloadUrl": "builtin://activate/kingbase" },
|
||||
"highgo": { "engine": "go", "version": "0.0.0-local", "checksumPolicy": "off", "downloadUrl": "builtin://activate/highgo" },
|
||||
@@ -271,7 +271,7 @@ var latestDriverVersionMap = map[string]string{
|
||||
"sphinx": "1.9.3",
|
||||
"sqlserver": "1.9.6",
|
||||
"sqlite": "1.46.1",
|
||||
"duckdb": "2.5.5",
|
||||
"duckdb": "2.5.6",
|
||||
"dameng": "1.8.22",
|
||||
"kingbase": "0.0.0-20201021123113-29bd62a876c3",
|
||||
"highgo": "0.0.0-local",
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -120,6 +121,78 @@ func (a *App) SelectSSHKeyFile(currentPath string) connection.QueryResult {
|
||||
return connection.QueryResult{Success: true, Data: map[string]interface{}{"path": selection}}
|
||||
}
|
||||
|
||||
func (a *App) SelectDatabaseFile(currentPath string, driverType string) connection.QueryResult {
|
||||
defaultDir := strings.TrimSpace(currentPath)
|
||||
if defaultDir == "" {
|
||||
if home, err := os.UserHomeDir(); err == nil {
|
||||
defaultDir = home
|
||||
}
|
||||
}
|
||||
if filepath.Ext(defaultDir) != "" {
|
||||
defaultDir = filepath.Dir(defaultDir)
|
||||
}
|
||||
if defaultDir != "" && !filepath.IsAbs(defaultDir) {
|
||||
if abs, err := filepath.Abs(defaultDir); err == nil {
|
||||
defaultDir = abs
|
||||
}
|
||||
}
|
||||
|
||||
normalizedType := strings.ToLower(strings.TrimSpace(driverType))
|
||||
filters := []runtime.FileFilter{
|
||||
{
|
||||
DisplayName: "数据库文件",
|
||||
Pattern: "*.db;*.sqlite;*.sqlite3;*.db3;*.duckdb;*.ddb",
|
||||
},
|
||||
{
|
||||
DisplayName: "所有文件",
|
||||
Pattern: "*",
|
||||
},
|
||||
}
|
||||
title := "选择数据库文件"
|
||||
switch normalizedType {
|
||||
case "sqlite":
|
||||
title = "选择 SQLite 数据文件"
|
||||
filters = []runtime.FileFilter{
|
||||
{
|
||||
DisplayName: "SQLite 文件",
|
||||
Pattern: "*.db;*.sqlite;*.sqlite3;*.db3",
|
||||
},
|
||||
{
|
||||
DisplayName: "所有文件",
|
||||
Pattern: "*",
|
||||
},
|
||||
}
|
||||
case "duckdb":
|
||||
title = "选择 DuckDB 数据文件"
|
||||
filters = []runtime.FileFilter{
|
||||
{
|
||||
DisplayName: "DuckDB 文件",
|
||||
Pattern: "*.duckdb;*.ddb;*.db",
|
||||
},
|
||||
{
|
||||
DisplayName: "所有文件",
|
||||
Pattern: "*",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
selection, err := runtime.OpenFileDialog(a.ctx, runtime.OpenDialogOptions{
|
||||
Title: title,
|
||||
DefaultDirectory: defaultDir,
|
||||
Filters: filters,
|
||||
})
|
||||
if err != nil {
|
||||
return connection.QueryResult{Success: false, Message: err.Error()}
|
||||
}
|
||||
if strings.TrimSpace(selection) == "" {
|
||||
return connection.QueryResult{Success: false, Message: "Cancelled"}
|
||||
}
|
||||
if abs, err := filepath.Abs(selection); err == nil {
|
||||
selection = abs
|
||||
}
|
||||
return connection.QueryResult{Success: true, Data: map[string]interface{}{"path": selection}}
|
||||
}
|
||||
|
||||
// PreviewImportFile 解析导入文件,返回字段列表、总行数、前 5 行预览数据
|
||||
func (a *App) PreviewImportFile(filePath string) connection.QueryResult {
|
||||
if filePath == "" {
|
||||
@@ -1527,7 +1600,11 @@ func writeRowsToFile(f *os.File, data []map[string]interface{}, columns []string
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := jsonEncoder.Encode(rowMap); err != nil {
|
||||
exportedRow := make(map[string]interface{}, len(columns))
|
||||
for _, col := range columns {
|
||||
exportedRow[col] = normalizeExportJSONValue(rowMap[col])
|
||||
}
|
||||
if err := jsonEncoder.Encode(exportedRow); err != nil {
|
||||
return err
|
||||
}
|
||||
isJsonFirstRow = false
|
||||
@@ -1567,11 +1644,102 @@ func formatExportCellText(val interface{}) string {
|
||||
return "NULL"
|
||||
}
|
||||
return v.Format("2006-01-02 15:04:05")
|
||||
case float32:
|
||||
f := float64(v)
|
||||
if math.IsNaN(f) || math.IsInf(f, 0) {
|
||||
return "NULL"
|
||||
}
|
||||
return strconv.FormatFloat(f, 'f', -1, 32)
|
||||
case float64:
|
||||
if math.IsNaN(v) || math.IsInf(v, 0) {
|
||||
return "NULL"
|
||||
}
|
||||
return strconv.FormatFloat(v, 'f', -1, 64)
|
||||
case json.Number:
|
||||
text := strings.TrimSpace(v.String())
|
||||
if text == "" {
|
||||
return "NULL"
|
||||
}
|
||||
return text
|
||||
default:
|
||||
return fmt.Sprintf("%v", val)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeExportJSONValue(val interface{}) interface{} {
|
||||
if val == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch v := val.(type) {
|
||||
case float32:
|
||||
f := float64(v)
|
||||
if math.IsNaN(f) || math.IsInf(f, 0) {
|
||||
return nil
|
||||
}
|
||||
return json.Number(strconv.FormatFloat(f, 'f', -1, 32))
|
||||
case float64:
|
||||
if math.IsNaN(v) || math.IsInf(v, 0) {
|
||||
return nil
|
||||
}
|
||||
return json.Number(strconv.FormatFloat(v, 'f', -1, 64))
|
||||
case json.Number:
|
||||
text := strings.TrimSpace(v.String())
|
||||
if text == "" {
|
||||
return nil
|
||||
}
|
||||
return json.Number(text)
|
||||
case map[string]interface{}:
|
||||
out := make(map[string]interface{}, len(v))
|
||||
for key, item := range v {
|
||||
out[key] = normalizeExportJSONValue(item)
|
||||
}
|
||||
return out
|
||||
case []interface{}:
|
||||
items := make([]interface{}, len(v))
|
||||
for i, item := range v {
|
||||
items[i] = normalizeExportJSONValue(item)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
rv := reflect.ValueOf(val)
|
||||
switch rv.Kind() {
|
||||
case reflect.Pointer, reflect.Interface:
|
||||
if rv.IsNil() {
|
||||
return nil
|
||||
}
|
||||
return normalizeExportJSONValue(rv.Elem().Interface())
|
||||
case reflect.Map:
|
||||
if rv.IsNil() {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]interface{}, rv.Len())
|
||||
iter := rv.MapRange()
|
||||
for iter.Next() {
|
||||
out[fmt.Sprint(iter.Key().Interface())] = normalizeExportJSONValue(iter.Value().Interface())
|
||||
}
|
||||
return out
|
||||
case reflect.Slice:
|
||||
if rv.IsNil() {
|
||||
return nil
|
||||
}
|
||||
if rv.Type().Elem().Kind() == reflect.Uint8 {
|
||||
return val
|
||||
}
|
||||
fallthrough
|
||||
case reflect.Array:
|
||||
size := rv.Len()
|
||||
items := make([]interface{}, size)
|
||||
for i := 0; i < size; i++ {
|
||||
items[i] = normalizeExportJSONValue(rv.Index(i).Interface())
|
||||
}
|
||||
return items
|
||||
default:
|
||||
return val
|
||||
}
|
||||
}
|
||||
|
||||
// writeRowsToXlsx 使用 excelize 写入真正的 xlsx 格式文件
|
||||
func writeRowsToXlsx(filename string, data []map[string]interface{}, columns []string) error {
|
||||
xlsx := excelize.NewFile()
|
||||
|
||||
89
internal/app/methods_file_export_test.go
Normal file
89
internal/app/methods_file_export_test.go
Normal file
@@ -0,0 +1,89 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFormatExportCellText_FloatNoScientificNotation(t *testing.T) {
|
||||
got := formatExportCellText(1.445663e+06)
|
||||
if strings.Contains(strings.ToLower(got), "e+") || strings.Contains(strings.ToLower(got), "e-") {
|
||||
t.Fatalf("不应输出科学计数法,got=%q", got)
|
||||
}
|
||||
if got != "1445663" {
|
||||
t.Fatalf("浮点整值导出异常,want=%q got=%q", "1445663", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteRowsToFile_Markdown_NumberKeepPlainText(t *testing.T) {
|
||||
f, err := os.CreateTemp("", "gonavi-export-*.md")
|
||||
if err != nil {
|
||||
t.Fatalf("创建临时文件失败: %v", err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer f.Close()
|
||||
|
||||
data := []map[string]interface{}{
|
||||
{"id": 1.445663e+06},
|
||||
}
|
||||
columns := []string{"id"}
|
||||
|
||||
if err := writeRowsToFile(f, data, columns, "md"); err != nil {
|
||||
t.Fatalf("写入 md 失败: %v", err)
|
||||
}
|
||||
|
||||
contentBytes, err := os.ReadFile(f.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("读取 md 失败: %v", err)
|
||||
}
|
||||
content := string(contentBytes)
|
||||
if strings.Contains(strings.ToLower(content), "e+") || strings.Contains(strings.ToLower(content), "e-") {
|
||||
t.Fatalf("md 导出包含科学计数法: %s", content)
|
||||
}
|
||||
if !strings.Contains(content, "| 1445663 |") {
|
||||
t.Fatalf("md 导出未保留整数字面量,content=%s", content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteRowsToFile_JSON_NumberKeepPlainText(t *testing.T) {
|
||||
f, err := os.CreateTemp("", "gonavi-export-*.json")
|
||||
if err != nil {
|
||||
t.Fatalf("创建临时文件失败: %v", err)
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer f.Close()
|
||||
|
||||
data := []map[string]interface{}{
|
||||
{"id": 1.445663e+06},
|
||||
}
|
||||
columns := []string{"id"}
|
||||
|
||||
if err := writeRowsToFile(f, data, columns, "json"); err != nil {
|
||||
t.Fatalf("写入 json 失败: %v", err)
|
||||
}
|
||||
|
||||
contentBytes, err := os.ReadFile(f.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("读取 json 失败: %v", err)
|
||||
}
|
||||
content := string(contentBytes)
|
||||
if strings.Contains(strings.ToLower(content), "e+") || strings.Contains(strings.ToLower(content), "e-") {
|
||||
t.Fatalf("json 导出包含科学计数法: %s", content)
|
||||
}
|
||||
|
||||
var decoded []map[string]json.Number
|
||||
decoder := json.NewDecoder(bytes.NewReader(contentBytes))
|
||||
decoder.UseNumber()
|
||||
if err := decoder.Decode(&decoded); err != nil {
|
||||
t.Fatalf("解析导出 json 失败: %v", err)
|
||||
}
|
||||
if len(decoded) != 1 {
|
||||
t.Fatalf("导出行数异常,got=%d", len(decoded))
|
||||
}
|
||||
if decoded[0]["id"].String() != "1445663" {
|
||||
t.Fatalf("json 数值格式异常,want=1445663 got=%s", decoded[0]["id"].String())
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package db
|
||||
import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
@@ -18,7 +19,70 @@ func normalizeQueryValueWithDBType(v interface{}, databaseTypeName string) inter
|
||||
if b, ok := v.([]byte); ok {
|
||||
return bytesToDisplayValue(b, databaseTypeName)
|
||||
}
|
||||
return v
|
||||
return normalizeCompositeQueryValue(v)
|
||||
}
|
||||
|
||||
func normalizeCompositeQueryValue(v interface{}) interface{} {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
switch typed := v.(type) {
|
||||
case []interface{}:
|
||||
items := make([]interface{}, len(typed))
|
||||
for i, item := range typed {
|
||||
items[i] = normalizeQueryValue(item)
|
||||
}
|
||||
return items
|
||||
case map[string]interface{}:
|
||||
out := make(map[string]interface{}, len(typed))
|
||||
for key, value := range typed {
|
||||
out[key] = normalizeQueryValue(value)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
rv := reflect.ValueOf(v)
|
||||
switch rv.Kind() {
|
||||
case reflect.Pointer:
|
||||
if rv.IsNil() {
|
||||
return nil
|
||||
}
|
||||
return normalizeQueryValue(rv.Elem().Interface())
|
||||
case reflect.Map:
|
||||
if rv.IsNil() {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]interface{}, rv.Len())
|
||||
iter := rv.MapRange()
|
||||
for iter.Next() {
|
||||
out[mapKeyToString(iter.Key().Interface())] = normalizeQueryValue(iter.Value().Interface())
|
||||
}
|
||||
return out
|
||||
case reflect.Slice, reflect.Array:
|
||||
// []byte 在上层已单独处理,这里保留对其它切片/数组的递归规整。
|
||||
if rv.Kind() == reflect.Slice && rv.IsNil() {
|
||||
return nil
|
||||
}
|
||||
size := rv.Len()
|
||||
items := make([]interface{}, size)
|
||||
for i := 0; i < size; i++ {
|
||||
items[i] = normalizeQueryValue(rv.Index(i).Interface())
|
||||
}
|
||||
return items
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
func mapKeyToString(key interface{}) string {
|
||||
if key == nil {
|
||||
return "null"
|
||||
}
|
||||
if s, ok := key.(string); ok {
|
||||
return s
|
||||
}
|
||||
return fmt.Sprintf("%v", key)
|
||||
}
|
||||
|
||||
func bytesToDisplayValue(b []byte, databaseTypeName string) interface{} {
|
||||
|
||||
@@ -2,6 +2,8 @@ package db
|
||||
|
||||
import "testing"
|
||||
|
||||
type duckMapLike map[any]any
|
||||
|
||||
func TestNormalizeQueryValueWithDBType_BitBytes(t *testing.T) {
|
||||
v := normalizeQueryValueWithDBType([]byte{0x00}, "BIT")
|
||||
if v != int64(0) {
|
||||
@@ -42,3 +44,40 @@ func TestNormalizeQueryValueWithDBType_ByteFallbacks(t *testing.T) {
|
||||
t.Fatalf("未知类型 0xff 期望返回 0xff,实际=%v(%T)", v, v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeQueryValueWithDBType_MapAnyAnyForJSON(t *testing.T) {
|
||||
input := duckMapLike{
|
||||
"id": int64(1),
|
||||
1: "one",
|
||||
true: []interface{}{duckMapLike{2: "two"}},
|
||||
"bytes": []byte("ok"),
|
||||
}
|
||||
|
||||
v := normalizeQueryValueWithDBType(input, "")
|
||||
root, ok := v.(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("期望转换为 map[string]interface{},实际=%T", v)
|
||||
}
|
||||
|
||||
if root["id"] != int64(1) {
|
||||
t.Fatalf("id 字段异常,实际=%v(%T)", root["id"], root["id"])
|
||||
}
|
||||
if root["1"] != "one" {
|
||||
t.Fatalf("数字 key 未被字符串化,实际=%v(%T)", root["1"], root["1"])
|
||||
}
|
||||
if root["bytes"] != "ok" {
|
||||
t.Fatalf("嵌套 []byte 未被转换,实际=%v(%T)", root["bytes"], root["bytes"])
|
||||
}
|
||||
|
||||
arr, ok := root["true"].([]interface{})
|
||||
if !ok || len(arr) != 1 {
|
||||
t.Fatalf("bool key 下的数组结构异常,实际=%v(%T)", root["true"], root["true"])
|
||||
}
|
||||
nested, ok := arr[0].(map[string]interface{})
|
||||
if !ok {
|
||||
t.Fatalf("嵌套 map 未被转换,实际=%v(%T)", arr[0], arr[0])
|
||||
}
|
||||
if nested["2"] != "two" {
|
||||
t.Fatalf("嵌套 map 数字 key 未转换,实际=%v(%T)", nested["2"], nested["2"])
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user