🐛 fix(data-viewer): 修复ClickHouse尾部分页异常并增强DuckDB复杂类型兼容

- DataViewer 新增 ClickHouse 反向分页策略,修复最后页与倒数页查询失败
- DuckDB 查询失败时按列类型生成安全 SELECT,复杂类型转 VARCHAR 重试
- 分页状态统一使用 currentPage 回填,避免页码与总数推导不一致
- 增强查询异常日志与重试路径,降低大表场景卡顿与误报
This commit is contained in:
Syngnat
2026-03-02 10:49:23 +08:00
parent ec59023736
commit 26b79adc5f
15 changed files with 853 additions and 56 deletions

View File

@@ -131,6 +131,24 @@ jobs:
- name: Install Wails
run: go install -v github.com/wailsapp/wails/v2/cmd/wails@latest
- name: Prepare MinGW For DuckDB (Windows)
if: ${{ matrix.build_optional_agents && contains(matrix.platform, 'windows') }}
shell: pwsh
run: |
$mingwBin = "C:\msys64\mingw64\bin"
if (!(Test-Path $mingwBin)) {
choco install mingw --yes --no-progress
$mingwBin = "C:\ProgramData\chocolatey\lib\mingw\tools\install\mingw64\bin"
}
if (!(Test-Path $mingwBin)) {
Write-Error "❌ 未找到 MinGW GCC 路径:$mingwBin"
exit 1
}
"$mingwBin" | Out-File -FilePath $env:GITHUB_PATH -Append -Encoding utf8
"CC=$mingwBin\gcc.exe" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
"CXX=$mingwBin\g++.exe" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
Write-Host "✅ 已配置 DuckDB cgo 编译器: $mingwBin"
- name: Build
shell: bash
run: |
@@ -166,20 +184,12 @@ jobs:
OUTPUT_PATH="${OUTDIR}/${OUTPUT}"
echo "🔧 构建 ${OUTPUT_PATH} (tag=${TAG})"
if [ "$DRIVER" = "duckdb" ]; then
set +e
CGO_ENABLED=1 GOOS="$GOOS" GOARCH="$GOARCH" go build \
-tags "${TAG}" \
-trimpath \
-ldflags "-s -w" \
-o "${OUTPUT_PATH}" \
./cmd/optional-driver-agent
DUCKDB_RC=$?
set -e
if [ "${DUCKDB_RC}" -ne 0 ]; then
echo "⚠️ DuckDB 代理构建失败(平台 ${GOOS}/${GOARCH}),跳过该资产,不阻断发布"
rm -f "${OUTPUT_PATH}"
continue
fi
else
CGO_ENABLED=0 GOOS="$GOOS" GOARCH="$GOARCH" go build \
-tags "${TAG}" \
@@ -369,6 +379,34 @@ jobs:
- name: List Assets
run: ls -R release-assets
- name: Verify DuckDB Driver Assets
shell: bash
run: |
set -euo pipefail
cd release-assets
REQUIRED_FILES=(
"drivers/Windows/duckdb-driver-agent-windows-amd64.exe"
"drivers/MacOS/duckdb-driver-agent-darwin-amd64"
"drivers/MacOS/duckdb-driver-agent-darwin-arm64"
"drivers/Linux/duckdb-driver-agent-linux-amd64"
)
missing=0
for file in "${REQUIRED_FILES[@]}"; do
if [ ! -f "$file" ]; then
echo "❌ 缺少 DuckDB 驱动资产:$file"
missing=1
else
echo "✅ 已找到 DuckDB 驱动资产:$file"
fi
done
if [ "$missing" -ne 0 ]; then
echo "❌ DuckDB 驱动资产不完整,终止发布"
exit 1
fi
- name: Package Driver Agents Bundle
shell: bash
run: |

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"os"
"reflect"
"strings"
"GoNavi-Wails/internal/connection"
@@ -218,7 +219,11 @@ func handleRequest(inst *db.Database, req agentRequest) agentResponse {
}
func writeResponse(writer *bufio.Writer, resp agentResponse) error {
payload, err := json.Marshal(resp)
// 对响应数据做统一 JSON 安全归一化:
// 将 map[any]any如 duckdb.Map递归转换为 map[string]any避免序列化失败导致代理进程退出。
safeResp := resp
safeResp.Data = normalizeAgentResponseData(resp.Data)
payload, err := json.Marshal(safeResp)
if err != nil {
return err
}
@@ -234,3 +239,51 @@ func fail(resp agentResponse, errText string) agentResponse {
resp.Error = strings.TrimSpace(errText)
return resp
}
func normalizeAgentResponseData(v interface{}) interface{} {
if v == nil {
return nil
}
rv := reflect.ValueOf(v)
switch rv.Kind() {
case reflect.Pointer, reflect.Interface:
if rv.IsNil() {
return nil
}
return normalizeAgentResponseData(rv.Elem().Interface())
case reflect.Map:
if rv.IsNil() {
return nil
}
out := make(map[string]interface{}, rv.Len())
iter := rv.MapRange()
for iter.Next() {
out[fmt.Sprint(iter.Key().Interface())] = normalizeAgentResponseData(iter.Value().Interface())
}
return out
case reflect.Slice:
if rv.IsNil() {
return nil
}
// 保持 []byte 原样,避免改变现有二进制列的 JSON 编码行为base64
if rv.Type().Elem().Kind() == reflect.Uint8 {
return v
}
size := rv.Len()
items := make([]interface{}, size)
for i := 0; i < size; i++ {
items[i] = normalizeAgentResponseData(rv.Index(i).Interface())
}
return items
case reflect.Array:
size := rv.Len()
items := make([]interface{}, size)
for i := 0; i < size; i++ {
items[i] = normalizeAgentResponseData(rv.Index(i).Interface())
}
return items
default:
return v
}
}

View File

@@ -0,0 +1,62 @@
package main
import (
"bufio"
"bytes"
"encoding/json"
"testing"
)
type duckMapLike map[any]any
func TestWriteResponse_NormalizesMapAnyAny(t *testing.T) {
resp := agentResponse{
ID: 1,
Success: true,
Data: []map[string]interface{}{
{
"id": int64(7),
"meta": duckMapLike{"k": "v", 2: "two"},
},
},
}
var out bytes.Buffer
writer := bufio.NewWriter(&out)
if err := writeResponse(writer, resp); err != nil {
t.Fatalf("writeResponse 返回错误: %v", err)
}
var decoded struct {
Data []map[string]interface{} `json:"data"`
}
if err := json.Unmarshal(bytes.TrimSpace(out.Bytes()), &decoded); err != nil {
t.Fatalf("解码响应失败: %v", err)
}
if len(decoded.Data) != 1 {
t.Fatalf("期望 1 行数据,实际 %d", len(decoded.Data))
}
meta, ok := decoded.Data[0]["meta"].(map[string]interface{})
if !ok {
t.Fatalf("meta 字段类型异常: %T", decoded.Data[0]["meta"])
}
if meta["k"] != "v" {
t.Fatalf("字符串 key 转换异常: %v", meta["k"])
}
if meta["2"] != "two" {
t.Fatalf("数字 key 未字符串化: %v", meta["2"])
}
}
func TestNormalizeAgentResponseData_KeepByteSlice(t *testing.T) {
raw := []byte{0x61, 0x62, 0x63}
normalized := normalizeAgentResponseData(raw)
out, ok := normalized.([]byte)
if !ok {
t.Fatalf("期望 []byte实际 %T", normalized)
}
if !bytes.Equal(out, raw) {
t.Fatalf("[]byte 内容被意外改写: %v", out)
}
}

View File

@@ -33,7 +33,7 @@
},
"duckdb": {
"engine": "go",
"version": "2.5.5",
"version": "2.5.6",
"checksumPolicy": "off",
"downloadUrl": "builtin://activate/duckdb"
},

View File

@@ -3,7 +3,7 @@ import { Modal, Form, Input, InputNumber, Button, message, Checkbox, Divider, Se
import { DatabaseOutlined, ConsoleSqlOutlined, FileTextOutlined, CloudServerOutlined, AppstoreAddOutlined, CloudOutlined, CheckCircleFilled, CloseCircleFilled } from '@ant-design/icons';
import { useStore } from '../store';
import { normalizeOpacityForPlatform } from '../utils/appearance';
import { DBGetDatabases, GetDriverStatusList, MongoDiscoverMembers, TestConnection, RedisConnect, SelectSSHKeyFile } from '../../wailsjs/go/app/App';
import { DBGetDatabases, GetDriverStatusList, MongoDiscoverMembers, TestConnection, RedisConnect, SelectDatabaseFile, SelectSSHKeyFile } from '../../wailsjs/go/app/App';
import { ConnectionConfig, MongoMemberInfo, SavedConnection } from '../types';
const { Meta } = Card;
@@ -80,6 +80,7 @@ const ConnectionModal: React.FC<{
const [typeSelectWarning, setTypeSelectWarning] = useState<{ driverName: string; reason: string } | null>(null);
const [driverStatusMap, setDriverStatusMap] = useState<Record<string, DriverStatusSnapshot>>({});
const [driverStatusLoaded, setDriverStatusLoaded] = useState(false);
const [selectingDbFile, setSelectingDbFile] = useState(false);
const [selectingSSHKey, setSelectingSSHKey] = useState(false);
const testInFlightRef = useRef(false);
const testTimerRef = useRef<number | null>(null);
@@ -665,6 +666,30 @@ const ConnectionModal: React.FC<{
}
};
const handleSelectDatabaseFile = async () => {
if (selectingDbFile) {
return;
}
try {
setSelectingDbFile(true);
const currentPath = String(form.getFieldValue('host') || '').trim();
const res = await SelectDatabaseFile(currentPath, dbType);
if (res?.success) {
const data = res.data || {};
const selectedPath = typeof data === 'string' ? data : String(data.path || '').trim();
if (selectedPath) {
form.setFieldValue('host', normalizeFileDbPath(selectedPath));
}
} else if (res?.message !== 'Cancelled') {
message.error(`选择数据库文件失败: ${res?.message || '未知错误'}`);
}
} catch (e: any) {
message.error(`选择数据库文件失败: ${e?.message || String(e)}`);
} finally {
setSelectingDbFile(false);
}
};
useEffect(() => {
if (open) {
setTestResult(null); // Reset test result
@@ -1392,6 +1417,13 @@ const ConnectionModal: React.FC<{
onDoubleClick={requestTest}
/>
</Form.Item>
{isFileDb && (
<Form.Item label=" " style={{ width: 120 }}>
<Button style={{ width: '100%' }} onClick={handleSelectDatabaseFile} loading={selectingDbFile}>
...
</Button>
</Form.Item>
)}
{!isFileDb && (
<Form.Item
name="port"

View File

@@ -2,9 +2,9 @@ import React, { useEffect, useState, useCallback, useRef } from 'react';
import { message } from 'antd';
import { TabData, ColumnDefinition } from '../types';
import { useStore } from '../store';
import { DBQuery, DBGetColumns, DBQueryIsolated } from '../../wailsjs/go/app/App';
import { DBQuery, DBGetColumns } from '../../wailsjs/go/app/App';
import DataGrid, { GONAVI_ROW_KEY } from './DataGrid';
import { buildOrderBySQL, buildWhereSQL, quoteQualifiedIdent, withSortBufferTuningSQL, type FilterCondition } from '../utils/sql';
import { buildOrderBySQL, buildWhereSQL, quoteIdentPart, quoteQualifiedIdent, withSortBufferTuningSQL, type FilterCondition } from '../utils/sql';
type ViewerPaginationState = {
current: number;
@@ -108,6 +108,33 @@ const resolveDuckDBSchemaAndTable = (dbName: string, tableName: string) => {
const escapeSQLLiteral = (value: string): string => String(value || '').replace(/'/g, "''");
const isDuckDBUnsupportedTypeError = (msg: string): boolean => /unsupported\s*type:\s*duckdb\./i.test(String(msg || ''));
const isDuckDBComplexColumnType = (columnType?: string): boolean => {
const raw = String(columnType || '').trim().toLowerCase();
if (!raw) return false;
return raw.includes('map') || raw.includes('struct') || raw.includes('union') || raw.includes('array') || raw.includes('list');
};
const reverseOrderBySQL = (orderBySQL: string): string => {
const raw = String(orderBySQL || '').trim();
if (!raw) return '';
const body = raw.replace(/^order\s+by\s+/i, '').trim();
if (!body) return '';
const parts = body
.split(',')
.map((part) => part.trim())
.filter(Boolean)
.map((part) => {
if (/\s+asc$/i.test(part)) return part.replace(/\s+asc$/i, ' DESC');
if (/\s+desc$/i.test(part)) return part.replace(/\s+desc$/i, ' ASC');
return `${part} DESC`;
});
if (parts.length === 0) return '';
return ` ORDER BY ${parts.join(', ')}`;
};
const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
const [data, setData] = useState<any[]>([]);
const [columnNames, setColumnNames] = useState<string[]>([]);
@@ -144,19 +171,17 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
const [showFilter, setShowFilter] = useState(false);
const [filterConditions, setFilterConditions] = useState<FilterCondition[]>([]);
const duckdbSafeSelectCacheRef = useRef<Record<string, string>>({});
const currentConnType = (connections.find(c => c.id === tab.connectionId)?.config?.type || '').toLowerCase();
const forceReadOnly = currentConnType === 'tdengine' || currentConnType === 'clickhouse';
const runIsolatedQuery = useCallback(async (queryConfig: any, dbName: string, sql: string) => {
return DBQueryIsolated(queryConfig as any, dbName, sql);
}, []);
useEffect(() => {
setPkColumns([]);
pkKeyRef.current = '';
countKeyRef.current = '';
duckdbApproxKeyRef.current = '';
manualCountKeyRef.current = '';
duckdbSafeSelectCacheRef.current = {};
latestConfigRef.current = null;
latestDbTypeRef.current = '';
latestDbNameRef.current = '';
@@ -194,7 +219,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
const countConfig: any = { ...(config as any), timeout: 120 };
try {
const resCount = await runIsolatedQuery(countConfig, dbName, countSql);
const resCount = await DBQuery(countConfig as any, dbName, countSql);
const countDuration = Date.now() - countStart;
addSqlLog({
id: `log-${Date.now()}-duckdb-manual-count`,
@@ -240,7 +265,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
setPagination(prev => ({ ...prev, totalCountLoading: false }));
message.error(`统计总数失败: ${String(e?.message || e)}`);
}
}, [addSqlLog, runIsolatedQuery]);
}, [addSqlLog]);
const handleDuckDBCancelManualCount = useCallback(() => {
manualCountSeqRef.current++;
@@ -277,35 +302,112 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
const countSql = `SELECT COUNT(*) as total FROM ${quoteQualifiedIdent(dbType, tableName)} ${whereSQL}`;
let sql = `SELECT * FROM ${quoteQualifiedIdent(dbType, tableName)} ${whereSQL}`;
sql += buildOrderBySQL(dbType, sortInfo, pkColumns);
const offset = (page - 1) * size;
// 大表性能:打开表不阻塞在 COUNT(*),先通过多取 1 条判断是否还有下一页;总数在后台统计并异步回填。
sql += ` LIMIT ${size + 1} OFFSET ${offset}`;
const baseSql = `SELECT * FROM ${quoteQualifiedIdent(dbType, tableName)} ${whereSQL}`;
const orderBySQL = buildOrderBySQL(dbType, sortInfo, pkColumns);
let sql = `${baseSql}${orderBySQL}`;
const totalRows = Number(pagination.total);
const hasFiniteTotal = Number.isFinite(totalRows) && totalRows >= 0;
const totalKnown = pagination.totalKnown && hasFiniteTotal;
const totalPages = hasFiniteTotal ? Math.max(1, Math.ceil(totalRows / size)) : 0;
const currentPage = totalPages > 0 ? Math.min(Math.max(1, page), totalPages) : Math.max(1, page);
const offset = (currentPage - 1) * size;
const isClickHouse = dbTypeLower === 'clickhouse';
const reverseOrderSQL = isClickHouse ? reverseOrderBySQL(orderBySQL) : '';
let useClickHouseReversePagination = false;
let clickHouseReverseLimit = 0;
let clickHouseReverseHasMore = false;
// ClickHouse 深分页在超大 OFFSET 下容易超时。对于总数已知且存在 ORDER BY 的场景,
// 当“尾部偏移”小于“头部偏移”时,改为反向 ORDER BY + 小 OFFSET并在前端翻转结果。
if (isClickHouse && totalKnown && offset > 0 && reverseOrderSQL) {
const pageRowCount = Math.max(0, Math.min(size, totalRows - offset));
if (pageRowCount > 0) {
const tailOffset = Math.max(0, totalRows - (offset + pageRowCount));
if (tailOffset < offset) {
sql = `${baseSql}${reverseOrderSQL} LIMIT ${pageRowCount} OFFSET ${tailOffset}`;
useClickHouseReversePagination = true;
clickHouseReverseLimit = pageRowCount;
clickHouseReverseHasMore = currentPage < totalPages;
}
}
}
if (!useClickHouseReversePagination) {
// 大表性能:打开表不阻塞在 COUNT(*),先通过多取 1 条判断是否还有下一页;总数在后台统计并异步回填。
sql += ` LIMIT ${size + 1} OFFSET ${offset}`;
}
const requestStartTime = Date.now();
let executedSql = sql;
try {
const executeDataQuery = async (querySql: string, attemptLabel: string) => {
const startTime = Date.now();
const result = await DBQuery(config as any, dbName, querySql);
addSqlLog({
id: `log-${Date.now()}-data`,
timestamp: Date.now(),
sql: querySql,
status: result.success ? 'success' : 'error',
duration: Date.now() - startTime,
message: result.success ? '' : `${attemptLabel}: ${result.message}`,
affectedRows: Array.isArray(result.data) ? result.data.length : undefined,
dbName
});
return result;
try {
const result = await DBQuery(config as any, dbName, querySql);
addSqlLog({
id: `log-${Date.now()}-data`,
timestamp: Date.now(),
sql: querySql,
status: result.success ? 'success' : 'error',
duration: Date.now() - startTime,
message: result.success ? '' : `${attemptLabel}: ${result.message}`,
affectedRows: Array.isArray(result.data) ? result.data.length : undefined,
dbName
});
return result;
} catch (e: any) {
const errMessage = String(e?.message || e || 'query failed');
addSqlLog({
id: `log-${Date.now()}-data`,
timestamp: Date.now(),
sql: querySql,
status: 'error',
duration: Date.now() - startTime,
message: `${attemptLabel}: ${errMessage}`,
dbName
});
return { success: false, message: errMessage, data: [], fields: [] };
}
};
const hasSort = !!sortInfo?.columnKey && (sortInfo?.order === 'ascend' || sortInfo?.order === 'descend');
const isSortMemoryErr = (msg: string) => /error\s*1038|out of sort memory/i.test(String(msg || ''));
let resData = await executeDataQuery(sql, '主查询');
if (!resData.success && dbTypeLower === 'duckdb' && isDuckDBUnsupportedTypeError(String(resData.message || ''))) {
const cacheKey = `${tab.connectionId}|${dbName}|${tableName}`;
let safeSelect = duckdbSafeSelectCacheRef.current[cacheKey] || '';
if (!safeSelect) {
try {
const resCols = await DBGetColumns(config as any, dbName, tableName);
if (resCols?.success && Array.isArray(resCols.data)) {
const columnDefs = resCols.data as ColumnDefinition[];
const selectParts = columnDefs.map((col) => {
const colName = String(col?.name || '').trim();
if (!colName) return '';
const quotedCol = quoteIdentPart(dbType, colName);
if (isDuckDBComplexColumnType(col?.type)) {
return `CAST(${quotedCol} AS VARCHAR) AS ${quotedCol}`;
}
return quotedCol;
}).filter(Boolean);
if (selectParts.length > 0) {
safeSelect = selectParts.join(', ');
duckdbSafeSelectCacheRef.current[cacheKey] = safeSelect;
}
}
} catch {
// ignore and keep original error path
}
}
if (safeSelect) {
let fallbackSql = `SELECT ${safeSelect} FROM ${quoteQualifiedIdent(dbType, tableName)} ${whereSQL}`;
fallbackSql += buildOrderBySQL(dbType, sortInfo, pkColumns);
fallbackSql += ` LIMIT ${size + 1} OFFSET ${offset}`;
executedSql = fallbackSql;
resData = await executeDataQuery(fallbackSql, '复杂类型降级重试');
}
}
if (!resData.success && isMySQLFamily && hasSort && isSortMemoryErr(resData.message)) {
const retrySql32MB = withSortBufferTuningSQL(dbType, sql, 32 * 1024 * 1024);
if (retrySql32MB !== sql) {
@@ -348,7 +450,12 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
let resultData = resData.data as any[];
if (!Array.isArray(resultData)) resultData = [];
const hasMore = resultData.length > size;
if (useClickHouseReversePagination) {
// 反向查询后恢复为原排序方向,保证用户看到的仍是“最后一页正序数据”。
resultData = resultData.slice(0, clickHouseReverseLimit).reverse();
}
const hasMore = useClickHouseReversePagination ? clickHouseReverseHasMore : resultData.length > size;
if (hasMore) resultData = resultData.slice(0, size);
let fieldNames = resData.fields || [];
@@ -363,7 +470,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
setData(resultData);
const countKey = `${tab.connectionId}|${dbName}|${tableName}|${whereSQL}`;
const derivedTotalKnown = !hasMore;
const derivedTotal = derivedTotalKnown ? offset + resultData.length : page * size + 1;
const derivedTotal = derivedTotalKnown ? offset + resultData.length : currentPage * size + 1;
const isDuckDB = dbTypeLower === 'duckdb';
const minExpectedTotal = hasMore ? offset + resultData.length + 1 : offset + resultData.length;
if (derivedTotalKnown) countKeyRef.current = countKey;
@@ -377,7 +484,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
if (derivedTotalKnown) {
return {
...prev,
current: page,
current: currentPage,
pageSize: size,
total: derivedTotal,
totalKnown: true,
@@ -388,19 +495,19 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
}
if (prev.totalKnown && countKeyRef.current === countKey) {
if (!isDuckDB) {
return { ...prev, current: page, pageSize: size };
return { ...prev, current: currentPage, pageSize: size };
}
// 当当前页存在“下一页”信号时,已知总数至少应大于当前页末尾。
// 若旧总数不满足该条件(例如历史统计值为 0降级为未知总数并回退到 derivedTotal。
if (Number.isFinite(prev.total) && prev.total >= minExpectedTotal) {
return { ...prev, current: page, pageSize: size };
return { ...prev, current: currentPage, pageSize: size };
}
}
const keepManualCounting = prev.totalCountLoading && manualCountKeyRef.current === countKey;
if (isDuckDB && prev.totalApprox && duckdbApproxKeyRef.current === countKey && Number.isFinite(prev.total) && prev.total >= minExpectedTotal) {
return {
...prev,
current: page,
current: currentPage,
pageSize: size,
totalKnown: false,
totalApprox: true,
@@ -410,7 +517,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
}
return {
...prev,
current: page,
current: currentPage,
pageSize: size,
total: derivedTotal,
totalKnown: false,
@@ -489,7 +596,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
(async () => {
for (const approxSql of approxSqlCandidates) {
try {
const approxRes = await runIsolatedQuery(approxConfig, dbName, approxSql);
const approxRes = await DBQuery(approxConfig as any, dbName, approxSql);
if (duckdbApproxSeqRef.current !== approxSeq) return;
if (countKeyRef.current !== countKey) return;
if (!approxRes?.success || !Array.isArray(approxRes.data) || approxRes.data.length === 0) continue;
@@ -534,7 +641,7 @@ const DataViewer: React.FC<{ tab: TabData }> = ({ tab }) => {
});
}
if (fetchSeqRef.current === seq) setLoading(false);
}, [connections, tab, sortInfo, filterConditions, pkColumns, runIsolatedQuery]);
}, [connections, tab, sortInfo, filterConditions, pkColumns, pagination.total, pagination.totalKnown]);
// 依赖 pkColumns在无手动排序时可回退到主键稳定排序。
// 主键信息只会在首次加载后更新一次,避免循环查询。

View File

@@ -164,6 +164,8 @@ export function ResolveDriverPackageDownloadURL(arg1:string,arg2:string):Promise
export function ResolveDriverRepositoryURL(arg1:string):Promise<connection.QueryResult>;
export function SelectDatabaseFile(arg1:string,arg2:string):Promise<connection.QueryResult>;
export function SelectDriverDownloadDirectory(arg1:string):Promise<connection.QueryResult>;
export function SelectDriverPackageDirectory(arg1:string):Promise<connection.QueryResult>;

View File

@@ -322,6 +322,10 @@ export function ResolveDriverRepositoryURL(arg1) {
return window['go']['app']['App']['ResolveDriverRepositoryURL'](arg1);
}
export function SelectDatabaseFile(arg1, arg2) {
return window['go']['app']['App']['SelectDatabaseFile'](arg1, arg2);
}
export function SelectDriverDownloadDirectory(arg1) {
return window['go']['app']['App']['SelectDriverDownloadDirectory'](arg1);
}

View File

@@ -74,16 +74,67 @@ func (a *App) Shutdown(ctx context.Context) {
logger.Close()
}
// Helper: Generate a unique key for the connection config
func getCacheKey(config connection.ConnectionConfig) string {
if !config.UseSSH {
config.SSH = connection.SSHConfig{}
func normalizeCacheKeyConfig(config connection.ConnectionConfig) connection.ConnectionConfig {
normalized := config
normalized.Type = strings.ToLower(strings.TrimSpace(normalized.Type))
// timeout 仅用于 Query/Ping 控制,不应作为物理连接复用键的一部分。
normalized.Timeout = 0
normalized.SavePassword = false
if !normalized.UseSSH {
normalized.SSH = connection.SSHConfig{}
}
if !config.UseProxy {
config.Proxy = connection.ProxyConfig{}
if !normalized.UseProxy {
normalized.Proxy = connection.ProxyConfig{}
}
b, _ := json.Marshal(config)
if isFileDatabaseType(normalized.Type) {
dsn := strings.TrimSpace(normalized.Host)
if dsn == "" {
dsn = strings.TrimSpace(normalized.Database)
}
if dsn == "" {
dsn = ":memory:"
}
// DuckDB/SQLite 仅基于文件来源识别连接,其他网络字段不参与键计算。
normalized.Host = dsn
normalized.Database = ""
normalized.Port = 0
normalized.User = ""
normalized.Password = ""
normalized.URI = ""
normalized.Hosts = nil
normalized.Topology = ""
normalized.MySQLReplicaUser = ""
normalized.MySQLReplicaPassword = ""
normalized.ReplicaSet = ""
normalized.AuthSource = ""
normalized.ReadPreference = ""
normalized.MongoSRV = false
normalized.MongoAuthMechanism = ""
normalized.MongoReplicaUser = ""
normalized.MongoReplicaPassword = ""
}
return normalized
}
func resolveFileDatabaseDSN(config connection.ConnectionConfig) string {
dsn := strings.TrimSpace(config.Host)
if dsn == "" {
dsn = strings.TrimSpace(config.Database)
}
if dsn == "" {
dsn = ":memory:"
}
return dsn
}
// Helper: Generate a unique key for the connection config
func getCacheKey(config connection.ConnectionConfig) string {
normalized := normalizeCacheKeyConfig(config)
b, _ := json.Marshal(normalized)
sum := sha256.Sum256(b)
return hex.EncodeToString(sum[:])
}
@@ -235,12 +286,19 @@ func (a *App) openDatabaseIsolated(config connection.ConnectionConfig) (db.Datab
func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing bool) (db.Database, error) {
effectiveConfig := applyGlobalProxyToConnection(config)
isFileDB := isFileDatabaseType(effectiveConfig.Type)
key := getCacheKey(effectiveConfig)
shortKey := key
if len(shortKey) > 12 {
shortKey = shortKey[:12]
}
if isFileDB {
rawDSN := resolveFileDatabaseDSN(effectiveConfig)
normalizedDSN := resolveFileDatabaseDSN(normalizeCacheKeyConfig(effectiveConfig))
logger.Infof("文件库连接缓存探测:类型=%s 原始DSN=%s 归一化DSN=%s timeout=%ds forcePing=%t 缓存Key=%s",
strings.TrimSpace(effectiveConfig.Type), rawDSN, normalizedDSN, effectiveConfig.Timeout, forcePing, shortKey)
}
if supported, reason := db.DriverRuntimeSupportStatus(effectiveConfig.Type); !supported {
if strings.TrimSpace(reason) == "" {
@@ -260,6 +318,9 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
entry, ok := a.dbCache[key]
a.mu.RUnlock()
if ok {
if isFileDB {
logger.Infof("命中文件库连接缓存:类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
}
needPing := forcePing
if !needPing {
lastPing := entry.lastPing
@@ -269,6 +330,9 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
}
if !needPing {
if isFileDB {
logger.Infof("复用文件库连接缓存(免 Ping类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
}
return entry.inst, nil
}
@@ -280,6 +344,9 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
a.dbCache[key] = cur
}
a.mu.Unlock()
if isFileDB {
logger.Infof("复用文件库连接缓存Ping 成功):类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
}
return entry.inst, nil
} else {
logger.Error(err, "缓存连接不可用,准备重建:%s 缓存Key=%s", formatConnSummary(effectiveConfig), shortKey)
@@ -294,6 +361,12 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
delete(a.dbCache, key)
}
a.mu.Unlock()
if isFileDB {
logger.Infof("文件库缓存连接已剔除,准备新建连接:类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
}
}
if isFileDB {
logger.Infof("未命中文件库连接缓存,开始创建连接:类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
}
logger.Infof("获取数据库连接:%s 缓存Key=%s", formatConnSummary(effectiveConfig), shortKey)
@@ -324,6 +397,9 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
a.mu.Unlock()
// Prefer existing cached connection to avoid cache racing duplicates.
_ = dbInst.Close()
if isFileDB {
logger.Infof("并发创建命中已存在文件库连接,关闭新建连接并复用缓存:类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
}
return existing.inst, nil
}
a.dbCache[key] = cachedDatabase{inst: dbInst, lastPing: now}

View File

@@ -0,0 +1,63 @@
package app
import (
"testing"
"GoNavi-Wails/internal/connection"
)
func TestGetCacheKey_IgnoreTimeout(t *testing.T) {
base := connection.ConnectionConfig{
Type: "duckdb",
Host: `C:\data\songs.duckdb`,
Timeout: 30,
UseProxy: false,
UseSSH: false,
}
modified := base
modified.Timeout = 120
left := getCacheKey(base)
right := getCacheKey(modified)
if left != right {
t.Fatalf("expected same cache key when only timeout differs, got %s vs %s", left, right)
}
}
func TestGetCacheKey_DuckDBHostAndDatabaseEquivalent(t *testing.T) {
withHost := connection.ConnectionConfig{
Type: "duckdb",
Host: `D:\music\songs.duckdb`,
}
withDatabase := connection.ConnectionConfig{
Type: "duckdb",
Database: `D:\music\songs.duckdb`,
}
left := getCacheKey(withHost)
right := getCacheKey(withDatabase)
if left != right {
t.Fatalf("expected same cache key for duckdb host/database path, got %s vs %s", left, right)
}
}
func TestGetCacheKey_KeepDatabaseIsolation(t *testing.T) {
a := connection.ConnectionConfig{
Type: "mysql",
Host: "127.0.0.1",
Port: 3306,
User: "root",
Password: "root",
Database: "db_a",
Timeout: 30,
}
b := a
b.Database = "db_b"
b.Timeout = 5
left := getCacheKey(a)
right := getCacheKey(b)
if left == right {
t.Fatalf("expected different cache key for different database targets")
}
}

View File

@@ -218,7 +218,7 @@ const builtinDriverManifestJSON = `{
"sphinx": { "engine": "go", "version": "1.9.3", "checksumPolicy": "off", "downloadUrl": "builtin://activate/sphinx" },
"sqlserver": { "engine": "go", "version": "1.9.6", "checksumPolicy": "off", "downloadUrl": "builtin://activate/sqlserver" },
"sqlite": { "engine": "go", "version": "1.44.3", "checksumPolicy": "off", "downloadUrl": "builtin://activate/sqlite" },
"duckdb": { "engine": "go", "version": "2.5.5", "checksumPolicy": "off", "downloadUrl": "builtin://activate/duckdb" },
"duckdb": { "engine": "go", "version": "2.5.6", "checksumPolicy": "off", "downloadUrl": "builtin://activate/duckdb" },
"dameng": { "engine": "go", "version": "1.8.22", "checksumPolicy": "off", "downloadUrl": "builtin://activate/dameng" },
"kingbase": { "engine": "go", "version": "0.0.0-20201021123113-29bd62a876c3", "checksumPolicy": "off", "downloadUrl": "builtin://activate/kingbase" },
"highgo": { "engine": "go", "version": "0.0.0-local", "checksumPolicy": "off", "downloadUrl": "builtin://activate/highgo" },
@@ -271,7 +271,7 @@ var latestDriverVersionMap = map[string]string{
"sphinx": "1.9.3",
"sqlserver": "1.9.6",
"sqlite": "1.46.1",
"duckdb": "2.5.5",
"duckdb": "2.5.6",
"dameng": "1.8.22",
"kingbase": "0.0.0-20201021123113-29bd62a876c3",
"highgo": "0.0.0-local",

View File

@@ -8,6 +8,7 @@ import (
"math"
"os"
"path/filepath"
"reflect"
"sort"
"strconv"
"strings"
@@ -120,6 +121,78 @@ func (a *App) SelectSSHKeyFile(currentPath string) connection.QueryResult {
return connection.QueryResult{Success: true, Data: map[string]interface{}{"path": selection}}
}
func (a *App) SelectDatabaseFile(currentPath string, driverType string) connection.QueryResult {
defaultDir := strings.TrimSpace(currentPath)
if defaultDir == "" {
if home, err := os.UserHomeDir(); err == nil {
defaultDir = home
}
}
if filepath.Ext(defaultDir) != "" {
defaultDir = filepath.Dir(defaultDir)
}
if defaultDir != "" && !filepath.IsAbs(defaultDir) {
if abs, err := filepath.Abs(defaultDir); err == nil {
defaultDir = abs
}
}
normalizedType := strings.ToLower(strings.TrimSpace(driverType))
filters := []runtime.FileFilter{
{
DisplayName: "数据库文件",
Pattern: "*.db;*.sqlite;*.sqlite3;*.db3;*.duckdb;*.ddb",
},
{
DisplayName: "所有文件",
Pattern: "*",
},
}
title := "选择数据库文件"
switch normalizedType {
case "sqlite":
title = "选择 SQLite 数据文件"
filters = []runtime.FileFilter{
{
DisplayName: "SQLite 文件",
Pattern: "*.db;*.sqlite;*.sqlite3;*.db3",
},
{
DisplayName: "所有文件",
Pattern: "*",
},
}
case "duckdb":
title = "选择 DuckDB 数据文件"
filters = []runtime.FileFilter{
{
DisplayName: "DuckDB 文件",
Pattern: "*.duckdb;*.ddb;*.db",
},
{
DisplayName: "所有文件",
Pattern: "*",
},
}
}
selection, err := runtime.OpenFileDialog(a.ctx, runtime.OpenDialogOptions{
Title: title,
DefaultDirectory: defaultDir,
Filters: filters,
})
if err != nil {
return connection.QueryResult{Success: false, Message: err.Error()}
}
if strings.TrimSpace(selection) == "" {
return connection.QueryResult{Success: false, Message: "Cancelled"}
}
if abs, err := filepath.Abs(selection); err == nil {
selection = abs
}
return connection.QueryResult{Success: true, Data: map[string]interface{}{"path": selection}}
}
// PreviewImportFile 解析导入文件,返回字段列表、总行数、前 5 行预览数据
func (a *App) PreviewImportFile(filePath string) connection.QueryResult {
if filePath == "" {
@@ -1527,7 +1600,11 @@ func writeRowsToFile(f *os.File, data []map[string]interface{}, columns []string
return err
}
}
if err := jsonEncoder.Encode(rowMap); err != nil {
exportedRow := make(map[string]interface{}, len(columns))
for _, col := range columns {
exportedRow[col] = normalizeExportJSONValue(rowMap[col])
}
if err := jsonEncoder.Encode(exportedRow); err != nil {
return err
}
isJsonFirstRow = false
@@ -1567,11 +1644,102 @@ func formatExportCellText(val interface{}) string {
return "NULL"
}
return v.Format("2006-01-02 15:04:05")
case float32:
f := float64(v)
if math.IsNaN(f) || math.IsInf(f, 0) {
return "NULL"
}
return strconv.FormatFloat(f, 'f', -1, 32)
case float64:
if math.IsNaN(v) || math.IsInf(v, 0) {
return "NULL"
}
return strconv.FormatFloat(v, 'f', -1, 64)
case json.Number:
text := strings.TrimSpace(v.String())
if text == "" {
return "NULL"
}
return text
default:
return fmt.Sprintf("%v", val)
}
}
func normalizeExportJSONValue(val interface{}) interface{} {
if val == nil {
return nil
}
switch v := val.(type) {
case float32:
f := float64(v)
if math.IsNaN(f) || math.IsInf(f, 0) {
return nil
}
return json.Number(strconv.FormatFloat(f, 'f', -1, 32))
case float64:
if math.IsNaN(v) || math.IsInf(v, 0) {
return nil
}
return json.Number(strconv.FormatFloat(v, 'f', -1, 64))
case json.Number:
text := strings.TrimSpace(v.String())
if text == "" {
return nil
}
return json.Number(text)
case map[string]interface{}:
out := make(map[string]interface{}, len(v))
for key, item := range v {
out[key] = normalizeExportJSONValue(item)
}
return out
case []interface{}:
items := make([]interface{}, len(v))
for i, item := range v {
items[i] = normalizeExportJSONValue(item)
}
return items
}
rv := reflect.ValueOf(val)
switch rv.Kind() {
case reflect.Pointer, reflect.Interface:
if rv.IsNil() {
return nil
}
return normalizeExportJSONValue(rv.Elem().Interface())
case reflect.Map:
if rv.IsNil() {
return nil
}
out := make(map[string]interface{}, rv.Len())
iter := rv.MapRange()
for iter.Next() {
out[fmt.Sprint(iter.Key().Interface())] = normalizeExportJSONValue(iter.Value().Interface())
}
return out
case reflect.Slice:
if rv.IsNil() {
return nil
}
if rv.Type().Elem().Kind() == reflect.Uint8 {
return val
}
fallthrough
case reflect.Array:
size := rv.Len()
items := make([]interface{}, size)
for i := 0; i < size; i++ {
items[i] = normalizeExportJSONValue(rv.Index(i).Interface())
}
return items
default:
return val
}
}
// writeRowsToXlsx 使用 excelize 写入真正的 xlsx 格式文件
func writeRowsToXlsx(filename string, data []map[string]interface{}, columns []string) error {
xlsx := excelize.NewFile()

View File

@@ -0,0 +1,89 @@
package app
import (
"bytes"
"encoding/json"
"os"
"strings"
"testing"
)
func TestFormatExportCellText_FloatNoScientificNotation(t *testing.T) {
got := formatExportCellText(1.445663e+06)
if strings.Contains(strings.ToLower(got), "e+") || strings.Contains(strings.ToLower(got), "e-") {
t.Fatalf("不应输出科学计数法got=%q", got)
}
if got != "1445663" {
t.Fatalf("浮点整值导出异常want=%q got=%q", "1445663", got)
}
}
func TestWriteRowsToFile_Markdown_NumberKeepPlainText(t *testing.T) {
f, err := os.CreateTemp("", "gonavi-export-*.md")
if err != nil {
t.Fatalf("创建临时文件失败: %v", err)
}
defer os.Remove(f.Name())
defer f.Close()
data := []map[string]interface{}{
{"id": 1.445663e+06},
}
columns := []string{"id"}
if err := writeRowsToFile(f, data, columns, "md"); err != nil {
t.Fatalf("写入 md 失败: %v", err)
}
contentBytes, err := os.ReadFile(f.Name())
if err != nil {
t.Fatalf("读取 md 失败: %v", err)
}
content := string(contentBytes)
if strings.Contains(strings.ToLower(content), "e+") || strings.Contains(strings.ToLower(content), "e-") {
t.Fatalf("md 导出包含科学计数法: %s", content)
}
if !strings.Contains(content, "| 1445663 |") {
t.Fatalf("md 导出未保留整数字面量content=%s", content)
}
}
func TestWriteRowsToFile_JSON_NumberKeepPlainText(t *testing.T) {
f, err := os.CreateTemp("", "gonavi-export-*.json")
if err != nil {
t.Fatalf("创建临时文件失败: %v", err)
}
defer os.Remove(f.Name())
defer f.Close()
data := []map[string]interface{}{
{"id": 1.445663e+06},
}
columns := []string{"id"}
if err := writeRowsToFile(f, data, columns, "json"); err != nil {
t.Fatalf("写入 json 失败: %v", err)
}
contentBytes, err := os.ReadFile(f.Name())
if err != nil {
t.Fatalf("读取 json 失败: %v", err)
}
content := string(contentBytes)
if strings.Contains(strings.ToLower(content), "e+") || strings.Contains(strings.ToLower(content), "e-") {
t.Fatalf("json 导出包含科学计数法: %s", content)
}
var decoded []map[string]json.Number
decoder := json.NewDecoder(bytes.NewReader(contentBytes))
decoder.UseNumber()
if err := decoder.Decode(&decoded); err != nil {
t.Fatalf("解析导出 json 失败: %v", err)
}
if len(decoded) != 1 {
t.Fatalf("导出行数异常got=%d", len(decoded))
}
if decoded[0]["id"].String() != "1445663" {
t.Fatalf("json 数值格式异常want=1445663 got=%s", decoded[0]["id"].String())
}
}

View File

@@ -3,6 +3,7 @@ package db
import (
"encoding/hex"
"fmt"
"reflect"
"strings"
"unicode"
"unicode/utf8"
@@ -18,7 +19,70 @@ func normalizeQueryValueWithDBType(v interface{}, databaseTypeName string) inter
if b, ok := v.([]byte); ok {
return bytesToDisplayValue(b, databaseTypeName)
}
return v
return normalizeCompositeQueryValue(v)
}
func normalizeCompositeQueryValue(v interface{}) interface{} {
if v == nil {
return nil
}
switch typed := v.(type) {
case []interface{}:
items := make([]interface{}, len(typed))
for i, item := range typed {
items[i] = normalizeQueryValue(item)
}
return items
case map[string]interface{}:
out := make(map[string]interface{}, len(typed))
for key, value := range typed {
out[key] = normalizeQueryValue(value)
}
return out
}
rv := reflect.ValueOf(v)
switch rv.Kind() {
case reflect.Pointer:
if rv.IsNil() {
return nil
}
return normalizeQueryValue(rv.Elem().Interface())
case reflect.Map:
if rv.IsNil() {
return nil
}
out := make(map[string]interface{}, rv.Len())
iter := rv.MapRange()
for iter.Next() {
out[mapKeyToString(iter.Key().Interface())] = normalizeQueryValue(iter.Value().Interface())
}
return out
case reflect.Slice, reflect.Array:
// []byte 在上层已单独处理,这里保留对其它切片/数组的递归规整。
if rv.Kind() == reflect.Slice && rv.IsNil() {
return nil
}
size := rv.Len()
items := make([]interface{}, size)
for i := 0; i < size; i++ {
items[i] = normalizeQueryValue(rv.Index(i).Interface())
}
return items
default:
return v
}
}
func mapKeyToString(key interface{}) string {
if key == nil {
return "null"
}
if s, ok := key.(string); ok {
return s
}
return fmt.Sprintf("%v", key)
}
func bytesToDisplayValue(b []byte, databaseTypeName string) interface{} {

View File

@@ -2,6 +2,8 @@ package db
import "testing"
type duckMapLike map[any]any
func TestNormalizeQueryValueWithDBType_BitBytes(t *testing.T) {
v := normalizeQueryValueWithDBType([]byte{0x00}, "BIT")
if v != int64(0) {
@@ -42,3 +44,40 @@ func TestNormalizeQueryValueWithDBType_ByteFallbacks(t *testing.T) {
t.Fatalf("未知类型 0xff 期望返回 0xff实际=%v(%T)", v, v)
}
}
func TestNormalizeQueryValueWithDBType_MapAnyAnyForJSON(t *testing.T) {
input := duckMapLike{
"id": int64(1),
1: "one",
true: []interface{}{duckMapLike{2: "two"}},
"bytes": []byte("ok"),
}
v := normalizeQueryValueWithDBType(input, "")
root, ok := v.(map[string]interface{})
if !ok {
t.Fatalf("期望转换为 map[string]interface{},实际=%T", v)
}
if root["id"] != int64(1) {
t.Fatalf("id 字段异常,实际=%v(%T)", root["id"], root["id"])
}
if root["1"] != "one" {
t.Fatalf("数字 key 未被字符串化,实际=%v(%T)", root["1"], root["1"])
}
if root["bytes"] != "ok" {
t.Fatalf("嵌套 []byte 未被转换,实际=%v(%T)", root["bytes"], root["bytes"])
}
arr, ok := root["true"].([]interface{})
if !ok || len(arr) != 1 {
t.Fatalf("bool key 下的数组结构异常,实际=%v(%T)", root["true"], root["true"])
}
nested, ok := arr[0].(map[string]interface{})
if !ok {
t.Fatalf("嵌套 map 未被转换,实际=%v(%T)", arr[0], arr[0])
}
if nested["2"] != "two" {
t.Fatalf("嵌套 map 数字 key 未转换,实际=%v(%T)", nested["2"], nested["2"])
}
}