🐛 fix(datasource): 修复 SQL Server 分页与 ClickHouse 22.8 连接兼容

- SQL Server 表数据分页改用旧版本兼容语法,避免 FETCH NEXT 报错

- ClickHouse HTTP 连接支持移除 client_protocol_version 后兼容重试

- 补充 SQL 分页与 ClickHouse 连接兼容回归测试

Refs #479
This commit is contained in:
Syngnat
2026-05-23 19:14:40 +08:00
parent 8615265ee1
commit cf0a216329
6 changed files with 347 additions and 42 deletions

View File

@@ -4,7 +4,7 @@ import { TabData, ColumnDefinition, IndexDefinition } from '../types';
import { useStore } from '../store';
import { DBQuery, DBGetColumns, DBGetIndexes } from '../../wailsjs/go/app/App';
import DataGrid, { GONAVI_ROW_KEY } from './DataGrid';
import { buildOrderBySQL, buildPaginatedSelectSQL, buildWhereSQL, hasExplicitSort, quoteIdentPart, quoteQualifiedIdent, withSortBufferTuningSQL, type FilterCondition } from '../utils/sql';
import { buildOrderBySQL, buildPaginatedSelectSQL, buildWhereSQL, hasExplicitSort, quoteIdentPart, quoteQualifiedIdent, reverseOrderBySQL, withSortBufferTuningSQL, type FilterCondition } from '../utils/sql';
import { buildMongoCountCommand, buildMongoFilter, buildMongoFindCommand, buildMongoSort } from '../utils/mongodb';
import { buildOracleApproximateTotalSql, parseApproximateTableCountRow, resolveApproximateTableCountStrategy } from '../utils/approximateTableCount';
import { getDataSourceCapabilities, resolveDataSourceType } from '../utils/dataSourceCapabilities';
@@ -219,25 +219,6 @@ const formatDataViewerQueryError = (dbType: string, messageText: unknown): strin
return rawMessage;
};
const reverseOrderBySQL = (orderBySQL: string): string => {
const raw = String(orderBySQL || '').trim();
if (!raw) return '';
const body = raw.replace(/^order\s+by\s+/i, '').trim();
if (!body) return '';
const parts = body
.split(',')
.map((part) => part.trim())
.filter(Boolean)
.map((part) => {
if (/\s+asc$/i.test(part)) return part.replace(/\s+asc$/i, ' DESC');
if (/\s+desc$/i.test(part)) return part.replace(/\s+desc$/i, ' ASC');
return `${part} DESC`;
});
if (parts.length === 0) return '';
return ` ORDER BY ${parts.join(', ')}`;
};
type ViewerFilterSnapshot = {
showFilter: boolean;
conditions: FilterCondition[];

View File

@@ -27,6 +27,11 @@ describe('buildAIReadonlyPreviewSQL', () => {
.toBe('SELECT * FROM users LIMIT 50 OFFSET 0');
});
it('limits SQL Server readonly SQL with TOP syntax', () => {
expect(buildAIReadonlyPreviewSQL('sqlserver', 'SELECT * FROM users', 50))
.toBe('SELECT TOP 50 * FROM users');
});
it('keeps PostgreSQL-compatible and ClickHouse SQL on LIMIT syntax', () => {
expect(buildAIReadonlyPreviewSQL('postgres', 'SELECT * FROM users', 50))
.toBe('SELECT * FROM users LIMIT 50 OFFSET 0');

View File

@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest';
import { buildOrderBySQL } from './sql';
import { buildOrderBySQL, buildPaginatedSelectSQL, reverseOrderBySQL } from './sql';
describe('buildOrderBySQL', () => {
it('does not add fallback ORDER BY for DuckDB without explicit sort', () => {
@@ -11,3 +11,44 @@ describe('buildOrderBySQL', () => {
expect(buildOrderBySQL('duckdb', { columnKey: 'ID', order: 'descend' }, ['NAME'])).toBe(' ORDER BY "ID" DESC');
});
});
describe('buildPaginatedSelectSQL', () => {
it('uses SQL Server TOP for the first page to support old compatibility levels', () => {
const sql = buildPaginatedSelectSQL('sqlserver', 'SELECT * FROM [Users]', ' ORDER BY [ID] ASC', 101, 0);
expect(sql).toBe('SELECT TOP 101 * FROM [Users] ORDER BY [ID] ASC');
expect(sql.toLowerCase()).not.toContain('fetch next');
expect(sql.toLowerCase()).not.toContain('offset');
});
it('adds SQL Server TOP after DISTINCT', () => {
expect(buildPaginatedSelectSQL('mssql', 'SELECT DISTINCT [Name] FROM [Users]', '', 50, 0))
.toBe('SELECT DISTINCT TOP 50 [Name] FROM [Users]');
});
it('does not add another SQL Server TOP when base SQL already has one', () => {
expect(buildPaginatedSelectSQL('sqlserver', 'SELECT TOP 10 * FROM [Users]', '', 50, 0))
.toBe('SELECT TOP 10 * FROM [Users]');
});
it('uses SQL Server TOP window pagination instead of OFFSET FETCH for sorted pages', () => {
const sql = buildPaginatedSelectSQL('sqlserver', 'SELECT * FROM [Users]', ' ORDER BY [ID] ASC', 25, 50);
expect(sql).toContain('SELECT TOP 25 * FROM (SELECT TOP 75 * FROM (SELECT * FROM [Users])');
expect(sql).toContain('ORDER BY [ID] DESC');
expect(sql.endsWith('ORDER BY [ID] ASC')).toBe(true);
expect(sql.toLowerCase()).not.toContain('fetch next');
});
it('keeps generic pagination for other databases', () => {
expect(buildPaginatedSelectSQL('postgres', 'SELECT * FROM users', ' ORDER BY id ASC', 20, 40))
.toBe('SELECT * FROM users ORDER BY id ASC LIMIT 20 OFFSET 40');
});
});
describe('reverseOrderBySQL', () => {
it('reverses comma separated order parts without splitting function arguments', () => {
expect(reverseOrderBySQL(' ORDER BY COALESCE([a], [b]) ASC, [id] DESC'))
.toBe(' ORDER BY COALESCE([a], [b]) DESC, [id] ASC');
});
});

View File

@@ -174,6 +174,129 @@ export const buildOrderBySQL = (
return '';
};
const splitOrderByParts = (body: string): string[] => {
const text = String(body || '');
const parts: string[] = [];
let start = 0;
let parenDepth = 0;
let inSingle = false;
let inDouble = false;
let inBracket = false;
for (let i = 0; i < text.length; i++) {
const ch = text[i];
const next = i + 1 < text.length ? text[i + 1] : '';
if (inSingle) {
if (ch === "'" && next === "'") {
i++;
} else if (ch === "'") {
inSingle = false;
}
continue;
}
if (inDouble) {
if (ch === '"' && next === '"') {
i++;
} else if (ch === '"') {
inDouble = false;
}
continue;
}
if (inBracket) {
if (ch === ']' && next === ']') {
i++;
} else if (ch === ']') {
inBracket = false;
}
continue;
}
if (ch === "'") {
inSingle = true;
continue;
}
if (ch === '"') {
inDouble = true;
continue;
}
if (ch === '[') {
inBracket = true;
continue;
}
if (ch === '(') {
parenDepth++;
continue;
}
if (ch === ')') {
if (parenDepth > 0) parenDepth--;
continue;
}
if (ch === ',' && parenDepth === 0) {
const part = text.slice(start, i).trim();
if (part) parts.push(part);
start = i + 1;
}
}
const tail = text.slice(start).trim();
if (tail) parts.push(tail);
return parts;
};
export const reverseOrderBySQL = (orderBySQL: string): string => {
const raw = String(orderBySQL || '').trim();
if (!raw) return '';
const body = raw.replace(/^order\s+by\s+/i, '').trim();
if (!body) return '';
const parts = splitOrderByParts(body)
.map((part) => {
if (/\s+asc$/i.test(part)) return part.replace(/\s+asc$/i, ' DESC');
if (/\s+desc$/i.test(part)) return part.replace(/\s+desc$/i, ' ASC');
return `${part} DESC`;
})
.filter(Boolean);
if (parts.length === 0) return '';
return ` ORDER BY ${parts.join(', ')}`;
};
const addSqlServerTopLimit = (sql: string, limit: number): string => {
const text = String(sql || '').trim();
if (!text) return text;
if (/^\s*select\s+(?:distinct\s+)?top\b/i.test(text)) {
return text;
}
return text.replace(
/^(\s*select\b)(\s+distinct\b)?/i,
(_match, selectKeyword: string, distinctKeyword = '') => `${selectKeyword}${distinctKeyword} TOP ${limit}`,
);
};
const buildSqlServerPaginatedSelectSQL = (
base: string,
orderBy: string,
limit: number,
offset: number,
): string => {
if (offset <= 0) {
return `${addSqlServerTopLimit(base, limit)}${orderBy}`;
}
const effectiveOrderBy = orderBy.trim();
if (effectiveOrderBy) {
const reverseOrderBy = reverseOrderBySQL(effectiveOrderBy);
if (reverseOrderBy) {
const upperBound = offset + limit;
return `SELECT * FROM (SELECT TOP ${limit} * FROM (SELECT TOP ${upperBound} * FROM (${base}) AS [__gonavi_page_base__] ${effectiveOrderBy}) AS [__gonavi_page_window__] ${reverseOrderBy}) AS [__gonavi_page_slice__] ${effectiveOrderBy}`;
}
}
const rowNumberOrderBy = effectiveOrderBy || 'ORDER BY (SELECT NULL)';
const upperBound = offset + limit;
return `SELECT * FROM (SELECT [__gonavi_page__].*, ROW_NUMBER() OVER (${rowNumberOrderBy}) AS [__gonavi_rn__] FROM (${base}) AS [__gonavi_page__]) AS [__gonavi_page_result__] WHERE [__gonavi_rn__] > ${offset} AND [__gonavi_rn__] <= ${upperBound} ORDER BY [__gonavi_rn__]`;
};
export const buildPaginatedSelectSQL = (
dbType: string,
baseSql: string,
@@ -203,8 +326,7 @@ export const buildPaginatedSelectSQL = (
}
case 'sqlserver':
case 'mssql': {
const effectiveOrderBy = orderBy.trim() ? orderBy : ' ORDER BY (SELECT NULL)';
return `${base}${effectiveOrderBy} OFFSET ${safeOffset} ROWS FETCH NEXT ${safeLimit} ROWS ONLY`;
return buildSqlServerPaginatedSelectSQL(base, orderBy, safeLimit, safeOffset);
}
default:
return `${base}${orderBy} LIMIT ${safeLimit} OFFSET ${safeOffset}`;

View File

@@ -7,6 +7,7 @@ import (
"database/sql"
"fmt"
"net"
"net/http"
"net/url"
"sort"
"strconv"
@@ -168,6 +169,10 @@ func defaultClickHousePortForScheme(scheme string) int {
}
func (c *ClickHouseDB) buildClickHouseOptions(config connection.ConnectionConfig) (*clickhouse.Options, error) {
return c.buildClickHouseOptionsWithHTTPCompatibility(config, false)
}
func (c *ClickHouseDB) buildClickHouseOptionsWithHTTPCompatibility(config connection.ConnectionConfig, stripHTTPClientProtocolVersion bool) (*clickhouse.Options, error) {
connectTimeout := getConnectTimeout(config)
readTimeout := connectTimeout
if readTimeout < minClickHouseReadTimeout {
@@ -195,9 +200,57 @@ func (c *ClickHouseDB) buildClickHouseOptions(config connection.ConnectionConfig
opts.TLS = tlsConfig
}
applyClickHouseConnectionParams(opts, config)
if stripHTTPClientProtocolVersion && protocol == clickhouse.HTTP {
installClickHouseHTTPClientProtocolVersionStripper(opts)
}
return opts, nil
}
type clickHouseHTTPClientProtocolVersionStripper struct {
next http.RoundTripper
}
func (rt clickHouseHTTPClientProtocolVersionStripper) RoundTrip(req *http.Request) (*http.Response, error) {
next := rt.next
if next == nil {
next = http.DefaultTransport
}
if req == nil || req.URL == nil {
return next.RoundTrip(req)
}
query := req.URL.Query()
if _, ok := query["client_protocol_version"]; !ok {
return next.RoundTrip(req)
}
cloned := req.Clone(req.Context())
clonedURL := *req.URL
query.Del("client_protocol_version")
clonedURL.RawQuery = query.Encode()
cloned.URL = &clonedURL
return next.RoundTrip(cloned)
}
func installClickHouseHTTPClientProtocolVersionStripper(opts *clickhouse.Options) {
if opts == nil {
return
}
previous := opts.TransportFunc
opts.TransportFunc = func(base *http.Transport) (http.RoundTripper, error) {
next := http.RoundTripper(base)
if previous != nil {
wrapped, err := previous(base)
if err != nil {
return nil, err
}
if wrapped != nil {
next = wrapped
}
}
return clickHouseHTTPClientProtocolVersionStripper{next: next}, nil
}
}
func parseClickHouseDurationParam(raw string) (time.Duration, bool) {
text := strings.TrimSpace(raw)
if text == "" {
@@ -396,6 +449,24 @@ func isClickHouseProtocolMismatch(err error) bool {
strings.Contains(text, "malformed http response")
}
func isClickHouseHTTPClientProtocolVersionUnsupported(err error) bool {
if err == nil {
return false
}
text := strings.ToLower(strings.TrimSpace(err.Error()))
if text == "" || !strings.Contains(text, "client_protocol_version") {
return false
}
return strings.Contains(text, "unknown setting") ||
strings.Contains(text, "unknown_setting") ||
strings.Contains(text, "code: 115")
}
func shouldTryNextClickHouseProtocol(protocol clickhouse.Protocol, err error) bool {
return isClickHouseProtocolMismatch(err) ||
(protocol == clickhouse.HTTP && isClickHouseHTTPClientProtocolVersionUnsupported(err))
}
func clickHouseProtocolName(protocol clickhouse.Protocol) string {
if protocol == clickhouse.HTTP {
return "HTTP"
@@ -436,6 +507,9 @@ func sanitizeClickHouseErrorMessage(err error) string {
}
func clickHouseAttemptFailureMessage(protocol clickhouse.Protocol, err error) string {
if protocol == clickhouse.HTTP && isClickHouseHTTPClientProtocolVersionUnsupported(err) {
return "当前 ClickHouse HTTP 端口不支持 client_protocol_version常见于 ClickHouse 22.8),将使用 HTTP 兼容模式重试;如仍失败请确认连接协议和端口"
}
if isClickHouseProtocolMismatch(err) {
if protocol == clickhouse.Native {
return "服务端响应不像 Native 握手,当前端口更像 HTTP/HTTPS 端口;请选择 HTTP 协议,或确认 ClickHouse Native 端口"
@@ -551,27 +625,52 @@ func (c *ClickHouseDB) Connect(config connection.ConnectionConfig) error {
protocols := clickHouseProtocolsForAttempt(attempt)
for pIdx, protocol := range protocols {
protocolConfig := withClickHouseProtocol(attempt, protocol)
logger.Infof("ClickHouse 连接尝试:第%d组/%d 协议=%s 地址=%s:%d SSL=%t",
idx+1, len(attempts), clickHouseProtocolName(protocol), protocolConfig.Host, protocolConfig.Port, protocolConfig.UseSSL)
opts, err := c.buildClickHouseOptions(protocolConfig)
if err != nil {
failures = append(failures, fmt.Sprintf("第%d次 TLS 配置失败(protocol=%s): %v", idx+1, protocol.String(), err))
logger.Warnf("ClickHouse TLS 配置失败:第%d组/%d 协议=%s 地址=%s:%d SSL=%t 原因=%v",
idx+1, len(attempts), clickHouseProtocolName(protocol), protocolConfig.Host, protocolConfig.Port, protocolConfig.UseSSL, err)
continue
compatibilityModes := []bool{false}
if protocol == clickhouse.HTTP {
compatibilityModes = append(compatibilityModes, true)
}
c.conn = clickhouse.OpenDB(opts)
if err := c.Ping(); err != nil {
failureMessage := clickHouseAttemptFailureMessage(protocol, err)
failures = append(failures, fmt.Sprintf("第%d次连接验证失败(protocol=%s): %s", idx+1, protocol.String(), failureMessage))
logger.Warnf("ClickHouse 连接尝试失败:第%d组/%d 协议=%s 地址=%s:%d SSL=%t 原因=%s",
idx+1, len(attempts), clickHouseProtocolName(protocol), protocolConfig.Host, protocolConfig.Port, protocolConfig.UseSSL, failureMessage)
if c.conn != nil {
_ = c.conn.Close()
c.conn = nil
protocolSuccess := false
var lastProtocolErr error
for compatIdx, stripHTTPClientProtocolVersion := range compatibilityModes {
logger.Infof("ClickHouse 连接尝试:第%d组/%d 协议=%s 地址=%s:%d SSL=%t HTTP兼容=%t",
idx+1, len(attempts), clickHouseProtocolName(protocol), protocolConfig.Host, protocolConfig.Port, protocolConfig.UseSSL, stripHTTPClientProtocolVersion)
opts, err := c.buildClickHouseOptionsWithHTTPCompatibility(protocolConfig, stripHTTPClientProtocolVersion)
if err != nil {
failures = append(failures, fmt.Sprintf("第%d次 TLS 配置失败(protocol=%s): %v", idx+1, protocol.String(), err))
logger.Warnf("ClickHouse TLS 配置失败:第%d组/%d 协议=%s 地址=%s:%d SSL=%t 原因=%v",
idx+1, len(attempts), clickHouseProtocolName(protocol), protocolConfig.Host, protocolConfig.Port, protocolConfig.UseSSL, err)
lastProtocolErr = err
break
}
if pIdx == 0 && !isClickHouseProtocolMismatch(err) {
// 首次连接不是协议误配特征,避免无谓重试次协议。
c.conn = clickhouse.OpenDB(opts)
if err := c.Ping(); err != nil {
lastProtocolErr = err
failureMessage := clickHouseAttemptFailureMessage(protocol, err)
failures = append(failures, fmt.Sprintf("第%d次连接验证失败(protocol=%s): %s", idx+1, protocol.String(), failureMessage))
logger.Warnf("ClickHouse 连接尝试失败:第%d组/%d 协议=%s 地址=%s:%d SSL=%t HTTP兼容=%t 原因=%s",
idx+1, len(attempts), clickHouseProtocolName(protocol), protocolConfig.Host, protocolConfig.Port, protocolConfig.UseSSL, stripHTTPClientProtocolVersion, failureMessage)
if c.conn != nil {
_ = c.conn.Close()
c.conn = nil
}
if protocol == clickhouse.HTTP &&
!stripHTTPClientProtocolVersion &&
isClickHouseHTTPClientProtocolVersionUnsupported(err) &&
compatIdx+1 < len(compatibilityModes) {
logger.Warnf("ClickHouse HTTP 端口不支持 client_protocol_version改用 HTTP 兼容模式重试")
continue
}
break
}
protocolSuccess = true
if stripHTTPClientProtocolVersion {
logger.Warnf("ClickHouse HTTP 兼容模式连接成功:已移除 client_protocol_version 参数")
}
break
}
if !protocolSuccess {
if pIdx == 0 && !shouldTryNextClickHouseProtocol(protocol, lastProtocolErr) {
// 首次连接不是协议误配或已知兼容性特征,避免无谓重试次协议。
break
}
continue

View File

@@ -8,6 +8,7 @@ import (
"database/sql/driver"
"errors"
"io"
"net/http"
"strings"
"sync"
"testing"
@@ -304,6 +305,56 @@ func TestClickHouseProtocolMismatchIncludesHTTPParseBinaryResponse(t *testing.T)
}
}
func TestClickHouseHTTPClientProtocolVersionUnsupportedEnablesCompatibilityRetry(t *testing.T) {
err := errors.New(`failed to query server hello: failed to query server hello info: sendQuery: [HTTP 404] response body: "Code: 115. DB::Exception: Unknown setting client_protocol_version. (UNKNOWN_SETTING)"`)
if !isClickHouseHTTPClientProtocolVersionUnsupported(err) {
t.Fatalf("expected client_protocol_version unknown setting to be treated as HTTP compatibility issue")
}
if !shouldTryNextClickHouseProtocol(clickhouse.HTTP, err) {
t.Fatalf("expected HTTP client_protocol_version issue to permit protocol fallback")
}
if shouldTryNextClickHouseProtocol(clickhouse.Native, err) {
t.Fatalf("native protocol should not treat HTTP client_protocol_version issue as retryable")
}
message := clickHouseAttemptFailureMessage(clickhouse.HTTP, err)
if !strings.Contains(message, "client_protocol_version") || !strings.Contains(message, "兼容模式") {
t.Fatalf("expected compatibility retry hint, got %q", message)
}
}
func TestClickHouseHTTPClientProtocolVersionStripperRemovesDriverQueryParam(t *testing.T) {
var seenQuery string
stripper := clickHouseHTTPClientProtocolVersionStripper{
next: roundTripFunc(func(req *http.Request) (*http.Response, error) {
seenQuery = req.URL.RawQuery
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(strings.NewReader("")),
Header: make(http.Header),
}, nil
}),
}
req, err := http.NewRequest(http.MethodPost, "http://clickhouse.local:8123/?database=default&client_protocol_version=54485", nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
res, err := stripper.RoundTrip(req)
if err != nil {
t.Fatalf("round trip: %v", err)
}
if res != nil && res.Body != nil {
res.Body.Close()
}
if strings.Contains(seenQuery, "client_protocol_version") {
t.Fatalf("expected client_protocol_version stripped from query, got %q", seenQuery)
}
if !strings.Contains(seenQuery, "database=default") {
t.Fatalf("expected other query parameters to remain, got %q", seenQuery)
}
}
func TestWithClickHouseProtocolForcesProtocolSelection(t *testing.T) {
httpConfig := withClickHouseProtocol(connection.ConnectionConfig{
Type: "clickhouse",
@@ -381,6 +432,12 @@ func TestClickHouseProtocolsForAttemptOnlyFallsBackInAutoMode(t *testing.T) {
}
}
type roundTripFunc func(*http.Request) (*http.Response, error)
func (fn roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return fn(req)
}
func protocolNames(protocols []clickhouse.Protocol) []string {
names := make([]string, 0, len(protocols))
for _, protocol := range protocols {