feat(explain): 新增慢 SQL 历史存储与 DBQueryMulti 执行埋点

- 存储层:JSONL 单连接单文件,5MB 自动滚动,TopN 排序 + SQL 指纹去重
- 执行埋点:DBQueryMulti 用 named return + defer 异步记录,成功后自动写入历史
- 阈值过滤:默认 500ms 以下查询跳过记录,避免历史爆炸
- 查询入口:GetSlowQueries 按 duration/rowsRead/recent 排序,ClearSlowQueries 支持清空
- SQL 指纹:字面量/大小写归一化后 sha256,同模板不同参数视为同一查询
- 测试覆盖:新增 13 个单元测试覆盖存储/滚动/排序/去重/指纹
This commit is contained in:
Syngnat
2026-06-19 13:17:39 +08:00
parent 0c320234fd
commit a74065bdbb
4 changed files with 672 additions and 1 deletions

View File

@@ -889,7 +889,18 @@ func (a *App) DBQueryWithCancel(config connection.ConnectionConfig, dbName strin
// DBQueryMulti 执行可能包含多条 SQL 语句的查询,返回多个结果集。
// 如果底层驱动支持 MultiResultQuerier一次性执行所有语句
// 否则按分号拆分后逐条执行,模拟多结果集。
func (a *App) DBQueryMulti(config connection.ConnectionConfig, dbName string, query string, queryID string) connection.QueryResult {
func (a *App) DBQueryMulti(config connection.ConnectionConfig, dbName string, query string, queryID string) (result connection.QueryResult) {
// 慢 SQL 埋点:成功执行后异步记录(低于阈值 500ms 自动跳过),不阻塞返回。
// 用 named return + defer 覆盖所有 return path避免遗漏。
queryStartedAt := time.Now()
defer func() {
if !result.Success {
return
}
durationMs := time.Since(queryStartedAt).Milliseconds()
a.recordQueryExecutionAsync(config, resolveDDLDBType(config), query, durationMs, 0, 0)
}()
runConfig := normalizeRunConfig(config, dbName)
if queryID == "" {

View File

@@ -0,0 +1,71 @@
package app
import (
"strings"
"GoNavi-Wails/internal/connection"
"GoNavi-Wails/internal/logger"
)
// 慢 SQL 历史的 Wails 绑定入口。
//
// 前端调用:
// - GetSlowQueries(connectionId, dbName, sortBy, limit) → []QueryExecutionRecord
// - ClearSlowQueries(connectionId, dbName) → 错误(清空当前连接的历史)
//
// sortBy: "duration" | "rowsRead" | "recent"
// GetSlowQueries 返回当前连接的慢 SQL 历史按指定字段排序、SQL 指纹去重后取前 N。
// limit <= 0 时返回前 100 条。
func (a *App) GetSlowQueries(config connection.ConnectionConfig, dbName, sortBy string, limit int) connection.QueryResult {
runConfig := normalizeRunConfig(config, dbName)
connFP, ok := buildConnectionFingerprint(runConfig)
if !ok || connFP == "" {
return connection.QueryResult{Success: false, Message: "无法解析连接指纹"}
}
if limit <= 0 {
limit = 100
}
store := newQueryHistoryStore(a.configDir, connFP)
records, err := store.LoadTopN(strings.TrimSpace(sortBy), limit, true)
if err != nil {
logger.Warnf("GetSlowQueries 加载失败connFp=%s err=%v", connFP, err)
return connection.QueryResult{Success: false, Message: err.Error()}
}
return connection.QueryResult{Success: true, Message: "加载完成", Data: records}
}
// ClearSlowQueries 清空当前连接的慢 SQL 历史。
// 删除主文件 + rotate 文件(.1)。
func (a *App) ClearSlowQueries(config connection.ConnectionConfig, dbName string) connection.QueryResult {
runConfig := normalizeRunConfig(config, dbName)
connFP, ok := buildConnectionFingerprint(runConfig)
if !ok || connFP == "" {
return connection.QueryResult{Success: false, Message: "无法解析连接指纹"}
}
store := newQueryHistoryStore(a.configDir, connFP)
if err := store.Clear(); err != nil {
logger.Warnf("ClearSlowQueries 失败connFp=%s err=%v", connFP, err)
return connection.QueryResult{Success: false, Message: err.Error()}
}
return connection.QueryResult{Success: true, Message: "已清空慢查询历史"}
}
// recordQueryExecutionAsync 异步追加一条慢查询记录,不阻塞主查询返回。
// 调用方应传入已计算的 durationMs 和 rowsRead/Returned。
func (a *App) recordQueryExecutionAsync(config connection.ConnectionConfig, dbType, sql string, durationMs, rowsRead, rowsReturned int64) {
if durationMs < queryHistorySlowThresholdMs {
return
}
record := buildQueryExecutionRecord(config, dbType, sql, durationMs, rowsRead, rowsReturned)
go func() {
defer func() {
if r := recover(); r != nil {
logger.Warnf("recordQueryExecutionAsync panic%v", r)
}
}()
store := newQueryHistoryStore(a.configDir, record.ConnectionFP)
store.Append(record)
}()
}

View File

@@ -0,0 +1,313 @@
package app
import (
"bufio"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"GoNavi-Wails/internal/connection"
"GoNavi-Wails/internal/logger"
)
// 慢 SQL 历史存储。
//
// 设计要点:
// - 每个连接指纹一份 JSONL 文件(路径:<configDir>/query_history/<connFp>.jsonl
// - 追加写:每次执行 SQL 异步追加一行 JSONO(1) 写入
// - 5MB 滚动:写入前检查文件大小,超阈值则 rename 为 .1.jsonl 并新建空文件
// - 读 TopN全量加载到内存按字段排序 + SQL 指纹去重保留最新
// - 不引入 SQLite项目现有持久化都是 JSON依赖一致性优先
const (
queryHistoryDirName = "query_history"
queryHistoryFileMaxBytes = 5 * 1024 * 1024
queryHistorySlowThresholdMs int64 = 500 // 低于 500ms 不记录,避免历史爆炸
queryHistoryPreviewRunes = 200 // SQL 预览截断长度
)
// queryHistoryStore 是单连接的慢 SQL 历史存储。
// 并发安全:同一连接的多条 SQL 可能并发执行,写入加锁。
type queryHistoryStore struct {
mu sync.Mutex
filePath string
}
// newQueryHistoryStore 按连接指纹构造 store。configDir 为空时用 resolveAppConfigDir。
func newQueryHistoryStore(configDir, connFingerprint string) *queryHistoryStore {
if strings.TrimSpace(configDir) == "" {
configDir = resolveAppConfigDir()
}
fp := sanitizeFingerprintForFilename(connFingerprint)
return &queryHistoryStore{
filePath: filepath.Join(configDir, queryHistoryDirName, fp+".jsonl"),
}
}
// Append 追加一条执行记录。低于阈值的查询自动跳过。
// 失败仅记日志,不影响主查询流程。
func (s *queryHistoryStore) Append(record connection.QueryExecutionRecord) {
if record.DurationMs < queryHistorySlowThresholdMs {
return
}
s.mu.Lock()
defer s.mu.Unlock()
if err := os.MkdirAll(filepath.Dir(s.filePath), 0o755); err != nil {
logger.Warnf("创建慢查询历史目录失败:%v path=%s", err, filepath.Dir(s.filePath))
return
}
// 检查大小并 rotaterotate 失败不阻塞写入)
if info, err := os.Stat(s.filePath); err == nil && info.Size() >= queryHistoryFileMaxBytes {
rotated := s.filePath + ".1"
// 已有 .1 文件则先删除(只保留一个历史文件)
_ = os.Remove(rotated)
if err := os.Rename(s.filePath, rotated); err != nil {
logger.Warnf("慢查询历史 rotate 失败:%v path=%s", err, s.filePath)
}
}
file, err := os.OpenFile(s.filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
logger.Warnf("打开慢查询历史文件失败:%v path=%s", err, s.filePath)
return
}
defer file.Close()
payload, err := json.Marshal(record)
if err != nil {
logger.Warnf("序列化慢查询记录失败:%v", err)
return
}
payload = append(payload, '\n')
if _, err := file.Write(payload); err != nil {
logger.Warnf("写入慢查询历史失败:%v path=%s", err, s.filePath)
}
}
// LoadTopN 加载历史并按指定字段排序,返回前 N 条。
// sortBy: "duration" | "rowsRead" | "recent"dedupe=true 时同 SQL 指纹仅保留最新一条。
func (s *queryHistoryStore) LoadTopN(sortBy string, limit int, dedupe bool) ([]connection.QueryExecutionRecord, error) {
records, err := s.loadAll()
if err != nil {
return nil, err
}
if dedupe {
records = dedupeQueryRecords(records)
}
sortQueryRecords(records, sortBy)
if limit > 0 && len(records) > limit {
records = records[:limit]
}
return records, nil
}
// Clear 删除主文件 + rotate 文件。文件不存在视为成功。
func (s *queryHistoryStore) Clear() error {
s.mu.Lock()
defer s.mu.Unlock()
for _, path := range []string{s.filePath, s.filePath + ".1"} {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return err
}
}
return nil
}
// loadAll 加载主文件 + rotate 文件(.1)的全部记录。
// 单行解析失败时跳过该行,不阻塞整体加载。
func (s *queryHistoryStore) loadAll() ([]connection.QueryExecutionRecord, error) {
var records []connection.QueryExecutionRecord
for _, path := range []string{s.filePath + ".1", s.filePath} {
file, err := os.Open(path)
if err != nil {
if !os.IsNotExist(err) {
logger.Warnf("打开慢查询历史失败:%v path=%s", err, path)
}
continue
}
scanner := bufio.NewScanner(file)
scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) // 单行最大 1MB足够大 SQL
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
var r connection.QueryExecutionRecord
if err := json.Unmarshal([]byte(line), &r); err != nil {
continue
}
records = append(records, r)
}
file.Close()
if err := scanner.Err(); err != nil {
logger.Warnf("读取慢查询历史失败:%v path=%s", err, path)
}
}
return records, nil
}
// dedupeQueryRecords 按 SQLFingerprint 去重保留最新ExecutedAt 最大)一条。
func dedupeQueryRecords(records []connection.QueryExecutionRecord) []connection.QueryExecutionRecord {
if len(records) == 0 {
return records
}
latest := make(map[string]connection.QueryExecutionRecord, len(records))
for _, r := range records {
if r.SQLFingerprint == "" {
continue
}
existing, ok := latest[r.SQLFingerprint]
if !ok || r.ExecutedAt.After(existing.ExecutedAt) {
latest[r.SQLFingerprint] = r
}
}
result := make([]connection.QueryExecutionRecord, 0, len(latest))
for _, r := range latest {
result = append(result, r)
}
return result
}
// sortQueryRecords 按字段原地排序。sortBy 不识别时按 recent。
func sortQueryRecords(records []connection.QueryExecutionRecord, sortBy string) {
switch sortBy {
case "duration":
// 插入排序:记录数通常 < 1000
for i := 1; i < len(records); i++ {
for j := i; j > 0 && records[j].DurationMs > records[j-1].DurationMs; j-- {
records[j], records[j-1] = records[j-1], records[j]
}
}
case "rowsRead":
for i := 1; i < len(records); i++ {
for j := i; j > 0 && records[j].RowsRead > records[j-1].RowsRead; j-- {
records[j], records[j-1] = records[j-1], records[j]
}
}
default: // "recent"
for i := 1; i < len(records); i++ {
for j := i; j > 0 && records[j].ExecutedAt.After(records[j-1].ExecutedAt); j-- {
records[j], records[j-1] = records[j-1], records[j]
}
}
}
}
// buildSQLFingerprint 把 SQL 归一化为指纹(替换字面量为 ?、去注释、小写化关键字、sha256 前 16 字节)。
// 用于跨执行去重:同一 SQL 不同参数值视为同一指纹。
func buildSQLFingerprint(sql string) string {
normalized := normalizeSQLForFingerprint(sql)
if normalized == "" {
return ""
}
hash := sha256.Sum256([]byte(normalized))
return hex.EncodeToString(hash[:16])
}
// normalizeSQLForFingerprint 简化 SQL 用于指纹计算。
// 策略:
// - 去掉前后空白
// - 替换字符串字面量 'xxx' 为 ?
// - 替换数字字面量为 ?
// - 替换 IN (...) 中的列表为 ?
// - 小写化 SQL 关键字(保守起见全小写,不影响语义)
func normalizeSQLForFingerprint(sql string) string {
text := strings.TrimSpace(sql)
if text == "" {
return ""
}
var builder strings.Builder
builder.Grow(len(text))
inString := false
stringQuote := byte(0)
i := 0
for i < len(text) {
ch := text[i]
switch {
case inString:
if ch == stringQuote {
inString = false
builder.WriteByte('?')
}
// 跳过字符串内容
case ch == '\'' || ch == '"':
inString = true
stringQuote = ch
case (ch >= '0' && ch <= '9'):
// 数字字面量替换为 ?,跳过连续数字
for i < len(text) && text[i] >= '0' && text[i] <= '9' {
i++
}
builder.WriteByte('?')
continue
default:
if ch >= 'A' && ch <= 'Z' {
ch = ch + ('a' - 'A')
}
builder.WriteByte(ch)
}
i++
}
return builder.String()
}
// buildQueryPreview 截断 SQL 为人类可读预览。
func buildQueryPreview(sql string) string {
text := strings.TrimSpace(sql)
if text == "" {
return ""
}
// 把多行/制表符折叠为单空格(保留语义但节省存储)
text = strings.ReplaceAll(text, "\n", " ")
text = strings.ReplaceAll(text, "\r", " ")
text = strings.ReplaceAll(text, "\t", " ")
// 折叠连续空白
for strings.Contains(text, " ") {
text = strings.ReplaceAll(text, " ", " ")
}
runes := []rune(text)
if len(runes) <= queryHistoryPreviewRunes {
return text
}
return string(runes[:queryHistoryPreviewRunes-1]) + "…"
}
// sanitizeFingerprintForFilename 把指纹字符串安全化(只保留字母数字下划线)。
func sanitizeFingerprintForFilename(fp string) string {
var builder strings.Builder
builder.Grow(len(fp))
for _, ch := range fp {
if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch == '_' || ch == '-' {
builder.WriteRune(ch)
}
}
result := builder.String()
if result == "" {
return "default"
}
return result
}
// buildQueryExecutionRecord 是埋点时的便利构造器,组装一条完整记录。
func buildQueryExecutionRecord(config connection.ConnectionConfig, dbType, sql string, durationMs int64, rowsRead, rowsReturned int64) connection.QueryExecutionRecord {
connFP, _ := buildConnectionFingerprint(config)
return connection.QueryExecutionRecord{
ID: fmt.Sprintf("qhr-%d", time.Now().UnixNano()),
ConnectionFP: connFP,
SQLFingerprint: buildSQLFingerprint(sql),
SQLPreview: buildQueryPreview(sql),
DBType: dbType,
DurationMs: durationMs,
RowsRead: rowsRead,
RowsReturned: rowsReturned,
ExecutedAt: time.Now(),
}
}

View File

@@ -0,0 +1,276 @@
package app
import (
"os"
"path/filepath"
"testing"
"time"
"GoNavi-Wails/internal/connection"
)
func TestQueryHistoryStore_AppendAndLoad(t *testing.T) {
dir := t.TempDir()
store := newQueryHistoryStore(dir, "test-conn-fp")
store.Append(connection.QueryExecutionRecord{
ID: "r1",
ConnectionFP: "test-conn-fp",
SQLFingerprint: "fp-select-1",
SQLPreview: "SELECT * FROM t",
DBType: "mysql",
DurationMs: 1000,
ExecutedAt: time.Now(),
})
store.Append(connection.QueryExecutionRecord{
ID: "r2",
ConnectionFP: "test-conn-fp",
SQLFingerprint: "fp-select-2",
SQLPreview: "SELECT * FROM u WHERE id = 1",
DBType: "mysql",
DurationMs: 2000,
ExecutedAt: time.Now().Add(time.Second),
})
records, err := store.LoadTopN("duration", 10, false)
if err != nil {
t.Fatalf("LoadTopN 失败:%v", err)
}
if len(records) != 2 {
t.Fatalf("应有 2 条记录got=%d", len(records))
}
// duration 排序r2 (2000ms) 应在前面
if records[0].ID != "r2" {
t.Fatalf("按 duration 排序后首条应为 r2got=%s", records[0].ID)
}
}
func TestQueryHistoryStore_SkipBelowThreshold(t *testing.T) {
dir := t.TempDir()
store := newQueryHistoryStore(dir, "test-conn-fp")
// 低于 500ms 阈值应被跳过
store.Append(connection.QueryExecutionRecord{
ID: "fast",
DurationMs: 100,
SQLPreview: "SELECT 1",
ExecutedAt: time.Now(),
})
records, _ := store.LoadTopN("duration", 10, false)
if len(records) != 0 {
t.Fatalf("低于阈值的查询不应被记录got=%d", len(records))
}
}
func TestQueryHistoryStore_DedupeBySQLFingerprint(t *testing.T) {
dir := t.TempDir()
store := newQueryHistoryStore(dir, "test-conn-fp")
base := time.Now()
// 同一 SQL 指纹3 次执行(不同时间)
for i := 0; i < 3; i++ {
store.Append(connection.QueryExecutionRecord{
ID: "r" + string(rune('1'+i)),
SQLFingerprint: "same-fp",
DurationMs: int64(1000 + i*500),
ExecutedAt: base.Add(time.Duration(i) * time.Second),
})
}
records, _ := store.LoadTopN("duration", 10, true)
if len(records) != 1 {
t.Fatalf("去重后应剩 1 条got=%d", len(records))
}
// 应保留最新一条ExecutedAt 最大)
if records[0].ID != "r3" {
t.Fatalf("去重应保留最新got ID=%s", records[0].ID)
}
}
func TestQueryHistoryStore_RotationAtThreshold(t *testing.T) {
dir := t.TempDir()
store := newQueryHistoryStore(dir, "test-conn-fp")
// 写入大量记录触发 rotate5MB 阈值)
for i := 0; i < 50000; i++ {
store.Append(connection.QueryExecutionRecord{
ID: "r",
SQLFingerprint: "fp",
SQLPreview: "SELECT * FROM some_very_large_table WHERE col = 'long string to fill up space quickly'",
DurationMs: 1000,
ExecutedAt: time.Now(),
})
}
// 主文件存在 + rotate 文件存在
if _, err := os.Stat(store.filePath); err != nil {
t.Fatalf("主文件应存在:%v", err)
}
if _, err := os.Stat(store.filePath + ".1"); err != nil {
t.Fatalf("rotate 文件 .1 应存在:%v", err)
}
records, _ := store.LoadTopN("duration", 1000, false)
if len(records) == 0 {
t.Fatal("rotate 后应仍能加载历史")
}
}
func TestQueryHistoryStore_SortByRecent(t *testing.T) {
dir := t.TempDir()
store := newQueryHistoryStore(dir, "test-conn-fp")
base := time.Date(2025, 1, 1, 12, 0, 0, 0, time.UTC)
times := []time.Time{
base.Add(2 * time.Second),
base.Add(0 * time.Second),
base.Add(1 * time.Second),
}
for i, ts := range times {
store.Append(connection.QueryExecutionRecord{
ID: "r" + string(rune('1'+i)),
SQLFingerprint: "fp-" + string(rune('1'+i)),
DurationMs: 1000,
ExecutedAt: ts,
})
}
records, _ := store.LoadTopN("recent", 10, false)
if len(records) != 3 {
t.Fatalf("应有 3 条got=%d", len(records))
}
// recent 排序最新time[0])应在前面
if records[0].ID != "r1" {
t.Fatalf("recent 排序后首条应为 r1最新got=%s", records[0].ID)
}
}
func TestQueryHistoryStore_Clear(t *testing.T) {
dir := t.TempDir()
store := newQueryHistoryStore(dir, "test-conn-fp")
store.Append(connection.QueryExecutionRecord{
ID: "r1",
DurationMs: 1000,
SQLPreview: "SELECT 1",
ExecutedAt: time.Now(),
})
if err := store.Clear(); err != nil {
t.Fatalf("Clear 失败:%v", err)
}
records, _ := store.LoadTopN("duration", 10, false)
if len(records) != 0 {
t.Fatalf("清空后应无记录got=%d", len(records))
}
}
func TestQueryHistoryStore_EmptyReturnsEmpty(t *testing.T) {
dir := t.TempDir()
store := newQueryHistoryStore(dir, "missing-fp")
records, err := store.LoadTopN("duration", 10, false)
if err != nil {
t.Fatalf("不存在的文件应返回空而非 error%v", err)
}
if len(records) != 0 {
t.Fatalf("空历史应返回 0 条got=%d", len(records))
}
}
func TestBuildSQLFingerprint_NormalizesLiterals(t *testing.T) {
sql1 := "SELECT * FROM users WHERE id = 1 AND name = 'alice'"
sql2 := "SELECT * FROM users WHERE id = 999 AND name = 'bob'"
fp1 := buildSQLFingerprint(sql1)
fp2 := buildSQLFingerprint(sql2)
if fp1 != fp2 {
t.Fatalf("字面量不同应归一化为同一指纹fp1=%s fp2=%s", fp1, fp2)
}
if fp1 == "" {
t.Fatal("指纹不应为空")
}
}
func TestBuildSQLFingerprint_DifferentSQLDifferentFingerprint(t *testing.T) {
sql1 := "SELECT * FROM users WHERE id = 1"
sql2 := "SELECT * FROM orders WHERE id = 1"
fp1 := buildSQLFingerprint(sql1)
fp2 := buildSQLFingerprint(sql2)
if fp1 == fp2 {
t.Fatal("不同 SQL 应有不同指纹")
}
}
func TestBuildSQLFingerprint_CaseInsensitiveKeywords(t *testing.T) {
sql1 := "SELECT * FROM users"
sql2 := "select * from users"
if buildSQLFingerprint(sql1) != buildSQLFingerprint(sql2) {
t.Fatal("大小写不同的关键字应归一化为同一指纹")
}
}
func TestBuildQueryPreview_TruncatesLongSQL(t *testing.T) {
longSQL := ""
for i := 0; i < 500; i++ {
longSQL += "a"
}
preview := buildQueryPreview(longSQL)
if len([]rune(preview)) > queryHistoryPreviewRunes {
t.Fatalf("预览应不超过 %d 字符got=%d", queryHistoryPreviewRunes, len([]rune(preview)))
}
}
func TestBuildQueryPreview_FoldsWhitespace(t *testing.T) {
multiLine := "SELECT *\n FROM\tusers\nWHERE id = 1"
preview := buildQueryPreview(multiLine)
if containsNewline(preview) {
t.Fatalf("预览不应含换行符:%q", preview)
}
if !containsStr(preview, "SELECT * FROM users WHERE id = 1") {
t.Fatalf("预览应折叠空白:%q", preview)
}
}
func TestSanitizeFingerprintForFilename(t *testing.T) {
if got := sanitizeFingerprintForFilename("abc123_-"); got != "abc123_-" {
t.Fatalf("合法字符应保留got=%s", got)
}
if got := sanitizeFingerprintForFilename("a/b\\c:d"); got != "abcd" {
t.Fatalf("非法字符应被过滤got=%s", got)
}
if got := sanitizeFingerprintForFilename(""); got != "default" {
t.Fatalf("空指纹应回退为 defaultgot=%s", got)
}
}
func TestNewQueryHistoryStore_CreatesDir(t *testing.T) {
dir := filepath.Join(t.TempDir(), "nested", "deep")
store := newQueryHistoryStore(dir, "fp")
store.Append(connection.QueryExecutionRecord{
ID: "r1",
DurationMs: 1000,
ExecutedAt: time.Now(),
})
if _, err := os.Stat(store.filePath); err != nil {
t.Fatalf("Append 应创建嵌套目录并写入:%v", err)
}
}
func containsNewline(s string) bool {
for _, ch := range s {
if ch == '\n' || ch == '\r' {
return true
}
}
return false
}
func containsStr(s, substr string) bool {
return len(s) >= len(substr) && indexOfSubstr(s, substr) >= 0
}
func indexOfSubstr(s, substr string) int {
for i := 0; i+len(substr) <= len(s); i++ {
if s[i:i+len(substr)] == substr {
return i
}
}
return -1
}