🐛 fix(connection): 修复失败连接高频重试并暂停后台自动元数据拉取 #331

- 后端为失败数据库连接增加冷却窗口,避免短时间内重复真实建连
- 补充失败冷却回归测试,覆盖重复失败、冷却后重试和成功后清理场景
- 前端在后台态暂停查询页、侧边栏和表概览的自动元数据拉取
- 保持手动刷新、手动展开等显式操作行为不变
This commit is contained in:
tianqijiuyun-latiao
2026-04-05 15:27:08 +08:00
parent 35944d58f8
commit 1a042321d2
7 changed files with 345 additions and 4 deletions

View File

@@ -23,6 +23,7 @@ import (
)
const dbCachePingInterval = 30 * time.Second
const dbConnectFailureCooldown = 30 * time.Second
const (
startupConnectRetryWindow = 20 * time.Second
@@ -40,6 +41,11 @@ type cachedDatabase struct {
lastPing time.Time
}
type cachedConnectFailure struct {
occurredAt time.Time
err error
}
type queryContext struct {
cancel context.CancelFunc
started time.Time
@@ -50,6 +56,7 @@ type App struct {
ctx context.Context
startedAt time.Time
dbCache map[string]cachedDatabase // Cache for DB connections
connectFailures map[string]cachedConnectFailure
mu sync.RWMutex // Mutex for cache access
updateMu sync.Mutex
updateState updateState
@@ -70,6 +77,7 @@ func NewAppWithSecretStore(store secretstore.SecretStore) *App {
}
return &App{
dbCache: make(map[string]cachedDatabase),
connectFailures: make(map[string]cachedConnectFailure),
runningQueries: make(map[string]queryContext),
configDir: resolveAppConfigDir(),
secretStore: store,
@@ -573,14 +581,28 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
if isFileDB {
logger.Infof("未命中文件库连接缓存,开始创建连接:类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
}
if failure, remaining, ok := a.getCachedConnectFailureByKey(key); ok {
message := fmt.Sprintf("连接最近失败,正在冷却中,请 %s 后重试;上次错误:%s",
formatConnectFailureCooldown(remaining),
normalizeErrorMessage(failure.err),
)
logger.Warnf("命中数据库连接失败冷却:%s 缓存Key=%s 剩余=%s 原因=%s",
formatConnSummary(effectiveConfig), shortKey, formatConnectFailureCooldown(remaining), normalizeErrorMessage(failure.err))
return nil, withLogHint{err: fmt.Errorf("%s", message), logPath: logger.Path()}
}
initialKey := key
dbInst, connectedConfig, err := a.connectDatabaseWithStartupRetry(resolvedConfig)
if err != nil {
failedKey := getCacheKey(connectedConfig)
a.recordConnectFailureByKey(failedKey, err)
return nil, err
}
a.clearConnectFailureByKey(initialKey)
effectiveConfig = connectedConfig
key = getCacheKey(effectiveConfig)
shortKey = shortenCacheKey(key)
a.clearConnectFailureByKey(key)
now := time.Now()
@@ -601,6 +623,62 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
return dbInst, nil
}
func (a *App) getCachedConnectFailureByKey(key string) (cachedConnectFailure, time.Duration, bool) {
if a == nil || strings.TrimSpace(key) == "" {
return cachedConnectFailure{}, 0, false
}
a.mu.RLock()
entry, exists := a.connectFailures[key]
a.mu.RUnlock()
if !exists || entry.err == nil || entry.occurredAt.IsZero() {
return cachedConnectFailure{}, 0, false
}
remaining := dbConnectFailureCooldown - time.Since(entry.occurredAt)
if remaining <= 0 {
a.clearConnectFailureByKey(key)
return cachedConnectFailure{}, 0, false
}
return entry, remaining, true
}
func (a *App) recordConnectFailureByKey(key string, err error) {
if a == nil || strings.TrimSpace(key) == "" || err == nil {
return
}
a.mu.Lock()
if a.connectFailures == nil {
a.connectFailures = make(map[string]cachedConnectFailure)
}
a.connectFailures[key] = cachedConnectFailure{
occurredAt: time.Now(),
err: err,
}
a.mu.Unlock()
}
func (a *App) clearConnectFailureByKey(key string) {
if a == nil || strings.TrimSpace(key) == "" {
return
}
a.mu.Lock()
if a.connectFailures != nil {
delete(a.connectFailures, key)
}
a.mu.Unlock()
}
func formatConnectFailureCooldown(remaining time.Duration) time.Duration {
if remaining <= time.Second {
return time.Second
}
return remaining.Truncate(time.Second)
}
func shortenCacheKey(key string) string {
if len(key) > 12 {
return key[:12]

View File

@@ -303,3 +303,167 @@ func TestIsTransientStartupConnectError(t *testing.T) {
t.Fatal("expected authentication failure to not be treated as transient startup connect error")
}
}
func TestGetDatabaseWithPing_CoolsDownRepeatedFailures(t *testing.T) {
originalNewDatabaseFunc := newDatabaseFunc
originalResolveDialConfigWithProxyFunc := resolveDialConfigWithProxyFunc
defer func() {
newDatabaseFunc = originalNewDatabaseFunc
resolveDialConfigWithProxyFunc = originalResolveDialConfigWithProxyFunc
}()
connectCalls := 0
newDatabaseFunc = func(dbType string) (db.Database, error) {
return &fakeStartupRetryDB{
connect: func(config connection.ConnectionConfig) error {
connectCalls++
return errors.New("dial tcp 10.1.131.86:5432: connect: connection refused")
},
}, nil
}
resolveDialConfigWithProxyFunc = func(raw connection.ConnectionConfig) (connection.ConnectionConfig, error) {
return raw, nil
}
a := &App{
startedAt: time.Now().Add(-startupConnectRetryWindow - time.Second),
dbCache: make(map[string]cachedDatabase),
connectFailures: make(map[string]cachedConnectFailure),
runningQueries: make(map[string]queryContext),
}
config := connection.ConnectionConfig{Type: "postgres", Host: "10.1.131.86", Port: 5432, User: "postgres"}
_, firstErr := a.getDatabaseWithPing(config, false)
if firstErr == nil {
t.Fatal("expected first connection attempt to fail")
}
if connectCalls != 2 {
t.Fatalf("expected first request to use 2 connect attempts, got %d", connectCalls)
}
_, secondErr := a.getDatabaseWithPing(config, false)
if secondErr == nil {
t.Fatal("expected second connection attempt to fail during cooldown")
}
if connectCalls != 2 {
t.Fatalf("expected repeated request during cooldown to avoid reconnecting, got %d connect attempts", connectCalls)
}
}
func TestGetDatabaseWithPing_AllowsRetryAfterFailureCooldown(t *testing.T) {
originalNewDatabaseFunc := newDatabaseFunc
originalResolveDialConfigWithProxyFunc := resolveDialConfigWithProxyFunc
defer func() {
newDatabaseFunc = originalNewDatabaseFunc
resolveDialConfigWithProxyFunc = originalResolveDialConfigWithProxyFunc
}()
connectCalls := 0
newDatabaseFunc = func(dbType string) (db.Database, error) {
return &fakeStartupRetryDB{
connect: func(config connection.ConnectionConfig) error {
connectCalls++
if connectCalls <= 2 {
return errors.New("dial tcp 10.1.131.86:5432: connect: connection refused")
}
return nil
},
}, nil
}
resolveDialConfigWithProxyFunc = func(raw connection.ConnectionConfig) (connection.ConnectionConfig, error) {
return raw, nil
}
a := &App{
startedAt: time.Now().Add(-startupConnectRetryWindow - time.Second),
dbCache: make(map[string]cachedDatabase),
connectFailures: make(map[string]cachedConnectFailure),
runningQueries: make(map[string]queryContext),
}
config := connection.ConnectionConfig{Type: "postgres", Host: "10.1.131.86", Port: 5432, User: "postgres"}
_, firstErr := a.getDatabaseWithPing(config, false)
if firstErr == nil {
t.Fatal("expected first connection attempt to fail")
}
if connectCalls != 2 {
t.Fatalf("expected first request to use 2 connect attempts, got %d", connectCalls)
}
key := getCacheKey(config)
a.mu.Lock()
a.connectFailures[key] = cachedConnectFailure{
occurredAt: time.Now().Add(-dbConnectFailureCooldown - time.Second),
err: errors.New("dial tcp 10.1.131.86:5432: connect: connection refused"),
}
a.mu.Unlock()
inst, secondErr := a.getDatabaseWithPing(config, false)
if secondErr != nil {
t.Fatalf("expected retry after cooldown to be allowed, got error: %v", secondErr)
}
if inst == nil {
t.Fatal("expected database instance after cooldown retry")
}
if connectCalls != 3 {
t.Fatalf("expected reconnect after cooldown expiration, got %d connect attempts", connectCalls)
}
}
func TestGetDatabaseWithPing_ClearsFailureCooldownAfterSuccess(t *testing.T) {
originalNewDatabaseFunc := newDatabaseFunc
originalResolveDialConfigWithProxyFunc := resolveDialConfigWithProxyFunc
defer func() {
newDatabaseFunc = originalNewDatabaseFunc
resolveDialConfigWithProxyFunc = originalResolveDialConfigWithProxyFunc
}()
connectCalls := 0
newDatabaseFunc = func(dbType string) (db.Database, error) {
return &fakeStartupRetryDB{
connect: func(config connection.ConnectionConfig) error {
connectCalls++
if connectCalls <= 2 {
return errors.New("dial tcp 10.1.131.86:5432: connect: connection refused")
}
return nil
},
}, nil
}
resolveDialConfigWithProxyFunc = func(raw connection.ConnectionConfig) (connection.ConnectionConfig, error) {
return raw, nil
}
a := &App{
startedAt: time.Now().Add(-startupConnectRetryWindow - time.Second),
dbCache: make(map[string]cachedDatabase),
connectFailures: make(map[string]cachedConnectFailure),
runningQueries: make(map[string]queryContext),
}
config := connection.ConnectionConfig{Type: "postgres", Host: "10.1.131.86", Port: 5432, User: "postgres"}
_, firstErr := a.getDatabaseWithPing(config, false)
if firstErr == nil {
t.Fatal("expected first connection attempt to fail")
}
key := getCacheKey(config)
a.mu.Lock()
a.connectFailures[key] = cachedConnectFailure{
occurredAt: time.Now().Add(-dbConnectFailureCooldown - time.Second),
err: errors.New("dial tcp 10.1.131.86:5432: connect: connection refused"),
}
a.mu.Unlock()
_, secondErr := a.getDatabaseWithPing(config, false)
if secondErr != nil {
t.Fatalf("expected retry after cooldown to succeed, got error: %v", secondErr)
}
a.mu.RLock()
_, exists := a.connectFailures[key]
a.mu.RUnlock()
if exists {
t.Fatal("expected successful connection to clear cached failure cooldown")
}
}