mirror of
https://github.com/Syngnat/GoNavi.git
synced 2026-08-25 02:00:09 +08:00
🐛 fix(connection): 修复失败连接高频重试并暂停后台自动元数据拉取 #331
- 后端为失败数据库连接增加冷却窗口,避免短时间内重复真实建连 - 补充失败冷却回归测试,覆盖重复失败、冷却后重试和成功后清理场景 - 前端在后台态暂停查询页、侧边栏和表概览的自动元数据拉取 - 保持手动刷新、手动展开等显式操作行为不变
This commit is contained in:
@@ -23,6 +23,7 @@ import (
|
||||
)
|
||||
|
||||
const dbCachePingInterval = 30 * time.Second
|
||||
const dbConnectFailureCooldown = 30 * time.Second
|
||||
|
||||
const (
|
||||
startupConnectRetryWindow = 20 * time.Second
|
||||
@@ -40,6 +41,11 @@ type cachedDatabase struct {
|
||||
lastPing time.Time
|
||||
}
|
||||
|
||||
type cachedConnectFailure struct {
|
||||
occurredAt time.Time
|
||||
err error
|
||||
}
|
||||
|
||||
type queryContext struct {
|
||||
cancel context.CancelFunc
|
||||
started time.Time
|
||||
@@ -50,6 +56,7 @@ type App struct {
|
||||
ctx context.Context
|
||||
startedAt time.Time
|
||||
dbCache map[string]cachedDatabase // Cache for DB connections
|
||||
connectFailures map[string]cachedConnectFailure
|
||||
mu sync.RWMutex // Mutex for cache access
|
||||
updateMu sync.Mutex
|
||||
updateState updateState
|
||||
@@ -70,6 +77,7 @@ func NewAppWithSecretStore(store secretstore.SecretStore) *App {
|
||||
}
|
||||
return &App{
|
||||
dbCache: make(map[string]cachedDatabase),
|
||||
connectFailures: make(map[string]cachedConnectFailure),
|
||||
runningQueries: make(map[string]queryContext),
|
||||
configDir: resolveAppConfigDir(),
|
||||
secretStore: store,
|
||||
@@ -573,14 +581,28 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
|
||||
if isFileDB {
|
||||
logger.Infof("未命中文件库连接缓存,开始创建连接:类型=%s 缓存Key=%s", strings.TrimSpace(effectiveConfig.Type), shortKey)
|
||||
}
|
||||
if failure, remaining, ok := a.getCachedConnectFailureByKey(key); ok {
|
||||
message := fmt.Sprintf("连接最近失败,正在冷却中,请 %s 后重试;上次错误:%s",
|
||||
formatConnectFailureCooldown(remaining),
|
||||
normalizeErrorMessage(failure.err),
|
||||
)
|
||||
logger.Warnf("命中数据库连接失败冷却:%s 缓存Key=%s 剩余=%s 原因=%s",
|
||||
formatConnSummary(effectiveConfig), shortKey, formatConnectFailureCooldown(remaining), normalizeErrorMessage(failure.err))
|
||||
return nil, withLogHint{err: fmt.Errorf("%s", message), logPath: logger.Path()}
|
||||
}
|
||||
|
||||
initialKey := key
|
||||
dbInst, connectedConfig, err := a.connectDatabaseWithStartupRetry(resolvedConfig)
|
||||
if err != nil {
|
||||
failedKey := getCacheKey(connectedConfig)
|
||||
a.recordConnectFailureByKey(failedKey, err)
|
||||
return nil, err
|
||||
}
|
||||
a.clearConnectFailureByKey(initialKey)
|
||||
effectiveConfig = connectedConfig
|
||||
key = getCacheKey(effectiveConfig)
|
||||
shortKey = shortenCacheKey(key)
|
||||
a.clearConnectFailureByKey(key)
|
||||
|
||||
now := time.Now()
|
||||
|
||||
@@ -601,6 +623,62 @@ func (a *App) getDatabaseWithPing(config connection.ConnectionConfig, forcePing
|
||||
return dbInst, nil
|
||||
}
|
||||
|
||||
func (a *App) getCachedConnectFailureByKey(key string) (cachedConnectFailure, time.Duration, bool) {
|
||||
if a == nil || strings.TrimSpace(key) == "" {
|
||||
return cachedConnectFailure{}, 0, false
|
||||
}
|
||||
|
||||
a.mu.RLock()
|
||||
entry, exists := a.connectFailures[key]
|
||||
a.mu.RUnlock()
|
||||
if !exists || entry.err == nil || entry.occurredAt.IsZero() {
|
||||
return cachedConnectFailure{}, 0, false
|
||||
}
|
||||
|
||||
remaining := dbConnectFailureCooldown - time.Since(entry.occurredAt)
|
||||
if remaining <= 0 {
|
||||
a.clearConnectFailureByKey(key)
|
||||
return cachedConnectFailure{}, 0, false
|
||||
}
|
||||
|
||||
return entry, remaining, true
|
||||
}
|
||||
|
||||
func (a *App) recordConnectFailureByKey(key string, err error) {
|
||||
if a == nil || strings.TrimSpace(key) == "" || err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
if a.connectFailures == nil {
|
||||
a.connectFailures = make(map[string]cachedConnectFailure)
|
||||
}
|
||||
a.connectFailures[key] = cachedConnectFailure{
|
||||
occurredAt: time.Now(),
|
||||
err: err,
|
||||
}
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
func (a *App) clearConnectFailureByKey(key string) {
|
||||
if a == nil || strings.TrimSpace(key) == "" {
|
||||
return
|
||||
}
|
||||
|
||||
a.mu.Lock()
|
||||
if a.connectFailures != nil {
|
||||
delete(a.connectFailures, key)
|
||||
}
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
func formatConnectFailureCooldown(remaining time.Duration) time.Duration {
|
||||
if remaining <= time.Second {
|
||||
return time.Second
|
||||
}
|
||||
return remaining.Truncate(time.Second)
|
||||
}
|
||||
|
||||
func shortenCacheKey(key string) string {
|
||||
if len(key) > 12 {
|
||||
return key[:12]
|
||||
|
||||
@@ -303,3 +303,167 @@ func TestIsTransientStartupConnectError(t *testing.T) {
|
||||
t.Fatal("expected authentication failure to not be treated as transient startup connect error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDatabaseWithPing_CoolsDownRepeatedFailures(t *testing.T) {
|
||||
originalNewDatabaseFunc := newDatabaseFunc
|
||||
originalResolveDialConfigWithProxyFunc := resolveDialConfigWithProxyFunc
|
||||
defer func() {
|
||||
newDatabaseFunc = originalNewDatabaseFunc
|
||||
resolveDialConfigWithProxyFunc = originalResolveDialConfigWithProxyFunc
|
||||
}()
|
||||
|
||||
connectCalls := 0
|
||||
newDatabaseFunc = func(dbType string) (db.Database, error) {
|
||||
return &fakeStartupRetryDB{
|
||||
connect: func(config connection.ConnectionConfig) error {
|
||||
connectCalls++
|
||||
return errors.New("dial tcp 10.1.131.86:5432: connect: connection refused")
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
resolveDialConfigWithProxyFunc = func(raw connection.ConnectionConfig) (connection.ConnectionConfig, error) {
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
a := &App{
|
||||
startedAt: time.Now().Add(-startupConnectRetryWindow - time.Second),
|
||||
dbCache: make(map[string]cachedDatabase),
|
||||
connectFailures: make(map[string]cachedConnectFailure),
|
||||
runningQueries: make(map[string]queryContext),
|
||||
}
|
||||
config := connection.ConnectionConfig{Type: "postgres", Host: "10.1.131.86", Port: 5432, User: "postgres"}
|
||||
|
||||
_, firstErr := a.getDatabaseWithPing(config, false)
|
||||
if firstErr == nil {
|
||||
t.Fatal("expected first connection attempt to fail")
|
||||
}
|
||||
if connectCalls != 2 {
|
||||
t.Fatalf("expected first request to use 2 connect attempts, got %d", connectCalls)
|
||||
}
|
||||
|
||||
_, secondErr := a.getDatabaseWithPing(config, false)
|
||||
if secondErr == nil {
|
||||
t.Fatal("expected second connection attempt to fail during cooldown")
|
||||
}
|
||||
if connectCalls != 2 {
|
||||
t.Fatalf("expected repeated request during cooldown to avoid reconnecting, got %d connect attempts", connectCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDatabaseWithPing_AllowsRetryAfterFailureCooldown(t *testing.T) {
|
||||
originalNewDatabaseFunc := newDatabaseFunc
|
||||
originalResolveDialConfigWithProxyFunc := resolveDialConfigWithProxyFunc
|
||||
defer func() {
|
||||
newDatabaseFunc = originalNewDatabaseFunc
|
||||
resolveDialConfigWithProxyFunc = originalResolveDialConfigWithProxyFunc
|
||||
}()
|
||||
|
||||
connectCalls := 0
|
||||
newDatabaseFunc = func(dbType string) (db.Database, error) {
|
||||
return &fakeStartupRetryDB{
|
||||
connect: func(config connection.ConnectionConfig) error {
|
||||
connectCalls++
|
||||
if connectCalls <= 2 {
|
||||
return errors.New("dial tcp 10.1.131.86:5432: connect: connection refused")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
resolveDialConfigWithProxyFunc = func(raw connection.ConnectionConfig) (connection.ConnectionConfig, error) {
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
a := &App{
|
||||
startedAt: time.Now().Add(-startupConnectRetryWindow - time.Second),
|
||||
dbCache: make(map[string]cachedDatabase),
|
||||
connectFailures: make(map[string]cachedConnectFailure),
|
||||
runningQueries: make(map[string]queryContext),
|
||||
}
|
||||
config := connection.ConnectionConfig{Type: "postgres", Host: "10.1.131.86", Port: 5432, User: "postgres"}
|
||||
|
||||
_, firstErr := a.getDatabaseWithPing(config, false)
|
||||
if firstErr == nil {
|
||||
t.Fatal("expected first connection attempt to fail")
|
||||
}
|
||||
if connectCalls != 2 {
|
||||
t.Fatalf("expected first request to use 2 connect attempts, got %d", connectCalls)
|
||||
}
|
||||
|
||||
key := getCacheKey(config)
|
||||
a.mu.Lock()
|
||||
a.connectFailures[key] = cachedConnectFailure{
|
||||
occurredAt: time.Now().Add(-dbConnectFailureCooldown - time.Second),
|
||||
err: errors.New("dial tcp 10.1.131.86:5432: connect: connection refused"),
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
inst, secondErr := a.getDatabaseWithPing(config, false)
|
||||
if secondErr != nil {
|
||||
t.Fatalf("expected retry after cooldown to be allowed, got error: %v", secondErr)
|
||||
}
|
||||
if inst == nil {
|
||||
t.Fatal("expected database instance after cooldown retry")
|
||||
}
|
||||
if connectCalls != 3 {
|
||||
t.Fatalf("expected reconnect after cooldown expiration, got %d connect attempts", connectCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDatabaseWithPing_ClearsFailureCooldownAfterSuccess(t *testing.T) {
|
||||
originalNewDatabaseFunc := newDatabaseFunc
|
||||
originalResolveDialConfigWithProxyFunc := resolveDialConfigWithProxyFunc
|
||||
defer func() {
|
||||
newDatabaseFunc = originalNewDatabaseFunc
|
||||
resolveDialConfigWithProxyFunc = originalResolveDialConfigWithProxyFunc
|
||||
}()
|
||||
|
||||
connectCalls := 0
|
||||
newDatabaseFunc = func(dbType string) (db.Database, error) {
|
||||
return &fakeStartupRetryDB{
|
||||
connect: func(config connection.ConnectionConfig) error {
|
||||
connectCalls++
|
||||
if connectCalls <= 2 {
|
||||
return errors.New("dial tcp 10.1.131.86:5432: connect: connection refused")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
resolveDialConfigWithProxyFunc = func(raw connection.ConnectionConfig) (connection.ConnectionConfig, error) {
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
a := &App{
|
||||
startedAt: time.Now().Add(-startupConnectRetryWindow - time.Second),
|
||||
dbCache: make(map[string]cachedDatabase),
|
||||
connectFailures: make(map[string]cachedConnectFailure),
|
||||
runningQueries: make(map[string]queryContext),
|
||||
}
|
||||
config := connection.ConnectionConfig{Type: "postgres", Host: "10.1.131.86", Port: 5432, User: "postgres"}
|
||||
|
||||
_, firstErr := a.getDatabaseWithPing(config, false)
|
||||
if firstErr == nil {
|
||||
t.Fatal("expected first connection attempt to fail")
|
||||
}
|
||||
|
||||
key := getCacheKey(config)
|
||||
a.mu.Lock()
|
||||
a.connectFailures[key] = cachedConnectFailure{
|
||||
occurredAt: time.Now().Add(-dbConnectFailureCooldown - time.Second),
|
||||
err: errors.New("dial tcp 10.1.131.86:5432: connect: connection refused"),
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
_, secondErr := a.getDatabaseWithPing(config, false)
|
||||
if secondErr != nil {
|
||||
t.Fatalf("expected retry after cooldown to succeed, got error: %v", secondErr)
|
||||
}
|
||||
|
||||
a.mu.RLock()
|
||||
_, exists := a.connectFailures[key]
|
||||
a.mu.RUnlock()
|
||||
if exists {
|
||||
t.Fatal("expected successful connection to clear cached failure cooldown")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user