mirror of
https://github.com/DullJZ/s3-balance.git
synced 2026-07-02 16:41:22 +08:00
Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d9258d4924 | ||
|
|
9738091c87 | ||
|
|
72fac92783 | ||
|
|
322d41fbb2 |
@@ -88,12 +88,12 @@ go build -o s3-balance cmd/s3-balance/main.go
|
||||
- `database`:GORM 支持 sqlite/mysql/postgres,并存储对象元数据、分片会话等。
|
||||
- `buckets`:列出真实与虚拟桶。`virtual: true` 的条目会对外暴露,真实桶为 `virtual: false`。可设置 `path_style` 与 `max_size`。
|
||||
- `balancer`:策略 (`round-robin`|`least-space`|`weighted`)、健康检查周期、重试次数与延迟。
|
||||
- `metrics`:是否启用 Prometheus 指标及路径。
|
||||
- `metrics`:是否启用 Prometheus 指标、路径与可选抓取 Token。
|
||||
- `s3api`:Access/Secret Key、`proxy_mode`(true=服务代理,false=重定向)、`auth_required`(SigV4 校验)、`virtual_host`(Host-style 路由)。
|
||||
|
||||
## API & 测试
|
||||
|
||||
- 默认监听 `http://localhost:8080`,支持 `GET /health` 健康检查、`GET /metrics` 指标。
|
||||
- 默认监听 `http://localhost:8080`,支持 `GET /health` 健康检查、`GET /metrics` 指标(配置 `metrics.token` 后需携带 `Authorization: Bearer <token>`)。
|
||||
- 可使用 AWS CLI、s3cmd、MinIO Client 或 `python3 test_virtual_bucket_s3.py` 验证兼容性;脚本运行前需修改 endpoint 与凭据。
|
||||
|
||||
## 项目结构
|
||||
|
||||
@@ -86,12 +86,12 @@ go build -o s3-balance cmd/s3-balance/main.go
|
||||
- `database`: GORM supports sqlite/mysql/postgres and stores object metadata, multipart sessions, etc.
|
||||
- `buckets`: Lists real and virtual buckets. Entries with `virtual: true` are exposed externally, while real buckets are marked as `virtual: false`. Supports `path_style` and `max_size` settings.
|
||||
- `balancer`: Strategy (`round-robin`|`least-space`|`weighted`), health check intervals, retry counts, and delays.
|
||||
- `metrics`: Whether to enable Prometheus metrics and their path.
|
||||
- `metrics`: Whether to enable Prometheus metrics, the path, and optional scrape token.
|
||||
- `s3api`: Access/Secret Key, `proxy_mode` (true=proxy, false=redirect), `auth_required` (SigV4 validation), `virtual_host` (host-style routing).
|
||||
|
||||
## API & Testing
|
||||
|
||||
- Default listening at `http://localhost:8080`, supports `GET /health` for health checks and `GET /metrics` for metrics.
|
||||
- Default listening at `http://localhost:8080`, supports `GET /health` for health checks and `GET /metrics` for metrics (if `metrics.token` is set, send `Authorization: Bearer <token>`).
|
||||
- Compatibility can be verified using AWS CLI, s3cmd, MinIO Client, or `python3 test_virtual_bucket_s3.py`. Modify the endpoint and credentials in the script before running.
|
||||
|
||||
## Project Structure
|
||||
@@ -106,4 +106,4 @@ internal/storage/ # GORM models and services
|
||||
pkg/presigner/ # Pre-signed URL utilities
|
||||
config/ # Example configurations and deployment manifests
|
||||
deploy/ # Docker/Kubernetes/Helm manifests
|
||||
```
|
||||
```
|
||||
|
||||
@@ -137,8 +137,16 @@ func main() {
|
||||
|
||||
// 添加指标端点
|
||||
if cfg.Metrics.Enabled {
|
||||
router.Path(cfg.Metrics.Path).Handler(promhttp.Handler())
|
||||
log.Printf("Metrics server enabled at %s", cfg.Metrics.Path)
|
||||
metricsHandler := promhttp.Handler()
|
||||
if cfg.Metrics.Token != "" {
|
||||
metricsHandler = middleware.TokenAuthMiddleware(cfg.Metrics.Token)(metricsHandler)
|
||||
}
|
||||
router.Path(cfg.Metrics.Path).Handler(metricsHandler)
|
||||
log.Printf(
|
||||
"Metrics server enabled at %s (auth required: %t)",
|
||||
cfg.Metrics.Path,
|
||||
cfg.Metrics.Token != "",
|
||||
)
|
||||
}
|
||||
|
||||
// 注册管理API路由(如果启用)
|
||||
|
||||
@@ -128,6 +128,8 @@ balancer:
|
||||
metrics:
|
||||
enabled: true
|
||||
path: "/metrics"
|
||||
# Prometheus 抓取时需要携带的 Token(可选,配置后需使用 Authorization: Bearer <token> 访问)
|
||||
token: ""
|
||||
|
||||
# S3兼容API配置
|
||||
s3api:
|
||||
|
||||
@@ -44,6 +44,7 @@ type BucketResponse struct {
|
||||
UsedSize int64 `json:"used_size"`
|
||||
AvailableSize int64 `json:"available_size"`
|
||||
UsagePercent float64 `json:"usage_percent"`
|
||||
ObjectCount int64 `json:"object_count"`
|
||||
Weight int `json:"weight"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Available bool `json:"available"`
|
||||
@@ -59,18 +60,18 @@ type BucketResponse struct {
|
||||
|
||||
// BucketsListResponse 存储桶列表响应结构
|
||||
type BucketsListResponse struct {
|
||||
Total int `json:"total"`
|
||||
Buckets []BucketResponse `json:"buckets"`
|
||||
Total int `json:"total"`
|
||||
Buckets []BucketResponse `json:"buckets"`
|
||||
}
|
||||
|
||||
// HealthResponse 健康状态响应结构
|
||||
type HealthResponse struct {
|
||||
Status string `json:"status"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
LoadBalancer string `json:"load_balancer_strategy"`
|
||||
TotalBuckets int `json:"total_buckets"`
|
||||
AvailableBuckets int `json:"available_buckets"`
|
||||
Database string `json:"database_type"`
|
||||
Status string `json:"status"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
LoadBalancer string `json:"load_balancer_strategy"`
|
||||
TotalBuckets int `json:"total_buckets"`
|
||||
AvailableBuckets int `json:"available_buckets"`
|
||||
Database string `json:"database_type"`
|
||||
}
|
||||
|
||||
// RegisterRoutes 注册管理API路由
|
||||
@@ -205,6 +206,7 @@ func (h *AdminHandler) convertBucketInfo(b *bucket.BucketInfo) BucketResponse {
|
||||
MaxSize: b.Config.MaxSize,
|
||||
MaxSizeBytes: b.Config.MaxSizeBytes,
|
||||
UsedSize: b.UsedSize,
|
||||
ObjectCount: b.GetObjectCount(),
|
||||
Weight: b.Config.Weight,
|
||||
Enabled: b.Config.Enabled,
|
||||
Available: b.Available,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/DullJZ/s3-balance/internal/balancer"
|
||||
@@ -73,6 +74,7 @@ func (h *S3Handler) initSettings(accessKey, secretKey string, proxyMode, authReq
|
||||
func (h *S3Handler) RegisterS3Routes(router *mux.Router) {
|
||||
// 公共路由(不需要认证)
|
||||
router.HandleFunc("/", h.handleListBuckets).Methods("GET")
|
||||
router.HandleFunc("/", h.handleOptions).Methods("OPTIONS")
|
||||
|
||||
// 带认证/虚拟主机的路由
|
||||
protected := router.NewRoute().PathPrefix("/{bucket}").Subrouter()
|
||||
@@ -80,7 +82,9 @@ func (h *S3Handler) RegisterS3Routes(router *mux.Router) {
|
||||
|
||||
// Bucket operations
|
||||
protected.HandleFunc("", h.handleBucketOperations).Methods("GET", "HEAD", "PUT", "DELETE")
|
||||
protected.HandleFunc("", h.handleOptions).Methods("OPTIONS")
|
||||
protected.HandleFunc("/", h.handleBucketOperations).Methods("GET", "HEAD", "PUT", "DELETE")
|
||||
protected.HandleFunc("/", h.handleOptions).Methods("OPTIONS")
|
||||
|
||||
// Multipart upload operations - must be registered before generic object operations
|
||||
protected.HandleFunc("/{key:.*}", h.handleUploadPart).Methods("PUT").Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId}")
|
||||
@@ -92,6 +96,7 @@ func (h *S3Handler) RegisterS3Routes(router *mux.Router) {
|
||||
|
||||
// Object operations - must be registered after multipart operations to avoid conflicts
|
||||
protected.HandleFunc("/{key:.*}", h.handleObjectOperations).Methods("GET", "HEAD", "PUT", "DELETE")
|
||||
protected.HandleFunc("/{key:.*}", h.handleOptions).Methods("OPTIONS")
|
||||
|
||||
// 添加中间件
|
||||
protected.Use(h.accessLogMiddleware)
|
||||
@@ -110,6 +115,10 @@ func (h *S3Handler) RegisterS3Routes(router *mux.Router) {
|
||||
}))
|
||||
}
|
||||
|
||||
func (h *S3Handler) handleOptions(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *S3Handler) loadSettings() handlerSettings {
|
||||
if v := h.settings.Load(); v != nil {
|
||||
return v.(handlerSettings)
|
||||
|
||||
@@ -32,6 +32,7 @@ type BucketInfo struct {
|
||||
Config config.BucketConfig
|
||||
Client *s3.Client
|
||||
UsedSize int64 // 已使用容量(字节)
|
||||
ObjectCount int64 // 对象数量
|
||||
Available bool // 是否可用(由health监控更新)
|
||||
LastChecked time.Time // 最后检查时间(由health监控更新)
|
||||
mu sync.RWMutex
|
||||
@@ -320,6 +321,13 @@ func (b *BucketInfo) GetUsedSize() int64 {
|
||||
return b.UsedSize
|
||||
}
|
||||
|
||||
// GetObjectCount 获取对象数量
|
||||
func (b *BucketInfo) GetObjectCount() int64 {
|
||||
b.mu.RLock()
|
||||
defer b.mu.RUnlock()
|
||||
return b.ObjectCount
|
||||
}
|
||||
|
||||
// UpdateUsedSize 更新已使用容量
|
||||
func (b *BucketInfo) UpdateUsedSize(delta int64) {
|
||||
b.mu.Lock()
|
||||
|
||||
@@ -60,6 +60,7 @@ func (r *MetricsReporter) ReportStats(stats *health.Stats) {
|
||||
if exists {
|
||||
bucket.mu.Lock()
|
||||
bucket.UsedSize = stats.UsedSize
|
||||
bucket.ObjectCount = stats.ObjectCount
|
||||
bucket.mu.Unlock()
|
||||
|
||||
// 更新 Prometheus 指标
|
||||
|
||||
@@ -63,6 +63,7 @@ type BalancerConfig struct {
|
||||
type MetricsConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Path string `yaml:"path"`
|
||||
Token string `yaml:"token"` // 可选Token,保护Prometheus端点
|
||||
// Port int `yaml:"port"` // 目前未使用,与主服务共享端口
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user