mirror of
https://github.com/DullJZ/s3-balance.git
synced 2026-07-06 18:41:24 +08:00
Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4ae2a54da3 | ||
|
|
a36b6a13f4 | ||
|
|
7e1f4bbee3 | ||
|
|
b55699513f | ||
|
|
908881574d | ||
|
|
4b44dc1a1b | ||
|
|
34a905defd | ||
|
|
d1d970705c | ||
|
|
42199c16c1 |
59
.github/workflows/build.yml
vendored
59
.github/workflows/build.yml
vendored
@@ -86,6 +86,7 @@ jobs:
|
|||||||
- name: Create Release
|
- name: Create Release
|
||||||
id: create_release
|
id: create_release
|
||||||
uses: actions/create-release@v1
|
uses: actions/create-release@v1
|
||||||
|
continue-on-error: true
|
||||||
env:
|
env:
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
with:
|
with:
|
||||||
@@ -94,10 +95,62 @@ jobs:
|
|||||||
draft: false
|
draft: false
|
||||||
prerelease: false
|
prerelease: false
|
||||||
|
|
||||||
- name: Upload All Release Assets
|
- name: Upload Linux AMD64 Release Asset
|
||||||
uses: actions/upload-release-asset@v1
|
uses: actions/upload-release-asset@v1
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
with:
|
with:
|
||||||
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||||
asset_path: ./build
|
asset_path: ./build/S3Balance_linux_amd64
|
||||||
asset_name: ""
|
asset_name: S3Balance_linux_amd64
|
||||||
|
asset_content_type: application/octet-stream
|
||||||
|
|
||||||
|
- name: Upload Linux ARM64 Release Asset
|
||||||
|
uses: actions/upload-release-asset@v1
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
with:
|
||||||
|
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||||
|
asset_path: ./build/S3Balance_linux_arm64
|
||||||
|
asset_name: S3Balance_linux_arm64
|
||||||
|
asset_content_type: application/octet-stream
|
||||||
|
|
||||||
|
- name: Upload Darwin AMD64 Release Asset
|
||||||
|
uses: actions/upload-release-asset@v1
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
with:
|
||||||
|
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||||
|
asset_path: ./build/S3Balance_darwin_amd64
|
||||||
|
asset_name: S3Balance_darwin_amd64
|
||||||
|
asset_content_type: application/octet-stream
|
||||||
|
|
||||||
|
- name: Upload Darwin ARM64 Release Asset
|
||||||
|
uses: actions/upload-release-asset@v1
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
with:
|
||||||
|
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||||
|
asset_path: ./build/S3Balance_darwin_arm64
|
||||||
|
asset_name: S3Balance_darwin_arm64
|
||||||
|
asset_content_type: application/octet-stream
|
||||||
|
|
||||||
|
- name: Upload Windows AMD64 Release Asset
|
||||||
|
uses: actions/upload-release-asset@v1
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
with:
|
||||||
|
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||||
|
asset_path: ./build/S3Balance_windows_amd64.exe
|
||||||
|
asset_name: S3Balance_windows_amd64.exe
|
||||||
|
asset_content_type: application/octet-stream
|
||||||
|
|
||||||
|
- name: Upload Windows ARM64 Release Asset
|
||||||
|
uses: actions/upload-release-asset@v1
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
with:
|
||||||
|
upload_url: ${{ steps.create_release.outputs.upload_url }}
|
||||||
|
asset_path: ./build/S3Balance_windows_arm64.exe
|
||||||
|
asset_name: S3Balance_windows_arm64.exe
|
||||||
asset_content_type: application/octet-stream
|
asset_content_type: application/octet-stream
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ S3 Balance 是一个用 Go 编写的 S3 兼容负载均衡器,可在多套对
|
|||||||
- **多桶调度**:支持轮询、剩余空间、加权和一致性哈希等策略,可热切换。
|
- **多桶调度**:支持轮询、剩余空间、加权和一致性哈希等策略,可热切换。
|
||||||
- **健康监控**:周期性探活与容量统计,Prometheus 指标暴露在 `/metrics`。
|
- **健康监控**:周期性探活与容量统计,Prometheus 指标暴露在 `/metrics`。
|
||||||
- **虚拟桶映射**:对外只暴露虚拟桶名称,真实桶在后端透明调度。
|
- **虚拟桶映射**:对外只暴露虚拟桶名称,真实桶在后端透明调度。
|
||||||
- **代理或重定向模式**:可选择由服务转发数据,或返回预签名 URL 让客户端直连。
|
- **代理或重定向模式**:可选择由服务转发数据,或返回302重定向让客户端直连。
|
||||||
- **SigV4 认证**:配置 `s3api.auth_required` 后,通过 `github.com/DullJZ/s3-validate` 校验 AWS Signature Version 4 请求。
|
- **SigV4 认证**:配置 `s3api.auth_required` 后,通过 `github.com/DullJZ/s3-validate` 校验 AWS Signature Version 4 请求。
|
||||||
|
|
||||||
## 快速开始
|
## 快速开始
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ S3 Balance is an S3-compatible load balancer written in Go. It automatically sel
|
|||||||
- **Multi-bucket Scheduling**: Supports strategies like round-robin, least-space, weighted, and consistent hashing, with hot-swapping capabilities.
|
- **Multi-bucket Scheduling**: Supports strategies like round-robin, least-space, weighted, and consistent hashing, with hot-swapping capabilities.
|
||||||
- **Health Monitoring**: Periodic health checks and capacity statistics, with Prometheus metrics exposed at `/metrics`.
|
- **Health Monitoring**: Periodic health checks and capacity statistics, with Prometheus metrics exposed at `/metrics`.
|
||||||
- **Virtual Bucket Mapping**: Only virtual bucket names are exposed externally, while real buckets are transparently scheduled in the backend.
|
- **Virtual Bucket Mapping**: Only virtual bucket names are exposed externally, while real buckets are transparently scheduled in the backend.
|
||||||
- **Proxy or Redirect Mode**: Choose between service-forwarded data or pre-signed URLs for direct client connections.
|
- **Proxy or Redirect Mode**: Choose between service-forwarded data or 302 redirects for direct client connections.
|
||||||
- **SigV4 Authentication**: When `s3api.auth_required` is enabled, AWS Signature Version 4 requests are validated using `github.com/DullJZ/s3-validate`.
|
- **SigV4 Authentication**: When `s3api.auth_required` is enabled, AWS Signature Version 4 requests are validated using `github.com/DullJZ/s3-validate`.
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
|
|||||||
@@ -92,6 +92,7 @@ func main() {
|
|||||||
cfg.S3API.ProxyMode,
|
cfg.S3API.ProxyMode,
|
||||||
cfg.S3API.AuthRequired,
|
cfg.S3API.AuthRequired,
|
||||||
cfg.S3API.VirtualHost,
|
cfg.S3API.VirtualHost,
|
||||||
|
cfg.S3API.Host,
|
||||||
)
|
)
|
||||||
|
|
||||||
// 注册配置热更新回调
|
// 注册配置热更新回调
|
||||||
|
|||||||
@@ -138,3 +138,9 @@ s3api:
|
|||||||
|
|
||||||
# 是否需要认证(开启后使用 Basic Auth,凭据来自 access_key/secret_key)
|
# 是否需要认证(开启后使用 Basic Auth,凭据来自 access_key/secret_key)
|
||||||
auth_required: true
|
auth_required: true
|
||||||
|
|
||||||
|
# 用于签名验证的Host(可选)
|
||||||
|
# 当服务前有 nginx 等反向代理时,可以设置此项为客户端实际访问的域名
|
||||||
|
# 留空则使用请求中的 Host 头
|
||||||
|
# 示例: "s3.example.com" 或 "s3.example.com:8080"
|
||||||
|
host: ""
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
# 多阶段构建用于减小镜像大小
|
# 多阶段构建用于减小镜像大小
|
||||||
FROM golang:1.24.5-alpine AS builder
|
FROM --platform=$BUILDPLATFORM golang:1.24.5-alpine AS builder
|
||||||
|
|
||||||
# 安装构建依赖
|
# 安装构建依赖
|
||||||
# 注意:不再需要 gcc, musl-dev, sqlite-dev,因为使用 modernc.org/sqlite 纯Go驱动
|
# 注意:不再需要 gcc, musl-dev, sqlite-dev,因为使用 modernc.org/sqlite 纯Go驱动
|
||||||
@@ -16,8 +16,8 @@ RUN go mod download && go mod tidy
|
|||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
# 添加构建参数以支持多架构
|
# 添加构建参数以支持多架构
|
||||||
ARG TARGETOS=linux
|
ARG TARGETOS
|
||||||
ARG TARGETARCH=amd64
|
ARG TARGETARCH
|
||||||
|
|
||||||
# 构建应用
|
# 构建应用
|
||||||
# CGO_ENABLED=0: 禁用CGO,使用纯Go SQLite驱动 (modernc.org/sqlite)
|
# CGO_ENABLED=0: 禁用CGO,使用纯Go SQLite驱动 (modernc.org/sqlite)
|
||||||
|
|||||||
@@ -56,4 +56,5 @@ s3api:
|
|||||||
secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||||
virtual_host: false
|
virtual_host: false
|
||||||
proxy_mode: true
|
proxy_mode: true
|
||||||
auth_required: true
|
auth_required: true
|
||||||
|
host: ""
|
||||||
2
go.mod
2
go.mod
@@ -3,7 +3,7 @@ module github.com/DullJZ/s3-balance
|
|||||||
go 1.24.5
|
go 1.24.5
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/DullJZ/s3-validate v0.0.0-20250930120412-fc4ea70939f6
|
github.com/DullJZ/s3-validate v0.0.0-20251103105435-c25eac6b580b
|
||||||
github.com/aws/aws-sdk-go-v2 v1.39.2
|
github.com/aws/aws-sdk-go-v2 v1.39.2
|
||||||
github.com/aws/aws-sdk-go-v2/config v1.31.1
|
github.com/aws/aws-sdk-go-v2/config v1.31.1
|
||||||
github.com/aws/aws-sdk-go-v2/credentials v1.18.5
|
github.com/aws/aws-sdk-go-v2/credentials v1.18.5
|
||||||
|
|||||||
4
go.sum
4
go.sum
@@ -1,7 +1,7 @@
|
|||||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||||
github.com/DullJZ/s3-validate v0.0.0-20250930120412-fc4ea70939f6 h1:UZ4i/MFU0ttINUch3GYyJayq6Y2ODm+RPawLgPna5L8=
|
github.com/DullJZ/s3-validate v0.0.0-20251103105435-c25eac6b580b h1:BHue7N77inSdaDUUZSO/gMmc3+4ZGdQA3ORdcLHnxtg=
|
||||||
github.com/DullJZ/s3-validate v0.0.0-20250930120412-fc4ea70939f6/go.mod h1:OEx+/bRlDdI0oj/Bb1Plsq+1+qU1qal3/g9phixhU6Y=
|
github.com/DullJZ/s3-validate v0.0.0-20251103105435-c25eac6b580b/go.mod h1:OEx+/bRlDdI0oj/Bb1Plsq+1+qU1qal3/g9phixhU6Y=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.39.2 h1:EJLg8IdbzgeD7xgvZ+I8M1e0fL0ptn/M47lianzth0I=
|
github.com/aws/aws-sdk-go-v2 v1.39.2 h1:EJLg8IdbzgeD7xgvZ+I8M1e0fL0ptn/M47lianzth0I=
|
||||||
github.com/aws/aws-sdk-go-v2 v1.39.2/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY=
|
github.com/aws/aws-sdk-go-v2 v1.39.2/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY=
|
||||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg=
|
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg=
|
||||||
|
|||||||
216
internal/api/access_log.go
Normal file
216
internal/api/access_log.go
Normal file
@@ -0,0 +1,216 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"log"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/gorilla/mux"
|
||||||
|
)
|
||||||
|
|
||||||
|
type accessLogContextKey string
|
||||||
|
|
||||||
|
const (
|
||||||
|
errorCodeKey accessLogContextKey = "errorCode"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (h *S3Handler) accessLogMiddleware(next http.Handler) http.Handler {
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if h.storage == nil {
|
||||||
|
next.ServeHTTP(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
lrw := newLoggingResponseWriter(w)
|
||||||
|
next.ServeHTTP(lrw, r)
|
||||||
|
|
||||||
|
vars := mux.Vars(r)
|
||||||
|
bucket := vars["bucket"]
|
||||||
|
key := vars["key"]
|
||||||
|
action := determineAccessAction(r, bucket, key)
|
||||||
|
if action == "" && bucket == "" && key == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
success := lrw.statusCode < 400
|
||||||
|
errMsg := ""
|
||||||
|
if !success {
|
||||||
|
// 优先使用context中的错误码(auth错误设置的)
|
||||||
|
if code, ok := r.Context().Value(errorCodeKey).(string); ok && code != "" {
|
||||||
|
errMsg = code
|
||||||
|
} else if code := lrw.Header().Get("X-Amz-Error-Code"); code != "" {
|
||||||
|
errMsg = code
|
||||||
|
} else {
|
||||||
|
errMsg = http.StatusText(lrw.statusCode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
size := calculateLogSize(r, lrw)
|
||||||
|
duration := time.Since(start)
|
||||||
|
h.recordAccessLog(r, action, bucket, key, size, success, errMsg, duration)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *S3Handler) recordAccessLog(r *http.Request, action, bucket, key string, size int64, success bool, errMsg string, duration time.Duration) {
|
||||||
|
clientIP := extractClientIP(r)
|
||||||
|
userAgent := r.UserAgent()
|
||||||
|
host := r.Host
|
||||||
|
// 异步记录日志,避免阻塞请求响应
|
||||||
|
go func() {
|
||||||
|
if err := h.storage.RecordAccessLog(action, key, bucket, clientIP, userAgent, host, size, success, errMsg, duration.Milliseconds()); err != nil {
|
||||||
|
log.Printf("failed to record access log: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *S3Handler) handleAuthError(w http.ResponseWriter, r *http.Request, code, message, resource string) {
|
||||||
|
// 将错误码存入context,供accessLogMiddleware使用
|
||||||
|
ctx := context.WithValue(r.Context(), errorCodeKey, code)
|
||||||
|
*r = *r.WithContext(ctx)
|
||||||
|
|
||||||
|
h.sendS3Error(w, code, message, resource)
|
||||||
|
}
|
||||||
|
|
||||||
|
type loggingResponseWriter struct {
|
||||||
|
http.ResponseWriter
|
||||||
|
statusCode int
|
||||||
|
bytesWritten int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func newLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
|
||||||
|
return &loggingResponseWriter{
|
||||||
|
ResponseWriter: w,
|
||||||
|
statusCode: http.StatusOK,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (lrw *loggingResponseWriter) WriteHeader(statusCode int) {
|
||||||
|
lrw.statusCode = statusCode
|
||||||
|
lrw.ResponseWriter.WriteHeader(statusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (lrw *loggingResponseWriter) Write(p []byte) (int, error) {
|
||||||
|
n, err := lrw.ResponseWriter.Write(p)
|
||||||
|
lrw.bytesWritten += int64(n)
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (lrw *loggingResponseWriter) Flush() {
|
||||||
|
if flusher, ok := lrw.ResponseWriter.(http.Flusher); ok {
|
||||||
|
flusher.Flush()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (lrw *loggingResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
|
||||||
|
if hijacker, ok := lrw.ResponseWriter.(http.Hijacker); ok {
|
||||||
|
return hijacker.Hijack()
|
||||||
|
}
|
||||||
|
return nil, nil, errors.New("http.Hijacker not supported")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (lrw *loggingResponseWriter) Push(target string, opts *http.PushOptions) error {
|
||||||
|
if pusher, ok := lrw.ResponseWriter.(http.Pusher); ok {
|
||||||
|
return pusher.Push(target, opts)
|
||||||
|
}
|
||||||
|
return http.ErrNotSupported
|
||||||
|
}
|
||||||
|
|
||||||
|
func determineAccessAction(r *http.Request, bucket, key string) string {
|
||||||
|
method := r.Method
|
||||||
|
query := r.URL.Query()
|
||||||
|
|
||||||
|
if bucket == "" && key == "" {
|
||||||
|
if method == http.MethodGet {
|
||||||
|
return "list_buckets"
|
||||||
|
}
|
||||||
|
return strings.ToLower(method)
|
||||||
|
}
|
||||||
|
|
||||||
|
if key == "" {
|
||||||
|
switch method {
|
||||||
|
case http.MethodGet:
|
||||||
|
if _, ok := query["uploads"]; ok {
|
||||||
|
return "list_multipart_uploads"
|
||||||
|
}
|
||||||
|
return "list_objects"
|
||||||
|
case http.MethodHead:
|
||||||
|
return "head_bucket"
|
||||||
|
case http.MethodPut:
|
||||||
|
return "create_bucket"
|
||||||
|
case http.MethodDelete:
|
||||||
|
return "delete_bucket"
|
||||||
|
}
|
||||||
|
return strings.ToLower(method)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch method {
|
||||||
|
case http.MethodGet:
|
||||||
|
if _, ok := query["uploads"]; ok {
|
||||||
|
return "list_multipart_uploads"
|
||||||
|
}
|
||||||
|
if _, ok := query["uploadId"]; ok {
|
||||||
|
return "list_multipart_parts"
|
||||||
|
}
|
||||||
|
return "download_object"
|
||||||
|
case http.MethodHead:
|
||||||
|
return "head_object"
|
||||||
|
case http.MethodPut:
|
||||||
|
if _, hasUploadID := query["uploadId"]; hasUploadID {
|
||||||
|
if _, hasPart := query["partNumber"]; hasPart {
|
||||||
|
return "upload_part"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "upload_object"
|
||||||
|
case http.MethodDelete:
|
||||||
|
if _, ok := query["uploadId"]; ok {
|
||||||
|
return "abort_multipart_upload"
|
||||||
|
}
|
||||||
|
return "delete_object"
|
||||||
|
case http.MethodPost:
|
||||||
|
if _, ok := query["uploads"]; ok {
|
||||||
|
return "initiate_multipart_upload"
|
||||||
|
}
|
||||||
|
if _, ok := query["uploadId"]; ok {
|
||||||
|
return "complete_multipart_upload"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return strings.ToLower(method)
|
||||||
|
}
|
||||||
|
|
||||||
|
func calculateLogSize(r *http.Request, lrw *loggingResponseWriter) int64 {
|
||||||
|
switch r.Method {
|
||||||
|
case http.MethodPut, http.MethodPost:
|
||||||
|
// 对于上传请求,优先使用请求体大小(如果可用)
|
||||||
|
if r.ContentLength > 0 {
|
||||||
|
return r.ContentLength
|
||||||
|
}
|
||||||
|
// 对于分块传输(chunked),ContentLength 为 -1,返回 0
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
// 对于 GET/HEAD 等请求,返回响应体大小
|
||||||
|
return lrw.bytesWritten
|
||||||
|
}
|
||||||
|
|
||||||
|
func extractClientIP(r *http.Request) string {
|
||||||
|
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
|
||||||
|
parts := strings.Split(xff, ",")
|
||||||
|
if len(parts) > 0 {
|
||||||
|
return strings.TrimSpace(parts[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if xrip := r.Header.Get("X-Real-IP"); xrip != "" {
|
||||||
|
return strings.TrimSpace(xrip)
|
||||||
|
}
|
||||||
|
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||||
|
if err != nil {
|
||||||
|
return r.RemoteAddr
|
||||||
|
}
|
||||||
|
return host
|
||||||
|
}
|
||||||
@@ -12,6 +12,14 @@ import (
|
|||||||
|
|
||||||
// handleListBuckets 处理列出所有存储桶请求
|
// handleListBuckets 处理列出所有存储桶请求
|
||||||
func (h *S3Handler) handleListBuckets(w http.ResponseWriter, r *http.Request) {
|
func (h *S3Handler) handleListBuckets(w http.ResponseWriter, r *http.Request) {
|
||||||
|
start := time.Now()
|
||||||
|
defer func() {
|
||||||
|
if h.storage == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
h.recordAccessLog(r, "list_buckets", "", "", 0, true, "", time.Since(start))
|
||||||
|
}()
|
||||||
|
|
||||||
buckets := h.bucketManager.GetAllBuckets()
|
buckets := h.bucketManager.GetAllBuckets()
|
||||||
|
|
||||||
result := ListBucketsResult{
|
result := ListBucketsResult{
|
||||||
|
|||||||
@@ -24,11 +24,12 @@ type S3Handler struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type handlerSettings struct {
|
type handlerSettings struct {
|
||||||
accessKey string
|
accessKey string
|
||||||
secretKey string
|
secretKey string
|
||||||
proxyMode bool
|
proxyMode bool
|
||||||
authRequired bool
|
authRequired bool
|
||||||
virtualHost bool
|
virtualHost bool
|
||||||
|
signatureHost string
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewS3Handler 创建新的S3兼容API处理器
|
// NewS3Handler 创建新的S3兼容API处理器
|
||||||
@@ -43,6 +44,7 @@ func NewS3Handler(
|
|||||||
proxyMode bool,
|
proxyMode bool,
|
||||||
authRequired bool,
|
authRequired bool,
|
||||||
virtualHost bool,
|
virtualHost bool,
|
||||||
|
signatureHost string,
|
||||||
|
|
||||||
) *S3Handler {
|
) *S3Handler {
|
||||||
handler := &S3Handler{
|
handler := &S3Handler{
|
||||||
@@ -52,17 +54,18 @@ func NewS3Handler(
|
|||||||
storage: storage,
|
storage: storage,
|
||||||
metrics: metrics,
|
metrics: metrics,
|
||||||
}
|
}
|
||||||
handler.initSettings(accessKey, secretKey, proxyMode, authRequired, virtualHost)
|
handler.initSettings(accessKey, secretKey, proxyMode, authRequired, virtualHost, signatureHost)
|
||||||
return handler
|
return handler
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *S3Handler) initSettings(accessKey, secretKey string, proxyMode, authRequired, virtualHost bool) {
|
func (h *S3Handler) initSettings(accessKey, secretKey string, proxyMode, authRequired, virtualHost bool, signatureHost string) {
|
||||||
h.settings.Store(handlerSettings{
|
h.settings.Store(handlerSettings{
|
||||||
accessKey: accessKey,
|
accessKey: accessKey,
|
||||||
secretKey: secretKey,
|
secretKey: secretKey,
|
||||||
proxyMode: proxyMode,
|
proxyMode: proxyMode,
|
||||||
authRequired: authRequired,
|
authRequired: authRequired,
|
||||||
virtualHost: virtualHost,
|
virtualHost: virtualHost,
|
||||||
|
signatureHost: signatureHost,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,7 +76,7 @@ func (h *S3Handler) RegisterS3Routes(router *mux.Router) {
|
|||||||
|
|
||||||
// 带认证/虚拟主机的路由
|
// 带认证/虚拟主机的路由
|
||||||
protected := router.NewRoute().PathPrefix("/{bucket}").Subrouter()
|
protected := router.NewRoute().PathPrefix("/{bucket}").Subrouter()
|
||||||
protected.StrictSlash(true)
|
// 注意:不使用 StrictSlash(true) 以避免 301 重定向,兼容WinSCP
|
||||||
|
|
||||||
// Bucket operations
|
// Bucket operations
|
||||||
protected.HandleFunc("", h.handleBucketOperations).Methods("GET", "HEAD", "PUT", "DELETE")
|
protected.HandleFunc("", h.handleBucketOperations).Methods("GET", "HEAD", "PUT", "DELETE")
|
||||||
@@ -91,6 +94,7 @@ func (h *S3Handler) RegisterS3Routes(router *mux.Router) {
|
|||||||
protected.HandleFunc("/{key:.*}", h.handleObjectOperations).Methods("GET", "HEAD", "PUT", "DELETE")
|
protected.HandleFunc("/{key:.*}", h.handleObjectOperations).Methods("GET", "HEAD", "PUT", "DELETE")
|
||||||
|
|
||||||
// 添加中间件
|
// 添加中间件
|
||||||
|
protected.Use(h.accessLogMiddleware)
|
||||||
protected.Use(middleware.VirtualHost(middleware.VirtualHostConfig{
|
protected.Use(middleware.VirtualHost(middleware.VirtualHostConfig{
|
||||||
Enabled: h.virtualHostEnabled,
|
Enabled: h.virtualHostEnabled,
|
||||||
BucketExists: func(name string) bool {
|
BucketExists: func(name string) bool {
|
||||||
@@ -99,9 +103,10 @@ func (h *S3Handler) RegisterS3Routes(router *mux.Router) {
|
|||||||
},
|
},
|
||||||
}))
|
}))
|
||||||
protected.Use(middleware.S3Signature(middleware.S3SignatureConfig{
|
protected.Use(middleware.S3Signature(middleware.S3SignatureConfig{
|
||||||
Required: h.authRequired,
|
Required: h.authRequired,
|
||||||
Credentials: h.credentials,
|
Credentials: h.credentials,
|
||||||
OnError: h.sendS3Error,
|
OnError: h.handleAuthError,
|
||||||
|
SignatureHost: h.signatureHost,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -129,15 +134,20 @@ func (h *S3Handler) proxyModeEnabled() bool {
|
|||||||
return h.loadSettings().proxyMode
|
return h.loadSettings().proxyMode
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *S3Handler) signatureHost() string {
|
||||||
|
return h.loadSettings().signatureHost
|
||||||
|
}
|
||||||
|
|
||||||
func (h *S3Handler) UpdateS3APIConfig(cfg *config.S3APIConfig) {
|
func (h *S3Handler) UpdateS3APIConfig(cfg *config.S3APIConfig) {
|
||||||
if cfg == nil {
|
if cfg == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
h.settings.Store(handlerSettings{
|
h.settings.Store(handlerSettings{
|
||||||
accessKey: cfg.AccessKey,
|
accessKey: cfg.AccessKey,
|
||||||
secretKey: cfg.SecretKey,
|
secretKey: cfg.SecretKey,
|
||||||
proxyMode: cfg.ProxyMode,
|
proxyMode: cfg.ProxyMode,
|
||||||
authRequired: cfg.AuthRequired,
|
authRequired: cfg.AuthRequired,
|
||||||
virtualHost: cfg.VirtualHost,
|
virtualHost: cfg.VirtualHost,
|
||||||
|
signatureHost: cfg.Host,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -36,6 +36,9 @@ func (h *S3Handler) sendS3Error(w http.ResponseWriter, code string, message stri
|
|||||||
RequestID: fmt.Sprintf("%d", time.Now().UnixNano()),
|
RequestID: fmt.Sprintf("%d", time.Now().UnixNano()),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
w.Header().Set("X-Amz-Error-Code", code)
|
||||||
|
w.Header().Set("X-Amz-Error-Message", message)
|
||||||
|
|
||||||
statusCode := http.StatusBadRequest
|
statusCode := http.StatusBadRequest
|
||||||
switch code {
|
switch code {
|
||||||
case "NoSuchBucket", "NoSuchKey":
|
case "NoSuchBucket", "NoSuchKey":
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ type S3APIConfig struct {
|
|||||||
VirtualHost bool `yaml:"virtual_host"` // 是否使用虚拟主机模式
|
VirtualHost bool `yaml:"virtual_host"` // 是否使用虚拟主机模式
|
||||||
ProxyMode bool `yaml:"proxy_mode"` // 是否使用代理模式(而非重定向)
|
ProxyMode bool `yaml:"proxy_mode"` // 是否使用代理模式(而非重定向)
|
||||||
AuthRequired bool `yaml:"auth_required"` // 是否需要认证
|
AuthRequired bool `yaml:"auth_required"` // 是否需要认证
|
||||||
|
Host string `yaml:"host"` // 用于签名验证的Host(为空则使用请求的Host)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DatabaseConfig 数据库配置
|
// DatabaseConfig 数据库配置
|
||||||
|
|||||||
@@ -10,9 +10,10 @@ import (
|
|||||||
|
|
||||||
// S3SignatureConfig controls S3 signature validation.
|
// S3SignatureConfig controls S3 signature validation.
|
||||||
type S3SignatureConfig struct {
|
type S3SignatureConfig struct {
|
||||||
Required func() bool
|
Required func() bool
|
||||||
Credentials func() (string, string)
|
Credentials func() (string, string)
|
||||||
OnError func(http.ResponseWriter, string, string, string)
|
OnError func(http.ResponseWriter, *http.Request, string, string, string)
|
||||||
|
SignatureHost func() string // 用于签名验证的Host(为空则使用请求的Host)
|
||||||
}
|
}
|
||||||
|
|
||||||
// credentialsProvider implements s3validate.CredentialsProvider interface.
|
// credentialsProvider implements s3validate.CredentialsProvider interface.
|
||||||
@@ -47,9 +48,16 @@ func S3Signature(cfg S3SignatureConfig) func(http.Handler) http.Handler {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 如果配置了签名验证的Host,覆盖请求的Host
|
||||||
|
if cfg.SignatureHost != nil {
|
||||||
|
if signatureHost := cfg.SignatureHost(); signatureHost != "" {
|
||||||
|
r.Host = signatureHost
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
result, err := verifier.Verify(r.Context(), r)
|
result, err := verifier.Verify(r.Context(), r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
invokeOnError(w, cfg, "SignatureDoesNotMatch", err.Error())
|
invokeOnError(w, r, cfg, "SignatureDoesNotMatch", err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -61,9 +69,9 @@ func S3Signature(cfg S3SignatureConfig) func(http.Handler) http.Handler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func invokeOnError(w http.ResponseWriter, cfg S3SignatureConfig, code, message string) {
|
func invokeOnError(w http.ResponseWriter, r *http.Request, cfg S3SignatureConfig, code, message string) {
|
||||||
if cfg.OnError != nil {
|
if cfg.OnError != nil {
|
||||||
cfg.OnError(w, code, message, "")
|
cfg.OnError(w, r, code, message, "")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
http.Error(w, message, http.StatusForbidden)
|
http.Error(w, message, http.StatusForbidden)
|
||||||
|
|||||||
@@ -87,7 +87,8 @@ type AccessLog struct {
|
|||||||
Size int64 `gorm:"default:0" json:"size"`
|
Size int64 `gorm:"default:0" json:"size"`
|
||||||
ClientIP string `gorm:"size:64" json:"client_ip"`
|
ClientIP string `gorm:"size:64" json:"client_ip"`
|
||||||
UserAgent string `gorm:"size:512" json:"user_agent"`
|
UserAgent string `gorm:"size:512" json:"user_agent"`
|
||||||
Success bool `gorm:"default:true" json:"success"`
|
Host string `gorm:"size:255" json:"host"`
|
||||||
|
Success bool `gorm:"not null" json:"success"`
|
||||||
ErrorMsg string `gorm:"type:text" json:"error_msg,omitempty"`
|
ErrorMsg string `gorm:"type:text" json:"error_msg,omitempty"`
|
||||||
ResponseTime int64 `gorm:"default:0" json:"response_time"` // 响应时间(毫秒)
|
ResponseTime int64 `gorm:"default:0" json:"response_time"` // 响应时间(毫秒)
|
||||||
CreatedAt time.Time `gorm:"index" json:"created_at"`
|
CreatedAt time.Time `gorm:"index" json:"created_at"`
|
||||||
|
|||||||
@@ -367,13 +367,14 @@ func (s *Service) CleanExpiredSessions() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// RecordAccessLog 记录访问日志
|
// RecordAccessLog 记录访问日志
|
||||||
func (s *Service) RecordAccessLog(action, key, bucketName, clientIP, userAgent string, size int64, success bool, errorMsg string, responseTime int64) error {
|
func (s *Service) RecordAccessLog(action, key, bucketName, clientIP, userAgent, host string, size int64, success bool, errorMsg string, responseTime int64) error {
|
||||||
log := &AccessLog{
|
log := &AccessLog{
|
||||||
Action: action,
|
Action: action,
|
||||||
Key: key,
|
Key: key,
|
||||||
BucketName: bucketName,
|
BucketName: bucketName,
|
||||||
ClientIP: clientIP,
|
ClientIP: clientIP,
|
||||||
UserAgent: userAgent,
|
UserAgent: userAgent,
|
||||||
|
Host: host,
|
||||||
Size: size,
|
Size: size,
|
||||||
Success: success,
|
Success: success,
|
||||||
ErrorMsg: errorMsg,
|
ErrorMsg: errorMsg,
|
||||||
|
|||||||
Reference in New Issue
Block a user