This commit is contained in:
DullJZ
2025-08-22 21:15:56 +08:00
commit 37b6adb6de
16 changed files with 3838 additions and 0 deletions

392
internal/api/handler.go Normal file
View File

@@ -0,0 +1,392 @@
package api
import (
"context"
"encoding/json"
"log"
"net/http"
"strconv"
"time"
"github.com/DullJZ/s3-balance/internal/balancer"
"github.com/DullJZ/s3-balance/internal/bucket"
"github.com/DullJZ/s3-balance/internal/storage"
"github.com/DullJZ/s3-balance/pkg/presigner"
"github.com/gorilla/mux"
)
// Handler API处理器
type Handler struct {
bucketManager *bucket.Manager
balancer *balancer.Balancer
presigner *presigner.Presigner
storage *storage.Service
}
// NewHandler 创建新的API处理器
func NewHandler(
bucketManager *bucket.Manager,
balancer *balancer.Balancer,
presigner *presigner.Presigner,
storage *storage.Service,
) *Handler {
return &Handler{
bucketManager: bucketManager,
balancer: balancer,
presigner: presigner,
storage: storage,
}
}
// RegisterRoutes 注册路由
func (h *Handler) RegisterRoutes(router *mux.Router) {
// 健康检查
router.HandleFunc("/health", h.handleHealth).Methods("GET")
// 存储桶状态
router.HandleFunc("/api/v1/buckets", h.handleListBuckets).Methods("GET")
router.HandleFunc("/api/v1/buckets/{bucket}/stats", h.handleBucketStats).Methods("GET")
// 预签名URL生成
router.HandleFunc("/api/v1/presign/upload", h.handlePresignUpload).Methods("POST")
router.HandleFunc("/api/v1/presign/download", h.handlePresignDownload).Methods("POST")
router.HandleFunc("/api/v1/presign/delete", h.handlePresignDelete).Methods("POST")
router.HandleFunc("/api/v1/presign/multipart", h.handlePresignMultipart).Methods("POST")
// 对象操作(记录元数据)
router.HandleFunc("/api/v1/objects", h.handleListObjects).Methods("GET")
router.HandleFunc("/api/v1/objects/{key:.*}", h.handleGetObjectInfo).Methods("GET")
router.HandleFunc("/api/v1/objects/{key:.*}", h.handleDeleteObject).Methods("DELETE")
}
// 健康检查
func (h *Handler) handleHealth(w http.ResponseWriter, r *http.Request) {
response := map[string]interface{}{
"status": "healthy",
"time": time.Now().Unix(),
}
h.sendJSON(w, http.StatusOK, response)
}
// 列出所有存储桶状态
func (h *Handler) handleListBuckets(w http.ResponseWriter, r *http.Request) {
buckets := h.bucketManager.GetAllBuckets()
var bucketList []map[string]interface{}
for _, b := range buckets {
bucketList = append(bucketList, map[string]interface{}{
"name": b.Config.Name,
"endpoint": b.Config.Endpoint,
"region": b.Config.Region,
"max_size": b.Config.MaxSize,
"max_size_bytes": b.Config.MaxSizeBytes,
"used_size": b.GetUsedSize(),
"available": b.IsAvailable(),
"weight": b.Config.Weight,
"enabled": b.Config.Enabled,
})
}
h.sendJSON(w, http.StatusOK, map[string]interface{}{
"buckets": bucketList,
"strategy": h.balancer.GetStrategy(),
})
}
// 获取单个存储桶统计
func (h *Handler) handleBucketStats(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucketName := vars["bucket"]
bucket, ok := h.bucketManager.GetBucket(bucketName)
if !ok {
h.sendError(w, http.StatusNotFound, "bucket not found")
return
}
stats := map[string]interface{}{
"name": bucket.Config.Name,
"max_size_bytes": bucket.Config.MaxSizeBytes,
"used_size": bucket.GetUsedSize(),
"available_space": bucket.GetAvailableSpace(),
"available": bucket.IsAvailable(),
"last_checked": bucket.LastChecked,
}
h.sendJSON(w, http.StatusOK, stats)
}
// PresignUploadRequest 上传预签名请求
type PresignUploadRequest struct {
Key string `json:"key"`
Size int64 `json:"size"`
ContentType string `json:"content_type,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
}
// 生成上传预签名URL
func (h *Handler) handlePresignUpload(w http.ResponseWriter, r *http.Request) {
var req PresignUploadRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
h.sendError(w, http.StatusBadRequest, "invalid request body")
return
}
if req.Key == "" {
h.sendError(w, http.StatusBadRequest, "key is required")
return
}
// 选择存储桶
bucket, err := h.balancer.SelectBucket(req.Key, req.Size)
if err != nil {
h.sendError(w, http.StatusServiceUnavailable, err.Error())
return
}
// 生成预签名URL
uploadURL, err := h.presigner.GenerateUploadURL(
context.Background(),
bucket,
req.Key,
req.ContentType,
req.Metadata,
)
if err != nil {
h.sendError(w, http.StatusInternalServerError, "failed to generate upload URL")
return
}
// 记录对象元数据
if err := h.storage.RecordObject(req.Key, bucket.Config.Name, req.Size, req.Metadata); err != nil {
log.Printf("Failed to record object metadata: %v", err)
}
// 更新存储桶使用量(预估)
bucket.UpdateUsedSize(req.Size)
h.sendJSON(w, http.StatusOK, uploadURL)
}
// PresignDownloadRequest 下载预签名请求
type PresignDownloadRequest struct {
Key string `json:"key"`
}
// 生成下载预签名URL
func (h *Handler) handlePresignDownload(w http.ResponseWriter, r *http.Request) {
var req PresignDownloadRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
h.sendError(w, http.StatusBadRequest, "invalid request body")
return
}
if req.Key == "" {
h.sendError(w, http.StatusBadRequest, "key is required")
return
}
// 查找对象所在的存储桶
bucketName, err := h.storage.FindObjectBucket(req.Key)
if err != nil {
h.sendError(w, http.StatusNotFound, "object not found")
return
}
bucket, ok := h.bucketManager.GetBucket(bucketName)
if !ok {
h.sendError(w, http.StatusNotFound, "bucket not found")
return
}
// 生成预签名URL
downloadURL, err := h.presigner.GenerateDownloadURL(
context.Background(),
bucket,
req.Key,
)
if err != nil {
h.sendError(w, http.StatusInternalServerError, "failed to generate download URL")
return
}
h.sendJSON(w, http.StatusOK, downloadURL)
}
// PresignDeleteRequest 删除预签名请求
type PresignDeleteRequest struct {
Key string `json:"key"`
}
// 生成删除预签名URL
func (h *Handler) handlePresignDelete(w http.ResponseWriter, r *http.Request) {
var req PresignDeleteRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
h.sendError(w, http.StatusBadRequest, "invalid request body")
return
}
if req.Key == "" {
h.sendError(w, http.StatusBadRequest, "key is required")
return
}
// 查找对象所在的存储桶
bucketName, err := h.storage.FindObjectBucket(req.Key)
if err != nil {
h.sendError(w, http.StatusNotFound, "object not found")
return
}
bucket, ok := h.bucketManager.GetBucket(bucketName)
if !ok {
h.sendError(w, http.StatusNotFound, "bucket not found")
return
}
// 生成预签名URL
deleteURL, err := h.presigner.GenerateDeleteURL(
context.Background(),
bucket,
req.Key,
)
if err != nil {
h.sendError(w, http.StatusInternalServerError, "failed to generate delete URL")
return
}
h.sendJSON(w, http.StatusOK, deleteURL)
}
// PresignMultipartRequest 分片上传预签名请求
type PresignMultipartRequest struct {
Key string `json:"key"`
PartCount int `json:"part_count"`
Size int64 `json:"size"`
}
// 生成分片上传预签名URLs
func (h *Handler) handlePresignMultipart(w http.ResponseWriter, r *http.Request) {
var req PresignMultipartRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
h.sendError(w, http.StatusBadRequest, "invalid request body")
return
}
if req.Key == "" || req.PartCount <= 0 {
h.sendError(w, http.StatusBadRequest, "invalid parameters")
return
}
// 选择存储桶
bucket, err := h.balancer.SelectBucket(req.Key, req.Size)
if err != nil {
h.sendError(w, http.StatusServiceUnavailable, err.Error())
return
}
// 生成预签名URLs
multipartURLs, err := h.presigner.GenerateMultipartUploadURLs(
context.Background(),
bucket,
req.Key,
req.PartCount,
)
if err != nil {
h.sendError(w, http.StatusInternalServerError, "failed to generate multipart URLs")
return
}
// 记录对象元数据
if err := h.storage.RecordObject(req.Key, bucket.Config.Name, req.Size, nil); err != nil {
log.Printf("Failed to record object metadata: %v", err)
}
// 更新存储桶使用量(预估)
bucket.UpdateUsedSize(req.Size)
h.sendJSON(w, http.StatusOK, multipartURLs)
}
// 列出对象
func (h *Handler) handleListObjects(w http.ResponseWriter, r *http.Request) {
prefix := r.URL.Query().Get("prefix")
bucketName := r.URL.Query().Get("bucket")
marker := r.URL.Query().Get("marker")
limitStr := r.URL.Query().Get("limit")
limit := 100
if limitStr != "" {
if l, err := strconv.Atoi(limitStr); err == nil && l > 0 {
limit = l
}
}
// 调用更新后的ListObjects方法传入所有必需的参数
objects, err := h.storage.ListObjects(bucketName, prefix, marker, limit)
if err != nil {
h.sendError(w, http.StatusInternalServerError, "failed to list objects")
return
}
h.sendJSON(w, http.StatusOK, map[string]interface{}{
"objects": objects,
"count": len(objects),
})
}
// 获取对象信息
func (h *Handler) handleGetObjectInfo(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
key := vars["key"]
info, err := h.storage.GetObjectInfo(key)
if err != nil {
h.sendError(w, http.StatusNotFound, "object not found")
return
}
h.sendJSON(w, http.StatusOK, info)
}
// 删除对象(只删除元数据记录)
func (h *Handler) handleDeleteObject(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
key := vars["key"]
// 获取对象信息以更新存储桶使用量
info, err := h.storage.GetObjectInfo(key)
if err != nil {
h.sendError(w, http.StatusNotFound, "object not found")
return
}
// 更新存储桶使用量
if bucket, ok := h.bucketManager.GetBucket(info.BucketName); ok {
bucket.UpdateUsedSize(-info.Size)
}
// 删除元数据记录
if err := h.storage.DeleteObject(key); err != nil {
h.sendError(w, http.StatusInternalServerError, "failed to delete object")
return
}
h.sendJSON(w, http.StatusOK, map[string]string{
"message": "object deleted successfully",
})
}
// 发送JSON响应
func (h *Handler) sendJSON(w http.ResponseWriter, status int, data interface{}) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
json.NewEncoder(w).Encode(data)
}
// 发送错误响应
func (h *Handler) sendError(w http.ResponseWriter, status int, message string) {
h.sendJSON(w, status, map[string]string{
"error": message,
})
}

780
internal/api/s3_handler.go Normal file
View File

@@ -0,0 +1,780 @@
package api
import (
"context"
"encoding/xml"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/DullJZ/s3-balance/internal/balancer"
"github.com/DullJZ/s3-balance/internal/bucket"
"github.com/DullJZ/s3-balance/internal/storage"
"github.com/DullJZ/s3-balance/pkg/presigner"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gorilla/mux"
)
// S3Handler S3兼容的API处理器
type S3Handler struct {
bucketManager *bucket.Manager
balancer *balancer.Balancer
presigner *presigner.Presigner
storage *storage.Service
accessKey string
secretKey string
}
// NewS3Handler 创建新的S3兼容API处理器
func NewS3Handler(
bucketManager *bucket.Manager,
balancer *balancer.Balancer,
presigner *presigner.Presigner,
storage *storage.Service,
accessKey string,
secretKey string,
) *S3Handler {
return &S3Handler{
bucketManager: bucketManager,
balancer: balancer,
presigner: presigner,
storage: storage,
accessKey: accessKey,
secretKey: secretKey,
}
}
// RegisterS3Routes 注册S3兼容的路由
func (h *S3Handler) RegisterS3Routes(router *mux.Router) {
// Service operations
router.HandleFunc("/", h.handleListBuckets).Methods("GET")
// Bucket operations
router.HandleFunc("/{bucket}", h.handleBucketOperations).Methods("GET", "HEAD", "PUT", "DELETE")
// Object operations
router.HandleFunc("/{bucket}/{key:.*}", h.handleObjectOperations).Methods("GET", "HEAD", "PUT", "DELETE")
// Multipart upload operations
router.HandleFunc("/{bucket}/{key:.*}", h.handleMultipartUpload).Methods("POST").Queries("uploads", "")
router.HandleFunc("/{bucket}/{key:.*}", h.handleListMultipartUploads).Methods("GET").Queries("uploads", "")
router.HandleFunc("/{bucket}/{key:.*}", h.handleListMultipartParts).Methods("GET").Queries("uploadId", "")
router.HandleFunc("/{bucket}/{key:.*}", h.handleCompleteMultipartUpload).Methods("POST").Queries("uploadId", "")
router.HandleFunc("/{bucket}/{key:.*}", h.handleAbortMultipartUpload).Methods("DELETE").Queries("uploadId", "")
// 添加认证中间件
router.Use(h.s3AuthMiddleware)
}
// S3 XML响应结构体定义
type ListBucketsResult struct {
XMLName xml.Name `xml:"ListAllMyBucketsResult"`
Xmlns string `xml:"xmlns,attr"`
Owner Owner `xml:"Owner"`
Buckets Buckets `xml:"Buckets"`
}
type Owner struct {
ID string `xml:"ID"`
DisplayName string `xml:"DisplayName"`
}
type Buckets struct {
Bucket []BucketInfo `xml:"Bucket"`
}
type BucketInfo struct {
Name string `xml:"Name"`
CreationDate time.Time `xml:"CreationDate"`
}
type ListBucketResult struct {
XMLName xml.Name `xml:"ListBucketResult"`
Xmlns string `xml:"xmlns,attr"`
Name string `xml:"Name"`
Prefix string `xml:"Prefix"`
Marker string `xml:"Marker"`
MaxKeys int `xml:"MaxKeys"`
IsTruncated bool `xml:"IsTruncated"`
Contents []ObjectInfo `xml:"Contents"`
CommonPrefixes []CommonPrefix `xml:"CommonPrefixes,omitempty"`
}
type ObjectInfo struct {
Key string `xml:"Key"`
LastModified time.Time `xml:"LastModified"`
ETag string `xml:"ETag"`
Size int64 `xml:"Size"`
StorageClass string `xml:"StorageClass"`
Owner Owner `xml:"Owner"`
}
type CommonPrefix struct {
Prefix string `xml:"Prefix"`
}
type InitiateMultipartUploadResult struct {
XMLName xml.Name `xml:"InitiateMultipartUploadResult"`
Xmlns string `xml:"xmlns,attr"`
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
}
type ListMultipartUploadsResult struct {
XMLName xml.Name `xml:"ListMultipartUploadsResult"`
Xmlns string `xml:"xmlns,attr"`
Bucket string `xml:"Bucket"`
KeyMarker string `xml:"KeyMarker"`
UploadIdMarker string `xml:"UploadIdMarker"`
NextKeyMarker string `xml:"NextKeyMarker"`
NextUploadIdMarker string `xml:"NextUploadIdMarker"`
MaxUploads int `xml:"MaxUploads"`
IsTruncated bool `xml:"IsTruncated"`
Uploads []Upload `xml:"Upload"`
CommonPrefixes []CommonPrefix `xml:"CommonPrefixes,omitempty"`
}
type Upload struct {
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
Initiator Owner `xml:"Initiator"`
Owner Owner `xml:"Owner"`
StorageClass string `xml:"StorageClass"`
Initiated time.Time `xml:"Initiated"`
}
type ListPartsResult struct {
XMLName xml.Name `xml:"ListPartsResult"`
Xmlns string `xml:"xmlns,attr"`
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
PartNumberMarker int `xml:"PartNumberMarker"`
NextPartNumberMarker int `xml:"NextPartNumberMarker"`
MaxParts int `xml:"MaxParts"`
IsTruncated bool `xml:"IsTruncated"`
Parts []Part `xml:"Part"`
}
type Part struct {
PartNumber int `xml:"PartNumber"`
LastModified time.Time `xml:"LastModified"`
ETag string `xml:"ETag"`
Size int64 `xml:"Size"`
}
type CompleteMultipartUpload struct {
XMLName xml.Name `xml:"CompleteMultipartUpload"`
Parts []Part `xml:"Part"`
}
type CompleteMultipartUploadResult struct {
XMLName xml.Name `xml:"CompleteMultipartUploadResult"`
Xmlns string `xml:"xmlns,attr"`
Location string `xml:"Location"`
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
ETag string `xml:"ETag"`
}
type ErrorResponse struct {
XMLName xml.Name `xml:"Error"`
Code string `xml:"Code"`
Message string `xml:"Message"`
Resource string `xml:"Resource"`
RequestID string `xml:"RequestId"`
}
// s3AuthMiddleware S3认证中间件简化版
func (h *S3Handler) s3AuthMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// 简化的认证实现实际应该验证AWS Signature
// 这里只做基本的header检查
authHeader := r.Header.Get("Authorization")
if authHeader == "" {
// 允许匿名访问(用于测试)
// 在生产环境中应该要求认证
}
next.ServeHTTP(w, r)
})
}
// handleListBuckets 处理列出所有存储桶请求
func (h *S3Handler) handleListBuckets(w http.ResponseWriter, r *http.Request) {
buckets := h.bucketManager.GetAllBuckets()
result := ListBucketsResult{
Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
Owner: Owner{
ID: "s3-balance",
DisplayName: "S3 Balance Service",
},
Buckets: Buckets{
Bucket: make([]BucketInfo, 0, len(buckets)),
},
}
for _, b := range buckets {
if b.IsAvailable() {
result.Buckets.Bucket = append(result.Buckets.Bucket, BucketInfo{
Name: b.Config.Name,
CreationDate: time.Now().Add(-24 * time.Hour), // 模拟创建时间
})
}
}
h.sendXMLResponse(w, http.StatusOK, result)
}
// handleBucketOperations 处理存储桶相关操作
func (h *S3Handler) handleBucketOperations(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucketName := vars["bucket"]
switch r.Method {
case "GET":
h.handleListObjects(w, r, bucketName)
case "HEAD":
h.handleHeadBucket(w, r, bucketName)
case "PUT":
h.handleCreateBucket(w, r, bucketName)
case "DELETE":
h.handleDeleteBucket(w, r, bucketName)
}
}
// handleListObjects 列出存储桶中的对象
func (h *S3Handler) handleListObjects(w http.ResponseWriter, r *http.Request, bucketName string) {
// 检查bucket是否存在
if _, ok := h.bucketManager.GetBucket(bucketName); !ok {
h.sendS3Error(w, "NoSuchBucket", "The specified bucket does not exist", bucketName)
return
}
// 解析查询参数
prefix := r.URL.Query().Get("prefix")
marker := r.URL.Query().Get("marker")
maxKeysStr := r.URL.Query().Get("max-keys")
delimiter := r.URL.Query().Get("delimiter")
maxKeys := 1000
if maxKeysStr != "" {
if mk, err := strconv.Atoi(maxKeysStr); err == nil {
maxKeys = mk
}
}
// 从存储中获取对象列表
objects, err := h.storage.ListObjects(bucketName, prefix, marker, maxKeys)
if err != nil {
h.sendS3Error(w, "InternalError", "Internal server error", bucketName)
return
}
result := ListBucketResult{
Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
Name: bucketName,
Prefix: prefix,
Marker: marker,
MaxKeys: maxKeys,
IsTruncated: false, // 简化实现
Contents: make([]ObjectInfo, 0),
}
// 处理分隔符逻辑
if delimiter != "" {
// 简化的分隔符处理
result.CommonPrefixes = make([]CommonPrefix, 0)
}
for _, obj := range objects {
result.Contents = append(result.Contents, ObjectInfo{
Key: obj.Key,
LastModified: obj.UpdatedAt,
ETag: fmt.Sprintf("\"%x\"", obj.ID), // 简化的ETag
Size: obj.Size,
StorageClass: "STANDARD",
Owner: Owner{
ID: "s3-balance",
DisplayName: "S3 Balance Service",
},
})
}
h.sendXMLResponse(w, http.StatusOK, result)
}
// handleHeadBucket 检查存储桶是否存在
func (h *S3Handler) handleHeadBucket(w http.ResponseWriter, r *http.Request, bucketName string) {
if _, ok := h.bucketManager.GetBucket(bucketName); !ok {
w.WriteHeader(http.StatusNotFound)
return
}
w.WriteHeader(http.StatusOK)
}
// handleCreateBucket 创建存储桶(虚拟实现)
func (h *S3Handler) handleCreateBucket(w http.ResponseWriter, r *http.Request, bucketName string) {
// 在负载均衡场景下不真正创建bucket只返回成功
// 实际的bucket应该在配置中预先定义
w.Header().Set("Location", "/" + bucketName)
w.WriteHeader(http.StatusOK)
}
// handleDeleteBucket 删除存储桶(虚拟实现)
func (h *S3Handler) handleDeleteBucket(w http.ResponseWriter, r *http.Request, bucketName string) {
// 在负载均衡场景下不真正删除bucket
w.WriteHeader(http.StatusNoContent)
}
// handleObjectOperations 处理对象相关操作
func (h *S3Handler) handleObjectOperations(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucketName := vars["bucket"]
key := vars["key"]
switch r.Method {
case "GET":
h.handleGetObject(w, r, bucketName, key)
case "HEAD":
h.handleHeadObject(w, r, bucketName, key)
case "PUT":
h.handlePutObject(w, r, bucketName, key)
case "DELETE":
h.handleDeleteObject(w, r, bucketName, key)
}
}
// handleGetObject 获取对象默认使用预签名URL重定向
func (h *S3Handler) handleGetObject(w http.ResponseWriter, r *http.Request, bucketName string, key string) {
// 查找对象所在的实际存储桶
actualBucketName, err := h.storage.FindObjectBucket(key)
if err != nil {
h.sendS3Error(w, "NoSuchKey", "The specified key does not exist", key)
return
}
bucket, ok := h.bucketManager.GetBucket(actualBucketName)
if !ok {
h.sendS3Error(w, "InternalError", "Internal server error", bucketName)
return
}
// 生成预签名下载URL
downloadInfo, err := h.presigner.GenerateDownloadURL(
context.Background(),
bucket,
key,
)
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to generate download URL", key)
return
}
// 默认使用预签名重定向模式,只有明确指定时才使用代理模式
if r.URL.Query().Get("proxy") == "true" {
// 代理模式:服务器下载内容并返回给客户端
resp, err := http.Get(downloadInfo.URL)
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to fetch object", key)
return
}
defer resp.Body.Close()
// 复制响应头
for k, v := range resp.Header {
w.Header()[k] = v
}
// 复制响应体
io.Copy(w, resp.Body)
} else {
// 重定向模式返回302重定向到预签名URL默认
http.Redirect(w, r, downloadInfo.URL, http.StatusFound)
}
}
// handleHeadObject 获取对象元数据
func (h *S3Handler) handleHeadObject(w http.ResponseWriter, r *http.Request, bucketName string, key string) {
// 从存储中获取对象信息
obj, err := h.storage.GetObjectInfo(key)
if err != nil {
w.WriteHeader(http.StatusNotFound)
return
}
w.Header().Set("Content-Length", strconv.FormatInt(obj.Size, 10))
w.Header().Set("Last-Modified", obj.UpdatedAt.Format(http.TimeFormat))
w.Header().Set("ETag", fmt.Sprintf("\"%x\"", obj.ID))
w.Header().Set("Content-Type", "application/octet-stream")
w.WriteHeader(http.StatusOK)
}
// handlePutObject 上传对象默认使用预签名URL重定向
func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, bucketName string, key string) {
// 获取内容长度
contentLength := r.ContentLength
if contentLength < 0 {
h.sendS3Error(w, "MissingContentLength", "Content-Length header is required", key)
return
}
// 选择目标存储桶
targetBucket, err := h.balancer.SelectBucket(key, contentLength)
if err != nil {
h.sendS3Error(w, "InsufficientStorage", "No bucket has enough space", key)
return
}
// 生成预签名上传URL
uploadInfo, err := h.presigner.GenerateUploadURL(
context.Background(),
targetBucket,
key,
r.Header.Get("Content-Type"),
nil, // metadata
)
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to generate upload URL", key)
return
}
// 默认使用预签名重定向模式,只有明确指定时才使用代理模式
if r.URL.Query().Get("proxy") == "true" {
// 代理模式读取请求体并上传到预签名URL
// 创建新的请求
req, err := http.NewRequest(uploadInfo.Method, uploadInfo.URL, r.Body)
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to create upload request", key)
return
}
// 设置必要的头
req.ContentLength = contentLength
if ct := r.Header.Get("Content-Type"); ct != "" {
req.Header.Set("Content-Type", ct)
}
// 添加预签名URL所需的额外头
for k, v := range uploadInfo.Headers {
req.Header.Set(k, v)
}
// 执行上传
client := &http.Client{Timeout: 30 * time.Minute}
resp, err := client.Do(req)
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to upload object", key)
return
}
defer resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
// 记录对象元数据
h.storage.RecordObject(key, targetBucket.Config.Name, contentLength, nil)
targetBucket.UpdateUsedSize(contentLength)
// 返回成功响应
w.Header().Set("ETag", fmt.Sprintf("\"%x\"", time.Now().UnixNano()))
w.WriteHeader(http.StatusOK)
} else {
h.sendS3Error(w, "InternalError", "Upload failed", key)
}
} else {
// 重定向模式返回307临时重定向让客户端直接上传默认
w.Header().Set("Location", uploadInfo.URL)
w.WriteHeader(http.StatusTemporaryRedirect)
}
}
// handleDeleteObject 删除对象
func (h *S3Handler) handleDeleteObject(w http.ResponseWriter, r *http.Request, bucketName string, key string) {
// 查找对象所在的实际存储桶
actualBucketName, err := h.storage.FindObjectBucket(key)
if err != nil {
// 对象不存在S3规范要求返回204
w.WriteHeader(http.StatusNoContent)
return
}
bucket, ok := h.bucketManager.GetBucket(actualBucketName)
if !ok {
h.sendS3Error(w, "InternalError", "Internal server error", bucketName)
return
}
// 生成预签名删除URL
deleteInfo, err := h.presigner.GenerateDeleteURL(
context.Background(),
bucket,
key,
)
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to generate delete URL", key)
return
}
// 执行删除
req, _ := http.NewRequest("DELETE", deleteInfo.URL, nil)
client := &http.Client{Timeout: 30 * time.Second}
resp, err := client.Do(req)
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to delete object", key)
return
}
defer resp.Body.Close()
// 从数据库中删除对象记录
h.storage.DeleteObject(key)
// S3规范要求删除操作总是返回204
w.WriteHeader(http.StatusNoContent)
}
// handleMultipartUpload 初始化分片上传
func (h *S3Handler) handleMultipartUpload(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
key := vars["key"]
// 选择目标存储桶
targetBucket, err := h.balancer.SelectBucket(key, 0) // 分片上传时不检查空间
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to select bucket for upload", key)
return
}
// 初始化分片上传
ctx := context.Background()
createResp, err := targetBucket.Client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
Bucket: aws.String(targetBucket.Config.Name),
Key: aws.String(key),
})
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to initiate multipart upload", key)
return
}
result := InitiateMultipartUploadResult{
Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
Bucket: targetBucket.Config.Name,
Key: key,
UploadID: *createResp.UploadId,
}
h.sendXMLResponse(w, http.StatusOK, result)
}
// handleListMultipartUploads 列出分片上传
func (h *S3Handler) handleListMultipartUploads(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucketName := vars["bucket"]
key := vars["key"]
// 检查bucket是否存在
if _, ok := h.bucketManager.GetBucket(bucketName); !ok {
h.sendS3Error(w, "NoSuchBucket", "The specified bucket does not exist", bucketName)
return
}
// 简化实现:返回空列表
result := ListMultipartUploadsResult{
Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
Bucket: bucketName,
KeyMarker: key,
MaxUploads: 1000,
IsTruncated: false,
Uploads: make([]Upload, 0),
}
h.sendXMLResponse(w, http.StatusOK, result)
}
// handleListMultipartParts 列出分片上传的分片
func (h *S3Handler) handleListMultipartParts(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucketName := vars["bucket"]
key := vars["key"]
uploadID := r.URL.Query().Get("uploadId")
// 检查bucket是否存在
if _, ok := h.bucketManager.GetBucket(bucketName); !ok {
h.sendS3Error(w, "NoSuchBucket", "The specified bucket does not exist", bucketName)
return
}
// 简化实现:返回空列表
result := ListPartsResult{
Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
Bucket: bucketName,
Key: key,
UploadID: uploadID,
PartNumberMarker: 0,
MaxParts: 1000,
IsTruncated: false,
Parts: make([]Part, 0),
}
h.sendXMLResponse(w, http.StatusOK, result)
}
// handleCompleteMultipartUpload 完成分片上传
func (h *S3Handler) handleCompleteMultipartUpload(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucketName := vars["bucket"]
key := vars["key"]
uploadID := r.URL.Query().Get("uploadId")
// 查找对象所在的实际存储桶简化实现使用配置的bucket
bucket, ok := h.bucketManager.GetBucket(bucketName)
if !ok {
h.sendS3Error(w, "NoSuchBucket", "The specified bucket does not exist", bucketName)
return
}
// 解析请求体以获取分片列表
var completeReq CompleteMultipartUpload
body, _ := io.ReadAll(r.Body)
xml.Unmarshal(body, &completeReq)
// 完成分片上传
ctx := context.Background()
var parts []types.CompletedPart
for _, part := range completeReq.Parts {
parts = append(parts, types.CompletedPart{
ETag: aws.String(part.ETag),
PartNumber: aws.Int32(int32(part.PartNumber)),
})
}
completeResp, err := bucket.Client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
Bucket: aws.String(bucket.Config.Name),
Key: aws.String(key),
UploadId: aws.String(uploadID),
MultipartUpload: &types.CompletedMultipartUpload{
Parts: parts,
},
})
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to complete multipart upload", key)
return
}
result := CompleteMultipartUploadResult{
Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
Location: "/" + bucket.Config.Name + "/" + key,
Bucket: bucket.Config.Name,
Key: key,
ETag: *completeResp.ETag,
}
// 记录对象元数据(简化:假设总大小)
h.storage.RecordObject(key, bucket.Config.Name, 0, nil)
h.sendXMLResponse(w, http.StatusOK, result)
}
// handleAbortMultipartUpload 中止分片上传
func (h *S3Handler) handleAbortMultipartUpload(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucketName := vars["bucket"]
key := vars["key"]
uploadID := r.URL.Query().Get("uploadId")
// 查找对象所在的实际存储桶简化实现使用配置的bucket
bucket, ok := h.bucketManager.GetBucket(bucketName)
if !ok {
h.sendS3Error(w, "NoSuchBucket", "The specified bucket does not exist", bucketName)
return
}
// 中止分片上传
ctx := context.Background()
_, err := bucket.Client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
Bucket: aws.String(bucket.Config.Name),
Key: aws.String(key),
UploadId: aws.String(uploadID),
})
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to abort multipart upload", key)
return
}
w.WriteHeader(http.StatusNoContent)
}
// sendXMLResponse 发送XML响应
func (h *S3Handler) sendXMLResponse(w http.ResponseWriter, statusCode int, data interface{}) {
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(statusCode)
encoder := xml.NewEncoder(w)
encoder.Indent("", " ")
// 写入XML声明
w.Write([]byte(xml.Header))
if err := encoder.Encode(data); err != nil {
// 如果编码失败,记录错误
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
}
}
// sendS3Error 发送S3错误响应
func (h *S3Handler) sendS3Error(w http.ResponseWriter, code string, message string, resource string) {
errorResp := ErrorResponse{
Code: code,
Message: message,
Resource: resource,
RequestID: fmt.Sprintf("%d", time.Now().UnixNano()),
}
statusCode := http.StatusBadRequest
switch code {
case "NoSuchBucket", "NoSuchKey":
statusCode = http.StatusNotFound
case "BucketAlreadyExists":
statusCode = http.StatusConflict
case "InvalidAccessKeyId", "SignatureDoesNotMatch":
statusCode = http.StatusForbidden
case "InternalError":
statusCode = http.StatusInternalServerError
case "InsufficientStorage":
statusCode = http.StatusInsufficientStorage
}
h.sendXMLResponse(w, statusCode, errorResp)
}
// 辅助函数解析S3路径
func parseS3Path(requestPath string) (bucket string, key string) {
requestPath = strings.TrimPrefix(requestPath, "/")
parts := strings.SplitN(requestPath, "/", 2)
if len(parts) > 0 {
bucket = parts[0]
}
if len(parts) > 1 {
key = parts[1]
}
return bucket, key
}
// 辅助函数URL编码/解码
func urlEncodePath(p string) string {
return strings.ReplaceAll(url.QueryEscape(p), "+", "%20")
}
func urlDecodePath(p string) string {
decoded, _ := url.QueryUnescape(p)
return decoded
}