GithubBackup/s3-balance
1
0
Fork 0
mirror of https://github.com/DullJZ/s3-balance.git synced 2026-09-05 15:56:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix the deleted key issue

Browse Source
This commit is contained in:
DullJZ
2025-08-25 23:12:35 +08:00
parent 1d6a94e5b8
commit 8a340f2fa2
3 changed files with 178 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/s3_handler.go
+77 -84
View File
@@ -120,54 +120,54 @@ type CommonPrefix struct {
} }
type InitiateMultipartUploadResult struct { type InitiateMultipartUploadResult struct {
XMLName xml.Name `xml:"InitiateMultipartUploadResult"` XMLName xml.Name `xml:"InitiateMultipartUploadResult"`
Xmlns string `xml:"xmlns,attr"` Xmlns string `xml:"xmlns,attr"`
Bucket string `xml:"Bucket"` Bucket string `xml:"Bucket"`
Key string `xml:"Key"` Key string `xml:"Key"`
UploadID string `xml:"UploadId"` UploadID string `xml:"UploadId"`
} }
type ListMultipartUploadsResult struct { type ListMultipartUploadsResult struct {
XMLName xml.Name `xml:"ListMultipartUploadsResult"` XMLName xml.Name `xml:"ListMultipartUploadsResult"`
Xmlns string `xml:"xmlns,attr"` Xmlns string `xml:"xmlns,attr"`
Bucket string `xml:"Bucket"` Bucket string `xml:"Bucket"`
KeyMarker string `xml:"KeyMarker"` KeyMarker string `xml:"KeyMarker"`
UploadIdMarker string `xml:"UploadIdMarker"` UploadIdMarker string `xml:"UploadIdMarker"`
NextKeyMarker string `xml:"NextKeyMarker"` NextKeyMarker string `xml:"NextKeyMarker"`
NextUploadIdMarker string `xml:"NextUploadIdMarker"` NextUploadIdMarker string `xml:"NextUploadIdMarker"`
MaxUploads int `xml:"MaxUploads"` MaxUploads int `xml:"MaxUploads"`
IsTruncated bool `xml:"IsTruncated"` IsTruncated bool `xml:"IsTruncated"`
Uploads []Upload `xml:"Upload"` Uploads []Upload `xml:"Upload"`
CommonPrefixes []CommonPrefix `xml:"CommonPrefixes,omitempty"` CommonPrefixes []CommonPrefix `xml:"CommonPrefixes,omitempty"`
} }
type Upload struct { type Upload struct {
Key string `xml:"Key"` Key string `xml:"Key"`
UploadID string `xml:"UploadId"` UploadID string `xml:"UploadId"`
Initiator Owner `xml:"Initiator"` Initiator Owner `xml:"Initiator"`
Owner Owner `xml:"Owner"` Owner Owner `xml:"Owner"`
StorageClass string `xml:"StorageClass"` StorageClass string `xml:"StorageClass"`
Initiated time.Time `xml:"Initiated"` Initiated time.Time `xml:"Initiated"`
} }
type ListPartsResult struct { type ListPartsResult struct {
XMLName xml.Name `xml:"ListPartsResult"` XMLName xml.Name `xml:"ListPartsResult"`
Xmlns string `xml:"xmlns,attr"` Xmlns string `xml:"xmlns,attr"`
Bucket string `xml:"Bucket"` Bucket string `xml:"Bucket"`
Key string `xml:"Key"` Key string `xml:"Key"`
UploadID string `xml:"UploadId"` UploadID string `xml:"UploadId"`
PartNumberMarker int `xml:"PartNumberMarker"` PartNumberMarker int `xml:"PartNumberMarker"`
NextPartNumberMarker int `xml:"NextPartNumberMarker"` NextPartNumberMarker int `xml:"NextPartNumberMarker"`
MaxParts int `xml:"MaxParts"` MaxParts int `xml:"MaxParts"`
IsTruncated bool `xml:"IsTruncated"` IsTruncated bool `xml:"IsTruncated"`
Parts []Part `xml:"Part"` Parts []Part `xml:"Part"`
} }
type Part struct { type Part struct {
PartNumber int `xml:"PartNumber"` PartNumber int `xml:"PartNumber"`
LastModified time.Time `xml:"LastModified"` LastModified time.Time `xml:"LastModified"`
ETag string `xml:"ETag"` ETag string `xml:"ETag"`
Size int64 `xml:"Size"` Size int64 `xml:"Size"`
} }
type CompleteMultipartUpload struct { type CompleteMultipartUpload struct {
@@ -388,7 +388,7 @@ func (h *S3Handler) handleCreateBucket(w http.ResponseWriter, r *http.Request, b
// 在负载均衡场景下，不真正创建bucket，只返回成功 // 在负载均衡场景下，不真正创建bucket，只返回成功
// 实际的bucket应该在配置中预先定义 // 实际的bucket应该在配置中预先定义
w.Header().Set("Location", "/" + bucketName) w.Header().Set("Location", "/"+bucketName)
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
} }
@@ -551,7 +551,7 @@ func (h *S3Handler) setObjectHeaders(w http.ResponseWriter, obj *storage.Object)
} }
} }
// handlePutObject 上传对象（默认使用预签名URL重定向） // handlePutObject 上传对象
func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, bucketName string, key string) { func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, bucketName string, key string) {
// 检查请求的存储桶是否为虚拟存储桶 // 检查请求的存储桶是否为虚拟存储桶
requestedBucket, ok := h.bucketManager.GetBucket(bucketName) requestedBucket, ok := h.bucketManager.GetBucket(bucketName)
@@ -614,51 +614,44 @@ func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, buck
return return
} }
// 默认使用预签名重定向模式，只有明确指定时才使用代理模式 // 只使用反向代理上传到真实预签名URL，不再返回307重定向
if r.URL.Query().Get("proxy") == "true" { // 创建新的请求
// 代理模式：读取请求体并上传到预签名URL req, err := http.NewRequest(uploadInfo.Method, uploadInfo.URL, r.Body)
// 创建新的请求 if err != nil {
req, err := http.NewRequest(uploadInfo.Method, uploadInfo.URL, r.Body) h.sendS3Error(w, "InternalError", "Failed to create upload request", key)
if err != nil { return
h.sendS3Error(w, "InternalError", "Failed to create upload request", key) }
return
}
// 设置必要的头 // 设置必要的头
req.ContentLength = contentLength req.ContentLength = contentLength
if ct := r.Header.Get("Content-Type"); ct != "" { if ct := r.Header.Get("Content-Type"); ct != "" {
req.Header.Set("Content-Type", ct) req.Header.Set("Content-Type", ct)
} }
// 添加预签名URL所需的额外头 // 添加预签名URL所需的额外头
for k, v := range uploadInfo.Headers { for k, v := range uploadInfo.Headers {
req.Header.Set(k, v) req.Header.Set(k, v)
} }
// 执行上传 // 执行上传
client := &http.Client{Timeout: 30 * time.Minute} client := &http.Client{Timeout: 30 * time.Minute}
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
h.sendS3Error(w, "InternalError", "Failed to upload object", key) h.sendS3Error(w, "InternalError", "Failed to upload object", key)
return return
} }
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 { if resp.StatusCode >= 200 && resp.StatusCode < 300 {
// 记录对象元数据 // 记录对象元数据
h.storage.RecordObject(key, targetBucket.Config.Name, contentLength, nil) h.storage.RecordObject(key, targetBucket.Config.Name, contentLength, nil)
targetBucket.UpdateUsedSize(contentLength) targetBucket.UpdateUsedSize(contentLength)
// 返回成功响应 // 返回成功响应
w.Header().Set("ETag", fmt.Sprintf("\"%x\"", time.Now().UnixNano())) w.Header().Set("ETag", fmt.Sprintf("\"%x\"", time.Now().UnixNano()))
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
} else {
h.sendS3Error(w, "InternalError", "Upload failed", key)
}
} else { } else {
// 重定向模式：返回307临时重定向让客户端直接上传（默认） h.sendS3Error(w, "InternalError", "Upload failed", key)
w.Header().Set("Location", uploadInfo.URL)
w.WriteHeader(http.StatusTemporaryRedirect)
} }
} }
@@ -801,14 +794,14 @@ func (h *S3Handler) handleListMultipartParts(w http.ResponseWriter, r *http.Requ
// 简化实现：返回空列表 // 简化实现：返回空列表
result := ListPartsResult{ result := ListPartsResult{
Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
Bucket: bucketName, Bucket: bucketName,
Key: key, Key: key,
UploadID: uploadID, UploadID: uploadID,
PartNumberMarker: 0, PartNumberMarker: 0,
MaxParts: 1000, MaxParts: 1000,
IsTruncated: false, IsTruncated: false,
Parts: make([]Part, 0), Parts: make([]Part, 0),
} }
h.sendXMLResponse(w, http.StatusOK, result) h.sendXMLResponse(w, http.StatusOK, result)
@@ -858,7 +851,7 @@ func (h *S3Handler) handleCompleteMultipartUpload(w http.ResponseWriter, r *http
result := CompleteMultipartUploadResult{ result := CompleteMultipartUploadResult{
Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
Location: "/" + bucket.Config.Name + "/" + key, Location: "/" + bucket.Config.Name + "/" + key,
Bucket: bucket.Config.Name, Bucket: bucket.Config.Name,
Key: key, Key: key,
ETag: *completeResp.ETag, ETag: *completeResp.ETag,
internal/storage/models.go
+1 -1
View File
@@ -11,7 +11,7 @@ import (
// Object 对象信息模型 // Object 对象信息模型
type Object struct { type Object struct {
ID uint `gorm:"primaryKey" json:"id"` ID uint `gorm:"primaryKey" json:"id"`
Key string `gorm:"uniqueIndex;size:512;not null" json:"key"` Key string `gorm:"size:512;not null" json:"key"`
BucketName string `gorm:"index;size:255;not null" json:"bucket_name"` BucketName string `gorm:"index;size:255;not null" json:"bucket_name"`
Size int64 `gorm:"not null;default:0" json:"size"` Size int64 `gorm:"not null;default:0" json:"size"`
Metadata JSON `gorm:"type:json" json:"metadata,omitempty"` Metadata JSON `gorm:"type:json" json:"metadata,omitempty"`
internal/storage/service.go
+14 -2
View File
@@ -21,6 +21,15 @@ func NewService(db *gorm.DB) *Service {
// RecordObject 记录对象信息 // RecordObject 记录对象信息
func (s *Service) RecordObject(key, bucketName string, size int64, metadata map[string]string) error { func (s *Service) RecordObject(key, bucketName string, size int64, metadata map[string]string) error {
// 首先检查是否存在已删除的同名对象
var deletedObj Object
if err := s.db.Unscoped().Where("`key` = ?", key).Where("`deleted_at` IS NOT NULL").First(&deletedObj).Error; err == nil {
// 存在已删除的同名对象，永久删除它
if err := s.db.Unscoped().Delete(&deletedObj).Error; err != nil {
return fmt.Errorf("failed to permanently delete soft-deleted object: %w", err)
}
}
obj := &Object{ obj := &Object{
Key: key, Key: key,
BucketName: bucketName, BucketName: bucketName,
@@ -37,7 +46,7 @@ func (s *Service) RecordObject(key, bucketName string, size int64, metadata map[
} }
// 使用 Upsert（更新或插入） // 使用 Upsert（更新或插入）
result := s.db.Where("`key` = ?", key).FirstOrCreate(&obj) result := s.db.Where("`key` = ?", key).Where("`deleted_at` IS NULL").FirstOrCreate(&obj)
if result.Error != nil { if result.Error != nil {
return fmt.Errorf("failed to record object: %w", result.Error) return fmt.Errorf("failed to record object: %w", result.Error)
} }
@@ -197,7 +206,10 @@ func (s *Service) updateBucketStats(bucketName string) error {
// 获取或创建统计记录 // 获取或创建统计记录
result := s.db.Where("bucket_name = ?", bucketName).FirstOrCreate(&stats, BucketStats{ result := s.db.Where("bucket_name = ?", bucketName).FirstOrCreate(&stats, BucketStats{
BucketName: bucketName, BucketName: bucketName,
LastCheckedAt: time.Now(),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}) })
if result.Error != nil { if result.Error != nil {
return fmt.Errorf("failed to get bucket stats: %w", result.Error) return fmt.Errorf("failed to get bucket stats: %w", result.Error)