GithubBackup/s3-balance
1
0
Fork 0
mirror of https://github.com/DullJZ/s3-balance.git synced 2026-09-08 01:16:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix the deleted key issue

Browse Source
This commit is contained in:
DullJZ
2025-08-25 23:12:35 +08:00
parent 1d6a94e5b8
commit 8a340f2fa2
3 changed files with 178 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/s3_handler.go
+77 -84
View File
@@ -120,54 +120,54 @@ type CommonPrefix struct {
}
type InitiateMultipartUploadResult struct {
XMLName xml.Name `xml:"InitiateMultipartUploadResult"`
Xmlns string `xml:"xmlns,attr"`
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
XMLName xml.Name `xml:"InitiateMultipartUploadResult"`
Xmlns string `xml:"xmlns,attr"`
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
}
type ListMultipartUploadsResult struct {
XMLName xml.Name `xml:"ListMultipartUploadsResult"`
Xmlns string `xml:"xmlns,attr"`
Bucket string `xml:"Bucket"`
KeyMarker string `xml:"KeyMarker"`
UploadIdMarker string `xml:"UploadIdMarker"`
NextKeyMarker string `xml:"NextKeyMarker"`
NextUploadIdMarker string `xml:"NextUploadIdMarker"`
MaxUploads int `xml:"MaxUploads"`
IsTruncated bool `xml:"IsTruncated"`
Uploads []Upload `xml:"Upload"`
CommonPrefixes []CommonPrefix `xml:"CommonPrefixes,omitempty"`
XMLName xml.Name `xml:"ListMultipartUploadsResult"`
Xmlns string `xml:"xmlns,attr"`
Bucket string `xml:"Bucket"`
KeyMarker string `xml:"KeyMarker"`
UploadIdMarker string `xml:"UploadIdMarker"`
NextKeyMarker string `xml:"NextKeyMarker"`
NextUploadIdMarker string `xml:"NextUploadIdMarker"`
MaxUploads int `xml:"MaxUploads"`
IsTruncated bool `xml:"IsTruncated"`
Uploads []Upload `xml:"Upload"`
CommonPrefixes []CommonPrefix `xml:"CommonPrefixes,omitempty"`
}
type Upload struct {
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
Initiator Owner `xml:"Initiator"`
Owner Owner `xml:"Owner"`
StorageClass string `xml:"StorageClass"`
Initiated time.Time `xml:"Initiated"`
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
Initiator Owner `xml:"Initiator"`
Owner Owner `xml:"Owner"`
StorageClass string `xml:"StorageClass"`
Initiated time.Time `xml:"Initiated"`
}
type ListPartsResult struct {
XMLName xml.Name `xml:"ListPartsResult"`
Xmlns string `xml:"xmlns,attr"`
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
PartNumberMarker int `xml:"PartNumberMarker"`
NextPartNumberMarker int `xml:"NextPartNumberMarker"`
MaxParts int `xml:"MaxParts"`
IsTruncated bool `xml:"IsTruncated"`
Parts []Part `xml:"Part"`
XMLName xml.Name `xml:"ListPartsResult"`
Xmlns string `xml:"xmlns,attr"`
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
UploadID string `xml:"UploadId"`
PartNumberMarker int `xml:"PartNumberMarker"`
NextPartNumberMarker int `xml:"NextPartNumberMarker"`
MaxParts int `xml:"MaxParts"`
IsTruncated bool `xml:"IsTruncated"`
Parts []Part `xml:"Part"`
}
type Part struct {
PartNumber int `xml:"PartNumber"`
LastModified time.Time `xml:"LastModified"`
ETag string `xml:"ETag"`
Size int64 `xml:"Size"`
ETag string `xml:"ETag"`
Size int64 `xml:"Size"`
}
type CompleteMultipartUpload struct {
@@ -388,7 +388,7 @@ func (h *S3Handler) handleCreateBucket(w http.ResponseWriter, r *http.Request, b
// 在负载均衡场景下，不真正创建bucket，只返回成功
// 实际的bucket应该在配置中预先定义
w.Header().Set("Location", "/" + bucketName)
w.Header().Set("Location", "/"+bucketName)
w.WriteHeader(http.StatusOK)
}
@@ -551,7 +551,7 @@ func (h *S3Handler) setObjectHeaders(w http.ResponseWriter, obj *storage.Object)
}
}
// handlePutObject 上传对象（默认使用预签名URL重定向）
// handlePutObject 上传对象
func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, bucketName string, key string) {
// 检查请求的存储桶是否为虚拟存储桶
requestedBucket, ok := h.bucketManager.GetBucket(bucketName)
@@ -614,51 +614,44 @@ func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, buck
return
}
// 默认使用预签名重定向模式，只有明确指定时才使用代理模式
if r.URL.Query().Get("proxy") == "true" {
// 代理模式：读取请求体并上传到预签名URL
// 创建新的请求
req, err := http.NewRequest(uploadInfo.Method, uploadInfo.URL, r.Body)
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to create upload request", key)
return
}
// 只使用反向代理上传到真实预签名URL，不再返回307重定向
// 创建新的请求
req, err := http.NewRequest(uploadInfo.Method, uploadInfo.URL, r.Body)
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to create upload request", key)
return
}
// 设置必要的头
req.ContentLength = contentLength
if ct := r.Header.Get("Content-Type"); ct != "" {
req.Header.Set("Content-Type", ct)
}
// 设置必要的头
req.ContentLength = contentLength
if ct := r.Header.Get("Content-Type"); ct != "" {
req.Header.Set("Content-Type", ct)
}
// 添加预签名URL所需的额外头
for k, v := range uploadInfo.Headers {
req.Header.Set(k, v)
}
// 添加预签名URL所需的额外头
for k, v := range uploadInfo.Headers {
req.Header.Set(k, v)
}
// 执行上传
client := &http.Client{Timeout: 30 * time.Minute}
resp, err := client.Do(req)
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to upload object", key)
return
}
defer resp.Body.Close()
// 执行上传
client := &http.Client{Timeout: 30 * time.Minute}
resp, err := client.Do(req)
if err != nil {
h.sendS3Error(w, "InternalError", "Failed to upload object", key)
return
}
defer resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
// 记录对象元数据
h.storage.RecordObject(key, targetBucket.Config.Name, contentLength, nil)
targetBucket.UpdateUsedSize(contentLength)
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
// 记录对象元数据
h.storage.RecordObject(key, targetBucket.Config.Name, contentLength, nil)
targetBucket.UpdateUsedSize(contentLength)
// 返回成功响应
w.Header().Set("ETag", fmt.Sprintf("\"%x\"", time.Now().UnixNano()))
w.WriteHeader(http.StatusOK)
} else {
h.sendS3Error(w, "InternalError", "Upload failed", key)
}
// 返回成功响应
w.Header().Set("ETag", fmt.Sprintf("\"%x\"", time.Now().UnixNano()))
w.WriteHeader(http.StatusOK)
} else {
// 重定向模式：返回307临时重定向让客户端直接上传（默认）
w.Header().Set("Location", uploadInfo.URL)
w.WriteHeader(http.StatusTemporaryRedirect)
h.sendS3Error(w, "InternalError", "Upload failed", key)
}
}
@@ -801,14 +794,14 @@ func (h *S3Handler) handleListMultipartParts(w http.ResponseWriter, r *http.Requ
// 简化实现：返回空列表
result := ListPartsResult{
Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
Bucket: bucketName,
Key: key,
UploadID: uploadID,
PartNumberMarker: 0,
MaxParts: 1000,
IsTruncated: false,
Parts: make([]Part, 0),
Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
Bucket: bucketName,
Key: key,
UploadID: uploadID,
PartNumberMarker: 0,
MaxParts: 1000,
IsTruncated: false,
Parts: make([]Part, 0),
}
h.sendXMLResponse(w, http.StatusOK, result)
@@ -858,7 +851,7 @@ func (h *S3Handler) handleCompleteMultipartUpload(w http.ResponseWriter, r *http
result := CompleteMultipartUploadResult{
Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/",
Location: "/" + bucket.Config.Name + "/" + key,
Location: "/" + bucket.Config.Name + "/" + key,
Bucket: bucket.Config.Name,
Key: key,
ETag: *completeResp.ETag,
internal/storage/models.go
+1 -1
View File
@@ -11,7 +11,7 @@ import (
// Object 对象信息模型
type Object struct {
ID uint `gorm:"primaryKey" json:"id"`
Key string `gorm:"uniqueIndex;size:512;not null" json:"key"`
Key string `gorm:"size:512;not null" json:"key"`
BucketName string `gorm:"index;size:255;not null" json:"bucket_name"`
Size int64 `gorm:"not null;default:0" json:"size"`
Metadata JSON `gorm:"type:json" json:"metadata,omitempty"`
internal/storage/service.go
+14 -2
View File
@@ -21,6 +21,15 @@ func NewService(db *gorm.DB) *Service {
// RecordObject 记录对象信息
func (s *Service) RecordObject(key, bucketName string, size int64, metadata map[string]string) error {
// 首先检查是否存在已删除的同名对象
var deletedObj Object
if err := s.db.Unscoped().Where("`key` = ?", key).Where("`deleted_at` IS NOT NULL").First(&deletedObj).Error; err == nil {
// 存在已删除的同名对象，永久删除它
if err := s.db.Unscoped().Delete(&deletedObj).Error; err != nil {
return fmt.Errorf("failed to permanently delete soft-deleted object: %w", err)
}
}
obj := &Object{
Key: key,
BucketName: bucketName,
@@ -37,7 +46,7 @@ func (s *Service) RecordObject(key, bucketName string, size int64, metadata map[
}
// 使用 Upsert（更新或插入）
result := s.db.Where("`key` = ?", key).FirstOrCreate(&obj)
result := s.db.Where("`key` = ?", key).Where("`deleted_at` IS NULL").FirstOrCreate(&obj)
if result.Error != nil {
return fmt.Errorf("failed to record object: %w", result.Error)
}
@@ -197,7 +206,10 @@ func (s *Service) updateBucketStats(bucketName string) error {
// 获取或创建统计记录
result := s.db.Where("bucket_name = ?", bucketName).FirstOrCreate(&stats, BucketStats{
BucketName: bucketName,
BucketName: bucketName,
LastCheckedAt: time.Now(),
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
})
if result.Error != nil {
return fmt.Errorf("failed to get bucket stats: %w", result.Error)