From 6b0d2e85960974f5cab01599180ea46e3ca2d007 Mon Sep 17 00:00:00 2001 From: DullJZ <79080562+DullJZ@users.noreply.github.com> Date: Sat, 15 Nov 2025 17:32:50 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8DCopyObject=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=EF=BC=8C=E6=B7=BB=E5=8A=A0RealObjectKey=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E7=9C=9F=E6=AD=A3=E9=9B=B6=E6=8B=B7=E8=B4=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/api/bucket_handler.go | 2 +- internal/api/multipart_handler.go | 8 +- internal/api/object_handler.go | 186 +++++++++++++++++++++++++----- internal/storage/models.go | 3 +- internal/storage/service.go | 111 ++++++++++++++++-- 5 files changed, 267 insertions(+), 43 deletions(-) diff --git a/internal/api/bucket_handler.go b/internal/api/bucket_handler.go index 4767b0b..98ecbe1 100644 --- a/internal/api/bucket_handler.go +++ b/internal/api/bucket_handler.go @@ -237,7 +237,7 @@ func (h *S3Handler) handleCreateBucket(w http.ResponseWriter, r *http.Request, b targetBucket := realBuckets[0] // 创建虚拟存储桶到真实存储桶的映射 - if err := h.storage.CreateVirtualBucketMapping(bucketName, "", targetBucket.Config.Name); err != nil { + if err := h.storage.CreateVirtualBucketMapping(bucketName, "", targetBucket.Config.Name, ""); err != nil { h.sendS3Error(w, "InternalError", "Failed to create virtual bucket mapping", bucketName) return } diff --git a/internal/api/multipart_handler.go b/internal/api/multipart_handler.go index 4aca3f3..b86c2c9 100644 --- a/internal/api/multipart_handler.go +++ b/internal/api/multipart_handler.go @@ -57,8 +57,8 @@ func (h *S3Handler) handleUploadPart(w http.ResponseWriter, r *http.Request) { return } - // 创建虚拟存储桶文件级映射 - if err := h.storage.CreateVirtualBucketMapping(bucketName, key, targetBucket.Config.Name); err != nil { + // 创建虚拟存储桶文件级映射（对于Multipart，虚拟key和真实key相同） + if err := h.storage.CreateVirtualBucketMapping(bucketName, key, targetBucket.Config.Name, key); err != nil { h.sendS3Error(w, "InternalError", "Failed to create virtual bucket file mapping", key) return } @@ -204,8 +204,8 @@ func (h *S3Handler) handleMultipartUpload(w http.ResponseWriter, r *http.Request return } - // 创建虚拟存储桶文件级映射 - if err := h.storage.CreateVirtualBucketMapping(bucketName, key, targetBucket.Config.Name); err != nil { + // 创建虚拟存储桶文件级映射（对于Multipart，虚拟key和真实key相同） + if err := h.storage.CreateVirtualBucketMapping(bucketName, key, targetBucket.Config.Name, key); err != nil { h.sendS3Error(w, "InternalError", "Failed to create virtual bucket file mapping", key) return } diff --git a/internal/api/object_handler.go b/internal/api/object_handler.go index 0655b19..c47eccf 100644 --- a/internal/api/object_handler.go +++ b/internal/api/object_handler.go @@ -6,7 +6,9 @@ import ( "io" "log" "net/http" + "net/url" "strconv" + "strings" "time" "github.com/DullJZ/s3-balance/internal/bucket" @@ -56,6 +58,7 @@ func (h *S3Handler) handleGetObject(w http.ResponseWriter, r *http.Request, buck var err error var bucket1 *bucket.BucketInfo + var realKey string if requestedBucket.IsVirtual() { // 获取虚拟存储桶映射 mapping, err := h.storage.GetVirtualBucketMapping(bucketName, key) @@ -64,21 +67,24 @@ func (h *S3Handler) handleGetObject(w http.ResponseWriter, r *http.Request, buck return } - // 获取映射到的真实存储桶 + // 获取映射到的真实存储桶和真实key bucket1, ok = h.bucketManager.GetBucket(mapping.RealBucketName) if !ok { h.sendS3Error(w, "InternalError", "Mapped real bucket not found", key) return } + realKey = mapping.RealObjectKey h.recordBackendOperation(bucket1, bucket.OperationTypeB) + } else { + realKey = key } - // 生成预签名下载URL + // 生成预签名下载URL（使用真实key） downloadInfo, err := h.presigner.GenerateDownloadURL( context.Background(), bucket1, - key, + realKey, ) if err != nil { h.sendS3Error(w, "InternalError", "Failed to generate download URL", key) @@ -182,6 +188,13 @@ func (h *S3Handler) handleHeadObject(w http.ResponseWriter, r *http.Request, buc // handlePutObject 上传对象 func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, bucketName string, key string) { + // 检查是否是复制操作（CopyObject） + copySource := r.Header.Get("x-amz-copy-source") + if copySource != "" { + h.handleCopyObject(w, r, bucketName, key, copySource) + return + } + // 检查请求的存储桶是否为虚拟存储桶 requestedBucket, ok := h.bucketManager.GetBucket(bucketName) if !ok { @@ -211,8 +224,8 @@ func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, buck return } - // 创建虚拟存储桶文件级映射 - if err := h.storage.CreateVirtualBucketMapping(bucketName, key, targetBucket.Config.Name); err != nil { + // 创建虚拟存储桶文件级映射（对于普通PUT，虚拟key和真实key相同） + if err := h.storage.CreateVirtualBucketMapping(bucketName, key, targetBucket.Config.Name, key); err != nil { h.sendS3Error(w, "InternalError", "Failed to create virtual bucket file mapping", key) return } @@ -290,6 +303,108 @@ func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, buck } } +// handleCopyObject 复制对象（只在数据库中创建新映射） +func (h *S3Handler) handleCopyObject(w http.ResponseWriter, r *http.Request, destBucket, destKey, copySource string) { + // 解析复制源 (格式: /source-bucket/source-key 或 source-bucket/source-key) + copySource = strings.TrimPrefix(copySource, "/") + parts := strings.SplitN(copySource, "/", 2) + if len(parts) != 2 { + h.sendS3Error(w, "InvalidArgument", "Invalid copy source format", copySource) + return + } + + sourceBucket := parts[0] + sourceKey := parts[1] + + // URL 解码源对象键 + sourceKey, err := url.QueryUnescape(sourceKey) + if err != nil { + h.sendS3Error(w, "InvalidArgument", "Invalid source key encoding", sourceKey) + return + } + + // 检查目标存储桶是否存在 + _, ok := h.bucketManager.GetBucket(destBucket) + if !ok { + h.sendS3Error(w, "NoSuchBucket", "The specified bucket does not exist", destBucket) + return + } + + // 检查源存储桶是否存在 + _, ok = h.bucketManager.GetBucket(sourceBucket) + if !ok { + h.sendS3Error(w, "NoSuchBucket", "The source bucket does not exist", sourceBucket) + return + } + + // 获取元数据指令 + metadataDirective := r.Header.Get("x-amz-metadata-directive") + var metadata map[string]string + + if metadataDirective == "REPLACE" { + // 使用请求中的新元数据 + metadata = make(map[string]string) + for k, v := range r.Header { + if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { + metaKey := strings.TrimPrefix(strings.ToLower(k), "x-amz-meta-") + metadata[metaKey] = v[0] + } + } + } + // 如果是 COPY 或未指定，则复制源对象的元数据（在 storage.CopyObject 中处理） + + // 获取源对象的映射信息 + sourceMapping, err := h.storage.GetVirtualBucketMapping(sourceBucket, sourceKey) + if err != nil { + log.Printf("Failed to get source object mapping %s: %v", sourceKey, err) + h.sendS3Error(w, "NoSuchKey", "The specified key does not exist", sourceKey) + return + } + + // 复制操作只创建新的虚拟映射，指向相同的真实对象（零拷贝） + destBucketInfo, destOk := h.bucketManager.GetBucket(destBucket) + if !destOk || !destBucketInfo.IsVirtual() { + h.sendS3Error(w, "NoSuchBucket", "The destination bucket does not exist", destBucket) + return + } + + // 创建新的虚拟存储桶映射，指向相同的真实bucket和真实key + if err := h.storage.CreateVirtualBucketMapping(destBucket, destKey, sourceMapping.RealBucketName, sourceMapping.RealObjectKey); err != nil { + log.Printf("Failed to create virtual bucket mapping for copied object %s: %v", destKey, err) + h.sendS3Error(w, "InternalError", "Failed to create virtual bucket file mapping", destKey) + return + } + + // 获取源对象信息用于响应 + sourceObj, err := h.storage.GetObjectInfo(sourceMapping.RealObjectKey) + if err != nil { + log.Printf("Failed to get source object info: %v", err) + } + + // 返回成功响应 + w.Header().Set("Content-Type", "application/xml") + + // 生成 ETag + etag := fmt.Sprintf("\"%x\"", time.Now().UnixNano()) + + // 构造 CopyObjectResult XML 响应 + lastModified := time.Now().UTC().Format(time.RFC3339) + if sourceObj != nil { + lastModified = sourceObj.UpdatedAt.UTC().Format(time.RFC3339) + } + + response := fmt.Sprintf(` + + %s + %s +`, lastModified, etag) + + w.WriteHeader(http.StatusOK) + w.Write([]byte(response)) + + log.Printf("Object copied successfully: %s -> %s", sourceKey, destKey) +} + // handleDeleteObject 删除对象 func (h *S3Handler) handleDeleteObject(w http.ResponseWriter, r *http.Request, bucketName string, key string) { // 检查请求的存储桶是否为虚拟存储桶 @@ -301,6 +416,7 @@ func (h *S3Handler) handleDeleteObject(w http.ResponseWriter, r *http.Request, b var targetBucket *bucket.BucketInfo var err error + var realKey string if requestedBucket.IsVirtual() { // 获取虚拟存储桶文件映射 @@ -311,47 +427,59 @@ func (h *S3Handler) handleDeleteObject(w http.ResponseWriter, r *http.Request, b return } - // 获取映射到的真实存储桶 + // 获取映射到的真实存储桶和真实key targetBucket, ok = h.bucketManager.GetBucket(mapping.RealBucketName) if !ok { h.sendS3Error(w, "InternalError", "Mapped real bucket not found", key) return } + + realKey = mapping.RealObjectKey } else { // 如果不是虚拟存储桶，拒绝客户端对真实存储桶的直接DELETE操作 w.WriteHeader(http.StatusNoContent) return } - h.recordBackendOperation(targetBucket, bucket.OperationTypeA) - - // 生成预签名删除URL - deleteInfo, err := h.presigner.GenerateDeleteURL( - context.Background(), - targetBucket, - key, - ) - if err != nil { - h.sendS3Error(w, "InternalError", "Failed to generate delete URL", key) - return + // 先删除虚拟存储桶映射 + if err := h.storage.DeleteVirtualBucketObjectMapping(bucketName, key); err != nil { + log.Printf("Failed to delete virtual bucket mapping for %s/%s: %v", bucketName, key, err) } - // 执行删除 - req, _ := http.NewRequest("DELETE", deleteInfo.URL, nil) - client := &http.Client{Timeout: 30 * time.Second} - resp, err := client.Do(req) + // 检查是否还有其他映射指向同一个真实对象 + count, err := h.storage.CountMappingsToRealObject(targetBucket.Config.Name, realKey) if err != nil { - h.sendS3Error(w, "InternalError", "Failed to delete object", key) - return + log.Printf("Failed to count mappings for real object %s: %v", realKey, err) } - defer resp.Body.Close() - // 从数据库中删除对象记录 - h.storage.DeleteObject(key) + // 只有当没有其他映射引用时，才删除真实S3对象 + if count == 0 { + h.recordBackendOperation(targetBucket, bucket.OperationTypeA) - // 如果是虚拟存储桶，还需要删除文件级别映射 - if requestedBucket.IsVirtual() { - h.storage.DeleteVirtualBucketFileMapping(bucketName, key) + // 生成预签名删除URL（使用真实key） + deleteInfo, err := h.presigner.GenerateDeleteURL( + context.Background(), + targetBucket, + realKey, + ) + if err != nil { + log.Printf("Failed to generate delete URL for %s: %v", realKey, err) + } else { + // 执行删除真实S3对象 + req, _ := http.NewRequest("DELETE", deleteInfo.URL, nil) + client := &http.Client{Timeout: 30 * time.Second} + resp, err := client.Do(req) + if err != nil { + log.Printf("Failed to delete real S3 object %s: %v", realKey, err) + } else { + defer resp.Body.Close() + } + } + + // 从数据库中删除对象记录 + if err := h.storage.DeleteObject(realKey); err != nil { + log.Printf("Failed to delete object record for %s: %v", realKey, err) + } } // S3规范要求删除操作总是返回204 diff --git a/internal/storage/models.go b/internal/storage/models.go index b87eef6..924ea6b 100644 --- a/internal/storage/models.go +++ b/internal/storage/models.go @@ -66,8 +66,9 @@ func (BucketMonthlyStats) TableName() string { type VirtualBucketMapping struct { ID uint `gorm:"primaryKey" json:"id"` VirtualBucketName string `gorm:"index;size:255;not null" json:"virtual_bucket_name"` - ObjectKey string `gorm:"index;size:512;not null" json:"object_key"` + ObjectKey string `gorm:"index;size:512;not null" json:"object_key"` // 虚拟对象key RealBucketName string `gorm:"index;size:255;not null" json:"real_bucket_name"` + RealObjectKey string `gorm:"size:512;not null" json:"real_object_key"` // 真实对象key CreatedAt time.Time `gorm:"not null" json:"created_at"` UpdatedAt time.Time `gorm:"not null" json:"updated_at"` } diff --git a/internal/storage/service.go b/internal/storage/service.go index e7ba263..7e75b02 100644 --- a/internal/storage/service.go +++ b/internal/storage/service.go @@ -101,6 +101,63 @@ func (s *Service) GetObjectInfo(key string) (*Object, error) { return &obj, nil } +// CopyObject 复制对象（只创建新的数据库映射记录，不复制实际数据） +func (s *Service) CopyObject(sourceKey, destKey string, metadata map[string]string) error { + // 获取源对象信息 + sourceObj, err := s.GetObjectInfo(sourceKey) + if err != nil { + return fmt.Errorf("source object not found: %w", err) + } + + // 检查目标对象是否已存在 + var existingObj Object + if err := s.db.Where("`key` = ?", destKey).Where("`deleted_at` IS NULL").First(&existingObj).Error; err == nil { + // 目标对象已存在，删除旧的 + if err := s.DeleteObject(destKey); err != nil { + return fmt.Errorf("failed to delete existing destination object: %w", err) + } + } + + // 清理已软删除的同名对象 + var deletedObj Object + if err := s.db.Unscoped().Where("`key` = ?", destKey).Where("`deleted_at` IS NOT NULL").First(&deletedObj).Error; err == nil { + if err := s.db.Unscoped().Delete(&deletedObj).Error; err != nil { + return fmt.Errorf("failed to permanently delete soft-deleted object: %w", err) + } + } + + // 创建新的对象记录，指向相同的真实存储桶 + newObj := &Object{ + Key: destKey, + BucketName: sourceObj.BucketName, // 指向相同的真实桶 + Size: sourceObj.Size, + } + + // 使用提供的元数据，如果没有则复制源对象的元数据 + if len(metadata) > 0 { + newObj.Metadata = make(JSON) + for k, v := range metadata { + newObj.Metadata[k] = v + } + } else { + // 复制源对象的元数据 + newObj.Metadata = make(JSON) + for k, v := range sourceObj.Metadata { + newObj.Metadata[k] = v + } + } + + // 插入新记录 + if err := s.db.Create(newObj).Error; err != nil { + return fmt.Errorf("failed to create copy object record: %w", err) + } + + // 更新存储桶统计 + s.updateBucketStats(sourceObj.BucketName) + + return nil +} + // DeleteObject 删除对象记录（软删除） func (s *Service) DeleteObject(key string) error { var obj Object @@ -527,11 +584,12 @@ func (s *Service) GetAccessLogs(filter *AccessLogFilter) ([]*AccessLog, error) { } // CreateVirtualBucketMapping 创建虚拟存储桶文件级映射 -func (s *Service) CreateVirtualBucketMapping(virtualBucketName, objectKey, realBucketName string) error { +func (s *Service) CreateVirtualBucketMapping(virtualBucketName, objectKey, realBucketName, realObjectKey string) error { mapping := &VirtualBucketMapping{ VirtualBucketName: virtualBucketName, ObjectKey: objectKey, RealBucketName: realBucketName, + RealObjectKey: realObjectKey, } if err := s.db.Create(mapping).Error; err != nil { @@ -553,6 +611,17 @@ func (s *Service) GetVirtualBucketMapping(virtualBucketName, objectKey string) ( return &mapping, nil } +// CountMappingsToRealObject 统计指向同一真实对象的映射数量 +func (s *Service) CountMappingsToRealObject(realBucketName, realObjectKey string) (int64, error) { + var count int64 + if err := s.db.Model(&VirtualBucketMapping{}). + Where("real_bucket_name = ? AND real_object_key = ?", realBucketName, realObjectKey). + Count(&count).Error; err != nil { + return 0, fmt.Errorf("failed to count mappings: %w", err) + } + return count, nil +} + // GetVirtualBucketMappings 获取所有虚拟存储桶映射 func (s *Service) GetVirtualBucketMappings() ([]*VirtualBucketMapping, error) { var mappings []*VirtualBucketMapping @@ -618,19 +687,45 @@ func (s *Service) GetVirtualBucketObjects(virtualBucketName string) ([]*Object, return []*Object{}, nil } - // 收集所有对象键 - objectKeys := make([]string, 0, len(mappings)) + // 收集所有真实对象键 + realObjectKeys := make([]string, 0, len(mappings)) for _, mapping := range mappings { - objectKeys = append(objectKeys, mapping.ObjectKey) + realObjectKeys = append(realObjectKeys, mapping.RealObjectKey) } - // 从对象表中查询这些对象 - var objects []*Object - if err := s.db.Where("`key` IN ?", objectKeys).Find(&objects).Error; err != nil { + // 从对象表中查询这些真实对象 + var realObjects []*Object + if err := s.db.Where("`key` IN ?", realObjectKeys).Find(&realObjects).Error; err != nil { return nil, fmt.Errorf("failed to get objects for virtual bucket: %w", err) } - return objects, nil + // 创建真实key到对象的映射 + realObjectMap := make(map[string]*Object) + for _, obj := range realObjects { + realObjectMap[obj.Key] = obj + } + + // 构建虚拟对象列表（使用虚拟key，但其他信息来自真实对象） + virtualObjects := make([]*Object, 0, len(mappings)) + for _, mapping := range mappings { + if realObj, exists := realObjectMap[mapping.RealObjectKey]; exists { + // 创建虚拟对象副本，使用虚拟key + virtualObj := &Object{ + ID: realObj.ID, + Key: mapping.ObjectKey, // 使用虚拟key + BucketName: realObj.BucketName, + Size: realObj.Size, + Metadata: realObj.Metadata, + ContentType: realObj.ContentType, + ETag: realObj.ETag, + CreatedAt: realObj.CreatedAt, + UpdatedAt: realObj.UpdatedAt, + } + virtualObjects = append(virtualObjects, virtualObj) + } + } + + return virtualObjects, nil } // DeleteVirtualBucketFileMapping 删除虚拟存储桶文件映射