From f911cc6951ce48a45dde8552274743de4629e003 Mon Sep 17 00:00:00 2001 From: DullJZ <79080562+DullJZ@users.noreply.github.com> Date: Sat, 23 Aug 2025 01:16:33 +0800 Subject: [PATCH] Fix mapping error --- internal/api/s3_handler.go | 169 +++++++++++----------------------- internal/database/database.go | 1 + internal/storage/models.go | 9 +- internal/storage/service.go | 66 ++++++++++--- 4 files changed, 114 insertions(+), 131 deletions(-) diff --git a/internal/api/s3_handler.go b/internal/api/s3_handler.go index 070a250..3f6b7c6 100644 --- a/internal/api/s3_handler.go +++ b/internal/api/s3_handler.go @@ -267,57 +267,8 @@ func (h *S3Handler) handleListObjects(w http.ResponseWriter, r *http.Request, bu return } - // 解析查询参数 - prefix := r.URL.Query().Get("prefix") - marker := r.URL.Query().Get("marker") - maxKeysStr := r.URL.Query().Get("max-keys") - delimiter := r.URL.Query().Get("delimiter") - - maxKeys := 1000 - if maxKeysStr != "" { - if mk, err := strconv.Atoi(maxKeysStr); err == nil { - maxKeys = mk - } - } - - // 从存储中获取对象列表 - objects, err := h.storage.ListObjects(bucketName, prefix, marker, maxKeys) - if err != nil { - h.sendS3Error(w, "InternalError", "Internal server error", bucketName) - return - } - - result := ListBucketResult{ - Xmlns: "http://s3.amazonaws.com/doc/2006-03-01/", - Name: bucketName, - Prefix: prefix, - Marker: marker, - MaxKeys: maxKeys, - IsTruncated: false, // 简化实现 - Contents: make([]ObjectInfo, 0), - } - - // 处理分隔符逻辑 - if delimiter != "" { - // 简化的分隔符处理 - result.CommonPrefixes = make([]CommonPrefix, 0) - } - - for _, obj := range objects { - result.Contents = append(result.Contents, ObjectInfo{ - Key: obj.Key, - LastModified: obj.UpdatedAt, - ETag: fmt.Sprintf("\"%x\"", obj.ID), // 简化的ETag - Size: obj.Size, - StorageClass: "STANDARD", - Owner: Owner{ - ID: "s3-balance", - DisplayName: "S3 Balance Service", - }, - }) - } - - h.sendXMLResponse(w, http.StatusOK, result) + // 如果不是虚拟存储桶，拒绝客户端访问真实存储桶 + h.sendS3Error(w, "NoSuchBucket", "The specified bucket does not exist", bucketName) } // handleListObjectsForVirtualBucket 列出虚拟存储桶中的对象 @@ -395,7 +346,8 @@ func (h *S3Handler) handleHeadBucket(w http.ResponseWriter, r *http.Request, buc return } - w.WriteHeader(http.StatusOK) + // 如果不是虚拟存储桶，拒绝客户端访问真实存储桶 + w.WriteHeader(http.StatusNotFound) } // handleCreateBucket 创建存储桶（虚拟实现） @@ -404,10 +356,9 @@ func (h *S3Handler) handleCreateBucket(w http.ResponseWriter, r *http.Request, b if bucket, exists := h.bucketManager.GetBucket(bucketName); exists { // 如果是虚拟存储桶，检查是否已经有映射 if bucket.IsVirtual() { - if _, err := h.storage.GetVirtualBucketMapping(bucketName); err == nil { - h.sendS3Error(w, "BucketAlreadyExists", "The requested bucket name is not available", bucketName) - return - } + // 虚拟存储桶不需要检查映射，文件级映射只在有文件时才创建 + h.sendS3Error(w, "BucketAlreadyExists", "The requested bucket name is not available", bucketName) + return } else { // 如果是真实存储桶，返回已存在错误 h.sendS3Error(w, "BucketAlreadyExists", "The requested bucket name is not available", bucketName) @@ -416,7 +367,7 @@ func (h *S3Handler) handleCreateBucket(w http.ResponseWriter, r *http.Request, b } // 检查是否为虚拟存储桶 - if bucket, exists := h.bucketManager.GetBucket(bucketName); exists && bucket.IsVirtual() { + if requestedBucket, exists := h.bucketManager.GetBucket(bucketName); exists && requestedBucket.IsVirtual() { // 虚拟存储桶需要选择一个真实存储桶进行映射 realBuckets := h.bucketManager.GetRealBuckets() if len(realBuckets) == 0 { @@ -429,7 +380,7 @@ func (h *S3Handler) handleCreateBucket(w http.ResponseWriter, r *http.Request, b targetBucket := realBuckets[0] // 创建虚拟存储桶到真实存储桶的映射 - if err := h.storage.CreateVirtualBucketMapping(bucketName, targetBucket.Config.Name); err != nil { + if err := h.storage.CreateVirtualBucketMapping(bucketName, "", targetBucket.Config.Name); err != nil { h.sendS3Error(w, "InternalError", "Failed to create virtual bucket mapping", bucketName) return } @@ -492,35 +443,29 @@ func (h *S3Handler) handleGetObject(w http.ResponseWriter, r *http.Request, buck } // 如果是虚拟存储桶，需要通过映射查找真实存储桶 - var actualBucketName string var err error - + var bucket1 *bucket.BucketInfo + if requestedBucket.IsVirtual() { // 获取虚拟存储桶映射 - _, err := h.storage.GetVirtualBucketMapping(bucketName) + mapping, err := h.storage.GetVirtualBucketMapping(bucketName, key) if err != nil { h.sendS3Error(w, "NoSuchKey", "The specified key does not exist", key) return } - } - - // 查找对象所在的实际存储桶 - actualBucketName, err = h.storage.FindObjectBucket(key) - if err != nil { - h.sendS3Error(w, "NoSuchKey", "The specified key does not exist", key) - return - } - - bucket, ok := h.bucketManager.GetBucket(actualBucketName) - if !ok { - h.sendS3Error(w, "InternalError", "Internal server error", bucketName) - return + + // 获取映射到的真实存储桶 + bucket1, ok = h.bucketManager.GetBucket(mapping.RealBucketName) + if !ok { + h.sendS3Error(w, "InternalError", "Mapped real bucket not found", key) + return + } } // 生成预签名下载URL downloadInfo, err := h.presigner.GenerateDownloadURL( context.Background(), - bucket, + bucket1, key, ) if err != nil { @@ -563,7 +508,7 @@ func (h *S3Handler) handleHeadObject(w http.ResponseWriter, r *http.Request, buc // 如果是虚拟存储桶，需要通过映射查找真实存储桶 if requestedBucket.IsVirtual() { // 获取虚拟存储桶映射 - mapping, err := h.storage.GetVirtualBucketMapping(bucketName) + mapping, err := h.storage.GetVirtualBucketMapping(bucketName, key) if err != nil { w.WriteHeader(http.StatusNotFound) return @@ -627,31 +572,23 @@ func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, buck // 如果是虚拟存储桶，需要选择真实存储桶并创建映射 if requestedBucket.IsVirtual() { - // 获取虚拟存储桶映射，如果不存在则创建 - _, mappingErr := h.storage.GetVirtualBucketMapping(bucketName) + // 获取虚拟存储桶文件映射，如果不存在则创建 + mapping, mappingErr := h.storage.GetVirtualBucketMapping(bucketName, key) if mappingErr != nil { - // 映射不存在，选择真实存储桶并创建映射 - realBuckets := h.bucketManager.GetRealBuckets() - if len(realBuckets) == 0 { - h.sendS3Error(w, "InternalError", "No real buckets available for virtual bucket mapping", key) + // 映射不存在，使用负载均衡器选择真实存储桶并创建映射 + targetBucket, err = h.balancer.SelectBucket(key, contentLength) + if err != nil { + h.sendS3Error(w, "InsufficientStorage", "No bucket has enough space", key) return } - // 简化：选择第一个可用的真实存储桶 - targetBucket = realBuckets[0] - - // 创建虚拟存储桶映射 - if err := h.storage.CreateVirtualBucketMapping(bucketName, targetBucket.Config.Name); err != nil { - h.sendS3Error(w, "InternalError", "Failed to create virtual bucket mapping", key) + // 创建虚拟存储桶文件级映射 + if err := h.storage.CreateVirtualBucketMapping(bucketName, key, targetBucket.Config.Name); err != nil { + h.sendS3Error(w, "InternalError", "Failed to create virtual bucket file mapping", key) return } } else { - // 映射已存在，从存储服务获取对应的真实存储桶 - mapping, err := h.storage.GetVirtualBucketMapping(bucketName) - if err != nil { - h.sendS3Error(w, "InternalError", "Failed to get virtual bucket mapping", key) - return - } + // 映射已存在，获取对应的真实存储桶 targetBucket, ok = h.bucketManager.GetBucket(mapping.RealBucketName) if !ok { h.sendS3Error(w, "InternalError", "Mapped real bucket not found", key) @@ -659,13 +596,9 @@ func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, buck } } } else { - // 真实存储桶的直接处理 - // 选择目标存储桶 - targetBucket, err = h.balancer.SelectBucket(key, contentLength) - if err != nil { - h.sendS3Error(w, "InsufficientStorage", "No bucket has enough space", key) - return - } + // 如果不是虚拟存储桶，拒绝客户端对真实存储桶的直接PUT操作 + h.sendS3Error(w, "NoSuchBucket", "The specified bucket does not exist", bucketName) + return } // 生成预签名上传URL @@ -738,25 +671,30 @@ func (h *S3Handler) handleDeleteObject(w http.ResponseWriter, r *http.Request, b return } - _ = requestedBucket // 使用requestedBucket变量，避免编译错误 - var bucket *bucket.BucketInfo var err error - // 查找对象所在的实际存储桶 - actualBucketName, err := h.storage.FindObjectBucket(key) - if err != nil { - // 对象不存在，S3规范要求返回204 + if requestedBucket.IsVirtual() { + // 获取虚拟存储桶文件映射 + mapping, err := h.storage.GetVirtualBucketMapping(bucketName, key) + if err != nil { + // 对象不存在，S3规范要求返回204 + w.WriteHeader(http.StatusNoContent) + return + } + + // 获取映射到的真实存储桶 + bucket, ok = h.bucketManager.GetBucket(mapping.RealBucketName) + if !ok { + h.sendS3Error(w, "InternalError", "Mapped real bucket not found", key) + return + } + } else { + // 如果不是虚拟存储桶，拒绝客户端对真实存储桶的直接DELETE操作 w.WriteHeader(http.StatusNoContent) return } - bucket, ok = h.bucketManager.GetBucket(actualBucketName) - if !ok { - h.sendS3Error(w, "InternalError", "Internal server error", bucketName) - return - } - // 生成预签名删除URL deleteInfo, err := h.presigner.GenerateDeleteURL( context.Background(), @@ -781,6 +719,11 @@ func (h *S3Handler) handleDeleteObject(w http.ResponseWriter, r *http.Request, b // 从数据库中删除对象记录 h.storage.DeleteObject(key) + // 如果是虚拟存储桶，还需要删除文件级别映射 + if requestedBucket.IsVirtual() { + h.storage.DeleteVirtualBucketFileMapping(bucketName, key) + } + // S3规范要求删除操作总是返回204 w.WriteHeader(http.StatusNoContent) } diff --git a/internal/database/database.go b/internal/database/database.go index 5436e6f..7580cb5 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -178,6 +178,7 @@ func AutoMigrate() error { &storage.BucketStats{}, &storage.UploadSession{}, &storage.AccessLog{}, + &storage.VirtualBucketMapping{}, } for _, model := range models { diff --git a/internal/storage/models.go b/internal/storage/models.go index 347dfe7..f2c4cb6 100644 --- a/internal/storage/models.go +++ b/internal/storage/models.go @@ -43,13 +43,14 @@ func (BucketStats) TableName() string { return "bucket_stats" } -// VirtualBucketMapping 虚拟存储桶映射模型 +// VirtualBucketMapping 虚拟存储桶文件级映射模型 type VirtualBucketMapping struct { ID uint `gorm:"primaryKey" json:"id"` - VirtualBucketName string `gorm:"uniqueIndex;size:255;not null" json:"virtual_bucket_name"` + VirtualBucketName string `gorm:"index;size:255;not null" json:"virtual_bucket_name"` + ObjectKey string `gorm:"index;size:512;not null" json:"object_key"` RealBucketName string `gorm:"index;size:255;not null" json:"real_bucket_name"` - CreatedAt time.Time `gorm:"not null;default:CURRENT_TIMESTAMP" json:"created_at"` - UpdatedAt time.Time `gorm:"not null;default:CURRENT_TIMESTAMP" json:"updated_at"` + CreatedAt time.Time `gorm:"not null" json:"created_at"` + UpdatedAt time.Time `gorm:"not null" json:"updated_at"` } // TableName 指定表名 diff --git a/internal/storage/service.go b/internal/storage/service.go index 4798437..3f9720b 100644 --- a/internal/storage/service.go +++ b/internal/storage/service.go @@ -349,10 +349,11 @@ func (s *Service) GetAccessLogs(filter *AccessLogFilter) ([]*AccessLog, error) { return logs, nil } -// CreateVirtualBucketMapping 创建虚拟存储桶映射 -func (s *Service) CreateVirtualBucketMapping(virtualBucketName, realBucketName string) error { +// CreateVirtualBucketMapping 创建虚拟存储桶文件级映射 +func (s *Service) CreateVirtualBucketMapping(virtualBucketName, objectKey, realBucketName string) error { mapping := &VirtualBucketMapping{ VirtualBucketName: virtualBucketName, + ObjectKey: objectKey, RealBucketName: realBucketName, } @@ -363,12 +364,12 @@ func (s *Service) CreateVirtualBucketMapping(virtualBucketName, realBucketName s return nil } -// GetVirtualBucketMapping 获取虚拟存储桶映射 -func (s *Service) GetVirtualBucketMapping(virtualBucketName string) (*VirtualBucketMapping, error) { +// GetVirtualBucketMapping 获取虚拟存储桶文件级映射 +func (s *Service) GetVirtualBucketMapping(virtualBucketName, objectKey string) (*VirtualBucketMapping, error) { var mapping VirtualBucketMapping - if err := s.db.Where("virtual_bucket_name = ?", virtualBucketName).First(&mapping).Error; err != nil { + if err := s.db.Where("virtual_bucket_name = ? AND object_key = ?", virtualBucketName, objectKey).First(&mapping).Error; err != nil { if err == gorm.ErrRecordNotFound { - return nil, fmt.Errorf("virtual bucket mapping not found: %s", virtualBucketName) + return nil, fmt.Errorf("virtual bucket mapping not found: %s/%s", virtualBucketName, objectKey) } return nil, fmt.Errorf("failed to get virtual bucket mapping: %w", err) } @@ -384,15 +385,24 @@ func (s *Service) GetVirtualBucketMappings() ([]*VirtualBucketMapping, error) { return mappings, nil } +// GetVirtualBucketMappingsForBucket 获取指定虚拟存储桶的所有映射 +func (s *Service) GetVirtualBucketMappingsForBucket(virtualBucketName string) ([]*VirtualBucketMapping, error) { + var mappings []*VirtualBucketMapping + if err := s.db.Where("virtual_bucket_name = ?", virtualBucketName).Find(&mappings).Error; err != nil { + return nil, fmt.Errorf("failed to get virtual bucket mappings for bucket %s: %w", virtualBucketName, err) + } + return mappings, nil +} + // UpdateVirtualBucketMapping 更新虚拟存储桶映射 -func (s *Service) UpdateVirtualBucketMapping(virtualBucketName, realBucketName string) error { +func (s *Service) UpdateVirtualBucketMapping(virtualBucketName, objectKey, realBucketName string) error { updates := map[string]interface{}{ "real_bucket_name": realBucketName, "updated_at": time.Now(), } if err := s.db.Model(&VirtualBucketMapping{}). - Where("virtual_bucket_name = ?", virtualBucketName). + Where("virtual_bucket_name = ? AND object_key = ?", virtualBucketName, objectKey). Updates(updates).Error; err != nil { return fmt.Errorf("failed to update virtual bucket mapping: %w", err) } @@ -402,6 +412,7 @@ func (s *Service) UpdateVirtualBucketMapping(virtualBucketName, realBucketName s // DeleteVirtualBucketMapping 删除虚拟存储桶映射 func (s *Service) DeleteVirtualBucketMapping(virtualBucketName string) error { + // 删除虚拟存储桶的所有映射 if err := s.db.Where("virtual_bucket_name = ?", virtualBucketName). Delete(&VirtualBucketMapping{}).Error; err != nil { return fmt.Errorf("failed to delete virtual bucket mapping: %w", err) @@ -409,20 +420,47 @@ func (s *Service) DeleteVirtualBucketMapping(virtualBucketName string) error { return nil } +// DeleteVirtualBucketObjectMapping 删除虚拟存储桶中特定对象的映射 +func (s *Service) DeleteVirtualBucketObjectMapping(virtualBucketName, objectKey string) error { + if err := s.db.Where("virtual_bucket_name = ? AND object_key = ?", virtualBucketName, objectKey). + Delete(&VirtualBucketMapping{}).Error; err != nil { + return fmt.Errorf("failed to delete virtual bucket object mapping: %w", err) + } + return nil +} + // GetVirtualBucketObjects 获取虚拟存储桶中的所有对象 func (s *Service) GetVirtualBucketObjects(virtualBucketName string) ([]*Object, error) { - // 首先找到虚拟存储桶对应的真实存储桶 - mapping, err := s.GetVirtualBucketMapping(virtualBucketName) + // 获取虚拟存储桶的所有文件映射 + mappings, err := s.GetVirtualBucketMappingsForBucket(virtualBucketName) if err != nil { return nil, err } - // 查找所有映射到该虚拟存储桶的对象 + if len(mappings) == 0 { + return []*Object{}, nil + } + + // 收集所有对象键 + objectKeys := make([]string, 0, len(mappings)) + for _, mapping := range mappings { + objectKeys = append(objectKeys, mapping.ObjectKey) + } + + // 从对象表中查询这些对象 var objects []*Object - if err := s.db.Where("bucket_name = ?", mapping.RealBucketName). - Find(&objects).Error; err != nil { - return nil, fmt.Errorf("failed to get virtual bucket objects: %w", err) + if err := s.db.Where("key IN ?", objectKeys).Find(&objects).Error; err != nil { + return nil, fmt.Errorf("failed to get objects for virtual bucket: %w", err) } return objects, nil } + +// DeleteVirtualBucketFileMapping 删除虚拟存储桶文件映射 +func (s *Service) DeleteVirtualBucketFileMapping(virtualBucketName, objectKey string) error { + if err := s.db.Where("virtual_bucket_name = ? AND object_key = ?", virtualBucketName, objectKey). + Delete(&VirtualBucketMapping{}).Error; err != nil { + return fmt.Errorf("failed to delete virtual bucket file mapping: %w", err) + } + return nil +}