From f331d42b48346d7e90f56930528b5d43a0164ebd Mon Sep 17 00:00:00 2001 From: DullJZ <79080562+DullJZ@users.noreply.github.com> Date: Wed, 10 Sep 2025 15:35:53 +0800 Subject: [PATCH] Add error log --- internal/api/handler.go | 392 ------------------------------------- internal/api/s3_handler.go | 7 +- test_s3_compatibility.py | 347 -------------------------------- 3 files changed, 6 insertions(+), 740 deletions(-) delete mode 100644 internal/api/handler.go delete mode 100644 test_s3_compatibility.py diff --git a/internal/api/handler.go b/internal/api/handler.go deleted file mode 100644 index c1448e6..0000000 --- a/internal/api/handler.go +++ /dev/null @@ -1,392 +0,0 @@ -package api - -import ( - "context" - "encoding/json" - "log" - "net/http" - "strconv" - "time" - - "github.com/DullJZ/s3-balance/internal/balancer" - "github.com/DullJZ/s3-balance/internal/bucket" - "github.com/DullJZ/s3-balance/internal/storage" - "github.com/DullJZ/s3-balance/pkg/presigner" - "github.com/gorilla/mux" -) - -// Handler API处理器 -type Handler struct { - bucketManager *bucket.Manager - balancer *balancer.Balancer - presigner *presigner.Presigner - storage *storage.Service -} - -// NewHandler 创建新的API处理器 -func NewHandler( - bucketManager *bucket.Manager, - balancer *balancer.Balancer, - presigner *presigner.Presigner, - storage *storage.Service, -) *Handler { - return &Handler{ - bucketManager: bucketManager, - balancer: balancer, - presigner: presigner, - storage: storage, - } -} - -// RegisterRoutes 注册路由 -func (h *Handler) RegisterRoutes(router *mux.Router) { - // 健康检查 - router.HandleFunc("/health", h.handleHealth).Methods("GET") - - // 存储桶状态 - router.HandleFunc("/api/v1/buckets", h.handleListBuckets).Methods("GET") - router.HandleFunc("/api/v1/buckets/{bucket}/stats", h.handleBucketStats).Methods("GET") - - // 预签名URL生成 - router.HandleFunc("/api/v1/presign/upload", h.handlePresignUpload).Methods("POST") - router.HandleFunc("/api/v1/presign/download", h.handlePresignDownload).Methods("POST") - router.HandleFunc("/api/v1/presign/delete", h.handlePresignDelete).Methods("POST") - router.HandleFunc("/api/v1/presign/multipart", h.handlePresignMultipart).Methods("POST") - - // 对象操作（记录元数据） - router.HandleFunc("/api/v1/objects", h.handleListObjects).Methods("GET") - router.HandleFunc("/api/v1/objects/{key:.*}", h.handleGetObjectInfo).Methods("GET") - router.HandleFunc("/api/v1/objects/{key:.*}", h.handleDeleteObject).Methods("DELETE") -} - -// 健康检查 -func (h *Handler) handleHealth(w http.ResponseWriter, r *http.Request) { - response := map[string]interface{}{ - "status": "healthy", - "time": time.Now().Unix(), - } - h.sendJSON(w, http.StatusOK, response) -} - -// 列出所有存储桶状态 -func (h *Handler) handleListBuckets(w http.ResponseWriter, r *http.Request) { - buckets := h.bucketManager.GetAllBuckets() - - var bucketList []map[string]interface{} - for _, b := range buckets { - bucketList = append(bucketList, map[string]interface{}{ - "name": b.Config.Name, - "endpoint": b.Config.Endpoint, - "region": b.Config.Region, - "max_size": b.Config.MaxSize, - "max_size_bytes": b.Config.MaxSizeBytes, - "used_size": b.GetUsedSize(), - "available": b.IsAvailable(), - "weight": b.Config.Weight, - "enabled": b.Config.Enabled, - }) - } - - h.sendJSON(w, http.StatusOK, map[string]interface{}{ - "buckets": bucketList, - "strategy": h.balancer.GetStrategy(), - }) -} - -// 获取单个存储桶统计 -func (h *Handler) handleBucketStats(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - bucketName := vars["bucket"] - - bucket, ok := h.bucketManager.GetBucket(bucketName) - if !ok { - h.sendError(w, http.StatusNotFound, "bucket not found") - return - } - - stats := map[string]interface{}{ - "name": bucket.Config.Name, - "max_size_bytes": bucket.Config.MaxSizeBytes, - "used_size": bucket.GetUsedSize(), - "available_space": bucket.GetAvailableSpace(), - "available": bucket.IsAvailable(), - "last_checked": bucket.LastChecked, - } - - h.sendJSON(w, http.StatusOK, stats) -} - -// PresignUploadRequest 上传预签名请求 -type PresignUploadRequest struct { - Key string `json:"key"` - Size int64 `json:"size"` - ContentType string `json:"content_type,omitempty"` - Metadata map[string]string `json:"metadata,omitempty"` -} - -// 生成上传预签名URL -func (h *Handler) handlePresignUpload(w http.ResponseWriter, r *http.Request) { - var req PresignUploadRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - h.sendError(w, http.StatusBadRequest, "invalid request body") - return - } - - if req.Key == "" { - h.sendError(w, http.StatusBadRequest, "key is required") - return - } - - // 选择存储桶 - bucket, err := h.balancer.SelectBucket(req.Key, req.Size) - if err != nil { - h.sendError(w, http.StatusServiceUnavailable, err.Error()) - return - } - - // 生成预签名URL - uploadURL, err := h.presigner.GenerateUploadURL( - context.Background(), - bucket, - req.Key, - req.ContentType, - req.Metadata, - ) - if err != nil { - h.sendError(w, http.StatusInternalServerError, "failed to generate upload URL") - return - } - - // 记录对象元数据 - if err := h.storage.RecordObject(req.Key, bucket.Config.Name, req.Size, req.Metadata); err != nil { - log.Printf("Failed to record object metadata: %v", err) - } - - // 更新存储桶使用量（预估） - bucket.UpdateUsedSize(req.Size) - - h.sendJSON(w, http.StatusOK, uploadURL) -} - -// PresignDownloadRequest 下载预签名请求 -type PresignDownloadRequest struct { - Key string `json:"key"` -} - -// 生成下载预签名URL -func (h *Handler) handlePresignDownload(w http.ResponseWriter, r *http.Request) { - var req PresignDownloadRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - h.sendError(w, http.StatusBadRequest, "invalid request body") - return - } - - if req.Key == "" { - h.sendError(w, http.StatusBadRequest, "key is required") - return - } - - // 查找对象所在的存储桶 - bucketName, err := h.storage.FindObjectBucket(req.Key) - if err != nil { - h.sendError(w, http.StatusNotFound, "object not found") - return - } - - bucket, ok := h.bucketManager.GetBucket(bucketName) - if !ok { - h.sendError(w, http.StatusNotFound, "bucket not found") - return - } - - // 生成预签名URL - downloadURL, err := h.presigner.GenerateDownloadURL( - context.Background(), - bucket, - req.Key, - ) - if err != nil { - h.sendError(w, http.StatusInternalServerError, "failed to generate download URL") - return - } - - h.sendJSON(w, http.StatusOK, downloadURL) -} - -// PresignDeleteRequest 删除预签名请求 -type PresignDeleteRequest struct { - Key string `json:"key"` -} - -// 生成删除预签名URL -func (h *Handler) handlePresignDelete(w http.ResponseWriter, r *http.Request) { - var req PresignDeleteRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - h.sendError(w, http.StatusBadRequest, "invalid request body") - return - } - - if req.Key == "" { - h.sendError(w, http.StatusBadRequest, "key is required") - return - } - - // 查找对象所在的存储桶 - bucketName, err := h.storage.FindObjectBucket(req.Key) - if err != nil { - h.sendError(w, http.StatusNotFound, "object not found") - return - } - - bucket, ok := h.bucketManager.GetBucket(bucketName) - if !ok { - h.sendError(w, http.StatusNotFound, "bucket not found") - return - } - - // 生成预签名URL - deleteURL, err := h.presigner.GenerateDeleteURL( - context.Background(), - bucket, - req.Key, - ) - if err != nil { - h.sendError(w, http.StatusInternalServerError, "failed to generate delete URL") - return - } - - h.sendJSON(w, http.StatusOK, deleteURL) -} - -// PresignMultipartRequest 分片上传预签名请求 -type PresignMultipartRequest struct { - Key string `json:"key"` - PartCount int `json:"part_count"` - Size int64 `json:"size"` -} - -// 生成分片上传预签名URLs -func (h *Handler) handlePresignMultipart(w http.ResponseWriter, r *http.Request) { - var req PresignMultipartRequest - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - h.sendError(w, http.StatusBadRequest, "invalid request body") - return - } - - if req.Key == "" || req.PartCount <= 0 { - h.sendError(w, http.StatusBadRequest, "invalid parameters") - return - } - - // 选择存储桶 - bucket, err := h.balancer.SelectBucket(req.Key, req.Size) - if err != nil { - h.sendError(w, http.StatusServiceUnavailable, err.Error()) - return - } - - // 生成预签名URLs - multipartURLs, err := h.presigner.GenerateMultipartUploadURLs( - context.Background(), - bucket, - req.Key, - req.PartCount, - ) - if err != nil { - h.sendError(w, http.StatusInternalServerError, "failed to generate multipart URLs") - return - } - - // 记录对象元数据 - if err := h.storage.RecordObject(req.Key, bucket.Config.Name, req.Size, nil); err != nil { - log.Printf("Failed to record object metadata: %v", err) - } - - // 更新存储桶使用量（预估） - bucket.UpdateUsedSize(req.Size) - - h.sendJSON(w, http.StatusOK, multipartURLs) -} - -// 列出对象 -func (h *Handler) handleListObjects(w http.ResponseWriter, r *http.Request) { - prefix := r.URL.Query().Get("prefix") - bucketName := r.URL.Query().Get("bucket") - marker := r.URL.Query().Get("marker") - limitStr := r.URL.Query().Get("limit") - - limit := 100 - if limitStr != "" { - if l, err := strconv.Atoi(limitStr); err == nil && l > 0 { - limit = l - } - } - - // 调用更新后的ListObjects方法，传入所有必需的参数 - objects, err := h.storage.ListObjects(bucketName, prefix, marker, limit) - if err != nil { - h.sendError(w, http.StatusInternalServerError, "failed to list objects") - return - } - - h.sendJSON(w, http.StatusOK, map[string]interface{}{ - "objects": objects, - "count": len(objects), - }) -} - -// 获取对象信息 -func (h *Handler) handleGetObjectInfo(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - key := vars["key"] - - info, err := h.storage.GetObjectInfo(key) - if err != nil { - h.sendError(w, http.StatusNotFound, "object not found") - return - } - - h.sendJSON(w, http.StatusOK, info) -} - -// 删除对象（只删除元数据记录） -func (h *Handler) handleDeleteObject(w http.ResponseWriter, r *http.Request) { - vars := mux.Vars(r) - key := vars["key"] - - // 获取对象信息以更新存储桶使用量 - info, err := h.storage.GetObjectInfo(key) - if err != nil { - h.sendError(w, http.StatusNotFound, "object not found") - return - } - - // 更新存储桶使用量 - if bucket, ok := h.bucketManager.GetBucket(info.BucketName); ok { - bucket.UpdateUsedSize(-info.Size) - } - - // 删除元数据记录 - if err := h.storage.DeleteObject(key); err != nil { - h.sendError(w, http.StatusInternalServerError, "failed to delete object") - return - } - - h.sendJSON(w, http.StatusOK, map[string]string{ - "message": "object deleted successfully", - }) -} - -// 发送JSON响应 -func (h *Handler) sendJSON(w http.ResponseWriter, status int, data interface{}) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - json.NewEncoder(w).Encode(data) -} - -// 发送错误响应 -func (h *Handler) sendError(w http.ResponseWriter, status int, message string) { - h.sendJSON(w, status, map[string]string{ - "error": message, - }) -} diff --git a/internal/api/s3_handler.go b/internal/api/s3_handler.go index 2832f8c..6eee3f6 100644 --- a/internal/api/s3_handler.go +++ b/internal/api/s3_handler.go @@ -5,6 +5,7 @@ import ( "encoding/xml" "fmt" "io" + "log" "net/http" "net/url" "strconv" @@ -610,6 +611,7 @@ func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, buck nil, // metadata ) if err != nil { + log.Printf("Failed to generate upload URL for key %s in bucket %s: %v", key, targetBucket.Config.Name, err) h.sendS3Error(w, "InternalError", "Failed to generate upload URL", key) return } @@ -651,7 +653,10 @@ func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, buck w.Header().Set("ETag", fmt.Sprintf("\"%x\"", time.Now().UnixNano())) w.WriteHeader(http.StatusOK) } else { - h.sendS3Error(w, "InternalError", "Upload failed", key) + // 读取错误响应体以获取详细信息 + body, _ := io.ReadAll(resp.Body) + log.Printf("Upload failed with status %d: %s", resp.StatusCode, string(body)) + h.sendS3Error(w, "InternalError", fmt.Sprintf("Upload failed with status %d", resp.StatusCode), key) } } diff --git a/test_s3_compatibility.py b/test_s3_compatibility.py deleted file mode 100644 index fdf8e7e..0000000 --- a/test_s3_compatibility.py +++ /dev/null @@ -1,347 +0,0 @@ -#!/usr/bin/env python3 -""" -S3 Balance - S3兼容性测试脚本 -使用boto3 AWS SDK测试S3 Balance的S3兼容性 -""" - -import os -import sys -import time -import hashlib -import tempfile -from datetime import datetime - -try: - import boto3 - from botocore.client import Config -except ImportError: - print("请先安装boto3: pip install boto3") - sys.exit(1) - -# S3 Balance服务配置 -S3_BALANCE_ENDPOINT = "http://localhost:8080" -ACCESS_KEY = "AKIAIOSFODNN7EXAMPLE" -SECRET_KEY = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - -# 测试配置 -TEST_BUCKET = "test-virtual-1" -TEST_KEY_PREFIX = f"test-{int(time.time())}" - -def create_s3_client(): - """创建S3客户端""" - return boto3.client( - 's3', - endpoint_url=S3_BALANCE_ENDPOINT, - aws_access_key_id=ACCESS_KEY, - aws_secret_access_key=SECRET_KEY, - config=Config( - signature_version='s3v4', - s3={'addressing_style': 'path'} - ), - region_name='us-east-1' - ) - -def test_list_buckets(s3_client): - """测试列出存储桶""" - print("\n1. 测试列出存储桶 (ListBuckets)...") - try: - response = s3_client.list_buckets() - buckets = response.get('Buckets', []) - print(f" ✓ 找到 {len(buckets)} 个存储桶") - for bucket in buckets: - print(f" - {bucket['Name']} (创建时间: {bucket['CreationDate']})") - return True - except Exception as e: - print(f" ✗ 失败: {e}") - return False - -def test_upload_object(s3_client): - """测试上传对象""" - print(f"\n2. 测试上传对象 (PutObject)...") - - # 创建测试文件 - test_data = b"Hello, S3 Balance! This is a test file." - test_key = f"{TEST_KEY_PREFIX}/test-upload.txt" - - try: - # 上传对象 - response = s3_client.put_object( - Bucket=TEST_BUCKET, - Key=test_key, - Body=test_data, - ContentType='text/plain', - Metadata={'test': 'true', 'timestamp': str(int(time.time()))} - ) - - etag = response.get('ETag', '').strip('"') - print(f" ✓ 成功上传对象: {test_key}") - print(f" ETag: {etag}") - return test_key - except Exception as e: - print(f" ✗ 失败: {e}") - return None - -def test_list_objects(s3_client): - """测试列出对象""" - print(f"\n3. 测试列出对象 (ListObjects)...") - - try: - response = s3_client.list_objects_v2( - Bucket=TEST_BUCKET, - Prefix=TEST_KEY_PREFIX, - MaxKeys=10 - ) - - objects = response.get('Contents', []) - print(f" ✓ 找到 {len(objects)} 个对象") - for obj in objects: - print(f" - {obj['Key']} (大小: {obj['Size']} bytes, 修改时间: {obj['LastModified']})") - return True - except Exception as e: - print(f" ✗ 失败: {e}") - return False - -def test_download_object(s3_client, key): - """测试下载对象""" - print(f"\n4. 测试下载对象 (GetObject)...") - - if not key: - print(" ⚠ 跳过: 没有可下载的对象") - return False - - try: - response = s3_client.get_object( - Bucket=TEST_BUCKET, - Key=key - ) - - data = response['Body'].read() - content_type = response.get('ContentType', '') - content_length = response.get('ContentLength', 0) - - print(f" ✓ 成功下载对象: {key}") - print(f" 内容类型: {content_type}") - print(f" 内容长度: {content_length} bytes") - print(f" 内容预览: {data[:50].decode('utf-8', errors='ignore')}...") - return True - except Exception as e: - print(f" ✗ 失败: {e}") - return False - -def test_head_object(s3_client, key): - """测试获取对象元数据""" - print(f"\n5. 测试获取对象元数据 (HeadObject)...") - - if not key: - print(" ⚠ 跳过: 没有可查询的对象") - return False - - try: - response = s3_client.head_object( - Bucket=TEST_BUCKET, - Key=key - ) - - print(f" ✓ 成功获取对象元数据: {key}") - print(f" 内容长度: {response.get('ContentLength', 0)} bytes") - print(f" 内容类型: {response.get('ContentType', '')}") - print(f" ETag: {response.get('ETag', '').strip('\"')}") - print(f" 最后修改: {response.get('LastModified', '')}") - return True - except Exception as e: - print(f" ✗ 失败: {e}") - return False - -def test_multipart_upload(s3_client): - """测试分片上传（大文件）""" - print(f"\n6. 测试分片上传 (Multipart Upload)...") - - # 创建一个5MB的测试文件 - test_key = f"{TEST_KEY_PREFIX}/test-multipart.bin" - part_size = 5 * 1024 * 1024 # 5MB per part - total_size = 10 * 1024 * 1024 # 10MB total - - try: - # 初始化分片上传 - response = s3_client.create_multipart_upload( - Bucket=TEST_BUCKET, - Key=test_key, - ContentType='application/octet-stream' - ) - upload_id = response['UploadId'] - print(f" ✓ 初始化分片上传，UploadId: {upload_id}") - - # 上传分片 - parts = [] - for i in range(2): # 上传2个5MB的分片 - part_number = i + 1 - part_data = os.urandom(part_size) # 生成随机数据 - - part_response = s3_client.upload_part( - Bucket=TEST_BUCKET, - Key=test_key, - PartNumber=part_number, - UploadId=upload_id, - Body=part_data - ) - - parts.append({ - 'ETag': part_response['ETag'], - 'PartNumber': part_number - }) - print(f" ✓ 上传分片 {part_number}/2 完成") - - # 完成分片上传 - s3_client.complete_multipart_upload( - Bucket=TEST_BUCKET, - Key=test_key, - UploadId=upload_id, - MultipartUpload={'Parts': parts} - ) - - print(f" ✓ 分片上传完成: {test_key}") - return test_key - except Exception as e: - print(f" ✗ 失败: {e}") - return None - -def test_delete_object(s3_client, key): - """测试删除对象""" - print(f"\n7. 测试删除对象 (DeleteObject)...") - - if not key: - print(" ⚠ 跳过: 没有可删除的对象") - return False - - try: - s3_client.delete_object( - Bucket=TEST_BUCKET, - Key=key - ) - - print(f" ✓ 成功删除对象: {key}") - return True - except Exception as e: - print(f" ✗ 失败: {e}") - return False - -def test_presigned_url(s3_client): - """测试预签名URL""" - print(f"\n8. 测试预签名URL...") - - test_key = f"{TEST_KEY_PREFIX}/test-presigned.txt" - - try: - # 生成上传预签名URL - upload_url = s3_client.generate_presigned_url( - 'put_object', - Params={'Bucket': TEST_BUCKET, 'Key': test_key}, - ExpiresIn=3600 - ) - print(f" ✓ 生成上传预签名URL") - print(f" URL: {upload_url[:80]}...") - - # 生成下载预签名URL - download_url = s3_client.generate_presigned_url( - 'get_object', - Params={'Bucket': TEST_BUCKET, 'Key': test_key}, - ExpiresIn=3600 - ) - print(f" ✓ 生成下载预签名URL") - print(f" URL: {download_url[:80]}...") - - return True - except Exception as e: - print(f" ✗ 失败: {e}") - return False - -def cleanup(s3_client): - """清理测试数据""" - print(f"\n清理测试数据...") - - try: - # 列出所有测试对象 - response = s3_client.list_objects_v2( - Bucket=TEST_BUCKET, - Prefix=TEST_KEY_PREFIX - ) - - objects = response.get('Contents', []) - if objects: - # 删除所有测试对象 - delete_objects = [{'Key': obj['Key']} for obj in objects] - s3_client.delete_objects( - Bucket=TEST_BUCKET, - Delete={'Objects': delete_objects} - ) - print(f" ✓ 删除了 {len(objects)} 个测试对象") - else: - print(" ✓ 没有需要清理的对象") - - return True - except Exception as e: - print(f" ✗ 清理失败: {e}") - return False - -def main(): - """主测试函数""" - print("=" * 60) - print("S3 Balance - S3兼容性测试") - print("=" * 60) - print(f"端点: {S3_BALANCE_ENDPOINT}") - print(f"测试桶: {TEST_BUCKET}") - print(f"测试前缀: {TEST_KEY_PREFIX}") - - # 创建S3客户端 - s3_client = create_s3_client() - - # 执行测试 - results = [] - - # 基础测试 - results.append(("ListBuckets", test_list_buckets(s3_client))) - - # 对象操作测试 - uploaded_key = test_upload_object(s3_client) - results.append(("PutObject", uploaded_key is not None)) - - results.append(("ListObjects", test_list_objects(s3_client))) - results.append(("GetObject", test_download_object(s3_client, uploaded_key))) - results.append(("HeadObject", test_head_object(s3_client, uploaded_key))) - - # 高级功能测试 - # multipart_key = test_multipart_upload(s3_client) - # results.append(("MultipartUpload", multipart_key is not None)) - - results.append(("PresignedURL", test_presigned_url(s3_client))) - - # 删除测试 - results.append(("DeleteObject", test_delete_object(s3_client, uploaded_key))) - - # 清理 - cleanup(s3_client) - - # 打印测试结果摘要 - print("\n" + "=" * 60) - print("测试结果摘要") - print("=" * 60) - - passed = sum(1 for _, result in results if result) - total = len(results) - - for test_name, result in results: - status = "✓ 通过" if result else "✗ 失败" - print(f"{test_name:20} {status}") - - print("-" * 60) - print(f"总计: {passed}/{total} 测试通过") - - if passed == total: - print("\n🎉 所有测试通过！S3 Balance S3兼容性良好。") - else: - print(f"\n⚠️ 有 {total - passed} 个测试失败，请检查服务配置。") - - return 0 if passed == total else 1 - -if __name__ == "__main__": - sys.exit(main())