mirror of
https://github.com/DullJZ/s3-balance.git
synced 2026-06-27 22:11:22 +08:00
Add error log
This commit is contained in:
@@ -1,392 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/DullJZ/s3-balance/internal/balancer"
|
||||
"github.com/DullJZ/s3-balance/internal/bucket"
|
||||
"github.com/DullJZ/s3-balance/internal/storage"
|
||||
"github.com/DullJZ/s3-balance/pkg/presigner"
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// Handler API处理器
|
||||
type Handler struct {
|
||||
bucketManager *bucket.Manager
|
||||
balancer *balancer.Balancer
|
||||
presigner *presigner.Presigner
|
||||
storage *storage.Service
|
||||
}
|
||||
|
||||
// NewHandler 创建新的API处理器
|
||||
func NewHandler(
|
||||
bucketManager *bucket.Manager,
|
||||
balancer *balancer.Balancer,
|
||||
presigner *presigner.Presigner,
|
||||
storage *storage.Service,
|
||||
) *Handler {
|
||||
return &Handler{
|
||||
bucketManager: bucketManager,
|
||||
balancer: balancer,
|
||||
presigner: presigner,
|
||||
storage: storage,
|
||||
}
|
||||
}
|
||||
|
||||
// RegisterRoutes 注册路由
|
||||
func (h *Handler) RegisterRoutes(router *mux.Router) {
|
||||
// 健康检查
|
||||
router.HandleFunc("/health", h.handleHealth).Methods("GET")
|
||||
|
||||
// 存储桶状态
|
||||
router.HandleFunc("/api/v1/buckets", h.handleListBuckets).Methods("GET")
|
||||
router.HandleFunc("/api/v1/buckets/{bucket}/stats", h.handleBucketStats).Methods("GET")
|
||||
|
||||
// 预签名URL生成
|
||||
router.HandleFunc("/api/v1/presign/upload", h.handlePresignUpload).Methods("POST")
|
||||
router.HandleFunc("/api/v1/presign/download", h.handlePresignDownload).Methods("POST")
|
||||
router.HandleFunc("/api/v1/presign/delete", h.handlePresignDelete).Methods("POST")
|
||||
router.HandleFunc("/api/v1/presign/multipart", h.handlePresignMultipart).Methods("POST")
|
||||
|
||||
// 对象操作(记录元数据)
|
||||
router.HandleFunc("/api/v1/objects", h.handleListObjects).Methods("GET")
|
||||
router.HandleFunc("/api/v1/objects/{key:.*}", h.handleGetObjectInfo).Methods("GET")
|
||||
router.HandleFunc("/api/v1/objects/{key:.*}", h.handleDeleteObject).Methods("DELETE")
|
||||
}
|
||||
|
||||
// 健康检查
|
||||
func (h *Handler) handleHealth(w http.ResponseWriter, r *http.Request) {
|
||||
response := map[string]interface{}{
|
||||
"status": "healthy",
|
||||
"time": time.Now().Unix(),
|
||||
}
|
||||
h.sendJSON(w, http.StatusOK, response)
|
||||
}
|
||||
|
||||
// 列出所有存储桶状态
|
||||
func (h *Handler) handleListBuckets(w http.ResponseWriter, r *http.Request) {
|
||||
buckets := h.bucketManager.GetAllBuckets()
|
||||
|
||||
var bucketList []map[string]interface{}
|
||||
for _, b := range buckets {
|
||||
bucketList = append(bucketList, map[string]interface{}{
|
||||
"name": b.Config.Name,
|
||||
"endpoint": b.Config.Endpoint,
|
||||
"region": b.Config.Region,
|
||||
"max_size": b.Config.MaxSize,
|
||||
"max_size_bytes": b.Config.MaxSizeBytes,
|
||||
"used_size": b.GetUsedSize(),
|
||||
"available": b.IsAvailable(),
|
||||
"weight": b.Config.Weight,
|
||||
"enabled": b.Config.Enabled,
|
||||
})
|
||||
}
|
||||
|
||||
h.sendJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"buckets": bucketList,
|
||||
"strategy": h.balancer.GetStrategy(),
|
||||
})
|
||||
}
|
||||
|
||||
// 获取单个存储桶统计
|
||||
func (h *Handler) handleBucketStats(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
bucketName := vars["bucket"]
|
||||
|
||||
bucket, ok := h.bucketManager.GetBucket(bucketName)
|
||||
if !ok {
|
||||
h.sendError(w, http.StatusNotFound, "bucket not found")
|
||||
return
|
||||
}
|
||||
|
||||
stats := map[string]interface{}{
|
||||
"name": bucket.Config.Name,
|
||||
"max_size_bytes": bucket.Config.MaxSizeBytes,
|
||||
"used_size": bucket.GetUsedSize(),
|
||||
"available_space": bucket.GetAvailableSpace(),
|
||||
"available": bucket.IsAvailable(),
|
||||
"last_checked": bucket.LastChecked,
|
||||
}
|
||||
|
||||
h.sendJSON(w, http.StatusOK, stats)
|
||||
}
|
||||
|
||||
// PresignUploadRequest 上传预签名请求
|
||||
type PresignUploadRequest struct {
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size"`
|
||||
ContentType string `json:"content_type,omitempty"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
// 生成上传预签名URL
|
||||
func (h *Handler) handlePresignUpload(w http.ResponseWriter, r *http.Request) {
|
||||
var req PresignUploadRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
h.sendError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.Key == "" {
|
||||
h.sendError(w, http.StatusBadRequest, "key is required")
|
||||
return
|
||||
}
|
||||
|
||||
// 选择存储桶
|
||||
bucket, err := h.balancer.SelectBucket(req.Key, req.Size)
|
||||
if err != nil {
|
||||
h.sendError(w, http.StatusServiceUnavailable, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// 生成预签名URL
|
||||
uploadURL, err := h.presigner.GenerateUploadURL(
|
||||
context.Background(),
|
||||
bucket,
|
||||
req.Key,
|
||||
req.ContentType,
|
||||
req.Metadata,
|
||||
)
|
||||
if err != nil {
|
||||
h.sendError(w, http.StatusInternalServerError, "failed to generate upload URL")
|
||||
return
|
||||
}
|
||||
|
||||
// 记录对象元数据
|
||||
if err := h.storage.RecordObject(req.Key, bucket.Config.Name, req.Size, req.Metadata); err != nil {
|
||||
log.Printf("Failed to record object metadata: %v", err)
|
||||
}
|
||||
|
||||
// 更新存储桶使用量(预估)
|
||||
bucket.UpdateUsedSize(req.Size)
|
||||
|
||||
h.sendJSON(w, http.StatusOK, uploadURL)
|
||||
}
|
||||
|
||||
// PresignDownloadRequest 下载预签名请求
|
||||
type PresignDownloadRequest struct {
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// 生成下载预签名URL
|
||||
func (h *Handler) handlePresignDownload(w http.ResponseWriter, r *http.Request) {
|
||||
var req PresignDownloadRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
h.sendError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.Key == "" {
|
||||
h.sendError(w, http.StatusBadRequest, "key is required")
|
||||
return
|
||||
}
|
||||
|
||||
// 查找对象所在的存储桶
|
||||
bucketName, err := h.storage.FindObjectBucket(req.Key)
|
||||
if err != nil {
|
||||
h.sendError(w, http.StatusNotFound, "object not found")
|
||||
return
|
||||
}
|
||||
|
||||
bucket, ok := h.bucketManager.GetBucket(bucketName)
|
||||
if !ok {
|
||||
h.sendError(w, http.StatusNotFound, "bucket not found")
|
||||
return
|
||||
}
|
||||
|
||||
// 生成预签名URL
|
||||
downloadURL, err := h.presigner.GenerateDownloadURL(
|
||||
context.Background(),
|
||||
bucket,
|
||||
req.Key,
|
||||
)
|
||||
if err != nil {
|
||||
h.sendError(w, http.StatusInternalServerError, "failed to generate download URL")
|
||||
return
|
||||
}
|
||||
|
||||
h.sendJSON(w, http.StatusOK, downloadURL)
|
||||
}
|
||||
|
||||
// PresignDeleteRequest 删除预签名请求
|
||||
type PresignDeleteRequest struct {
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// 生成删除预签名URL
|
||||
func (h *Handler) handlePresignDelete(w http.ResponseWriter, r *http.Request) {
|
||||
var req PresignDeleteRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
h.sendError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.Key == "" {
|
||||
h.sendError(w, http.StatusBadRequest, "key is required")
|
||||
return
|
||||
}
|
||||
|
||||
// 查找对象所在的存储桶
|
||||
bucketName, err := h.storage.FindObjectBucket(req.Key)
|
||||
if err != nil {
|
||||
h.sendError(w, http.StatusNotFound, "object not found")
|
||||
return
|
||||
}
|
||||
|
||||
bucket, ok := h.bucketManager.GetBucket(bucketName)
|
||||
if !ok {
|
||||
h.sendError(w, http.StatusNotFound, "bucket not found")
|
||||
return
|
||||
}
|
||||
|
||||
// 生成预签名URL
|
||||
deleteURL, err := h.presigner.GenerateDeleteURL(
|
||||
context.Background(),
|
||||
bucket,
|
||||
req.Key,
|
||||
)
|
||||
if err != nil {
|
||||
h.sendError(w, http.StatusInternalServerError, "failed to generate delete URL")
|
||||
return
|
||||
}
|
||||
|
||||
h.sendJSON(w, http.StatusOK, deleteURL)
|
||||
}
|
||||
|
||||
// PresignMultipartRequest 分片上传预签名请求
|
||||
type PresignMultipartRequest struct {
|
||||
Key string `json:"key"`
|
||||
PartCount int `json:"part_count"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// 生成分片上传预签名URLs
|
||||
func (h *Handler) handlePresignMultipart(w http.ResponseWriter, r *http.Request) {
|
||||
var req PresignMultipartRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
h.sendError(w, http.StatusBadRequest, "invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
if req.Key == "" || req.PartCount <= 0 {
|
||||
h.sendError(w, http.StatusBadRequest, "invalid parameters")
|
||||
return
|
||||
}
|
||||
|
||||
// 选择存储桶
|
||||
bucket, err := h.balancer.SelectBucket(req.Key, req.Size)
|
||||
if err != nil {
|
||||
h.sendError(w, http.StatusServiceUnavailable, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// 生成预签名URLs
|
||||
multipartURLs, err := h.presigner.GenerateMultipartUploadURLs(
|
||||
context.Background(),
|
||||
bucket,
|
||||
req.Key,
|
||||
req.PartCount,
|
||||
)
|
||||
if err != nil {
|
||||
h.sendError(w, http.StatusInternalServerError, "failed to generate multipart URLs")
|
||||
return
|
||||
}
|
||||
|
||||
// 记录对象元数据
|
||||
if err := h.storage.RecordObject(req.Key, bucket.Config.Name, req.Size, nil); err != nil {
|
||||
log.Printf("Failed to record object metadata: %v", err)
|
||||
}
|
||||
|
||||
// 更新存储桶使用量(预估)
|
||||
bucket.UpdateUsedSize(req.Size)
|
||||
|
||||
h.sendJSON(w, http.StatusOK, multipartURLs)
|
||||
}
|
||||
|
||||
// 列出对象
|
||||
func (h *Handler) handleListObjects(w http.ResponseWriter, r *http.Request) {
|
||||
prefix := r.URL.Query().Get("prefix")
|
||||
bucketName := r.URL.Query().Get("bucket")
|
||||
marker := r.URL.Query().Get("marker")
|
||||
limitStr := r.URL.Query().Get("limit")
|
||||
|
||||
limit := 100
|
||||
if limitStr != "" {
|
||||
if l, err := strconv.Atoi(limitStr); err == nil && l > 0 {
|
||||
limit = l
|
||||
}
|
||||
}
|
||||
|
||||
// 调用更新后的ListObjects方法,传入所有必需的参数
|
||||
objects, err := h.storage.ListObjects(bucketName, prefix, marker, limit)
|
||||
if err != nil {
|
||||
h.sendError(w, http.StatusInternalServerError, "failed to list objects")
|
||||
return
|
||||
}
|
||||
|
||||
h.sendJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"objects": objects,
|
||||
"count": len(objects),
|
||||
})
|
||||
}
|
||||
|
||||
// 获取对象信息
|
||||
func (h *Handler) handleGetObjectInfo(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
key := vars["key"]
|
||||
|
||||
info, err := h.storage.GetObjectInfo(key)
|
||||
if err != nil {
|
||||
h.sendError(w, http.StatusNotFound, "object not found")
|
||||
return
|
||||
}
|
||||
|
||||
h.sendJSON(w, http.StatusOK, info)
|
||||
}
|
||||
|
||||
// 删除对象(只删除元数据记录)
|
||||
func (h *Handler) handleDeleteObject(w http.ResponseWriter, r *http.Request) {
|
||||
vars := mux.Vars(r)
|
||||
key := vars["key"]
|
||||
|
||||
// 获取对象信息以更新存储桶使用量
|
||||
info, err := h.storage.GetObjectInfo(key)
|
||||
if err != nil {
|
||||
h.sendError(w, http.StatusNotFound, "object not found")
|
||||
return
|
||||
}
|
||||
|
||||
// 更新存储桶使用量
|
||||
if bucket, ok := h.bucketManager.GetBucket(info.BucketName); ok {
|
||||
bucket.UpdateUsedSize(-info.Size)
|
||||
}
|
||||
|
||||
// 删除元数据记录
|
||||
if err := h.storage.DeleteObject(key); err != nil {
|
||||
h.sendError(w, http.StatusInternalServerError, "failed to delete object")
|
||||
return
|
||||
}
|
||||
|
||||
h.sendJSON(w, http.StatusOK, map[string]string{
|
||||
"message": "object deleted successfully",
|
||||
})
|
||||
}
|
||||
|
||||
// 发送JSON响应
|
||||
func (h *Handler) sendJSON(w http.ResponseWriter, status int, data interface{}) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
json.NewEncoder(w).Encode(data)
|
||||
}
|
||||
|
||||
// 发送错误响应
|
||||
func (h *Handler) sendError(w http.ResponseWriter, status int, message string) {
|
||||
h.sendJSON(w, status, map[string]string{
|
||||
"error": message,
|
||||
})
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
@@ -610,6 +611,7 @@ func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, buck
|
||||
nil, // metadata
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("Failed to generate upload URL for key %s in bucket %s: %v", key, targetBucket.Config.Name, err)
|
||||
h.sendS3Error(w, "InternalError", "Failed to generate upload URL", key)
|
||||
return
|
||||
}
|
||||
@@ -651,7 +653,10 @@ func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, buck
|
||||
w.Header().Set("ETag", fmt.Sprintf("\"%x\"", time.Now().UnixNano()))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
} else {
|
||||
h.sendS3Error(w, "InternalError", "Upload failed", key)
|
||||
// 读取错误响应体以获取详细信息
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
log.Printf("Upload failed with status %d: %s", resp.StatusCode, string(body))
|
||||
h.sendS3Error(w, "InternalError", fmt.Sprintf("Upload failed with status %d", resp.StatusCode), key)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,347 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
S3 Balance - S3兼容性测试脚本
|
||||
使用boto3 AWS SDK测试S3 Balance的S3兼容性
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import hashlib
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
|
||||
try:
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
except ImportError:
|
||||
print("请先安装boto3: pip install boto3")
|
||||
sys.exit(1)
|
||||
|
||||
# S3 Balance服务配置
|
||||
S3_BALANCE_ENDPOINT = "http://localhost:8080"
|
||||
ACCESS_KEY = "AKIAIOSFODNN7EXAMPLE"
|
||||
SECRET_KEY = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
|
||||
|
||||
# 测试配置
|
||||
TEST_BUCKET = "test-virtual-1"
|
||||
TEST_KEY_PREFIX = f"test-{int(time.time())}"
|
||||
|
||||
def create_s3_client():
|
||||
"""创建S3客户端"""
|
||||
return boto3.client(
|
||||
's3',
|
||||
endpoint_url=S3_BALANCE_ENDPOINT,
|
||||
aws_access_key_id=ACCESS_KEY,
|
||||
aws_secret_access_key=SECRET_KEY,
|
||||
config=Config(
|
||||
signature_version='s3v4',
|
||||
s3={'addressing_style': 'path'}
|
||||
),
|
||||
region_name='us-east-1'
|
||||
)
|
||||
|
||||
def test_list_buckets(s3_client):
|
||||
"""测试列出存储桶"""
|
||||
print("\n1. 测试列出存储桶 (ListBuckets)...")
|
||||
try:
|
||||
response = s3_client.list_buckets()
|
||||
buckets = response.get('Buckets', [])
|
||||
print(f" ✓ 找到 {len(buckets)} 个存储桶")
|
||||
for bucket in buckets:
|
||||
print(f" - {bucket['Name']} (创建时间: {bucket['CreationDate']})")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" ✗ 失败: {e}")
|
||||
return False
|
||||
|
||||
def test_upload_object(s3_client):
|
||||
"""测试上传对象"""
|
||||
print(f"\n2. 测试上传对象 (PutObject)...")
|
||||
|
||||
# 创建测试文件
|
||||
test_data = b"Hello, S3 Balance! This is a test file."
|
||||
test_key = f"{TEST_KEY_PREFIX}/test-upload.txt"
|
||||
|
||||
try:
|
||||
# 上传对象
|
||||
response = s3_client.put_object(
|
||||
Bucket=TEST_BUCKET,
|
||||
Key=test_key,
|
||||
Body=test_data,
|
||||
ContentType='text/plain',
|
||||
Metadata={'test': 'true', 'timestamp': str(int(time.time()))}
|
||||
)
|
||||
|
||||
etag = response.get('ETag', '').strip('"')
|
||||
print(f" ✓ 成功上传对象: {test_key}")
|
||||
print(f" ETag: {etag}")
|
||||
return test_key
|
||||
except Exception as e:
|
||||
print(f" ✗ 失败: {e}")
|
||||
return None
|
||||
|
||||
def test_list_objects(s3_client):
|
||||
"""测试列出对象"""
|
||||
print(f"\n3. 测试列出对象 (ListObjects)...")
|
||||
|
||||
try:
|
||||
response = s3_client.list_objects_v2(
|
||||
Bucket=TEST_BUCKET,
|
||||
Prefix=TEST_KEY_PREFIX,
|
||||
MaxKeys=10
|
||||
)
|
||||
|
||||
objects = response.get('Contents', [])
|
||||
print(f" ✓ 找到 {len(objects)} 个对象")
|
||||
for obj in objects:
|
||||
print(f" - {obj['Key']} (大小: {obj['Size']} bytes, 修改时间: {obj['LastModified']})")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" ✗ 失败: {e}")
|
||||
return False
|
||||
|
||||
def test_download_object(s3_client, key):
|
||||
"""测试下载对象"""
|
||||
print(f"\n4. 测试下载对象 (GetObject)...")
|
||||
|
||||
if not key:
|
||||
print(" ⚠ 跳过: 没有可下载的对象")
|
||||
return False
|
||||
|
||||
try:
|
||||
response = s3_client.get_object(
|
||||
Bucket=TEST_BUCKET,
|
||||
Key=key
|
||||
)
|
||||
|
||||
data = response['Body'].read()
|
||||
content_type = response.get('ContentType', '')
|
||||
content_length = response.get('ContentLength', 0)
|
||||
|
||||
print(f" ✓ 成功下载对象: {key}")
|
||||
print(f" 内容类型: {content_type}")
|
||||
print(f" 内容长度: {content_length} bytes")
|
||||
print(f" 内容预览: {data[:50].decode('utf-8', errors='ignore')}...")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" ✗ 失败: {e}")
|
||||
return False
|
||||
|
||||
def test_head_object(s3_client, key):
|
||||
"""测试获取对象元数据"""
|
||||
print(f"\n5. 测试获取对象元数据 (HeadObject)...")
|
||||
|
||||
if not key:
|
||||
print(" ⚠ 跳过: 没有可查询的对象")
|
||||
return False
|
||||
|
||||
try:
|
||||
response = s3_client.head_object(
|
||||
Bucket=TEST_BUCKET,
|
||||
Key=key
|
||||
)
|
||||
|
||||
print(f" ✓ 成功获取对象元数据: {key}")
|
||||
print(f" 内容长度: {response.get('ContentLength', 0)} bytes")
|
||||
print(f" 内容类型: {response.get('ContentType', '')}")
|
||||
print(f" ETag: {response.get('ETag', '').strip('\"')}")
|
||||
print(f" 最后修改: {response.get('LastModified', '')}")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" ✗ 失败: {e}")
|
||||
return False
|
||||
|
||||
def test_multipart_upload(s3_client):
|
||||
"""测试分片上传(大文件)"""
|
||||
print(f"\n6. 测试分片上传 (Multipart Upload)...")
|
||||
|
||||
# 创建一个5MB的测试文件
|
||||
test_key = f"{TEST_KEY_PREFIX}/test-multipart.bin"
|
||||
part_size = 5 * 1024 * 1024 # 5MB per part
|
||||
total_size = 10 * 1024 * 1024 # 10MB total
|
||||
|
||||
try:
|
||||
# 初始化分片上传
|
||||
response = s3_client.create_multipart_upload(
|
||||
Bucket=TEST_BUCKET,
|
||||
Key=test_key,
|
||||
ContentType='application/octet-stream'
|
||||
)
|
||||
upload_id = response['UploadId']
|
||||
print(f" ✓ 初始化分片上传,UploadId: {upload_id}")
|
||||
|
||||
# 上传分片
|
||||
parts = []
|
||||
for i in range(2): # 上传2个5MB的分片
|
||||
part_number = i + 1
|
||||
part_data = os.urandom(part_size) # 生成随机数据
|
||||
|
||||
part_response = s3_client.upload_part(
|
||||
Bucket=TEST_BUCKET,
|
||||
Key=test_key,
|
||||
PartNumber=part_number,
|
||||
UploadId=upload_id,
|
||||
Body=part_data
|
||||
)
|
||||
|
||||
parts.append({
|
||||
'ETag': part_response['ETag'],
|
||||
'PartNumber': part_number
|
||||
})
|
||||
print(f" ✓ 上传分片 {part_number}/2 完成")
|
||||
|
||||
# 完成分片上传
|
||||
s3_client.complete_multipart_upload(
|
||||
Bucket=TEST_BUCKET,
|
||||
Key=test_key,
|
||||
UploadId=upload_id,
|
||||
MultipartUpload={'Parts': parts}
|
||||
)
|
||||
|
||||
print(f" ✓ 分片上传完成: {test_key}")
|
||||
return test_key
|
||||
except Exception as e:
|
||||
print(f" ✗ 失败: {e}")
|
||||
return None
|
||||
|
||||
def test_delete_object(s3_client, key):
|
||||
"""测试删除对象"""
|
||||
print(f"\n7. 测试删除对象 (DeleteObject)...")
|
||||
|
||||
if not key:
|
||||
print(" ⚠ 跳过: 没有可删除的对象")
|
||||
return False
|
||||
|
||||
try:
|
||||
s3_client.delete_object(
|
||||
Bucket=TEST_BUCKET,
|
||||
Key=key
|
||||
)
|
||||
|
||||
print(f" ✓ 成功删除对象: {key}")
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" ✗ 失败: {e}")
|
||||
return False
|
||||
|
||||
def test_presigned_url(s3_client):
|
||||
"""测试预签名URL"""
|
||||
print(f"\n8. 测试预签名URL...")
|
||||
|
||||
test_key = f"{TEST_KEY_PREFIX}/test-presigned.txt"
|
||||
|
||||
try:
|
||||
# 生成上传预签名URL
|
||||
upload_url = s3_client.generate_presigned_url(
|
||||
'put_object',
|
||||
Params={'Bucket': TEST_BUCKET, 'Key': test_key},
|
||||
ExpiresIn=3600
|
||||
)
|
||||
print(f" ✓ 生成上传预签名URL")
|
||||
print(f" URL: {upload_url[:80]}...")
|
||||
|
||||
# 生成下载预签名URL
|
||||
download_url = s3_client.generate_presigned_url(
|
||||
'get_object',
|
||||
Params={'Bucket': TEST_BUCKET, 'Key': test_key},
|
||||
ExpiresIn=3600
|
||||
)
|
||||
print(f" ✓ 生成下载预签名URL")
|
||||
print(f" URL: {download_url[:80]}...")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" ✗ 失败: {e}")
|
||||
return False
|
||||
|
||||
def cleanup(s3_client):
|
||||
"""清理测试数据"""
|
||||
print(f"\n清理测试数据...")
|
||||
|
||||
try:
|
||||
# 列出所有测试对象
|
||||
response = s3_client.list_objects_v2(
|
||||
Bucket=TEST_BUCKET,
|
||||
Prefix=TEST_KEY_PREFIX
|
||||
)
|
||||
|
||||
objects = response.get('Contents', [])
|
||||
if objects:
|
||||
# 删除所有测试对象
|
||||
delete_objects = [{'Key': obj['Key']} for obj in objects]
|
||||
s3_client.delete_objects(
|
||||
Bucket=TEST_BUCKET,
|
||||
Delete={'Objects': delete_objects}
|
||||
)
|
||||
print(f" ✓ 删除了 {len(objects)} 个测试对象")
|
||||
else:
|
||||
print(" ✓ 没有需要清理的对象")
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f" ✗ 清理失败: {e}")
|
||||
return False
|
||||
|
||||
def main():
|
||||
"""主测试函数"""
|
||||
print("=" * 60)
|
||||
print("S3 Balance - S3兼容性测试")
|
||||
print("=" * 60)
|
||||
print(f"端点: {S3_BALANCE_ENDPOINT}")
|
||||
print(f"测试桶: {TEST_BUCKET}")
|
||||
print(f"测试前缀: {TEST_KEY_PREFIX}")
|
||||
|
||||
# 创建S3客户端
|
||||
s3_client = create_s3_client()
|
||||
|
||||
# 执行测试
|
||||
results = []
|
||||
|
||||
# 基础测试
|
||||
results.append(("ListBuckets", test_list_buckets(s3_client)))
|
||||
|
||||
# 对象操作测试
|
||||
uploaded_key = test_upload_object(s3_client)
|
||||
results.append(("PutObject", uploaded_key is not None))
|
||||
|
||||
results.append(("ListObjects", test_list_objects(s3_client)))
|
||||
results.append(("GetObject", test_download_object(s3_client, uploaded_key)))
|
||||
results.append(("HeadObject", test_head_object(s3_client, uploaded_key)))
|
||||
|
||||
# 高级功能测试
|
||||
# multipart_key = test_multipart_upload(s3_client)
|
||||
# results.append(("MultipartUpload", multipart_key is not None))
|
||||
|
||||
results.append(("PresignedURL", test_presigned_url(s3_client)))
|
||||
|
||||
# 删除测试
|
||||
results.append(("DeleteObject", test_delete_object(s3_client, uploaded_key)))
|
||||
|
||||
# 清理
|
||||
cleanup(s3_client)
|
||||
|
||||
# 打印测试结果摘要
|
||||
print("\n" + "=" * 60)
|
||||
print("测试结果摘要")
|
||||
print("=" * 60)
|
||||
|
||||
passed = sum(1 for _, result in results if result)
|
||||
total = len(results)
|
||||
|
||||
for test_name, result in results:
|
||||
status = "✓ 通过" if result else "✗ 失败"
|
||||
print(f"{test_name:20} {status}")
|
||||
|
||||
print("-" * 60)
|
||||
print(f"总计: {passed}/{total} 测试通过")
|
||||
|
||||
if passed == total:
|
||||
print("\n🎉 所有测试通过!S3 Balance S3兼容性良好。")
|
||||
else:
|
||||
print(f"\n⚠️ 有 {total - passed} 个测试失败,请检查服务配置。")
|
||||
|
||||
return 0 if passed == total else 1
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user