GithubBackup/BackupX
1
0
Fork 0
mirror of https://github.com/Awuqing/BackupX.git synced 2026-09-07 08:26:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into feature/demo-showcase

Browse Source
This commit is contained in:
Awuqing
2026-08-11 15:46:58 +08:00
parent 3a181c7416 0aa3cc92df
commit 2e20c511b9
218 changed files with 14685 additions and 6184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/cmd/backupx/agent.go
+15 -1
View File
@@ -24,7 +24,10 @@ func runAgent(args []string) {
configPath := fs.String("config", "", "path to agent config YAML (optional)")
master := fs.String("master", "", "master URL, e.g. http://master.example.com:8340")
token := fs.String("token", "", "agent authentication token")
tokenFile := fs.String("token-file", "", "read the agent authentication token from a file")
tempDir := fs.String("temp-dir", "", "local temp directory for backup artifacts")
proxyURL := fs.String("proxy-url", "", "HTTP(S) or SOCKS5 proxy used to reach the master")
caCertFile := fs.String("ca-cert", "", "PEM CA certificate used to verify the master")
insecureTLS := fs.Bool("insecure-tls", false, "skip TLS verification (testing only)")
if err := fs.Parse(args); err != nil {
@@ -36,10 +39,21 @@ func runAgent(args []string) {
fmt.Fprintf(os.Stderr, "agent: load config: %v\n", err)
os.Exit(2)
}
cfg.MergeWithFlags(*master, *token, *tempDir)
cfg.ApplyOverrides(agent.Overrides{
Master: *master,
Token: *token,
TokenFile: *tokenFile,
TempDir: *tempDir,
ProxyURL: *proxyURL,
CACertFile: *caCertFile,
})
if *insecureTLS {
cfg.InsecureSkipTLSVerify = true
}
if err := cfg.ResolveToken(); err != nil {
fmt.Fprintf(os.Stderr, "agent: %v\n", err)
os.Exit(2)
}
if err := cfg.Validate(); err != nil {
fmt.Fprintf(os.Stderr, "agent: %v\n", err)
os.Exit(2)
server/cmd/backupx/backint.go
-1
View File
@@ -95,4 +95,3 @@ func redirectStderr(path string) (func(), error) {
_ = f.Close()
}, nil
}
server/config.example.yaml
+3
View File
@@ -4,6 +4,9 @@ server:
port: 8340
mode: "release" # debug | release
external_url: "" # 可选：Master 对 Agent 可达的 URL，例如 https://backup.example.com
trusted_proxies: # 仅这些代理可提供 X-Forwarded-For；跨容器代理需加入其网段
- "127.0.0.1"
- "::1"
web_root: "" # 前端静态目录；留空自动探测（./web、/opt/backupx/web 等）。
# 命中后后端直接托管 Web 控制台，无需额外 nginx 反向代理。
server/internal/agent/agent.go
+6 -1
View File
@@ -28,10 +28,16 @@ type Agent struct {
// New 构造 Agent。
func New(cfg *Config, version string) (*Agent, error) {
if err := cfg.ResolveToken(); err != nil {
return nil, err
}
if err := cfg.Validate(); err != nil {
return nil, err
}
client := NewMasterClient(cfg.Master, cfg.Token, cfg.InsecureSkipTLSVerify)
if err := client.ConfigureTransport(cfg.ProxyURL, cfg.CACertFile); err != nil {
return nil, fmt.Errorf("configure master connection: %w", err)
}
executor := NewExecutor(client, cfg.TempDir)
return &Agent{
cfg: cfg,
@@ -93,7 +99,6 @@ func (a *Agent) heartbeatLoop(ctx context.Context, interval time.Duration) {
func (a *Agent) heartbeatOnce(ctx context.Context) error {
hostname, _ := os.Hostname()
req := HeartbeatRequest{
Token: a.cfg.Token,
Hostname: hostname,
IPAddress: detectLocalIP(),
AgentVersion: a.version,
server/internal/agent/client.go
+110 -8
View File
@@ -4,10 +4,14 @@ import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
)
@@ -21,23 +25,64 @@ type MasterClient struct {
// NewMasterClient 构造 Master 客户端。
func NewMasterClient(baseURL, token string, insecureTLS bool) *MasterClient {
transport := &http.Transport{}
if insecureTLS {
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
transport := http.DefaultTransport.(*http.Transport).Clone()
tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12}
if transport.TLSClientConfig != nil {
tlsConfig = transport.TLSClientConfig.Clone()
tlsConfig.MinVersion = tls.VersionTLS12
}
// 仅用于用户显式开启的测试模式。生产环境应配置受信 CA。
tlsConfig.InsecureSkipVerify = insecureTLS // #nosec G402
transport.TLSClientConfig = tlsConfig
return &MasterClient{
baseURL: strings.TrimRight(baseURL, "/"),
token: token,
httpClient: &http.Client{
Timeout: 120 * time.Second,
Transport: transport,
// Agent Token 是自定义认证头。禁止自动重定向，避免代理或错误
// 配置把它转发到另一个主机；Master URL 必须直接指向 API。
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
return http.ErrUseLastResponse
},
},
}
}
// ConfigureTransport 应用显式代理和私有 CA。默认 Transport 已保留
// ProxyFromEnvironment，因此 ProxyURL 留空时 HTTP_PROXY/HTTPS_PROXY/NO_PROXY 生效。
func (c *MasterClient) ConfigureTransport(proxyURL, caCertFile string) error {
transport, ok := c.httpClient.Transport.(*http.Transport)
if !ok {
return errors.New("agent http transport has unexpected type")
}
if strings.TrimSpace(proxyURL) != "" {
parsedProxy, err := url.Parse(strings.TrimSpace(proxyURL))
if err != nil {
return fmt.Errorf("parse proxy URL: %w", err)
}
transport.Proxy = http.ProxyURL(parsedProxy)
}
if strings.TrimSpace(caCertFile) == "" {
return nil
}
pemData, err := os.ReadFile(strings.TrimSpace(caCertFile))
if err != nil {
return fmt.Errorf("read CA certificate: %w", err)
}
roots, err := x509.SystemCertPool()
if err != nil || roots == nil {
roots = x509.NewCertPool()
}
if !roots.AppendCertsFromPEM(pemData) {
return errors.New("CA certificate file does not contain a valid PEM certificate")
}
transport.TLSClientConfig.RootCAs = roots
return nil
}
// HeartbeatRequest Agent 上报心跳的请求
type HeartbeatRequest struct {
Token string `json:"token"`
Hostname string `json:"hostname,omitempty"`
IPAddress string `json:"ipAddress,omitempty"`
AgentVersion string `json:"agentVersion,omitempty"`
@@ -125,10 +170,11 @@ type TaskSpec struct {
// StorageTargetConfig 与 service.AgentStorageTargetConfig 对齐
type StorageTargetConfig struct {
ID uint `json:"id"`
Type string `json:"type"`
Name string `json:"name"`
Config json.RawMessage `json:"config"`
ID uint `json:"id"`
Type string `json:"type"`
Name string `json:"name"`
Config json.RawMessage `json:"config"`
TransferMode string `json:"transferMode"`
}
// GetTaskSpec 拉取任务规格
@@ -149,6 +195,7 @@ type RecordUpdate struct {
Checksum string `json:"checksum,omitempty"`
StoragePath string `json:"storagePath,omitempty"`
StorageTargetID uint `json:"storageTargetId,omitempty"`
StorageTransferMode string `json:"storageTransferMode,omitempty"`
StorageUploadResults []StorageResultItem `json:"storageUploadResults,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
LogAppend string `json:"logAppend,omitempty"`
@@ -160,6 +207,7 @@ type StorageResultItem struct {
Status string `json:"status"`
StoragePath string `json:"storagePath,omitempty"`
FileSize int64 `json:"fileSize,omitempty"`
TransferMode string `json:"transferMode,omitempty"`
Error string `json:"error,omitempty"`
}
@@ -169,6 +217,39 @@ func (c *MasterClient) UpdateRecord(ctx context.Context, recordID uint, update R
return c.do(ctx, http.MethodPost, path, update, nil)
}
// UploadArtifact streams an artifact through the Master for storage targets
// that are not directly reachable from the Agent.
func (c *MasterClient) UploadArtifact(ctx context.Context, recordID, targetID uint, objectKey string, size int64, checksum string, reader io.Reader) error {
path := fmt.Sprintf("/api/agent/records/%d/artifacts/%d", recordID, targetID)
req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.baseURL+path, reader)
if err != nil {
return err
}
// The executor owns and closes the artifact file. Prevent net/http from
// closing that underlying reader when it finishes the request body.
req.Body = io.NopCloser(reader)
req.ContentLength = size
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("X-Agent-Token", c.token)
req.Header.Set("X-BackupX-Object-Key", objectKey)
req.Header.Set("X-BackupX-SHA256", checksum)
client := *c.httpClient
client.Timeout = 0
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("relay artifact to Master: %w", err)
}
data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
closeErr := resp.Body.Close()
if readErr != nil || closeErr != nil {
return errors.Join(readErr, closeErr)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return fmt.Errorf("relay artifact to Master: http %d: %s", resp.StatusCode, string(data))
}
return nil
}
// RestoreSpec 与 service.AgentRestoreSpec 对齐
type RestoreSpec struct {
RestoreRecordID uint `json:"restoreRecordId"`
@@ -210,6 +291,27 @@ func (c *MasterClient) GetRestoreSpec(ctx context.Context, restoreRecordID uint)
return &spec, nil
}
func (c *MasterClient) DownloadRestoreArtifact(ctx context.Context, restoreRecordID uint) (io.ReadCloser, error) {
path := fmt.Sprintf("/api/agent/restores/%d/artifact", restoreRecordID)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil)
if err != nil {
return nil, err
}
req.Header.Set("X-Agent-Token", c.token)
client := *c.httpClient
client.Timeout = 0
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("download relayed artifact from Master: %w", err)
}
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
return resp.Body, nil
}
data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
closeErr := resp.Body.Close()
return nil, errors.Join(fmt.Errorf("download relayed artifact from Master: http %d: %s", resp.StatusCode, string(data)), readErr, closeErr)
}
// UpdateRestore 上报恢复记录的状态/日志
func (c *MasterClient) UpdateRestore(ctx context.Context, restoreRecordID uint, update RestoreUpdate) error {
path := fmt.Sprintf("/api/agent/restores/%d", restoreRecordID)
server/internal/agent/client_test.go
+93
View File
@@ -0,0 +1,93 @@
package agent
import (
"context"
"io"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"testing"
)
func TestMasterClientKeepsEnvironmentProxySupport(t *testing.T) {
client := NewMasterClient("https://master.example.com", "token", false)
transport := client.httpClient.Transport.(*http.Transport)
if transport.Proxy == nil {
t.Fatal("default transport proxy function must be preserved")
}
}
func TestMasterClientConfiguresExplicitProxy(t *testing.T) {
client := NewMasterClient("https://master.example.com", "token", false)
if err := client.ConfigureTransport("socks5h://127.0.0.1:1080", ""); err != nil {
t.Fatal(err)
}
transport := client.httpClient.Transport.(*http.Transport)
requestURL, _ := url.Parse("https://master.example.com")
proxyURL, err := transport.Proxy(&http.Request{URL: requestURL})
if err != nil {
t.Fatal(err)
}
if proxyURL == nil || proxyURL.String() != "socks5h://127.0.0.1:1080" {
t.Fatalf("proxy URL = %v", proxyURL)
}
}
func TestMasterClientRejectsInvalidCACertificate(t *testing.T) {
path := filepath.Join(t.TempDir(), "invalid.pem")
if err := os.WriteFile(path, []byte("not a certificate"), 0o600); err != nil {
t.Fatal(err)
}
client := NewMasterClient("https://master.example.com", "token", false)
if err := client.ConfigureTransport("", path); err == nil {
t.Fatal("expected invalid CA certificate error")
}
}
func TestMasterClientDoesNotForwardTokenThroughRedirects(t *testing.T) {
receivedToken := ""
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
receivedToken = r.Header.Get("X-Agent-Token")
w.WriteHeader(http.StatusOK)
}))
defer target.Close()
redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, target.URL, http.StatusTemporaryRedirect)
}))
defer redirector.Close()
client := NewMasterClient(redirector.URL, "secret-agent-token", false)
if _, err := client.Heartbeat(context.Background(), HeartbeatRequest{}); err == nil {
t.Fatal("redirect response should not be accepted as a Master API response")
}
if receivedToken != "" {
t.Fatalf("Agent token leaked through redirect: %q", receivedToken)
}
}
func TestHeartbeatSendsTokenOnlyInAuthenticationHeader(t *testing.T) {
requestBody := ""
receivedHeader := ""
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
requestBody = string(body)
receivedHeader = r.Header.Get("X-Agent-Token")
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"data":{"status":"ok","nodeId":1,"name":"node"}}`))
}))
defer server.Close()
client := NewMasterClient(server.URL, "secret-agent-token", false)
if _, err := client.Heartbeat(context.Background(), HeartbeatRequest{Hostname: "node"}); err != nil {
t.Fatal(err)
}
if receivedHeader != "secret-agent-token" {
t.Fatalf("authentication header = %q", receivedHeader)
}
if strings.Contains(requestBody, "secret-agent-token") || strings.Contains(requestBody, `"token"`) {
t.Fatalf("heartbeat body exposed the Agent token: %s", requestBody)
}
}
server/internal/agent/config.go
+100 -9
View File
@@ -10,8 +10,10 @@ package agent
import (
"errors"
"fmt"
"net/url"
"os"
"strings"
"time"
"gopkg.in/yaml.v3"
)
@@ -22,16 +24,34 @@ type Config struct {
Master string `yaml:"master"`
// Token 节点认证令牌（在 Master 创建节点时生成）
Token string `yaml:"token"`
// TokenFile 从文件读取节点认证令牌；适合 systemd 凭据和容器 secret。
// Token 与 TokenFile 同时设置时优先使用 Token。
TokenFile string `yaml:"tokenFile"`
// HeartbeatInterval 心跳间隔，默认 15s
HeartbeatInterval string `yaml:"heartbeatInterval"`
// PollInterval 命令轮询间隔，默认 5s
PollInterval string `yaml:"pollInterval"`
// TempDir 备份临时目录，默认 /var/lib/backupx-agent/tmp
TempDir string `yaml:"tempDir"`
// ProxyURL Agent 访问 Master 使用的显式代理。留空时遵循
// HTTP_PROXY、HTTPS_PROXY 与 NO_PROXY；支持 http(s) 和 socks5(h)。
ProxyURL string `yaml:"proxyUrl"`
// CACertFile 私有 CA 的 PEM 文件路径，用于安全连接内网 HTTPS Master。
CACertFile string `yaml:"caCertFile"`
// InsecureSkipTLSVerify 测试环境允许跳过 TLS 证书校验
InsecureSkipTLSVerify bool `yaml:"insecureSkipTlsVerify"`
}
// Overrides 表示命令行显式提供的 Agent 配置覆盖项。
type Overrides struct {
Master string
Token string
TokenFile string
TempDir string
ProxyURL string
CACertFile string
}
// LoadConfigFile 从 YAML 文件加载 Agent 配置。
func LoadConfigFile(path string) (*Config, error) {
data, err := os.ReadFile(path)
@@ -50,42 +70,109 @@ func LoadConfigFile(path string) (*Config, error) {
// 支持的环境变量：
// - BACKUPX_AGENT_MASTER Master URL
// - BACKUPX_AGENT_TOKEN 节点认证令牌
// - BACKUPX_AGENT_TOKEN_FILE 节点认证令牌文件
// - BACKUPX_AGENT_HEARTBEAT 心跳间隔（如 15s）
// - BACKUPX_AGENT_POLL 命令轮询间隔（如 5s）
// - BACKUPX_AGENT_TEMP_DIR 临时目录
// - BACKUPX_AGENT_PROXY_URL 显式 HTTP(S)/SOCKS5 代理
// - BACKUPX_AGENT_CA_CERT_FILE 私有 CA PEM 文件
// - BACKUPX_AGENT_INSECURE_TLS true / 1 跳过 TLS 校验
func LoadConfigFromEnv() (*Config, error) {
cfg := &Config{
Master: strings.TrimSpace(os.Getenv("BACKUPX_AGENT_MASTER")),
Token: strings.TrimSpace(os.Getenv("BACKUPX_AGENT_TOKEN")),
TokenFile: strings.TrimSpace(os.Getenv("BACKUPX_AGENT_TOKEN_FILE")),
HeartbeatInterval: strings.TrimSpace(os.Getenv("BACKUPX_AGENT_HEARTBEAT")),
PollInterval: strings.TrimSpace(os.Getenv("BACKUPX_AGENT_POLL")),
TempDir: strings.TrimSpace(os.Getenv("BACKUPX_AGENT_TEMP_DIR")),
ProxyURL: strings.TrimSpace(os.Getenv("BACKUPX_AGENT_PROXY_URL")),
CACertFile: strings.TrimSpace(os.Getenv("BACKUPX_AGENT_CA_CERT_FILE")),
InsecureSkipTLSVerify: strings.EqualFold(os.Getenv("BACKUPX_AGENT_INSECURE_TLS"), "true") || os.Getenv("BACKUPX_AGENT_INSECURE_TLS") == "1",
}
return applyConfigDefaults(cfg)
}
// MergeWithFlags 把命令行覆盖值合并入配置（非空覆盖）。
func (c *Config) MergeWithFlags(master, token, tempDir string) {
if strings.TrimSpace(master) != "" {
c.Master = master
// ApplyOverrides 把命令行覆盖值合并入配置（非空覆盖）。
func (c *Config) ApplyOverrides(overrides Overrides) {
if strings.TrimSpace(overrides.Master) != "" {
c.Master = strings.TrimSpace(overrides.Master)
}
if strings.TrimSpace(token) != "" {
c.Token = token
tokenProvided := strings.TrimSpace(overrides.Token) != ""
if strings.TrimSpace(overrides.TokenFile) != "" {
c.TokenFile = strings.TrimSpace(overrides.TokenFile)
if !tokenProvided {
// An explicit --token-file must override a token inherited from YAML.
c.Token = ""
}
}
if strings.TrimSpace(tempDir) != "" {
c.TempDir = tempDir
if tokenProvided {
c.Token = strings.TrimSpace(overrides.Token)
}
if strings.TrimSpace(overrides.TempDir) != "" {
c.TempDir = strings.TrimSpace(overrides.TempDir)
}
if strings.TrimSpace(overrides.ProxyURL) != "" {
c.ProxyURL = strings.TrimSpace(overrides.ProxyURL)
}
if strings.TrimSpace(overrides.CACertFile) != "" {
c.CACertFile = strings.TrimSpace(overrides.CACertFile)
}
}
// ResolveToken 在所有配置源合并完成后读取 token 文件。
func (c *Config) ResolveToken() error {
if strings.TrimSpace(c.Token) != "" || strings.TrimSpace(c.TokenFile) == "" {
c.Token = strings.TrimSpace(c.Token)
return nil
}
data, err := os.ReadFile(strings.TrimSpace(c.TokenFile))
if err != nil {
return fmt.Errorf("read agent token file: %w", err)
}
c.Token = strings.TrimSpace(string(data))
if c.Token == "" {
return errors.New("agent token file is empty")
}
return nil
}
// Validate 校验必填字段。
func (c *Config) Validate() error {
masterURL, err := url.Parse(strings.TrimSpace(c.Master))
if strings.TrimSpace(c.Master) == "" {
return errors.New("master url is required (set via --master, BACKUPX_AGENT_MASTER or config file)")
}
if err != nil || (masterURL.Scheme != "http" && masterURL.Scheme != "https") || masterURL.Host == "" || masterURL.User != nil || masterURL.RawQuery != "" || masterURL.Fragment != "" {
return errors.New("master url must be an absolute http(s) URL without credentials, query or fragment")
}
if strings.TrimSpace(c.Token) == "" {
return errors.New("token is required (set via --token, BACKUPX_AGENT_TOKEN or config file)")
return errors.New("token is required (set via --token, --token-file, environment or config file)")
}
if c.ProxyURL != "" {
proxyURL, proxyErr := url.Parse(c.ProxyURL)
if proxyErr != nil || proxyURL.Host == "" {
return errors.New("proxy url must be an absolute URL")
}
switch proxyURL.Scheme {
case "http", "https", "socks5", "socks5h":
default:
return errors.New("proxy url scheme must be http, https, socks5 or socks5h")
}
if proxyURL.RawQuery != "" || proxyURL.Fragment != "" || (proxyURL.Path != "" && proxyURL.Path != "/") {
return errors.New("proxy url must not contain a path, query or fragment")
}
}
if c.CACertFile != "" && c.InsecureSkipTLSVerify {
return errors.New("ca cert file and insecure TLS cannot be enabled together")
}
for name, value := range map[string]string{
"heartbeat interval": c.HeartbeatInterval,
"poll interval": c.PollInterval,
} {
duration, durationErr := time.ParseDuration(value)
if durationErr != nil || duration <= 0 {
return fmt.Errorf("%s must be a positive duration", name)
}
}
return nil
}
@@ -101,5 +188,9 @@ func applyConfigDefaults(cfg *Config) (*Config, error) {
cfg.TempDir = "/var/lib/backupx-agent/tmp"
}
cfg.Master = strings.TrimRight(strings.TrimSpace(cfg.Master), "/")
cfg.Token = strings.TrimSpace(cfg.Token)
cfg.TokenFile = strings.TrimSpace(cfg.TokenFile)
cfg.ProxyURL = strings.TrimSpace(cfg.ProxyURL)
cfg.CACertFile = strings.TrimSpace(cfg.CACertFile)
return cfg, nil
}
server/internal/agent/config_test.go
+48 -2
View File
@@ -11,9 +11,12 @@ func TestLoadConfigFile(t *testing.T) {
path := filepath.Join(dir, "agent.yaml")
content := `master: http://master.example.com:8340/
token: abc123
tokenFile: /run/secrets/backupx_agent_token
heartbeatInterval: 20s
pollInterval: 3s
tempDir: /var/backupx-agent
proxyUrl: socks5h://127.0.0.1:1080
caCertFile: /etc/backupx-agent/ca.pem
insecureSkipTlsVerify: true
`
if err := os.WriteFile(path, []byte(content), 0644); err != nil {
@@ -35,6 +38,9 @@ insecureSkipTlsVerify: true
if !cfg.InsecureSkipTLSVerify {
t.Errorf("insecure should be true")
}
if cfg.ProxyURL != "socks5h://127.0.0.1:1080" || cfg.CACertFile != "/etc/backupx-agent/ca.pem" {
t.Errorf("connection options not loaded: %+v", cfg)
}
}
func TestLoadConfigDefaults(t *testing.T) {
@@ -64,8 +70,15 @@ func TestConfigValidate(t *testing.T) {
{"valid", Config{Master: "http://m", Token: "t"}, false},
{"missing master", Config{Token: "t"}, true},
{"missing token", Config{Master: "http://m"}, true},
{"invalid master scheme", Config{Master: "ssh://m", Token: "t"}, true},
{"master credentials rejected", Config{Master: "https://user:pass@m", Token: "t"}, true},
{"valid socks proxy", Config{Master: "https://m", Token: "t", ProxyURL: "socks5h://127.0.0.1:1080"}, false},
{"invalid proxy", Config{Master: "https://m", Token: "t", ProxyURL: "ftp://proxy"}, true},
{"proxy path rejected", Config{Master: "https://m", Token: "t", ProxyURL: "http://proxy/connect"}, true},
{"invalid heartbeat", Config{Master: "https://m", Token: "t", HeartbeatInterval: "never"}, true},
}
for _, c := range cases {
_, _ = applyConfigDefaults(&c.cfg)
err := c.cfg.Validate()
if (err != nil) != c.wantErr {
t.Errorf("%s: err=%v wantErr=%v", c.name, err, c.wantErr)
@@ -75,7 +88,7 @@ func TestConfigValidate(t *testing.T) {
func TestMergeWithFlags(t *testing.T) {
cfg := &Config{Master: "http://old", Token: "old"}
cfg.MergeWithFlags("http://new", "", "/tmp/x")
cfg.ApplyOverrides(Overrides{Master: "http://new", TempDir: "/tmp/x", ProxyURL: "http://proxy:3128"})
if cfg.Master != "http://new" {
t.Errorf("master not overridden: %q", cfg.Master)
}
@@ -85,17 +98,50 @@ func TestMergeWithFlags(t *testing.T) {
if cfg.TempDir != "/tmp/x" {
t.Errorf("tempDir: %q", cfg.TempDir)
}
if cfg.ProxyURL != "http://proxy:3128" {
t.Errorf("proxyUrl: %q", cfg.ProxyURL)
}
}
func TestTokenFileOverrideReplacesConfiguredToken(t *testing.T) {
path := filepath.Join(t.TempDir(), "agent.token")
if err := os.WriteFile(path, []byte("file-token\n"), 0o600); err != nil {
t.Fatal(err)
}
cfg := &Config{Token: "yaml-token", TokenFile: "/old/token"}
cfg.ApplyOverrides(Overrides{TokenFile: " " + path + " "})
if err := cfg.ResolveToken(); err != nil {
t.Fatal(err)
}
if cfg.Token != "file-token" || cfg.TokenFile != path {
t.Fatalf("token file override was not applied: %+v", cfg)
}
}
func TestLoadConfigFromEnv(t *testing.T) {
t.Setenv("BACKUPX_AGENT_MASTER", "http://env-master")
t.Setenv("BACKUPX_AGENT_TOKEN", "env-token")
t.Setenv("BACKUPX_AGENT_PROXY_URL", "http://env-proxy:8080")
t.Setenv("BACKUPX_AGENT_INSECURE_TLS", "true")
cfg, err := LoadConfigFromEnv()
if err != nil {
t.Fatal(err)
}
if cfg.Master != "http://env-master" || cfg.Token != "env-token" || !cfg.InsecureSkipTLSVerify {
if cfg.Master != "http://env-master" || cfg.Token != "env-token" || cfg.ProxyURL != "http://env-proxy:8080" || !cfg.InsecureSkipTLSVerify {
t.Errorf("env not picked up: %+v", cfg)
}
}
func TestResolveTokenFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "agent.token")
if err := os.WriteFile(path, []byte(" file-token\n"), 0o600); err != nil {
t.Fatal(err)
}
cfg := &Config{TokenFile: path}
if err := cfg.ResolveToken(); err != nil {
t.Fatal(err)
}
if cfg.Token != "file-token" {
t.Fatalf("token = %q", cfg.Token)
}
}
server/internal/agent/executor.go
+53 -38
View File
@@ -5,6 +5,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
@@ -123,7 +124,7 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er
}
fileName := filepath.Base(finalPath)
fileSize := info.Size()
storagePath := backup.BuildStorageKey(spec.Type, startedAt, fileName)
storagePath := backup.BuildRecordStorageKey(spec.Type, startedAt, recordID, fileName)
// 5) 计算 checksum（一次读一次）并上传到所有目标
checksum, err := computeFileSHA256(finalPath)
@@ -137,13 +138,15 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er
}
uploadResults := make([]StorageResultItem, 0, len(spec.StorageTargets))
selectedStorageTargetID := uint(0)
selectedStorageTransferMode := ""
var uploadErrors []string
for _, target := range spec.StorageTargets {
if err := e.uploadToTarget(ctx, recordID, target, finalPath, storagePath, fileSize, spec.TaskID); err != nil {
if err := e.uploadToTarget(ctx, recordID, target, finalPath, storagePath, fileSize, checksum, spec.TaskID); err != nil {
uploadResults = append(uploadResults, StorageResultItem{
StorageTargetID: target.ID,
StorageTargetName: target.Name,
Status: "failed",
TransferMode: target.TransferMode,
Error: err.Error(),
})
uploadErrors = append(uploadErrors, fmt.Sprintf("%s: %v", target.Name, err))
@@ -152,6 +155,7 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er
}
if selectedStorageTargetID == 0 {
selectedStorageTargetID = target.ID
selectedStorageTransferMode = target.TransferMode
}
uploadResults = append(uploadResults, StorageResultItem{
StorageTargetID: target.ID,
@@ -159,6 +163,7 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er
Status: "success",
StoragePath: storagePath,
FileSize: fileSize,
TransferMode: target.TransferMode,
})
e.appendLog(ctx, recordID, fmt.Sprintf("[agent] 已上传到存储目标 %s\n", target.Name))
}
@@ -179,34 +184,40 @@ func (e *Executor) ExecuteRunTask(ctx context.Context, taskID, recordID uint) er
Checksum: checksum,
StoragePath: storagePath,
StorageTargetID: selectedStorageTargetID,
StorageTransferMode: selectedStorageTransferMode,
StorageUploadResults: uploadResults,
LogAppend: fmt.Sprintf("[agent] 任务完成，总计 %d 字节\n", fileSize),
})
}
// uploadToTarget 上传单个目标。为保持简化不做上传级重试（rclone 本身已有 low-level 重试）。
func (e *Executor) uploadToTarget(ctx context.Context, recordID uint, target StorageTargetConfig, filePath, objectKey string, fileSize int64, taskID uint) error {
var rawConfig map[string]any
if len(target.Config) > 0 {
// DecodeRawConfig 通过 json 解析
if err := jsonUnmarshalMap(target.Config, &rawConfig); err != nil {
return fmt.Errorf("parse storage config: %w", err)
}
}
provider, err := e.storageRegistry.Create(ctx, target.Type, rawConfig)
if err != nil {
return fmt.Errorf("create provider: %w", err)
}
func (e *Executor) uploadToTarget(ctx context.Context, recordID uint, target StorageTargetConfig, filePath, objectKey string, fileSize int64, checksum string, taskID uint) error {
f, err := os.Open(filePath)
if err != nil {
return fmt.Errorf("open artifact: %w", err)
}
defer f.Close()
if target.TransferMode == storage.TransferModeMasterRelay {
uploadErr := e.client.UploadArtifact(ctx, recordID, target.ID, objectKey, fileSize, checksum, f)
return errors.Join(uploadErr, f.Close())
}
var rawConfig map[string]any
if len(target.Config) > 0 {
// DecodeRawConfig 通过 json 解析
if err := jsonUnmarshalMap(target.Config, &rawConfig); err != nil {
return errors.Join(fmt.Errorf("parse storage config: %w", err), f.Close())
}
}
provider, err := e.storageRegistry.Create(ctx, target.Type, rawConfig)
if err != nil {
closeErr := f.Close()
return errors.Join(fmt.Errorf("create provider: %w", err), closeErr)
}
meta := map[string]string{
"taskId": fmt.Sprintf("%d", taskID),
"recordId": fmt.Sprintf("%d", recordID),
}
return provider.Upload(ctx, objectKey, f, fileSize, meta)
uploadErr := provider.Upload(ctx, objectKey, f, fileSize, meta)
return errors.Join(uploadErr, f.Close())
}
// appendLog 追加日志到 Master 记录（尽力而为，失败不中断主流程）
@@ -328,7 +339,7 @@ func (e *Executor) DeleteStorageObject(ctx context.Context, targetType string, t
// ExecuteRestore 处理 restore_record 命令：拉规格 → 下载 → 解压 → 执行 runner.Restore → 上报结果。
//
// 与 ExecuteRunTask 对称，但方向相反：
// - 下载：通过 spec.Storage 创建 provider → Download(spec.StoragePath)
// - 下载：直连共享存储，或通过 Master 中转其本地磁盘对象
// - 解密：当前 Agent 不支持加密恢复（密钥未下发），spec.Encrypt=true 会直接失败
// - 执行：backup.Registry.Runner(spec.Type).Restore
// - 上报：通过 UpdateRestore（status/logAppend）
@@ -357,28 +368,31 @@ func (e *Executor) ExecuteRestore(ctx context.Context, restoreRecordID uint) err
}
defer os.RemoveAll(tmpDir)
// 1) 创建 storage provider
var rawConfig map[string]any
if len(spec.Storage.Config) > 0 {
if err := jsonUnmarshalMap(spec.Storage.Config, &rawConfig); err != nil {
e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("解析存储配置失败: %v", err))
return err
}
}
provider, err := e.storageRegistry.Create(ctx, spec.Storage.Type, rawConfig)
if err != nil {
e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("创建存储客户端失败: %v", err))
return err
}
// 2) 下载
// 1) 下载
fileName := spec.FileName
if strings.TrimSpace(fileName) == "" {
fileName = filepath.Base(spec.StoragePath)
}
artifactPath := filepath.Join(tmpDir, filepath.Base(fileName))
e.appendRestoreLog(ctx, restoreRecordID, fmt.Sprintf("[agent] 下载备份文件 %s\n", spec.StoragePath))
reader, err := provider.Download(ctx, spec.StoragePath)
var reader io.ReadCloser
if spec.Storage.TransferMode == storage.TransferModeMasterRelay {
reader, err = e.client.DownloadRestoreArtifact(ctx, restoreRecordID)
} else {
var rawConfig map[string]any
if len(spec.Storage.Config) > 0 {
if err := jsonUnmarshalMap(spec.Storage.Config, &rawConfig); err != nil {
e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("解析存储配置失败: %v", err))
return err
}
}
provider, providerErr := e.storageRegistry.Create(ctx, spec.Storage.Type, rawConfig)
if providerErr != nil {
e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("创建存储客户端失败: %v", providerErr))
return providerErr
}
reader, err = provider.Download(ctx, spec.StoragePath)
}
if err != nil {
e.reportRestoreFailure(ctx, restoreRecordID, fmt.Sprintf("下载备份失败: %v", err))
return err
@@ -489,8 +503,10 @@ func buildRestoreBackupTaskSpec(spec *RestoreSpec, startedAt time.Time, tempDir
}
// writeReaderToLocal 把 reader 写到本地文件（Agent 侧工具函数）。
func writeReaderToLocal(targetPath string, reader io.ReadCloser) error {
defer reader.Close()
func writeReaderToLocal(targetPath string, reader io.ReadCloser) (err error) {
defer func() {
err = errors.Join(err, reader.Close())
}()
if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil {
return err
}
@@ -498,9 +514,8 @@ func writeReaderToLocal(targetPath string, reader io.ReadCloser) error {
if err != nil {
return err
}
defer file.Close()
_, err = io.Copy(file, reader)
return err
_, copyErr := io.Copy(file, reader)
return errors.Join(copyErr, file.Close())
}
// 辅助函数
server/internal/agent/executor_test.go
+147
View File
@@ -1,7 +1,10 @@
package agent
import (
"archive/tar"
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"io"
@@ -109,6 +112,150 @@ func TestExecuteRunTaskRecordsPerTargetUploadResults(t *testing.T) {
}
}
func TestExecuteRunTaskRelaysMasterLocalDiskTarget(t *testing.T) {
sourceDir := t.TempDir()
if err := os.WriteFile(filepath.Join(sourceDir, "index.html"), []byte("centralize me"), 0o644); err != nil {
t.Fatalf("WriteFile returned error: %v", err)
}
var relayed []byte
var finalUpdate RecordUpdate
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/api/agent/tasks/1":
writeAgentEnvelope(t, w, TaskSpec{
TaskID: 1,
Name: "remote-source",
Type: "file",
SourcePath: sourceDir,
Compression: "gzip",
StorageTargets: []StorageTargetConfig{{
ID: 11, Name: "master-disk", Type: storage.TypeLocalDisk, TransferMode: storage.TransferModeMasterRelay,
}},
})
case r.Method == http.MethodPut && r.URL.Path == "/api/agent/records/99/artifacts/11":
body, err := io.ReadAll(r.Body)
if err != nil {
t.Fatalf("ReadAll relayed body: %v", err)
}
digest := sha256.Sum256(body)
if got := r.Header.Get("X-BackupX-SHA256"); got != fmt.Sprintf("%x", digest[:]) {
t.Fatalf("relay checksum header = %q", got)
}
objectKey := r.Header.Get("X-BackupX-Object-Key")
if !strings.Contains(objectKey, "/records/99/") || r.ContentLength != int64(len(body)) {
t.Fatalf("invalid relay metadata: key=%q length=%d body=%d", objectKey, r.ContentLength, len(body))
}
relayed = append([]byte(nil), body...)
writeAgentEnvelope(t, w, map[string]string{"status": "ok"})
case r.Method == http.MethodPost && r.URL.Path == "/api/agent/records/99":
var update RecordUpdate
if err := json.NewDecoder(r.Body).Decode(&update); err != nil {
t.Fatalf("Decode update returned error: %v", err)
}
if update.Status != "" {
finalUpdate = update
}
writeAgentEnvelope(t, w, map[string]string{"status": "ok"})
default:
http.NotFound(w, r)
}
}))
defer server.Close()
executor := NewExecutor(NewMasterClient(server.URL, "token", false), filepath.Join(t.TempDir(), "tmp"))
if err := executor.ExecuteRunTask(context.Background(), 1, 99); err != nil {
t.Fatalf("ExecuteRunTask returned error: %v", err)
}
if len(relayed) == 0 {
t.Fatal("expected artifact bytes to be streamed through Master")
}
if finalUpdate.Status != "success" || finalUpdate.StorageTransferMode != storage.TransferModeMasterRelay {
t.Fatalf("unexpected final relay update: %#v", finalUpdate)
}
if len(finalUpdate.StorageUploadResults) != 1 || finalUpdate.StorageUploadResults[0].TransferMode != storage.TransferModeMasterRelay {
t.Fatalf("unexpected relay target result: %#v", finalUpdate.StorageUploadResults)
}
}
func TestExecuteRestoreDownloadsMasterRelayedArtifact(t *testing.T) {
var archive bytes.Buffer
tarWriter := tar.NewWriter(&archive)
content := []byte("restored through Master")
header := &tar.Header{Name: "site/index.html", Mode: 0o644, Size: int64(len(content)), Typeflag: tar.TypeReg}
if err := tarWriter.WriteHeader(header); err != nil {
t.Fatalf("WriteHeader returned error: %v", err)
}
if _, err := tarWriter.Write(content); err != nil {
t.Fatalf("Write returned error: %v", err)
}
if err := tarWriter.Close(); err != nil {
t.Fatalf("Close returned error: %v", err)
}
artifact := append([]byte(nil), archive.Bytes()...)
digest := sha256.Sum256(artifact)
restoreRoot := t.TempDir()
restoreSource := filepath.Join(restoreRoot, "site")
artifactRequests := 0
var finalUpdate RestoreUpdate
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/api/agent/restores/77/spec":
writeAgentEnvelope(t, w, RestoreSpec{
RestoreRecordID: 77,
BackupRecordID: 99,
TaskID: 1,
TaskName: "remote-source",
Type: "file",
SourcePath: restoreSource,
Storage: StorageTargetConfig{
ID: 11, Name: "master-disk", Type: storage.TypeLocalDisk, TransferMode: storage.TransferModeMasterRelay,
},
StoragePath: "BackupX/file/site.tar",
FileName: "site.tar",
Checksum: fmt.Sprintf("%x", digest[:]),
})
case r.Method == http.MethodGet && r.URL.Path == "/api/agent/restores/77/artifact":
artifactRequests++
if r.Header.Get("X-Agent-Token") != "token" {
t.Fatalf("missing Agent token on relay download")
}
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(artifact)))
_, _ = w.Write(artifact)
case r.Method == http.MethodPost && r.URL.Path == "/api/agent/restores/77":
var update RestoreUpdate
if err := json.NewDecoder(r.Body).Decode(&update); err != nil {
t.Fatalf("Decode update returned error: %v", err)
}
if update.Status != "" {
finalUpdate = update
}
writeAgentEnvelope(t, w, map[string]string{"status": "ok"})
default:
http.NotFound(w, r)
}
}))
defer server.Close()
executor := NewExecutor(NewMasterClient(server.URL, "token", false), filepath.Join(t.TempDir(), "tmp"))
if err := executor.ExecuteRestore(context.Background(), 77); err != nil {
t.Fatalf("ExecuteRestore returned error: %v", err)
}
if artifactRequests != 1 {
t.Fatalf("expected one relay artifact request, got %d", artifactRequests)
}
restored, err := os.ReadFile(filepath.Join(restoreRoot, "site", "index.html"))
if err != nil {
t.Fatalf("ReadFile returned error: %v", err)
}
if string(restored) != string(content) {
t.Fatalf("restored content = %q, want %q", restored, content)
}
if finalUpdate.Status != "success" {
t.Fatalf("unexpected final restore update: %#v", finalUpdate)
}
}
func TestExecuteRunTaskReportsPerTargetUploadResultsWhenAllTargetsFail(t *testing.T) {
sourceDir := t.TempDir()
if err := os.WriteFile(filepath.Join(sourceDir, "index.html"), []byte("hello"), 0o644); err != nil {
server/internal/app/app.go
+2 -2
View File
@@ -84,7 +84,7 @@ func New(ctx context.Context, cfg config.Config, version string) (*Application,
// nodeRepo 在下方 Cluster 节点管理区块才实例化，这里延后注入
backupRunnerRegistry := backup.NewRegistry(backup.NewFileRunner(), backup.NewSQLiteRunner(), backup.NewMySQLRunner(nil), backup.NewPostgreSQLRunner(nil), backup.NewSAPHANARunner(nil), backup.NewMongoDBRunner(nil))
logHub := backup.NewLogHub()
retentionService := backupretention.NewService(backupRecordRepo)
retentionService := backupretention.NewService(backupRecordRepo, configCipher.Key())
notifyRegistry := notify.NewRegistry(notify.NewEmailNotifier(), notify.NewWebhookNotifier(), notify.NewTelegramNotifier())
notificationService := service.NewNotificationService(notificationRepo, notifyRegistry, configCipher)
authService.SetNotificationService(notificationService)
@@ -135,7 +135,7 @@ func New(ctx context.Context, cfg config.Config, version string) (*Application,
// Agent 协议服务：命令队列 + 任务下发 + 记录上报
agentCmdRepo := repository.NewAgentCommandRepository(db)
nodeService.SetAgentCommandRepository(agentCmdRepo)
agentService := service.NewAgentService(nodeRepo, backupTaskRepo, backupRecordRepo, storageTargetRepo, agentCmdRepo, configCipher)
agentService := service.NewAgentService(nodeRepo, backupTaskRepo, backupRecordRepo, storageTargetRepo, agentCmdRepo, configCipher, storageRegistry)
agentService.SetRestoreRepository(restoreRecordRepo)
agentService.StartCommandTimeoutMonitor(ctx, 30*time.Second, 10*time.Minute)
server/internal/backint/agent.go
-1
View File
@@ -357,4 +357,3 @@ func buildStorageRegistry() *storage.Registry {
storageRclone.RegisterAllBackends(registry)
return registry
}
server/internal/backint/protocol.go
+4 -4
View File
@@ -21,10 +21,10 @@ import (
type Function string
const (
FunctionBackup Function = "backup"
FunctionRestore Function = "restore"
FunctionInquire Function = "inquire"
FunctionDelete Function = "delete"
FunctionBackup Function = "backup"
FunctionRestore Function = "restore"
FunctionInquire Function = "inquire"
FunctionDelete Function = "delete"
)
// BackupRequest 是 BACKUP 操作的单条请求。
server/internal/backup/file_runner.go
+2 -2
View File
@@ -84,7 +84,7 @@ func (r *FileRunner) Run(_ context.Context, task TaskSpec, writer LogWriter) (*R
walkErr := filepath.Walk(sourcePath, func(currentPath string, currentInfo os.FileInfo, walkErr error) error {
if walkErr != nil {
writer.WriteLine(fmt.Sprintf("⚠ 无法访问 %s: %v", currentPath, walkErr))
writer.WriteLine(fmt.Sprintf("[WARN] 无法访问 %s: %v", currentPath, walkErr))
return nil
}
relPath, err := filepath.Rel(baseParent, currentPath)
@@ -115,7 +115,7 @@ func (r *FileRunner) Run(_ context.Context, task TaskSpec, writer LogWriter) (*R
if currentInfo.IsDir() {
dirCount++
writer.WriteLine(fmt.Sprintf("📁 进入目录 %s", archiveName))
writer.WriteLine(fmt.Sprintf("[DIR] 进入目录 %s", archiveName))
}
header, err := tar.FileInfoHeader(currentInfo, "")
server/internal/backup/file_runner_diff_test.go
+1 -1
View File
@@ -160,7 +160,7 @@ func TestFileRunnerSelectiveRestore(t *testing.T) {
}
diffAssertContent(t, filepath.Join(restoreSrc, "a.txt"), "alpha")
diffAssertContent(t, filepath.Join(restoreSrc, "sub", "c.txt"), "charlie") // 选中目录 → 子项一并恢复
diffAssertAbsent(t, filepath.Join(restoreSrc, "b.txt")) // 未选中 → 不恢复
diffAssertAbsent(t, filepath.Join(restoreSrc, "b.txt")) // 未选中 → 不恢复
}
// TestFileRunnerDifferentialWithoutBaseIsFull 验证无基线时差异请求回退为全量（产出清单、含全部文件）。
server/internal/backup/mysql_runner.go
-1
View File
@@ -160,4 +160,3 @@ func formatFileSize(size int64) string {
return fmt.Sprintf("%d B", size)
}
}
server/internal/backup/repository.go
+1356
View File
File diff suppressed because it is too large Load Diff
server/internal/backup/repository_chunker.go
+131
View File
@@ -0,0 +1,131 @@
package backup
import (
"context"
"fmt"
"io"
)
const (
repositoryChunkMin = 512 << 10
repositoryChunkAvg = 1 << 20
repositoryChunkMax = 4 << 20
)
// contentDefinedChunker implements the normalized FastCDC cut-point strategy.
// The rolling Gear hash only retains the latest 64 bytes through uint64
// overflow, so chunk boundaries re-synchronize after insertions or deletions.
type contentDefinedChunker struct {
minSize int
avgSize int
maxSize int
smallMask uint64
largeMask uint64
gear [256]uint64
}
func newContentDefinedChunker() *contentDefinedChunker {
chunker := &contentDefinedChunker{
minSize: repositoryChunkMin,
avgSize: repositoryChunkAvg,
maxSize: repositoryChunkMax,
smallMask: (1 << 21) - 1,
largeMask: (1 << 19) - 1,
}
// SplitMix64 produces a stable, well-distributed Gear table. The seed and
// generation algorithm are part of repository format v1 and must not change.
seed := uint64(0x6a09e667f3bcc909)
for i := range chunker.gear {
seed += 0x9e3779b97f4a7c15
value := seed
value = (value ^ (value >> 30)) * 0xbf58476d1ce4e5b9
value = (value ^ (value >> 27)) * 0x94d049bb133111eb
chunker.gear[i] = value ^ (value >> 31)
}
return chunker
}
func (c *contentDefinedChunker) Split(ctx context.Context, reader io.Reader, emit func([]byte) error) error {
if reader == nil || emit == nil {
return fmt.Errorf("chunk reader and emitter are required")
}
pending := make([]byte, 0, c.maxSize+(256<<10))
readBuffer := make([]byte, 256<<10)
eof := false
for {
if !eof {
readCount, readErr := reader.Read(readBuffer)
if readCount > 0 {
pending = append(pending, readBuffer[:readCount]...)
}
switch readErr {
case nil:
case io.EOF:
eof = true
default:
return fmt.Errorf("read source for chunking: %w", readErr)
}
if readCount == 0 && readErr == nil {
continue
}
}
for len(pending) > 0 {
if err := ctx.Err(); err != nil {
return err
}
cut := c.findCutPoint(pending, eof)
if cut == 0 {
break
}
chunk := make([]byte, cut)
copy(chunk, pending[:cut])
if err := emit(chunk); err != nil {
return err
}
copy(pending, pending[cut:])
pending = pending[:len(pending)-cut]
}
if eof {
if len(pending) != 0 {
return fmt.Errorf("chunker stopped with %d buffered bytes", len(pending))
}
return nil
}
}
}
func (c *contentDefinedChunker) findCutPoint(data []byte, eof bool) int {
if len(data) < c.minSize {
if eof {
return len(data)
}
return 0
}
limit := len(data)
if limit > c.maxSize {
limit = c.maxSize
}
var hash uint64
for index := c.minSize; index < limit; index++ {
hash = (hash << 1) + c.gear[data[index]]
mask := c.largeMask
if index < c.avgSize {
mask = c.smallMask
}
if hash&mask == 0 {
return index + 1
}
}
if len(data) >= c.maxSize {
return c.maxSize
}
if eof {
return len(data)
}
return 0
}
server/internal/backup/repository_test.go
+414
View File
@@ -0,0 +1,414 @@
package backup
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"io"
"math/rand"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"testing"
"time"
"backupx/server/internal/storage"
)
func TestContentDefinedChunkerResynchronizesAfterInsertion(t *testing.T) {
source := make([]byte, 8<<20)
if _, err := rand.New(rand.NewSource(42)).Read(source); err != nil {
t.Fatalf("generate source: %v", err)
}
modified := make([]byte, 0, len(source)+4096)
modified = append(modified, source[:2<<20]...)
modified = append(modified, bytes.Repeat([]byte("inserted"), 512)...)
modified = append(modified, source[2<<20:]...)
chunker := newContentDefinedChunker()
collect := func(data []byte) map[string]struct{} {
t.Helper()
ids := make(map[string]struct{})
err := chunker.Split(context.Background(), bytes.NewReader(data), func(chunk []byte) error {
digest := sha256.Sum256(chunk)
ids[fmt.Sprintf("%x", digest[:])] = struct{}{}
if len(chunk) > repositoryChunkMax {
return fmt.Errorf("chunk exceeds maximum: %d", len(chunk))
}
return nil
})
if err != nil {
t.Fatalf("split chunks: %v", err)
}
return ids
}
originalChunks := collect(source)
modifiedChunks := collect(modified)
shared := 0
for chunkID := range originalChunks {
if _, ok := modifiedChunks[chunkID]; ok {
shared++
}
}
if shared < len(originalChunks)/2 {
t.Fatalf("content-defined boundaries did not resynchronize: shared=%d original=%d", shared, len(originalChunks))
}
}
func TestRepositoryRoundTripDedupAndPrune(t *testing.T) {
ctx := context.Background()
tempDir := t.TempDir()
sourceDir := filepath.Join(tempDir, "dataset")
if err := os.MkdirAll(filepath.Join(sourceDir, "empty"), 0o755); err != nil {
t.Fatalf("create source: %v", err)
}
original := make([]byte, 6<<20)
if _, err := rand.New(rand.NewSource(7)).Read(original); err != nil {
t.Fatalf("generate fixture: %v", err)
}
primaryPath := filepath.Join(sourceDir, "primary.bin")
duplicatePath := filepath.Join(sourceDir, "duplicate.bin")
if err := os.WriteFile(primaryPath, original, 0o640); err != nil {
t.Fatalf("write primary: %v", err)
}
if err := os.WriteFile(duplicatePath, original, 0o640); err != nil {
t.Fatalf("write duplicate: %v", err)
}
key := sha256.Sum256([]byte("repository-test-key"))
store := NewRepositoryStore(key[:])
provider := newMemoryRepositoryProvider()
task := TaskSpec{
ID: 12,
Name: "repository-test",
Type: "file",
SourcePaths: []string{sourceDir},
Compression: "zstd",
Encrypt: true,
StartedAt: time.Date(2026, 8, 6, 1, 2, 3, 0, time.UTC),
TempDir: tempDir,
}
firstPlan, err := store.BuildPlan(ctx, task, NopLogWriter{})
if err != nil {
t.Fatalf("build first plan: %v", err)
}
firstKey := store.SnapshotKey(task.ID, 1, task.StartedAt)
firstResult, err := store.Upload(ctx, provider, firstPlan, firstKey)
if closeErr := firstPlan.Close(); closeErr != nil {
t.Fatalf("close first plan: %v", closeErr)
}
if err != nil {
t.Fatalf("upload first snapshot: %v", err)
}
if firstResult.NewChunks == 0 || firstResult.UniqueChunks == 0 {
t.Fatalf("first upload did not create chunks: %+v", firstResult)
}
if firstPlan.UniqueSize >= firstPlan.LogicalSize {
t.Fatalf("duplicate file was not deduplicated within plan: unique=%d logical=%d", firstPlan.UniqueSize, firstPlan.LogicalSize)
}
modified := make([]byte, 0, len(original)+4096)
modified = append(modified, original[:2<<20]...)
modified = append(modified, bytes.Repeat([]byte("changed!"), 512)...)
modified = append(modified, original[2<<20:]...)
if err := os.WriteFile(primaryPath, modified, 0o640); err != nil {
t.Fatalf("modify primary: %v", err)
}
task.StartedAt = task.StartedAt.Add(time.Hour)
secondPlan, err := store.BuildPlan(ctx, task, NopLogWriter{})
if err != nil {
t.Fatalf("build second plan: %v", err)
}
secondKey := store.SnapshotKey(task.ID, 2, task.StartedAt)
secondResult, err := store.Upload(ctx, provider, secondPlan, secondKey)
if closeErr := secondPlan.Close(); closeErr != nil {
t.Fatalf("close second plan: %v", closeErr)
}
if err != nil {
t.Fatalf("upload second snapshot: %v", err)
}
if secondResult.ReusedBytes <= secondResult.LogicalSize/3 {
t.Fatalf("second snapshot reused too little data: %+v", secondResult)
}
if secondResult.UploadedBytes >= secondResult.LogicalSize {
t.Fatalf("incremental upload was not smaller than logical data: %+v", secondResult)
}
verify, err := store.Verify(ctx, provider, secondKey, secondResult.Checksum)
if err != nil {
t.Fatalf("verify repository: %v", err)
}
if verify.Chunks == 0 || verify.Bytes == 0 {
t.Fatalf("empty verification result: %+v", verify)
}
restoreRoot := filepath.Join(tempDir, "restore")
restoreTask := task
restoreTask.RestoreTargetPath = restoreRoot
if err := store.Restore(ctx, provider, secondKey, strings.Repeat("0", sha256.Size*2), restoreTask, NopLogWriter{}); err == nil {
t.Fatal("restore accepted a mismatched snapshot checksum")
}
if err := store.Restore(ctx, provider, secondKey, "", restoreTask, NopLogWriter{}); err == nil {
t.Fatal("restore accepted a missing snapshot checksum")
}
if err := store.Restore(ctx, provider, secondKey, secondResult.Checksum, restoreTask, NopLogWriter{}); err != nil {
t.Fatalf("restore snapshot: %v", err)
}
restored, err := os.ReadFile(filepath.Join(restoreRoot, filepath.Base(sourceDir), "primary.bin"))
if err != nil {
t.Fatalf("read restored primary: %v", err)
}
if !bytes.Equal(restored, modified) {
t.Fatalf("restored primary differs from source")
}
if info, err := os.Stat(filepath.Join(restoreRoot, filepath.Base(sourceDir), "empty")); err != nil || !info.IsDir() {
t.Fatalf("empty directory was not restored: info=%v err=%v", info, err)
}
if err := provider.Delete(ctx, firstKey); err != nil {
t.Fatalf("delete first snapshot: %v", err)
}
if _, err := store.Prune(ctx, provider); err != nil {
t.Fatalf("prune with live snapshot: %v", err)
}
if err := provider.Delete(ctx, secondKey); err != nil {
t.Fatalf("delete second snapshot: %v", err)
}
pruned, err := store.Prune(ctx, provider)
if err != nil {
t.Fatalf("prune empty repository: %v", err)
}
if pruned.DeletedPacks == 0 || pruned.DeletedIndexes == 0 {
t.Fatalf("prune did not reclaim repository data: %+v", pruned)
}
if objects, err := provider.List(ctx, repositoryPackPrefix); err != nil || len(objects) != 0 {
t.Fatalf("packs remain after prune: objects=%v err=%v", objects, err)
}
}
func TestRepositoryRestoreRejectsUnsafeSnapshotMetadata(t *testing.T) {
cases := []struct {
name string
entries []repositoryEntry
}{
{
name: "path traversal",
entries: []repositoryEntry{{Path: "../escape", Kind: "directory", Mode: 0o755}},
},
{
name: "entry below symlink",
entries: []repositoryEntry{
{Path: "link", Kind: "symlink", Mode: 0o777, LinkTarget: "inside"},
{Path: "link/payload", Kind: "file", Mode: 0o600, Size: 1, Chunks: []string{"p-" + strings.Repeat("0", sha256.Size*2)}},
},
},
{
name: "escaping symlink target",
entries: []repositoryEntry{{Path: "escape", Kind: "symlink", Mode: 0o777, LinkTarget: "../outside"}},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
store := NewRepositoryStore(nil)
provider := newMemoryRepositoryProvider()
snapshot := repositorySnapshot{
Version: repositoryFormatVersion,
TaskID: 1,
CreatedAt: time.Now().UTC(),
Compression: "none",
Entries: tc.entries,
}
data, err := store.encodeSnapshot(snapshot)
if err != nil {
t.Fatalf("encodeSnapshot returned error: %v", err)
}
key := store.SnapshotKey(1, 1, snapshot.CreatedAt)
if err := provider.Upload(ctx, key, bytes.NewReader(data), int64(len(data)), nil); err != nil {
t.Fatalf("Upload snapshot returned error: %v", err)
}
digest := sha256.Sum256(data)
task := TaskSpec{SourcePath: filepath.Join(t.TempDir(), "source"), RestoreTargetPath: filepath.Join(t.TempDir(), "restore")}
if err := store.Restore(ctx, provider, key, fmt.Sprintf("%x", digest[:]), task, NopLogWriter{}); err == nil {
t.Fatal("restore accepted unsafe snapshot metadata")
}
})
}
}
func TestRepositoryRestorePreservesDirectoryWhenSnapshotContainsSymlink(t *testing.T) {
ctx := context.Background()
store := NewRepositoryStore(nil)
provider := newMemoryRepositoryProvider()
snapshot := repositorySnapshot{
Version: repositoryFormatVersion,
TaskID: 1,
CreatedAt: time.Now().UTC(),
Compression: "none",
Entries: []repositoryEntry{{Path: "link", Kind: "symlink", Mode: 0o777, LinkTarget: "inside"}},
}
data, err := store.encodeSnapshot(snapshot)
if err != nil {
t.Fatalf("encodeSnapshot returned error: %v", err)
}
key := store.SnapshotKey(1, 1, snapshot.CreatedAt)
if err := provider.Upload(ctx, key, bytes.NewReader(data), int64(len(data)), nil); err != nil {
t.Fatalf("Upload snapshot returned error: %v", err)
}
digest := sha256.Sum256(data)
restoreRoot := filepath.Join(t.TempDir(), "restore")
markerPath := filepath.Join(restoreRoot, "link", "keep.txt")
if err := os.MkdirAll(filepath.Dir(markerPath), 0o755); err != nil {
t.Fatalf("MkdirAll marker parent: %v", err)
}
if err := os.WriteFile(markerPath, []byte("keep"), 0o600); err != nil {
t.Fatalf("WriteFile marker: %v", err)
}
task := TaskSpec{SourcePath: filepath.Join(t.TempDir(), "source"), RestoreTargetPath: restoreRoot}
if err := store.Restore(ctx, provider, key, fmt.Sprintf("%x", digest[:]), task, NopLogWriter{}); err == nil {
t.Fatal("restore replaced an existing directory with a symlink")
}
if data, err := os.ReadFile(markerPath); err != nil || string(data) != "keep" {
t.Fatalf("existing directory content changed: data=%q err=%v", data, err)
}
}
func TestRepositoryRejectsOversizedChunkLocation(t *testing.T) {
ctx := context.Background()
store := NewRepositoryStore(nil)
provider := newMemoryRepositoryProvider()
chunkID := "p-" + strings.Repeat("0", sha256.Size*2)
packID := strings.Repeat("a", sha256.Size*2)
segment := repositoryIndexSegment{
Version: repositoryFormatVersion,
Pack: fmt.Sprintf("%s/%s/%s.pack", repositoryPackPrefix, packID[:2], packID),
Chunks: map[string]repositoryChunkLocation{
chunkID: {Pack: fmt.Sprintf("%s/%s/%s.pack", repositoryPackPrefix, packID[:2], packID), Offset: 0, Length: repositoryMaxEncoded + 1, PlainSize: 1, Compression: "none"},
},
}
data, err := json.Marshal(segment)
if err != nil {
t.Fatalf("Marshal index returned error: %v", err)
}
indexKey := fmt.Sprintf("%s/%s.json", repositoryIndexPrefix, packID)
if err := provider.Upload(ctx, indexKey, bytes.NewReader(data), int64(len(data)), nil); err != nil {
t.Fatalf("Upload index returned error: %v", err)
}
if _, err := store.loadIndex(ctx, provider); err == nil {
t.Fatal("loadIndex accepted an oversized encoded chunk")
}
}
func TestRepositoryRejectsIndexPackMismatch(t *testing.T) {
ctx := context.Background()
store := NewRepositoryStore(nil)
provider := newMemoryRepositoryProvider()
chunkID := "p-" + strings.Repeat("0", sha256.Size*2)
indexID := strings.Repeat("a", sha256.Size*2)
otherPackID := strings.Repeat("b", sha256.Size*2)
expectedPack := fmt.Sprintf("%s/%s/%s.pack", repositoryPackPrefix, indexID[:2], indexID)
segment := repositoryIndexSegment{
Version: repositoryFormatVersion,
Pack: expectedPack,
Chunks: map[string]repositoryChunkLocation{
chunkID: {
Pack: fmt.Sprintf("%s/%s/%s.pack", repositoryPackPrefix, otherPackID[:2], otherPackID),
Offset: 0,
Length: 1,
PlainSize: 1,
Compression: "none",
},
},
}
data, err := json.Marshal(segment)
if err != nil {
t.Fatalf("Marshal index returned error: %v", err)
}
indexKey := fmt.Sprintf("%s/%s.json", repositoryIndexPrefix, indexID)
if err := provider.Upload(ctx, indexKey, bytes.NewReader(data), int64(len(data)), nil); err != nil {
t.Fatalf("Upload index returned error: %v", err)
}
if _, err := store.loadIndex(ctx, provider); err == nil {
t.Fatal("loadIndex accepted a chunk location pointing to a different pack")
}
}
type memoryRepositoryProvider struct {
mu sync.RWMutex
objects map[string][]byte
times map[string]time.Time
}
func newMemoryRepositoryProvider() *memoryRepositoryProvider {
return &memoryRepositoryProvider{objects: make(map[string][]byte), times: make(map[string]time.Time)}
}
func (p *memoryRepositoryProvider) Type() storage.ProviderType { return "memory" }
func (p *memoryRepositoryProvider) TestConnection(context.Context) error { return nil }
func (p *memoryRepositoryProvider) Upload(_ context.Context, key string, reader io.Reader, size int64, _ map[string]string) error {
data, err := io.ReadAll(reader)
if err != nil {
return err
}
if int64(len(data)) != size {
return fmt.Errorf("size mismatch for %s: %d != %d", key, len(data), size)
}
p.mu.Lock()
defer p.mu.Unlock()
p.objects[key] = append([]byte(nil), data...)
p.times[key] = time.Now().UTC()
return nil
}
func (p *memoryRepositoryProvider) Download(_ context.Context, key string) (io.ReadCloser, error) {
p.mu.RLock()
defer p.mu.RUnlock()
data, ok := p.objects[key]
if !ok {
return nil, fmt.Errorf("object %s not found", key)
}
return io.NopCloser(bytes.NewReader(append([]byte(nil), data...))), nil
}
func (p *memoryRepositoryProvider) DownloadRange(_ context.Context, key string, offset, length int64) (io.ReadCloser, error) {
p.mu.RLock()
defer p.mu.RUnlock()
data, ok := p.objects[key]
if !ok {
return nil, fmt.Errorf("object %s not found", key)
}
if offset < 0 || length <= 0 || offset+length > int64(len(data)) {
return nil, fmt.Errorf("invalid range %d:%d for %s", offset, length, key)
}
return io.NopCloser(bytes.NewReader(append([]byte(nil), data[offset:offset+length]...))), nil
}
func (p *memoryRepositoryProvider) Delete(_ context.Context, key string) error {
p.mu.Lock()
defer p.mu.Unlock()
if _, ok := p.objects[key]; !ok {
return fmt.Errorf("object %s not found", key)
}
delete(p.objects, key)
delete(p.times, key)
return nil
}
func (p *memoryRepositoryProvider) List(_ context.Context, prefix string) ([]storage.ObjectInfo, error) {
p.mu.RLock()
defer p.mu.RUnlock()
result := make([]storage.ObjectInfo, 0)
for key, data := range p.objects {
if strings.HasPrefix(key, prefix) {
result = append(result, storage.ObjectInfo{Key: key, Size: int64(len(data)), UpdatedAt: p.times[key]})
}
}
sort.Slice(result, func(i, j int) bool { return result[i].Key < result[j].Key })
return result, nil
}
server/internal/backup/retention/service.go
+113 -12
View File
@@ -2,11 +2,13 @@ package retention
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"backupx/server/internal/backup"
"backupx/server/internal/model"
"backupx/server/internal/repository"
"backupx/server/internal/storage"
@@ -40,16 +42,48 @@ type CleanupResult struct {
Warnings []string
}
type Service struct {
records repository.BackupRecordRepository
now func() time.Time
type cleanupObject struct {
targetID uint
path string
}
func NewService(records repository.BackupRecordRepository) *Service {
return &Service{records: records, now: func() time.Time { return time.Now().UTC() }}
type storedUploadResult struct {
StorageTargetID uint `json:"storageTargetId"`
Status string `json:"status"`
StoragePath string `json:"storagePath"`
}
type Service struct {
records repository.BackupRecordRepository
repositoryKey []byte
now func() time.Time
}
func NewService(records repository.BackupRecordRepository, repositoryKey ...[]byte) *Service {
var key []byte
if len(repositoryKey) > 0 {
key = append([]byte(nil), repositoryKey[0]...)
}
return &Service{records: records, repositoryKey: key, now: func() time.Time { return time.Now().UTC() }}
}
func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider storage.StorageProvider) (*CleanupResult, error) {
return s.cleanup(ctx, task, func(uint) (storage.StorageProvider, bool) {
return provider, provider != nil
})
}
// CleanupProviders applies one retention decision to every successful copy of
// a record before deleting its database row. This prevents multi-target tasks
// from leaving stale objects after the first target removes the shared record.
func (s *Service) CleanupProviders(ctx context.Context, task *model.BackupTask, providers map[uint]storage.StorageProvider) (*CleanupResult, error) {
return s.cleanup(ctx, task, func(targetID uint) (storage.StorageProvider, bool) {
provider, ok := providers[targetID]
return provider, ok && provider != nil
})
}
func (s *Service) cleanup(ctx context.Context, task *model.BackupTask, resolveProvider func(uint) (storage.StorageProvider, bool)) (*CleanupResult, error) {
if task == nil {
return nil, fmt.Errorf("backup task is required")
}
@@ -67,17 +101,35 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
// 差异链保护：保留仍被存活差异依赖的全量，避免删除基线后差异无法恢复。
candidates = protectDifferentialBases(records, candidates)
result := &CleanupResult{}
repositoryProviders := make(map[uint]storage.StorageProvider)
touchedProviders := make(map[uint]storage.StorageProvider)
for _, record := range candidates {
if strings.TrimSpace(record.StoragePath) != "" {
if provider == nil {
result.Warnings = append(result.Warnings, fmt.Sprintf("record %d missing storage provider for cleanup", record.ID))
objects, objectErr := cleanupObjectsForRecord(record)
if objectErr != nil {
result.Warnings = append(result.Warnings, fmt.Sprintf("decode storage copies for record %d failed: %v", record.ID, objectErr))
continue
}
allObjectsDeleted := true
for _, object := range objects {
provider, ok := resolveProvider(object.targetID)
if !ok {
result.Warnings = append(result.Warnings, fmt.Sprintf("record %d missing storage provider %d for cleanup", record.ID, object.targetID))
allObjectsDeleted = false
continue
}
if err := provider.Delete(ctx, record.StoragePath); err != nil {
result.Warnings = append(result.Warnings, fmt.Sprintf("delete storage object %s failed: %v", record.StoragePath, err))
if err := provider.Delete(ctx, object.path); err != nil {
result.Warnings = append(result.Warnings, fmt.Sprintf("delete storage object %s from target %d failed: %v", object.path, object.targetID, err))
allObjectsDeleted = false
continue
}
result.DeletedObjects++
touchedProviders[object.targetID] = provider
if record.BackupKind == model.BackupKindRepository {
repositoryProviders[object.targetID] = provider
}
}
if !allObjectsDeleted {
continue
}
if err := s.records.Delete(ctx, record.ID); err != nil {
result.Warnings = append(result.Warnings, fmt.Sprintf("delete backup record %d failed: %v", record.ID, err))
@@ -85,13 +137,25 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
}
result.DeletedRecords++
}
for targetID, provider := range repositoryProviders {
pruned, pruneErr := backup.NewRepositoryStore(s.repositoryKey).Prune(ctx, provider)
if pruneErr != nil {
result.Warnings = append(result.Warnings, fmt.Sprintf("prune CDC repository on target %d failed: %v", targetID, pruneErr))
} else {
result.DeletedObjects += pruned.DeletedPacks + pruned.DeletedIndexes
}
}
// 清理空目录：收集被删除文件的父目录，尝试移除空目录
if dirCleaner, ok := provider.(storage.StorageDirCleaner); ok && result.DeletedObjects > 0 {
for targetID, provider := range touchedProviders {
dirCleaner, ok := provider.(storage.StorageDirCleaner)
if !ok {
continue
}
prefixes := collectDirPrefixes(candidates)
for _, prefix := range prefixes {
if err := dirCleaner.RemoveEmptyDirs(ctx, prefix); err != nil {
result.Warnings = append(result.Warnings, fmt.Sprintf("cleanup empty dirs for %s: %v", prefix, err))
result.Warnings = append(result.Warnings, fmt.Sprintf("cleanup empty dirs for %s on target %d: %v", prefix, targetID, err))
}
}
}
@@ -99,6 +163,43 @@ func (s *Service) Cleanup(ctx context.Context, task *model.BackupTask, provider
return result, nil
}
func cleanupObjectsForRecord(record model.BackupRecord) ([]cleanupObject, error) {
defaultPath := strings.TrimSpace(record.StoragePath)
if strings.TrimSpace(record.StorageUploadResults) == "" {
if defaultPath == "" {
return nil, nil
}
return []cleanupObject{{targetID: record.StorageTargetID, path: defaultPath}}, nil
}
var results []storedUploadResult
if err := json.Unmarshal([]byte(record.StorageUploadResults), &results); err != nil {
return nil, err
}
objects := make([]cleanupObject, 0, len(results))
seen := make(map[uint]struct{}, len(results))
for _, result := range results {
if !strings.EqualFold(strings.TrimSpace(result.Status), model.BackupRecordStatusSuccess) {
continue
}
objectPath := strings.TrimSpace(result.StoragePath)
if objectPath == "" {
objectPath = defaultPath
}
if objectPath == "" {
continue
}
if _, exists := seen[result.StorageTargetID]; exists {
continue
}
seen[result.StorageTargetID] = struct{}{}
objects = append(objects, cleanupObject{targetID: result.StorageTargetID, path: objectPath})
}
if len(objects) == 0 && defaultPath != "" {
return nil, fmt.Errorf("successful record has no successful storage copy")
}
return objects, nil
}
// protectDifferentialBases 从删除候选中剔除「仍被存活差异依赖的全量」，
// 避免删除基线后其差异备份失去依据、无法恢复。全量仅当其全部差异都已过期/删除时才会被清理。
func protectDifferentialBases(all []model.BackupRecord, candidates []model.BackupRecord) []model.BackupRecord {
server/internal/backup/retention/service_test.go
+63
View File
@@ -221,3 +221,66 @@ func TestCleanupDeletesExpiredRecords(t *testing.T) {
t.Fatalf("unexpected deleted objects: %#v", provider.deleted)
}
}
func TestCleanupProvidersDeletesEverySuccessfulCopyBeforeRecord(t *testing.T) {
now := time.Date(2026, 3, 7, 16, 0, 0, 0, time.UTC)
completedNew := now.Add(-time.Hour)
completedOld := now.Add(-24 * time.Hour)
repo := &fakeRecordRepository{records: []model.BackupRecord{
{ID: 2, TaskID: 1, StoragePath: "records/2", Status: model.BackupRecordStatusSuccess, CompletedAt: &completedNew},
{
ID: 1, TaskID: 1, StoragePath: "records/1", Status: model.BackupRecordStatusSuccess, CompletedAt: &completedOld,
StorageUploadResults: `[{"storageTargetId":11,"status":"success","storagePath":"first/1"},{"storageTargetId":12,"status":"success","storagePath":"second/1"},{"storageTargetId":13,"status":"failed"}]`,
},
}}
first := &fakeProvider{}
second := &fakeProvider{}
service := NewService(repo)
service.now = func() time.Time { return now }
result, err := service.CleanupProviders(context.Background(), &model.BackupTask{ID: 1, MaxBackups: 1}, map[uint]storage.StorageProvider{
11: first,
12: second,
})
if err != nil {
t.Fatalf("CleanupProviders returned error: %v", err)
}
if result.DeletedRecords != 1 || result.DeletedObjects != 2 || len(result.Warnings) != 0 {
t.Fatalf("unexpected cleanup result: %#v", result)
}
if len(repo.deleted) != 1 || repo.deleted[0] != 1 {
t.Fatalf("unexpected deleted records: %#v", repo.deleted)
}
if len(first.deleted) != 1 || first.deleted[0] != "first/1" {
t.Fatalf("unexpected first-target deletes: %#v", first.deleted)
}
if len(second.deleted) != 1 || second.deleted[0] != "second/1" {
t.Fatalf("unexpected second-target deletes: %#v", second.deleted)
}
}
func TestCleanupProvidersKeepsRecordWhenCopyProviderIsUnavailable(t *testing.T) {
now := time.Date(2026, 3, 7, 16, 0, 0, 0, time.UTC)
completedNew := now.Add(-time.Hour)
completedOld := now.Add(-24 * time.Hour)
repo := &fakeRecordRepository{records: []model.BackupRecord{
{ID: 2, TaskID: 1, StoragePath: "records/2", Status: model.BackupRecordStatusSuccess, CompletedAt: &completedNew},
{
ID: 1, TaskID: 1, StoragePath: "records/1", Status: model.BackupRecordStatusSuccess, CompletedAt: &completedOld,
StorageUploadResults: `[{"storageTargetId":11,"status":"success","storagePath":"first/1"},{"storageTargetId":12,"status":"success","storagePath":"second/1"}]`,
},
}}
first := &fakeProvider{}
service := NewService(repo)
result, err := service.CleanupProviders(context.Background(), &model.BackupTask{ID: 1, MaxBackups: 1}, map[uint]storage.StorageProvider{11: first})
if err != nil {
t.Fatalf("CleanupProviders returned error: %v", err)
}
if result.DeletedRecords != 0 || result.DeletedObjects != 1 || len(result.Warnings) != 1 {
t.Fatalf("unexpected safe partial-cleanup result: %#v", result)
}
if len(repo.deleted) != 0 {
t.Fatalf("record must remain until all copies are deleted: %#v", repo.deleted)
}
}
server/internal/backup/temp_files.go
+11
View File
@@ -55,6 +55,17 @@ func BuildStorageKey(backupType string, startedAt time.Time, fileName string) st
return filepath.ToSlash(filepath.Join("BackupX", typeName, startedAt.UTC().Format("060102"), fileName))
}
// BuildRecordStorageKey gives remote-Agent artifacts a record-owned namespace.
// The Master validates this namespace before accepting a relayed upload, so one
// Agent cannot overwrite another record's object on centrally mounted storage.
func BuildRecordStorageKey(backupType string, startedAt time.Time, recordID uint, fileName string) string {
typeName := strings.TrimSpace(strings.ToLower(backupType))
if typeName == "" {
typeName = "file"
}
return filepath.ToSlash(filepath.Join("BackupX", typeName, startedAt.UTC().Format("060102"), "records", fmt.Sprintf("%d", recordID), fileName))
}
func sanitizeTaskName(value string) string {
trimmed := strings.TrimSpace(strings.ToLower(value))
trimmed = strings.ReplaceAll(trimmed, " ", "-")
server/internal/backup/verify_test.go
+2 -2
View File
@@ -32,8 +32,8 @@ func writeTestTar(t *testing.T, entries map[string][]byte) string {
func TestVerifyTarArchive_Valid(t *testing.T) {
path := writeTestTar(t, map[string][]byte{
"readme.md": []byte("hello"),
"data.bin": []byte("world!!!"),
"readme.md": []byte("hello"),
"data.bin": []byte("world!!!"),
})
report, err := VerifyTarArchive(path, "")
if err != nil {
server/internal/backup/window.go
+5 -4
View File
@@ -24,10 +24,11 @@ type MaintenanceWindow struct {
// 简化语法：多个窗口以 ';' 分隔，每个窗口按 "[days=xxx;]time=HH:MM-HH:MM" 格式。
// Days 缺省 = 全周；若不合法，跳过该段而非抛错（让调用方尽力工作）。
// 示例：
// "time=01:00-05:00" 每天 1 点到 5 点
// "days=sat,sun;time=00:00-23:59" 仅周末全天
// "time=22:00-06:00" 每天跨夜
// "days=mon,tue,wed,thu,fri;time=22:00-06:00" 工作日跨夜
//
// "time=01:00-05:00" 每天 1 点到 5 点
// "days=sat,sun;time=00:00-23:59" 仅周末全天
// "time=22:00-06:00" 每天跨夜
// "days=mon,tue,wed,thu,fri;time=22:00-06:00" 工作日跨夜
func ParseMaintenanceWindows(value string) []MaintenanceWindow {
v := strings.TrimSpace(value)
if v == "" {
server/internal/config/config.go
+27
View File
@@ -2,6 +2,8 @@ package config
import (
"fmt"
"net"
"net/url"
"strings"
"time"
@@ -21,6 +23,9 @@ type ServerConfig struct {
Port int `mapstructure:"port"`
Mode string `mapstructure:"mode"`
ExternalURL string `mapstructure:"external_url"`
// TrustedProxies 限定可提供 X-Forwarded-For 等头部的反向代理地址。
// 默认仅信任本机代理；空列表表示不信任任何代理头。
TrustedProxies []string `mapstructure:"trusted_proxies"`
// WebRoot 指向前端构建产物目录。留空时后端会按部署惯例自动探测
// （./web、./web/dist、/opt/backupx/web 等）。探测命中后后端直接托管
// 前端 SPA，无需额外的 nginx 反向代理即可访问 Web 控制台。
@@ -91,6 +96,25 @@ func Load(configPath string) (Config, error) {
if cfg.Server.Mode == "" {
cfg.Server.Mode = "release"
}
cfg.Server.ExternalURL = strings.TrimRight(strings.TrimSpace(cfg.Server.ExternalURL), "/")
if cfg.Server.ExternalURL != "" {
externalURL, parseErr := url.Parse(cfg.Server.ExternalURL)
if parseErr != nil || (externalURL.Scheme != "http" && externalURL.Scheme != "https") || externalURL.Host == "" || externalURL.User != nil || externalURL.RawQuery != "" || externalURL.Fragment != "" {
return Config{}, fmt.Errorf("server.external_url must be an absolute http(s) URL without credentials, query or fragment")
}
}
if len(cfg.Server.TrustedProxies) == 1 && strings.Contains(cfg.Server.TrustedProxies[0], ",") {
cfg.Server.TrustedProxies = strings.Split(cfg.Server.TrustedProxies[0], ",")
}
for index := range cfg.Server.TrustedProxies {
proxy := strings.TrimSpace(cfg.Server.TrustedProxies[index])
cfg.Server.TrustedProxies[index] = proxy
if net.ParseIP(proxy) == nil {
if _, _, parseErr := net.ParseCIDR(proxy); parseErr != nil {
return Config{}, fmt.Errorf("server.trusted_proxies contains invalid IP or CIDR %q", proxy)
}
}
}
if cfg.Database.Path == "" {
cfg.Database.Path = "./data/backupx.db"
}
@@ -142,9 +166,12 @@ func applyDefaults(v *viper.Viper) {
v.SetDefault("server.port", 8340)
v.SetDefault("server.mode", "release")
v.SetDefault("server.external_url", "")
v.SetDefault("server.trusted_proxies", []string{"127.0.0.1", "::1"})
v.SetDefault("server.web_root", "")
v.SetDefault("database.path", "./data/backupx.db")
v.SetDefault("security.jwt_secret", "")
v.SetDefault("security.jwt_expire", "24h")
v.SetDefault("security.encryption_key", "")
v.SetDefault("backup.temp_dir", "/tmp/backupx")
v.SetDefault("backup.max_concurrent", 2)
v.SetDefault("backup.retries", 10)
server/internal/config/config_test.go
+60
View File
@@ -21,6 +21,25 @@ func TestLoadUsesDefaultsWithoutConfigFile(t *testing.T) {
if cfg.Database.Path != "./data/backupx.db" {
t.Fatalf("expected default database path, got %s", cfg.Database.Path)
}
if len(cfg.Server.TrustedProxies) != 2 {
t.Fatalf("expected loopback trusted proxies, got %#v", cfg.Server.TrustedProxies)
}
}
func TestLoadRejectsInvalidExternalURLAndTrustedProxy(t *testing.T) {
tests := []string{
"server:\n external_url: \"ssh://master.example.com\"\n",
"server:\n trusted_proxies: [\"not-an-ip\"]\n",
}
for _, content := range tests {
configPath := filepath.Join(t.TempDir(), "config.yaml")
if err := os.WriteFile(configPath, []byte(content), 0o600); err != nil {
t.Fatal(err)
}
if _, err := Load(configPath); err == nil {
t.Fatalf("expected invalid configuration to fail: %s", content)
}
}
}
func TestLoadReadsServerExternalURLFromFile(t *testing.T) {
@@ -52,3 +71,44 @@ func TestLoadReadsServerExternalURLFromEnv(t *testing.T) {
t.Fatalf("expected external URL from env, got %q", cfg.Server.ExternalURL)
}
}
func TestLoadReadsSecuritySecretsFromEnv(t *testing.T) {
t.Setenv("BACKUPX_SECURITY_JWT_SECRET", "test-jwt-secret")
t.Setenv("BACKUPX_SECURITY_ENCRYPTION_KEY", "test-encryption-key")
cfg, err := Load("")
if err != nil {
t.Fatalf("Load returned error: %v", err)
}
if cfg.Security.JWTSecret != "test-jwt-secret" {
t.Fatalf("expected JWT secret from env, got %q", cfg.Security.JWTSecret)
}
if cfg.Security.EncryptionKey != "test-encryption-key" {
t.Fatalf("expected encryption key from env, got %q", cfg.Security.EncryptionKey)
}
}
func TestLoadReadsTrustedProxiesFromEnv(t *testing.T) {
t.Setenv("BACKUPX_SERVER_TRUSTED_PROXIES", "127.0.0.1,172.18.0.0/16")
cfg, err := Load("")
if err != nil {
t.Fatal(err)
}
if len(cfg.Server.TrustedProxies) != 2 || cfg.Server.TrustedProxies[1] != "172.18.0.0/16" {
t.Fatalf("trusted proxies = %#v", cfg.Server.TrustedProxies)
}
}
func TestLoadAllowsTrustedProxiesToBeDisabled(t *testing.T) {
configPath := filepath.Join(t.TempDir(), "config.yaml")
if err := os.WriteFile(configPath, []byte("server:\n trusted_proxies: []\n"), 0o600); err != nil {
t.Fatal(err)
}
cfg, err := Load(configPath)
if err != nil {
t.Fatal(err)
}
if len(cfg.Server.TrustedProxies) != 0 {
t.Fatalf("trusted proxies should be disabled, got %#v", cfg.Server.TrustedProxies)
}
}
server/internal/database/database.go
+9 -1
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"path/filepath"
"strings"
"backupx/server/internal/config"
"backupx/server/internal/model"
@@ -18,7 +19,14 @@ func Open(cfg config.DatabaseConfig, logger *zap.Logger) (*gorm.DB, error) {
return nil, fmt.Errorf("create database dir: %w", err)
}
db, err := gorm.Open(sqlite.Open(cfg.Path), &gorm.Config{Logger: gormlogger.Default.LogMode(gormlogger.Silent)})
separator := "?"
if strings.Contains(cfg.Path, "?") {
separator = "&"
}
// busy_timeout 减少 Agent 轮询、心跳和任务写入同时发生时的瞬时锁错误。
// 维持默认回滚日志模式，保证当前嵌入式 SQLite 依赖的数据完整性。
dsn := cfg.Path + separator + "_pragma=busy_timeout(5000)"
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: gormlogger.Default.LogMode(gormlogger.Silent)})
if err != nil {
return nil, fmt.Errorf("open sqlite: %w", err)
}
server/internal/database/database_test.go
+40
View File
@@ -0,0 +1,40 @@
package database
import (
"path/filepath"
"testing"
"backupx/server/internal/config"
"backupx/server/internal/logger"
)
func TestOpenConfiguresSQLiteForSingleMasterConcurrency(t *testing.T) {
log, err := logger.New(config.LogConfig{Level: "error"})
if err != nil {
t.Fatal(err)
}
db, err := Open(config.DatabaseConfig{Path: filepath.Join(t.TempDir(), "backupx.db")}, log)
if err != nil {
t.Fatal(err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
var journalMode string
if err := db.Raw("PRAGMA journal_mode").Scan(&journalMode).Error; err != nil {
t.Fatal(err)
}
if journalMode != "delete" {
t.Fatalf("journal_mode = %q, want delete", journalMode)
}
var busyTimeout int
if err := db.Raw("PRAGMA busy_timeout").Scan(&busyTimeout).Error; err != nil {
t.Fatal(err)
}
if busyTimeout != 5000 {
t.Fatalf("busy_timeout = %d, want 5000", busyTimeout)
}
}
server/internal/http/agent_handler.go
+71 -5
View File
@@ -23,7 +23,7 @@ func NewAgentHandler(agentService *service.AgentService, nodeService *service.No
return &AgentHandler{agentService: agentService, nodeService: nodeService, restoreService: restoreService}
}
// extractToken 从请求头或 JSON body 中提取 Agent Token。
// extractToken 从认证请求头中提取 Agent Token。
func extractToken(c *gin.Context) string {
if t := strings.TrimSpace(c.GetHeader("X-Agent-Token")); t != "" {
return t
@@ -46,10 +46,10 @@ func (h *AgentHandler) Heartbeat(c *gin.Context) {
Arch string `json:"arch"`
}
_ = c.ShouldBindJSON(&input)
// token 优先走 body（向后兼容），否则从 header 读
token := input.Token
// 新版 Agent 只通过请求头发送 Token；JSON body 仅保留旧版本兼容。
token := extractToken(c)
if token == "" {
token = extractToken(c)
token = input.Token
}
if token == "" {
c.JSON(stdhttp.StatusBadRequest, gin.H{"code": "INVALID_INPUT", "message": "missing token"})
@@ -72,7 +72,7 @@ func (h *AgentHandler) Heartbeat(c *gin.Context) {
})
}
// Poll Agent 长轮询获取下一条待执行命令。
// Poll Agent 获取下一条待执行命令；Agent 按配置间隔主动轮询。
// 无命令时返回 {command: null}。
func (h *AgentHandler) Poll(c *gin.Context) {
node, err := h.agentService.AuthenticatedNode(c.Request.Context(), extractToken(c))
@@ -156,6 +156,44 @@ func (h *AgentHandler) UpdateRecord(c *gin.Context) {
response.Success(c, gin.H{"status": "ok"})
}
// UploadArtifact streams a remote source artifact into storage mounted only on
// the Master. The request body is never buffered as a whole in memory or disk.
func (h *AgentHandler) UploadArtifact(c *gin.Context) {
node, err := h.agentService.AuthenticatedNode(c.Request.Context(), extractToken(c))
if err != nil {
response.Error(c, err)
return
}
recordID, err := strconv.ParseUint(c.Param("id"), 10, 32)
if err != nil {
response.Error(c, err)
return
}
targetID, err := strconv.ParseUint(c.Param("targetId"), 10, 32)
if err != nil {
response.Error(c, err)
return
}
if c.Request.ContentLength < 0 {
c.JSON(stdhttp.StatusLengthRequired, gin.H{"code": "CONTENT_LENGTH_REQUIRED", "message": "artifact content length is required"})
return
}
if err := h.agentService.UploadArtifact(
c.Request.Context(),
node,
uint(recordID),
uint(targetID),
c.GetHeader("X-BackupX-Object-Key"),
c.Request.ContentLength,
c.GetHeader("X-BackupX-SHA256"),
c.Request.Body,
); err != nil {
response.Error(c, err)
return
}
response.Success(c, gin.H{"status": "ok"})
}
// GetRestoreSpec Agent 拉取恢复规格。
func (h *AgentHandler) GetRestoreSpec(c *gin.Context) {
if h.restoreService == nil {
@@ -208,6 +246,34 @@ func (h *AgentHandler) UpdateRestore(c *gin.Context) {
response.Success(c, gin.H{"status": "ok"})
}
// DownloadRestoreArtifact streams a Master-local backup back to its source
// Agent for restore without exposing the local storage configuration.
func (h *AgentHandler) DownloadRestoreArtifact(c *gin.Context) {
if h.restoreService == nil {
c.JSON(stdhttp.StatusServiceUnavailable, gin.H{"code": "RESTORE_SERVICE_DISABLED", "message": "restore service is not enabled"})
return
}
node, err := h.agentService.AuthenticatedNode(c.Request.Context(), extractToken(c))
if err != nil {
response.Error(c, err)
return
}
restoreID, err := strconv.ParseUint(c.Param("id"), 10, 32)
if err != nil {
response.Error(c, err)
return
}
artifact, err := h.restoreService.DownloadAgentArtifact(c.Request.Context(), node, uint(restoreID))
if err != nil {
response.Error(c, err)
return
}
c.DataFromReader(stdhttp.StatusOK, artifact.Size, "application/octet-stream", artifact.Reader, nil)
if err := artifact.Reader.Close(); err != nil {
_ = c.Error(err)
}
}
// Self 返回当前 Agent token 所属节点的状态，供安装脚本末尾探活。
func (h *AgentHandler) Self(c *gin.Context) {
node, err := h.agentService.AuthenticatedNode(c.Request.Context(), extractToken(c))
server/internal/http/forwarded_headers_test.go
+49
View File
@@ -0,0 +1,49 @@
package http
import (
stdhttp "net/http"
"net/http/httptest"
"testing"
"github.com/gin-gonic/gin"
)
func TestForwardedHeadersMiddlewareRejectsUntrustedHeaders(t *testing.T) {
gin.SetMode(gin.TestMode)
engine := gin.New()
engine.Use(ForwardedHeadersMiddleware([]string{"127.0.0.1", "10.0.0.0/8"}))
engine.GET("/master-url", func(c *gin.Context) {
c.String(stdhttp.StatusOK, resolveMasterURL(c, ""))
})
request := httptest.NewRequest(stdhttp.MethodGet, "http://master.example.com/master-url", nil)
request.RemoteAddr = "203.0.113.10:54321"
request.Header.Set("X-Forwarded-Host", "attacker.example.com")
request.Header.Set("X-Forwarded-Proto", "https")
recorder := httptest.NewRecorder()
engine.ServeHTTP(recorder, request)
if recorder.Body.String() != "http://master.example.com" {
t.Fatalf("untrusted forwarding headers changed URL: %q", recorder.Body.String())
}
}
func TestForwardedHeadersMiddlewareAcceptsTrustedProxy(t *testing.T) {
gin.SetMode(gin.TestMode)
engine := gin.New()
engine.Use(ForwardedHeadersMiddleware([]string{"10.0.0.0/8"}))
engine.GET("/master-url", func(c *gin.Context) {
c.String(stdhttp.StatusOK, resolveMasterURL(c, ""))
})
request := httptest.NewRequest(stdhttp.MethodGet, "http://backupx:8340/master-url", nil)
request.RemoteAddr = "10.10.0.5:43210"
request.Header.Set("X-Forwarded-Host", "backup.example.com")
request.Header.Set("X-Forwarded-Proto", "https")
recorder := httptest.NewRecorder()
engine.ServeHTTP(recorder, request)
if recorder.Body.String() != "https://backup.example.com" {
t.Fatalf("trusted forwarding headers were ignored: %q", recorder.Body.String())
}
}
server/internal/http/install_flow_test.go
+29 -15
View File
@@ -115,7 +115,7 @@ func setupInstallFlowRouterWithExternalURL(t *testing.T, externalURL string) (ht
return router, setupResp.Data.Token
}
func TestInstallTokenUsesConfiguredExternalURL(t *testing.T) {
func TestInstallTokenUsesAgentSpecificURLAndConnectionSettings(t *testing.T) {
const externalURL = "https://public.example.com/base"
router, jwt := setupInstallFlowRouterWithExternalURL(t, externalURL)
@@ -141,11 +141,14 @@ func TestInstallTokenUsesConfiguredExternalURL(t *testing.T) {
}
genBody, _ := json.Marshal(map[string]any{
"mode": "systemd",
"arch": "auto",
"agentVersion": "v1.7.0",
"downloadSrc": "github",
"ttlSeconds": 900,
"mode": "systemd",
"arch": "auto",
"agentVersion": "v1.7.0",
"downloadSrc": "github",
"ttlSeconds": 900,
"agentMasterUrl": "http://127.0.0.1:18340",
"proxyUrl": "socks5h://127.0.0.1:1080",
"caCertFile": "/etc/pki/internal-ca.pem",
})
genReq := httptest.NewRequest(http.MethodPost,
"/api/nodes/"+formatUint(batchResp.Data[0].ID)+"/install-tokens", bytes.NewBuffer(genBody))
@@ -156,6 +159,9 @@ func TestInstallTokenUsesConfiguredExternalURL(t *testing.T) {
if genRec.Code != 200 {
t.Fatalf("install-tokens failed: %d %s", genRec.Code, genRec.Body.String())
}
if genRec.Header().Get("Cache-Control") != "no-store" {
t.Fatalf("install-token response must not be cached: %#v", genRec.Header())
}
var genResp struct {
Data struct {
InstallToken string `json:"installToken"`
@@ -167,18 +173,21 @@ func TestInstallTokenUsesConfiguredExternalURL(t *testing.T) {
if err := json.Unmarshal(genRec.Body.Bytes(), &genResp); err != nil {
t.Fatalf("unmarshal gen: %v", err)
}
if genResp.Data.URL != externalURL+"/api/install/"+genResp.Data.InstallToken {
t.Fatalf("url should use external URL, got %q", genResp.Data.URL)
if genResp.Data.URL != "http://127.0.0.1:18340/api/install/"+genResp.Data.InstallToken {
t.Fatalf("url should use Agent-specific URL, got %q", genResp.Data.URL)
}
if genResp.Data.FallbackURL != externalURL+"/install/"+genResp.Data.InstallToken {
t.Fatalf("fallbackUrl should use external URL, got %q", genResp.Data.FallbackURL)
if genResp.Data.FallbackURL != "http://127.0.0.1:18340/install/"+genResp.Data.InstallToken {
t.Fatalf("fallbackUrl should use Agent-specific URL, got %q", genResp.Data.FallbackURL)
}
decodedScript, err := base64.StdEncoding.DecodeString(genResp.Data.ScriptBase64)
if err != nil {
t.Fatalf("scriptBase64 should be valid base64: %v", err)
}
if !strings.Contains(string(decodedScript), `MASTER_URL="`+externalURL+`"`) {
t.Fatalf("script should use external MASTER_URL:\n%s", string(decodedScript))
if !strings.Contains(string(decodedScript), `MASTER_URL="http://127.0.0.1:18340"`) {
t.Fatalf("script should use the Agent-specific Master URL:\n%s", string(decodedScript))
}
if !strings.Contains(string(decodedScript), `PROXY_URL="socks5h://127.0.0.1:1080"`) || !strings.Contains(string(decodedScript), `CA_CERT_FILE="/etc/pki/internal-ca.pem"`) {
t.Fatalf("script should include restricted-network settings:\n%s", string(decodedScript))
}
}
@@ -258,8 +267,9 @@ func TestOneClickInstallFlow(t *testing.T) {
if scriptRec.Code != 200 {
t.Fatalf("script fetch failed: %d %s", scriptRec.Code, scriptRec.Body.String())
}
if !strings.Contains(scriptRec.Body.String(), "systemctl enable --now backupx-agent") {
t.Fatalf("script missing systemctl enable:\n%s", scriptRec.Body.String())
if !strings.Contains(scriptRec.Body.String(), "systemctl enable backupx-agent") ||
!strings.Contains(scriptRec.Body.String(), "systemctl restart backupx-agent") {
t.Fatalf("script missing systemctl enable/restart:\n%s", scriptRec.Body.String())
}
// Issue #46 防嗅探 headers：text/plain + nosniff + no-store + Content-Disposition
if ct := scriptRec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
@@ -354,7 +364,8 @@ func TestInstallScriptAliasUnderAPI(t *testing.T) {
if aliasRec.Code != 200 {
t.Fatalf("/api/install alias failed: %d %s", aliasRec.Code, aliasRec.Body.String())
}
if !strings.Contains(aliasRec.Body.String(), "systemctl enable --now backupx-agent") {
if !strings.Contains(aliasRec.Body.String(), "systemctl enable backupx-agent") ||
!strings.Contains(aliasRec.Body.String(), "systemctl restart backupx-agent") {
t.Errorf("alias should return rendered script, got:\n%s", aliasRec.Body.String())
}
if ct := aliasRec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
@@ -557,6 +568,9 @@ func TestInstallFlowComposeSuccessConsumesToken(t *testing.T) {
if !strings.Contains(composeRec.Body.String(), "BACKUPX_AGENT_TOKEN") {
t.Fatalf("compose missing token env:\n%s", composeRec.Body.String())
}
if composeRec.Header().Get("Cache-Control") != "no-store" || composeRec.Header().Get("X-Content-Type-Options") != "nosniff" {
t.Fatalf("compose response missing secret-safe headers: %#v", composeRec.Header())
}
scriptReq := httptest.NewRequest(http.MethodGet, "/api/install/"+genResp.Data.InstallToken, nil)
scriptRec := httptest.NewRecorder()
server/internal/http/install_handler.go
+17 -2
View File
@@ -104,16 +104,22 @@ func (h *InstallHandler) Compose(c *gin.Context) {
}
h.recordConsumeAudit(c, consumed, "compose")
yaml, err := installscript.RenderComposeYaml(installscript.Context{
MasterURL: resolveMasterURL(c, h.externalURL),
MasterURL: installMasterURL(resolveMasterURL(c, h.externalURL), consumed.Record),
AgentToken: consumed.Node.Token,
AgentVersion: consumed.Record.AgentVer,
Mode: model.InstallModeDocker,
Arch: consumed.Record.Arch,
NodeID: consumed.Node.ID,
ProxyURL: consumed.Record.ProxyURL,
CACertFile: consumed.Record.CACertFile,
})
if err != nil {
c.String(stdhttp.StatusInternalServerError, "render error\n")
return
}
c.Header("X-Content-Type-Options", "nosniff")
c.Header("Cache-Control", "no-store")
c.Header("Content-Disposition", `attachment; filename="backupx-agent-compose.yml"`)
c.Data(stdhttp.StatusOK, "text/yaml; charset=utf-8", []byte(yaml))
}
@@ -134,7 +140,7 @@ func (h *InstallHandler) recordConsumeAudit(c *gin.Context, consumed *service.Co
func renderInstallScript(masterURL string, node *model.Node, record *model.AgentInstallToken) (string, error) {
return installscript.RenderScript(installscript.Context{
MasterURL: masterURL,
MasterURL: installMasterURL(masterURL, record),
AgentToken: node.Token,
AgentVersion: record.AgentVer,
Mode: record.Mode,
@@ -142,9 +148,18 @@ func renderInstallScript(masterURL string, node *model.Node, record *model.Agent
DownloadBase: installscript.DownloadBaseFor(record.DownloadSrc),
InstallPrefix: "/opt/backupx-agent",
NodeID: node.ID,
ProxyURL: record.ProxyURL,
CACertFile: record.CACertFile,
})
}
func installMasterURL(fallback string, record *model.AgentInstallToken) string {
if record != nil && strings.TrimSpace(record.AgentMasterURL) != "" {
return strings.TrimRight(strings.TrimSpace(record.AgentMasterURL), "/")
}
return fallback
}
// resolveMasterURL 按优先级推导 Master URL：外部配置 > X-Forwarded-* > Request.Host。
// 此为包级 helper，供 install_handler 和 node_handler 共用。
func resolveMasterURL(c *gin.Context, externalURL string) string {
server/internal/http/middleware.go
+41
View File
@@ -3,6 +3,7 @@ package http
import (
"context"
stdhttp "net/http"
"net/netip"
"strings"
"backupx/server/internal/apperror"
@@ -11,6 +12,46 @@ import (
"github.com/gin-gonic/gin"
)
// ForwardedHeadersMiddleware 只允许配置中的反向代理提供转发头。
// Gin 的 trusted_proxies 保护 ClientIP；这里同步保护安装命令使用的
// X-Forwarded-Host 与 X-Forwarded-Proto，避免直连请求伪造 Agent 地址。
func ForwardedHeadersMiddleware(trustedProxies []string) gin.HandlerFunc {
trustedPrefixes := make([]netip.Prefix, 0, len(trustedProxies))
for _, raw := range trustedProxies {
raw = strings.TrimSpace(raw)
if prefix, err := netip.ParsePrefix(raw); err == nil {
trustedPrefixes = append(trustedPrefixes, prefix)
continue
}
if addr, err := netip.ParseAddr(raw); err == nil {
trustedPrefixes = append(trustedPrefixes, netip.PrefixFrom(addr, addr.BitLen()))
}
}
return func(c *gin.Context) {
remote, err := netip.ParseAddrPort(c.Request.RemoteAddr)
trusted := false
if err == nil {
remoteAddr := remote.Addr().Unmap()
for _, prefix := range trustedPrefixes {
if prefix.Contains(remoteAddr) {
trusted = true
break
}
}
}
if !trusted {
for _, header := range []string{
"Forwarded", "X-Forwarded-For", "X-Forwarded-Host",
"X-Forwarded-Port", "X-Forwarded-Proto", "X-Real-IP",
} {
c.Request.Header.Del(header)
}
}
c.Next()
}
}
// CORSMiddleware handles Cross-Origin Resource Sharing for the API.
func CORSMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
server/internal/http/node_handler.go
+28 -14
View File
@@ -204,6 +204,7 @@ func (h *NodeHandler) RotateToken(c *gin.Context) {
recordAudit(c, h.auditService, "node", "rotate_token", "node",
fmt.Sprintf("%d", id), "",
fmt.Sprintf("轮换节点 Token (ID: %d)", id))
c.Header("Cache-Control", "no-store")
response.Success(c, gin.H{"newToken": tok})
}
@@ -220,11 +221,14 @@ func (h *NodeHandler) CreateInstallToken(c *gin.Context) {
return
}
var input struct {
Mode string `json:"mode"`
Arch string `json:"arch"`
AgentVersion string `json:"agentVersion"`
DownloadSrc string `json:"downloadSrc"`
TTLSeconds int `json:"ttlSeconds"`
Mode string `json:"mode"`
Arch string `json:"arch"`
AgentVersion string `json:"agentVersion"`
DownloadSrc string `json:"downloadSrc"`
TTLSeconds int `json:"ttlSeconds"`
AgentMasterURL string `json:"agentMasterUrl"`
ProxyURL string `json:"proxyUrl"`
CACertFile string `json:"caCertFile"`
}
if err := c.ShouldBindJSON(&input); err != nil {
c.JSON(stdhttp.StatusBadRequest, gin.H{"code": "INVALID_INPUT", "message": err.Error()})
@@ -246,13 +250,16 @@ func (h *NodeHandler) CreateInstallToken(c *gin.Context) {
out, err := h.installTokenSvc.CreateCommand(c.Request.Context(), service.InstallCommandInput{
InstallTokenInput: service.InstallTokenInput{
NodeID: uint(id),
Mode: input.Mode,
Arch: input.Arch,
AgentVersion: input.AgentVersion,
DownloadSrc: input.DownloadSrc,
TTLSeconds: input.TTLSeconds,
CreatedByID: h.resolveCurrentUserID(c),
NodeID: uint(id),
Mode: input.Mode,
Arch: input.Arch,
AgentVersion: input.AgentVersion,
DownloadSrc: input.DownloadSrc,
TTLSeconds: input.TTLSeconds,
CreatedByID: h.resolveCurrentUserID(c),
AgentMasterURL: input.AgentMasterURL,
ProxyURL: input.ProxyURL,
CACertFile: input.CACertFile,
},
MasterURL: resolveMasterURL(c, h.externalURL),
})
@@ -278,6 +285,7 @@ func (h *NodeHandler) CreateInstallToken(c *gin.Context) {
"composeUrl": out.ComposeURL,
"fallbackComposeUrl": out.FallbackComposeURL,
}
c.Header("Cache-Control", "no-store")
response.Success(c, body)
}
@@ -292,18 +300,24 @@ func (h *NodeHandler) PreviewScript(c *gin.Context) {
return
}
src := c.DefaultQuery("downloadSrc", "github")
agentMasterURL := c.Query("agentMasterUrl")
if agentMasterURL == "" {
agentMasterURL = resolveMasterURL(c, h.externalURL)
}
ctx := installscript.Context{
MasterURL: resolveMasterURL(c, h.externalURL),
MasterURL: agentMasterURL,
AgentToken: "<AGENT_TOKEN>",
AgentVersion: ver,
Mode: mode,
Arch: arch,
DownloadBase: installscript.DownloadBaseFor(src),
InstallPrefix: "/opt/backupx-agent",
ProxyURL: c.Query("proxyUrl"),
CACertFile: c.Query("caCertFile"),
}
script, err := installscript.RenderScript(ctx)
if err != nil {
response.Error(c, err)
response.Error(c, apperror.BadRequest("INSTALL_TOKEN_INVALID", "Agent 连接配置无效", err))
return
}
c.Data(stdhttp.StatusOK, "text/x-shellscript; charset=utf-8", []byte(script))
server/internal/http/router.go
+6
View File
@@ -61,7 +61,11 @@ type RouterDependencies struct {
func NewRouter(deps RouterDependencies) *gin.Engine {
gin.SetMode(deps.Config.Server.Mode)
engine := gin.New()
if err := engine.SetTrustedProxies(deps.Config.Server.TrustedProxies); err != nil {
panic("invalid trusted proxy configuration: " + err.Error())
}
engine.Use(gin.Recovery())
engine.Use(ForwardedHeadersMiddleware(deps.Config.Server.TrustedProxies))
engine.Use(CORSMiddleware())
engine.Use(requestLogger(deps.Logger))
@@ -322,7 +326,9 @@ func NewRouter(deps RouterDependencies) *gin.Engine {
agent.POST("/commands/:id/result", agentHandler.SubmitCommandResult)
agent.GET("/tasks/:id", agentHandler.GetTaskSpec)
agent.POST("/records/:id", agentHandler.UpdateRecord)
agent.PUT("/records/:id/artifacts/:targetId", agentHandler.UploadArtifact)
agent.GET("/restores/:id/spec", agentHandler.GetRestoreSpec)
agent.GET("/restores/:id/artifact", agentHandler.DownloadRestoreArtifact)
agent.POST("/restores/:id", agentHandler.UpdateRestore)
// Agent v1（安装脚本探活用），仅 Self 端点
server/internal/installscript/deploy_install_test.go
+95 -2
View File
@@ -6,11 +6,17 @@ import (
"path/filepath"
"strings"
"testing"
"gopkg.in/yaml.v3"
)
func TestDeployInstallScriptSyntax(t *testing.T) {
scriptPath := filepath.Join("..", "..", "..", "deploy", "install.sh")
cmd := exec.Command("sh", "-n", scriptPath)
sh, err := exec.LookPath("sh")
if err != nil {
t.Skip("POSIX sh is not available on this platform")
}
cmd := exec.Command(sh, "-n", scriptPath)
output, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("install.sh syntax invalid: %v\n%s", err, output)
@@ -30,12 +36,99 @@ func TestDeployInstallScriptSupportsReleasePackageLayout(t *testing.T) {
`BIN_SOURCE="${BIN_SOURCE:-$SCRIPT_DIR/backupx}"`,
`WEB_SOURCE="${WEB_SOURCE:-$SCRIPT_DIR/web}"`,
`CONFIG_TEMPLATE="${CONFIG_TEMPLATE:-$SCRIPT_DIR/config.example.yaml}"`,
`SERVICE_SOURCE_DEFAULT="$SCRIPT_DIR/backupx.service"`,
`发布包安装请确认当前目录包含 ./backupx、./web 和 ./install.sh。`,
`cat > "/etc/systemd/system/$SERVICE_NAME.service" <<UNIT`,
`if [ -d "/etc/nginx/conf.d" ] && [ -f "$NGINX_SOURCE" ]; then`,
`if [ "$INSTALL_NGINX" = "1" ]; then`,
`[ "$PREFIX" = "/opt/backupx" ] && [ "$ETC_DIR" = "/etc/backupx" ]`,
`validate_install_path PREFIX "$PREFIX"`,
`拒绝通过符号链接写入受管目录`,
} {
if !strings.Contains(script, want) {
t.Fatalf("install.sh missing %q", want)
}
}
}
func TestDeployInstallScriptSupportsSourceBuildAndVerifiesFirstSetup(t *testing.T) {
scriptPath := filepath.Join("..", "..", "..", "deploy", "install.sh")
data, err := os.ReadFile(scriptPath)
if err != nil {
t.Fatal(err)
}
script := string(data)
for _, want := range []string{
`SOURCE_BIN_DEFAULT="$PROJECT_ROOT/server/bin/backupx"`,
`For a source install, run 'make build' in the repository root first.`,
`HEALTH_URL="${HEALTH_URL:-http://127.0.0.1:8340/api/auth/setup/status}"`,
`systemctl is-active --quiet "$SERVICE_NAME"`,
`System setup`,
`chown -R root:root "$PREFIX/bin" "$PREFIX/web"`,
`find "$PREFIX/web" -type f -exec chmod 0644`,
`chown root:"$APP_GROUP" "$ETC_DIR/config.yaml"`,
`systemctl restart "$SERVICE_NAME"`,
} {
if !strings.Contains(script, want) {
t.Fatalf("install.sh missing %q", want)
}
}
}
func TestDockerDeploymentUsesSingleUnprivilegedProcess(t *testing.T) {
root := filepath.Join("..", "..", "..")
dockerfileData, err := os.ReadFile(filepath.Join(root, "Dockerfile"))
if err != nil {
t.Fatal(err)
}
dockerfile := string(dockerfileData)
for _, want := range []string{"su-exec", "BACKUPX_SERVER_WEB_ROOT=/app/web", "HEALTHCHECK"} {
if !strings.Contains(dockerfile, want) {
t.Fatalf("Dockerfile missing %q", want)
}
}
for _, forbidden := range []string{"docker-cli", "COPY deploy/docker/nginx.conf"} {
if strings.Contains(dockerfile, forbidden) {
t.Fatalf("Dockerfile still contains %q", forbidden)
}
}
composeData, err := os.ReadFile(filepath.Join(root, "docker-compose.yml"))
if err != nil {
t.Fatal(err)
}
var compose map[string]any
if err := yaml.Unmarshal(composeData, &compose); err != nil {
t.Fatalf("docker-compose.yml is not valid YAML: %v", err)
}
composeText := string(composeData)
for _, want := range []string{"no-new-privileges:true", "cap_drop:", "cap_add:", "DAC_OVERRIDE", "SETGID", "SETUID", "/ready"} {
if !strings.Contains(composeText, want) {
t.Fatalf("docker-compose.yml missing %q", want)
}
}
if strings.Contains(composeText, "docker.sock") {
t.Fatal("docker-compose.yml must not expose the Docker socket")
}
}
func TestReleaseWorkflowPublishesChecksums(t *testing.T) {
workflowPath := filepath.Join("..", "..", "..", ".github", "workflows", "release.yml")
workflowData, err := os.ReadFile(workflowPath)
if err != nil {
t.Fatal(err)
}
var workflow map[string]any
if err := yaml.Unmarshal(workflowData, &workflow); err != nil {
t.Fatalf("release workflow is not valid YAML: %v", err)
}
workflowText := string(workflowData)
for _, want := range []string{
`cp deploy/backupx.service "${ARCHIVE_NAME}/"`,
`sha256sum "${ARCHIVE_NAME}.tar.gz"`,
`backupx-${{ matrix.goos }}-${{ matrix.goarch }}.tar.gz.sha256`,
} {
if !strings.Contains(workflowText, want) {
t.Fatalf("release workflow missing %q", want)
}
}
}
server/internal/installscript/issue46_test.go
+1 -1
View File
@@ -44,7 +44,7 @@ func TestRenderScriptUsesRootForBareMetalBackups(t *testing.T) {
}
for _, want := range []string{
"/var/lib/backupx-agent/tmp",
"install -d -m 0700 /var/lib/backupx-agent /var/lib/backupx-agent/tmp",
"install -d -m 0700 \"$CONFIG_DIR\" /var/lib/backupx-agent /var/lib/backupx-agent/tmp",
} {
if !strings.Contains(got, want) {
t.Errorf("script missing %q:\n%s", want, got)
server/internal/installscript/renderer.go
+85
View File
@@ -8,6 +8,7 @@ import (
_ "embed"
"fmt"
"net/url"
"path"
"strings"
"text/template"
@@ -30,6 +31,8 @@ type Context struct {
DownloadBase string
InstallPrefix string
NodeID uint
ProxyURL string
CACertFile string
}
// DownloadBaseFor 将下载源枚举转换为具体 URL 前缀。
@@ -84,6 +87,12 @@ func RenderComposeYaml(ctx Context) (string, error) {
// 这些字段被直接写入 shell 双引号字符串和 YAML 双引号值；不做校验会带来
// 注入风险（如 MasterURL 含 `"\nCOMMAND:` 可逃逸 YAML 结构）。
func validateContext(ctx Context) error {
if ctx.Mode != model.InstallModeSystemd && ctx.Mode != model.InstallModeDocker && ctx.Mode != model.InstallModeForeground {
return fmt.Errorf("unsupported install mode %q", ctx.Mode)
}
if ctx.Arch != model.InstallArchAmd64 && ctx.Arch != model.InstallArchArm64 && ctx.Arch != model.InstallArchAuto {
return fmt.Errorf("unsupported install architecture %q", ctx.Arch)
}
if err := validateMasterURL(ctx.MasterURL); err != nil {
return err
}
@@ -93,6 +102,42 @@ func validateContext(ctx Context) error {
if err := validateAgentVersion(ctx.AgentVersion); err != nil {
return err
}
if err := validateProxyURL(ctx.ProxyURL); err != nil {
return err
}
if err := validateCACertFile(ctx.CACertFile); err != nil {
return err
}
if !path.IsAbs(ctx.InstallPrefix) || path.Clean(ctx.InstallPrefix) != ctx.InstallPrefix {
return fmt.Errorf("install prefix must be a clean absolute path without shell metacharacters")
}
for _, c := range ctx.InstallPrefix {
switch {
case c >= '0' && c <= '9':
case c >= 'a' && c <= 'z':
case c >= 'A' && c <= 'Z':
case c == '/' || c == '.' || c == '_' || c == '-' || c == '+':
default:
return fmt.Errorf("install prefix contains illegal character %q", c)
}
}
if err := validateDownloadBase(ctx.DownloadBase); err != nil {
return err
}
return nil
}
func validateDownloadBase(raw string) error {
if strings.ContainsAny(raw, " \t\r\n\"'`$\\") {
return fmt.Errorf("download base contains illegal characters")
}
u, err := url.Parse(raw)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return fmt.Errorf("download base must be an absolute http(s) URL")
}
if u.User != nil || u.RawQuery != "" || u.Fragment != "" {
return fmt.Errorf("download base must not contain credentials, query or fragment")
}
return nil
}
@@ -114,6 +159,43 @@ func validateMasterURL(raw string) error {
if u.Host == "" {
return fmt.Errorf("master URL missing host")
}
if u.User != nil || u.RawQuery != "" || u.Fragment != "" {
return fmt.Errorf("master URL must not contain credentials, query or fragment")
}
return nil
}
func validateProxyURL(raw string) error {
raw = strings.TrimSpace(raw)
if raw == "" {
return nil
}
if strings.ContainsAny(raw, " \t\r\n\"'`$\\") {
return fmt.Errorf("proxy URL contains illegal characters")
}
u, err := url.Parse(raw)
if err != nil || u.Host == "" {
return fmt.Errorf("invalid proxy URL")
}
switch u.Scheme {
case "http", "https", "socks5", "socks5h":
default:
return fmt.Errorf("proxy URL scheme must be http, https, socks5 or socks5h")
}
if u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") {
return fmt.Errorf("proxy URL must not contain credentials, path, query or fragment")
}
return nil
}
func validateCACertFile(file string) error {
file = strings.TrimSpace(file)
if file == "" {
return nil
}
if !path.IsAbs(file) || path.Clean(file) != file || strings.ContainsAny(file, " \t\r\n\"'`$\\") {
return fmt.Errorf("CA certificate path must be a clean absolute path without shell metacharacters")
}
return nil
}
@@ -160,6 +242,9 @@ func validateAgentVersion(v string) error {
}
func withDefaults(ctx Context) Context {
if ctx.Arch == "" {
ctx.Arch = model.InstallArchAuto
}
if ctx.InstallPrefix == "" {
ctx.InstallPrefix = "/opt/backupx-agent"
}
server/internal/installscript/renderer_test.go
+145 -8
View File
@@ -2,11 +2,13 @@ package installscript
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"backupx/server/internal/model"
"gopkg.in/yaml.v3"
)
// 使用合法 hex token（32 字节 = 64 字符）以通过 validateAgentToken 校验
@@ -27,11 +29,13 @@ func TestRenderScriptSystemd(t *testing.T) {
t.Fatalf("render err: %v", err)
}
mustContain := []string{
"BACKUPX_AGENT_MASTER=${MASTER_URL}",
`Environment="BACKUPX_AGENT_TOKEN=${AGENT_TOKEN}"`,
`master: "${MASTER_URL}"`,
`tokenFile: "${TOKEN_FILE}"`,
`ExecStart=${INSTALL_PREFIX}/backupx agent --config ${CONFIG_FILE}`,
"/var/lib/backupx-agent/tmp",
"systemctl daemon-reload",
"systemctl enable --now backupx-agent",
"systemctl enable backupx-agent",
"systemctl restart backupx-agent",
"systemctl status backupx-agent",
"X-Agent-Token: ${AGENT_TOKEN}",
"MASTER_URL=\"https://master.example.com\"",
@@ -48,6 +52,29 @@ func TestRenderScriptSystemd(t *testing.T) {
t.Errorf("systemd script unexpectedly contains %q", s)
}
}
if strings.Contains(got, `Environment="BACKUPX_AGENT_TOKEN=`) {
t.Errorf("systemd unit must not expose the agent token in its environment:\n%s", got)
}
}
func TestRenderedInstallScriptSyntax(t *testing.T) {
sh, err := exec.LookPath("sh")
if err != nil {
t.Skip("POSIX sh is not available on this platform")
}
for _, mode := range []string{model.InstallModeSystemd, model.InstallModeDocker, model.InstallModeForeground} {
ctx := testCtx
ctx.Mode = mode
script, renderErr := RenderScript(ctx)
if renderErr != nil {
t.Fatal(renderErr)
}
cmd := exec.Command(sh, "-n")
cmd.Stdin = strings.NewReader(script)
if output, syntaxErr := cmd.CombinedOutput(); syntaxErr != nil {
t.Fatalf("%s installer syntax invalid: %v\n%s", mode, syntaxErr, output)
}
}
}
func TestRenderScriptForeground(t *testing.T) {
@@ -90,8 +117,11 @@ func TestRenderScriptDocker(t *testing.T) {
if !strings.Contains(got, `"awuqing/backupx:${AGENT_VERSION}" agent`) {
t.Errorf("docker script must start image in agent mode:\n%s", got)
}
if !strings.Contains(got, `-e "BACKUPX_AGENT_TEMP_DIR=/var/lib/backupx-agent/tmp"`) {
t.Errorf("docker script missing temp dir env:\n%s", got)
if !strings.Contains(got, `-v /etc/backupx-agent:/etc/backupx-agent:ro`) {
t.Errorf("docker script missing protected config mount:\n%s", got)
}
if !strings.Contains(got, `agent --config /etc/backupx-agent/config.yaml`) {
t.Errorf("docker script must load the protected config file:\n%s", got)
}
if !strings.Contains(got, `docker logs --tail=100 backupx-agent`) {
t.Errorf("docker script missing diagnostic log command:\n%s", got)
@@ -102,6 +132,9 @@ func TestRenderScriptDocker(t *testing.T) {
if strings.Contains(got, "systemctl daemon-reload") {
t.Errorf("docker script should not reference systemctl:\n%s", got)
}
if strings.Contains(got, `-e "BACKUPX_AGENT_TOKEN=`) {
t.Errorf("docker inspect must not expose the agent token:\n%s", got)
}
}
func TestDockerEntrypointForwardsAgentSubcommand(t *testing.T) {
@@ -111,12 +144,18 @@ func TestDockerEntrypointForwardsAgentSubcommand(t *testing.T) {
t.Fatalf("read docker entrypoint: %v", err)
}
script := string(got)
if !strings.Contains(script, `"${1:-}" = "agent"`) {
t.Fatalf("entrypoint must detect the agent subcommand before starting server:\n%s", script)
}
if !strings.Contains(script, `exec /app/bin/backupx "$@"`) {
t.Fatalf("entrypoint must exec backupx with forwarded args:\n%s", script)
}
if !strings.Contains(script, `exec su-exec backupx:backupx /app/bin/backupx "$@"`) {
t.Fatalf("master entrypoint must drop privileges after data migration:\n%s", script)
}
if !strings.Contains(script, `export HOME=/app`) {
t.Fatalf("master entrypoint must set the service user's home directory:\n%s", script)
}
if strings.Contains(script, "nginx") || strings.Contains(script, "wait -n") {
t.Fatalf("entrypoint should run a single foreground process:\n%s", script)
}
}
func TestRenderComposeYaml(t *testing.T) {
@@ -138,9 +177,40 @@ func TestRenderComposeYaml(t *testing.T) {
if !strings.Contains(got, `BACKUPX_AGENT_TEMP_DIR: "/var/lib/backupx-agent/tmp"`) {
t.Errorf("compose missing temp dir env:\n%s", got)
}
if !strings.Contains(got, `user: "0:0"`) || !strings.Contains(got, "no-new-privileges:true") {
t.Errorf("compose missing root execution declaration or security option:\n%s", got)
}
if !strings.Contains(got, "/var/lib/backupx-agent:/var/lib/backupx-agent") {
t.Errorf("compose missing agent data volume:\n%s", got)
}
var document map[string]any
if err := yaml.Unmarshal([]byte(got), &document); err != nil {
t.Fatalf("compose is not valid YAML: %v\n%s", err, got)
}
}
func TestRenderComposeYamlIncludesRestrictedNetworkSettings(t *testing.T) {
ctx := testCtx
ctx.Mode = model.InstallModeDocker
ctx.ProxyURL = "socks5h://127.0.0.1:1080"
ctx.CACertFile = "/etc/backupx-agent/ca.pem"
got, err := RenderComposeYaml(ctx)
if err != nil {
t.Fatal(err)
}
for _, want := range []string{
`BACKUPX_AGENT_PROXY_URL: "socks5h://127.0.0.1:1080"`,
`BACKUPX_AGENT_CA_CERT_FILE: "/etc/backupx-agent/ca.pem"`,
`- /etc/backupx-agent/ca.pem:/etc/backupx-agent/ca.pem:ro`,
} {
if !strings.Contains(got, want) {
t.Errorf("compose missing %q:\n%s", want, got)
}
}
var document map[string]any
if err := yaml.Unmarshal([]byte(got), &document); err != nil {
t.Fatalf("compose is not valid YAML: %v\n%s", err, got)
}
}
func TestRenderScriptRejectsInjectedMasterURL(t *testing.T) {
@@ -159,6 +229,73 @@ func TestRenderScriptRejectsInjectedMasterURL(t *testing.T) {
}
}
func TestRenderScriptIncludesRestrictedNetworkSettings(t *testing.T) {
ctx := testCtx
ctx.MasterURL = "http://127.0.0.1:18340"
ctx.ProxyURL = "socks5h://127.0.0.1:1080"
ctx.CACertFile = "/etc/pki/internal-ca.pem"
got, err := RenderScript(ctx)
if err != nil {
t.Fatal(err)
}
for _, want := range []string{
`PROXY_URL="socks5h://127.0.0.1:1080"`,
`CA_CERT_FILE="/etc/pki/internal-ca.pem"`,
`proxyUrl: "${PROXY_URL}"`,
`caCertFile: "${CA_CERT_FILE}"`,
`curl -fsSL --retry 3 --retry-delay 2 --connect-timeout 15 --proxy "$PROXY_URL"`,
} {
if !strings.Contains(got, want) {
t.Errorf("restricted network script missing %q", want)
}
}
}
func TestRenderScriptRejectsUnsafeRestrictedNetworkSettings(t *testing.T) {
for _, mutate := range []func(*Context){
func(ctx *Context) { ctx.ProxyURL = "ftp://proxy.example.com" },
func(ctx *Context) { ctx.ProxyURL = "http://user:pass@proxy.example.com" },
func(ctx *Context) { ctx.ProxyURL = "http://proxy.example.com/connect" },
func(ctx *Context) { ctx.CACertFile = "relative-ca.pem" },
} {
ctx := testCtx
mutate(&ctx)
if _, err := RenderScript(ctx); err == nil {
t.Fatalf("expected restricted network settings to be rejected: %+v", ctx)
}
}
}
func TestRenderScriptRejectsUnsafeDeploymentSettings(t *testing.T) {
for _, mutate := range []func(*Context){
func(ctx *Context) { ctx.Mode = "unknown" },
func(ctx *Context) { ctx.Arch = "386" },
func(ctx *Context) { ctx.InstallPrefix = "/opt/backupx;touch/tmp/pwned" },
func(ctx *Context) { ctx.DownloadBase = "https://user:pass@example.com/releases" },
} {
ctx := testCtx
mutate(&ctx)
if _, err := RenderScript(ctx); err == nil {
t.Fatalf("expected unsafe deployment settings to be rejected: %+v", ctx)
}
}
}
func TestRenderScriptVerifiesReleaseChecksumWithoutEmoji(t *testing.T) {
got, err := RenderScript(testCtx)
if err != nil {
t.Fatal(err)
}
for _, want := range []string{"sha256sum -c", `download_file "${URL}.sha256"`, "[OK] 节点已上线", "[WARN]"} {
if !strings.Contains(got, want) {
t.Errorf("script missing %q", want)
}
}
if strings.ContainsAny(got, "\u2713\u26a0") {
t.Fatal("installer output must not use emoji symbols")
}
}
func TestRenderComposeYamlRejectsInjectedMasterURL(t *testing.T) {
ctx := testCtx
ctx.Mode = model.InstallModeDocker
server/internal/installscript/templates/agent-compose.yml.tmpl
+11 -3
View File
@@ -1,14 +1,22 @@
# BackupX Agent docker-compose 片段
# BackupX Agent Compose 片段
# 生成于 {{.MasterURL}} · 节点 ID {{.NodeID}}
version: "3.8"
# 文件包含长期节点 Token，请保存为 0600 权限并在部署完成后限制访问。
services:
backupx-agent:
image: awuqing/backupx:{{.AgentVersion}}
command: ["agent"]
user: "0:0"
restart: unless-stopped
security_opt:
- no-new-privileges:true
environment:
BACKUPX_AGENT_MASTER: "{{.MasterURL}}"
BACKUPX_AGENT_TOKEN: "{{.AgentToken}}"
BACKUPX_AGENT_TEMP_DIR: "/var/lib/backupx-agent/tmp"
volumes:
{{if .ProxyURL}} BACKUPX_AGENT_PROXY_URL: "{{.ProxyURL}}"
{{end}}{{if .CACertFile}} BACKUPX_AGENT_CA_CERT_FILE: "{{.CACertFile}}"
{{end}} volumes:
- /var/lib/backupx-agent:/var/lib/backupx-agent
{{if .CACertFile}} - {{.CACertFile}}:{{.CACertFile}}:ro
{{end}} # 备份宿主机文件时，必须按需添加只读源目录挂载：
# - /srv/data:/srv/data:ro
server/internal/installscript/templates/agent-install.sh.tmpl
+125 -45
View File
@@ -1,12 +1,11 @@
#!/bin/sh
# BackupX Agent 一键安装脚本（由 Master 动态渲染）
# Magic: BACKUPX_AGENT_INSTALL_V1 —— 若 `head -3 脚本` 看不到此行，说明反向代理/CDN 改写了响应
# Magic: BACKUPX_AGENT_INSTALL_V1 —— 若 `head -3 脚本` 看不到此行，说明反向代理或 CDN 改写了响应
# 模式: {{.Mode}} | 架构: {{.Arch}} | 版本: {{.AgentVersion}}
set -eu
umask 077
# 自举到 bash（文件执行模式下生效；管道模式 $0 不是文件，exec 会静默失败，继续用 sh）。
# 动机：部分 Debian/Ubuntu 用户通过 `curl | sudo sh` 触发时，dash 对本脚本报语法错误；
# 若目标机装有 bash，优先切换到 bash 获得更一致的行为。
# 文件执行模式下优先使用 bash；管道模式继续使用 POSIX sh。
if [ -z "${BASH_VERSION:-}" ] && command -v bash >/dev/null 2>&1 && [ -f "$0" ]; then
exec bash "$0" "$@"
fi
@@ -17,14 +16,56 @@ AGENT_VERSION="{{.AgentVersion}}"
DOWNLOAD_BASE="{{.DownloadBase}}"
INSTALL_PREFIX="{{.InstallPrefix}}"
ARCH="{{.Arch}}"
PROXY_URL="{{.ProxyURL}}"
CA_CERT_FILE="{{.CACertFile}}"
CONFIG_DIR="/etc/backupx-agent"
CONFIG_FILE="${CONFIG_DIR}/config.yaml"
TOKEN_FILE="${CONFIG_DIR}/agent.token"
download_file() {
source_url="$1"
destination="$2"
if command -v curl >/dev/null 2>&1; then
if [ -n "$PROXY_URL" ]; then
curl -fsSL --retry 3 --retry-delay 2 --connect-timeout 15 --proxy "$PROXY_URL" "$source_url" -o "$destination"
else
curl -fsSL --retry 3 --retry-delay 2 --connect-timeout 15 "$source_url" -o "$destination"
fi
else
wget -q -T 30 -O "$destination" "$source_url"
fi
}
agent_is_online() {
if command -v curl >/dev/null 2>&1; then
if [ -n "$PROXY_URL" ] && [ -n "$CA_CERT_FILE" ]; then
response=$(curl -fsS --max-time 5 --proxy "$PROXY_URL" --cacert "$CA_CERT_FILE" -H "X-Agent-Token: ${AGENT_TOKEN}" "${MASTER_URL}/api/v1/agent/self" 2>/dev/null) || return 1
elif [ -n "$PROXY_URL" ]; then
response=$(curl -fsS --max-time 5 --proxy "$PROXY_URL" -H "X-Agent-Token: ${AGENT_TOKEN}" "${MASTER_URL}/api/v1/agent/self" 2>/dev/null) || return 1
elif [ -n "$CA_CERT_FILE" ]; then
response=$(curl -fsS --max-time 5 --cacert "$CA_CERT_FILE" -H "X-Agent-Token: ${AGENT_TOKEN}" "${MASTER_URL}/api/v1/agent/self" 2>/dev/null) || return 1
else
response=$(curl -fsS --max-time 5 -H "X-Agent-Token: ${AGENT_TOKEN}" "${MASTER_URL}/api/v1/agent/self" 2>/dev/null) || return 1
fi
else
response=$(wget -q -T 5 --max-redirect=0 -O - --header="X-Agent-Token: ${AGENT_TOKEN}" "${MASTER_URL}/api/v1/agent/self" 2>/dev/null) || return 1
fi
printf '%s' "$response" | grep -q '"status":"online"'
}
# 1. 前置检查
[ "$(id -u)" -eq 0 ] || { echo "请使用 root 或 sudo 执行" >&2; exit 1; }
command -v curl >/dev/null || command -v wget >/dev/null \
command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1 \
|| { echo "需要 curl 或 wget" >&2; exit 1; }
{{if eq .Mode "systemd"}}command -v systemctl >/dev/null || { echo "不支持非 systemd 系统" >&2; exit 1; }
{{end}}{{if eq .Mode "docker"}}command -v docker >/dev/null || { echo "需要先安装 docker" >&2; exit 1; }
command -v grep >/dev/null 2>&1 || { echo "需要 grep" >&2; exit 1; }
if { [ -n "$PROXY_URL" ] || [ -n "$CA_CERT_FILE" ]; } && ! command -v curl >/dev/null 2>&1; then
echo "显式代理或自定义 CA 场景需要 curl" >&2
exit 1
fi
{{if eq .Mode "systemd"}}command -v systemctl >/dev/null 2>&1 || { echo "当前系统不支持 systemd" >&2; exit 1; }
{{end}}{{if eq .Mode "docker"}}command -v docker >/dev/null 2>&1 || { echo "需要先安装 Docker" >&2; exit 1; }
{{end}}
# 2. 架构检测
if [ "$ARCH" = "auto" ]; then
case "$(uname -m)" in
@@ -34,24 +75,54 @@ if [ "$ARCH" = "auto" ]; then
esac
fi
# 3. 安全写入 Agent 配置。systemd unit 与容器元数据中不保存节点 Token。
install -d -m 0700 "$CONFIG_DIR" /var/lib/backupx-agent /var/lib/backupx-agent/tmp
printf '%s\n' "$AGENT_TOKEN" > "$TOKEN_FILE"
chmod 0600 "$TOKEN_FILE"
if [ -n "$CA_CERT_FILE" ]; then
[ -r "$CA_CERT_FILE" ] || { echo "无法读取 CA 证书: $CA_CERT_FILE" >&2; exit 1; }
if [ "$CA_CERT_FILE" != "$CONFIG_DIR/ca.pem" ]; then
install -m 0644 "$CA_CERT_FILE" "$CONFIG_DIR/ca.pem"
fi
CA_CERT_FILE="$CONFIG_DIR/ca.pem"
fi
cat > "$CONFIG_FILE" <<CONFIG
master: "${MASTER_URL}"
tokenFile: "${TOKEN_FILE}"
heartbeatInterval: "15s"
pollInterval: "5s"
tempDir: "/var/lib/backupx-agent/tmp"
proxyUrl: "${PROXY_URL}"
caCertFile: "${CA_CERT_FILE}"
CONFIG
chmod 0600 "$CONFIG_FILE"
{{if ne .Mode "docker"}}
# 3. 下载二进制（systemd / foreground 模式）
# 4. 下载并安装二进制（systemd / foreground 模式）
ARCHIVE="backupx-${AGENT_VERSION}-linux-${ARCH}.tar.gz"
URL="${DOWNLOAD_BASE}/${AGENT_VERSION}/${ARCHIVE}"
TMPDIR="$(mktemp -d)"; trap 'rm -rf "$TMPDIR"' EXIT
TMPDIR="$(mktemp -d)"
trap 'rm -rf "$TMPDIR"' EXIT HUP INT TERM
echo "[1/4] 下载 ${URL}"
if command -v curl >/dev/null; then
curl -fsSL "$URL" -o "$TMPDIR/pkg.tar.gz"
download_file "$URL" "$TMPDIR/$ARCHIVE"
if download_file "${URL}.sha256" "$TMPDIR/$ARCHIVE.sha256" 2>/dev/null; then
if command -v sha256sum >/dev/null 2>&1; then
(cd "$TMPDIR" && sha256sum -c "$ARCHIVE.sha256")
elif command -v shasum >/dev/null 2>&1; then
(cd "$TMPDIR" && shasum -a 256 -c "$ARCHIVE.sha256")
else
echo "已下载校验文件，但系统缺少 sha256sum 或 shasum，拒绝未校验安装" >&2
exit 1
fi
else
wget -qO "$TMPDIR/pkg.tar.gz" "$URL"
echo "[WARN] 当前版本未提供 SHA-256 校验文件，继续兼容安装" >&2
fi
tar xzf "$TMPDIR/pkg.tar.gz" -C "$TMPDIR"
tar xzf "$TMPDIR/$ARCHIVE" -C "$TMPDIR"
# 4. 安装二进制 + 数据目录
echo "[2/4] 安装到 ${INSTALL_PREFIX}"
install -d -m 0755 "$INSTALL_PREFIX"
install -d -m 0700 /var/lib/backupx-agent /var/lib/backupx-agent/tmp
install -m 0755 "$TMPDIR/backupx-${AGENT_VERSION}-linux-${ARCH}/backupx" "$INSTALL_PREFIX/backupx"
install -m 0755 "$TMPDIR/backupx-${AGENT_VERSION}-linux-${ARCH}/backupx" "$INSTALL_PREFIX/backupx.new"
mv -f "$INSTALL_PREFIX/backupx.new" "$INSTALL_PREFIX/backupx"
{{end}}
{{if eq .Mode "systemd"}}
@@ -62,69 +133,78 @@ cat > /etc/systemd/system/backupx-agent.service <<UNIT
Description=BackupX Agent
After=network-online.target
Wants=network-online.target
StartLimitIntervalSec=300
StartLimitBurst=10
[Service]
Type=simple
Environment="BACKUPX_AGENT_MASTER=${MASTER_URL}"
Environment="BACKUPX_AGENT_TOKEN=${AGENT_TOKEN}"
ExecStart=${INSTALL_PREFIX}/backupx agent --temp-dir /var/lib/backupx-agent/tmp
ExecStart=${INSTALL_PREFIX}/backupx agent --config ${CONFIG_FILE}
Restart=on-failure
RestartSec=10s
# Agent 需以 root 运行以读取任意源数据；与单机服务端保持一致的资源/句柄上限。
TimeoutStopSec=30s
UMask=0077
# Agent 以 root 运行，以便读取备份源及执行恢复；节点 Token 仅保存在 0600 文件中。
LimitNOFILE=65535
[Install]
WantedBy=multi-user.target
UNIT
chmod 0644 /etc/systemd/system/backupx-agent.service
systemctl daemon-reload
systemctl enable --now backupx-agent
if ! systemctl enable backupx-agent || ! systemctl restart backupx-agent; then
echo "BackupX Agent 服务启动失败" >&2
systemctl status backupx-agent --no-pager >&2 || true
journalctl -u backupx-agent -n 100 --no-pager >&2 || true
exit 1
fi
# 6. 等待上线
echo "[4/4] 等待节点上线"
for i in $(seq 1 15); do
i=0
while [ "$i" -lt 15 ]; do
sleep 2
if curl -fsSL -H "X-Agent-Token: ${AGENT_TOKEN}" "${MASTER_URL}/api/v1/agent/self" 2>/dev/null \
| grep -q '"status":"online"'; then
echo "✓ 节点已上线"
if agent_is_online; then
echo "[OK] 节点已上线"
exit 0
fi
i=$((i + 1))
done
echo "⚠ 30s 内未收到上线心跳，请检查防火墙或 journalctl -u backupx-agent"
echo "提示：systemd 服务名是 backupx-agent，可执行 systemctl status backupx-agent 查看状态。"
echo "[WARN] 30 秒内未收到上线心跳，请检查网络、代理或 SSH 隧道" >&2
echo "排查命令: systemctl status backupx-agent" >&2
echo "排查命令: journalctl -u backupx-agent -n 100 --no-pager" >&2
exit 2
{{end}}
{{if eq .Mode "foreground"}}
# 5. 前台运行
echo "[3/3] 前台启动 agent（Ctrl+C 退出）"
export BACKUPX_AGENT_MASTER="${MASTER_URL}"
export BACKUPX_AGENT_TOKEN="${AGENT_TOKEN}"
exec "${INSTALL_PREFIX}/backupx" agent --temp-dir /var/lib/backupx-agent/tmp
echo "[3/3] 前台启动 Agent（Ctrl+C 退出）"
exec "${INSTALL_PREFIX}/backupx" agent --config "$CONFIG_FILE"
{{end}}
{{if eq .Mode "docker"}}
# Docker 模式：直接用镜像启动容器
# Docker 模式：配置文件只读挂载，避免 Token 出现在 docker inspect 环境变量中。
echo "[1/2] 拉取镜像 awuqing/backupx:${AGENT_VERSION}"
docker pull "awuqing/backupx:${AGENT_VERSION}"
echo "[2/2] 启动容器 backupx-agent"
docker rm -f backupx-agent >/dev/null 2>&1 || true
docker run -d --name backupx-agent --restart=unless-stopped \
-e "BACKUPX_AGENT_MASTER=${MASTER_URL}" \
-e "BACKUPX_AGENT_TOKEN=${AGENT_TOKEN}" \
-e "BACKUPX_AGENT_TEMP_DIR=/var/lib/backupx-agent/tmp" \
docker run -d --name backupx-agent --restart=unless-stopped --user 0:0 \
--security-opt no-new-privileges:true \
-v /etc/backupx-agent:/etc/backupx-agent:ro \
-v /var/lib/backupx-agent:/var/lib/backupx-agent \
"awuqing/backupx:${AGENT_VERSION}" agent
echo "✓ 容器已启动，等待节点上线"
for i in $(seq 1 15); do
"awuqing/backupx:${AGENT_VERSION}" agent --config /etc/backupx-agent/config.yaml
echo "提示: Docker Agent 只能访问显式挂载的目录；备份宿主机路径前请按文档添加只读 -v 挂载。"
echo "容器已启动，等待节点上线"
i=0
while [ "$i" -lt 15 ]; do
sleep 2
if curl -fsSL -H "X-Agent-Token: ${AGENT_TOKEN}" "${MASTER_URL}/api/v1/agent/self" 2>/dev/null \
| grep -q '"status":"online"'; then
echo "✓ 节点已上线"
if agent_is_online; then
echo "[OK] 节点已上线"
exit 0
fi
i=$((i + 1))
done
echo "⚠ 30s 内未收到上线心跳，请检查容器状态、网络与 Master URL。"
echo "排查命令：docker ps -a --filter name=backupx-agent"
echo "排查命令：docker logs --tail=100 backupx-agent"
echo "[WARN] 30 秒内未收到上线心跳，请检查容器、代理、隧道与 Master 地址" >&2
echo "排查命令: docker ps -a --filter name=backupx-agent" >&2
echo "排查命令: docker logs --tail=100 backupx-agent" >&2
exit 2
{{end}}
server/internal/model/agent_command.go
+16 -16
View File
@@ -4,11 +4,11 @@ import "time"
// AgentCommand 状态常量
const (
AgentCommandStatusPending = "pending" // 待 Agent 拉取
AgentCommandStatusPending = "pending" // 待 Agent 拉取
AgentCommandStatusDispatched = "dispatched" // Agent 已领取，正在执行
AgentCommandStatusSucceeded = "succeeded" // 执行成功
AgentCommandStatusFailed = "failed" // 执行失败
AgentCommandStatusTimeout = "timeout" // 超时未完成
AgentCommandStatusSucceeded = "succeeded" // 执行成功
AgentCommandStatusFailed = "failed" // 执行失败
AgentCommandStatusTimeout = "timeout" // 超时未完成
)
// AgentCommand 类型常量
@@ -36,20 +36,20 @@ const (
)
// AgentCommand 代表 Master 发给某个 Agent 节点的待执行命令。
// 使用简单的数据库队列实现：Agent 通过 token 长轮询拉取本节点 pending 命令，
// 使用简单的数据库队列实现：Agent 通过 token 定期轮询本节点 pending 命令，
// 执行后回写状态与结果。Master 侧通过定时检查把超时的命令标记为 timeout。
type AgentCommand struct {
ID uint `gorm:"primaryKey" json:"id"`
NodeID uint `gorm:"column:node_id;index;not null" json:"nodeId"`
Type string `gorm:"size:32;index;not null" json:"type"`
Status string `gorm:"size:20;index;not null;default:'pending'" json:"status"`
Payload string `gorm:"type:text" json:"payload"` // JSON
Result string `gorm:"type:text" json:"result"` // JSON（成功结果）
ErrorMessage string `gorm:"column:error_message;type:text" json:"errorMessage"`
DispatchedAt *time.Time `gorm:"column:dispatched_at" json:"dispatchedAt,omitempty"`
CompletedAt *time.Time `gorm:"column:completed_at" json:"completedAt,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
ID uint `gorm:"primaryKey" json:"id"`
NodeID uint `gorm:"column:node_id;not null;index:idx_agent_commands_node_status,priority:1" json:"nodeId"`
Type string `gorm:"size:32;index;not null" json:"type"`
Status string `gorm:"size:20;not null;default:'pending';index:idx_agent_commands_node_status,priority:2;index:idx_agent_commands_status_dispatched,priority:1;index:idx_agent_commands_status_created,priority:1" json:"status"`
Payload string `gorm:"type:text" json:"payload"` // JSON
Result string `gorm:"type:text" json:"result"` // JSON（成功结果）
ErrorMessage string `gorm:"column:error_message;type:text" json:"errorMessage"`
DispatchedAt *time.Time `gorm:"column:dispatched_at;index:idx_agent_commands_status_dispatched,priority:2" json:"dispatchedAt,omitempty"`
CompletedAt *time.Time `gorm:"column:completed_at" json:"completedAt,omitempty"`
CreatedAt time.Time `gorm:"index:idx_agent_commands_status_created,priority:2" json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
func (AgentCommand) TableName() string {
server/internal/model/agent_install_token.go
+15 -11
View File
@@ -6,17 +6,21 @@ import "time"
//
// 生命周期：创建 → 消费（ConsumedAt 非空即作废）→ 超过 ExpiresAt 后被 GC 硬删除。
type AgentInstallToken struct {
ID uint `gorm:"primaryKey" json:"id"`
Token string `gorm:"size:64;uniqueIndex;not null" json:"token"`
NodeID uint `gorm:"not null;index" json:"nodeId"`
Mode string `gorm:"size:16;not null" json:"mode"` // systemd|docker|foreground
Arch string `gorm:"size:16;not null" json:"arch"` // amd64|arm64|auto
AgentVer string `gorm:"size:32;not null" json:"agentVersion"`
DownloadSrc string `gorm:"size:16;not null;default:'github'" json:"downloadSrc"`
ExpiresAt time.Time `gorm:"not null;index" json:"expiresAt"`
ConsumedAt *time.Time `json:"consumedAt,omitempty"`
CreatedByID uint `gorm:"not null" json:"createdById"`
CreatedAt time.Time `json:"createdAt"`
ID uint `gorm:"primaryKey" json:"id"`
Token string `gorm:"size:64;uniqueIndex;not null" json:"token"`
NodeID uint `gorm:"not null;index" json:"nodeId"`
Mode string `gorm:"size:16;not null" json:"mode"` // systemd|docker|foreground
Arch string `gorm:"size:16;not null" json:"arch"` // amd64|arm64|auto
AgentVer string `gorm:"size:32;not null" json:"agentVersion"`
DownloadSrc string `gorm:"size:16;not null;default:'github'" json:"downloadSrc"`
// AgentMasterURL 可覆盖公开安装地址，支持代理或 SSH 隧道后的节点专用入口。
AgentMasterURL string `gorm:"size:2048" json:"agentMasterUrl,omitempty"`
ProxyURL string `gorm:"size:2048" json:"proxyUrl,omitempty"`
CACertFile string `gorm:"size:512" json:"caCertFile,omitempty"`
ExpiresAt time.Time `gorm:"not null;index" json:"expiresAt"`
ConsumedAt *time.Time `json:"consumedAt,omitempty"`
CreatedByID uint `gorm:"not null" json:"createdById"`
CreatedAt time.Time `json:"createdAt"`
}
func (AgentInstallToken) TableName() string { return "agent_install_tokens" }
server/internal/model/backup_record.go
+14 -10
View File
@@ -9,9 +9,11 @@ const (
)
const (
// BackupKindFull 全量备份；BackupKindDifferential 差异备份（仅含自基线全量以来的变更）。
// BackupKindFull 全量归档；BackupKindDifferential 差异归档；
// BackupKindRepository 为可独立恢复的 CDC 内容寻址快照。
BackupKindFull = "full"
BackupKindDifferential = "differential"
BackupKindRepository = "repository"
)
type BackupRecord struct {
@@ -20,20 +22,22 @@ type BackupRecord struct {
Task BackupTask `json:"task,omitempty"`
StorageTargetID uint `gorm:"column:storage_target_id;index;not null" json:"storageTargetId"`
StorageTarget StorageTarget `json:"storageTarget,omitempty"`
// NodeID 执行该次备份的节点（0 = 本机 Master）。用于集群中识别 local_disk 类型
// 存储的归属节点，避免 Master 端试图跨节点访问远程 Agent 的本地存储。
NodeID uint `gorm:"column:node_id;index;default:0" json:"nodeId"`
Status string `gorm:"size:20;index;not null" json:"status"`
FileName string `gorm:"column:file_name;size:255" json:"fileName"`
FileSize int64 `gorm:"column:file_size;not null;default:0" json:"fileSize"`
Checksum string `gorm:"column:checksum;size:64" json:"checksum"`
StoragePath string `gorm:"column:storage_path;size:500" json:"storagePath"`
// NodeID 执行该次备份的节点（0 = 本机 Master）。StorageTransferMode 进一步
// 区分远程 Agent 直写与 Master 中转，避免在错误节点访问 local_disk。
NodeID uint `gorm:"column:node_id;index;default:0" json:"nodeId"`
Status string `gorm:"size:20;index;not null" json:"status"`
FileName string `gorm:"column:file_name;size:255" json:"fileName"`
FileSize int64 `gorm:"column:file_size;not null;default:0" json:"fileSize"`
Checksum string `gorm:"column:checksum;size:64" json:"checksum"`
StoragePath string `gorm:"column:storage_path;size:500" json:"storagePath"`
// 空值表示旧版 Agent 直写；direct / master_relay 记录新协议的实际数据路径。
StorageTransferMode string `gorm:"column:storage_transfer_mode;size:20" json:"storageTransferMode,omitempty"`
StorageUploadResults string `gorm:"column:storage_upload_results;type:text" json:"-"`
DurationSeconds int `gorm:"column:duration_seconds;not null;default:0" json:"durationSeconds"`
// Locked 保留锁定（法律保留）：为 true 时该备份不参与保留期/数量自动清理，
// 且禁止手动删除，直到显式解锁。用于保护合规快照、迁移前基线等关键备份。
Locked bool `gorm:"column:locked;not null;default:false;index" json:"locked"`
// BackupKind 备份类型：full（全量）/ differential（差异）。
// BackupKind 备份类型：full（全量）/ differential（差异）/ repository（CDC 快照）。
BackupKind string `gorm:"column:backup_kind;size:16;not null;default:'full';index" json:"backupKind"`
// BaseRecordID 差异备份所基于的全量备份记录 ID（全量记录为 0）。
BaseRecordID uint `gorm:"column:base_record_id;index;not null;default:0" json:"baseRecordId"`
server/internal/model/backup_task.go
+5 -2
View File
@@ -12,9 +12,11 @@ const (
)
const (
// BackupModeFull 全量模式（默认）；BackupModeDifferential 差异模式（仅文件类型本机任务）。
// BackupModeFull 全量模式（默认）；BackupModeDifferential 差异归档；
// BackupModeRepository 为 CDC 内容寻址仓库模式（仅文件类型本机任务）。
BackupModeFull = "full"
BackupModeDifferential = "differential"
BackupModeRepository = "repository"
)
const (
@@ -55,7 +57,8 @@ type BackupTask struct {
Compression string `gorm:"size:10;not null;default:'gzip'" json:"compression"`
Encrypt bool `gorm:"not null;default:false" json:"encrypt"`
MaxBackups int `gorm:"column:max_backups;not null;default:10" json:"maxBackups"`
// BackupMode 备份模式：full（全量，默认）/ differential（差异）。差异仅支持本机文件任务。
// BackupMode 备份模式：full（全量，默认）/ differential（差异归档）/
// repository（FastCDC 分块、全局去重快照）。后两者仅支持本机文件任务。
BackupMode string `gorm:"column:backup_mode;size:16;not null;default:'full'" json:"backupMode"`
// DiffFullIntervalDays 差异模式下强制全量的间隔（天）：最近全量超过该天数则本次自动改为全量，
// 限制差异链跨度与单个差异体积。默认 7。
server/internal/model/notification.go
+11 -11
View File
@@ -6,12 +6,12 @@ import "time"
// 任一 Notification 可订阅多个事件，EventTypes 字段存 CSV。
// 空 EventTypes + OnSuccess/OnFailure=true 时沿用旧语义（仅备份成功/失败）。
const (
NotificationEventBackupSuccess = "backup_success"
NotificationEventBackupFailed = "backup_failed"
NotificationEventBackupSuccess = "backup_success"
NotificationEventBackupFailed = "backup_failed"
NotificationEventRestoreSuccess = "restore_success"
NotificationEventRestoreFailed = "restore_failed"
NotificationEventVerifyFailed = "verify_failed"
NotificationEventSLAViolation = "sla_violation"
NotificationEventVerifyFailed = "verify_failed"
NotificationEventSLAViolation = "sla_violation"
// NotificationEventStorageUnhealthy 存储目标连接失败（后台健康扫描触发）。
NotificationEventStorageUnhealthy = "storage_unhealthy"
// NotificationEventReplicationFailed 备份复制失败。
@@ -23,13 +23,13 @@ const (
)
type Notification struct {
ID uint `gorm:"primaryKey" json:"id"`
Type string `gorm:"size:20;index;not null" json:"type"`
Name string `gorm:"size:100;uniqueIndex;not null" json:"name"`
ConfigCiphertext string `gorm:"column:config_ciphertext;type:text;not null" json:"-"`
Enabled bool `gorm:"not null;default:true" json:"enabled"`
OnSuccess bool `gorm:"column:on_success;not null;default:false" json:"onSuccess"`
OnFailure bool `gorm:"column:on_failure;not null;default:true" json:"onFailure"`
ID uint `gorm:"primaryKey" json:"id"`
Type string `gorm:"size:20;index;not null" json:"type"`
Name string `gorm:"size:100;uniqueIndex;not null" json:"name"`
ConfigCiphertext string `gorm:"column:config_ciphertext;type:text;not null" json:"-"`
Enabled bool `gorm:"not null;default:true" json:"enabled"`
OnSuccess bool `gorm:"column:on_success;not null;default:false" json:"onSuccess"`
OnFailure bool `gorm:"column:on_failure;not null;default:true" json:"onFailure"`
// EventTypes 逗号分隔，订阅的事件类型。
// 空 = 仅监听备份成功/失败（兼容旧配置）；非空则严格按订阅触发。
EventTypes string `gorm:"column:event_types;size:500" json:"eventTypes"`
server/internal/model/replication_record.go
+13 -13
View File
@@ -24,19 +24,19 @@ type ReplicationRecord struct {
SourceTargetID uint `gorm:"column:source_target_id;index;not null" json:"sourceTargetId"`
SourceTarget StorageTarget `gorm:"foreignKey:SourceTargetID;references:ID" json:"sourceTarget,omitempty"`
// DestTargetID 目标存储（复制过去）
DestTargetID uint `gorm:"column:dest_target_id;index;not null" json:"destTargetId"`
DestTarget StorageTarget `gorm:"foreignKey:DestTargetID;references:ID" json:"destTarget,omitempty"`
Status string `gorm:"size:20;index;not null" json:"status"`
StoragePath string `gorm:"column:storage_path;size:500" json:"storagePath"`
FileSize int64 `gorm:"column:file_size;not null;default:0" json:"fileSize"`
Checksum string `gorm:"column:checksum;size:64" json:"checksum"`
ErrorMessage string `gorm:"column:error_message;size:2000" json:"errorMessage"`
DurationSeconds int `gorm:"column:duration_seconds;not null;default:0" json:"durationSeconds"`
TriggeredBy string `gorm:"column:triggered_by;size:100" json:"triggeredBy"`
StartedAt time.Time `gorm:"column:started_at;index;not null" json:"startedAt"`
CompletedAt *time.Time `gorm:"column:completed_at;index" json:"completedAt,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
DestTargetID uint `gorm:"column:dest_target_id;index;not null" json:"destTargetId"`
DestTarget StorageTarget `gorm:"foreignKey:DestTargetID;references:ID" json:"destTarget,omitempty"`
Status string `gorm:"size:20;index;not null" json:"status"`
StoragePath string `gorm:"column:storage_path;size:500" json:"storagePath"`
FileSize int64 `gorm:"column:file_size;not null;default:0" json:"fileSize"`
Checksum string `gorm:"column:checksum;size:64" json:"checksum"`
ErrorMessage string `gorm:"column:error_message;size:2000" json:"errorMessage"`
DurationSeconds int `gorm:"column:duration_seconds;not null;default:0" json:"durationSeconds"`
TriggeredBy string `gorm:"column:triggered_by;size:100" json:"triggeredBy"`
StartedAt time.Time `gorm:"column:started_at;index;not null" json:"startedAt"`
CompletedAt *time.Time `gorm:"column:completed_at;index" json:"completedAt,omitempty"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
func (ReplicationRecord) TableName() string {
server/internal/model/task_template.go
+4 -4
View File
@@ -11,10 +11,10 @@ import "time"
// - name
// - sourcePath / sourcePaths 中的 {{.Host}} / {{.Env}} 等占位符
type TaskTemplate struct {
ID uint `gorm:"primaryKey" json:"id"`
Name string `gorm:"size:128;uniqueIndex;not null" json:"name"`
Description string `gorm:"size:500" json:"description"`
TaskType string `gorm:"column:task_type;size:20;not null" json:"taskType"`
ID uint `gorm:"primaryKey" json:"id"`
Name string `gorm:"size:128;uniqueIndex;not null" json:"name"`
Description string `gorm:"size:500" json:"description"`
TaskType string `gorm:"column:task_type;size:20;not null" json:"taskType"`
// Payload JSON，存完整 BackupTaskUpsertInput 的序列化
Payload string `gorm:"type:text;not null" json:"payload"`
CreatedBy string `gorm:"column:created_by;size:128" json:"createdBy"`
server/internal/repository/agent_command_repository_test.go
+18
View File
@@ -17,12 +17,30 @@ func newTestDB(t *testing.T) *gorm.DB {
if err != nil {
t.Fatalf("open: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("get sql database: %v", err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
if err := db.AutoMigrate(&model.AgentCommand{}); err != nil {
t.Fatalf("migrate: %v", err)
}
return db
}
func TestAgentCommandQueueIndexes(t *testing.T) {
db := newTestDB(t)
for _, name := range []string{
"idx_agent_commands_node_status",
"idx_agent_commands_status_dispatched",
"idx_agent_commands_status_created",
} {
if !db.Migrator().HasIndex(&model.AgentCommand{}, name) {
t.Fatalf("missing Agent command queue index %s", name)
}
}
}
func TestAgentCommandRepository_ClaimPending(t *testing.T) {
db := newTestDB(t)
repo := NewAgentCommandRepository(db)
server/internal/repository/agent_install_token_repository_test.go
+5
View File
@@ -20,6 +20,11 @@ func openTestInstallTokenDB(t *testing.T) *gorm.DB {
if err != nil {
t.Fatalf("open: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("get sql database: %v", err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
if err := db.AutoMigrate(&model.AgentInstallToken{}); err != nil {
t.Fatalf("migrate: %v", err)
}
server/internal/repository/backup_task_repository_test.go
+5
View File
@@ -21,6 +21,11 @@ func newBackupTaskTestRepository(t *testing.T) *GormBackupTaskRepository {
if err != nil {
t.Fatalf("database.Open returned error: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("db.DB returned error: %v", err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
if err := db.Create(&model.StorageTarget{Name: "local", Type: "local_disk", Enabled: true, ConfigCiphertext: "{}", ConfigVersion: 1, LastTestStatus: "unknown"}).Error; err != nil {
t.Fatalf("seed storage target error: %v", err)
}
server/internal/repository/node_repository_test.go
+5
View File
@@ -19,6 +19,11 @@ func openTestNodeDB(t *testing.T) *gorm.DB {
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("get sql database: %v", err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
if err := db.AutoMigrate(&model.Node{}); err != nil {
t.Fatalf("migrate: %v", err)
}
server/internal/repository/notification_repository_test.go
+5
View File
@@ -21,6 +21,11 @@ func newNotificationTestRepository(t *testing.T) *GormNotificationRepository {
if err != nil {
t.Fatalf("database.Open returned error: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("db.DB returned error: %v", err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
return NewNotificationRepository(db)
}
server/internal/repository/oauth_session_repository_test.go
+5
View File
@@ -22,6 +22,11 @@ func newOAuthSessionTestRepository(t *testing.T) *GormOAuthSessionRepository {
if err != nil {
t.Fatalf("database.Open returned error: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("db.DB returned error: %v", err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
return NewOAuthSessionRepository(db)
}
server/internal/repository/restore_record_repository_test.go
+5
View File
@@ -22,6 +22,11 @@ func newRestoreRecordTestRepository(t *testing.T) (*GormRestoreRecordRepository,
if err != nil {
t.Fatalf("database.Open returned error: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("db.DB returned error: %v", err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
storageTarget := &model.StorageTarget{Name: "local", Type: "local_disk", Enabled: true, ConfigCiphertext: "{}", ConfigVersion: 1, LastTestStatus: "unknown"}
if err := db.Create(storageTarget).Error; err != nil {
t.Fatalf("seed storage target error: %v", err)
server/internal/repository/storage_target_repository_test.go
+5
View File
@@ -26,6 +26,11 @@ func newStorageTestRepository(t *testing.T) *GormStorageTargetRepository {
if err != nil {
t.Fatalf("database.Open returned error: %v", err)
}
sqlDB, err := db.DB()
if err != nil {
t.Fatalf("db.DB returned error: %v", err)
}
t.Cleanup(func() { _ = sqlDB.Close() })
return NewStorageTargetRepository(db)
}
server/internal/service/agent_service.go
+242 -8
View File
@@ -2,15 +2,22 @@ package service
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"math"
"path"
"strings"
"time"
"backupx/server/internal/apperror"
"backupx/server/internal/backup"
"backupx/server/internal/model"
"backupx/server/internal/repository"
"backupx/server/internal/storage"
"backupx/server/internal/storage/codec"
)
@@ -23,6 +30,7 @@ type AgentService struct {
storageRepo repository.StorageTargetRepository
cmdRepo repository.AgentCommandRepository
restoreRepo repository.RestoreRecordRepository
registry *storage.Registry
cipher *codec.ConfigCipher
}
@@ -33,6 +41,7 @@ func NewAgentService(
storageRepo repository.StorageTargetRepository,
cmdRepo repository.AgentCommandRepository,
cipher *codec.ConfigCipher,
registry *storage.Registry,
) *AgentService {
return &AgentService{
nodeRepo: nodeRepo,
@@ -40,6 +49,7 @@ func NewAgentService(
recordRepo: recordRepo,
storageRepo: storageRepo,
cmdRepo: cmdRepo,
registry: registry,
cipher: cipher,
}
}
@@ -145,10 +155,11 @@ type AgentTaskSpec struct {
// AgentStorageTargetConfig 存储目标配置（已解密）
type AgentStorageTargetConfig struct {
ID uint `json:"id"`
Type string `json:"type"`
Name string `json:"name"`
Config json.RawMessage `json:"config"`
ID uint `json:"id"`
Type string `json:"type"`
Name string `json:"name"`
Config json.RawMessage `json:"config"`
TransferMode string `json:"transferMode"`
}
// GetTaskSpec 返回 Agent 执行任务所需的完整规格。
@@ -187,11 +198,22 @@ func (s *AgentService) GetTaskSpec(ctx context.Context, node *model.Node, taskID
if err != nil {
return nil, fmt.Errorf("decrypt storage config: %w", err)
}
transferMode := storage.TransferModeDirect
if strings.EqualFold(target.Type, storage.TypeLocalDisk) {
var localConfig storage.LocalDiskConfig
if err := json.Unmarshal(configRaw, &localConfig); err != nil {
return nil, fmt.Errorf("decode local disk config: %w", err)
}
if localConfig.MasterRelay {
transferMode = storage.TransferModeMasterRelay
}
}
storageTargets = append(storageTargets, AgentStorageTargetConfig{
ID: target.ID,
Type: target.Type,
Name: target.Name,
Config: json.RawMessage(configRaw),
ID: target.ID,
Type: target.Type,
Name: target.Name,
Config: json.RawMessage(configRaw),
TransferMode: transferMode,
})
}
return &AgentTaskSpec{
@@ -214,6 +236,102 @@ func (s *AgentService) GetTaskSpec(ctx context.Context, node *model.Node, taskID
}, nil
}
// UploadArtifact receives a remote Agent artifact as a stream and writes it
// with a provider created on the Master. The first supported use is local_disk,
// whose configured path belongs to the Master rather than the source Agent.
func (s *AgentService) UploadArtifact(ctx context.Context, node *model.Node, recordID, targetID uint, objectKey string, size int64, checksum string, reader io.Reader) error {
if node == nil || reader == nil || s.registry == nil {
return apperror.BadRequest("AGENT_ARTIFACT_INVALID", "中转上传参数不完整", nil)
}
record, err := s.recordRepo.FindByID(ctx, recordID)
if err != nil {
return err
}
if record == nil {
return apperror.New(404, "BACKUP_RECORD_NOT_FOUND", "记录不存在", nil)
}
task, err := s.taskRepo.FindByID(ctx, record.TaskID)
if err != nil {
return err
}
if task == nil || !recordBelongsToNode(record, task, node.ID) {
return apperror.Unauthorized("BACKUP_RECORD_FORBIDDEN", "记录不属于当前节点", nil)
}
if isBackupRecordTerminal(record.Status) {
return apperror.BadRequest("BACKUP_RECORD_TERMINAL", "备份记录已结束，不能继续上传产物", nil)
}
allowedTarget := false
for _, configuredTargetID := range collectTargetIDs(task) {
if configuredTargetID == targetID {
allowedTarget = true
break
}
}
if !allowedTarget {
return apperror.Unauthorized("BACKUP_STORAGE_TARGET_FORBIDDEN", "存储目标不属于该任务", nil)
}
target, err := s.storageRepo.FindByID(ctx, targetID)
if err != nil {
return err
}
if target == nil || !strings.EqualFold(target.Type, storage.TypeLocalDisk) {
return apperror.BadRequest("AGENT_ARTIFACT_RELAY_UNSUPPORTED", "仅 Master 本地磁盘目标需要中转上传", nil)
}
configMap := map[string]any{}
if err := s.cipher.DecryptJSON(target.ConfigCiphertext, &configMap); err != nil {
return fmt.Errorf("decrypt storage config: %w", err)
}
masterRelay, _ := configMap["masterRelay"].(bool)
if !masterRelay {
return apperror.BadRequest("AGENT_ARTIFACT_RELAY_UNSUPPORTED", "该本地磁盘目标配置为 Agent 直接写入", nil)
}
cleanKey, keyErr := s.validateArtifactKey(record, task, objectKey, true)
if keyErr != nil {
return keyErr
}
checksum = strings.TrimSpace(checksum)
checksumBytes, checksumErr := hex.DecodeString(checksum)
if size < 0 || size == math.MaxInt64 || checksumErr != nil || len(checksumBytes) != sha256.Size {
return apperror.BadRequest("AGENT_ARTIFACT_INVALID", "中转上传需要有效的大小和 SHA-256", checksumErr)
}
if target.QuotaBytes > 0 {
usage, usageErr := s.recordRepo.StorageUsage(ctx)
if usageErr != nil {
return fmt.Errorf("read storage usage: %w", usageErr)
}
currentUsed := int64(0)
for _, item := range usage {
if item.StorageTargetID == targetID {
currentUsed = item.TotalSize
break
}
}
if currentUsed >= target.QuotaBytes || size > target.QuotaBytes-currentUsed {
return apperror.BadRequest("BACKUP_STORAGE_QUOTA_EXCEEDED", fmt.Sprintf("超出存储目标配额（当前 %d，新增 %d，配额 %d）", currentUsed, size, target.QuotaBytes), nil)
}
}
provider, err := s.registry.Create(ctx, target.Type, configMap)
if err != nil {
return fmt.Errorf("create master relay provider: %w", err)
}
limited := io.LimitReader(reader, size+1)
hashed := newHashingReader(limited)
metadata := map[string]string{
"taskId": fmt.Sprintf("%d", task.ID),
"recordId": fmt.Sprintf("%d", record.ID),
"sourceNodeId": fmt.Sprintf("%d", node.ID),
"transferMode": storage.TransferModeMasterRelay,
}
if err := provider.Upload(ctx, cleanKey, hashed, size, metadata); err != nil {
return errors.Join(fmt.Errorf("relay artifact to master storage: %w", err), provider.Delete(ctx, cleanKey))
}
if hashed.n != size || !strings.EqualFold(hashed.Sum(), checksum) {
deleteErr := provider.Delete(ctx, cleanKey)
return errors.Join(fmt.Errorf("relayed artifact integrity mismatch: received %d of %d bytes", hashed.n, size), deleteErr)
}
return nil
}
func (s *AgentService) ensureTaskSpecAccess(ctx context.Context, node *model.Node, task *model.BackupTask) error {
if task.NodeID == node.ID {
return nil
@@ -236,6 +354,7 @@ type AgentRecordUpdate struct {
Checksum string `json:"checksum,omitempty"`
StoragePath string `json:"storagePath,omitempty"`
StorageTargetID uint `json:"storageTargetId,omitempty"`
StorageTransferMode string `json:"storageTransferMode,omitempty"`
StorageUploadResults []StorageUploadResultItem `json:"storageUploadResults,omitempty"`
ErrorMessage string `json:"errorMessage,omitempty"`
LogAppend string `json:"logAppend,omitempty"` // 增量日志，追加到 record.log_content
@@ -260,6 +379,99 @@ func (s *AgentService) UpdateRecord(ctx context.Context, node *model.Node, recor
if isBackupRecordTerminal(record.Status) {
return nil
}
allowedTargets := make(map[uint]struct{})
for _, targetID := range collectTargetIDs(task) {
allowedTargets[targetID] = struct{}{}
}
targetCache := make(map[uint]*model.StorageTarget)
validateTransferMode := func(targetID uint, transferMode string) (string, error) {
if _, ok := allowedTargets[targetID]; !ok {
return "", apperror.Unauthorized("BACKUP_STORAGE_TARGET_FORBIDDEN", "存储目标不属于该任务", nil)
}
target := targetCache[targetID]
if target == nil {
var findErr error
target, findErr = s.storageRepo.FindByID(ctx, targetID)
if findErr != nil {
return "", findErr
}
if target == nil {
return "", apperror.BadRequest("BACKUP_STORAGE_TARGET_INVALID", "存储目标不存在", nil)
}
targetCache[targetID] = target
}
expectedMode := storage.TransferModeDirect
if strings.EqualFold(target.Type, storage.TypeLocalDisk) {
var localConfig storage.LocalDiskConfig
if err := s.cipher.DecryptJSON(target.ConfigCiphertext, &localConfig); err != nil {
return "", fmt.Errorf("decrypt storage config: %w", err)
}
if localConfig.MasterRelay {
expectedMode = storage.TransferModeMasterRelay
}
}
if transferMode != "" && transferMode != expectedMode {
return "", apperror.BadRequest("AGENT_STORAGE_TRANSFER_MODE_INVALID", "Agent 上报的存储传输模式与目标配置不一致", nil)
}
return expectedMode, nil
}
selectedTransferMode := ""
if update.StorageTargetID > 0 {
if _, ok := allowedTargets[update.StorageTargetID]; !ok {
return apperror.Unauthorized("BACKUP_STORAGE_TARGET_FORBIDDEN", "存储目标不属于该任务", nil)
}
var modeErr error
selectedTransferMode, modeErr = validateTransferMode(update.StorageTargetID, update.StorageTransferMode)
if modeErr != nil {
return modeErr
}
} else if update.StorageTransferMode != "" {
return apperror.BadRequest("AGENT_STORAGE_TRANSFER_MODE_INVALID", "传输模式缺少对应的存储目标", nil)
}
for index := range update.StorageUploadResults {
result := &update.StorageUploadResults[index]
expectedMode, err := validateTransferMode(result.StorageTargetID, result.TransferMode)
if err != nil {
return err
}
if result.FileSize < 0 || (result.Status != "" && result.Status != "success" && result.Status != "failed") {
return apperror.BadRequest("AGENT_ARTIFACT_INVALID", "Agent 上报的存储结果无效", nil)
}
if result.StoragePath != "" {
normalizedPath, err := s.validateArtifactKey(record, task, result.StoragePath, expectedMode == storage.TransferModeMasterRelay)
if err != nil {
return err
}
result.StoragePath = normalizedPath
}
result.TransferMode = expectedMode
}
if update.StoragePath != "" {
if update.StorageTargetID == 0 {
return apperror.BadRequest("AGENT_ARTIFACT_INVALID_PATH", "存储路径缺少对应的存储目标", nil)
}
cleanStoragePath, err := s.validateArtifactKey(record, task, update.StoragePath, selectedTransferMode == storage.TransferModeMasterRelay)
if err != nil {
return err
}
if update.FileName != "" && path.Base(cleanStoragePath) != update.FileName {
return apperror.BadRequest("AGENT_ARTIFACT_INVALID_PATH", "Agent 上报的文件名与存储路径不一致", nil)
}
update.StoragePath = cleanStoragePath
}
if update.Status != "" && update.Status != model.BackupRecordStatusRunning && update.Status != model.BackupRecordStatusSuccess && update.Status != model.BackupRecordStatusFailed {
return apperror.BadRequest("BACKUP_RECORD_STATUS_INVALID", "Agent 上报的备份状态无效", nil)
}
if update.FileSize < 0 || (update.FileName != "" && (path.Base(update.FileName) != update.FileName || strings.Contains(update.FileName, "\\"))) {
return apperror.BadRequest("AGENT_ARTIFACT_INVALID", "Agent 上报的备份文件信息无效", nil)
}
if update.Checksum != "" {
checksumBytes, checksumErr := hex.DecodeString(strings.TrimSpace(update.Checksum))
if checksumErr != nil || len(checksumBytes) != sha256.Size {
return apperror.BadRequest("AGENT_ARTIFACT_INVALID", "Agent 上报的 SHA-256 无效", checksumErr)
}
update.Checksum = strings.ToLower(strings.TrimSpace(update.Checksum))
}
if update.Status != "" {
record.Status = update.Status
}
@@ -277,6 +489,7 @@ func (s *AgentService) UpdateRecord(ctx context.Context, node *model.Node, recor
}
if update.StorageTargetID > 0 {
record.StorageTargetID = update.StorageTargetID
record.StorageTransferMode = selectedTransferMode
}
if len(update.StorageUploadResults) > 0 {
if resultsJSON, marshalErr := json.Marshal(update.StorageUploadResults); marshalErr == nil {
@@ -312,6 +525,27 @@ func (s *AgentService) UpdateRecord(ctx context.Context, node *model.Node, recor
return nil
}
func (s *AgentService) validateArtifactKey(record *model.BackupRecord, task *model.BackupTask, objectKey string, requireRecordNamespace bool) (string, error) {
if record == nil || task == nil {
return "", apperror.BadRequest("AGENT_ARTIFACT_INVALID_PATH", "无法确认中转对象归属", nil)
}
rawKey := objectKey
cleanKey := path.Clean(rawKey)
fileName := path.Base(cleanKey)
if rawKey == "" || strings.TrimSpace(rawKey) != rawKey || cleanKey == "." || path.IsAbs(cleanKey) || strings.HasPrefix(cleanKey, "../") || cleanKey != rawKey || strings.Contains(rawKey, "\\") || fileName == "." || fileName == "/" {
return "", apperror.BadRequest("AGENT_ARTIFACT_INVALID_PATH", "中转上传对象路径不安全", nil)
}
expectedKey := backup.BuildRecordStorageKey(task.Type, record.StartedAt, record.ID, fileName)
if requireRecordNamespace {
legacyKey := backup.BuildStorageKey(task.Type, record.StartedAt, fileName)
if cleanKey != expectedKey && cleanKey != legacyKey {
return "", apperror.BadRequest("AGENT_ARTIFACT_INVALID_PATH", "中转上传对象不属于当前备份记录", nil)
}
return expectedKey, nil
}
return cleanKey, nil
}
func recordBelongsToNode(record *model.BackupRecord, task *model.BackupTask, nodeID uint) bool {
if record.NodeID != 0 {
return record.NodeID == nodeID
server/internal/service/agent_service_test.go
+182 -12
View File
@@ -1,19 +1,26 @@
package service
import (
"bytes"
"context"
"crypto/sha256"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"backupx/server/internal/backup"
"backupx/server/internal/config"
"backupx/server/internal/database"
"backupx/server/internal/logger"
"backupx/server/internal/model"
"backupx/server/internal/repository"
"backupx/server/internal/storage"
"backupx/server/internal/storage/codec"
storageRclone "backupx/server/internal/storage/rclone"
"gorm.io/gorm"
)
@@ -42,7 +49,7 @@ func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repo
if err := nodeRepo.Create(context.Background(), other); err != nil {
t.Fatalf("create other node: %v", err)
}
targetConfig, err := cipher.EncryptJSON(map[string]any{"basePath": t.TempDir()})
targetConfig, err := cipher.EncryptJSON(map[string]any{"basePath": t.TempDir(), "masterRelay": true})
if err != nil {
t.Fatalf("EncryptJSON returned error: %v", err)
}
@@ -76,7 +83,8 @@ func newAgentServicePoolTestHarness(t *testing.T) (*AgentService, *gorm.DB, repo
if err := recordRepo.Create(context.Background(), record); err != nil {
t.Fatalf("create record: %v", err)
}
return NewAgentService(nodeRepo, taskRepo, recordRepo, storageRepo, cmdRepo, cipher), db, recordRepo, cmdRepo, owner, other
storageRegistry := storage.NewRegistry(storageRclone.NewLocalDiskFactory())
return NewAgentService(nodeRepo, taskRepo, recordRepo, storageRepo, cmdRepo, cipher, storageRegistry), db, recordRepo, cmdRepo, owner, other
}
func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.T) {
@@ -90,19 +98,27 @@ func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.
if spec.TaskID != 1 || len(spec.StorageTargets) != 1 {
t.Fatalf("unexpected spec: %#v", spec)
}
if spec.StorageTargets[0].TransferMode != storage.TransferModeMasterRelay {
t.Fatalf("expected local disk to use Master relay, got %#v", spec.StorageTargets[0])
}
if _, err := svc.GetTaskSpec(ctx, other, 1); err == nil {
t.Fatal("expected non-owner node to be forbidden from pooled task spec")
}
record, err := records.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
storagePath := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID, "backup.tar.gz")
if err := svc.UpdateRecord(ctx, owner, 1, AgentRecordUpdate{
Status: model.BackupRecordStatusSuccess,
FileName: "backup.tar.gz",
FileSize: 123,
StoragePath: "tasks/1/backup.tar.gz",
StorageTargetID: 2,
Status: model.BackupRecordStatusSuccess,
FileName: "backup.tar.gz",
FileSize: 123,
StoragePath: storagePath,
StorageTargetID: 1,
StorageTransferMode: storage.TransferModeMasterRelay,
StorageUploadResults: []StorageUploadResultItem{
{StorageTargetID: 1, StorageTargetName: "first", Status: "failed", Error: "boom"},
{StorageTargetID: 2, StorageTargetName: "second", Status: "success", StoragePath: "tasks/1/backup.tar.gz", FileSize: 123},
{StorageTargetID: 1, StorageTargetName: "local", Status: "success", StoragePath: storagePath, FileSize: 123, TransferMode: storage.TransferModeMasterRelay},
},
}); err != nil {
t.Fatalf("owner UpdateRecord returned error: %v", err)
@@ -114,10 +130,13 @@ func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.
if updated.Status != model.BackupRecordStatusSuccess || updated.NodeID != owner.ID {
t.Fatalf("unexpected updated record: %#v", updated)
}
if updated.StorageTargetID != 2 {
t.Fatalf("expected successful storage target id 2, got %d", updated.StorageTargetID)
if updated.StorageTargetID != 1 {
t.Fatalf("expected successful storage target id 1, got %d", updated.StorageTargetID)
}
if !strings.Contains(updated.StorageUploadResults, `"storageTargetName":"second"`) {
if updated.StorageTransferMode != storage.TransferModeMasterRelay {
t.Fatalf("expected Master relay transfer mode, got %q", updated.StorageTransferMode)
}
if !strings.Contains(updated.StorageUploadResults, `"storageTargetName":"local"`) {
t.Fatalf("expected upload results to be persisted, got %q", updated.StorageUploadResults)
}
if err := svc.UpdateRecord(ctx, other, 1, AgentRecordUpdate{LogAppend: "bad"}); err == nil {
@@ -125,6 +144,157 @@ func TestAgentServicePooledTaskUsesRecordNodeForSpecAndRecordUpdates(t *testing.
}
}
func TestAgentServiceRelaysRemoteArtifactToMasterLocalDisk(t *testing.T) {
svc, _, records, _, owner, other := newAgentServicePoolTestHarness(t)
ctx := context.Background()
payload := []byte("artifact from remote source server")
digest := sha256.Sum256(payload)
checksum := fmt.Sprintf("%x", digest[:])
record, err := records.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
objectKey := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID, "remote-source.tar")
if err := svc.UploadArtifact(ctx, owner, 1, 1, objectKey, int64(len(payload)), checksum, bytes.NewReader(payload)); err != nil {
t.Fatalf("UploadArtifact returned error: %v", err)
}
target, err := svc.storageRepo.FindByID(ctx, 1)
if err != nil || target == nil {
t.Fatalf("FindByID target: target=%#v err=%v", target, err)
}
config := map[string]any{}
if err := svc.cipher.DecryptJSON(target.ConfigCiphertext, &config); err != nil {
t.Fatalf("DecryptJSON target config: %v", err)
}
basePath, _ := config["basePath"].(string)
stored, err := os.ReadFile(filepath.Join(basePath, filepath.FromSlash(objectKey)))
if err != nil {
t.Fatalf("read relayed artifact: %v", err)
}
if !bytes.Equal(stored, payload) {
t.Fatalf("relayed artifact differs: got %q", stored)
}
if err := svc.UploadArtifact(ctx, other, 1, 1, objectKey, int64(len(payload)), checksum, bytes.NewReader(payload)); err == nil {
t.Fatal("expected a different node to be forbidden from relaying the artifact")
}
legacyPayload := []byte("artifact from an older Agent")
legacyDigest := sha256.Sum256(legacyPayload)
legacyKey := backup.BuildStorageKey("file", record.StartedAt, "legacy-agent.tar")
canonicalKey := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID, "legacy-agent.tar")
if err := svc.UploadArtifact(ctx, owner, record.ID, target.ID, legacyKey, int64(len(legacyPayload)), fmt.Sprintf("%x", legacyDigest[:]), bytes.NewReader(legacyPayload)); err != nil {
t.Fatalf("UploadArtifact legacy key returned error: %v", err)
}
stored, err = os.ReadFile(filepath.Join(basePath, filepath.FromSlash(canonicalKey)))
if err != nil || !bytes.Equal(stored, legacyPayload) {
t.Fatalf("legacy Agent artifact was not normalized: data=%q err=%v", stored, err)
}
if _, err := os.Stat(filepath.Join(basePath, filepath.FromSlash(legacyKey))); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("legacy object key should not be written directly: %v", err)
}
if err := svc.UpdateRecord(ctx, owner, record.ID, AgentRecordUpdate{
Status: model.BackupRecordStatusSuccess,
FileName: "legacy-agent.tar",
FileSize: int64(len(legacyPayload)),
Checksum: fmt.Sprintf("%x", legacyDigest[:]),
StoragePath: legacyKey,
StorageTargetID: target.ID,
StorageUploadResults: []StorageUploadResultItem{{
StorageTargetID: target.ID,
Status: "success",
StoragePath: legacyKey,
FileSize: int64(len(legacyPayload)),
}},
}); err != nil {
t.Fatalf("UpdateRecord legacy key returned error: %v", err)
}
updated, err := records.FindByID(ctx, record.ID)
if err != nil {
t.Fatalf("FindByID updated record returned error: %v", err)
}
if updated.StoragePath != canonicalKey || updated.StorageTransferMode != storage.TransferModeMasterRelay || !strings.Contains(updated.StorageUploadResults, canonicalKey) {
t.Fatalf("legacy Agent record was not normalized: %#v", updated)
}
}
func TestAgentServiceRejectsArtifactOutsideRecordNamespace(t *testing.T) {
svc, _, records, _, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
record, err := records.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID record returned error: %v", err)
}
target, err := svc.storageRepo.FindByID(ctx, 1)
if err != nil || target == nil {
t.Fatalf("FindByID target: target=%#v err=%v", target, err)
}
config := map[string]any{}
if err := svc.cipher.DecryptJSON(target.ConfigCiphertext, &config); err != nil {
t.Fatalf("DecryptJSON target config: %v", err)
}
basePath, _ := config["basePath"].(string)
victimKey := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID+1, "victim.tar")
victimPath := filepath.Join(basePath, filepath.FromSlash(victimKey))
if err := os.MkdirAll(filepath.Dir(victimPath), 0o755); err != nil {
t.Fatalf("MkdirAll victim parent: %v", err)
}
if err := os.WriteFile(victimPath, []byte("keep me"), 0o600); err != nil {
t.Fatalf("WriteFile victim: %v", err)
}
payload := []byte("overwrite")
digest := sha256.Sum256(payload)
if err := svc.UploadArtifact(ctx, owner, record.ID, target.ID, victimKey, int64(len(payload)), fmt.Sprintf("%x", digest[:]), bytes.NewReader(payload)); err == nil {
t.Fatal("expected another record namespace to be rejected")
}
stored, err := os.ReadFile(victimPath)
if err != nil {
t.Fatalf("ReadFile victim: %v", err)
}
if string(stored) != "keep me" {
t.Fatalf("victim object changed: %q", stored)
}
if err := svc.UpdateRecord(ctx, owner, record.ID, AgentRecordUpdate{StoragePath: victimKey, StorageTargetID: target.ID, StorageTransferMode: storage.TransferModeMasterRelay}); err == nil {
t.Fatal("expected another record namespace in status update to be rejected")
}
}
func TestAgentServiceKeepsExistingLocalDiskTargetsAgentLocal(t *testing.T) {
svc, _, records, _, owner, _ := newAgentServicePoolTestHarness(t)
ctx := context.Background()
target, err := svc.storageRepo.FindByID(ctx, 1)
if err != nil || target == nil {
t.Fatalf("FindByID target: target=%#v err=%v", target, err)
}
legacyConfig, err := svc.cipher.EncryptJSON(map[string]any{"basePath": t.TempDir()})
if err != nil {
t.Fatalf("EncryptJSON legacy target: %v", err)
}
target.ConfigCiphertext = legacyConfig
if err := svc.storageRepo.Update(ctx, target); err != nil {
t.Fatalf("Update legacy target: %v", err)
}
spec, err := svc.GetTaskSpec(ctx, owner, 1)
if err != nil {
t.Fatalf("GetTaskSpec returned error: %v", err)
}
if len(spec.StorageTargets) != 1 || spec.StorageTargets[0].TransferMode != storage.TransferModeDirect {
t.Fatalf("expected legacy local disk to stay Agent-local, got %#v", spec.StorageTargets)
}
payload := []byte("must not be relayed")
digest := sha256.Sum256(payload)
record, findErr := records.FindByID(ctx, 1)
if findErr != nil {
t.Fatalf("FindByID record returned error: %v", findErr)
}
objectKey := backup.BuildRecordStorageKey("file", record.StartedAt, record.ID, "legacy.tar")
err = svc.UploadArtifact(ctx, owner, 1, 1, objectKey, int64(len(payload)), fmt.Sprintf("%x", digest[:]), bytes.NewReader(payload))
if err == nil {
t.Fatal("expected relay upload to be rejected for an Agent-local target")
}
}
func TestAgentServiceUpdateRecordRefreshesTaskSummaryOnTerminalStatus(t *testing.T) {
for _, status := range []string{model.BackupRecordStatusSuccess, model.BackupRecordStatusFailed} {
t.Run(status, func(t *testing.T) {
server/internal/service/backup_execution_service.go
+309 -4
View File
@@ -5,11 +5,13 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"hash"
"io"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"time"
@@ -49,6 +51,7 @@ type StorageUploadResultItem struct {
Status string `json:"status"`
StoragePath string `json:"storagePath,omitempty"`
FileSize int64 `json:"fileSize,omitempty"`
TransferMode string `json:"transferMode,omitempty"`
Error string `json:"error,omitempty"`
}
@@ -62,6 +65,17 @@ type DownloadedArtifact struct {
Reader io.ReadCloser
}
type temporaryArtifactReader struct {
*os.File
directory string
}
func (r *temporaryArtifactReader) Close() error {
closeErr := r.File.Close()
removeErr := os.RemoveAll(r.directory)
return errors.Join(closeErr, removeErr)
}
// collectTargetIDs 获取任务关联的所有存储目标 ID
func collectTargetIDs(task *model.BackupTask) []uint {
if len(task.StorageTargets) > 0 {
@@ -102,6 +116,10 @@ type BackupExecutionService struct {
bandwidthLimit string // rclone 带宽限制（全局默认，节点配置可覆盖）
metrics *metrics.Metrics
taskLocks sync.Map
// repositoryLocks serializes immutable index updates per storage target.
// Repository mode is intentionally single-writer in v1 to avoid orphaned
// duplicate packs when two local tasks discover the same missing chunk.
repositoryLocks sync.Map
}
// SetMetrics 注入 Prometheus 采集器。nil 时所有埋点退化为 no-op。
@@ -211,6 +229,25 @@ func (s *BackupExecutionService) DownloadRecord(ctx context.Context, recordID ui
if err != nil {
return nil, err
}
if record.BackupKind == model.BackupKindRepository {
tempDir, err := os.MkdirTemp(s.tempDir, "repository-download-*")
if err != nil {
return nil, apperror.Internal("BACKUP_RECORD_DOWNLOAD_FAILED", "无法创建 CDC 导出目录", err)
}
exportName := fmt.Sprintf("backupx-record-%d.tar", record.ID)
exportPath := filepath.Join(tempDir, exportName)
store := backup.NewRepositoryStore(s.cipher.Key())
if err := store.ExportTar(ctx, provider, record.StoragePath, record.Checksum, exportPath); err != nil {
cleanupErr := os.RemoveAll(tempDir)
return nil, apperror.Internal("BACKUP_RECORD_DOWNLOAD_FAILED", "无法从 CDC 仓库导出归档", errors.Join(err, cleanupErr))
}
file, err := os.Open(exportPath)
if err != nil {
cleanupErr := os.RemoveAll(tempDir)
return nil, apperror.Internal("BACKUP_RECORD_DOWNLOAD_FAILED", "无法打开 CDC 导出归档", errors.Join(err, cleanupErr))
}
return &DownloadedArtifact{FileName: exportName, Reader: &temporaryArtifactReader{File: file, directory: tempDir}}, nil
}
reader, err := provider.Download(ctx, record.StoragePath)
if err != nil {
return nil, apperror.Internal("BACKUP_RECORD_DOWNLOAD_FAILED", "无法下载备份文件", err)
@@ -234,6 +271,16 @@ func (s *BackupExecutionService) RestoreRecord(ctx context.Context, recordID uin
if task == nil {
return apperror.New(404, "BACKUP_TASK_NOT_FOUND", "关联的备份任务不存在，无法执行恢复", fmt.Errorf("backup task %d not found", record.TaskID))
}
if record.BackupKind == model.BackupKindRepository {
spec, specErr := s.buildTaskSpec(task, record.StartedAt)
if specErr != nil {
return specErr
}
if err := backup.NewRepositoryStore(s.cipher.Key()).Restore(ctx, provider, record.StoragePath, record.Checksum, spec, backup.NopLogWriter{}); err != nil {
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "从 CDC 仓库恢复备份失败", err)
}
return nil
}
tempDir, err := os.MkdirTemp("", "backupx-restore-*")
if err != nil {
return apperror.Internal("BACKUP_RECORD_RESTORE_FAILED", "无法创建恢复目录", err)
@@ -291,6 +338,58 @@ func (s *BackupExecutionService) DeleteRecord(ctx context.Context, recordID uint
fmt.Sprintf("该全量备份仍有 %d 个差异备份依赖它，删除会导致这些差异无法恢复。请先删除相关差异备份或等待其过期。", deps), nil)
}
}
if record.BackupKind == model.BackupKindRepository {
copies := []StorageUploadResultItem{{
StorageTargetID: record.StorageTargetID,
Status: model.BackupRecordStatusSuccess,
StoragePath: record.StoragePath,
}}
if strings.TrimSpace(record.StorageUploadResults) != "" {
if err := json.Unmarshal([]byte(record.StorageUploadResults), &copies); err != nil {
return apperror.Internal("BACKUP_RECORD_DELETE_FAILED", "无法解析 CDC 仓库副本信息，已停止删除以避免遗留数据", err)
}
}
copyPaths := make(map[uint]string, len(copies))
for _, copy := range copies {
if strings.EqualFold(copy.Status, model.BackupRecordStatusSuccess) && strings.TrimSpace(copy.StoragePath) != "" {
copyPaths[copy.StorageTargetID] = copy.StoragePath
}
}
targetIDs := make([]uint, 0, len(copyPaths))
for targetID := range copyPaths {
targetIDs = append(targetIDs, targetID)
}
sort.Slice(targetIDs, func(i, j int) bool { return targetIDs[i] < targetIDs[j] })
unlocks := make([]func(), 0, len(targetIDs))
for _, targetID := range targetIDs {
unlocks = append(unlocks, s.acquireRepositoryLock(targetID))
}
defer func() {
for index := len(unlocks) - 1; index >= 0; index-- {
unlocks[index]()
}
}()
providers := make(map[uint]storage.StorageProvider, len(targetIDs))
for _, targetID := range targetIDs {
provider, resolveErr := s.resolveProvider(ctx, targetID)
if resolveErr != nil {
return resolveErr
}
if deleteErr := provider.Delete(ctx, copyPaths[targetID]); deleteErr != nil {
return apperror.Internal("BACKUP_RECORD_DELETE_FAILED", "无法删除 CDC 仓库快照", deleteErr)
}
providers[targetID] = provider
}
for targetID, provider := range providers {
if _, pruneErr := backup.NewRepositoryStore(s.cipher.Key()).Prune(ctx, provider); pruneErr != nil {
return apperror.Internal("BACKUP_REPOSITORY_PRUNE_FAILED", fmt.Sprintf("无法清理存储目标 %d 的 CDC 仓库；记录暂时保留以便重试", targetID), pruneErr)
}
}
if err := s.records.Delete(ctx, recordID); err != nil {
return apperror.Internal("BACKUP_RECORD_DELETE_FAILED", "无法删除备份记录", err)
}
return nil
}
if remote, err := s.deleteRemoteLocalDiskObject(ctx, record); err != nil {
return err
} else if !remote && strings.TrimSpace(record.StoragePath) != "" {
@@ -312,6 +411,9 @@ func (s *BackupExecutionService) deleteRemoteLocalDiskObject(ctx context.Context
if strings.TrimSpace(record.StoragePath) == "" || s.nodeRepo == nil {
return false, nil
}
if record.StorageTransferMode == storage.TransferModeMasterRelay {
return false, nil
}
node, err := s.nodeRepo.FindByID(ctx, record.NodeID)
if err != nil || node == nil || node.IsLocal {
return false, nil
@@ -384,6 +486,9 @@ func (s *BackupExecutionService) startTask(ctx context.Context, id uint, async b
return nil, perr
}
}
if strings.EqualFold(task.BackupMode, model.BackupModeRepository) && s.resolveRemoteNode(ctx, resolvedNodeID) != nil {
return nil, apperror.BadRequest("BACKUP_TASK_REPOSITORY_REMOTE_UNSUPPORTED", "CDC 仓库模式当前仅支持 Master 本机单写者执行", nil)
}
startedAt := s.now()
// 取第一个存储目标 ID 做兼容
primaryTargetID := task.StorageTargetID
@@ -630,6 +735,141 @@ func (s *BackupExecutionService) resolveDifferentialBase(ctx context.Context, ta
return 0, backup.Manifest{}, false
}
type repositoryTaskResult struct {
fileName string
logicalSize int64
checksum string
storagePath string
storageTargetID uint
manifestJSON string
uploadResults []StorageUploadResultItem
providers map[uint]storage.StorageProvider
}
func (s *BackupExecutionService) executeRepositoryTask(ctx context.Context, task *model.BackupTask, recordID uint, startedAt time.Time, spec backup.TaskSpec, logger *backup.ExecutionLogger) (*repositoryTaskResult, error) {
store := backup.NewRepositoryStore(s.cipher.Key())
plan, err := store.BuildPlan(ctx, spec, logger)
if err != nil {
return nil, err
}
defer func() {
if closeErr := plan.Close(); closeErr != nil {
logger.Warnf("清理 CDC 临时计划失败：%v", closeErr)
}
}()
manifestBytes, err := backup.EncodeManifest(plan.Manifest)
if err != nil {
return nil, fmt.Errorf("encode repository manifest: %w", err)
}
targetIDs := collectTargetIDs(task)
if len(targetIDs) == 0 {
return nil, fmt.Errorf("没有关联的存储目标")
}
storageUsage, usageErr := s.storageUsageSnapshot(ctx)
if usageErr != nil {
logger.Warnf("读取存储目标用量失败，跳过本次软配额校验：%v", usageErr)
storageUsage = map[uint]int64{}
}
snapshotKey := store.SnapshotKey(task.ID, recordID, startedAt)
result := &repositoryTaskResult{
fileName: filepath.Base(snapshotKey),
logicalSize: plan.LogicalSize,
storagePath: snapshotKey,
manifestJSON: string(manifestBytes),
uploadResults: make([]StorageUploadResultItem, 0, len(targetIDs)),
providers: make(map[uint]storage.StorageProvider),
}
var failures []string
for _, targetID := range targetIDs {
target, findErr := s.targets.FindByID(ctx, targetID)
targetName := fmt.Sprintf("target-%d", targetID)
if findErr == nil && target != nil {
targetName = target.Name
}
if findErr != nil || target == nil {
message := "存储目标不存在"
if findErr != nil {
message = findErr.Error()
}
result.uploadResults = append(result.uploadResults, StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: message})
failures = append(failures, fmt.Sprintf("%s: %s", targetName, message))
continue
}
provider, resolveErr := s.resolveProviderForNode(ctx, targetID, task.NodeID)
if resolveErr != nil {
result.uploadResults = append(result.uploadResults, StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: resolveErr.Error()})
failures = append(failures, fmt.Sprintf("%s: %v", targetName, resolveErr))
continue
}
logger.Infof("同步 CDC 仓库到存储目标：%s", targetName)
unlock := s.acquireRepositoryLock(targetID)
estimatedSize, estimateErr := store.EstimateUploadSize(ctx, provider, plan)
if estimateErr != nil {
unlock()
result.uploadResults = append(result.uploadResults, StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: estimateErr.Error()})
failures = append(failures, fmt.Sprintf("%s: %v", targetName, estimateErr))
continue
}
if target.QuotaBytes > 0 && storageUsage[targetID]+estimatedSize > target.QuotaBytes {
unlock()
message := fmt.Sprintf("超出存储目标配额（%d + 预计 %d > %d）", storageUsage[targetID], estimatedSize, target.QuotaBytes)
result.uploadResults = append(result.uploadResults, StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: message})
failures = append(failures, fmt.Sprintf("%s: %s", targetName, message))
continue
}
upload, uploadErr := store.Upload(ctx, provider, plan, snapshotKey)
unlock()
if uploadErr != nil {
result.uploadResults = append(result.uploadResults, StorageUploadResultItem{StorageTargetID: targetID, StorageTargetName: targetName, Status: "failed", Error: uploadErr.Error()})
failures = append(failures, fmt.Sprintf("%s: %v", targetName, uploadErr))
logger.Warnf("存储目标 %s CDC 仓库同步失败：%v", targetName, uploadErr)
continue
}
result.uploadResults = append(result.uploadResults, StorageUploadResultItem{
StorageTargetID: targetID, StorageTargetName: targetName, Status: "success",
StoragePath: upload.SnapshotKey, FileSize: upload.UploadedBytes,
})
result.providers[targetID] = provider
if result.storageTargetID == 0 {
result.storageTargetID = targetID
result.checksum = upload.Checksum
}
logger.Infof("存储目标 %s CDC 同步完成：新块 %d/%d，复用 %d bytes，实际上传 %d bytes", targetName, upload.NewChunks, upload.UniqueChunks, upload.ReusedBytes, upload.UploadedBytes)
}
if result.storageTargetID == 0 {
return nil, fmt.Errorf("所有存储目标 CDC 仓库同步均失败：%s", strings.Join(failures, "; "))
}
if len(failures) > 0 {
logger.Warnf("部分存储目标 CDC 仓库同步失败：%s", strings.Join(failures, "; "))
}
if s.dependentsResolver != nil {
go func(upstreamID uint, upstreamName string) {
dependents, resolveErr := s.dependentsResolver.TriggerDependents(context.Background(), upstreamID)
if resolveErr != nil {
logger.Warnf("解析任务 %s 的下游依赖失败：%v", upstreamName, resolveErr)
return
}
for _, dependentID := range dependents {
if _, runErr := s.RunTaskByID(context.Background(), dependentID); runErr != nil {
logger.Warnf("触发下游任务 #%d 失败（上游: %s）：%v", dependentID, upstreamName, runErr)
} else {
logger.Infof("已触发下游任务 #%d（上游: %s）", dependentID, upstreamName)
}
}
}(task.ID, task.Name)
}
return result, nil
}
func (s *BackupExecutionService) acquireRepositoryLock(targetID uint) func() {
created := &sync.Mutex{}
actual, _ := s.repositoryLocks.LoadOrStore(targetID, created)
lock := actual.(*sync.Mutex)
lock.Lock()
return lock.Unlock
}
func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.BackupTask, recordID uint, startedAt time.Time) {
// 节点级并发限流：当任务绑定节点且节点配置了 MaxConcurrent>0，
// 该节点上所有任务共享一个节点专属 semaphore，互相排队
@@ -658,18 +898,33 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
backupKind := model.BackupKindFull
var baseRecordID uint
var manifestJSON string
var repositoryProviders map[uint]storage.StorageProvider
completeRecord := func() {
readyForRepositoryRetention := status == model.BackupRecordStatusSuccess
if finalizeErr := s.finalizeRecord(ctx, task, recordID, startedAt, status, errMessage, logger.String(), fileName, fileSize, checksum, storagePath, selectedStorageTargetID); finalizeErr != nil {
logger.Errorf("写回备份记录失败：%v", finalizeErr)
readyForRepositoryRetention = false
}
// 采集任务执行结果到 Prometheus（耗时 + 产出字节 + 状态计数）
s.metrics.ObserveTaskRun(task.Type, status, time.Since(startedAt).Seconds(), fileSize)
// 写入多目标上传结果
if len(uploadResults) > 0 {
if resultsJSON, marshalErr := json.Marshal(uploadResults); marshalErr == nil {
if record, findErr := s.records.FindByID(ctx, recordID); findErr == nil && record != nil {
record.StorageUploadResults = string(resultsJSON)
_ = s.records.Update(ctx, record)
resultsJSON, marshalErr := json.Marshal(uploadResults)
if marshalErr != nil {
logger.Warnf("序列化多目标上传结果失败：%v", marshalErr)
readyForRepositoryRetention = false
} else if record, findErr := s.records.FindByID(ctx, recordID); findErr != nil || record == nil {
if findErr != nil {
logger.Warnf("读取备份记录以写回多目标结果失败：%v", findErr)
} else {
logger.Warnf("备份记录 #%d 不存在，无法写回多目标结果", recordID)
}
readyForRepositoryRetention = false
} else {
record.StorageUploadResults = string(resultsJSON)
if updateErr := s.records.Update(ctx, record); updateErr != nil {
logger.Warnf("写回多目标上传结果失败：%v", updateErr)
readyForRepositoryRetention = false
}
}
}
@@ -681,6 +936,36 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
record.Manifest = manifestJSON
if updErr := s.records.Update(ctx, record); updErr != nil {
logger.Warnf("写回差异链信息失败：%v", updErr)
readyForRepositoryRetention = false
}
} else {
if findErr != nil {
logger.Warnf("读取备份记录以写回备份类型失败：%v", findErr)
} else {
logger.Warnf("备份记录 #%d 不存在，无法写回备份类型", recordID)
}
readyForRepositoryRetention = false
}
}
if readyForRepositoryRetention && backupKind == model.BackupKindRepository && s.retention != nil && len(repositoryProviders) > 0 {
targetIDs := make([]uint, 0, len(repositoryProviders))
for targetID := range repositoryProviders {
targetIDs = append(targetIDs, targetID)
}
sort.Slice(targetIDs, func(i, j int) bool { return targetIDs[i] < targetIDs[j] })
unlocks := make([]func(), 0, len(targetIDs))
for _, targetID := range targetIDs {
unlocks = append(unlocks, s.acquireRepositoryLock(targetID))
}
cleanupResult, cleanupErr := s.retention.CleanupProviders(ctx, task, repositoryProviders)
for index := len(unlocks) - 1; index >= 0; index-- {
unlocks[index]()
}
if cleanupErr != nil {
logger.Warnf("执行 CDC 仓库保留策略失败：%v", cleanupErr)
} else {
for _, warning := range cleanupResult.Warnings {
logger.Warnf("CDC 仓库保留策略警告：%s", warning)
}
}
}
@@ -701,6 +986,26 @@ func (s *BackupExecutionService) executeTask(ctx context.Context, task *model.Ba
logger.Errorf("构建任务运行时配置失败：%v", err)
return
}
if task.Type == model.BackupTaskTypeFile && strings.EqualFold(task.BackupMode, model.BackupModeRepository) {
backupKind = model.BackupKindRepository
repositoryResult, repositoryErr := s.executeRepositoryTask(ctx, task, recordID, startedAt, spec, logger)
if repositoryErr != nil {
errMessage = repositoryErr.Error()
logger.Errorf("执行 CDC 仓库备份失败：%v", repositoryErr)
return
}
fileName = repositoryResult.fileName
fileSize = repositoryResult.logicalSize
checksum = repositoryResult.checksum
storagePath = repositoryResult.storagePath
selectedStorageTargetID = repositoryResult.storageTargetID
uploadResults = repositoryResult.uploadResults
repositoryProviders = repositoryResult.providers
manifestJSON = repositoryResult.manifestJSON
status = model.BackupRecordStatusSuccess
logger.Infof("CDC 仓库备份执行完成")
return
}
// 差异备份：解析基线全量，命中则切换为差异模式（仅本机文件任务）。
if baseID, baseManifest, ok := s.resolveDifferentialBase(ctx, task); ok {
spec.Differential = true
server/internal/service/backup_execution_service_test.go
+153
View File
@@ -1,6 +1,7 @@
package service
import (
"bytes"
"context"
"errors"
"fmt"
@@ -188,6 +189,108 @@ func TestBackupExecutionServiceSQLiteBackupRemainsFull(t *testing.T) {
}
}
func TestBackupExecutionServiceRepositoryModeRoundTrip(t *testing.T) {
executionService, recordService, tasks, _, records, sourceDir, storageDir := newExecutionTestServices(t)
ctx := context.Background()
task, err := tasks.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID task returned error: %v", err)
}
task.BackupMode = model.BackupModeRepository
task.Compression = "zstd"
if err := tasks.Update(ctx, task); err != nil {
t.Fatalf("Update repository task returned error: %v", err)
}
large := make([]byte, 4<<20)
for index := range large {
large[index] = byte((index * 31) % 251)
}
largePath := filepath.Join(sourceDir, "large.bin")
if err := os.WriteFile(largePath, large, 0o640); err != nil {
t.Fatalf("write large fixture: %v", err)
}
first, err := executionService.RunTaskByIDSync(ctx, task.ID)
if err != nil {
t.Fatalf("first repository backup returned error: %v", err)
}
if first.Status != model.BackupRecordStatusSuccess || first.BackupKind != model.BackupKindRepository {
t.Fatalf("unexpected first repository record: %#v", first)
}
if !strings.HasPrefix(first.StoragePath, ".backupx/repository/v1/snapshots/") {
t.Fatalf("unexpected repository snapshot path: %s", first.StoragePath)
}
large[2<<20] ^= 0xff
if err := os.WriteFile(largePath, large, 0o640); err != nil {
t.Fatalf("modify large fixture: %v", err)
}
second, err := executionService.RunTaskByIDSync(ctx, task.ID)
if err != nil {
t.Fatalf("second repository backup returned error: %v", err)
}
stored, err := records.FindByID(ctx, second.ID)
if err != nil {
t.Fatalf("FindByID repository record returned error: %v", err)
}
if stored == nil || stored.BackupKind != model.BackupKindRepository || stored.Manifest == "" {
t.Fatalf("repository metadata was not persisted: %#v", stored)
}
download, err := recordService.Download(ctx, second.ID)
if err != nil {
t.Fatalf("export repository snapshot returned error: %v", err)
}
exported, readErr := io.ReadAll(download.Reader)
closeErr := download.Reader.Close()
if readErr != nil || closeErr != nil {
t.Fatalf("read repository export: read=%v close=%v", readErr, closeErr)
}
if len(exported) == 0 || !strings.HasSuffix(download.FileName, ".tar") {
t.Fatalf("unexpected repository export: name=%s size=%d", download.FileName, len(exported))
}
if err := os.WriteFile(largePath, bytes.Repeat([]byte{0}, len(large)), 0o640); err != nil {
t.Fatalf("damage source before restore: %v", err)
}
if err := executionService.RestoreRecord(ctx, second.ID); err != nil {
t.Fatalf("restore repository record returned error: %v", err)
}
restored, err := os.ReadFile(largePath)
if err != nil {
t.Fatalf("read restored source: %v", err)
}
if !bytes.Equal(restored, large) {
t.Fatalf("repository restore did not reproduce the source")
}
if err := recordService.Delete(ctx, first.ID); err != nil {
t.Fatalf("delete first repository record: %v", err)
}
if err := recordService.Delete(ctx, second.ID); err != nil {
t.Fatalf("delete second repository record: %v", err)
}
packRoot := filepath.Join(storageDir, filepath.FromSlash(".backupx/repository/v1/packs"))
remainingPacks := 0
if err := filepath.Walk(packRoot, func(_ string, info os.FileInfo, walkErr error) error {
if walkErr != nil {
if os.IsNotExist(walkErr) {
return nil
}
return walkErr
}
if info != nil && !info.IsDir() {
remainingPacks++
}
return nil
}); err != nil {
t.Fatalf("inspect repository packs: %v", err)
}
if remainingPacks != 0 {
t.Fatalf("repository prune left %d packs after deleting all snapshots", remainingPacks)
}
}
func TestBackupExecutionServiceNodePoolSelectionDoesNotPersistTaskNodeID(t *testing.T) {
executionService, _, tasks, _, records, _, _ := newExecutionTestServices(t)
ctx := context.Background()
@@ -359,6 +462,56 @@ func TestBackupExecutionServiceRestoreRecordRejectsRemoteLocalDisk(t *testing.T)
}
}
func TestBackupExecutionServiceDownloadsMasterRelayedLocalDiskRecord(t *testing.T) {
executionService, _, tasks, _, records, _, storageDir := newExecutionTestServices(t)
ctx := context.Background()
executionService.SetClusterDependencies(&nodeRepoStub{nodes: []model.Node{
{ID: 10, Name: "edge-a", Token: "edge-a-token", Status: model.NodeStatusOnline},
}}, &fakeDispatcher{})
task, err := tasks.FindByID(ctx, 1)
if err != nil {
t.Fatalf("FindByID task returned error: %v", err)
}
storagePath := "file/2026/05/09/relayed.tar"
artifactPath := filepath.Join(storageDir, filepath.FromSlash(storagePath))
if err := os.MkdirAll(filepath.Dir(artifactPath), 0o755); err != nil {
t.Fatalf("MkdirAll artifact parent returned error: %v", err)
}
content := []byte("stored on Master")
if err := os.WriteFile(artifactPath, content, 0o600); err != nil {
t.Fatalf("WriteFile artifact returned error: %v", err)
}
completedAt := time.Now().UTC()
record := &model.BackupRecord{
TaskID: task.ID,
StorageTargetID: task.StorageTargetID,
NodeID: 10,
Status: model.BackupRecordStatusSuccess,
FileName: "relayed.tar",
FileSize: int64(len(content)),
StoragePath: storagePath,
StorageTransferMode: storage.TransferModeMasterRelay,
StartedAt: completedAt.Add(-time.Second),
CompletedAt: &completedAt,
}
if err := records.Create(ctx, record); err != nil {
t.Fatalf("Create record returned error: %v", err)
}
download, err := executionService.DownloadRecord(ctx, record.ID)
if err != nil {
t.Fatalf("DownloadRecord returned error: %v", err)
}
got, readErr := io.ReadAll(download.Reader)
closeErr := download.Reader.Close()
if readErr != nil || closeErr != nil {
t.Fatalf("read relayed artifact: read=%v close=%v", readErr, closeErr)
}
if !bytes.Equal(got, content) {
t.Fatalf("downloaded content = %q, want %q", got, content)
}
}
func TestBackupExecutionServiceRecordsFirstSuccessfulStorageTarget(t *testing.T) {
executionService, _, tasks, targets, records, _, _ := newExecutionTestServices(t)
ctx := context.Background()
server/internal/service/backup_record_service.go
+34 -32
View File
@@ -23,22 +23,23 @@ type BackupRecordListInput struct {
}
type BackupRecordSummary struct {
ID uint `json:"id"`
TaskID uint `json:"taskId"`
TaskName string `json:"taskName"`
StorageTargetID uint `json:"storageTargetId"`
StorageTargetName string `json:"storageTargetName"`
Status string `json:"status"`
FileName string `json:"fileName"`
FileSize int64 `json:"fileSize"`
Checksum string `json:"checksum"`
StoragePath string `json:"storagePath"`
DurationSeconds int `json:"durationSeconds"`
ErrorMessage string `json:"errorMessage"`
StartedAt time.Time `json:"startedAt"`
CompletedAt *time.Time `json:"completedAt,omitempty"`
Locked bool `json:"locked"`
BackupKind string `json:"backupKind"`
ID uint `json:"id"`
TaskID uint `json:"taskId"`
TaskName string `json:"taskName"`
StorageTargetID uint `json:"storageTargetId"`
StorageTargetName string `json:"storageTargetName"`
Status string `json:"status"`
FileName string `json:"fileName"`
FileSize int64 `json:"fileSize"`
Checksum string `json:"checksum"`
StoragePath string `json:"storagePath"`
StorageTransferMode string `json:"storageTransferMode,omitempty"`
DurationSeconds int `json:"durationSeconds"`
ErrorMessage string `json:"errorMessage"`
StartedAt time.Time `json:"startedAt"`
CompletedAt *time.Time `json:"completedAt,omitempty"`
Locked bool `json:"locked"`
BackupKind string `json:"backupKind"`
}
type BackupRecordDetail struct {
@@ -184,22 +185,23 @@ func (s *BackupRecordService) SetLock(ctx context.Context, id uint, locked bool)
func toBackupRecordSummary(item *model.BackupRecord) BackupRecordSummary {
return BackupRecordSummary{
ID: item.ID,
TaskID: item.TaskID,
TaskName: item.Task.Name,
StorageTargetID: item.StorageTargetID,
StorageTargetName: item.StorageTarget.Name,
Status: item.Status,
FileName: item.FileName,
FileSize: item.FileSize,
Checksum: item.Checksum,
StoragePath: item.StoragePath,
DurationSeconds: item.DurationSeconds,
ErrorMessage: item.ErrorMessage,
StartedAt: item.StartedAt,
CompletedAt: item.CompletedAt,
Locked: item.Locked,
BackupKind: item.BackupKind,
ID: item.ID,
TaskID: item.TaskID,
TaskName: item.Task.Name,
StorageTargetID: item.StorageTargetID,
StorageTargetName: item.StorageTarget.Name,
Status: item.Status,
FileName: item.FileName,
FileSize: item.FileSize,
Checksum: item.Checksum,
StoragePath: item.StoragePath,
StorageTransferMode: item.StorageTransferMode,
DurationSeconds: item.DurationSeconds,
ErrorMessage: item.ErrorMessage,
StartedAt: item.StartedAt,
CompletedAt: item.CompletedAt,
Locked: item.Locked,
BackupKind: item.BackupKind,
}
}
server/internal/service/backup_task_service.go
+60 -13
View File
@@ -57,8 +57,8 @@ type BackupTaskUpsertInput struct {
KeepWeekly int `json:"keepWeekly"`
KeepMonthly int `json:"keepMonthly"`
KeepYearly int `json:"keepYearly"`
// BackupMode 备份模式：full（默认）/ differential（差异，仅文件类型本机任务）
BackupMode string `json:"backupMode" binding:"omitempty,oneof=full differential"`
// BackupMode 备份模式：full（默认）/ differential（差异归档）/ repository（CDC 去重仓库）
BackupMode string `json:"backupMode" binding:"omitempty,oneof=full differential repository"`
DiffFullIntervalDays int `json:"diffFullIntervalDays"`
// 备份复制目标存储 ID 列表（3-2-1 规则）
ReplicationTargetIDs []uint `json:"replicationTargetIds"`
@@ -414,21 +414,50 @@ func (s *BackupTaskService) cleanupRemoteFiles(ctx context.Context, taskID uint)
recordCount = len(records)
// 缓存 provider 避免同一存储目标重复创建连接
providerCache := make(map[uint]storage.StorageProvider)
repositoryProviders := make(map[uint]storage.StorageProvider)
for _, record := range records {
if strings.TrimSpace(record.StoragePath) == "" {
continue
copies := []StorageUploadResultItem{{
StorageTargetID: record.StorageTargetID,
Status: model.BackupRecordStatusSuccess,
StoragePath: record.StoragePath,
}}
if strings.TrimSpace(record.StorageUploadResults) != "" {
var storedCopies []StorageUploadResultItem
if unmarshalErr := json.Unmarshal([]byte(record.StorageUploadResults), &storedCopies); unmarshalErr == nil {
copies = storedCopies
}
}
provider, ok := providerCache[record.StorageTargetID]
if !ok {
provider, err = s.resolveStorageProvider(ctx, record.StorageTargetID)
if err != nil {
seenTargets := make(map[uint]struct{}, len(copies))
for _, copy := range copies {
if !strings.EqualFold(copy.Status, model.BackupRecordStatusSuccess) || strings.TrimSpace(copy.StoragePath) == "" {
continue
}
providerCache[record.StorageTargetID] = provider
if _, seen := seenTargets[copy.StorageTargetID]; seen {
continue
}
seenTargets[copy.StorageTargetID] = struct{}{}
provider, ok := providerCache[copy.StorageTargetID]
if !ok {
provider, err = s.resolveStorageProvider(ctx, copy.StorageTargetID)
if err != nil {
continue
}
providerCache[copy.StorageTargetID] = provider
}
if err := provider.Delete(ctx, copy.StoragePath); err == nil {
cleanedFiles++
if record.BackupKind == model.BackupKindRepository {
repositoryProviders[copy.StorageTargetID] = provider
}
}
}
if err := provider.Delete(ctx, record.StoragePath); err == nil {
cleanedFiles++
}
for _, provider := range repositoryProviders {
pruned, pruneErr := backup.NewRepositoryStore(s.cipher.Key()).Prune(ctx, provider)
if pruneErr != nil {
continue
}
cleanedFiles += pruned.DeletedIndexes + pruned.DeletedPacks
}
return recordCount, cleanedFiles
}
@@ -530,6 +559,17 @@ func (s *BackupTaskService) validateInput(ctx context.Context, existing *model.B
return apperror.BadRequest("BACKUP_TASK_DIFF_REMOTE_UNSUPPORTED", "差异备份当前仅支持本机 Master 执行，请将任务固定在本机或改用全量备份。", nil)
}
}
if strings.EqualFold(strings.TrimSpace(input.BackupMode), model.BackupModeRepository) {
if input.Type != model.BackupTaskTypeFile {
return apperror.BadRequest("BACKUP_TASK_REPOSITORY_UNSUPPORTED", "CDC 仓库模式仅支持文件目录类型任务", nil)
}
if strings.TrimSpace(input.NodePoolTag) != "" || (fixedNode != nil && !fixedNode.IsLocal) {
return apperror.BadRequest("BACKUP_TASK_REPOSITORY_REMOTE_UNSUPPORTED", "CDC 仓库模式当前采用单写者索引，仅支持 Master 本机执行。远程服务器备份请暂用全量模式。", nil)
}
if len(input.ReplicationTargetIDs) > 0 {
return apperror.BadRequest("BACKUP_TASK_REPOSITORY_REPLICATION_UNSUPPORTED", "CDC 仓库快照不能使用对象级复制；请直接为任务选择多个存储目标以生成完整仓库副本。", nil)
}
}
if input.RetentionDays < 0 {
return apperror.BadRequest("BACKUP_TASK_INVALID", "保留天数不能小于 0", nil)
}
@@ -935,10 +975,17 @@ func decodeExtraConfig(value string) (map[string]any, error) {
return result, nil
}
// normalizeBackupMode 归一化备份模式：仅文件类型可启用差异，其余一律全量（双保险，防绕过校验）。
// normalizeBackupMode 归一化备份模式：仅文件类型可启用差异或 CDC 仓库，
// 其余一律全量（双保险，防绕过校验）。
func normalizeBackupMode(mode, taskType string) string {
if strings.EqualFold(strings.TrimSpace(mode), model.BackupModeDifferential) && normalizeBackupTaskType(taskType) == model.BackupTaskTypeFile {
if normalizeBackupTaskType(taskType) != model.BackupTaskTypeFile {
return model.BackupModeFull
}
switch strings.ToLower(strings.TrimSpace(mode)) {
case model.BackupModeDifferential:
return model.BackupModeDifferential
case model.BackupModeRepository:
return model.BackupModeRepository
}
return model.BackupModeFull
}
server/internal/service/dashboard_service.go
+20 -20
View File
@@ -126,14 +126,14 @@ func (s *DashboardService) Timeline(ctx context.Context, days int) ([]repository
// 判定规则：任务设置了 SLAHoursRPO > 0，且距最近一次 success 备份的时间 > SLAHoursRPO。
// 从未成功过的任务（LastSuccessAt = nil）若启用也视为违约（from createdAt 起算）。
type SLAViolation struct {
TaskID uint `json:"taskId"`
TaskName string `json:"taskName"`
NodeID uint `json:"nodeId"`
NodeName string `json:"nodeName,omitempty"`
SLAHoursRPO int `json:"slaHoursRpo"`
LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
HoursSinceLastSuccess float64 `json:"hoursSinceLastSuccess"`
NeverSucceeded bool `json:"neverSucceeded"`
TaskID uint `json:"taskId"`
TaskName string `json:"taskName"`
NodeID uint `json:"nodeId"`
NodeName string `json:"nodeName,omitempty"`
SLAHoursRPO int `json:"slaHoursRpo"`
LastSuccessAt *time.Time `json:"lastSuccessAt,omitempty"`
HoursSinceLastSuccess float64 `json:"hoursSinceLastSuccess"`
NeverSucceeded bool `json:"neverSucceeded"`
}
// SLAComplianceReport Dashboard 的 SLA 合规概览。
@@ -204,15 +204,15 @@ func roundHours(value float64) float64 {
// ClusterNodeSummary 集群节点简报（Dashboard 用）。
type ClusterNodeSummary struct {
ID uint `json:"id"`
Name string `json:"name"`
Hostname string `json:"hostname"`
Status string `json:"status"`
IsLocal bool `json:"isLocal"`
AgentVersion string `json:"agentVersion"`
VersionStatus string `json:"versionStatus"` // current | outdated | unknown
LastSeen time.Time `json:"lastSeen"`
TaskCount int64 `json:"taskCount"`
ID uint `json:"id"`
Name string `json:"name"`
Hostname string `json:"hostname"`
Status string `json:"status"`
IsLocal bool `json:"isLocal"`
AgentVersion string `json:"agentVersion"`
VersionStatus string `json:"versionStatus"` // current | outdated | unknown
LastSeen time.Time `json:"lastSeen"`
TaskCount int64 `json:"taskCount"`
}
// ClusterOverview Dashboard 集群概览卡片。
@@ -312,9 +312,9 @@ func (s *DashboardService) Breakdown(ctx context.Context, days int) (*BreakdownS
}
}
result := &BreakdownStats{
ByType: makeBreakdown(typeCounts, typeLabel),
ByNode: makeBreakdownByUint(nodeCounts, nodeNames, "节点 #"),
ByStatus: []BreakdownItem{},
ByType: makeBreakdown(typeCounts, typeLabel),
ByNode: makeBreakdownByUint(nodeCounts, nodeNames, "节点 #"),
ByStatus: []BreakdownItem{},
ByStorage: []BreakdownItem{},
}
// 按状态（最近 days 天记录）
server/internal/service/execution_helpers.go
+5
View File
@@ -140,6 +140,11 @@ func validateCrossNodeLocalDisk(ctx context.Context, nodeRepo repository.NodeRep
if record == nil || record.NodeID == 0 || nodeRepo == nil {
return nil
}
// 中转模式的对象实际落在 Master 配置的本地磁盘，Master 可以安全访问。
// 空值和 direct 均按旧版 Agent 本地落盘处理，保持升级兼容。
if record.StorageTransferMode == storage.TransferModeMasterRelay {
return nil
}
node, err := nodeRepo.FindByID(ctx, record.NodeID)
if err != nil || node == nil || node.IsLocal {
return nil
server/internal/service/install_token_service.go
+52 -27
View File
@@ -6,6 +6,7 @@ import (
"encoding/base64"
"encoding/hex"
"fmt"
"net/url"
"strings"
"time"
@@ -27,13 +28,16 @@ func NewInstallTokenService(repo repository.AgentInstallTokenRepository, nodeRep
// InstallTokenInput 生成一次性安装令牌的输入。
type InstallTokenInput struct {
NodeID uint
Mode string
Arch string
AgentVersion string
DownloadSrc string
TTLSeconds int
CreatedByID uint
NodeID uint
Mode string
Arch string
AgentVersion string
DownloadSrc string
TTLSeconds int
CreatedByID uint
AgentMasterURL string
ProxyURL string
CACertFile string
}
// InstallTokenOutput 生成结果。
@@ -112,14 +116,17 @@ func (s *InstallTokenService) Create(ctx context.Context, in InstallTokenInput)
}
expiresAt := time.Now().UTC().Add(time.Duration(in.TTLSeconds) * time.Second)
record := &model.AgentInstallToken{
Token: token,
NodeID: in.NodeID,
Mode: in.Mode,
Arch: in.Arch,
AgentVer: in.AgentVersion,
DownloadSrc: in.DownloadSrc,
ExpiresAt: expiresAt,
CreatedByID: in.CreatedByID,
Token: token,
NodeID: in.NodeID,
Mode: in.Mode,
Arch: in.Arch,
AgentVer: in.AgentVersion,
DownloadSrc: in.DownloadSrc,
AgentMasterURL: strings.TrimRight(strings.TrimSpace(in.AgentMasterURL), "/"),
ProxyURL: strings.TrimSpace(in.ProxyURL),
CACertFile: strings.TrimSpace(in.CACertFile),
ExpiresAt: expiresAt,
CreatedByID: in.CreatedByID,
}
if err := s.repo.Create(ctx, record); err != nil {
return nil, err
@@ -130,9 +137,15 @@ func (s *InstallTokenService) Create(ctx context.Context, in InstallTokenInput)
// CreateCommand 创建 install token，并返回 UI 展示安装命令所需的 URL 与嵌入式脚本。
func (s *InstallTokenService) CreateCommand(ctx context.Context, in InstallCommandInput) (*InstallCommandOutput, error) {
masterURL := strings.TrimRight(strings.TrimSpace(in.MasterURL), "/")
if masterURL == "" {
return nil, apperror.BadRequest("INSTALL_TOKEN_INVALID", "masterURL 必填", nil)
deliveryURL, parseErr := url.Parse(masterURL)
if masterURL == "" || parseErr != nil || (deliveryURL.Scheme != "http" && deliveryURL.Scheme != "https") || deliveryURL.Host == "" || deliveryURL.User != nil || deliveryURL.RawQuery != "" || deliveryURL.Fragment != "" || strings.ContainsAny(masterURL, " \t\r\n\"'`$\\") {
return nil, apperror.BadRequest("INSTALL_TOKEN_INVALID", "masterURL 必须是安全的完整 HTTP(S) 地址", parseErr)
}
agentMasterURL := strings.TrimRight(strings.TrimSpace(in.AgentMasterURL), "/")
if agentMasterURL == "" {
agentMasterURL = masterURL
}
in.AgentMasterURL = agentMasterURL
if err := s.validate(in.InstallTokenInput); err != nil {
return nil, err
}
@@ -144,12 +157,15 @@ func (s *InstallTokenService) CreateCommand(ctx context.Context, in InstallComma
return nil, apperror.New(404, "NODE_NOT_FOUND", "节点不存在", nil)
}
if _, err := renderInstallCommandScript(masterURL, node, &model.AgentInstallToken{
Mode: in.Mode,
Arch: in.Arch,
AgentVer: in.AgentVersion,
DownloadSrc: in.DownloadSrc,
Mode: in.Mode,
Arch: in.Arch,
AgentVer: in.AgentVersion,
DownloadSrc: in.DownloadSrc,
AgentMasterURL: in.AgentMasterURL,
ProxyURL: in.ProxyURL,
CACertFile: in.CACertFile,
}); err != nil {
return nil, err
return nil, apperror.BadRequest("INSTALL_TOKEN_INVALID", "Agent 连接配置无效", err)
}
out, err := s.Create(ctx, in.InstallTokenInput)
if err != nil {
@@ -164,20 +180,24 @@ func (s *InstallTokenService) CreateCommand(ctx context.Context, in InstallComma
ExpiresAt: out.ExpiresAt,
Node: out.Node,
Record: out.Record,
URL: masterURL + "/api/install/" + out.Token,
FallbackURL: masterURL + "/install/" + out.Token,
URL: agentMasterURL + "/api/install/" + out.Token,
FallbackURL: agentMasterURL + "/install/" + out.Token,
ScriptBase64: base64.StdEncoding.EncodeToString([]byte(script)),
}
if out.Record.Mode == model.InstallModeDocker {
result.ComposeURL = masterURL + "/api/install/" + out.Token + "/compose.yml"
result.FallbackComposeURL = masterURL + "/install/" + out.Token + "/compose.yml"
result.ComposeURL = agentMasterURL + "/api/install/" + out.Token + "/compose.yml"
result.FallbackComposeURL = agentMasterURL + "/install/" + out.Token + "/compose.yml"
}
return result, nil
}
func renderInstallCommandScript(masterURL string, node *model.Node, record *model.AgentInstallToken) (string, error) {
agentMasterURL := strings.TrimRight(strings.TrimSpace(record.AgentMasterURL), "/")
if agentMasterURL == "" {
agentMasterURL = masterURL
}
return installscript.RenderScript(installscript.Context{
MasterURL: masterURL,
MasterURL: agentMasterURL,
AgentToken: node.Token,
AgentVersion: record.AgentVer,
Mode: record.Mode,
@@ -185,6 +205,8 @@ func renderInstallCommandScript(masterURL string, node *model.Node, record *mode
DownloadBase: installscript.DownloadBaseFor(record.DownloadSrc),
InstallPrefix: "/opt/backupx-agent",
NodeID: node.ID,
ProxyURL: record.ProxyURL,
CACertFile: record.CACertFile,
})
}
@@ -259,6 +281,9 @@ func (s *InstallTokenService) validate(in InstallTokenInput) error {
return apperror.BadRequest("INSTALL_TOKEN_INVALID",
fmt.Sprintf("ttlSeconds 需在 %d-%d", InstallTokenMinTTL, InstallTokenMaxTTL), nil)
}
if len(in.AgentMasterURL) > 2048 || len(in.ProxyURL) > 2048 || len(in.CACertFile) > 512 {
return apperror.BadRequest("INSTALL_TOKEN_INVALID", "连接配置过长", nil)
}
return nil
}
server/internal/service/install_token_service_test.go
+66 -10
View File
@@ -2,7 +2,9 @@ package service
import (
"context"
"encoding/base64"
"path/filepath"
"strings"
"testing"
"time"
@@ -177,13 +179,16 @@ func TestInstallTokenServiceCreateCommandBuildsURLsAndScript(t *testing.T) {
out, err := svc.CreateCommand(context.Background(), InstallCommandInput{
InstallTokenInput: InstallTokenInput{
NodeID: node.ID,
Mode: model.InstallModeDocker,
Arch: model.InstallArchAuto,
AgentVersion: "v1.7.0",
DownloadSrc: model.InstallSourceGitHub,
TTLSeconds: 900,
CreatedByID: 1,
NodeID: node.ID,
Mode: model.InstallModeDocker,
Arch: model.InstallArchAuto,
AgentVersion: "v1.7.0",
DownloadSrc: model.InstallSourceGitHub,
TTLSeconds: 900,
CreatedByID: 1,
AgentMasterURL: "http://127.0.0.1:18340",
ProxyURL: "socks5h://127.0.0.1:1080",
CACertFile: "/etc/pki/internal-ca.pem",
},
MasterURL: "https://public.example.com/base",
})
@@ -193,15 +198,66 @@ func TestInstallTokenServiceCreateCommandBuildsURLsAndScript(t *testing.T) {
if out.Token == "" || out.ScriptBase64 == "" {
t.Fatalf("missing token or script: %+v", out)
}
if out.URL != "https://public.example.com/base/api/install/"+out.Token {
if out.URL != "http://127.0.0.1:18340/api/install/"+out.Token {
t.Fatalf("bad url: %s", out.URL)
}
if out.FallbackURL != "https://public.example.com/base/install/"+out.Token {
if out.FallbackURL != "http://127.0.0.1:18340/install/"+out.Token {
t.Fatalf("bad fallback url: %s", out.FallbackURL)
}
if out.ComposeURL != "https://public.example.com/base/api/install/"+out.Token+"/compose.yml" {
if out.ComposeURL != "http://127.0.0.1:18340/api/install/"+out.Token+"/compose.yml" {
t.Fatalf("bad compose url: %s", out.ComposeURL)
}
script, err := base64.StdEncoding.DecodeString(out.ScriptBase64)
if err != nil {
t.Fatal(err)
}
for _, want := range []string{
`MASTER_URL="http://127.0.0.1:18340"`,
`PROXY_URL="socks5h://127.0.0.1:1080"`,
`CA_CERT_FILE="/etc/pki/internal-ca.pem"`,
} {
if !strings.Contains(string(script), want) {
t.Fatalf("script missing %q", want)
}
}
}
func TestInstallTokenServiceRejectsUnsafeConnectionBeforeCreate(t *testing.T) {
db := openInstallTokenTestDB(t)
nodeRepo := repository.NewNodeRepository(db)
node := &model.Node{Name: "restricted", Token: "deadbeefcafebabe0123456789abcdef0123456789abcdef0123456789abcdef"}
if err := nodeRepo.Create(context.Background(), node); err != nil {
t.Fatal(err)
}
tokenRepo := repository.NewAgentInstallTokenRepository(db)
svc := NewInstallTokenService(tokenRepo, nodeRepo)
_, err := svc.CreateCommand(context.Background(), InstallCommandInput{
InstallTokenInput: InstallTokenInput{
NodeID: node.ID, Mode: model.InstallModeSystemd, Arch: model.InstallArchAuto,
AgentVersion: "v2.4.0", DownloadSrc: model.InstallSourceGitHub, TTLSeconds: 900,
ProxyURL: "http://user:pass@proxy.example.com",
},
MasterURL: "https://public.example.com",
})
if err == nil {
t.Fatal("expected proxy credentials to be rejected")
}
count, countErr := tokenRepo.CountCreatedSince(context.Background(), node.ID, time.Now().UTC().Add(-time.Hour))
if countErr != nil || count != 0 {
t.Fatalf("invalid connection created token records: count=%d err=%v", count, countErr)
}
_, err = svc.CreateCommand(context.Background(), InstallCommandInput{
InstallTokenInput: InstallTokenInput{
NodeID: node.ID, Mode: model.InstallModeSystemd, Arch: model.InstallArchAuto,
AgentVersion: "v2.4.0", DownloadSrc: model.InstallSourceGitHub, TTLSeconds: 900,
AgentMasterURL: "https://master.internal",
},
MasterURL: "http://public.example.com/$unsafe",
})
if err == nil {
t.Fatal("expected unsafe public delivery URL to be rejected")
}
}
func TestInstallTokenServiceRateLimit(t *testing.T) {
server/internal/service/restore_service.go
+84 -8
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strings"
@@ -321,6 +322,13 @@ func (s *RestoreService) restoreArtifact(ctx context.Context, record *model.Back
if err != nil {
return fmt.Errorf("创建存储客户端失败：%w", err)
}
if record.BackupKind == model.BackupKindRepository {
logger.Infof("读取 CDC 仓库快照：%s", record.StoragePath)
if err := backup.NewRepositoryStore(s.cipher.Key()).Restore(ctx, provider, record.StoragePath, record.Checksum, spec, logger); err != nil {
return fmt.Errorf("恢复 CDC 仓库快照失败：%w", err)
}
return nil
}
recDir, err := os.MkdirTemp(parentTempDir, fmt.Sprintf("rec-%d-*", record.ID))
if err != nil {
return fmt.Errorf("创建恢复子目录失败：%w", err)
@@ -368,6 +376,9 @@ func (s *RestoreService) buildRestoreChain(ctx context.Context, record *model.Ba
}
func backupKindLabel(kind string) string {
if kind == model.BackupKindRepository {
return "CDC 仓库快照"
}
if kind == model.BackupKindDifferential {
return "差异"
}
@@ -591,15 +602,22 @@ func (s *RestoreService) GetAgentRestoreSpec(ctx context.Context, node *model.No
if target == nil {
return nil, apperror.BadRequest("BACKUP_STORAGE_TARGET_INVALID", "存储目标不存在", nil)
}
configRaw, err := s.cipher.Decrypt(target.ConfigCiphertext)
if err != nil {
return nil, fmt.Errorf("decrypt storage config: %w", err)
}
// 拆开 sourcePaths
sourcePaths := []string{}
if strings.TrimSpace(task.SourcePaths) != "" {
_ = json.Unmarshal([]byte(task.SourcePaths), &sourcePaths)
}
transferMode := storage.TransferModeDirect
if backupRecord.StorageTransferMode == storage.TransferModeMasterRelay {
transferMode = storage.TransferModeMasterRelay
}
var configRaw []byte
if transferMode == storage.TransferModeDirect {
configRaw, err = s.cipher.Decrypt(target.ConfigCiphertext)
if err != nil {
return nil, fmt.Errorf("decrypt storage config: %w", err)
}
}
return &AgentRestoreSpec{
RestoreRecordID: restore.ID,
BackupRecordID: backupRecord.ID,
@@ -618,10 +636,11 @@ func (s *RestoreService) GetAgentRestoreSpec(ctx context.Context, node *model.No
Compression: task.Compression,
Encrypt: task.Encrypt,
Storage: AgentStorageTargetConfig{
ID: target.ID,
Type: target.Type,
Name: target.Name,
Config: json.RawMessage(configRaw),
ID: target.ID,
Type: target.Type,
Name: target.Name,
Config: json.RawMessage(configRaw),
TransferMode: transferMode,
},
StoragePath: backupRecord.StoragePath,
FileName: backupRecord.FileName,
@@ -629,6 +648,63 @@ func (s *RestoreService) GetAgentRestoreSpec(ctx context.Context, node *model.No
}, nil
}
type AgentArtifactDownload struct {
Reader io.ReadCloser
Size int64
}
// DownloadAgentArtifact opens a Master-local object for authenticated streaming
// back to the Agent that owns the restore record.
func (s *RestoreService) DownloadAgentArtifact(ctx context.Context, node *model.Node, restoreID uint) (*AgentArtifactDownload, error) {
if node == nil {
return nil, apperror.Unauthorized("RESTORE_RECORD_FORBIDDEN", "恢复记录不属于当前节点", nil)
}
restore, err := s.restores.FindByID(ctx, restoreID)
if err != nil {
return nil, err
}
if restore == nil {
return nil, apperror.New(404, "RESTORE_RECORD_NOT_FOUND", "恢复记录不存在", nil)
}
if restore.NodeID != node.ID {
return nil, apperror.Unauthorized("RESTORE_RECORD_FORBIDDEN", "恢复记录不属于当前节点", nil)
}
if isRestoreRecordTerminal(restore.Status) {
return nil, apperror.BadRequest("RESTORE_RECORD_TERMINAL", "恢复记录已结束，不能继续下载产物", nil)
}
record, err := s.records.FindByID(ctx, restore.BackupRecordID)
if err != nil {
return nil, err
}
if record == nil {
return nil, apperror.New(404, "BACKUP_RECORD_NOT_FOUND", "源备份记录不存在", nil)
}
target, err := s.targets.FindByID(ctx, record.StorageTargetID)
if err != nil {
return nil, err
}
if target == nil || !strings.EqualFold(target.Type, storage.TypeLocalDisk) || record.StorageTransferMode != storage.TransferModeMasterRelay {
return nil, apperror.BadRequest("AGENT_ARTIFACT_RELAY_UNSUPPORTED", "该存储目标应由 Agent 直接下载", nil)
}
configMap := map[string]any{}
if err := s.cipher.DecryptJSON(target.ConfigCiphertext, &configMap); err != nil {
return nil, fmt.Errorf("decrypt storage config: %w", err)
}
provider, err := s.storageRegistry.Create(ctx, target.Type, configMap)
if err != nil {
return nil, fmt.Errorf("create master relay provider: %w", err)
}
reader, err := provider.Download(ctx, record.StoragePath)
if err != nil {
return nil, fmt.Errorf("open master relay artifact: %w", err)
}
size := record.FileSize
if size <= 0 {
size = -1
}
return &AgentArtifactDownload{Reader: reader, Size: size}, nil
}
// UpdateAgentRestore Agent 回传状态/日志。
func (s *RestoreService) UpdateAgentRestore(ctx context.Context, node *model.Node, restoreID uint, update AgentRestoreUpdate) error {
restore, err := s.restores.FindByID(ctx, restoreID)
server/internal/service/restore_service_test.go
+37 -9
View File
@@ -1,8 +1,10 @@
package service
import (
"bytes"
"context"
"encoding/json"
"io"
"os"
"path/filepath"
"strings"
@@ -427,16 +429,27 @@ func TestRestoreServiceAgentRestoreAccessUsesRestoreRecordNode(t *testing.T) {
}
startedAt := time.Now().UTC()
completedAt := startedAt.Add(time.Second)
artifact := []byte("central backup artifact")
storagePath := "file/2026/05/09/remote.tar.gz"
artifactPath := filepath.Join(h.storageDir, filepath.FromSlash(storagePath))
if err := os.MkdirAll(filepath.Dir(artifactPath), 0o755); err != nil {
t.Fatalf("MkdirAll artifact parent: %v", err)
}
if err := os.WriteFile(artifactPath, artifact, 0o600); err != nil {
t.Fatalf("WriteFile artifact: %v", err)
}
backupRecord := &model.BackupRecord{
TaskID: task.ID,
StorageTargetID: task.StorageTargetID,
NodeID: owner.ID,
Status: model.BackupRecordStatusSuccess,
FileName: "remote.tar.gz",
StoragePath: "file/2026/05/09/remote.tar.gz",
Checksum: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
StartedAt: startedAt,
CompletedAt: &completedAt,
TaskID: task.ID,
StorageTargetID: task.StorageTargetID,
NodeID: owner.ID,
Status: model.BackupRecordStatusSuccess,
FileName: "remote.tar.gz",
StoragePath: storagePath,
FileSize: int64(len(artifact)),
StorageTransferMode: storage.TransferModeMasterRelay,
Checksum: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
StartedAt: startedAt,
CompletedAt: &completedAt,
}
if err := h.records.Create(ctx, backupRecord); err != nil {
t.Fatalf("Create backup record: %v", err)
@@ -464,6 +477,21 @@ func TestRestoreServiceAgentRestoreAccessUsesRestoreRecordNode(t *testing.T) {
if spec.Checksum != backupRecord.Checksum {
t.Fatalf("expected spec.Checksum=%q, got %q", backupRecord.Checksum, spec.Checksum)
}
if spec.Storage.TransferMode != storage.TransferModeMasterRelay {
t.Fatalf("expected Master relay restore, got %#v", spec.Storage)
}
download, err := h.service.DownloadAgentArtifact(ctx, owner, restore.ID)
if err != nil {
t.Fatalf("DownloadAgentArtifact returned error: %v", err)
}
downloaded, readErr := io.ReadAll(download.Reader)
closeErr := download.Reader.Close()
if readErr != nil || closeErr != nil {
t.Fatalf("read relayed restore artifact: read=%v close=%v", readErr, closeErr)
}
if !bytes.Equal(downloaded, artifact) {
t.Fatalf("relayed restore artifact differs: %q", downloaded)
}
if _, err := h.service.GetAgentRestoreSpec(ctx, other, restore.ID); err == nil {
t.Fatal("expected non-owner node to be forbidden from restore spec")
}
server/internal/service/task_export_service.go
+85 -85
View File
@@ -14,9 +14,9 @@ import (
// TaskExportService 管理备份任务的 JSON 导入 / 导出。
// 用途：
// 1. 集群迁移（旧 Master → 新 Master 的任务配置搬迁）
// 2. 灾备恢复（任务配置本地文件化，Master 宕机后重建）
// 3. 配置审计（版本化 Git 管理 JSON 快照）
// 1. 集群迁移（旧 Master → 新 Master 的任务配置搬迁）
// 2. 灾备恢复（任务配置本地文件化，Master 宕机后重建）
// 3. 配置审计（版本化 Git 管理 JSON 快照）
//
// 出于安全考虑，导出/导入不包含任何敏感字段：
// - 数据库密码（DBPasswordCiphertext）：跳过，导入后需人工填补
@@ -40,35 +40,35 @@ func NewTaskExportService(
// ExportedTask 导出格式：按名称引用存储/节点，不含敏感数据。
type ExportedTask struct {
Name string `json:"name"`
Type string `json:"type"`
Enabled bool `json:"enabled"`
CronExpr string `json:"cronExpr,omitempty"`
SourcePath string `json:"sourcePath,omitempty"`
SourcePaths []string `json:"sourcePaths,omitempty"`
ExcludePatterns []string `json:"excludePatterns,omitempty"`
DBHost string `json:"dbHost,omitempty"`
DBPort int `json:"dbPort,omitempty"`
DBUser string `json:"dbUser,omitempty"`
DBName string `json:"dbName,omitempty"`
DBPath string `json:"dbPath,omitempty"`
ExtraConfig map[string]any `json:"extraConfig,omitempty"`
Name string `json:"name"`
Type string `json:"type"`
Enabled bool `json:"enabled"`
CronExpr string `json:"cronExpr,omitempty"`
SourcePath string `json:"sourcePath,omitempty"`
SourcePaths []string `json:"sourcePaths,omitempty"`
ExcludePatterns []string `json:"excludePatterns,omitempty"`
DBHost string `json:"dbHost,omitempty"`
DBPort int `json:"dbPort,omitempty"`
DBUser string `json:"dbUser,omitempty"`
DBName string `json:"dbName,omitempty"`
DBPath string `json:"dbPath,omitempty"`
ExtraConfig map[string]any `json:"extraConfig,omitempty"`
// 按名称引用：导入时按名称查找对应 ID
StorageTargetNames []string `json:"storageTargetNames"`
ReplicationTargetNames []string `json:"replicationTargetNames,omitempty"`
NodeName string `json:"nodeName,omitempty"`
DependsOnTaskNames []string `json:"dependsOnTaskNames,omitempty"`
Tags string `json:"tags,omitempty"`
Compression string `json:"compression,omitempty"`
Encrypt bool `json:"encrypt,omitempty"`
RetentionDays int `json:"retentionDays,omitempty"`
MaxBackups int `json:"maxBackups,omitempty"`
VerifyEnabled bool `json:"verifyEnabled,omitempty"`
VerifyCronExpr string `json:"verifyCronExpr,omitempty"`
VerifyMode string `json:"verifyMode,omitempty"`
SLAHoursRPO int `json:"slaHoursRpo,omitempty"`
AlertOnConsecutiveFails int `json:"alertOnConsecutiveFails,omitempty"`
MaintenanceWindows string `json:"maintenanceWindows,omitempty"`
StorageTargetNames []string `json:"storageTargetNames"`
ReplicationTargetNames []string `json:"replicationTargetNames,omitempty"`
NodeName string `json:"nodeName,omitempty"`
DependsOnTaskNames []string `json:"dependsOnTaskNames,omitempty"`
Tags string `json:"tags,omitempty"`
Compression string `json:"compression,omitempty"`
Encrypt bool `json:"encrypt,omitempty"`
RetentionDays int `json:"retentionDays,omitempty"`
MaxBackups int `json:"maxBackups,omitempty"`
VerifyEnabled bool `json:"verifyEnabled,omitempty"`
VerifyCronExpr string `json:"verifyCronExpr,omitempty"`
VerifyMode string `json:"verifyMode,omitempty"`
SLAHoursRPO int `json:"slaHoursRpo,omitempty"`
AlertOnConsecutiveFails int `json:"alertOnConsecutiveFails,omitempty"`
MaintenanceWindows string `json:"maintenanceWindows,omitempty"`
}
// ExportPayload 导出整体结构，带元信息。
@@ -233,67 +233,67 @@ func (s *TaskExportService) toExported(item *model.BackupTask, targetNames, node
nodeName = nodeNames[item.NodeID]
}
return ExportedTask{
Name: item.Name,
Type: item.Type,
Enabled: item.Enabled,
CronExpr: item.CronExpr,
SourcePath: item.SourcePath,
SourcePaths: sourcePaths,
ExcludePatterns: excludes,
DBHost: item.DBHost,
DBPort: item.DBPort,
DBUser: item.DBUser,
DBName: item.DBName,
DBPath: item.DBPath,
ExtraConfig: extra,
StorageTargetNames: storageNames,
ReplicationTargetNames: replicationNames,
NodeName: nodeName,
DependsOnTaskNames: dependsOnNames,
Tags: item.Tags,
Compression: item.Compression,
Encrypt: item.Encrypt,
RetentionDays: item.RetentionDays,
MaxBackups: item.MaxBackups,
VerifyEnabled: item.VerifyEnabled,
VerifyCronExpr: item.VerifyCronExpr,
VerifyMode: item.VerifyMode,
SLAHoursRPO: item.SLAHoursRPO,
Name: item.Name,
Type: item.Type,
Enabled: item.Enabled,
CronExpr: item.CronExpr,
SourcePath: item.SourcePath,
SourcePaths: sourcePaths,
ExcludePatterns: excludes,
DBHost: item.DBHost,
DBPort: item.DBPort,
DBUser: item.DBUser,
DBName: item.DBName,
DBPath: item.DBPath,
ExtraConfig: extra,
StorageTargetNames: storageNames,
ReplicationTargetNames: replicationNames,
NodeName: nodeName,
DependsOnTaskNames: dependsOnNames,
Tags: item.Tags,
Compression: item.Compression,
Encrypt: item.Encrypt,
RetentionDays: item.RetentionDays,
MaxBackups: item.MaxBackups,
VerifyEnabled: item.VerifyEnabled,
VerifyCronExpr: item.VerifyCronExpr,
VerifyMode: item.VerifyMode,
SLAHoursRPO: item.SLAHoursRPO,
AlertOnConsecutiveFails: item.AlertOnConsecutiveFails,
MaintenanceWindows: item.MaintenanceWindows,
MaintenanceWindows: item.MaintenanceWindows,
}
}
func (s *TaskExportService) toUpsertInput(t ExportedTask, targetsByName, nodesByName map[string]uint, deps []uint) BackupTaskUpsertInput {
return BackupTaskUpsertInput{
Name: t.Name,
Type: t.Type,
Enabled: t.Enabled,
CronExpr: t.CronExpr,
SourcePath: t.SourcePath,
SourcePaths: t.SourcePaths,
ExcludePatterns: t.ExcludePatterns,
DBHost: t.DBHost,
DBPort: t.DBPort,
DBUser: t.DBUser,
DBName: t.DBName,
DBPath: t.DBPath,
ExtraConfig: t.ExtraConfig,
StorageTargetIDs: idsFromNames(t.StorageTargetNames, targetsByName),
ReplicationTargetIDs: idsFromNames(t.ReplicationTargetNames, targetsByName),
NodeID: nodesByName[t.NodeName],
Tags: t.Tags,
Compression: t.Compression,
Encrypt: t.Encrypt,
RetentionDays: t.RetentionDays,
MaxBackups: t.MaxBackups,
VerifyEnabled: t.VerifyEnabled,
VerifyCronExpr: t.VerifyCronExpr,
VerifyMode: t.VerifyMode,
SLAHoursRPO: t.SLAHoursRPO,
Name: t.Name,
Type: t.Type,
Enabled: t.Enabled,
CronExpr: t.CronExpr,
SourcePath: t.SourcePath,
SourcePaths: t.SourcePaths,
ExcludePatterns: t.ExcludePatterns,
DBHost: t.DBHost,
DBPort: t.DBPort,
DBUser: t.DBUser,
DBName: t.DBName,
DBPath: t.DBPath,
ExtraConfig: t.ExtraConfig,
StorageTargetIDs: idsFromNames(t.StorageTargetNames, targetsByName),
ReplicationTargetIDs: idsFromNames(t.ReplicationTargetNames, targetsByName),
NodeID: nodesByName[t.NodeName],
Tags: t.Tags,
Compression: t.Compression,
Encrypt: t.Encrypt,
RetentionDays: t.RetentionDays,
MaxBackups: t.MaxBackups,
VerifyEnabled: t.VerifyEnabled,
VerifyCronExpr: t.VerifyCronExpr,
VerifyMode: t.VerifyMode,
SLAHoursRPO: t.SLAHoursRPO,
AlertOnConsecutiveFails: t.AlertOnConsecutiveFails,
MaintenanceWindows: t.MaintenanceWindows,
DependsOnTaskIDs: deps,
MaintenanceWindows: t.MaintenanceWindows,
DependsOnTaskIDs: deps,
}
}
server/internal/service/verification_service.go
+14
View File
@@ -299,6 +299,20 @@ func (s *VerificationService) executeLocally(ctx context.Context, verID uint, ta
logger.Errorf("创建存储客户端失败：%v", err)
return
}
if backupRecord.BackupKind == model.BackupKindRepository {
logger.Infof("验证 CDC 仓库快照及全部引用块：%s", backupRecord.StoragePath)
report, verifyErr := backup.NewRepositoryStore(s.cipher.Key()).Verify(ctx, provider, backupRecord.StoragePath, backupRecord.Checksum)
if verifyErr != nil {
errMessage = verifyErr.Error()
summary = "CDC 仓库完整性校验失败"
logger.Errorf("验证未通过：%v", verifyErr)
return
}
status = model.VerificationRecordStatusSuccess
summary = fmt.Sprintf("CDC 仓库完整性校验通过：%d 个条目、%d 个唯一块、%d bytes", report.Entries, report.Chunks, report.Bytes)
logger.Infof("%s", summary)
return
}
fileName := backupRecord.FileName
if strings.TrimSpace(fileName) == "" {
fileName = filepath.Base(backupRecord.StoragePath)
server/internal/storage/rclone/factory.go
+16 -10
View File
@@ -43,7 +43,7 @@ type LocalDiskFactory struct{}
func NewLocalDiskFactory() LocalDiskFactory { return LocalDiskFactory{} }
func (LocalDiskFactory) Type() storage.ProviderType { return storage.ProviderTypeLocalDisk }
func (LocalDiskFactory) SensitiveFields() []string { return nil }
func (LocalDiskFactory) SensitiveFields() []string { return nil }
func (LocalDiskFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
cfg, err := storage.DecodeConfig[storage.LocalDiskConfig](rawConfig)
@@ -66,7 +66,7 @@ type S3Factory struct{}
func NewS3Factory() S3Factory { return S3Factory{} }
func (S3Factory) Type() storage.ProviderType { return storage.ProviderTypeS3 }
func (S3Factory) SensitiveFields() []string { return []string{"accessKeyId", "secretAccessKey"} }
func (S3Factory) SensitiveFields() []string { return []string{"accessKeyId", "secretAccessKey"} }
func (S3Factory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
cfg, err := storage.DecodeConfig[storage.S3Config](rawConfig)
@@ -116,7 +116,7 @@ type WebDAVFactory struct{}
func NewWebDAVFactory() WebDAVFactory { return WebDAVFactory{} }
func (WebDAVFactory) Type() storage.ProviderType { return storage.ProviderTypeWebDAV }
func (WebDAVFactory) SensitiveFields() []string { return []string{"username", "password"} }
func (WebDAVFactory) SensitiveFields() []string { return []string{"username", "password"} }
func (WebDAVFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
cfg, err := storage.DecodeConfig[storage.WebDAVConfig](rawConfig)
@@ -187,7 +187,7 @@ type FTPFactory struct{}
func NewFTPFactory() FTPFactory { return FTPFactory{} }
func (FTPFactory) Type() storage.ProviderType { return storage.ProviderTypeFTP }
func (FTPFactory) SensitiveFields() []string { return []string{"username", "password"} }
func (FTPFactory) SensitiveFields() []string { return []string{"username", "password"} }
func (FTPFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
cfg, err := storage.DecodeConfig[storage.FTPConfig](rawConfig)
@@ -228,7 +228,7 @@ type AliyunOSSFactory struct{}
func NewAliyunOSSFactory() AliyunOSSFactory { return AliyunOSSFactory{} }
func (AliyunOSSFactory) Type() storage.ProviderType { return storage.ProviderTypeAliyunOSS }
func (AliyunOSSFactory) SensitiveFields() []string { return []string{"accessKeyId", "secretAccessKey"} }
func (AliyunOSSFactory) SensitiveFields() []string { return []string{"accessKeyId", "secretAccessKey"} }
// AliyunConfig 是阿里云 OSS 的用户配置。
type AliyunConfig struct {
@@ -269,7 +269,9 @@ type TencentCOSFactory struct{}
func NewTencentCOSFactory() TencentCOSFactory { return TencentCOSFactory{} }
func (TencentCOSFactory) Type() storage.ProviderType { return storage.ProviderTypeTencentCOS }
func (TencentCOSFactory) SensitiveFields() []string { return []string{"accessKeyId", "secretAccessKey"} }
func (TencentCOSFactory) SensitiveFields() []string {
return []string{"accessKeyId", "secretAccessKey"}
}
// TencentConfig 是腾讯云 COS 的用户配置。
type TencentConfig struct {
@@ -305,7 +307,7 @@ type QiniuKodoFactory struct{}
func NewQiniuKodoFactory() QiniuKodoFactory { return QiniuKodoFactory{} }
func (QiniuKodoFactory) Type() storage.ProviderType { return storage.ProviderTypeQiniuKodo }
func (QiniuKodoFactory) SensitiveFields() []string { return []string{"accessKeyId", "secretAccessKey"} }
func (QiniuKodoFactory) SensitiveFields() []string { return []string{"accessKeyId", "secretAccessKey"} }
// QiniuConfig 是七牛云 Kodo 的用户配置。
type QiniuConfig struct {
@@ -355,7 +357,9 @@ type RcloneFactory struct{}
func NewRcloneFactory() RcloneFactory { return RcloneFactory{} }
func (RcloneFactory) Type() storage.ProviderType { return storage.ProviderTypeRclone }
func (RcloneFactory) SensitiveFields() []string { return []string{"pass", "password", "secret_access_key", "client_secret", "token"} }
func (RcloneFactory) SensitiveFields() []string {
return []string{"pass", "password", "secret_access_key", "client_secret", "token"}
}
func (RcloneFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
backend, _ := rawConfig["backend"].(string)
@@ -462,8 +466,10 @@ func NewBackendFactory(backendType string) GenericBackendFactory {
return GenericBackendFactory{backendType: backendType, sensitive: sensitive}
}
func (f GenericBackendFactory) Type() storage.ProviderType { return storage.ProviderType(f.backendType) }
func (f GenericBackendFactory) SensitiveFields() []string { return f.sensitive }
func (f GenericBackendFactory) Type() storage.ProviderType {
return storage.ProviderType(f.backendType)
}
func (f GenericBackendFactory) SensitiveFields() []string { return f.sensitive }
func (f GenericBackendFactory) New(ctx context.Context, rawConfig map[string]any) (storage.StorageProvider, error) {
root, _ := rawConfig["root"].(string)
server/internal/storage/rclone/provider.go
+47
View File
@@ -2,6 +2,7 @@ package rclone
import (
"context"
"errors"
"fmt"
"io"
"sort"
@@ -68,13 +69,56 @@ func (p *Provider) Download(ctx context.Context, objectKey string) (io.ReadClose
return reader, nil
}
// DownloadRange reads one slice from an object. Most object-storage backends
// map this to a native HTTP Range request. Backends that reject ranged reads
// fall back to a full stream while preserving the same interface contract.
func (p *Provider) DownloadRange(ctx context.Context, objectKey string, offset, length int64) (io.ReadCloser, error) {
if offset < 0 || length <= 0 {
return nil, fmt.Errorf("rclone download range %s: invalid offset=%d length=%d", objectKey, offset, length)
}
obj, err := p.rfs.NewObject(ctx, objectKey)
if err != nil {
return nil, fmt.Errorf("rclone find object %s: %w", objectKey, err)
}
reader, rangeErr := obj.Open(ctx, &fs.RangeOption{Start: offset, End: offset + length - 1})
if rangeErr == nil {
return reader, nil
}
reader, err = obj.Open(ctx)
if err != nil {
return nil, fmt.Errorf("rclone download range %s (range: %v; fallback: %w)", objectKey, rangeErr, err)
}
if offset > 0 {
if _, err := io.CopyN(io.Discard, reader, offset); err != nil {
closeErr := reader.Close()
return nil, errors.Join(fmt.Errorf("rclone seek object %s: %w", objectKey, err), closeErr)
}
}
return &limitedReadCloser{Reader: io.LimitReader(reader, length), closer: reader}, nil
}
type limitedReadCloser struct {
io.Reader
closer io.Closer
}
func (r *limitedReadCloser) Close() error {
return r.closer.Close()
}
// Delete 通过 rclone 删除远端对象。
func (p *Provider) Delete(ctx context.Context, objectKey string) error {
obj, err := p.rfs.NewObject(ctx, objectKey)
if err != nil {
if errors.Is(err, fs.ErrorObjectNotFound) || errors.Is(err, fs.ErrorDirNotFound) {
return nil
}
return fmt.Errorf("rclone find object %s: %w", objectKey, err)
}
if err := obj.Remove(ctx); err != nil {
if errors.Is(err, fs.ErrorObjectNotFound) || errors.Is(err, fs.ErrorDirNotFound) {
return nil
}
return fmt.Errorf("rclone delete %s: %w", objectKey, err)
}
return nil
@@ -102,6 +146,9 @@ func (p *Provider) List(ctx context.Context, prefix string) ([]storage.ObjectInf
return nil
})
if err != nil {
if errors.Is(err, fs.ErrorDirNotFound) || errors.Is(err, fs.ErrorObjectNotFound) {
return []storage.ObjectInfo{}, nil
}
return nil, fmt.Errorf("rclone list %s: %w", prefix, err)
}
return items, nil
server/internal/storage/types.go
+17 -2
View File
@@ -34,6 +34,14 @@ const (
TypeFTP = string(ProviderTypeFTP)
)
const (
// TransferModeDirect lets an Agent write to a network-accessible backend.
TransferModeDirect = "direct"
// TransferModeMasterRelay streams an artifact through the authenticated
// Agent API so a remote source can use storage mounted only on the Master.
TransferModeMasterRelay = "master_relay"
)
type ObjectInfo struct {
Key string `json:"key"`
Size int64 `json:"size"`
@@ -49,6 +57,13 @@ type StorageProvider interface {
List(ctx context.Context, prefix string) ([]ObjectInfo, error)
}
// StorageRangeDownloader is an optional capability used by packed repository
// backups. Implementations return exactly the requested byte range when the
// backend supports ranged reads and may transparently fall back to a full read.
type StorageRangeDownloader interface {
DownloadRange(ctx context.Context, objectKey string, offset, length int64) (io.ReadCloser, error)
}
type ProviderFactory interface {
Type() ProviderType
}
@@ -92,7 +107,8 @@ func ParseProviderType(value string) ProviderType {
}
type LocalDiskConfig struct {
BasePath string `json:"basePath"`
BasePath string `json:"basePath"`
MasterRelay bool `json:"masterRelay"`
}
type S3Config struct {
@@ -151,4 +167,3 @@ type FTPConfig struct {
type StorageDirCleaner interface {
RemoveEmptyDirs(ctx context.Context, prefix string) error
}