Compare commits
21 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
da6cf42355 | ||
|
|
8c76516953 | ||
|
|
15c4fffb98 | ||
|
|
b5cdf1e880 | ||
|
|
264dd9f9ed | ||
|
|
b25df2e214 | ||
|
|
5999ddbe1d | ||
|
|
7424190ee5 | ||
|
|
87e8836c78 | ||
|
|
1a7747c2d2 | ||
|
|
ca0fd67fba | ||
|
|
4d736b925b | ||
|
|
ead2b20f4e | ||
|
|
080d474714 | ||
|
|
f453205fde | ||
|
|
407677f270 | ||
|
|
958bfd1dbe | ||
|
|
debe33d84d | ||
|
|
52eead3bf5 | ||
|
|
0af049a507 | ||
|
|
8752dd865c |
130
.github/workflows/build-docker.yml
vendored
130
.github/workflows/build-docker.yml
vendored
@@ -7,33 +7,56 @@ on:
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
IMAGE_NAME: krau/saveany-bot
|
||||
|
||||
concurrency:
|
||||
group: docker-build-${{ github.repository }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
prepare:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
version: ${{ steps.vars.outputs.version }}
|
||||
major_minor: ${{ steps.vars.outputs.major_minor }}
|
||||
short_sha: ${{ steps.vars.outputs.short_sha }}
|
||||
build_time: ${{ steps.vars.outputs.build_time }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Extract Version Components
|
||||
id: vars
|
||||
run: |
|
||||
VERSION=${GITHUB_REF#refs/tags/v}
|
||||
|
||||
MAJOR_MINOR=$(echo "$VERSION" | cut -d. -f1,2)
|
||||
|
||||
SHORT_SHA=$(git rev-parse --short HEAD)
|
||||
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
echo "major_minor=$MAJOR_MINOR" >> "$GITHUB_OUTPUT"
|
||||
echo "short_sha=$SHORT_SHA" >> "$GITHUB_OUTPUT"
|
||||
echo "build_time=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
build:
|
||||
needs: prepare
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
strategy:
|
||||
matrix:
|
||||
arch: [amd64, arm64]
|
||||
type: [default, micro, pico]
|
||||
fail-fast: false
|
||||
runs-on: ${{ matrix.arch == 'amd64' && 'ubuntu-latest' || 'ubuntu-24.04-arm' }}
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Extract metadata for Docker
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
@@ -44,26 +67,63 @@ jobs:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract Dockerfile args
|
||||
id: args
|
||||
run: |
|
||||
echo "git_commit=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
|
||||
echo "build_time=$(git show -s --format=%cI)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Build and push Docker image
|
||||
id: build-and-push
|
||||
- name: Build and push (Temporary Tags)
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: |
|
||||
type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
|
||||
type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
# 根据类型选择 Dockerfile
|
||||
file: ${{ matrix.type == 'default' && './Dockerfile' || format('./Dockerfile.{0}', matrix.type) }}
|
||||
# 仅构建当前架构
|
||||
platforms: ${{ matrix.arch == 'amd64' && 'linux/amd64' || 'linux/arm64' }}
|
||||
push: true
|
||||
# 推送带有架构后缀的临时标签,供后续合并使用
|
||||
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:tmp-${{ matrix.type }}-${{ matrix.arch }}
|
||||
build-args: |
|
||||
VERSION=${{ steps.meta.outputs.version }}
|
||||
GitCommit=${{ steps.args.outputs.git_commit }}
|
||||
BuildTime=${{ steps.args.outputs.build_time }}
|
||||
VERSION=${{ needs.prepare.outputs.version }}
|
||||
GitCommit=${{ needs.prepare.outputs.short_sha }}
|
||||
BuildTime=${{ needs.prepare.outputs.build_time }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
create-manifest:
|
||||
needs: [prepare, build]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
packages: write
|
||||
strategy:
|
||||
matrix:
|
||||
type: [default, micro, pico]
|
||||
steps:
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Create and push manifest lists
|
||||
run: |
|
||||
REPO="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
|
||||
VERSION="${{ needs.prepare.outputs.version }}"
|
||||
MAJOR_MINOR="${{ needs.prepare.outputs.major_minor }}"
|
||||
SHA="${{ needs.prepare.outputs.short_sha }}"
|
||||
TYPE="${{ matrix.type }}"
|
||||
TAGS=()
|
||||
|
||||
if [ "$TYPE" == "default" ]; then
|
||||
TAGS+=("$REPO:latest")
|
||||
TAGS+=("$REPO:$VERSION")
|
||||
TAGS+=("$REPO:$MAJOR_MINOR")
|
||||
TAGS+=("$REPO:sha-$SHA")
|
||||
else
|
||||
TAGS+=("$REPO:$TYPE")
|
||||
TAGS+=("$REPO:$TYPE-latest")
|
||||
TAGS+=("$REPO:$TYPE-$VERSION")
|
||||
fi
|
||||
|
||||
SRC_AMD64="$REPO:tmp-$TYPE-amd64"
|
||||
SRC_ARM64="$REPO:tmp-$TYPE-arm64"
|
||||
|
||||
for TAG in "${TAGS[@]}"; do
|
||||
docker buildx imagetools create -t "$TAG" "$SRC_AMD64" "$SRC_ARM64"
|
||||
done
|
||||
|
||||
@@ -20,6 +20,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
-X 'github.com/krau/SaveAny-Bot/config.Version=${VERSION}' \
|
||||
-X 'github.com/krau/SaveAny-Bot/config.GitCommit=${GitCommit}' \
|
||||
-X 'github.com/krau/SaveAny-Bot/config.BuildTime=${BuildTime}' \
|
||||
-X 'github.com/krau/SaveAny-Bot/config.Docker=true' \
|
||||
" \
|
||||
-o saveany-bot .
|
||||
|
||||
|
||||
41
Dockerfile.micro
Normal file
41
Dockerfile.micro
Normal file
@@ -0,0 +1,41 @@
|
||||
FROM golang:alpine AS builder
|
||||
|
||||
ARG VERSION="dev"
|
||||
ARG GitCommit="Unknown"
|
||||
ARG BuildTime="Unknown"
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
go mod download
|
||||
|
||||
COPY . .
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
--mount=type=cache,target=/go/pkg \
|
||||
CGO_ENABLED=0 \
|
||||
go build -trimpath \
|
||||
-tags=no_jsparser,no_minio \
|
||||
-ldflags=" \
|
||||
-s -w \
|
||||
-X 'github.com/krau/SaveAny-Bot/config.Version=${VERSION}' \
|
||||
-X 'github.com/krau/SaveAny-Bot/config.GitCommit=${GitCommit}' \
|
||||
-X 'github.com/krau/SaveAny-Bot/config.BuildTime=${BuildTime}' \
|
||||
-X 'github.com/krau/SaveAny-Bot/config.Docker=true' \
|
||||
" \
|
||||
-o saveany-bot .
|
||||
|
||||
FROM alpine:latest
|
||||
|
||||
RUN apk add --no-cache curl
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /app/saveany-bot .
|
||||
COPY entrypoint.sh .
|
||||
|
||||
RUN chmod +x /app/saveany-bot && \
|
||||
chmod +x /app/entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/app/entrypoint.sh"]
|
||||
|
||||
35
Dockerfile.pico
Normal file
35
Dockerfile.pico
Normal file
@@ -0,0 +1,35 @@
|
||||
# pico is the minimum build of SaveAnyBot, which disables all the optional features like JS parsing and MinIO support.
|
||||
FROM golang:alpine AS builder
|
||||
|
||||
ARG VERSION="dev"
|
||||
ARG GitCommit="Unknown"
|
||||
ARG BuildTime="Unknown"
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
go mod download
|
||||
|
||||
COPY . .
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
--mount=type=cache,target=/go/pkg \
|
||||
CGO_ENABLED=0 \
|
||||
go build -trimpath \
|
||||
-tags=no_jsparser,no_minio,sqlite_glebarez \
|
||||
-ldflags=" \
|
||||
-s -w \
|
||||
-X 'github.com/krau/SaveAny-Bot/config.Version=${VERSION}' \
|
||||
-X 'github.com/krau/SaveAny-Bot/config.GitCommit=${GitCommit}' \
|
||||
-X 'github.com/krau/SaveAny-Bot/config.BuildTime=${BuildTime}' \
|
||||
-X 'github.com/krau/SaveAny-Bot/config.Docker=true' \
|
||||
" \
|
||||
-o saveany-bot . && chmod +x saveany-bot
|
||||
|
||||
FROM scratch
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /app/saveany-bot .
|
||||
|
||||
ENTRYPOINT ["/app/saveany-bot"]
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"github.com/krau/SaveAny-Bot/client/middleware"
|
||||
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/ncruces/go-sqlite3/gormlite"
|
||||
"github.com/krau/SaveAny-Bot/database"
|
||||
)
|
||||
|
||||
func Init(ctx context.Context) <-chan struct{} {
|
||||
@@ -39,7 +39,7 @@ func Init(ctx context.Context) <-chan struct{} {
|
||||
config.C().Telegram.AppHash,
|
||||
gotgproto.ClientTypeBot(config.C().Telegram.Token),
|
||||
&gotgproto.ClientOpts{
|
||||
Session: sessionMaker.SqlSession(gormlite.Open(config.C().DB.Session)),
|
||||
Session: sessionMaker.SqlSession(database.GetDialect(config.C().DB.Session)),
|
||||
DisableCopyright: true,
|
||||
Middlewares: middleware.NewDefaultMiddlewares(ctx, 5*time.Minute),
|
||||
Resolver: resolver,
|
||||
|
||||
@@ -38,6 +38,7 @@ func handleUpdateCmd(ctx *ext.Context, u *ext.Update) error {
|
||||
ctx.Reply(u, ext.ReplyTextString(fmt.Sprintf("当前已经是最新版本: %s", config.Version)), nil)
|
||||
return dispatcher.EndGroups
|
||||
}
|
||||
indocker := config.Docker == "true"
|
||||
ctx.Sender.To(u.GetUserChat().AsInputPeer()).StyledText(ctx, html.String(nil, func() string {
|
||||
md := latest.ReleaseNotes
|
||||
md = regexp.MustCompile(`(?m)^###\s+ (.+)$`).ReplaceAllString(md, "<b>$1</b>")
|
||||
@@ -53,6 +54,15 @@ func handleUpdateCmd(ctx *ext.Context, u *ext.Update) error {
|
||||
|
||||
return `<blockquote expandable>` + md + `</blockquote>`
|
||||
}()))
|
||||
if indocker {
|
||||
text := fmt.Sprintf("发现新版本: %s\n当前版本: %s\n发布时间: %s\n由于您正在使用 Docker 部署, 请自行在部署平台上执行更新命令",
|
||||
latest.Version,
|
||||
config.Version,
|
||||
latest.PublishedAt.Format("2006-01-02 15:04:05"),
|
||||
)
|
||||
ctx.Reply(u, ext.ReplyTextString(text), nil)
|
||||
return dispatcher.EndGroups
|
||||
}
|
||||
text := fmt.Sprintf(`发现新版本: %s
|
||||
当前版本: %s
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ import (
|
||||
"github.com/krau/SaveAny-Bot/common/utils/tgutil"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
"github.com/krau/SaveAny-Bot/database"
|
||||
"github.com/ncruces/go-sqlite3/gormlite"
|
||||
)
|
||||
|
||||
var uc *gotgproto.Client
|
||||
@@ -64,7 +63,7 @@ func Login(ctx context.Context) (*gotgproto.Client, error) {
|
||||
config.C().Telegram.AppHash,
|
||||
gotgproto.ClientTypePhone(""),
|
||||
&gotgproto.ClientOpts{
|
||||
Session: sessionMaker.SqlSession(gormlite.Open(config.C().Telegram.Userbot.Session)),
|
||||
Session: sessionMaker.SqlSession(database.GetDialect(config.C().Telegram.Userbot.Session)),
|
||||
AuthConversator: &terminalAuthConversator{},
|
||||
Context: ctx,
|
||||
DisableCopyright: true,
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
// [TODO] complete the i18n support
|
||||
|
||||
package i18n
|
||||
|
||||
import (
|
||||
|
||||
@@ -6,8 +6,9 @@ var (
|
||||
Version string = "dev"
|
||||
BuildTime string = "unknown"
|
||||
GitCommit string = "unknown"
|
||||
Docker string = "false" // whether built inside Docker
|
||||
)
|
||||
|
||||
const (
|
||||
GitRepo = "krau/SaveAny-Bot"
|
||||
)
|
||||
)
|
||||
|
||||
@@ -9,8 +9,6 @@ import (
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/krau/SaveAny-Bot/config"
|
||||
_ "github.com/ncruces/go-sqlite3/embed"
|
||||
"github.com/ncruces/go-sqlite3/gormlite"
|
||||
"gorm.io/gorm"
|
||||
glogger "gorm.io/gorm/logger"
|
||||
)
|
||||
@@ -23,7 +21,7 @@ func Init(ctx context.Context) {
|
||||
logger.Fatal("Failed to create data directory: ", err)
|
||||
}
|
||||
var err error
|
||||
db, err = gorm.Open(gormlite.Open(config.C().DB.Path), &gorm.Config{
|
||||
db, err = gorm.Open(GetDialect(config.C().DB.Path), &gorm.Config{
|
||||
Logger: glogger.New(logger, glogger.Config{
|
||||
Colorful: true,
|
||||
SlowThreshold: time.Second * 5,
|
||||
|
||||
13
database/driver.go
Normal file
13
database/driver.go
Normal file
@@ -0,0 +1,13 @@
|
||||
//go:build !sqlite_glebarez
|
||||
|
||||
package database
|
||||
|
||||
import (
|
||||
_ "github.com/ncruces/go-sqlite3/embed"
|
||||
"github.com/ncruces/go-sqlite3/gormlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func GetDialect(dsn string) gorm.Dialector {
|
||||
return gormlite.Open(dsn)
|
||||
}
|
||||
12
database/driver_glebarez.go
Normal file
12
database/driver_glebarez.go
Normal file
@@ -0,0 +1,12 @@
|
||||
//go:build sqlite_glebarez
|
||||
|
||||
package database
|
||||
|
||||
import (
|
||||
"github.com/glebarez/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func GetDialect(dsn string) gorm.Dialector {
|
||||
return sqlite.Open(dsn)
|
||||
}
|
||||
@@ -44,6 +44,15 @@ Stream 模式对于磁盘空间有限的部署环境十分有用, 但也有一
|
||||
- `workers`: 同时处理任务数量, 默认为 3
|
||||
- `threads`: 下载文件时使用的线程数, 默认为 4. 仅在未启用 Stream 模式时生效.
|
||||
- `retry`: 任务失败时的重试次数, 默认为 3.
|
||||
- `proxy`: 全局代理配置, 配置后程序内一切网络连接将会尝试使用该代理, 可选.
|
||||
|
||||
```toml
|
||||
stream = false
|
||||
workers = 3
|
||||
threads = 4
|
||||
retry = 3
|
||||
proxy = "socks5://127.0.0.1:7890"
|
||||
```
|
||||
|
||||
### Telegram 配置
|
||||
|
||||
|
||||
@@ -46,15 +46,29 @@ base_path = "/path/to/webdav" # WebDAV 中的基础路径, 所有文件将存储
|
||||
`type=s3`
|
||||
|
||||
```toml
|
||||
endpoint = "s3.example.com" # S3 的端点
|
||||
endpoint = "s3.example.com" # S3 的端点, 默认为 aws S3 的端点
|
||||
region = "us-east-1" # S3 的区域
|
||||
access_key_id = "your_access_key_id" # S3 的访问密钥 ID
|
||||
secret_access_key = "your_secret_access_key" # S3 的秘密访问密钥
|
||||
bucket_name = "your_bucket_name" # S3 的存储桶名称
|
||||
use_ssl = true # 是否使用 SSL, 默认为 true
|
||||
base_path = "/path/to/s3" # S3 中的基础路径, 所有文件将存储在此路径下
|
||||
virtual_host = false # 使用虚拟主机风格的 URL, 默认为 false
|
||||
```
|
||||
|
||||
虚拟主机风格的 URL 示例:
|
||||
|
||||
```
|
||||
https://your_bucket_name.s3.example.com/path/to/s3/your_file
|
||||
```
|
||||
|
||||
路径风格(关闭 virtual_host)的 URL 示例:
|
||||
|
||||
```
|
||||
https://s3.example.com/your_bucket_name/path/to/s3/your_file
|
||||
```
|
||||
|
||||
如果你使用的是第三方的兼容 S3 的服务, 一般使用的是路径风格的 URL. 而 AWS S3 则通常使用虚拟主机风格的 URL. 详情请参考你所使用的 S3 兼容服务的文档.
|
||||
|
||||
## Telegram
|
||||
|
||||
`type=telegram`
|
||||
@@ -63,4 +77,7 @@ base_path = "/path/to/s3" # S3 中的基础路径, 所有文件将存储在此
|
||||
|
||||
```toml
|
||||
chat_id = "123456789" # Telegram 聊天 ID, Bot 将把文件发送到这个聊天
|
||||
force_file = false # 是否强制使用文件方式发送, 默认为 false.
|
||||
skip_large = true # 是否跳过大文件, 默认为 true. 如果启用, 超过 Telegram 限制的文件将不会上传.
|
||||
spilt_size_mb = 2000 # 分卷大小, 单位 MB, 默认为 2000 MB (2 GB). 超过该大小的文件将被分割成多个部分上传.(使用 zip 格式)
|
||||
```
|
||||
@@ -129,17 +129,39 @@ docker run -d --name saveany-bot \
|
||||
ghcr.io/krau/saveany-bot:latest
|
||||
```
|
||||
|
||||
{{< hint info >}}
|
||||
关于 docker 镜像的变体版本
|
||||
<br />
|
||||
<ul>
|
||||
<li>默认版本: 包含所有功能和依赖, 体积较大. 如果没有特殊需要, 请使用此版本</li>
|
||||
<li>micro: 精简版本, 去除部分可选依赖, 体积较小</li>
|
||||
<li>pico: 极简版本, 仅包含核心功能, 体积最小</li>
|
||||
</ul>
|
||||
你可以根据需要, 通过指定不同的标签来拉取合适的版本, 例如: <code>ghcr.io/krau/saveany-bot:micro</code>
|
||||
<br />
|
||||
关于变体版本的更详细的区别, 请参考项目根目录下的 Dockerfile 文件.
|
||||
{{< /hint >}}
|
||||
|
||||
## 更新
|
||||
|
||||
向 Bot 发送 `/update` 指令检查更新并升级, 或者使用 CLI 命令更新:
|
||||
若使用预编译二进制文件部署, 使用以下 CLI 命令更新:
|
||||
|
||||
```bash
|
||||
./saveany-bot up
|
||||
```
|
||||
|
||||
如果是 Docker 部署, 还可以使用以下命令更新:
|
||||
如果是 Docker 部署, 使用以下命令更新:
|
||||
|
||||
docker:
|
||||
|
||||
```bash
|
||||
docker pull ghcr.io/krau/saveany-bot:latest
|
||||
docker restart saveany-bot
|
||||
```
|
||||
|
||||
docker compose:
|
||||
|
||||
```bash
|
||||
docker compose pull
|
||||
docker compose restart
|
||||
```
|
||||
20
go.mod
20
go.mod
@@ -3,10 +3,6 @@ module github.com/krau/SaveAny-Bot
|
||||
go 1.24.0
|
||||
|
||||
require (
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.3
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0
|
||||
github.com/blang/semver v3.5.1+incompatible
|
||||
github.com/celestix/gotgproto v1.0.0-beta22
|
||||
github.com/cenkalti/backoff/v4 v4.3.0
|
||||
@@ -31,20 +27,6 @@ require (
|
||||
|
||||
require (
|
||||
github.com/AnimeKaizoku/cacher v1.0.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
@@ -125,7 +107,7 @@ require (
|
||||
github.com/dop251/goja v0.0.0-20251008123653-cf18d89f3cf6
|
||||
github.com/duke-git/lancet/v2 v2.3.7
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/glebarez/sqlite v1.11.0 // indirect
|
||||
github.com/glebarez/sqlite v1.11.0
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
|
||||
58
go.sum
58
go.sum
@@ -4,44 +4,30 @@ github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg
|
||||
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
||||
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
|
||||
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1 h1:difXb4maDZkRH0x//Qkwcfpdg1XQVXEAEs2DdXldFFc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.40.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.3 h1:cpz7H2uMNTDa0h/5CYL5dLUEzPSLo2g0NkbxTRJtSSU=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.3/go.mod h1:srtPKaJJe3McW6T/+GMBZyIPc+SeqJsNPJsd4mOYZ6s=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3 h1:01Ym72hK43hjwDeJUfi1l2oYLXBAOR8gNSZNmXmvuas=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.3/go.mod h1:55nWF/Sr9Zvls0bGnWkRxUdhzKqj9uRNlPvgV1vgxKc=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15 h1:utxLraaifrSBkeyII9mIbVwXXWrZdlPO7FIKmyLCEcY=
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.15/go.mod h1:hW6zjYUDQwfz3icf4g2O41PHi77u10oAzJ84iSzR/lo=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM=
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI=
|
||||
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15 h1:Y5YXgygXwDI5P4RkteB5yF7v35neH7LfJKBG+hzIons=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.15/go.mod h1:K+/1EpG42dFSY7CBj+Fruzm8PsCGWTXJ3jdeJ659oGQ=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15 h1:AvltKnW9ewxX2hFmQS0FyJH93aSvJVUEFvXfU+HWtSE=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.15/go.mod h1:3I4oCdZdmgrREhU74qS1dK9yZ62yumob+58AbFR4cQA=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15 h1:NLYTEyZmVZo0Qh183sC8nC+ydJXOOeIL/qI/sS3PdLY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.15/go.mod h1:Z803iB3B0bc8oJV8zH2PERLRfQUJ2n2BXISpsA4+O1M=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6 h1:P1MU/SuhadGvg2jtviDXPEejU3jBNhoeeAlRadHzvHI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.6/go.mod h1:5KYaMG6wmVKMFBSfWoyG/zH8pWwzQFnKgpoSRlXHKdQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15 h1:3/u/4yZOffg5jdNk1sDpOQ4Y+R6Xbh+GzpDrSZjuy3U=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.15/go.mod h1:4Zkjq0FKjE78NKjabuM4tRXKFzUJWXgP0ItEZK8l7JU=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15 h1:wsSQ4SVz5YE1crz0Ap7VBZrV4nNqZt4CIBBT8mnwoNc=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.15/go.mod h1:I7sditnFGtYMIqPRU1QoHZAUrXkGp4SczmlLwrNPlD0=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0 h1:IrbE3B8O9pm3lsg96AXIN5MXX4pECEuExh/A0Du3AuI=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.93.0/go.mod h1:/sJLzHtiiZvs6C1RbxS/anSAFwZD6oC6M/kotQzOiLw=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.3 h1:d/6xOGIllc/XW1lzG9a4AUBMmpLA9PXcQnVPTuHHcik=
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.3/go.mod h1:fQ7E7Qj9GiW8y0ClD7cUJk3Bz5Iw8wZkWDHsTe8vDKs=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6 h1:8sTTiw+9yuNXcfWeqKF2x01GqCF49CpP4Z9nKrrk/ts=
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.6/go.mod h1:8WYg+Y40Sn3X2hioaaWAAIngndR8n1XFdRPPX+7QBaM=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11 h1:E+KqWoVsSrj1tJ6I/fjDIu5xoS2Zacuu1zT+H7KtiIk=
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.11/go.mod h1:qyWHz+4lvkXcr3+PoGlGHEI+3DLLiU6/GdrFfMaAhB0=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3 h1:tzMkjh0yTChUqJDgGkcDdxvZDSrJ/WB6R6ymI5ehqJI=
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.3/go.mod h1:T270C0R5sZNLbWUe8ueiAF42XSZxxPocTaGSgs5c/60=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM=
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 h1:4nm2G6A4pV9rdlWzGMPv4BNtQp22v1hg3yrtkYpeLl8=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ=
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k=
|
||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build !no_jsparser
|
||||
|
||||
package js
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build !no_jsparser && !no_playwright
|
||||
|
||||
package js
|
||||
|
||||
import (
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//go:build no_playwright
|
||||
//go:build no_playwright && !no_jsparser
|
||||
|
||||
package js
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build !no_jsparser
|
||||
|
||||
package js
|
||||
|
||||
import (
|
||||
|
||||
16
parsers/js/js_stub.go
Normal file
16
parsers/js/js_stub.go
Normal file
@@ -0,0 +1,16 @@
|
||||
//go:build no_jsparser
|
||||
|
||||
package js
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
)
|
||||
|
||||
func LoadPlugins(ctx context.Context, dir string) error {
|
||||
return errors.New("JS parser plugins are not supported in this build")
|
||||
}
|
||||
|
||||
func AddPlugin(ctx context.Context, code string, name string) error {
|
||||
return errors.New("JS parser plugins are not supported in this build")
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build !no_jsparser
|
||||
|
||||
package js
|
||||
|
||||
import "github.com/blang/semver"
|
||||
|
||||
221
pkg/s3/client.go
Normal file
221
pkg/s3/client.go
Normal file
@@ -0,0 +1,221 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
endpoint string
|
||||
region string
|
||||
bucket string
|
||||
accessKey string
|
||||
secretKey string
|
||||
httpClient *http.Client
|
||||
pathStyle bool
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
Region string
|
||||
BucketName string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
PathStyle bool
|
||||
HttpClient *http.Client
|
||||
}
|
||||
|
||||
func (c *Config) ApplyDefaults() {
|
||||
if c.HttpClient == nil {
|
||||
c.HttpClient = http.DefaultClient
|
||||
}
|
||||
if c.Endpoint == "" {
|
||||
switch c.Region {
|
||||
case "us-east-1", "":
|
||||
c.Endpoint = "https://s3.amazonaws.com"
|
||||
default:
|
||||
c.Endpoint = fmt.Sprintf("https://s3.%s.amazonaws.com", c.Region)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func NewClient(cfg *Config) (*Client, error) {
|
||||
cfg.ApplyDefaults()
|
||||
return &Client{
|
||||
endpoint: cfg.Endpoint,
|
||||
region: cfg.Region,
|
||||
bucket: cfg.BucketName,
|
||||
accessKey: cfg.AccessKeyID,
|
||||
secretKey: cfg.SecretAccessKey,
|
||||
httpClient: cfg.HttpClient,
|
||||
pathStyle: cfg.PathStyle,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Client) HeadBucket(ctx context.Context) error {
|
||||
url, err := c.buildURL("")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "HEAD", url, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := signRequest(req, c.region, c.accessKey, c.secretKey, hashSHA256(nil)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("head bucket failed: %s", resp.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) Exists(ctx context.Context, key string) bool {
|
||||
url, err := c.buildURL(key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "HEAD", url, nil)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if err := signRequest(req, c.region, c.accessKey, c.secretKey, hashSHA256(nil)); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
return resp.StatusCode == http.StatusOK
|
||||
}
|
||||
|
||||
func (c *Client) Put(ctx context.Context, key string, r io.Reader, size int64) error {
|
||||
url, err := c.buildURL(key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "PUT", url, r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if size >= 0 {
|
||||
req.ContentLength = size
|
||||
}
|
||||
|
||||
if err := signRequest(req, c.region, c.accessKey, c.secretKey, "UNSIGNED-PAYLOAD"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("put object failed: %s", resp.Status)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) buildURL(key string) (string, error) {
|
||||
if c.pathStyle {
|
||||
return fmt.Sprintf("%s/%s/%s", c.endpoint, c.bucket, key), nil
|
||||
}
|
||||
u, err := url.Parse(c.endpoint)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
u.Host = c.bucket + "." + u.Host
|
||||
u.Path = "/" + key
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func hmacSHA256(key []byte, data string) []byte {
|
||||
h := hmac.New(sha256.New, key)
|
||||
h.Write([]byte(data))
|
||||
return h.Sum(nil)
|
||||
}
|
||||
|
||||
func hashSHA256(data []byte) string {
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func signRequest(req *http.Request, region, accessKey, secretKey string, payloadHash string) error {
|
||||
now := time.Now().UTC()
|
||||
amzDate := now.Format("20060102T150405Z")
|
||||
date := now.Format("20060102")
|
||||
|
||||
req.Header.Set("x-amz-date", amzDate)
|
||||
req.Header.Set("x-amz-content-sha256", payloadHash)
|
||||
|
||||
// Canonical headers
|
||||
var headers []string
|
||||
for k := range req.Header {
|
||||
headers = append(headers, strings.ToLower(k))
|
||||
}
|
||||
sort.Strings(headers)
|
||||
|
||||
var canonicalHeaders strings.Builder
|
||||
for _, k := range headers {
|
||||
canonicalHeaders.WriteString(k)
|
||||
canonicalHeaders.WriteString(":")
|
||||
canonicalHeaders.WriteString(strings.TrimSpace(req.Header.Get(k)))
|
||||
canonicalHeaders.WriteString("\n")
|
||||
}
|
||||
|
||||
signedHeaders := strings.Join(headers, ";")
|
||||
|
||||
canonicalRequest := strings.Join([]string{
|
||||
req.Method,
|
||||
req.URL.EscapedPath(),
|
||||
req.URL.RawQuery,
|
||||
canonicalHeaders.String(),
|
||||
signedHeaders,
|
||||
payloadHash,
|
||||
}, "\n")
|
||||
|
||||
scope := fmt.Sprintf("%s/%s/s3/aws4_request", date, region)
|
||||
stringToSign := strings.Join([]string{
|
||||
"AWS4-HMAC-SHA256",
|
||||
amzDate,
|
||||
scope,
|
||||
hashSHA256([]byte(canonicalRequest)),
|
||||
}, "\n")
|
||||
|
||||
kDate := hmacSHA256([]byte("AWS4"+secretKey), date)
|
||||
kRegion := hmacSHA256(kDate, region)
|
||||
kService := hmacSHA256(kRegion, "s3")
|
||||
kSigning := hmacSHA256(kService, "aws4_request")
|
||||
|
||||
signature := hex.EncodeToString(hmacSHA256(kSigning, stringToSign))
|
||||
|
||||
auth := fmt.Sprintf(
|
||||
"AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s",
|
||||
accessKey, scope, signedHeaders, signature,
|
||||
)
|
||||
|
||||
req.Header.Set("Authorization", auth)
|
||||
return nil
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build !no_minio
|
||||
|
||||
package minio
|
||||
|
||||
import (
|
||||
|
||||
41
storage/minio/client_stub.go
Normal file
41
storage/minio/client_stub.go
Normal file
@@ -0,0 +1,41 @@
|
||||
//go:build no_minio
|
||||
|
||||
package minio
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
config "github.com/krau/SaveAny-Bot/config/storage"
|
||||
storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
|
||||
)
|
||||
|
||||
type Minio struct {
|
||||
}
|
||||
|
||||
func (m *Minio) Init(_ context.Context, _ config.StorageConfig) error {
|
||||
return fmt.Errorf("minio storage is not supported in this build")
|
||||
}
|
||||
|
||||
func (m *Minio) Type() storenum.StorageType {
|
||||
return storenum.Minio
|
||||
}
|
||||
|
||||
func (m *Minio) Name() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (m *Minio) JoinStoragePath(p string) string {
|
||||
return strings.TrimPrefix(path.Join("", p), "/")
|
||||
}
|
||||
|
||||
func (m *Minio) Save(_ context.Context, _ io.Reader, _ string) error {
|
||||
return fmt.Errorf("minio storage is not supported in this build")
|
||||
}
|
||||
|
||||
func (m *Minio) Exists(_ context.Context, _ string) bool {
|
||||
return false
|
||||
}
|
||||
@@ -4,18 +4,14 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/charmbracelet/log"
|
||||
storconfig "github.com/krau/SaveAny-Bot/config/storage"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
|
||||
storenum "github.com/krau/SaveAny-Bot/pkg/enums/storage"
|
||||
"github.com/krau/SaveAny-Bot/pkg/s3"
|
||||
"github.com/rs/xid"
|
||||
)
|
||||
|
||||
@@ -26,71 +22,32 @@ type S3 struct {
|
||||
}
|
||||
|
||||
func (m *S3) Init(ctx context.Context, cfg storconfig.StorageConfig) error {
|
||||
s3Config, ok := cfg.(*storconfig.S3StorageConfig)
|
||||
s3cfg, ok := cfg.(*storconfig.S3StorageConfig)
|
||||
if !ok {
|
||||
return fmt.Errorf("failed to cast s3 config")
|
||||
}
|
||||
if err := s3Config.Validate(); err != nil {
|
||||
if err := s3cfg.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m.config = *s3Config
|
||||
m.config = *s3cfg
|
||||
m.logger = log.FromContext(ctx).WithPrefix(fmt.Sprintf("s3[%s]", m.config.Name))
|
||||
loadOpts := make([]config.LoadOptionsFunc, 0)
|
||||
if m.config.Region != "" {
|
||||
loadOpts = append(loadOpts, config.WithRegion(m.config.Region))
|
||||
}
|
||||
if endpoint := m.config.Endpoint; endpoint != "" {
|
||||
if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") {
|
||||
if m.config.UseSSL {
|
||||
endpoint = "https://" + endpoint
|
||||
} else {
|
||||
endpoint = "http://" + endpoint
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := url.Parse(endpoint); err != nil {
|
||||
return fmt.Errorf("invalid s3 endpoint %q: %w", m.config.Endpoint, err)
|
||||
}
|
||||
loadOpts = append(loadOpts, config.WithBaseEndpoint(endpoint))
|
||||
}
|
||||
loadOpts = append(loadOpts, config.WithCredentialsProvider(
|
||||
credentials.NewStaticCredentialsProvider(
|
||||
m.config.AccessKeyID,
|
||||
m.config.SecretAccessKey,
|
||||
"",
|
||||
),
|
||||
))
|
||||
awsCfg, err := config.LoadDefaultConfig(
|
||||
ctx,
|
||||
func() []func(*config.LoadOptions) error {
|
||||
// wtf aws sdk
|
||||
// https://github.com/aws/aws-sdk-go-v2/issues/2193
|
||||
funcs := make([]func(*config.LoadOptions) error, 0)
|
||||
for _, fn := range loadOpts {
|
||||
funcs = append(funcs, fn)
|
||||
}
|
||||
return funcs
|
||||
}()...,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load AWS config: %w", err)
|
||||
}
|
||||
|
||||
m.client = s3.NewFromConfig(awsCfg, func(o *s3.Options) {
|
||||
// Path style: https://s3.amazonaws.com/mybucket/path/to/file.jpg
|
||||
// virtual hosted style: https://mybucket.s3.amazonaws.com/path/to/file.jpg
|
||||
o.UsePathStyle = !m.config.VirtualHost
|
||||
client, err := s3.NewClient(&s3.Config{
|
||||
Endpoint: m.config.Endpoint,
|
||||
Region: m.config.Region,
|
||||
AccessKeyID: m.config.AccessKeyID,
|
||||
SecretAccessKey: m.config.SecretAccessKey,
|
||||
BucketName: m.config.BucketName,
|
||||
PathStyle: !m.config.VirtualHost,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create s3 client: %w", err)
|
||||
}
|
||||
m.client = client
|
||||
|
||||
// Check if bucket exists
|
||||
_, err = m.client.HeadBucket(ctx, &s3.HeadBucketInput{
|
||||
Bucket: aws.String(m.config.BucketName),
|
||||
})
|
||||
if err != nil {
|
||||
if err := m.client.HeadBucket(ctx); err != nil {
|
||||
return fmt.Errorf("bucket %s not accessible: %w", m.config.BucketName, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -131,18 +88,7 @@ func (m *S3) Save(ctx context.Context, r io.Reader, storagePath string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// S3 PutObject needs either size or StreamingBody
|
||||
input := &s3.PutObjectInput{
|
||||
Bucket: aws.String(m.config.BucketName),
|
||||
Key: aws.String(candidate),
|
||||
Body: r,
|
||||
}
|
||||
|
||||
if size >= 0 {
|
||||
input.ContentLength = &size
|
||||
}
|
||||
|
||||
_, err := m.client.PutObject(ctx, input)
|
||||
err := m.client.Put(ctx, candidate, r, size)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload file to S3: %w", err)
|
||||
}
|
||||
@@ -153,10 +99,5 @@ func (m *S3) Save(ctx context.Context, r io.Reader, storagePath string) error {
|
||||
func (m *S3) Exists(ctx context.Context, storagePath string) bool {
|
||||
m.logger.Debugf("Checking if file exists at %s", storagePath)
|
||||
|
||||
_, err := m.client.HeadObject(ctx, &s3.HeadObjectInput{
|
||||
Bucket: aws.String(m.config.BucketName),
|
||||
Key: aws.String(storagePath),
|
||||
})
|
||||
|
||||
return err == nil
|
||||
return m.client.Exists(ctx, storagePath)
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package s3
|
||||
package s3_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"github.com/johannesboyne/gofakes3"
|
||||
"github.com/johannesboyne/gofakes3/backend/s3mem"
|
||||
storconfig "github.com/krau/SaveAny-Bot/config/storage"
|
||||
"github.com/krau/SaveAny-Bot/pkg/enums/ctxkey"
|
||||
"github.com/krau/SaveAny-Bot/storage/s3"
|
||||
)
|
||||
|
||||
func newTestContext(t *testing.T) context.Context {
|
||||
@@ -19,7 +21,7 @@ func newTestContext(t *testing.T) context.Context {
|
||||
return log.WithContext(ctx, logger)
|
||||
}
|
||||
|
||||
func newFakeS3(t *testing.T) (*S3, *storconfig.S3StorageConfig) {
|
||||
func newFakeS3(t *testing.T) (*s3.S3, *storconfig.S3StorageConfig) {
|
||||
t.Helper()
|
||||
|
||||
backend := s3mem.New()
|
||||
@@ -45,7 +47,7 @@ func newFakeS3(t *testing.T) (*S3, *storconfig.S3StorageConfig) {
|
||||
t.Fatalf("failed to create fake bucket: %v", err)
|
||||
}
|
||||
|
||||
s := &S3{}
|
||||
s := &s3.S3{}
|
||||
ctx := newTestContext(t)
|
||||
if err := s.Init(ctx, cfg); err != nil {
|
||||
t.Fatalf("init s3 failed: %v", err)
|
||||
@@ -54,9 +56,9 @@ func newFakeS3(t *testing.T) (*S3, *storconfig.S3StorageConfig) {
|
||||
return s, cfg
|
||||
}
|
||||
|
||||
func TestS3_SaveAndExists(t *testing.T) {
|
||||
func TestS3(t *testing.T) {
|
||||
s, _ := newFakeS3(t)
|
||||
ctx := context.Background()
|
||||
ctx := t.Context()
|
||||
|
||||
content := []byte("hello world")
|
||||
reader := bytes.NewReader(content)
|
||||
@@ -69,4 +71,26 @@ func TestS3_SaveAndExists(t *testing.T) {
|
||||
if !s.Exists(ctx, key) {
|
||||
t.Fatalf("Exists should return true for saved key")
|
||||
}
|
||||
|
||||
if s.Exists(ctx, "nonexistent.txt") {
|
||||
t.Fatalf("Exists should return false for nonexistent key")
|
||||
}
|
||||
|
||||
if err := s.Save(ctx, bytes.NewReader(content), key); err != nil {
|
||||
t.Fatalf("Save with existing key failed: %v", err)
|
||||
}
|
||||
|
||||
if !s.Exists(ctx, "foo/bar_1.txt") {
|
||||
t.Fatalf("Exists should return true for unique renamed key")
|
||||
}
|
||||
|
||||
var length int64 = int64(len(content))
|
||||
ctx = context.WithValue(ctx, ctxkey.ContentLength, length)
|
||||
if err := s.Save(ctx, bytes.NewReader(content), "size_test.txt"); err != nil {
|
||||
t.Fatalf("Save with content length failed: %v", err)
|
||||
}
|
||||
|
||||
if !s.Exists(ctx, "size_test.txt") {
|
||||
t.Fatalf("Exists should return true for size_test.txt")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user