mirror of
https://github.com/halfwaystudent/douyin-sparkflow.git
synced 2026-09-07 00:17:20 +08:00
fix: prevent duplicate scheduled sends
This commit is contained in:
+6
-2
@@ -5,6 +5,10 @@ WEB_BIND_ADDRESS=0.0.0.0
|
|||||||
WEB_PORT=8787
|
WEB_PORT=8787
|
||||||
SPARKFLOW_SESSION_COOKIE_SECURE=0
|
SPARKFLOW_SESSION_COOKIE_SECURE=0
|
||||||
|
|
||||||
|
# Optional compatibility override for older Docker Engine hosts. Leave empty
|
||||||
|
# unless the host daemon only supports API 1.43 or another explicitly tested version.
|
||||||
|
DOCKER_API_VERSION=
|
||||||
|
|
||||||
# Keep noVNC and proxy control ports local by default. Use an SSH tunnel for remote access.
|
# Keep noVNC and proxy control ports local by default. Use an SSH tunnel for remote access.
|
||||||
LOGIN_DESKTOP_BIND_ADDRESS=127.0.0.1
|
LOGIN_DESKTOP_BIND_ADDRESS=127.0.0.1
|
||||||
LOGIN_DESKTOP_WEB_PORT=8788
|
LOGIN_DESKTOP_WEB_PORT=8788
|
||||||
@@ -16,8 +20,8 @@ LOGIN_DESKTOP_IDLE_TIMEOUT_SECONDS=1800
|
|||||||
LOGIN_DESKTOP_STOP_AFTER_EXPORT_SECONDS=60
|
LOGIN_DESKTOP_STOP_AFTER_EXPORT_SECONDS=60
|
||||||
LOGIN_DESKTOP_STATUS_CACHE_SECONDS=15
|
LOGIN_DESKTOP_STATUS_CACHE_SECONDS=15
|
||||||
|
|
||||||
# Login browser uses direct access by default; Mihomo is an advanced option.
|
# Login browser uses direct access first; Mihomo is a fallback when direct access fails.
|
||||||
LOGIN_DESKTOP_PROXY_MODE=direct
|
LOGIN_DESKTOP_PROXY_MODE=auto
|
||||||
LOGIN_DESKTOP_PROXY=http://proxy:7890
|
LOGIN_DESKTOP_PROXY=http://proxy:7890
|
||||||
LOGIN_DESKTOP_PREFLIGHT_TIMEOUT_SECONDS=15
|
LOGIN_DESKTOP_PREFLIGHT_TIMEOUT_SECONDS=15
|
||||||
LOGIN_DESKTOP_NETWORK_CACHE_SECONDS=30
|
LOGIN_DESKTOP_NETWORK_CACHE_SECONDS=30
|
||||||
|
|||||||
@@ -61,11 +61,11 @@ def _douyin_network_mode():
|
|||||||
|
|
||||||
|
|
||||||
def douyin_network_modes():
|
def douyin_network_modes():
|
||||||
# Direct is the default; Mihomo is used only when explicitly selected.
|
# Direct is the default; Mihomo is the fallback unless explicitly selected.
|
||||||
mode = _douyin_network_mode()
|
mode = _douyin_network_mode()
|
||||||
if mode == "mihomo":
|
if mode == "mihomo":
|
||||||
return ("mihomo",)
|
return ("mihomo",)
|
||||||
return ("direct",)
|
return ("direct", "mihomo")
|
||||||
|
|
||||||
|
|
||||||
def _douyin_browser_proxy(network_mode=None):
|
def _douyin_browser_proxy(network_mode=None):
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
@@ -5,6 +6,61 @@ from datetime import datetime
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
LEGACY_DOCKER_COMMAND_MARKERS = (
|
||||||
|
"docker ps --format",
|
||||||
|
"docker exec",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_legacy_line(line):
|
||||||
|
"""Replace old Docker-in-Docker task commands with a local task runner."""
|
||||||
|
if not line.strip() or line.lstrip().startswith("#"):
|
||||||
|
return line
|
||||||
|
if not all(marker in line for marker in LEGACY_DOCKER_COMMAND_MARKERS):
|
||||||
|
return line
|
||||||
|
|
||||||
|
parts = line.split(maxsplit=5)
|
||||||
|
if len(parts) < 6:
|
||||||
|
return line
|
||||||
|
|
||||||
|
schedule = " ".join(parts[:5])
|
||||||
|
command = parts[5]
|
||||||
|
redirect_match = re.search(r"(\s+>>\s+\S+(?:\s+2>&1)?)\s*$", command)
|
||||||
|
redirect = redirect_match.group(1) if redirect_match else ""
|
||||||
|
|
||||||
|
if "SPARKFLOW_MANUAL_UNSENT_ONLY=1" in command:
|
||||||
|
env_prefix = (
|
||||||
|
"SPARKFLOW_TRIGGER_LABEL='unsent fallback' "
|
||||||
|
"SPARKFLOW_MANUAL_RUN=1 "
|
||||||
|
"SPARKFLOW_MANUAL_UNSENT_ONLY=1 "
|
||||||
|
"PYTHONUNBUFFERED=1"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
env_prefix = "SPARKFLOW_TRIGGER_LABEL='scheduled send'"
|
||||||
|
|
||||||
|
return f"{schedule} env {env_prefix} bash /app/scripts/run_scheduled_task.sh{redirect}"
|
||||||
|
|
||||||
|
|
||||||
|
def migrate_legacy_crontab(path):
|
||||||
|
"""Rewrite persisted legacy task lines once, preserving other cron entries."""
|
||||||
|
if not path.exists():
|
||||||
|
return False
|
||||||
|
|
||||||
|
raw = path.read_text(encoding="utf-8-sig", errors="replace")
|
||||||
|
lines = raw.splitlines()
|
||||||
|
migrated_lines = [migrate_legacy_line(line) for line in lines]
|
||||||
|
if migrated_lines == lines:
|
||||||
|
return False
|
||||||
|
|
||||||
|
updated = "\n".join(migrated_lines)
|
||||||
|
if updated:
|
||||||
|
updated += "\n"
|
||||||
|
temporary = path.with_name(f".{path.name}.migration.tmp")
|
||||||
|
temporary.write_text(updated, encoding="utf-8")
|
||||||
|
temporary.replace(path)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
def expand_field(field, minimum, maximum, current):
|
def expand_field(field, minimum, maximum, current):
|
||||||
values = set()
|
values = set()
|
||||||
for part in str(field).split(","):
|
for part in str(field).split(","):
|
||||||
@@ -68,6 +124,8 @@ def read_crontab(path):
|
|||||||
|
|
||||||
def run_loop(crontab_path):
|
def run_loop(crontab_path):
|
||||||
crontab_path.parent.mkdir(parents=True, exist_ok=True)
|
crontab_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
if migrate_legacy_crontab(crontab_path):
|
||||||
|
print(f"[cron_runner] migrated legacy task commands in {crontab_path}", flush=True)
|
||||||
last_minute_key = None
|
last_minute_key = None
|
||||||
print(f"[cron_runner] watching {crontab_path}", flush=True)
|
print(f"[cron_runner] watching {crontab_path}", flush=True)
|
||||||
while True:
|
while True:
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -u
|
||||||
|
|
||||||
|
script_dir="$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
app_root="$(CDPATH= cd -- "$script_dir/.." && pwd)"
|
||||||
|
trigger_label="${SPARKFLOW_TRIGGER_LABEL:-scheduled task}"
|
||||||
|
|
||||||
|
echo "[AUTO_TRIGGER] $(date -Iseconds) ${trigger_label} start"
|
||||||
|
|
||||||
|
cd "$app_root"
|
||||||
|
cd_rc=$?
|
||||||
|
if [ "$cd_rc" -ne 0 ]; then
|
||||||
|
echo "[AUTO_TRIGGER] $(date -Iseconds) ${trigger_label} failed to enter app directory rc=${cd_rc}"
|
||||||
|
exit "$cd_rc"
|
||||||
|
fi
|
||||||
|
|
||||||
|
python main.py --doTask
|
||||||
|
task_rc=$?
|
||||||
|
echo "[AUTO_TRIGGER] $(date -Iseconds) ${trigger_label} exit rc=${task_rc}"
|
||||||
|
exit "$task_rc"
|
||||||
@@ -33,6 +33,18 @@ class DeploymentContractTests(unittest.TestCase):
|
|||||||
self.assertTrue((SOURCE_ROOT / "config.example.json").is_file())
|
self.assertTrue((SOURCE_ROOT / "config.example.json").is_file())
|
||||||
self.assertIn("config.json", (SOURCE_ROOT / ".gitignore").read_text(encoding="utf-8"))
|
self.assertIn("config.json", (SOURCE_ROOT / ".gitignore").read_text(encoding="utf-8"))
|
||||||
|
|
||||||
|
def test_scheduler_uses_local_task_runner_and_migrates_legacy_commands(self):
|
||||||
|
compose = (REPO_ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||||
|
runner_path = SOURCE_ROOT / "scripts" / "run_scheduled_task.sh"
|
||||||
|
runner = runner_path.read_text(encoding="utf-8")
|
||||||
|
cron_runner = (SOURCE_ROOT / "scripts" / "cron_runner.py").read_text(encoding="utf-8")
|
||||||
|
scheduler = compose.split(" scheduler:", 1)[1].split("\n task:", 1)[0]
|
||||||
|
|
||||||
|
self.assertTrue(runner_path.is_file())
|
||||||
|
self.assertIn("python main.py --doTask", runner)
|
||||||
|
self.assertIn("migrate_legacy_crontab", cron_runner)
|
||||||
|
self.assertNotIn("/var/run/docker.sock:/var/run/docker.sock", scheduler)
|
||||||
|
|
||||||
def test_compose_runtime_mounts_follow_least_privilege(self):
|
def test_compose_runtime_mounts_follow_least_privilege(self):
|
||||||
text = (REPO_ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
text = (REPO_ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||||
web = text.split(" web:", 1)[1].split("\n login-desktop:", 1)[0]
|
web = text.split(" web:", 1)[1].split("\n login-desktop:", 1)[0]
|
||||||
@@ -41,7 +53,7 @@ class DeploymentContractTests(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertIn("/var/run/docker.sock:/var/run/docker.sock", web)
|
self.assertIn("/var/run/docker.sock:/var/run/docker.sock", web)
|
||||||
self.assertIn(".:/opt/douyin-sparkflow", web)
|
self.assertIn(".:/opt/douyin-sparkflow", web)
|
||||||
self.assertIn("/var/run/docker.sock:/var/run/docker.sock", scheduler)
|
self.assertNotIn("/var/run/docker.sock:/var/run/docker.sock", scheduler)
|
||||||
self.assertNotIn(".:/opt/douyin-sparkflow", scheduler)
|
self.assertNotIn(".:/opt/douyin-sparkflow", scheduler)
|
||||||
self.assertNotIn("/var/run/docker.sock:/var/run/docker.sock", task)
|
self.assertNotIn("/var/run/docker.sock:/var/run/docker.sock", task)
|
||||||
self.assertNotIn(".:/opt/douyin-sparkflow", task)
|
self.assertNotIn(".:/opt/douyin-sparkflow", task)
|
||||||
@@ -67,6 +79,23 @@ class DeploymentContractTests(unittest.TestCase):
|
|||||||
self.assertEqual(len(lines), 1)
|
self.assertEqual(len(lines), 1)
|
||||||
self.assertTrue(lines[0].startswith("*/20 "))
|
self.assertTrue(lines[0].startswith("*/20 "))
|
||||||
|
|
||||||
|
def test_legacy_cron_command_migrates_without_touching_schedule(self):
|
||||||
|
from scripts.cron_runner import migrate_legacy_line
|
||||||
|
|
||||||
|
legacy = (
|
||||||
|
"20 18 * * * /bin/bash -lc 'docker ps --format \"{{.Names}}\" | "
|
||||||
|
"grep douyin-web; docker exec douyin-web sh -lc \"cd /app && "
|
||||||
|
"env SPARKFLOW_MANUAL_RUN=1 SPARKFLOW_MANUAL_UNSENT_ONLY=1 "
|
||||||
|
"python main.py --doTask\"' >> /var/log/douyin-sparkflow.log 2>&1"
|
||||||
|
)
|
||||||
|
migrated = migrate_legacy_line(legacy)
|
||||||
|
|
||||||
|
self.assertTrue(migrated.startswith("20 18 * * * "))
|
||||||
|
self.assertIn("run_scheduled_task.sh", migrated)
|
||||||
|
self.assertIn("SPARKFLOW_MANUAL_UNSENT_ONLY=1", migrated)
|
||||||
|
self.assertNotIn("docker ps", migrated)
|
||||||
|
self.assertNotIn("docker exec", migrated)
|
||||||
|
|
||||||
def test_build_proxy_does_not_leak_into_runtime_and_runtime_proxy_is_explicit(self):
|
def test_build_proxy_does_not_leak_into_runtime_and_runtime_proxy_is_explicit(self):
|
||||||
dockerfile = (SOURCE_ROOT / "Dockerfile.server").read_text(encoding="utf-8")
|
dockerfile = (SOURCE_ROOT / "Dockerfile.server").read_text(encoding="utf-8")
|
||||||
compose = (REPO_ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
compose = (REPO_ROOT / "docker-compose.yml").read_text(encoding="utf-8")
|
||||||
@@ -101,6 +130,9 @@ class DeploymentContractTests(unittest.TestCase):
|
|||||||
windows = (REPO_ROOT / "deploy" / "install-local.ps1").read_text(encoding="utf-8")
|
windows = (REPO_ROOT / "deploy" / "install-local.ps1").read_text(encoding="utf-8")
|
||||||
self.assertIn("runtime_config_backup", server)
|
self.assertIn("runtime_config_backup", server)
|
||||||
self.assertIn("Restored runtime config.json", server)
|
self.assertIn("Restored runtime config.json", server)
|
||||||
|
self.assertIn("remove_legacy_host_cron", server)
|
||||||
|
self.assertIn("host-crontab-", server)
|
||||||
|
self.assertIn("docker ps --format", server)
|
||||||
self.assertNotIn("bash ./refresh_proxy.sh", windows)
|
self.assertNotIn("bash ./refresh_proxy.sh", windows)
|
||||||
self.assertIn("Initialize-ProxyConfig", windows)
|
self.assertIn("Initialize-ProxyConfig", windows)
|
||||||
|
|
||||||
@@ -124,15 +156,15 @@ class DeploymentContractTests(unittest.TestCase):
|
|||||||
env_example = (REPO_ROOT / ".env.example").read_text(encoding="utf-8")
|
env_example = (REPO_ROOT / ".env.example").read_text(encoding="utf-8")
|
||||||
server = (SOURCE_ROOT / "login_desktop_server.py").read_text(encoding="utf-8")
|
server = (SOURCE_ROOT / "login_desktop_server.py").read_text(encoding="utf-8")
|
||||||
login_block = compose.split(" login-desktop:", 1)[1].split(" scheduler:", 1)[0]
|
login_block = compose.split(" login-desktop:", 1)[1].split(" scheduler:", 1)[0]
|
||||||
self.assertIn("LOGIN_DESKTOP_PROXY_MODE: ${LOGIN_DESKTOP_PROXY_MODE:-direct}", login_block)
|
self.assertIn("LOGIN_DESKTOP_PROXY_MODE: ${LOGIN_DESKTOP_PROXY_MODE:-auto}", login_block)
|
||||||
self.assertIn("LOGIN_DESKTOP_PROXY: ${LOGIN_DESKTOP_PROXY:-http://proxy:7890}", login_block)
|
self.assertIn("LOGIN_DESKTOP_PROXY: ${LOGIN_DESKTOP_PROXY:-http://proxy:7890}", login_block)
|
||||||
self.assertNotIn("HTTP_PROXY: http://proxy:7890", login_block)
|
self.assertNotIn("HTTP_PROXY: http://proxy:7890", login_block)
|
||||||
self.assertIn("LOGIN_DESKTOP_PROXY_MODE=direct", env_example)
|
self.assertIn("LOGIN_DESKTOP_PROXY_MODE=auto", env_example)
|
||||||
self.assertIn('candidates.append(("direct", None))', server)
|
self.assertIn('candidates.append(("direct", None))', server)
|
||||||
self.assertIn('candidates.append(("proxy", LOGIN_PROXY_SERVER))', server)
|
self.assertIn('candidates.append(("proxy", LOGIN_PROXY_SERVER))', server)
|
||||||
self.assertIn('"--no-proxy-server"', server)
|
self.assertIn('"--no-proxy-server"', server)
|
||||||
self.assertIn('"/preflight"', server)
|
self.assertIn('"/preflight"', server)
|
||||||
self.assertIn('LOGIN_DESKTOP_PROXY_MODE: ${LOGIN_DESKTOP_PROXY_MODE:-direct}', login_block)
|
self.assertIn('LOGIN_DESKTOP_PROXY_MODE: ${LOGIN_DESKTOP_PROXY_MODE:-auto}', login_block)
|
||||||
dashboard = (SOURCE_ROOT / "webui" / "templates" / "dashboard.html").read_text(encoding="utf-8")
|
dashboard = (SOURCE_ROOT / "webui" / "templates" / "dashboard.html").read_text(encoding="utf-8")
|
||||||
self.assertIn('name="douyin_network_mode"', dashboard)
|
self.assertIn('name="douyin_network_mode"', dashboard)
|
||||||
self.assertIn('name="douyin_proxy_url"', dashboard)
|
self.assertIn('name="douyin_proxy_url"', dashboard)
|
||||||
@@ -176,4 +208,4 @@ class DeploymentContractTests(unittest.TestCase):
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -305,7 +305,8 @@ class WebUiSafetyTests(unittest.TestCase):
|
|||||||
self.assertIn("*/20 10-17 * * *", text)
|
self.assertIn("*/20 10-17 * * *", text)
|
||||||
self.assertIn("0 18 * * *", text)
|
self.assertIn("0 18 * * *", text)
|
||||||
self.assertIn("20 18 * * *", text)
|
self.assertIn("20 18 * * *", text)
|
||||||
self.assertIn("docker exec", text)
|
self.assertIn("run_scheduled_task.sh", text)
|
||||||
|
self.assertNotIn("docker exec", text)
|
||||||
|
|
||||||
def test_overview_api_requires_authentication_and_disables_cache(self):
|
def test_overview_api_requires_authentication_and_disables_cache(self):
|
||||||
client = TestClient(app_module.app)
|
client = TestClient(app_module.app)
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ TASK_SCHEDULE_MARKERS = (
|
|||||||
"docker compose run --rm task",
|
"docker compose run --rm task",
|
||||||
"docker compose run --rm douyin",
|
"docker compose run --rm douyin",
|
||||||
"main.py --doTask",
|
"main.py --doTask",
|
||||||
|
"run_scheduled_task.sh",
|
||||||
)
|
)
|
||||||
HOST_CRONTAB_PATH = Path("/host-spool-cron/root")
|
HOST_CRONTAB_PATH = Path("/host-spool-cron/root")
|
||||||
WINDOWED_SCHEDULE_RE = re.compile(r"^(\d{2}):(\d{2})-(\d{2}):(\d{2})/(\d+)m$", re.IGNORECASE)
|
WINDOWED_SCHEDULE_RE = re.compile(r"^(\d{2}):(\d{2})-(\d{2}):(\d{2})/(\d+)m$", re.IGNORECASE)
|
||||||
@@ -199,20 +200,9 @@ def _compose_env_args(extra_env=None):
|
|||||||
|
|
||||||
def build_scheduled_task_command(extra_env=None, trigger_label="scheduled send"):
|
def build_scheduled_task_command(extra_env=None, trigger_label="scheduled send"):
|
||||||
if running_in_container():
|
if running_in_container():
|
||||||
task_command = _with_env_prefix("python main.py --doTask", extra_env)
|
task_env = dict(extra_env or {})
|
||||||
return (
|
task_env["SPARKFLOW_TRIGGER_LABEL"] = trigger_label
|
||||||
"/bin/bash -lc 'timestamp=$(date -Iseconds); "
|
return _with_env_prefix("bash /app/scripts/run_scheduled_task.sh", task_env)
|
||||||
f"echo \"[AUTO_TRIGGER] $timestamp {trigger_label} start\"; "
|
|
||||||
"container=$(docker ps --format \"{{.Names}}\" | "
|
|
||||||
"grep -E \"^(douyin-web-hostfix|douyin-web)$\" | head -n 1); "
|
|
||||||
"if [ -z \"$container\" ]; then "
|
|
||||||
"echo \"[AUTO_TRIGGER] $timestamp no matching container found\"; "
|
|
||||||
"exit 1; "
|
|
||||||
"fi; "
|
|
||||||
"echo \"[AUTO_TRIGGER] $timestamp container=$container\"; "
|
|
||||||
"docker exec \"$container\" sh -lc "
|
|
||||||
f"\"cd /app && {task_command}\"'"
|
|
||||||
)
|
|
||||||
if compose_file_path():
|
if compose_file_path():
|
||||||
compose_root_quoted = shlex.quote(str(compose_root()))
|
compose_root_quoted = shlex.quote(str(compose_root()))
|
||||||
compose_env_args = _compose_env_args(extra_env)
|
compose_env_args = _compose_env_args(extra_env)
|
||||||
|
|||||||
@@ -238,8 +238,8 @@ LOGIN_DESKTOP_BIND_ADDRESS=127.0.0.1
|
|||||||
LOGIN_DESKTOP_WEB_PORT=8788
|
LOGIN_DESKTOP_WEB_PORT=8788
|
||||||
LOGIN_DESKTOP_PUBLIC_URL=/login-desktop/proxy/vnc.html?autoconnect=1&resize=scale&view_only=0&path=login-desktop/proxy/websockify
|
LOGIN_DESKTOP_PUBLIC_URL=/login-desktop/proxy/vnc.html?autoconnect=1&resize=scale&view_only=0&path=login-desktop/proxy/websockify
|
||||||
|
|
||||||
# 登录浏览器默认直连;Mihomo 仅作为高级选项
|
# 登录浏览器默认先直连抖音,直连失败时再尝试 Mihomo
|
||||||
LOGIN_DESKTOP_PROXY_MODE=direct
|
LOGIN_DESKTOP_PROXY_MODE=auto
|
||||||
LOGIN_DESKTOP_PROXY=http://proxy:7890
|
LOGIN_DESKTOP_PROXY=http://proxy:7890
|
||||||
|
|
||||||
|
|
||||||
@@ -252,7 +252,7 @@ PROXY_SUB_URL=
|
|||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
默认抖音业务网络使用直连。登录、好友刷新和浏览器发送会显式禁用环境代理,避免未配置的 Mihomo 影响正常使用。高级用户可在 Web UI「系统设置」中选择 Mihomo 并填写代理地址;登录浏览器仍可通过 `LOGIN_DESKTOP_PROXY_MODE=proxy` 强制使用代理。
|
登录、好友刷新和发送任务默认都采用“直连优先,Mihomo 回退”的网络策略。`LOGIN_DESKTOP_PROXY_MODE=auto` 时,登录浏览器先直连 `creator.douyin.com`,直连预检失败后才使用 `LOGIN_DESKTOP_PROXY`。好友刷新和续火花任务也会在任务开始前选择可用出口;发送动作开始后不会因响应不明确而盲目切换代理重发。没有配置有效订阅时,Mihomo 仅提供 DIRECT-only 配置,回退不会凭空产生代理节点。
|
||||||
|
|
||||||
|
|
||||||
#### `config.example.json` 与 `config.json` - 应用配置
|
#### `config.example.json` 与 `config.json` - 应用配置
|
||||||
@@ -304,6 +304,12 @@ PROXY_SUB_URL=
|
|||||||
- **scheduler**: 发送窗口定时调度服务
|
- **scheduler**: 发送窗口定时调度服务
|
||||||
- **task**: 一次性发送任务服务
|
- **task**: 一次性发送任务服务
|
||||||
|
|
||||||
|
定时任务由 `scheduler` 容器直接运行 `/app/main.py --doTask`,不再通过
|
||||||
|
`docker ps` 和 `docker exec` 控制 `web` 容器,因此 scheduler 不需要挂载宿主机
|
||||||
|
Docker socket。旧部署中已经写入的共享定时文件,会在 scheduler 启动时自动迁移;
|
||||||
|
服务器更新脚本还会备份并清理宿主机 root crontab 中旧的 Docker 发送任务,避免两套
|
||||||
|
定时器同时运行。
|
||||||
|
|
||||||
#### 快速部署
|
#### 快速部署
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -394,6 +400,10 @@ mode: rule
|
|||||||
|
|
||||||
如果 `PROXY_SUB_URL` 不为空,`refresh_proxy.sh` 会下载订阅并更新本地配置;如果为空,则生成 DIRECT-only 配置。不要在 Git 中提交包含订阅 token 的 `proxy/config.yaml`。首次部署不要跳过初始化步骤直接执行 `docker compose up -d`,否则 Docker 可能把缺失的配置文件创建成目录。
|
如果 `PROXY_SUB_URL` 不为空,`refresh_proxy.sh` 会下载订阅并更新本地配置;如果为空,则生成 DIRECT-only 配置。不要在 Git 中提交包含订阅 token 的 `proxy/config.yaml`。首次部署不要跳过初始化步骤直接执行 `docker compose up -d`,否则 Docker 可能把缺失的配置文件创建成目录。
|
||||||
|
|
||||||
|
如果宿主机 Docker Engine 较旧,Web 容器中的 Docker 运维功能可能需要显式指定
|
||||||
|
`DOCKER_API_VERSION`。默认留空即可;只有确认宿主机 API 版本后,才在 `.env` 中设置,
|
||||||
|
例如 `DOCKER_API_VERSION=1.43`。
|
||||||
|
|
||||||
|
|
||||||
### 默认网络安全
|
### 默认网络安全
|
||||||
|
|
||||||
|
|||||||
@@ -130,6 +130,40 @@ read_env_value() {
|
|||||||
grep -E "^${key}=" "$file" | tail -n 1 | cut -d= -f2- || true
|
grep -E "^${key}=" "$file" | tail -n 1 | cut -d= -f2- || true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
remove_legacy_host_cron() {
|
||||||
|
local current_file filtered_file backup_dir backup_file
|
||||||
|
current_file="$(mktemp)"
|
||||||
|
filtered_file="$(mktemp)"
|
||||||
|
|
||||||
|
if ! run_root crontab -l > "$current_file" 2>/dev/null; then
|
||||||
|
rm -f "$current_file" "$filtered_file"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
awk '
|
||||||
|
function is_legacy_sparkflow_job(line) {
|
||||||
|
return line ~ /main\.py --doTask/ \
|
||||||
|
&& line ~ /docker ps --format/ \
|
||||||
|
&& line ~ /docker exec/ \
|
||||||
|
&& line ~ /douyin-web/
|
||||||
|
}
|
||||||
|
!is_legacy_sparkflow_job($0) { print }
|
||||||
|
' "$current_file" > "$filtered_file"
|
||||||
|
|
||||||
|
if cmp -s "$current_file" "$filtered_file"; then
|
||||||
|
rm -f "$current_file" "$filtered_file"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
backup_dir="$APP_ROOT/backups"
|
||||||
|
backup_file="$backup_dir/host-crontab-$(date +%Y%m%d-%H%M%S).bak"
|
||||||
|
run_root mkdir -p "$backup_dir"
|
||||||
|
run_root cp "$current_file" "$backup_file"
|
||||||
|
run_root crontab "$filtered_file"
|
||||||
|
rm -f "$current_file" "$filtered_file"
|
||||||
|
log "Removed legacy Docker-based SparkFlow host cron jobs; backup: $backup_file"
|
||||||
|
}
|
||||||
|
|
||||||
write_default_cron() {
|
write_default_cron() {
|
||||||
local cron_file="$APP_ROOT/state/cron/root"
|
local cron_file="$APP_ROOT/state/cron/root"
|
||||||
if [ -s "$cron_file" ]; then
|
if [ -s "$cron_file" ]; then
|
||||||
@@ -139,9 +173,9 @@ write_default_cron() {
|
|||||||
echo "DEFAULT_SCHEDULE=$DEFAULT_SCHEDULE will be saved to .env. The initial cron file uses the built-in 10:00-18:00/20m schedule; adjust it from the Web UI after first login." >&2
|
echo "DEFAULT_SCHEDULE=$DEFAULT_SCHEDULE will be saved to .env. The initial cron file uses the built-in 10:00-18:00/20m schedule; adjust it from the Web UI after first login." >&2
|
||||||
fi
|
fi
|
||||||
cat > /tmp/douyin-sparkflow-cron <<'CRON'
|
cat > /tmp/douyin-sparkflow-cron <<'CRON'
|
||||||
*/20 10-17 * * * cd /app && python main.py --doTask >> /app/logs/app.log 2>&1
|
*/20 10-17 * * * env SPARKFLOW_TRIGGER_LABEL='scheduled send' bash /app/scripts/run_scheduled_task.sh >> /var/log/douyin-sparkflow.log 2>&1
|
||||||
0 18 * * * cd /app && python main.py --doTask >> /app/logs/app.log 2>&1
|
0 18 * * * env SPARKFLOW_TRIGGER_LABEL='scheduled send' bash /app/scripts/run_scheduled_task.sh >> /var/log/douyin-sparkflow.log 2>&1
|
||||||
20 18 * * * cd /app && env SPARKFLOW_MANUAL_RUN=1 SPARKFLOW_MANUAL_UNSENT_ONLY=1 PYTHONUNBUFFERED=1 python main.py --doTask >> /app/logs/app.log 2>&1
|
20 18 * * * env SPARKFLOW_MANUAL_RUN=1 SPARKFLOW_MANUAL_UNSENT_ONLY=1 PYTHONUNBUFFERED=1 SPARKFLOW_TRIGGER_LABEL='unsent fallback' bash /app/scripts/run_scheduled_task.sh >> /var/log/douyin-sparkflow.log 2>&1
|
||||||
CRON
|
CRON
|
||||||
run_root cp /tmp/douyin-sparkflow-cron "$cron_file"
|
run_root cp /tmp/douyin-sparkflow-cron "$cron_file"
|
||||||
rm -f /tmp/douyin-sparkflow-cron
|
rm -f /tmp/douyin-sparkflow-cron
|
||||||
@@ -156,7 +190,7 @@ prepare_runtime_files() {
|
|||||||
set_env_value "$env_file" "APP_ROOT" "$APP_ROOT"
|
set_env_value "$env_file" "APP_ROOT" "$APP_ROOT"
|
||||||
set_env_value "$env_file" "DEFAULT_SCHEDULE" "$DEFAULT_SCHEDULE"
|
set_env_value "$env_file" "DEFAULT_SCHEDULE" "$DEFAULT_SCHEDULE"
|
||||||
|
|
||||||
for key in TZ WEB_BIND_ADDRESS WEB_PORT SPARKFLOW_SESSION_COOKIE_SECURE LOGIN_DESKTOP_BIND_ADDRESS LOGIN_DESKTOP_WEB_PORT LOGIN_DESKTOP_PUBLIC_URL PROXY_BIND_ADDRESS PROXY_HTTP_PORT PROXY_CONTROLLER_PORT PROXY_SUB_URL PROXY_USER_AGENT PLAYWRIGHT_BASE_IMAGE NODE_RUNTIME_IMAGE HTTP_PROXY_BUILD HTTPS_PROXY_BUILD ALL_PROXY_BUILD PIP_INDEX_URL PIP_TRUSTED_HOST; do
|
for key in TZ WEB_BIND_ADDRESS WEB_PORT SPARKFLOW_SESSION_COOKIE_SECURE DOCKER_API_VERSION LOGIN_DESKTOP_BIND_ADDRESS LOGIN_DESKTOP_WEB_PORT LOGIN_DESKTOP_PUBLIC_URL PROXY_BIND_ADDRESS PROXY_HTTP_PORT PROXY_CONTROLLER_PORT PROXY_SUB_URL PROXY_USER_AGENT PLAYWRIGHT_BASE_IMAGE NODE_RUNTIME_IMAGE HTTP_PROXY_BUILD HTTPS_PROXY_BUILD ALL_PROXY_BUILD PIP_INDEX_URL PIP_TRUSTED_HOST; do
|
||||||
if [ -n "${!key:-}" ]; then
|
if [ -n "${!key:-}" ]; then
|
||||||
set_env_value "$env_file" "$key" "${!key}"
|
set_env_value "$env_file" "$key" "${!key}"
|
||||||
fi
|
fi
|
||||||
@@ -245,6 +279,7 @@ main() {
|
|||||||
ensure_docker
|
ensure_docker
|
||||||
prepare_repo
|
prepare_repo
|
||||||
prepare_runtime_files
|
prepare_runtime_files
|
||||||
|
remove_legacy_host_cron
|
||||||
compose_up
|
compose_up
|
||||||
print_summary
|
print_summary
|
||||||
}
|
}
|
||||||
|
|||||||
+6
-3
@@ -22,6 +22,9 @@ services:
|
|||||||
args:
|
args:
|
||||||
PLAYWRIGHT_BASE_IMAGE: ${PLAYWRIGHT_BASE_IMAGE:-swr.cn-north-4.myhuaweicloud.com/ddn-k8s/mcr.microsoft.com/playwright/python:v1.56.0-jammy}
|
PLAYWRIGHT_BASE_IMAGE: ${PLAYWRIGHT_BASE_IMAGE:-swr.cn-north-4.myhuaweicloud.com/ddn-k8s/mcr.microsoft.com/playwright/python:v1.56.0-jammy}
|
||||||
NODE_RUNTIME_IMAGE: ${NODE_RUNTIME_IMAGE:-node:22-bookworm-slim}
|
NODE_RUNTIME_IMAGE: ${NODE_RUNTIME_IMAGE:-node:22-bookworm-slim}
|
||||||
|
HTTP_PROXY: ${HTTP_PROXY_BUILD:-}
|
||||||
|
HTTPS_PROXY: ${HTTPS_PROXY_BUILD:-}
|
||||||
|
ALL_PROXY: ${ALL_PROXY_BUILD:-}
|
||||||
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.tuna.tsinghua.edu.cn/simple}
|
PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.tuna.tsinghua.edu.cn/simple}
|
||||||
PIP_TRUSTED_HOST: ${PIP_TRUSTED_HOST:-pypi.tuna.tsinghua.edu.cn}
|
PIP_TRUSTED_HOST: ${PIP_TRUSTED_HOST:-pypi.tuna.tsinghua.edu.cn}
|
||||||
image: douyin-sparkflow:local
|
image: douyin-sparkflow:local
|
||||||
@@ -38,6 +41,7 @@ services:
|
|||||||
SPARKFLOW_LOGIN_DESKTOP_NOVNC_URL: http://login-desktop:6080
|
SPARKFLOW_LOGIN_DESKTOP_NOVNC_URL: http://login-desktop:6080
|
||||||
SPARKFLOW_LOGIN_DESKTOP_NOVNC_WS_URL: ws://login-desktop:6080/websockify
|
SPARKFLOW_LOGIN_DESKTOP_NOVNC_WS_URL: ws://login-desktop:6080/websockify
|
||||||
SPARKFLOW_SESSION_COOKIE_SECURE: ${SPARKFLOW_SESSION_COOKIE_SECURE:-0}
|
SPARKFLOW_SESSION_COOKIE_SECURE: ${SPARKFLOW_SESSION_COOKIE_SECURE:-0}
|
||||||
|
DOCKER_API_VERSION: ${DOCKER_API_VERSION:-}
|
||||||
ports:
|
ports:
|
||||||
- "${WEB_BIND_ADDRESS:-0.0.0.0}:${WEB_PORT:-8787}:8787"
|
- "${WEB_BIND_ADDRESS:-0.0.0.0}:${WEB_PORT:-8787}:8787"
|
||||||
command: python main.py --web --host 0.0.0.0 --port 8787
|
command: python main.py --web --host 0.0.0.0 --port 8787
|
||||||
@@ -64,7 +68,7 @@ services:
|
|||||||
LOGIN_DESKTOP_STOP_AFTER_EXPORT_SECONDS: ${LOGIN_DESKTOP_STOP_AFTER_EXPORT_SECONDS:-60}
|
LOGIN_DESKTOP_STOP_AFTER_EXPORT_SECONDS: ${LOGIN_DESKTOP_STOP_AFTER_EXPORT_SECONDS:-60}
|
||||||
LOGIN_DESKTOP_STATUS_CACHE_SECONDS: ${LOGIN_DESKTOP_STATUS_CACHE_SECONDS:-15}
|
LOGIN_DESKTOP_STATUS_CACHE_SECONDS: ${LOGIN_DESKTOP_STATUS_CACHE_SECONDS:-15}
|
||||||
|
|
||||||
LOGIN_DESKTOP_PROXY_MODE: ${LOGIN_DESKTOP_PROXY_MODE:-direct}
|
LOGIN_DESKTOP_PROXY_MODE: ${LOGIN_DESKTOP_PROXY_MODE:-auto}
|
||||||
LOGIN_DESKTOP_PROXY: ${LOGIN_DESKTOP_PROXY:-http://proxy:7890}
|
LOGIN_DESKTOP_PROXY: ${LOGIN_DESKTOP_PROXY:-http://proxy:7890}
|
||||||
LOGIN_DESKTOP_PREFLIGHT_TIMEOUT_SECONDS: ${LOGIN_DESKTOP_PREFLIGHT_TIMEOUT_SECONDS:-15}
|
LOGIN_DESKTOP_PREFLIGHT_TIMEOUT_SECONDS: ${LOGIN_DESKTOP_PREFLIGHT_TIMEOUT_SECONDS:-15}
|
||||||
LOGIN_DESKTOP_NETWORK_CACHE_SECONDS: ${LOGIN_DESKTOP_NETWORK_CACHE_SECONDS:-30}
|
LOGIN_DESKTOP_NETWORK_CACHE_SECONDS: ${LOGIN_DESKTOP_NETWORK_CACHE_SECONDS:-30}
|
||||||
@@ -85,7 +89,7 @@ services:
|
|||||||
container_name: douyin-scheduler
|
container_name: douyin-scheduler
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
depends_on:
|
depends_on:
|
||||||
- web
|
- proxy
|
||||||
environment:
|
environment:
|
||||||
TZ: ${TZ:-Asia/Shanghai}
|
TZ: ${TZ:-Asia/Shanghai}
|
||||||
PYTHONUNBUFFERED: "1"
|
PYTHONUNBUFFERED: "1"
|
||||||
@@ -95,7 +99,6 @@ services:
|
|||||||
- ./DouYinSparkFlow/logs:/app/logs
|
- ./DouYinSparkFlow/logs:/app/logs
|
||||||
- ./state/cron:/host-spool-cron
|
- ./state/cron:/host-spool-cron
|
||||||
- ./state/browser-profiles:/opt/douyin-sparkflow/state/browser-profiles
|
- ./state/browser-profiles:/opt/douyin-sparkflow/state/browser-profiles
|
||||||
- /var/run/docker.sock:/var/run/docker.sock
|
|
||||||
|
|
||||||
task:
|
task:
|
||||||
image: douyin-sparkflow:local
|
image: douyin-sparkflow:local
|
||||||
|
|||||||
@@ -101,4 +101,9 @@ docker compose exec login-desktop curl -fsS http://127.0.0.1:18090/preflight
|
|||||||
docker compose logs -f scheduler
|
docker compose logs -f scheduler
|
||||||
```
|
```
|
||||||
|
|
||||||
|
正常情况下可以看到 `scheduled send start` 和 `scheduled send exit rc=0`。
|
||||||
|
如果仍看到 `docker ps` 或 `docker exec`,说明定时文件还是旧格式;重启 scheduler
|
||||||
|
后会自动迁移共享定时文件。使用服务器更新脚本时,还会清理宿主机 root crontab 中
|
||||||
|
旧的 Docker 发送任务,防止同一时间运行两遍。
|
||||||
|
|
||||||
如果只是少量目标失败,先使用 **补发未成功目标**,不要直接补发全部对象。
|
如果只是少量目标失败,先使用 **补发未成功目标**,不要直接补发全部对象。
|
||||||
|
|||||||
Reference in New Issue
Block a user