From 6eb132eafb778c5720eeedd856769c5f9a9f1f84 Mon Sep 17 00:00:00 2001 From: Rixuan Shao <2023311022@bipt.edu.cn> Date: Tue, 25 Aug 2026 18:42:43 +0800 Subject: [PATCH] feat: add direct-first Mihomo fallback --- .env.example | 5 + DouYinSparkFlow/core/browser.py | 86 ++++- DouYinSparkFlow/core/friends.py | 78 +++- DouYinSparkFlow/core/tasks.py | 31 +- DouYinSparkFlow/login_desktop_server.py | 364 ++++++++++++++++-- .../tests/test_network_fallback.py | 32 ++ README.md | 30 +- docker-compose.yml | 43 +-- 8 files changed, 548 insertions(+), 121 deletions(-) create mode 100644 DouYinSparkFlow/tests/test_network_fallback.py diff --git a/.env.example b/.env.example index a86b621..5ad6d11 100644 --- a/.env.example +++ b/.env.example @@ -15,6 +15,11 @@ LOGIN_DESKTOP_PIDS_LIMIT=256 LOGIN_DESKTOP_IDLE_TIMEOUT_SECONDS=1800 LOGIN_DESKTOP_STOP_AFTER_EXPORT_SECONDS=60 LOGIN_DESKTOP_STATUS_CACHE_SECONDS=15 +# Login browser uses direct access first; Mihomo is a fallback when direct access fails. +LOGIN_DESKTOP_PROXY_MODE=auto +LOGIN_DESKTOP_PROXY=http://proxy:7890 +LOGIN_DESKTOP_PREFLIGHT_TIMEOUT_SECONDS=15 +LOGIN_DESKTOP_NETWORK_CACHE_SECONDS=30 PROXY_BIND_ADDRESS=127.0.0.1 PROXY_HTTP_PORT=7890 PROXY_CONTROLLER_PORT=9090 diff --git a/DouYinSparkFlow/core/browser.py b/DouYinSparkFlow/core/browser.py index f501ccc..dfd8949 100644 --- a/DouYinSparkFlow/core/browser.py +++ b/DouYinSparkFlow/core/browser.py @@ -8,7 +8,7 @@ from pathlib import Path from playwright.async_api import async_playwright from rich.console import Console -from utils.config import DEBUG, Environment, get_environment +from utils.config import DEBUG, Environment, get_app_settings, get_environment console = Console() @@ -52,6 +52,75 @@ def _browser_args(): ] +def _douyin_network_mode(): + settings = get_app_settings(force_reload=True) + return str( + os.getenv("SPARKFLOW_DOUYIN_NETWORK_MODE") + or settings.get("douyin_network_mode", "direct") + ).strip().lower() + + +def douyin_network_modes(): + # Direct is the default; Mihomo is the fallback unless explicitly selected. + mode = _douyin_network_mode() + if mode == "mihomo": + return ("mihomo",) + return ("direct", "mihomo") + + +def _douyin_browser_proxy(network_mode=None): + # Return an explicit proxy URL for Douyin traffic, or None for direct. + settings = get_app_settings(force_reload=True) + mode = str(network_mode or _douyin_network_mode()).strip().lower() + if mode != "mihomo": + return None + return str( + os.getenv("SPARKFLOW_DOUYIN_PROXY_URL") + or settings.get("douyin_proxy_url", "http://proxy:7890") + ).strip() or None + + +def _browser_launch_options(GUI=False, network_mode=None): + args = _browser_args() + proxy = _douyin_browser_proxy(network_mode=network_mode) + if proxy: + return { + "headless": _headless_for(GUI), + "args": args, + "proxy": {"server": proxy}, + } + args.append("--no-proxy-server") + return { + "headless": _headless_for(GUI), + "args": args, + } + + +async def select_douyin_network_mode(target_url): + # Select the first route that can load the target before a task starts. + failures = [] + for network_mode in douyin_network_modes(): + playwright = browser = page = None + try: + playwright, browser = await get_browser(network_mode=network_mode) + page = await browser.new_page() + response = await page.goto(target_url, wait_until="commit", timeout=30000) + status = response.status if response is not None else None + if status is not None and status < 500: + return network_mode + failures.append(f"{network_mode}: HTTP {status}") + except Exception as exc: + failures.append(f"{network_mode}: {exc}") + finally: + if page: + await page.close() + if browser: + await browser.close() + if playwright: + await playwright.stop() + raise RuntimeError(f"Douyin network preflight failed: {'; '.join(failures)}") + + def sanitize_profile_name(value): raw = str(value or "").strip() if not raw: @@ -78,15 +147,12 @@ async def install_browser(): console.print(f"[bold red]Browser install failed: {exc}[/bold red]") -async def get_browser(GUI=False): +async def get_browser(GUI=False, network_mode=None): configure_playwright_environment() try: playwright = await async_playwright().start() - browser = await playwright.chromium.launch( - headless=_headless_for(GUI), - args=_browser_args(), - ) + browser = await playwright.chromium.launch(**_browser_launch_options(GUI, network_mode=network_mode)) return playwright, browser except Exception as exc: if "Executable doesn't exist" in str(exc) and get_environment() != Environment.GITHUBACTION: @@ -97,7 +163,7 @@ async def get_browser(GUI=False): raise -async def get_persistent_browser_context(profile_name, GUI=False, root=None): +async def get_persistent_browser_context(profile_name, GUI=False, root=None, network_mode=None): configure_playwright_environment() profile_dir = browser_profile_root(root) / sanitize_profile_name(profile_name) @@ -105,11 +171,11 @@ async def get_persistent_browser_context(profile_name, GUI=False, root=None): try: playwright = await async_playwright().start() + launch_options = _browser_launch_options(GUI, network_mode=network_mode) + launch_options["viewport"] = {"width": 1600, "height": 1000} context = await playwright.chromium.launch_persistent_context( str(profile_dir), - headless=_headless_for(GUI), - viewport={"width": 1600, "height": 1000}, - args=_browser_args(), + **launch_options, ) return playwright, context, profile_dir except Exception as exc: diff --git a/DouYinSparkFlow/core/friends.py b/DouYinSparkFlow/core/friends.py index cc9165d..fb3a5e5 100644 --- a/DouYinSparkFlow/core/friends.py +++ b/DouYinSparkFlow/core/friends.py @@ -1,5 +1,10 @@ import asyncio -from core.browser import get_browser +import logging + +from core.browser import douyin_network_modes, get_browser + + +logger = logging.getLogger(__name__) CHAT_PAGE_URL = "https://creator.douyin.com/creator-micro/data/following/chat" @@ -141,7 +146,21 @@ async def _wait_for_first_friend_or_empty(page): return False +async def _wait_for_chat_or_login(page, timeout_seconds=30): + deadline = asyncio.get_running_loop().time() + timeout_seconds + while asyncio.get_running_loop().time() < deadline: + await _ensure_logged_in(page) + try: + if await page.locator("#sub-app").count() > 0: + return + except Exception: + pass + await asyncio.sleep(0.5) + raise RuntimeError("chat page did not load within timeout") + + async def collect_friend_names(page): + await _wait_for_chat_or_login(page) await _click_friends_tab(page) await asyncio.sleep(1) @@ -204,31 +223,19 @@ async def collect_friend_names(page): return found_names -async def fetch_account_friends(account): +async def _fetch_account_friends_once(account, network_mode): cookies = list(account.get("cookies") or []) - if not cookies: - raise RuntimeError("账号没有可用 cookies,请重新扫码登录") - playwright = browser = context = page = None try: - playwright, browser = await get_browser(GUI=False) + playwright, browser = await get_browser(GUI=False, network_mode=network_mode) context = await browser.new_context() context.set_default_navigation_timeout(120000) context.set_default_timeout(120000) page = await context.new_page() - - await page.goto("https://creator.douyin.com/", wait_until="domcontentloaded", timeout=60000) await context.add_cookies(cookies) - await page.goto(CHAT_PAGE_URL, wait_until="domcontentloaded", timeout=60000) - await asyncio.sleep(2) - - await _ensure_logged_in(page) - friends = await collect_friend_names(page) - return friends - except RuntimeError: - raise - except Exception as exc: - raise RuntimeError(f"刷新好友列表失败,请重试:{exc}") from exc + await page.goto(CHAT_PAGE_URL, wait_until="commit", timeout=30000) + await asyncio.sleep(1) + return await collect_friend_names(page) finally: if page: await page.close() @@ -238,3 +245,38 @@ async def fetch_account_friends(account): await browser.close() if playwright: await playwright.stop() + + +async def fetch_account_friends(account): + cookies = list(account.get("cookies") or []) + if not cookies: + raise RuntimeError("account has no cookies; scan login QR code first") + + modes = douyin_network_modes() + last_error = None + for index, network_mode in enumerate(modes): + try: + friends = await _fetch_account_friends_once(account, network_mode) + logger.info( + "Friend refresh route=%s count=%s attempt=%s/%s", + network_mode, + len(friends), + index + 1, + len(modes), + ) + if friends or index == len(modes) - 1: + return friends + logger.warning( + "Friend refresh route=%s returned zero friends; trying next route", + network_mode, + ) + except RuntimeError as exc: + text = str(exc).lower() + if any(marker in text for marker in ("login", "cookie", "scan", "登录", "扫码")): + raise + last_error = exc + logger.warning("Friend refresh route=%s failed; trying next route: %s", network_mode, exc) + except Exception as exc: + last_error = exc + logger.warning("Friend refresh route=%s failed; trying next route: %s", network_mode, exc) + raise RuntimeError(f"friend refresh failed after routes {modes}: {last_error}") diff --git a/DouYinSparkFlow/core/tasks.py b/DouYinSparkFlow/core/tasks.py index bf19271..f5b7e5e 100644 --- a/DouYinSparkFlow/core/tasks.py +++ b/DouYinSparkFlow/core/tasks.py @@ -12,7 +12,12 @@ from datetime import datetime, timedelta, timezone from pathlib import Path from zoneinfo import ZoneInfo -from core.browser import get_browser, get_persistent_browser_context, sanitize_profile_name +from core.browser import ( + get_browser, + get_persistent_browser_context, + sanitize_profile_name, + select_douyin_network_mode, +) from core.msg_builder import build_message, build_message_candidates from core.protocol_dispatch import run_protocol_tasks from core.send_state import parse_sent_at, target_is_strong_confirmed_today @@ -1625,6 +1630,13 @@ def _manual_run_unsent_only(): return _is_manual_run() and os.getenv("SPARKFLOW_MANUAL_UNSENT_ONLY") == "1" +def _requested_account_refs(): + raw = os.getenv("SPARKFLOW_ACCOUNT_REFS") + if raw is None: + return None + return {item.strip() for item in raw.split(",") if item.strip()} + + def _unsent_retry_max_attempts(): raw_value = str(os.getenv("SPARKFLOW_UNSENT_RETRY_MAX_ATTEMPTS") or "3").strip() try: @@ -2347,6 +2359,8 @@ async def run_browser_tasks(active_config, browser_user_data): send_strategy = _normalize_send_strategy(active_config) friend_scan_config = _normalize_friend_list_scan_config(active_config) profile_config = _normalize_persistent_profile_config(active_config) + network_mode = await select_douyin_network_mode(CREATOR_HOME_URL) + logger.info("Selected Douyin task network route=%s", network_mode) semaphore = asyncio.Semaphore(active_config["taskCount"] if active_config["multiTask"] else 1) tasks = [] @@ -2364,11 +2378,11 @@ async def run_browser_tasks(active_config, browser_user_data): user.get("username", "unknown"), len(user["targets"]), ) - tasks.append(do_user_task(None, user, semaphore, send_strategy, profile_config, friend_scan_config)) + tasks.append(do_user_task(None, user, semaphore, send_strategy, profile_config, friend_scan_config, network_mode)) await asyncio.gather(*tasks) return - playwright, browser = await get_browser() + playwright, browser = await get_browser(network_mode=network_mode) try: for user in browser_user_data: logger.info( @@ -2376,7 +2390,7 @@ async def run_browser_tasks(active_config, browser_user_data): user.get("username", "unknown"), len(user["targets"]), ) - tasks.append(do_user_task(browser, user, semaphore, send_strategy, profile_config, friend_scan_config)) + tasks.append(do_user_task(browser, user, semaphore, send_strategy, profile_config, friend_scan_config, network_mode)) await asyncio.gather(*tasks) finally: @@ -2384,7 +2398,7 @@ async def run_browser_tasks(active_config, browser_user_data): await browser.close() -async def do_user_task(browser, user, semaphore, send_strategy, profile_config, friend_scan_config): +async def do_user_task(browser, user, semaphore, send_strategy, profile_config, friend_scan_config, network_mode): async with semaphore: account_name = user.get("username", "unknown") account_lock_handle = None @@ -2409,6 +2423,7 @@ async def do_user_task(browser, user, semaphore, send_strategy, profile_config, profile_config, friend_scan_config, account_name, + network_mode, ), timeout=timeout_seconds, ) @@ -2430,7 +2445,7 @@ async def do_user_task(browser, user, semaphore, send_strategy, profile_config, _release_browser_account_lock(account_lock_handle, account_lock_path, account_name) -async def _do_user_task_locked(browser, user, send_strategy, profile_config, friend_scan_config, account_name): +async def _do_user_task_locked(browser, user, send_strategy, profile_config, friend_scan_config, account_name, network_mode): cookies = user["cookies"] targets = user["targets"] start_delay = _random_delay_seconds( @@ -2446,6 +2461,7 @@ async def _do_user_task_locked(browser, user, send_strategy, profile_config, fri owned_playwright, context, profile_dir = await get_persistent_browser_context( _account_profile_name(user), root=profile_config["root"], + network_mode=network_mode, ) logger.info("Opened persistent browser profile for %s at %s", account_name, profile_dir) if profile_config["syncStoredCookiesBeforeRun"]: @@ -2719,6 +2735,9 @@ async def _do_user_task_locked(browser, user, send_strategy, profile_config, fri async def runTasks(): active_config = get_config(force_reload=True) all_user_data = get_userData(force_reload=True) + requested_refs = _requested_account_refs() + if requested_refs is not None: + all_user_data = [user for user in all_user_data if user.get("account_ref") in requested_refs] active_user_data = [user for user in all_user_data if user.get("enabled", True)] disabled_user_data = [user for user in all_user_data if not user.get("enabled", True)] diff --git a/DouYinSparkFlow/login_desktop_server.py b/DouYinSparkFlow/login_desktop_server.py index 1100faf..9a0fbb0 100644 --- a/DouYinSparkFlow/login_desktop_server.py +++ b/DouYinSparkFlow/login_desktop_server.py @@ -2,8 +2,10 @@ import asyncio import os import shutil import time +import urllib.request from contextlib import asynccontextmanager from pathlib import Path +from urllib.parse import urlsplit from fastapi import FastAPI, HTTPException, Request, Response import uvicorn @@ -14,10 +16,30 @@ from core.login import collect_login_result REMOTE_LOGIN_URL = "https://creator.douyin.com/" WWW_SELF_URL = "https://www.douyin.com/user/self" -PROFILE_DIR = Path("/data/login-profile") +DEFAULT_PROFILE_DIR = ( + Path(__file__).resolve().parents[1] / "state" / "login-profile" + if os.name == "nt" + else Path("/data/login-profile") +) +PROFILE_DIR = Path(os.getenv("LOGIN_PROFILE_DIR", str(DEFAULT_PROFILE_DIR))).expanduser() +LOGIN_DESKTOP_MODE = str( + os.getenv("LOGIN_DESKTOP_MODE", "native" if os.name == "nt" else "novnc") +).strip().lower() +if LOGIN_DESKTOP_MODE not in {"native", "novnc"}: + LOGIN_DESKTOP_MODE = "native" if os.name == "nt" else "novnc" IDLE_TIMEOUT_SECONDS = max(300, int(os.getenv("LOGIN_DESKTOP_IDLE_TIMEOUT_SECONDS", "1800"))) STOP_AFTER_EXPORT_SECONDS = max(0, int(os.getenv("LOGIN_DESKTOP_STOP_AFTER_EXPORT_SECONDS", "60"))) STATUS_CACHE_SECONDS = max(1, int(os.getenv("LOGIN_DESKTOP_STATUS_CACHE_SECONDS", "15"))) +LOGIN_NETWORK_MODE = str(os.getenv("LOGIN_DESKTOP_PROXY_MODE", "auto")).strip().lower() +if LOGIN_NETWORK_MODE not in {"auto", "direct", "proxy"}: + LOGIN_NETWORK_MODE = "auto" +LOGIN_PROXY_SERVER = str(os.getenv("LOGIN_DESKTOP_PROXY", "http://proxy:7890")).strip() +LOGIN_PREFLIGHT_TIMEOUT_SECONDS = max( + 3, int(os.getenv("LOGIN_DESKTOP_PREFLIGHT_TIMEOUT_SECONDS", "15")) +) +LOGIN_NETWORK_CACHE_SECONDS = max( + 0, int(os.getenv("LOGIN_DESKTOP_NETWORK_CACHE_SECONDS", "30")) +) GENERIC_WWW_NAMES = { "", "我的", @@ -41,9 +63,68 @@ GENERIC_WWW_NAMES = { } +class LoginNetworkError(RuntimeError): + """Raised when the login browser cannot reach Douyin.""" + + def __init__(self, message, *, checks=None): + super().__init__(message) + self.checks = checks or {} + + +def _safe_proxy_label(proxy_server): + if not proxy_server: + return "" + try: + parsed = urlsplit(proxy_server) + host = parsed.hostname or "" + port = f":{parsed.port}" if parsed.port else "" + return f"{host}{port}" if host else "configured proxy" + except ValueError: + return "configured proxy" + + +def _probe_login_target(proxy_server=None, timeout_seconds=15): + """Probe Douyin without inheriting the process proxy environment.""" + if proxy_server: + handlers = [ + urllib.request.ProxyHandler( + {"http": proxy_server, "https": proxy_server} + ) + ] + else: + handlers = [urllib.request.ProxyHandler({})] + opener = urllib.request.build_opener(*handlers) + request = urllib.request.Request( + REMOTE_LOGIN_URL, + headers={"User-Agent": "DouYinSparkFlow-login-preflight/1"}, + ) + started = time.monotonic() + try: + with opener.open(request, timeout=timeout_seconds) as response: + response.read(256) + status = int(getattr(response, "status", 200)) + if status >= 400: + raise RuntimeError(f"HTTP {status}") + return { + "ok": True, + "status": status, + "latency_ms": round((time.monotonic() - started) * 1000), + } + except Exception as exc: + error = str(exc) or exc.__class__.__name__ + if proxy_server: + error = error.replace(proxy_server, _safe_proxy_label(proxy_server)) + return { + "ok": False, + "error": error[:240], + "latency_ms": round((time.monotonic() - started) * 1000), + } + + class LoginDesktopManager: def __init__(self): self._lock = asyncio.Lock() + self._page_operation_lock = asyncio.Lock() self.playwright = None self.context = None self.page = None @@ -52,6 +133,83 @@ class LoginDesktopManager: self._status_checked_at = 0.0 self._idle_monitor_task = None self._scheduled_stop_task = None + self._network_route = None + self._network_checks = {} + self._network_checked_at = 0.0 + + def _network_payload(self): + route = dict(self._network_route or {}) + return { + "mode": LOGIN_NETWORK_MODE, + "selected": route.get("mode", ""), + "proxy": _safe_proxy_label(LOGIN_PROXY_SERVER) if route.get("mode") == "proxy" else "", + "checked_at": route.get("checked_at", ""), + "checks": dict(self._network_checks), + } + + def _invalidate_network_route(self): + self._network_route = None + self._network_checked_at = 0.0 + + async def _select_network_route(self, *, force=False): + now = time.monotonic() + if ( + not force + and self._network_route + and now - self._network_checked_at < LOGIN_NETWORK_CACHE_SECONDS + ): + return dict(self._network_route) + + checks = {} + candidates = [] + if LOGIN_NETWORK_MODE in {"auto", "direct"}: + candidates.append(("direct", None)) + if LOGIN_NETWORK_MODE in {"auto", "proxy"} and LOGIN_PROXY_SERVER: + candidates.append(("proxy", LOGIN_PROXY_SERVER)) + + for mode, proxy_server in candidates: + result = await asyncio.to_thread( + _probe_login_target, + proxy_server, + LOGIN_PREFLIGHT_TIMEOUT_SECONDS, + ) + checks[mode] = result + if result.get("ok"): + route = { + "mode": mode, + "proxy": _safe_proxy_label(proxy_server), + "checked_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + } + self._network_checks = checks + self._network_route = route + self._network_checked_at = now + return dict(route) + + self._network_checks = checks + self._network_route = None + self._network_checked_at = now + if LOGIN_NETWORK_MODE == "direct": + message = "无法直连抖音创作者中心,请检查服务器网络出口" + elif LOGIN_NETWORK_MODE == "proxy": + message = f"代理 {_safe_proxy_label(LOGIN_PROXY_SERVER)} 无法访问抖音创作者中心" + else: + message = "直连和代理都无法访问抖音创作者中心" + raise LoginNetworkError(message, checks=checks) + + async def network_preflight(self, *, force=True): + try: + route = await self._select_network_route(force=force) + return { + "ok": True, + "route": route, + "network": self._network_payload(), + } + except LoginNetworkError as exc: + return { + "ok": False, + "error": str(exc), + "network": self._network_payload(), + } def mark_activity(self): self._last_activity = time.monotonic() @@ -125,25 +283,41 @@ class LoginDesktopManager: except Exception: pass self.playwright = None + route = await self._select_network_route() self.playwright = await async_playwright().start() + launch_args = [ + "--start-maximized", + "--window-position=0,0", + "--window-size=1600,1000", + "--disable-background-networking", + "--disable-sync", + "--disable-features=Translate,MediaRouter,OptimizationHints,AutofillServerCommunication", + ] + if os.name != "nt": + launch_args.extend( + [ + "--disable-dev-shm-usage", + "--no-sandbox", + "--disable-gpu", + "--disable-gpu-compositing", + "--disable-software-rasterizer", + "--disable-accelerated-2d-canvas", + "--disable-accelerated-video-decode", + "--renderer-process-limit=2", + ] + ) + launch_options = { + "headless": False, + "viewport": {"width": 1600, "height": 1000}, + "args": launch_args, + } + if route["mode"] == "proxy": + launch_options["proxy"] = {"server": LOGIN_PROXY_SERVER} + else: + launch_args.append("--no-proxy-server") self.context = await self.playwright.chromium.launch_persistent_context( str(PROFILE_DIR), - headless=False, - viewport={"width": 1600, "height": 1000}, - args=[ - "--disable-dev-shm-usage", - "--no-sandbox", - "--start-maximized", - "--disable-gpu", - "--disable-gpu-compositing", - "--disable-software-rasterizer", - "--disable-accelerated-2d-canvas", - "--disable-accelerated-video-decode", - "--renderer-process-limit=2", - "--disable-background-networking", - "--disable-sync", - "--disable-features=Translate,MediaRouter,OptimizationHints,AutofillServerCommunication", - ], + **launch_options, ) self.page = self.context.pages[0] if self.context.pages else await self.context.new_page() @@ -170,13 +344,22 @@ class LoginDesktopManager: async def stop(self, clear_profile=False): async with self._lock: if self.page: - await self.page.close() + try: + await self.page.close() + except Exception: + pass self.page = None if self.context: - await self.context.close() + try: + await self.context.close() + except Exception: + pass self.context = None if self.playwright: - await self.playwright.stop() + try: + await self.playwright.stop() + except Exception: + pass self.playwright = None if clear_profile and PROFILE_DIR.exists(): shutil.rmtree(PROFILE_DIR, ignore_errors=True) @@ -192,6 +375,20 @@ class LoginDesktopManager: if not self.context or self._context_is_closed(): await self.start() + async def focus_browser(self): + self.mark_activity() + page = await self._get_active_page() + try: + await page.bring_to_front() + except Exception: + pass + return { + "ok": True, + "url": page.url, + "mode": LOGIN_DESKTOP_MODE, + "network": self._network_payload(), + } + async def status(self): now = time.monotonic() if self._status_cache is not None and now - self._status_checked_at < STATUS_CACHE_SECONDS: @@ -210,6 +407,7 @@ class LoginDesktopManager: "unique_id": "", "current_url": "", "profile_dir": str(PROFILE_DIR), + "network": self._network_payload(), } self._status_cache = payload self._status_checked_at = now @@ -235,6 +433,7 @@ class LoginDesktopManager: "unique_id": "", "current_url": "", "profile_dir": str(PROFILE_DIR), + "network": self._network_payload(), } self._status_cache = payload self._status_checked_at = now @@ -242,13 +441,14 @@ class LoginDesktopManager: if page: current_url = page.url - try: - result = await collect_login_result(page, self.context, timeout_ms=1000) - logged_in = True - username = result["username"] - unique_id = result["unique_id"] - except Exception: - pass + if not self._page_operation_lock.locked(): + try: + result = await collect_login_result(page, self.context, timeout_ms=1000) + logged_in = True + username = result["username"] + unique_id = result["unique_id"] + except Exception: + pass payload = { "running": True, @@ -257,36 +457,80 @@ class LoginDesktopManager: "unique_id": unique_id, "current_url": current_url, "profile_dir": str(PROFILE_DIR), + "network": self._network_payload(), } self._status_cache = payload self._status_checked_at = now return dict(payload) async def open_login(self): - await self.refresh_login_qr() + self.mark_activity() + try: + await asyncio.wait_for(self._page_operation_lock.acquire(), timeout=5) + except asyncio.TimeoutError as exc: + raise RuntimeError("login page is busy; retry shortly") from exc + try: + page = await self._get_active_page() + if page.url.startswith(REMOTE_LOGIN_URL): + return {"ok": True, "url": page.url, "network": self._network_payload()} + refresh_url = f"{REMOTE_LOGIN_URL}?qr_refresh={int(time.time() * 1000)}" + try: + await page.goto(refresh_url, wait_until="commit", timeout=30000) + except Exception: + await self.stop(clear_profile=False) + await self.start() + page = await self._get_active_page() + await page.goto(refresh_url, wait_until="commit", timeout=30000) + return {"ok": True, "url": page.url, "network": self._network_payload()} + finally: + self._page_operation_lock.release() async def refresh_login_qr(self): self.mark_activity() + try: + await asyncio.wait_for(self._page_operation_lock.acquire(), timeout=5) + except asyncio.TimeoutError as exc: + raise RuntimeError("login page is busy; retry shortly") from exc + try: + return await self._refresh_login_qr_locked() + finally: + self._page_operation_lock.release() + + async def _refresh_login_qr_locked(self): refresh_url = f"{REMOTE_LOGIN_URL}?qr_refresh={int(time.time() * 1000)}" try: page = await self._get_active_page() - await page.goto(refresh_url, wait_until="domcontentloaded", timeout=60000) + await page.goto(refresh_url, wait_until="commit", timeout=30000) except Exception: await self.reset() page = await self._get_active_page() - await page.goto(refresh_url, wait_until="domcontentloaded", timeout=60000) + await page.goto(refresh_url, wait_until="commit", timeout=30000) deadline = asyncio.get_running_loop().time() + 45 logged_in = False qr_ready = False while asyncio.get_running_loop().time() < deadline: - qr = page.locator('img[class*="qrcode"]').first - try: - if await qr.count() and await qr.is_visible(): - qr_ready = True + for selector in ( + 'img[class*="qrcode"]', + 'img[src^="data:image/png;base64"]', + ): + candidates = page.locator(selector) + for index in range(await candidates.count()): + qr = candidates.nth(index) + try: + if not await qr.is_visible(): + continue + box = await qr.bounding_box() + if not box or box["width"] < 120 or box["height"] < 120: + continue + ratio = box["width"] / max(1, box["height"]) + if 0.8 <= ratio <= 1.25: + qr_ready = True + break + except Exception: + pass + if qr_ready: break - except Exception: - pass if "/creator-micro/" in page.url: logged_in = True @@ -435,6 +679,14 @@ async def health(): return {"ok": True} +@app.get("/preflight") +async def preflight(): + result = await manager.network_preflight(force=True) + if not result.get("ok"): + raise HTTPException(status_code=502, detail=result) + return result + + @app.get("/status") async def status(): return await manager.status() @@ -442,8 +694,13 @@ async def status(): @app.post("/open-login") async def open_login(): - await manager.open_login() - return {"ok": True} + try: + return await manager.open_login() + except LoginNetworkError as exc: + raise HTTPException( + status_code=502, + detail={"code": "LOGIN_NETWORK_UNAVAILABLE", "message": str(exc), "checks": exc.checks}, + ) from exc @app.post("/reset") @@ -452,9 +709,26 @@ async def reset(): return {"ok": True} +@app.post("/close") +async def close(): + await manager.stop(clear_profile=True) + return {"ok": True} + + +@app.post("/focus") +async def focus(): + return await manager.focus_browser() + + @app.post("/refresh-qr") async def refresh_qr(): - return await manager.refresh_login_qr() + try: + return await manager.refresh_login_qr() + except LoginNetworkError as exc: + raise HTTPException( + status_code=502, + detail={"code": "LOGIN_NETWORK_UNAVAILABLE", "message": str(exc), "checks": exc.checks}, + ) from exc @app.post("/export") @@ -472,7 +746,15 @@ async def export(): @app.get("/qr") async def login_qr(): - page = await manager._get_active_page() + if manager._page_operation_lock.locked(): + raise HTTPException(status_code=503, detail="login page is busy; retry shortly") + try: + page = await manager._get_active_page() + except LoginNetworkError as exc: + raise HTTPException( + status_code=502, + detail={"code": "LOGIN_NETWORK_UNAVAILABLE", "message": str(exc), "checks": exc.checks}, + ) from exc expired = await page.locator('[class*="qrcode_expired"]').count() if expired and await page.locator('[class*="qrcode_expired"]').first.is_visible(): raise HTTPException(status_code=409, detail="login QR code has expired") @@ -499,7 +781,7 @@ async def login_qr(): ) except Exception: continue - raise HTTPException(status_code=404, detail="login QR code is not ready") + raise HTTPException(status_code=202, detail="login QR code is still starting", headers={"Retry-After": "2"}) @app.get("/debug/screenshot") diff --git a/DouYinSparkFlow/tests/test_network_fallback.py b/DouYinSparkFlow/tests/test_network_fallback.py new file mode 100644 index 0000000..331b78d --- /dev/null +++ b/DouYinSparkFlow/tests/test_network_fallback.py @@ -0,0 +1,32 @@ +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +SOURCE_ROOT = REPO_ROOT / "DouYinSparkFlow" + +class NetworkFallbackContractTests(unittest.TestCase): + def test_browser_exposes_direct_first_routes_and_preflight(self): + browser = (SOURCE_ROOT / "core" / "browser.py").read_text(encoding="utf-8") + self.assertIn("def douyin_network_modes", browser) + self.assertIn("return (\"direct\", \"mihomo\")", browser) + self.assertIn("async def select_douyin_network_mode", browser) + self.assertIn("get_browser(network_mode=network_mode)", browser) + + def test_friend_refresh_can_try_the_next_route(self): + friends = (SOURCE_ROOT / "core" / "friends.py").read_text(encoding="utf-8") + self.assertIn("for index, network_mode in enumerate(modes)", friends) + self.assertIn("returned zero friends; trying next route", friends) + self.assertIn("get_browser(GUI=False, network_mode=network_mode)", friends) + + def test_tasks_select_route_before_browser_creation(self): + tasks = (SOURCE_ROOT / "core" / "tasks.py").read_text(encoding="utf-8") + self.assertIn("select_douyin_network_mode(CREATOR_HOME_URL)", tasks) + self.assertIn("get_browser(network_mode=network_mode)", tasks) + self.assertIn("network_mode=network_mode", tasks) + + def test_login_defaults_to_auto_in_compose(self): + compose = (REPO_ROOT / "docker-compose.yml").read_text(encoding="utf-8") + self.assertIn("LOGIN_DESKTOP_PROXY_MODE: ${LOGIN_DESKTOP_PROXY_MODE:-auto}", compose) + +if __name__ == "__main__": + unittest.main() diff --git a/README.md b/README.md index eca1061..f1b54ec 100644 --- a/README.md +++ b/README.md @@ -85,12 +85,17 @@ git clone https://github.com/halfwaystudent/douyin-sparkflow.git cd douyin-sparkflow -# 2. 配置环境变量 +# 2. 创建本地环境变量 cp .env.example .env nano .env # 根据需要修改配置 +# 可选:在本地 .env 中填写 PROXY_SUB_URL,不要提交真实订阅地址 -# 3. 启动服务 -docker compose up -d +# 3. 初始化运行时文件并启动服务 +# 会创建 proxy/config.yaml;没有订阅时使用 DIRECT-only 配置 +bash ./deploy/install-local.sh + +# Windows PowerShell 使用: +# powershell -ExecutionPolicy Bypass -File .\deploy\install-local.ps1 # 4. 访问 Web 界面 # 浏览器打开 http://localhost:8787 @@ -176,7 +181,8 @@ douyin-sparkflow/ │ └── login_desktop_server.py # 登录桌面服务 ├── .github/workflows/ # GitHub Actions 定时任务 ├── proxy/ # 代理配置 -│ └── config.yaml # Mihomo 代理配置 +│ ├── config.example.yaml # Git 跟踪的安全模板 +│ └── config.yaml # 本地生成,Git 忽略 ├── docker-compose.yml # 容器编排配置 ├── .env.example # 环境变量模板 ├── refresh_proxy.sh # 代理刷新脚本 @@ -222,14 +228,20 @@ WEB_PORT=8787 LOGIN_DESKTOP_BIND_ADDRESS=127.0.0.1 LOGIN_DESKTOP_WEB_PORT=8788 LOGIN_DESKTOP_PUBLIC_URL=/login-desktop/proxy/vnc.html?autoconnect=1&resize=scale&view_only=0&path=login-desktop/proxy/websockify +# 登录浏览器默认先直连抖音,直连失败时再尝试 Mihomo +LOGIN_DESKTOP_PROXY_MODE=auto +LOGIN_DESKTOP_PROXY=http://proxy:7890 # Mihomo 代理和控制端口默认仅绑定本机 PROXY_BIND_ADDRESS=127.0.0.1 PROXY_HTTP_PORT=7890 PROXY_CONTROLLER_PORT=9090 +# 可选:Mihomo/Clash 订阅地址。通常包含敏感 token,只写入本地 .env。 PROXY_SUB_URL= ``` +登录、好友刷新和发送任务默认都采用“直连优先,Mihomo 回退”的网络策略。`LOGIN_DESKTOP_PROXY_MODE=auto` 时,登录浏览器先直连 `creator.douyin.com`,直连预检失败后才使用 `LOGIN_DESKTOP_PROXY`。好友刷新和续火花任务也会在任务开始前选择可用出口;发送动作开始后不会因响应不明确而盲目切换代理重发。没有配置有效订阅时,Mihomo 仅提供 DIRECT-only 配置,回退不会凭空产生代理节点。 + #### `config.example.json` 与 `config.json` - 应用配置 仓库跟踪 `DouYinSparkFlow/config.example.json`;首次运行会生成被 Git 忽略的 `DouYinSparkFlow/config.json`。常用配置示例: @@ -285,8 +297,8 @@ PROXY_SUB_URL= # 1. 准备环境变量 cp .env.example .env -# 2. 启动所有服务 -docker compose up -d +# 2. 初始化 proxy/config.yaml 并启动所有服务 +bash ./deploy/install-local.sh # 3. 查看日志 docker compose logs -f @@ -358,15 +370,17 @@ server { ### 代理配置 -项目支持通过代理访问抖音服务,配置文件位于 `proxy/config.yaml`: +项目支持通过代理访问抖音服务。仓库提供 `proxy/config.example.yaml` 作为安全模板,部署脚本会在启动前生成本地的 `proxy/config.yaml`: ```yaml mixed-port: 7890 allow-lan: true mode: rule -# ... 更多配置见配置文件 +# ... 更多配置见 proxy/config.example.yaml ``` +如果 `PROXY_SUB_URL` 不为空,`refresh_proxy.sh` 会下载订阅并更新本地配置;如果为空,则生成 DIRECT-only 配置。不要在 Git 中提交包含订阅 token 的 `proxy/config.yaml`。首次部署不要跳过初始化步骤直接执行 `docker compose up -d`,否则 Docker 可能把缺失的配置文件创建成目录。 + ### 默认网络安全 diff --git a/docker-compose.yml b/docker-compose.yml index 1042e81..a9ad446 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,12 +22,6 @@ services: args: PLAYWRIGHT_BASE_IMAGE: ${PLAYWRIGHT_BASE_IMAGE:-swr.cn-north-4.myhuaweicloud.com/ddn-k8s/mcr.microsoft.com/playwright/python:v1.56.0-jammy} NODE_RUNTIME_IMAGE: ${NODE_RUNTIME_IMAGE:-node:22-bookworm-slim} - HTTP_PROXY: ${HTTP_PROXY_BUILD:-} - HTTPS_PROXY: ${HTTPS_PROXY_BUILD:-} - ALL_PROXY: ${ALL_PROXY_BUILD:-} - http_proxy: ${HTTP_PROXY_BUILD:-} - https_proxy: ${HTTPS_PROXY_BUILD:-} - all_proxy: ${ALL_PROXY_BUILD:-} PIP_INDEX_URL: ${PIP_INDEX_URL:-https://pypi.tuna.tsinghua.edu.cn/simple} PIP_TRUSTED_HOST: ${PIP_TRUSTED_HOST:-pypi.tuna.tsinghua.edu.cn} image: douyin-sparkflow:local @@ -38,14 +32,6 @@ services: - login-desktop environment: TZ: ${TZ:-Asia/Shanghai} - HTTP_PROXY: http://proxy:7890 - HTTPS_PROXY: http://proxy:7890 - ALL_PROXY: socks5://proxy:7890 - http_proxy: http://proxy:7890 - https_proxy: http://proxy:7890 - all_proxy: socks5://proxy:7890 - NO_PROXY: localhost,127.0.0.1,login-desktop,douyin.com,amemv.com,snssdk.com,bytedance.com,pstatp.com,volccdn.com,bytescm.com,byted.net,douyinstatic.com,bytecdn.cn,byteimg.com,bytegoofy.com,toutiaostatic.com - no_proxy: localhost,127.0.0.1,login-desktop,douyin.com,amemv.com,snssdk.com,bytedance.com,pstatp.com,volccdn.com,bytescm.com,byted.net,douyinstatic.com,bytecdn.cn,byteimg.com,bytegoofy.com,toutiaostatic.com SPARKFLOW_LOGIN_DESKTOP_API_URL: http://login-desktop:18090 LOGIN_DESKTOP_PUBLIC_PORT: ${LOGIN_DESKTOP_WEB_PORT:-8788} SPARKFLOW_LOGIN_DESKTOP_PUBLIC_URL: ${LOGIN_DESKTOP_PUBLIC_URL:-/login-desktop/proxy/vnc.html?autoconnect=1&resize=scale&view_only=0&path=login-desktop/proxy/websockify} @@ -66,6 +52,7 @@ services: image: douyin-sparkflow:local container_name: login-desktop restart: unless-stopped + init: true depends_on: - proxy environment: @@ -76,14 +63,10 @@ services: LOGIN_DESKTOP_IDLE_TIMEOUT_SECONDS: ${LOGIN_DESKTOP_IDLE_TIMEOUT_SECONDS:-1800} LOGIN_DESKTOP_STOP_AFTER_EXPORT_SECONDS: ${LOGIN_DESKTOP_STOP_AFTER_EXPORT_SECONDS:-60} LOGIN_DESKTOP_STATUS_CACHE_SECONDS: ${LOGIN_DESKTOP_STATUS_CACHE_SECONDS:-15} - HTTP_PROXY: http://proxy:7890 - HTTPS_PROXY: http://proxy:7890 - ALL_PROXY: socks5://proxy:7890 - http_proxy: http://proxy:7890 - https_proxy: http://proxy:7890 - all_proxy: socks5://proxy:7890 - NO_PROXY: localhost,127.0.0.1,login-desktop,douyin.com,amemv.com,snssdk.com,bytedance.com,pstatp.com,volccdn.com,bytescm.com,byted.net,douyinstatic.com,bytecdn.cn,byteimg.com,bytegoofy.com,toutiaostatic.com - no_proxy: localhost,127.0.0.1,login-desktop,douyin.com,amemv.com,snssdk.com,bytedance.com,pstatp.com,volccdn.com,bytescm.com,byted.net,douyinstatic.com,bytecdn.cn,byteimg.com,bytegoofy.com,toutiaostatic.com + LOGIN_DESKTOP_PROXY_MODE: ${LOGIN_DESKTOP_PROXY_MODE:-auto} + LOGIN_DESKTOP_PROXY: ${LOGIN_DESKTOP_PROXY:-http://proxy:7890} + LOGIN_DESKTOP_PREFLIGHT_TIMEOUT_SECONDS: ${LOGIN_DESKTOP_PREFLIGHT_TIMEOUT_SECONDS:-15} + LOGIN_DESKTOP_NETWORK_CACHE_SECONDS: ${LOGIN_DESKTOP_NETWORK_CACHE_SECONDS:-30} ports: - "${LOGIN_DESKTOP_BIND_ADDRESS:-127.0.0.1}:${LOGIN_DESKTOP_WEB_PORT:-8788}:6080" command: bash /app/scripts/start_login_desktop.sh @@ -104,14 +87,6 @@ services: environment: TZ: ${TZ:-Asia/Shanghai} PYTHONUNBUFFERED: "1" - HTTP_PROXY: http://proxy:7890 - HTTPS_PROXY: http://proxy:7890 - ALL_PROXY: socks5://proxy:7890 - http_proxy: http://proxy:7890 - https_proxy: http://proxy:7890 - all_proxy: socks5://proxy:7890 - NO_PROXY: localhost,127.0.0.1,douyin.com,amemv.com,snssdk.com,bytedance.com,pstatp.com,volccdn.com,bytescm.com,byted.net,douyinstatic.com,bytecdn.cn,byteimg.com,bytegoofy.com,toutiaostatic.com - no_proxy: localhost,127.0.0.1,douyin.com,amemv.com,snssdk.com,bytedance.com,pstatp.com,volccdn.com,bytescm.com,byted.net,douyinstatic.com,bytecdn.cn,byteimg.com,bytegoofy.com,toutiaostatic.com command: python /app/scripts/cron_runner.py /host-spool-cron/root volumes: - ./DouYinSparkFlow:/app @@ -128,14 +103,6 @@ services: environment: TZ: ${TZ:-Asia/Shanghai} PYTHONUNBUFFERED: "1" - HTTP_PROXY: http://proxy:7890 - HTTPS_PROXY: http://proxy:7890 - ALL_PROXY: socks5://proxy:7890 - http_proxy: http://proxy:7890 - https_proxy: http://proxy:7890 - all_proxy: socks5://proxy:7890 - NO_PROXY: localhost,127.0.0.1,douyin.com,amemv.com,snssdk.com,bytedance.com,pstatp.com,volccdn.com,bytescm.com,byted.net,douyinstatic.com,bytecdn.cn,byteimg.com,bytegoofy.com,toutiaostatic.com - no_proxy: localhost,127.0.0.1,douyin.com,amemv.com,snssdk.com,bytedance.com,pstatp.com,volccdn.com,bytescm.com,byted.net,douyinstatic.com,bytecdn.cn,byteimg.com,bytegoofy.com,toutiaostatic.com command: python main.py --doTask volumes: - ./DouYinSparkFlow:/app