Sync deployed SparkFlow reliability updates

DouYinSparkFlow/core/browser.py

@@ -1,4 +1,5 @@
 import os
+import re
 import subprocess
 import sys
 import traceback
@@ -12,6 +13,7 @@ from utils.config import DEBUG, Environment, get_environment
 
 console = Console()
 PLAYWRIGHT_BROWSERS_PATH = "../chrome"
+DEFAULT_PROFILE_ROOT = "/opt/douyin-sparkflow/state/browser-profiles"
 
 
 def _local_browser_bundle_path():
@@ -32,6 +34,38 @@ def configure_playwright_environment():
         os.environ["PLAYWRIGHT_BROWSERS_PATH"] = str(bundle_path.resolve())
 
 
+def _headless_for(GUI=False):
+    headless = not GUI
+    if get_environment() == Environment.LOCAL and DEBUG:
+        headless = False
+    return headless
+
+
+def _browser_args():
+    return [
+        "--disable-dev-shm-usage",
+        "--no-sandbox",
+    ]
+
+
+def sanitize_profile_name(value):
+    raw = str(value or "").strip()
+    if not raw:
+        raw = "unknown"
+    safe = re.sub(r"[^0-9A-Za-z._-]+", "_", raw)
+    safe = safe.strip("._-") or "unknown"
+    return safe[:80]
+
+
+def browser_profile_root(root=None):
+    configured = (
+        root
+        or os.getenv("SPARKFLOW_BROWSER_PROFILE_ROOT")
+        or DEFAULT_PROFILE_ROOT
+    )
+    return Path(configured)
+
+
 async def install_browser():
     try:
         subprocess.run([sys.executable, "-m", "playwright", "install", "chromium"], check=True)
@@ -43,15 +77,11 @@ async def install_browser():
 async def get_browser(GUI=False):
     configure_playwright_environment()
 
-    headless = not GUI
-    if get_environment() == Environment.LOCAL and DEBUG:
-        headless = False
-
     try:
         playwright = await async_playwright().start()
         browser = await playwright.chromium.launch(
-            headless=headless,
-            args=["--disable-dev-shm-usage"],
+            headless=_headless_for(GUI),
+            args=_browser_args(),
         )
         return playwright, browser
     except Exception as exc:
@@ -61,3 +91,27 @@ async def get_browser(GUI=False):
             sys.exit(1)
         traceback.print_exc()
         raise
+
+
+async def get_persistent_browser_context(profile_name, GUI=False, root=None):
+    configure_playwright_environment()
+
+    profile_dir = browser_profile_root(root) / sanitize_profile_name(profile_name)
+    profile_dir.mkdir(parents=True, exist_ok=True)
+
+    try:
+        playwright = await async_playwright().start()
+        context = await playwright.chromium.launch_persistent_context(
+            str(profile_dir),
+            headless=_headless_for(GUI),
+            viewport={"width": 1600, "height": 1000},
+            args=_browser_args(),
+        )
+        return playwright, context, profile_dir
+    except Exception as exc:
+        if "Executable doesn't exist" in str(exc) and get_environment() != Environment.GITHUBACTION:
+            console.print("[bold red]Playwright browser is missing.[/bold red]")
+            await install_browser()
+            sys.exit(1)
+        traceback.print_exc()
+        raise

DouYinSparkFlow/core/friends.py

@@ -1,27 +1,25 @@
 import asyncio
 from pathlib import Path
-import time
 
 from core.browser import get_browser
 
 
 CHAT_PAGE_URL = "https://creator.douyin.com/creator-micro/data/following/chat"
-FRIENDS_TAB_SELECTORS = (
-    'xpath=//*[@id="sub-app"]/div/div/div[1]/div[2]',
-    'xpath=//*[@id="sub-app"]//*[self::div or self::span or self::button][contains(normalize-space(.), "朋友私信") and string-length(normalize-space(.)) <= 20]',
-    'xpath=//*[@id="sub-app"]//*[self::div or self::span or self::button][normalize-space()="朋友"]',
+FRIENDS_TAB_SELECTOR = 'xpath=//*[@id="sub-app"]/div/div/div[1]/div[2]'
+TARGET_SELECTOR = (
+    'xpath=//*[@id="sub-app"]/div/div[1]/div[2]/div[2]'
+    '//div[contains(@class, "semi-list-item-body semi-list-item-body-flex-start")]'
 )
-FRIEND_NAME_SELECTOR = 'xpath=//*[@id="sub-app"]//span[contains(@class, "item-header-name-")]'
-SCROLLABLE_FRIENDS_SELECTORS = (
-    'xpath=//*[@id="sub-app"]/div/div[1]/div[2]/div[2]/div/div/div[3]/div/div/div/ul/div',
-    'xpath=//*[@id="sub-app"]//ul/div',
-    'xpath=//*[@id="sub-app"]//ul',
+SCROLLABLE_FRIENDS_SELECTOR = (
+    'xpath=//*[@id="sub-app"]/div/div[1]/div[2]/div[2]/div/div/div[3]/div/div/div/ul/div'
 )
 NO_MORE_SELECTOR = 'xpath=//div[contains(@class, "no-more-tip-ftdJnu")]'
 LOADING_SELECTOR = 'xpath=//div[contains(@class, "semi-spin")]'
+FIRST_FRIEND_SELECTOR = (
+    'xpath=//*[@id="sub-app"]/div/div/div[2]/div[2]/div/div/div[1]/div/div/div/ul/div/div/div[1]/li/div'
+)
+FRIEND_NAME_SELECTOR = """xpath=.//span[contains(@class, "item-header-name-")]"""
 LOGIN_MASK_SELECTORS = [".login-mask", ".login-guide-container", ".login-img-code-wrapper"]
-EMPTY_LIST_KEYWORDS = ("暂无", "没有", "空空", "还没有")
-LOGIN_KEYWORDS = ("扫码登录", "登录抖音", "请登录", "登录已过期", "重新登录")
 
 
 def update_collection_progress(new_names_count, no_more_visible, scroll_moved, idle_rounds, stuck_rounds, idle_limit=5, stuck_limit=2):
@@ -32,10 +30,6 @@ def update_collection_progress(new_names_count, no_more_visible, scroll_moved, i
 
 
 async def _ensure_logged_in(page):
-    current_url = page.url or ""
-    if "login" in current_url or "passport" in current_url:
-        raise RuntimeError("账号登录已失效，请重新扫码登录")
-
     for selector in LOGIN_MASK_SELECTORS:
         try:
             locator = page.locator(selector).first
@@ -47,115 +41,12 @@ async def _ensure_logged_in(page):
             continue
 
 
-async def _body_text(page, limit=600):
-    try:
-        text = await page.locator("body").inner_text(timeout=2000)
-    except Exception:
-        return ""
-    return " ".join(text.split())[:limit]
-
-
-async def _page_diagnosis(page):
-    await _ensure_logged_in(page)
-    body_text = await _body_text(page)
-    if any(keyword in body_text for keyword in LOGIN_KEYWORDS):
-        raise RuntimeError("账号登录已失效或页面要求重新登录，请重新扫码登录")
-    if any(keyword in body_text for keyword in EMPTY_LIST_KEYWORDS):
-        return "页面提示当前没有可读取的朋友私信好友"
-    return f"未等到好友列表。当前URL={page.url}，页面提示={body_text or '空'}"
-
-
-async def _open_friends_tab(page):
-    if await page.locator(FRIEND_NAME_SELECTOR).count() > 0:
-        return
-
-    last_error = None
-    for selector in FRIENDS_TAB_SELECTORS:
-        locator = page.locator(selector).first
-        try:
-            await locator.wait_for(state="visible", timeout=10000)
-            await locator.click(timeout=5000)
-            await asyncio.sleep(1.5)
-            return
-        except Exception as exc:
-            last_error = exc
-
-    raise RuntimeError(f"未找到“朋友私信”入口，可能页面结构变化或账号未登录。最后错误：{last_error}")
-
-
-async def _wait_for_friend_name_or_empty(page, timeout_ms=45000):
-    deadline = time.monotonic() + timeout_ms / 1000
-    while time.monotonic() < deadline:
-        await _ensure_logged_in(page)
-        names = page.locator(FRIEND_NAME_SELECTOR)
-        if await names.count() > 0:
-            first = names.first
-            try:
-                if await first.is_visible():
-                    return True
-            except Exception:
-                return True
-
-        body_text = await _body_text(page, limit=300)
-        if any(keyword in body_text for keyword in EMPTY_LIST_KEYWORDS):
-            return False
-        if any(keyword in body_text for keyword in LOGIN_KEYWORDS):
-            raise RuntimeError("账号登录已失效或页面要求重新登录，请重新扫码登录")
-
-        loading = page.locator(LOADING_SELECTOR).first
-        if await loading.count() > 0 and await loading.is_visible():
-            await asyncio.sleep(1.5)
-        else:
-            await asyncio.sleep(1)
-
-    diagnosis = await _page_diagnosis(page)
-    raise RuntimeError(diagnosis)
-
-
-async def _collect_visible_friend_names(page):
-    names = []
-    for raw_name in await page.locator(FRIEND_NAME_SELECTOR).all_inner_texts():
-        name = raw_name.strip()
-        if name:
-            names.append(name)
-    return names
-
-
-async def _find_scrollable_friends_element(page):
-    for selector in SCROLLABLE_FRIENDS_SELECTORS:
-        try:
-            handle = await page.locator(selector).first.element_handle(timeout=2000)
-            if handle:
-                return handle
-        except Exception:
-            continue
-
-    try:
-        handle = await page.evaluate_handle(
-            """() => {
-                const firstName = document.querySelector('#sub-app span[class*="item-header-name-"]');
-                let node = firstName;
-                while (node && node !== document.body) {
-                    const style = window.getComputedStyle(node);
-                    const overflow = `${style.overflow} ${style.overflowY}`;
-                    if (node.scrollHeight > node.clientHeight + 20 && /(auto|scroll|overlay)/.test(overflow)) {
-                        return node;
-                    }
-                    node = node.parentElement;
-                }
-                return document.scrollingElement || document.documentElement;
-            }"""
-        )
-        return handle.as_element()
-    except Exception:
-        return None
-
-
 async def collect_friend_names(page):
-    await _open_friends_tab(page)
-    has_friends = await _wait_for_friend_name_or_empty(page)
-    if not has_friends:
-        return []
+    await page.wait_for_selector(FRIENDS_TAB_SELECTOR, timeout=30000)
+    await page.locator(FRIENDS_TAB_SELECTOR).click()
+
+    await page.wait_for_selector(FIRST_FRIEND_SELECTOR, timeout=30000)
+    await page.locator(FIRST_FRIEND_SELECTOR).click()
     await asyncio.sleep(2)
 
     found_names = []
@@ -164,8 +55,13 @@ async def collect_friend_names(page):
     stuck_rounds = 0
 
     while True:
+        target_elements = await page.locator(TARGET_SELECTOR).all()
         new_names_count = 0
-        for name in await _collect_visible_friend_names(page):
+        for element in target_elements:
+            try:
+                name = (await element.locator(FRIEND_NAME_SELECTOR).inner_text()).strip()
+            except Exception:
+                continue
             if not name or name in seen_names:
                 continue
             seen_names.add(name)
@@ -180,12 +76,11 @@ async def collect_friend_names(page):
         if await loading.count() > 0 and await loading.is_visible():
             await asyncio.sleep(1.5)
 
-        scrollable_element = await _find_scrollable_friends_element(page)
+        scrollable_element = await page.locator(SCROLLABLE_FRIENDS_SELECTOR).element_handle()
         if not scrollable_element:
             if found_names:
                 return found_names
-            diagnosis = await _page_diagnosis(page)
-            raise RuntimeError(f"未找到好友列表滚动容器；{diagnosis}")
+            raise RuntimeError("未找到好友列表滚动容器")
 
         before_top = await page.evaluate("(element) => element.scrollTop", scrollable_element)
         await page.evaluate("(element) => element.scrollTop += 800", scrollable_element)