From 89bf89c02d533071f98fb7f9f6bb8eb42d42ef22 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Wed, 22 Apr 2026 16:22:10 +0800 Subject: [PATCH] feat: add clawhub skill registry source --- app/chain/skills.py | 12 +- app/core/config.py | 1 + app/helper/skill.py | 364 ++++++++++++++++++++++++++++++++++- tests/test_skills_command.py | 135 +++++++++++++ 4 files changed, 502 insertions(+), 10 deletions(-) diff --git a/app/chain/skills.py b/app/chain/skills.py index 503d7796..6ef2f18f 100644 --- a/app/chain/skills.py +++ b/app/chain/skills.py @@ -585,7 +585,7 @@ class SkillsChain(ChainBase): sources = self.skillhelper.get_market_sources() source_lines = [] for index, source in enumerate(sources, start=1): - source_lines.append(f"{index}. {source}") + source_lines.append(f"{index}. {self.skillhelper.describe_market_source(source)}") text_lines = [ f"已安装技能：{len(local_skills)}", @@ -708,6 +708,16 @@ class SkillsChain(ChainBase): self._truncate(skill.description), ] ) + if skill.source_type == "registry": + text_lines.append("社区源，安装前请自行甄别安全性") + + if any(skill.source_type == "registry" for skill in page_items): + text_lines.extend( + [ + "", + "提示：ClawHub 属于社区注册表，技能质量与安全性需要自行甄别。", + ] + ) text_lines.extend( [ diff --git a/app/core/config.py b/app/core/config.py index 8344cb0f..bb8f42e0 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -423,6 +423,7 @@ class ConfigModel(BaseModel): # ==================== 技能配置 ==================== # 技能市场仓库地址，多个地址使用,分隔 SKILL_MARKET: str = ( + "https://clawhub.ai," "https://github.com/openai/skills," "https://github.com/anthropics/skills," "https://github.com/vercel-labs/agent-skills" diff --git a/app/helper/skill.py b/app/helper/skill.py index 1cba16c2..04a10ec2 100644 --- a/app/helper/skill.py +++ b/app/helper/skill.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import Dict, List, Optional, Tuple -from urllib.parse import urlparse +from urllib.parse import urlencode, urlparse from app.agent.middleware.skills import _parse_skill_metadata from app.core.cache import cached, fresh @@ -19,6 +19,13 @@ from app.utils.url import UrlUtils _SOURCE_META_FILENAME = ".moviepilot-skill-source.json" _DEFAULT_BRANCHES = ("main", "master") _MARKET_CACHE_TTL = 60 * 30 +_CLAWHUB_HOSTS = {"clawhub.ai", "www.clawhub.ai"} +_OFFICIAL_SKILL_REPOS = { + "openai/skills", + "anthropics/skills", + "vercel-labs/agent-skills", + "NousResearch/hermes-agent", +} @dataclass @@ -33,6 +40,10 @@ class SkillInfo: repo_url: Optional[str] = None repo_name: Optional[str] = None skill_path: Optional[str] = None + registry_url: Optional[str] = None + registry_name: Optional[str] = None + registry_slug: Optional[str] = None + download_url: Optional[str] = None installed: bool = False removable: bool = False @@ -74,6 +85,41 @@ class SkillHelper(metaclass=WeakSingleton): skill_dir.mkdir(parents=True, exist_ok=True) return skill_dir + @staticmethod + def _build_repo_source_label(repo_name: Optional[str]) -> str: + """ + 根据仓库名称生成展示标签。 + """ + repo_name = (repo_name or "").strip() + if not repo_name: + return "仓库来源" + if repo_name in _OFFICIAL_SKILL_REPOS: + return f"官方仓库 · {repo_name}" + return f"仓库来源 · {repo_name}" + + @staticmethod + def _build_registry_source_label(registry_name: Optional[str]) -> str: + """ + 根据注册表名称生成展示标签。 + """ + registry_name = (registry_name or "").strip() + if not registry_name: + return "社区注册表" + return f"社区注册表 · {registry_name}" + + def describe_market_source(self, source: str) -> str: + """ + 将配置中的市场源地址转换为更适合用户阅读的描述。 + """ + registry = self._parse_market_registry(source) + if registry: + return self._build_registry_source_label(registry.get("registry_name")) + + repo = self._parse_market_repo(source) + if repo: + return self._build_repo_source_label(repo.get("repo_name")) + return source + @staticmethod def _normalize_repo_url(repo_url: str) -> Optional[str]: """ @@ -121,6 +167,32 @@ class SkillHelper(metaclass=WeakSingleton): "root_path": root_path.strip("/") or "skills", } + @staticmethod + def _parse_market_registry(source_url: str) -> Optional[dict]: + """ + 解析注册表市场地址，目前支持 ClawHub。 + """ + normalized = (source_url or "").strip() + if not normalized.startswith(("http://", "https://")): + return None + + parsed = urlparse(normalized) + if parsed.netloc.lower() not in _CLAWHUB_HOSTS: + return None + + base_url = f"{parsed.scheme}://{parsed.netloc}".rstrip("/") + path = parsed.path.rstrip("/") + api_base = ( + f"{base_url}{path}" + if path.endswith("/api/v1") + else f"{base_url}/api/v1" + ) + return { + "registry_url": base_url, + "registry_name": "ClawHub", + "api_base": api_base.rstrip("/"), + } + @staticmethod def _read_source_meta(skill_dir: Path) -> dict: """ @@ -181,8 +253,13 @@ class SkillHelper(metaclass=WeakSingleton): source_meta = self._read_source_meta(path) source_type = "bundled" if bundled else source_meta.get("source", "local") if source_type == "market": - repo_name = source_meta.get("repo_name") or source_meta.get("repo_url") - source_label = f"市场 · {repo_name}" if repo_name else "市场" + source_label = self._build_repo_source_label( + source_meta.get("repo_name") or source_meta.get("repo_url") + ) + elif source_type == "registry": + source_label = self._build_registry_source_label( + source_meta.get("registry_name") or source_meta.get("registry_url") + ) elif source_type == "bundled": source_label = "内置" else: @@ -200,6 +277,10 @@ class SkillHelper(metaclass=WeakSingleton): repo_url=source_meta.get("repo_url"), repo_name=source_meta.get("repo_name"), skill_path=source_meta.get("skill_path"), + registry_url=source_meta.get("registry_url"), + registry_name=source_meta.get("registry_name"), + registry_slug=source_meta.get("registry_slug"), + download_url=source_meta.get("download_url"), installed=True, removable=not bundled, ) @@ -218,7 +299,7 @@ class SkillHelper(metaclass=WeakSingleton): deduped: Dict[str, SkillInfo] = {} for source in self.get_market_sources(): with fresh(force): - market_skills = self._list_market_repo_skills(source) + market_skills = self._list_market_source_skills(source) for skill in market_skills: key = skill.name or skill.id if key in deduped: @@ -230,11 +311,21 @@ class SkillHelper(metaclass=WeakSingleton): deduped.values(), key=lambda item: ( item.installed, - (item.repo_name or "").lower(), + (item.repo_name or item.registry_name or "").lower(), item.name.lower(), ), ) + @cached(maxsize=24, ttl=_MARKET_CACHE_TTL, skip_empty=True) + def _list_market_source_skills(self, source: str) -> List[SkillInfo]: + """ + 根据市场源类型分发到仓库扫描或注册表读取。 + """ + registry = self._parse_market_registry(source) + if registry: + return self._list_market_registry_skills(registry) + return self._list_market_repo_skills(source) + @cached(maxsize=16, ttl=_MARKET_CACHE_TTL, skip_empty=True) def _list_market_repo_skills(self, repo_url: str) -> List[SkillInfo]: """ @@ -297,7 +388,9 @@ class SkillHelper(metaclass=WeakSingleton): version=metadata["version"], path=f"{repo['repo_url']}/tree/{repo['branch']}/{skill_dir}", source_type="market", - source_label=f"市场 · {repo['repo_name']}", + source_label=self._build_repo_source_label( + repo["repo_name"] + ), repo_url=repo["repo_url"], repo_name=repo["repo_name"], skill_path=skill_dir, @@ -310,13 +403,30 @@ class SkillHelper(metaclass=WeakSingleton): logger.error("解析技能市场压缩包失败：%s", e) return [] + def _list_market_registry_skills(self, registry: dict) -> List[SkillInfo]: + """ + 从注册表拉取技能列表，目前用于 ClawHub。 + """ + response = self._request_registry( + url=f"{registry['api_base']}/skills", + params={"limit": 200, "sort": "installsAllTime"}, + ) + if not response: + return [] + + payload = self._load_json_response(response) + items = self._extract_registry_items(payload) + results: List[SkillInfo] = [] + for item in items: + skill = self._build_registry_skill(item, registry) + if skill: + results.append(skill) + return results + def install_market_skill(self, skill: SkillInfo) -> Tuple[bool, str]: """ 将市场技能安装到用户技能目录，并记录来源元数据。 """ - if not skill.repo_url or not skill.skill_path: - return False, "技能来源信息不完整，无法安装" - target_root = self._ensure_user_skills_dir() target_dir = target_root / skill.id if target_dir.exists(): @@ -324,6 +434,12 @@ class SkillHelper(metaclass=WeakSingleton): if self._is_bundled_skill(skill.id): return False, f"技能 {skill.id} 是 MoviePilot 内置技能，不能覆盖安装" + if skill.registry_url: + return self._install_registry_skill(skill, target_dir) + + if not skill.repo_url or not skill.skill_path: + return False, "技能来源信息不完整，无法安装" + repo = self._parse_market_repo(skill.repo_url) if not repo: return False, "技能市场地址无效" @@ -382,6 +498,50 @@ class SkillHelper(metaclass=WeakSingleton): logger.error("安装市场技能失败：%s", e) return False, f"安装技能失败：{e}" + def _install_registry_skill( + self, skill: SkillInfo, target_dir: Path + ) -> Tuple[bool, str]: + """ + 从注册表下载并安装技能包。 + """ + if not skill.registry_url or not (skill.registry_slug or skill.id): + return False, "注册表技能来源信息不完整，无法安装" + + archive_bytes = self._download_registry_archive(skill) + if not archive_bytes: + return False, "下载注册表技能失败，请检查网络连接" + + try: + with zipfile.ZipFile(io.BytesIO(archive_bytes)) as zf: + if not zf.namelist(): + return False, "注册表技能压缩包为空" + + target_dir.mkdir(parents=True, exist_ok=False) + try: + wrote = self._extract_skill_archive(zf, target_dir) + if not wrote or not (target_dir / "SKILL.md").exists(): + shutil.rmtree(target_dir, ignore_errors=True) + return False, "注册表技能内容不完整，安装失败" + + self._write_source_meta( + target_dir, + { + "source": "registry", + "registry_url": skill.registry_url, + "registry_name": skill.registry_name, + "registry_slug": skill.registry_slug or skill.id, + "download_url": skill.download_url, + "installed_at": datetime.now(timezone.utc).isoformat(), + }, + ) + return True, f"技能 {skill.id} 已安装到 {target_dir}" + except Exception: + shutil.rmtree(target_dir, ignore_errors=True) + raise + except Exception as e: + logger.error("安装注册表技能失败：%s", e) + return False, f"安装技能失败：{e}" + def remove_local_skill(self, skill_id: str) -> Tuple[bool, str]: """ 删除一个本地技能目录，内置技能会被显式拦截。 @@ -404,6 +564,165 @@ class SkillHelper(metaclass=WeakSingleton): logger.error("删除技能失败：%s", e) return False, f"删除技能失败：{e}" + @staticmethod + def _load_json_response(response) -> dict: + """ + 读取 HTTP 响应中的 JSON 数据，异常时回退为空对象。 + """ + try: + payload = response.json() + return payload if isinstance(payload, dict) else {"items": payload} + except Exception as e: + logger.warning("解析技能市场 JSON 响应失败：%s", e) + return {} + + @staticmethod + def _extract_registry_items(payload: dict) -> List[dict]: + """ + 从不同响应结构中提取技能列表。 + """ + if isinstance(payload, list): + return [item for item in payload if isinstance(item, dict)] + + for key in ("items", "skills", "results"): + items = payload.get(key) + if isinstance(items, list): + return [item for item in items if isinstance(item, dict)] + + for key in ("data", "result"): + nested = payload.get(key) + if not isinstance(nested, dict): + continue + for list_key in ("items", "skills", "results"): + items = nested.get(list_key) + if isinstance(items, list): + return [item for item in items if isinstance(item, dict)] + return [] + + def _build_registry_skill( + self, item: dict, registry: dict + ) -> Optional[SkillInfo]: + """ + 将注册表返回的条目转换为统一的 SkillInfo。 + """ + slug = ( + item.get("slug") + or item.get("id") + or item.get("name") + or item.get("title") + ) + if not slug: + return None + + name = item.get("name") or item.get("title") or slug + description = ( + item.get("description") + or item.get("summary") + or item.get("excerpt") + or "" + ) + owner_handle = self._extract_registry_owner_handle(item) + page_path = f"/{owner_handle}/{slug}" if owner_handle else f"/skills/{slug}" + + return SkillInfo( + id=str(slug), + name=str(name), + description=str(description), + version=0, + path=f"{registry['registry_url']}{page_path}", + source_type="registry", + source_label=self._build_registry_source_label( + registry["registry_name"] + ), + registry_url=registry["registry_url"], + registry_name=registry["registry_name"], + registry_slug=str(slug), + download_url=item.get("downloadUrl") + or self._build_registry_download_url(registry["api_base"], str(slug)), + installed=False, + removable=False, + ) + + @staticmethod + def _extract_registry_owner_handle(item: dict) -> Optional[str]: + """ + 尽量从注册表条目中提取作者/拥有者 handle。 + """ + for key in ("ownerHandle", "authorHandle", "handle", "username"): + value = item.get(key) + if isinstance(value, str) and value.strip(): + return value.strip().lstrip("@") + + for key in ("owner", "author", "user", "publisher"): + nested = item.get(key) + if not isinstance(nested, dict): + continue + for nested_key in ("handle", "username", "login", "name"): + value = nested.get(nested_key) + if isinstance(value, str) and value.strip(): + return value.strip().lstrip("@") + return None + + @staticmethod + def _build_registry_download_url(api_base: str, slug: str) -> str: + """ + 根据官方文档约定构造注册表 ZIP 下载地址。 + """ + query = urlencode({"slug": slug}) + return f"{api_base.rstrip('/')}/download?{query}" + + def _download_registry_archive(self, skill: SkillInfo) -> Optional[bytes]: + """ + 下载注册表技能包 ZIP。 + """ + download_url = skill.download_url or self._build_registry_download_url( + f"{skill.registry_url.rstrip('/')}/api/v1", + skill.registry_slug or skill.id, + ) + response = self._request_registry(url=download_url) + if response is None or response.status_code != 200: + logger.warning("下载注册表技能失败：%s", download_url) + return None + return response.content + + @staticmethod + def _extract_skill_archive(zf: zipfile.ZipFile, target_dir: Path) -> bool: + """ + 从技能压缩包中提取单个技能目录。 + + 兼容 `package/`、`skill-name/` 或直接根目录三种常见打包形式。 + """ + names = zf.namelist() + skill_md_names = [ + name + for name in names + if name.endswith("SKILL.md") and "/.system/" not in f"/{name}/" + ] + if not skill_md_names: + return False + + skill_md_name = min(skill_md_names, key=lambda name: (name.count("/"), len(name))) + prefix = skill_md_name[: -len("SKILL.md")] + wrote = False + for archive_name in names: + if prefix and not archive_name.startswith(prefix): + continue + + rel_name = archive_name[len(prefix):] if prefix else archive_name + if not rel_name: + continue + + output_path = target_dir / rel_name + if archive_name.endswith("/"): + output_path.mkdir(parents=True, exist_ok=True) + continue + + output_path.parent.mkdir(parents=True, exist_ok=True) + with zf.open(archive_name, "r") as src, open(output_path, "wb") as dst: + shutil.copyfileobj(src, dst) + wrote = True + return wrote + def _download_repo_archive(self, repo: dict) -> Optional[bytes]: """ 下载市场仓库压缩包，并在缺省分支之间做回退尝试。 @@ -425,6 +744,33 @@ class SkillHelper(metaclass=WeakSingleton): logger.warning("下载技能市场仓库失败：%s", repo["repo_url"]) return None + @staticmethod + def _request_registry( + url: str, + params: Optional[dict] = None, + timeout: int = 30, + ): + """ + 请求注册表 API，兼容代理和直连场景。 + """ + strategies = [] + if settings.PROXY_HOST: + strategies.append(({"proxies": settings.PROXY, "timeout": timeout}, url)) + strategies.append(({"timeout": timeout}, url)) + + for kwargs, target_url in strategies: + try: + response = RequestUtils(**kwargs).get_res( + url=target_url, + params=params, + raise_exception=True, + ) + if response is not None and response.status_code == 200: + return response + except Exception as e: + logger.warning("请求注册表技能市场失败：%s - %s", target_url, e) + return None + @staticmethod def _request_github( url: str, diff --git a/tests/test_skills_command.py b/tests/test_skills_command.py index 212fec0d..234c934e 100644 --- a/tests/test_skills_command.py +++ b/tests/test_skills_command.py @@ -37,6 +37,16 @@ def _build_skill_zip(skill_dir: str, skill_name: str) -> bytes: return buf.getvalue() +class _FakeResponse: + def __init__(self, payload=None, content: bytes = b"", status_code: int = 200): + self._payload = payload + self.content = content + self.status_code = status_code + + def json(self): + return self._payload + + class TestSkillsCommand(unittest.TestCase): def tearDown(self): skills_interaction_manager.clear() @@ -148,6 +158,131 @@ class TestSkillsCommand(unittest.TestCase): self.assertFalse(removed) self.assertIn("内置技能", remove_message) + def test_skillhelper_lists_clawhub_registry_skills(self): + helper = SkillHelper() + response = _FakeResponse( + payload={ + "items": [ + { + "slug": "weather-forecast", + "name": "Weather Forecast", + "summary": "Forecast weather from ClawHub", + "owner": {"handle": "openclaw"}, + } + ] + } + ) + + with patch.object(helper, "_request_registry", return_value=response): + skills = helper._list_market_source_skills("https://clawhub.ai") + + self.assertEqual(len(skills), 1) + self.assertEqual(skills[0].id, "weather-forecast") + self.assertEqual(skills[0].source_type, "registry") + self.assertEqual(skills[0].registry_name, "ClawHub") + self.assertEqual(skills[0].source_label, "社区注册表 · ClawHub") + self.assertIn("/openclaw/weather-forecast", skills[0].path) + + def test_skillhelper_installs_registry_skill(self): + helper = SkillHelper() + skill = SkillInfo( + id="registry-demo", + name="Registry Demo", + description="registry demo", + source_type="registry", + source_label="注册表 · ClawHub", + registry_url="https://clawhub.ai", + registry_name="ClawHub", + registry_slug="registry-demo", + download_url="https://clawhub.ai/api/v1/download?slug=registry-demo", + ) + zip_bytes = _build_skill_zip("package", "registry-demo") + + with tempfile.TemporaryDirectory() as tempdir: + user_root = Path(tempdir) / "user-skills" + bundled_root = Path(tempdir) / "bundled-skills" + user_root.mkdir(parents=True, exist_ok=True) + bundled_root.mkdir(parents=True, exist_ok=True) + + with patch.object( + SkillHelper, "get_user_skills_dir", return_value=user_root + ), patch.object( + SkillHelper, "get_bundled_skills_dir", return_value=bundled_root + ), patch.object( + helper, "_request_registry", return_value=_FakeResponse(content=zip_bytes) + ): + success, message = helper.install_market_skill(skill) + self.assertTrue(success, message) + self.assertTrue((user_root / "registry-demo" / "SKILL.md").exists()) + self.assertTrue( + ( + user_root + / "registry-demo" + / ".moviepilot-skill-source.json" + ).exists() + ) + + local_skills = helper.list_local_skills() + self.assertEqual(len(local_skills), 1) + self.assertEqual(local_skills[0].source_type, "registry") + self.assertEqual(local_skills[0].registry_name, "ClawHub") + self.assertEqual(local_skills[0].source_label, "社区注册表 · ClawHub") + + def test_skills_chain_market_view_marks_clawhub_as_community_source(self): + chain = SkillsChain() + request = skills_interaction_manager.create_or_replace( + user_id="10001", + channel=MessageChannel.Telegram, + source="telegram-test", + username="tester", + ) + request.view = "market" + + with patch.object( + chain.skillhelper, + "list_market_skills", + return_value=[ + SkillInfo( + id="weather-forecast", + name="Weather Forecast", + description="Forecast weather from ClawHub", + source_type="registry", + source_label="社区注册表 · ClawHub", + registry_name="ClawHub", + registry_url="https://clawhub.ai", + registry_slug="weather-forecast", + ) + ], + ): + title, text, _buttons = chain._build_market_view(request=request) + + self.assertEqual(title, "技能市场") + self.assertIn("社区注册表 · ClawHub", text) + self.assertIn("社区源，安装前请自行甄别安全性", text) + self.assertIn("ClawHub 属于社区注册表", text) + + def test_skills_chain_root_view_uses_friendly_source_labels(self): + chain = SkillsChain() + request = skills_interaction_manager.create_or_replace( + user_id="10001", + channel=MessageChannel.Telegram, + source="telegram-test", + username="tester", + ) + + with patch.object(chain.skillhelper, "list_local_skills", return_value=[]), patch.object( + chain.skillhelper, "list_market_skills", return_value=[] + ), patch.object( + chain.skillhelper, + "get_market_sources", + return_value=["https://clawhub.ai", "https://github.com/openai/skills"], + ): + title, text, _buttons = chain._build_root_view(request=request) + + self.assertEqual(title, "技能管理") + self.assertIn("社区注册表 · ClawHub", text) + self.assertIn("官方仓库 · openai/skills", text) + def test_skills_chain_updates_buttons_via_edit_message(self): chain = SkillsChain() buttons = [[{"text": "安装 1", "callback_data": "skills:req:install:1"}]]