mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-09 01:16:50 +08:00
fix: 无损合并旧分类枚举并隔离迁移校验失败
This commit is contained in:
@@ -121,7 +121,7 @@ class LegacyClassificationMigrationResult:
|
|||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class _LegacyToken:
|
class _LegacyToken:
|
||||||
"""保留一个旧逗号项展开后的值集合及其排除语义。"""
|
"""保留旧字段展开后的同向值集合及其排除语义。"""
|
||||||
|
|
||||||
negative: bool
|
negative: bool
|
||||||
values: tuple[str, ...]
|
values: tuple[str, ...]
|
||||||
@@ -530,26 +530,20 @@ def _legacy_field_definition(
|
|||||||
|
|
||||||
|
|
||||||
def _parse_legacy_tokens(value: str) -> tuple[tuple[_LegacyToken, ...], bool]:
|
def _parse_legacy_tokens(value: str) -> tuple[tuple[_LegacyToken, ...], bool]:
|
||||||
"""逐项复现旧逗号、排除前缀和连字符范围展开算法。"""
|
"""沿用旧范围展开语义,并合并同向枚举,避免值数量膨胀为叶子数量。"""
|
||||||
raw_tokens = [item for item in value.split(",") if item]
|
raw_tokens = [item for item in value.split(",") if item]
|
||||||
parsed: list[_LegacyToken] = []
|
values_by_sign: dict[bool, list[str]] = {}
|
||||||
requires_exists = not raw_tokens
|
requires_exists = not raw_tokens
|
||||||
for raw_token in raw_tokens:
|
for raw_token in raw_tokens:
|
||||||
expanded = _expand_legacy_token(raw_token)
|
expanded = _expand_legacy_token(raw_token)
|
||||||
if not expanded:
|
if not expanded:
|
||||||
requires_exists = True
|
requires_exists = True
|
||||||
continue
|
continue
|
||||||
grouped: list[_LegacyToken] = []
|
|
||||||
for expanded_value in expanded:
|
for expanded_value in expanded:
|
||||||
negative = expanded_value.startswith("!")
|
negative = expanded_value.startswith("!")
|
||||||
plain_value = expanded_value[1:] if negative else expanded_value
|
plain_value = expanded_value[1:] if negative else expanded_value
|
||||||
if grouped and grouped[-1].negative == negative:
|
values_by_sign.setdefault(negative, []).append(plain_value)
|
||||||
previous = grouped[-1]
|
return tuple(_LegacyToken(negative, tuple(values)) for negative, values in values_by_sign.items()), requires_exists
|
||||||
grouped[-1] = _LegacyToken(negative, (*previous.values, plain_value))
|
|
||||||
else:
|
|
||||||
grouped.append(_LegacyToken(negative, (plain_value,)))
|
|
||||||
parsed.extend(grouped)
|
|
||||||
return tuple(parsed), requires_exists
|
|
||||||
|
|
||||||
|
|
||||||
def _expand_legacy_token(value: str) -> tuple[str, ...]:
|
def _expand_legacy_token(value: str) -> tuple[str, ...]:
|
||||||
|
|||||||
@@ -219,11 +219,11 @@ def _project_rule(
|
|||||||
continue
|
continue
|
||||||
if not values:
|
if not values:
|
||||||
continue
|
continue
|
||||||
token = _render_legacy_token(values)
|
|
||||||
rendered = f"!{token}" if negative else token
|
|
||||||
field_tokens = tokens_by_field.setdefault(field_name, [])
|
field_tokens = tokens_by_field.setdefault(field_name, [])
|
||||||
if rendered not in field_tokens:
|
for value in values:
|
||||||
field_tokens.append(rendered)
|
rendered = f"!{value}" if negative else value
|
||||||
|
if rendered not in field_tokens:
|
||||||
|
field_tokens.append(rendered)
|
||||||
return (
|
return (
|
||||||
{field_name: ",".join(tokens) for field_name, tokens in tokens_by_field.items()},
|
{field_name: ",".join(tokens) for field_name, tokens in tokens_by_field.items()},
|
||||||
diagnostics,
|
diagnostics,
|
||||||
@@ -300,17 +300,6 @@ def _original_alias_value(value: str, aliases: Mapping[str, str]) -> str:
|
|||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
def _render_legacy_token(values: Sequence[str]) -> str:
|
|
||||||
"""把一个迁移时保留边界的值集合恢复为单个旧逗号项。"""
|
|
||||||
if len(values) == 1:
|
|
||||||
return values[0]
|
|
||||||
if values and all(value.isdigit() for value in values):
|
|
||||||
numbers = [int(value) for value in values]
|
|
||||||
if numbers == list(range(numbers[0], numbers[-1] + 1)):
|
|
||||||
return f"{numbers[0]}-{numbers[-1]}"
|
|
||||||
return "-".join(values)
|
|
||||||
|
|
||||||
|
|
||||||
def _projection_warning(
|
def _projection_warning(
|
||||||
code: str,
|
code: str,
|
||||||
message: str,
|
message: str,
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from pydantic import ValidationError
|
|||||||
|
|
||||||
from app.application.classification.configuration import (
|
from app.application.classification.configuration import (
|
||||||
ClassificationPolicyConfigurationService,
|
ClassificationPolicyConfigurationService,
|
||||||
|
ClassificationPolicyValidationError,
|
||||||
)
|
)
|
||||||
from app.application.classification.contract import (
|
from app.application.classification.contract import (
|
||||||
ClassificationPolicyStateCorruptError,
|
ClassificationPolicyStateCorruptError,
|
||||||
@@ -196,7 +197,18 @@ async def compose_classification(
|
|||||||
)
|
)
|
||||||
|
|
||||||
service.register_extra_fields(migration.extra_fields)
|
service.register_extra_fields(migration.extra_fields)
|
||||||
await service.async_initialize(migration.policy)
|
try:
|
||||||
|
await service.async_initialize(migration.policy)
|
||||||
|
except ClassificationPolicyValidationError as error:
|
||||||
|
logger.error("旧分类策略未通过发布校验,继续保留 legacy 只读兼容行为")
|
||||||
|
return finish(
|
||||||
|
ClassificationRuntime(
|
||||||
|
service,
|
||||||
|
legacy_config=legacy_config,
|
||||||
|
diagnostics=(*_migration_issues(migration.issues), *error.result.issues),
|
||||||
|
),
|
||||||
|
migrated=False,
|
||||||
|
)
|
||||||
_log_migration_issues(migration)
|
_log_migration_issues(migration)
|
||||||
logger.info("已将 category.yaml 无损迁移为 MediaClassificationPolicy revision 1")
|
logger.info("已将 category.yaml 无损迁移为 MediaClassificationPolicy revision 1")
|
||||||
return finish(
|
return finish(
|
||||||
|
|||||||
@@ -934,12 +934,17 @@ API 常规读取只返回 `active`,历史接口按需读取 `history`。选择
|
|||||||
8. 首个空规则分类转换为该媒体类型的全局 `fallbacks`,其后的 legacy 项在旧实现中本就不可达,迁移时保持禁用并向管理员报告。
|
8. 首个空规则分类转换为该媒体类型的全局 `fallbacks`,其后的 legacy 项在旧实现中本就不可达,迁移时保持禁用并向管理员报告。
|
||||||
9. 新策略只按媒体类型配置通用兜底;数据源只能作为规则的筛选条件,不能单独决定默认分类。
|
9. 新策略只按媒体类型配置通用兜底;数据源只能作为规则的筛选条件,不能单独决定默认分类。
|
||||||
10. 保存新策略 revision 1,并保留原 YAML 文件只读备份,不再继续写入。
|
10. 保存新策略 revision 1,并保留原 YAML 文件只读备份,不再继续写入。
|
||||||
|
11. 同一字段的正向枚举合并为 `contains_any`,排除枚举合并为 `contains_none`;国家、语言等值数量
|
||||||
|
不应展开为叶子数量。已知和未知 Genre ID 仍分别使用标准字段与受控扩展字段,保持原有 OR 语义。
|
||||||
|
12. 迁移后的策略仍需通过完整发布校验;超出真实条件复杂度或引用约束时不写入新策略,保留旧分类
|
||||||
|
只读兼容行为和结构化错误诊断,不得让发布校验异常中断宿主启动。
|
||||||
|
|
||||||
### 13.2 行为兼容
|
### 13.2 行为兼容
|
||||||
|
|
||||||
- 迁移测试必须证明同一批 TMDB fixture 在旧分类器和新分类器下得到完全相同的目录分类。
|
- 迁移测试必须证明同一批 TMDB fixture 在旧分类器和新分类器下得到完全相同的目录分类。
|
||||||
- 旧配置中的 `!值` 转换为明确的排除操作符。
|
- 旧配置中的 `!值` 转换为明确的排除操作符。
|
||||||
- 旧逗号字符串转换为数组,后续 API 不再使用逗号编码多值。
|
- 旧逗号字符串转换为数组,后续 API 不再使用逗号编码多值。
|
||||||
|
- 旧只读 API 将集合逐值投影为逗号项,每个排除值独立添加 `!`;范围允许展开为等价枚举,不改写原 YAML。
|
||||||
- 迁移后 TMDB 模块不再实例化 `CategoryHelper`,也不再写 `MediaInfo.category`。
|
- 迁移后 TMDB 模块不再实例化 `CategoryHelper`,也不再写 `MediaInfo.category`。
|
||||||
|
|
||||||
### 13.3 字段迁移
|
### 13.3 字段迁移
|
||||||
|
|||||||
@@ -273,6 +273,37 @@ def test_mixed_genre_ids_keep_positive_or_and_negative_and_semantics() -> None:
|
|||||||
assert _category_name(result, _tmdb_facts(result, {}, "电影")) == "兜底"
|
assert _category_name(result, _tmdb_facts(result, {}, "电影")) == "兜底"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("negative", [False, True])
|
||||||
|
@pytest.mark.parametrize("field_name", ["origin_country", "genre_ids"])
|
||||||
|
def test_large_value_sets_remain_compact_and_preserve_legacy_semantics(
|
||||||
|
field_name: str,
|
||||||
|
negative: bool,
|
||||||
|
) -> None:
|
||||||
|
"""大枚举只产生集合条件,旧 API 投影后每个正向或排除值仍保持原语义。"""
|
||||||
|
values = [str(index) for index in range(1, 61)]
|
||||||
|
prefix = "!" if negative else ""
|
||||||
|
config = {
|
||||||
|
"movie": {
|
||||||
|
"目标": {field_name: ",".join(f"{prefix}{value}" for value in values)},
|
||||||
|
"兜底": None,
|
||||||
|
},
|
||||||
|
"tv": {},
|
||||||
|
}
|
||||||
|
result = migrate_legacy_category_config(config)
|
||||||
|
projection = project_policy_to_legacy_category_projection(result.policy)
|
||||||
|
remigrated = migrate_legacy_category_config(projection.config)
|
||||||
|
|
||||||
|
assert result.valid
|
||||||
|
assert ClassificationPolicyValidator.validate(result.policy, result.extra_fields).valid
|
||||||
|
assert len(_leaf_fields(result.policy.rules[0].when)) <= 2
|
||||||
|
assert projection.exact
|
||||||
|
for actual in [*([value] for value in values), ["999"], ["1", "999"], [], None]:
|
||||||
|
tmdb_info = {field_name: actual, "id": 1}
|
||||||
|
expected = _legacy_tmdb_category(config["movie"], tmdb_info)
|
||||||
|
assert _category_name(result, _tmdb_facts(result, tmdb_info, "电影")) == expected
|
||||||
|
assert _category_name(remigrated, _tmdb_facts(remigrated, tmdb_info, "电影")) == expected
|
||||||
|
|
||||||
|
|
||||||
def test_safe_unknown_field_is_declared_but_unsafe_field_blocks_publish() -> None:
|
def test_safe_unknown_field_is_declared_but_unsafe_field_blocks_publish() -> None:
|
||||||
"""合法未知 TMDB 一级字段可迁移,非法字段段必须产生阻断错误。"""
|
"""合法未知 TMDB 一级字段可迁移,非法字段段必须产生阻断错误。"""
|
||||||
safe = migrate_legacy_category_config(
|
safe = migrate_legacy_category_config(
|
||||||
@@ -387,7 +418,15 @@ def test_release_year_supports_values_ranges_and_non_numeric_hyphen_endpoints()
|
|||||||
assert _category_name(result, _tmdb_facts(result, {"release_date": "2023-01-01"}, "电影")) == "兜底"
|
assert _category_name(result, _tmdb_facts(result, {"release_date": "2023-01-01"}, "电影")) == "兜底"
|
||||||
projection = project_policy_to_legacy_category_projection(result.policy)
|
projection = project_policy_to_legacy_category_projection(result.policy)
|
||||||
assert projection.exact
|
assert projection.exact
|
||||||
assert projection.config.movie == CategoryConfig.model_validate(config).movie
|
assert projection.config.movie == CategoryConfig.model_validate(
|
||||||
|
{
|
||||||
|
"movie": {
|
||||||
|
"近年": {"release_year": "2020,2021,2022,2024"},
|
||||||
|
"字母年": {"release_year": "ABCD,EFGH"},
|
||||||
|
"兜底": None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
).movie
|
||||||
|
|
||||||
|
|
||||||
def test_stable_ids_are_repeatable_ascii_and_media_type_scoped() -> None:
|
def test_stable_ids_are_repeatable_ascii_and_media_type_scoped() -> None:
|
||||||
@@ -485,5 +524,9 @@ def test_migrated_policy_round_trips_to_category_config() -> None:
|
|||||||
projected = project_policy_to_legacy_category_projection(migrated.policy)
|
projected = project_policy_to_legacy_category_projection(migrated.policy)
|
||||||
|
|
||||||
assert projected.exact
|
assert projected.exact
|
||||||
assert projected.config == CategoryConfig.model_validate(config)
|
expected = CategoryConfig.model_validate(config)
|
||||||
|
assert expected.movie is not None
|
||||||
|
assert expected.movie["组合"] is not None
|
||||||
|
expected.movie["组合"].release_year = "2020,2021,2022,2024"
|
||||||
|
assert projected.config == expected
|
||||||
assert project_policy_to_legacy_category_config(migrated.policy) == projected.config
|
assert project_policy_to_legacy_category_config(migrated.policy) == projected.config
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from app.application.classification.configuration import (
|
|||||||
from app.application.classification.contract import ClassificationPolicyConflictError
|
from app.application.classification.contract import ClassificationPolicyConflictError
|
||||||
from app.application.classification.runtime import ClassificationRuntime
|
from app.application.classification.runtime import ClassificationRuntime
|
||||||
from app.schemas.category import (
|
from app.schemas.category import (
|
||||||
|
ClassificationConditionGroup,
|
||||||
ClassificationPolicyState,
|
ClassificationPolicyState,
|
||||||
)
|
)
|
||||||
from app.schemas.types import SystemConfigKey
|
from app.schemas.types import SystemConfigKey
|
||||||
@@ -152,6 +153,83 @@ async def test_absent_policy_migrates_yaml_once_without_rewriting_file(
|
|||||||
assert legacy_path.read_text(encoding="utf-8") == content
|
assert legacy_path.read_text(encoding="utf-8") == content
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_large_legacy_enumerations_migrate_and_reload_without_losing_values(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""复现 56/31 个旧枚举值的升级场景,验证首次发布、重启和旧接口均保留全部值。"""
|
||||||
|
languages = ",".join(f"language{index}" for index in range(26))
|
||||||
|
countries = ",".join(f"country{index}" for index in range(30))
|
||||||
|
content = (
|
||||||
|
f"movie:\n 欧美电影:\n original_language: '{languages}'\n"
|
||||||
|
f" production_countries: '{countries}'\n 未分类:\n"
|
||||||
|
f"tv:\n 欧美漫:\n genre_ids: '16'\n origin_country: '{countries}'\n 未分类:\n"
|
||||||
|
)
|
||||||
|
legacy_path = tmp_path / "category.yaml"
|
||||||
|
legacy_path.write_text(content, encoding="utf-8")
|
||||||
|
store = _MemoryPolicyStore()
|
||||||
|
monkeypatch.setattr(classification_composition, "SystemConfigClassificationPolicyStore", lambda *_args: store)
|
||||||
|
system_config = _SystemConfig({})
|
||||||
|
|
||||||
|
composition = await classification_composition.compose_classification(
|
||||||
|
executor=cast(Any, _InlineExecutor()),
|
||||||
|
settings=cast(Any, SimpleNamespace(CONFIG_PATH=tmp_path)),
|
||||||
|
system_config=cast(Any, system_config),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert composition.migrated is True
|
||||||
|
policy = composition.runtime.require_policy()
|
||||||
|
assert policy.revision == 1
|
||||||
|
assert len(policy.rules) == 2
|
||||||
|
for rule in policy.rules:
|
||||||
|
assert isinstance(rule.when, ClassificationConditionGroup)
|
||||||
|
assert rule.when.all is not None and len(rule.when.all) == 2
|
||||||
|
assert store.state is not None
|
||||||
|
system_config.publish_many({SystemConfigKey.MediaClassificationPolicy: store.state.model_dump(mode="json")})
|
||||||
|
reloaded = await classification_composition.compose_classification(
|
||||||
|
executor=cast(Any, _InlineExecutor()),
|
||||||
|
settings=cast(Any, SimpleNamespace(CONFIG_PATH=tmp_path)),
|
||||||
|
system_config=cast(Any, system_config),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert reloaded.migrated is False
|
||||||
|
assert reloaded.runtime.require_policy() == policy
|
||||||
|
projected = reloaded.runtime.legacy_config().model_dump(exclude_none=True)
|
||||||
|
assert projected["movie"]["欧美电影"]["original_language"] == languages
|
||||||
|
assert projected["movie"]["欧美电影"]["production_countries"] == countries
|
||||||
|
assert projected["tv"]["欧美漫"]["origin_country"] == countries
|
||||||
|
assert store.write_count == 1
|
||||||
|
assert legacy_path.read_text(encoding="utf-8") == content
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_publish_validation_failure_preserves_legacy_runtime_without_writes(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
"""真正超出叶子上限的不同字段保留错误诊断和旧配置,不得中断启动或写入新策略。"""
|
||||||
|
content = "movie:\n 复杂规则:\n" + "".join(f" field{index}: 'X'\n" for index in range(31))
|
||||||
|
legacy_path = tmp_path / "category.yaml"
|
||||||
|
legacy_path.write_text(content, encoding="utf-8")
|
||||||
|
store = _MemoryPolicyStore()
|
||||||
|
monkeypatch.setattr(classification_composition, "SystemConfigClassificationPolicyStore", lambda *_args: store)
|
||||||
|
|
||||||
|
composition = await classification_composition.compose_classification(
|
||||||
|
executor=cast(Any, _InlineExecutor()),
|
||||||
|
settings=cast(Any, SimpleNamespace(CONFIG_PATH=tmp_path)),
|
||||||
|
system_config=cast(Any, _SystemConfig({})),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert composition.migrated is False
|
||||||
|
assert composition.runtime.active_policy() is None
|
||||||
|
assert any(issue.code == "max_conditions_exceeded" for issue in composition.runtime.diagnostics())
|
||||||
|
assert "复杂规则" in (composition.runtime.legacy_config().movie or {})
|
||||||
|
assert store.write_count == 0
|
||||||
|
assert store.state is None
|
||||||
|
assert legacy_path.read_text(encoding="utf-8") == content
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio # type: ignore[misc]
|
@pytest.mark.asyncio # type: ignore[misc]
|
||||||
async def test_invalid_legacy_config_keeps_runtime_uninitialized_and_writes_nothing(
|
async def test_invalid_legacy_config_keeps_runtime_uninitialized_and_writes_nothing(
|
||||||
monkeypatch: pytest.MonkeyPatch,
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
|||||||
Reference in New Issue
Block a user