From cd5e6933021f6b640a508f539a7e8c91173a5f07 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Sat, 9 May 2026 20:04:05 +0800 Subject: [PATCH] refactor: adjust database indexes by adding high-frequency composite indexes and removing redundant id indexes --- app/db/__init__.py | 4 +- app/db/models/downloadhistory.py | 18 +- app/db/models/mediaserver.py | 8 +- app/db/models/message.py | 8 +- app/db/models/plugindata.py | 10 +- app/db/models/siteuserdata.py | 11 +- app/db/models/subscribe.py | 8 +- app/db/models/subscribehistory.py | 6 +- app/db/models/transferhistory.py | 9 +- app/db/models/userconfig.py | 5 +- app/db/models/workflow.py | 6 +- database/versions/93f8cb6a4d1e_2_2_4.py | 278 ++++++++++++++++++++++++ 12 files changed, 346 insertions(+), 25 deletions(-) create mode 100644 database/versions/93f8cb6a4d1e_2_2_4.py diff --git a/app/db/__init__.py b/app/db/__init__.py index fc473aa5..efc09916 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -15,10 +15,10 @@ def get_id_column(): """ if settings.DB_TYPE.lower() == "postgresql": # PostgreSQL使用SERIAL类型，让数据库自动处理序列 - return Column(Integer, Identity(start=1, cycle=True), primary_key=True, index=True) + return Column(Integer, Identity(start=1, cycle=True), primary_key=True) else: # SQLite使用Sequence - return Column(Integer, Sequence('id'), primary_key=True, index=True) + return Column(Integer, Sequence('id'), primary_key=True) def _get_database_engine(is_async: bool = False): diff --git a/app/db/models/downloadhistory.py b/app/db/models/downloadhistory.py index 4b9d2f17..26a147be 100644 --- a/app/db/models/downloadhistory.py +++ b/app/db/models/downloadhistory.py @@ -1,7 +1,7 @@ import time from typing import List, Optional -from sqlalchemy import Column, Integer, String, JSON, select, func +from sqlalchemy import Column, Integer, String, JSON, Index, select, func from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session @@ -35,7 +35,7 @@ class DownloadHistory(Base): # 下载器 downloader = Column(String) # 下载任务Hash - download_hash = Column(String, index=True) + download_hash = Column(String) # 种子名称 torrent_name = Column(String) # 种子描述 @@ -59,6 +59,11 @@ class DownloadHistory(Base): # 自定义识别词（用于整理时应用） custom_words = Column(String) + __table_args__ = ( + Index('ix_downloadhistory_download_hash_date', 'download_hash', 'date'), + Index('ix_downloadhistory_date_id', 'date', 'id'), + ) + @classmethod @db_query def get_by_hash(cls, db: Session, download_hash: str): @@ -373,9 +378,9 @@ class DownloadFiles(Base): # 下载器 downloader = Column(String) # 下载任务Hash - download_hash = Column(String, index=True) + download_hash = Column(String) # 完整路径 - fullpath = Column(String, index=True) + fullpath = Column(String) # 保存路径 savepath = Column(String, index=True) # 文件相对路径/名称 @@ -385,6 +390,11 @@ class DownloadFiles(Base): # 状态 0-已删除 1-正常 state = Column(Integer, nullable=False, default=1) + __table_args__ = ( + Index('ix_downloadfiles_download_hash_state', 'download_hash', 'state'), + Index('ix_downloadfiles_fullpath_id', 'fullpath', 'id'), + ) + @classmethod @db_query def get_by_hash(cls, db: Session, download_hash: str, state: Optional[int] = None): diff --git a/app/db/models/mediaserver.py b/app/db/models/mediaserver.py index c7340664..69c4db84 100644 --- a/app/db/models/mediaserver.py +++ b/app/db/models/mediaserver.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Optional -from sqlalchemy import Column, Integer, String, JSON +from sqlalchemy import Column, Integer, String, JSON, Index from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session @@ -29,7 +29,7 @@ class MediaServerItem(Base): # 年份 year = Column(String) # TMDBID - tmdbid = Column(Integer, index=True) + tmdbid = Column(Integer) # IMDBID imdbid = Column(String, index=True) # TVDBID @@ -43,6 +43,10 @@ class MediaServerItem(Base): # 同步时间 lst_mod_date = Column(String, default=datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + __table_args__ = ( + Index('ix_mediaserveritem_tmdbid_item_type', 'tmdbid', 'item_type'), + ) + @classmethod @db_query def get_by_itemid(cls, db: Session, item_id: str): diff --git a/app/db/models/message.py b/app/db/models/message.py index 67d5c21d..e27e9675 100644 --- a/app/db/models/message.py +++ b/app/db/models/message.py @@ -1,6 +1,6 @@ from typing import Optional -from sqlalchemy import Column, Integer, String, JSON, select +from sqlalchemy import Column, Integer, String, JSON, Index, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session @@ -29,12 +29,16 @@ class Message(Base): # 用户ID userid = Column(String) # 登记时间 - reg_time = Column(String, index=True) + reg_time = Column(String) # 消息方向：0-接收息，1-发送消息 action = Column(Integer) # 附件json note = Column(JSON) + __table_args__ = ( + Index('ix_message_reg_time_id', 'reg_time', 'id'), + ) + @classmethod @db_query def list_by_page(cls, db: Session, page: Optional[int] = 1, count: Optional[int] = 30): diff --git a/app/db/models/plugindata.py b/app/db/models/plugindata.py index dfeb94a2..bfd882b8 100644 --- a/app/db/models/plugindata.py +++ b/app/db/models/plugindata.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, String, JSON, select +from sqlalchemy import Column, String, JSON, Index, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session @@ -16,10 +16,14 @@ class PluginData(Base): 插件数据表 """ id = get_id_column() - plugin_id = Column(String, nullable=False, index=True) - key = Column(String, index=True, nullable=False) + plugin_id = Column(String, nullable=False) + key = Column(String, nullable=False) value = Column(JSON) + __table_args__ = ( + Index('ix_plugindata_plugin_id_key', 'plugin_id', 'key'), + ) + @classmethod @db_query def get_plugin_data(cls, db: Session, plugin_id: str): diff --git a/app/db/models/siteuserdata.py b/app/db/models/siteuserdata.py index c3d1f58f..c35d910a 100644 --- a/app/db/models/siteuserdata.py +++ b/app/db/models/siteuserdata.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Optional -from sqlalchemy import Column, Integer, String, Float, JSON, func, or_, select +from sqlalchemy import Column, Integer, String, Float, JSON, Index, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session @@ -14,7 +14,7 @@ class SiteUserData(Base): """ id = get_id_column() # 站点域名 - domain = Column(String, index=True) + domain = Column(String) # 站点名称 name = Column(String) # 用户名 @@ -50,10 +50,15 @@ class SiteUserData(Base): # 错误信息 err_msg = Column(String) # 更新日期 - updated_day = Column(String, index=True, default=datetime.now().strftime('%Y-%m-%d')) + updated_day = Column(String, default=datetime.now().strftime('%Y-%m-%d')) # 更新时间 updated_time = Column(String, default=datetime.now().strftime('%H:%M:%S')) + __table_args__ = ( + Index('ix_siteuserdata_updated_day_id', 'updated_day', 'id'), + Index('ix_siteuserdata_domain_updated_day_updated_time', 'domain', 'updated_day', 'updated_time'), + ) + @classmethod @db_query def get_by_domain(cls, db: Session, domain: str, workdate: Optional[str] = None, worktime: Optional[str] = None): diff --git a/app/db/models/subscribe.py b/app/db/models/subscribe.py index c4e58b5e..b8305624 100644 --- a/app/db/models/subscribe.py +++ b/app/db/models/subscribe.py @@ -1,7 +1,7 @@ import time from typing import Optional -from sqlalchemy import Column, Integer, String, Float, JSON, select +from sqlalchemy import Column, Integer, String, Float, JSON, Index, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session @@ -64,7 +64,7 @@ class Subscribe(Base): # 创建时间 date = Column(String) # 订阅用户 - username = Column(String) + username = Column(String, index=True) # 订阅站点 sites = Column(JSON, default=list) # 下载器 @@ -88,6 +88,10 @@ class Subscribe(Base): # 选择的剧集组 episode_group = Column(String) + __table_args__ = ( + Index('ix_subscribe_type_date', 'type', 'date'), + ) + @classmethod @db_query def exists(cls, db: Session, tmdbid: Optional[int] = None, doubanid: Optional[str] = None, diff --git a/app/db/models/subscribehistory.py b/app/db/models/subscribehistory.py index bf9c7b3d..698c516e 100644 --- a/app/db/models/subscribehistory.py +++ b/app/db/models/subscribehistory.py @@ -1,6 +1,6 @@ from typing import Optional -from sqlalchemy import Column, Integer, String, Float, JSON, select +from sqlalchemy import Column, Integer, String, Float, JSON, Index, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session @@ -73,6 +73,10 @@ class SubscribeHistory(Base): # 剧集组 episode_group = Column(String) + __table_args__ = ( + Index('ix_subscribehistory_type_date', 'type', 'date'), + ) + @classmethod @db_query def list_by_type(cls, db: Session, mtype: str, page: Optional[int] = 1, count: Optional[int] = 30): diff --git a/app/db/models/transferhistory.py b/app/db/models/transferhistory.py index 2f94f4f6..a02c80a5 100644 --- a/app/db/models/transferhistory.py +++ b/app/db/models/transferhistory.py @@ -1,7 +1,7 @@ import time from typing import Optional -from sqlalchemy import Column, Integer, String, Boolean, func, or_, JSON, select +from sqlalchemy import Column, Integer, String, Boolean, Index, func, or_, JSON, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session @@ -54,12 +54,17 @@ class TransferHistory(Base): # 转移失败信息 errmsg = Column(String) # 时间 - date = Column(String, index=True) + date = Column(String) # 文件清单，以JSON存储 files = Column(JSON, default=list) # 剧集组 episode_group = Column(String) + __table_args__ = ( + Index('ix_transferhistory_status_date', 'status', 'date'), + Index('ix_transferhistory_date_id', 'date', 'id'), + ) + @classmethod @db_query def list_by_title(cls, db: Session, title: str, page: Optional[int] = 1, count: Optional[int] = 30, diff --git a/app/db/models/userconfig.py b/app/db/models/userconfig.py index b8b6eedf..c9942401 100644 --- a/app/db/models/userconfig.py +++ b/app/db/models/userconfig.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, String, UniqueConstraint, Index, JSON +from sqlalchemy import Column, String, UniqueConstraint, JSON from sqlalchemy.orm import Session from app.db import db_query, db_update, get_id_column, Base @@ -10,7 +10,7 @@ class UserConfig(Base): """ id = get_id_column() # 用户名 - username = Column(String, index=True) + username = Column(String) # 配置键 key = Column(String) # 值 @@ -19,7 +19,6 @@ class UserConfig(Base): __table_args__ = ( # 用户名和配置键联合唯一 UniqueConstraint('username', 'key'), - Index('ix_userconfig_username_key', 'username', 'key'), ) @classmethod diff --git a/app/db/models/workflow.py b/app/db/models/workflow.py index 69bf4f5f..b1837d79 100644 --- a/app/db/models/workflow.py +++ b/app/db/models/workflow.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Optional -from sqlalchemy import Column, Integer, JSON, String, and_, or_, select +from sqlalchemy import Column, Integer, JSON, String, Index, and_, or_, select from sqlalchemy.ext.asyncio import AsyncSession from app.db import Base, db_query, get_id_column, db_update, async_db_query, async_db_update @@ -44,6 +44,10 @@ class Workflow(Base): # 最后执行时间 last_time = Column(String) + __table_args__ = ( + Index('ix_workflow_trigger_type_state', 'trigger_type', 'state'), + ) + @classmethod @db_query def list(cls, db): diff --git a/database/versions/93f8cb6a4d1e_2_2_4.py b/database/versions/93f8cb6a4d1e_2_2_4.py new file mode 100644 index 00000000..97c194f2 --- /dev/null +++ b/database/versions/93f8cb6a4d1e_2_2_4.py @@ -0,0 +1,278 @@ +"""2.2.4 +调整数据库索引，补充高频组合索引并移除冗余 id 索引 + +Revision ID: 93f8cb6a4d1e +Revises: 58edfac72c32 +Create Date: 2026-05-09 +""" + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = "93f8cb6a4d1e" +down_revision = "58edfac72c32" +branch_labels = None +depends_on = None + + +REDUNDANT_ID_INDEXES = { + "downloadfiles": [("ix_downloadfiles_id", ["id"])], + "downloadhistory": [("ix_downloadhistory_id", ["id"])], + "mediaserveritem": [("ix_mediaserveritem_id", ["id"])], + "message": [("ix_message_id", ["id"])], + "passkey": [("ix_passkey_id", ["id"])], + "plugindata": [("ix_plugindata_id", ["id"])], + "site": [("ix_site_id", ["id"])], + "siteicon": [("ix_siteicon_id", ["id"])], + "sitestatistic": [("ix_sitestatistic_id", ["id"])], + "siteuserdata": [("ix_siteuserdata_id", ["id"])], + "subscribe": [("ix_subscribe_id", ["id"])], + "subscribehistory": [("ix_subscribehistory_id", ["id"])], + "systemconfig": [("ix_systemconfig_id", ["id"])], + "transferhistory": [("ix_transferhistory_id", ["id"])], + "user": [("ix_user_id", ["id"])], + "userconfig": [("ix_userconfig_id", ["id"])], + "workflow": [("ix_workflow_id", ["id"])], +} + + +DROP_INDEXES = { + "plugindata": [ + ("ix_plugindata_plugin_id", ["plugin_id"]), + ("ix_plugindata_key", ["key"]), + ], + "message": [ + ("ix_message_reg_time", ["reg_time"]), + ], + "siteuserdata": [ + ("ix_siteuserdata_domain", ["domain"]), + ("ix_siteuserdata_updated_day", ["updated_day"]), + ], + "downloadhistory": [ + ("ix_downloadhistory_download_hash", ["download_hash"]), + ], + "downloadfiles": [ + ("ix_downloadfiles_download_hash", ["download_hash"]), + ("ix_downloadfiles_fullpath", ["fullpath"]), + ], + "mediaserveritem": [ + ("ix_mediaserveritem_tmdbid", ["tmdbid"]), + ], + "transferhistory": [ + ("ix_transferhistory_date", ["date"]), + ], + "userconfig": [ + ("ix_userconfig_username", ["username"]), + ("ix_userconfig_username_key", ["username", "key"]), + ], +} + + +CREATE_INDEXES = { + "plugindata": [ + ("ix_plugindata_plugin_id_key", ["plugin_id", "key"]), + ], + "message": [ + ("ix_message_reg_time_id", ["reg_time", "id"]), + ], + "siteuserdata": [ + ("ix_siteuserdata_updated_day_id", ["updated_day", "id"]), + ( + "ix_siteuserdata_domain_updated_day_updated_time", + ["domain", "updated_day", "updated_time"], + ), + ], + "downloadhistory": [ + ("ix_downloadhistory_download_hash_date", ["download_hash", "date"]), + ("ix_downloadhistory_date_id", ["date", "id"]), + ], + "downloadfiles": [ + ("ix_downloadfiles_download_hash_state", ["download_hash", "state"]), + ("ix_downloadfiles_fullpath_id", ["fullpath", "id"]), + ], + "mediaserveritem": [ + ("ix_mediaserveritem_tmdbid_item_type", ["tmdbid", "item_type"]), + ], + "subscribe": [ + ("ix_subscribe_username", ["username"]), + ("ix_subscribe_type_date", ["type", "date"]), + ], + "subscribehistory": [ + ("ix_subscribehistory_type_date", ["type", "date"]), + ], + "transferhistory": [ + ("ix_transferhistory_status_date", ["status", "date"]), + ("ix_transferhistory_date_id", ["date", "id"]), + ], + "workflow": [ + ("ix_workflow_trigger_type_state", ["trigger_type", "state"]), + ], +} + + +DOWNGRADE_RESTORE_INDEXES = { + "plugindata": [ + ("ix_plugindata_plugin_id", ["plugin_id"]), + ("ix_plugindata_key", ["key"]), + ], + "message": [ + ("ix_message_reg_time", ["reg_time"]), + ], + "siteuserdata": [ + ("ix_siteuserdata_domain", ["domain"]), + ("ix_siteuserdata_updated_day", ["updated_day"]), + ], + "downloadhistory": [ + ("ix_downloadhistory_download_hash", ["download_hash"]), + ], + "downloadfiles": [ + ("ix_downloadfiles_download_hash", ["download_hash"]), + ("ix_downloadfiles_fullpath", ["fullpath"]), + ], + "mediaserveritem": [ + ("ix_mediaserveritem_tmdbid", ["tmdbid"]), + ], + "transferhistory": [ + ("ix_transferhistory_date", ["date"]), + ], + "userconfig": [ + ("ix_userconfig_username", ["username"]), + ("ix_userconfig_username_key", ["username", "key"]), + ], +} + + +def _load_schema_state(inspector: sa.Inspector): + tables = set(inspector.get_table_names()) + table_indexes = { + table_name: { + index["name"]: { + "columns": tuple(index.get("column_names") or []), + "unique": bool(index.get("unique")), + } + for index in inspector.get_indexes(table_name) + } + for table_name in tables + } + return tables, table_indexes + + +def _drop_index( + table_name: str, + index_name: str, + tables: set[str], + table_indexes: dict[str, dict[str, dict[str, object]]], +) -> None: + if table_name not in tables: + return + if index_name not in table_indexes[table_name]: + return + op.drop_index(index_name, table_name=table_name) + table_indexes[table_name].pop(index_name, None) + + +def _drop_index_by_signature( + table_name: str, + columns: list[str], + tables: set[str], + table_indexes: dict[str, dict[str, dict[str, object]]], + expected_name: str | None = None, + unique: bool = False, +) -> None: + if table_name not in tables: + return + + target_columns = tuple(columns) + for index_name, index_meta in list(table_indexes[table_name].items()): + if expected_name and index_name == expected_name: + _drop_index(table_name, index_name, tables, table_indexes) + return + if index_meta.get("columns") == target_columns and index_meta.get("unique") == unique: + _drop_index(table_name, index_name, tables, table_indexes) + return + + +def _has_index_signature( + table_name: str, + columns: list[str], + tables: set[str], + table_indexes: dict[str, dict[str, dict[str, object]]], + unique: bool = False, +) -> bool: + if table_name not in tables: + return False + + target_columns = tuple(columns) + for index_meta in table_indexes[table_name].values(): + if index_meta.get("columns") == target_columns and index_meta.get("unique") == unique: + return True + return False + + +def _create_index( + table_name: str, + index_name: str, + columns: list[str], + tables: set[str], + table_indexes: dict[str, dict[str, dict[str, object]]], +) -> None: + if table_name not in tables: + return + if index_name in table_indexes[table_name]: + return + if _has_index_signature(table_name, columns, tables, table_indexes, unique=False): + return + op.create_index(index_name, table_name, columns, unique=False) + table_indexes[table_name][index_name] = { + "columns": tuple(columns), + "unique": False, + } + + +def upgrade() -> None: + inspector = sa.inspect(op.get_bind()) + tables, table_indexes = _load_schema_state(inspector) + + for table_name, index_specs in REDUNDANT_ID_INDEXES.items(): + for index_name, columns in index_specs: + _drop_index_by_signature( + table_name, + columns, + tables, + table_indexes, + expected_name=index_name, + unique=False, + ) + + for table_name, index_specs in DROP_INDEXES.items(): + for index_name, columns in index_specs: + _drop_index_by_signature( + table_name, + columns, + tables, + table_indexes, + expected_name=index_name, + unique=False, + ) + + for table_name, index_specs in CREATE_INDEXES.items(): + for index_name, columns in index_specs: + _create_index(table_name, index_name, columns, tables, table_indexes) + + +def downgrade() -> None: + inspector = sa.inspect(op.get_bind()) + tables, table_indexes = _load_schema_state(inspector) + + for table_name, index_specs in CREATE_INDEXES.items(): + for index_name, _ in index_specs: + _drop_index(table_name, index_name, tables, table_indexes) + + for table_name, index_specs in DOWNGRADE_RESTORE_INDEXES.items(): + for index_name, columns in index_specs: + _create_index(table_name, index_name, columns, tables, table_indexes) + + for table_name, index_specs in REDUNDANT_ID_INDEXES.items(): + for index_name, columns in index_specs: + _create_index(table_name, index_name, columns, tables, table_indexes)