refactor: adjust database indexes by adding high-frequency composite indexes and removing redundant id indexes

This commit is contained in:
jxxghp
2026-05-09 20:04:05 +08:00
parent ac11b303b3
commit cd5e693302
12 changed files with 346 additions and 25 deletions

View File

@@ -15,10 +15,10 @@ def get_id_column():
"""
if settings.DB_TYPE.lower() == "postgresql":
# PostgreSQL使用SERIAL类型让数据库自动处理序列
return Column(Integer, Identity(start=1, cycle=True), primary_key=True, index=True)
return Column(Integer, Identity(start=1, cycle=True), primary_key=True)
else:
# SQLite使用Sequence
return Column(Integer, Sequence('id'), primary_key=True, index=True)
return Column(Integer, Sequence('id'), primary_key=True)
def _get_database_engine(is_async: bool = False):

View File

@@ -1,7 +1,7 @@
import time
from typing import List, Optional
from sqlalchemy import Column, Integer, String, JSON, select, func
from sqlalchemy import Column, Integer, String, JSON, Index, select, func
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session
@@ -35,7 +35,7 @@ class DownloadHistory(Base):
# 下载器
downloader = Column(String)
# 下载任务Hash
download_hash = Column(String, index=True)
download_hash = Column(String)
# 种子名称
torrent_name = Column(String)
# 种子描述
@@ -59,6 +59,11 @@ class DownloadHistory(Base):
# 自定义识别词(用于整理时应用)
custom_words = Column(String)
__table_args__ = (
Index('ix_downloadhistory_download_hash_date', 'download_hash', 'date'),
Index('ix_downloadhistory_date_id', 'date', 'id'),
)
@classmethod
@db_query
def get_by_hash(cls, db: Session, download_hash: str):
@@ -373,9 +378,9 @@ class DownloadFiles(Base):
# 下载器
downloader = Column(String)
# 下载任务Hash
download_hash = Column(String, index=True)
download_hash = Column(String)
# 完整路径
fullpath = Column(String, index=True)
fullpath = Column(String)
# 保存路径
savepath = Column(String, index=True)
# 文件相对路径/名称
@@ -385,6 +390,11 @@ class DownloadFiles(Base):
# 状态 0-已删除 1-正常
state = Column(Integer, nullable=False, default=1)
__table_args__ = (
Index('ix_downloadfiles_download_hash_state', 'download_hash', 'state'),
Index('ix_downloadfiles_fullpath_id', 'fullpath', 'id'),
)
@classmethod
@db_query
def get_by_hash(cls, db: Session, download_hash: str, state: Optional[int] = None):

View File

@@ -1,7 +1,7 @@
from datetime import datetime
from typing import Optional
from sqlalchemy import Column, Integer, String, JSON
from sqlalchemy import Column, Integer, String, JSON, Index
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session
@@ -29,7 +29,7 @@ class MediaServerItem(Base):
# 年份
year = Column(String)
# TMDBID
tmdbid = Column(Integer, index=True)
tmdbid = Column(Integer)
# IMDBID
imdbid = Column(String, index=True)
# TVDBID
@@ -43,6 +43,10 @@ class MediaServerItem(Base):
# 同步时间
lst_mod_date = Column(String, default=datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
__table_args__ = (
Index('ix_mediaserveritem_tmdbid_item_type', 'tmdbid', 'item_type'),
)
@classmethod
@db_query
def get_by_itemid(cls, db: Session, item_id: str):

View File

@@ -1,6 +1,6 @@
from typing import Optional
from sqlalchemy import Column, Integer, String, JSON, select
from sqlalchemy import Column, Integer, String, JSON, Index, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session
@@ -29,12 +29,16 @@ class Message(Base):
# 用户ID
userid = Column(String)
# 登记时间
reg_time = Column(String, index=True)
reg_time = Column(String)
# 消息方向0-接收息1-发送消息
action = Column(Integer)
# 附件json
note = Column(JSON)
__table_args__ = (
Index('ix_message_reg_time_id', 'reg_time', 'id'),
)
@classmethod
@db_query
def list_by_page(cls, db: Session, page: Optional[int] = 1, count: Optional[int] = 30):

View File

@@ -1,4 +1,4 @@
from sqlalchemy import Column, String, JSON, select
from sqlalchemy import Column, String, JSON, Index, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session
@@ -16,10 +16,14 @@ class PluginData(Base):
插件数据表
"""
id = get_id_column()
plugin_id = Column(String, nullable=False, index=True)
key = Column(String, index=True, nullable=False)
plugin_id = Column(String, nullable=False)
key = Column(String, nullable=False)
value = Column(JSON)
__table_args__ = (
Index('ix_plugindata_plugin_id_key', 'plugin_id', 'key'),
)
@classmethod
@db_query
def get_plugin_data(cls, db: Session, plugin_id: str):

View File

@@ -1,7 +1,7 @@
from datetime import datetime
from typing import Optional
from sqlalchemy import Column, Integer, String, Float, JSON, func, or_, select
from sqlalchemy import Column, Integer, String, Float, JSON, Index, func, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session
@@ -14,7 +14,7 @@ class SiteUserData(Base):
"""
id = get_id_column()
# 站点域名
domain = Column(String, index=True)
domain = Column(String)
# 站点名称
name = Column(String)
# 用户名
@@ -50,10 +50,15 @@ class SiteUserData(Base):
# 错误信息
err_msg = Column(String)
# 更新日期
updated_day = Column(String, index=True, default=datetime.now().strftime('%Y-%m-%d'))
updated_day = Column(String, default=datetime.now().strftime('%Y-%m-%d'))
# 更新时间
updated_time = Column(String, default=datetime.now().strftime('%H:%M:%S'))
__table_args__ = (
Index('ix_siteuserdata_updated_day_id', 'updated_day', 'id'),
Index('ix_siteuserdata_domain_updated_day_updated_time', 'domain', 'updated_day', 'updated_time'),
)
@classmethod
@db_query
def get_by_domain(cls, db: Session, domain: str, workdate: Optional[str] = None, worktime: Optional[str] = None):

View File

@@ -1,7 +1,7 @@
import time
from typing import Optional
from sqlalchemy import Column, Integer, String, Float, JSON, select
from sqlalchemy import Column, Integer, String, Float, JSON, Index, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session
@@ -64,7 +64,7 @@ class Subscribe(Base):
# 创建时间
date = Column(String)
# 订阅用户
username = Column(String)
username = Column(String, index=True)
# 订阅站点
sites = Column(JSON, default=list)
# 下载器
@@ -88,6 +88,10 @@ class Subscribe(Base):
# 选择的剧集组
episode_group = Column(String)
__table_args__ = (
Index('ix_subscribe_type_date', 'type', 'date'),
)
@classmethod
@db_query
def exists(cls, db: Session, tmdbid: Optional[int] = None, doubanid: Optional[str] = None,

View File

@@ -1,6 +1,6 @@
from typing import Optional
from sqlalchemy import Column, Integer, String, Float, JSON, select
from sqlalchemy import Column, Integer, String, Float, JSON, Index, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session
@@ -73,6 +73,10 @@ class SubscribeHistory(Base):
# 剧集组
episode_group = Column(String)
__table_args__ = (
Index('ix_subscribehistory_type_date', 'type', 'date'),
)
@classmethod
@db_query
def list_by_type(cls, db: Session, mtype: str, page: Optional[int] = 1, count: Optional[int] = 30):

View File

@@ -1,7 +1,7 @@
import time
from typing import Optional
from sqlalchemy import Column, Integer, String, Boolean, func, or_, JSON, select
from sqlalchemy import Column, Integer, String, Boolean, Index, func, or_, JSON, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session
@@ -54,12 +54,17 @@ class TransferHistory(Base):
# 转移失败信息
errmsg = Column(String)
# 时间
date = Column(String, index=True)
date = Column(String)
# 文件清单以JSON存储
files = Column(JSON, default=list)
# 剧集组
episode_group = Column(String)
__table_args__ = (
Index('ix_transferhistory_status_date', 'status', 'date'),
Index('ix_transferhistory_date_id', 'date', 'id'),
)
@classmethod
@db_query
def list_by_title(cls, db: Session, title: str, page: Optional[int] = 1, count: Optional[int] = 30,

View File

@@ -1,4 +1,4 @@
from sqlalchemy import Column, String, UniqueConstraint, Index, JSON
from sqlalchemy import Column, String, UniqueConstraint, JSON
from sqlalchemy.orm import Session
from app.db import db_query, db_update, get_id_column, Base
@@ -10,7 +10,7 @@ class UserConfig(Base):
"""
id = get_id_column()
# 用户名
username = Column(String, index=True)
username = Column(String)
# 配置键
key = Column(String)
# 值
@@ -19,7 +19,6 @@ class UserConfig(Base):
__table_args__ = (
# 用户名和配置键联合唯一
UniqueConstraint('username', 'key'),
Index('ix_userconfig_username_key', 'username', 'key'),
)
@classmethod

View File

@@ -1,7 +1,7 @@
from datetime import datetime
from typing import Optional
from sqlalchemy import Column, Integer, JSON, String, and_, or_, select
from sqlalchemy import Column, Integer, JSON, String, Index, and_, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import Base, db_query, get_id_column, db_update, async_db_query, async_db_update
@@ -44,6 +44,10 @@ class Workflow(Base):
# 最后执行时间
last_time = Column(String)
__table_args__ = (
Index('ix_workflow_trigger_type_state', 'trigger_type', 'state'),
)
@classmethod
@db_query
def list(cls, db):

View File

@@ -0,0 +1,278 @@
"""2.2.4
调整数据库索引,补充高频组合索引并移除冗余 id 索引
Revision ID: 93f8cb6a4d1e
Revises: 58edfac72c32
Create Date: 2026-05-09
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = "93f8cb6a4d1e"
down_revision = "58edfac72c32"
branch_labels = None
depends_on = None
REDUNDANT_ID_INDEXES = {
"downloadfiles": [("ix_downloadfiles_id", ["id"])],
"downloadhistory": [("ix_downloadhistory_id", ["id"])],
"mediaserveritem": [("ix_mediaserveritem_id", ["id"])],
"message": [("ix_message_id", ["id"])],
"passkey": [("ix_passkey_id", ["id"])],
"plugindata": [("ix_plugindata_id", ["id"])],
"site": [("ix_site_id", ["id"])],
"siteicon": [("ix_siteicon_id", ["id"])],
"sitestatistic": [("ix_sitestatistic_id", ["id"])],
"siteuserdata": [("ix_siteuserdata_id", ["id"])],
"subscribe": [("ix_subscribe_id", ["id"])],
"subscribehistory": [("ix_subscribehistory_id", ["id"])],
"systemconfig": [("ix_systemconfig_id", ["id"])],
"transferhistory": [("ix_transferhistory_id", ["id"])],
"user": [("ix_user_id", ["id"])],
"userconfig": [("ix_userconfig_id", ["id"])],
"workflow": [("ix_workflow_id", ["id"])],
}
DROP_INDEXES = {
"plugindata": [
("ix_plugindata_plugin_id", ["plugin_id"]),
("ix_plugindata_key", ["key"]),
],
"message": [
("ix_message_reg_time", ["reg_time"]),
],
"siteuserdata": [
("ix_siteuserdata_domain", ["domain"]),
("ix_siteuserdata_updated_day", ["updated_day"]),
],
"downloadhistory": [
("ix_downloadhistory_download_hash", ["download_hash"]),
],
"downloadfiles": [
("ix_downloadfiles_download_hash", ["download_hash"]),
("ix_downloadfiles_fullpath", ["fullpath"]),
],
"mediaserveritem": [
("ix_mediaserveritem_tmdbid", ["tmdbid"]),
],
"transferhistory": [
("ix_transferhistory_date", ["date"]),
],
"userconfig": [
("ix_userconfig_username", ["username"]),
("ix_userconfig_username_key", ["username", "key"]),
],
}
CREATE_INDEXES = {
"plugindata": [
("ix_plugindata_plugin_id_key", ["plugin_id", "key"]),
],
"message": [
("ix_message_reg_time_id", ["reg_time", "id"]),
],
"siteuserdata": [
("ix_siteuserdata_updated_day_id", ["updated_day", "id"]),
(
"ix_siteuserdata_domain_updated_day_updated_time",
["domain", "updated_day", "updated_time"],
),
],
"downloadhistory": [
("ix_downloadhistory_download_hash_date", ["download_hash", "date"]),
("ix_downloadhistory_date_id", ["date", "id"]),
],
"downloadfiles": [
("ix_downloadfiles_download_hash_state", ["download_hash", "state"]),
("ix_downloadfiles_fullpath_id", ["fullpath", "id"]),
],
"mediaserveritem": [
("ix_mediaserveritem_tmdbid_item_type", ["tmdbid", "item_type"]),
],
"subscribe": [
("ix_subscribe_username", ["username"]),
("ix_subscribe_type_date", ["type", "date"]),
],
"subscribehistory": [
("ix_subscribehistory_type_date", ["type", "date"]),
],
"transferhistory": [
("ix_transferhistory_status_date", ["status", "date"]),
("ix_transferhistory_date_id", ["date", "id"]),
],
"workflow": [
("ix_workflow_trigger_type_state", ["trigger_type", "state"]),
],
}
DOWNGRADE_RESTORE_INDEXES = {
"plugindata": [
("ix_plugindata_plugin_id", ["plugin_id"]),
("ix_plugindata_key", ["key"]),
],
"message": [
("ix_message_reg_time", ["reg_time"]),
],
"siteuserdata": [
("ix_siteuserdata_domain", ["domain"]),
("ix_siteuserdata_updated_day", ["updated_day"]),
],
"downloadhistory": [
("ix_downloadhistory_download_hash", ["download_hash"]),
],
"downloadfiles": [
("ix_downloadfiles_download_hash", ["download_hash"]),
("ix_downloadfiles_fullpath", ["fullpath"]),
],
"mediaserveritem": [
("ix_mediaserveritem_tmdbid", ["tmdbid"]),
],
"transferhistory": [
("ix_transferhistory_date", ["date"]),
],
"userconfig": [
("ix_userconfig_username", ["username"]),
("ix_userconfig_username_key", ["username", "key"]),
],
}
def _load_schema_state(inspector: sa.Inspector):
tables = set(inspector.get_table_names())
table_indexes = {
table_name: {
index["name"]: {
"columns": tuple(index.get("column_names") or []),
"unique": bool(index.get("unique")),
}
for index in inspector.get_indexes(table_name)
}
for table_name in tables
}
return tables, table_indexes
def _drop_index(
table_name: str,
index_name: str,
tables: set[str],
table_indexes: dict[str, dict[str, dict[str, object]]],
) -> None:
if table_name not in tables:
return
if index_name not in table_indexes[table_name]:
return
op.drop_index(index_name, table_name=table_name)
table_indexes[table_name].pop(index_name, None)
def _drop_index_by_signature(
table_name: str,
columns: list[str],
tables: set[str],
table_indexes: dict[str, dict[str, dict[str, object]]],
expected_name: str | None = None,
unique: bool = False,
) -> None:
if table_name not in tables:
return
target_columns = tuple(columns)
for index_name, index_meta in list(table_indexes[table_name].items()):
if expected_name and index_name == expected_name:
_drop_index(table_name, index_name, tables, table_indexes)
return
if index_meta.get("columns") == target_columns and index_meta.get("unique") == unique:
_drop_index(table_name, index_name, tables, table_indexes)
return
def _has_index_signature(
table_name: str,
columns: list[str],
tables: set[str],
table_indexes: dict[str, dict[str, dict[str, object]]],
unique: bool = False,
) -> bool:
if table_name not in tables:
return False
target_columns = tuple(columns)
for index_meta in table_indexes[table_name].values():
if index_meta.get("columns") == target_columns and index_meta.get("unique") == unique:
return True
return False
def _create_index(
table_name: str,
index_name: str,
columns: list[str],
tables: set[str],
table_indexes: dict[str, dict[str, dict[str, object]]],
) -> None:
if table_name not in tables:
return
if index_name in table_indexes[table_name]:
return
if _has_index_signature(table_name, columns, tables, table_indexes, unique=False):
return
op.create_index(index_name, table_name, columns, unique=False)
table_indexes[table_name][index_name] = {
"columns": tuple(columns),
"unique": False,
}
def upgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables, table_indexes = _load_schema_state(inspector)
for table_name, index_specs in REDUNDANT_ID_INDEXES.items():
for index_name, columns in index_specs:
_drop_index_by_signature(
table_name,
columns,
tables,
table_indexes,
expected_name=index_name,
unique=False,
)
for table_name, index_specs in DROP_INDEXES.items():
for index_name, columns in index_specs:
_drop_index_by_signature(
table_name,
columns,
tables,
table_indexes,
expected_name=index_name,
unique=False,
)
for table_name, index_specs in CREATE_INDEXES.items():
for index_name, columns in index_specs:
_create_index(table_name, index_name, columns, tables, table_indexes)
def downgrade() -> None:
inspector = sa.inspect(op.get_bind())
tables, table_indexes = _load_schema_state(inspector)
for table_name, index_specs in CREATE_INDEXES.items():
for index_name, _ in index_specs:
_drop_index(table_name, index_name, tables, table_indexes)
for table_name, index_specs in DOWNGRADE_RESTORE_INDEXES.items():
for index_name, columns in index_specs:
_create_index(table_name, index_name, columns, tables, table_indexes)
for table_name, index_specs in REDUNDANT_ID_INDEXES.items():
for index_name, columns in index_specs:
_create_index(table_name, index_name, columns, tables, table_indexes)