mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-04 23:17:20 +08:00
refactor: reorganize backend module boundaries
This commit is contained in:
jxxghp
Vendored
+1
@@ -0,0 +1 @@
|
||||
"""缓存持久化适配器。"""
|
||||
Vendored
+341
@@ -0,0 +1,341 @@
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncGenerator, Generator, Optional, Tuple
|
||||
|
||||
import aiofiles
|
||||
import aioshutil
|
||||
from anyio import Path as AsyncPath
|
||||
|
||||
from app.adapters.cache.redis import AsyncRedisHelper, RedisHelper
|
||||
from app.runtime.cache import (
|
||||
AsyncCacheBackend,
|
||||
CacheBackend,
|
||||
DEFAULT_CACHE_REGION,
|
||||
configure_cache_factories,
|
||||
)
|
||||
from app.runtime.config import settings
|
||||
|
||||
|
||||
class RedisBackend(CacheBackend):
|
||||
"""通过同步 Redis 客户端实现缓存后端。"""
|
||||
|
||||
def __init__(self, ttl: Optional[int] = None) -> None:
|
||||
"""初始化 Redis 缓存并保存默认 TTL。"""
|
||||
self.ttl = ttl
|
||||
self.redis_helper = RedisHelper()
|
||||
|
||||
def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
ttl: Optional[int] = None,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""写入缓存，非正 TTL 视为立即删除。"""
|
||||
ttl = self.ttl if ttl is None else ttl
|
||||
if ttl is not None and ttl <= 0:
|
||||
self.redis_helper.delete(key, region=region)
|
||||
return
|
||||
self.redis_helper.set(key, value, ttl=ttl, region=region, **kwargs)
|
||||
|
||||
def exists(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> bool:
|
||||
"""判断缓存键是否存在。"""
|
||||
return self.redis_helper.exists(key, region=region)
|
||||
|
||||
def get(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Optional[Any]:
|
||||
"""读取缓存值，不存在时返回空值。"""
|
||||
return self.redis_helper.get(key, region=region)
|
||||
|
||||
def delete(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> None:
|
||||
"""删除缓存键。"""
|
||||
self.redis_helper.delete(key, region=region)
|
||||
|
||||
def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None:
|
||||
"""清空指定缓存区或全部缓存。"""
|
||||
self.redis_helper.clear(region=region)
|
||||
|
||||
def items(
|
||||
self,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Generator[Tuple[str, Any], None, None]:
|
||||
"""遍历指定缓存区的键值对。"""
|
||||
return self.redis_helper.items(region=region)
|
||||
|
||||
def close(self) -> None:
|
||||
"""关闭同步 Redis 连接池。"""
|
||||
self.redis_helper.close()
|
||||
|
||||
@staticmethod
|
||||
def is_redis() -> bool:
|
||||
"""标记当前后端为 Redis。"""
|
||||
return True
|
||||
|
||||
|
||||
class AsyncRedisBackend(AsyncCacheBackend):
|
||||
"""通过异步 Redis 客户端实现缓存后端。"""
|
||||
|
||||
def __init__(self, ttl: Optional[int] = None) -> None:
|
||||
"""初始化异步 Redis 缓存并保存默认 TTL。"""
|
||||
self.ttl = ttl
|
||||
self.redis_helper = AsyncRedisHelper()
|
||||
|
||||
async def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
ttl: Optional[int] = None,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""异步写入缓存，非正 TTL 视为立即删除。"""
|
||||
ttl = self.ttl if ttl is None else ttl
|
||||
if ttl is not None and ttl <= 0:
|
||||
await self.redis_helper.delete(key, region=region)
|
||||
return
|
||||
await self.redis_helper.set(key, value, ttl=ttl, region=region, **kwargs)
|
||||
|
||||
async def exists(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> bool:
|
||||
"""异步判断缓存键是否存在。"""
|
||||
return await self.redis_helper.exists(key, region=region)
|
||||
|
||||
async def get(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Optional[Any]:
|
||||
"""异步读取缓存值，不存在时返回空值。"""
|
||||
return await self.redis_helper.get(key, region=region)
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> None:
|
||||
"""异步删除缓存键。"""
|
||||
await self.redis_helper.delete(key, region=region)
|
||||
|
||||
async def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None:
|
||||
"""异步清空指定缓存区或全部缓存。"""
|
||||
await self.redis_helper.clear(region=region)
|
||||
|
||||
async def items(
|
||||
self,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> AsyncGenerator[Tuple[str, Any], None]:
|
||||
"""异步遍历指定缓存区的键值对。"""
|
||||
async for item in self.redis_helper.items(region=region):
|
||||
yield item
|
||||
|
||||
async def close(self) -> None:
|
||||
"""关闭异步 Redis 连接池。"""
|
||||
await self.redis_helper.close()
|
||||
|
||||
@staticmethod
|
||||
def is_redis() -> bool:
|
||||
"""标记当前后端为 Redis。"""
|
||||
return True
|
||||
|
||||
|
||||
class FileBackend(CacheBackend):
|
||||
"""通过本地文件系统保存二进制缓存。"""
|
||||
|
||||
def __init__(self, base: Path) -> None:
|
||||
"""初始化缓存根目录。"""
|
||||
self.base = base
|
||||
self.base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
**_kwargs,
|
||||
) -> None:
|
||||
"""原子写入一个二进制缓存文件。"""
|
||||
cache_path = self.base / region / key
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
dir=cache_path.parent,
|
||||
delete=False,
|
||||
) as tmp_file:
|
||||
tmp_file.write(value)
|
||||
temp_path = Path(tmp_file.name)
|
||||
temp_path.replace(cache_path)
|
||||
|
||||
def exists(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> bool:
|
||||
"""判断缓存文件是否存在。"""
|
||||
return (self.base / region / key).exists()
|
||||
|
||||
def get(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Optional[Any]:
|
||||
"""读取二进制缓存文件。"""
|
||||
cache_path = self.base / region / key
|
||||
if not cache_path.exists():
|
||||
return None
|
||||
with cache_path.open("rb") as file_handle:
|
||||
return file_handle.read()
|
||||
|
||||
def delete(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> None:
|
||||
"""删除缓存文件或缓存子目录。"""
|
||||
cache_path = self.base / region / key
|
||||
if cache_path.is_file():
|
||||
cache_path.unlink()
|
||||
elif cache_path.exists():
|
||||
shutil.rmtree(cache_path, ignore_errors=True)
|
||||
|
||||
def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None:
|
||||
"""清空指定缓存区或缓存根目录。"""
|
||||
cache_path = self.base / region if region else self.base
|
||||
if not cache_path.exists():
|
||||
return
|
||||
for item in cache_path.iterdir():
|
||||
if item.is_file():
|
||||
item.unlink()
|
||||
else:
|
||||
shutil.rmtree(item, ignore_errors=True)
|
||||
|
||||
def items(
|
||||
self,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Generator[Tuple[str, Any], None, None]:
|
||||
"""按相对键遍历指定缓存区中的二进制文件。"""
|
||||
cache_path = self.base / region
|
||||
if not cache_path.exists():
|
||||
return
|
||||
for item in sorted(cache_path.rglob("*")):
|
||||
if item.is_file():
|
||||
with item.open("rb") as file_handle:
|
||||
yield item.relative_to(cache_path).as_posix(), file_handle.read()
|
||||
|
||||
def close(self) -> None:
|
||||
"""文件缓存没有需要关闭的持久连接。"""
|
||||
|
||||
|
||||
class AsyncFileBackend(AsyncCacheBackend):
|
||||
"""通过异步文件接口保存二进制缓存。"""
|
||||
|
||||
def __init__(self, base: Path) -> None:
|
||||
"""初始化异步缓存根目录。"""
|
||||
self.base = base
|
||||
self.base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
async def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
**_kwargs,
|
||||
) -> None:
|
||||
"""异步原子写入一个二进制缓存文件。"""
|
||||
cache_path = AsyncPath(self.base) / region / key
|
||||
await cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
async with aiofiles.tempfile.NamedTemporaryFile(
|
||||
dir=cache_path.parent,
|
||||
delete=False,
|
||||
) as tmp_file:
|
||||
await tmp_file.write(value)
|
||||
temp_path = AsyncPath(tmp_file.name)
|
||||
await temp_path.replace(cache_path)
|
||||
|
||||
async def exists(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> bool:
|
||||
"""异步判断缓存文件是否存在。"""
|
||||
return await (AsyncPath(self.base) / region / key).exists()
|
||||
|
||||
async def get(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Optional[Any]:
|
||||
"""异步读取二进制缓存文件。"""
|
||||
cache_path = AsyncPath(self.base) / region / key
|
||||
if not await cache_path.exists():
|
||||
return None
|
||||
async with aiofiles.open(cache_path, "rb") as file_handle:
|
||||
return await file_handle.read()
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> None:
|
||||
"""异步删除缓存文件或缓存子目录。"""
|
||||
cache_path = AsyncPath(self.base) / region / key
|
||||
if await cache_path.is_file():
|
||||
await cache_path.unlink()
|
||||
elif await cache_path.exists():
|
||||
await aioshutil.rmtree(cache_path, ignore_errors=True)
|
||||
|
||||
async def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None:
|
||||
"""异步清空指定缓存区或缓存根目录。"""
|
||||
cache_path = AsyncPath(self.base) / region if region else AsyncPath(self.base)
|
||||
if not await cache_path.exists():
|
||||
return
|
||||
async for item in cache_path.iterdir():
|
||||
if await item.is_file():
|
||||
await item.unlink()
|
||||
else:
|
||||
await aioshutil.rmtree(item, ignore_errors=True)
|
||||
|
||||
async def items(
|
||||
self,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> AsyncGenerator[Tuple[str, Any], None]:
|
||||
"""异步按相对键遍历指定缓存区中的二进制文件。"""
|
||||
cache_path = AsyncPath(self.base) / region
|
||||
if not await cache_path.exists():
|
||||
return
|
||||
async for item in cache_path.rglob("*"):
|
||||
if await item.is_file():
|
||||
key = Path(str(item)).relative_to(Path(str(cache_path))).as_posix()
|
||||
async with aiofiles.open(item, "rb") as file_handle:
|
||||
yield key, await file_handle.read()
|
||||
|
||||
async def close(self) -> None:
|
||||
"""异步文件缓存没有需要关闭的持久连接。"""
|
||||
|
||||
|
||||
def configure_platform_cache() -> None:
|
||||
"""把配置感知的 Redis 与文件适配器注册到平台缓存工厂。"""
|
||||
configure_cache_factories(
|
||||
backend_type_provider=lambda: settings.CACHE_BACKEND_TYPE,
|
||||
redis_factory=lambda ttl: RedisBackend(ttl=ttl),
|
||||
async_redis_factory=lambda ttl: AsyncRedisBackend(ttl=ttl),
|
||||
file_factory=lambda base: FileBackend(base=base or settings.TEMP_PATH),
|
||||
async_file_factory=lambda base: AsyncFileBackend(
|
||||
base=base or settings.TEMP_PATH
|
||||
),
|
||||
file_ttl_provider=lambda: settings.TEMP_FILE_DAYS * 24 * 3600,
|
||||
)
|
||||
Vendored
+621
@@ -0,0 +1,621 @@
|
||||
import asyncio
|
||||
import json
|
||||
import pickle
|
||||
import threading
|
||||
from typing import Any, Optional, Generator, Tuple, AsyncGenerator, Union
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
import redis
|
||||
from redis.asyncio import BlockingConnectionPool as AsyncBlockingConnectionPool
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from app.runtime.config import settings
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.reload import ConfigReloadMixin
|
||||
from app.foundation.singleton import Singleton
|
||||
|
||||
# 类型缓存集合，针对非容器简单类型
|
||||
_complex_serializable_types = set()
|
||||
_simple_serializable_types = set()
|
||||
|
||||
# 默认连接参数
|
||||
_socket_timeout = 30
|
||||
_socket_connect_timeout = 5
|
||||
_health_check_interval = 60
|
||||
|
||||
|
||||
def serialize(value: Any) -> bytes:
|
||||
"""
|
||||
将值序列化为二进制数据，根据序列化方式标识格式
|
||||
"""
|
||||
|
||||
def _is_container_type(t):
|
||||
"""
|
||||
判断是否为容器类型
|
||||
"""
|
||||
return t in (list, dict, tuple, set)
|
||||
|
||||
vt = type(value)
|
||||
# 针对非容器类型使用缓存策略
|
||||
if not _is_container_type(vt):
|
||||
# 如果已知需要复杂序列化
|
||||
if vt in _complex_serializable_types:
|
||||
return b"PICKLE" + b"\x00" + pickle.dumps(value)
|
||||
# 如果已知可以简单序列化
|
||||
if vt in _simple_serializable_types:
|
||||
json_data = json.dumps(value).encode("utf-8")
|
||||
return b"JSON" + b"\x00" + json_data
|
||||
# 对于未知的非容器类型，尝试简单序列化，如抛出异常，再使用复杂序列化
|
||||
try:
|
||||
json_data = json.dumps(value).encode("utf-8")
|
||||
_simple_serializable_types.add(vt)
|
||||
return b"JSON" + b"\x00" + json_data
|
||||
except TypeError:
|
||||
_complex_serializable_types.add(vt)
|
||||
return b"PICKLE" + b"\x00" + pickle.dumps(value)
|
||||
else:
|
||||
# 针对容器类型，每次尝试简单序列化，不使用缓存
|
||||
try:
|
||||
json_data = json.dumps(value).encode("utf-8")
|
||||
return b"JSON" + b"\x00" + json_data
|
||||
except TypeError:
|
||||
return b"PICKLE" + b"\x00" + pickle.dumps(value)
|
||||
|
||||
|
||||
def deserialize(value: bytes) -> Any:
|
||||
"""
|
||||
将二进制数据反序列化为原始值，根据格式标识区分序列化方式
|
||||
"""
|
||||
format_marker, data = value.split(b"\x00", 1)
|
||||
if format_marker == b"JSON":
|
||||
return json.loads(data.decode("utf-8"))
|
||||
elif format_marker == b"PICKLE":
|
||||
return pickle.loads(data)
|
||||
else:
|
||||
raise ValueError("Unknown serialization format")
|
||||
|
||||
|
||||
class RedisHelper(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""
|
||||
Redis连接和操作助手类，单例模式
|
||||
|
||||
特性：
|
||||
- 管理Redis连接池和客户端
|
||||
- 提供序列化和反序列化功能
|
||||
- 支持内存限制和淘汰策略设置
|
||||
- 提供键名生成和区域管理功能
|
||||
"""
|
||||
CONFIG_WATCH = {
|
||||
"CACHE_BACKEND_TYPE",
|
||||
"CACHE_BACKEND_URL",
|
||||
"CACHE_REDIS_MAXMEMORY",
|
||||
"CACHE_REDIS_MAX_CONNECTIONS",
|
||||
"CACHE_REDIS_POOL_TIMEOUT",
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化Redis助手实例
|
||||
"""
|
||||
self.redis_url = settings.CACHE_BACKEND_URL
|
||||
self.client = None
|
||||
self._connect_lock = threading.RLock()
|
||||
|
||||
def _connect(self):
|
||||
"""
|
||||
建立Redis连接
|
||||
"""
|
||||
if self.client is not None:
|
||||
return
|
||||
client = None
|
||||
try:
|
||||
with self._connect_lock:
|
||||
if self.client is not None:
|
||||
return
|
||||
self.redis_url = settings.CACHE_BACKEND_URL
|
||||
connection_pool = redis.BlockingConnectionPool.from_url(
|
||||
self.redis_url,
|
||||
decode_responses=False,
|
||||
socket_timeout=_socket_timeout,
|
||||
socket_connect_timeout=_socket_connect_timeout,
|
||||
health_check_interval=_health_check_interval,
|
||||
max_connections=settings.CACHE_REDIS_MAX_CONNECTIONS,
|
||||
timeout=settings.CACHE_REDIS_POOL_TIMEOUT,
|
||||
)
|
||||
client = redis.Redis(connection_pool=connection_pool)
|
||||
# 测试连接，确保Redis可用
|
||||
client.ping()
|
||||
self.client = client
|
||||
logger.info(f"Successfully connected to Redis：{self.redis_url}")
|
||||
self.set_memory_limit()
|
||||
except Exception as e:
|
||||
if client:
|
||||
client.close()
|
||||
logger.error(f"Failed to connect to Redis: {e}")
|
||||
self.client = None
|
||||
raise RuntimeError("Redis connection failed") from e
|
||||
|
||||
def on_config_changed(self):
|
||||
"""缓存配置变化后重建同步 Redis 连接。"""
|
||||
with self._connect_lock:
|
||||
self.redis_url = settings.CACHE_BACKEND_URL
|
||||
self.close()
|
||||
self._connect()
|
||||
|
||||
def get_reload_name(self):
|
||||
"""返回同步 Redis 配置重载名称。"""
|
||||
return "Redis"
|
||||
|
||||
def set_memory_limit(self, policy: Optional[str] = "allkeys-lru"):
|
||||
"""
|
||||
动态设置Redis最大内存和内存淘汰策略
|
||||
|
||||
:param policy: 淘汰策略（如'allkeys-lru'）
|
||||
"""
|
||||
try:
|
||||
# 如果有显式值，则直接使用，为0时说明不限制，如果未配置，开启BIG_MEMORY_MODE时为"1024mb"，未开启时为"256mb"
|
||||
maxmemory = settings.CACHE_REDIS_MAXMEMORY or ("1024mb" if settings.BIG_MEMORY_MODE else "256mb")
|
||||
self.client.config_set("maxmemory", maxmemory)
|
||||
self.client.config_set("maxmemory-policy", policy)
|
||||
logger.debug(f"Redis maxmemory set to {maxmemory}, policy: {policy}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set Redis maxmemory or policy: {e}")
|
||||
|
||||
@staticmethod
|
||||
def __get_region(region: Optional[str] = None):
|
||||
"""
|
||||
获取缓存的区
|
||||
"""
|
||||
return f"region:{region}" if region else "region:DEFAULT"
|
||||
|
||||
def __make_redis_key(self, region: str, key: str) -> str:
|
||||
"""
|
||||
获取缓存Key
|
||||
"""
|
||||
# 使用region作为缓存键的一部分
|
||||
region = self.__get_region(region)
|
||||
return f"{region}:key:{quote(key)}"
|
||||
|
||||
@staticmethod
|
||||
def __get_original_key(redis_key: Union[str, bytes]) -> str:
|
||||
"""
|
||||
从Redis键中提取原始key
|
||||
"""
|
||||
try:
|
||||
if isinstance(redis_key, bytes):
|
||||
redis_key = redis_key.decode("utf-8")
|
||||
parts = redis_key.split(":key:")
|
||||
return unquote(parts[-1])
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to parse redis key: {redis_key}, error: {e}")
|
||||
return redis_key
|
||||
|
||||
def set(self, key: str, value: Any, ttl: Optional[int] = None,
|
||||
region: Optional[str] = "DEFAULT", **kwargs) -> None:
|
||||
"""
|
||||
设置缓存
|
||||
|
||||
:param key: 缓存的键
|
||||
:param value: 缓存的值
|
||||
:param ttl: 缓存的存活时间，单位秒
|
||||
:param region: 缓存的区
|
||||
:param kwargs: 其他参数
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
# 对值进行序列化
|
||||
serialized_value = serialize(value)
|
||||
kwargs.pop("maxsize", None)
|
||||
self.client.set(redis_key, serialized_value, ex=ttl, **kwargs)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set key: {key} in region: {region}, error: {e}")
|
||||
|
||||
def exists(self, key: str, region: Optional[str] = "DEFAULT") -> bool:
|
||||
"""
|
||||
判断缓存键是否存在
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
:return: 存在返回True，否则返回False
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
return self.client.exists(redis_key) == 1
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to exists key: {key} region: {region}, error: {e}")
|
||||
return False
|
||||
|
||||
def get(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]:
|
||||
"""
|
||||
获取缓存的值
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
:return: 返回缓存的值，如果缓存不存在返回None
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
value = self.client.get(redis_key)
|
||||
if value is not None:
|
||||
return deserialize(value)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get key: {key} in region: {region}, error: {e}")
|
||||
return None
|
||||
|
||||
def pop(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]:
|
||||
"""原子读取并删除缓存值。"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
value = self.client.getdel(redis_key)
|
||||
return deserialize(value) if value is not None else None
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to pop key: {key} in region: {region}, error: {e}"
|
||||
)
|
||||
return None
|
||||
|
||||
def delete(self, key: str, region: Optional[str] = "DEFAULT") -> None:
|
||||
"""
|
||||
删除缓存
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
self.client.delete(redis_key)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete key: {key} in region: {region}, error: {e}")
|
||||
|
||||
def clear(self, region: Optional[str] = None) -> None:
|
||||
"""
|
||||
清除指定区域的缓存或全部缓存
|
||||
|
||||
:param region: 缓存的区
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
if region:
|
||||
cache_region = self.__get_region(region)
|
||||
redis_key = f"{cache_region}:key:*"
|
||||
with self.client.pipeline() as pipe:
|
||||
for key in self.client.scan_iter(redis_key):
|
||||
pipe.delete(key)
|
||||
pipe.execute()
|
||||
logger.debug(f"Cleared Redis cache for region: {region}")
|
||||
else:
|
||||
self.client.flushdb()
|
||||
logger.info("All Redis cache Cleared！")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clear cache, region: {region}, error: {e}")
|
||||
|
||||
def items(self, region: Optional[str] = None) -> Generator[Tuple[str, Any], None, None]:
|
||||
"""
|
||||
获取指定区域的所有缓存键值对
|
||||
|
||||
:param region: 缓存的区
|
||||
:return: 返回键值对生成器
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
if region:
|
||||
cache_region = self.__get_region(region)
|
||||
redis_key = f"{cache_region}:key:*"
|
||||
for key in self.client.scan_iter(redis_key):
|
||||
value = self.client.get(key)
|
||||
if value is not None:
|
||||
yield self.__get_original_key(key), deserialize(value)
|
||||
else:
|
||||
for key in self.client.scan_iter("*"):
|
||||
value = self.client.get(key)
|
||||
if value is not None:
|
||||
yield self.__get_original_key(key), deserialize(value)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get items from Redis, region: {region}, error: {e}")
|
||||
|
||||
def test(self) -> bool:
|
||||
"""
|
||||
测试Redis连接性
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Redis connection test failed: {e}")
|
||||
return False
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
关闭Redis客户端的连接池
|
||||
"""
|
||||
with self._connect_lock:
|
||||
if self.client:
|
||||
self.client.close()
|
||||
self.client = None
|
||||
logger.debug("Redis connection closed")
|
||||
|
||||
|
||||
class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""
|
||||
异步Redis连接和操作助手类，单例模式
|
||||
|
||||
特性：
|
||||
- 管理异步Redis连接池和客户端
|
||||
- 提供序列化和反序列化功能
|
||||
- 支持内存限制和淘汰策略设置
|
||||
- 提供键名生成和区域管理功能
|
||||
- 所有操作都是异步的
|
||||
"""
|
||||
CONFIG_WATCH = {
|
||||
"CACHE_BACKEND_TYPE",
|
||||
"CACHE_BACKEND_URL",
|
||||
"CACHE_REDIS_MAXMEMORY",
|
||||
"CACHE_REDIS_MAX_CONNECTIONS",
|
||||
"CACHE_REDIS_POOL_TIMEOUT",
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化异步Redis助手实例
|
||||
"""
|
||||
self.redis_url = settings.CACHE_BACKEND_URL
|
||||
self.client: Optional[Redis] = None
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._connect_lock: Optional[asyncio.Lock] = None
|
||||
self._connect_lock_loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
|
||||
def _get_connect_lock(self, current_loop: asyncio.AbstractEventLoop) -> asyncio.Lock:
|
||||
"""
|
||||
获取当前事件循环对应的异步连接锁
|
||||
"""
|
||||
if self._connect_lock is None or self._connect_lock_loop is not current_loop:
|
||||
self._connect_lock = asyncio.Lock()
|
||||
self._connect_lock_loop = current_loop
|
||||
return self._connect_lock
|
||||
|
||||
async def _connect(self):
|
||||
"""
|
||||
建立异步Redis连接
|
||||
"""
|
||||
current_loop = asyncio.get_running_loop()
|
||||
connect_lock = self._get_connect_lock(current_loop)
|
||||
client = None
|
||||
try:
|
||||
async with connect_lock:
|
||||
# 检测事件循环是否发生变化，如果变化则重新连接
|
||||
if self.client is not None and self._loop is not current_loop:
|
||||
logger.debug("Event loop changed, reconnecting Redis (async)")
|
||||
await self._close_client()
|
||||
if self.client is not None:
|
||||
return
|
||||
self.redis_url = settings.CACHE_BACKEND_URL
|
||||
connection_pool = AsyncBlockingConnectionPool.from_url(
|
||||
self.redis_url,
|
||||
decode_responses=False,
|
||||
socket_timeout=_socket_timeout,
|
||||
socket_connect_timeout=_socket_connect_timeout,
|
||||
health_check_interval=_health_check_interval,
|
||||
max_connections=settings.CACHE_REDIS_MAX_CONNECTIONS,
|
||||
timeout=settings.CACHE_REDIS_POOL_TIMEOUT,
|
||||
)
|
||||
client = Redis(connection_pool=connection_pool)
|
||||
self._loop = current_loop
|
||||
# 测试连接，确保Redis可用
|
||||
await client.ping()
|
||||
self.client = client
|
||||
logger.info(f"Successfully connected to Redis (async)：{self.redis_url}")
|
||||
await self.set_memory_limit()
|
||||
except Exception as e:
|
||||
if client:
|
||||
await client.close()
|
||||
logger.error(f"Failed to connect to Redis (async): {e}")
|
||||
self.client = None
|
||||
self._loop = None
|
||||
raise RuntimeError("Redis async connection failed") from e
|
||||
|
||||
async def _close_client(self):
|
||||
"""
|
||||
关闭当前Redis客户端连接
|
||||
"""
|
||||
if self.client:
|
||||
try:
|
||||
await self.client.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.client = None
|
||||
self._loop = None
|
||||
|
||||
async def on_config_changed(self):
|
||||
"""缓存配置变化后异步重建 Redis 连接。"""
|
||||
self.redis_url = settings.CACHE_BACKEND_URL
|
||||
await self._close_client()
|
||||
await self._connect()
|
||||
|
||||
def get_reload_name(self):
|
||||
"""返回异步 Redis 配置重载名称。"""
|
||||
return "Redis (async)"
|
||||
|
||||
async def set_memory_limit(self, policy: Optional[str] = "allkeys-lru"):
|
||||
"""
|
||||
动态设置Redis最大内存和内存淘汰策略
|
||||
|
||||
:param policy: 淘汰策略（如'allkeys-lru'）
|
||||
"""
|
||||
try:
|
||||
# 如果有显式值，则直接使用，为0时说明不限制，如果未配置，开启BIG_MEMORY_MODE时为"1024mb"，未开启时为"256mb"
|
||||
maxmemory = settings.CACHE_REDIS_MAXMEMORY or ("1024mb" if settings.BIG_MEMORY_MODE else "256mb")
|
||||
await self.client.config_set("maxmemory", maxmemory)
|
||||
await self.client.config_set("maxmemory-policy", policy)
|
||||
logger.debug(f"Redis maxmemory set to {maxmemory}, policy: {policy} (async)")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set Redis maxmemory or policy (async): {e}")
|
||||
|
||||
@staticmethod
|
||||
def __get_region(region: Optional[str] = "DEFAULT"):
|
||||
"""
|
||||
获取缓存的区
|
||||
"""
|
||||
return f"region:{region}" if region else "region:DEFAULT"
|
||||
|
||||
def __make_redis_key(self, region: str, key: str) -> str:
|
||||
"""
|
||||
获取缓存Key
|
||||
"""
|
||||
# 使用region作为缓存键的一部分
|
||||
region = self.__get_region(region)
|
||||
return f"{region}:key:{quote(key)}"
|
||||
|
||||
@staticmethod
|
||||
def __get_original_key(redis_key: Union[str, bytes]) -> str:
|
||||
"""
|
||||
从Redis键中提取原始key
|
||||
"""
|
||||
try:
|
||||
if isinstance(redis_key, bytes):
|
||||
redis_key = redis_key.decode("utf-8")
|
||||
parts = redis_key.split(":key:")
|
||||
return unquote(parts[-1])
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to parse redis key: {redis_key}, error: {e}")
|
||||
return redis_key
|
||||
|
||||
async def set(self, key: str, value: Any, ttl: Optional[int] = None,
|
||||
region: Optional[str] = "DEFAULT", **kwargs) -> None:
|
||||
"""
|
||||
异步设置缓存
|
||||
|
||||
:param key: 缓存的键
|
||||
:param value: 缓存的值
|
||||
:param ttl: 缓存的存活时间，单位秒
|
||||
:param region: 缓存的区
|
||||
:param kwargs: 其他参数
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
# 对值进行序列化
|
||||
serialized_value = serialize(value)
|
||||
kwargs.pop("maxsize", None)
|
||||
await self.client.set(redis_key, serialized_value, ex=ttl, **kwargs)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set key (async): {key} in region: {region}, error: {e}")
|
||||
|
||||
async def exists(self, key: str, region: Optional[str] = "DEFAULT") -> bool:
|
||||
"""
|
||||
异步判断缓存键是否存在
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
:return: 存在返回True，否则返回False
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
result = await self.client.exists(redis_key)
|
||||
return result == 1
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to exists key (async): {key} region: {region}, error: {e}")
|
||||
return False
|
||||
|
||||
async def get(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]:
|
||||
"""
|
||||
异步获取缓存的值
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
:return: 返回缓存的值，如果缓存不存在返回None
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
value = await self.client.get(redis_key)
|
||||
if value is not None:
|
||||
return deserialize(value)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get key (async): {key} in region: {region}, error: {e}")
|
||||
return None
|
||||
|
||||
async def delete(self, key: str, region: Optional[str] = "DEFAULT") -> None:
|
||||
"""
|
||||
异步删除缓存
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
await self.client.delete(redis_key)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete key (async): {key} in region: {region}, error: {e}")
|
||||
|
||||
async def clear(self, region: Optional[str] = None) -> None:
|
||||
"""
|
||||
异步清除指定区域的缓存或全部缓存
|
||||
|
||||
:param region: 缓存的区
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
if region:
|
||||
cache_region = self.__get_region(region)
|
||||
redis_key = f"{cache_region}:key:*"
|
||||
async with self.client.pipeline() as pipe:
|
||||
async for key in self.client.scan_iter(redis_key):
|
||||
await pipe.delete(key)
|
||||
await pipe.execute()
|
||||
logger.debug(f"Cleared Redis cache for region (async): {region}")
|
||||
else:
|
||||
await self.client.flushdb()
|
||||
logger.info("Cleared all Redis cache (async)")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clear cache (async), region: {region}, error: {e}")
|
||||
|
||||
async def items(self, region: Optional[str] = None) -> AsyncGenerator[Tuple[str, Any], None]:
|
||||
"""
|
||||
获取指定区域的所有缓存键值对
|
||||
|
||||
:param region: 缓存的区
|
||||
:return: 返回键值对生成器
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
if region:
|
||||
cache_region = self.__get_region(region)
|
||||
redis_key = f"{cache_region}:key:*"
|
||||
async for key in self.client.scan_iter(redis_key):
|
||||
value = await self.client.get(key)
|
||||
if value is not None:
|
||||
yield self.__get_original_key(key), deserialize(value)
|
||||
else:
|
||||
async for key in self.client.scan_iter("*"):
|
||||
value = await self.client.get(key)
|
||||
if value is not None:
|
||||
yield self.__get_original_key(key), deserialize(value)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get items from Redis, region: {region}, error: {e}")
|
||||
|
||||
async def test(self) -> bool:
|
||||
"""
|
||||
异步测试Redis连接性
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Redis async connection test failed: {e}")
|
||||
return False
|
||||
|
||||
async def close(self) -> None:
|
||||
"""
|
||||
关闭异步Redis客户端的连接池
|
||||
"""
|
||||
await self._close_client()
|
||||
logger.debug("Redis async connection closed")
|
||||
Reference in New Issue
Block a user