diff --git a/app/chain/search.py b/app/chain/search.py index 5f7f0a87..3286f007 100644 --- a/app/chain/search.py +++ b/app/chain/search.py @@ -202,16 +202,15 @@ class SearchChain(ChainBase): # 过滤完成 progress.update(value=50, text=f'过滤完成，剩余 {len(torrents)} 个资源', key=ProgressKey.Search) - # 开始匹配 - _match_torrents = [] # 总数 _total = len(torrents) # 已处理数 _count = 0 + # 开始匹配 + _match_torrents = [] torrenthelper = TorrentHelper() - - if mediainfo: + try: # 英文标题应该在别名/原标题中，不需要再匹配 logger.info(f"开始匹配结果 标题：{mediainfo.title}，原标题：{mediainfo.original_title}，别名：{mediainfo.names}") progress.update(value=51, text=f'开始匹配，总 {_total} 个资源 ...', key=ProgressKey.Search) @@ -256,16 +255,16 @@ class SearchChain(ChainBase): progress.update(value=97, text=f'匹配完成，共匹配到 {len(_match_torrents)} 个资源', key=ProgressKey.Search) - else: - _match_torrents = [(t, MetaInfo(title=t.title, subtitle=t.description)) for t in torrents] - # 去掉mediainfo中多余的数据 - mediainfo.clear() - - # 组装上下文 - contexts = [Context(torrent_info=t[0], - media_info=mediainfo, - meta_info=t[1]) for t in _match_torrents] + # 去掉mediainfo中多余的数据 + mediainfo.clear() + # 组装上下文 + contexts = [Context(torrent_info=t[0], + media_info=mediainfo, + meta_info=t[1]) for t in _match_torrents] + finally: + torrents.clear() + _match_torrents.clear() # 排序 progress.update(value=99, diff --git a/app/chain/site.py b/app/chain/site.py index 12bddd31..4ca13d2a 100644 --- a/app/chain/site.py +++ b/app/chain/site.py @@ -92,10 +92,9 @@ class SiteChain(ChainBase): """ 刷新所有站点的用户数据 """ - sites = SitesHelper().get_indexers() any_site_updated = False result = {} - for site in sites: + for site in SitesHelper().get_indexers(): if global_vars.is_system_stopped: return None if site.get("is_active"): diff --git a/app/chain/subscribe.py b/app/chain/subscribe.py index 48f4a63b..9ddff28b 100644 --- a/app/chain/subscribe.py +++ b/app/chain/subscribe.py @@ -286,147 +286,152 @@ class SubscribeChain(ChainBase): subscribes = [subscribe] if subscribe else [] else: subscribes = subscribeoper.list(self.get_states_for_search(state)) - # 遍历订阅 - for subscribe in subscribes: - if global_vars.is_system_stopped: - break - mediakey = subscribe.tmdbid or subscribe.doubanid - custom_word_list = subscribe.custom_words.split("\n") if subscribe.custom_words else None - # 校验当前时间减订阅创建时间是否大于1分钟，否则跳过先，留出编辑订阅的时间 - if subscribe.date: - now = datetime.now() - subscribe_time = datetime.strptime(subscribe.date, '%Y-%m-%d %H:%M:%S') - if (now - subscribe_time).total_seconds() < 60: - logger.debug(f"订阅标题：{subscribe.name} 新增小于1分钟，暂不搜索...") - continue - # 随机休眠1-5分钟 - if not sid and state in ['R', 'P']: - sleep_time = random.randint(60, 300) - logger.info(f'订阅搜索随机休眠 {sleep_time} 秒 ...') - time.sleep(sleep_time) - try: - logger.info(f'开始搜索订阅，标题：{subscribe.name} ...') - # 生成元数据 - meta = MetaInfo(subscribe.name) - meta.year = subscribe.year - meta.begin_season = subscribe.season or None + + try: + # 遍历订阅 + for subscribe in subscribes: + if global_vars.is_system_stopped: + break + mediakey = subscribe.tmdbid or subscribe.doubanid + custom_word_list = subscribe.custom_words.split("\n") if subscribe.custom_words else None + # 校验当前时间减订阅创建时间是否大于1分钟，否则跳过先，留出编辑订阅的时间 + if subscribe.date: + now = datetime.now() + subscribe_time = datetime.strptime(subscribe.date, '%Y-%m-%d %H:%M:%S') + if (now - subscribe_time).total_seconds() < 60: + logger.debug(f"订阅标题：{subscribe.name} 新增小于1分钟，暂不搜索...") + continue + # 随机休眠1-5分钟 + if not sid and state in ['R', 'P']: + sleep_time = random.randint(60, 300) + logger.info(f'订阅搜索随机休眠 {sleep_time} 秒 ...') + time.sleep(sleep_time) try: - meta.type = MediaType(subscribe.type) - except ValueError: - logger.error(f'订阅 {subscribe.name} 类型错误：{subscribe.type}') - continue - # 识别媒体信息 - mediainfo: MediaInfo = self.recognize_media(meta=meta, mtype=meta.type, - tmdbid=subscribe.tmdbid, - doubanid=subscribe.doubanid, - episode_group=subscribe.episode_group, - cache=False) - if not mediainfo: - logger.warn( - f'未识别到媒体信息，标题：{subscribe.name}，tmdbid：{subscribe.tmdbid}，doubanid：{subscribe.doubanid}') - continue + logger.info(f'开始搜索订阅，标题：{subscribe.name} ...') + # 生成元数据 + meta = MetaInfo(subscribe.name) + meta.year = subscribe.year + meta.begin_season = subscribe.season or None + try: + meta.type = MediaType(subscribe.type) + except ValueError: + logger.error(f'订阅 {subscribe.name} 类型错误：{subscribe.type}') + continue + # 识别媒体信息 + mediainfo: MediaInfo = self.recognize_media(meta=meta, mtype=meta.type, + tmdbid=subscribe.tmdbid, + doubanid=subscribe.doubanid, + episode_group=subscribe.episode_group, + cache=False) + if not mediainfo: + logger.warn( + f'未识别到媒体信息，标题：{subscribe.name}，tmdbid：{subscribe.tmdbid}，doubanid：{subscribe.doubanid}') + continue - # 如果媒体已存在或已下载完毕，跳过当前订阅处理 - exist_flag, no_exists = self.check_and_handle_existing_media(subscribe=subscribe, - meta=meta, - mediainfo=mediainfo, - mediakey=mediakey) - if exist_flag: - continue + # 如果媒体已存在或已下载完毕，跳过当前订阅处理 + exist_flag, no_exists = self.check_and_handle_existing_media(subscribe=subscribe, + meta=meta, + mediainfo=mediainfo, + mediakey=mediakey) + if exist_flag: + continue - # 站点范围 - sites = self.get_sub_sites(subscribe) + # 站点范围 + sites = self.get_sub_sites(subscribe) - # 优先级过滤规则 - if subscribe.best_version: - rule_groups = subscribe.filter_groups \ - or SystemConfigOper().get(SystemConfigKey.BestVersionFilterRuleGroups) or [] - else: - rule_groups = subscribe.filter_groups \ - or SystemConfigOper().get(SystemConfigKey.SubscribeFilterRuleGroups) or [] - - # 搜索，同时电视剧会过滤掉不需要的剧集 - contexts = SearchChain().process(mediainfo=mediainfo, - keyword=subscribe.keyword, - no_exists=no_exists, - sites=sites, - rule_groups=rule_groups, - area="imdbid" if subscribe.search_imdbid else "title", - custom_words=custom_word_list, - filter_params=self.get_params(subscribe)) - if not contexts: - logger.warn(f'订阅 {subscribe.keyword or subscribe.name} 未搜索到资源') - self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, - mediainfo=mediainfo, lefts=no_exists) - continue - - # 过滤搜索结果 - matched_contexts = [] - for context in contexts: - if global_vars.is_system_stopped: - break - torrent_meta = context.meta_info - torrent_info = context.torrent_info - torrent_mediainfo = context.media_info - - # 洗版 + # 优先级过滤规则 if subscribe.best_version: - # 洗版时，非整季不要 - if torrent_mediainfo.type == MediaType.TV: - if torrent_meta.episode_list: - logger.info(f'{subscribe.name} 正在洗版，{torrent_info.title} 不是整季') + rule_groups = subscribe.filter_groups \ + or SystemConfigOper().get(SystemConfigKey.BestVersionFilterRuleGroups) or [] + else: + rule_groups = subscribe.filter_groups \ + or SystemConfigOper().get(SystemConfigKey.SubscribeFilterRuleGroups) or [] + + # 搜索，同时电视剧会过滤掉不需要的剧集 + contexts = SearchChain().process(mediainfo=mediainfo, + keyword=subscribe.keyword, + no_exists=no_exists, + sites=sites, + rule_groups=rule_groups, + area="imdbid" if subscribe.search_imdbid else "title", + custom_words=custom_word_list, + filter_params=self.get_params(subscribe)) + if not contexts: + logger.warn(f'订阅 {subscribe.keyword or subscribe.name} 未搜索到资源') + self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, + mediainfo=mediainfo, lefts=no_exists) + continue + + # 过滤搜索结果 + matched_contexts = [] + for context in contexts: + if global_vars.is_system_stopped: + break + torrent_meta = context.meta_info + torrent_info = context.torrent_info + torrent_mediainfo = context.media_info + + # 洗版 + if subscribe.best_version: + # 洗版时，非整季不要 + if torrent_mediainfo.type == MediaType.TV: + if torrent_meta.episode_list: + logger.info(f'{subscribe.name} 正在洗版，{torrent_info.title} 不是整季') + continue + # 洗版时，优先级小于等于已下载优先级的不要 + if subscribe.current_priority \ + and torrent_info.pri_order <= subscribe.current_priority: + logger.info( + f'{subscribe.name} 正在洗版，{torrent_info.title} 优先级低于或等于已下载优先级') continue - # 洗版时，优先级小于等于已下载优先级的不要 - if subscribe.current_priority \ - and torrent_info.pri_order <= subscribe.current_priority: - logger.info( - f'{subscribe.name} 正在洗版，{torrent_info.title} 优先级低于或等于已下载优先级') - continue - # 更新订阅自定义属性 - if subscribe.media_category: - torrent_mediainfo.category = subscribe.media_category - if subscribe.episode_group: - torrent_mediainfo.episode_group = subscribe.episode_group - matched_contexts.append(context) + # 更新订阅自定义属性 + if subscribe.media_category: + torrent_mediainfo.category = subscribe.media_category + if subscribe.episode_group: + torrent_mediainfo.episode_group = subscribe.episode_group + matched_contexts.append(context) - if not matched_contexts: - logger.warn(f'订阅 {subscribe.name} 没有符合过滤条件的资源') - self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, - mediainfo=mediainfo, lefts=no_exists) - continue + if not matched_contexts: + logger.warn(f'订阅 {subscribe.name} 没有符合过滤条件的资源') + self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, + mediainfo=mediainfo, lefts=no_exists) + continue - # 自动下载 - downloads, lefts = DownloadChain().batch_download( - contexts=matched_contexts, - no_exists=no_exists, - username=subscribe.username, - save_path=subscribe.save_path, - downloader=subscribe.downloader, - source=self.get_subscribe_source_keyword(subscribe) - ) + # 自动下载 + downloads, lefts = DownloadChain().batch_download( + contexts=matched_contexts, + no_exists=no_exists, + username=subscribe.username, + save_path=subscribe.save_path, + downloader=subscribe.downloader, + source=self.get_subscribe_source_keyword(subscribe) + ) - # 同步外部修改，更新订阅信息 - subscribe = subscribeoper.get(subscribe.id) + # 同步外部修改，更新订阅信息 + subscribe = subscribeoper.get(subscribe.id) - # 判断是否应完成订阅 - if subscribe: - self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, mediainfo=mediainfo, - downloads=downloads, lefts=lefts) - finally: - # 如果状态为N则更新为R - if subscribe and subscribe.state == 'N': - subscribeoper.update(subscribe.id, {'state': 'R'}) + # 判断是否应完成订阅 + if subscribe: + self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, mediainfo=mediainfo, + downloads=downloads, lefts=lefts) + finally: + # 如果状态为N则更新为R + if subscribe and subscribe.state == 'N': + subscribeoper.update(subscribe.id, {'state': 'R'}) - # 手动触发时发送系统消息 - if manual: - if subscribes: - if sid: - self.messagehelper.put(f'{subscribes[0].name} 搜索完成！', title="订阅搜索", role="system") + # 手动触发时发送系统消息 + if manual: + if subscribes: + if sid: + self.messagehelper.put(f'{subscribes[0].name} 搜索完成！', title="订阅搜索", role="system") + else: + self.messagehelper.put('所有订阅搜索完成！', title="订阅搜索", role="system") else: - self.messagehelper.put('所有订阅搜索完成！', title="订阅搜索", role="system") - else: - self.messagehelper.put('没有找到订阅！', title="订阅搜索", role="system") - logger.debug(f"search Lock released at {datetime.now()}") + self.messagehelper.put('没有找到订阅！', title="订阅搜索", role="system") + + logger.debug(f"search Lock released at {datetime.now()}") + finally: + subscribes.clear() def update_subscribe_priority(self, subscribe: Subscribe, meta: MetaBase, mediainfo: MediaInfo, downloads: Optional[List[Context]]): @@ -527,13 +532,9 @@ class SubscribeChain(ChainBase): 获取订阅中涉及的所有站点清单（节约资源） :return: 返回[]代表所有站点命中，返回None代表没有订阅 """ - # 查询所有订阅 - subscribes = SubscribeOper().list(self.get_states_for_search('R')) - if not subscribes: - return None ret_sites = [] # 刷新订阅选中的Rss站点 - for subscribe in subscribes: + for subscribe in SubscribeOper().list(self.get_states_for_search('R')): # 刷新选中的站点 ret_sites.extend(self.get_sub_sites(subscribe)) # 去重 @@ -551,258 +552,261 @@ class SubscribeChain(ChainBase): return with self._rlock: - logger.debug(f"match lock acquired at {datetime.now()}") - # 所有订阅 - subscribes = SubscribeOper().list(self.get_states_for_search('R')) + try: + logger.debug(f"match lock acquired at {datetime.now()}") - # 预识别所有未识别的种子 - processed_torrents: Dict[str, List[Context]] = {} - for domain, contexts in torrents.items(): - if global_vars.is_system_stopped: - break - processed_torrents[domain] = [] - for context in contexts: + # 预识别所有未识别的种子 + processed_torrents: Dict[str, List[Context]] = {} + for domain, contexts in torrents.items(): if global_vars.is_system_stopped: break - # 如果种子未识别，尝试识别 - if not context.media_info or (not context.media_info.tmdb_id - and not context.media_info.douban_id): - re_mediainfo = self.recognize_media(meta=context.meta_info) - if re_mediainfo: - # 清理多余信息 - re_mediainfo.clear() - # 更新种子缓存 - context.media_info = re_mediainfo - # 添加已预处理 - processed_torrents[domain].append(context) - - # 遍历订阅 - for subscribe in subscribes: - if global_vars.is_system_stopped: - break - logger.info(f'开始匹配订阅，标题：{subscribe.name} ...') - mediakey = subscribe.tmdbid or subscribe.doubanid - # 生成元数据 - meta = MetaInfo(subscribe.name) - meta.year = subscribe.year - meta.begin_season = subscribe.season or None - try: - meta.type = MediaType(subscribe.type) - except ValueError: - logger.error(f'订阅 {subscribe.name} 类型错误：{subscribe.type}') - continue - # 订阅的站点域名列表 - domains = [] - if subscribe.sites: - domains = SiteOper().get_domains_by_ids(subscribe.sites) - # 识别媒体信息 - mediainfo: MediaInfo = self.recognize_media(meta=meta, mtype=meta.type, - tmdbid=subscribe.tmdbid, - doubanid=subscribe.doubanid, - episode_group=subscribe.episode_group, - cache=False) - if not mediainfo: - logger.warn( - f'未识别到媒体信息，标题：{subscribe.name}，tmdbid：{subscribe.tmdbid}，doubanid：{subscribe.doubanid}') - continue - - # 如果媒体已存在或已下载完毕，跳过当前订阅处理 - exist_flag, no_exists = self.check_and_handle_existing_media(subscribe=subscribe, meta=meta, - mediainfo=mediainfo, - mediakey=mediakey) - if exist_flag: - continue - - # 清理多余信息 - mediainfo.clear() - - # 订阅识别词 - if subscribe.custom_words: - custom_words_list = subscribe.custom_words.split("\n") - else: - custom_words_list = None - - # 遍历预识别后的种子 - _match_context = [] - torrenthelper = TorrentHelper() - systemconfig = SystemConfigOper() - wordsmatcher = WordsMatcher() - for domain, contexts in processed_torrents.items(): - if global_vars.is_system_stopped: - break - if domains and domain not in domains: - continue - logger.debug(f'开始匹配站点：{domain}，共缓存了 {len(contexts)} 个种子...') + processed_torrents[domain] = [] for context in contexts: if global_vars.is_system_stopped: break - # 提取信息 - _context = copy.copy(context) - torrent_meta = _context.meta_info - torrent_mediainfo = _context.media_info - torrent_info = _context.torrent_info - - # 不在订阅站点范围的不处理 - sub_sites = self.get_sub_sites(subscribe) - if sub_sites and torrent_info.site not in sub_sites: - logger.debug(f"{torrent_info.site_name} - {torrent_info.title} 不符合订阅站点要求") - continue - - # 有自定义识别词时，需要判断是否需要重新识别 - if custom_words_list: - # 使用org_string，应用一次后理论上不能再次应用 - _, apply_words = wordsmatcher.prepare(torrent_meta.org_string, - custom_words=custom_words_list) - if apply_words: - logger.info( - f'{torrent_info.site_name} - {torrent_info.title} 因订阅存在自定义识别词，重新识别元数据...') - # 重新识别元数据 - torrent_meta = MetaInfo(title=torrent_info.title, subtitle=torrent_info.description, - custom_words=custom_words_list) - # 更新元数据缓存 - _context.meta_info = torrent_meta - # 重新识别媒体信息 - torrent_mediainfo = self.recognize_media(meta=torrent_meta, - episode_group=subscribe.episode_group) - if torrent_mediainfo: - # 清理多余信息 - torrent_mediainfo.clear() - # 更新种子缓存 - _context.media_info = torrent_mediainfo - - # 如果仍然没有识别到媒体信息，尝试标题匹配 - if not torrent_mediainfo or (not torrent_mediainfo.tmdb_id and not torrent_mediainfo.douban_id): - logger.info( - f'{torrent_info.site_name} - {torrent_info.title} 重新识别失败，尝试通过标题匹配...') - if torrenthelper.match_torrent(mediainfo=mediainfo, - torrent_meta=torrent_meta, - torrent=torrent_info): - # 匹配成功 - logger.info( - f'{mediainfo.title_year} 通过标题匹配到可选资源：{torrent_info.site_name} - {torrent_info.title}') - torrent_mediainfo = mediainfo + # 如果种子未识别，尝试识别 + if not context.media_info or (not context.media_info.tmdb_id + and not context.media_info.douban_id): + re_mediainfo = self.recognize_media(meta=context.meta_info) + if re_mediainfo: + # 清理多余信息 + re_mediainfo.clear() # 更新种子缓存 - _context.media_info = mediainfo + context.media_info = re_mediainfo + # 添加已预处理 + processed_torrents[domain].append(context) + + # 所有订阅 + subscribes = SubscribeOper().list(self.get_states_for_search('R')) + for subscribe in subscribes: + if global_vars.is_system_stopped: + break + logger.info(f'开始匹配订阅，标题：{subscribe.name} ...') + mediakey = subscribe.tmdbid or subscribe.doubanid + # 生成元数据 + meta = MetaInfo(subscribe.name) + meta.year = subscribe.year + meta.begin_season = subscribe.season or None + try: + meta.type = MediaType(subscribe.type) + except ValueError: + logger.error(f'订阅 {subscribe.name} 类型错误：{subscribe.type}') + continue + # 订阅的站点域名列表 + domains = [] + if subscribe.sites: + domains = SiteOper().get_domains_by_ids(subscribe.sites) + # 识别媒体信息 + mediainfo: MediaInfo = self.recognize_media(meta=meta, mtype=meta.type, + tmdbid=subscribe.tmdbid, + doubanid=subscribe.doubanid, + episode_group=subscribe.episode_group, + cache=False) + if not mediainfo: + logger.warn( + f'未识别到媒体信息，标题：{subscribe.name}，tmdbid：{subscribe.tmdbid}，doubanid：{subscribe.doubanid}') + continue + + # 如果媒体已存在或已下载完毕，跳过当前订阅处理 + exist_flag, no_exists = self.check_and_handle_existing_media(subscribe=subscribe, meta=meta, + mediainfo=mediainfo, + mediakey=mediakey) + if exist_flag: + continue + + # 清理多余信息 + mediainfo.clear() + + # 订阅识别词 + if subscribe.custom_words: + custom_words_list = subscribe.custom_words.split("\n") + else: + custom_words_list = None + + # 遍历预识别后的种子 + _match_context = [] + torrenthelper = TorrentHelper() + systemconfig = SystemConfigOper() + wordsmatcher = WordsMatcher() + for domain, contexts in processed_torrents.items(): + if global_vars.is_system_stopped: + break + if domains and domain not in domains: + continue + logger.debug(f'开始匹配站点：{domain}，共缓存了 {len(contexts)} 个种子...') + for context in contexts: + if global_vars.is_system_stopped: + break + # 提取信息 + _context = copy.copy(context) + torrent_meta = _context.meta_info + torrent_mediainfo = _context.media_info + torrent_info = _context.torrent_info + + # 不在订阅站点范围的不处理 + sub_sites = self.get_sub_sites(subscribe) + if sub_sites and torrent_info.site not in sub_sites: + logger.debug(f"{torrent_info.site_name} - {torrent_info.title} 不符合订阅站点要求") + continue + + # 有自定义识别词时，需要判断是否需要重新识别 + if custom_words_list: + # 使用org_string，应用一次后理论上不能再次应用 + _, apply_words = wordsmatcher.prepare(torrent_meta.org_string, + custom_words=custom_words_list) + if apply_words: + logger.info( + f'{torrent_info.site_name} - {torrent_info.title} 因订阅存在自定义识别词，重新识别元数据...') + # 重新识别元数据 + torrent_meta = MetaInfo(title=torrent_info.title, subtitle=torrent_info.description, + custom_words=custom_words_list) + # 更新元数据缓存 + _context.meta_info = torrent_meta + # 重新识别媒体信息 + torrent_mediainfo = self.recognize_media(meta=torrent_meta, + episode_group=subscribe.episode_group) + if torrent_mediainfo: + # 清理多余信息 + torrent_mediainfo.clear() + # 更新种子缓存 + _context.media_info = torrent_mediainfo + + # 如果仍然没有识别到媒体信息，尝试标题匹配 + if not torrent_mediainfo or (not torrent_mediainfo.tmdb_id and not torrent_mediainfo.douban_id): + logger.info( + f'{torrent_info.site_name} - {torrent_info.title} 重新识别失败，尝试通过标题匹配...') + if torrenthelper.match_torrent(mediainfo=mediainfo, + torrent_meta=torrent_meta, + torrent=torrent_info): + # 匹配成功 + logger.info( + f'{mediainfo.title_year} 通过标题匹配到可选资源：{torrent_info.site_name} - {torrent_info.title}') + torrent_mediainfo = mediainfo + # 更新种子缓存 + _context.media_info = mediainfo + else: + continue + + # 直接比对媒体信息 + if torrent_mediainfo and (torrent_mediainfo.tmdb_id or torrent_mediainfo.douban_id): + if torrent_mediainfo.type != mediainfo.type: + continue + if torrent_mediainfo.tmdb_id \ + and torrent_mediainfo.tmdb_id != mediainfo.tmdb_id: + continue + if torrent_mediainfo.douban_id \ + and torrent_mediainfo.douban_id != mediainfo.douban_id: + continue + logger.info( + f'{mediainfo.title_year} 通过媒体信ID匹配到可选资源：{torrent_info.site_name} - {torrent_info.title}') else: continue - # 直接比对媒体信息 - if torrent_mediainfo and (torrent_mediainfo.tmdb_id or torrent_mediainfo.douban_id): - if torrent_mediainfo.type != mediainfo.type: - continue - if torrent_mediainfo.tmdb_id \ - and torrent_mediainfo.tmdb_id != mediainfo.tmdb_id: - continue - if torrent_mediainfo.douban_id \ - and torrent_mediainfo.douban_id != mediainfo.douban_id: - continue - logger.info( - f'{mediainfo.title_year} 通过媒体信ID匹配到可选资源：{torrent_info.site_name} - {torrent_info.title}') - else: - continue - - # 如果是电视剧 - if torrent_mediainfo.type == MediaType.TV: - # 有多季的不要 - if len(torrent_meta.season_list) > 1: - logger.debug(f'{torrent_info.title} 有多季，不处理') - continue - # 比对季 - if torrent_meta.begin_season: - if meta.begin_season != torrent_meta.begin_season: + # 如果是电视剧 + if torrent_mediainfo.type == MediaType.TV: + # 有多季的不要 + if len(torrent_meta.season_list) > 1: + logger.debug(f'{torrent_info.title} 有多季，不处理') + continue + # 比对季 + if torrent_meta.begin_season: + if meta.begin_season != torrent_meta.begin_season: + logger.debug(f'{torrent_info.title} 季不匹配') + continue + elif meta.begin_season != 1: logger.debug(f'{torrent_info.title} 季不匹配') continue - elif meta.begin_season != 1: - logger.debug(f'{torrent_info.title} 季不匹配') - continue - # 非洗版 - if not subscribe.best_version: - # 不是缺失的剧集不要 - if no_exists and no_exists.get(mediakey): - # 缺失集 - no_exists_info = no_exists.get(mediakey).get(subscribe.season) - if no_exists_info: - # 是否有交集 - if no_exists_info.episodes and \ - torrent_meta.episode_list and \ - not set(no_exists_info.episodes).intersection( - set(torrent_meta.episode_list) - ): - logger.debug( - f'{torrent_info.title} 对应剧集 {torrent_meta.episode_list} 未包含缺失的剧集' - ) + # 非洗版 + if not subscribe.best_version: + # 不是缺失的剧集不要 + if no_exists and no_exists.get(mediakey): + # 缺失集 + no_exists_info = no_exists.get(mediakey).get(subscribe.season) + if no_exists_info: + # 是否有交集 + if no_exists_info.episodes and \ + torrent_meta.episode_list and \ + not set(no_exists_info.episodes).intersection( + set(torrent_meta.episode_list) + ): + logger.debug( + f'{torrent_info.title} 对应剧集 {torrent_meta.episode_list} 未包含缺失的剧集' + ) + continue + else: + # 洗版时，非整季不要 + if meta.type == MediaType.TV: + if torrent_meta.episode_list: + logger.debug(f'{subscribe.name} 正在洗版，{torrent_info.title} 不是整季') continue - else: - # 洗版时，非整季不要 - if meta.type == MediaType.TV: - if torrent_meta.episode_list: - logger.debug(f'{subscribe.name} 正在洗版，{torrent_info.title} 不是整季') - continue - # 匹配订阅附加参数 - if not torrenthelper.filter_torrent(torrent_info=torrent_info, - filter_params=self.get_params(subscribe)): - continue - - # 优先级过滤规则 - if subscribe.best_version: - rule_groups = subscribe.filter_groups \ - or systemconfig.get(SystemConfigKey.BestVersionFilterRuleGroups) - else: - rule_groups = subscribe.filter_groups \ - or systemconfig.get(SystemConfigKey.SubscribeFilterRuleGroups) - result: List[TorrentInfo] = self.filter_torrents( - rule_groups=rule_groups, - torrent_list=[torrent_info], - mediainfo=torrent_mediainfo) - if result is not None and not result: - # 不符合过滤规则 - logger.debug(f"{torrent_info.title} 不匹配过滤规则") - continue - - # 洗版时，优先级小于已下载优先级的不要 - if subscribe.best_version: - if subscribe.current_priority \ - and torrent_info.pri_order <= subscribe.current_priority: - logger.info( - f'{subscribe.name} 正在洗版，{torrent_info.title} 优先级低于或等于已下载优先级') + # 匹配订阅附加参数 + if not torrenthelper.filter_torrent(torrent_info=torrent_info, + filter_params=self.get_params(subscribe)): continue - # 匹配成功 - logger.info(f'{mediainfo.title_year} 匹配成功：{torrent_info.title}') - # 自定义属性 - if subscribe.media_category: - torrent_mediainfo.category = subscribe.media_category - if subscribe.episode_group: - torrent_mediainfo.episode_group = subscribe.episode_group - _match_context.append(_context) + # 优先级过滤规则 + if subscribe.best_version: + rule_groups = subscribe.filter_groups \ + or systemconfig.get(SystemConfigKey.BestVersionFilterRuleGroups) + else: + rule_groups = subscribe.filter_groups \ + or systemconfig.get(SystemConfigKey.SubscribeFilterRuleGroups) + result: List[TorrentInfo] = self.filter_torrents( + rule_groups=rule_groups, + torrent_list=[torrent_info], + mediainfo=torrent_mediainfo) + if result is not None and not result: + # 不符合过滤规则 + logger.debug(f"{torrent_info.title} 不匹配过滤规则") + continue - if not _match_context: - # 未匹配到资源 - logger.info(f'{mediainfo.title_year} 未匹配到符合条件的资源') - self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, - mediainfo=mediainfo, lefts=no_exists) - continue + # 洗版时，优先级小于已下载优先级的不要 + if subscribe.best_version: + if subscribe.current_priority \ + and torrent_info.pri_order <= subscribe.current_priority: + logger.info( + f'{subscribe.name} 正在洗版，{torrent_info.title} 优先级低于或等于已下载优先级') + continue - # 开始批量择优下载 - logger.info(f'{mediainfo.title_year} 匹配完成，共匹配到{len(_match_context)}个资源') - downloads, lefts = DownloadChain().batch_download(contexts=_match_context, - no_exists=no_exists, - username=subscribe.username, - save_path=subscribe.save_path, - downloader=subscribe.downloader, - source=self.get_subscribe_source_keyword(subscribe) - ) + # 匹配成功 + logger.info(f'{mediainfo.title_year} 匹配成功：{torrent_info.title}') + # 自定义属性 + if subscribe.media_category: + torrent_mediainfo.category = subscribe.media_category + if subscribe.episode_group: + torrent_mediainfo.episode_group = subscribe.episode_group + _match_context.append(_context) - # 同步外部修改，更新订阅信息 - subscribe = SubscribeOper().get(subscribe.id) + if not _match_context: + # 未匹配到资源 + logger.info(f'{mediainfo.title_year} 未匹配到符合条件的资源') + self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, + mediainfo=mediainfo, lefts=no_exists) + continue - # 判断是否要完成订阅 - if subscribe: - self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, mediainfo=mediainfo, - downloads=downloads, lefts=lefts) + # 开始批量择优下载 + logger.info(f'{mediainfo.title_year} 匹配完成，共匹配到{len(_match_context)}个资源') + downloads, lefts = DownloadChain().batch_download(contexts=_match_context, + no_exists=no_exists, + username=subscribe.username, + save_path=subscribe.save_path, + downloader=subscribe.downloader, + source=self.get_subscribe_source_keyword(subscribe) + ) - logger.debug(f"match Lock released at {datetime.now()}") + # 同步外部修改，更新订阅信息 + subscribe = SubscribeOper().get(subscribe.id) + + # 判断是否要完成订阅 + if subscribe: + self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, mediainfo=mediainfo, + downloads=downloads, lefts=lefts) + + logger.debug(f"match Lock released at {datetime.now()}") + finally: + subscribes.clear() + processed_torrents.clear() def check(self): """ @@ -810,12 +814,8 @@ class SubscribeChain(ChainBase): """ # 查询所有订阅 subscribeoper = SubscribeOper() - subscribes = subscribeoper.list() - if not subscribes: - # 没有订阅不运行 - return # 遍历订阅 - for subscribe in subscribes: + for subscribe in subscribeoper.list(): if global_vars.is_system_stopped: break logger.info(f'开始更新订阅元数据：{subscribe.name} ...') @@ -871,11 +871,10 @@ class SubscribeChain(ChainBase): follow_users: List[str] = SystemConfigOper().get(SystemConfigKey.FollowSubscribers) if not follow_users: return - share_subs = SubscribeHelper().get_shares() logger.info(f'开始刷新follow用户分享订阅 ...') success_count = 0 subscribeoper = SubscribeOper() - for share_sub in share_subs: + for share_sub in SubscribeHelper().get_shares(): if global_vars.is_system_stopped: break uid = share_sub.get("share_uid") diff --git a/app/chain/torrents.py b/app/chain/torrents.py index d4e767e6..e3902336 100644 --- a/app/chain/torrents.py +++ b/app/chain/torrents.py @@ -98,6 +98,7 @@ class TorrentsChain(ChainBase): if not site.get("rss"): logger.error(f'站点 {domain} 未配置RSS地址！') return [] + # 解析RSS rss_items = RssHelper().parse(site.get("rss"), True if site.get("proxy") else False, timeout=int(site.get("timeout") or 30)) if rss_items is None: @@ -109,25 +110,27 @@ class TorrentsChain(ChainBase): return [] # 组装种子 ret_torrents: List[TorrentInfo] = [] - for item in rss_items: - if not item.get("title"): - continue - torrentinfo = TorrentInfo( - site=site.get("id"), - site_name=site.get("name"), - site_cookie=site.get("cookie"), - site_ua=site.get("ua") or settings.USER_AGENT, - site_proxy=site.get("proxy"), - site_order=site.get("pri"), - site_downloader=site.get("downloader"), - title=item.get("title"), - enclosure=item.get("enclosure"), - page_url=item.get("link"), - size=item.get("size"), - pubdate=item["pubdate"].strftime("%Y-%m-%d %H:%M:%S") if item.get("pubdate") else None, - ) - ret_torrents.append(torrentinfo) - + try: + for item in rss_items: + if not item.get("title"): + continue + torrentinfo = TorrentInfo( + site=site.get("id"), + site_name=site.get("name"), + site_cookie=site.get("cookie"), + site_ua=site.get("ua") or settings.USER_AGENT, + site_proxy=site.get("proxy"), + site_order=site.get("pri"), + site_downloader=site.get("downloader"), + title=item.get("title"), + enclosure=item.get("enclosure"), + page_url=item.get("link"), + size=item.get("size"), + pubdate=item["pubdate"].strftime("%Y-%m-%d %H:%M:%S") if item.get("pubdate") else None, + ) + ret_torrents.append(torrentinfo) + finally: + rss_items.clear() return ret_torrents def refresh(self, stype: Optional[str] = None, sites: List[int] = None) -> Dict[str, List[Context]]: @@ -152,13 +155,10 @@ class TorrentsChain(ChainBase): torrents_cache[_domain] = [_torrent for _torrent in _torrents if not TorrentHelper().is_invalid(_torrent.torrent_info.enclosure)] - # 所有站点索引 - indexers = SitesHelper().get_indexers() # 需要刷新的站点domain domains = [] - # 遍历站点缓存资源 - for indexer in indexers: + for indexer in SitesHelper().get_indexers(): if global_vars.is_system_stopped: break # 未开启的站点不刷新 @@ -187,36 +187,39 @@ class TorrentsChain(ChainBase): else: logger.info(f'{indexer.get("name")} 没有新种子') continue - for torrent in torrents: - if global_vars.is_system_stopped: - break - logger.info(f'处理资源：{torrent.title} ...') - # 识别 - meta = MetaInfo(title=torrent.title, subtitle=torrent.description) - if torrent.title != meta.org_string: - logger.info(f'种子名称应用识别词后发生改变：{torrent.title} => {meta.org_string}') - # 使用站点种子分类，校正类型识别 - if meta.type != MediaType.TV \ - and torrent.category == MediaType.TV.value: - meta.type = MediaType.TV - # 识别媒体信息 - mediainfo: MediaInfo = MediaChain().recognize_by_meta(meta) - if not mediainfo: - logger.warn(f'{torrent.title} 未识别到媒体信息') - # 存储空的媒体信息 - mediainfo = MediaInfo() - # 清理多余数据，减少内存占用 - mediainfo.clear() - # 上下文 - context = Context(meta_info=meta, media_info=mediainfo, torrent_info=torrent) - # 添加到缓存 - if not torrents_cache.get(domain): - torrents_cache[domain] = [context] - else: - torrents_cache[domain].append(context) - # 如果超过了限制条数则移除掉前面的 - if len(torrents_cache[domain]) > settings.CONF["torrents"]: - torrents_cache[domain] = torrents_cache[domain][-settings.CONF["torrents"]:] + try: + for torrent in torrents: + if global_vars.is_system_stopped: + break + logger.info(f'处理资源：{torrent.title} ...') + # 识别 + meta = MetaInfo(title=torrent.title, subtitle=torrent.description) + if torrent.title != meta.org_string: + logger.info(f'种子名称应用识别词后发生改变：{torrent.title} => {meta.org_string}') + # 使用站点种子分类，校正类型识别 + if meta.type != MediaType.TV \ + and torrent.category == MediaType.TV.value: + meta.type = MediaType.TV + # 识别媒体信息 + mediainfo: MediaInfo = MediaChain().recognize_by_meta(meta) + if not mediainfo: + logger.warn(f'{torrent.title} 未识别到媒体信息') + # 存储空的媒体信息 + mediainfo = MediaInfo() + # 清理多余数据，减少内存占用 + mediainfo.clear() + # 上下文 + context = Context(meta_info=meta, media_info=mediainfo, torrent_info=torrent) + # 添加到缓存 + if not torrents_cache.get(domain): + torrents_cache[domain] = [context] + else: + torrents_cache[domain].append(context) + # 如果超过了限制条数则移除掉前面的 + if len(torrents_cache[domain]) > settings.CONF["torrents"]: + torrents_cache[domain] = torrents_cache[domain][-settings.CONF["torrents"]:] + finally: + torrents.clear() else: logger.info(f'{indexer.get("name")} 没有获取到种子') diff --git a/app/chain/transfer.py b/app/chain/transfer.py index e5910e60..1977da9f 100755 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -788,6 +788,7 @@ class TransferChain(ChainBase, metaclass=Singleton): for dir_info in download_dirs): return True logger.info("开始整理下载器中已经完成下载的文件 ...") + # 从下载器获取种子列表 torrents: Optional[List[TransferTorrent]] = self.list_torrents(status=TorrentStatus.TRANSFER) if not torrents: @@ -796,70 +797,73 @@ class TransferChain(ChainBase, metaclass=Singleton): logger.info(f"获取到 {len(torrents)} 个已完成的下载任务") - for torrent in torrents: - if global_vars.is_system_stopped: - break - # 文件路径 - file_path = torrent.path - if not file_path.exists(): - logger.warn(f"文件不存在：{file_path}") - continue - # 检查是否为下载器监控目录中的文件 - is_downloader_monitor = False - for dir_info in download_dirs: - if dir_info.monitor_type != "downloader": - continue - if not dir_info.download_path: - continue - if file_path.is_relative_to(Path(dir_info.download_path)): - is_downloader_monitor = True + try: + for torrent in torrents: + if global_vars.is_system_stopped: break - if not is_downloader_monitor: - logger.debug(f"文件 {file_path} 不在下载器监控目录中，不通过下载器进行整理") - continue - # 查询下载记录识别情况 - downloadhis: DownloadHistory = DownloadHistoryOper().get_by_hash(torrent.hash) - if downloadhis: - # 类型 - try: - mtype = MediaType(downloadhis.type) - except ValueError: - mtype = MediaType.TV - # 按TMDBID识别 - mediainfo = self.recognize_media(mtype=mtype, - tmdbid=downloadhis.tmdbid, - doubanid=downloadhis.doubanid, - episode_group=downloadhis.episode_group) - if mediainfo: - # 补充图片 - self.obtain_images(mediainfo) - # 更新自定义媒体类别 - if downloadhis.media_category: - mediainfo.category = downloadhis.media_category - else: - # 非MoviePilot下载的任务，按文件识别 - mediainfo = None + # 文件路径 + file_path = torrent.path + if not file_path.exists(): + logger.warn(f"文件不存在：{file_path}") + continue + # 检查是否为下载器监控目录中的文件 + is_downloader_monitor = False + for dir_info in download_dirs: + if dir_info.monitor_type != "downloader": + continue + if not dir_info.download_path: + continue + if file_path.is_relative_to(Path(dir_info.download_path)): + is_downloader_monitor = True + break + if not is_downloader_monitor: + logger.debug(f"文件 {file_path} 不在下载器监控目录中，不通过下载器进行整理") + continue + # 查询下载记录识别情况 + downloadhis: DownloadHistory = DownloadHistoryOper().get_by_hash(torrent.hash) + if downloadhis: + # 类型 + try: + mtype = MediaType(downloadhis.type) + except ValueError: + mtype = MediaType.TV + # 按TMDBID识别 + mediainfo = self.recognize_media(mtype=mtype, + tmdbid=downloadhis.tmdbid, + doubanid=downloadhis.doubanid, + episode_group=downloadhis.episode_group) + if mediainfo: + # 补充图片 + self.obtain_images(mediainfo) + # 更新自定义媒体类别 + if downloadhis.media_category: + mediainfo.category = downloadhis.media_category + else: + # 非MoviePilot下载的任务，按文件识别 + mediainfo = None - # 执行实时整理，匹配源目录 - state, errmsg = self.do_transfer( - fileitem=FileItem( - storage="local", - path=str(file_path).replace("\\", "/"), - type="dir" if not file_path.is_file() else "file", - name=file_path.name, - size=file_path.stat().st_size, - extension=file_path.suffix.lstrip('.'), - ), - mediainfo=mediainfo, - downloader=torrent.downloader, - download_hash=torrent.hash, - background=False, - ) + # 执行实时整理，匹配源目录 + state, errmsg = self.do_transfer( + fileitem=FileItem( + storage="local", + path=str(file_path).replace("\\", "/"), + type="dir" if not file_path.is_file() else "file", + name=file_path.name, + size=file_path.stat().st_size, + extension=file_path.suffix.lstrip('.'), + ), + mediainfo=mediainfo, + downloader=torrent.downloader, + download_hash=torrent.hash, + background=False, + ) - # 设置下载任务状态 - if not state: - logger.warn(f"整理下载器任务失败：{torrent.hash} - {errmsg}") - self.transfer_completed(hashs=torrent.hash, downloader=torrent.downloader) + # 设置下载任务状态 + if not state: + logger.warn(f"整理下载器任务失败：{torrent.hash} - {errmsg}") + self.transfer_completed(hashs=torrent.hash, downloader=torrent.downloader) + finally: + torrents.clear() # 结束 logger.info("所有下载器中下载完成的文件已整理完成") @@ -1032,111 +1036,114 @@ class TransferChain(ChainBase, metaclass=Singleton): # 整理所有文件 transfer_tasks: List[TransferTask] = [] - for file_item, bluray_dir in file_items: - if global_vars.is_system_stopped: - break - if continue_callback and not continue_callback(): - break - file_path = Path(file_item.path) - # 回收站及隐藏的文件不处理 - if file_item.path.find('/@Recycle/') != -1 \ - or file_item.path.find('/#recycle/') != -1 \ - or file_item.path.find('/.') != -1 \ - or file_item.path.find('/@eaDir') != -1: - logger.debug(f"{file_item.path} 是回收站或隐藏的文件") - continue - - # 整理屏蔽词不处理 - is_blocked = False - if transfer_exclude_words: - for keyword in transfer_exclude_words: - if not keyword: - continue - if keyword and re.search(r"%s" % keyword, file_item.path, re.IGNORECASE): - logger.info(f"{file_item.path} 命中整理屏蔽词 {keyword}，不处理") - is_blocked = True - break - if is_blocked: - continue - - # 整理成功的不再处理 - if not force: - transferd = TransferHistoryOper().get_by_src(file_item.path, storage=file_item.storage) - if transferd: - if not transferd.status: - all_success = False - logger.info(f"{file_item.path} 已整理过，如需重新处理，请删除整理记录。") - err_msgs.append(f"{file_item.name} 已整理过") + try: + for file_item, bluray_dir in file_items: + if global_vars.is_system_stopped: + break + if continue_callback and not continue_callback(): + break + file_path = Path(file_item.path) + # 回收站及隐藏的文件不处理 + if file_item.path.find('/@Recycle/') != -1 \ + or file_item.path.find('/#recycle/') != -1 \ + or file_item.path.find('/.') != -1 \ + or file_item.path.find('/@eaDir') != -1: + logger.debug(f"{file_item.path} 是回收站或隐藏的文件") continue - if not meta: - # 文件元数据 - file_meta = MetaInfoPath(file_path) - else: - file_meta = meta + # 整理屏蔽词不处理 + is_blocked = False + if transfer_exclude_words: + for keyword in transfer_exclude_words: + if not keyword: + continue + if keyword and re.search(r"%s" % keyword, file_item.path, re.IGNORECASE): + logger.info(f"{file_item.path} 命中整理屏蔽词 {keyword}，不处理") + is_blocked = True + break + if is_blocked: + continue - # 合并季 - if season is not None: - file_meta.begin_season = season + # 整理成功的不再处理 + if not force: + transferd = TransferHistoryOper().get_by_src(file_item.path, storage=file_item.storage) + if transferd: + if not transferd.status: + all_success = False + logger.info(f"{file_item.path} 已整理过，如需重新处理，请删除整理记录。") + err_msgs.append(f"{file_item.name} 已整理过") + continue - if not file_meta: - all_success = False - logger.error(f"{file_path.name} 无法识别有效信息") - err_msgs.append(f"{file_path.name} 无法识别有效信息") - continue + if not meta: + # 文件元数据 + file_meta = MetaInfoPath(file_path) + else: + file_meta = meta - # 自定义识别 - if formaterHandler: - # 开始集、结束集、PART - begin_ep, end_ep, part = formaterHandler.split_episode(file_name=file_path.name, file_meta=file_meta) - if begin_ep is not None: - file_meta.begin_episode = begin_ep - file_meta.part = part - if end_ep is not None: - file_meta.end_episode = end_ep + # 合并季 + if season is not None: + file_meta.begin_season = season - # 根据父路径获取下载历史 - download_history = None - downloadhis = DownloadHistoryOper() - if bluray_dir: - # 蓝光原盘，按目录名查询 - download_history = downloadhis.get_by_path(str(file_path)) - else: - # 按文件全路径查询 - download_file = downloadhis.get_file_by_fullpath(str(file_path)) - if download_file: - download_history = downloadhis.get_by_hash(download_file.download_hash) + if not file_meta: + all_success = False + logger.error(f"{file_path.name} 无法识别有效信息") + err_msgs.append(f"{file_path.name} 无法识别有效信息") + continue - # 获取下载Hash - if download_history and (not downloader or not download_hash): - downloader = download_history.downloader - download_hash = download_history.download_hash + # 自定义识别 + if formaterHandler: + # 开始集、结束集、PART + begin_ep, end_ep, part = formaterHandler.split_episode(file_name=file_path.name, file_meta=file_meta) + if begin_ep is not None: + file_meta.begin_episode = begin_ep + file_meta.part = part + if end_ep is not None: + file_meta.end_episode = end_ep - # 后台整理 - transfer_task = TransferTask( - fileitem=file_item, - meta=file_meta, - mediainfo=mediainfo, - target_directory=target_directory, - target_storage=target_storage, - target_path=target_path, - transfer_type=transfer_type, - scrape=scrape, - library_type_folder=library_type_folder, - library_category_folder=library_category_folder, - downloader=downloader, - download_hash=download_hash, - download_history=download_history, - manual=manual, - background=background - ) - if background: - self.put_to_queue(task=transfer_task) - logger.info(f"{file_path.name} 已添加到整理队列") - else: - # 加入列表 - self.__put_to_jobview(transfer_task) - transfer_tasks.append(transfer_task) + # 根据父路径获取下载历史 + download_history = None + downloadhis = DownloadHistoryOper() + if bluray_dir: + # 蓝光原盘，按目录名查询 + download_history = downloadhis.get_by_path(str(file_path)) + else: + # 按文件全路径查询 + download_file = downloadhis.get_file_by_fullpath(str(file_path)) + if download_file: + download_history = downloadhis.get_by_hash(download_file.download_hash) + + # 获取下载Hash + if download_history and (not downloader or not download_hash): + downloader = download_history.downloader + download_hash = download_history.download_hash + + # 后台整理 + transfer_task = TransferTask( + fileitem=file_item, + meta=file_meta, + mediainfo=mediainfo, + target_directory=target_directory, + target_storage=target_storage, + target_path=target_path, + transfer_type=transfer_type, + scrape=scrape, + library_type_folder=library_type_folder, + library_category_folder=library_category_folder, + downloader=downloader, + download_hash=download_hash, + download_history=download_history, + manual=manual, + background=background + ) + if background: + self.put_to_queue(task=transfer_task) + logger.info(f"{file_path.name} 已添加到整理队列") + else: + # 加入列表 + self.__put_to_jobview(transfer_task) + transfer_tasks.append(transfer_task) + finally: + file_items.clear() # 实时整理 if transfer_tasks: @@ -1155,29 +1162,31 @@ class TransferChain(ChainBase, metaclass=Singleton): progress.update(value=0, text=__process_msg, key=ProgressKey.FileTransfer) - - for transfer_task in transfer_tasks: - if global_vars.is_system_stopped: - break - if continue_callback and not continue_callback(): - break - # 更新进度 - __process_msg = f"正在整理 （{processed_num + fail_num + 1}/{total_num}）{transfer_task.fileitem.name} ..." - logger.info(__process_msg) - progress.update(value=(processed_num + fail_num) / total_num * 100, - text=__process_msg, - key=ProgressKey.FileTransfer) - state, err_msg = self.__handle_transfer( - task=transfer_task, - callback=self.__default_callback - ) - if not state: - all_success = False - logger.warn(f"{transfer_task.fileitem.name} {err_msg}") - err_msgs.append(f"{transfer_task.fileitem.name} {err_msg}") - fail_num += 1 - else: - processed_num += 1 + try: + for transfer_task in transfer_tasks: + if global_vars.is_system_stopped: + break + if continue_callback and not continue_callback(): + break + # 更新进度 + __process_msg = f"正在整理 （{processed_num + fail_num + 1}/{total_num}）{transfer_task.fileitem.name} ..." + logger.info(__process_msg) + progress.update(value=(processed_num + fail_num) / total_num * 100, + text=__process_msg, + key=ProgressKey.FileTransfer) + state, err_msg = self.__handle_transfer( + task=transfer_task, + callback=self.__default_callback + ) + if not state: + all_success = False + logger.warn(f"{transfer_task.fileitem.name} {err_msg}") + err_msgs.append(f"{transfer_task.fileitem.name} {err_msg}") + fail_num += 1 + else: + processed_num += 1 + finally: + transfer_tasks.clear() # 整理结束 __end_msg = f"整理队列处理完成，共整理 {total_num} 个文件，失败 {fail_num} 个" diff --git a/app/db/__init__.py b/app/db/__init__.py index cf7eda25..4fe29d0c 100644 --- a/app/db/__init__.py +++ b/app/db/__init__.py @@ -221,8 +221,7 @@ class Base: @classmethod @db_query def list(cls, db: Session) -> List[Self]: - result = db.query(cls).all() - return list(result) + return db.query(cls).all() def to_dict(self): return {c.name: getattr(self, c.name, None) for c in self.__table__.columns} # noqa