From ae96e20cc0161f99d49e83dc37a8518c4934ac85 Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Thu, 7 Dec 2023 18:27:07 +0800 Subject: [PATCH] =?UTF-8?q?v2.4.7:=20=E5=AE=9E=E7=8E=B0=E6=94=B6=E8=97=8F?= =?UTF-8?q?=E6=9C=AC=E5=AD=90=E5=8A=9F=E8=83=BD=EF=BC=8C=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E6=94=B6=E8=97=8F=E5=A4=B9=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=E4=BC=A0=E5=8F=82=EF=BC=8C=E4=BC=98=E5=8C=96Headers=E5=92=8C?= =?UTF-8?q?=E5=9F=9F=E5=90=8D=E7=9A=84=E5=A4=84=E7=90=86;=20=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E6=96=87=E6=A1=A3=E3=80=81dispatch=E5=B7=A5=E4=BD=9C?= =?UTF-8?q?=E6=B5=81=E6=96=87=E6=A1=88+=E8=84=9A=E6=9C=AC.=20(#176)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/close_specific_pr.yml | 6 +- .github/workflows/download_dispatch.yml | 6 +- assets/docs/sources/tutorial/5_filter.md | 14 +- src/jmcomic/__init__.py | 2 +- src/jmcomic/jm_client_impl.py | 172 ++++++++++++++++------- src/jmcomic/jm_client_interface.py | 9 ++ src/jmcomic/jm_config.py | 10 +- src/jmcomic/jm_downloader.py | 21 ++- src/jmcomic/jm_entity.py | 16 ++- src/jmcomic/jm_option.py | 59 +++----- src/jmcomic/jm_plugin.py | 2 +- src/jmcomic/jm_toolkit.py | 5 +- tests/test_jmcomic/test_jm_custom.py | 8 +- usage/workflow_download.py | 26 +++- 14 files changed, 233 insertions(+), 123 deletions(-) diff --git a/.github/workflows/close_specific_pr.yml b/.github/workflows/close_specific_pr.yml index 0123c16..eaf35b6 100644 --- a/.github/workflows/close_specific_pr.yml +++ b/.github/workflows/close_specific_pr.yml @@ -2,10 +2,12 @@ name: Close specific PR on: pull_request: + types: [opened, ] + # Note: If you use both the branches filter and the paths filter, the workflow will only run when both filters are satisfied. + branches: + - master paths: - 'usage/workflow_download.py' - types: [opened, ] - jobs: close_pr: env: diff --git a/.github/workflows/download_dispatch.yml b/.github/workflows/download_dispatch.yml index b5b64c9..9556758 100644 --- a/.github/workflows/download_dispatch.yml +++ b/.github/workflows/download_dispatch.yml @@ -15,19 +15,19 @@ on: CLIENT_IMPL: type: string - description: 客户端类型(client.impl),[api]=移动端,[html]=网页端。下载失败时,你可以尝试填入此项重试。 + description: 客户端类型(client.impl),下载失败时,你可以尝试填入此项重试。'api' 表示移动端,'html' 表示网页端。 default: '' required: false IMAGE_SUFFIX: type: string - description: 图片后缀(download.cache.suffix),默认为空,表示不做图片格式转换。可填入例如 "png" "jpg"。 + description: 图片后缀(download.cache.suffix),默认为空,表示不做图片格式转换。可填入例如 'png' 'jpg' default: '' required: false DIR_RULE: type: string - description: 下载文件夹规则(dir_rule.rule)。此处可以不填,默认使用配置文件的'Bd_Aauthor_Atitle_Pindex'。 + description: 下载文件夹规则(dir_rule.rule)。默认使用配置文件的 'Bd_Aauthor_Atitle_Pindex'。 default: '' required: false diff --git a/assets/docs/sources/tutorial/5_filter.md b/assets/docs/sources/tutorial/5_filter.md index af88323..ae6db71 100644 --- a/assets/docs/sources/tutorial/5_filter.md +++ b/assets/docs/sources/tutorial/5_filter.md @@ -6,10 +6,10 @@ filter(过滤器)是v2.1.12新引入的机制, 使用filter的步骤如下: ``` -1. 自定义class,继承JmDownloader,重写filter_iter_objs方法,即: +1. 自定义class,继承JmDownloader,重写do_filter方法,即: class MyDownloader(JmDownloader): - def filter_iter_objs(self, iter_objs: DownloadIterObjs): - # 如何重写?参考JmDownloader.filter_iter_objs和下面的示例 + def do_filter(self, detail): + # 如何重写?参考JmDownloader.do_filter和下面的示例 ... 2. 让你的class生效,使用如下代码: @@ -30,8 +30,8 @@ from jmcomic import * class First3ImageDownloader(JmDownloader): - def filter_iter_objs(self, iter_objs: DownloadIterObjs): - if isinstance(iter_objs, JmPhotoDetail): + def do_filter(self, detail): + if detail.is_photo(): photo: JmPhotoDetail = iter_objs # 支持[start,end,step] return photo[:3] @@ -52,8 +52,8 @@ class FindUpdateDownloader(JmDownloader): 'xxx': 'yyy' } - def filter_iter_objs(self, iter_objs: DownloadIterObjs): - if not isinstance(iter_objs, JmAlbumDetail): + def do_filter(self, detail): + if not detail.is_album(): return iter_objs return self.find_update(iter_objs) diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index 626b9b2..a53b58f 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,7 +2,7 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.4.6' +__version__ = '2.4.7' from .api import * from .jm_plugin import * diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py index 918612d..2716421 100644 --- a/src/jmcomic/jm_client_impl.py +++ b/src/jmcomic/jm_client_impl.py @@ -1,3 +1,5 @@ +from threading import Lock + from .jm_client_interface import * @@ -25,6 +27,7 @@ class AbstractJmClient( self.retry_times = retry_times self.domain_list = domain_list self.CLIENT_CACHE = None + self.__username = None # help for favorite_folder method self.enable_cache() self.after_init() @@ -50,7 +53,7 @@ class AbstractJmClient( resp.require_success() return resp - return self.get(img_url, judge=judge) + return self.get(img_url, judge=judge, headers=JmModuleConfig.new_html_headers()) def request_with_retry(self, request, @@ -61,7 +64,12 @@ class AbstractJmClient( **kwargs, ): """ - 统一请求,支持重试 + 支持重试和切换域名的机制 + + 如果url包含了指定域名,则不会切换域名,例如图片URL。 + + 如果需要拿到域名进行回调处理,可以重写 self.update_request_with_specify_domain 方法,例如更新headers + :param request: 请求方法 :param url: 图片url / path (/album/xxx) :param domain_index: 域名下标 @@ -74,10 +82,11 @@ class AbstractJmClient( if url.startswith('/'): # path → url - url = self.of_api_url( - api_path=url, - domain=self.domain_list[domain_index], - ) + domain = self.domain_list[domain_index] + url = self.of_api_url(url, domain) + + self.update_request_with_specify_domain(kwargs, domain) + jm_log(self.log_topic(), self.decode(url)) else: # 图片url @@ -96,6 +105,8 @@ class AbstractJmClient( try: resp = request(url, **kwargs) return judge(resp) + except KeyboardInterrupt as e: + raise e except Exception as e: if self.retry_times == 0: raise e @@ -107,6 +118,12 @@ class AbstractJmClient( else: return self.request_with_retry(request, url, domain_index + 1, 0, judge, **kwargs) + def update_request_with_specify_domain(self, kwargs: dict, domain: str): + """ + 域名自动切换时,用于更新请求参数的回调 + """ + pass + # noinspection PyMethodMayBeStatic def log_topic(self): return self.client_key @@ -205,6 +222,34 @@ class JmHtmlClient(AbstractJmClient): func_to_cache = ['search', 'fetch_detail_entity'] + def add_favorite_album(self, + album_id, + folder_id='0', + ): + data = { + 'album_id': album_id, + 'fid': folder_id, + } + + resp = self.get_jm_html( + '/ajax/favorite_album', + data=data, + ) + + res = resp.json() + + if res['status'] != 1: + msg = parse_unicode_escape_text(res['msg']) + error_msg = PatternTool.match_or_default(msg, JmcomicText.pattern_ajax_favorite_msg, msg) + # 此圖片已經在您最喜愛的清單! + + self.raise_request_error( + resp, + error_msg + ) + + return resp + def get_album_detail(self, album_id) -> JmAlbumDetail: return self.fetch_detail_entity(album_id, 'album') @@ -301,6 +346,7 @@ class JmHtmlClient(AbstractJmClient): return resp self['cookies'] = new_cookies + self.__username = username return resp @@ -311,7 +357,8 @@ class JmHtmlClient(AbstractJmClient): username='', ) -> JmFavoritePage: if username == '': - username = self.get_username_or_raise() + ExceptionTool.require_true(self.__username is not None, 'favorite_folder方法需要传username参数') + username = self.__username resp = self.get_jm_html( f'/user/{username}/favorite/albums', @@ -325,13 +372,12 @@ class JmHtmlClient(AbstractJmClient): return JmPageTool.parse_html_to_favorite_page(resp.text) # noinspection PyTypeChecker - def get_username_or_raise(self) -> str: - cookies = self.get_meta_data('cookies', None) - if not cookies: - ExceptionTool.raises('未登录,无法获取到对应的用户名,需要传username参数') - + def get_username_from_cookies(self) -> str: + # cookies = self.get_meta_data('cookies', None) + # if not cookies: + # ExceptionTool.raises('未登录,无法获取到对应的用户名,请给favorite方法传入username参数') # 解析cookies,可能需要用到 phpserialize,比较麻烦,暂不实现 - ExceptionTool.raises('需要传username参数') + pass def get_jm_html(self, url, require_200=True, **kwargs): """ @@ -351,6 +397,12 @@ class JmHtmlClient(AbstractJmClient): return resp + def update_request_with_specify_domain(self, kwargs: dict, domain: Optional[str]): + latest_headers = kwargs.get('headers', None) + base_headers = self.get_meta_data('headers', None) or JmModuleConfig.new_html_headers(domain) + base_headers.update(latest_headers or {}) + kwargs['headers'] = base_headers + @classmethod def raise_request_error(cls, resp, msg: Optional[str] = None): """ @@ -393,10 +445,7 @@ class JmHtmlClient(AbstractJmClient): (f' to ({comment_id})' if comment_id is not None else '') ) - resp = self.post('/ajax/album_comment', - headers=self.album_comment_headers, - data=data, - ) + resp = self.post('/ajax/album_comment', data=data) ret = JmAlbumCommentResp(resp) jm_log('album.comment', f'{video_id}: [{comment}] ← ({ret.model().cid})') @@ -469,26 +518,6 @@ class JmHtmlClient(AbstractJmClient): + (f'URL=[{url}]' if url is not None else '') ) - album_comment_headers = { - 'authority': '18comic.vip', - 'accept': 'application/json, text/javascript, */*; q=0.01', - 'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7', - 'cache-control': 'no-cache', - 'content-type': 'application/x-www-form-urlencoded; charset=UTF-8', - 'origin': 'https://18comic.vip', - 'pragma': 'no-cache', - 'referer': 'https://18comic.vip/album/248965/', - 'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Windows"', - 'sec-fetch-dest': 'empty', - 'sec-fetch-mode': 'cors', - 'sec-fetch-site': 'same-origin', - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' - 'Chrome/114.0.0.0 Safari/537.36', - 'x-requested-with': 'XMLHttpRequest', - } - # 基于禁漫移动端(APP)实现的JmClient class JmApiClient(AbstractJmClient): @@ -581,8 +610,6 @@ class JmApiClient(AbstractJmClient): }, ) - self.require_resp_success(resp, url) - return JmApiAdaptTool.parse_entity(resp.res_data, clazz) def fetch_scramble_id(self, photo_id): @@ -600,6 +627,7 @@ class JmApiClient(AbstractJmClient): 'express': 'off', 'v': time_stamp(), }, + require_success=False, ) scramble_id = PatternTool.match_or_default(resp.text, @@ -712,7 +740,6 @@ class JmApiClient(AbstractJmClient): 'password': password, }) - resp.require_success() cookies = dict(resp.resp.cookies) cookies.update({'AVS': resp.res_data['s']}) self['cookies'] = cookies @@ -736,7 +763,34 @@ class JmApiClient(AbstractJmClient): return JmPageTool.parse_api_to_favorite_page(resp.model_data) - def req_api(self, url, get=True, **kwargs) -> JmApiResp: + def add_favorite_album(self, + album_id, + folder_id='0', + ): + """ + 移动端没有提供folder_id参数 + """ + resp = self.req_api( + '/favorite', + data={ + 'aid': album_id, + }, + ) + + self.require_resp_status_ok(resp) + + return resp + + # noinspection PyMethodMayBeStatic + def require_resp_status_ok(self, resp: JmApiResp): + """ + 检查返回数据中的status字段是否为ok + """ + data = resp.model_data + if data.status == 'ok': + ExceptionTool.raises_resp(data.msg, resp) + + def req_api(self, url, get=True, require_success=True, **kwargs) -> JmApiResp: ts = self.decide_headers_and_ts(kwargs, url) if get: @@ -744,7 +798,15 @@ class JmApiClient(AbstractJmClient): else: resp = self.post(url, **kwargs) - return JmApiResp(resp, ts) + resp = JmApiResp(resp, ts) + + if require_success: + self.require_resp_success(resp, url) + + return resp + + def update_request_with_specify_domain(self, kwargs: dict, domain: str): + pass # noinspection PyMethodMayBeStatic def decide_headers_and_ts(self, kwargs, url): @@ -791,7 +853,6 @@ class JmApiClient(AbstractJmClient): if JmModuleConfig.flag_api_client_require_cookies: self.ensure_have_cookies() - from threading import Lock client_init_cookies_lock = Lock() def ensure_have_cookies(self): @@ -826,9 +887,6 @@ class FutureClientProxy(JmcomicClient): ``` """ client_key = 'cl_proxy_future' - proxy_methods = ['album_comment', 'enable_cache', 'get_domain_list', - 'get_html_domain', 'get_html_domain_all', 'get_jm_image', - 'set_cache_dict', 'get_cache_dict', 'set_domain_list', ] class FutureWrapper: def __init__(self, future, after_done_callback): @@ -855,8 +913,7 @@ class FutureClientProxy(JmcomicClient): executors=None, ): self.client = client - for method in self.proxy_methods: - setattr(self, method, getattr(client, method)) + self.route_notimpl_method_to_internal_client(client) if executors is None: from concurrent.futures import ThreadPoolExecutor @@ -867,6 +924,25 @@ class FutureClientProxy(JmcomicClient): from threading import Lock self.lock = Lock() + def route_notimpl_method_to_internal_client(self, client): + + impl_methods = str_to_set(''' + get_album_detail + get_photo_detail + search + ''') + + # 获取对象的所有属性和方法的名称列表 + attributes_and_methods = dir(client) + # 遍历属性和方法列表,并访问每个方法 + for method in attributes_and_methods: + # 判断是否为方法(可调用对象) + if (not method.startswith('_') + and callable(getattr(client, method)) + and method not in impl_methods + ): + setattr(self, method, getattr(client, method)) + def get_album_detail(self, album_id) -> JmAlbumDetail: album_id = JmcomicText.parse_to_jm_id(album_id) cache_key = f'album_{album_id}' diff --git a/src/jmcomic/jm_client_interface.py b/src/jmcomic/jm_client_interface.py index 9e89fce..f4e052e 100644 --- a/src/jmcomic/jm_client_interface.py +++ b/src/jmcomic/jm_client_interface.py @@ -229,6 +229,15 @@ class JmUserClient: """ raise NotImplementedError + def add_favorite_album(self, + album_id, + folder_id='0', + ): + """ + 把漫画加入收藏夹 + """ + raise NotImplementedError + class JmImageClient: diff --git a/src/jmcomic/jm_config.py b/src/jmcomic/jm_config.py index 2c45323..6e1c46a 100644 --- a/src/jmcomic/jm_config.py +++ b/src/jmcomic/jm_config.py @@ -260,6 +260,7 @@ class JmModuleConfig: headers = JmMagicConstants.HTML_HEADERS_TEMPLATE.copy() headers.update({ 'authority': domain, + 'origin': f'https://{domain}', 'referer': f'https://{domain}', }) return headers @@ -296,9 +297,12 @@ class JmModuleConfig: return Postmans.new_postman(**kwargs) # option 相关的默认配置 + # 一般情况下,建议使用option配置文件来定制配置 + # 而如果只想修改几个简单常用的配置,也可以下方的DEFAULT_XXX属性 JM_OPTION_VER = '2.1' - DEFAULT_CLIENT_IMPL = 'html' - DEFAULT_PROXIES = ProxyBuilder.system_proxy() # use system proxy by default + DEFAULT_CLIENT_IMPL = 'html' # 默认Client实现类型为网页端 + DEFAULT_CLIENT_CACHE = True # 默认开启Client缓存,缓存级别是level_option,详见CacheRegistry + DEFAULT_PROXIES = ProxyBuilder.system_proxy() # 默认使用系统代理 default_option_dict: dict = { 'log': None, @@ -355,7 +359,7 @@ class JmModuleConfig: # client cache client = option_dict['client'] if client['cache'] is None: - client['cache'] = True + client['cache'] = cls.DEFAULT_CLIENT_CACHE # client impl if client['impl'] is None: diff --git a/src/jmcomic/jm_downloader.py b/src/jmcomic/jm_downloader.py index 733380b..9a3d718 100644 --- a/src/jmcomic/jm_downloader.py +++ b/src/jmcomic/jm_downloader.py @@ -115,7 +115,7 @@ class JmDownloader(DownloadCallback): """ 调度本子/章节的下载 """ - iter_objs = self.filter_iter_objs(iter_objs) + iter_objs = self.do_filter(iter_objs) count_real = len(iter_objs) if count_real == 0: @@ -136,14 +136,14 @@ class JmDownloader(DownloadCallback): ) # noinspection PyMethodMayBeStatic - def filter_iter_objs(self, detail: DetailEntity): + def do_filter(self, detail: DetailEntity): """ 该方法可用于过滤本子/章节,默认不会做过滤。 例如: 只想下载 本子的最新一章,返回 [album[-1]] 只想下载 章节的前10张图片,返回 [photo[:10]] - :param detail: 可能是本子或者章节,需要自行使用 isinstance / is_xxx 判断 + :param detail: 可能是本子或者章节,需要自行使用 isinstance / detail.is_xxx 判断 :returns: 只想要下载的 本子的章节 或 章节的图片 """ return detail @@ -198,3 +198,18 @@ class JmDownloader(DownloadCallback): jm_log('dler.exception', f'{self.__class__.__name__} Exit with exception: {exc_type, exc_val}' ) + + +class DoNotDownloadImage(JmDownloader): + """ + 本类仅用于测试 + + 用法: + + JmModuleConfig.CLASS_DOWNLOADER = DoNotDownloadImage + """ + + def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient): + # ensure make dir + self.option.decide_image_filepath(image) + pass diff --git a/src/jmcomic/jm_entity.py b/src/jmcomic/jm_entity.py index 7a6b6e8..f060e2d 100644 --- a/src/jmcomic/jm_entity.py +++ b/src/jmcomic/jm_entity.py @@ -175,8 +175,8 @@ class JmImageDetail(JmBaseEntity): self.img_file_suffix: str = img_file_suffix self.from_photo: Optional[JmPhotoDetail] = from_photo - self.query_params: StrNone = query_params - self.index = index # 从1开始 + self.query_params: Optional[str] = query_params + self.index = index # 从1开始 # temp fields, in order to simplify passing parameter self.save_path: str = '' @@ -266,7 +266,7 @@ class JmPhotoDetail(DetailEntity): self._tags: str = tags self._series_id: int = int(series_id) - self._author: StrNone = author + self._author: Optional[str] = author self.from_album: Optional[JmAlbumDetail] = from_album self.index = self.album_index @@ -278,7 +278,7 @@ class JmPhotoDetail(DetailEntity): # page_arr存放了该photo的所有图片文件名 img_name self.page_arr: List[str] = page_arr # 图片的cdn域名 - self.data_original_domain: StrNone = data_original_domain + self.data_original_domain: Optional[str] = data_original_domain # 第一张图的URL self.data_original_0 = data_original_0 @@ -372,7 +372,7 @@ class JmPhotoDetail(DetailEntity): return f'{JmModuleConfig.PROT}{domain}/media/photos/{self.photo_id}/{img_name}' # noinspection PyMethodMayBeStatic - def get_data_original_query_params(self, data_original_0: StrNone) -> str: + def get_data_original_query_params(self, data_original_0: Optional[str]) -> str: if data_original_0 is None: return f'v={time_stamp()}' @@ -534,12 +534,18 @@ class JmPageContent(JmBaseEntity, IndexedEntity): @property def page_count(self) -> int: + """ + 页数 + """ page_size = self.page_size import math return math.ceil(int(self.total) / page_size) @property def page_size(self) -> int: + """ + 页大小 + """ raise NotImplementedError def iter_id(self) -> Generator[str, None, None]: diff --git a/src/jmcomic/jm_option.py b/src/jmcomic/jm_option.py index eb79e2e..379e391 100644 --- a/src/jmcomic/jm_option.py +++ b/src/jmcomic/jm_option.py @@ -276,22 +276,10 @@ class JmOption: return JmModuleConfig.option_default_dict() @classmethod - def default(cls, proxies=None, domain=None) -> 'JmOption': + def default(cls) -> 'JmOption': """ 使用默认的 JmOption - proxies, domain 为常用配置项,为了方便起见直接支持参数配置。 - 其他配置项建议还是使用配置文件 - :param proxies: clash; 127.0.0.1:7890; v2ray - :param domain: 18comic.vip; ["18comic.vip"] """ - if proxies is not None or domain is not None: - return cls.construct({ - 'client': { - 'domain': [domain] if isinstance(domain, str) else domain, - 'postman': {'meta_data': {'proxies': ProxyBuilder.build_by_str(proxies)}}, - }, - }) - return cls.construct({}) @classmethod @@ -372,7 +360,7 @@ class JmOption: """ return self.new_jm_client(**kwargs) - def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient: + def new_jm_client(self, domain_list=None, impl=None, cache=None, **kwargs) -> JmcomicClient: """ 创建新的Client(客户端),不同Client之间的元数据不共享 """ @@ -380,10 +368,15 @@ class JmOption: # 所有需要用到的 self.client 配置项如下 postman_conf: dict = deepcopy(self.client.postman.src_dict) # postman dsl 配置 + meta_data: dict = postman_conf['meta_data'] # 元数据 + retry_times: int = self.client.retry_times # 重试次数 + cache: str = cache if cache is not None else self.client.cache # 启用缓存 + impl: str = impl or self.client.impl # client_key + if isinstance(impl, type): # eg: impl = JmHtmlClient # noinspection PyUnresolvedReferences @@ -392,28 +385,30 @@ class JmOption: # start construct client # domain - def decide_domain(): - domain_list: Union[List[str], DictModel, dict] = domain if domain is not None \ - else self.client.domain # 域名 + def decide_domain_list(): + nonlocal domain_list - if not isinstance(domain_list, list): + if domain_list is None: + domain_list = self.client.domain + + if not isinstance(domain_list, (list, str)): + # dict domain_list = domain_list.get(impl, []) + if isinstance(domain_list, str): + # multi-lines text + domain_list = str_to_list(domain_list) + + # list or str if len(domain_list) == 0: domain_list = self.decide_client_domain(impl) return domain_list - domain: List[str] = decide_domain() - # support kwargs overwrite meta_data if len(kwargs) != 0: meta_data.update(kwargs) - # headers - if meta_data['headers'] is None: - meta_data['headers'] = self.decide_postman_headers(impl, domain[0]) - # postman postman = Postmans.create(data=postman_conf) @@ -424,7 +419,7 @@ class JmOption: client: AbstractJmClient = clazz( postman=postman, - domain_list=domain, + domain_list=decide_domain_list(), retry_times=retry_times, ) @@ -459,20 +454,6 @@ class JmOption: ExceptionTool.raises(f'没有配置域名,且是无法识别的client类型: {client_key}') - def decide_postman_headers(self, client_key, domain): - is_client_type = lambda ctype: self.client_key_is_given_type(client_key, ctype) - - if is_client_type(JmApiClient): - # 移动端 - # 不配置headers,由client每次请求前创建headers - return None - - if is_client_type(JmHtmlClient): - # 网页端 - return JmModuleConfig.new_html_headers(domain) - - ExceptionTool.raises(f'没有配置域名,且是无法识别的client类型: {client_key}') - @classmethod def client_key_is_given_type(cls, client_key, ctype: Type[JmcomicClient]): if client_key == ctype.client_key: diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py index f707890..6ef9960 100644 --- a/src/jmcomic/jm_plugin.py +++ b/src/jmcomic/jm_plugin.py @@ -208,7 +208,7 @@ class FindUpdatePlugin(JmOptionPlugin): return photo_ls class FindUpdateDownloader(JmDownloader): - def filter_iter_objs(self, detail): + def do_filter(self, detail): if not detail.is_album(): return detail diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py index ec4b0f1..90f64fd 100644 --- a/src/jmcomic/jm_toolkit.py +++ b/src/jmcomic/jm_toolkit.py @@ -55,6 +55,9 @@ class JmcomicText: # 評論(div) pattern_html_album_comment_count = compile(r'
]*?id="total_video_comments">(\d+)
'), 0 + # 提取接口返回值信息 + pattern_ajax_favorite_msg = compile(r'(.*?)') + @classmethod def parse_to_jm_domain(cls, text: str): if text.startswith(JmModuleConfig.PROT): @@ -308,7 +311,7 @@ class PatternTool: def require_match(cls, html: str, pattern: Pattern, msg, rindex=1): match = pattern.search(html) if match is not None: - return match[rindex] + return match[rindex] if rindex is not None else match ExceptionTool.raises_regex( msg, diff --git a/tests/test_jmcomic/test_jm_custom.py b/tests/test_jmcomic/test_jm_custom.py index d1946dc..a7e050b 100644 --- a/tests/test_jmcomic/test_jm_custom.py +++ b/tests/test_jmcomic/test_jm_custom.py @@ -45,7 +45,7 @@ class Test_Custom(JmTestConfigurable): self.assertListEqual( JmModuleConfig.DOMAIN_API_LIST, - self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list() + self.option.new_jm_client(domain_list=[], impl=MyClient.client_key).get_domain_list() ) def test_extends_html_client(self): @@ -59,7 +59,7 @@ class Test_Custom(JmTestConfigurable): self.assertListEqual( JmModuleConfig.DOMAIN_HTML_LIST, - self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list() + self.option.new_jm_client(domain_list=[], impl=MyClient.client_key).get_domain_list() ) def test_client_key_missing(self): @@ -74,7 +74,7 @@ class Test_Custom(JmTestConfigurable): ) def test_custom_client_empty_domain(self): - class MyClient(JmcomicClient): + class MyClient(AbstractJmClient): client_key = 'myclient' pass @@ -95,6 +95,6 @@ class Test_Custom(JmTestConfigurable): JmModuleConfig.register_client(MyClient) self.assertListEqual( JmModuleConfig.DOMAIN_API_LIST, - self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list(), + self.option.new_jm_client(domain_list=[], impl=MyClient.client_key).get_domain_list(), msg='继承client,不配置域名', ) diff --git a/usage/workflow_download.py b/usage/workflow_download.py index 4baa98f..7f33453 100644 --- a/usage/workflow_download.py +++ b/usage/workflow_download.py @@ -1,5 +1,5 @@ from jmcomic import * -from jmcomic.cl import get_env, JmcomicUI +from jmcomic.cl import JmcomicUI # 下方填入你要下载的本子的id,一行一个,每行的首尾可以有空白字符 jm_albums = ''' @@ -16,11 +16,24 @@ jm_photos = ''' ''' +def env(name, default, trim=('[]', '""', "''")): + import os + value = os.getenv(name, None) + if value is None or value == '': + return default + + for pair in trim: + if value.startswith(pair[0]) and value.endswith(pair[1]): + value = value[1:-1] + + return value + + def get_id_set(env_name): aid_set = set() for text in [ jm_albums, - (get_env(env_name, '')).replace('-', '\n'), + (env(env_name, '')).replace('-', '\n'), ]: aid_set.update(str_to_set(text)) @@ -39,6 +52,7 @@ def main(): helper.run(option) option.call_all_plugin('after_download') + def get_option(): # 读取 option 配置文件 option = create_option('../assets/option/option_workflow_download.yml') @@ -53,23 +67,23 @@ def get_option(): def cover_option_config(option: JmOption): - dir_rule = get_env('DIR_RULE', None) + dir_rule = env('DIR_RULE', None) if dir_rule is not None: the_old = option.dir_rule the_new = DirRule(dir_rule, base_dir=the_old.base_dir) option.dir_rule = the_new - impl = get_env('CLIENT_IMPL', None) + impl = env('CLIENT_IMPL', None) if impl is not None: option.client.impl = impl - suffix = get_env('IMAGE_SUFFIX', None) + suffix = env('IMAGE_SUFFIX', None) if suffix is not None: option.download.image.suffix = fix_suffix(suffix) def log_before_raise(): - jm_download_dir = get_env('JM_DOWNLOAD_DIR', workspace()) + jm_download_dir = env('JM_DOWNLOAD_DIR', workspace()) mkdir_if_not_exists(jm_download_dir) # 自定义异常抛出函数,在抛出前把HTML响应数据写到下载文件夹(日志留痕)