From 87ed6bce83a298f90a2dd83d76ad93a9053b50dd Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Mon, 30 Oct 2023 20:35:27 +0800 Subject: [PATCH] =?UTF-8?q?v2.3.15:=20=E4=BF=AE=E5=A4=8D=E7=A7=BB=E5=8A=A8?= =?UTF-8?q?=E7=AB=AF=E6=9C=AA=E6=90=BA=E5=B8=A6cookies=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E8=B7=B3=E8=BD=AC=E5=90=8C=E6=9C=AC=E9=97=AE=E9=A2=98;=20?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8F=92=E4=BB=B6=E3=80=90debug=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E4=B8=BB=E9=A2=98=E8=BF=87=E6=BB=A4=E5=99=A8=E3=80=91?= =?UTF-8?q?;=20=E6=96=B0=E5=A2=9E=E6=8F=92=E4=BB=B6=E5=8F=82=E6=95=B0?= =?UTF-8?q?=E6=A0=A1=E9=AA=8C=E5=BC=82=E5=B8=B8=E7=B1=BB=E5=8F=8A=E5=85=B6?= =?UTF-8?q?=E7=9B=B8=E5=BA=94=E7=9A=84=E5=BC=82=E5=B8=B8=E5=A4=84=E7=90=86?= =?UTF-8?q?=E7=AD=96=E7=95=A5=20(ignore,=20debug,=20raise)=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/release.py | 43 +++++++++++---- .github/workflows/release_auto.yml | 2 +- README.md | 6 +-- assets/docs/sources/index.md | 2 +- requirements-dev.txt | 3 +- src/jmcomic/__init__.py | 2 +- src/jmcomic/jm_client_impl.py | 12 ++++- src/jmcomic/jm_config.py | 10 ++-- src/jmcomic/jm_downloader.py | 4 -- src/jmcomic/jm_option.py | 87 ++++++++++++++++++++++-------- src/jmcomic/jm_plugin.py | 57 ++++++++++++++++---- 11 files changed, 174 insertions(+), 54 deletions(-) diff --git a/.github/release.py b/.github/release.py index 0df89d7..4493df4 100644 --- a/.github/release.py +++ b/.github/release.py @@ -4,17 +4,42 @@ import re def add_output(k, v): - cmd = f'echo {k}="{v}" >> $GITHUB_OUTPUT' + cmd = f'echo "{k}={v}" >> $GITHUB_OUTPUT' print(cmd, os.system(cmd)) -msg = sys.argv[1] -print(f'msg: {msg}') -p = re.compile('(.*?): ?(.*)') -match = p.search(msg) -assert match is not None, f'commit message format is wrong: {msg}' +def parse_body(body): + if ';' not in body: + return body -tag, body = match[1], match[2] + parts = body.split(";") + points = [] + for i, e in enumerate(parts): + e: str = e.strip() + if e == '': + continue + points.append(f'{i}. {e}') -add_output('tag', tag) -add_output('body', body) + return '\n'.join(points) + + +def get_tag_and_body(): + msg = sys.argv[1] + print(f'msg: {msg}') + p = re.compile('(.*?): ?(.*)') + match = p.search(msg) + assert match is not None, f'commit message format is wrong: {msg}' + tag, body = match[1], match[2] + return body, tag + + +def main(): + body, tag = get_tag_and_body() + + add_output('tag', tag) + + with open('release_body.txt', 'w', encoding='utf-8') as f: + f.write(parse_body(body)) + + +main() diff --git a/.github/workflows/release_auto.yml b/.github/workflows/release_auto.yml index 83e4bd0..061b92e 100644 --- a/.github/workflows/release_auto.yml +++ b/.github/workflows/release_auto.yml @@ -31,7 +31,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ steps.tb.outputs.tag }} - body: ${{ steps.tb.outputs.body }} + body_path: release_body.txt - name: Build run: | diff --git a/README.md b/README.md index ef89959..998134f 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ $ jmcomic 422866 - GitHub Actions:网页上直接输入本子id就能下载([教程:使用GitHub Actions下载禁漫本子](./assets/docs/sources/tutorial/1_github_actions.md)) - 命令行:无需写Python代码,简单易用([教程:使用命令行下载禁漫本子](./assets/docs/sources/tutorial/2_command_line.md)) - - Python代码:最直接的使用方式,需要你有一定的python编程基础 + - Python代码:最本质、最强大的使用方式,需要你有一定的python编程基础 - 支持**网页端**和**移动端**两种客户端实现,可通过配置切换(**移动端不限ip兼容性好,网页端限制ip地区但效率高**) - 支持**自动重试和域名切换**机制 - **多线程下载**(可细化到一图一线程,效率极高) @@ -54,9 +54,9 @@ $ jmcomic 422866 - **可扩展性强** - **支持Plugin插件,可以方便地扩展功能,以及使用别人的插件** - - 目前内置支持的插件有:`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` `下载特定后缀图片插件` `发送QQ邮件插件` + - 目前内置支持的插件有:`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` `下载特定后缀图片插件` `发送QQ邮件插件` `日志主题过滤插件` - 支持自定义本子/章节/图片下载前后的回调函数 - - 支持自定义debug日志 + - 支持自定义debug/logging - 支持自定义类:`Downloader(负责调度)` `Option(负责配置)` `Client(负责请求)` `实体类`等 ## 进阶使用 diff --git a/assets/docs/sources/index.md b/assets/docs/sources/index.md index 2e777c6..4a9aa40 100644 --- a/assets/docs/sources/index.md +++ b/assets/docs/sources/index.md @@ -26,7 +26,7 @@ Python API for JMComic(禁漫天堂) - Supports Plugin plugins for easy functionality extension and use of other plugins. - Currently built-in - plugins: `login plugin`, `hardware usage monitoring plugin`, `only download new chapters plugin`, `zip compression plugin`, `image suffix filter plugin` `send qq email plugin`. + plugins: `login plugin`, `hardware usage monitoring plugin`, `only download new chapters plugin`, `zip compression plugin`, `image suffix filter plugin` `send qq email plugin` `debug logging topic filter plugin`. - Supports custom callback functions before and after downloading album/chapter/images. - Supports custom debug logging. - Supports custom core diff --git a/requirements-dev.txt b/requirements-dev.txt index a3fbf81..816cd87 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,4 +3,5 @@ curl_cffi PyYAML Pillow psutil -pycryptodome \ No newline at end of file +pycryptodome +requests \ No newline at end of file diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index dc43756..0a3ab30 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,7 +2,7 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.3.14' +__version__ = '2.3.15' from .api import * from .jm_plugin import * diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py index 365927a..3847455 100644 --- a/src/jmcomic/jm_client_impl.py +++ b/src/jmcomic/jm_client_impl.py @@ -584,7 +584,7 @@ class JmApiClient(AbstractJmClient): def get_decode(self, url, **kwargs) -> JmApiResp: # set headers headers, key_ts = self.headers_key_ts - kwargs.setdefault('headers', headers) + kwargs['headers'] = headers resp = self.get(url, **kwargs) return JmApiResp.wrap(resp, key_ts) @@ -610,6 +610,16 @@ class JmApiClient(AbstractJmClient): # 2. 是否是特殊的内容 # 暂无 + @classmethod + @field_cache('__init_cookies__') + def fetch_init_cookies(cls, client: 'JmApiClient'): + resp = client.setting() + return dict(resp.resp.cookies) + + def after_init(self): + cookies = self.__class__.fetch_init_cookies(self) + self.get_root_postman().get_meta_data()['cookies'] = cookies + class FutureClientProxy(JmcomicClient): """ diff --git a/src/jmcomic/jm_config.py b/src/jmcomic/jm_config.py index ecbefeb..467aa8b 100644 --- a/src/jmcomic/jm_config.py +++ b/src/jmcomic/jm_config.py @@ -64,7 +64,7 @@ class JmModuleConfig: # 移动端API的相关配置 # API密钥 - MAGIC_18COMICAPPCONTENT = '18comicAPPContent' + APP_SECRET = '18comicAPP' # 域名配置 - 移动端 # 图片域名 @@ -247,7 +247,7 @@ class JmModuleConfig: key_ts = time_stamp() import hashlib - token = hashlib.md5(f"{key_ts}{cls.MAGIC_18COMICAPPCONTENT}".encode()).hexdigest() + token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode()).hexdigest() return { 'token': token, @@ -331,7 +331,11 @@ class JmModuleConfig: 'impl': None, 'retry_times': 5 }, - 'plugins': {}, + 'plugins': { + # 如果插件抛出参数校验异常,只debug。(全局配置,可以被插件的局部配置覆盖) + # 可选值:ignore(忽略),debug(打印日志),raise(抛异常)。 + 'valid': 'debug', + }, } @classmethod diff --git a/src/jmcomic/jm_downloader.py b/src/jmcomic/jm_downloader.py index b7de67a..b016d6f 100644 --- a/src/jmcomic/jm_downloader.py +++ b/src/jmcomic/jm_downloader.py @@ -9,10 +9,6 @@ DownloadIterObjs = Union[ ] -class JmDownloadException(Exception): - pass - - # noinspection PyMethodMayBeStatic class DownloadCallback: diff --git a/src/jmcomic/jm_option.py b/src/jmcomic/jm_option.py index 2426676..72501a4 100644 --- a/src/jmcomic/jm_option.py +++ b/src/jmcomic/jm_option.py @@ -311,12 +311,12 @@ class JmOption: """ return self.new_jm_client(**kwargs) - def new_jm_client(self, domain=None, impl=None, **kwargs) -> JmcomicClient: + def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient: # 所有需要用到的 self.client 配置项如下 postman_conf: dict = self.client.postman.src_dict # postman dsl 配置 impl: str = impl or self.client.impl # client_key retry_times: int = self.client.retry_times # 重试次数 - cache: str = self.client.cache # 启用缓存 + cache: str = cache or self.client.cache # 启用缓存 # domain def decide_domain(): @@ -340,7 +340,10 @@ class JmOption: # headers meta_data = postman_conf['meta_data'] if meta_data['headers'] is None: - meta_data['headers'] = self.decide_postman_headers(impl, domain[0]) + headers = self.decide_postman_headers(impl, domain[0]) + # if headers is None: + # postman_conf['type'] = 'requests' + meta_data['headers'] = headers # postman postman = Postmans.create(data=postman_conf) @@ -349,7 +352,8 @@ class JmOption: clazz = JmModuleConfig.client_impl_class(impl) if clazz == AbstractJmClient or not issubclass(clazz, AbstractJmClient): raise NotImplementedError(clazz) - client = clazz( + + client: AbstractJmClient = clazz( postman, retry_times, fallback_domain_list=decide_domain(), @@ -442,36 +446,77 @@ class JmOption: ExceptionTool.require_true(plugin_class is not None, f'[{group}] 未注册的plugin: {key}') - self.invoke_plugin(plugin_class, kwargs, extra) + self.invoke_plugin(plugin_class, kwargs, extra, pinfo) + + def invoke_plugin(self, plugin_class, kwargs: Any, extra: dict, pinfo: dict): + # 检查插件的参数类型 + kwargs = self.fix_kwargs(kwargs) + # 把插件的配置数据kwargs和附加数据extra合并,extra会覆盖kwargs + if len(extra) != 0: + kwargs.update(extra) - def invoke_plugin(self, plugin_class, kwargs: Any, extra: dict): # 保证 jm_plugin.py 被加载 - from .jm_plugin import JmOptionPlugin + from .jm_plugin import JmOptionPlugin, PluginValidationException + plugin = plugin_class plugin_class: Type[JmOptionPlugin] - pkey = plugin_class.plugin_key try: - # 检查插件的参数类型 - kwargs = self.fix_kwargs(kwargs) - # 把插件的配置数据kwargs和附加数据extra合并 - # extra会覆盖kwargs - if len(extra) != 0: - kwargs.update(extra) # 构建插件对象 - plugin = plugin_class.build(self) + plugin: JmOptionPlugin = plugin_class.build(self) + + jm_debug('plugin.invoke', f'调用插件: [{plugin_class.plugin_key}]') # 调用插件功能 - jm_debug('plugin.invoke', f'调用插件: [{pkey}]') plugin.invoke(**kwargs) + + except PluginValidationException as e: + # 插件抛出的参数校验异常 + self.handle_plugin_valid_exception(e, pinfo, kwargs, plugin) + except JmcomicException as e: - msg = str(e) - jm_debug('plugin.exception', f'插件[{pkey}]调用失败,异常信息: {msg}') - raise e + # 模块内部异常,通过不是插件抛出的,而是插件调用了例如Client,Client请求失败抛出的 + self.handle_plugin_exception(e, pinfo, kwargs, plugin) + except BaseException as e: - msg = str(e) - jm_debug('plugin.error', f'插件[{pkey}]运行遇到未捕获异常,异常信息: {msg}') + # 为插件兜底,捕获其他所有异常 + self.handle_plugin_unexpected_error(e, pinfo, kwargs, plugin) + + # noinspection PyMethodMayBeStatic,PyUnusedLocal + def handle_plugin_valid_exception(self, e, pinfo: dict, kwargs: dict, plugin): + from .jm_plugin import PluginValidationException + e: PluginValidationException + + mode = pinfo.get('valid', self.plugins.valid) + + if mode == 'ignore': + # ignore + return + + if mode == 'debug': + # debug + jm_debug('plugin.validation', + f'插件 [{e.plugin.plugin_key}] 参数校验异常:{e.msg}' + ) + return + + if mode == 'raise': + # raise raise e + # 其他的mode可以通过继承+方法重写来扩展 + + # noinspection PyMethodMayBeStatic,PyUnusedLocal + def handle_plugin_unexpected_error(self, e, pinfo: dict, kwargs: dict, plugin): + msg = str(e) + jm_debug('plugin.error', f'插件 [{plugin.plugin_key}],运行遇到未捕获异常,异常信息: {msg}') + raise e + + # noinspection PyMethodMayBeStatic,PyUnusedLocal + def handle_plugin_exception(self, e, pinfo: dict, kwargs: dict, plugin): + msg = str(e) + jm_debug('plugin.exception', f'插件 [{plugin.plugin_key}],调用失败,异常信息: {msg}') + raise e + # noinspection PyMethodMayBeStatic def fix_kwargs(self, kwargs) -> Dict[str, Any]: """ diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py index dd19250..d936a61 100644 --- a/src/jmcomic/jm_plugin.py +++ b/src/jmcomic/jm_plugin.py @@ -5,6 +5,13 @@ from .jm_option import * +class PluginValidationException(Exception): + + def __init__(self, plugin: 'JmOptionPlugin', msg: str): + self.plugin = plugin + self.msg = msg + + class JmOptionPlugin: plugin_key: str @@ -33,6 +40,15 @@ class JmOptionPlugin: msg=msg ) + def require_true(self, case: Any, msg: str): + """ + 独立于ExceptionTool的一套异常抛出体系 + """ + if case: + return + + raise PluginValidationException(self, msg) + class JmLoginPlugin(JmOptionPlugin): """ @@ -44,8 +60,8 @@ class JmLoginPlugin(JmOptionPlugin): username: str, password: str, ) -> None: - if not (username and password): - return + self.require_true(username, '用户名不能为空') + self.require_true(password, '密码不能为空') client = self.option.new_jm_client() client.login(username, password) @@ -326,14 +342,19 @@ class ZipPlugin(JmOptionPlugin): 删除所有文件和文件夹 """ import os - for album, photo_dict in all_downloaded.items(): - for photo, image_list in photo_dict.items(): - for f, image in image_list: + for photo_dict in all_downloaded.values(): + for image_list in photo_dict.values(): + for f, _ in image_list: + # check not exist + if file_not_exists(f): + continue + os.remove(f) self.debug(f'删除原文件: {f}', 'remove') for d in dir_list: - if len(os.listdir(d)) == 0: + # check exist + if file_exists(d) and len(os.listdir(d)) == 0: os.removedirs(d) self.debug(f'删除文件夹: {d}', 'remove') @@ -403,11 +424,29 @@ class SendQQEmailPlugin(JmOptionPlugin): album=None, downloader=None, ) -> None: - if not (msg_from and msg_to and password): - self.debug('发送邮件的相关参数为空,不处理') - return + self.require_true(msg_from and msg_to and password, '发件人、收件人、授权码都不能为空') + from common import EmailConfig econfig = EmailConfig(msg_from, msg_to, password) epostman = econfig.create_email_postman() epostman.send(content, title) + self.debug('Email sent successfully') + + +class DebugTopicFilterPlugin(JmOptionPlugin): + plugin_key = 'debug_topic_filter' + + def invoke(self, whitelist) -> None: + if whitelist is not None: + whitelist = set(whitelist) + + old_jm_debug = JmModuleConfig.debug_executor + + def new_jm_debug(topic, msg): + if whitelist is not None and topic not in whitelist: + return + + old_jm_debug(topic, msg) + + JmModuleConfig.debug_executor = new_jm_debug