v2.3.15: 修复移动端未携带cookies导致跳转同本问题; 新增插件【debug日志主题过滤器】; 新增插件参数校验异常类及其相应的异常处理策略 (ignore, debug, raise)。

This commit is contained in:
hect0x7 2023-10-30 20:35:27 +08:00
parent de0e0ba729
commit 87ed6bce83
11 changed files with 174 additions and 54 deletions

43
.github/release.py vendored
View File

@ -4,17 +4,42 @@ import re
def add_output(k, v):
cmd = f'echo {k}="{v}" >> $GITHUB_OUTPUT'
cmd = f'echo "{k}={v}" >> $GITHUB_OUTPUT'
print(cmd, os.system(cmd))
msg = sys.argv[1]
print(f'msg: {msg}')
p = re.compile('(.*?): ?(.*)')
match = p.search(msg)
assert match is not None, f'commit message format is wrong: {msg}'
def parse_body(body):
if ';' not in body:
return body
tag, body = match[1], match[2]
parts = body.split(";")
points = []
for i, e in enumerate(parts):
e: str = e.strip()
if e == '':
continue
points.append(f'{i}. {e}')
add_output('tag', tag)
add_output('body', body)
return '\n'.join(points)
def get_tag_and_body():
msg = sys.argv[1]
print(f'msg: {msg}')
p = re.compile('(.*?): ?(.*)')
match = p.search(msg)
assert match is not None, f'commit message format is wrong: {msg}'
tag, body = match[1], match[2]
return body, tag
def main():
body, tag = get_tag_and_body()
add_output('tag', tag)
with open('release_body.txt', 'w', encoding='utf-8') as f:
f.write(parse_body(body))
main()

View File

@ -31,7 +31,7 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ steps.tb.outputs.tag }}
body: ${{ steps.tb.outputs.body }}
body_path: release_body.txt
- name: Build
run: |

View File

@ -42,7 +42,7 @@ $ jmcomic 422866
- GitHub Actions网页上直接输入本子id就能下载[教程使用GitHub Actions下载禁漫本子](./assets/docs/sources/tutorial/1_github_actions.md)
- 命令行无需写Python代码简单易用[教程:使用命令行下载禁漫本子](./assets/docs/sources/tutorial/2_command_line.md)
- Python代码直接的使用方式需要你有一定的python编程基础
- Python代码本质、最强大的使用方式需要你有一定的python编程基础
- 支持**网页端**和**移动端**两种客户端实现,可通过配置切换(**移动端不限ip兼容性好网页端限制ip地区但效率高**
- 支持**自动重试和域名切换**机制
- **多线程下载**(可细化到一图一线程,效率极高)
@ -54,9 +54,9 @@ $ jmcomic 422866
- **可扩展性强**
- **支持Plugin插件可以方便地扩展功能以及使用别人的插件**
- 目前内置支持的插件有:`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` `下载特定后缀图片插件` `发送QQ邮件插件`
- 目前内置支持的插件有:`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` `下载特定后缀图片插件` `发送QQ邮件插件` `日志主题过滤插件`
- 支持自定义本子/章节/图片下载前后的回调函数
- 支持自定义debug日志
- 支持自定义debug/logging
- 支持自定义类:`Downloader负责调度` `Option负责配置` `Client负责请求` `实体类`
## 进阶使用

View File

@ -26,7 +26,7 @@ Python API for JMComic禁漫天堂
- Supports Plugin plugins for easy functionality extension and use of other plugins.
- Currently built-in
plugins: `login plugin`, `hardware usage monitoring plugin`, `only download new chapters plugin`, `zip compression plugin`, `image suffix filter plugin` `send qq email plugin`.
plugins: `login plugin`, `hardware usage monitoring plugin`, `only download new chapters plugin`, `zip compression plugin`, `image suffix filter plugin` `send qq email plugin` `debug logging topic filter plugin`.
- Supports custom callback functions before and after downloading album/chapter/images.
- Supports custom debug logging.
- Supports custom core

View File

@ -3,4 +3,5 @@ curl_cffi
PyYAML
Pillow
psutil
pycryptodome
pycryptodome
requests

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.3.14'
__version__ = '2.3.15'
from .api import *
from .jm_plugin import *

View File

@ -584,7 +584,7 @@ class JmApiClient(AbstractJmClient):
def get_decode(self, url, **kwargs) -> JmApiResp:
# set headers
headers, key_ts = self.headers_key_ts
kwargs.setdefault('headers', headers)
kwargs['headers'] = headers
resp = self.get(url, **kwargs)
return JmApiResp.wrap(resp, key_ts)
@ -610,6 +610,16 @@ class JmApiClient(AbstractJmClient):
# 2. 是否是特殊的内容
# 暂无
@classmethod
@field_cache('__init_cookies__')
def fetch_init_cookies(cls, client: 'JmApiClient'):
resp = client.setting()
return dict(resp.resp.cookies)
def after_init(self):
cookies = self.__class__.fetch_init_cookies(self)
self.get_root_postman().get_meta_data()['cookies'] = cookies
class FutureClientProxy(JmcomicClient):
"""

View File

@ -64,7 +64,7 @@ class JmModuleConfig:
# 移动端API的相关配置
# API密钥
MAGIC_18COMICAPPCONTENT = '18comicAPPContent'
APP_SECRET = '18comicAPP'
# 域名配置 - 移动端
# 图片域名
@ -247,7 +247,7 @@ class JmModuleConfig:
key_ts = time_stamp()
import hashlib
token = hashlib.md5(f"{key_ts}{cls.MAGIC_18COMICAPPCONTENT}".encode()).hexdigest()
token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode()).hexdigest()
return {
'token': token,
@ -331,7 +331,11 @@ class JmModuleConfig:
'impl': None,
'retry_times': 5
},
'plugins': {},
'plugins': {
# 如果插件抛出参数校验异常只debug。全局配置可以被插件的局部配置覆盖
# 可选值ignore忽略debug打印日志raise抛异常
'valid': 'debug',
},
}
@classmethod

View File

@ -9,10 +9,6 @@ DownloadIterObjs = Union[
]
class JmDownloadException(Exception):
pass
# noinspection PyMethodMayBeStatic
class DownloadCallback:

View File

@ -311,12 +311,12 @@ class JmOption:
"""
return self.new_jm_client(**kwargs)
def new_jm_client(self, domain=None, impl=None, **kwargs) -> JmcomicClient:
def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient:
# 所有需要用到的 self.client 配置项如下
postman_conf: dict = self.client.postman.src_dict # postman dsl 配置
impl: str = impl or self.client.impl # client_key
retry_times: int = self.client.retry_times # 重试次数
cache: str = self.client.cache # 启用缓存
cache: str = cache or self.client.cache # 启用缓存
# domain
def decide_domain():
@ -340,7 +340,10 @@ class JmOption:
# headers
meta_data = postman_conf['meta_data']
if meta_data['headers'] is None:
meta_data['headers'] = self.decide_postman_headers(impl, domain[0])
headers = self.decide_postman_headers(impl, domain[0])
# if headers is None:
# postman_conf['type'] = 'requests'
meta_data['headers'] = headers
# postman
postman = Postmans.create(data=postman_conf)
@ -349,7 +352,8 @@ class JmOption:
clazz = JmModuleConfig.client_impl_class(impl)
if clazz == AbstractJmClient or not issubclass(clazz, AbstractJmClient):
raise NotImplementedError(clazz)
client = clazz(
client: AbstractJmClient = clazz(
postman,
retry_times,
fallback_domain_list=decide_domain(),
@ -442,36 +446,77 @@ class JmOption:
ExceptionTool.require_true(plugin_class is not None, f'[{group}] 未注册的plugin: {key}')
self.invoke_plugin(plugin_class, kwargs, extra)
self.invoke_plugin(plugin_class, kwargs, extra, pinfo)
def invoke_plugin(self, plugin_class, kwargs: Any, extra: dict, pinfo: dict):
# 检查插件的参数类型
kwargs = self.fix_kwargs(kwargs)
# 把插件的配置数据kwargs和附加数据extra合并extra会覆盖kwargs
if len(extra) != 0:
kwargs.update(extra)
def invoke_plugin(self, plugin_class, kwargs: Any, extra: dict):
# 保证 jm_plugin.py 被加载
from .jm_plugin import JmOptionPlugin
from .jm_plugin import JmOptionPlugin, PluginValidationException
plugin = plugin_class
plugin_class: Type[JmOptionPlugin]
pkey = plugin_class.plugin_key
try:
# 检查插件的参数类型
kwargs = self.fix_kwargs(kwargs)
# 把插件的配置数据kwargs和附加数据extra合并
# extra会覆盖kwargs
if len(extra) != 0:
kwargs.update(extra)
# 构建插件对象
plugin = plugin_class.build(self)
plugin: JmOptionPlugin = plugin_class.build(self)
jm_debug('plugin.invoke', f'调用插件: [{plugin_class.plugin_key}]')
# 调用插件功能
jm_debug('plugin.invoke', f'调用插件: [{pkey}]')
plugin.invoke(**kwargs)
except PluginValidationException as e:
# 插件抛出的参数校验异常
self.handle_plugin_valid_exception(e, pinfo, kwargs, plugin)
except JmcomicException as e:
msg = str(e)
jm_debug('plugin.exception', f'插件[{pkey}]调用失败,异常信息: {msg}')
raise e
# 模块内部异常通过不是插件抛出的而是插件调用了例如ClientClient请求失败抛出的
self.handle_plugin_exception(e, pinfo, kwargs, plugin)
except BaseException as e:
msg = str(e)
jm_debug('plugin.error', f'插件[{pkey}]运行遇到未捕获异常,异常信息: {msg}')
# 为插件兜底,捕获其他所有异常
self.handle_plugin_unexpected_error(e, pinfo, kwargs, plugin)
# noinspection PyMethodMayBeStatic,PyUnusedLocal
def handle_plugin_valid_exception(self, e, pinfo: dict, kwargs: dict, plugin):
from .jm_plugin import PluginValidationException
e: PluginValidationException
mode = pinfo.get('valid', self.plugins.valid)
if mode == 'ignore':
# ignore
return
if mode == 'debug':
# debug
jm_debug('plugin.validation',
f'插件 [{e.plugin.plugin_key}] 参数校验异常:{e.msg}'
)
return
if mode == 'raise':
# raise
raise e
# 其他的mode可以通过继承+方法重写来扩展
# noinspection PyMethodMayBeStatic,PyUnusedLocal
def handle_plugin_unexpected_error(self, e, pinfo: dict, kwargs: dict, plugin):
msg = str(e)
jm_debug('plugin.error', f'插件 [{plugin.plugin_key}],运行遇到未捕获异常,异常信息: {msg}')
raise e
# noinspection PyMethodMayBeStatic,PyUnusedLocal
def handle_plugin_exception(self, e, pinfo: dict, kwargs: dict, plugin):
msg = str(e)
jm_debug('plugin.exception', f'插件 [{plugin.plugin_key}],调用失败,异常信息: {msg}')
raise e
# noinspection PyMethodMayBeStatic
def fix_kwargs(self, kwargs) -> Dict[str, Any]:
"""

View File

@ -5,6 +5,13 @@
from .jm_option import *
class PluginValidationException(Exception):
def __init__(self, plugin: 'JmOptionPlugin', msg: str):
self.plugin = plugin
self.msg = msg
class JmOptionPlugin:
plugin_key: str
@ -33,6 +40,15 @@ class JmOptionPlugin:
msg=msg
)
def require_true(self, case: Any, msg: str):
"""
独立于ExceptionTool的一套异常抛出体系
"""
if case:
return
raise PluginValidationException(self, msg)
class JmLoginPlugin(JmOptionPlugin):
"""
@ -44,8 +60,8 @@ class JmLoginPlugin(JmOptionPlugin):
username: str,
password: str,
) -> None:
if not (username and password):
return
self.require_true(username, '用户名不能为空')
self.require_true(password, '密码不能为空')
client = self.option.new_jm_client()
client.login(username, password)
@ -326,14 +342,19 @@ class ZipPlugin(JmOptionPlugin):
删除所有文件和文件夹
"""
import os
for album, photo_dict in all_downloaded.items():
for photo, image_list in photo_dict.items():
for f, image in image_list:
for photo_dict in all_downloaded.values():
for image_list in photo_dict.values():
for f, _ in image_list:
# check not exist
if file_not_exists(f):
continue
os.remove(f)
self.debug(f'删除原文件: {f}', 'remove')
for d in dir_list:
if len(os.listdir(d)) == 0:
# check exist
if file_exists(d) and len(os.listdir(d)) == 0:
os.removedirs(d)
self.debug(f'删除文件夹: {d}', 'remove')
@ -403,11 +424,29 @@ class SendQQEmailPlugin(JmOptionPlugin):
album=None,
downloader=None,
) -> None:
if not (msg_from and msg_to and password):
self.debug('发送邮件的相关参数为空,不处理')
return
self.require_true(msg_from and msg_to and password, '发件人、收件人、授权码都不能为空')
from common import EmailConfig
econfig = EmailConfig(msg_from, msg_to, password)
epostman = econfig.create_email_postman()
epostman.send(content, title)
self.debug('Email sent successfully')
class DebugTopicFilterPlugin(JmOptionPlugin):
plugin_key = 'debug_topic_filter'
def invoke(self, whitelist) -> None:
if whitelist is not None:
whitelist = set(whitelist)
old_jm_debug = JmModuleConfig.debug_executor
def new_jm_debug(topic, msg):
if whitelist is not None and topic not in whitelist:
return
old_jm_debug(topic, msg)
JmModuleConfig.debug_executor = new_jm_debug