mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v2.3.15: 修复移动端未携带cookies导致跳转同本问题; 新增插件【debug日志主题过滤器】; 新增插件参数校验异常类及其相应的异常处理策略 (ignore, debug, raise)。
This commit is contained in:
parent
de0e0ba729
commit
87ed6bce83
43
.github/release.py
vendored
43
.github/release.py
vendored
@ -4,17 +4,42 @@ import re
|
||||
|
||||
|
||||
def add_output(k, v):
|
||||
cmd = f'echo {k}="{v}" >> $GITHUB_OUTPUT'
|
||||
cmd = f'echo "{k}={v}" >> $GITHUB_OUTPUT'
|
||||
print(cmd, os.system(cmd))
|
||||
|
||||
|
||||
msg = sys.argv[1]
|
||||
print(f'msg: {msg}')
|
||||
p = re.compile('(.*?): ?(.*)')
|
||||
match = p.search(msg)
|
||||
assert match is not None, f'commit message format is wrong: {msg}'
|
||||
def parse_body(body):
|
||||
if ';' not in body:
|
||||
return body
|
||||
|
||||
tag, body = match[1], match[2]
|
||||
parts = body.split(";")
|
||||
points = []
|
||||
for i, e in enumerate(parts):
|
||||
e: str = e.strip()
|
||||
if e == '':
|
||||
continue
|
||||
points.append(f'{i}. {e}')
|
||||
|
||||
add_output('tag', tag)
|
||||
add_output('body', body)
|
||||
return '\n'.join(points)
|
||||
|
||||
|
||||
def get_tag_and_body():
|
||||
msg = sys.argv[1]
|
||||
print(f'msg: {msg}')
|
||||
p = re.compile('(.*?): ?(.*)')
|
||||
match = p.search(msg)
|
||||
assert match is not None, f'commit message format is wrong: {msg}'
|
||||
tag, body = match[1], match[2]
|
||||
return body, tag
|
||||
|
||||
|
||||
def main():
|
||||
body, tag = get_tag_and_body()
|
||||
|
||||
add_output('tag', tag)
|
||||
|
||||
with open('release_body.txt', 'w', encoding='utf-8') as f:
|
||||
f.write(parse_body(body))
|
||||
|
||||
|
||||
main()
|
||||
|
||||
2
.github/workflows/release_auto.yml
vendored
2
.github/workflows/release_auto.yml
vendored
@ -31,7 +31,7 @@ jobs:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
tag_name: ${{ steps.tb.outputs.tag }}
|
||||
body: ${{ steps.tb.outputs.body }}
|
||||
body_path: release_body.txt
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
|
||||
@ -42,7 +42,7 @@ $ jmcomic 422866
|
||||
|
||||
- GitHub Actions:网页上直接输入本子id就能下载([教程:使用GitHub Actions下载禁漫本子](./assets/docs/sources/tutorial/1_github_actions.md))
|
||||
- 命令行:无需写Python代码,简单易用([教程:使用命令行下载禁漫本子](./assets/docs/sources/tutorial/2_command_line.md))
|
||||
- Python代码:最直接的使用方式,需要你有一定的python编程基础
|
||||
- Python代码:最本质、最强大的使用方式,需要你有一定的python编程基础
|
||||
- 支持**网页端**和**移动端**两种客户端实现,可通过配置切换(**移动端不限ip兼容性好,网页端限制ip地区但效率高**)
|
||||
- 支持**自动重试和域名切换**机制
|
||||
- **多线程下载**(可细化到一图一线程,效率极高)
|
||||
@ -54,9 +54,9 @@ $ jmcomic 422866
|
||||
- **可扩展性强**
|
||||
|
||||
- **支持Plugin插件,可以方便地扩展功能,以及使用别人的插件**
|
||||
- 目前内置支持的插件有:`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` `下载特定后缀图片插件` `发送QQ邮件插件`
|
||||
- 目前内置支持的插件有:`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` `下载特定后缀图片插件` `发送QQ邮件插件` `日志主题过滤插件`
|
||||
- 支持自定义本子/章节/图片下载前后的回调函数
|
||||
- 支持自定义debug日志
|
||||
- 支持自定义debug/logging
|
||||
- 支持自定义类:`Downloader(负责调度)` `Option(负责配置)` `Client(负责请求)` `实体类`等
|
||||
|
||||
## 进阶使用
|
||||
|
||||
@ -26,7 +26,7 @@ Python API for JMComic(禁漫天堂)
|
||||
|
||||
- Supports Plugin plugins for easy functionality extension and use of other plugins.
|
||||
- Currently built-in
|
||||
plugins: `login plugin`, `hardware usage monitoring plugin`, `only download new chapters plugin`, `zip compression plugin`, `image suffix filter plugin` `send qq email plugin`.
|
||||
plugins: `login plugin`, `hardware usage monitoring plugin`, `only download new chapters plugin`, `zip compression plugin`, `image suffix filter plugin` `send qq email plugin` `debug logging topic filter plugin`.
|
||||
- Supports custom callback functions before and after downloading album/chapter/images.
|
||||
- Supports custom debug logging.
|
||||
- Supports custom core
|
||||
|
||||
@ -3,4 +3,5 @@ curl_cffi
|
||||
PyYAML
|
||||
Pillow
|
||||
psutil
|
||||
pycryptodome
|
||||
pycryptodome
|
||||
requests
|
||||
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.3.14'
|
||||
__version__ = '2.3.15'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
||||
@ -584,7 +584,7 @@ class JmApiClient(AbstractJmClient):
|
||||
def get_decode(self, url, **kwargs) -> JmApiResp:
|
||||
# set headers
|
||||
headers, key_ts = self.headers_key_ts
|
||||
kwargs.setdefault('headers', headers)
|
||||
kwargs['headers'] = headers
|
||||
|
||||
resp = self.get(url, **kwargs)
|
||||
return JmApiResp.wrap(resp, key_ts)
|
||||
@ -610,6 +610,16 @@ class JmApiClient(AbstractJmClient):
|
||||
# 2. 是否是特殊的内容
|
||||
# 暂无
|
||||
|
||||
@classmethod
|
||||
@field_cache('__init_cookies__')
|
||||
def fetch_init_cookies(cls, client: 'JmApiClient'):
|
||||
resp = client.setting()
|
||||
return dict(resp.resp.cookies)
|
||||
|
||||
def after_init(self):
|
||||
cookies = self.__class__.fetch_init_cookies(self)
|
||||
self.get_root_postman().get_meta_data()['cookies'] = cookies
|
||||
|
||||
|
||||
class FutureClientProxy(JmcomicClient):
|
||||
"""
|
||||
|
||||
@ -64,7 +64,7 @@ class JmModuleConfig:
|
||||
|
||||
# 移动端API的相关配置
|
||||
# API密钥
|
||||
MAGIC_18COMICAPPCONTENT = '18comicAPPContent'
|
||||
APP_SECRET = '18comicAPP'
|
||||
|
||||
# 域名配置 - 移动端
|
||||
# 图片域名
|
||||
@ -247,7 +247,7 @@ class JmModuleConfig:
|
||||
key_ts = time_stamp()
|
||||
|
||||
import hashlib
|
||||
token = hashlib.md5(f"{key_ts}{cls.MAGIC_18COMICAPPCONTENT}".encode()).hexdigest()
|
||||
token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode()).hexdigest()
|
||||
|
||||
return {
|
||||
'token': token,
|
||||
@ -331,7 +331,11 @@ class JmModuleConfig:
|
||||
'impl': None,
|
||||
'retry_times': 5
|
||||
},
|
||||
'plugins': {},
|
||||
'plugins': {
|
||||
# 如果插件抛出参数校验异常,只debug。(全局配置,可以被插件的局部配置覆盖)
|
||||
# 可选值:ignore(忽略),debug(打印日志),raise(抛异常)。
|
||||
'valid': 'debug',
|
||||
},
|
||||
}
|
||||
|
||||
@classmethod
|
||||
|
||||
@ -9,10 +9,6 @@ DownloadIterObjs = Union[
|
||||
]
|
||||
|
||||
|
||||
class JmDownloadException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
class DownloadCallback:
|
||||
|
||||
|
||||
@ -311,12 +311,12 @@ class JmOption:
|
||||
"""
|
||||
return self.new_jm_client(**kwargs)
|
||||
|
||||
def new_jm_client(self, domain=None, impl=None, **kwargs) -> JmcomicClient:
|
||||
def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient:
|
||||
# 所有需要用到的 self.client 配置项如下
|
||||
postman_conf: dict = self.client.postman.src_dict # postman dsl 配置
|
||||
impl: str = impl or self.client.impl # client_key
|
||||
retry_times: int = self.client.retry_times # 重试次数
|
||||
cache: str = self.client.cache # 启用缓存
|
||||
cache: str = cache or self.client.cache # 启用缓存
|
||||
|
||||
# domain
|
||||
def decide_domain():
|
||||
@ -340,7 +340,10 @@ class JmOption:
|
||||
# headers
|
||||
meta_data = postman_conf['meta_data']
|
||||
if meta_data['headers'] is None:
|
||||
meta_data['headers'] = self.decide_postman_headers(impl, domain[0])
|
||||
headers = self.decide_postman_headers(impl, domain[0])
|
||||
# if headers is None:
|
||||
# postman_conf['type'] = 'requests'
|
||||
meta_data['headers'] = headers
|
||||
|
||||
# postman
|
||||
postman = Postmans.create(data=postman_conf)
|
||||
@ -349,7 +352,8 @@ class JmOption:
|
||||
clazz = JmModuleConfig.client_impl_class(impl)
|
||||
if clazz == AbstractJmClient or not issubclass(clazz, AbstractJmClient):
|
||||
raise NotImplementedError(clazz)
|
||||
client = clazz(
|
||||
|
||||
client: AbstractJmClient = clazz(
|
||||
postman,
|
||||
retry_times,
|
||||
fallback_domain_list=decide_domain(),
|
||||
@ -442,36 +446,77 @@ class JmOption:
|
||||
|
||||
ExceptionTool.require_true(plugin_class is not None, f'[{group}] 未注册的plugin: {key}')
|
||||
|
||||
self.invoke_plugin(plugin_class, kwargs, extra)
|
||||
self.invoke_plugin(plugin_class, kwargs, extra, pinfo)
|
||||
|
||||
def invoke_plugin(self, plugin_class, kwargs: Any, extra: dict, pinfo: dict):
|
||||
# 检查插件的参数类型
|
||||
kwargs = self.fix_kwargs(kwargs)
|
||||
# 把插件的配置数据kwargs和附加数据extra合并,extra会覆盖kwargs
|
||||
if len(extra) != 0:
|
||||
kwargs.update(extra)
|
||||
|
||||
def invoke_plugin(self, plugin_class, kwargs: Any, extra: dict):
|
||||
# 保证 jm_plugin.py 被加载
|
||||
from .jm_plugin import JmOptionPlugin
|
||||
from .jm_plugin import JmOptionPlugin, PluginValidationException
|
||||
|
||||
plugin = plugin_class
|
||||
plugin_class: Type[JmOptionPlugin]
|
||||
pkey = plugin_class.plugin_key
|
||||
|
||||
try:
|
||||
# 检查插件的参数类型
|
||||
kwargs = self.fix_kwargs(kwargs)
|
||||
# 把插件的配置数据kwargs和附加数据extra合并
|
||||
# extra会覆盖kwargs
|
||||
if len(extra) != 0:
|
||||
kwargs.update(extra)
|
||||
# 构建插件对象
|
||||
plugin = plugin_class.build(self)
|
||||
plugin: JmOptionPlugin = plugin_class.build(self)
|
||||
|
||||
jm_debug('plugin.invoke', f'调用插件: [{plugin_class.plugin_key}]')
|
||||
# 调用插件功能
|
||||
jm_debug('plugin.invoke', f'调用插件: [{pkey}]')
|
||||
plugin.invoke(**kwargs)
|
||||
|
||||
except PluginValidationException as e:
|
||||
# 插件抛出的参数校验异常
|
||||
self.handle_plugin_valid_exception(e, pinfo, kwargs, plugin)
|
||||
|
||||
except JmcomicException as e:
|
||||
msg = str(e)
|
||||
jm_debug('plugin.exception', f'插件[{pkey}]调用失败,异常信息: {msg}')
|
||||
raise e
|
||||
# 模块内部异常,通过不是插件抛出的,而是插件调用了例如Client,Client请求失败抛出的
|
||||
self.handle_plugin_exception(e, pinfo, kwargs, plugin)
|
||||
|
||||
except BaseException as e:
|
||||
msg = str(e)
|
||||
jm_debug('plugin.error', f'插件[{pkey}]运行遇到未捕获异常,异常信息: {msg}')
|
||||
# 为插件兜底,捕获其他所有异常
|
||||
self.handle_plugin_unexpected_error(e, pinfo, kwargs, plugin)
|
||||
|
||||
# noinspection PyMethodMayBeStatic,PyUnusedLocal
|
||||
def handle_plugin_valid_exception(self, e, pinfo: dict, kwargs: dict, plugin):
|
||||
from .jm_plugin import PluginValidationException
|
||||
e: PluginValidationException
|
||||
|
||||
mode = pinfo.get('valid', self.plugins.valid)
|
||||
|
||||
if mode == 'ignore':
|
||||
# ignore
|
||||
return
|
||||
|
||||
if mode == 'debug':
|
||||
# debug
|
||||
jm_debug('plugin.validation',
|
||||
f'插件 [{e.plugin.plugin_key}] 参数校验异常:{e.msg}'
|
||||
)
|
||||
return
|
||||
|
||||
if mode == 'raise':
|
||||
# raise
|
||||
raise e
|
||||
|
||||
# 其他的mode可以通过继承+方法重写来扩展
|
||||
|
||||
# noinspection PyMethodMayBeStatic,PyUnusedLocal
|
||||
def handle_plugin_unexpected_error(self, e, pinfo: dict, kwargs: dict, plugin):
|
||||
msg = str(e)
|
||||
jm_debug('plugin.error', f'插件 [{plugin.plugin_key}],运行遇到未捕获异常,异常信息: {msg}')
|
||||
raise e
|
||||
|
||||
# noinspection PyMethodMayBeStatic,PyUnusedLocal
|
||||
def handle_plugin_exception(self, e, pinfo: dict, kwargs: dict, plugin):
|
||||
msg = str(e)
|
||||
jm_debug('plugin.exception', f'插件 [{plugin.plugin_key}],调用失败,异常信息: {msg}')
|
||||
raise e
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def fix_kwargs(self, kwargs) -> Dict[str, Any]:
|
||||
"""
|
||||
|
||||
@ -5,6 +5,13 @@
|
||||
from .jm_option import *
|
||||
|
||||
|
||||
class PluginValidationException(Exception):
|
||||
|
||||
def __init__(self, plugin: 'JmOptionPlugin', msg: str):
|
||||
self.plugin = plugin
|
||||
self.msg = msg
|
||||
|
||||
|
||||
class JmOptionPlugin:
|
||||
plugin_key: str
|
||||
|
||||
@ -33,6 +40,15 @@ class JmOptionPlugin:
|
||||
msg=msg
|
||||
)
|
||||
|
||||
def require_true(self, case: Any, msg: str):
|
||||
"""
|
||||
独立于ExceptionTool的一套异常抛出体系
|
||||
"""
|
||||
if case:
|
||||
return
|
||||
|
||||
raise PluginValidationException(self, msg)
|
||||
|
||||
|
||||
class JmLoginPlugin(JmOptionPlugin):
|
||||
"""
|
||||
@ -44,8 +60,8 @@ class JmLoginPlugin(JmOptionPlugin):
|
||||
username: str,
|
||||
password: str,
|
||||
) -> None:
|
||||
if not (username and password):
|
||||
return
|
||||
self.require_true(username, '用户名不能为空')
|
||||
self.require_true(password, '密码不能为空')
|
||||
|
||||
client = self.option.new_jm_client()
|
||||
client.login(username, password)
|
||||
@ -326,14 +342,19 @@ class ZipPlugin(JmOptionPlugin):
|
||||
删除所有文件和文件夹
|
||||
"""
|
||||
import os
|
||||
for album, photo_dict in all_downloaded.items():
|
||||
for photo, image_list in photo_dict.items():
|
||||
for f, image in image_list:
|
||||
for photo_dict in all_downloaded.values():
|
||||
for image_list in photo_dict.values():
|
||||
for f, _ in image_list:
|
||||
# check not exist
|
||||
if file_not_exists(f):
|
||||
continue
|
||||
|
||||
os.remove(f)
|
||||
self.debug(f'删除原文件: {f}', 'remove')
|
||||
|
||||
for d in dir_list:
|
||||
if len(os.listdir(d)) == 0:
|
||||
# check exist
|
||||
if file_exists(d) and len(os.listdir(d)) == 0:
|
||||
os.removedirs(d)
|
||||
self.debug(f'删除文件夹: {d}', 'remove')
|
||||
|
||||
@ -403,11 +424,29 @@ class SendQQEmailPlugin(JmOptionPlugin):
|
||||
album=None,
|
||||
downloader=None,
|
||||
) -> None:
|
||||
if not (msg_from and msg_to and password):
|
||||
self.debug('发送邮件的相关参数为空,不处理')
|
||||
return
|
||||
self.require_true(msg_from and msg_to and password, '发件人、收件人、授权码都不能为空')
|
||||
|
||||
from common import EmailConfig
|
||||
econfig = EmailConfig(msg_from, msg_to, password)
|
||||
epostman = econfig.create_email_postman()
|
||||
epostman.send(content, title)
|
||||
|
||||
self.debug('Email sent successfully')
|
||||
|
||||
|
||||
class DebugTopicFilterPlugin(JmOptionPlugin):
|
||||
plugin_key = 'debug_topic_filter'
|
||||
|
||||
def invoke(self, whitelist) -> None:
|
||||
if whitelist is not None:
|
||||
whitelist = set(whitelist)
|
||||
|
||||
old_jm_debug = JmModuleConfig.debug_executor
|
||||
|
||||
def new_jm_debug(topic, msg):
|
||||
if whitelist is not None and topic not in whitelist:
|
||||
return
|
||||
|
||||
old_jm_debug(topic, msg)
|
||||
|
||||
JmModuleConfig.debug_executor = new_jm_debug
|
||||
|
||||
Loading…
Reference in New Issue
Block a user