mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.5.3: 紧急修复域名切换重试机制,优化异常机制和GitHub Actions的异常处理 (#206)
This commit is contained in:
parent
684754af57
commit
fb8a390423
@ -69,8 +69,8 @@ $ jmcomic 422866
|
||||
- **可扩展性强**
|
||||
|
||||
- 支持自定义本子/章节/图片下载前后的回调函数
|
||||
- 支持自定义日志
|
||||
- 支持自定义类:`Downloader(负责调度)` `Option(负责配置)` `Client(负责请求)` `实体类`等
|
||||
- 支持自定义日志、异常监听器
|
||||
- **支持Plugin插件,可以方便地扩展功能,以及使用别人的插件,目前内置插件有**:
|
||||
- `登录插件`
|
||||
- `硬件占用监控插件`
|
||||
|
@ -164,3 +164,31 @@ def custom_jm_log():
|
||||
# 2. 让my_log生效
|
||||
JmModuleConfig.log_executor = my_log
|
||||
```
|
||||
|
||||
|
||||
|
||||
## 自定义异常监听器/回调
|
||||
|
||||
```python
|
||||
def custom_exception_listener():
|
||||
"""
|
||||
该函数演示jmcomic的异常监听器机制
|
||||
"""
|
||||
|
||||
# 1. 选一个可能会发生的、你感兴趣的异常
|
||||
etype = ResponseUnexpectedException
|
||||
|
||||
|
||||
def listener(e):
|
||||
"""
|
||||
你的监听器方法
|
||||
该方法无需返回值
|
||||
:param e: 异常实例
|
||||
"""
|
||||
print(f'my exception listener invoke !!! exception happened: {e}')
|
||||
|
||||
|
||||
# 注册监听器/回调
|
||||
# 这个异常类(或者这个异常的子类)的实例将要被raise前,你的listener方法会被调用
|
||||
JmModuleConfig.register_exception_listener(etype, listener)
|
||||
```
|
@ -45,7 +45,7 @@ print(f'获取到{len(domain_set)}个域名,开始测试')
|
||||
|
||||
|
||||
def test_domain(domain: str):
|
||||
client = option.new_jm_client(domain_list=[domain], **meta_data)
|
||||
client = option.new_jm_client(impl='html', domain_list=[domain], **meta_data)
|
||||
status = 'ok'
|
||||
|
||||
try:
|
||||
|
@ -11,6 +11,7 @@ client:
|
||||
timeout: 7
|
||||
domain:
|
||||
html:
|
||||
- 18comic.org
|
||||
- jmcomic1.me
|
||||
- jmcomic.me
|
||||
|
||||
|
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.5.2'
|
||||
__version__ = '2.5.3'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
@ -78,7 +78,9 @@ class AbstractJmClient(
|
||||
:param kwargs: 请求方法的kwargs
|
||||
"""
|
||||
if domain_index >= len(self.domain_list):
|
||||
self.fallback(request, url, domain_index, retry_count, **kwargs)
|
||||
return self.fallback(request, url, domain_index, retry_count, **kwargs)
|
||||
|
||||
url_backup = url
|
||||
|
||||
if url.startswith('/'):
|
||||
# path → url
|
||||
@ -120,9 +122,9 @@ class AbstractJmClient(
|
||||
self.before_retry(e, kwargs, retry_count, url)
|
||||
|
||||
if retry_count < self.retry_times:
|
||||
return self.request_with_retry(request, url, domain_index, retry_count + 1, callback, **kwargs)
|
||||
return self.request_with_retry(request, url_backup, domain_index, retry_count + 1, callback, **kwargs)
|
||||
else:
|
||||
return self.request_with_retry(request, url, domain_index + 1, 0, callback, **kwargs)
|
||||
return self.request_with_retry(request, url_backup, domain_index + 1, 0, callback, **kwargs)
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def raise_if_resp_should_retry(self, resp):
|
||||
@ -209,7 +211,7 @@ class AbstractJmClient(
|
||||
def fallback(self, request, url, domain_index, retry_count, **kwargs):
|
||||
msg = f"请求重试全部失败: [{url}], {self.domain_list}"
|
||||
jm_log('req.fallback', msg)
|
||||
ExceptionTool.raises(msg)
|
||||
ExceptionTool.raises(msg, {}, RequestRetryAllFailException)
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def append_params_to_url(self, url, params):
|
||||
|
@ -117,10 +117,10 @@ class JmModuleConfig:
|
||||
|
||||
# 移动端API域名
|
||||
DOMAIN_API_LIST = str_to_list('''
|
||||
www.jmapinode.biz
|
||||
www.jmapinode1.top
|
||||
www.jmapinode2.top
|
||||
www.jmapinode3.top
|
||||
www.jmapinode.biz
|
||||
www.jmapinode.top
|
||||
|
||||
''')
|
||||
@ -144,8 +144,11 @@ class JmModuleConfig:
|
||||
REGISTRY_CLIENT = {}
|
||||
# 插件注册表
|
||||
REGISTRY_PLUGIN = {}
|
||||
# 异常处理器
|
||||
REGISTRY_EXCEPTION_ADVICE = {}
|
||||
# 异常监听器
|
||||
# key: 异常类
|
||||
# value: 函数,参数只有异常对象,无需返回值
|
||||
# 这个异常类(或者这个异常的子类)的实例将要被raise前,你的listener方法会被调用
|
||||
REGISTRY_EXCEPTION_LISTENER = {}
|
||||
|
||||
# 执行log的函数
|
||||
executor_log = default_jm_logging
|
||||
@ -311,7 +314,7 @@ class JmModuleConfig:
|
||||
# 而如果只想修改几个简单常用的配置,也可以下方的DEFAULT_XXX属性
|
||||
JM_OPTION_VER = '2.1'
|
||||
DEFAULT_CLIENT_IMPL = 'api' # 默认Client实现类型为网页端
|
||||
DEFAULT_CLIENT_CACHE = True # 默认开启Client缓存,缓存级别是level_option,详见CacheRegistry
|
||||
DEFAULT_CLIENT_CACHE = None # 默认关闭Client缓存。缓存的配置详见 CacheRegistry
|
||||
DEFAULT_PROXIES = ProxyBuilder.system_proxy() # 默认使用系统代理
|
||||
|
||||
default_option_dict: dict = {
|
||||
@ -404,8 +407,8 @@ class JmModuleConfig:
|
||||
cls.REGISTRY_CLIENT[client_class.client_key] = client_class
|
||||
|
||||
@classmethod
|
||||
def register_exception_advice(cls, etype, eadvice):
|
||||
cls.REGISTRY_EXCEPTION_ADVICE[etype] = eadvice
|
||||
def register_exception_listener(cls, etype, listener):
|
||||
cls.REGISTRY_EXCEPTION_LISTENER[etype] = listener
|
||||
|
||||
|
||||
jm_log = JmModuleConfig.jm_log
|
||||
|
@ -3,9 +3,7 @@ from .jm_entity import *
|
||||
|
||||
|
||||
class JmcomicException(Exception):
|
||||
"""
|
||||
jmcomic 模块异常
|
||||
"""
|
||||
description = 'jmcomic 模块异常'
|
||||
|
||||
def __init__(self, msg: str, context: dict):
|
||||
self.msg = msg
|
||||
@ -16,19 +14,22 @@ class JmcomicException(Exception):
|
||||
|
||||
|
||||
class ResponseUnexpectedException(JmcomicException):
|
||||
"""
|
||||
响应不符合预期异常
|
||||
"""
|
||||
description = '响应不符合预期异常'
|
||||
|
||||
@property
|
||||
def resp(self):
|
||||
return self.from_context(ExceptionTool.CONTEXT_KEY_RESP)
|
||||
|
||||
|
||||
class RegularNotMatchException(ResponseUnexpectedException):
|
||||
class RegularNotMatchException(JmcomicException):
|
||||
description = '正则表达式不匹配异常'
|
||||
|
||||
@property
|
||||
def resp(self):
|
||||
"""
|
||||
正则表达式不匹配异常
|
||||
可能为None
|
||||
"""
|
||||
return self.context.get(ExceptionTool.CONTEXT_KEY_RESP, None)
|
||||
|
||||
@property
|
||||
def error_text(self):
|
||||
@ -40,19 +41,23 @@ class RegularNotMatchException(ResponseUnexpectedException):
|
||||
|
||||
|
||||
class JsonResolveFailException(ResponseUnexpectedException):
|
||||
description = 'Json解析异常'
|
||||
pass
|
||||
|
||||
|
||||
class MissingAlbumPhotoException(ResponseUnexpectedException):
|
||||
"""
|
||||
缺少本子/章节异常
|
||||
"""
|
||||
description = '不存在本子或章节异常'
|
||||
|
||||
@property
|
||||
def error_jmid(self) -> str:
|
||||
return self.from_context(ExceptionTool.CONTEXT_KEY_MISSING_JM_ID)
|
||||
|
||||
|
||||
class RequestRetryAllFailException(JmcomicException):
|
||||
description = '请求重试全部失败异常'
|
||||
pass
|
||||
|
||||
|
||||
class ExceptionTool:
|
||||
"""
|
||||
抛异常的工具
|
||||
@ -95,10 +100,7 @@ class ExceptionTool:
|
||||
e = etype(msg, context)
|
||||
|
||||
# 异常处理建议
|
||||
advice = JmModuleConfig.REGISTRY_EXCEPTION_ADVICE.get(etype, None)
|
||||
|
||||
if advice is not None:
|
||||
advice(e)
|
||||
cls.notify_all_listeners(e)
|
||||
|
||||
raise e
|
||||
|
||||
@ -174,3 +176,13 @@ class ExceptionTool:
|
||||
raises(old, msg, context)
|
||||
|
||||
cls.raises = new
|
||||
|
||||
@classmethod
|
||||
def notify_all_listeners(cls, e):
|
||||
registry: Dict[Type, Callable[Type]] = JmModuleConfig.REGISTRY_EXCEPTION_LISTENER
|
||||
if not registry:
|
||||
return None
|
||||
|
||||
for accept_type, listener in registry.items():
|
||||
if isinstance(e, accept_type):
|
||||
listener(e)
|
||||
|
@ -17,8 +17,11 @@ class CacheRegistry:
|
||||
return registry[client]
|
||||
|
||||
@classmethod
|
||||
def enable_client_cache_on_condition(cls, option: 'JmOption', client: JmcomicClient,
|
||||
cache: Union[None, bool, str, Callable]):
|
||||
def enable_client_cache_on_condition(cls,
|
||||
option: 'JmOption',
|
||||
client: JmcomicClient,
|
||||
cache: Union[None, bool, str, Callable],
|
||||
):
|
||||
"""
|
||||
cache parameter
|
||||
|
||||
|
@ -48,7 +48,8 @@ class JmTestConfigurable(unittest.TestCase):
|
||||
# 设置 JmOption,JmcomicClient
|
||||
option = cls.new_option()
|
||||
cls.option = option
|
||||
cls.client = option.build_jm_client()
|
||||
# 设置缓存级别为option,可以减少请求次数
|
||||
cls.client = option.build_jm_client(cache='level_option')
|
||||
|
||||
# 跨平台设置
|
||||
cls.adapt_os()
|
||||
|
@ -86,20 +86,50 @@ def log_before_raise():
|
||||
jm_download_dir = env('JM_DOWNLOAD_DIR', workspace())
|
||||
mkdir_if_not_exists(jm_download_dir)
|
||||
|
||||
# 自定义异常抛出函数,在抛出前把HTML响应数据写到下载文件夹(日志留痕)
|
||||
def raises(old, msg, extra: dict):
|
||||
if ExceptionTool.EXTRA_KEY_RESP not in extra:
|
||||
return old(msg, extra)
|
||||
def decide_filepath(e):
|
||||
resp = e.context.get(ExceptionTool.CONTEXT_KEY_RESP, None)
|
||||
|
||||
if resp is None:
|
||||
suffix = str(time_stamp())
|
||||
else:
|
||||
suffix = resp.url
|
||||
|
||||
name = '-'.join(
|
||||
fix_windir_name(it)
|
||||
for it in [
|
||||
e.description,
|
||||
current_thread().name,
|
||||
suffix
|
||||
]
|
||||
)
|
||||
|
||||
path = f'{jm_download_dir}/【出错了】{name}.log'
|
||||
return path
|
||||
|
||||
def exception_listener(e: JmcomicException):
|
||||
"""
|
||||
异常监听器,实现了在 GitHub Actions 下,把请求错误的信息下载到文件,方便调试和通知使用者
|
||||
"""
|
||||
# 决定要写入的文件路径
|
||||
path = decide_filepath(e)
|
||||
|
||||
# 准备内容
|
||||
content = [
|
||||
str(type(e)),
|
||||
e.msg,
|
||||
]
|
||||
for k, v in e.context.items():
|
||||
content.append(f'{k}: {v}')
|
||||
|
||||
# resp.text
|
||||
resp = e.context.get(ExceptionTool.CONTEXT_KEY_RESP, None)
|
||||
if resp:
|
||||
content.append(f'响应文本: {resp.text}')
|
||||
|
||||
resp = extra[ExceptionTool.EXTRA_KEY_RESP]
|
||||
# 写文件
|
||||
from common import write_text, fix_windir_name
|
||||
write_text(f'{jm_download_dir}/{fix_windir_name(resp.url)}', resp.text)
|
||||
write_text(path, '\n'.join(content))
|
||||
|
||||
return old(msg, extra)
|
||||
|
||||
# 应用函数
|
||||
ExceptionTool.replace_old_exception_executor(raises)
|
||||
JmModuleConfig.register_exception_listener(JmcomicException, exception_listener)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
Loading…
Reference in New Issue
Block a user