v2.5.3: 紧急修复域名切换重试机制,优化异常机制和GitHub Actions的异常处理 (#206)

This commit is contained in:
hect0x7 2024-01-30 19:59:33 +08:00 committed by GitHub
parent 684754af57
commit fb8a390423
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 126 additions and 46 deletions

View File

@ -69,8 +69,8 @@ $ jmcomic 422866
- **可扩展性强** - **可扩展性强**
- 支持自定义本子/章节/图片下载前后的回调函数 - 支持自定义本子/章节/图片下载前后的回调函数
- 支持自定义日志
- 支持自定义类:`Downloader负责调度` `Option负责配置` `Client负责请求` `实体类` - 支持自定义类:`Downloader负责调度` `Option负责配置` `Client负责请求` `实体类`
- 支持自定义日志、异常监听器
- **支持Plugin插件可以方便地扩展功能以及使用别人的插件目前内置插件有** - **支持Plugin插件可以方便地扩展功能以及使用别人的插件目前内置插件有**
- `登录插件` - `登录插件`
- `硬件占用监控插件` - `硬件占用监控插件`

View File

@ -164,3 +164,31 @@ def custom_jm_log():
# 2. 让my_log生效 # 2. 让my_log生效
JmModuleConfig.log_executor = my_log JmModuleConfig.log_executor = my_log
``` ```
## 自定义异常监听器/回调
```python
def custom_exception_listener():
"""
该函数演示jmcomic的异常监听器机制
"""
# 1. 选一个可能会发生的、你感兴趣的异常
etype = ResponseUnexpectedException
def listener(e):
"""
你的监听器方法
该方法无需返回值
:param e: 异常实例
"""
print(f'my exception listener invoke !!! exception happened: {e}')
# 注册监听器/回调
# 这个异常类或者这个异常的子类的实例将要被raise前你的listener方法会被调用
JmModuleConfig.register_exception_listener(etype, listener)
```

View File

@ -45,7 +45,7 @@ print(f'获取到{len(domain_set)}个域名,开始测试')
def test_domain(domain: str): def test_domain(domain: str):
client = option.new_jm_client(domain_list=[domain], **meta_data) client = option.new_jm_client(impl='html', domain_list=[domain], **meta_data)
status = 'ok' status = 'ok'
try: try:

View File

@ -11,6 +11,7 @@ client:
timeout: 7 timeout: 7
domain: domain:
html: html:
- 18comic.org
- jmcomic1.me - jmcomic1.me
- jmcomic.me - jmcomic.me

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方 # 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader # config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.5.2' __version__ = '2.5.3'
from .api import * from .api import *
from .jm_plugin import * from .jm_plugin import *

View File

@ -78,7 +78,9 @@ class AbstractJmClient(
:param kwargs: 请求方法的kwargs :param kwargs: 请求方法的kwargs
""" """
if domain_index >= len(self.domain_list): if domain_index >= len(self.domain_list):
self.fallback(request, url, domain_index, retry_count, **kwargs) return self.fallback(request, url, domain_index, retry_count, **kwargs)
url_backup = url
if url.startswith('/'): if url.startswith('/'):
# path → url # path → url
@ -120,9 +122,9 @@ class AbstractJmClient(
self.before_retry(e, kwargs, retry_count, url) self.before_retry(e, kwargs, retry_count, url)
if retry_count < self.retry_times: if retry_count < self.retry_times:
return self.request_with_retry(request, url, domain_index, retry_count + 1, callback, **kwargs) return self.request_with_retry(request, url_backup, domain_index, retry_count + 1, callback, **kwargs)
else: else:
return self.request_with_retry(request, url, domain_index + 1, 0, callback, **kwargs) return self.request_with_retry(request, url_backup, domain_index + 1, 0, callback, **kwargs)
# noinspection PyMethodMayBeStatic # noinspection PyMethodMayBeStatic
def raise_if_resp_should_retry(self, resp): def raise_if_resp_should_retry(self, resp):
@ -209,7 +211,7 @@ class AbstractJmClient(
def fallback(self, request, url, domain_index, retry_count, **kwargs): def fallback(self, request, url, domain_index, retry_count, **kwargs):
msg = f"请求重试全部失败: [{url}], {self.domain_list}" msg = f"请求重试全部失败: [{url}], {self.domain_list}"
jm_log('req.fallback', msg) jm_log('req.fallback', msg)
ExceptionTool.raises(msg) ExceptionTool.raises(msg, {}, RequestRetryAllFailException)
# noinspection PyMethodMayBeStatic # noinspection PyMethodMayBeStatic
def append_params_to_url(self, url, params): def append_params_to_url(self, url, params):

View File

@ -117,10 +117,10 @@ class JmModuleConfig:
# 移动端API域名 # 移动端API域名
DOMAIN_API_LIST = str_to_list(''' DOMAIN_API_LIST = str_to_list('''
www.jmapinode.biz
www.jmapinode1.top www.jmapinode1.top
www.jmapinode2.top www.jmapinode2.top
www.jmapinode3.top www.jmapinode3.top
www.jmapinode.biz
www.jmapinode.top www.jmapinode.top
''') ''')
@ -144,8 +144,11 @@ class JmModuleConfig:
REGISTRY_CLIENT = {} REGISTRY_CLIENT = {}
# 插件注册表 # 插件注册表
REGISTRY_PLUGIN = {} REGISTRY_PLUGIN = {}
# 异常处理器 # 异常监听器
REGISTRY_EXCEPTION_ADVICE = {} # key: 异常类
# value: 函数,参数只有异常对象,无需返回值
# 这个异常类或者这个异常的子类的实例将要被raise前你的listener方法会被调用
REGISTRY_EXCEPTION_LISTENER = {}
# 执行log的函数 # 执行log的函数
executor_log = default_jm_logging executor_log = default_jm_logging
@ -311,7 +314,7 @@ class JmModuleConfig:
# 而如果只想修改几个简单常用的配置也可以下方的DEFAULT_XXX属性 # 而如果只想修改几个简单常用的配置也可以下方的DEFAULT_XXX属性
JM_OPTION_VER = '2.1' JM_OPTION_VER = '2.1'
DEFAULT_CLIENT_IMPL = 'api' # 默认Client实现类型为网页端 DEFAULT_CLIENT_IMPL = 'api' # 默认Client实现类型为网页端
DEFAULT_CLIENT_CACHE = True # 默认开启Client缓存缓存级别是level_option详见CacheRegistry DEFAULT_CLIENT_CACHE = None # 默认关闭Client缓存。缓存的配置详见 CacheRegistry
DEFAULT_PROXIES = ProxyBuilder.system_proxy() # 默认使用系统代理 DEFAULT_PROXIES = ProxyBuilder.system_proxy() # 默认使用系统代理
default_option_dict: dict = { default_option_dict: dict = {
@ -404,8 +407,8 @@ class JmModuleConfig:
cls.REGISTRY_CLIENT[client_class.client_key] = client_class cls.REGISTRY_CLIENT[client_class.client_key] = client_class
@classmethod @classmethod
def register_exception_advice(cls, etype, eadvice): def register_exception_listener(cls, etype, listener):
cls.REGISTRY_EXCEPTION_ADVICE[etype] = eadvice cls.REGISTRY_EXCEPTION_LISTENER[etype] = listener
jm_log = JmModuleConfig.jm_log jm_log = JmModuleConfig.jm_log

View File

@ -3,9 +3,7 @@ from .jm_entity import *
class JmcomicException(Exception): class JmcomicException(Exception):
""" description = 'jmcomic 模块异常'
jmcomic 模块异常
"""
def __init__(self, msg: str, context: dict): def __init__(self, msg: str, context: dict):
self.msg = msg self.msg = msg
@ -16,19 +14,22 @@ class JmcomicException(Exception):
class ResponseUnexpectedException(JmcomicException): class ResponseUnexpectedException(JmcomicException):
""" description = '响应不符合预期异常'
响应不符合预期异常
"""
@property @property
def resp(self): def resp(self):
return self.from_context(ExceptionTool.CONTEXT_KEY_RESP) return self.from_context(ExceptionTool.CONTEXT_KEY_RESP)
class RegularNotMatchException(ResponseUnexpectedException): class RegularNotMatchException(JmcomicException):
""" description = '正则表达式不匹配异常'
正则表达式不匹配异常
""" @property
def resp(self):
"""
可能为None
"""
return self.context.get(ExceptionTool.CONTEXT_KEY_RESP, None)
@property @property
def error_text(self): def error_text(self):
@ -40,19 +41,23 @@ class RegularNotMatchException(ResponseUnexpectedException):
class JsonResolveFailException(ResponseUnexpectedException): class JsonResolveFailException(ResponseUnexpectedException):
description = 'Json解析异常'
pass pass
class MissingAlbumPhotoException(ResponseUnexpectedException): class MissingAlbumPhotoException(ResponseUnexpectedException):
""" description = '不存在本子或章节异常'
缺少本子/章节异常
"""
@property @property
def error_jmid(self) -> str: def error_jmid(self) -> str:
return self.from_context(ExceptionTool.CONTEXT_KEY_MISSING_JM_ID) return self.from_context(ExceptionTool.CONTEXT_KEY_MISSING_JM_ID)
class RequestRetryAllFailException(JmcomicException):
description = '请求重试全部失败异常'
pass
class ExceptionTool: class ExceptionTool:
""" """
抛异常的工具 抛异常的工具
@ -95,10 +100,7 @@ class ExceptionTool:
e = etype(msg, context) e = etype(msg, context)
# 异常处理建议 # 异常处理建议
advice = JmModuleConfig.REGISTRY_EXCEPTION_ADVICE.get(etype, None) cls.notify_all_listeners(e)
if advice is not None:
advice(e)
raise e raise e
@ -174,3 +176,13 @@ class ExceptionTool:
raises(old, msg, context) raises(old, msg, context)
cls.raises = new cls.raises = new
@classmethod
def notify_all_listeners(cls, e):
registry: Dict[Type, Callable[Type]] = JmModuleConfig.REGISTRY_EXCEPTION_LISTENER
if not registry:
return None
for accept_type, listener in registry.items():
if isinstance(e, accept_type):
listener(e)

View File

@ -17,8 +17,11 @@ class CacheRegistry:
return registry[client] return registry[client]
@classmethod @classmethod
def enable_client_cache_on_condition(cls, option: 'JmOption', client: JmcomicClient, def enable_client_cache_on_condition(cls,
cache: Union[None, bool, str, Callable]): option: 'JmOption',
client: JmcomicClient,
cache: Union[None, bool, str, Callable],
):
""" """
cache parameter cache parameter

View File

@ -48,7 +48,8 @@ class JmTestConfigurable(unittest.TestCase):
# 设置 JmOptionJmcomicClient # 设置 JmOptionJmcomicClient
option = cls.new_option() option = cls.new_option()
cls.option = option cls.option = option
cls.client = option.build_jm_client() # 设置缓存级别为option可以减少请求次数
cls.client = option.build_jm_client(cache='level_option')
# 跨平台设置 # 跨平台设置
cls.adapt_os() cls.adapt_os()

View File

@ -86,20 +86,50 @@ def log_before_raise():
jm_download_dir = env('JM_DOWNLOAD_DIR', workspace()) jm_download_dir = env('JM_DOWNLOAD_DIR', workspace())
mkdir_if_not_exists(jm_download_dir) mkdir_if_not_exists(jm_download_dir)
# 自定义异常抛出函数在抛出前把HTML响应数据写到下载文件夹日志留痕 def decide_filepath(e):
def raises(old, msg, extra: dict): resp = e.context.get(ExceptionTool.CONTEXT_KEY_RESP, None)
if ExceptionTool.EXTRA_KEY_RESP not in extra:
return old(msg, extra) if resp is None:
suffix = str(time_stamp())
else:
suffix = resp.url
name = '-'.join(
fix_windir_name(it)
for it in [
e.description,
current_thread().name,
suffix
]
)
path = f'{jm_download_dir}/【出错了】{name}.log'
return path
def exception_listener(e: JmcomicException):
"""
异常监听器实现了在 GitHub Actions 把请求错误的信息下载到文件方便调试和通知使用者
"""
# 决定要写入的文件路径
path = decide_filepath(e)
# 准备内容
content = [
str(type(e)),
e.msg,
]
for k, v in e.context.items():
content.append(f'{k}: {v}')
# resp.text
resp = e.context.get(ExceptionTool.CONTEXT_KEY_RESP, None)
if resp:
content.append(f'响应文本: {resp.text}')
resp = extra[ExceptionTool.EXTRA_KEY_RESP]
# 写文件 # 写文件
from common import write_text, fix_windir_name write_text(path, '\n'.join(content))
write_text(f'{jm_download_dir}/{fix_windir_name(resp.url)}', resp.text)
return old(msg, extra) JmModuleConfig.register_exception_listener(JmcomicException, exception_listener)
# 应用函数
ExceptionTool.replace_old_exception_executor(raises)
if __name__ == '__main__': if __name__ == '__main__':