v2.5.3: 紧急修复域名切换重试机制,优化异常机制和GitHub Actions的异常处理 (#206)

This commit is contained in:
hect0x7 2024-01-30 19:59:33 +08:00 committed by GitHub
parent 684754af57
commit fb8a390423
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 126 additions and 46 deletions

View File

@ -69,8 +69,8 @@ $ jmcomic 422866
- **可扩展性强**
- 支持自定义本子/章节/图片下载前后的回调函数
- 支持自定义日志
- 支持自定义类:`Downloader负责调度` `Option负责配置` `Client负责请求` `实体类`
- 支持自定义日志、异常监听器
- **支持Plugin插件可以方便地扩展功能以及使用别人的插件目前内置插件有**
- `登录插件`
- `硬件占用监控插件`

View File

@ -164,3 +164,31 @@ def custom_jm_log():
# 2. 让my_log生效
JmModuleConfig.log_executor = my_log
```
## 自定义异常监听器/回调
```python
def custom_exception_listener():
"""
该函数演示jmcomic的异常监听器机制
"""
# 1. 选一个可能会发生的、你感兴趣的异常
etype = ResponseUnexpectedException
def listener(e):
"""
你的监听器方法
该方法无需返回值
:param e: 异常实例
"""
print(f'my exception listener invoke !!! exception happened: {e}')
# 注册监听器/回调
# 这个异常类或者这个异常的子类的实例将要被raise前你的listener方法会被调用
JmModuleConfig.register_exception_listener(etype, listener)
```

View File

@ -45,7 +45,7 @@ print(f'获取到{len(domain_set)}个域名,开始测试')
def test_domain(domain: str):
client = option.new_jm_client(domain_list=[domain], **meta_data)
client = option.new_jm_client(impl='html', domain_list=[domain], **meta_data)
status = 'ok'
try:

View File

@ -11,6 +11,7 @@ client:
timeout: 7
domain:
html:
- 18comic.org
- jmcomic1.me
- jmcomic.me

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.5.2'
__version__ = '2.5.3'
from .api import *
from .jm_plugin import *

View File

@ -78,7 +78,9 @@ class AbstractJmClient(
:param kwargs: 请求方法的kwargs
"""
if domain_index >= len(self.domain_list):
self.fallback(request, url, domain_index, retry_count, **kwargs)
return self.fallback(request, url, domain_index, retry_count, **kwargs)
url_backup = url
if url.startswith('/'):
# path → url
@ -120,9 +122,9 @@ class AbstractJmClient(
self.before_retry(e, kwargs, retry_count, url)
if retry_count < self.retry_times:
return self.request_with_retry(request, url, domain_index, retry_count + 1, callback, **kwargs)
return self.request_with_retry(request, url_backup, domain_index, retry_count + 1, callback, **kwargs)
else:
return self.request_with_retry(request, url, domain_index + 1, 0, callback, **kwargs)
return self.request_with_retry(request, url_backup, domain_index + 1, 0, callback, **kwargs)
# noinspection PyMethodMayBeStatic
def raise_if_resp_should_retry(self, resp):
@ -209,7 +211,7 @@ class AbstractJmClient(
def fallback(self, request, url, domain_index, retry_count, **kwargs):
msg = f"请求重试全部失败: [{url}], {self.domain_list}"
jm_log('req.fallback', msg)
ExceptionTool.raises(msg)
ExceptionTool.raises(msg, {}, RequestRetryAllFailException)
# noinspection PyMethodMayBeStatic
def append_params_to_url(self, url, params):

View File

@ -117,10 +117,10 @@ class JmModuleConfig:
# 移动端API域名
DOMAIN_API_LIST = str_to_list('''
www.jmapinode.biz
www.jmapinode1.top
www.jmapinode2.top
www.jmapinode3.top
www.jmapinode.biz
www.jmapinode.top
''')
@ -144,8 +144,11 @@ class JmModuleConfig:
REGISTRY_CLIENT = {}
# 插件注册表
REGISTRY_PLUGIN = {}
# 异常处理器
REGISTRY_EXCEPTION_ADVICE = {}
# 异常监听器
# key: 异常类
# value: 函数,参数只有异常对象,无需返回值
# 这个异常类或者这个异常的子类的实例将要被raise前你的listener方法会被调用
REGISTRY_EXCEPTION_LISTENER = {}
# 执行log的函数
executor_log = default_jm_logging
@ -311,7 +314,7 @@ class JmModuleConfig:
# 而如果只想修改几个简单常用的配置也可以下方的DEFAULT_XXX属性
JM_OPTION_VER = '2.1'
DEFAULT_CLIENT_IMPL = 'api' # 默认Client实现类型为网页端
DEFAULT_CLIENT_CACHE = True # 默认开启Client缓存缓存级别是level_option详见CacheRegistry
DEFAULT_CLIENT_CACHE = None # 默认关闭Client缓存。缓存的配置详见 CacheRegistry
DEFAULT_PROXIES = ProxyBuilder.system_proxy() # 默认使用系统代理
default_option_dict: dict = {
@ -404,8 +407,8 @@ class JmModuleConfig:
cls.REGISTRY_CLIENT[client_class.client_key] = client_class
@classmethod
def register_exception_advice(cls, etype, eadvice):
cls.REGISTRY_EXCEPTION_ADVICE[etype] = eadvice
def register_exception_listener(cls, etype, listener):
cls.REGISTRY_EXCEPTION_LISTENER[etype] = listener
jm_log = JmModuleConfig.jm_log

View File

@ -3,9 +3,7 @@ from .jm_entity import *
class JmcomicException(Exception):
"""
jmcomic 模块异常
"""
description = 'jmcomic 模块异常'
def __init__(self, msg: str, context: dict):
self.msg = msg
@ -16,19 +14,22 @@ class JmcomicException(Exception):
class ResponseUnexpectedException(JmcomicException):
"""
响应不符合预期异常
"""
description = '响应不符合预期异常'
@property
def resp(self):
return self.from_context(ExceptionTool.CONTEXT_KEY_RESP)
class RegularNotMatchException(ResponseUnexpectedException):
"""
正则表达式不匹配异常
"""
class RegularNotMatchException(JmcomicException):
description = '正则表达式不匹配异常'
@property
def resp(self):
"""
可能为None
"""
return self.context.get(ExceptionTool.CONTEXT_KEY_RESP, None)
@property
def error_text(self):
@ -40,19 +41,23 @@ class RegularNotMatchException(ResponseUnexpectedException):
class JsonResolveFailException(ResponseUnexpectedException):
description = 'Json解析异常'
pass
class MissingAlbumPhotoException(ResponseUnexpectedException):
"""
缺少本子/章节异常
"""
description = '不存在本子或章节异常'
@property
def error_jmid(self) -> str:
return self.from_context(ExceptionTool.CONTEXT_KEY_MISSING_JM_ID)
class RequestRetryAllFailException(JmcomicException):
description = '请求重试全部失败异常'
pass
class ExceptionTool:
"""
抛异常的工具
@ -95,10 +100,7 @@ class ExceptionTool:
e = etype(msg, context)
# 异常处理建议
advice = JmModuleConfig.REGISTRY_EXCEPTION_ADVICE.get(etype, None)
if advice is not None:
advice(e)
cls.notify_all_listeners(e)
raise e
@ -174,3 +176,13 @@ class ExceptionTool:
raises(old, msg, context)
cls.raises = new
@classmethod
def notify_all_listeners(cls, e):
registry: Dict[Type, Callable[Type]] = JmModuleConfig.REGISTRY_EXCEPTION_LISTENER
if not registry:
return None
for accept_type, listener in registry.items():
if isinstance(e, accept_type):
listener(e)

View File

@ -17,8 +17,11 @@ class CacheRegistry:
return registry[client]
@classmethod
def enable_client_cache_on_condition(cls, option: 'JmOption', client: JmcomicClient,
cache: Union[None, bool, str, Callable]):
def enable_client_cache_on_condition(cls,
option: 'JmOption',
client: JmcomicClient,
cache: Union[None, bool, str, Callable],
):
"""
cache parameter

View File

@ -48,7 +48,8 @@ class JmTestConfigurable(unittest.TestCase):
# 设置 JmOptionJmcomicClient
option = cls.new_option()
cls.option = option
cls.client = option.build_jm_client()
# 设置缓存级别为option可以减少请求次数
cls.client = option.build_jm_client(cache='level_option')
# 跨平台设置
cls.adapt_os()

View File

@ -86,20 +86,50 @@ def log_before_raise():
jm_download_dir = env('JM_DOWNLOAD_DIR', workspace())
mkdir_if_not_exists(jm_download_dir)
# 自定义异常抛出函数在抛出前把HTML响应数据写到下载文件夹日志留痕
def raises(old, msg, extra: dict):
if ExceptionTool.EXTRA_KEY_RESP not in extra:
return old(msg, extra)
def decide_filepath(e):
resp = e.context.get(ExceptionTool.CONTEXT_KEY_RESP, None)
if resp is None:
suffix = str(time_stamp())
else:
suffix = resp.url
name = '-'.join(
fix_windir_name(it)
for it in [
e.description,
current_thread().name,
suffix
]
)
path = f'{jm_download_dir}/【出错了】{name}.log'
return path
def exception_listener(e: JmcomicException):
"""
异常监听器实现了在 GitHub Actions 把请求错误的信息下载到文件方便调试和通知使用者
"""
# 决定要写入的文件路径
path = decide_filepath(e)
# 准备内容
content = [
str(type(e)),
e.msg,
]
for k, v in e.context.items():
content.append(f'{k}: {v}')
# resp.text
resp = e.context.get(ExceptionTool.CONTEXT_KEY_RESP, None)
if resp:
content.append(f'响应文本: {resp.text}')
resp = extra[ExceptionTool.EXTRA_KEY_RESP]
# 写文件
from common import write_text, fix_windir_name
write_text(f'{jm_download_dir}/{fix_windir_name(resp.url)}', resp.text)
write_text(path, '\n'.join(content))
return old(msg, extra)
# 应用函数
ExceptionTool.replace_old_exception_executor(raises)
JmModuleConfig.register_exception_listener(JmcomicException, exception_listener)
if __name__ == '__main__':