mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.5.3: 紧急修复域名切换重试机制,优化异常机制和GitHub Actions的异常处理 (#206)
This commit is contained in:
parent
684754af57
commit
fb8a390423
@ -69,8 +69,8 @@ $ jmcomic 422866
|
|||||||
- **可扩展性强**
|
- **可扩展性强**
|
||||||
|
|
||||||
- 支持自定义本子/章节/图片下载前后的回调函数
|
- 支持自定义本子/章节/图片下载前后的回调函数
|
||||||
- 支持自定义日志
|
|
||||||
- 支持自定义类:`Downloader(负责调度)` `Option(负责配置)` `Client(负责请求)` `实体类`等
|
- 支持自定义类:`Downloader(负责调度)` `Option(负责配置)` `Client(负责请求)` `实体类`等
|
||||||
|
- 支持自定义日志、异常监听器
|
||||||
- **支持Plugin插件,可以方便地扩展功能,以及使用别人的插件,目前内置插件有**:
|
- **支持Plugin插件,可以方便地扩展功能,以及使用别人的插件,目前内置插件有**:
|
||||||
- `登录插件`
|
- `登录插件`
|
||||||
- `硬件占用监控插件`
|
- `硬件占用监控插件`
|
||||||
|
@ -163,4 +163,32 @@ def custom_jm_log():
|
|||||||
|
|
||||||
# 2. 让my_log生效
|
# 2. 让my_log生效
|
||||||
JmModuleConfig.log_executor = my_log
|
JmModuleConfig.log_executor = my_log
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## 自定义异常监听器/回调
|
||||||
|
|
||||||
|
```python
|
||||||
|
def custom_exception_listener():
|
||||||
|
"""
|
||||||
|
该函数演示jmcomic的异常监听器机制
|
||||||
|
"""
|
||||||
|
|
||||||
|
# 1. 选一个可能会发生的、你感兴趣的异常
|
||||||
|
etype = ResponseUnexpectedException
|
||||||
|
|
||||||
|
|
||||||
|
def listener(e):
|
||||||
|
"""
|
||||||
|
你的监听器方法
|
||||||
|
该方法无需返回值
|
||||||
|
:param e: 异常实例
|
||||||
|
"""
|
||||||
|
print(f'my exception listener invoke !!! exception happened: {e}')
|
||||||
|
|
||||||
|
|
||||||
|
# 注册监听器/回调
|
||||||
|
# 这个异常类(或者这个异常的子类)的实例将要被raise前,你的listener方法会被调用
|
||||||
|
JmModuleConfig.register_exception_listener(etype, listener)
|
||||||
```
|
```
|
@ -45,7 +45,7 @@ print(f'获取到{len(domain_set)}个域名,开始测试')
|
|||||||
|
|
||||||
|
|
||||||
def test_domain(domain: str):
|
def test_domain(domain: str):
|
||||||
client = option.new_jm_client(domain_list=[domain], **meta_data)
|
client = option.new_jm_client(impl='html', domain_list=[domain], **meta_data)
|
||||||
status = 'ok'
|
status = 'ok'
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
@ -11,6 +11,7 @@ client:
|
|||||||
timeout: 7
|
timeout: 7
|
||||||
domain:
|
domain:
|
||||||
html:
|
html:
|
||||||
|
- 18comic.org
|
||||||
- jmcomic1.me
|
- jmcomic1.me
|
||||||
- jmcomic.me
|
- jmcomic.me
|
||||||
|
|
||||||
|
@ -2,7 +2,7 @@
|
|||||||
# 被依赖方 <--- 使用方
|
# 被依赖方 <--- 使用方
|
||||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||||
|
|
||||||
__version__ = '2.5.2'
|
__version__ = '2.5.3'
|
||||||
|
|
||||||
from .api import *
|
from .api import *
|
||||||
from .jm_plugin import *
|
from .jm_plugin import *
|
||||||
|
@ -78,8 +78,10 @@ class AbstractJmClient(
|
|||||||
:param kwargs: 请求方法的kwargs
|
:param kwargs: 请求方法的kwargs
|
||||||
"""
|
"""
|
||||||
if domain_index >= len(self.domain_list):
|
if domain_index >= len(self.domain_list):
|
||||||
self.fallback(request, url, domain_index, retry_count, **kwargs)
|
return self.fallback(request, url, domain_index, retry_count, **kwargs)
|
||||||
|
|
||||||
|
url_backup = url
|
||||||
|
|
||||||
if url.startswith('/'):
|
if url.startswith('/'):
|
||||||
# path → url
|
# path → url
|
||||||
domain = self.domain_list[domain_index]
|
domain = self.domain_list[domain_index]
|
||||||
@ -120,9 +122,9 @@ class AbstractJmClient(
|
|||||||
self.before_retry(e, kwargs, retry_count, url)
|
self.before_retry(e, kwargs, retry_count, url)
|
||||||
|
|
||||||
if retry_count < self.retry_times:
|
if retry_count < self.retry_times:
|
||||||
return self.request_with_retry(request, url, domain_index, retry_count + 1, callback, **kwargs)
|
return self.request_with_retry(request, url_backup, domain_index, retry_count + 1, callback, **kwargs)
|
||||||
else:
|
else:
|
||||||
return self.request_with_retry(request, url, domain_index + 1, 0, callback, **kwargs)
|
return self.request_with_retry(request, url_backup, domain_index + 1, 0, callback, **kwargs)
|
||||||
|
|
||||||
# noinspection PyMethodMayBeStatic
|
# noinspection PyMethodMayBeStatic
|
||||||
def raise_if_resp_should_retry(self, resp):
|
def raise_if_resp_should_retry(self, resp):
|
||||||
@ -209,7 +211,7 @@ class AbstractJmClient(
|
|||||||
def fallback(self, request, url, domain_index, retry_count, **kwargs):
|
def fallback(self, request, url, domain_index, retry_count, **kwargs):
|
||||||
msg = f"请求重试全部失败: [{url}], {self.domain_list}"
|
msg = f"请求重试全部失败: [{url}], {self.domain_list}"
|
||||||
jm_log('req.fallback', msg)
|
jm_log('req.fallback', msg)
|
||||||
ExceptionTool.raises(msg)
|
ExceptionTool.raises(msg, {}, RequestRetryAllFailException)
|
||||||
|
|
||||||
# noinspection PyMethodMayBeStatic
|
# noinspection PyMethodMayBeStatic
|
||||||
def append_params_to_url(self, url, params):
|
def append_params_to_url(self, url, params):
|
||||||
|
@ -117,10 +117,10 @@ class JmModuleConfig:
|
|||||||
|
|
||||||
# 移动端API域名
|
# 移动端API域名
|
||||||
DOMAIN_API_LIST = str_to_list('''
|
DOMAIN_API_LIST = str_to_list('''
|
||||||
|
www.jmapinode.biz
|
||||||
www.jmapinode1.top
|
www.jmapinode1.top
|
||||||
www.jmapinode2.top
|
www.jmapinode2.top
|
||||||
www.jmapinode3.top
|
www.jmapinode3.top
|
||||||
www.jmapinode.biz
|
|
||||||
www.jmapinode.top
|
www.jmapinode.top
|
||||||
|
|
||||||
''')
|
''')
|
||||||
@ -144,8 +144,11 @@ class JmModuleConfig:
|
|||||||
REGISTRY_CLIENT = {}
|
REGISTRY_CLIENT = {}
|
||||||
# 插件注册表
|
# 插件注册表
|
||||||
REGISTRY_PLUGIN = {}
|
REGISTRY_PLUGIN = {}
|
||||||
# 异常处理器
|
# 异常监听器
|
||||||
REGISTRY_EXCEPTION_ADVICE = {}
|
# key: 异常类
|
||||||
|
# value: 函数,参数只有异常对象,无需返回值
|
||||||
|
# 这个异常类(或者这个异常的子类)的实例将要被raise前,你的listener方法会被调用
|
||||||
|
REGISTRY_EXCEPTION_LISTENER = {}
|
||||||
|
|
||||||
# 执行log的函数
|
# 执行log的函数
|
||||||
executor_log = default_jm_logging
|
executor_log = default_jm_logging
|
||||||
@ -311,7 +314,7 @@ class JmModuleConfig:
|
|||||||
# 而如果只想修改几个简单常用的配置,也可以下方的DEFAULT_XXX属性
|
# 而如果只想修改几个简单常用的配置,也可以下方的DEFAULT_XXX属性
|
||||||
JM_OPTION_VER = '2.1'
|
JM_OPTION_VER = '2.1'
|
||||||
DEFAULT_CLIENT_IMPL = 'api' # 默认Client实现类型为网页端
|
DEFAULT_CLIENT_IMPL = 'api' # 默认Client实现类型为网页端
|
||||||
DEFAULT_CLIENT_CACHE = True # 默认开启Client缓存,缓存级别是level_option,详见CacheRegistry
|
DEFAULT_CLIENT_CACHE = None # 默认关闭Client缓存。缓存的配置详见 CacheRegistry
|
||||||
DEFAULT_PROXIES = ProxyBuilder.system_proxy() # 默认使用系统代理
|
DEFAULT_PROXIES = ProxyBuilder.system_proxy() # 默认使用系统代理
|
||||||
|
|
||||||
default_option_dict: dict = {
|
default_option_dict: dict = {
|
||||||
@ -404,8 +407,8 @@ class JmModuleConfig:
|
|||||||
cls.REGISTRY_CLIENT[client_class.client_key] = client_class
|
cls.REGISTRY_CLIENT[client_class.client_key] = client_class
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def register_exception_advice(cls, etype, eadvice):
|
def register_exception_listener(cls, etype, listener):
|
||||||
cls.REGISTRY_EXCEPTION_ADVICE[etype] = eadvice
|
cls.REGISTRY_EXCEPTION_LISTENER[etype] = listener
|
||||||
|
|
||||||
|
|
||||||
jm_log = JmModuleConfig.jm_log
|
jm_log = JmModuleConfig.jm_log
|
||||||
|
@ -3,9 +3,7 @@ from .jm_entity import *
|
|||||||
|
|
||||||
|
|
||||||
class JmcomicException(Exception):
|
class JmcomicException(Exception):
|
||||||
"""
|
description = 'jmcomic 模块异常'
|
||||||
jmcomic 模块异常
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, msg: str, context: dict):
|
def __init__(self, msg: str, context: dict):
|
||||||
self.msg = msg
|
self.msg = msg
|
||||||
@ -16,19 +14,22 @@ class JmcomicException(Exception):
|
|||||||
|
|
||||||
|
|
||||||
class ResponseUnexpectedException(JmcomicException):
|
class ResponseUnexpectedException(JmcomicException):
|
||||||
"""
|
description = '响应不符合预期异常'
|
||||||
响应不符合预期异常
|
|
||||||
"""
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def resp(self):
|
def resp(self):
|
||||||
return self.from_context(ExceptionTool.CONTEXT_KEY_RESP)
|
return self.from_context(ExceptionTool.CONTEXT_KEY_RESP)
|
||||||
|
|
||||||
|
|
||||||
class RegularNotMatchException(ResponseUnexpectedException):
|
class RegularNotMatchException(JmcomicException):
|
||||||
"""
|
description = '正则表达式不匹配异常'
|
||||||
正则表达式不匹配异常
|
|
||||||
"""
|
@property
|
||||||
|
def resp(self):
|
||||||
|
"""
|
||||||
|
可能为None
|
||||||
|
"""
|
||||||
|
return self.context.get(ExceptionTool.CONTEXT_KEY_RESP, None)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def error_text(self):
|
def error_text(self):
|
||||||
@ -40,19 +41,23 @@ class RegularNotMatchException(ResponseUnexpectedException):
|
|||||||
|
|
||||||
|
|
||||||
class JsonResolveFailException(ResponseUnexpectedException):
|
class JsonResolveFailException(ResponseUnexpectedException):
|
||||||
|
description = 'Json解析异常'
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class MissingAlbumPhotoException(ResponseUnexpectedException):
|
class MissingAlbumPhotoException(ResponseUnexpectedException):
|
||||||
"""
|
description = '不存在本子或章节异常'
|
||||||
缺少本子/章节异常
|
|
||||||
"""
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def error_jmid(self) -> str:
|
def error_jmid(self) -> str:
|
||||||
return self.from_context(ExceptionTool.CONTEXT_KEY_MISSING_JM_ID)
|
return self.from_context(ExceptionTool.CONTEXT_KEY_MISSING_JM_ID)
|
||||||
|
|
||||||
|
|
||||||
|
class RequestRetryAllFailException(JmcomicException):
|
||||||
|
description = '请求重试全部失败异常'
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class ExceptionTool:
|
class ExceptionTool:
|
||||||
"""
|
"""
|
||||||
抛异常的工具
|
抛异常的工具
|
||||||
@ -95,10 +100,7 @@ class ExceptionTool:
|
|||||||
e = etype(msg, context)
|
e = etype(msg, context)
|
||||||
|
|
||||||
# 异常处理建议
|
# 异常处理建议
|
||||||
advice = JmModuleConfig.REGISTRY_EXCEPTION_ADVICE.get(etype, None)
|
cls.notify_all_listeners(e)
|
||||||
|
|
||||||
if advice is not None:
|
|
||||||
advice(e)
|
|
||||||
|
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
@ -174,3 +176,13 @@ class ExceptionTool:
|
|||||||
raises(old, msg, context)
|
raises(old, msg, context)
|
||||||
|
|
||||||
cls.raises = new
|
cls.raises = new
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def notify_all_listeners(cls, e):
|
||||||
|
registry: Dict[Type, Callable[Type]] = JmModuleConfig.REGISTRY_EXCEPTION_LISTENER
|
||||||
|
if not registry:
|
||||||
|
return None
|
||||||
|
|
||||||
|
for accept_type, listener in registry.items():
|
||||||
|
if isinstance(e, accept_type):
|
||||||
|
listener(e)
|
||||||
|
@ -17,8 +17,11 @@ class CacheRegistry:
|
|||||||
return registry[client]
|
return registry[client]
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def enable_client_cache_on_condition(cls, option: 'JmOption', client: JmcomicClient,
|
def enable_client_cache_on_condition(cls,
|
||||||
cache: Union[None, bool, str, Callable]):
|
option: 'JmOption',
|
||||||
|
client: JmcomicClient,
|
||||||
|
cache: Union[None, bool, str, Callable],
|
||||||
|
):
|
||||||
"""
|
"""
|
||||||
cache parameter
|
cache parameter
|
||||||
|
|
||||||
@ -539,7 +542,7 @@ class JmOption:
|
|||||||
|
|
||||||
pclass: Type[JmOptionPlugin]
|
pclass: Type[JmOptionPlugin]
|
||||||
plugin: Optional[JmOptionPlugin] = None
|
plugin: Optional[JmOptionPlugin] = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 构建插件对象
|
# 构建插件对象
|
||||||
plugin: JmOptionPlugin = pclass.build(self)
|
plugin: JmOptionPlugin = pclass.build(self)
|
||||||
|
@ -48,7 +48,8 @@ class JmTestConfigurable(unittest.TestCase):
|
|||||||
# 设置 JmOption,JmcomicClient
|
# 设置 JmOption,JmcomicClient
|
||||||
option = cls.new_option()
|
option = cls.new_option()
|
||||||
cls.option = option
|
cls.option = option
|
||||||
cls.client = option.build_jm_client()
|
# 设置缓存级别为option,可以减少请求次数
|
||||||
|
cls.client = option.build_jm_client(cache='level_option')
|
||||||
|
|
||||||
# 跨平台设置
|
# 跨平台设置
|
||||||
cls.adapt_os()
|
cls.adapt_os()
|
||||||
|
@ -86,20 +86,50 @@ def log_before_raise():
|
|||||||
jm_download_dir = env('JM_DOWNLOAD_DIR', workspace())
|
jm_download_dir = env('JM_DOWNLOAD_DIR', workspace())
|
||||||
mkdir_if_not_exists(jm_download_dir)
|
mkdir_if_not_exists(jm_download_dir)
|
||||||
|
|
||||||
# 自定义异常抛出函数,在抛出前把HTML响应数据写到下载文件夹(日志留痕)
|
def decide_filepath(e):
|
||||||
def raises(old, msg, extra: dict):
|
resp = e.context.get(ExceptionTool.CONTEXT_KEY_RESP, None)
|
||||||
if ExceptionTool.EXTRA_KEY_RESP not in extra:
|
|
||||||
return old(msg, extra)
|
if resp is None:
|
||||||
|
suffix = str(time_stamp())
|
||||||
|
else:
|
||||||
|
suffix = resp.url
|
||||||
|
|
||||||
|
name = '-'.join(
|
||||||
|
fix_windir_name(it)
|
||||||
|
for it in [
|
||||||
|
e.description,
|
||||||
|
current_thread().name,
|
||||||
|
suffix
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
path = f'{jm_download_dir}/【出错了】{name}.log'
|
||||||
|
return path
|
||||||
|
|
||||||
|
def exception_listener(e: JmcomicException):
|
||||||
|
"""
|
||||||
|
异常监听器,实现了在 GitHub Actions 下,把请求错误的信息下载到文件,方便调试和通知使用者
|
||||||
|
"""
|
||||||
|
# 决定要写入的文件路径
|
||||||
|
path = decide_filepath(e)
|
||||||
|
|
||||||
|
# 准备内容
|
||||||
|
content = [
|
||||||
|
str(type(e)),
|
||||||
|
e.msg,
|
||||||
|
]
|
||||||
|
for k, v in e.context.items():
|
||||||
|
content.append(f'{k}: {v}')
|
||||||
|
|
||||||
|
# resp.text
|
||||||
|
resp = e.context.get(ExceptionTool.CONTEXT_KEY_RESP, None)
|
||||||
|
if resp:
|
||||||
|
content.append(f'响应文本: {resp.text}')
|
||||||
|
|
||||||
resp = extra[ExceptionTool.EXTRA_KEY_RESP]
|
|
||||||
# 写文件
|
# 写文件
|
||||||
from common import write_text, fix_windir_name
|
write_text(path, '\n'.join(content))
|
||||||
write_text(f'{jm_download_dir}/{fix_windir_name(resp.url)}', resp.text)
|
|
||||||
|
|
||||||
return old(msg, extra)
|
JmModuleConfig.register_exception_listener(JmcomicException, exception_listener)
|
||||||
|
|
||||||
# 应用函数
|
|
||||||
ExceptionTool.replace_old_exception_executor(raises)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
Loading…
Reference in New Issue
Block a user