mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.5.1: 更新文档,增加option配置和log自定义说明; 优化download api,有返回值对外界更友好; 重构异常处理,异常类别更清晰; 扩大请求重试的范围,加入json格式检查; (#197) [skip ci]
This commit is contained in:
parent
14050d4657
commit
a2d29994ca
@ -15,6 +15,10 @@ JmOption.default().to_file('./option.yml') # 创建默认option,导出为optio
|
||||
## 2. option常规配置项
|
||||
|
||||
```yml
|
||||
# 开启jmcomic的日志输入,默认为true
|
||||
# 对日志有需求的可进一步参考文档
|
||||
log: true
|
||||
|
||||
# 配置客户端相关
|
||||
client:
|
||||
# impl: 客户端实现类,不配置默认会使用JmModuleConfig.DEFAULT_CLIENT_IMPL
|
||||
@ -32,6 +36,10 @@ client:
|
||||
- 18comic.vip
|
||||
- 18comic.org
|
||||
|
||||
# retry_times: 请求失败重试次数,默认为5
|
||||
retry_times: 5
|
||||
|
||||
# postman: 请求配置
|
||||
postman:
|
||||
meta_data:
|
||||
# proxies: 代理配置,默认是 system,表示使用系统代理。
|
||||
|
55
assets/docs/sources/tutorial/11_log_custom.md
Normal file
55
assets/docs/sources/tutorial/11_log_custom.md
Normal file
@ -0,0 +1,55 @@
|
||||
# 日志自定义 - 如果你不想看到那么多的日志
|
||||
|
||||
本文档缘起于 GitHub Discussions: [discussions/195](https://github.com/hect0x7/JMComic-Crawler-Python/discussions/195)
|
||||
|
||||
下面是这个问题的解决方法:
|
||||
|
||||
## 1. 日志完全开启/关闭
|
||||
|
||||
使用代码:
|
||||
|
||||
```
|
||||
from jmcomic import disable_jm_log
|
||||
disable_jm_log()
|
||||
```
|
||||
|
||||
使用配置:
|
||||
|
||||
```yml
|
||||
log: false
|
||||
```
|
||||
|
||||
## 2. 日志过滤,只保留特定topic
|
||||
|
||||
使用插件配置
|
||||
|
||||
```yml
|
||||
log: true
|
||||
|
||||
plugins:
|
||||
after_init:
|
||||
- plugin: log_topic_filter # 日志topic过滤插件
|
||||
kwargs:
|
||||
whitelist: [ # 只保留api和html,这两个是Client发请求时会打的日志topic
|
||||
'api',
|
||||
'html',
|
||||
]
|
||||
```
|
||||
|
||||
## 3. 屏蔽插件的日志
|
||||
|
||||
给插件配置加上一个`log`配置项即可
|
||||
|
||||
```yml
|
||||
plugins:
|
||||
after_init:
|
||||
- plugin: client_proxy # 提高移动端的请求效率的插件
|
||||
log: false # 插件自身不打印日志
|
||||
kwargs:
|
||||
proxy_client_key: cl_proxy_future
|
||||
whitelist: [ api, ]
|
||||
```
|
||||
|
||||
## 4. 完全自定义 jmcomic 日志
|
||||
|
||||
你可以自定义jmcomic的模块日志打印函数,参考文档:[模块自定义](./4_module_custom.md#自定义log)
|
@ -102,7 +102,7 @@ JmModuleConfig.PFIELD_ADVICE['myname'] = lambda photo: f'【{photo.id}】{photo.
|
||||
|
||||
### 文件夹名=第x话+标题
|
||||
|
||||
```python
|
||||
```yml
|
||||
# 直接使用内置字段 indextitle 即可
|
||||
dir_rule:
|
||||
rule: Bd_Pindextitle
|
||||
|
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.5.0'
|
||||
__version__ = '2.5.1'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
@ -11,8 +11,8 @@ from .jm_plugin import *
|
||||
gb = dict(filter(lambda pair: isinstance(pair[1], type), globals().items()))
|
||||
|
||||
|
||||
def register_jmcomic_component(gb: dict, method, valid_interface: type):
|
||||
for v in gb.values():
|
||||
def register_jmcomic_component(variables: Dict[str, Any], method, valid_interface: type):
|
||||
for v in variables.values():
|
||||
if v != valid_interface and issubclass(v, valid_interface):
|
||||
method(v)
|
||||
|
||||
|
@ -5,7 +5,7 @@ def download_batch(download_api,
|
||||
jm_id_iter: Union[Iterable, Generator],
|
||||
option=None,
|
||||
downloader=None,
|
||||
):
|
||||
) -> Set[Tuple[JmAlbumDetail, JmDownloader]]:
|
||||
"""
|
||||
批量下载 album / photo
|
||||
|
||||
@ -21,34 +21,60 @@ def download_batch(download_api,
|
||||
if option is None:
|
||||
option = JmModuleConfig.option_class().default()
|
||||
|
||||
return multi_thread_launcher(
|
||||
result = set()
|
||||
|
||||
def callback(*ret):
|
||||
result.add(ret)
|
||||
|
||||
multi_thread_launcher(
|
||||
iter_objs=set(
|
||||
JmcomicText.parse_to_jm_id(jmid)
|
||||
for jmid in jm_id_iter
|
||||
),
|
||||
apply_each_obj_func=lambda aid: download_api(aid, option, downloader),
|
||||
apply_each_obj_func=lambda aid: download_api(aid,
|
||||
option,
|
||||
downloader,
|
||||
callback=callback,
|
||||
),
|
||||
wait_finish=True
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def download_album(jm_album_id, option=None, downloader=None):
|
||||
|
||||
def download_album(jm_album_id,
|
||||
option=None,
|
||||
downloader=None,
|
||||
callback=None,
|
||||
):
|
||||
"""
|
||||
下载一个本子(album),包含其所有的章节(photo)
|
||||
|
||||
当jm_album_id不是str或int时,相当于调用 download_batch(download_album, jm_album_id, option, downloader)
|
||||
当jm_album_id不是str或int时,视为批量下载,相当于调用 download_batch(download_album, jm_album_id, option, downloader)
|
||||
|
||||
:param jm_album_id: 本子的禁漫车号
|
||||
:param option: 下载选项
|
||||
:param downloader: 下载器类
|
||||
:param callback: 返回值回调函数,可以拿到 album 和 downloader
|
||||
:return: 对于的本子实体类,下载器(如果是上述的批量情况,返回值为download_batch的返回值)
|
||||
"""
|
||||
|
||||
if not isinstance(jm_album_id, (str, int)):
|
||||
return download_batch(download_album, jm_album_id, option, downloader)
|
||||
|
||||
with new_downloader(option, downloader) as dler:
|
||||
dler.download_album(jm_album_id)
|
||||
album = dler.download_album(jm_album_id)
|
||||
|
||||
if callback is not None:
|
||||
callback(album, dler)
|
||||
|
||||
return album, dler
|
||||
|
||||
|
||||
def download_photo(jm_photo_id, option=None, downloader=None):
|
||||
def download_photo(jm_photo_id,
|
||||
option=None,
|
||||
downloader=None,
|
||||
callback=None):
|
||||
"""
|
||||
下载一个章节(photo),参数同 download_album
|
||||
"""
|
||||
@ -56,7 +82,12 @@ def download_photo(jm_photo_id, option=None, downloader=None):
|
||||
return download_batch(download_photo, jm_photo_id, option)
|
||||
|
||||
with new_downloader(option, downloader) as dler:
|
||||
dler.download_photo(jm_photo_id)
|
||||
photo = dler.download_photo(jm_photo_id)
|
||||
|
||||
if callback is not None:
|
||||
callback(photo, dler)
|
||||
|
||||
return photo, dler
|
||||
|
||||
|
||||
def new_downloader(option=None, downloader=None) -> JmDownloader:
|
||||
|
@ -45,7 +45,7 @@ class AbstractJmClient(
|
||||
|
||||
def get_jm_image(self, img_url) -> JmImageResp:
|
||||
|
||||
def judge(resp):
|
||||
def callback(resp):
|
||||
"""
|
||||
使用此方法包装 self.get,使得图片数据为空时,判定为请求失败时,走重试逻辑
|
||||
"""
|
||||
@ -53,14 +53,14 @@ class AbstractJmClient(
|
||||
resp.require_success()
|
||||
return resp
|
||||
|
||||
return self.get(img_url, judge=judge, headers=JmModuleConfig.new_html_headers())
|
||||
return self.get(img_url, callback=callback, headers=JmModuleConfig.new_html_headers())
|
||||
|
||||
def request_with_retry(self,
|
||||
request,
|
||||
url,
|
||||
domain_index=0,
|
||||
retry_count=0,
|
||||
judge=lambda resp: resp,
|
||||
callback=None,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
@ -74,7 +74,7 @@ class AbstractJmClient(
|
||||
:param url: 图片url / path (/album/xxx)
|
||||
:param domain_index: 域名下标
|
||||
:param retry_count: 重试次数
|
||||
:param judge: 判定响应是否成功
|
||||
:param callback: 回调,可以接收resp返回新的resp,也可以抛出异常强制重试
|
||||
:param kwargs: 请求方法的kwargs
|
||||
"""
|
||||
if domain_index >= len(self.domain_list):
|
||||
@ -104,9 +104,15 @@ class AbstractJmClient(
|
||||
|
||||
try:
|
||||
resp = request(url, **kwargs)
|
||||
return judge(resp)
|
||||
except KeyboardInterrupt as e:
|
||||
raise e
|
||||
|
||||
# 回调,可以接收resp返回新的resp,也可以抛出异常强制重试
|
||||
if callback is not None:
|
||||
resp = callback(resp)
|
||||
|
||||
# 依然是回调,在最后返回之前,还可以判断resp是否重试
|
||||
resp = self.raise_if_resp_should_retry(resp)
|
||||
|
||||
return resp
|
||||
except Exception as e:
|
||||
if self.retry_times == 0:
|
||||
raise e
|
||||
@ -114,9 +120,16 @@ class AbstractJmClient(
|
||||
self.before_retry(e, kwargs, retry_count, url)
|
||||
|
||||
if retry_count < self.retry_times:
|
||||
return self.request_with_retry(request, url, domain_index, retry_count + 1, judge, **kwargs)
|
||||
return self.request_with_retry(request, url, domain_index, retry_count + 1, callback, **kwargs)
|
||||
else:
|
||||
return self.request_with_retry(request, url, domain_index + 1, 0, judge, **kwargs)
|
||||
return self.request_with_retry(request, url, domain_index + 1, 0, callback, **kwargs)
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def raise_if_resp_should_retry(self, resp):
|
||||
"""
|
||||
依然是回调,在最后返回之前,还可以判断resp是否重试
|
||||
"""
|
||||
return resp
|
||||
|
||||
def update_request_with_specify_domain(self, kwargs: dict, domain: str):
|
||||
"""
|
||||
@ -269,12 +282,12 @@ class JmHtmlClient(AbstractJmClient):
|
||||
|
||||
return photo
|
||||
|
||||
def fetch_detail_entity(self, apid, prefix):
|
||||
def fetch_detail_entity(self, jmid, prefix):
|
||||
# 参数校验
|
||||
apid = JmcomicText.parse_to_jm_id(apid)
|
||||
jmid = JmcomicText.parse_to_jm_id(jmid)
|
||||
|
||||
# 请求
|
||||
resp = self.get_jm_html(f"/{prefix}/{apid}")
|
||||
resp = self.get_jm_html(f"/{prefix}/{jmid}")
|
||||
|
||||
# 用 JmcomicText 解析 html,返回实体类
|
||||
if prefix == 'album':
|
||||
@ -474,10 +487,10 @@ class JmHtmlClient(AbstractJmClient):
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def require_resp_success_else_raise(cls, resp, orig_req_url: str):
|
||||
def require_resp_success_else_raise(cls, resp, url: str):
|
||||
"""
|
||||
:param resp: 响应对象
|
||||
:param orig_req_url: /photo/12412312
|
||||
:param url: /photo/12412312
|
||||
"""
|
||||
resp_url: str = resp.url
|
||||
|
||||
@ -490,11 +503,11 @@ class JmHtmlClient(AbstractJmClient):
|
||||
|
||||
# 3. 检查错误类型
|
||||
def match_case(error_path):
|
||||
return resp_url.endswith(error_path) and not orig_req_url.endswith(error_path)
|
||||
return resp_url.endswith(error_path) and not url.endswith(error_path)
|
||||
|
||||
# 3.1 album_missing
|
||||
if match_case('/error/album_missing'):
|
||||
ExceptionTool.raise_missing(resp, orig_req_url)
|
||||
ExceptionTool.raise_missing(resp, JmcomicText.parse_to_jm_id(url))
|
||||
|
||||
# 3.2 user_missing
|
||||
if match_case('/error/user_missing'):
|
||||
@ -639,17 +652,17 @@ class JmApiClient(AbstractJmClient):
|
||||
|
||||
return scramble_id
|
||||
|
||||
def fetch_detail_entity(self, apid, clazz):
|
||||
def fetch_detail_entity(self, jmid, clazz):
|
||||
"""
|
||||
请求实体类
|
||||
"""
|
||||
apid = JmcomicText.parse_to_jm_id(apid)
|
||||
jmid = JmcomicText.parse_to_jm_id(jmid)
|
||||
url = self.API_ALBUM if issubclass(clazz, JmAlbumDetail) else self.API_CHAPTER
|
||||
resp = self.req_api(
|
||||
resp = self.req_api(self.append_params_to_url(
|
||||
url,
|
||||
params={
|
||||
'id': apid,
|
||||
},
|
||||
{
|
||||
'id': jmid
|
||||
})
|
||||
)
|
||||
|
||||
return JmApiAdaptTool.parse_entity(resp.res_data, clazz)
|
||||
@ -886,18 +899,57 @@ class JmApiClient(AbstractJmClient):
|
||||
return ts
|
||||
|
||||
@classmethod
|
||||
def require_resp_success(cls, resp: JmApiResp, orig_req_url: str):
|
||||
def require_resp_success(cls, resp: JmApiResp, url: Optional[str] = None):
|
||||
"""
|
||||
|
||||
:param resp: 响应对象
|
||||
:param url: 请求路径,例如 /setting
|
||||
"""
|
||||
resp.require_success()
|
||||
|
||||
# 1. 检查是否 album_missing
|
||||
# json: {'code': 200, 'data': []}
|
||||
data = resp.model().data
|
||||
if isinstance(data, list) and len(data) == 0:
|
||||
ExceptionTool.raise_missing(resp, orig_req_url)
|
||||
ExceptionTool.raise_missing(resp, JmcomicText.parse_to_jm_id(url))
|
||||
|
||||
# 2. 是否是特殊的内容
|
||||
# 暂无
|
||||
|
||||
def raise_if_resp_should_retry(self, resp):
|
||||
"""
|
||||
该方法会判断resp返回值是否是json格式,
|
||||
如果不是,大概率是禁漫内部异常,需要进行重试
|
||||
|
||||
由于完整的json格式校验会有性能开销,所以只做简单的检查,
|
||||
只校验第一个有效字符是不是 '{',如果不是,就认为异常数据,需要重试
|
||||
|
||||
:param resp: 响应对象
|
||||
:return: resp
|
||||
"""
|
||||
if isinstance(resp, JmResp):
|
||||
# 不对包装过的resp对象做校验,包装者自行校验
|
||||
# 例如图片请求
|
||||
return resp
|
||||
|
||||
url = resp.request.url
|
||||
|
||||
if self.API_SCRAMBLE in url:
|
||||
# /chapter_view_template 这个接口不是返回json数据,不做检查
|
||||
return resp
|
||||
|
||||
text = resp.text
|
||||
for char in text:
|
||||
if char not in (' ', '\n', '\t'):
|
||||
# 找到第一个有效字符
|
||||
ExceptionTool.require_true(
|
||||
char == '{',
|
||||
f'请求不是json格式,强制重试!响应文本: [{resp.text}]'
|
||||
)
|
||||
return resp
|
||||
|
||||
ExceptionTool.raises_resp(f'响应无数据!request_url=[{url}]', resp)
|
||||
|
||||
def after_init(self):
|
||||
# 保证拥有cookies,因为移动端要求必须携带cookies,否则会直接跳转同一本子【禁漫娘】
|
||||
if JmModuleConfig.flag_api_client_require_cookies:
|
||||
|
@ -41,16 +41,15 @@ class JmResp:
|
||||
|
||||
def require_success(self):
|
||||
if self.is_not_success:
|
||||
ExceptionTool.raises_resp(self.text, self)
|
||||
ExceptionTool.raises_resp(self.error_msg(), self)
|
||||
|
||||
def error_msg(self):
|
||||
return self.text
|
||||
|
||||
|
||||
class JmImageResp(JmResp):
|
||||
|
||||
def require_success(self):
|
||||
if self.is_not_success:
|
||||
ExceptionTool.raises_resp(self.get_error_msg(), self)
|
||||
|
||||
def get_error_msg(self):
|
||||
def error_msg(self):
|
||||
msg = f'禁漫图片获取失败: [{self.url}]'
|
||||
if self.http_code != 200:
|
||||
msg += f',http状态码={self.http_code}'
|
||||
@ -77,7 +76,7 @@ class JmImageResp(JmResp):
|
||||
# 解密图片并保存文件
|
||||
JmImageTool.decode_and_save(
|
||||
JmImageTool.get_num_by_url(scramble_id, img_url),
|
||||
JmImageTool.open_Image(self.content),
|
||||
JmImageTool.open_image(self.content),
|
||||
path,
|
||||
)
|
||||
|
||||
@ -86,7 +85,10 @@ class JmJsonResp(JmResp):
|
||||
|
||||
@field_cache()
|
||||
def json(self) -> Dict:
|
||||
return self.resp.json()
|
||||
try:
|
||||
return self.resp.json()
|
||||
except Exception:
|
||||
ExceptionTool.raises_resp('json解析失败', self, JsonResolveFailException)
|
||||
|
||||
def model(self) -> DictModel:
|
||||
return DictModel(self.json())
|
||||
|
@ -6,14 +6,6 @@ def default_jm_logging(topic: str, msg: str):
|
||||
print(f'{format_ts()}:【{topic}】{msg}')
|
||||
|
||||
|
||||
def default_raise_exception_executor(msg, _extra):
|
||||
raise JmModuleConfig.CLASS_EXCEPTION(msg)
|
||||
|
||||
|
||||
class JmcomicException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# 禁漫常量
|
||||
class JmMagicConstants:
|
||||
# 搜索参数-排序
|
||||
@ -147,16 +139,16 @@ class JmModuleConfig:
|
||||
CLASS_ALBUM = None
|
||||
CLASS_PHOTO = None
|
||||
CLASS_IMAGE = None
|
||||
CLASS_EXCEPTION = JmcomicException
|
||||
|
||||
# 客户端注册表
|
||||
REGISTRY_CLIENT = {}
|
||||
# 插件注册表
|
||||
REGISTRY_PLUGIN = {}
|
||||
# 异常处理器
|
||||
REGISTRY_EXCEPTION_ADVICE = {}
|
||||
|
||||
# 执行log的函数
|
||||
executor_log = default_jm_logging
|
||||
# 网页正则表达式解析失败时,执行抛出异常的函数,可以替换掉用于log
|
||||
executor_raise_exception = default_raise_exception_executor
|
||||
|
||||
# 使用固定时间戳
|
||||
flag_use_fix_timestamp = True
|
||||
@ -411,6 +403,10 @@ class JmModuleConfig:
|
||||
f'未配置client_key, class: {client_class}')
|
||||
cls.REGISTRY_CLIENT[client_class.client_key] = client_class
|
||||
|
||||
@classmethod
|
||||
def register_exception_advice(cls, etype, eadvice):
|
||||
cls.REGISTRY_EXCEPTION_ADVICE[etype] = eadvice
|
||||
|
||||
|
||||
jm_log = JmModuleConfig.jm_log
|
||||
disable_jm_log = JmModuleConfig.disable_jm_log
|
||||
|
@ -57,6 +57,7 @@ class JmDownloader(DownloadCallback):
|
||||
client = self.client_for_album(album_id)
|
||||
album = client.get_album_detail(album_id)
|
||||
self.download_by_album_detail(album, client)
|
||||
return album
|
||||
|
||||
def download_by_album_detail(self, album: JmAlbumDetail, client: JmcomicClient):
|
||||
self.before_album(album)
|
||||
@ -71,6 +72,7 @@ class JmDownloader(DownloadCallback):
|
||||
client = self.client_for_photo(photo_id)
|
||||
photo = client.get_photo_detail(photo_id)
|
||||
self.download_by_photo_detail(photo, client)
|
||||
return photo
|
||||
|
||||
def download_by_photo_detail(self, photo: JmPhotoDetail, client: JmcomicClient):
|
||||
client.check_photo(photo)
|
||||
|
@ -186,6 +186,10 @@ class JmImageDetail(JmBaseEntity):
|
||||
def filename_without_suffix(self):
|
||||
return self.img_file_name
|
||||
|
||||
@property
|
||||
def filename(self):
|
||||
return self.img_file_name + self.img_file_suffix
|
||||
|
||||
@property
|
||||
def is_gif(self):
|
||||
return self.img_file_suffix == '.gif'
|
||||
@ -478,7 +482,7 @@ class JmAlbumDetail(DetailEntity):
|
||||
|
||||
return ret
|
||||
|
||||
def create_photo_detail(self, index) -> Tuple[JmPhotoDetail, Tuple]:
|
||||
def create_photo_detail(self, index) -> JmPhotoDetail:
|
||||
# 校验参数
|
||||
length = len(self.episode_list)
|
||||
|
||||
@ -497,10 +501,10 @@ class JmAlbumDetail(DetailEntity):
|
||||
from_album=self,
|
||||
)
|
||||
|
||||
return photo, (self.episode_list[index])
|
||||
return photo
|
||||
|
||||
def getindex(self, item) -> JmPhotoDetail:
|
||||
return self.create_photo_detail(item)[0]
|
||||
return self.create_photo_detail(item)
|
||||
|
||||
def __getitem__(self, item) -> Union[JmPhotoDetail, List[JmPhotoDetail]]:
|
||||
return super().__getitem__(item)
|
||||
|
158
src/jmcomic/jm_exception.py
Normal file
158
src/jmcomic/jm_exception.py
Normal file
@ -0,0 +1,158 @@
|
||||
# 该文件存放jmcomic的异常机制设计和实现
|
||||
from .jm_entity import *
|
||||
|
||||
|
||||
class JmcomicException(Exception):
|
||||
"""
|
||||
jmcomic 模块异常
|
||||
"""
|
||||
|
||||
def __init__(self, msg: str, context: dict):
|
||||
self.msg = msg
|
||||
self.context = context
|
||||
|
||||
def from_context(self, key):
|
||||
return self.context[key]
|
||||
|
||||
|
||||
class ResponseUnexpectedException(JmcomicException):
|
||||
"""
|
||||
响应不符合预期异常
|
||||
"""
|
||||
|
||||
@property
|
||||
def resp(self):
|
||||
return self.from_context(ExceptionTool.CONTEXT_KEY_RESP)
|
||||
|
||||
|
||||
class RegularNotMatchException(ResponseUnexpectedException):
|
||||
"""
|
||||
正则表达式不匹配异常
|
||||
"""
|
||||
|
||||
@property
|
||||
def error_text(self):
|
||||
return self.from_context(ExceptionTool.CONTEXT_KEY_HTML)
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
return self.from_context(ExceptionTool.CONTEXT_KEY_RE_PATTERN)
|
||||
|
||||
|
||||
class JsonResolveFailException(ResponseUnexpectedException):
|
||||
pass
|
||||
|
||||
|
||||
class MissingAlbumPhotoException(ResponseUnexpectedException):
|
||||
"""
|
||||
缺少本子/章节异常
|
||||
"""
|
||||
|
||||
@property
|
||||
def error_jmid(self) -> str:
|
||||
return self.from_context(ExceptionTool.CONTEXT_KEY_MISSING_JM_ID)
|
||||
|
||||
|
||||
class ExceptionTool:
|
||||
"""
|
||||
抛异常的工具
|
||||
1: 能简化 if-raise 语句的编写
|
||||
2: 有更好的上下文信息传递方式
|
||||
"""
|
||||
|
||||
CONTEXT_KEY_RESP = 'resp'
|
||||
CONTEXT_KEY_HTML = 'html'
|
||||
CONTEXT_KEY_RE_PATTERN = 'pattern'
|
||||
CONTEXT_KEY_MISSING_JM_ID = 'missing_jm_id'
|
||||
|
||||
@classmethod
|
||||
def raises(cls,
|
||||
msg: str,
|
||||
context: dict = None,
|
||||
etype: Optional[Type[Exception]] = None,
|
||||
):
|
||||
"""
|
||||
抛出异常
|
||||
|
||||
:param msg: 异常消息
|
||||
:param context: 异常上下文数据
|
||||
:param etype: 异常类型,默认使用 JmcomicException
|
||||
"""
|
||||
if context is None:
|
||||
context = {}
|
||||
|
||||
if etype is None:
|
||||
etype = JmcomicException
|
||||
|
||||
# 异常对象
|
||||
e = etype(msg, context)
|
||||
|
||||
# 异常处理建议
|
||||
advice = JmModuleConfig.REGISTRY_EXCEPTION_ADVICE.get(etype, None)
|
||||
|
||||
if advice is not None:
|
||||
advice(e)
|
||||
|
||||
raise e
|
||||
|
||||
@classmethod
|
||||
def raises_regex(cls,
|
||||
msg: str,
|
||||
html: str,
|
||||
pattern: Pattern,
|
||||
):
|
||||
cls.raises(
|
||||
msg,
|
||||
{
|
||||
cls.CONTEXT_KEY_HTML: html,
|
||||
cls.CONTEXT_KEY_RE_PATTERN: pattern,
|
||||
},
|
||||
RegularNotMatchException,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def raises_resp(cls,
|
||||
msg: str,
|
||||
resp,
|
||||
etype=ResponseUnexpectedException
|
||||
):
|
||||
cls.raises(
|
||||
msg, {
|
||||
cls.CONTEXT_KEY_RESP: resp
|
||||
},
|
||||
etype,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def raise_missing(cls,
|
||||
resp,
|
||||
jmid: str,
|
||||
):
|
||||
"""
|
||||
抛出本子/章节的异常
|
||||
:param resp: 响应对象
|
||||
:param jmid: 禁漫本子/章节id
|
||||
"""
|
||||
url = resp.url
|
||||
|
||||
req_type = "本子" if "album" in url else "章节"
|
||||
cls.raises(
|
||||
(
|
||||
f'请求的{req_type}不存在!({url})\n'
|
||||
'原因可能为:\n'
|
||||
f'1. id有误,检查你的{req_type}id\n'
|
||||
'2. 该漫画只对登录用户可见,请配置你的cookies,或者使用移动端Client(api)\n'
|
||||
),
|
||||
{
|
||||
cls.CONTEXT_KEY_RESP: resp,
|
||||
cls.CONTEXT_KEY_MISSING_JM_ID: jmid,
|
||||
},
|
||||
MissingAlbumPhotoException,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def require_true(cls, case: bool, msg: str):
|
||||
if case:
|
||||
return
|
||||
|
||||
cls.raises(msg)
|
@ -17,7 +17,8 @@ class CacheRegistry:
|
||||
return registry[client]
|
||||
|
||||
@classmethod
|
||||
def enable_client_cache_on_condition(cls, option: 'JmOption', client: JmcomicClient, cache: Union[None, bool, str, Callable]):
|
||||
def enable_client_cache_on_condition(cls, option: 'JmOption', client: JmcomicClient,
|
||||
cache: Union[None, bool, str, Callable]):
|
||||
"""
|
||||
cache parameter
|
||||
|
||||
@ -132,7 +133,9 @@ class DirRule:
|
||||
|
||||
# Axxx or Pyyy
|
||||
key = 1 if rule[0] == 'A' else 2
|
||||
solve_func = lambda detail, ref=rule[1:]: fix_windir_name(str(DetailEntity.get_dirname(detail, ref)))
|
||||
|
||||
def solve_func(detail):
|
||||
return fix_windir_name(str(DetailEntity.get_dirname(detail, rule[1:])))
|
||||
|
||||
# 保存缓存
|
||||
rule_solver = (key, solve_func, rule)
|
||||
@ -367,7 +370,7 @@ class JmOption:
|
||||
"""
|
||||
return self.new_jm_client(**kwargs)
|
||||
|
||||
def new_jm_client(self, domain_list=None, impl=None, cache=None, **kwargs) -> JmcomicClient:
|
||||
def new_jm_client(self, domain_list=None, impl=None, cache=None, **kwargs) -> Union[JmHtmlClient, JmApiClient]:
|
||||
"""
|
||||
创建新的Client(客户端),不同Client之间的元数据不共享
|
||||
"""
|
||||
@ -433,6 +436,7 @@ class JmOption:
|
||||
# enable cache
|
||||
CacheRegistry.enable_client_cache_on_condition(self, client, cache)
|
||||
|
||||
# noinspection PyTypeChecker
|
||||
return client
|
||||
|
||||
def update_cookies(self, cookies: dict):
|
||||
@ -446,7 +450,8 @@ class JmOption:
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def decide_client_domain(self, client_key: str) -> List[str]:
|
||||
is_client_type = lambda ctype: self.client_key_is_given_type(client_key, ctype)
|
||||
def is_client_type(ctype) -> bool:
|
||||
return self.client_key_is_given_type(client_key, ctype)
|
||||
|
||||
if is_client_type(JmApiClient):
|
||||
# 移动端
|
||||
@ -510,19 +515,19 @@ class JmOption:
|
||||
plugin_registry = JmModuleConfig.REGISTRY_PLUGIN
|
||||
for pinfo in plugin_list:
|
||||
key, kwargs = pinfo['plugin'], pinfo.get('kwargs', None) # kwargs为None
|
||||
plugin_class: Optional[Type[JmOptionPlugin]] = plugin_registry.get(key, None)
|
||||
pclass: Optional[Type[JmOptionPlugin]] = plugin_registry.get(key, None)
|
||||
|
||||
ExceptionTool.require_true(plugin_class is not None, f'[{group}] 未注册的plugin: {key}')
|
||||
ExceptionTool.require_true(pclass is not None, f'[{group}] 未注册的plugin: {key}')
|
||||
|
||||
try:
|
||||
self.invoke_plugin(plugin_class, kwargs, extra, pinfo)
|
||||
self.invoke_plugin(pclass, kwargs, extra, pinfo)
|
||||
except BaseException as e:
|
||||
if safe is True:
|
||||
traceback_print_exec()
|
||||
else:
|
||||
raise e
|
||||
|
||||
def invoke_plugin(self, plugin_class, kwargs: Optional[Dict], extra: dict, pinfo: dict):
|
||||
def invoke_plugin(self, pclass, kwargs: Optional[Dict], extra: dict, pinfo: dict):
|
||||
# 检查插件的参数类型
|
||||
kwargs = self.fix_kwargs(kwargs)
|
||||
# 把插件的配置数据kwargs和附加数据extra合并,extra会覆盖kwargs
|
||||
@ -532,35 +537,36 @@ class JmOption:
|
||||
# 保证 jm_plugin.py 被加载
|
||||
from .jm_plugin import JmOptionPlugin, PluginValidationException
|
||||
|
||||
plugin = plugin_class
|
||||
plugin_class: Type[JmOptionPlugin]
|
||||
|
||||
pclass: Type[JmOptionPlugin]
|
||||
plugin: Optional[JmOptionPlugin] = None
|
||||
|
||||
try:
|
||||
# 构建插件对象
|
||||
plugin: JmOptionPlugin = plugin_class.build(self)
|
||||
plugin: JmOptionPlugin = pclass.build(self)
|
||||
|
||||
# 设置日志开关
|
||||
if pinfo.get('log', True) is not True:
|
||||
plugin.log_enable = False
|
||||
|
||||
jm_log('plugin.invoke', f'调用插件: [{plugin_class.plugin_key}]')
|
||||
jm_log('plugin.invoke', f'调用插件: [{pclass.plugin_key}]')
|
||||
|
||||
# 调用插件功能
|
||||
plugin.invoke(**kwargs)
|
||||
|
||||
except PluginValidationException as e:
|
||||
# 插件抛出的参数校验异常
|
||||
self.handle_plugin_valid_exception(e, pinfo, kwargs, plugin)
|
||||
self.handle_plugin_valid_exception(e, pinfo, kwargs, plugin, pclass)
|
||||
|
||||
except JmcomicException as e:
|
||||
# 模块内部异常,通过不是插件抛出的,而是插件调用了例如Client,Client请求失败抛出的
|
||||
self.handle_plugin_jmcomic_exception(e, pinfo, kwargs, plugin)
|
||||
self.handle_plugin_jmcomic_exception(e, pinfo, kwargs, plugin, pclass)
|
||||
|
||||
except BaseException as e:
|
||||
# 为插件兜底,捕获其他所有异常
|
||||
self.handle_plugin_unexpected_error(e, pinfo, kwargs, plugin)
|
||||
self.handle_plugin_unexpected_error(e, pinfo, kwargs, plugin, pclass)
|
||||
|
||||
# noinspection PyMethodMayBeStatic,PyUnusedLocal
|
||||
def handle_plugin_valid_exception(self, e, pinfo: dict, kwargs: dict, plugin):
|
||||
def handle_plugin_valid_exception(self, e, pinfo: dict, kwargs: dict, _plugin, _pclass):
|
||||
from .jm_plugin import PluginValidationException
|
||||
e: PluginValidationException
|
||||
|
||||
@ -584,15 +590,15 @@ class JmOption:
|
||||
# 其他的mode可以通过继承+方法重写来扩展
|
||||
|
||||
# noinspection PyMethodMayBeStatic,PyUnusedLocal
|
||||
def handle_plugin_unexpected_error(self, e, pinfo: dict, kwargs: dict, plugin):
|
||||
def handle_plugin_unexpected_error(self, e, pinfo: dict, kwargs: dict, _plugin, pclass):
|
||||
msg = str(e)
|
||||
jm_log('plugin.error', f'插件 [{plugin.plugin_key}],运行遇到未捕获异常,异常信息: [{msg}]')
|
||||
jm_log('plugin.error', f'插件 [{pclass.plugin_key}],运行遇到未捕获异常,异常信息: [{msg}]')
|
||||
raise e
|
||||
|
||||
# noinspection PyMethodMayBeStatic,PyUnusedLocal
|
||||
def handle_plugin_jmcomic_exception(self, e, pinfo: dict, kwargs: dict, plugin):
|
||||
def handle_plugin_jmcomic_exception(self, e, pinfo: dict, kwargs: dict, _plugin, pclass):
|
||||
msg = str(e)
|
||||
jm_log('plugin.exception', f'插件 [{plugin.plugin_key}] 调用失败,异常信息: [{msg}]')
|
||||
jm_log('plugin.exception', f'插件 [{pclass.plugin_key}] 调用失败,异常信息: [{msg}]')
|
||||
raise e
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
|
@ -409,13 +409,12 @@ class ClientProxyPlugin(JmOptionPlugin):
|
||||
def invoke(self,
|
||||
proxy_client_key,
|
||||
whitelist=None,
|
||||
**kwargs,
|
||||
**clazz_init_kwargs,
|
||||
) -> None:
|
||||
if whitelist is not None:
|
||||
whitelist = set(whitelist)
|
||||
|
||||
proxy_clazz = JmModuleConfig.client_impl_class(proxy_client_key)
|
||||
clazz_init_kwargs = kwargs
|
||||
new_jm_client: Callable = self.option.new_jm_client
|
||||
|
||||
def hook_new_jm_client(*args, **kwargs):
|
||||
@ -700,14 +699,14 @@ class ConvertJpgToPdfPlugin(JmOptionPlugin):
|
||||
filename_rule='Pid',
|
||||
quality=100,
|
||||
delete_original_file=False,
|
||||
overwrite_cmd=None,
|
||||
overwrite_jpg=None,
|
||||
override_cmd=None,
|
||||
override_jpg=None,
|
||||
**kwargs,
|
||||
):
|
||||
self.delete_original_file = delete_original_file
|
||||
|
||||
# 检查图片后缀配置
|
||||
suffix = overwrite_jpg or '.jpg'
|
||||
suffix = override_jpg or '.jpg'
|
||||
self.check_image_suffix_is_valid(suffix)
|
||||
|
||||
# 处理文件夹配置
|
||||
@ -726,7 +725,7 @@ class ConvertJpgToPdfPlugin(JmOptionPlugin):
|
||||
# 生成命令
|
||||
def generate_cmd():
|
||||
return (
|
||||
overwrite_cmd or
|
||||
override_cmd or
|
||||
'magick convert -quality {quality} "{photo_dir}*{suffix}" "{pdf_filepath}"'
|
||||
).format(
|
||||
quality=quality,
|
||||
@ -742,7 +741,7 @@ class ConvertJpgToPdfPlugin(JmOptionPlugin):
|
||||
ExceptionTool.require_true(
|
||||
code == 0,
|
||||
'jpg图片合并为pdf失败!'
|
||||
'请确认你是否安装了magick,安装网站: [http://www.imagemagick.org/]',
|
||||
'请确认你是否安装了magick,安装网站: [https://www.imagemagick.org/]',
|
||||
)
|
||||
|
||||
self.log(f'Convert Successfully: JM{photo.id} → {pdf_filepath}')
|
||||
|
@ -1,11 +1,14 @@
|
||||
from PIL import Image
|
||||
|
||||
from .jm_entity import *
|
||||
from .jm_exception import *
|
||||
|
||||
|
||||
class JmcomicText:
|
||||
pattern_jm_domain = compile(r'https://([\w.-]+)')
|
||||
pattern_jm_pa_id = compile(r'(photos?|album)/(\d+)')
|
||||
pattern_jm_pa_id = [
|
||||
(compile(r'(photos?|album)/(\d+)'), 2),
|
||||
(compile(r'id=(\d+)'), 1),
|
||||
]
|
||||
pattern_html_jm_pub_domain = compile(r'[\w-]+\.\w+/?\w+')
|
||||
|
||||
pattern_html_photo_photo_id = compile(r'<meta property="og:url" content=".*?/photo/(\d+)/?.*?">')
|
||||
@ -87,9 +90,13 @@ class JmcomicText:
|
||||
return text[2:]
|
||||
else:
|
||||
# https://xxx/photo/412038
|
||||
match = cls.pattern_jm_pa_id.search(text)
|
||||
ExceptionTool.require_true(match is not None, f"无法解析jm车号, 文本为: {text}")
|
||||
return match[2]
|
||||
# https://xxx/album/?id=412038
|
||||
for p, i in cls.pattern_jm_pa_id:
|
||||
match = p.search(text)
|
||||
if match is not None:
|
||||
return match[i]
|
||||
|
||||
ExceptionTool.raises(f"无法解析jm车号, 文本为: {text}")
|
||||
|
||||
@classmethod
|
||||
def analyse_jm_pub_html(cls, html: str, domain_keyword=('jm', 'comic')) -> List[str]:
|
||||
@ -162,7 +169,7 @@ class JmcomicText:
|
||||
if field_value is None:
|
||||
if default is None:
|
||||
ExceptionTool.raises_regex(
|
||||
f"文本没有匹配上字段:字段名为'{field_name}',pattern: [{pattern}]" \
|
||||
f"文本没有匹配上字段:字段名为'{field_name}',pattern: [{pattern}]"
|
||||
+ (f"\n响应文本=[{html}]" if len(html) < 200 else
|
||||
f'响应文本过长(len={len(html)}),不打印'
|
||||
),
|
||||
@ -712,7 +719,7 @@ class JmImageTool:
|
||||
if need_convert is False:
|
||||
cls.save_directly(resp, filepath)
|
||||
else:
|
||||
cls.save_image(cls.open_Image(resp.content), filepath)
|
||||
cls.save_image(cls.open_image(resp.content), filepath)
|
||||
|
||||
@classmethod
|
||||
def save_image(cls, image: Image, filepath: str):
|
||||
@ -773,7 +780,7 @@ class JmImageTool:
|
||||
cls.save_image(img_decode, decoded_save_path)
|
||||
|
||||
@classmethod
|
||||
def open_Image(cls, fp: Union[str, bytes]):
|
||||
def open_image(cls, fp: Union[str, bytes]):
|
||||
from io import BytesIO
|
||||
fp = fp if isinstance(fp, str) else BytesIO(fp)
|
||||
return Image.open(fp)
|
||||
@ -821,86 +828,6 @@ class JmImageTool:
|
||||
return cls.get_num(detail.scramble_id, detail.aid, detail.img_file_name)
|
||||
|
||||
|
||||
class ExceptionTool:
|
||||
"""
|
||||
抛异常的工具
|
||||
1: 能简化 if-raise 语句的编写
|
||||
2: 有更好的上下文信息传递方式
|
||||
"""
|
||||
|
||||
EXTRA_KEY_RESP = 'resp'
|
||||
EXTRA_KEY_HTML = 'html'
|
||||
EXTRA_KEY_RE_PATTERN = 'pattern'
|
||||
|
||||
@classmethod
|
||||
def raises(cls, msg: str, extra: dict = None):
|
||||
if extra is None:
|
||||
extra = {}
|
||||
|
||||
JmModuleConfig.executor_raise_exception(msg, extra)
|
||||
|
||||
@classmethod
|
||||
def raises_regex(cls,
|
||||
msg: str,
|
||||
html: str,
|
||||
pattern: Pattern,
|
||||
):
|
||||
cls.raises(
|
||||
msg, {
|
||||
cls.EXTRA_KEY_HTML: html,
|
||||
cls.EXTRA_KEY_RE_PATTERN: pattern,
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def raises_resp(cls,
|
||||
msg: str,
|
||||
resp,
|
||||
):
|
||||
cls.raises(
|
||||
msg, {
|
||||
cls.EXTRA_KEY_RESP: resp
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def raise_missing(cls,
|
||||
resp,
|
||||
orig_req_url=None,
|
||||
):
|
||||
"""
|
||||
抛出本子/章节的异常
|
||||
:param resp: 响应对象
|
||||
:param orig_req_url: 原始请求url,可不传
|
||||
"""
|
||||
if orig_req_url is None:
|
||||
orig_req_url = resp.url
|
||||
|
||||
req_type = "本子" if "album" in orig_req_url else "章节"
|
||||
cls.raises_resp((
|
||||
f'请求的{req_type}不存在!({orig_req_url})\n'
|
||||
'原因可能为:\n'
|
||||
f'1. id有误,检查你的{req_type}id\n'
|
||||
'2. 该漫画只对登录用户可见,请配置你的cookies,或者使用移动端Client(api)\n'
|
||||
), resp)
|
||||
|
||||
@classmethod
|
||||
def require_true(cls, case: bool, msg: str):
|
||||
if case:
|
||||
return
|
||||
|
||||
cls.raises(msg)
|
||||
|
||||
@classmethod
|
||||
def replace_old_exception_executor(cls, raises: Callable[[Callable, str, dict], None]):
|
||||
old = JmModuleConfig.executor_raise_exception
|
||||
|
||||
def new(msg, extra):
|
||||
raises(old, msg, extra)
|
||||
|
||||
JmModuleConfig.executor_raise_exception = new
|
||||
|
||||
|
||||
class JmCryptoTool:
|
||||
"""
|
||||
禁漫加解密相关逻辑
|
||||
|
@ -38,31 +38,12 @@ class Test_Client(JmTestConfigurable):
|
||||
self.client.download_by_image_detail(image, self.option.decide_image_filepath(image))
|
||||
|
||||
def test_album_missing(self):
|
||||
class A(BaseException):
|
||||
pass
|
||||
|
||||
JmModuleConfig.CLASS_EXCEPTION = A
|
||||
self.assertRaises(
|
||||
A,
|
||||
MissingAlbumPhotoException,
|
||||
self.client.get_album_detail,
|
||||
'0'
|
||||
)
|
||||
|
||||
def test_raise_exception(self):
|
||||
|
||||
class B(BaseException):
|
||||
pass
|
||||
|
||||
def raises(old, _msg, _extra):
|
||||
self.assertEqual(old, default_raise_exception_executor)
|
||||
raise B()
|
||||
|
||||
JmModuleConfig.executor_raise_exception = default_raise_exception_executor
|
||||
ExceptionTool.replace_old_exception_executor(raises)
|
||||
self.assertRaises(B, JmcomicText.parse_to_jm_id, 'asdhasjhkd')
|
||||
# 还原
|
||||
JmModuleConfig.executor_raise_exception = default_raise_exception_executor
|
||||
|
||||
def test_detail_property_list(self):
|
||||
album = self.client.get_album_detail(410090)
|
||||
|
||||
|
@ -68,7 +68,7 @@ class Test_Custom(JmTestConfigurable):
|
||||
|
||||
# '不重写 client_key'
|
||||
self.assertRaises(
|
||||
JmModuleConfig.CLASS_EXCEPTION,
|
||||
JmcomicException,
|
||||
JmModuleConfig.register_client,
|
||||
MyClient,
|
||||
)
|
||||
@ -81,7 +81,7 @@ class Test_Custom(JmTestConfigurable):
|
||||
JmModuleConfig.register_client(MyClient)
|
||||
# '自定义client,不配置域名'
|
||||
self.assertRaises(
|
||||
JmModuleConfig.CLASS_EXCEPTION,
|
||||
JmcomicException,
|
||||
self.option.new_jm_client,
|
||||
domain_list=[],
|
||||
impl=MyClient.client_key,
|
||||
|
Loading…
Reference in New Issue
Block a user