mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.1.17: 增加模块级别配置类,优化异常类型、代码示例 (#103)
This commit is contained in:
parent
6948743e24
commit
2c50ae4c51
@ -2,6 +2,6 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.1.16'
|
||||
__version__ = '2.1.17'
|
||||
|
||||
from .api import *
|
||||
|
@ -143,7 +143,7 @@ class AbstractJmClient(
|
||||
def fallback(self, request, url, domain_index, retry_count, **kwargs):
|
||||
msg = f"请求重试全部失败: [{url}], {self.domain_list}"
|
||||
jm_debug('req.fallback', msg)
|
||||
raise AssertionError(msg)
|
||||
raise JmModuleConfig.exception(msg)
|
||||
|
||||
|
||||
# 基于网页实现的JmClient
|
||||
@ -216,7 +216,7 @@ class JmHtmlClient(AbstractJmClient):
|
||||
)
|
||||
|
||||
if resp.status_code != 301:
|
||||
raise AssertionError(f'登录失败,状态码为{resp.status_code}')
|
||||
raise JmModuleConfig.exception(f'登录失败,状态码为{resp.status_code}')
|
||||
|
||||
if refresh_client_cookies is True:
|
||||
self['cookies'] = resp.cookies
|
||||
@ -251,7 +251,7 @@ class JmHtmlClient(AbstractJmClient):
|
||||
+ (f"响应文本=[{resp.text}]" if len(resp.text) < 200 else
|
||||
f'响应文本过长(len={len(resp.text)}),不打印'
|
||||
)
|
||||
raise AssertionError(msg)
|
||||
raise JmModuleConfig.exception(msg)
|
||||
|
||||
def get_jm_image(self, img_url) -> JmImageResp:
|
||||
|
||||
|
@ -23,13 +23,13 @@ class JmResp(CommonResp):
|
||||
class JmImageResp(JmResp):
|
||||
|
||||
def json(self, **kwargs) -> Dict:
|
||||
raise AssertionError
|
||||
raise NotImplementedError
|
||||
|
||||
def require_success(self):
|
||||
if self.is_success:
|
||||
return
|
||||
|
||||
raise AssertionError(self.get_error_msg())
|
||||
raise JmModuleConfig.exception(self.get_error_msg())
|
||||
|
||||
def get_error_msg(self):
|
||||
msg = f'禁漫图片获取失败: [{self.url}]'
|
||||
@ -68,7 +68,7 @@ class JmApiResp(JmResp):
|
||||
@classmethod
|
||||
def wrap(cls, resp, key_ts):
|
||||
if isinstance(resp, JmApiResp):
|
||||
raise AssertionError('重复包装')
|
||||
raise JmModuleConfig.exception('重复包装')
|
||||
|
||||
return cls(resp, key_ts)
|
||||
|
||||
|
@ -18,7 +18,11 @@ def default_postman_constructor(session, **kwargs):
|
||||
|
||||
|
||||
def default_raise_regex_error(msg, *_args, **_kwargs):
|
||||
raise AssertionError(msg)
|
||||
raise JmModuleConfig.exception(msg)
|
||||
|
||||
|
||||
class JmcomicException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class JmModuleConfig:
|
||||
@ -60,7 +64,9 @@ class JmModuleConfig:
|
||||
CLASS_OPTION = None
|
||||
CLASS_ALBUM = None
|
||||
CLASS_PHOTO = None
|
||||
CLASS_IMAGE = None
|
||||
CLASS_CLIENT_IMPL = {}
|
||||
CLASS_EXCEPTION = None
|
||||
|
||||
# 执行debug的函数
|
||||
debug_executor = default_jm_debug
|
||||
@ -104,6 +110,14 @@ class JmModuleConfig:
|
||||
from .jm_entity import JmPhotoDetail
|
||||
return JmPhotoDetail
|
||||
|
||||
@classmethod
|
||||
def image_class(cls):
|
||||
if cls.CLASS_IMAGE is not None:
|
||||
return cls.CLASS_IMAGE
|
||||
|
||||
from .jm_entity import JmImageDetail
|
||||
return JmImageDetail
|
||||
|
||||
@classmethod
|
||||
def client_impl_class(cls, client_key: str):
|
||||
client_impl_dict = cls.CLASS_CLIENT_IMPL
|
||||
@ -114,6 +128,13 @@ class JmModuleConfig:
|
||||
|
||||
return impl_class
|
||||
|
||||
@classmethod
|
||||
def exception(cls, msg: str):
|
||||
if cls.CLASS_EXCEPTION is not None:
|
||||
return cls.CLASS_EXCEPTION(msg)
|
||||
|
||||
return JmcomicException(msg)
|
||||
|
||||
@classmethod
|
||||
@field_cache("DOMAIN")
|
||||
def domain(cls, postman=None):
|
||||
@ -185,7 +206,7 @@ class JmModuleConfig:
|
||||
|
||||
resp = postman.get(cls.JM_PUB_URL)
|
||||
if resp.status_code != 200:
|
||||
raise AssertionError(resp.text)
|
||||
raise JmModuleConfig.exception(resp.text)
|
||||
|
||||
from .jm_toolkit import JmcomicText
|
||||
domain_list = JmcomicText.analyse_jm_pub_html(resp.text)
|
||||
|
@ -241,11 +241,11 @@ class JmPhotoDetail(DetailEntity):
|
||||
# 校验参数
|
||||
length = len(self.page_arr)
|
||||
if index >= length:
|
||||
raise AssertionError(f'创建JmImageDetail失败,{index} >= {length}')
|
||||
raise JmModuleConfig.exception(f'创建JmImageDetail失败,{index} >= {length}')
|
||||
|
||||
data_original = self.get_img_data_original(self.page_arr[index])
|
||||
|
||||
return JmImageDetail.of(
|
||||
return JmModuleConfig.image_class().of(
|
||||
self.photo_id,
|
||||
self.scramble_id,
|
||||
data_original,
|
||||
@ -262,7 +262,7 @@ class JmPhotoDetail(DetailEntity):
|
||||
"""
|
||||
data_original_domain = self.data_original_domain
|
||||
if data_original_domain is None:
|
||||
raise AssertionError(f'图片域名为空: {self.__dict__}')
|
||||
raise JmModuleConfig.exception(f'图片域名为空: {self.__dict__}')
|
||||
|
||||
return f'https://{data_original_domain}/media/photos/{self.photo_id}/{img_name}'
|
||||
|
||||
@ -330,13 +330,13 @@ class JmAlbumDetail(DetailEntity):
|
||||
length = len(self.episode_list)
|
||||
|
||||
if index >= length:
|
||||
raise AssertionError(f'创建JmPhotoDetail失败,{index} >= {length}')
|
||||
raise JmModuleConfig.exception(f'创建JmPhotoDetail失败,{index} >= {length}')
|
||||
|
||||
# episode_info: ('212214', '81', '94 突然打來', '2020-08-29')
|
||||
episode_info: tuple = self.episode_list[index]
|
||||
photo_id, photo_index, photo_title, photo_pub_date = episode_info
|
||||
|
||||
photo = JmPhotoDetail(
|
||||
photo = JmModuleConfig.photo_class()(
|
||||
photo_id=photo_id,
|
||||
scramble_id=self.scramble_id,
|
||||
title=photo_title,
|
||||
|
@ -10,7 +10,8 @@ class DirRule:
|
||||
# 根目录 / Photo-序号&标题 /
|
||||
'Bd_Pindextitle',
|
||||
# 根目录 / Photo-自定义类属性 /
|
||||
'Bd_Aauthor_Atitle_Pcustomfield', # 使用自定义类属性前,需替换 JmcomicText的 PhotoClass / AlbumClass
|
||||
'Bd_Aauthor_Atitle_Pcustomfield',
|
||||
# 需要替换JmModuleConfig.CLASS_ALBUM / CLASS_PHOTO才能让自定义属性生效
|
||||
]
|
||||
|
||||
RuleFunc = Callable[[Union[JmAlbumDetail, JmPhotoDetail, None]], str]
|
||||
@ -45,8 +46,10 @@ class DirRule:
|
||||
path_ls.append(str(ret))
|
||||
except BaseException as e:
|
||||
# noinspection PyUnboundLocalVariable
|
||||
raise AssertionError(f'路径规则"{self.rule_dsl}"的第{i + 1}个解析出错: {e},'
|
||||
f'param is {param}')
|
||||
raise JmModuleConfig.exception(
|
||||
f'路径规则"{self.rule_dsl}"的第{i + 1}个解析出错: {e},'
|
||||
f'param is {param}'
|
||||
)
|
||||
|
||||
return fix_filepath('/'.join(path_ls), is_dir=True)
|
||||
|
||||
@ -214,7 +217,7 @@ class JmOption:
|
||||
filepath = self.filepath
|
||||
|
||||
if filepath is None:
|
||||
raise AssertionError("未指定JmOption的保存路径")
|
||||
raise JmModuleConfig.exception("未指定JmOption的保存路径")
|
||||
|
||||
PackerUtil.pack(self.deconstruct(), filepath)
|
||||
|
||||
|
@ -52,7 +52,7 @@ class JmcomicText:
|
||||
return str(text)
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise AssertionError(f"无法解析jm车号, 参数类型为: {type(text)}")
|
||||
raise JmModuleConfig.exception(f"无法解析jm车号, 参数类型为: {type(text)}")
|
||||
|
||||
# 43210
|
||||
if text.isdigit():
|
||||
@ -60,7 +60,7 @@ class JmcomicText:
|
||||
|
||||
# Jm43210
|
||||
if len(text) <= 2:
|
||||
raise AssertionError(f"无法解析jm车号, 文本为: {text}")
|
||||
raise JmModuleConfig.exception(f"无法解析jm车号, 文本为: {text}")
|
||||
|
||||
# text: JM12341
|
||||
c0 = text[0]
|
||||
@ -72,7 +72,7 @@ class JmcomicText:
|
||||
# https://xxx/photo/412038
|
||||
match = cls.pattern_jm_pa_id.search(text)
|
||||
if match is None:
|
||||
raise AssertionError(f"无法解析jm车号, 文本为: {text}")
|
||||
raise JmModuleConfig.exception(f"无法解析jm车号, 文本为: {text}")
|
||||
return match[2]
|
||||
|
||||
@classmethod
|
||||
|
@ -52,8 +52,9 @@ class Test_Client(JmTestConfigurable):
|
||||
self.client.download_by_image_detail(image, workspace('3000.png'))
|
||||
|
||||
def test_album_missing(self):
|
||||
JmModuleConfig.CLASS_EXCEPTION = JmcomicException
|
||||
self.assertRaises(
|
||||
AssertionError,
|
||||
JmcomicException,
|
||||
self.client.get_album_detail,
|
||||
'332583'
|
||||
)
|
||||
|
@ -19,8 +19,8 @@
|
||||
** 本文件下面的示例只演示步骤1 **
|
||||
|
||||
本文件包含如下示例:
|
||||
- 只下载本子的特定章节以后的章节
|
||||
- 只下载章节的前三张图
|
||||
- 只下载本子的特定章节以后的章节
|
||||
|
||||
|
||||
"""
|
||||
@ -28,6 +28,18 @@
|
||||
from jmcomic import *
|
||||
|
||||
|
||||
# 示例:只下载章节的前三张图
|
||||
class First3ImageDownloader(JmDownloader):
|
||||
|
||||
def filter_iter_objs(self, iter_objs: DownloadIterObjs):
|
||||
if isinstance(iter_objs, JmPhotoDetail):
|
||||
photo: JmPhotoDetail = iter_objs
|
||||
# 支持[start,end,step]
|
||||
return photo[:3]
|
||||
|
||||
return iter_objs
|
||||
|
||||
|
||||
# 示例:只下载本子的特定章节以后的章节
|
||||
# 参考:https://github.com/hect0x7/JMComic-Crawler-Python/issues/95
|
||||
class FindUpdateDownloader(JmDownloader):
|
||||
@ -58,15 +70,3 @@ class FindUpdateDownloader(JmDownloader):
|
||||
is_new_photo = True
|
||||
|
||||
return photo_ls
|
||||
|
||||
|
||||
# 示例:只下载章节的前三张图
|
||||
class First3ImageDownloader(JmDownloader):
|
||||
|
||||
def filter_iter_objs(self, iter_objs: DownloadIterObjs):
|
||||
if isinstance(iter_objs, JmPhotoDetail):
|
||||
photo: JmPhotoDetail = iter_objs
|
||||
# 支持[start,end,step]
|
||||
return photo[:3]
|
||||
|
||||
return iter_objs
|
||||
|
@ -10,9 +10,11 @@
|
||||
|
||||
from jmcomic import *
|
||||
|
||||
# 核心下载配置
|
||||
option = create_option(
|
||||
f'你的配置文件路径,例如: D:/a/b/c/jmcomic/config.yml'
|
||||
)
|
||||
# 提供请求功能的客户端对象
|
||||
client = option.build_jm_client()
|
||||
|
||||
|
||||
@ -24,8 +26,7 @@ def download_jm_album():
|
||||
|
||||
''')
|
||||
|
||||
download_album(ls, option) # 效果同下面的代码
|
||||
# download_album_batch(ls, op)
|
||||
download_album(ls, option)
|
||||
|
||||
|
||||
@timeit('获取实体类: ')
|
||||
|
Loading…
Reference in New Issue
Block a user