mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v1.3.0: 优化实体类、缓存,简化from_album的使用、decide_image_save_dir的api。 (#10)
* 增加get_photo_detail、get_album_detail的缓存。 * 优化实体类,简化from_album的使用、decide_image_save_dir的api。
This commit is contained in:
parent
d8349d863d
commit
6e465ec94f
@ -2,6 +2,6 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- service <--- option
|
||||
|
||||
__version__ = '1.2.0'
|
||||
__version__ = '1.3.0'
|
||||
|
||||
from .api import *
|
||||
|
@ -23,7 +23,7 @@ def download_album(jm_album_id, option=None):
|
||||
photo_detail: JmPhotoDetail,
|
||||
debug_topic='download_album_photo',
|
||||
):
|
||||
jm_client.update(photo_detail)
|
||||
jm_client.ensure_photo_can_use(photo_detail)
|
||||
|
||||
jm_debug(debug_topic,
|
||||
f"下载第[{index + 1}]集: "
|
||||
@ -90,12 +90,7 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
|
||||
# 下载准备
|
||||
use_cache = option.download_use_disk_cache
|
||||
decode_image = option.download_image_then_decode
|
||||
|
||||
if photo_detail.from_album is None:
|
||||
jm_client.fill_from_album(photo_detail)
|
||||
|
||||
if photo_detail.page_arr is None:
|
||||
jm_client.update(photo_detail)
|
||||
jm_client.ensure_photo_can_use(photo_detail)
|
||||
|
||||
# 下载每个图片的函数
|
||||
def download_image(index, img_detail, debug_topic='download_images_of_photo'):
|
||||
@ -114,6 +109,7 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
|
||||
img_save_path,
|
||||
decode_image=decode_image,
|
||||
)
|
||||
|
||||
jm_debug(debug_topic, f'photo-{img_detail.aid}: '
|
||||
f'图片{img_detail.filename}下载完成:'
|
||||
f'[{img_detail.img_url}] → [{img_save_path}]')
|
||||
|
@ -78,28 +78,32 @@ class JmcomicClient(PostmanProxy):
|
||||
# 用 JmcomicText 解析 html,返回实体类
|
||||
return JmcomicText.analyse_jm_album_html(resp.text)
|
||||
|
||||
def get_photo_detail(self, jm_photo_id: str) -> JmPhotoDetail:
|
||||
def get_photo_detail(self, photo_id: str, album=True) -> JmPhotoDetail:
|
||||
# 参数校验
|
||||
photo_id = JmcomicText.parse_to_photo_id(jm_photo_id)
|
||||
photo_id = JmcomicText.parse_to_photo_id(photo_id)
|
||||
|
||||
# 请求
|
||||
resp = self.jm_get(f"/photo/{photo_id}")
|
||||
|
||||
# 用 JmcomicText 解析 html,返回实体类
|
||||
return JmcomicText.analyse_jm_photo_html(resp.text)
|
||||
photo_detail = JmcomicText.analyse_jm_photo_html(resp.text)
|
||||
|
||||
def fill_from_album(self, photo_detail: JmPhotoDetail) -> JmAlbumDetail:
|
||||
"""
|
||||
获取 photo_detail 所在的 album。
|
||||
并把 album_detail 赋值给 photo_detail.from_album
|
||||
"""
|
||||
album_detail = self.get_album_detail(photo_detail.album_id)
|
||||
photo_detail.from_album = album_detail
|
||||
return album_detail
|
||||
# 一并获取该章节的所处本子
|
||||
if album is True:
|
||||
photo_detail.from_album = self.get_album_detail(photo_detail.album_id)
|
||||
|
||||
def update(self, photo_detail: JmPhotoDetail):
|
||||
new = self.get_photo_detail(photo_detail.photo_id)
|
||||
photo_detail.__dict__.update(new.__dict__)
|
||||
return photo_detail
|
||||
|
||||
def ensure_photo_can_use(self, photo_detail: JmPhotoDetail):
|
||||
# 检查 from_album
|
||||
if photo_detail.from_album is None:
|
||||
photo_detail.from_album = self.get_album_detail(photo_detail.album_id)
|
||||
|
||||
# 检查 page_arr 和 data_original_domain
|
||||
if photo_detail.page_arr is None or photo_detail.data_original_domain is None:
|
||||
new = self.get_photo_detail(photo_detail.photo_id, False)
|
||||
photo_detail.page_arr = new.page_arr
|
||||
photo_detail.data_original_domain = new.data_original_domain
|
||||
|
||||
# -- search --
|
||||
|
||||
|
@ -21,20 +21,16 @@ class WorkEntity(JmBaseEntity, SaveableEntity, IterableEntity):
|
||||
return self.detail_save_base_dir
|
||||
|
||||
def save_file_name(self) -> str:
|
||||
return f"【{self.get_id_prefix_of_filename()}{self.get_id()}】{self.get_title()}{self.detail_save_file_suffix}"
|
||||
def jm_type():
|
||||
# "JmAlbumDetail" -> "album"
|
||||
cls_name = self.__class__.__name__
|
||||
return cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")].lower()
|
||||
|
||||
def get_id_prefix_of_filename(self):
|
||||
# "JmAlbumDetail" -> "album"
|
||||
cls_name = self.__class__.__name__
|
||||
id_prefix = cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")]
|
||||
return f'{id_prefix}-'
|
||||
return '[{}]{}{}'.format(jm_type(), self.get_id(), self.detail_save_file_suffix)
|
||||
|
||||
def get_id(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_title(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def __len__(self):
|
||||
raise NotImplementedError
|
||||
|
||||
@ -152,15 +148,14 @@ class JmPhotoDetail(WorkEntity):
|
||||
|
||||
@property
|
||||
def author(self) -> str:
|
||||
# self._author 不为空字符串
|
||||
if self._author is not None and self._author != '':
|
||||
return self._author.strip()
|
||||
|
||||
# self._author 为空,先向上找
|
||||
# 优先使用 from_album
|
||||
if self.from_album is not None:
|
||||
return self.from_album.author
|
||||
|
||||
# 无向上元素,使用默认
|
||||
if self._author is not None and self._author != '':
|
||||
return self._author.strip()
|
||||
|
||||
# 使用默认
|
||||
return JmModuleConfig.default_author
|
||||
|
||||
def create_image_detail(self, index) -> JmImageDetail:
|
||||
@ -196,9 +191,6 @@ class JmPhotoDetail(WorkEntity):
|
||||
def get_id(self):
|
||||
return self.photo_id
|
||||
|
||||
def get_title(self):
|
||||
return self.title
|
||||
|
||||
def __len__(self):
|
||||
return len(self.page_arr)
|
||||
|
||||
@ -213,13 +205,15 @@ class JmAlbumDetail(WorkEntity):
|
||||
page_count,
|
||||
author_list,
|
||||
pub_date,
|
||||
update_date,
|
||||
):
|
||||
self.album_id: str = album_id
|
||||
self.scramble_id: str = scramble_id
|
||||
self.title: str = title
|
||||
self.page_count = int(page_count)
|
||||
self._author_list: List[str] = author_list
|
||||
self.pub_date = pub_date
|
||||
self.pub_date: str = pub_date # 发布日期
|
||||
self.update_date: str = update_date # 更新日期
|
||||
|
||||
# 有的 album 没有章节,则自成一章。
|
||||
if len(episode_list) == 0:
|
||||
@ -263,9 +257,6 @@ class JmAlbumDetail(WorkEntity):
|
||||
def get_id(self):
|
||||
return self.album_id
|
||||
|
||||
def get_title(self):
|
||||
return self.title
|
||||
|
||||
def __len__(self):
|
||||
return len(self.episode_list)
|
||||
|
||||
|
@ -5,13 +5,11 @@ class JmOptionAdvice:
|
||||
|
||||
def decide_image_save_dir(self,
|
||||
option: 'JmOption',
|
||||
album_detail: Optional[JmAlbumDetail],
|
||||
photo_detail: JmPhotoDetail,
|
||||
) -> StrNone:
|
||||
"""
|
||||
决定一个本子图片的下载文件夹
|
||||
@param option: JmOption对象
|
||||
@param album_detail: 本子集实体类
|
||||
@param photo_detail: 本子章节实体类
|
||||
@return: 下载文件夹,为空表示不处理
|
||||
"""
|
||||
@ -245,19 +243,16 @@ class JmOption(SaveableEntity):
|
||||
下面是决定图片保存路径的方法
|
||||
"""
|
||||
|
||||
def decide_image_save_dir(self, album_detail, photo_detail) -> str:
|
||||
if album_detail is None and photo_detail is None:
|
||||
raise AssertionError('album和photo不能同时为None')
|
||||
|
||||
def decide_image_save_dir(self, photo_detail) -> str:
|
||||
# 先检查advice的回调,如果回调支持,则优先使用回调
|
||||
for advice in JmAdviceRegistry.get_advice(self):
|
||||
save_dir = advice.decide_image_save_dir(self, album_detail, photo_detail)
|
||||
save_dir = advice.decide_image_save_dir(self, photo_detail)
|
||||
if save_dir is not None:
|
||||
return save_dir
|
||||
|
||||
# 使用 self.dir_tree 决定 save_dir
|
||||
save_dir = self.dir_tree.deside_image_save_dir(
|
||||
album_detail or photo_detail.from_album,
|
||||
photo_detail.from_album,
|
||||
photo_detail
|
||||
)
|
||||
|
||||
@ -287,7 +282,7 @@ class JmOption(SaveableEntity):
|
||||
return filepath
|
||||
|
||||
# 通过拼接生成绝对路径
|
||||
save_dir = self.decide_image_save_dir(photo_detail.from_album, photo_detail)
|
||||
save_dir = self.decide_image_save_dir(photo_detail)
|
||||
image: JmImageDetail = photo_detail[index]
|
||||
suffix = self.decide_image_suffix(image)
|
||||
return save_dir + image.img_file_name + suffix
|
||||
@ -451,15 +446,13 @@ class JmOption(SaveableEntity):
|
||||
|
||||
if use_all_default_save_path is True:
|
||||
# 不通过请求获取 photo 的信息,相当于使用【空本子】和【空集】
|
||||
photo_detail, album_detail = None, None
|
||||
photo_detail = None
|
||||
else:
|
||||
# 通过请求获得 photo 的本子信息
|
||||
client = self.build_jm_client()
|
||||
photo_detail = client.get_photo_detail(photo_id)
|
||||
album_detail = client.fill_from_album(photo_detail)
|
||||
photo_detail = self.build_jm_client().get_photo_detail(photo_id)
|
||||
|
||||
def save_path_provider(url, _suffix: str, _index, _is_decode):
|
||||
return '{0}{1}{2}'.format(self.decide_image_save_dir(album_detail, photo_detail),
|
||||
return '{0}{1}{2}'.format(self.decide_image_save_dir(photo_detail),
|
||||
of_file_name(url, trim_suffix=True),
|
||||
self.download_convert_image_suffix or _suffix)
|
||||
|
||||
|
@ -24,7 +24,8 @@ class JmcomicText:
|
||||
'<li.*>\n第(\d+)話\n(.*)\n'
|
||||
'<[\s\S]*?>(\d+-\d+-\d+).*?')
|
||||
pattern_html_album_page_count = compile('<span class="pagecount">.*?:(\d+)</span>')
|
||||
pattern_html_album_pub_date = compile('itemprop="datePublished" content=".*?">更新日期 : (.*?)</span>')
|
||||
pattern_html_album_pub_date = compile('>上架日期 : (.*?)</span>')
|
||||
pattern_html_album_update_date = compile('>更新日期 : (.*?)</span>')
|
||||
|
||||
# album 作者
|
||||
pattern_html_album_author_list = [
|
||||
|
@ -35,7 +35,12 @@ class JmTestConfigurable(unittest.TestCase):
|
||||
# 设置 JmOption,JmcomicClient
|
||||
option = cls.use_option('option_test.yml')
|
||||
cls.option = option
|
||||
cls.client = option.build_jm_client()
|
||||
|
||||
client = option.build_jm_client()
|
||||
# enable cache
|
||||
client.get_photo_detail = enable_cache()(client.get_photo_detail)
|
||||
client.get_album_detail = enable_cache()(client.get_album_detail)
|
||||
cls.client = client
|
||||
|
||||
# 跨平台设置
|
||||
cls.adapt_os()
|
||||
|
@ -57,12 +57,12 @@ class Test_Api(JmTestConfigurable):
|
||||
) -> StrNone:
|
||||
return workspace(f'{time_stamp()}_{photo_detail[index].img_file_name}.test.png')
|
||||
|
||||
option = JmOption.default()
|
||||
option = self.option
|
||||
option.register_advice(MyAdvice())
|
||||
jmcomic.download_album('366867', option)
|
||||
|
||||
def test_photo_sort(self):
|
||||
client = JmOption.default().build_jm_client()
|
||||
client = self.option.build_jm_client()
|
||||
|
||||
# 测试用例 - 单章本子
|
||||
single_photo_album_is = str_to_list('''
|
||||
|
Loading…
Reference in New Issue
Block a user