v1.3.0: 优化实体类、缓存,简化from_album的使用、decide_image_save_dir的api。 (#10)

* 增加get_photo_detail、get_album_detail的缓存。
* 优化实体类,简化from_album的使用、decide_image_save_dir的api。
This commit is contained in:
hect0x7 2023-04-06 22:04:26 +08:00 committed by GitHub
parent d8349d863d
commit 6e465ec94f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 52 additions and 62 deletions

View File

@ -2,6 +2,6 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- service <--- option
__version__ = '1.2.0'
__version__ = '1.3.0'
from .api import *

View File

@ -23,7 +23,7 @@ def download_album(jm_album_id, option=None):
photo_detail: JmPhotoDetail,
debug_topic='download_album_photo',
):
jm_client.update(photo_detail)
jm_client.ensure_photo_can_use(photo_detail)
jm_debug(debug_topic,
f"下载第[{index + 1}]集: "
@ -90,12 +90,7 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
# 下载准备
use_cache = option.download_use_disk_cache
decode_image = option.download_image_then_decode
if photo_detail.from_album is None:
jm_client.fill_from_album(photo_detail)
if photo_detail.page_arr is None:
jm_client.update(photo_detail)
jm_client.ensure_photo_can_use(photo_detail)
# 下载每个图片的函数
def download_image(index, img_detail, debug_topic='download_images_of_photo'):
@ -114,6 +109,7 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
img_save_path,
decode_image=decode_image,
)
jm_debug(debug_topic, f'photo-{img_detail.aid}: '
f'图片{img_detail.filename}下载完成:'
f'[{img_detail.img_url}] → [{img_save_path}]')

View File

@ -78,28 +78,32 @@ class JmcomicClient(PostmanProxy):
# 用 JmcomicText 解析 html返回实体类
return JmcomicText.analyse_jm_album_html(resp.text)
def get_photo_detail(self, jm_photo_id: str) -> JmPhotoDetail:
def get_photo_detail(self, photo_id: str, album=True) -> JmPhotoDetail:
# 参数校验
photo_id = JmcomicText.parse_to_photo_id(jm_photo_id)
photo_id = JmcomicText.parse_to_photo_id(photo_id)
# 请求
resp = self.jm_get(f"/photo/{photo_id}")
# 用 JmcomicText 解析 html返回实体类
return JmcomicText.analyse_jm_photo_html(resp.text)
photo_detail = JmcomicText.analyse_jm_photo_html(resp.text)
def fill_from_album(self, photo_detail: JmPhotoDetail) -> JmAlbumDetail:
"""
获取 photo_detail 所在的 album
并把 album_detail 赋值给 photo_detail.from_album
"""
album_detail = self.get_album_detail(photo_detail.album_id)
photo_detail.from_album = album_detail
return album_detail
# 一并获取该章节的所处本子
if album is True:
photo_detail.from_album = self.get_album_detail(photo_detail.album_id)
def update(self, photo_detail: JmPhotoDetail):
new = self.get_photo_detail(photo_detail.photo_id)
photo_detail.__dict__.update(new.__dict__)
return photo_detail
def ensure_photo_can_use(self, photo_detail: JmPhotoDetail):
# 检查 from_album
if photo_detail.from_album is None:
photo_detail.from_album = self.get_album_detail(photo_detail.album_id)
# 检查 page_arr 和 data_original_domain
if photo_detail.page_arr is None or photo_detail.data_original_domain is None:
new = self.get_photo_detail(photo_detail.photo_id, False)
photo_detail.page_arr = new.page_arr
photo_detail.data_original_domain = new.data_original_domain
# -- search --

View File

@ -21,20 +21,16 @@ class WorkEntity(JmBaseEntity, SaveableEntity, IterableEntity):
return self.detail_save_base_dir
def save_file_name(self) -> str:
return f"{self.get_id_prefix_of_filename()}{self.get_id()}{self.get_title()}{self.detail_save_file_suffix}"
def jm_type():
# "JmAlbumDetail" -> "album"
cls_name = self.__class__.__name__
return cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")].lower()
def get_id_prefix_of_filename(self):
# "JmAlbumDetail" -> "album"
cls_name = self.__class__.__name__
id_prefix = cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")]
return f'{id_prefix}-'
return '[{}]{}{}'.format(jm_type(), self.get_id(), self.detail_save_file_suffix)
def get_id(self) -> str:
raise NotImplementedError
def get_title(self) -> str:
raise NotImplementedError
def __len__(self):
raise NotImplementedError
@ -152,15 +148,14 @@ class JmPhotoDetail(WorkEntity):
@property
def author(self) -> str:
# self._author 不为空字符串
if self._author is not None and self._author != '':
return self._author.strip()
# self._author 为空,先向上找
# 优先使用 from_album
if self.from_album is not None:
return self.from_album.author
# 无向上元素,使用默认
if self._author is not None and self._author != '':
return self._author.strip()
# 使用默认
return JmModuleConfig.default_author
def create_image_detail(self, index) -> JmImageDetail:
@ -196,9 +191,6 @@ class JmPhotoDetail(WorkEntity):
def get_id(self):
return self.photo_id
def get_title(self):
return self.title
def __len__(self):
return len(self.page_arr)
@ -213,13 +205,15 @@ class JmAlbumDetail(WorkEntity):
page_count,
author_list,
pub_date,
update_date,
):
self.album_id: str = album_id
self.scramble_id: str = scramble_id
self.title: str = title
self.page_count = int(page_count)
self._author_list: List[str] = author_list
self.pub_date = pub_date
self.pub_date: str = pub_date # 发布日期
self.update_date: str = update_date # 更新日期
# 有的 album 没有章节,则自成一章。
if len(episode_list) == 0:
@ -263,9 +257,6 @@ class JmAlbumDetail(WorkEntity):
def get_id(self):
return self.album_id
def get_title(self):
return self.title
def __len__(self):
return len(self.episode_list)

View File

@ -5,13 +5,11 @@ class JmOptionAdvice:
def decide_image_save_dir(self,
option: 'JmOption',
album_detail: Optional[JmAlbumDetail],
photo_detail: JmPhotoDetail,
) -> StrNone:
"""
决定一个本子图片的下载文件夹
@param option: JmOption对象
@param album_detail: 本子集实体类
@param photo_detail: 本子章节实体类
@return: 下载文件夹为空表示不处理
"""
@ -245,19 +243,16 @@ class JmOption(SaveableEntity):
下面是决定图片保存路径的方法
"""
def decide_image_save_dir(self, album_detail, photo_detail) -> str:
if album_detail is None and photo_detail is None:
raise AssertionError('album和photo不能同时为None')
def decide_image_save_dir(self, photo_detail) -> str:
# 先检查advice的回调如果回调支持则优先使用回调
for advice in JmAdviceRegistry.get_advice(self):
save_dir = advice.decide_image_save_dir(self, album_detail, photo_detail)
save_dir = advice.decide_image_save_dir(self, photo_detail)
if save_dir is not None:
return save_dir
# 使用 self.dir_tree 决定 save_dir
save_dir = self.dir_tree.deside_image_save_dir(
album_detail or photo_detail.from_album,
photo_detail.from_album,
photo_detail
)
@ -287,7 +282,7 @@ class JmOption(SaveableEntity):
return filepath
# 通过拼接生成绝对路径
save_dir = self.decide_image_save_dir(photo_detail.from_album, photo_detail)
save_dir = self.decide_image_save_dir(photo_detail)
image: JmImageDetail = photo_detail[index]
suffix = self.decide_image_suffix(image)
return save_dir + image.img_file_name + suffix
@ -451,15 +446,13 @@ class JmOption(SaveableEntity):
if use_all_default_save_path is True:
# 不通过请求获取 photo 的信息,相当于使用【空本子】和【空集】
photo_detail, album_detail = None, None
photo_detail = None
else:
# 通过请求获得 photo 的本子信息
client = self.build_jm_client()
photo_detail = client.get_photo_detail(photo_id)
album_detail = client.fill_from_album(photo_detail)
photo_detail = self.build_jm_client().get_photo_detail(photo_id)
def save_path_provider(url, _suffix: str, _index, _is_decode):
return '{0}{1}{2}'.format(self.decide_image_save_dir(album_detail, photo_detail),
return '{0}{1}{2}'.format(self.decide_image_save_dir(photo_detail),
of_file_name(url, trim_suffix=True),
self.download_convert_image_suffix or _suffix)

View File

@ -24,7 +24,8 @@ class JmcomicText:
'<li.*>\n第(\d+)話\n(.*)\n'
'<[\s\S]*?>(\d+-\d+-\d+).*?')
pattern_html_album_page_count = compile('<span class="pagecount">.*?:(\d+)</span>')
pattern_html_album_pub_date = compile('itemprop="datePublished" content=".*?">更新日期 : (.*?)</span>')
pattern_html_album_pub_date = compile('>上架日期 : (.*?)</span>')
pattern_html_album_update_date = compile('>更新日期 : (.*?)</span>')
# album 作者
pattern_html_album_author_list = [

View File

@ -35,7 +35,12 @@ class JmTestConfigurable(unittest.TestCase):
# 设置 JmOptionJmcomicClient
option = cls.use_option('option_test.yml')
cls.option = option
cls.client = option.build_jm_client()
client = option.build_jm_client()
# enable cache
client.get_photo_detail = enable_cache()(client.get_photo_detail)
client.get_album_detail = enable_cache()(client.get_album_detail)
cls.client = client
# 跨平台设置
cls.adapt_os()

View File

@ -57,12 +57,12 @@ class Test_Api(JmTestConfigurable):
) -> StrNone:
return workspace(f'{time_stamp()}_{photo_detail[index].img_file_name}.test.png')
option = JmOption.default()
option = self.option
option.register_advice(MyAdvice())
jmcomic.download_album('366867', option)
def test_photo_sort(self):
client = JmOption.default().build_jm_client()
client = self.option.build_jm_client()
# 测试用例 - 单章本子
single_photo_album_is = str_to_list('''