v2.1.5: 重构代码结构,使得更简洁可扩展 (#81)

This commit is contained in:
hect0x7 2023-07-26 17:42:40 +08:00 committed by GitHub
parent 7b2e997c6a
commit 9a5c5d12cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 219 additions and 153 deletions

View File

@ -2,6 +2,3 @@ commonX
curl_cffi
PyYAML
Pillow
# for test
pyperclip

View File

@ -2,6 +2,6 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option
__version__ = '2.1.4'
__version__ = '2.1.5'
from .api import *

View File

@ -15,36 +15,61 @@ def download_album(jm_album_id, option=None):
option, jm_client = build_client(option)
album: JmAlbumDetail = jm_client.get_album_detail(jm_album_id)
jm_debug('album',
f'本子获取成功: [{album.id}], '
f'作者: [{album.author}], '
f'章节数: [{len(album)}], '
f'标题: [{album.title}], '
)
def download_photo(photo: JmPhotoDetail,
debug_topic='photo',
):
jm_client.check_photo(photo)
jm_debug(debug_topic,
f'开始下载章节: {photo.id} ({photo.album_id}[{photo.index}/{len(album)}]), '
f'标题: [{photo.title}], '
f'图片数为[{len(photo)}]'
)
download_by_photo_detail(photo, option)
jm_debug(debug_topic,
f'章节下载完成: {photo.id} ({photo.album_id}[{photo.index}/{len(album)}])'
)
thread_pool_executor(
iter_objs=album,
apply_each_obj_func=download_photo,
option.before_album(album)
execute_by_condition(
iter_obj=album,
apply=lambda photo: download_by_photo_detail(photo, option),
count_batch=option.decide_photo_batch_count(album)
)
option.after_album(album)
jm_debug('album', f'本子下载完成: [{album.id}]')
def download_photo(jm_photo_id, option=None):
"""
下载一个本子的一章入口api
"""
option, jm_client = build_client(option)
photo = jm_client.get_photo_detail(jm_photo_id)
download_by_photo_detail(photo, option)
def download_by_photo_detail(photo: JmPhotoDetail, option=None):
"""
下载一个本子的一章根据 photo
@param photo: 本子章节信息
@param option: 选项
"""
option, jm_client = build_client(option)
# 下载准备
use_cache = option.download_cache
decode_image = option.download_image_decode
jm_client.check_photo(photo)
# 下载每个图片的函数
def download_image(image: JmImageDetail):
img_save_path = option.decide_image_filepath(image)
# 已下载过,缓存命中
if use_cache is True and file_exists(img_save_path):
image.is_exists = True
return
option.before_image(image, img_save_path)
jm_client.download_by_image_detail(
image,
img_save_path,
decode_image=decode_image,
)
option.after_image(image, img_save_path)
option.before_photo(photo)
execute_by_condition(
iter_obj=photo,
apply=download_image,
count_batch=option.decide_image_batch_count(photo)
)
option.before_photo(photo)
def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
@ -69,64 +94,24 @@ def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
)
def download_photo(jm_photo_id, option=None):
def execute_by_condition(iter_obj, apply: Callable, count_batch: int):
"""
下载一个本子的一章入口api
章节/图片的下载调度逻辑
"""
option, jm_client = build_client(option)
photo_detail = jm_client.get_photo_detail(jm_photo_id)
download_by_photo_detail(photo_detail, option)
count_real = len(iter_obj)
def download_by_photo_detail(photo_detail: JmPhotoDetail,
option=None,
):
"""
下载一个本子的一章根据 photo_detail
@param photo_detail: 本子章节信息
@param option: 选项
"""
option, jm_client = build_client(option)
# 下载准备
use_cache = option.download_cache
decode_image = option.download_image_decode
jm_client.check_photo(photo_detail)
# 下载每个图片的函数
def download_image(index, image: JmImageDetail, debug_topic='image'):
img_save_path = option.decide_image_filepath(photo_detail, index)
debug_tag = f'{image.aid}/{image.filename} [{index + 1}/{len(photo_detail)}]'
# 已下载过,缓存命中
if use_cache is True and file_exists(img_save_path):
jm_debug(debug_topic,
f'图片已存在: {debug_tag} ← [{img_save_path}]'
)
return
# 开始下载
jm_client.download_by_image_detail(
image,
img_save_path,
decode_image=decode_image,
)
jm_debug(debug_topic,
f'图片下载完成: {debug_tag}, [{image.img_url}] → [{img_save_path}]'
)
batch = option.download_threading_batch_count
if batch <= 0:
if count_batch >= count_real:
# 一图一线程
multi_thread_launcher(
iter_objs=enumerate(photo_detail),
apply_each_obj_func=download_image,
iter_objs=iter_obj,
apply_each_obj_func=apply,
)
else:
# 创建batch个线程的线程池当图片数>batch时要等待。
thread_pool_executor(
iter_objs=enumerate(photo_detail),
apply_each_obj_func=download_image,
max_workers=batch,
iter_objs=iter_obj,
apply_each_obj_func=apply,
max_workers=count_batch,
)

View File

@ -31,7 +31,7 @@ class AbstractJmClient(
return self.request_with_retry(self.postman.post, url, **kwargs)
def of_api_url(self, api_path, domain):
return f'{JmModuleConfig.PROT}{domain}{api_path}'
return JmcomicText.format_url(api_path, domain)
def request_with_retry(self,
request,
@ -52,9 +52,11 @@ class AbstractJmClient(
self.fallback(request, url, domain_index, retry_count, **kwargs)
if url.startswith('/'):
# path
domain = self.domain_list[domain_index]
url = self.of_api_url(url, domain)
# path → url
url = self.of_api_url(
api_path=url,
domain=self.domain_list[domain_index],
)
jm_debug('api', url)
else:
# 图片url
@ -154,13 +156,13 @@ class JmHtmlClient(AbstractJmClient):
resp = self.get_jm_html(f"/photo/{photo_id}")
# 用 JmcomicText 解析 html返回实体类
photo_detail = JmcomicText.analyse_jm_photo_html(resp.text)
photo = JmcomicText.analyse_jm_photo_html(resp.text)
# 一并获取该章节的所处本子
if fetch_album is True:
photo_detail.from_album = self.get_album_detail(photo_detail.album_id)
photo.from_album = self.get_album_detail(photo.album_id)
return photo_detail
return photo
def search_album(self, search_query, main_tag=0, page=1) -> JmSearchPage:
params = {

View File

@ -160,16 +160,28 @@ class JmDetailClient:
def enable_cache(self, debug=False):
raise NotImplementedError
def check_photo(self, photo_detail: JmPhotoDetail):
def check_photo(self, photo: JmPhotoDetail):
"""
photo来源有两种:
1. album[?]
2. client.get_photo_detail(?)
其中只有[2]是可以包含下载图片的url信息的
本方法会检查photo是不是[1]
如果是[1]通过请求获取[2]然后把2中的一些重要字段更新到1中
@param photo: 被检查的JmPhotoDetail对象
"""
# 检查 from_album
if photo_detail.from_album is None:
photo_detail.from_album = self.get_album_detail(photo_detail.album_id)
if photo.from_album is None:
photo.from_album = self.get_album_detail(photo.album_id)
# 检查 page_arr 和 data_original_domain
if photo_detail.page_arr is None or photo_detail.data_original_domain is None:
new = self.get_photo_detail(photo_detail.photo_id, False)
new.from_album = photo_detail.from_album
photo_detail.__dict__.update(new.__dict__)
if photo.page_arr is None or photo.data_original_domain is None:
new = self.get_photo_detail(photo.photo_id, False)
new.from_album = photo.from_album
photo.__dict__.update(new.__dict__)
class JmUserClient:
@ -232,14 +244,14 @@ class JmImageClient:
resp.transfer_to(img_save_path, scramble_id, decode_image, img_url)
def download_by_image_detail(self,
img_detail: JmImageDetail,
image: JmImageDetail,
img_save_path,
decode_image=True,
):
self.download_image(
img_detail.download_url,
image.download_url,
img_save_path,
img_detail.scramble_id,
image.scramble_id,
decode_image=decode_image,
)

View File

@ -7,37 +7,33 @@ class JmBaseEntity:
pass
class WorkEntity(JmBaseEntity, SaveableEntity, IterableEntity):
when_del_save_file = False
after_save_print_info = True
attr_char = '_'
cache_getitem_result = True
cache_field_name = '__cache_items_dict__'
detail_save_base_dir = workspace()
detail_save_file_suffix = '.yml'
def save_base_dir(self):
return self.detail_save_base_dir
def save_file_name(self) -> str:
def jm_type():
# "JmAlbumDetail" -> "album"
cls_name = self.__class__.__name__
return cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")].lower()
return '[{}]{}{}'.format(jm_type(), self.id, self.detail_save_file_suffix)
class DetailEntity(JmBaseEntity, IterableEntity):
@property
def id(self) -> str:
raise NotImplementedError
def __len__(self):
raise NotImplementedError
@property
def name(self) -> str:
return getattr(self, 'title')
def save_to_file(self, filepath):
from common import PackerUtil
PackerUtil.pack(self, filepath)
@classmethod
def __jm_type__(cls):
# "JmAlbumDetail" -> "album" (本子)
# "JmPhotoDetail" -> "photo" (章节)
cls_name = cls.__name__
return cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")].lower()
def __getitem__(self, item) -> Union['JmAlbumDetail', 'JmPhotoDetail']:
raise NotImplementedError
def __str__(self):
return f'{self.__class__.__name__}({self.id}-{self.name})'
class JmImageDetail(JmBaseEntity):
@ -49,6 +45,7 @@ class JmImageDetail(JmBaseEntity):
img_file_suffix,
from_photo=None,
query_params=None,
index=-1,
) -> None:
self.aid: str = aid
self.scramble_id: str = scramble_id
@ -58,6 +55,8 @@ class JmImageDetail(JmBaseEntity):
self.from_photo: Optional[JmPhotoDetail] = from_photo
self.query_params: StrNone = query_params
self.is_exists: bool = False
self.index = index
@property
def filename(self) -> str:
@ -82,6 +81,7 @@ class JmImageDetail(JmBaseEntity):
data_original: str,
from_photo=None,
query_params=None,
index=-1,
) -> 'JmImageDetail':
"""
该方法用于创建 JmImageDetail 对象
@ -101,10 +101,19 @@ class JmImageDetail(JmBaseEntity):
img_file_suffix=data_original[y:],
from_photo=from_photo,
query_params=query_params,
index=index,
)
"""
below help for debug method
"""
class JmPhotoDetail(WorkEntity):
@property
def tag(self) -> str:
return f'{self.aid}/{self.filename} [{self.index + 1}/{len(self.from_photo)}]'
class JmPhotoDetail(DetailEntity):
def __init__(self,
photo_id,
@ -209,6 +218,7 @@ class JmPhotoDetail(WorkEntity):
data_original,
from_photo=self,
query_params=self.data_original_query_params,
index=index,
)
def get_img_data_original(self, img_name: str) -> str:
@ -248,7 +258,7 @@ class JmPhotoDetail(WorkEntity):
return super().__iter__()
class JmAlbumDetail(WorkEntity):
class JmAlbumDetail(DetailEntity):
def __init__(self,
album_id,
@ -288,7 +298,7 @@ class JmAlbumDetail(WorkEntity):
episode_info: tuple = self.episode_list[index]
photo_id, photo_index_of_album, photo_title, photo_pub_date = episode_info
photo_detail = JmPhotoDetail(
photo = JmPhotoDetail(
photo_id=photo_id,
scramble_id=self.scramble_id,
title=photo_title,
@ -301,7 +311,7 @@ class JmAlbumDetail(WorkEntity):
data_original_domain=None
)
return photo_detail, episode_info
return photo, episode_info
@property
def author(self):
@ -344,7 +354,7 @@ class JmAlbumDetail(WorkEntity):
return super().__iter__()
class JmSearchPage(IterableEntity):
class JmSearchPage(JmBaseEntity, IterableEntity):
def __init__(self, album_info_list: List[Tuple[str, str, StrNone, StrNone, List[str]]]):
# (album_id, title, category_none, label_sub_none, tag_list)

View File

@ -1,6 +1,46 @@
from .jm_client_impl import *
# noinspection PyMethodMayBeStatic
class DownloadCallback:
def before_album(self, album: JmAlbumDetail):
jm_debug('album-before',
f'本子获取成功: [{album.id}], '
f'作者: [{album.author}], '
f'章节数: [{len(album)}], '
f'标题: [{album.title}], '
)
def after_album(self, album: JmAlbumDetail):
jm_debug('album-after', f'本子下载完成: [{album.id}]')
def before_photo(self, photo: JmPhotoDetail):
jm_debug('photo-before',
f'开始下载章节: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}]), '
f'标题: [{photo.title}], '
f'图片数为[{len(photo)}]'
)
def after_photo(self, photo: JmPhotoDetail):
jm_debug('photo-after',
f'章节下载完成: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}])')
def before_image(self, image: JmImageDetail, img_save_path):
if image.is_exists:
jm_debug('image-before',
f'图片已存在: {image.tag} ← [{img_save_path}]'
)
else:
jm_debug('image_before',
f'图片准备下载: {image.tag}, [{image.img_url}] → [{img_save_path}]'
)
def after_image(self, image: JmImageDetail, img_save_path):
jm_debug('image-after',
f'图片下载完成: {image.tag}, [{image.img_url}] → [{img_save_path}]')
class DirRule:
rule_sample = [
# 根目录 / Album-id / Photo-序号 /
@ -99,7 +139,7 @@ class DirRule:
return base_dir
class JmOption:
class JmOption(DownloadCallback):
JM_OP_VER = '2.0'
def __init__(self,
@ -142,29 +182,36 @@ class JmOption:
下面是决定图片保存路径的方法
"""
def decide_image_save_dir(self, photo_detail) -> str:
# noinspection PyUnusedLocal
def decide_image_batch_count(self, photo: JmPhotoDetail):
return self.download_threading_batch_count
# noinspection PyMethodMayBeStatic
def decide_photo_batch_count(self, album: JmAlbumDetail):
return len(album)
def decide_image_save_dir(self, photo) -> str:
# 使用 self.dir_rule 决定 save_dir
save_dir = self.dir_rule.deside_image_save_dir(
photo_detail.from_album,
photo_detail
photo.from_album,
photo
)
mkdir_if_not_exists(save_dir)
return save_dir
def decide_image_suffix(self, img_detail: JmImageDetail):
def decide_image_suffix(self, image: JmImageDetail):
# 动图则使用原后缀
suffix = img_detail.img_file_suffix
suffix = image.img_file_suffix
if suffix.endswith("gif"):
return suffix
# 非动图,以配置为先
return self.download_image_suffix or suffix
def decide_image_filepath(self, photo_detail: JmPhotoDetail, index: int) -> str:
def decide_image_filepath(self, image: JmImageDetail) -> str:
# 通过拼接生成绝对路径
save_dir = self.decide_image_save_dir(photo_detail)
image: JmImageDetail = photo_detail[index]
save_dir = self.decide_image_save_dir(image.from_photo)
suffix = self.decide_image_suffix(image)
return save_dir + image.img_file_name + suffix

View File

@ -152,6 +152,21 @@ class JmcomicText:
return clazz(**field_dict)
@classmethod
def format_photo_url(cls, photo_id, domain=None):
return cls.format_url(f'/photo/{cls.parse_to_photo_id(photo_id)}', domain)
@classmethod
def format_album_url(cls, album_id, domain=None):
return cls.format_url(f'/album/{cls.parse_to_album_id(album_id)}', domain)
@classmethod
def format_url(cls, path, domain=None):
if domain is None:
domain = JmModuleConfig.domain()
return f'{JmModuleConfig.PROT}{domain}{path}'
class JmSearchSupport:
# 用来缩减html的长度
@ -206,23 +221,23 @@ class JmImageSupport:
@classmethod
def save_resp_decoded_img(cls,
resp: Any,
img_detail: JmImageDetail,
image: JmImageDetail,
filepath: str
) -> None:
cls.decode_and_save(
cls.get_num_by_detail(img_detail),
cls.get_num_by_detail(image),
cls.open_Image(resp.content),
filepath
)
@classmethod
def decode_disk_img(cls,
img_detail: JmImageDetail,
image: JmImageDetail,
img_filepath: str,
decoded_save_path: str
) -> None:
cls.decode_and_save(
cls.get_num_by_detail(img_detail),
cls.get_num_by_detail(image),
cls.open_Image(img_filepath),
decoded_save_path
)

View File

@ -28,8 +28,6 @@ class JmTestConfigurable(unittest.TestCase):
# 设置 workspace → assets/
set_application_workspace(f'{application_workspace}/assets/')
# 设置 实体类的save_dir → assets/download
WorkEntity.detail_save_base_dir = workspace("/download/", is_dir=True)
# 设置 JmOptionJmcomicClient
option = cls.use_option('option_test.yml')

View File

@ -10,9 +10,9 @@ class Test_Client(JmTestConfigurable):
def test_download_image(self):
jm_photo_id = 'JM438516'
photo_detail = self.client.get_photo_detail(jm_photo_id, False)
photo = self.client.get_photo_detail(jm_photo_id, False)
self.client.download_by_image_detail(
photo_detail[0],
photo[0],
img_save_path=workspace('test_download_image.png')
)
@ -25,10 +25,10 @@ class Test_Client(JmTestConfigurable):
测试通过 JmcomicClient jm_photo_id 获取 JmPhotoDetail对象
"""
jm_photo_id = 'JM438516'
photo_detail = self.client.get_photo_detail(jm_photo_id, False)
photo_detail.when_del_save_file = True
photo_detail.after_save_print_info = True
del photo_detail
photo = self.client.get_photo_detail(jm_photo_id, False)
photo.when_del_save_file = True
photo.after_save_print_info = True
del photo
def test_multi_album_and_single_album(self):
multi_photo_album_id = [
@ -36,18 +36,18 @@ class Test_Client(JmTestConfigurable):
]
for album_id in multi_photo_album_id:
album_detail: JmAlbumDetail = self.client.get_album_detail(album_id)
print(f'本子: [{album_detail.title}] 一共有{album_detail.page_count}页图')
album: JmAlbumDetail = self.client.get_album_detail(album_id)
print(f'本子: [{album.title}] 一共有{album.page_count}页图')
def test_search(self):
jm_search_page: JmSearchPage = self.client.search_album('MANA')
jm_search_page: JmSearchPage = self.client.search_album('+无修正 +中文 -全彩')
for album_id, title in reversed(jm_search_page):
print(album_id, title)
def test_gt_300_photo(self):
photo_id = '147643'
photo_detail: JmPhotoDetail = self.client.get_photo_detail(photo_id, False)
image = photo_detail[3000]
photo: JmPhotoDetail = self.client.get_photo_detail(photo_id, False)
image = photo[3000]
print(image.img_url)
self.client.download_by_image_detail(image, workspace('3000.png'))