v2.1.12: 支持实体类的切片语法,JmDownloader增加下载前Filter过滤机制 (#96)

This commit is contained in:
hect0x7 2023-08-16 20:38:49 +08:00 committed by GitHub
parent 476efc66d3
commit fc5194015f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 137 additions and 34 deletions

View File

@ -2,6 +2,6 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.1.11'
__version__ = '2.1.12'
from .api import *

View File

@ -1,24 +1,29 @@
from .jm_downloader import *
def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
option=None,
):
def download_batch(download_api,
jm_id_iter: Union[Iterable, Generator],
option=None,
):
"""
批量下载album.
一个album对应一个线程对应一个option
批量下载 album / photo
一个album/photo对应一个线程对应一个option
@param jm_album_id_iter: album_id的迭代器
@param option: 下载选项为空默认是 JmOption.default()
@param download_api: 下载api
@param jm_id_iter: jmid (album_id, photo_id) 的迭代器
@param option: 下载选项对所有的jmid使用同一个默认是 JmOption.default()
"""
from common import multi_thread_launcher
if option is None:
option = JmOption.default()
return multi_thread_launcher(
iter_objs=set(
JmcomicText.parse_to_album_id(album_id)
for album_id in jm_album_id_iter
JmcomicText.parse_to_album_id(jmid)
for jmid in jm_id_iter
),
apply_each_obj_func=lambda aid: download_album(aid, option),
apply_each_obj_func=lambda aid: download_api(aid, option),
)
@ -31,7 +36,7 @@ def download_album(jm_album_id, option=None):
"""
if not isinstance(jm_album_id, (str, int)):
return download_album_batch(jm_album_id, option)
return download_batch(download_album, jm_album_id, option)
with new_downloader(option) as dler:
dler.download_album(jm_album_id)
@ -41,6 +46,9 @@ def download_photo(jm_photo_id, option=None):
"""
下载一个章节
"""
if not isinstance(jm_photo_id, (str, int)):
return download_batch(download_photo, jm_photo_id, option)
with new_downloader(option) as dler:
dler.download_photo(jm_photo_id)

View File

@ -1,5 +1,13 @@
from .jm_option import *
# help for typing
DownloadIterObjs = Union[
JmAlbumDetail,
Sequence[JmPhotoDetail],
JmPhotoDetail,
Sequence[JmImageDetail],
]
class JmDownloadException(Exception):
pass
@ -100,10 +108,15 @@ class JmDownloader(DownloadCallback):
self.after_image(image, img_save_path)
# noinspection PyMethodMayBeStatic
def execute_by_condition(self, iter_objs, apply: Callable, count_batch: int):
def execute_by_condition(self,
iter_objs: DownloadIterObjs,
apply: Callable,
count_batch: int,
):
"""
章节/图片的下载调度逻辑
调度本子/章节的下载
"""
iter_objs = self.filter_iter_objs(iter_objs)
count_real = len(iter_objs)
if count_batch >= count_real:
@ -120,6 +133,19 @@ class JmDownloader(DownloadCallback):
max_workers=count_batch,
)
# noinspection PyMethodMayBeStatic
def filter_iter_objs(self, iter_objs: DownloadIterObjs):
"""
该方法可用于过滤本子/章节默认不会做过滤
例如:
只想下载 本子的最新一章返回 [album[-1]]
只想下载 章节的前10张图片返回 [photo[:10]]
@param iter_objs: 可能是本子或者章节需要自行使用 isinstance 判断
@return: 只想要下载的 本子的章节 章节的图片
"""
return iter_objs
# noinspection PyUnusedLocal
def client_for_album(self, jm_album_id) -> JmcomicClient:
"""

View File

@ -21,7 +21,7 @@ class JmBaseEntity:
PackerUtil.pack(self, filepath)
class DetailEntity(JmBaseEntity, IterableEntity):
class DetailEntity(JmBaseEntity):
@property
def id(self) -> str:
@ -31,19 +31,42 @@ class DetailEntity(JmBaseEntity, IterableEntity):
def name(self) -> str:
return getattr(self, 'title')
# help for typing
JMPI = Union['JmPhotoDetail', 'JmImageDetail']
def getindex(self, index: int) -> JMPI:
raise NotImplementedError
def __len__(self):
raise NotImplementedError
def __iter__(self) -> Generator[JMPI, Any, None]:
for index in range(len(self)):
yield self.getindex(index)
def __str__(self):
return f'{self.__class__.__name__}({self.id}-{self.name})'
def __getitem__(self, item) -> Union[JMPI, List[JMPI]]:
if isinstance(item, slice):
start = item.start or 0
stop = item.stop or len(self)
step = item.step or 1
return [self.getindex(index) for index in range(start, stop, step)]
elif isinstance(item, int):
return self.getindex(item)
else:
raise TypeError(f"Invalid item type for {self.__class__}")
@classmethod
def __jm_type__(cls):
def __alias__(cls):
# "JmAlbumDetail" -> "album" (本子)
# "JmPhotoDetail" -> "photo" (章节)
cls_name = cls.__name__
return cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")].lower()
def __getitem__(self, item) -> Union['JmAlbumDetail', 'JmPhotoDetail']:
raise NotImplementedError
def __str__(self):
return f'{self.__class__.__name__}({self.id}-{self.name})'
class JmImageDetail(JmBaseEntity):
@ -254,13 +277,16 @@ class JmPhotoDetail(DetailEntity):
return data_original_0[index + 1:]
def __getitem__(self, item) -> JmImageDetail:
return self.create_image_detail(item)
@property
def id(self):
return self.photo_id
def getindex(self, index) -> JmImageDetail:
return self.create_image_detail(index)
def __getitem__(self, item) -> Union[JmImageDetail, List[JmImageDetail]]:
return super().__getitem__(item)
def __len__(self):
return len(self.page_arr)
@ -337,12 +363,6 @@ class JmAlbumDetail(DetailEntity):
def id(self):
return self.album_id
def __len__(self):
return len(self.episode_list)
def __getitem__(self, item) -> JmPhotoDetail:
return self.create_photo_detail(item)[0]
@staticmethod
def distinct_episode(episode_list):
ret = []
@ -360,6 +380,15 @@ class JmAlbumDetail(DetailEntity):
return ret
def getindex(self, item) -> JmPhotoDetail:
return self.create_photo_detail(item)[0]
def __getitem__(self, item) -> Union[JmPhotoDetail, List[JmPhotoDetail]]:
return super().__getitem__(item)
def __len__(self):
return len(self.episode_list)
def __iter__(self) -> Generator[JmPhotoDetail, Any, None]:
return super().__iter__()

View File

@ -1,6 +1,7 @@
from .jm_entity import *
from PIL import Image
from .jm_entity import *
class JmcomicText:
pattern_jm_domain = compile('https://([\w.-]+)')

View File

@ -41,11 +41,11 @@ class Test_Api(JmTestConfigurable):
ret1 = jmcomic.download_album(case, self.option)
self.assertEqual(len(ret1), len(album_ls), str(case))
ret2 = jmcomic.download_album_batch(case, self.option)
ret2 = jmcomic.download_album(case, self.option)
self.assertEqual(len(ret2), len(album_ls), str(case))
# 测试 Generator
ret2 = jmcomic.download_album_batch((e for e in album_ls), self.option)
ret2 = jmcomic.download_album((e for e in album_ls), self.option)
self.assertEqual(len(ret2), len(album_ls), 'Generator')
def test_photo_sort(self):
@ -130,3 +130,42 @@ class Test_Api(JmTestConfigurable):
print(e)
raise AssertionError(exception_list)
def test_getitem_and_slice(self):
cl: JmcomicClient = self.client
cases = [
['400222', 0, [400222]],
['400222', 1, [413446]],
['400222', (None, 1), [400222]],
['400222', (1, 3), [413446, 413447]],
['413447', (1, 3), [2, 3], []],
]
for [jmid, slicearg, *args] in cases:
ans = args[0]
if len(args) == 1:
func = cl.get_album_detail
else:
func = cl.get_photo_detail
jmentity = func(jmid)
ls: List[Union[JmPhotoDetail, JmImageDetail]]
if isinstance(slicearg, int):
ls = [jmentity[slicearg]]
elif len(slicearg) == 2:
ls = jmentity[slicearg[0]: slicearg[1]]
else:
ls = jmentity[slicearg[0]: slicearg[1]: slicearg[2]]
if len(args) == 1:
self.assertListEqual(
list1=[int(e.id) for e in ls],
list2=ans,
)
else:
self.assertListEqual(
list1=[int(e.img_file_name) for e in ls],
list2=ans,
)