From fc5194015f318a9cf0f298e662ae7d7e9223ad09 Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Wed, 16 Aug 2023 20:38:49 +0800 Subject: [PATCH] =?UTF-8?q?v2.1.12:=20=E6=94=AF=E6=8C=81=E5=AE=9E=E4=BD=93?= =?UTF-8?q?=E7=B1=BB=E7=9A=84=E5=88=87=E7=89=87=E8=AF=AD=E6=B3=95=EF=BC=8C?= =?UTF-8?q?JmDownloader=E5=A2=9E=E5=8A=A0=E4=B8=8B=E8=BD=BD=E5=89=8DFilter?= =?UTF-8?q?=E8=BF=87=E6=BB=A4=E6=9C=BA=E5=88=B6=20(#96)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/jmcomic/__init__.py | 2 +- src/jmcomic/api.py | 30 +++++++++------ src/jmcomic/jm_downloader.py | 30 ++++++++++++++- src/jmcomic/jm_entity.py | 63 ++++++++++++++++++++++--------- src/jmcomic/jm_toolkit.py | 3 +- tests/test_jmcomic/test_jm_api.py | 43 ++++++++++++++++++++- 6 files changed, 137 insertions(+), 34 deletions(-) diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index bce8d57..7bd371b 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,6 +2,6 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.1.11' +__version__ = '2.1.12' from .api import * diff --git a/src/jmcomic/api.py b/src/jmcomic/api.py index 46ba1e4..36e6ec0 100644 --- a/src/jmcomic/api.py +++ b/src/jmcomic/api.py @@ -1,24 +1,29 @@ from .jm_downloader import * -def download_album_batch(jm_album_id_iter: Union[Iterable, Generator], - option=None, - ): +def download_batch(download_api, + jm_id_iter: Union[Iterable, Generator], + option=None, + ): """ - 批量下载album. - 一个album,对应一个线程,对应一个option + 批量下载 album / photo + 一个album/photo,对应一个线程,对应一个option - @param jm_album_id_iter: album_id的迭代器 - @param option: 下载选项,为空默认是 JmOption.default() + @param download_api: 下载api + @param jm_id_iter: jmid (album_id, photo_id) 的迭代器 + @param option: 下载选项,对所有的jmid使用同一个,默认是 JmOption.default() """ from common import multi_thread_launcher + if option is None: + option = JmOption.default() + return multi_thread_launcher( iter_objs=set( - JmcomicText.parse_to_album_id(album_id) - for album_id in jm_album_id_iter + JmcomicText.parse_to_album_id(jmid) + for jmid in jm_id_iter ), - apply_each_obj_func=lambda aid: download_album(aid, option), + apply_each_obj_func=lambda aid: download_api(aid, option), ) @@ -31,7 +36,7 @@ def download_album(jm_album_id, option=None): """ if not isinstance(jm_album_id, (str, int)): - return download_album_batch(jm_album_id, option) + return download_batch(download_album, jm_album_id, option) with new_downloader(option) as dler: dler.download_album(jm_album_id) @@ -41,6 +46,9 @@ def download_photo(jm_photo_id, option=None): """ 下载一个章节 """ + if not isinstance(jm_photo_id, (str, int)): + return download_batch(download_photo, jm_photo_id, option) + with new_downloader(option) as dler: dler.download_photo(jm_photo_id) diff --git a/src/jmcomic/jm_downloader.py b/src/jmcomic/jm_downloader.py index 812dba3..54faa7c 100644 --- a/src/jmcomic/jm_downloader.py +++ b/src/jmcomic/jm_downloader.py @@ -1,5 +1,13 @@ from .jm_option import * +# help for typing +DownloadIterObjs = Union[ + JmAlbumDetail, + Sequence[JmPhotoDetail], + JmPhotoDetail, + Sequence[JmImageDetail], +] + class JmDownloadException(Exception): pass @@ -100,10 +108,15 @@ class JmDownloader(DownloadCallback): self.after_image(image, img_save_path) # noinspection PyMethodMayBeStatic - def execute_by_condition(self, iter_objs, apply: Callable, count_batch: int): + def execute_by_condition(self, + iter_objs: DownloadIterObjs, + apply: Callable, + count_batch: int, + ): """ - 章节/图片的下载调度逻辑 + 调度本子/章节的下载 """ + iter_objs = self.filter_iter_objs(iter_objs) count_real = len(iter_objs) if count_batch >= count_real: @@ -120,6 +133,19 @@ class JmDownloader(DownloadCallback): max_workers=count_batch, ) + # noinspection PyMethodMayBeStatic + def filter_iter_objs(self, iter_objs: DownloadIterObjs): + """ + 该方法可用于过滤本子/章节,默认不会做过滤。 + 例如: + 只想下载 本子的最新一章,返回 [album[-1]] + 只想下载 章节的前10张图片,返回 [photo[:10]] + + @param iter_objs: 可能是本子或者章节,需要自行使用 isinstance 判断 + @return: 只想要下载的 本子的章节 或 章节的图片 + """ + return iter_objs + # noinspection PyUnusedLocal def client_for_album(self, jm_album_id) -> JmcomicClient: """ diff --git a/src/jmcomic/jm_entity.py b/src/jmcomic/jm_entity.py index 033b597..f5236f0 100644 --- a/src/jmcomic/jm_entity.py +++ b/src/jmcomic/jm_entity.py @@ -21,7 +21,7 @@ class JmBaseEntity: PackerUtil.pack(self, filepath) -class DetailEntity(JmBaseEntity, IterableEntity): +class DetailEntity(JmBaseEntity): @property def id(self) -> str: @@ -31,19 +31,42 @@ class DetailEntity(JmBaseEntity, IterableEntity): def name(self) -> str: return getattr(self, 'title') + # help for typing + JMPI = Union['JmPhotoDetail', 'JmImageDetail'] + + def getindex(self, index: int) -> JMPI: + raise NotImplementedError + + def __len__(self): + raise NotImplementedError + + def __iter__(self) -> Generator[JMPI, Any, None]: + for index in range(len(self)): + yield self.getindex(index) + + def __str__(self): + return f'{self.__class__.__name__}({self.id}-{self.name})' + + def __getitem__(self, item) -> Union[JMPI, List[JMPI]]: + if isinstance(item, slice): + start = item.start or 0 + stop = item.stop or len(self) + step = item.step or 1 + return [self.getindex(index) for index in range(start, stop, step)] + + elif isinstance(item, int): + return self.getindex(item) + + else: + raise TypeError(f"Invalid item type for {self.__class__}") + @classmethod - def __jm_type__(cls): + def __alias__(cls): # "JmAlbumDetail" -> "album" (本子) # "JmPhotoDetail" -> "photo" (章节) cls_name = cls.__name__ return cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")].lower() - def __getitem__(self, item) -> Union['JmAlbumDetail', 'JmPhotoDetail']: - raise NotImplementedError - - def __str__(self): - return f'{self.__class__.__name__}({self.id}-{self.name})' - class JmImageDetail(JmBaseEntity): @@ -254,13 +277,16 @@ class JmPhotoDetail(DetailEntity): return data_original_0[index + 1:] - def __getitem__(self, item) -> JmImageDetail: - return self.create_image_detail(item) - @property def id(self): return self.photo_id + def getindex(self, index) -> JmImageDetail: + return self.create_image_detail(index) + + def __getitem__(self, item) -> Union[JmImageDetail, List[JmImageDetail]]: + return super().__getitem__(item) + def __len__(self): return len(self.page_arr) @@ -337,12 +363,6 @@ class JmAlbumDetail(DetailEntity): def id(self): return self.album_id - def __len__(self): - return len(self.episode_list) - - def __getitem__(self, item) -> JmPhotoDetail: - return self.create_photo_detail(item)[0] - @staticmethod def distinct_episode(episode_list): ret = [] @@ -360,6 +380,15 @@ class JmAlbumDetail(DetailEntity): return ret + def getindex(self, item) -> JmPhotoDetail: + return self.create_photo_detail(item)[0] + + def __getitem__(self, item) -> Union[JmPhotoDetail, List[JmPhotoDetail]]: + return super().__getitem__(item) + + def __len__(self): + return len(self.episode_list) + def __iter__(self) -> Generator[JmPhotoDetail, Any, None]: return super().__iter__() diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py index 5d820b6..7ac4d5e 100644 --- a/src/jmcomic/jm_toolkit.py +++ b/src/jmcomic/jm_toolkit.py @@ -1,6 +1,7 @@ -from .jm_entity import * from PIL import Image +from .jm_entity import * + class JmcomicText: pattern_jm_domain = compile('https://([\w.-]+)') diff --git a/tests/test_jmcomic/test_jm_api.py b/tests/test_jmcomic/test_jm_api.py index 9551916..1a8e33a 100644 --- a/tests/test_jmcomic/test_jm_api.py +++ b/tests/test_jmcomic/test_jm_api.py @@ -41,11 +41,11 @@ class Test_Api(JmTestConfigurable): ret1 = jmcomic.download_album(case, self.option) self.assertEqual(len(ret1), len(album_ls), str(case)) - ret2 = jmcomic.download_album_batch(case, self.option) + ret2 = jmcomic.download_album(case, self.option) self.assertEqual(len(ret2), len(album_ls), str(case)) # 测试 Generator - ret2 = jmcomic.download_album_batch((e for e in album_ls), self.option) + ret2 = jmcomic.download_album((e for e in album_ls), self.option) self.assertEqual(len(ret2), len(album_ls), 'Generator') def test_photo_sort(self): @@ -130,3 +130,42 @@ class Test_Api(JmTestConfigurable): print(e) raise AssertionError(exception_list) + + def test_getitem_and_slice(self): + cl: JmcomicClient = self.client + cases = [ + ['400222', 0, [400222]], + ['400222', 1, [413446]], + ['400222', (None, 1), [400222]], + ['400222', (1, 3), [413446, 413447]], + ['413447', (1, 3), [2, 3], []], + ] + + for [jmid, slicearg, *args] in cases: + ans = args[0] + + if len(args) == 1: + func = cl.get_album_detail + else: + func = cl.get_photo_detail + + jmentity = func(jmid) + + ls: List[Union[JmPhotoDetail, JmImageDetail]] + if isinstance(slicearg, int): + ls = [jmentity[slicearg]] + elif len(slicearg) == 2: + ls = jmentity[slicearg[0]: slicearg[1]] + else: + ls = jmentity[slicearg[0]: slicearg[1]: slicearg[2]] + + if len(args) == 1: + self.assertListEqual( + list1=[int(e.id) for e in ls], + list2=ans, + ) + else: + self.assertListEqual( + list1=[int(e.img_file_name) for e in ls], + list2=ans, + )