mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v2.1.12: 支持实体类的切片语法,JmDownloader增加下载前Filter过滤机制 (#96)
This commit is contained in:
parent
476efc66d3
commit
fc5194015f
@ -2,6 +2,6 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.1.11'
|
||||
__version__ = '2.1.12'
|
||||
|
||||
from .api import *
|
||||
|
||||
@ -1,24 +1,29 @@
|
||||
from .jm_downloader import *
|
||||
|
||||
|
||||
def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
|
||||
option=None,
|
||||
):
|
||||
def download_batch(download_api,
|
||||
jm_id_iter: Union[Iterable, Generator],
|
||||
option=None,
|
||||
):
|
||||
"""
|
||||
批量下载album.
|
||||
一个album,对应一个线程,对应一个option
|
||||
批量下载 album / photo
|
||||
一个album/photo,对应一个线程,对应一个option
|
||||
|
||||
@param jm_album_id_iter: album_id的迭代器
|
||||
@param option: 下载选项,为空默认是 JmOption.default()
|
||||
@param download_api: 下载api
|
||||
@param jm_id_iter: jmid (album_id, photo_id) 的迭代器
|
||||
@param option: 下载选项,对所有的jmid使用同一个,默认是 JmOption.default()
|
||||
"""
|
||||
from common import multi_thread_launcher
|
||||
|
||||
if option is None:
|
||||
option = JmOption.default()
|
||||
|
||||
return multi_thread_launcher(
|
||||
iter_objs=set(
|
||||
JmcomicText.parse_to_album_id(album_id)
|
||||
for album_id in jm_album_id_iter
|
||||
JmcomicText.parse_to_album_id(jmid)
|
||||
for jmid in jm_id_iter
|
||||
),
|
||||
apply_each_obj_func=lambda aid: download_album(aid, option),
|
||||
apply_each_obj_func=lambda aid: download_api(aid, option),
|
||||
)
|
||||
|
||||
|
||||
@ -31,7 +36,7 @@ def download_album(jm_album_id, option=None):
|
||||
"""
|
||||
|
||||
if not isinstance(jm_album_id, (str, int)):
|
||||
return download_album_batch(jm_album_id, option)
|
||||
return download_batch(download_album, jm_album_id, option)
|
||||
|
||||
with new_downloader(option) as dler:
|
||||
dler.download_album(jm_album_id)
|
||||
@ -41,6 +46,9 @@ def download_photo(jm_photo_id, option=None):
|
||||
"""
|
||||
下载一个章节
|
||||
"""
|
||||
if not isinstance(jm_photo_id, (str, int)):
|
||||
return download_batch(download_photo, jm_photo_id, option)
|
||||
|
||||
with new_downloader(option) as dler:
|
||||
dler.download_photo(jm_photo_id)
|
||||
|
||||
|
||||
@ -1,5 +1,13 @@
|
||||
from .jm_option import *
|
||||
|
||||
# help for typing
|
||||
DownloadIterObjs = Union[
|
||||
JmAlbumDetail,
|
||||
Sequence[JmPhotoDetail],
|
||||
JmPhotoDetail,
|
||||
Sequence[JmImageDetail],
|
||||
]
|
||||
|
||||
|
||||
class JmDownloadException(Exception):
|
||||
pass
|
||||
@ -100,10 +108,15 @@ class JmDownloader(DownloadCallback):
|
||||
self.after_image(image, img_save_path)
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def execute_by_condition(self, iter_objs, apply: Callable, count_batch: int):
|
||||
def execute_by_condition(self,
|
||||
iter_objs: DownloadIterObjs,
|
||||
apply: Callable,
|
||||
count_batch: int,
|
||||
):
|
||||
"""
|
||||
章节/图片的下载调度逻辑
|
||||
调度本子/章节的下载
|
||||
"""
|
||||
iter_objs = self.filter_iter_objs(iter_objs)
|
||||
count_real = len(iter_objs)
|
||||
|
||||
if count_batch >= count_real:
|
||||
@ -120,6 +133,19 @@ class JmDownloader(DownloadCallback):
|
||||
max_workers=count_batch,
|
||||
)
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def filter_iter_objs(self, iter_objs: DownloadIterObjs):
|
||||
"""
|
||||
该方法可用于过滤本子/章节,默认不会做过滤。
|
||||
例如:
|
||||
只想下载 本子的最新一章,返回 [album[-1]]
|
||||
只想下载 章节的前10张图片,返回 [photo[:10]]
|
||||
|
||||
@param iter_objs: 可能是本子或者章节,需要自行使用 isinstance 判断
|
||||
@return: 只想要下载的 本子的章节 或 章节的图片
|
||||
"""
|
||||
return iter_objs
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
def client_for_album(self, jm_album_id) -> JmcomicClient:
|
||||
"""
|
||||
|
||||
@ -21,7 +21,7 @@ class JmBaseEntity:
|
||||
PackerUtil.pack(self, filepath)
|
||||
|
||||
|
||||
class DetailEntity(JmBaseEntity, IterableEntity):
|
||||
class DetailEntity(JmBaseEntity):
|
||||
|
||||
@property
|
||||
def id(self) -> str:
|
||||
@ -31,19 +31,42 @@ class DetailEntity(JmBaseEntity, IterableEntity):
|
||||
def name(self) -> str:
|
||||
return getattr(self, 'title')
|
||||
|
||||
# help for typing
|
||||
JMPI = Union['JmPhotoDetail', 'JmImageDetail']
|
||||
|
||||
def getindex(self, index: int) -> JMPI:
|
||||
raise NotImplementedError
|
||||
|
||||
def __len__(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def __iter__(self) -> Generator[JMPI, Any, None]:
|
||||
for index in range(len(self)):
|
||||
yield self.getindex(index)
|
||||
|
||||
def __str__(self):
|
||||
return f'{self.__class__.__name__}({self.id}-{self.name})'
|
||||
|
||||
def __getitem__(self, item) -> Union[JMPI, List[JMPI]]:
|
||||
if isinstance(item, slice):
|
||||
start = item.start or 0
|
||||
stop = item.stop or len(self)
|
||||
step = item.step or 1
|
||||
return [self.getindex(index) for index in range(start, stop, step)]
|
||||
|
||||
elif isinstance(item, int):
|
||||
return self.getindex(item)
|
||||
|
||||
else:
|
||||
raise TypeError(f"Invalid item type for {self.__class__}")
|
||||
|
||||
@classmethod
|
||||
def __jm_type__(cls):
|
||||
def __alias__(cls):
|
||||
# "JmAlbumDetail" -> "album" (本子)
|
||||
# "JmPhotoDetail" -> "photo" (章节)
|
||||
cls_name = cls.__name__
|
||||
return cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")].lower()
|
||||
|
||||
def __getitem__(self, item) -> Union['JmAlbumDetail', 'JmPhotoDetail']:
|
||||
raise NotImplementedError
|
||||
|
||||
def __str__(self):
|
||||
return f'{self.__class__.__name__}({self.id}-{self.name})'
|
||||
|
||||
|
||||
class JmImageDetail(JmBaseEntity):
|
||||
|
||||
@ -254,13 +277,16 @@ class JmPhotoDetail(DetailEntity):
|
||||
|
||||
return data_original_0[index + 1:]
|
||||
|
||||
def __getitem__(self, item) -> JmImageDetail:
|
||||
return self.create_image_detail(item)
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self.photo_id
|
||||
|
||||
def getindex(self, index) -> JmImageDetail:
|
||||
return self.create_image_detail(index)
|
||||
|
||||
def __getitem__(self, item) -> Union[JmImageDetail, List[JmImageDetail]]:
|
||||
return super().__getitem__(item)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.page_arr)
|
||||
|
||||
@ -337,12 +363,6 @@ class JmAlbumDetail(DetailEntity):
|
||||
def id(self):
|
||||
return self.album_id
|
||||
|
||||
def __len__(self):
|
||||
return len(self.episode_list)
|
||||
|
||||
def __getitem__(self, item) -> JmPhotoDetail:
|
||||
return self.create_photo_detail(item)[0]
|
||||
|
||||
@staticmethod
|
||||
def distinct_episode(episode_list):
|
||||
ret = []
|
||||
@ -360,6 +380,15 @@ class JmAlbumDetail(DetailEntity):
|
||||
|
||||
return ret
|
||||
|
||||
def getindex(self, item) -> JmPhotoDetail:
|
||||
return self.create_photo_detail(item)[0]
|
||||
|
||||
def __getitem__(self, item) -> Union[JmPhotoDetail, List[JmPhotoDetail]]:
|
||||
return super().__getitem__(item)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.episode_list)
|
||||
|
||||
def __iter__(self) -> Generator[JmPhotoDetail, Any, None]:
|
||||
return super().__iter__()
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
from .jm_entity import *
|
||||
from PIL import Image
|
||||
|
||||
from .jm_entity import *
|
||||
|
||||
|
||||
class JmcomicText:
|
||||
pattern_jm_domain = compile('https://([\w.-]+)')
|
||||
|
||||
@ -41,11 +41,11 @@ class Test_Api(JmTestConfigurable):
|
||||
ret1 = jmcomic.download_album(case, self.option)
|
||||
self.assertEqual(len(ret1), len(album_ls), str(case))
|
||||
|
||||
ret2 = jmcomic.download_album_batch(case, self.option)
|
||||
ret2 = jmcomic.download_album(case, self.option)
|
||||
self.assertEqual(len(ret2), len(album_ls), str(case))
|
||||
|
||||
# 测试 Generator
|
||||
ret2 = jmcomic.download_album_batch((e for e in album_ls), self.option)
|
||||
ret2 = jmcomic.download_album((e for e in album_ls), self.option)
|
||||
self.assertEqual(len(ret2), len(album_ls), 'Generator')
|
||||
|
||||
def test_photo_sort(self):
|
||||
@ -130,3 +130,42 @@ class Test_Api(JmTestConfigurable):
|
||||
print(e)
|
||||
|
||||
raise AssertionError(exception_list)
|
||||
|
||||
def test_getitem_and_slice(self):
|
||||
cl: JmcomicClient = self.client
|
||||
cases = [
|
||||
['400222', 0, [400222]],
|
||||
['400222', 1, [413446]],
|
||||
['400222', (None, 1), [400222]],
|
||||
['400222', (1, 3), [413446, 413447]],
|
||||
['413447', (1, 3), [2, 3], []],
|
||||
]
|
||||
|
||||
for [jmid, slicearg, *args] in cases:
|
||||
ans = args[0]
|
||||
|
||||
if len(args) == 1:
|
||||
func = cl.get_album_detail
|
||||
else:
|
||||
func = cl.get_photo_detail
|
||||
|
||||
jmentity = func(jmid)
|
||||
|
||||
ls: List[Union[JmPhotoDetail, JmImageDetail]]
|
||||
if isinstance(slicearg, int):
|
||||
ls = [jmentity[slicearg]]
|
||||
elif len(slicearg) == 2:
|
||||
ls = jmentity[slicearg[0]: slicearg[1]]
|
||||
else:
|
||||
ls = jmentity[slicearg[0]: slicearg[1]: slicearg[2]]
|
||||
|
||||
if len(args) == 1:
|
||||
self.assertListEqual(
|
||||
list1=[int(e.id) for e in ls],
|
||||
list2=ans,
|
||||
)
|
||||
else:
|
||||
self.assertListEqual(
|
||||
list1=[int(e.img_file_name) for e in ls],
|
||||
list2=ans,
|
||||
)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user