mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v2.1.12: 支持实体类的切片语法,JmDownloader增加下载前Filter过滤机制 (#96)
This commit is contained in:
parent
476efc66d3
commit
fc5194015f
@ -2,6 +2,6 @@
|
|||||||
# 被依赖方 <--- 使用方
|
# 被依赖方 <--- 使用方
|
||||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||||
|
|
||||||
__version__ = '2.1.11'
|
__version__ = '2.1.12'
|
||||||
|
|
||||||
from .api import *
|
from .api import *
|
||||||
|
|||||||
@ -1,24 +1,29 @@
|
|||||||
from .jm_downloader import *
|
from .jm_downloader import *
|
||||||
|
|
||||||
|
|
||||||
def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
|
def download_batch(download_api,
|
||||||
option=None,
|
jm_id_iter: Union[Iterable, Generator],
|
||||||
):
|
option=None,
|
||||||
|
):
|
||||||
"""
|
"""
|
||||||
批量下载album.
|
批量下载 album / photo
|
||||||
一个album,对应一个线程,对应一个option
|
一个album/photo,对应一个线程,对应一个option
|
||||||
|
|
||||||
@param jm_album_id_iter: album_id的迭代器
|
@param download_api: 下载api
|
||||||
@param option: 下载选项,为空默认是 JmOption.default()
|
@param jm_id_iter: jmid (album_id, photo_id) 的迭代器
|
||||||
|
@param option: 下载选项,对所有的jmid使用同一个,默认是 JmOption.default()
|
||||||
"""
|
"""
|
||||||
from common import multi_thread_launcher
|
from common import multi_thread_launcher
|
||||||
|
|
||||||
|
if option is None:
|
||||||
|
option = JmOption.default()
|
||||||
|
|
||||||
return multi_thread_launcher(
|
return multi_thread_launcher(
|
||||||
iter_objs=set(
|
iter_objs=set(
|
||||||
JmcomicText.parse_to_album_id(album_id)
|
JmcomicText.parse_to_album_id(jmid)
|
||||||
for album_id in jm_album_id_iter
|
for jmid in jm_id_iter
|
||||||
),
|
),
|
||||||
apply_each_obj_func=lambda aid: download_album(aid, option),
|
apply_each_obj_func=lambda aid: download_api(aid, option),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -31,7 +36,7 @@ def download_album(jm_album_id, option=None):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
if not isinstance(jm_album_id, (str, int)):
|
if not isinstance(jm_album_id, (str, int)):
|
||||||
return download_album_batch(jm_album_id, option)
|
return download_batch(download_album, jm_album_id, option)
|
||||||
|
|
||||||
with new_downloader(option) as dler:
|
with new_downloader(option) as dler:
|
||||||
dler.download_album(jm_album_id)
|
dler.download_album(jm_album_id)
|
||||||
@ -41,6 +46,9 @@ def download_photo(jm_photo_id, option=None):
|
|||||||
"""
|
"""
|
||||||
下载一个章节
|
下载一个章节
|
||||||
"""
|
"""
|
||||||
|
if not isinstance(jm_photo_id, (str, int)):
|
||||||
|
return download_batch(download_photo, jm_photo_id, option)
|
||||||
|
|
||||||
with new_downloader(option) as dler:
|
with new_downloader(option) as dler:
|
||||||
dler.download_photo(jm_photo_id)
|
dler.download_photo(jm_photo_id)
|
||||||
|
|
||||||
|
|||||||
@ -1,5 +1,13 @@
|
|||||||
from .jm_option import *
|
from .jm_option import *
|
||||||
|
|
||||||
|
# help for typing
|
||||||
|
DownloadIterObjs = Union[
|
||||||
|
JmAlbumDetail,
|
||||||
|
Sequence[JmPhotoDetail],
|
||||||
|
JmPhotoDetail,
|
||||||
|
Sequence[JmImageDetail],
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
class JmDownloadException(Exception):
|
class JmDownloadException(Exception):
|
||||||
pass
|
pass
|
||||||
@ -100,10 +108,15 @@ class JmDownloader(DownloadCallback):
|
|||||||
self.after_image(image, img_save_path)
|
self.after_image(image, img_save_path)
|
||||||
|
|
||||||
# noinspection PyMethodMayBeStatic
|
# noinspection PyMethodMayBeStatic
|
||||||
def execute_by_condition(self, iter_objs, apply: Callable, count_batch: int):
|
def execute_by_condition(self,
|
||||||
|
iter_objs: DownloadIterObjs,
|
||||||
|
apply: Callable,
|
||||||
|
count_batch: int,
|
||||||
|
):
|
||||||
"""
|
"""
|
||||||
章节/图片的下载调度逻辑
|
调度本子/章节的下载
|
||||||
"""
|
"""
|
||||||
|
iter_objs = self.filter_iter_objs(iter_objs)
|
||||||
count_real = len(iter_objs)
|
count_real = len(iter_objs)
|
||||||
|
|
||||||
if count_batch >= count_real:
|
if count_batch >= count_real:
|
||||||
@ -120,6 +133,19 @@ class JmDownloader(DownloadCallback):
|
|||||||
max_workers=count_batch,
|
max_workers=count_batch,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# noinspection PyMethodMayBeStatic
|
||||||
|
def filter_iter_objs(self, iter_objs: DownloadIterObjs):
|
||||||
|
"""
|
||||||
|
该方法可用于过滤本子/章节,默认不会做过滤。
|
||||||
|
例如:
|
||||||
|
只想下载 本子的最新一章,返回 [album[-1]]
|
||||||
|
只想下载 章节的前10张图片,返回 [photo[:10]]
|
||||||
|
|
||||||
|
@param iter_objs: 可能是本子或者章节,需要自行使用 isinstance 判断
|
||||||
|
@return: 只想要下载的 本子的章节 或 章节的图片
|
||||||
|
"""
|
||||||
|
return iter_objs
|
||||||
|
|
||||||
# noinspection PyUnusedLocal
|
# noinspection PyUnusedLocal
|
||||||
def client_for_album(self, jm_album_id) -> JmcomicClient:
|
def client_for_album(self, jm_album_id) -> JmcomicClient:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@ -21,7 +21,7 @@ class JmBaseEntity:
|
|||||||
PackerUtil.pack(self, filepath)
|
PackerUtil.pack(self, filepath)
|
||||||
|
|
||||||
|
|
||||||
class DetailEntity(JmBaseEntity, IterableEntity):
|
class DetailEntity(JmBaseEntity):
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def id(self) -> str:
|
def id(self) -> str:
|
||||||
@ -31,19 +31,42 @@ class DetailEntity(JmBaseEntity, IterableEntity):
|
|||||||
def name(self) -> str:
|
def name(self) -> str:
|
||||||
return getattr(self, 'title')
|
return getattr(self, 'title')
|
||||||
|
|
||||||
|
# help for typing
|
||||||
|
JMPI = Union['JmPhotoDetail', 'JmImageDetail']
|
||||||
|
|
||||||
|
def getindex(self, index: int) -> JMPI:
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def __iter__(self) -> Generator[JMPI, Any, None]:
|
||||||
|
for index in range(len(self)):
|
||||||
|
yield self.getindex(index)
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return f'{self.__class__.__name__}({self.id}-{self.name})'
|
||||||
|
|
||||||
|
def __getitem__(self, item) -> Union[JMPI, List[JMPI]]:
|
||||||
|
if isinstance(item, slice):
|
||||||
|
start = item.start or 0
|
||||||
|
stop = item.stop or len(self)
|
||||||
|
step = item.step or 1
|
||||||
|
return [self.getindex(index) for index in range(start, stop, step)]
|
||||||
|
|
||||||
|
elif isinstance(item, int):
|
||||||
|
return self.getindex(item)
|
||||||
|
|
||||||
|
else:
|
||||||
|
raise TypeError(f"Invalid item type for {self.__class__}")
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def __jm_type__(cls):
|
def __alias__(cls):
|
||||||
# "JmAlbumDetail" -> "album" (本子)
|
# "JmAlbumDetail" -> "album" (本子)
|
||||||
# "JmPhotoDetail" -> "photo" (章节)
|
# "JmPhotoDetail" -> "photo" (章节)
|
||||||
cls_name = cls.__name__
|
cls_name = cls.__name__
|
||||||
return cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")].lower()
|
return cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")].lower()
|
||||||
|
|
||||||
def __getitem__(self, item) -> Union['JmAlbumDetail', 'JmPhotoDetail']:
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
def __str__(self):
|
|
||||||
return f'{self.__class__.__name__}({self.id}-{self.name})'
|
|
||||||
|
|
||||||
|
|
||||||
class JmImageDetail(JmBaseEntity):
|
class JmImageDetail(JmBaseEntity):
|
||||||
|
|
||||||
@ -254,13 +277,16 @@ class JmPhotoDetail(DetailEntity):
|
|||||||
|
|
||||||
return data_original_0[index + 1:]
|
return data_original_0[index + 1:]
|
||||||
|
|
||||||
def __getitem__(self, item) -> JmImageDetail:
|
|
||||||
return self.create_image_detail(item)
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def id(self):
|
def id(self):
|
||||||
return self.photo_id
|
return self.photo_id
|
||||||
|
|
||||||
|
def getindex(self, index) -> JmImageDetail:
|
||||||
|
return self.create_image_detail(index)
|
||||||
|
|
||||||
|
def __getitem__(self, item) -> Union[JmImageDetail, List[JmImageDetail]]:
|
||||||
|
return super().__getitem__(item)
|
||||||
|
|
||||||
def __len__(self):
|
def __len__(self):
|
||||||
return len(self.page_arr)
|
return len(self.page_arr)
|
||||||
|
|
||||||
@ -337,12 +363,6 @@ class JmAlbumDetail(DetailEntity):
|
|||||||
def id(self):
|
def id(self):
|
||||||
return self.album_id
|
return self.album_id
|
||||||
|
|
||||||
def __len__(self):
|
|
||||||
return len(self.episode_list)
|
|
||||||
|
|
||||||
def __getitem__(self, item) -> JmPhotoDetail:
|
|
||||||
return self.create_photo_detail(item)[0]
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def distinct_episode(episode_list):
|
def distinct_episode(episode_list):
|
||||||
ret = []
|
ret = []
|
||||||
@ -360,6 +380,15 @@ class JmAlbumDetail(DetailEntity):
|
|||||||
|
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
|
def getindex(self, item) -> JmPhotoDetail:
|
||||||
|
return self.create_photo_detail(item)[0]
|
||||||
|
|
||||||
|
def __getitem__(self, item) -> Union[JmPhotoDetail, List[JmPhotoDetail]]:
|
||||||
|
return super().__getitem__(item)
|
||||||
|
|
||||||
|
def __len__(self):
|
||||||
|
return len(self.episode_list)
|
||||||
|
|
||||||
def __iter__(self) -> Generator[JmPhotoDetail, Any, None]:
|
def __iter__(self) -> Generator[JmPhotoDetail, Any, None]:
|
||||||
return super().__iter__()
|
return super().__iter__()
|
||||||
|
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
from .jm_entity import *
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
|
from .jm_entity import *
|
||||||
|
|
||||||
|
|
||||||
class JmcomicText:
|
class JmcomicText:
|
||||||
pattern_jm_domain = compile('https://([\w.-]+)')
|
pattern_jm_domain = compile('https://([\w.-]+)')
|
||||||
|
|||||||
@ -41,11 +41,11 @@ class Test_Api(JmTestConfigurable):
|
|||||||
ret1 = jmcomic.download_album(case, self.option)
|
ret1 = jmcomic.download_album(case, self.option)
|
||||||
self.assertEqual(len(ret1), len(album_ls), str(case))
|
self.assertEqual(len(ret1), len(album_ls), str(case))
|
||||||
|
|
||||||
ret2 = jmcomic.download_album_batch(case, self.option)
|
ret2 = jmcomic.download_album(case, self.option)
|
||||||
self.assertEqual(len(ret2), len(album_ls), str(case))
|
self.assertEqual(len(ret2), len(album_ls), str(case))
|
||||||
|
|
||||||
# 测试 Generator
|
# 测试 Generator
|
||||||
ret2 = jmcomic.download_album_batch((e for e in album_ls), self.option)
|
ret2 = jmcomic.download_album((e for e in album_ls), self.option)
|
||||||
self.assertEqual(len(ret2), len(album_ls), 'Generator')
|
self.assertEqual(len(ret2), len(album_ls), 'Generator')
|
||||||
|
|
||||||
def test_photo_sort(self):
|
def test_photo_sort(self):
|
||||||
@ -130,3 +130,42 @@ class Test_Api(JmTestConfigurable):
|
|||||||
print(e)
|
print(e)
|
||||||
|
|
||||||
raise AssertionError(exception_list)
|
raise AssertionError(exception_list)
|
||||||
|
|
||||||
|
def test_getitem_and_slice(self):
|
||||||
|
cl: JmcomicClient = self.client
|
||||||
|
cases = [
|
||||||
|
['400222', 0, [400222]],
|
||||||
|
['400222', 1, [413446]],
|
||||||
|
['400222', (None, 1), [400222]],
|
||||||
|
['400222', (1, 3), [413446, 413447]],
|
||||||
|
['413447', (1, 3), [2, 3], []],
|
||||||
|
]
|
||||||
|
|
||||||
|
for [jmid, slicearg, *args] in cases:
|
||||||
|
ans = args[0]
|
||||||
|
|
||||||
|
if len(args) == 1:
|
||||||
|
func = cl.get_album_detail
|
||||||
|
else:
|
||||||
|
func = cl.get_photo_detail
|
||||||
|
|
||||||
|
jmentity = func(jmid)
|
||||||
|
|
||||||
|
ls: List[Union[JmPhotoDetail, JmImageDetail]]
|
||||||
|
if isinstance(slicearg, int):
|
||||||
|
ls = [jmentity[slicearg]]
|
||||||
|
elif len(slicearg) == 2:
|
||||||
|
ls = jmentity[slicearg[0]: slicearg[1]]
|
||||||
|
else:
|
||||||
|
ls = jmentity[slicearg[0]: slicearg[1]: slicearg[2]]
|
||||||
|
|
||||||
|
if len(args) == 1:
|
||||||
|
self.assertListEqual(
|
||||||
|
list1=[int(e.id) for e in ls],
|
||||||
|
list2=ans,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.assertListEqual(
|
||||||
|
list1=[int(e.img_file_name) for e in ls],
|
||||||
|
list2=ans,
|
||||||
|
)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user