mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.1.7: 修复永久网域的重定向bug,重构api提高可扩展性 (#85)
This commit is contained in:
parent
5e013e1818
commit
929dd46552
@ -1,7 +1,9 @@
|
||||
# 模块依赖关系如下:
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.1.6'
|
||||
__version__ = '2.1.7'
|
||||
|
||||
# noinspection PyUnresolvedReferences
|
||||
from common import *
|
||||
from .api import *
|
||||
|
@ -1,129 +1,56 @@
|
||||
from .jm_option import *
|
||||
from .jm_downloader import *
|
||||
|
||||
|
||||
def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
|
||||
option=None,
|
||||
):
|
||||
"""
|
||||
批量下载album.
|
||||
一个album,对应一个线程,对应一个option
|
||||
|
||||
@param jm_album_id_iter: album_id的迭代器
|
||||
@param option: 下载选项,为空默认是 JmOption.default()
|
||||
"""
|
||||
from common import multi_thread_launcher
|
||||
|
||||
return multi_thread_launcher(
|
||||
iter_objs=set(
|
||||
JmcomicText.parse_to_album_id(album_id)
|
||||
for album_id in jm_album_id_iter
|
||||
),
|
||||
apply_each_obj_func=lambda aid: download_album(aid, option),
|
||||
)
|
||||
|
||||
|
||||
def download_album(jm_album_id, option=None):
|
||||
"""
|
||||
下载一个本子集,入口api
|
||||
下载一个本子
|
||||
@param jm_album_id: 禁漫的本子的id,类型可以是str/int/iterable[str]。
|
||||
如果是iterable[str],则会调用批量下载方法 download_album_batch
|
||||
如果是iterable[str],则会调用 download_album_batch
|
||||
@param option: 下载选项,为空默认是 JmOption.default()
|
||||
"""
|
||||
|
||||
if not isinstance(jm_album_id, (str, int)):
|
||||
return download_album_batch(jm_album_id, option)
|
||||
|
||||
option, jm_client = build_client(option)
|
||||
album: JmAlbumDetail = jm_client.get_album_detail(jm_album_id)
|
||||
|
||||
option.before_album(album)
|
||||
execute_by_condition(
|
||||
iter_objs=album,
|
||||
apply=lambda photo: download_by_photo_detail(photo, option),
|
||||
count_batch=option.decide_photo_batch_count(album)
|
||||
)
|
||||
option.after_album(album)
|
||||
with new_downloader(option) as dler:
|
||||
dler.download_album(jm_album_id)
|
||||
|
||||
|
||||
def download_photo(jm_photo_id, option=None):
|
||||
"""
|
||||
下载一个本子的一章,入口api
|
||||
下载一个章节
|
||||
"""
|
||||
option, jm_client = build_client(option)
|
||||
photo = jm_client.get_photo_detail(jm_photo_id)
|
||||
download_by_photo_detail(photo, option)
|
||||
with new_downloader(option) as dler:
|
||||
dler.download_photo(jm_photo_id)
|
||||
|
||||
|
||||
def download_by_photo_detail(photo: JmPhotoDetail, option=None):
|
||||
"""
|
||||
下载一个本子的一章,根据 photo
|
||||
@param photo: 本子章节信息
|
||||
@param option: 选项
|
||||
"""
|
||||
option, jm_client = build_client(option)
|
||||
|
||||
# 下载准备
|
||||
use_cache = option.download_cache
|
||||
decode_image = option.download_image_decode
|
||||
jm_client.check_photo(photo)
|
||||
|
||||
# 下载每个图片的函数
|
||||
def download_image(image: JmImageDetail):
|
||||
img_save_path = option.decide_image_filepath(image)
|
||||
image.is_exists = file_exists(img_save_path)
|
||||
|
||||
option.before_image(image, img_save_path)
|
||||
if use_cache is True and image.is_exists:
|
||||
return
|
||||
jm_client.download_by_image_detail(
|
||||
image,
|
||||
img_save_path,
|
||||
decode_image=decode_image,
|
||||
)
|
||||
option.after_image(image, img_save_path)
|
||||
|
||||
option.before_photo(photo)
|
||||
execute_by_condition(
|
||||
iter_objs=photo,
|
||||
apply=download_image,
|
||||
count_batch=option.decide_image_batch_count(photo)
|
||||
)
|
||||
option.before_photo(photo)
|
||||
|
||||
|
||||
def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
|
||||
option=None,
|
||||
wait_finish=True,
|
||||
) -> List[Thread]:
|
||||
"""
|
||||
批量下载album,每个album一个线程,使用的是同一个option。
|
||||
|
||||
@param jm_album_id_iter: album_id的可迭代对象
|
||||
@param option: 下载选项,为空默认是 JmOption.default()
|
||||
@param wait_finish: 是否要等待这些下载线程全部完成
|
||||
@return 返回值是List[Thread],里面是每个下载漫画的线程。
|
||||
"""
|
||||
def new_downloader(option=None):
|
||||
if option is None:
|
||||
option = JmOption.default()
|
||||
option = JmModuleConfig.option_class().default()
|
||||
|
||||
return thread_pool_executor(
|
||||
iter_objs=set(JmcomicText.parse_to_album_id(album_id) for album_id in jm_album_id_iter),
|
||||
apply_each_obj_func=lambda album_id: download_album(album_id, option),
|
||||
wait_finish=wait_finish,
|
||||
)
|
||||
return JmModuleConfig.downloader_class()(option)
|
||||
|
||||
|
||||
def execute_by_condition(iter_objs, apply: Callable, count_batch: int):
|
||||
"""
|
||||
章节/图片的下载调度逻辑
|
||||
"""
|
||||
count_real = len(iter_objs)
|
||||
|
||||
if count_batch >= count_real:
|
||||
# 一个图/章节 对应 一个线程
|
||||
multi_thread_launcher(
|
||||
iter_objs=iter_objs,
|
||||
apply_each_obj_func=apply,
|
||||
)
|
||||
else:
|
||||
# 创建batch个线程的线程池
|
||||
thread_pool_executor(
|
||||
iter_objs=iter_objs,
|
||||
apply_each_obj_func=apply,
|
||||
max_workers=count_batch,
|
||||
)
|
||||
|
||||
|
||||
def build_client(option: Optional[JmOption]) -> Tuple[JmOption, JmcomicClient]:
|
||||
"""
|
||||
处理option的判空,并且创建jm_client
|
||||
"""
|
||||
if option is None:
|
||||
option = JmOption.default()
|
||||
|
||||
jm_client = option.build_jm_client()
|
||||
return option, jm_client
|
||||
|
||||
|
||||
def create_option(filepath: str) -> JmOption:
|
||||
option = JmOption.from_file(filepath)
|
||||
return option
|
||||
def create_option(filepath):
|
||||
return JmModuleConfig.option_class().from_file(filepath)
|
||||
|
@ -85,7 +85,7 @@ class AbstractJmClient(
|
||||
|
||||
# noinspection PyMethodMayBeStatic, PyUnusedLocal
|
||||
def before_retry(self, e, kwargs, retry_count, url):
|
||||
jm_debug('error', str(e))
|
||||
jm_debug('retry', str(e))
|
||||
|
||||
def enable_cache(self, debug=False):
|
||||
def wrap_func_cache(func_name, cache_dict_name):
|
||||
@ -131,8 +131,11 @@ class AbstractJmClient(
|
||||
def get_jmcomic_domain_all(self, postman=None):
|
||||
return JmModuleConfig.get_jmcomic_domain_all(postman or self.get_root_postman())
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
def fallback(self, request, url, domain_index, retry_count, **kwargs):
|
||||
raise AssertionError(f"请求重试全部失败: [{url}], {self.domain_list}")
|
||||
msg = f"请求重试全部失败: [{url}], {self.domain_list}"
|
||||
jm_debug('fallback', "msg")
|
||||
raise AssertionError(msg)
|
||||
|
||||
|
||||
# 基于网页实现的JmClient
|
||||
@ -406,3 +409,15 @@ class JmApiClient(AbstractJmClient):
|
||||
"user-agent": "okhttp/3.12.1",
|
||||
"accept-encoding": "gzip",
|
||||
}, key_ts
|
||||
|
||||
|
||||
class AsyncSaveImageClient(JmImageClient):
|
||||
|
||||
def __init__(self, workers=None) -> None:
|
||||
from concurrent.futures import ThreadPoolExecutor, Future
|
||||
self.executor = ThreadPoolExecutor(max_workers=workers)
|
||||
self.future_list: List[Future] = []
|
||||
|
||||
def save_image_resp(self, *args, **kwargs):
|
||||
future = self.executor.submit(lambda: super().save_image_resp(*args, **kwargs))
|
||||
self.future_list.append(future)
|
||||
|
@ -236,8 +236,10 @@ class JmImageClient:
|
||||
|
||||
resp.require_success()
|
||||
|
||||
# gif图无需加解密,需要最先判断
|
||||
return self.save_image_resp(decode_image, img_save_path, img_url, resp, scramble_id)
|
||||
|
||||
def save_image_resp(self, decode_image, img_save_path, img_url, resp, scramble_id):
|
||||
# gif图无需加解密,需要最先判断
|
||||
if self.img_is_not_need_to_decode(img_url, resp):
|
||||
JmImageSupport.save_resp_img(resp, img_save_path, False)
|
||||
else:
|
||||
@ -248,7 +250,7 @@ class JmImageClient:
|
||||
img_save_path,
|
||||
decode_image=True,
|
||||
):
|
||||
self.download_image(
|
||||
return self.download_image(
|
||||
image.download_url,
|
||||
img_save_path,
|
||||
image.scramble_id,
|
||||
|
@ -57,6 +57,24 @@ class JmModuleConfig:
|
||||
enable_jm_debug = True
|
||||
debug_executor = default_jm_debug
|
||||
postman_constructor = default_postman_constructor
|
||||
DOWNLOADER_CLASS = None
|
||||
OPTION_CLASS = None
|
||||
|
||||
@classmethod
|
||||
def downloader_class(cls):
|
||||
if cls.DOWNLOADER_CLASS is not None:
|
||||
return cls.DOWNLOADER_CLASS
|
||||
|
||||
from .jm_downloader import JmDownloader
|
||||
return JmDownloader
|
||||
|
||||
@classmethod
|
||||
def option_class(cls):
|
||||
if cls.OPTION_CLASS is not None:
|
||||
return cls.OPTION_CLASS
|
||||
|
||||
from .jm_option import JmOption
|
||||
return JmOption
|
||||
|
||||
@classmethod
|
||||
@field_cache("DOMAIN")
|
||||
@ -115,8 +133,7 @@ class JmModuleConfig:
|
||||
"""
|
||||
postman = postman or cls.new_postman(session=True)
|
||||
|
||||
resp = postman.get(cls.JM_REDIRECT_URL)
|
||||
url = resp.url
|
||||
url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL)
|
||||
cls.jm_debug('获取禁漫地址', f'[{cls.JM_REDIRECT_URL}] → [{url}]')
|
||||
return url
|
||||
|
||||
|
148
src/jmcomic/jm_downloader.py
Normal file
148
src/jmcomic/jm_downloader.py
Normal file
@ -0,0 +1,148 @@
|
||||
from .jm_option import *
|
||||
|
||||
|
||||
class JmDownloadException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
class DownloadCallback:
|
||||
|
||||
def before_album(self, album: JmAlbumDetail):
|
||||
jm_debug('album-before',
|
||||
f'本子获取成功: [{album.id}], '
|
||||
f'作者: [{album.author}], '
|
||||
f'章节数: [{len(album)}], '
|
||||
f'标题: [{album.title}], '
|
||||
)
|
||||
|
||||
def after_album(self, album: JmAlbumDetail):
|
||||
jm_debug('album-after', f'本子下载完成: [{album.id}]')
|
||||
|
||||
def before_photo(self, photo: JmPhotoDetail):
|
||||
jm_debug('photo-before',
|
||||
f'开始下载章节: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}]), '
|
||||
f'标题: [{photo.title}], '
|
||||
f'图片数为[{len(photo)}]'
|
||||
)
|
||||
|
||||
def after_photo(self, photo: JmPhotoDetail):
|
||||
jm_debug('photo-after',
|
||||
f'章节下载完成: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}])')
|
||||
|
||||
def before_image(self, image: JmImageDetail, img_save_path):
|
||||
if image.is_exists:
|
||||
jm_debug('image-before',
|
||||
f'图片已存在: {image.tag} ← [{img_save_path}]'
|
||||
)
|
||||
else:
|
||||
jm_debug('image-before',
|
||||
f'图片准备下载: {image.tag}, [{image.img_url}] → [{img_save_path}]'
|
||||
)
|
||||
|
||||
def after_image(self, image: JmImageDetail, img_save_path):
|
||||
jm_debug('image-after',
|
||||
f'图片下载完成: {image.tag}, [{image.img_url}] → [{img_save_path}]')
|
||||
|
||||
|
||||
class JmDownloader(DownloadCallback):
|
||||
"""
|
||||
JmDownloader = JmOption + 调度逻辑
|
||||
"""
|
||||
|
||||
def __init__(self, option) -> None:
|
||||
self.option = option
|
||||
self.use_cache = self.option.download_cache
|
||||
self.decode_image = self.option.download_image_decode
|
||||
|
||||
def download_album(self, album_id):
|
||||
client = self.client_for_album(album_id)
|
||||
album = client.get_album_detail(album_id)
|
||||
|
||||
self.before_album(album)
|
||||
self.download_by_album_detail(album, client)
|
||||
self.after_album(album)
|
||||
|
||||
def download_by_album_detail(self, album: JmAlbumDetail, client: JmcomicClient):
|
||||
self.execute_by_condition(
|
||||
iter_objs=album,
|
||||
apply=lambda photo: self.download_by_photo_detail(photo, client),
|
||||
count_batch=self.option.decide_photo_batch_count(album)
|
||||
)
|
||||
|
||||
def download_photo(self, photo_id):
|
||||
client = self.client_for_photo(photo_id)
|
||||
photo = client.get_photo_detail(photo_id)
|
||||
|
||||
self.before_photo(photo)
|
||||
self.download_by_photo_detail(photo, client)
|
||||
self.after_photo(photo)
|
||||
|
||||
def download_by_photo_detail(self, photo: JmPhotoDetail, client: JmcomicClient):
|
||||
client.check_photo(photo)
|
||||
|
||||
self.execute_by_condition(
|
||||
iter_objs=photo,
|
||||
apply=lambda image: self.download_by_image_detail(image, client),
|
||||
count_batch=self.option.decide_image_batch_count(photo)
|
||||
)
|
||||
|
||||
def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
|
||||
img_save_path = self.option.decide_image_filepath(image)
|
||||
image.is_exists = file_exists(img_save_path)
|
||||
|
||||
self.before_image(image, img_save_path)
|
||||
if self.use_cache is True and image.is_exists:
|
||||
return
|
||||
client.download_by_image_detail(
|
||||
image,
|
||||
img_save_path,
|
||||
decode_image=self.decode_image,
|
||||
)
|
||||
self.after_image(image, img_save_path)
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def execute_by_condition(self, iter_objs, apply: Callable, count_batch: int):
|
||||
"""
|
||||
章节/图片的下载调度逻辑
|
||||
"""
|
||||
count_real = len(iter_objs)
|
||||
|
||||
if count_batch >= count_real:
|
||||
# 一个图/章节 对应 一个线程
|
||||
multi_thread_launcher(
|
||||
iter_objs=iter_objs,
|
||||
apply_each_obj_func=apply,
|
||||
)
|
||||
else:
|
||||
# 创建batch个线程的线程池
|
||||
thread_pool_executor(
|
||||
iter_objs=iter_objs,
|
||||
apply_each_obj_func=apply,
|
||||
max_workers=count_batch,
|
||||
)
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
def client_for_album(self, jm_album_id):
|
||||
"""
|
||||
默认情况下,每次调用JmDownloader的download_album或download_photo,
|
||||
都会使用一个新的 JmcomicClient
|
||||
"""
|
||||
return self.option.new_jm_client()
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
def client_for_photo(self, jm_photo_id):
|
||||
"""
|
||||
默认情况下,每次调用JmDownloader的download_album或download_photo,
|
||||
都会使用一个新的 JmcomicClient
|
||||
"""
|
||||
return self.option.new_jm_client()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if exc_type is not None:
|
||||
jm_debug('exception',
|
||||
f'{self.__class__.__name__} Exit with exception: {exc_type, exc_val}'
|
||||
)
|
@ -1,46 +1,6 @@
|
||||
from .jm_client_impl import *
|
||||
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
class DownloadCallback:
|
||||
|
||||
def before_album(self, album: JmAlbumDetail):
|
||||
jm_debug('album-before',
|
||||
f'本子获取成功: [{album.id}], '
|
||||
f'作者: [{album.author}], '
|
||||
f'章节数: [{len(album)}], '
|
||||
f'标题: [{album.title}], '
|
||||
)
|
||||
|
||||
def after_album(self, album: JmAlbumDetail):
|
||||
jm_debug('album-after', f'本子下载完成: [{album.id}]')
|
||||
|
||||
def before_photo(self, photo: JmPhotoDetail):
|
||||
jm_debug('photo-before',
|
||||
f'开始下载章节: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}]), '
|
||||
f'标题: [{photo.title}], '
|
||||
f'图片数为[{len(photo)}]'
|
||||
)
|
||||
|
||||
def after_photo(self, photo: JmPhotoDetail):
|
||||
jm_debug('photo-after',
|
||||
f'章节下载完成: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}])')
|
||||
|
||||
def before_image(self, image: JmImageDetail, img_save_path):
|
||||
if image.is_exists:
|
||||
jm_debug('image-before',
|
||||
f'图片已存在: {image.tag} ← [{img_save_path}]'
|
||||
)
|
||||
else:
|
||||
jm_debug('image-before',
|
||||
f'图片准备下载: {image.tag}, [{image.img_url}] → [{img_save_path}]'
|
||||
)
|
||||
|
||||
def after_image(self, image: JmImageDetail, img_save_path):
|
||||
jm_debug('image-after',
|
||||
f'图片下载完成: {image.tag}, [{image.img_url}] → [{img_save_path}]')
|
||||
|
||||
|
||||
class DirRule:
|
||||
rule_sample = [
|
||||
# 根目录 / Album-id / Photo-序号 /
|
||||
@ -139,7 +99,7 @@ class DirRule:
|
||||
return base_dir
|
||||
|
||||
|
||||
class JmOption(DownloadCallback):
|
||||
class JmOption:
|
||||
JM_OP_VER = '2.0'
|
||||
|
||||
def __init__(self,
|
||||
|
Loading…
Reference in New Issue
Block a user