v2.1.7: 修复永久网域的重定向bug,重构api提高可扩展性 (#85)

This commit is contained in:
hect0x7 2023-08-03 17:01:16 +08:00 committed by GitHub
parent 5e013e1818
commit 929dd46552
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 227 additions and 156 deletions

View File

@ -1,7 +1,9 @@
# 模块依赖关系如下:
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.1.6'
__version__ = '2.1.7'
# noinspection PyUnresolvedReferences
from common import *
from .api import *

View File

@ -1,129 +1,56 @@
from .jm_option import *
from .jm_downloader import *
def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
option=None,
):
"""
批量下载album.
一个album对应一个线程对应一个option
@param jm_album_id_iter: album_id的迭代器
@param option: 下载选项为空默认是 JmOption.default()
"""
from common import multi_thread_launcher
return multi_thread_launcher(
iter_objs=set(
JmcomicText.parse_to_album_id(album_id)
for album_id in jm_album_id_iter
),
apply_each_obj_func=lambda aid: download_album(aid, option),
)
def download_album(jm_album_id, option=None):
"""
下载一个本子集入口api
下载一个本子
@param jm_album_id: 禁漫的本子的id类型可以是str/int/iterable[str]
如果是iterable[str]则会调用批量下载方法 download_album_batch
如果是iterable[str]则会调用 download_album_batch
@param option: 下载选项为空默认是 JmOption.default()
"""
if not isinstance(jm_album_id, (str, int)):
return download_album_batch(jm_album_id, option)
option, jm_client = build_client(option)
album: JmAlbumDetail = jm_client.get_album_detail(jm_album_id)
option.before_album(album)
execute_by_condition(
iter_objs=album,
apply=lambda photo: download_by_photo_detail(photo, option),
count_batch=option.decide_photo_batch_count(album)
)
option.after_album(album)
with new_downloader(option) as dler:
dler.download_album(jm_album_id)
def download_photo(jm_photo_id, option=None):
"""
下载一个本子的一章入口api
下载一个章节
"""
option, jm_client = build_client(option)
photo = jm_client.get_photo_detail(jm_photo_id)
download_by_photo_detail(photo, option)
with new_downloader(option) as dler:
dler.download_photo(jm_photo_id)
def download_by_photo_detail(photo: JmPhotoDetail, option=None):
"""
下载一个本子的一章根据 photo
@param photo: 本子章节信息
@param option: 选项
"""
option, jm_client = build_client(option)
# 下载准备
use_cache = option.download_cache
decode_image = option.download_image_decode
jm_client.check_photo(photo)
# 下载每个图片的函数
def download_image(image: JmImageDetail):
img_save_path = option.decide_image_filepath(image)
image.is_exists = file_exists(img_save_path)
option.before_image(image, img_save_path)
if use_cache is True and image.is_exists:
return
jm_client.download_by_image_detail(
image,
img_save_path,
decode_image=decode_image,
)
option.after_image(image, img_save_path)
option.before_photo(photo)
execute_by_condition(
iter_objs=photo,
apply=download_image,
count_batch=option.decide_image_batch_count(photo)
)
option.before_photo(photo)
def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
option=None,
wait_finish=True,
) -> List[Thread]:
"""
批量下载album每个album一个线程使用的是同一个option
@param jm_album_id_iter: album_id的可迭代对象
@param option: 下载选项为空默认是 JmOption.default()
@param wait_finish: 是否要等待这些下载线程全部完成
@return 返回值是List[Thread]里面是每个下载漫画的线程
"""
def new_downloader(option=None):
if option is None:
option = JmOption.default()
option = JmModuleConfig.option_class().default()
return thread_pool_executor(
iter_objs=set(JmcomicText.parse_to_album_id(album_id) for album_id in jm_album_id_iter),
apply_each_obj_func=lambda album_id: download_album(album_id, option),
wait_finish=wait_finish,
)
return JmModuleConfig.downloader_class()(option)
def execute_by_condition(iter_objs, apply: Callable, count_batch: int):
"""
章节/图片的下载调度逻辑
"""
count_real = len(iter_objs)
if count_batch >= count_real:
# 一个图/章节 对应 一个线程
multi_thread_launcher(
iter_objs=iter_objs,
apply_each_obj_func=apply,
)
else:
# 创建batch个线程的线程池
thread_pool_executor(
iter_objs=iter_objs,
apply_each_obj_func=apply,
max_workers=count_batch,
)
def build_client(option: Optional[JmOption]) -> Tuple[JmOption, JmcomicClient]:
"""
处理option的判空并且创建jm_client
"""
if option is None:
option = JmOption.default()
jm_client = option.build_jm_client()
return option, jm_client
def create_option(filepath: str) -> JmOption:
option = JmOption.from_file(filepath)
return option
def create_option(filepath):
return JmModuleConfig.option_class().from_file(filepath)

View File

@ -85,7 +85,7 @@ class AbstractJmClient(
# noinspection PyMethodMayBeStatic, PyUnusedLocal
def before_retry(self, e, kwargs, retry_count, url):
jm_debug('error', str(e))
jm_debug('retry', str(e))
def enable_cache(self, debug=False):
def wrap_func_cache(func_name, cache_dict_name):
@ -131,8 +131,11 @@ class AbstractJmClient(
def get_jmcomic_domain_all(self, postman=None):
return JmModuleConfig.get_jmcomic_domain_all(postman or self.get_root_postman())
# noinspection PyUnusedLocal
def fallback(self, request, url, domain_index, retry_count, **kwargs):
raise AssertionError(f"请求重试全部失败: [{url}], {self.domain_list}")
msg = f"请求重试全部失败: [{url}], {self.domain_list}"
jm_debug('fallback', "msg")
raise AssertionError(msg)
# 基于网页实现的JmClient
@ -406,3 +409,15 @@ class JmApiClient(AbstractJmClient):
"user-agent": "okhttp/3.12.1",
"accept-encoding": "gzip",
}, key_ts
class AsyncSaveImageClient(JmImageClient):
def __init__(self, workers=None) -> None:
from concurrent.futures import ThreadPoolExecutor, Future
self.executor = ThreadPoolExecutor(max_workers=workers)
self.future_list: List[Future] = []
def save_image_resp(self, *args, **kwargs):
future = self.executor.submit(lambda: super().save_image_resp(*args, **kwargs))
self.future_list.append(future)

View File

@ -236,8 +236,10 @@ class JmImageClient:
resp.require_success()
# gif图无需加解密需要最先判断
return self.save_image_resp(decode_image, img_save_path, img_url, resp, scramble_id)
def save_image_resp(self, decode_image, img_save_path, img_url, resp, scramble_id):
# gif图无需加解密需要最先判断
if self.img_is_not_need_to_decode(img_url, resp):
JmImageSupport.save_resp_img(resp, img_save_path, False)
else:
@ -248,7 +250,7 @@ class JmImageClient:
img_save_path,
decode_image=True,
):
self.download_image(
return self.download_image(
image.download_url,
img_save_path,
image.scramble_id,

View File

@ -57,6 +57,24 @@ class JmModuleConfig:
enable_jm_debug = True
debug_executor = default_jm_debug
postman_constructor = default_postman_constructor
DOWNLOADER_CLASS = None
OPTION_CLASS = None
@classmethod
def downloader_class(cls):
if cls.DOWNLOADER_CLASS is not None:
return cls.DOWNLOADER_CLASS
from .jm_downloader import JmDownloader
return JmDownloader
@classmethod
def option_class(cls):
if cls.OPTION_CLASS is not None:
return cls.OPTION_CLASS
from .jm_option import JmOption
return JmOption
@classmethod
@field_cache("DOMAIN")
@ -115,8 +133,7 @@ class JmModuleConfig:
"""
postman = postman or cls.new_postman(session=True)
resp = postman.get(cls.JM_REDIRECT_URL)
url = resp.url
url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL)
cls.jm_debug('获取禁漫地址', f'[{cls.JM_REDIRECT_URL}] → [{url}]')
return url

View File

@ -0,0 +1,148 @@
from .jm_option import *
class JmDownloadException(Exception):
pass
# noinspection PyMethodMayBeStatic
class DownloadCallback:
def before_album(self, album: JmAlbumDetail):
jm_debug('album-before',
f'本子获取成功: [{album.id}], '
f'作者: [{album.author}], '
f'章节数: [{len(album)}], '
f'标题: [{album.title}], '
)
def after_album(self, album: JmAlbumDetail):
jm_debug('album-after', f'本子下载完成: [{album.id}]')
def before_photo(self, photo: JmPhotoDetail):
jm_debug('photo-before',
f'开始下载章节: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}]), '
f'标题: [{photo.title}], '
f'图片数为[{len(photo)}]'
)
def after_photo(self, photo: JmPhotoDetail):
jm_debug('photo-after',
f'章节下载完成: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}])')
def before_image(self, image: JmImageDetail, img_save_path):
if image.is_exists:
jm_debug('image-before',
f'图片已存在: {image.tag} ← [{img_save_path}]'
)
else:
jm_debug('image-before',
f'图片准备下载: {image.tag}, [{image.img_url}] → [{img_save_path}]'
)
def after_image(self, image: JmImageDetail, img_save_path):
jm_debug('image-after',
f'图片下载完成: {image.tag}, [{image.img_url}] → [{img_save_path}]')
class JmDownloader(DownloadCallback):
"""
JmDownloader = JmOption + 调度逻辑
"""
def __init__(self, option) -> None:
self.option = option
self.use_cache = self.option.download_cache
self.decode_image = self.option.download_image_decode
def download_album(self, album_id):
client = self.client_for_album(album_id)
album = client.get_album_detail(album_id)
self.before_album(album)
self.download_by_album_detail(album, client)
self.after_album(album)
def download_by_album_detail(self, album: JmAlbumDetail, client: JmcomicClient):
self.execute_by_condition(
iter_objs=album,
apply=lambda photo: self.download_by_photo_detail(photo, client),
count_batch=self.option.decide_photo_batch_count(album)
)
def download_photo(self, photo_id):
client = self.client_for_photo(photo_id)
photo = client.get_photo_detail(photo_id)
self.before_photo(photo)
self.download_by_photo_detail(photo, client)
self.after_photo(photo)
def download_by_photo_detail(self, photo: JmPhotoDetail, client: JmcomicClient):
client.check_photo(photo)
self.execute_by_condition(
iter_objs=photo,
apply=lambda image: self.download_by_image_detail(image, client),
count_batch=self.option.decide_image_batch_count(photo)
)
def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
img_save_path = self.option.decide_image_filepath(image)
image.is_exists = file_exists(img_save_path)
self.before_image(image, img_save_path)
if self.use_cache is True and image.is_exists:
return
client.download_by_image_detail(
image,
img_save_path,
decode_image=self.decode_image,
)
self.after_image(image, img_save_path)
# noinspection PyMethodMayBeStatic
def execute_by_condition(self, iter_objs, apply: Callable, count_batch: int):
"""
章节/图片的下载调度逻辑
"""
count_real = len(iter_objs)
if count_batch >= count_real:
# 一个图/章节 对应 一个线程
multi_thread_launcher(
iter_objs=iter_objs,
apply_each_obj_func=apply,
)
else:
# 创建batch个线程的线程池
thread_pool_executor(
iter_objs=iter_objs,
apply_each_obj_func=apply,
max_workers=count_batch,
)
# noinspection PyUnusedLocal
def client_for_album(self, jm_album_id):
"""
默认情况下每次调用JmDownloader的download_album或download_photo,
都会使用一个新的 JmcomicClient
"""
return self.option.new_jm_client()
# noinspection PyUnusedLocal
def client_for_photo(self, jm_photo_id):
"""
默认情况下每次调用JmDownloader的download_album或download_photo,
都会使用一个新的 JmcomicClient
"""
return self.option.new_jm_client()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is not None:
jm_debug('exception',
f'{self.__class__.__name__} Exit with exception: {exc_type, exc_val}'
)

View File

@ -1,46 +1,6 @@
from .jm_client_impl import *
# noinspection PyMethodMayBeStatic
class DownloadCallback:
def before_album(self, album: JmAlbumDetail):
jm_debug('album-before',
f'本子获取成功: [{album.id}], '
f'作者: [{album.author}], '
f'章节数: [{len(album)}], '
f'标题: [{album.title}], '
)
def after_album(self, album: JmAlbumDetail):
jm_debug('album-after', f'本子下载完成: [{album.id}]')
def before_photo(self, photo: JmPhotoDetail):
jm_debug('photo-before',
f'开始下载章节: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}]), '
f'标题: [{photo.title}], '
f'图片数为[{len(photo)}]'
)
def after_photo(self, photo: JmPhotoDetail):
jm_debug('photo-after',
f'章节下载完成: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}])')
def before_image(self, image: JmImageDetail, img_save_path):
if image.is_exists:
jm_debug('image-before',
f'图片已存在: {image.tag} ← [{img_save_path}]'
)
else:
jm_debug('image-before',
f'图片准备下载: {image.tag}, [{image.img_url}] → [{img_save_path}]'
)
def after_image(self, image: JmImageDetail, img_save_path):
jm_debug('image-after',
f'图片下载完成: {image.tag}, [{image.img_url}] → [{img_save_path}]')
class DirRule:
rule_sample = [
# 根目录 / Album-id / Photo-序号 /
@ -139,7 +99,7 @@ class DirRule:
return base_dir
class JmOption(DownloadCallback):
class JmOption:
JM_OP_VER = '2.0'
def __init__(self,