mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.5.35: 优化部分下载失败时异常处理机制,更新文档 (#412)
Some checks failed
Auto Release & Publish / release (push) Has been cancelled
Some checks failed
Auto Release & Publish / release (push) Has been cancelled
This commit is contained in:
parent
e4584d09bd
commit
ebf0ea602d
@ -82,25 +82,38 @@ from jmcomic import *
|
||||
# 客户端
|
||||
client = JmOption.default().new_jm_client()
|
||||
|
||||
# 捕获jmcomic可能出现的异常
|
||||
# 捕获获取本子/章节详情时可能出现的异常
|
||||
try:
|
||||
# 请求本子实体类
|
||||
album: JmAlbumDetail = client.get_album_detail('427413')
|
||||
except MissingAlbumPhotoException as e:
|
||||
print(f'id={e.error_jmid}的本子不存在')
|
||||
|
||||
|
||||
except JsonResolveFailException as e:
|
||||
print(f'解析json失败')
|
||||
# 响应对象
|
||||
resp = e.resp
|
||||
print(f'resp.text: {resp.text}, resp.status_code: {resp.status_code}')
|
||||
|
||||
|
||||
except RequestRetryAllFailException as e:
|
||||
print(f'请求失败,重试次数耗尽')
|
||||
|
||||
|
||||
except JmcomicException as e:
|
||||
# 捕获所有异常,用作兜底
|
||||
print(f'jmcomic遇到异常: {e}')
|
||||
|
||||
# 多线程下载时,可能出现非当前线程下载失败,抛出异常,
|
||||
# 而JmDownloader有对应字段记录了这些线程发生的异常
|
||||
# 使用check_exception=True参数可以使downloader主动检查是否存在下载异常
|
||||
# 如果有,则当前线程会主动上抛一个PartialDownloadFailedException异常
|
||||
# 该参数主要用于主动检查部分下载失败的情况,
|
||||
# 因为非当前线程抛出的异常(比如下载章节的线程和下载图片的线程),这些线程如果抛出异常,
|
||||
# 当前线程是感知不到的,try-catch下载方法download_album不能捕获到其他线程发生的异常。
|
||||
try:
|
||||
album, downloader = download_album(123, check_exception=True)
|
||||
except PartialDownloadFailedException as e:
|
||||
downloader: JmDownloader = e.downloader
|
||||
print(f'下载出现部分失败, 下载失败的章节: {downloader.download_failed_photo}, 下载失败的图片: {downloader.download_failed_image}')
|
||||
```
|
||||
|
||||
|
||||
|
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.5.34'
|
||||
__version__ = '2.5.35'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
@ -48,6 +48,7 @@ def download_album(jm_album_id,
|
||||
option=None,
|
||||
downloader=None,
|
||||
callback=None,
|
||||
check_exception=True,
|
||||
) -> Union[__DOWNLOAD_API_RET, Set[__DOWNLOAD_API_RET]]:
|
||||
"""
|
||||
下载一个本子(album),包含其所有的章节(photo)
|
||||
@ -58,6 +59,7 @@ def download_album(jm_album_id,
|
||||
:param option: 下载选项
|
||||
:param downloader: 下载器类
|
||||
:param callback: 返回值回调函数,可以拿到 album 和 downloader
|
||||
:param check_exception: 是否检查异常, 如果为True,会检查downloader是否有下载异常,并上抛PartialDownloadFailedException
|
||||
:return: 对于的本子实体类,下载器(如果是上述的批量情况,返回值为download_batch的返回值)
|
||||
"""
|
||||
|
||||
@ -69,14 +71,17 @@ def download_album(jm_album_id,
|
||||
|
||||
if callback is not None:
|
||||
callback(album, dler)
|
||||
|
||||
if check_exception:
|
||||
dler.raise_if_has_exception()
|
||||
return album, dler
|
||||
|
||||
|
||||
def download_photo(jm_photo_id,
|
||||
option=None,
|
||||
downloader=None,
|
||||
callback=None):
|
||||
callback=None,
|
||||
check_exception=True,
|
||||
):
|
||||
"""
|
||||
下载一个章节(photo),参数同 download_album
|
||||
"""
|
||||
@ -88,7 +93,8 @@ def download_photo(jm_photo_id,
|
||||
|
||||
if callback is not None:
|
||||
callback(photo, dler)
|
||||
|
||||
if check_exception:
|
||||
dler.raise_if_has_exception()
|
||||
return photo, dler
|
||||
|
||||
|
||||
|
@ -566,7 +566,7 @@ class JmHtmlClient(AbstractJmClient):
|
||||
|
||||
cls.raise_request_error(
|
||||
resp,
|
||||
f'{reason}'
|
||||
f'{reason}({content})'
|
||||
+ (f': {url}' if url is not None else '')
|
||||
)
|
||||
|
||||
|
@ -1,6 +1,31 @@
|
||||
from .jm_option import *
|
||||
|
||||
|
||||
def catch_exception(func):
|
||||
from functools import wraps
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(self, *args, **kwargs):
|
||||
self: JmDownloader
|
||||
try:
|
||||
return func(self, *args, **kwargs)
|
||||
except Exception as e:
|
||||
detail: JmBaseEntity = args[0]
|
||||
if detail.is_image():
|
||||
detail: JmImageDetail
|
||||
jm_log('image.failed', f'图片下载失败: [{detail.download_url}], 异常: [{e}]')
|
||||
self.download_failed_image.append((detail, e))
|
||||
|
||||
elif detail.is_photo():
|
||||
detail: JmPhotoDetail
|
||||
jm_log('photo.failed', f'章节下载失败: [{detail.id}], 异常: [{e}]')
|
||||
self.download_failed_photo.append((detail, e))
|
||||
|
||||
raise e
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
class DownloadCallback:
|
||||
|
||||
@ -50,48 +75,50 @@ class JmDownloader(DownloadCallback):
|
||||
|
||||
def __init__(self, option: JmOption) -> None:
|
||||
self.option = option
|
||||
self.client = option.build_jm_client()
|
||||
# 下载成功的记录dict
|
||||
self.download_success_dict: Dict[JmAlbumDetail, Dict[JmPhotoDetail, List[Tuple[str, JmImageDetail]]]] = {}
|
||||
# 下载失败的记录list
|
||||
self.download_failed_list: List[Tuple[JmImageDetail, BaseException]] = []
|
||||
self.download_failed_image: List[Tuple[JmImageDetail, BaseException]] = []
|
||||
self.download_failed_photo: List[Tuple[JmPhotoDetail, BaseException]] = []
|
||||
|
||||
def download_album(self, album_id):
|
||||
client = self.client_for_album(album_id)
|
||||
album = client.get_album_detail(album_id)
|
||||
self.download_by_album_detail(album, client)
|
||||
album = self.client.get_album_detail(album_id)
|
||||
self.download_by_album_detail(album)
|
||||
return album
|
||||
|
||||
def download_by_album_detail(self, album: JmAlbumDetail, client: JmcomicClient):
|
||||
def download_by_album_detail(self, album: JmAlbumDetail):
|
||||
self.before_album(album)
|
||||
if album.skip:
|
||||
return
|
||||
self.execute_by_condition(
|
||||
self.execute_on_condition(
|
||||
iter_objs=album,
|
||||
apply=lambda photo: self.download_by_photo_detail(photo, client),
|
||||
apply=self.download_by_photo_detail,
|
||||
count_batch=self.option.decide_photo_batch_count(album)
|
||||
)
|
||||
self.after_album(album)
|
||||
|
||||
def download_photo(self, photo_id):
|
||||
client = self.client_for_photo(photo_id)
|
||||
photo = client.get_photo_detail(photo_id)
|
||||
self.download_by_photo_detail(photo, client)
|
||||
photo = self.client.get_photo_detail(photo_id)
|
||||
self.download_by_photo_detail(photo)
|
||||
return photo
|
||||
|
||||
def download_by_photo_detail(self, photo: JmPhotoDetail, client: JmcomicClient):
|
||||
client.check_photo(photo)
|
||||
@catch_exception
|
||||
def download_by_photo_detail(self, photo: JmPhotoDetail):
|
||||
self.client.check_photo(photo)
|
||||
|
||||
self.before_photo(photo)
|
||||
if photo.skip:
|
||||
return
|
||||
self.execute_by_condition(
|
||||
self.execute_on_condition(
|
||||
iter_objs=photo,
|
||||
apply=lambda image: self.download_by_image_detail(image, client),
|
||||
apply=self.download_by_image_detail,
|
||||
count_batch=self.option.decide_image_batch_count(photo)
|
||||
)
|
||||
self.after_photo(photo)
|
||||
|
||||
def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
|
||||
@catch_exception
|
||||
def download_by_image_detail(self, image: JmImageDetail):
|
||||
img_save_path = self.option.decide_image_filepath(image)
|
||||
|
||||
image.save_path = img_save_path
|
||||
@ -110,22 +137,15 @@ class JmDownloader(DownloadCallback):
|
||||
if use_cache is True and image.exists:
|
||||
return
|
||||
|
||||
try:
|
||||
client.download_by_image_detail(
|
||||
image,
|
||||
img_save_path,
|
||||
decode_image=decode_image,
|
||||
)
|
||||
except BaseException as e:
|
||||
jm_log('image.failed', f'图片下载失败: [{image.download_url}], 异常: {e}')
|
||||
# 保存失败记录
|
||||
self.download_failed_list.append((image, e))
|
||||
raise
|
||||
self.client.download_by_image_detail(
|
||||
image,
|
||||
img_save_path,
|
||||
decode_image=decode_image,
|
||||
)
|
||||
|
||||
self.after_image(image, img_save_path)
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def execute_by_condition(self,
|
||||
def execute_on_condition(self,
|
||||
iter_objs: DetailEntity,
|
||||
apply: Callable,
|
||||
count_batch: int,
|
||||
@ -166,20 +186,6 @@ class JmDownloader(DownloadCallback):
|
||||
"""
|
||||
return detail
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
def client_for_album(self, jm_album_id) -> JmcomicClient:
|
||||
"""
|
||||
默认情况下,所有的JmDownloader共用一个JmcomicClient
|
||||
"""
|
||||
return self.option.build_jm_client()
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
def client_for_photo(self, jm_photo_id) -> JmcomicClient:
|
||||
"""
|
||||
默认情况下,所有的JmDownloader共用一个JmcomicClient
|
||||
"""
|
||||
return self.option.build_jm_client()
|
||||
|
||||
@property
|
||||
def all_success(self) -> bool:
|
||||
"""
|
||||
@ -189,7 +195,7 @@ class JmDownloader(DownloadCallback):
|
||||
|
||||
注意!如果使用了filter机制,例如通过filter只下载3张图片,那么all_success也会为False
|
||||
"""
|
||||
if len(self.download_failed_list) != 0:
|
||||
if self.has_download_failures:
|
||||
return False
|
||||
|
||||
for album, photo_dict in self.download_success_dict.items():
|
||||
@ -202,6 +208,10 @@ class JmDownloader(DownloadCallback):
|
||||
|
||||
return True
|
||||
|
||||
@property
|
||||
def has_download_failures(self):
|
||||
return len(self.download_failed_image) != 0 or len(self.download_failed_photo) != 0
|
||||
|
||||
# 下面是回调方法
|
||||
|
||||
def before_album(self, album: JmAlbumDetail):
|
||||
@ -259,6 +269,23 @@ class JmDownloader(DownloadCallback):
|
||||
downloader=self,
|
||||
)
|
||||
|
||||
def raise_if_has_exception(self):
|
||||
if not self.has_download_failures:
|
||||
return
|
||||
msg_ls = ['部分下载失败', '', '']
|
||||
|
||||
if len(self.download_failed_photo) != 0:
|
||||
msg_ls[1] = f'共{len(self.download_failed_photo)}个章节下载失败: {self.download_failed_photo}'
|
||||
|
||||
if len(self.download_failed_image) != 0:
|
||||
msg_ls[2] = f'共{len(self.download_failed_image)}个图片下载失败: {self.download_failed_image}'
|
||||
|
||||
ExceptionTool.raises(
|
||||
'\n'.join(msg_ls),
|
||||
{'downloader': self},
|
||||
PartialDownloadFailedException,
|
||||
)
|
||||
|
||||
# 下面是对with语法的支持
|
||||
|
||||
def __enter__(self):
|
||||
@ -283,7 +310,7 @@ class DoNotDownloadImage(JmDownloader):
|
||||
不会下载任何图片的Downloader,用作测试
|
||||
"""
|
||||
|
||||
def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
|
||||
def download_by_image_detail(self, image: JmImageDetail):
|
||||
# ensure make dir
|
||||
self.option.decide_image_filepath(image)
|
||||
|
||||
@ -297,12 +324,13 @@ class JustDownloadSpecificCountImage(JmDownloader):
|
||||
count_lock = Lock()
|
||||
count = 0
|
||||
|
||||
def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
|
||||
@catch_exception
|
||||
def download_by_image_detail(self, image: JmImageDetail):
|
||||
# ensure make dir
|
||||
self.option.decide_image_filepath(image)
|
||||
|
||||
if self.try_countdown():
|
||||
return super().download_by_image_detail(image, client)
|
||||
return super().download_by_image_detail(image)
|
||||
|
||||
def try_countdown(self):
|
||||
if self.count < 0:
|
||||
|
@ -125,10 +125,9 @@ class DetailEntity(JmBaseEntity, IndexedEntity):
|
||||
return f'[{self.id}] {self.oname}'
|
||||
|
||||
def __str__(self):
|
||||
return f'{self.__class__.__name__}' \
|
||||
'{' \
|
||||
f'{self.id}: {self.title}' \
|
||||
'}'
|
||||
return f'''{self.__class__.__name__}({self.__alias__()}-{self.id}: "{self.title}")'''
|
||||
|
||||
__repr__ = __str__
|
||||
|
||||
@classmethod
|
||||
def __alias__(cls):
|
||||
@ -258,6 +257,11 @@ class JmImageDetail(JmBaseEntity, Downloadable):
|
||||
def is_image(cls):
|
||||
return True
|
||||
|
||||
def __str__(self):
|
||||
return f'''{self.__class__.__name__}(image-[{self.download_url}])'''
|
||||
|
||||
__repr__ = __str__
|
||||
|
||||
|
||||
class JmPhotoDetail(DetailEntity, Downloadable):
|
||||
|
||||
|
@ -15,6 +15,7 @@ class JmcomicException(Exception):
|
||||
def __str__(self):
|
||||
return self.msg
|
||||
|
||||
|
||||
class ResponseUnexpectedException(JmcomicException):
|
||||
description = '响应不符合预期异常'
|
||||
|
||||
@ -44,7 +45,6 @@ class RegularNotMatchException(JmcomicException):
|
||||
|
||||
class JsonResolveFailException(ResponseUnexpectedException):
|
||||
description = 'Json解析异常'
|
||||
pass
|
||||
|
||||
|
||||
class MissingAlbumPhotoException(ResponseUnexpectedException):
|
||||
@ -57,9 +57,15 @@ class MissingAlbumPhotoException(ResponseUnexpectedException):
|
||||
|
||||
class RequestRetryAllFailException(JmcomicException):
|
||||
description = '请求重试全部失败异常'
|
||||
pass
|
||||
|
||||
|
||||
class PartialDownloadFailedException(JmcomicException):
|
||||
description = '部分章节或图片下载失败异常'
|
||||
|
||||
@property
|
||||
def downloader(self):
|
||||
return self.from_context(ExceptionTool.CONTEXT_KEY_DOWNLOADER)
|
||||
|
||||
class ExceptionTool:
|
||||
"""
|
||||
抛异常的工具
|
||||
@ -71,6 +77,7 @@ class ExceptionTool:
|
||||
CONTEXT_KEY_HTML = 'html'
|
||||
CONTEXT_KEY_RE_PATTERN = 'pattern'
|
||||
CONTEXT_KEY_MISSING_JM_ID = 'missing_jm_id'
|
||||
CONTEXT_KEY_DOWNLOADER = 'downloader'
|
||||
|
||||
@classmethod
|
||||
def raises(cls,
|
||||
|
@ -76,3 +76,24 @@ class Test_Api(JmTestConfigurable):
|
||||
print(e)
|
||||
|
||||
raise AssertionError(exception_list)
|
||||
|
||||
def test_partial_exception(self):
|
||||
class TestDownloader(JmDownloader):
|
||||
@catch_exception
|
||||
def download_by_image_detail(self, image: JmImageDetail):
|
||||
raise Exception('test_partial_exception')
|
||||
|
||||
@catch_exception
|
||||
def download_by_photo_detail(self, photo: JmPhotoDetail):
|
||||
if photo.index != 2:
|
||||
raise Exception('test_partial_exception')
|
||||
return super().download_by_photo_detail(photo)
|
||||
|
||||
self.assertRaises(
|
||||
PartialDownloadFailedException,
|
||||
lambda: download_album(182150, downloader=TestDownloader, check_exception=True)
|
||||
)
|
||||
self.assertRaises(
|
||||
PartialDownloadFailedException,
|
||||
lambda: download_photo(182151, downloader=TestDownloader, check_exception=True)
|
||||
)
|
||||
|
Loading…
Reference in New Issue
Block a user