v2.5.35: 优化部分下载失败时异常处理机制,更新文档 (#412)
Some checks failed
Auto Release & Publish / release (push) Has been cancelled

This commit is contained in:
hect0x7 2025-04-09 01:24:11 +08:00 committed by GitHub
parent e4584d09bd
commit ebf0ea602d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 140 additions and 61 deletions

View File

@ -82,25 +82,38 @@ from jmcomic import *
# 客户端
client = JmOption.default().new_jm_client()
# 捕获jmcomic可能出现的异常
# 捕获获取本子/章节详情时可能出现的异常
try:
# 请求本子实体类
album: JmAlbumDetail = client.get_album_detail('427413')
except MissingAlbumPhotoException as e:
print(f'id={e.error_jmid}的本子不存在')
except JsonResolveFailException as e:
print(f'解析json失败')
# 响应对象
resp = e.resp
print(f'resp.text: {resp.text}, resp.status_code: {resp.status_code}')
except RequestRetryAllFailException as e:
print(f'请求失败,重试次数耗尽')
except JmcomicException as e:
# 捕获所有异常,用作兜底
print(f'jmcomic遇到异常: {e}')
# 多线程下载时,可能出现非当前线程下载失败,抛出异常,
# 而JmDownloader有对应字段记录了这些线程发生的异常
# 使用check_exception=True参数可以使downloader主动检查是否存在下载异常
# 如果有则当前线程会主动上抛一个PartialDownloadFailedException异常
# 该参数主要用于主动检查部分下载失败的情况,
# 因为非当前线程抛出的异常(比如下载章节的线程和下载图片的线程),这些线程如果抛出异常,
# 当前线程是感知不到的try-catch下载方法download_album不能捕获到其他线程发生的异常。
try:
album, downloader = download_album(123, check_exception=True)
except PartialDownloadFailedException as e:
downloader: JmDownloader = e.downloader
print(f'下载出现部分失败, 下载失败的章节: {downloader.download_failed_photo}, 下载失败的图片: {downloader.download_failed_image}')
```

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.5.34'
__version__ = '2.5.35'
from .api import *
from .jm_plugin import *

View File

@ -48,6 +48,7 @@ def download_album(jm_album_id,
option=None,
downloader=None,
callback=None,
check_exception=True,
) -> Union[__DOWNLOAD_API_RET, Set[__DOWNLOAD_API_RET]]:
"""
下载一个本子album包含其所有的章节photo
@ -58,6 +59,7 @@ def download_album(jm_album_id,
:param option: 下载选项
:param downloader: 下载器类
:param callback: 返回值回调函数可以拿到 album downloader
:param check_exception: 是否检查异常, 如果为True会检查downloader是否有下载异常并上抛PartialDownloadFailedException
:return: 对于的本子实体类下载器如果是上述的批量情况返回值为download_batch的返回值
"""
@ -69,14 +71,17 @@ def download_album(jm_album_id,
if callback is not None:
callback(album, dler)
if check_exception:
dler.raise_if_has_exception()
return album, dler
def download_photo(jm_photo_id,
option=None,
downloader=None,
callback=None):
callback=None,
check_exception=True,
):
"""
下载一个章节photo参数同 download_album
"""
@ -88,7 +93,8 @@ def download_photo(jm_photo_id,
if callback is not None:
callback(photo, dler)
if check_exception:
dler.raise_if_has_exception()
return photo, dler

View File

@ -566,7 +566,7 @@ class JmHtmlClient(AbstractJmClient):
cls.raise_request_error(
resp,
f'{reason}'
f'{reason}({content})'
+ (f': {url}' if url is not None else '')
)

View File

@ -1,6 +1,31 @@
from .jm_option import *
def catch_exception(func):
from functools import wraps
@wraps(func)
def wrapper(self, *args, **kwargs):
self: JmDownloader
try:
return func(self, *args, **kwargs)
except Exception as e:
detail: JmBaseEntity = args[0]
if detail.is_image():
detail: JmImageDetail
jm_log('image.failed', f'图片下载失败: [{detail.download_url}], 异常: [{e}]')
self.download_failed_image.append((detail, e))
elif detail.is_photo():
detail: JmPhotoDetail
jm_log('photo.failed', f'章节下载失败: [{detail.id}], 异常: [{e}]')
self.download_failed_photo.append((detail, e))
raise e
return wrapper
# noinspection PyMethodMayBeStatic
class DownloadCallback:
@ -50,48 +75,50 @@ class JmDownloader(DownloadCallback):
def __init__(self, option: JmOption) -> None:
self.option = option
self.client = option.build_jm_client()
# 下载成功的记录dict
self.download_success_dict: Dict[JmAlbumDetail, Dict[JmPhotoDetail, List[Tuple[str, JmImageDetail]]]] = {}
# 下载失败的记录list
self.download_failed_list: List[Tuple[JmImageDetail, BaseException]] = []
self.download_failed_image: List[Tuple[JmImageDetail, BaseException]] = []
self.download_failed_photo: List[Tuple[JmPhotoDetail, BaseException]] = []
def download_album(self, album_id):
client = self.client_for_album(album_id)
album = client.get_album_detail(album_id)
self.download_by_album_detail(album, client)
album = self.client.get_album_detail(album_id)
self.download_by_album_detail(album)
return album
def download_by_album_detail(self, album: JmAlbumDetail, client: JmcomicClient):
def download_by_album_detail(self, album: JmAlbumDetail):
self.before_album(album)
if album.skip:
return
self.execute_by_condition(
self.execute_on_condition(
iter_objs=album,
apply=lambda photo: self.download_by_photo_detail(photo, client),
apply=self.download_by_photo_detail,
count_batch=self.option.decide_photo_batch_count(album)
)
self.after_album(album)
def download_photo(self, photo_id):
client = self.client_for_photo(photo_id)
photo = client.get_photo_detail(photo_id)
self.download_by_photo_detail(photo, client)
photo = self.client.get_photo_detail(photo_id)
self.download_by_photo_detail(photo)
return photo
def download_by_photo_detail(self, photo: JmPhotoDetail, client: JmcomicClient):
client.check_photo(photo)
@catch_exception
def download_by_photo_detail(self, photo: JmPhotoDetail):
self.client.check_photo(photo)
self.before_photo(photo)
if photo.skip:
return
self.execute_by_condition(
self.execute_on_condition(
iter_objs=photo,
apply=lambda image: self.download_by_image_detail(image, client),
apply=self.download_by_image_detail,
count_batch=self.option.decide_image_batch_count(photo)
)
self.after_photo(photo)
def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
@catch_exception
def download_by_image_detail(self, image: JmImageDetail):
img_save_path = self.option.decide_image_filepath(image)
image.save_path = img_save_path
@ -110,22 +137,15 @@ class JmDownloader(DownloadCallback):
if use_cache is True and image.exists:
return
try:
client.download_by_image_detail(
image,
img_save_path,
decode_image=decode_image,
)
except BaseException as e:
jm_log('image.failed', f'图片下载失败: [{image.download_url}], 异常: {e}')
# 保存失败记录
self.download_failed_list.append((image, e))
raise
self.client.download_by_image_detail(
image,
img_save_path,
decode_image=decode_image,
)
self.after_image(image, img_save_path)
# noinspection PyMethodMayBeStatic
def execute_by_condition(self,
def execute_on_condition(self,
iter_objs: DetailEntity,
apply: Callable,
count_batch: int,
@ -166,20 +186,6 @@ class JmDownloader(DownloadCallback):
"""
return detail
# noinspection PyUnusedLocal
def client_for_album(self, jm_album_id) -> JmcomicClient:
"""
默认情况下所有的JmDownloader共用一个JmcomicClient
"""
return self.option.build_jm_client()
# noinspection PyUnusedLocal
def client_for_photo(self, jm_photo_id) -> JmcomicClient:
"""
默认情况下所有的JmDownloader共用一个JmcomicClient
"""
return self.option.build_jm_client()
@property
def all_success(self) -> bool:
"""
@ -189,7 +195,7 @@ class JmDownloader(DownloadCallback):
注意如果使用了filter机制例如通过filter只下载3张图片那么all_success也会为False
"""
if len(self.download_failed_list) != 0:
if self.has_download_failures:
return False
for album, photo_dict in self.download_success_dict.items():
@ -202,6 +208,10 @@ class JmDownloader(DownloadCallback):
return True
@property
def has_download_failures(self):
return len(self.download_failed_image) != 0 or len(self.download_failed_photo) != 0
# 下面是回调方法
def before_album(self, album: JmAlbumDetail):
@ -259,6 +269,23 @@ class JmDownloader(DownloadCallback):
downloader=self,
)
def raise_if_has_exception(self):
if not self.has_download_failures:
return
msg_ls = ['部分下载失败', '', '']
if len(self.download_failed_photo) != 0:
msg_ls[1] = f'{len(self.download_failed_photo)}个章节下载失败: {self.download_failed_photo}'
if len(self.download_failed_image) != 0:
msg_ls[2] = f'{len(self.download_failed_image)}个图片下载失败: {self.download_failed_image}'
ExceptionTool.raises(
'\n'.join(msg_ls),
{'downloader': self},
PartialDownloadFailedException,
)
# 下面是对with语法的支持
def __enter__(self):
@ -283,7 +310,7 @@ class DoNotDownloadImage(JmDownloader):
不会下载任何图片的Downloader用作测试
"""
def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
def download_by_image_detail(self, image: JmImageDetail):
# ensure make dir
self.option.decide_image_filepath(image)
@ -297,12 +324,13 @@ class JustDownloadSpecificCountImage(JmDownloader):
count_lock = Lock()
count = 0
def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
@catch_exception
def download_by_image_detail(self, image: JmImageDetail):
# ensure make dir
self.option.decide_image_filepath(image)
if self.try_countdown():
return super().download_by_image_detail(image, client)
return super().download_by_image_detail(image)
def try_countdown(self):
if self.count < 0:

View File

@ -125,10 +125,9 @@ class DetailEntity(JmBaseEntity, IndexedEntity):
return f'[{self.id}] {self.oname}'
def __str__(self):
return f'{self.__class__.__name__}' \
'{' \
f'{self.id}: {self.title}' \
'}'
return f'''{self.__class__.__name__}({self.__alias__()}-{self.id}: "{self.title}")'''
__repr__ = __str__
@classmethod
def __alias__(cls):
@ -258,6 +257,11 @@ class JmImageDetail(JmBaseEntity, Downloadable):
def is_image(cls):
return True
def __str__(self):
return f'''{self.__class__.__name__}(image-[{self.download_url}])'''
__repr__ = __str__
class JmPhotoDetail(DetailEntity, Downloadable):

View File

@ -15,6 +15,7 @@ class JmcomicException(Exception):
def __str__(self):
return self.msg
class ResponseUnexpectedException(JmcomicException):
description = '响应不符合预期异常'
@ -44,7 +45,6 @@ class RegularNotMatchException(JmcomicException):
class JsonResolveFailException(ResponseUnexpectedException):
description = 'Json解析异常'
pass
class MissingAlbumPhotoException(ResponseUnexpectedException):
@ -57,9 +57,15 @@ class MissingAlbumPhotoException(ResponseUnexpectedException):
class RequestRetryAllFailException(JmcomicException):
description = '请求重试全部失败异常'
pass
class PartialDownloadFailedException(JmcomicException):
description = '部分章节或图片下载失败异常'
@property
def downloader(self):
return self.from_context(ExceptionTool.CONTEXT_KEY_DOWNLOADER)
class ExceptionTool:
"""
抛异常的工具
@ -71,6 +77,7 @@ class ExceptionTool:
CONTEXT_KEY_HTML = 'html'
CONTEXT_KEY_RE_PATTERN = 'pattern'
CONTEXT_KEY_MISSING_JM_ID = 'missing_jm_id'
CONTEXT_KEY_DOWNLOADER = 'downloader'
@classmethod
def raises(cls,

View File

@ -76,3 +76,24 @@ class Test_Api(JmTestConfigurable):
print(e)
raise AssertionError(exception_list)
def test_partial_exception(self):
class TestDownloader(JmDownloader):
@catch_exception
def download_by_image_detail(self, image: JmImageDetail):
raise Exception('test_partial_exception')
@catch_exception
def download_by_photo_detail(self, photo: JmPhotoDetail):
if photo.index != 2:
raise Exception('test_partial_exception')
return super().download_by_photo_detail(photo)
self.assertRaises(
PartialDownloadFailedException,
lambda: download_album(182150, downloader=TestDownloader, check_exception=True)
)
self.assertRaises(
PartialDownloadFailedException,
lambda: download_photo(182151, downloader=TestDownloader, check_exception=True)
)