v2.5.9: 新增插件【跳过下载图片数少的章节】,可过滤一些公告章节; 新增Downloadable基类,对插件扩展更友好; 修复本子名过长超出导致oserror的错误; 升级curl_cffi. (#222)

This commit is contained in:
hect0x7 2024-04-06 10:56:47 +08:00 committed by GitHub
parent ee23037f94
commit dec2d54459
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 102 additions and 48 deletions

View File

@ -198,6 +198,11 @@ plugins:
zip_enable: true # 对收藏夹进行压缩
zip_filepath: ${JM_DOWNLOAD_DIR}/export.zip # 压缩文件路径
zip_password: ${ZIP_PASSWORD} # 压缩密码
before_photo:
- plugin: skip_photo_with_few_images # 跳过下载章节图片数量过少的章节。一些韩漫的章节是公告,没有实际内容,就可以用该插件来跳过下载这些章节。
kwargs:
at_least_image_count: 3
after_photo:
- plugin: j2p # jpg图片合成为一个pdf插件

View File

@ -1,5 +1,5 @@
commonX
curl_cffi
commonX
PyYAML
Pillow
psutil

View File

@ -27,8 +27,8 @@ setup(
package_dir={"": "src"},
python_requires=">=3.7",
install_requires=[
'commonX>=0.6.4',
'curl_cffi',
'commonX',
'PyYAML',
'Pillow',
'pycryptodome',

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.5.8'
__version__ = '2.5.9'
from .api import *
from .jm_plugin import *

View File

@ -224,7 +224,7 @@ class AbstractJmClient(
# noinspection PyMethodMayBeStatic
def decode(self, url: str):
if not JmModuleConfig.flag_decode_url_when_logging or '/search/' not in url:
if not JmModuleConfig.FLAG_DECODE_URL_WHEN_LOGGING or '/search/' not in url:
return url
from urllib.parse import unquote
@ -767,7 +767,7 @@ class JmApiClient(AbstractJmClient):
# 检查禁漫最新的版本号
setting_ver = str(resp.model_data.version)
# 禁漫接口的版本 > jmcomic库内置版本
if setting_ver > JmMagicConstants.APP_VERSION and JmModuleConfig.flag_use_version_newer_if_behind:
if setting_ver > JmMagicConstants.APP_VERSION and JmModuleConfig.FLAG_USE_VERSION_NEWER_IF_BEHIND:
jm_log('api.setting', f'change APP_VERSION from [{JmMagicConstants.APP_VERSION}] to [{setting_ver}]')
JmMagicConstants.APP_VERSION = setting_ver
@ -883,7 +883,7 @@ class JmApiClient(AbstractJmClient):
ts = time_stamp()
token, tokenparam = JmCryptoTool.token_and_tokenparam(ts, secret=JmMagicConstants.APP_TOKEN_SECRET_2)
elif JmModuleConfig.flag_use_fix_timestamp:
elif JmModuleConfig.FLAG_USE_FIX_TIMESTAMP:
ts, token, tokenparam = JmModuleConfig.get_fix_ts_token_tokenparam()
else:
@ -954,7 +954,7 @@ class JmApiClient(AbstractJmClient):
def after_init(self):
# 保证拥有cookies因为移动端要求必须携带cookies否则会直接跳转同一本子【禁漫娘】
if JmModuleConfig.flag_api_client_require_cookies:
if JmModuleConfig.FLAG_API_CLIENT_REQUIRE_COOKIES:
self.ensure_have_cookies()
client_init_cookies_lock = Lock()

View File

@ -146,18 +146,18 @@ class JmModuleConfig:
REGISTRY_EXCEPTION_LISTENER = {}
# 执行log的函数
executor_log = default_jm_logging
EXECUTOR_LOG = default_jm_logging
# 使用固定时间戳
flag_use_fix_timestamp = True
FLAG_USE_FIX_TIMESTAMP = True
# 移动端Client初始化cookies
flag_api_client_require_cookies = True
FLAG_API_CLIENT_REQUIRE_COOKIES = True
# log开关标记
flag_enable_jm_log = True
FLAG_ENABLE_JM_LOG = True
# log时解码url
flag_decode_url_when_logging = True
FLAG_DECODE_URL_WHEN_LOGGING = True
# 当内置的版本号落后时使用最新的禁漫app版本号
flag_use_version_newer_if_behind = True
FLAG_USE_VERSION_NEWER_IF_BEHIND = True
# 关联dir_rule的自定义字段与对应的处理函数
# 例如:
@ -165,6 +165,10 @@ class JmModuleConfig:
AFIELD_ADVICE = dict()
PFIELD_ADVICE = dict()
# 当发生 oserror: [Errno 36] File name too long 时,
# 把文件名限制在指定个字符以内
VAR_FILE_NAME_LENGTH_LIMIT = 100
@classmethod
def downloader_class(cls):
if cls.CLASS_DOWNLOADER is not None:
@ -319,12 +323,12 @@ class JmModuleConfig:
# noinspection PyUnusedLocal
@classmethod
def jm_log(cls, topic: str, msg: str):
if cls.flag_enable_jm_log is True:
cls.executor_log(topic, msg)
if cls.FLAG_ENABLE_JM_LOG is True:
cls.EXECUTOR_LOG(topic, msg)
@classmethod
def disable_jm_log(cls):
cls.flag_enable_jm_log = False
cls.FLAG_ENABLE_JM_LOG = False
@classmethod
def new_postman(cls, session=False, **kwargs):
@ -347,7 +351,7 @@ class JmModuleConfig:
DEFAULT_CLIENT_CACHE = None # 默认关闭Client缓存。缓存的配置详见 CacheRegistry
DEFAULT_PROXIES = ProxyBuilder.system_proxy() # 默认使用系统代理
default_option_dict: dict = {
DEFAULT_OPTION_DICT: dict = {
'log': None,
'dir_rule': {'rule': 'Bd_Pname', 'base_dir': None},
'download': {
@ -364,7 +368,7 @@ class JmModuleConfig:
'postman': {
'type': 'cffi',
'meta_data': {
'impersonate': 'chrome110',
'impersonate': 'chrome',
'headers': None,
'proxies': None,
}
@ -387,11 +391,11 @@ class JmModuleConfig:
"""
from copy import deepcopy
option_dict = deepcopy(cls.default_option_dict)
option_dict = deepcopy(cls.DEFAULT_OPTION_DICT)
# log
if option_dict['log'] is None:
option_dict['log'] = cls.flag_enable_jm_log
option_dict['log'] = cls.FLAG_ENABLE_JM_LOG
# dir_rule.base_dir
dir_rule = option_dict['dir_rule']

View File

@ -29,7 +29,7 @@ class DownloadCallback:
f'章节下载完成: [{photo.id}] ({photo.album_id}[{photo.index}/{len(photo.from_album)}])')
def before_image(self, image: JmImageDetail, img_save_path):
if image.is_exists:
if image.exists:
jm_log('image.before',
f'图片已存在: {image.tag} ← [{img_save_path}]'
)
@ -63,6 +63,8 @@ class JmDownloader(DownloadCallback):
def download_by_album_detail(self, album: JmAlbumDetail, client: JmcomicClient):
self.before_album(album)
if album.skip:
return
self.execute_by_condition(
iter_objs=album,
apply=lambda photo: self.download_by_photo_detail(photo, client),
@ -80,6 +82,8 @@ class JmDownloader(DownloadCallback):
client.check_photo(photo)
self.before_photo(photo)
if photo.skip:
return
self.execute_by_condition(
iter_objs=photo,
apply=lambda image: self.download_by_image_detail(image, client),
@ -91,16 +95,19 @@ class JmDownloader(DownloadCallback):
img_save_path = self.option.decide_image_filepath(image)
image.save_path = img_save_path
image.is_exists = file_exists(img_save_path)
image.exists = file_exists(img_save_path)
self.before_image(image, img_save_path)
if image.skip:
return
# let option decide use_cache and decode_image
use_cache = self.option.decide_download_cache(image)
decode_image = self.option.decide_download_image_decode(image)
# skip download
if use_cache is True and image.is_exists:
if use_cache is True and image.exists:
return
e = None

View File

@ -3,6 +3,14 @@ from common import *
from .jm_config import *
class Downloadable:
def __init__(self):
self.save_path: str = ''
self.exists: bool = False
self.skip = False
class JmBaseEntity:
def to_file(self, filepath):
@ -117,7 +125,7 @@ class DetailEntity(JmBaseEntity, IndexedEntity):
def __str__(self):
return f'{self.__class__.__name__}' \
'{' \
f'{self.id}: {self.title}'\
f'{self.id}: {self.title}' \
'}'
@classmethod
@ -156,7 +164,7 @@ class DetailEntity(JmBaseEntity, IndexedEntity):
return getattr(detail, ref)
class JmImageDetail(JmBaseEntity):
class JmImageDetail(JmBaseEntity, Downloadable):
def __init__(self,
aid,
@ -167,7 +175,8 @@ class JmImageDetail(JmBaseEntity):
from_photo=None,
query_params=None,
index=-1,
) -> None:
):
super().__init__()
if scramble_id is None or (isinstance(scramble_id, str) and scramble_id == ''):
from .jm_toolkit import ExceptionTool
ExceptionTool.raises(f'图片的scramble_id不能为空')
@ -182,10 +191,6 @@ class JmImageDetail(JmBaseEntity):
self.query_params: Optional[str] = query_params
self.index = index # 从1开始
# temp fields, in order to simplify passing parameter
self.save_path: str = ''
self.is_exists: bool = False
@property
def filename_without_suffix(self):
return self.img_file_name
@ -252,7 +257,7 @@ class JmImageDetail(JmBaseEntity):
return True
class JmPhotoDetail(DetailEntity):
class JmPhotoDetail(DetailEntity, Downloadable):
def __init__(self,
photo_id,
@ -267,6 +272,7 @@ class JmPhotoDetail(DetailEntity):
author=None,
from_album=None,
):
super().__init__()
self.photo_id: str = str(photo_id)
self.scramble_id: str = str(scramble_id)
self.name: str = str(name).strip()
@ -411,7 +417,7 @@ class JmPhotoDetail(DetailEntity):
return True
class JmAlbumDetail(DetailEntity):
class JmAlbumDetail(DetailEntity, Downloadable):
def __init__(self,
album_id,
@ -430,6 +436,7 @@ class JmAlbumDetail(DetailEntity):
tags,
related_list=None,
):
super().__init__()
self.album_id: str = str(album_id)
self.scramble_id: str = str(scramble_id)
self.name: str = name

View File

@ -72,13 +72,6 @@ class ExceptionTool:
CONTEXT_KEY_RE_PATTERN = 'pattern'
CONTEXT_KEY_MISSING_JM_ID = 'missing_jm_id'
# 兼容旧版本
EXTRA_KEY_RESP = 'resp'
EXTRA_KEY_HTML = 'html'
EXTRA_KEY_RE_PATTERN = 'pattern'
EXTRA_KEY_MISSING_JM_ID = 'missing_jm_id'
@classmethod
def raises(cls,
msg: str,
@ -144,7 +137,8 @@ class ExceptionTool:
:param resp: 响应对象
:param jmid: 禁漫本子/章节id
"""
url = resp.url
from .jm_toolkit import JmcomicText
url = JmcomicText.format_album_url(jmid)
req_type = "本子" if "album" in url else "章节"
cls.raises(

View File

@ -285,7 +285,15 @@ class JmOption:
)
if ensure_exists:
mkdir_if_not_exists(save_dir)
try:
mkdir_if_not_exists(save_dir)
except OSError as e:
if e.errno == 36:
# 目录名过长
limit = JmModuleConfig.VAR_FILE_NAME_LENGTH_LIMIT
jm_log('error', f'目录名过长,无法创建目录,强制缩短到{limit}个字符并重试')
save_dir = save_dir[0:limit]
mkdir_if_not_exists(save_dir)
return save_dir
@ -359,7 +367,7 @@ class JmOption:
def deconstruct(self) -> Dict:
return {
'version': JmModuleConfig.JM_OPTION_VER,
'log': JmModuleConfig.flag_enable_jm_log,
'log': JmModuleConfig.FLAG_ENABLE_JM_LOG,
'dir_rule': {
'rule': self.dir_rule.rule_dsl,
'base_dir': self.dir_rule.base_dir,

View File

@ -445,9 +445,7 @@ class ImageSuffixFilterPlugin(JmOptionPlugin):
if image.img_file_suffix not in allowed_suffix_set:
self.log(f'跳过下载图片: {image.tag}'
f'因为其后缀\'{image.img_file_suffix}\'不在允许的后缀集合{allowed_suffix_set}')
# hook is_exists True to skip download
image.is_exists = True
return True
image.skip = True
# let option decide
return option_decide_cache(image)
@ -484,7 +482,7 @@ class LogTopicFilterPlugin(JmOptionPlugin):
if whitelist is not None:
whitelist = set(whitelist)
old_jm_log = JmModuleConfig.executor_log
old_jm_log = JmModuleConfig.EXECUTOR_LOG
def new_jm_log(topic, msg):
if whitelist is not None and topic not in whitelist:
@ -492,7 +490,7 @@ class LogTopicFilterPlugin(JmOptionPlugin):
old_jm_log(topic, msg)
JmModuleConfig.executor_log = new_jm_log
JmModuleConfig.EXECUTOR_LOG = new_jm_log
class AutoSetBrowserCookiesPlugin(JmOptionPlugin):
@ -963,3 +961,34 @@ class SubscribeAlbumUpdatePlugin(JmOptionPlugin):
is_new_photo = True
return len(photo_new_list) != 0, photo_new_list
class SkipPhotoWithFewImagesPlugin(JmOptionPlugin):
plugin_key = 'skip_photo_with_few_images'
def invoke(self,
at_least_image_count: int,
photo: Optional[JmPhotoDetail] = None,
image: Optional[JmImageDetail] = None,
album: Optional[JmAlbumDetail] = None,
**kwargs
):
self.try_mark_photo_skip_and_log(photo, at_least_image_count)
if image is not None:
self.try_mark_photo_skip_and_log(image.from_photo, at_least_image_count)
def try_mark_photo_skip_and_log(self, photo: JmPhotoDetail, at_least_image_count: int):
if photo is None:
return
if len(photo) >= at_least_image_count:
return
self.log(f'跳过下载章节: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}])'
f'因为其图片数: {len(photo)} < {at_least_image_count} (at_least_image_count)')
photo.skip = True
@classmethod
@field_cache() # 单例
def build(cls, option: JmOption) -> 'JmOptionPlugin':
return super().build(option)

View File

@ -234,7 +234,7 @@ class Test_Client(JmTestConfigurable):
self.assertEqual(ans, id(photo))
def test_search_generator(self):
JmModuleConfig.flag_decode_url_when_logging = False
JmModuleConfig.FLAG_DECODE_URL_WHEN_LOGGING = False
gen = self.client.search_gen('MANA')
for i, page in enumerate(gen):