mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.5.9: 新增插件【跳过下载图片数少的章节】,可过滤一些公告章节; 新增Downloadable基类,对插件扩展更友好; 修复本子名过长超出导致oserror的错误; 升级curl_cffi. (#222)
This commit is contained in:
parent
ee23037f94
commit
dec2d54459
@ -198,6 +198,11 @@ plugins:
|
||||
zip_enable: true # 对收藏夹进行压缩
|
||||
zip_filepath: ${JM_DOWNLOAD_DIR}/export.zip # 压缩文件路径
|
||||
zip_password: ${ZIP_PASSWORD} # 压缩密码
|
||||
|
||||
before_photo:
|
||||
- plugin: skip_photo_with_few_images # 跳过下载章节图片数量过少的章节。一些韩漫的章节是公告,没有实际内容,就可以用该插件来跳过下载这些章节。
|
||||
kwargs:
|
||||
at_least_image_count: 3
|
||||
|
||||
after_photo:
|
||||
- plugin: j2p # jpg图片合成为一个pdf插件
|
||||
|
@ -1,5 +1,5 @@
|
||||
commonX
|
||||
curl_cffi
|
||||
commonX
|
||||
PyYAML
|
||||
Pillow
|
||||
psutil
|
||||
|
2
setup.py
2
setup.py
@ -27,8 +27,8 @@ setup(
|
||||
package_dir={"": "src"},
|
||||
python_requires=">=3.7",
|
||||
install_requires=[
|
||||
'commonX>=0.6.4',
|
||||
'curl_cffi',
|
||||
'commonX',
|
||||
'PyYAML',
|
||||
'Pillow',
|
||||
'pycryptodome',
|
||||
|
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.5.8'
|
||||
__version__ = '2.5.9'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
@ -224,7 +224,7 @@ class AbstractJmClient(
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def decode(self, url: str):
|
||||
if not JmModuleConfig.flag_decode_url_when_logging or '/search/' not in url:
|
||||
if not JmModuleConfig.FLAG_DECODE_URL_WHEN_LOGGING or '/search/' not in url:
|
||||
return url
|
||||
|
||||
from urllib.parse import unquote
|
||||
@ -767,7 +767,7 @@ class JmApiClient(AbstractJmClient):
|
||||
# 检查禁漫最新的版本号
|
||||
setting_ver = str(resp.model_data.version)
|
||||
# 禁漫接口的版本 > jmcomic库内置版本
|
||||
if setting_ver > JmMagicConstants.APP_VERSION and JmModuleConfig.flag_use_version_newer_if_behind:
|
||||
if setting_ver > JmMagicConstants.APP_VERSION and JmModuleConfig.FLAG_USE_VERSION_NEWER_IF_BEHIND:
|
||||
jm_log('api.setting', f'change APP_VERSION from [{JmMagicConstants.APP_VERSION}] to [{setting_ver}]')
|
||||
JmMagicConstants.APP_VERSION = setting_ver
|
||||
|
||||
@ -883,7 +883,7 @@ class JmApiClient(AbstractJmClient):
|
||||
ts = time_stamp()
|
||||
token, tokenparam = JmCryptoTool.token_and_tokenparam(ts, secret=JmMagicConstants.APP_TOKEN_SECRET_2)
|
||||
|
||||
elif JmModuleConfig.flag_use_fix_timestamp:
|
||||
elif JmModuleConfig.FLAG_USE_FIX_TIMESTAMP:
|
||||
ts, token, tokenparam = JmModuleConfig.get_fix_ts_token_tokenparam()
|
||||
|
||||
else:
|
||||
@ -954,7 +954,7 @@ class JmApiClient(AbstractJmClient):
|
||||
|
||||
def after_init(self):
|
||||
# 保证拥有cookies,因为移动端要求必须携带cookies,否则会直接跳转同一本子【禁漫娘】
|
||||
if JmModuleConfig.flag_api_client_require_cookies:
|
||||
if JmModuleConfig.FLAG_API_CLIENT_REQUIRE_COOKIES:
|
||||
self.ensure_have_cookies()
|
||||
|
||||
client_init_cookies_lock = Lock()
|
||||
|
@ -146,18 +146,18 @@ class JmModuleConfig:
|
||||
REGISTRY_EXCEPTION_LISTENER = {}
|
||||
|
||||
# 执行log的函数
|
||||
executor_log = default_jm_logging
|
||||
EXECUTOR_LOG = default_jm_logging
|
||||
|
||||
# 使用固定时间戳
|
||||
flag_use_fix_timestamp = True
|
||||
FLAG_USE_FIX_TIMESTAMP = True
|
||||
# 移动端Client初始化cookies
|
||||
flag_api_client_require_cookies = True
|
||||
FLAG_API_CLIENT_REQUIRE_COOKIES = True
|
||||
# log开关标记
|
||||
flag_enable_jm_log = True
|
||||
FLAG_ENABLE_JM_LOG = True
|
||||
# log时解码url
|
||||
flag_decode_url_when_logging = True
|
||||
FLAG_DECODE_URL_WHEN_LOGGING = True
|
||||
# 当内置的版本号落后时,使用最新的禁漫app版本号
|
||||
flag_use_version_newer_if_behind = True
|
||||
FLAG_USE_VERSION_NEWER_IF_BEHIND = True
|
||||
|
||||
# 关联dir_rule的自定义字段与对应的处理函数
|
||||
# 例如:
|
||||
@ -165,6 +165,10 @@ class JmModuleConfig:
|
||||
AFIELD_ADVICE = dict()
|
||||
PFIELD_ADVICE = dict()
|
||||
|
||||
# 当发生 oserror: [Errno 36] File name too long 时,
|
||||
# 把文件名限制在指定个字符以内
|
||||
VAR_FILE_NAME_LENGTH_LIMIT = 100
|
||||
|
||||
@classmethod
|
||||
def downloader_class(cls):
|
||||
if cls.CLASS_DOWNLOADER is not None:
|
||||
@ -319,12 +323,12 @@ class JmModuleConfig:
|
||||
# noinspection PyUnusedLocal
|
||||
@classmethod
|
||||
def jm_log(cls, topic: str, msg: str):
|
||||
if cls.flag_enable_jm_log is True:
|
||||
cls.executor_log(topic, msg)
|
||||
if cls.FLAG_ENABLE_JM_LOG is True:
|
||||
cls.EXECUTOR_LOG(topic, msg)
|
||||
|
||||
@classmethod
|
||||
def disable_jm_log(cls):
|
||||
cls.flag_enable_jm_log = False
|
||||
cls.FLAG_ENABLE_JM_LOG = False
|
||||
|
||||
@classmethod
|
||||
def new_postman(cls, session=False, **kwargs):
|
||||
@ -347,7 +351,7 @@ class JmModuleConfig:
|
||||
DEFAULT_CLIENT_CACHE = None # 默认关闭Client缓存。缓存的配置详见 CacheRegistry
|
||||
DEFAULT_PROXIES = ProxyBuilder.system_proxy() # 默认使用系统代理
|
||||
|
||||
default_option_dict: dict = {
|
||||
DEFAULT_OPTION_DICT: dict = {
|
||||
'log': None,
|
||||
'dir_rule': {'rule': 'Bd_Pname', 'base_dir': None},
|
||||
'download': {
|
||||
@ -364,7 +368,7 @@ class JmModuleConfig:
|
||||
'postman': {
|
||||
'type': 'cffi',
|
||||
'meta_data': {
|
||||
'impersonate': 'chrome110',
|
||||
'impersonate': 'chrome',
|
||||
'headers': None,
|
||||
'proxies': None,
|
||||
}
|
||||
@ -387,11 +391,11 @@ class JmModuleConfig:
|
||||
"""
|
||||
from copy import deepcopy
|
||||
|
||||
option_dict = deepcopy(cls.default_option_dict)
|
||||
option_dict = deepcopy(cls.DEFAULT_OPTION_DICT)
|
||||
|
||||
# log
|
||||
if option_dict['log'] is None:
|
||||
option_dict['log'] = cls.flag_enable_jm_log
|
||||
option_dict['log'] = cls.FLAG_ENABLE_JM_LOG
|
||||
|
||||
# dir_rule.base_dir
|
||||
dir_rule = option_dict['dir_rule']
|
||||
|
@ -29,7 +29,7 @@ class DownloadCallback:
|
||||
f'章节下载完成: [{photo.id}] ({photo.album_id}[{photo.index}/{len(photo.from_album)}])')
|
||||
|
||||
def before_image(self, image: JmImageDetail, img_save_path):
|
||||
if image.is_exists:
|
||||
if image.exists:
|
||||
jm_log('image.before',
|
||||
f'图片已存在: {image.tag} ← [{img_save_path}]'
|
||||
)
|
||||
@ -63,6 +63,8 @@ class JmDownloader(DownloadCallback):
|
||||
|
||||
def download_by_album_detail(self, album: JmAlbumDetail, client: JmcomicClient):
|
||||
self.before_album(album)
|
||||
if album.skip:
|
||||
return
|
||||
self.execute_by_condition(
|
||||
iter_objs=album,
|
||||
apply=lambda photo: self.download_by_photo_detail(photo, client),
|
||||
@ -80,6 +82,8 @@ class JmDownloader(DownloadCallback):
|
||||
client.check_photo(photo)
|
||||
|
||||
self.before_photo(photo)
|
||||
if photo.skip:
|
||||
return
|
||||
self.execute_by_condition(
|
||||
iter_objs=photo,
|
||||
apply=lambda image: self.download_by_image_detail(image, client),
|
||||
@ -91,16 +95,19 @@ class JmDownloader(DownloadCallback):
|
||||
img_save_path = self.option.decide_image_filepath(image)
|
||||
|
||||
image.save_path = img_save_path
|
||||
image.is_exists = file_exists(img_save_path)
|
||||
image.exists = file_exists(img_save_path)
|
||||
|
||||
self.before_image(image, img_save_path)
|
||||
|
||||
if image.skip:
|
||||
return
|
||||
|
||||
# let option decide use_cache and decode_image
|
||||
use_cache = self.option.decide_download_cache(image)
|
||||
decode_image = self.option.decide_download_image_decode(image)
|
||||
|
||||
# skip download
|
||||
if use_cache is True and image.is_exists:
|
||||
if use_cache is True and image.exists:
|
||||
return
|
||||
|
||||
e = None
|
||||
|
@ -3,6 +3,14 @@ from common import *
|
||||
from .jm_config import *
|
||||
|
||||
|
||||
class Downloadable:
|
||||
|
||||
def __init__(self):
|
||||
self.save_path: str = ''
|
||||
self.exists: bool = False
|
||||
self.skip = False
|
||||
|
||||
|
||||
class JmBaseEntity:
|
||||
|
||||
def to_file(self, filepath):
|
||||
@ -117,7 +125,7 @@ class DetailEntity(JmBaseEntity, IndexedEntity):
|
||||
def __str__(self):
|
||||
return f'{self.__class__.__name__}' \
|
||||
'{' \
|
||||
f'{self.id}: {self.title}'\
|
||||
f'{self.id}: {self.title}' \
|
||||
'}'
|
||||
|
||||
@classmethod
|
||||
@ -156,7 +164,7 @@ class DetailEntity(JmBaseEntity, IndexedEntity):
|
||||
return getattr(detail, ref)
|
||||
|
||||
|
||||
class JmImageDetail(JmBaseEntity):
|
||||
class JmImageDetail(JmBaseEntity, Downloadable):
|
||||
|
||||
def __init__(self,
|
||||
aid,
|
||||
@ -167,7 +175,8 @@ class JmImageDetail(JmBaseEntity):
|
||||
from_photo=None,
|
||||
query_params=None,
|
||||
index=-1,
|
||||
) -> None:
|
||||
):
|
||||
super().__init__()
|
||||
if scramble_id is None or (isinstance(scramble_id, str) and scramble_id == ''):
|
||||
from .jm_toolkit import ExceptionTool
|
||||
ExceptionTool.raises(f'图片的scramble_id不能为空')
|
||||
@ -182,10 +191,6 @@ class JmImageDetail(JmBaseEntity):
|
||||
self.query_params: Optional[str] = query_params
|
||||
self.index = index # 从1开始
|
||||
|
||||
# temp fields, in order to simplify passing parameter
|
||||
self.save_path: str = ''
|
||||
self.is_exists: bool = False
|
||||
|
||||
@property
|
||||
def filename_without_suffix(self):
|
||||
return self.img_file_name
|
||||
@ -252,7 +257,7 @@ class JmImageDetail(JmBaseEntity):
|
||||
return True
|
||||
|
||||
|
||||
class JmPhotoDetail(DetailEntity):
|
||||
class JmPhotoDetail(DetailEntity, Downloadable):
|
||||
|
||||
def __init__(self,
|
||||
photo_id,
|
||||
@ -267,6 +272,7 @@ class JmPhotoDetail(DetailEntity):
|
||||
author=None,
|
||||
from_album=None,
|
||||
):
|
||||
super().__init__()
|
||||
self.photo_id: str = str(photo_id)
|
||||
self.scramble_id: str = str(scramble_id)
|
||||
self.name: str = str(name).strip()
|
||||
@ -411,7 +417,7 @@ class JmPhotoDetail(DetailEntity):
|
||||
return True
|
||||
|
||||
|
||||
class JmAlbumDetail(DetailEntity):
|
||||
class JmAlbumDetail(DetailEntity, Downloadable):
|
||||
|
||||
def __init__(self,
|
||||
album_id,
|
||||
@ -430,6 +436,7 @@ class JmAlbumDetail(DetailEntity):
|
||||
tags,
|
||||
related_list=None,
|
||||
):
|
||||
super().__init__()
|
||||
self.album_id: str = str(album_id)
|
||||
self.scramble_id: str = str(scramble_id)
|
||||
self.name: str = name
|
||||
|
@ -72,13 +72,6 @@ class ExceptionTool:
|
||||
CONTEXT_KEY_RE_PATTERN = 'pattern'
|
||||
CONTEXT_KEY_MISSING_JM_ID = 'missing_jm_id'
|
||||
|
||||
# 兼容旧版本
|
||||
|
||||
EXTRA_KEY_RESP = 'resp'
|
||||
EXTRA_KEY_HTML = 'html'
|
||||
EXTRA_KEY_RE_PATTERN = 'pattern'
|
||||
EXTRA_KEY_MISSING_JM_ID = 'missing_jm_id'
|
||||
|
||||
@classmethod
|
||||
def raises(cls,
|
||||
msg: str,
|
||||
@ -144,7 +137,8 @@ class ExceptionTool:
|
||||
:param resp: 响应对象
|
||||
:param jmid: 禁漫本子/章节id
|
||||
"""
|
||||
url = resp.url
|
||||
from .jm_toolkit import JmcomicText
|
||||
url = JmcomicText.format_album_url(jmid)
|
||||
|
||||
req_type = "本子" if "album" in url else "章节"
|
||||
cls.raises(
|
||||
|
@ -285,7 +285,15 @@ class JmOption:
|
||||
)
|
||||
|
||||
if ensure_exists:
|
||||
mkdir_if_not_exists(save_dir)
|
||||
try:
|
||||
mkdir_if_not_exists(save_dir)
|
||||
except OSError as e:
|
||||
if e.errno == 36:
|
||||
# 目录名过长
|
||||
limit = JmModuleConfig.VAR_FILE_NAME_LENGTH_LIMIT
|
||||
jm_log('error', f'目录名过长,无法创建目录,强制缩短到{limit}个字符并重试')
|
||||
save_dir = save_dir[0:limit]
|
||||
mkdir_if_not_exists(save_dir)
|
||||
|
||||
return save_dir
|
||||
|
||||
@ -359,7 +367,7 @@ class JmOption:
|
||||
def deconstruct(self) -> Dict:
|
||||
return {
|
||||
'version': JmModuleConfig.JM_OPTION_VER,
|
||||
'log': JmModuleConfig.flag_enable_jm_log,
|
||||
'log': JmModuleConfig.FLAG_ENABLE_JM_LOG,
|
||||
'dir_rule': {
|
||||
'rule': self.dir_rule.rule_dsl,
|
||||
'base_dir': self.dir_rule.base_dir,
|
||||
|
@ -445,9 +445,7 @@ class ImageSuffixFilterPlugin(JmOptionPlugin):
|
||||
if image.img_file_suffix not in allowed_suffix_set:
|
||||
self.log(f'跳过下载图片: {image.tag},'
|
||||
f'因为其后缀\'{image.img_file_suffix}\'不在允许的后缀集合{allowed_suffix_set}内')
|
||||
# hook is_exists True to skip download
|
||||
image.is_exists = True
|
||||
return True
|
||||
image.skip = True
|
||||
|
||||
# let option decide
|
||||
return option_decide_cache(image)
|
||||
@ -484,7 +482,7 @@ class LogTopicFilterPlugin(JmOptionPlugin):
|
||||
if whitelist is not None:
|
||||
whitelist = set(whitelist)
|
||||
|
||||
old_jm_log = JmModuleConfig.executor_log
|
||||
old_jm_log = JmModuleConfig.EXECUTOR_LOG
|
||||
|
||||
def new_jm_log(topic, msg):
|
||||
if whitelist is not None and topic not in whitelist:
|
||||
@ -492,7 +490,7 @@ class LogTopicFilterPlugin(JmOptionPlugin):
|
||||
|
||||
old_jm_log(topic, msg)
|
||||
|
||||
JmModuleConfig.executor_log = new_jm_log
|
||||
JmModuleConfig.EXECUTOR_LOG = new_jm_log
|
||||
|
||||
|
||||
class AutoSetBrowserCookiesPlugin(JmOptionPlugin):
|
||||
@ -963,3 +961,34 @@ class SubscribeAlbumUpdatePlugin(JmOptionPlugin):
|
||||
is_new_photo = True
|
||||
|
||||
return len(photo_new_list) != 0, photo_new_list
|
||||
|
||||
|
||||
class SkipPhotoWithFewImagesPlugin(JmOptionPlugin):
|
||||
plugin_key = 'skip_photo_with_few_images'
|
||||
|
||||
def invoke(self,
|
||||
at_least_image_count: int,
|
||||
photo: Optional[JmPhotoDetail] = None,
|
||||
image: Optional[JmImageDetail] = None,
|
||||
album: Optional[JmAlbumDetail] = None,
|
||||
**kwargs
|
||||
):
|
||||
self.try_mark_photo_skip_and_log(photo, at_least_image_count)
|
||||
if image is not None:
|
||||
self.try_mark_photo_skip_and_log(image.from_photo, at_least_image_count)
|
||||
|
||||
def try_mark_photo_skip_and_log(self, photo: JmPhotoDetail, at_least_image_count: int):
|
||||
if photo is None:
|
||||
return
|
||||
|
||||
if len(photo) >= at_least_image_count:
|
||||
return
|
||||
|
||||
self.log(f'跳过下载章节: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}]),'
|
||||
f'因为其图片数: {len(photo)} < {at_least_image_count} (at_least_image_count)')
|
||||
photo.skip = True
|
||||
|
||||
@classmethod
|
||||
@field_cache() # 单例
|
||||
def build(cls, option: JmOption) -> 'JmOptionPlugin':
|
||||
return super().build(option)
|
||||
|
@ -234,7 +234,7 @@ class Test_Client(JmTestConfigurable):
|
||||
self.assertEqual(ans, id(photo))
|
||||
|
||||
def test_search_generator(self):
|
||||
JmModuleConfig.flag_decode_url_when_logging = False
|
||||
JmModuleConfig.FLAG_DECODE_URL_WHEN_LOGGING = False
|
||||
|
||||
gen = self.client.search_gen('MANA')
|
||||
for i, page in enumerate(gen):
|
||||
|
Loading…
Reference in New Issue
Block a user