v2.4.13: 更新禁漫网页的正则表达式,完善【多图合并pdf】插件的一些细节。 (#187)

This commit is contained in:
hect0x7 2024-01-04 22:29:43 +08:00 committed by GitHub
parent d0f408203d
commit 06237f83f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 142 additions and 57 deletions

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.4.12'
__version__ = '2.4.13'
from .api import *
from .jm_plugin import *

View File

@ -71,7 +71,7 @@ class JmImageResp(JmResp):
JmImageTool.save_resp_img(
self,
path,
need_convert=suffix_not_equal(img_url, path),
need_convert=suffix_not_equal(img_url[:img_url.find("?")], path),
)
else:
# 解密图片并保存文件

View File

@ -242,14 +242,16 @@ class JmOption:
# 非动图,以配置为先
return self.download.image.suffix or image.img_file_suffix
def decide_image_save_dir(self, photo) -> str:
def decide_image_save_dir(self, photo, ensure_exists=True) -> str:
# 使用 self.dir_rule 决定 save_dir
save_dir = self.dir_rule.decide_image_save_dir(
photo.from_album,
photo
)
mkdir_if_not_exists(save_dir)
if ensure_exists:
mkdir_if_not_exists(save_dir)
return save_dir
def decide_image_filepath(self, image: JmImageDetail, consider_custom_suffix=True) -> str:
@ -504,7 +506,7 @@ class JmOption:
plugin_registry = JmModuleConfig.REGISTRY_PLUGIN
for pinfo in plugin_list:
key, kwargs = pinfo['plugin'], pinfo['kwargs']
key, kwargs = pinfo['plugin'], pinfo.get('kwargs', None) # kwargs为None
plugin_class: Optional[Type[JmOptionPlugin]] = plugin_registry.get(key, None)
ExceptionTool.require_true(plugin_class is not None, f'[{group}] 未注册的plugin: {key}')
@ -517,7 +519,7 @@ class JmOption:
else:
raise e
def invoke_plugin(self, plugin_class, kwargs: Any, extra: dict, pinfo: dict):
def invoke_plugin(self, plugin_class, kwargs: Optional[Dict], extra: dict, pinfo: dict):
# 检查插件的参数类型
kwargs = self.fix_kwargs(kwargs)
# 把插件的配置数据kwargs和附加数据extra合并extra会覆盖kwargs
@ -591,15 +593,18 @@ class JmOption:
raise e
# noinspection PyMethodMayBeStatic
def fix_kwargs(self, kwargs) -> Dict[str, Any]:
def fix_kwargs(self, kwargs: Optional[Dict]) -> Dict[str, Any]:
"""
kwargs将来要传给方法参数这要求kwargs的key是str类型
该方法检查kwargs的key的类型如果不是str尝试转为str不行则抛异常
"""
ExceptionTool.require_true(
isinstance(kwargs, dict),
f'插件的kwargs参数必须为dict类型而不能是类型: {type(kwargs)}'
)
if kwargs is None:
kwargs = {}
else:
ExceptionTool.require_true(
isinstance(kwargs, dict),
f'插件的kwargs参数必须为dict类型而不能是类型: {type(kwargs)}'
)
kwargs: dict
new_kwargs: Dict[str, Any] = {}

View File

@ -18,6 +18,7 @@ class JmOptionPlugin:
def __init__(self, option: JmOption):
self.option = option
self.log_enable = True
self.delete_original_file = False
def invoke(self, **kwargs) -> None:
"""
@ -66,6 +67,33 @@ class JmOptionPlugin:
import warnings
warnings.warn(msg)
def execute_deletion(self, paths: List[str]):
"""
删除文件和文件夹
:param paths: 路径列表
"""
if self.delete_original_file is not True:
return
for p in paths:
if file_not_exists(p):
continue
if os.path.isdir(p):
os.rmdir(p)
self.log(f'删除文件夹: {p}', 'remove')
else:
os.remove(p)
self.log(f'删除原文件: {p}', 'remove')
# noinspection PyMethodMayBeStatic
def execute_cmd(self, cmd):
"""
执行shell命令这里采用简单的实现
:param cmd: shell命令
"""
return os.system(cmd)
class JmLoginPlugin(JmOptionPlugin):
"""
@ -342,12 +370,16 @@ class ZipPlugin(JmOptionPlugin):
return self.unified_path(source_dir)
def after_zip(self, dir_zip_dict: Dict[str, Optional[str]]):
# 是否要删除所有原文件
if self.delete_original_file is True:
self.delete_all_files_and_empty_dir(
all_downloaded=self.downloader.all_downloaded,
dir_list=list(dir_zip_dict.keys())
)
# 删除所有原文件
dirs = sorted(dir_zip_dict.keys(), reverse=True)
image_paths = [
path
for photo_dict in self.downloader.all_downloaded.values()
for image_list in photo_dict.values()
for path, image in image_list
]
self.execute_deletion(image_paths)
self.execute_deletion(dirs)
# noinspection PyMethodMayBeStatic
def get_zip_path(self, album, photo, filename_rule, suffix, zip_dir):
@ -361,28 +393,6 @@ class ZipPlugin(JmOptionPlugin):
filename + fix_suffix(suffix),
)
# noinspection PyMethodMayBeStatic
def delete_all_files_and_empty_dir(self, all_downloaded: dict, dir_list: List[str]):
"""
删除所有文件和文件夹
"""
import os
for photo_dict in all_downloaded.values():
for image_list in photo_dict.values():
for f, _ in image_list:
# check not exist
if file_not_exists(f):
continue
os.remove(f)
self.log(f'删除原文件: {f}', 'remove')
for d in sorted(dir_list, reverse=True):
# check exist
if file_exists(d):
os.rmdir(d)
self.log(f'删除文件夹: {d}', 'remove')
class ClientProxyPlugin(JmOptionPlugin):
plugin_key = 'client_proxy'
@ -581,10 +591,7 @@ class FavoriteFolderExportPlugin(JmOptionPlugin):
else:
self.zip_with_password()
# 删除导出的原文件
if self.delete_original_file is True:
for f in self.files:
os.remove(f)
self.execute_deletion(self.files)
def handle_folder(self, fid: str, fname: str):
self.log(f'【收藏夹: {fname}, fid: {fid}】开始获取数据')
@ -646,7 +653,7 @@ class FavoriteFolderExportPlugin(JmOptionPlugin):
os.chdir(self.save_dir)
cmd = f'7z a "{self.zip_filepath}" "{self.save_dir}" -p{self.zip_password} -mhe=on'
self.require_true(
0 == os.system(cmd),
0 == self.execute_cmd(cmd),
'加密压缩文件失败'
)
@ -654,27 +661,69 @@ class FavoriteFolderExportPlugin(JmOptionPlugin):
class ConvertJpgToPdfPlugin(JmOptionPlugin):
plugin_key = 'j2p'
def check_image_suffix_is_valid(self, std_suffix):
"""
检查option配置的图片后缀转换目前限制使用Magick时只能搭配jpg
暂不探究Magick是否支持更多图片格式
"""
cur_suffix: Optional[str] = self.option.download.image.suffix
ExceptionTool.require_true(
cur_suffix is not None and cur_suffix.endswith(std_suffix),
'请把图片的后缀转换配置为jpg不然无法使用Magick'
f'(当前配置是[{cur_suffix}]\n'
f'配置模板如下: \n'
f'```\n'
f'download:\n'
f' image:\n'
f' suffix: {std_suffix} # 当前配置是{cur_suffix}\n'
f'```'
)
def invoke(self,
photo: JmPhotoDetail,
downloader=None,
pdf_dir=None,
filename_rule='Pid',
quality=100,
delete_original_file=False,
overwrite_cmd=None,
overwrite_jpg=None,
**kwargs,
):
self.delete_original_file = delete_original_file
# 检查图片后缀配置
suffix = overwrite_jpg or '.jpg'
self.check_image_suffix_is_valid(suffix)
# 处理文件夹配置
filename = DirRule.apply_rule_directly(None, photo, filename_rule)
photo_dir = self.option.decide_image_save_dir(photo)
# 处理生成的pdf文件的路径
if pdf_dir is None:
pdf_dir = photo_dir
else:
pdf_dir = fix_filepath(pdf_dir, True)
mkdir_if_not_exists(pdf_dir)
pdf_filepath = f'{pdf_dir}{filename}.pdf'
pdf_filepath = os.path.join(pdf_dir, f'{filename}.pdf')
def get_cmd(suffix='.jpg'):
return f'magick convert -quality {quality} "{photo_dir}*{suffix}" "{pdf_filepath}"'
# 生成命令
def generate_cmd():
return (
overwrite_cmd or
'magick convert -quality {quality} "{photo_dir}*{suffix}" "{pdf_filepath}"'
).format(
quality=quality,
photo_dir=photo_dir,
suffix=suffix,
pdf_filepath=pdf_filepath,
)
cmd = get_cmd()
self.log(f'execute command: {cmd}')
cmd = generate_cmd()
self.log(f'Execute Command: [{cmd}]')
code = self.execute_cmd(cmd)
self.require_true(
@ -686,6 +735,14 @@ class ConvertJpgToPdfPlugin(JmOptionPlugin):
self.log(f'Convert Successfully: JM{photo.id}{pdf_filepath}')
# noinspection PyMethodMayBeStatic
def execute_cmd(self, cmd):
return os.system(cmd)
if downloader is not None:
from .jm_downloader import JmDownloader
downloader: JmDownloader
paths = [
path
for path, image in downloader.all_downloaded[photo.from_album][photo]
]
paths.append(self.option.decide_image_save_dir(photo, ensure_exists=False))
self.execute_deletion(paths)

View File

@ -51,7 +51,7 @@ class JmcomicText:
# 點擊喜歡
pattern_html_album_likes = compile(r'<span id="albim_likes_\d+">(.*?)</span>')
# 觀看
pattern_html_album_views = compile(r'<span>(.*?)</span> (次觀看|观看次数)')
pattern_html_album_views = compile(r'<span>(.*?)</span>\n<span>(次觀看|观看次数)</span>')
# 評論(div)
pattern_html_album_comment_count = compile(r'<div class="badge"[^>]*?id="total_video_comments">(\d+)</div>'), 0
@ -266,6 +266,24 @@ class JmcomicText:
ret.append(w)
char_list.clear()
def find_right_pair(left_pair, i):
stack = [left_pair]
j = i + 1
while j < length and len(stack) != 0:
c = title[j]
if c in bracket_map:
stack.append(c)
elif c == bracket_map[stack[-1]]:
stack.pop()
j += 1
if len(stack) == 0:
return j
else:
return -1
while i < length:
c = title[i]
@ -273,12 +291,12 @@ class JmcomicText:
# 上一个单词结束
add()
# 定位右括号
j = title.find(bracket_map[c], i)
j = find_right_pair(c, i)
ExceptionTool.require_true(j != -1, f'未闭合的 {c}{bracket_map[c]}: {title[i:]}')
# 整个括号的单词结束
add(title[i:j + 1])
add(title[i:j])
# 移动指针
i = j + 1
i = j
else:
char_list.append(c)
i += 1

View File

@ -55,7 +55,7 @@ def main():
def get_option():
# 读取 option 配置文件
option = create_option('../assets/option/option_workflow_download.yml')
option = create_option(os.path.abspath(os.path.join(__file__, '../../assets/option/option_workflow_download.yml')))
# 支持工作流覆盖配置文件的配置
cover_option_config(option)

View File

@ -2,6 +2,11 @@ from jmcomic import *
def prepare_actions_input_and_secrets():
"""
本函数替代对配置文件中的 ${} 的解析函数
目的是为了支持当没有配置环境变量时可以找另一个环境变量来用
"""
def env(match: Match) -> str:
name = match[1]
value = os.getenv(name, '')