v2.2.3: 实现DirRule文件夹规则支持调用env环境变量,优化文档和代码示例 (#119)

This commit is contained in:
hect0x7 2023-09-01 17:00:05 +08:00 committed by GitHub
parent a847b922ee
commit a2c0a86646
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 62 additions and 48 deletions

View File

@ -2,7 +2,7 @@
plugin:
after_init:
- plugin: usage-log # 实时打印硬件占用率的插件
- plugin: usage_log # 实时打印硬件占用率的插件
kwargs:
interval: 0.5 # 间隔时间
enable_warning: true # 占用过大时发出预警
@ -21,5 +21,5 @@ plugin:
kwargs:
level: photo # 按照章节,一个章节一个压缩文件
filename_rule: Ptitle # 压缩文件的命名规则
zip_dir: D:/GitProject/dev/pip/jmcomic/ # 压缩文件存放的文件夹
zip_dir: D:/jmcomic/zip/ # 压缩文件存放的文件夹
delete_original_file: true # 压缩成功后,删除所有原文件和文件夹

View File

@ -9,7 +9,7 @@ client:
# 插件配置
plugin:
after_init:
- plugin: usage-log # 实时打印硬件占用率的插件
- plugin: usage_log # 实时打印硬件占用率的插件
kwargs:
interval: 0.5 # 间隔时间
enable_warning: false # 不告警

View File

@ -2,7 +2,7 @@
version: '2.0'
dir_rule:
base_dir: /home/runner/work/jmcomic/download/
base_dir: ${JM_DOWNLOAD_DIR}
rule: Bd_Aauthor_Atitle_Pindex
@ -14,7 +14,14 @@ client:
# 插件配置
plugin:
after_init:
- plugin: usage-log # 实时打印硬件占用率的插件
- plugin: usage_log # 实时打印硬件占用率的插件
kwargs:
interval: 0.5 # 间隔时间
enable_warning: false # 不告警
# after_album:
# - plugin: zip # 压缩文件插件
# kwargs:
# level: album # 按照章节,一个章节一个压缩文件
# filename_rule: Atitle # 压缩文件的命名规则
# zip_dir: ${JM_ZIP_DIR} # 压缩文件存放的文件夹

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.2.2'
__version__ = '2.2.3'
from .api import *
from .jm_plugin import *

View File

@ -23,7 +23,7 @@ class DownloadCallback:
f'章节数: [{len(album)}], '
f'总页数: [{album.page_count}], '
f'标题: [{album.title}], '
f'关键词: [{album.keywords}], '
f'关键词: [{album.keywords}]'
)
def after_album(self, album: JmAlbumDetail):

View File

@ -22,7 +22,7 @@ class DirRule:
rule_solver_cache: Dict[str, RuleSolver] = {}
def __init__(self, rule: str, base_dir=None):
base_dir = self.parse_dsl(base_dir)
base_dir = JmcomicText.parse_to_abspath(base_dir)
self.base_dir = base_dir
self.rule_dsl = rule
self.solver_list = self.get_role_solver_list(rule, base_dir)
@ -114,16 +114,6 @@ class DirRule:
def apply_rule_directly(cls, album, photo, rule: str) -> str:
return cls.apply_rule_solver(album, photo, cls.get_rule_solver(rule))
dsl_support = {
'${workspace}': lambda text: workspace(),
}
def parse_dsl(self, base_dir: str):
for k, func in self.dsl_support.items():
if k in base_dir:
base_dir = base_dir.replace(k, func(base_dir))
return base_dir
class JmOption:
JM_OP_VER = '2.0'

View File

@ -51,7 +51,7 @@ class JmLoginPlugin(JmOptionPlugin):
class UsageLogPlugin(JmOptionPlugin):
plugin_key = 'usage-log'
plugin_key = 'usage_log'
def invoke(self, **kwargs) -> None:
import threading
@ -199,6 +199,7 @@ class ZipPlugin(JmOptionPlugin):
from .jm_downloader import JmDownloader
downloader: JmDownloader
zip_dir = JmcomicText.parse_to_abspath(zip_dir)
mkdir_if_not_exists(zip_dir)
photo_dict = downloader.all_downloaded[album]
original_file_dir_list = []

View File

@ -166,6 +166,38 @@ class JmcomicText:
assert isinstance(domain, str) and len(domain) != 0
return f'{JmModuleConfig.PROT}{domain}{path}'
class DSLReplacer:
def __init__(self):
self.dsl_dict: Dict[Pattern, Callable[[Match], str]] = {}
def parse_dsl_text(self, text) -> str:
for pattern, replacer in self.dsl_dict.items():
text = pattern.sub(replacer, text)
return text
def add_dsl_and_replacer(self, dsl: str, replacer: Callable[[Match], str]):
pattern = compile(dsl)
self.dsl_dict[pattern] = replacer
@classmethod
def match_os_env(cls, match: Match) -> str:
name = match[1]
value = os.getenv(name, None)
assert value is not None, f"未配置环境变量: {name}"
return os.path.abspath(value)
dsl_replacer = DSLReplacer()
@classmethod
def parse_to_abspath(cls, dsl_text: str) -> str:
path = cls.dsl_replacer.parse_dsl_text(dsl_text)
return os.path.abspath(path)
# 支持dsl: #{???} -> os.getenv(???)
JmcomicText.dsl_replacer.add_dsl_and_replacer('\$\{(.*?)\}', JmcomicText.match_os_env)
class JmSearchSupport:
# 用来缩减html的长度

View File

@ -14,6 +14,7 @@ sys.stderr = io.TextIOWrapper(sys.stderr.buffer, 'utf-8')
class JmTestConfigurable(unittest.TestCase):
option: JmOption = None
client: JmcomicClient = None
project_dir: str = None
def setUp(self) -> None:
print_sep('>')
@ -24,10 +25,8 @@ class JmTestConfigurable(unittest.TestCase):
@classmethod
def setUpClass(cls):
# 获取项目根目录
application_workspace = os.path.abspath(os.path.dirname(__file__) + '/../..')
# 设置 workspace → assets/
set_application_workspace(f'{application_workspace}/assets/')
cls.project_dir = os.path.abspath(os.path.dirname(__file__) + '/../..')
os.chdir(cls.project_dir)
# 设置 JmOptionJmcomicClient
option = cls.use_option('option_test.yml')
@ -37,13 +36,9 @@ class JmTestConfigurable(unittest.TestCase):
# 跨平台设置
cls.adapt_os()
@staticmethod
def use_option(op_filename: str) -> JmOption:
return create_option(workspace(f"/config/{op_filename}"))
@staticmethod
def move_workspace(new_dir: str):
set_application_workspace(workspace(f"/{new_dir}/", is_dir=True))
@classmethod
def use_option(cls, op_filename: str) -> JmOption:
return create_option(f'./assets/config/{op_filename}')
@classmethod
def adapt_os(cls):

View File

@ -3,11 +3,6 @@ from test_jmcomic import *
class Test_Api(JmTestConfigurable):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.move_workspace('download')
def test_download_photo_by_id(self):
"""
测试jmcomic模块的api的使用

View File

@ -3,18 +3,11 @@ from test_jmcomic import *
class Test_Client(JmTestConfigurable):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.move_workspace('download')
def test_download_image(self):
jm_photo_id = 'JM438516'
photo = self.client.get_photo_detail(jm_photo_id, False)
self.client.download_by_image_detail(
photo[0],
img_save_path=workspace('test_download_image.png')
)
image = photo[0]
self.client.download_by_image_detail(image, self.option.decide_image_filepath(image))
def test_get_album_detail_by_jm_photo_id(self):
album_id = "JM438516"
@ -49,7 +42,7 @@ class Test_Client(JmTestConfigurable):
photo: JmPhotoDetail = self.client.get_photo_detail(photo_id, False)
image = photo[3000]
print(image.img_url)
self.client.download_by_image_detail(image, workspace('3000.png'))
self.client.download_by_image_detail(image, self.option.decide_image_filepath(image))
def test_album_missing(self):
JmModuleConfig.CLASS_EXCEPTION = JmcomicException

View File

@ -50,7 +50,7 @@ plugin:
kwargs:
level: photo # 按照章节,一个章节一个压缩文件
filename_rule: Ptitle # 压缩文件的命名规则
zip_dir: D:/GitProject/dev/pip/jmcomic/ # 压缩文件存放的文件夹
zip_dir: D:/jmcomic # 压缩文件存放的文件夹
delete_original_file: true # 压缩成功后,删除所有原文件和文件夹
"""
@ -64,7 +64,8 @@ plugin:
2023-08-01 02:23:58:plugin.kwargs插件参数类型转换: 145504 (<class 'int'>) -> 145504 (<class 'str'>)
2023-08-01 02:23:58:plugin.invoke调用插件: [find_update]
2023-08-01 02:23:58:htmlhttps://18comic.vip/album/145504
2023-08-01 02:23:59:album.before本子获取成功: [145504], 作者: [G.HO], 章节数: [104], 总页数: [3598], 标题: [健身教练 / もしも幼馴染を抱いたなら / Fitness], 关键词: [['完結', '韩漫', '上柱香再走', '以晨:我爹呢', '你妈真香', '連載中', '肌肉豪名不虚传']], 2023-08-01 02:23:59:htmlhttps://18comic.vip/photo/291640
2023-08-01 02:23:59:album.before本子获取成功: [145504], 作者: [G.HO], 章节数: [104], 总页数: [3598], 标题: [健身教练 / もしも幼馴染を抱いたなら / Fitness], 关键词: [['完結', '韩漫', '上柱香再走', '以晨:我爹呢', '你妈真香', '連載中', '肌肉豪名不虚传']],
2023-08-01 02:23:59:htmlhttps://18comic.vip/photo/291640
2023-08-01 02:24:00:photo.before开始下载章节: 291640 (145504[104/104]), 标题: [健身教练 完結], 图片数为[41]
2023-08-01 02:24:00:image.before图片准备下载: 291640/00001.webp [1/41], [https://cdn-msp.18comic.vip/media/photos/291640/00001.webp] [D:/jmcomic/145504/00001.jpg]
2023-08-01 02:24:00:image.before图片准备下载: 291640/00004.webp [4/41], [https://cdn-msp.18comic.vip/media/photos/291640/00004.webp] [D:/jmcomic/145504/00004.jpg]
@ -151,7 +152,7 @@ plugin:
2023-08-01 02:24:08:photo.after章节下载完成: [291640] (145504[104/104])
2023-08-01 02:24:08:album.after本子下载完成: [145504]
2023-08-01 02:24:08:plugin.invoke调用插件: [zip]
2023-08-01 02:24:08:plugin.zip.finish压缩章节[291640]成功 D:/GitProject/dev/pip/jmcomic/健身教练 完結.zip
2023-08-01 02:24:08:plugin.zip.finish压缩章节[291640]成功 D:/jmcomic/健身教练 完結.zip
2023-08-01 02:24:08:plugin.zip.remove移除原文件: D:/jmcomic/145504/00017.jpg
2023-08-01 02:24:08:plugin.zip.remove移除原文件: D:/jmcomic/145504/00007.jpg
2023-08-01 02:24:08:plugin.zip.remove移除原文件: D:/jmcomic/145504/00021.jpg