mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v2.2.3: 实现DirRule文件夹规则支持调用env环境变量,优化文档和代码示例 (#119)
This commit is contained in:
parent
a847b922ee
commit
a2c0a86646
@ -2,7 +2,7 @@
|
||||
|
||||
plugin:
|
||||
after_init:
|
||||
- plugin: usage-log # 实时打印硬件占用率的插件
|
||||
- plugin: usage_log # 实时打印硬件占用率的插件
|
||||
kwargs:
|
||||
interval: 0.5 # 间隔时间
|
||||
enable_warning: true # 占用过大时发出预警
|
||||
@ -21,5 +21,5 @@ plugin:
|
||||
kwargs:
|
||||
level: photo # 按照章节,一个章节一个压缩文件
|
||||
filename_rule: Ptitle # 压缩文件的命名规则
|
||||
zip_dir: D:/GitProject/dev/pip/jmcomic/ # 压缩文件存放的文件夹
|
||||
zip_dir: D:/jmcomic/zip/ # 压缩文件存放的文件夹
|
||||
delete_original_file: true # 压缩成功后,删除所有原文件和文件夹
|
||||
@ -9,7 +9,7 @@ client:
|
||||
# 插件配置
|
||||
plugin:
|
||||
after_init:
|
||||
- plugin: usage-log # 实时打印硬件占用率的插件
|
||||
- plugin: usage_log # 实时打印硬件占用率的插件
|
||||
kwargs:
|
||||
interval: 0.5 # 间隔时间
|
||||
enable_warning: false # 不告警
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
version: '2.0'
|
||||
|
||||
dir_rule:
|
||||
base_dir: /home/runner/work/jmcomic/download/
|
||||
base_dir: ${JM_DOWNLOAD_DIR}
|
||||
rule: Bd_Aauthor_Atitle_Pindex
|
||||
|
||||
|
||||
@ -14,7 +14,14 @@ client:
|
||||
# 插件配置
|
||||
plugin:
|
||||
after_init:
|
||||
- plugin: usage-log # 实时打印硬件占用率的插件
|
||||
- plugin: usage_log # 实时打印硬件占用率的插件
|
||||
kwargs:
|
||||
interval: 0.5 # 间隔时间
|
||||
enable_warning: false # 不告警
|
||||
|
||||
# after_album:
|
||||
# - plugin: zip # 压缩文件插件
|
||||
# kwargs:
|
||||
# level: album # 按照章节,一个章节一个压缩文件
|
||||
# filename_rule: Atitle # 压缩文件的命名规则
|
||||
# zip_dir: ${JM_ZIP_DIR} # 压缩文件存放的文件夹
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.2.2'
|
||||
__version__ = '2.2.3'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
||||
@ -23,7 +23,7 @@ class DownloadCallback:
|
||||
f'章节数: [{len(album)}], '
|
||||
f'总页数: [{album.page_count}], '
|
||||
f'标题: [{album.title}], '
|
||||
f'关键词: [{album.keywords}], '
|
||||
f'关键词: [{album.keywords}]'
|
||||
)
|
||||
|
||||
def after_album(self, album: JmAlbumDetail):
|
||||
|
||||
@ -22,7 +22,7 @@ class DirRule:
|
||||
rule_solver_cache: Dict[str, RuleSolver] = {}
|
||||
|
||||
def __init__(self, rule: str, base_dir=None):
|
||||
base_dir = self.parse_dsl(base_dir)
|
||||
base_dir = JmcomicText.parse_to_abspath(base_dir)
|
||||
self.base_dir = base_dir
|
||||
self.rule_dsl = rule
|
||||
self.solver_list = self.get_role_solver_list(rule, base_dir)
|
||||
@ -114,16 +114,6 @@ class DirRule:
|
||||
def apply_rule_directly(cls, album, photo, rule: str) -> str:
|
||||
return cls.apply_rule_solver(album, photo, cls.get_rule_solver(rule))
|
||||
|
||||
dsl_support = {
|
||||
'${workspace}': lambda text: workspace(),
|
||||
}
|
||||
|
||||
def parse_dsl(self, base_dir: str):
|
||||
for k, func in self.dsl_support.items():
|
||||
if k in base_dir:
|
||||
base_dir = base_dir.replace(k, func(base_dir))
|
||||
return base_dir
|
||||
|
||||
|
||||
class JmOption:
|
||||
JM_OP_VER = '2.0'
|
||||
|
||||
@ -51,7 +51,7 @@ class JmLoginPlugin(JmOptionPlugin):
|
||||
|
||||
|
||||
class UsageLogPlugin(JmOptionPlugin):
|
||||
plugin_key = 'usage-log'
|
||||
plugin_key = 'usage_log'
|
||||
|
||||
def invoke(self, **kwargs) -> None:
|
||||
import threading
|
||||
@ -199,6 +199,7 @@ class ZipPlugin(JmOptionPlugin):
|
||||
from .jm_downloader import JmDownloader
|
||||
downloader: JmDownloader
|
||||
|
||||
zip_dir = JmcomicText.parse_to_abspath(zip_dir)
|
||||
mkdir_if_not_exists(zip_dir)
|
||||
photo_dict = downloader.all_downloaded[album]
|
||||
original_file_dir_list = []
|
||||
|
||||
@ -166,6 +166,38 @@ class JmcomicText:
|
||||
assert isinstance(domain, str) and len(domain) != 0
|
||||
return f'{JmModuleConfig.PROT}{domain}{path}'
|
||||
|
||||
class DSLReplacer:
|
||||
|
||||
def __init__(self):
|
||||
self.dsl_dict: Dict[Pattern, Callable[[Match], str]] = {}
|
||||
|
||||
def parse_dsl_text(self, text) -> str:
|
||||
for pattern, replacer in self.dsl_dict.items():
|
||||
text = pattern.sub(replacer, text)
|
||||
return text
|
||||
|
||||
def add_dsl_and_replacer(self, dsl: str, replacer: Callable[[Match], str]):
|
||||
pattern = compile(dsl)
|
||||
self.dsl_dict[pattern] = replacer
|
||||
|
||||
@classmethod
|
||||
def match_os_env(cls, match: Match) -> str:
|
||||
name = match[1]
|
||||
value = os.getenv(name, None)
|
||||
assert value is not None, f"未配置环境变量: {name}"
|
||||
return os.path.abspath(value)
|
||||
|
||||
dsl_replacer = DSLReplacer()
|
||||
|
||||
@classmethod
|
||||
def parse_to_abspath(cls, dsl_text: str) -> str:
|
||||
path = cls.dsl_replacer.parse_dsl_text(dsl_text)
|
||||
return os.path.abspath(path)
|
||||
|
||||
|
||||
# 支持dsl: #{???} -> os.getenv(???)
|
||||
JmcomicText.dsl_replacer.add_dsl_and_replacer('\$\{(.*?)\}', JmcomicText.match_os_env)
|
||||
|
||||
|
||||
class JmSearchSupport:
|
||||
# 用来缩减html的长度
|
||||
|
||||
@ -14,6 +14,7 @@ sys.stderr = io.TextIOWrapper(sys.stderr.buffer, 'utf-8')
|
||||
class JmTestConfigurable(unittest.TestCase):
|
||||
option: JmOption = None
|
||||
client: JmcomicClient = None
|
||||
project_dir: str = None
|
||||
|
||||
def setUp(self) -> None:
|
||||
print_sep('>')
|
||||
@ -24,10 +25,8 @@ class JmTestConfigurable(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# 获取项目根目录
|
||||
application_workspace = os.path.abspath(os.path.dirname(__file__) + '/../..')
|
||||
|
||||
# 设置 workspace → assets/
|
||||
set_application_workspace(f'{application_workspace}/assets/')
|
||||
cls.project_dir = os.path.abspath(os.path.dirname(__file__) + '/../..')
|
||||
os.chdir(cls.project_dir)
|
||||
|
||||
# 设置 JmOption,JmcomicClient
|
||||
option = cls.use_option('option_test.yml')
|
||||
@ -37,13 +36,9 @@ class JmTestConfigurable(unittest.TestCase):
|
||||
# 跨平台设置
|
||||
cls.adapt_os()
|
||||
|
||||
@staticmethod
|
||||
def use_option(op_filename: str) -> JmOption:
|
||||
return create_option(workspace(f"/config/{op_filename}"))
|
||||
|
||||
@staticmethod
|
||||
def move_workspace(new_dir: str):
|
||||
set_application_workspace(workspace(f"/{new_dir}/", is_dir=True))
|
||||
@classmethod
|
||||
def use_option(cls, op_filename: str) -> JmOption:
|
||||
return create_option(f'./assets/config/{op_filename}')
|
||||
|
||||
@classmethod
|
||||
def adapt_os(cls):
|
||||
|
||||
@ -3,11 +3,6 @@ from test_jmcomic import *
|
||||
|
||||
class Test_Api(JmTestConfigurable):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
cls.move_workspace('download')
|
||||
|
||||
def test_download_photo_by_id(self):
|
||||
"""
|
||||
测试jmcomic模块的api的使用
|
||||
|
||||
@ -3,18 +3,11 @@ from test_jmcomic import *
|
||||
|
||||
class Test_Client(JmTestConfigurable):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
cls.move_workspace('download')
|
||||
|
||||
def test_download_image(self):
|
||||
jm_photo_id = 'JM438516'
|
||||
photo = self.client.get_photo_detail(jm_photo_id, False)
|
||||
self.client.download_by_image_detail(
|
||||
photo[0],
|
||||
img_save_path=workspace('test_download_image.png')
|
||||
)
|
||||
image = photo[0]
|
||||
self.client.download_by_image_detail(image, self.option.decide_image_filepath(image))
|
||||
|
||||
def test_get_album_detail_by_jm_photo_id(self):
|
||||
album_id = "JM438516"
|
||||
@ -49,7 +42,7 @@ class Test_Client(JmTestConfigurable):
|
||||
photo: JmPhotoDetail = self.client.get_photo_detail(photo_id, False)
|
||||
image = photo[3000]
|
||||
print(image.img_url)
|
||||
self.client.download_by_image_detail(image, workspace('3000.png'))
|
||||
self.client.download_by_image_detail(image, self.option.decide_image_filepath(image))
|
||||
|
||||
def test_album_missing(self):
|
||||
JmModuleConfig.CLASS_EXCEPTION = JmcomicException
|
||||
|
||||
@ -50,7 +50,7 @@ plugin:
|
||||
kwargs:
|
||||
level: photo # 按照章节,一个章节一个压缩文件
|
||||
filename_rule: Ptitle # 压缩文件的命名规则
|
||||
zip_dir: D:/GitProject/dev/pip/jmcomic/ # 压缩文件存放的文件夹
|
||||
zip_dir: D:/jmcomic # 压缩文件存放的文件夹
|
||||
delete_original_file: true # 压缩成功后,删除所有原文件和文件夹
|
||||
"""
|
||||
|
||||
@ -64,7 +64,8 @@ plugin:
|
||||
2023-08-01 02:23:58:【plugin.kwargs】插件参数类型转换: 145504 (<class 'int'>) -> 145504 (<class 'str'>)
|
||||
2023-08-01 02:23:58:【plugin.invoke】调用插件: [find_update]
|
||||
2023-08-01 02:23:58:【html】https://18comic.vip/album/145504
|
||||
2023-08-01 02:23:59:【album.before】本子获取成功: [145504], 作者: [G.HO], 章节数: [104], 总页数: [3598], 标题: [健身教练 / もしも、幼馴染を抱いたなら / Fitness], 关键词: [['完結', '韩漫', '上柱香再走', '以晨:我爹呢', '你妈真香', '連載中', '肌肉豪名不虚传']], 2023-08-01 02:23:59:【html】https://18comic.vip/photo/291640
|
||||
2023-08-01 02:23:59:【album.before】本子获取成功: [145504], 作者: [G.HO], 章节数: [104], 总页数: [3598], 标题: [健身教练 / もしも、幼馴染を抱いたなら / Fitness], 关键词: [['完結', '韩漫', '上柱香再走', '以晨:我爹呢', '你妈真香', '連載中', '肌肉豪名不虚传']],
|
||||
2023-08-01 02:23:59:【html】https://18comic.vip/photo/291640
|
||||
2023-08-01 02:24:00:【photo.before】开始下载章节: 291640 (145504[104/104]), 标题: [健身教练 完結], 图片数为[41]
|
||||
2023-08-01 02:24:00:【image.before】图片准备下载: 291640/00001.webp [1/41], [https://cdn-msp.18comic.vip/media/photos/291640/00001.webp] → [D:/jmcomic/145504/00001.jpg]
|
||||
2023-08-01 02:24:00:【image.before】图片准备下载: 291640/00004.webp [4/41], [https://cdn-msp.18comic.vip/media/photos/291640/00004.webp] → [D:/jmcomic/145504/00004.jpg]
|
||||
@ -151,7 +152,7 @@ plugin:
|
||||
2023-08-01 02:24:08:【photo.after】章节下载完成: [291640] (145504[104/104])
|
||||
2023-08-01 02:24:08:【album.after】本子下载完成: [145504]
|
||||
2023-08-01 02:24:08:【plugin.invoke】调用插件: [zip]
|
||||
2023-08-01 02:24:08:【plugin.zip.finish】压缩章节[291640]成功 → D:/GitProject/dev/pip/jmcomic/健身教练 完結.zip
|
||||
2023-08-01 02:24:08:【plugin.zip.finish】压缩章节[291640]成功 → D:/jmcomic/健身教练 完結.zip
|
||||
2023-08-01 02:24:08:【plugin.zip.remove】移除原文件: D:/jmcomic/145504/00017.jpg
|
||||
2023-08-01 02:24:08:【plugin.zip.remove】移除原文件: D:/jmcomic/145504/00007.jpg
|
||||
2023-08-01 02:24:08:【plugin.zip.remove】移除原文件: D:/jmcomic/145504/00021.jpg
|
||||
|
||||
Loading…
Reference in New Issue
Block a user