mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.2.0: 引入plugin插件机制,增加相关文档,大幅提升可扩展性,希望构建一个插件生态 (#116)
This commit is contained in:
parent
b537a0e683
commit
d019d97a34
@ -40,6 +40,9 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
||||
- 使用API的Filter过滤功能: `usage_feature_filter.py`
|
||||
- 测试你的ip可以访问哪些禁漫域名: `pick_domain.py`
|
||||
- 基于GitHub Actions下载本子: `workflow_download.py`
|
||||
- 演示jmcomic模块的自定义功能点: `usage_custom.py`
|
||||
- 演示jmcomic模块的Plugin插件体系: `usage_plugin.py`
|
||||
|
||||
|
||||
## 项目特点
|
||||
|
||||
@ -50,6 +53,7 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
||||
- 配置可以从**配置文件**生成,支持多种文件格式,无需写Python代码
|
||||
- 配置点有:`是否使用磁盘缓存` `并发下载图片数` `图片类型转换` `下载路径` `请求元信息(headers,cookies,proxies)`等
|
||||
- **可扩展性强**
|
||||
- **支持Plugin插件,可以方便地扩展功能,以及使用别人的插件**
|
||||
- 支持自定义本子/章节/图片下载前后的回调函数
|
||||
- 支持自定义debug日志的开关/格式
|
||||
- 支持自定义Downloader/Option/Client/实体类
|
||||
|
7
assets/config/option_plugin.yml
Normal file
7
assets/config/option_plugin.yml
Normal file
@ -0,0 +1,7 @@
|
||||
# 插件的配置示例
|
||||
|
||||
plugin:
|
||||
after_init:
|
||||
login:
|
||||
username: un
|
||||
password: pw
|
@ -2,6 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.1.21'
|
||||
__version__ = '2.2.0'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
@ -87,7 +87,7 @@ class AbstractJmClient(
|
||||
|
||||
# noinspection PyMethodMayBeStatic, PyUnusedLocal
|
||||
def before_retry(self, e, kwargs, retry_count, url):
|
||||
jm_debug('req.err', str(e))
|
||||
jm_debug('req.error', str(e))
|
||||
|
||||
def enable_cache(self, debug=False):
|
||||
def wrap_func_cache(func_name, cache_dict_name):
|
||||
@ -235,7 +235,6 @@ class JmHtmlClient(AbstractJmClient):
|
||||
resp = self.get(url, **kwargs)
|
||||
|
||||
if require_200 is True and resp.status_code != 200:
|
||||
# write_text('./resp.html', resp.text)
|
||||
self.check_special_http_code(resp)
|
||||
self.raise_request_error(resp)
|
||||
|
||||
@ -342,7 +341,6 @@ class JmHtmlClient(AbstractJmClient):
|
||||
if content not in html:
|
||||
continue
|
||||
|
||||
write_text('./resp.html', html)
|
||||
cls.raise_request_error(
|
||||
resp,
|
||||
f'{reason}'
|
||||
|
@ -78,6 +78,9 @@ class JmModuleConfig:
|
||||
# debug开关标记
|
||||
enable_jm_debug = True
|
||||
|
||||
# 插件注册表
|
||||
plugin_registry = {}
|
||||
|
||||
@classmethod
|
||||
def downloader_class(cls):
|
||||
if cls.CLASS_DOWNLOADER is not None:
|
||||
@ -256,7 +259,8 @@ class JmModuleConfig:
|
||||
},
|
||||
'impl': 'html',
|
||||
'retry_times': 5
|
||||
}
|
||||
},
|
||||
'plugin': {},
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@ -289,6 +293,10 @@ class JmModuleConfig:
|
||||
|
||||
return option_dict
|
||||
|
||||
@classmethod
|
||||
def register_plugin(cls, plugin_class):
|
||||
cls.plugin_registry[plugin_class.plugin_key] = plugin_class
|
||||
|
||||
|
||||
jm_debug = JmModuleConfig.jm_debug
|
||||
disable_jm_debug = JmModuleConfig.disable_jm_debug
|
||||
|
@ -21,7 +21,9 @@ class DownloadCallback:
|
||||
f'本子获取成功: [{album.id}], '
|
||||
f'作者: [{album.author}], '
|
||||
f'章节数: [{len(album)}], '
|
||||
f'总页数: [{album.page_count}], '
|
||||
f'标题: [{album.title}], '
|
||||
f'关键词: [{album.keywords}], '
|
||||
)
|
||||
|
||||
def after_album(self, album: JmAlbumDetail):
|
||||
|
@ -109,6 +109,7 @@ class JmOption:
|
||||
dir_rule: Dict,
|
||||
download: Dict,
|
||||
client: Dict,
|
||||
plugin: Dict,
|
||||
filepath=None,
|
||||
):
|
||||
# 版本号
|
||||
@ -119,9 +120,13 @@ class JmOption:
|
||||
self.client = DictModel(client)
|
||||
# 下载配置
|
||||
self.download = DictModel(download)
|
||||
# 插件配置
|
||||
self.plugin = DictModel(plugin)
|
||||
# 其他配置
|
||||
self.filepath = filepath
|
||||
|
||||
self.call_all_plugin('after_init')
|
||||
|
||||
@property
|
||||
def download_cache(self):
|
||||
return self.download.cache
|
||||
@ -286,3 +291,48 @@ class JmOption:
|
||||
else:
|
||||
default_dict[key] = value
|
||||
return default_dict
|
||||
|
||||
# 下面的方法提供面向对象的调用风格
|
||||
|
||||
def download_album(self, album_id):
|
||||
from .api import download_album
|
||||
download_album(album_id, self)
|
||||
|
||||
def download_album(self, photo_id):
|
||||
from .api import download_album
|
||||
download_album(photo_id, self)
|
||||
|
||||
# 下面的方法为调用插件提供支持
|
||||
def call_all_plugin(self, key: str):
|
||||
plugin_dict: dict = self.plugin.get(key, {})
|
||||
if plugin_dict is None or len(plugin_dict) == 0:
|
||||
return
|
||||
|
||||
# 保证 jm_plugin.py 被加载
|
||||
from .jm_plugin import JmOptionPlugin
|
||||
|
||||
plugin_registry = JmModuleConfig.plugin_registry
|
||||
for name, kwargs in plugin_dict.items():
|
||||
plugin_class: Optional[Type[JmOptionPlugin]] = plugin_registry.get(name, None)
|
||||
|
||||
if plugin_class is None:
|
||||
raise JmModuleConfig.exception(f'[{key}] 未注册的plugin: {name}')
|
||||
|
||||
self.invoke_plugin(plugin_class, kwargs)
|
||||
|
||||
def invoke_plugin(self, plugin_class, kwargs: dict):
|
||||
# 保证 jm_plugin.py 被加载
|
||||
from .jm_plugin import JmOptionPlugin
|
||||
|
||||
plugin_class: Type[JmOptionPlugin]
|
||||
try:
|
||||
plugin = plugin_class.build(self)
|
||||
plugin.invoke(**kwargs)
|
||||
except JmcomicException as e:
|
||||
msg = str(e)
|
||||
jm_debug('plugin.exception', msg)
|
||||
raise JmModuleConfig.exception(msg)
|
||||
except BaseException as e:
|
||||
msg = str(e)
|
||||
jm_debug('plugin.error', msg)
|
||||
raise e
|
||||
|
53
src/jmcomic/jm_plugin.py
Normal file
53
src/jmcomic/jm_plugin.py
Normal file
@ -0,0 +1,53 @@
|
||||
"""
|
||||
该文件存放的是option插件类
|
||||
"""
|
||||
|
||||
from .jm_option import *
|
||||
|
||||
|
||||
class JmOptionPlugin:
|
||||
plugin_key: str
|
||||
|
||||
def __init__(self, option: JmOption):
|
||||
self.option = option
|
||||
|
||||
def invoke(self, **kwargs) -> None:
|
||||
"""
|
||||
执行插件的功能
|
||||
@param kwargs: 给插件的参数
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def build(cls, option: JmOption) -> 'JmOptionPlugin':
|
||||
"""
|
||||
创建插件实例
|
||||
@param option: JmOption对象
|
||||
"""
|
||||
return cls(option)
|
||||
|
||||
|
||||
"""
|
||||
插件功能:登录禁漫,并保存登录后的cookies,让所有client都带上此cookies
|
||||
"""
|
||||
|
||||
|
||||
class LoginPlugin(JmOptionPlugin):
|
||||
plugin_key = 'login'
|
||||
|
||||
def invoke(self, username, password) -> None:
|
||||
assert isinstance(username, str), '用户名必须是str'
|
||||
assert isinstance(password, str), '密码必须是str'
|
||||
|
||||
client = self.option.new_jm_client()
|
||||
client.login(username, password)
|
||||
cookies = client['cookies']
|
||||
|
||||
postman: dict = self.option.client.postman.src_dict
|
||||
meta_data = postman.get('meta_data', {})
|
||||
meta_data['cookies'] = cookies
|
||||
postman['meta_data'] = meta_data
|
||||
jm_debug('plugin.login', '登录成功')
|
||||
|
||||
|
||||
JmModuleConfig.register_plugin(LoginPlugin)
|
162
usage/usage_custom.py
Normal file
162
usage/usage_custom.py
Normal file
@ -0,0 +1,162 @@
|
||||
"""
|
||||
本文件演示对jmcomic模块进行自定义功能的方式,下面的每个函数都是一个独立的演示单元。
|
||||
本文件不演示【自定义配置】,有关配置的教程文档请见 ``
|
||||
"""
|
||||
from jmcomic import *
|
||||
|
||||
option = JmOption.default()
|
||||
client: JmcomicClient = option.build_jm_client()
|
||||
|
||||
|
||||
def custom_download_callback():
|
||||
"""
|
||||
该函数演示自定义下载时的回调函数
|
||||
"""
|
||||
|
||||
# jmcomic的下载功能由 JmModuleConfig.CLASS_DOWNLOADER 这个类来负责执行
|
||||
# 这个类默认是 JmDownloader,继承了DownloadCallback
|
||||
# 你可以写一个自定义类,继承JmDownloader,覆盖属于DownloadCallback的方法,来实现自定义回调
|
||||
class MyDownloader(JmDownloader):
|
||||
# 覆盖 album 下载完成后的回调
|
||||
def after_album(self, album: JmAlbumDetail):
|
||||
print(f'album下载完毕: {album}')
|
||||
pass
|
||||
|
||||
# 同样的,最后要让你的自定义类生效
|
||||
JmModuleConfig.CLASS_DOWNLOADER = MyDownloader
|
||||
|
||||
|
||||
def custom_option_class():
|
||||
"""
|
||||
该函数演示自定义option
|
||||
"""
|
||||
|
||||
# jmcomic模块支持自定义Option类,
|
||||
# 你可以写一个自己的类,继承JmOption,然后覆盖其中的一些方法。
|
||||
class MyOption(JmOption):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
print('MyOption 初始化开始')
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def default(cls):
|
||||
print('调用了MyOption.default()')
|
||||
return super().default()
|
||||
|
||||
# 最后,替换默认Option类即可
|
||||
JmModuleConfig.CLASS_OPTION = MyOption
|
||||
|
||||
|
||||
def custom_client_class():
|
||||
"""
|
||||
该文件演示自定义client类
|
||||
"""
|
||||
|
||||
# 默认情况下,JmOption使用client类是根据配置项 `client.impl` 决定的
|
||||
# JmOption会根据`client.impl`到 JmModuleConfig.CLASS_CLIENT_IMPL 中查找
|
||||
|
||||
# 你可以自定义一个`client.impl`,例如 'my-client',
|
||||
# 或者使用jmcomic内置 'html' 和 'api',
|
||||
# 然后把你的`client.impl`和类一起配置到JmModuleConfig中
|
||||
|
||||
# 1. 自定义Client类
|
||||
class MyClient(JmHtmlClient):
|
||||
pass
|
||||
|
||||
# 2. 让你的配置类生效
|
||||
JmModuleConfig.CLASS_CLIENT_IMPL['my-client'] = MyClient
|
||||
|
||||
# 3. 在配置文件中使用你定义的client.impl,后续使用这个option即可
|
||||
"""
|
||||
client:
|
||||
impl: 'my-client'
|
||||
"""
|
||||
|
||||
|
||||
def custom_album_photo_image_detail_class():
|
||||
"""
|
||||
该函数演示替换实体类(本子/章节/图片)
|
||||
"""
|
||||
|
||||
# 在使用路径规则 DirRule 时,可能会遇到需要自定义实体类属性的情况,例如:
|
||||
"""
|
||||
dir_rule:
|
||||
base_dir: ${workspace}
|
||||
rule: Bd_Acustom_Pcustom
|
||||
"""
|
||||
|
||||
# 上面的Acustom,Pcustom都是自定义字段
|
||||
# 如果你想要使用这种自定义字段,你就需要替换默认的实体类,例如
|
||||
|
||||
# 自定义本子实体类
|
||||
class MyAlbum(JmAlbumDetail):
|
||||
# 自定义 custom 属性
|
||||
@property
|
||||
def custom(self):
|
||||
return f'custom_{self.title}'
|
||||
|
||||
# 自定义章节实体类
|
||||
class MyPhoto(JmPhotoDetail):
|
||||
# 自定义 custom 属性
|
||||
@property
|
||||
def custom(self):
|
||||
return f'custom_{self.title}'
|
||||
|
||||
# 自定义图片实体类
|
||||
class MyImage(JmImageDetail):
|
||||
pass
|
||||
|
||||
# 最后,替换默认实体类来让你的自定义类生效
|
||||
JmModuleConfig.CLASS_ALBUM = MyAlbum
|
||||
JmModuleConfig.CLASS_PHOTO = MyPhoto
|
||||
JmModuleConfig.CLASS_IMAGE = MyImage
|
||||
|
||||
|
||||
def custom_jm_debug():
|
||||
"""
|
||||
该函数演示自定义debug
|
||||
"""
|
||||
|
||||
# jmcomic模块在运行过程中会使用 jm_debug() 这个函数进行打印信息
|
||||
# jm_debug() 这个函数 最后会调用 JmModuleConfig.debug_executor 函数
|
||||
# 你可以写一个自己的函数,替换 JmModuleConfig.debug_executor,实现自定义debug
|
||||
|
||||
# 1. 自定义debug函数
|
||||
def my_debug(topic: str, msg: str):
|
||||
"""
|
||||
这个debug函数的参数列表必须包含两个参数,topic和msg
|
||||
@param topic: debug主题,例如 'album.before', 'req.error', 'plugin.error'
|
||||
@param msg: 具体debug的信息
|
||||
"""
|
||||
pass
|
||||
|
||||
# 2. 让my_debug生效
|
||||
JmModuleConfig.debug_executor = my_debug
|
||||
|
||||
|
||||
def custom_exception_raise():
|
||||
"""
|
||||
该函数演示jmcomic的异常机制
|
||||
"""
|
||||
|
||||
# jmcomic 代码在运行过程中可能抛出异常,以获取album实体类为例:
|
||||
album = client.get_album_detail('999999')
|
||||
|
||||
# 上面这行代码用于获取本子id为 999999 的JmAlbumDetail
|
||||
# 如果本子不存在,则会抛出异常,异常类默认是 JmcomicException
|
||||
|
||||
# 你可以自定义抛出的异常类,做法如下:
|
||||
# 1. 自定义异常类
|
||||
class MyExceptionClass(Exception):
|
||||
pass
|
||||
|
||||
# 2. 替换默认异常类
|
||||
JmModuleConfig.CLASS_EXCEPTION = MyExceptionClass
|
||||
|
||||
# 这样一来,抛出的异常类就是 MyExceptionClass
|
||||
try:
|
||||
album = client.get_album_detail('999999')
|
||||
except MyExceptionClass as e:
|
||||
print('捕获MyExceptionClass异常')
|
||||
pass
|
58
usage/usage_plugin.py
Normal file
58
usage/usage_plugin.py
Normal file
@ -0,0 +1,58 @@
|
||||
"""
|
||||
plugin(扩展/插件)是jmcomic=2.2.0新引入的机制,
|
||||
plugin机制可以实现在`特定时间` 回调 `特定插件`,实现灵活无感知的功能增强。
|
||||
|
||||
|
||||
目前仅支持一个时机: after_init,表示在option对象的 __init__ 初始化方法的最后
|
||||
目前仅内置一个插件: login,实现的功能为:登录禁漫,并保存登录后的cookies,让所有client都带上此cookies。实现类是
|
||||
|
||||
你可以在option配置文件当中,配置如下内容,来实现在 after_init 时机,调用 login 插件
|
||||
|
||||
|
||||
plugin:
|
||||
after_init: # 时机
|
||||
login: # 插件的key
|
||||
# 下面是给插件的参数 (kwargs),由插件类自定义
|
||||
username: un
|
||||
password: pw
|
||||
|
||||
|
||||
你也可以自定义插件和插件时机
|
||||
自定义插件时机需要你重写Option类,示例请见 usage_custom
|
||||
下面演示自定义插件,分为3步:
|
||||
|
||||
1. 自定义plugin类
|
||||
2. 让plugin类失效
|
||||
3. 使用plugin的key
|
||||
|
||||
如果你有好的plugin想法,也欢迎向我提PR,将你的plugin内置到jmcomic模块中
|
||||
|
||||
"""
|
||||
|
||||
# 1. 自定义plugin类
|
||||
from jmcomic import JmOptionPlugin, JmModuleConfig, create_option
|
||||
|
||||
|
||||
class MyPlugin(JmOptionPlugin):
|
||||
# 指定你的插件的key
|
||||
plugin_key = 'myplugin'
|
||||
|
||||
# 覆盖invoke方法,设定方法只有一个参数,名为`hello_plugin`
|
||||
def invoke(self, hello_plugin) -> None:
|
||||
print(hello_plugin)
|
||||
|
||||
|
||||
# 2. 让plugin类失效
|
||||
JmModuleConfig.register_plugin(MyPlugin)
|
||||
|
||||
# 3. 使用plugin的key
|
||||
"""
|
||||
plugin:
|
||||
after_init: # 时机
|
||||
myplugin: # 插件的key
|
||||
hello_plugin: this is my plugin invoke method's parameter # 你自定义的插件的参数
|
||||
"""
|
||||
|
||||
# 当你使用上述配置文件创建option时,
|
||||
# 在option初始化完成后,你的plugin会被调用,控制台就会打印出 `this is my plugin invoke method's parameter`
|
||||
option = create_option('xxx')
|
Loading…
Reference in New Issue
Block a user