mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v2.3.7: 优化代码,实现组件(Client、Plugin)的自动注册机制,支持dict类型的域名配置方便统一配置文件 (#148)
This commit is contained in:
parent
8070951feb
commit
ad9427aceb
@ -1,5 +1,7 @@
|
||||
# GitHub Actions 测试用
|
||||
# 移动端配置
|
||||
dir_rule:
|
||||
rule: Bd_Aauthor_Aid_Pindextitle
|
||||
|
||||
client:
|
||||
impl: api
|
||||
@ -7,6 +9,10 @@ client:
|
||||
postman:
|
||||
meta_data:
|
||||
timeout: 7
|
||||
domain:
|
||||
html:
|
||||
- jmcomic1.me
|
||||
- jmcomic.me
|
||||
|
||||
# 插件配置
|
||||
plugin:
|
||||
|
||||
@ -1,15 +1,18 @@
|
||||
# GitHub Actions 测试用
|
||||
# 网页端配置
|
||||
dir_rule:
|
||||
rule: Bd_Aauthor_Aid_Pindextitle
|
||||
|
||||
client:
|
||||
impl: html
|
||||
retry_times: 3
|
||||
postman:
|
||||
meta_data:
|
||||
timeout: 5
|
||||
timeout: 7
|
||||
domain:
|
||||
- jmcomic1.me
|
||||
- jmcomic.me
|
||||
html:
|
||||
- jmcomic1.me
|
||||
- jmcomic.me
|
||||
|
||||
# 插件配置
|
||||
plugin:
|
||||
@ -18,3 +21,8 @@ plugin:
|
||||
kwargs:
|
||||
interval: 0.5 # 间隔时间
|
||||
enable_warning: false # 不告警
|
||||
|
||||
- plugin: client_proxy
|
||||
kwargs:
|
||||
proxy_client_key: cl_proxy_future
|
||||
whitelist: [ api, ]
|
||||
@ -2,7 +2,28 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.3.6'
|
||||
__version__ = '2.3.7'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
||||
# 下面进行注册组件(客户端、插件)
|
||||
gb = dict(filter(lambda pair: isinstance(pair[1], type), globals().items()))
|
||||
|
||||
|
||||
def register_jmcomic_component(gb: dict, method, valid_interface: type):
|
||||
for v in gb.values():
|
||||
if v != valid_interface and issubclass(v, valid_interface):
|
||||
method(v)
|
||||
|
||||
|
||||
# 注册客户端
|
||||
register_jmcomic_component(gb,
|
||||
JmModuleConfig.register_client,
|
||||
JmcomicClient,
|
||||
)
|
||||
# 注册插件
|
||||
register_jmcomic_component(gb,
|
||||
JmModuleConfig.register_plugin,
|
||||
JmOptionPlugin,
|
||||
)
|
||||
|
||||
@ -6,7 +6,7 @@ class AbstractJmClient(
|
||||
JmcomicClient,
|
||||
PostmanProxy,
|
||||
):
|
||||
client_key = None
|
||||
client_key = '__just_for_placeholder_do_not_use_me__'
|
||||
func_to_cache = []
|
||||
|
||||
def __init__(self,
|
||||
@ -324,7 +324,7 @@ class JmHtmlClient(AbstractJmClient):
|
||||
(f' to ({comment_id})' if comment_id is not None else '')
|
||||
)
|
||||
|
||||
resp = self.post('https://18comic.vip/ajax/album_comment',
|
||||
resp = self.post('/ajax/album_comment',
|
||||
headers=JmModuleConfig.album_comment_headers,
|
||||
data=data,
|
||||
)
|
||||
@ -679,8 +679,3 @@ class FutureClientProxy(JmcomicClient):
|
||||
cache_key = f'search_query_{search_query}_page_{page}_main_tag_{main_tag}_order_by_{order_by}_time_{time}'
|
||||
future = self.get_future(cache_key, task=lambda: self.client.search(search_query, page, main_tag, order_by, time))
|
||||
return future.result()
|
||||
|
||||
|
||||
JmModuleConfig.register_client(JmHtmlClient)
|
||||
JmModuleConfig.register_client(JmApiClient)
|
||||
JmModuleConfig.register_client(FutureClientProxy)
|
||||
|
||||
@ -78,11 +78,11 @@ class JmModuleConfig:
|
||||
CLASS_ALBUM = None
|
||||
CLASS_PHOTO = None
|
||||
CLASS_IMAGE = None
|
||||
CLASS_CLIENT_IMPL = {}
|
||||
CLASS_EXCEPTION = JmcomicException
|
||||
|
||||
# 客户端注册表
|
||||
REGISTRY_CLIENT = {}
|
||||
# 插件注册表
|
||||
PLUGIN_REGISTRY = {}
|
||||
REGISTRY_PLUGIN = {}
|
||||
|
||||
# 执行debug的函数
|
||||
debug_executor = default_jm_debug
|
||||
@ -140,7 +140,7 @@ class JmModuleConfig:
|
||||
|
||||
@classmethod
|
||||
def client_impl_class(cls, client_key: str):
|
||||
clazz_dict = cls.CLASS_CLIENT_IMPL
|
||||
clazz_dict = cls.REGISTRY_CLIENT
|
||||
|
||||
clazz = clazz_dict.get(client_key, None)
|
||||
if clazz is None:
|
||||
@ -320,16 +320,17 @@ class JmModuleConfig:
|
||||
|
||||
@classmethod
|
||||
def register_plugin(cls, plugin_class):
|
||||
cls.PLUGIN_REGISTRY[plugin_class.plugin_key] = plugin_class
|
||||
from .jm_toolkit import ExceptionTool
|
||||
ExceptionTool.require_true(getattr(plugin_class, 'plugin_key', None) is not None,
|
||||
f'未配置plugin_key, class: {plugin_class}')
|
||||
cls.REGISTRY_PLUGIN[plugin_class.plugin_key] = plugin_class
|
||||
|
||||
@classmethod
|
||||
def register_client(cls, client_class):
|
||||
client_key = getattr(client_class, 'client_key', None)
|
||||
if client_key is None:
|
||||
from .jm_toolkit import ExceptionTool
|
||||
ExceptionTool.raises(f'未配置client_key, class: {client_class}')
|
||||
|
||||
cls.CLASS_CLIENT_IMPL[client_key] = client_class
|
||||
from .jm_toolkit import ExceptionTool
|
||||
ExceptionTool.require_true(getattr(client_class, 'client_key', None) is not None,
|
||||
f'未配置client_key, class: {client_class}')
|
||||
cls.REGISTRY_CLIENT[client_class.client_key] = client_class
|
||||
|
||||
|
||||
jm_debug = JmModuleConfig.jm_debug
|
||||
|
||||
@ -84,7 +84,7 @@ class JmImageDetail(JmBaseEntity):
|
||||
) -> None:
|
||||
if scramble_id is None or (isinstance(scramble_id, str) and scramble_id == ''):
|
||||
from .jm_toolkit import ExceptionTool
|
||||
ExceptionTool.raises(f'图片的scramble_id不能为空D')
|
||||
ExceptionTool.raises(f'图片的scramble_id不能为空')
|
||||
|
||||
self.aid: str = str(aid)
|
||||
self.scramble_id: str = str(scramble_id)
|
||||
|
||||
@ -298,21 +298,27 @@ class JmOption:
|
||||
"""
|
||||
return self.new_jm_client(**kwargs)
|
||||
|
||||
def new_jm_client(self, domain_list=None, impl=None, **kwargs) -> JmcomicClient:
|
||||
postman_conf: dict = self.client.postman.src_dict
|
||||
impl = impl or self.client.impl
|
||||
def new_jm_client(self, domain=None, impl=None, **kwargs) -> JmcomicClient:
|
||||
# 所有需要用到的 self.client 配置项如下
|
||||
postman_conf: dict = self.client.postman.src_dict # postman dsl 配置
|
||||
impl: str = impl or self.client.impl # client_key
|
||||
retry_times: int = self.client.retry_times # 重试次数
|
||||
cache: str = self.client.cache # 启用缓存
|
||||
|
||||
# domain_list
|
||||
if domain_list is None:
|
||||
domain_list = self.client.domain
|
||||
# domain
|
||||
def decide_domain():
|
||||
domain_list: Union[List[str], DictModel, dict] = domain if domain is not None \
|
||||
else self.client.domain # 域名
|
||||
|
||||
domain_list: Union[List[str], DictModel]
|
||||
if not isinstance(domain_list, list):
|
||||
domain_list_dict: DictModel = domain_list
|
||||
domain_list = domain_list_dict.get(impl, [])
|
||||
if not isinstance(domain_list, list):
|
||||
domain_list = domain_list.get(impl, [])
|
||||
|
||||
if len(domain_list) == 0:
|
||||
domain_list = self.decide_client_domain(impl)
|
||||
if len(domain_list) == 0:
|
||||
domain_list = self.decide_client_domain(impl)
|
||||
|
||||
return domain_list
|
||||
|
||||
domain: List[str] = decide_domain()
|
||||
|
||||
# support kwargs overwrite meta_data
|
||||
if len(kwargs) != 0:
|
||||
@ -321,20 +327,23 @@ class JmOption:
|
||||
# headers
|
||||
meta_data = postman_conf['meta_data']
|
||||
if meta_data['headers'] is None:
|
||||
meta_data['headers'] = JmModuleConfig.headers(domain_list[0])
|
||||
meta_data['headers'] = JmModuleConfig.headers(domain[0])
|
||||
|
||||
# postman
|
||||
postman = Postmans.create(data=postman_conf)
|
||||
|
||||
# client
|
||||
client = JmModuleConfig.client_impl_class(impl)(
|
||||
clazz = JmModuleConfig.client_impl_class(impl)
|
||||
if clazz == AbstractJmClient or not issubclass(clazz, AbstractJmClient):
|
||||
raise NotImplementedError(clazz)
|
||||
client = clazz(
|
||||
postman,
|
||||
self.client.retry_times,
|
||||
fallback_domain_list=domain_list,
|
||||
retry_times,
|
||||
fallback_domain_list=decide_domain(),
|
||||
)
|
||||
|
||||
# enable cache
|
||||
if self.client.cache is True:
|
||||
if cache is True:
|
||||
client.enable_cache()
|
||||
|
||||
return client
|
||||
@ -396,7 +405,7 @@ class JmOption:
|
||||
# 保证 jm_plugin.py 被加载
|
||||
from .jm_plugin import JmOptionPlugin
|
||||
|
||||
plugin_registry = JmModuleConfig.PLUGIN_REGISTRY
|
||||
plugin_registry = JmModuleConfig.REGISTRY_PLUGIN
|
||||
for pinfo in plugin_list:
|
||||
key, kwargs = pinfo['plugin'], pinfo['kwargs']
|
||||
plugin_class: Optional[Type[JmOptionPlugin]] = plugin_registry.get(key, None)
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
"""
|
||||
该文件存放的是option扩展功能类
|
||||
该文件存放的是option插件
|
||||
"""
|
||||
|
||||
from .jm_option import *
|
||||
@ -335,9 +335,7 @@ class ClientProxyPlugin(JmOptionPlugin):
|
||||
whitelist=None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
if whitelist is None:
|
||||
whitelist = set()
|
||||
else:
|
||||
if whitelist is not None:
|
||||
whitelist = set(whitelist)
|
||||
|
||||
clazz = JmModuleConfig.client_impl_class(proxy_client_key)
|
||||
@ -346,17 +344,10 @@ class ClientProxyPlugin(JmOptionPlugin):
|
||||
|
||||
def hook_new_jm_client(*args, **kwargs):
|
||||
client = new_jm_client(*args, **kwargs)
|
||||
if client.client_key not in whitelist:
|
||||
if whitelist is not None and client.client_key not in whitelist:
|
||||
return client
|
||||
|
||||
jm_debug('plugin.client_proxy', f'proxy client {client} with {clazz}')
|
||||
jm_debug('plugin.client_proxy', f'proxy client {client} with {proxy_client_key}')
|
||||
return clazz(client, **clazz_init_kwargs)
|
||||
|
||||
self.option.new_jm_client = hook_new_jm_client
|
||||
|
||||
|
||||
JmModuleConfig.register_plugin(JmLoginPlugin)
|
||||
JmModuleConfig.register_plugin(UsageLogPlugin)
|
||||
JmModuleConfig.register_plugin(FindUpdatePlugin)
|
||||
JmModuleConfig.register_plugin(ZipPlugin)
|
||||
JmModuleConfig.register_plugin(ClientProxyPlugin)
|
||||
|
||||
@ -45,7 +45,7 @@ class Test_Custom(JmTestConfigurable):
|
||||
|
||||
self.assertListEqual(
|
||||
JmModuleConfig.DOMAIN_API_LIST,
|
||||
self.option.new_jm_client(domain_list=[], impl=MyClient.client_key).get_domain_list()
|
||||
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list()
|
||||
)
|
||||
|
||||
def test_extends_html_client(self):
|
||||
@ -57,7 +57,7 @@ class Test_Custom(JmTestConfigurable):
|
||||
html_domain = self.client.get_html_domain()
|
||||
self.assertListEqual(
|
||||
[html_domain],
|
||||
self.option.new_jm_client(domain_list=[], impl=MyClient.client_key).get_domain_list()
|
||||
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list()
|
||||
)
|
||||
|
||||
def test_client_key_missing(self):
|
||||
@ -93,6 +93,6 @@ class Test_Custom(JmTestConfigurable):
|
||||
JmModuleConfig.register_client(MyClient)
|
||||
self.assertListEqual(
|
||||
JmModuleConfig.DOMAIN_API_LIST,
|
||||
self.option.new_jm_client(domain_list=[], impl=MyClient.client_key).get_domain_list(),
|
||||
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list(),
|
||||
msg='继承client,不配置域名',
|
||||
)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user