v2.3.7: 优化代码,实现组件(Client、Plugin)的自动注册机制,支持dict类型的域名配置方便统一配置文件 (#148)

This commit is contained in:
hect0x7 2023-10-13 18:24:28 +08:00 committed by GitHub
parent 8070951feb
commit ad9427aceb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 88 additions and 57 deletions

View File

@ -1,5 +1,7 @@
# GitHub Actions 测试用
# 移动端配置
dir_rule:
rule: Bd_Aauthor_Aid_Pindextitle
client:
impl: api
@ -7,6 +9,10 @@ client:
postman:
meta_data:
timeout: 7
domain:
html:
- jmcomic1.me
- jmcomic.me
# 插件配置
plugin:

View File

@ -1,15 +1,18 @@
# GitHub Actions 测试用
# 网页端配置
dir_rule:
rule: Bd_Aauthor_Aid_Pindextitle
client:
impl: html
retry_times: 3
postman:
meta_data:
timeout: 5
timeout: 7
domain:
- jmcomic1.me
- jmcomic.me
html:
- jmcomic1.me
- jmcomic.me
# 插件配置
plugin:
@ -18,3 +21,8 @@ plugin:
kwargs:
interval: 0.5 # 间隔时间
enable_warning: false # 不告警
- plugin: client_proxy
kwargs:
proxy_client_key: cl_proxy_future
whitelist: [ api, ]

View File

@ -2,7 +2,28 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.3.6'
__version__ = '2.3.7'
from .api import *
from .jm_plugin import *
# 下面进行注册组件(客户端、插件)
gb = dict(filter(lambda pair: isinstance(pair[1], type), globals().items()))
def register_jmcomic_component(gb: dict, method, valid_interface: type):
for v in gb.values():
if v != valid_interface and issubclass(v, valid_interface):
method(v)
# 注册客户端
register_jmcomic_component(gb,
JmModuleConfig.register_client,
JmcomicClient,
)
# 注册插件
register_jmcomic_component(gb,
JmModuleConfig.register_plugin,
JmOptionPlugin,
)

View File

@ -6,7 +6,7 @@ class AbstractJmClient(
JmcomicClient,
PostmanProxy,
):
client_key = None
client_key = '__just_for_placeholder_do_not_use_me__'
func_to_cache = []
def __init__(self,
@ -324,7 +324,7 @@ class JmHtmlClient(AbstractJmClient):
(f' to ({comment_id})' if comment_id is not None else '')
)
resp = self.post('https://18comic.vip/ajax/album_comment',
resp = self.post('/ajax/album_comment',
headers=JmModuleConfig.album_comment_headers,
data=data,
)
@ -679,8 +679,3 @@ class FutureClientProxy(JmcomicClient):
cache_key = f'search_query_{search_query}_page_{page}_main_tag_{main_tag}_order_by_{order_by}_time_{time}'
future = self.get_future(cache_key, task=lambda: self.client.search(search_query, page, main_tag, order_by, time))
return future.result()
JmModuleConfig.register_client(JmHtmlClient)
JmModuleConfig.register_client(JmApiClient)
JmModuleConfig.register_client(FutureClientProxy)

View File

@ -78,11 +78,11 @@ class JmModuleConfig:
CLASS_ALBUM = None
CLASS_PHOTO = None
CLASS_IMAGE = None
CLASS_CLIENT_IMPL = {}
CLASS_EXCEPTION = JmcomicException
# 客户端注册表
REGISTRY_CLIENT = {}
# 插件注册表
PLUGIN_REGISTRY = {}
REGISTRY_PLUGIN = {}
# 执行debug的函数
debug_executor = default_jm_debug
@ -140,7 +140,7 @@ class JmModuleConfig:
@classmethod
def client_impl_class(cls, client_key: str):
clazz_dict = cls.CLASS_CLIENT_IMPL
clazz_dict = cls.REGISTRY_CLIENT
clazz = clazz_dict.get(client_key, None)
if clazz is None:
@ -320,16 +320,17 @@ class JmModuleConfig:
@classmethod
def register_plugin(cls, plugin_class):
cls.PLUGIN_REGISTRY[plugin_class.plugin_key] = plugin_class
from .jm_toolkit import ExceptionTool
ExceptionTool.require_true(getattr(plugin_class, 'plugin_key', None) is not None,
f'未配置plugin_key, class: {plugin_class}')
cls.REGISTRY_PLUGIN[plugin_class.plugin_key] = plugin_class
@classmethod
def register_client(cls, client_class):
client_key = getattr(client_class, 'client_key', None)
if client_key is None:
from .jm_toolkit import ExceptionTool
ExceptionTool.raises(f'未配置client_key, class: {client_class}')
cls.CLASS_CLIENT_IMPL[client_key] = client_class
from .jm_toolkit import ExceptionTool
ExceptionTool.require_true(getattr(client_class, 'client_key', None) is not None,
f'未配置client_key, class: {client_class}')
cls.REGISTRY_CLIENT[client_class.client_key] = client_class
jm_debug = JmModuleConfig.jm_debug

View File

@ -84,7 +84,7 @@ class JmImageDetail(JmBaseEntity):
) -> None:
if scramble_id is None or (isinstance(scramble_id, str) and scramble_id == ''):
from .jm_toolkit import ExceptionTool
ExceptionTool.raises(f'图片的scramble_id不能为空D')
ExceptionTool.raises(f'图片的scramble_id不能为空')
self.aid: str = str(aid)
self.scramble_id: str = str(scramble_id)

View File

@ -298,21 +298,27 @@ class JmOption:
"""
return self.new_jm_client(**kwargs)
def new_jm_client(self, domain_list=None, impl=None, **kwargs) -> JmcomicClient:
postman_conf: dict = self.client.postman.src_dict
impl = impl or self.client.impl
def new_jm_client(self, domain=None, impl=None, **kwargs) -> JmcomicClient:
# 所有需要用到的 self.client 配置项如下
postman_conf: dict = self.client.postman.src_dict # postman dsl 配置
impl: str = impl or self.client.impl # client_key
retry_times: int = self.client.retry_times # 重试次数
cache: str = self.client.cache # 启用缓存
# domain_list
if domain_list is None:
domain_list = self.client.domain
# domain
def decide_domain():
domain_list: Union[List[str], DictModel, dict] = domain if domain is not None \
else self.client.domain # 域名
domain_list: Union[List[str], DictModel]
if not isinstance(domain_list, list):
domain_list_dict: DictModel = domain_list
domain_list = domain_list_dict.get(impl, [])
if not isinstance(domain_list, list):
domain_list = domain_list.get(impl, [])
if len(domain_list) == 0:
domain_list = self.decide_client_domain(impl)
if len(domain_list) == 0:
domain_list = self.decide_client_domain(impl)
return domain_list
domain: List[str] = decide_domain()
# support kwargs overwrite meta_data
if len(kwargs) != 0:
@ -321,20 +327,23 @@ class JmOption:
# headers
meta_data = postman_conf['meta_data']
if meta_data['headers'] is None:
meta_data['headers'] = JmModuleConfig.headers(domain_list[0])
meta_data['headers'] = JmModuleConfig.headers(domain[0])
# postman
postman = Postmans.create(data=postman_conf)
# client
client = JmModuleConfig.client_impl_class(impl)(
clazz = JmModuleConfig.client_impl_class(impl)
if clazz == AbstractJmClient or not issubclass(clazz, AbstractJmClient):
raise NotImplementedError(clazz)
client = clazz(
postman,
self.client.retry_times,
fallback_domain_list=domain_list,
retry_times,
fallback_domain_list=decide_domain(),
)
# enable cache
if self.client.cache is True:
if cache is True:
client.enable_cache()
return client
@ -396,7 +405,7 @@ class JmOption:
# 保证 jm_plugin.py 被加载
from .jm_plugin import JmOptionPlugin
plugin_registry = JmModuleConfig.PLUGIN_REGISTRY
plugin_registry = JmModuleConfig.REGISTRY_PLUGIN
for pinfo in plugin_list:
key, kwargs = pinfo['plugin'], pinfo['kwargs']
plugin_class: Optional[Type[JmOptionPlugin]] = plugin_registry.get(key, None)

View File

@ -1,5 +1,5 @@
"""
该文件存放的是option扩展功能类
该文件存放的是option插件
"""
from .jm_option import *
@ -335,9 +335,7 @@ class ClientProxyPlugin(JmOptionPlugin):
whitelist=None,
**kwargs,
) -> None:
if whitelist is None:
whitelist = set()
else:
if whitelist is not None:
whitelist = set(whitelist)
clazz = JmModuleConfig.client_impl_class(proxy_client_key)
@ -346,17 +344,10 @@ class ClientProxyPlugin(JmOptionPlugin):
def hook_new_jm_client(*args, **kwargs):
client = new_jm_client(*args, **kwargs)
if client.client_key not in whitelist:
if whitelist is not None and client.client_key not in whitelist:
return client
jm_debug('plugin.client_proxy', f'proxy client {client} with {clazz}')
jm_debug('plugin.client_proxy', f'proxy client {client} with {proxy_client_key}')
return clazz(client, **clazz_init_kwargs)
self.option.new_jm_client = hook_new_jm_client
JmModuleConfig.register_plugin(JmLoginPlugin)
JmModuleConfig.register_plugin(UsageLogPlugin)
JmModuleConfig.register_plugin(FindUpdatePlugin)
JmModuleConfig.register_plugin(ZipPlugin)
JmModuleConfig.register_plugin(ClientProxyPlugin)

View File

@ -45,7 +45,7 @@ class Test_Custom(JmTestConfigurable):
self.assertListEqual(
JmModuleConfig.DOMAIN_API_LIST,
self.option.new_jm_client(domain_list=[], impl=MyClient.client_key).get_domain_list()
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list()
)
def test_extends_html_client(self):
@ -57,7 +57,7 @@ class Test_Custom(JmTestConfigurable):
html_domain = self.client.get_html_domain()
self.assertListEqual(
[html_domain],
self.option.new_jm_client(domain_list=[], impl=MyClient.client_key).get_domain_list()
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list()
)
def test_client_key_missing(self):
@ -93,6 +93,6 @@ class Test_Custom(JmTestConfigurable):
JmModuleConfig.register_client(MyClient)
self.assertListEqual(
JmModuleConfig.DOMAIN_API_LIST,
self.option.new_jm_client(domain_list=[], impl=MyClient.client_key).get_domain_list(),
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list(),
msg='继承client不配置域名',
)