v2.3.17: 支持从浏览器获取cookies自动登录禁漫【插件】; 重构Client缓存机制、支持配置缓存级别 (#161)

This commit is contained in:
hect0x7 2023-11-01 18:18:13 +08:00 committed by GitHub
parent 9e3fa7f17a
commit bf2ca6efbc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 259 additions and 39 deletions

View File

@ -54,7 +54,7 @@ $ jmcomic 422866
- **可扩展性强**
- **支持Plugin插件可以方便地扩展功能以及使用别人的插件**
- 目前内置支持的插件有:`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` `下载特定后缀图片插件` `发送QQ邮件插件` `日志主题过滤插件`
- 目前内置支持的插件有:`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` `下载特定后缀图片插件` `发送QQ邮件插件` `日志主题过滤插件` `自动使用浏览器cookies插件`
- 支持自定义本子/章节/图片下载前后的回调函数
- 支持自定义debug/logging
- 支持自定义类:`Downloader负责调度` `Option负责配置` `Client负责请求` `实体类`

View File

@ -25,8 +25,7 @@ Python API for JMComic禁漫天堂
- Highly extensible:
- Supports Plugin plugins for easy functionality extension and use of other plugins.
- Currently built-in
plugins: `login plugin`, `hardware usage monitoring plugin`, `only download new chapters plugin`, `zip compression plugin`, `image suffix filter plugin` `send qq email plugin` `debug logging topic filter plugin`.
- Currently built-in plugins: `login plugin`, `hardware usage monitoring plugin`, `only download new chapters plugin`, `zip compression plugin`, `image suffix filter plugin` `send qq email plugin` `debug logging topic filter plugin` `auto set browser cookies plugin`.
- Supports custom callback functions before and after downloading album/chapter/images.
- Supports custom debug logging.
- Supports custom core

View File

@ -120,6 +120,11 @@ plugins:
proxy_client_key: cl_proxy_future # 代理类的client_key
whitelist: [ api, ] # 白名单当client.impl匹配白名单时才代理
- plugin: auto_set_browser_cookies # 自动获取浏览器cookies详见插件类
kwargs:
browser: chrome
domain: 18comic.vip
after_album:
- plugin: zip # 压缩文件插件
kwargs:

View File

@ -27,7 +27,7 @@ setup(
package_dir={"": "src"},
python_requires=">=3.7",
install_requires=[
'commonX>=0.5.7',
'commonX>=0.6.2',
'curl_cffi',
'PyYAML',
'Pillow',

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.3.16'
__version__ = '2.3.17'
from .api import *
from .jm_plugin import *

View File

@ -25,6 +25,8 @@ class AbstractJmClient(
fallback_domain_list.insert(0, domain)
self.domain_list = fallback_domain_list
self.CLIENT_CACHE = None
self.enable_cache()
self.after_init()
def after_init(self):
@ -111,31 +113,59 @@ class AbstractJmClient(
def before_retry(self, e, kwargs, retry_count, url):
jm_debug('req.error', str(e))
def enable_cache(self, debug=False):
if self.is_cache_enabled():
return
def enable_cache(self):
# noinspection PyDefaultArgument,PyShadowingBuiltins
def make_key(args, kwds, typed,
kwd_mark=(object(),),
fasttypes={int, str},
tuple=tuple, type=type, len=len):
key = args
if kwds:
key += kwd_mark
for item in kwds.items():
key += item
if typed:
key += tuple(type(v) for v in args)
if kwds:
key += tuple(type(v) for v in kwds.values())
elif len(key) == 1 and type(key[0]) in fasttypes:
return key[0]
return hash(key)
def wrap_func_with_cache(func_name, cache_field_name):
if hasattr(self, cache_field_name):
return
if sys.version_info > (3, 9):
import functools
cache = functools.cache
else:
from functools import lru_cache
cache = lru_cache()
func = getattr(self, func_name)
setattr(self, func_name, cache(func))
def cache_wrapper(*args, **kwargs):
cache = self.CLIENT_CACHE
# Equivalent to not enable cache
if cache is None:
return func(*args, **kwargs)
key = make_key(args, kwargs, False)
sentinel = object() # unique object used to signal cache misses
result = cache.get(key, sentinel)
if result is not sentinel:
return result
result = func(*args, **kwargs)
cache[key] = result
return result
setattr(self, func_name, cache_wrapper)
for func_name in self.func_to_cache:
wrap_func_with_cache(func_name, f'__{func_name}.cache.dict__')
setattr(self, '__enable_cache__', True)
def set_cache_dict(self, cache_dict: Optional[Dict]):
self.CLIENT_CACHE = cache_dict
def is_cache_enabled(self) -> bool:
return getattr(self, '__enable_cache__', False)
def get_cache_dict(self):
return self.CLIENT_CACHE
def get_domain_list(self):
return self.domain_list
@ -635,7 +665,7 @@ class FutureClientProxy(JmcomicClient):
client_key = 'cl_proxy_future'
proxy_methods = ['album_comment', 'enable_cache', 'get_domain_list',
'get_html_domain', 'get_html_domain_all', 'get_jm_image',
'is_cache_enabled', 'set_domain_list', ]
'set_cache_dict', 'get_cache_dict', 'set_domain_list', ]
class FutureWrapper:
def __init__(self, future):

View File

@ -160,10 +160,10 @@ class JmDetailClient:
def of_api_url(self, api_path, domain):
raise NotImplementedError
def enable_cache(self, debug=False):
def set_cache_dict(self, cache_dict: Optional[Dict]):
raise NotImplementedError
def is_cache_enabled(self) -> bool:
def get_cache_dict(self) -> Optional[Dict]:
raise NotImplementedError
def check_photo(self, photo: JmPhotoDetail):

View File

@ -335,7 +335,7 @@ class JmModuleConfig:
},
},
'client': {
'cache': None,
'cache': None, # see CacheRegistry
'domain': [],
'postman': {
'type': 'cffi',

View File

@ -1,6 +1,58 @@
from .jm_client_impl import *
class CacheRegistry:
REGISTRY = {}
@classmethod
def level_option(cls, option, _client):
registry = cls.REGISTRY
registry.setdefault(option, {})
return registry[option]
@classmethod
def level_client(cls, _option, client):
registry = cls.REGISTRY
registry.setdefault(client, {})
return registry[client]
@classmethod
def enable_client_cache_on_condition(cls, option: 'JmOption', client: JmcomicClient, cache: Union[None, bool, str, Callable]):
"""
cache parameter
if None: no cache
if bool:
true: level_option
false: no cache
if str:
(invoke corresponding Cache class method)
:param option: JmOption
:param client: JmcomicClient
:param cache: config dsl
"""
if cache is None:
return
elif isinstance(cache, bool):
if cache is False:
return
else:
cache = cls.level_option
elif isinstance(cache, str):
func = getattr(cls, cache, None)
assert func is not None, f'未实现的cache配置名: {cache}'
cache = func
cache: Callable
client.set_cache_dict(cache(option, client))
class DirRule:
rule_sample = [
# 根目录 / Album-id / Photo-序号 /
@ -312,12 +364,17 @@ class JmOption:
return self.new_jm_client(**kwargs)
def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient:
"""
创建新的Client客户端不同Client之间的元数据不共享
"""
from copy import deepcopy
# 所有需要用到的 self.client 配置项如下
postman_conf: dict = self.client.postman.src_dict # postman dsl 配置
meta_data: dict = postman_conf['meta_data'] # 请求元信息
postman_conf: dict = deepcopy(self.client.postman.src_dict) # postman dsl 配置
meta_data: dict = postman_conf['meta_data'] # 元数据
impl: str = impl or self.client.impl # client_key
retry_times: int = self.client.retry_times # 重试次数
cache: str = cache or self.client.cache # 启用缓存
cache: str = cache if cache is not None else self.client.cache # 启用缓存
# domain
def decide_domain():
@ -357,11 +414,19 @@ class JmOption:
)
# enable cache
if cache is True:
client.enable_cache()
CacheRegistry.enable_client_cache_on_condition(self, client, cache)
return client
def update_cookies(self, cookies: dict):
metadata: dict = self.client.postman.meta_data.src_dict
orig_cookies: Optional[Dict] = metadata.get('cookies', None)
if orig_cookies is None:
metadata['cookies'] = cookies
else:
orig_cookies.update(cookies)
metadata['cookies'] = orig_cookies
# noinspection PyMethodMayBeStatic
def decide_client_domain(self, client_key: str) -> List[str]:
is_client_type = lambda ctype: self.client_key_is_given_type(client_key, ctype)

View File

@ -49,6 +49,12 @@ class JmOptionPlugin:
raise PluginValidationException(self, msg)
def warning_lib_not_install(self, lib='psutil'):
msg = (f'插件`{self.plugin_key}`依赖库: {lib},请先安装{lib}再使用。'
f'安装命令: [pip install {lib}]')
import warnings
warnings.warn(msg)
class JmLoginPlugin(JmOptionPlugin):
"""
@ -111,12 +117,7 @@ class UsageLogPlugin(JmOptionPlugin):
try:
import psutil
except ImportError:
msg = (f'插件`{self.plugin_key}`依赖psutil库请先安装psutil再使用。'
f'安装命令: [pip install psutil]')
import warnings
warnings.warn(msg)
# import sys
# print(msg, file=sys.stderr)
self.warning_lib_not_install('psutil')
return
from time import sleep
@ -464,3 +465,55 @@ class DebugTopicFilterPlugin(JmOptionPlugin):
old_jm_debug(topic, msg)
JmModuleConfig.debug_executor = new_jm_debug
class AutoSetBrowserCookiesPlugin(JmOptionPlugin):
plugin_key = 'auto_set_browser_cookies'
accepted_cookies_keys = str_to_set('''
yuo1
remember_id
remember
''')
def invoke(self,
browser: str,
domain: str,
) -> None:
"""
坑点预警由于禁漫需要校验同一设备使用该插件需要配置自己浏览器的headers例如
```yml
client:
postman:
meta_data:
headers: {
# 浏览器headers
}
# 插件配置如下:
plugins:
after_init:
- plugin: auto_set_browser_cookies
kwargs:
browser: chrome
domain: 18comic.vip
```
:param browser: chrome/edge/...
:param domain: 18comic.vip/...
:return: cookies
"""
cookies, e = get_browser_cookies(browser, domain, safe=True)
if cookies is None:
if isinstance(e, ImportError):
self.warning_lib_not_install('browser_cookie3')
else:
self.debug('获取浏览器cookies失败请关闭浏览器重试')
return
self.option.update_cookies(
{k: v for k, v in cookies.items() if k in self.accepted_cookies_keys}
)
self.debug('获取浏览器cookies成功')

View File

@ -46,11 +46,7 @@ class JmTestConfigurable(unittest.TestCase):
@classmethod
def setUpClass(cls):
# 设置 JmOptionJmcomicClient
try:
option = create_option_by_env('JM_OPTION_PATH_TEST')
except JmcomicException:
option = create_option('./assets/option/option_test.yml')
option = cls.new_option()
cls.option = option
cls.client = option.build_jm_client()
@ -61,6 +57,13 @@ class JmTestConfigurable(unittest.TestCase):
return
cost_time_dict[cls.__name__] = ts()
@classmethod
def new_option(cls):
try:
return create_option_by_env('JM_OPTION_PATH_TEST')
except JmcomicException:
return create_option('./assets/option/option_test.yml')
@classmethod
def tearDownClass(cls) -> None:
if skip_time_cost_debug:

View File

@ -264,3 +264,68 @@ class Test_Client(JmTestConfigurable):
})
print(page.page_count)
break
def test_cache_level(self):
def get(cl):
return cl.get_album_detail('123')
def assertEqual(first_cl, second_cl, msg):
return self.assertEqual(
get(first_cl),
get(second_cl),
msg,
)
def assertNotEqual(first_cl, second_cl, msg):
return self.assertNotEqual(
get(first_cl),
get(second_cl),
msg,
)
cases = [
(
CacheRegistry.level_option,
CacheRegistry.level_option,
CacheRegistry.level_client,
CacheRegistry.level_client,
),
(
True,
'level_option',
'level_client',
CacheRegistry.level_client,
)
]
def run(arg1, arg2, arg3, arg4):
op = self.new_option()
c1 = op.new_jm_client(cache=arg1)
c2 = op.new_jm_client(cache=arg2)
c3 = op.new_jm_client(cache=arg3)
c4 = op.new_jm_client(cache=arg4)
c5 = op.new_jm_client(cache=False)
# c1 == c2
# c3 == c4
# c1 != c3
# c5 != c1, c2, c3, c4
invoke_all(
args_func_list=[
(None, func) for func in [
lambda: assertEqual(c1, c2, 'equals in same option level'),
lambda: assertNotEqual(c3, c4, 'not equals in client level'),
lambda: assertNotEqual(c1, c3, 'not equals in different level'),
lambda: assertNotEqual(c1, c5, 'not equals for None level'),
lambda: assertNotEqual(c3, c5, 'not equals for None level'),
]
]
)
future_ls = thread_pool_executor(
iter_objs=cases,
apply_each_obj_func=run,
)
return [f.result() for f in future_ls] # 等待执行完毕