v1.5.0: 优化JmcomicClient的配置流程、缓存启用,模块配置更加清晰。 (#12)

This commit is contained in:
hect0x7 2023-04-07 18:04:39 +08:00 committed by GitHub
parent ca59399a3e
commit 12351dd454
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 100 additions and 59 deletions

View File

@ -2,6 +2,6 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- service <--- option
__version__ = '1.4.0'
__version__ = '1.5.0'
from .api import *

View File

@ -131,37 +131,17 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
)
def renew_jm_default_domain():
"""
由于禁漫的域名经常变化调用此方法可以获取一个当前可用的最新的域名 domain
并且设置把 domain 设置为禁漫模块的默认域名
这样一来配置文件也不用配置域名了一切都在运行时动态获取
"""
domain = JmcomicText.parse_to_jm_domain(JmModuleConfig.get_jmcomic_url())
JmModuleConfig.DOMAIN = domain
return domain
def build_client(option: Optional[JmOption]) -> Tuple[JmOption, JmcomicClient]:
"""
处理option的判空并且创建jm_client
"""
if option is None:
option = JmOption.default()
option.client_config['domain'] = renew_jm_default_domain()
jm_client = option.build_jm_client()
return option, jm_client
def create_option(filepath: str) -> JmOption:
"""
创建 JmOption同时检查域名是否配置未配置则补上配置
@param filepath:
@return:
"""
option = JmOption.create_from_file(filepath)
client_config = option.client_config
key = 'domain'
if client_config.get(key, None) is None or client_config[key] is None:
client_config[key] = renew_jm_default_domain()
return option

View File

@ -120,7 +120,7 @@ class JmcomicClient(PostmanProxy):
# -- 对象方法 --
def of_api_url(self, api_path):
return f"{JmModuleConfig.HTTP}{self.domain}{api_path}"
return f"{JmModuleConfig.PROT}{self.domain}{api_path}"
def jm_get(self, url, is_api=True, require_200=True, **kwargs):
"""
@ -157,6 +157,28 @@ class JmcomicClient(PostmanProxy):
def img_is_not_need_to_decode(cls, data_original: str, _resp):
return data_original.endswith('.gif')
# noinspection PyAttributeOutsideInit
def enable_cache(self):
def wrap_func_cache(func_name, cache_dict_name):
if hasattr(self, cache_dict_name):
return
cache_dict = {}
setattr(self, cache_dict_name, cache_dict)
# 重载本对象的方法
func = getattr(self, func_name)
wrap_func = enable_cache(
cache_dict=cache_dict,
cache_hit_msg=f'命中 {cache_dict_name} ' + '→ [{}]]',
cache_miss_msg=f'缺失 {cache_dict_name} ' + '← [{}]',
)(func)
setattr(self, func_name, wrap_func)
wrap_func_cache('get_photo_detail', 'album_cache_dict')
wrap_func_cache('get_album_detail', 'photo_cache_dict')
# 爬取策略
class FetchStrategy:

View File

@ -1,10 +1,10 @@
class JmModuleConfig:
# 网站相关
HTTP = "https://"
DOMAIN = "jmcomic1.group" # jmcomic默认域名
JM_REDIRECT_URL = f'{HTTP}jm365.xyz/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来
JM_PUB_URL = f'{HTTP}jmcomic1.bet'
JM_CDN_IMAGE_URL_TEMPLATE = HTTP + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始
PROT = "https://"
_DOMAIN = None
JM_REDIRECT_URL = f'{PROT}jm365.xyz/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来
JM_PUB_URL = f'{PROT}jmcomic1.bet'
JM_CDN_IMAGE_URL_TEMPLATE = PROT + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始
JM_SERVER_ERROR_HTML = "Could not connect to mysql! Please check your database settings!"
JM_IMAGE_SUFFIX = ['.jpg', '.webp', '.png', '.gif']
@ -26,9 +26,21 @@ class JmModuleConfig:
jm_client_caches = {}
@classmethod
def default_headers(cls):
def domain(cls, postman=None):
"""
由于禁漫的域名经常变化调用此方法可以获取一个当前可用的最新的域名 domain
并且设置把 domain 设置为禁漫模块的默认域名
这样一来配置文件也不用配置域名了一切都在运行时动态获取
"""
if cls._DOMAIN is None:
cls._DOMAIN = cls.get_jmcomic_url(postman).replace(cls.PROT, '')
return cls._DOMAIN # jmcomic默认域名
@classmethod
def headers(cls, authority=None):
return {
'authority': cls.DOMAIN,
'authority': authority or cls.domain(),
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
'application/signed-exchange;v=b3;q=0.7',
'accept-language': 'zh-CN,zh;q=0.9',
@ -55,16 +67,16 @@ class JmModuleConfig:
cls.enable_jm_debug = False
@classmethod
def get_jmcomic_url(cls):
def get_jmcomic_url(cls, postman=None):
"""
访问禁漫的永久网域从而得到一个可用的禁漫网址
"""
from common import Postmans
if postman is None:
from common import Postmans
postman = Postmans.get_impl_clazz('cffi') \
.create(headers=cls.headers(cls.JM_REDIRECT_URL))
domain = Postmans \
.get_impl_clazz('cffi') \
.create(headers=cls.default_headers()) \
.with_wrap_resp() \
domain = postman.with_wrap_resp() \
.get(cls.JM_REDIRECT_URL, allow_redirects=False) \
.redirect_url

View File

@ -131,7 +131,7 @@ class JmPhotoDetail(WorkEntity):
@property
def album_id(self) -> str:
return self.photo_id if self.is_single_album else self._series_id
return self.photo_id if self.is_single_album else str(self._series_id)
@property
def album_index(self) -> int:

View File

@ -317,10 +317,10 @@ class JmOption(SaveableEntity):
@classmethod
def default_client_config(cls):
return {
'domain': JmModuleConfig.DOMAIN,
'domain': JmModuleConfig.domain(),
'meta_data': {
'cookies': None,
'headers': JmModuleConfig.default_headers(),
'headers': JmModuleConfig.headers(),
'allow_redirects': True,
},
'postman_type_list': [
@ -364,9 +364,13 @@ class JmOption(SaveableEntity):
def new_jm_client(self) -> JmcomicClient:
meta_data = self.client_config['meta_data']
postman_clazz = Postmans.get_impl_clazz(self.client_config.get('postman_type', 'cffi'))
proxies = None
domain = None
postman: Optional[Postman] = None
# 处理代理
def handle_proxies(key='proxies'):
def decide_proxies(key='proxies'):
nonlocal proxies
proxies = meta_data.get(key, None)
# 无代理,或代理已配置好好的
@ -381,25 +385,46 @@ class JmOption(SaveableEntity):
meta_data[key] = proxies
# 处理 headers
def decide_domain(key='domain') -> str:
nonlocal domain
domain = self.client_config.get(key, None)
if domain is None:
temp_postman = postman_clazz.create(
headers=JmModuleConfig.headers(JmModuleConfig.JM_REDIRECT_URL),
proxies=proxies,
)
domain = JmModuleConfig.domain(temp_postman)
domain = JmcomicText.parse_to_jm_domain(domain)
self.client_config[key] = domain
return domain
def handle_headers(key='headers'):
headers = meta_data.get(key, None)
if headers is None or (not isinstance(headers, dict)) or len(headers) == 0:
meta_data[key] = JmModuleConfig.default_headers()
# 未配置headers使用默认headers
headers = JmModuleConfig.headers()
# 处理【特殊配置项】
handle_proxies()
meta_data[key] = headers
def handle_postman():
nonlocal postman
postman = postman_clazz(meta_data)
# 1. 决定 代理
decide_proxies()
# 2. 指定 JM域名
decide_domain()
# 3. 处理 headers
handle_headers()
# 决定Postman的实现类根据配置项 client_config.postman
postman_clazz = Postmans.get_impl_clazz(self.client_config.get('postman_type', 'cffi'))
# 决定域名
domain = self.client_config.get('domain', JmModuleConfig.DOMAIN)
# 4. 创建 postman
handle_postman()
jm_debug('创建JmcomicClient', f'使用域名: {domain}使用Postman实现: {postman_clazz}')
# 创建 JmcomicClient 实例
# 创建 JmcomicClient 对象
client = JmcomicClient(
postman=postman_clazz(meta_data),
postman=postman,
domain=domain,
retry_times=self.client_config.get('retry_times', None)
)
@ -413,7 +438,7 @@ class JmOption(SaveableEntity):
def build_cdn_option(self, use_multi_thread_strategy=True):
return CdnConfig.create(
cdn_domain=self.client_config.get('domain', JmModuleConfig.DOMAIN),
cdn_domain=self.client_config.get('domain', JmModuleConfig.domain()),
fetch_strategy=MultiThreadFetch if use_multi_thread_strategy else InOrderFetch,
cdn_image_suffix=None,
use_cache=self.download_use_disk_cache,

View File

@ -35,12 +35,10 @@ class JmTestConfigurable(unittest.TestCase):
# 设置 JmOptionJmcomicClient
option = cls.use_option('option_test.yml')
cls.option = option
cls.client = option.build_jm_client()
client = option.build_jm_client()
# enable cache
client.get_photo_detail = enable_cache()(client.get_photo_detail)
client.get_album_detail = enable_cache()(client.get_album_detail)
cls.client = client
# 启用 JmClientClient 缓存
cls.enable_client_cache()
# 跨平台设置
cls.adapt_os()
@ -79,3 +77,7 @@ class JmTestConfigurable(unittest.TestCase):
@classmethod
def adapt_macos(cls):
pass
@classmethod
def enable_client_cache(cls):
cls.client.enable_cache()

View File

@ -55,7 +55,7 @@ class Test_Api(JmTestConfigurable):
photo_detail: JmPhotoDetail,
index: int,
) -> StrNone:
return workspace(f'{time_stamp()}_{photo_detail[index].img_file_name}.test.png')
return workspace(f'advice_{photo_detail[index].img_file_name}.test.png')
option = self.option
option.register_advice(MyAdvice())