v1.5.0: 优化JmcomicClient的配置流程、缓存启用,模块配置更加清晰。 (#12)

This commit is contained in:
hect0x7 2023-04-07 18:04:39 +08:00 committed by GitHub
parent ca59399a3e
commit 12351dd454
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 100 additions and 59 deletions

View File

@ -2,6 +2,6 @@
# 被依赖方 <--- 使用方 # 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- service <--- option # config <--- entity <--- toolkit <--- client <--- service <--- option
__version__ = '1.4.0' __version__ = '1.5.0'
from .api import * from .api import *

View File

@ -131,37 +131,17 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
) )
def renew_jm_default_domain():
"""
由于禁漫的域名经常变化调用此方法可以获取一个当前可用的最新的域名 domain
并且设置把 domain 设置为禁漫模块的默认域名
这样一来配置文件也不用配置域名了一切都在运行时动态获取
"""
domain = JmcomicText.parse_to_jm_domain(JmModuleConfig.get_jmcomic_url())
JmModuleConfig.DOMAIN = domain
return domain
def build_client(option: Optional[JmOption]) -> Tuple[JmOption, JmcomicClient]: def build_client(option: Optional[JmOption]) -> Tuple[JmOption, JmcomicClient]:
""" """
处理option的判空并且创建jm_client 处理option的判空并且创建jm_client
""" """
if option is None: if option is None:
option = JmOption.default() option = JmOption.default()
option.client_config['domain'] = renew_jm_default_domain()
jm_client = option.build_jm_client() jm_client = option.build_jm_client()
return option, jm_client return option, jm_client
def create_option(filepath: str) -> JmOption: def create_option(filepath: str) -> JmOption:
"""
创建 JmOption同时检查域名是否配置未配置则补上配置
@param filepath:
@return:
"""
option = JmOption.create_from_file(filepath) option = JmOption.create_from_file(filepath)
client_config = option.client_config
key = 'domain'
if client_config.get(key, None) is None or client_config[key] is None:
client_config[key] = renew_jm_default_domain()
return option return option

View File

@ -120,7 +120,7 @@ class JmcomicClient(PostmanProxy):
# -- 对象方法 -- # -- 对象方法 --
def of_api_url(self, api_path): def of_api_url(self, api_path):
return f"{JmModuleConfig.HTTP}{self.domain}{api_path}" return f"{JmModuleConfig.PROT}{self.domain}{api_path}"
def jm_get(self, url, is_api=True, require_200=True, **kwargs): def jm_get(self, url, is_api=True, require_200=True, **kwargs):
""" """
@ -157,6 +157,28 @@ class JmcomicClient(PostmanProxy):
def img_is_not_need_to_decode(cls, data_original: str, _resp): def img_is_not_need_to_decode(cls, data_original: str, _resp):
return data_original.endswith('.gif') return data_original.endswith('.gif')
# noinspection PyAttributeOutsideInit
def enable_cache(self):
def wrap_func_cache(func_name, cache_dict_name):
if hasattr(self, cache_dict_name):
return
cache_dict = {}
setattr(self, cache_dict_name, cache_dict)
# 重载本对象的方法
func = getattr(self, func_name)
wrap_func = enable_cache(
cache_dict=cache_dict,
cache_hit_msg=f'命中 {cache_dict_name} ' + '→ [{}]]',
cache_miss_msg=f'缺失 {cache_dict_name} ' + '← [{}]',
)(func)
setattr(self, func_name, wrap_func)
wrap_func_cache('get_photo_detail', 'album_cache_dict')
wrap_func_cache('get_album_detail', 'photo_cache_dict')
# 爬取策略 # 爬取策略
class FetchStrategy: class FetchStrategy:

View File

@ -1,10 +1,10 @@
class JmModuleConfig: class JmModuleConfig:
# 网站相关 # 网站相关
HTTP = "https://" PROT = "https://"
DOMAIN = "jmcomic1.group" # jmcomic默认域名 _DOMAIN = None
JM_REDIRECT_URL = f'{HTTP}jm365.xyz/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来 JM_REDIRECT_URL = f'{PROT}jm365.xyz/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来
JM_PUB_URL = f'{HTTP}jmcomic1.bet' JM_PUB_URL = f'{PROT}jmcomic1.bet'
JM_CDN_IMAGE_URL_TEMPLATE = HTTP + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始 JM_CDN_IMAGE_URL_TEMPLATE = PROT + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始
JM_SERVER_ERROR_HTML = "Could not connect to mysql! Please check your database settings!" JM_SERVER_ERROR_HTML = "Could not connect to mysql! Please check your database settings!"
JM_IMAGE_SUFFIX = ['.jpg', '.webp', '.png', '.gif'] JM_IMAGE_SUFFIX = ['.jpg', '.webp', '.png', '.gif']
@ -26,9 +26,21 @@ class JmModuleConfig:
jm_client_caches = {} jm_client_caches = {}
@classmethod @classmethod
def default_headers(cls): def domain(cls, postman=None):
"""
由于禁漫的域名经常变化调用此方法可以获取一个当前可用的最新的域名 domain
并且设置把 domain 设置为禁漫模块的默认域名
这样一来配置文件也不用配置域名了一切都在运行时动态获取
"""
if cls._DOMAIN is None:
cls._DOMAIN = cls.get_jmcomic_url(postman).replace(cls.PROT, '')
return cls._DOMAIN # jmcomic默认域名
@classmethod
def headers(cls, authority=None):
return { return {
'authority': cls.DOMAIN, 'authority': authority or cls.domain(),
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,' 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
'application/signed-exchange;v=b3;q=0.7', 'application/signed-exchange;v=b3;q=0.7',
'accept-language': 'zh-CN,zh;q=0.9', 'accept-language': 'zh-CN,zh;q=0.9',
@ -55,16 +67,16 @@ class JmModuleConfig:
cls.enable_jm_debug = False cls.enable_jm_debug = False
@classmethod @classmethod
def get_jmcomic_url(cls): def get_jmcomic_url(cls, postman=None):
""" """
访问禁漫的永久网域从而得到一个可用的禁漫网址 访问禁漫的永久网域从而得到一个可用的禁漫网址
""" """
from common import Postmans if postman is None:
from common import Postmans
postman = Postmans.get_impl_clazz('cffi') \
.create(headers=cls.headers(cls.JM_REDIRECT_URL))
domain = Postmans \ domain = postman.with_wrap_resp() \
.get_impl_clazz('cffi') \
.create(headers=cls.default_headers()) \
.with_wrap_resp() \
.get(cls.JM_REDIRECT_URL, allow_redirects=False) \ .get(cls.JM_REDIRECT_URL, allow_redirects=False) \
.redirect_url .redirect_url

View File

@ -131,7 +131,7 @@ class JmPhotoDetail(WorkEntity):
@property @property
def album_id(self) -> str: def album_id(self) -> str:
return self.photo_id if self.is_single_album else self._series_id return self.photo_id if self.is_single_album else str(self._series_id)
@property @property
def album_index(self) -> int: def album_index(self) -> int:

View File

@ -317,10 +317,10 @@ class JmOption(SaveableEntity):
@classmethod @classmethod
def default_client_config(cls): def default_client_config(cls):
return { return {
'domain': JmModuleConfig.DOMAIN, 'domain': JmModuleConfig.domain(),
'meta_data': { 'meta_data': {
'cookies': None, 'cookies': None,
'headers': JmModuleConfig.default_headers(), 'headers': JmModuleConfig.headers(),
'allow_redirects': True, 'allow_redirects': True,
}, },
'postman_type_list': [ 'postman_type_list': [
@ -364,9 +364,13 @@ class JmOption(SaveableEntity):
def new_jm_client(self) -> JmcomicClient: def new_jm_client(self) -> JmcomicClient:
meta_data = self.client_config['meta_data'] meta_data = self.client_config['meta_data']
postman_clazz = Postmans.get_impl_clazz(self.client_config.get('postman_type', 'cffi'))
proxies = None
domain = None
postman: Optional[Postman] = None
# 处理代理 def decide_proxies(key='proxies'):
def handle_proxies(key='proxies'): nonlocal proxies
proxies = meta_data.get(key, None) proxies = meta_data.get(key, None)
# 无代理,或代理已配置好好的 # 无代理,或代理已配置好好的
@ -381,25 +385,46 @@ class JmOption(SaveableEntity):
meta_data[key] = proxies meta_data[key] = proxies
# 处理 headers def decide_domain(key='domain') -> str:
nonlocal domain
domain = self.client_config.get(key, None)
if domain is None:
temp_postman = postman_clazz.create(
headers=JmModuleConfig.headers(JmModuleConfig.JM_REDIRECT_URL),
proxies=proxies,
)
domain = JmModuleConfig.domain(temp_postman)
domain = JmcomicText.parse_to_jm_domain(domain)
self.client_config[key] = domain
return domain
def handle_headers(key='headers'): def handle_headers(key='headers'):
headers = meta_data.get(key, None) headers = meta_data.get(key, None)
if headers is None or (not isinstance(headers, dict)) or len(headers) == 0: if headers is None or (not isinstance(headers, dict)) or len(headers) == 0:
meta_data[key] = JmModuleConfig.default_headers() # 未配置headers使用默认headers
headers = JmModuleConfig.headers()
# 处理【特殊配置项】 meta_data[key] = headers
handle_proxies()
def handle_postman():
nonlocal postman
postman = postman_clazz(meta_data)
# 1. 决定 代理
decide_proxies()
# 2. 指定 JM域名
decide_domain()
# 3. 处理 headers
handle_headers() handle_headers()
# 4. 创建 postman
# 决定Postman的实现类根据配置项 client_config.postman handle_postman()
postman_clazz = Postmans.get_impl_clazz(self.client_config.get('postman_type', 'cffi'))
# 决定域名
domain = self.client_config.get('domain', JmModuleConfig.DOMAIN)
jm_debug('创建JmcomicClient', f'使用域名: {domain}使用Postman实现: {postman_clazz}') jm_debug('创建JmcomicClient', f'使用域名: {domain}使用Postman实现: {postman_clazz}')
# 创建 JmcomicClient 实例
# 创建 JmcomicClient 对象
client = JmcomicClient( client = JmcomicClient(
postman=postman_clazz(meta_data), postman=postman,
domain=domain, domain=domain,
retry_times=self.client_config.get('retry_times', None) retry_times=self.client_config.get('retry_times', None)
) )
@ -413,7 +438,7 @@ class JmOption(SaveableEntity):
def build_cdn_option(self, use_multi_thread_strategy=True): def build_cdn_option(self, use_multi_thread_strategy=True):
return CdnConfig.create( return CdnConfig.create(
cdn_domain=self.client_config.get('domain', JmModuleConfig.DOMAIN), cdn_domain=self.client_config.get('domain', JmModuleConfig.domain()),
fetch_strategy=MultiThreadFetch if use_multi_thread_strategy else InOrderFetch, fetch_strategy=MultiThreadFetch if use_multi_thread_strategy else InOrderFetch,
cdn_image_suffix=None, cdn_image_suffix=None,
use_cache=self.download_use_disk_cache, use_cache=self.download_use_disk_cache,

View File

@ -35,12 +35,10 @@ class JmTestConfigurable(unittest.TestCase):
# 设置 JmOptionJmcomicClient # 设置 JmOptionJmcomicClient
option = cls.use_option('option_test.yml') option = cls.use_option('option_test.yml')
cls.option = option cls.option = option
cls.client = option.build_jm_client()
client = option.build_jm_client() # 启用 JmClientClient 缓存
# enable cache cls.enable_client_cache()
client.get_photo_detail = enable_cache()(client.get_photo_detail)
client.get_album_detail = enable_cache()(client.get_album_detail)
cls.client = client
# 跨平台设置 # 跨平台设置
cls.adapt_os() cls.adapt_os()
@ -79,3 +77,7 @@ class JmTestConfigurable(unittest.TestCase):
@classmethod @classmethod
def adapt_macos(cls): def adapt_macos(cls):
pass pass
@classmethod
def enable_client_cache(cls):
cls.client.enable_cache()

View File

@ -55,7 +55,7 @@ class Test_Api(JmTestConfigurable):
photo_detail: JmPhotoDetail, photo_detail: JmPhotoDetail,
index: int, index: int,
) -> StrNone: ) -> StrNone:
return workspace(f'{time_stamp()}_{photo_detail[index].img_file_name}.test.png') return workspace(f'advice_{photo_detail[index].img_file_name}.test.png')
option = self.option option = self.option
option.register_advice(MyAdvice()) option.register_advice(MyAdvice())