mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v1.5.0: 优化JmcomicClient的配置流程、缓存启用,模块配置更加清晰。 (#12)
This commit is contained in:
parent
ca59399a3e
commit
12351dd454
@ -2,6 +2,6 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- service <--- option
|
||||
|
||||
__version__ = '1.4.0'
|
||||
__version__ = '1.5.0'
|
||||
|
||||
from .api import *
|
||||
|
||||
@ -131,37 +131,17 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
|
||||
)
|
||||
|
||||
|
||||
def renew_jm_default_domain():
|
||||
"""
|
||||
由于禁漫的域名经常变化,调用此方法可以获取一个当前可用的最新的域名 domain,
|
||||
并且设置把 domain 设置为禁漫模块的默认域名。
|
||||
这样一来,配置文件也不用配置域名了,一切都在运行时动态获取。
|
||||
"""
|
||||
domain = JmcomicText.parse_to_jm_domain(JmModuleConfig.get_jmcomic_url())
|
||||
JmModuleConfig.DOMAIN = domain
|
||||
return domain
|
||||
|
||||
|
||||
def build_client(option: Optional[JmOption]) -> Tuple[JmOption, JmcomicClient]:
|
||||
"""
|
||||
处理option的判空,并且创建jm_client
|
||||
"""
|
||||
if option is None:
|
||||
option = JmOption.default()
|
||||
option.client_config['domain'] = renew_jm_default_domain()
|
||||
|
||||
jm_client = option.build_jm_client()
|
||||
return option, jm_client
|
||||
|
||||
|
||||
def create_option(filepath: str) -> JmOption:
|
||||
"""
|
||||
创建 JmOption,同时检查域名是否配置,未配置则补上配置
|
||||
@param filepath:
|
||||
@return:
|
||||
"""
|
||||
option = JmOption.create_from_file(filepath)
|
||||
client_config = option.client_config
|
||||
key = 'domain'
|
||||
if client_config.get(key, None) is None or client_config[key] is None:
|
||||
client_config[key] = renew_jm_default_domain()
|
||||
return option
|
||||
|
||||
@ -120,7 +120,7 @@ class JmcomicClient(PostmanProxy):
|
||||
# -- 对象方法 --
|
||||
|
||||
def of_api_url(self, api_path):
|
||||
return f"{JmModuleConfig.HTTP}{self.domain}{api_path}"
|
||||
return f"{JmModuleConfig.PROT}{self.domain}{api_path}"
|
||||
|
||||
def jm_get(self, url, is_api=True, require_200=True, **kwargs):
|
||||
"""
|
||||
@ -157,6 +157,28 @@ class JmcomicClient(PostmanProxy):
|
||||
def img_is_not_need_to_decode(cls, data_original: str, _resp):
|
||||
return data_original.endswith('.gif')
|
||||
|
||||
# noinspection PyAttributeOutsideInit
|
||||
def enable_cache(self):
|
||||
def wrap_func_cache(func_name, cache_dict_name):
|
||||
if hasattr(self, cache_dict_name):
|
||||
return
|
||||
|
||||
cache_dict = {}
|
||||
setattr(self, cache_dict_name, cache_dict)
|
||||
|
||||
# 重载本对象的方法
|
||||
func = getattr(self, func_name)
|
||||
wrap_func = enable_cache(
|
||||
cache_dict=cache_dict,
|
||||
cache_hit_msg=f'命中 {cache_dict_name} ' + '→ [{}]]',
|
||||
cache_miss_msg=f'缺失 {cache_dict_name} ' + '← [{}]',
|
||||
)(func)
|
||||
|
||||
setattr(self, func_name, wrap_func)
|
||||
|
||||
wrap_func_cache('get_photo_detail', 'album_cache_dict')
|
||||
wrap_func_cache('get_album_detail', 'photo_cache_dict')
|
||||
|
||||
|
||||
# 爬取策略
|
||||
class FetchStrategy:
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
class JmModuleConfig:
|
||||
# 网站相关
|
||||
HTTP = "https://"
|
||||
DOMAIN = "jmcomic1.group" # jmcomic默认域名
|
||||
JM_REDIRECT_URL = f'{HTTP}jm365.xyz/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来↓
|
||||
JM_PUB_URL = f'{HTTP}jmcomic1.bet'
|
||||
JM_CDN_IMAGE_URL_TEMPLATE = HTTP + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始
|
||||
PROT = "https://"
|
||||
_DOMAIN = None
|
||||
JM_REDIRECT_URL = f'{PROT}jm365.xyz/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来
|
||||
JM_PUB_URL = f'{PROT}jmcomic1.bet'
|
||||
JM_CDN_IMAGE_URL_TEMPLATE = PROT + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始
|
||||
JM_SERVER_ERROR_HTML = "Could not connect to mysql! Please check your database settings!"
|
||||
JM_IMAGE_SUFFIX = ['.jpg', '.webp', '.png', '.gif']
|
||||
|
||||
@ -26,9 +26,21 @@ class JmModuleConfig:
|
||||
jm_client_caches = {}
|
||||
|
||||
@classmethod
|
||||
def default_headers(cls):
|
||||
def domain(cls, postman=None):
|
||||
"""
|
||||
由于禁漫的域名经常变化,调用此方法可以获取一个当前可用的最新的域名 domain,
|
||||
并且设置把 domain 设置为禁漫模块的默认域名。
|
||||
这样一来,配置文件也不用配置域名了,一切都在运行时动态获取。
|
||||
"""
|
||||
if cls._DOMAIN is None:
|
||||
cls._DOMAIN = cls.get_jmcomic_url(postman).replace(cls.PROT, '')
|
||||
|
||||
return cls._DOMAIN # jmcomic默认域名
|
||||
|
||||
@classmethod
|
||||
def headers(cls, authority=None):
|
||||
return {
|
||||
'authority': cls.DOMAIN,
|
||||
'authority': authority or cls.domain(),
|
||||
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
|
||||
'application/signed-exchange;v=b3;q=0.7',
|
||||
'accept-language': 'zh-CN,zh;q=0.9',
|
||||
@ -55,16 +67,16 @@ class JmModuleConfig:
|
||||
cls.enable_jm_debug = False
|
||||
|
||||
@classmethod
|
||||
def get_jmcomic_url(cls):
|
||||
def get_jmcomic_url(cls, postman=None):
|
||||
"""
|
||||
访问禁漫的永久网域,从而得到一个可用的禁漫网址,
|
||||
"""
|
||||
from common import Postmans
|
||||
if postman is None:
|
||||
from common import Postmans
|
||||
postman = Postmans.get_impl_clazz('cffi') \
|
||||
.create(headers=cls.headers(cls.JM_REDIRECT_URL))
|
||||
|
||||
domain = Postmans \
|
||||
.get_impl_clazz('cffi') \
|
||||
.create(headers=cls.default_headers()) \
|
||||
.with_wrap_resp() \
|
||||
domain = postman.with_wrap_resp() \
|
||||
.get(cls.JM_REDIRECT_URL, allow_redirects=False) \
|
||||
.redirect_url
|
||||
|
||||
|
||||
@ -131,7 +131,7 @@ class JmPhotoDetail(WorkEntity):
|
||||
|
||||
@property
|
||||
def album_id(self) -> str:
|
||||
return self.photo_id if self.is_single_album else self._series_id
|
||||
return self.photo_id if self.is_single_album else str(self._series_id)
|
||||
|
||||
@property
|
||||
def album_index(self) -> int:
|
||||
|
||||
@ -317,10 +317,10 @@ class JmOption(SaveableEntity):
|
||||
@classmethod
|
||||
def default_client_config(cls):
|
||||
return {
|
||||
'domain': JmModuleConfig.DOMAIN,
|
||||
'domain': JmModuleConfig.domain(),
|
||||
'meta_data': {
|
||||
'cookies': None,
|
||||
'headers': JmModuleConfig.default_headers(),
|
||||
'headers': JmModuleConfig.headers(),
|
||||
'allow_redirects': True,
|
||||
},
|
||||
'postman_type_list': [
|
||||
@ -364,9 +364,13 @@ class JmOption(SaveableEntity):
|
||||
|
||||
def new_jm_client(self) -> JmcomicClient:
|
||||
meta_data = self.client_config['meta_data']
|
||||
postman_clazz = Postmans.get_impl_clazz(self.client_config.get('postman_type', 'cffi'))
|
||||
proxies = None
|
||||
domain = None
|
||||
postman: Optional[Postman] = None
|
||||
|
||||
# 处理代理
|
||||
def handle_proxies(key='proxies'):
|
||||
def decide_proxies(key='proxies'):
|
||||
nonlocal proxies
|
||||
proxies = meta_data.get(key, None)
|
||||
|
||||
# 无代理,或代理已配置好好的
|
||||
@ -381,25 +385,46 @@ class JmOption(SaveableEntity):
|
||||
|
||||
meta_data[key] = proxies
|
||||
|
||||
# 处理 headers
|
||||
def decide_domain(key='domain') -> str:
|
||||
nonlocal domain
|
||||
domain = self.client_config.get(key, None)
|
||||
if domain is None:
|
||||
temp_postman = postman_clazz.create(
|
||||
headers=JmModuleConfig.headers(JmModuleConfig.JM_REDIRECT_URL),
|
||||
proxies=proxies,
|
||||
)
|
||||
domain = JmModuleConfig.domain(temp_postman)
|
||||
|
||||
domain = JmcomicText.parse_to_jm_domain(domain)
|
||||
self.client_config[key] = domain
|
||||
return domain
|
||||
|
||||
def handle_headers(key='headers'):
|
||||
headers = meta_data.get(key, None)
|
||||
if headers is None or (not isinstance(headers, dict)) or len(headers) == 0:
|
||||
meta_data[key] = JmModuleConfig.default_headers()
|
||||
# 未配置headers,使用默认headers
|
||||
headers = JmModuleConfig.headers()
|
||||
|
||||
# 处理【特殊配置项】
|
||||
handle_proxies()
|
||||
meta_data[key] = headers
|
||||
|
||||
def handle_postman():
|
||||
nonlocal postman
|
||||
postman = postman_clazz(meta_data)
|
||||
|
||||
# 1. 决定 代理
|
||||
decide_proxies()
|
||||
# 2. 指定 JM域名
|
||||
decide_domain()
|
||||
# 3. 处理 headers
|
||||
handle_headers()
|
||||
|
||||
# 决定Postman的实现类,根据配置项 client_config.postman
|
||||
postman_clazz = Postmans.get_impl_clazz(self.client_config.get('postman_type', 'cffi'))
|
||||
# 决定域名
|
||||
domain = self.client_config.get('domain', JmModuleConfig.DOMAIN)
|
||||
# 4. 创建 postman
|
||||
handle_postman()
|
||||
|
||||
jm_debug('创建JmcomicClient', f'使用域名: {domain},使用Postman实现: {postman_clazz}')
|
||||
# 创建 JmcomicClient 实例
|
||||
|
||||
# 创建 JmcomicClient 对象
|
||||
client = JmcomicClient(
|
||||
postman=postman_clazz(meta_data),
|
||||
postman=postman,
|
||||
domain=domain,
|
||||
retry_times=self.client_config.get('retry_times', None)
|
||||
)
|
||||
@ -413,7 +438,7 @@ class JmOption(SaveableEntity):
|
||||
def build_cdn_option(self, use_multi_thread_strategy=True):
|
||||
|
||||
return CdnConfig.create(
|
||||
cdn_domain=self.client_config.get('domain', JmModuleConfig.DOMAIN),
|
||||
cdn_domain=self.client_config.get('domain', JmModuleConfig.domain()),
|
||||
fetch_strategy=MultiThreadFetch if use_multi_thread_strategy else InOrderFetch,
|
||||
cdn_image_suffix=None,
|
||||
use_cache=self.download_use_disk_cache,
|
||||
|
||||
@ -35,12 +35,10 @@ class JmTestConfigurable(unittest.TestCase):
|
||||
# 设置 JmOption,JmcomicClient
|
||||
option = cls.use_option('option_test.yml')
|
||||
cls.option = option
|
||||
cls.client = option.build_jm_client()
|
||||
|
||||
client = option.build_jm_client()
|
||||
# enable cache
|
||||
client.get_photo_detail = enable_cache()(client.get_photo_detail)
|
||||
client.get_album_detail = enable_cache()(client.get_album_detail)
|
||||
cls.client = client
|
||||
# 启用 JmClientClient 缓存
|
||||
cls.enable_client_cache()
|
||||
|
||||
# 跨平台设置
|
||||
cls.adapt_os()
|
||||
@ -79,3 +77,7 @@ class JmTestConfigurable(unittest.TestCase):
|
||||
@classmethod
|
||||
def adapt_macos(cls):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def enable_client_cache(cls):
|
||||
cls.client.enable_cache()
|
||||
|
||||
@ -55,7 +55,7 @@ class Test_Api(JmTestConfigurable):
|
||||
photo_detail: JmPhotoDetail,
|
||||
index: int,
|
||||
) -> StrNone:
|
||||
return workspace(f'{time_stamp()}_{photo_detail[index].img_file_name}.test.png')
|
||||
return workspace(f'advice_{photo_detail[index].img_file_name}.test.png')
|
||||
|
||||
option = self.option
|
||||
option.register_advice(MyAdvice())
|
||||
|
||||
Loading…
Reference in New Issue
Block a user