mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v1.5.0: 优化JmcomicClient的配置流程、缓存启用,模块配置更加清晰。 (#12)
This commit is contained in:
parent
ca59399a3e
commit
12351dd454
@ -2,6 +2,6 @@
|
|||||||
# 被依赖方 <--- 使用方
|
# 被依赖方 <--- 使用方
|
||||||
# config <--- entity <--- toolkit <--- client <--- service <--- option
|
# config <--- entity <--- toolkit <--- client <--- service <--- option
|
||||||
|
|
||||||
__version__ = '1.4.0'
|
__version__ = '1.5.0'
|
||||||
|
|
||||||
from .api import *
|
from .api import *
|
||||||
|
|||||||
@ -131,37 +131,17 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def renew_jm_default_domain():
|
|
||||||
"""
|
|
||||||
由于禁漫的域名经常变化,调用此方法可以获取一个当前可用的最新的域名 domain,
|
|
||||||
并且设置把 domain 设置为禁漫模块的默认域名。
|
|
||||||
这样一来,配置文件也不用配置域名了,一切都在运行时动态获取。
|
|
||||||
"""
|
|
||||||
domain = JmcomicText.parse_to_jm_domain(JmModuleConfig.get_jmcomic_url())
|
|
||||||
JmModuleConfig.DOMAIN = domain
|
|
||||||
return domain
|
|
||||||
|
|
||||||
|
|
||||||
def build_client(option: Optional[JmOption]) -> Tuple[JmOption, JmcomicClient]:
|
def build_client(option: Optional[JmOption]) -> Tuple[JmOption, JmcomicClient]:
|
||||||
"""
|
"""
|
||||||
处理option的判空,并且创建jm_client
|
处理option的判空,并且创建jm_client
|
||||||
"""
|
"""
|
||||||
if option is None:
|
if option is None:
|
||||||
option = JmOption.default()
|
option = JmOption.default()
|
||||||
option.client_config['domain'] = renew_jm_default_domain()
|
|
||||||
jm_client = option.build_jm_client()
|
jm_client = option.build_jm_client()
|
||||||
return option, jm_client
|
return option, jm_client
|
||||||
|
|
||||||
|
|
||||||
def create_option(filepath: str) -> JmOption:
|
def create_option(filepath: str) -> JmOption:
|
||||||
"""
|
|
||||||
创建 JmOption,同时检查域名是否配置,未配置则补上配置
|
|
||||||
@param filepath:
|
|
||||||
@return:
|
|
||||||
"""
|
|
||||||
option = JmOption.create_from_file(filepath)
|
option = JmOption.create_from_file(filepath)
|
||||||
client_config = option.client_config
|
|
||||||
key = 'domain'
|
|
||||||
if client_config.get(key, None) is None or client_config[key] is None:
|
|
||||||
client_config[key] = renew_jm_default_domain()
|
|
||||||
return option
|
return option
|
||||||
|
|||||||
@ -120,7 +120,7 @@ class JmcomicClient(PostmanProxy):
|
|||||||
# -- 对象方法 --
|
# -- 对象方法 --
|
||||||
|
|
||||||
def of_api_url(self, api_path):
|
def of_api_url(self, api_path):
|
||||||
return f"{JmModuleConfig.HTTP}{self.domain}{api_path}"
|
return f"{JmModuleConfig.PROT}{self.domain}{api_path}"
|
||||||
|
|
||||||
def jm_get(self, url, is_api=True, require_200=True, **kwargs):
|
def jm_get(self, url, is_api=True, require_200=True, **kwargs):
|
||||||
"""
|
"""
|
||||||
@ -157,6 +157,28 @@ class JmcomicClient(PostmanProxy):
|
|||||||
def img_is_not_need_to_decode(cls, data_original: str, _resp):
|
def img_is_not_need_to_decode(cls, data_original: str, _resp):
|
||||||
return data_original.endswith('.gif')
|
return data_original.endswith('.gif')
|
||||||
|
|
||||||
|
# noinspection PyAttributeOutsideInit
|
||||||
|
def enable_cache(self):
|
||||||
|
def wrap_func_cache(func_name, cache_dict_name):
|
||||||
|
if hasattr(self, cache_dict_name):
|
||||||
|
return
|
||||||
|
|
||||||
|
cache_dict = {}
|
||||||
|
setattr(self, cache_dict_name, cache_dict)
|
||||||
|
|
||||||
|
# 重载本对象的方法
|
||||||
|
func = getattr(self, func_name)
|
||||||
|
wrap_func = enable_cache(
|
||||||
|
cache_dict=cache_dict,
|
||||||
|
cache_hit_msg=f'命中 {cache_dict_name} ' + '→ [{}]]',
|
||||||
|
cache_miss_msg=f'缺失 {cache_dict_name} ' + '← [{}]',
|
||||||
|
)(func)
|
||||||
|
|
||||||
|
setattr(self, func_name, wrap_func)
|
||||||
|
|
||||||
|
wrap_func_cache('get_photo_detail', 'album_cache_dict')
|
||||||
|
wrap_func_cache('get_album_detail', 'photo_cache_dict')
|
||||||
|
|
||||||
|
|
||||||
# 爬取策略
|
# 爬取策略
|
||||||
class FetchStrategy:
|
class FetchStrategy:
|
||||||
|
|||||||
@ -1,10 +1,10 @@
|
|||||||
class JmModuleConfig:
|
class JmModuleConfig:
|
||||||
# 网站相关
|
# 网站相关
|
||||||
HTTP = "https://"
|
PROT = "https://"
|
||||||
DOMAIN = "jmcomic1.group" # jmcomic默认域名
|
_DOMAIN = None
|
||||||
JM_REDIRECT_URL = f'{HTTP}jm365.xyz/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来↓
|
JM_REDIRECT_URL = f'{PROT}jm365.xyz/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来
|
||||||
JM_PUB_URL = f'{HTTP}jmcomic1.bet'
|
JM_PUB_URL = f'{PROT}jmcomic1.bet'
|
||||||
JM_CDN_IMAGE_URL_TEMPLATE = HTTP + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始
|
JM_CDN_IMAGE_URL_TEMPLATE = PROT + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始
|
||||||
JM_SERVER_ERROR_HTML = "Could not connect to mysql! Please check your database settings!"
|
JM_SERVER_ERROR_HTML = "Could not connect to mysql! Please check your database settings!"
|
||||||
JM_IMAGE_SUFFIX = ['.jpg', '.webp', '.png', '.gif']
|
JM_IMAGE_SUFFIX = ['.jpg', '.webp', '.png', '.gif']
|
||||||
|
|
||||||
@ -26,9 +26,21 @@ class JmModuleConfig:
|
|||||||
jm_client_caches = {}
|
jm_client_caches = {}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def default_headers(cls):
|
def domain(cls, postman=None):
|
||||||
|
"""
|
||||||
|
由于禁漫的域名经常变化,调用此方法可以获取一个当前可用的最新的域名 domain,
|
||||||
|
并且设置把 domain 设置为禁漫模块的默认域名。
|
||||||
|
这样一来,配置文件也不用配置域名了,一切都在运行时动态获取。
|
||||||
|
"""
|
||||||
|
if cls._DOMAIN is None:
|
||||||
|
cls._DOMAIN = cls.get_jmcomic_url(postman).replace(cls.PROT, '')
|
||||||
|
|
||||||
|
return cls._DOMAIN # jmcomic默认域名
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def headers(cls, authority=None):
|
||||||
return {
|
return {
|
||||||
'authority': cls.DOMAIN,
|
'authority': authority or cls.domain(),
|
||||||
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
|
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
|
||||||
'application/signed-exchange;v=b3;q=0.7',
|
'application/signed-exchange;v=b3;q=0.7',
|
||||||
'accept-language': 'zh-CN,zh;q=0.9',
|
'accept-language': 'zh-CN,zh;q=0.9',
|
||||||
@ -55,16 +67,16 @@ class JmModuleConfig:
|
|||||||
cls.enable_jm_debug = False
|
cls.enable_jm_debug = False
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_jmcomic_url(cls):
|
def get_jmcomic_url(cls, postman=None):
|
||||||
"""
|
"""
|
||||||
访问禁漫的永久网域,从而得到一个可用的禁漫网址,
|
访问禁漫的永久网域,从而得到一个可用的禁漫网址,
|
||||||
"""
|
"""
|
||||||
from common import Postmans
|
if postman is None:
|
||||||
|
from common import Postmans
|
||||||
|
postman = Postmans.get_impl_clazz('cffi') \
|
||||||
|
.create(headers=cls.headers(cls.JM_REDIRECT_URL))
|
||||||
|
|
||||||
domain = Postmans \
|
domain = postman.with_wrap_resp() \
|
||||||
.get_impl_clazz('cffi') \
|
|
||||||
.create(headers=cls.default_headers()) \
|
|
||||||
.with_wrap_resp() \
|
|
||||||
.get(cls.JM_REDIRECT_URL, allow_redirects=False) \
|
.get(cls.JM_REDIRECT_URL, allow_redirects=False) \
|
||||||
.redirect_url
|
.redirect_url
|
||||||
|
|
||||||
|
|||||||
@ -131,7 +131,7 @@ class JmPhotoDetail(WorkEntity):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def album_id(self) -> str:
|
def album_id(self) -> str:
|
||||||
return self.photo_id if self.is_single_album else self._series_id
|
return self.photo_id if self.is_single_album else str(self._series_id)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def album_index(self) -> int:
|
def album_index(self) -> int:
|
||||||
|
|||||||
@ -317,10 +317,10 @@ class JmOption(SaveableEntity):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def default_client_config(cls):
|
def default_client_config(cls):
|
||||||
return {
|
return {
|
||||||
'domain': JmModuleConfig.DOMAIN,
|
'domain': JmModuleConfig.domain(),
|
||||||
'meta_data': {
|
'meta_data': {
|
||||||
'cookies': None,
|
'cookies': None,
|
||||||
'headers': JmModuleConfig.default_headers(),
|
'headers': JmModuleConfig.headers(),
|
||||||
'allow_redirects': True,
|
'allow_redirects': True,
|
||||||
},
|
},
|
||||||
'postman_type_list': [
|
'postman_type_list': [
|
||||||
@ -364,9 +364,13 @@ class JmOption(SaveableEntity):
|
|||||||
|
|
||||||
def new_jm_client(self) -> JmcomicClient:
|
def new_jm_client(self) -> JmcomicClient:
|
||||||
meta_data = self.client_config['meta_data']
|
meta_data = self.client_config['meta_data']
|
||||||
|
postman_clazz = Postmans.get_impl_clazz(self.client_config.get('postman_type', 'cffi'))
|
||||||
|
proxies = None
|
||||||
|
domain = None
|
||||||
|
postman: Optional[Postman] = None
|
||||||
|
|
||||||
# 处理代理
|
def decide_proxies(key='proxies'):
|
||||||
def handle_proxies(key='proxies'):
|
nonlocal proxies
|
||||||
proxies = meta_data.get(key, None)
|
proxies = meta_data.get(key, None)
|
||||||
|
|
||||||
# 无代理,或代理已配置好好的
|
# 无代理,或代理已配置好好的
|
||||||
@ -381,25 +385,46 @@ class JmOption(SaveableEntity):
|
|||||||
|
|
||||||
meta_data[key] = proxies
|
meta_data[key] = proxies
|
||||||
|
|
||||||
# 处理 headers
|
def decide_domain(key='domain') -> str:
|
||||||
|
nonlocal domain
|
||||||
|
domain = self.client_config.get(key, None)
|
||||||
|
if domain is None:
|
||||||
|
temp_postman = postman_clazz.create(
|
||||||
|
headers=JmModuleConfig.headers(JmModuleConfig.JM_REDIRECT_URL),
|
||||||
|
proxies=proxies,
|
||||||
|
)
|
||||||
|
domain = JmModuleConfig.domain(temp_postman)
|
||||||
|
|
||||||
|
domain = JmcomicText.parse_to_jm_domain(domain)
|
||||||
|
self.client_config[key] = domain
|
||||||
|
return domain
|
||||||
|
|
||||||
def handle_headers(key='headers'):
|
def handle_headers(key='headers'):
|
||||||
headers = meta_data.get(key, None)
|
headers = meta_data.get(key, None)
|
||||||
if headers is None or (not isinstance(headers, dict)) or len(headers) == 0:
|
if headers is None or (not isinstance(headers, dict)) or len(headers) == 0:
|
||||||
meta_data[key] = JmModuleConfig.default_headers()
|
# 未配置headers,使用默认headers
|
||||||
|
headers = JmModuleConfig.headers()
|
||||||
|
|
||||||
# 处理【特殊配置项】
|
meta_data[key] = headers
|
||||||
handle_proxies()
|
|
||||||
|
def handle_postman():
|
||||||
|
nonlocal postman
|
||||||
|
postman = postman_clazz(meta_data)
|
||||||
|
|
||||||
|
# 1. 决定 代理
|
||||||
|
decide_proxies()
|
||||||
|
# 2. 指定 JM域名
|
||||||
|
decide_domain()
|
||||||
|
# 3. 处理 headers
|
||||||
handle_headers()
|
handle_headers()
|
||||||
|
# 4. 创建 postman
|
||||||
# 决定Postman的实现类,根据配置项 client_config.postman
|
handle_postman()
|
||||||
postman_clazz = Postmans.get_impl_clazz(self.client_config.get('postman_type', 'cffi'))
|
|
||||||
# 决定域名
|
|
||||||
domain = self.client_config.get('domain', JmModuleConfig.DOMAIN)
|
|
||||||
|
|
||||||
jm_debug('创建JmcomicClient', f'使用域名: {domain},使用Postman实现: {postman_clazz}')
|
jm_debug('创建JmcomicClient', f'使用域名: {domain},使用Postman实现: {postman_clazz}')
|
||||||
# 创建 JmcomicClient 实例
|
|
||||||
|
# 创建 JmcomicClient 对象
|
||||||
client = JmcomicClient(
|
client = JmcomicClient(
|
||||||
postman=postman_clazz(meta_data),
|
postman=postman,
|
||||||
domain=domain,
|
domain=domain,
|
||||||
retry_times=self.client_config.get('retry_times', None)
|
retry_times=self.client_config.get('retry_times', None)
|
||||||
)
|
)
|
||||||
@ -413,7 +438,7 @@ class JmOption(SaveableEntity):
|
|||||||
def build_cdn_option(self, use_multi_thread_strategy=True):
|
def build_cdn_option(self, use_multi_thread_strategy=True):
|
||||||
|
|
||||||
return CdnConfig.create(
|
return CdnConfig.create(
|
||||||
cdn_domain=self.client_config.get('domain', JmModuleConfig.DOMAIN),
|
cdn_domain=self.client_config.get('domain', JmModuleConfig.domain()),
|
||||||
fetch_strategy=MultiThreadFetch if use_multi_thread_strategy else InOrderFetch,
|
fetch_strategy=MultiThreadFetch if use_multi_thread_strategy else InOrderFetch,
|
||||||
cdn_image_suffix=None,
|
cdn_image_suffix=None,
|
||||||
use_cache=self.download_use_disk_cache,
|
use_cache=self.download_use_disk_cache,
|
||||||
|
|||||||
@ -35,12 +35,10 @@ class JmTestConfigurable(unittest.TestCase):
|
|||||||
# 设置 JmOption,JmcomicClient
|
# 设置 JmOption,JmcomicClient
|
||||||
option = cls.use_option('option_test.yml')
|
option = cls.use_option('option_test.yml')
|
||||||
cls.option = option
|
cls.option = option
|
||||||
|
cls.client = option.build_jm_client()
|
||||||
|
|
||||||
client = option.build_jm_client()
|
# 启用 JmClientClient 缓存
|
||||||
# enable cache
|
cls.enable_client_cache()
|
||||||
client.get_photo_detail = enable_cache()(client.get_photo_detail)
|
|
||||||
client.get_album_detail = enable_cache()(client.get_album_detail)
|
|
||||||
cls.client = client
|
|
||||||
|
|
||||||
# 跨平台设置
|
# 跨平台设置
|
||||||
cls.adapt_os()
|
cls.adapt_os()
|
||||||
@ -79,3 +77,7 @@ class JmTestConfigurable(unittest.TestCase):
|
|||||||
@classmethod
|
@classmethod
|
||||||
def adapt_macos(cls):
|
def adapt_macos(cls):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def enable_client_cache(cls):
|
||||||
|
cls.client.enable_cache()
|
||||||
|
|||||||
@ -55,7 +55,7 @@ class Test_Api(JmTestConfigurable):
|
|||||||
photo_detail: JmPhotoDetail,
|
photo_detail: JmPhotoDetail,
|
||||||
index: int,
|
index: int,
|
||||||
) -> StrNone:
|
) -> StrNone:
|
||||||
return workspace(f'{time_stamp()}_{photo_detail[index].img_file_name}.test.png')
|
return workspace(f'advice_{photo_detail[index].img_file_name}.test.png')
|
||||||
|
|
||||||
option = self.option
|
option = self.option
|
||||||
option.register_advice(MyAdvice())
|
option.register_advice(MyAdvice())
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user