mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v2.3.11: 更新禁漫移动端的域名、headers (#153)
This commit is contained in:
parent
4eeee1e8d2
commit
a0e8af9fe5
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.3.10'
|
||||
__version__ = '2.3.11'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
||||
@ -484,7 +484,7 @@ class JmApiClient(AbstractJmClient):
|
||||
"id": photo_id,
|
||||
"mode": "vertical",
|
||||
"page": "0",
|
||||
"app_img_shunt": "NaN",
|
||||
"app_img_shunt": "1",
|
||||
}
|
||||
)
|
||||
|
||||
@ -528,14 +528,7 @@ class JmApiClient(AbstractJmClient):
|
||||
@property
|
||||
def headers_key_ts(self):
|
||||
key_ts = time_stamp()
|
||||
import hashlib
|
||||
token = hashlib.md5(f"{key_ts}{JmModuleConfig.MAGIC_18COMICAPPCONTENT}".encode()).hexdigest()
|
||||
return {
|
||||
"token": token,
|
||||
"tokenparam": f"{key_ts},1.5.2",
|
||||
"user-agent": "okhttp/3.12.1",
|
||||
"accept-encoding": "gzip",
|
||||
}, key_ts
|
||||
return JmModuleConfig.new_api_headers(key_ts), key_ts
|
||||
|
||||
def debug_topic_request(self):
|
||||
return 'api'
|
||||
|
||||
@ -65,12 +65,7 @@ class JmModuleConfig:
|
||||
# 图片域名
|
||||
DOMAIN_API_IMAGE_LIST = [f"cdn-msp.jmapiproxy{i}.cc" for i in range(1, 4)]
|
||||
# API域名
|
||||
DOMAIN_API_LIST = [
|
||||
"www.jmapinode1.cc",
|
||||
"www.jmapinode2.cc",
|
||||
"www.jmapinode3.cc",
|
||||
"www.jmapibranch2.cc",
|
||||
]
|
||||
DOMAIN_API_LIST = [f'www.jmapinode{i}.top' for i in range(1, 4)]
|
||||
|
||||
# 域名配置 - 网页端
|
||||
# 无需配置,默认为None,需要的时候会发起请求获得
|
||||
@ -199,7 +194,10 @@ class JmModuleConfig:
|
||||
return domain_list
|
||||
|
||||
@classmethod
|
||||
def headers(cls, domain='18comic.vip'):
|
||||
def new_html_headers(cls, domain='18comic.vip'):
|
||||
"""
|
||||
网页端的headers
|
||||
"""
|
||||
return {
|
||||
'authority': domain,
|
||||
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
|
||||
@ -218,6 +216,33 @@ class JmModuleConfig:
|
||||
'Safari/537.36',
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def new_api_headers(cls, key_ts):
|
||||
"""
|
||||
根据key_ts生成移动端的headers
|
||||
"""
|
||||
if key_ts is None:
|
||||
from common import time_stamp
|
||||
key_ts = time_stamp()
|
||||
|
||||
import hashlib
|
||||
token = hashlib.md5(f"{key_ts}{cls.MAGIC_18COMICAPPCONTENT}".encode()).hexdigest()
|
||||
|
||||
return {
|
||||
'token': token,
|
||||
'tokenparam': f"{key_ts},1.6.0",
|
||||
'User-Agent': 'Mozilla/5.0 (Linux; Android 9; V1938CT Build/PQ3A.190705.09211555; wv) AppleWebKit/537.36 (KHTML, '
|
||||
'like Gecko) Version/4.0 Chrome/91.0.4472.114 Safari/537.36',
|
||||
'X-Requested-With': 'com.jiaohua_browser',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
|
||||
'application/signed-exchange;v=b3;q=0.9',
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-User': '?1',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
|
||||
}
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
@classmethod
|
||||
def jm_debug(cls, topic: str, msg: str):
|
||||
@ -231,7 +256,7 @@ class JmModuleConfig:
|
||||
@classmethod
|
||||
def new_postman(cls, session=False, **kwargs):
|
||||
kwargs.setdefault('impersonate', 'chrome110')
|
||||
kwargs.setdefault('headers', JmModuleConfig.headers())
|
||||
kwargs.setdefault('headers', JmModuleConfig.new_html_headers())
|
||||
kwargs.setdefault('proxies', JmModuleConfig.DEFAULT_PROXIES)
|
||||
return cls.postman_constructor(session, **kwargs)
|
||||
|
||||
|
||||
@ -128,7 +128,7 @@ class JmOption:
|
||||
self.version = JmModuleConfig.JM_OPTION_VER
|
||||
# 路径规则配置
|
||||
self.dir_rule = DirRule(**dir_rule)
|
||||
# 请求配置
|
||||
# 客户端配置
|
||||
self.client = AdvancedEasyAccessDict(client)
|
||||
# 下载配置
|
||||
self.download = AdvancedEasyAccessDict(download)
|
||||
@ -340,7 +340,7 @@ class JmOption:
|
||||
# headers
|
||||
meta_data = postman_conf['meta_data']
|
||||
if meta_data['headers'] is None:
|
||||
meta_data['headers'] = JmModuleConfig.headers(domain[0])
|
||||
meta_data['headers'] = self.decide_postman_headers(impl, domain[0])
|
||||
|
||||
# postman
|
||||
postman = Postmans.create(data=postman_conf)
|
||||
@ -363,15 +363,7 @@ class JmOption:
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def decide_client_domain(self, client_key: str) -> List[str]:
|
||||
def is_client_type(ct: Type[JmcomicClient]):
|
||||
if client_key == ct:
|
||||
return True
|
||||
|
||||
clazz = JmModuleConfig.client_impl_class(client_key)
|
||||
if issubclass(clazz, ct):
|
||||
return True
|
||||
|
||||
return False
|
||||
is_client_type = lambda ctype: self.client_key_is_given_type(client_key, ctype)
|
||||
|
||||
if is_client_type(JmApiClient):
|
||||
# 移动端
|
||||
@ -383,6 +375,31 @@ class JmOption:
|
||||
|
||||
ExceptionTool.raises(f'没有配置域名,且是无法识别的client类型: {client_key}')
|
||||
|
||||
def decide_postman_headers(self, client_key, domain):
|
||||
is_client_type = lambda ctype: self.client_key_is_given_type(client_key, ctype)
|
||||
|
||||
if is_client_type(JmApiClient):
|
||||
# 移动端
|
||||
# 不配置headers,由client每次请求前创建headers
|
||||
return {}
|
||||
|
||||
if is_client_type(JmHtmlClient):
|
||||
# 网页端
|
||||
return JmModuleConfig.new_html_headers(domain)
|
||||
|
||||
ExceptionTool.raises(f'没有配置域名,且是无法识别的client类型: {client_key}')
|
||||
|
||||
@classmethod
|
||||
def client_key_is_given_type(cls, client_key, ctype: Type[JmcomicClient]):
|
||||
if client_key == ctype.client_key:
|
||||
return True
|
||||
|
||||
clazz = JmModuleConfig.client_impl_class(client_key)
|
||||
if issubclass(clazz, ctype):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def merge_default_dict(cls, user_dict, default_dict=None):
|
||||
"""
|
||||
|
||||
@ -78,6 +78,8 @@ class Test_Client(JmTestConfigurable):
|
||||
|
||||
def test_photo_sort(self):
|
||||
client = self.option.build_jm_client()
|
||||
get_photo_detail = lambda *args: client.get_photo_detail(*args, fetch_album=False, fetch_scramble_id=False)
|
||||
get_album_detail = client.get_album_detail
|
||||
|
||||
# 测试用例 - 单章本子
|
||||
single_photo_album_is = str_to_list('''
|
||||
@ -92,8 +94,8 @@ class Test_Client(JmTestConfigurable):
|
||||
122061
|
||||
''')
|
||||
|
||||
photo_dict: Dict[str, JmPhotoDetail] = multi_call(client.get_photo_detail, single_photo_album_is)
|
||||
album_dict: Dict[str, JmAlbumDetail] = multi_call(client.get_album_detail, single_photo_album_is)
|
||||
photo_dict: Dict[str, JmPhotoDetail] = multi_call(get_photo_detail, single_photo_album_is)
|
||||
album_dict: Dict[str, JmAlbumDetail] = multi_call(get_album_detail, single_photo_album_is)
|
||||
|
||||
for each in photo_dict.values():
|
||||
each: JmPhotoDetail
|
||||
@ -107,10 +109,10 @@ class Test_Client(JmTestConfigurable):
|
||||
multi_photo_album_dict: Dict[JmAlbumDetail, List[JmPhotoDetail]] = {}
|
||||
|
||||
def run(aid):
|
||||
album = client.get_album_detail(aid)
|
||||
album = get_album_detail(aid)
|
||||
|
||||
photo_dict = multi_call(
|
||||
client.get_photo_detail,
|
||||
get_photo_detail,
|
||||
(photo.photo_id for photo in album),
|
||||
launcher=thread_pool_executor,
|
||||
)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user