mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.5.17: 更新禁漫APP端图片请求headers (#272)
This commit is contained in:
parent
710b229876
commit
fb260f2131
@ -8,7 +8,7 @@ client:
|
||||
retry_times: 3
|
||||
postman:
|
||||
meta_data:
|
||||
timeout: 7
|
||||
timeout: 15
|
||||
domain:
|
||||
html:
|
||||
- jmcomic1.me
|
||||
|
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.5.16'
|
||||
__version__ = '2.5.17'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
@ -92,7 +92,7 @@ class AbstractJmClient(
|
||||
jm_log(self.log_topic(), self.decode(url))
|
||||
else:
|
||||
# 图片url
|
||||
pass
|
||||
self.update_request_with_specify_domain(kwargs, None, True)
|
||||
|
||||
if domain_index != 0 or retry_count != 0:
|
||||
jm_log(f'req.retry',
|
||||
@ -133,7 +133,7 @@ class AbstractJmClient(
|
||||
"""
|
||||
return resp
|
||||
|
||||
def update_request_with_specify_domain(self, kwargs: dict, domain: str):
|
||||
def update_request_with_specify_domain(self, kwargs: dict, domain: Optional[str], is_image: bool = False):
|
||||
"""
|
||||
域名自动切换时,用于更新请求参数的回调
|
||||
"""
|
||||
@ -463,7 +463,10 @@ class JmHtmlClient(AbstractJmClient):
|
||||
|
||||
return resp
|
||||
|
||||
def update_request_with_specify_domain(self, kwargs: dict, domain: Optional[str]):
|
||||
def update_request_with_specify_domain(self, kwargs: dict, domain: Optional[str], is_image=False):
|
||||
if is_image:
|
||||
return
|
||||
|
||||
latest_headers = kwargs.get('headers', None)
|
||||
base_headers = self.get_meta_data('headers', None) or JmModuleConfig.new_html_headers(domain)
|
||||
base_headers.update(latest_headers or {})
|
||||
@ -909,8 +912,10 @@ class JmApiClient(AbstractJmClient):
|
||||
|
||||
return resp
|
||||
|
||||
def update_request_with_specify_domain(self, kwargs: dict, domain: str):
|
||||
pass
|
||||
def update_request_with_specify_domain(self, kwargs: dict, domain: Optional[str], is_image=False):
|
||||
if is_image:
|
||||
# 设置APP端的图片请求headers
|
||||
kwargs['headers'] = {**JmModuleConfig.APP_HEADERS_TEMPLATE, **JmModuleConfig.APP_HEADERS_IMAGE}
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def decide_headers_and_ts(self, kwargs, url):
|
||||
@ -930,7 +935,7 @@ class JmApiClient(AbstractJmClient):
|
||||
token, tokenparam = JmCryptoTool.token_and_tokenparam(ts)
|
||||
|
||||
# 设置headers
|
||||
headers = kwargs.get('headers', None) or JmMagicConstants.APP_HEADERS_TEMPLATE.copy()
|
||||
headers = kwargs.get('headers', None) or JmModuleConfig.APP_HEADERS_TEMPLATE.copy()
|
||||
headers.update({
|
||||
'token': token,
|
||||
'tokenparam': tokenparam,
|
||||
|
@ -59,29 +59,64 @@ class JmMagicConstants:
|
||||
SUB_SINGLE_JAPANESE = SUB_JAPANESE
|
||||
SUB_SINGLE_YOUTH = 'youth'
|
||||
|
||||
# 分页大小
|
||||
PAGE_SIZE_SEARCH = 80
|
||||
PAGE_SIZE_FAVORITE = 20
|
||||
|
||||
# 图片分割参数
|
||||
SCRAMBLE_220980 = 220980
|
||||
SCRAMBLE_268850 = 268850
|
||||
SCRAMBLE_421926 = 421926 # 2023-02-08后改了图片切割算法
|
||||
|
||||
# 当本子没有作者名字时,顶替作者名字
|
||||
DEFAULT_AUTHOR = 'default_author'
|
||||
|
||||
# 移动端API密钥
|
||||
APP_TOKEN_SECRET = '18comicAPP'
|
||||
APP_TOKEN_SECRET_2 = '18comicAPPContent'
|
||||
APP_DATA_SECRET = '185Hcomic3PAPP7R'
|
||||
APP_VERSION = '1.7.0'
|
||||
APP_VERSION = '1.7.1'
|
||||
|
||||
|
||||
# 模块级别共用配置
|
||||
class JmModuleConfig:
|
||||
# 网站相关
|
||||
PROT = "https://"
|
||||
JM_REDIRECT_URL = f'{PROT}jm365.work/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来
|
||||
JM_PUB_URL = f'{PROT}jmcomic-fb.vip'
|
||||
JM_CDN_IMAGE_URL_TEMPLATE = PROT + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始
|
||||
JM_IMAGE_SUFFIX = ['.jpg', '.webp', '.png', '.gif']
|
||||
|
||||
# JM的异常网页内容
|
||||
JM_ERROR_RESPONSE_TEXT = {
|
||||
"Could not connect to mysql! Please check your database settings!": "禁漫服务器内部报错",
|
||||
"Restricted Access!": "禁漫拒绝你所在ip地区的访问,你可以选择: 换域名/换代理",
|
||||
}
|
||||
|
||||
# JM的异常网页code
|
||||
JM_ERROR_STATUS_CODE = {
|
||||
403: 'ip地区禁止访问/爬虫被识别',
|
||||
500: '500: 禁漫服务器内部异常(可能是服务器过载,可以切换ip或稍后重试)',
|
||||
520: '520: Web server is returning an unknown error (禁漫服务器内部报错)',
|
||||
524: '524: The origin web server timed out responding to this request. (禁漫服务器处理超时)',
|
||||
}
|
||||
|
||||
# 分页大小
|
||||
PAGE_SIZE_SEARCH = 80
|
||||
PAGE_SIZE_FAVORITE = 20
|
||||
|
||||
# 图片分隔相关
|
||||
SCRAMBLE_CACHE = {}
|
||||
|
||||
# 当本子没有作者名字时,顶替作者名字
|
||||
DEFAULT_AUTHOR = 'default_author'
|
||||
|
||||
APP_HEADERS_TEMPLATE = {
|
||||
'Accept-Encoding': 'gzip',
|
||||
'Accept-Encoding': 'gzip, deflate',
|
||||
'user-agent': 'Mozilla/5.0 (Linux; Android 9; V1938CT Build/PQ3A.190705.11211812; wv) AppleWebKit/537.36 (KHTML, '
|
||||
'like Gecko) Version/4.0 Chrome/91.0.4472.114 Safari/537.36',
|
||||
}
|
||||
|
||||
APP_HEADERS_IMAGE = {
|
||||
'Accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
|
||||
'X-Requested-With': 'com.jiaohua_browser',
|
||||
'Referer': 'https://www.jmfreedomproxy.xyz/',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
|
||||
}
|
||||
|
||||
# 网页端headers
|
||||
HTML_HEADERS_TEMPLATE = {
|
||||
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
|
||||
@ -103,34 +138,6 @@ class JmMagicConstants:
|
||||
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 '
|
||||
'Safari/537.36',
|
||||
}
|
||||
|
||||
|
||||
# 模块级别共用配置
|
||||
class JmModuleConfig:
|
||||
# 网站相关
|
||||
PROT = "https://"
|
||||
JM_REDIRECT_URL = f'{PROT}jm365.work/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来
|
||||
JM_PUB_URL = f'{PROT}jmcomic-fb.vip'
|
||||
JM_CDN_IMAGE_URL_TEMPLATE = PROT + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始
|
||||
JM_IMAGE_SUFFIX = ['.jpg', '.webp', '.png', '.gif']
|
||||
|
||||
# JM的异常网页内容
|
||||
JM_ERROR_RESPONSE_TEXT = {
|
||||
"Could not connect to mysql! Please check your database settings!": "禁漫服务器内部报错",
|
||||
"Restricted Access!": "禁漫拒绝你所在ip地区的访问,你可以选择: 换域名/换代理",
|
||||
}
|
||||
|
||||
# JM的异常网页code
|
||||
JM_ERROR_STATUS_CODE = {
|
||||
403: 'ip地区禁止访问/爬虫被识别',
|
||||
500: '500: 禁漫服务器内部异常(可能是服务器过载,可以换个ip或稍后重试)',
|
||||
520: '520: Web server is returning an unknown error (禁漫服务器内部报错)',
|
||||
524: '524: The origin web server timed out responding to this request. (禁漫服务器处理超时)',
|
||||
}
|
||||
|
||||
# 图片分隔相关
|
||||
SCRAMBLE_CACHE = {}
|
||||
|
||||
# cookies,目前只在移动端使用,因为移动端请求接口须携带,但不会校验cookies的内容。
|
||||
APP_COOKIES = None
|
||||
|
||||
@ -336,7 +343,7 @@ class JmModuleConfig:
|
||||
"""
|
||||
网页端的headers
|
||||
"""
|
||||
headers = JmMagicConstants.HTML_HEADERS_TEMPLATE.copy()
|
||||
headers = cls.HTML_HEADERS_TEMPLATE.copy()
|
||||
headers.update({
|
||||
'authority': domain,
|
||||
'origin': f'https://{domain}',
|
||||
|
@ -305,7 +305,8 @@ class JmPhotoDetail(DetailEntity, Downloadable):
|
||||
# 2. 值目前在网页端只在photo页面的图片标签的data-original属性出现
|
||||
# 这里的模拟思路是,获取到第一个图片标签的data-original,
|
||||
# 取出其query参数 → self.data_original_query_params, 该值未来会传递给 JmImageDetail
|
||||
self.data_original_query_params = self.get_data_original_query_params(data_original_0)
|
||||
# self.data_original_query_params = self.get_data_original_query_params(data_original_0)
|
||||
self.data_original_query_params = None
|
||||
|
||||
@property
|
||||
def is_single_album(self) -> bool:
|
||||
@ -355,7 +356,7 @@ class JmPhotoDetail(DetailEntity, Downloadable):
|
||||
return self._author.strip()
|
||||
|
||||
# 使用默认
|
||||
return JmMagicConstants.DEFAULT_AUTHOR
|
||||
return JmModuleConfig.DEFAULT_AUTHOR
|
||||
|
||||
def create_image_detail(self, index) -> JmImageDetail:
|
||||
# 校验参数
|
||||
@ -475,7 +476,7 @@ class JmAlbumDetail(DetailEntity, Downloadable):
|
||||
if len(self.authors) >= 1:
|
||||
return self.authors[0]
|
||||
|
||||
return JmMagicConstants.DEFAULT_AUTHOR
|
||||
return JmModuleConfig.DEFAULT_AUTHOR
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
@ -612,7 +613,7 @@ class JmSearchPage(JmPageContent):
|
||||
|
||||
@property
|
||||
def page_size(self) -> int:
|
||||
return JmMagicConstants.PAGE_SIZE_SEARCH
|
||||
return JmModuleConfig.PAGE_SIZE_SEARCH
|
||||
|
||||
# 下面的方法是对单个album的包装
|
||||
|
||||
@ -653,7 +654,7 @@ class JmFavoritePage(JmPageContent):
|
||||
|
||||
@property
|
||||
def page_size(self) -> int:
|
||||
return JmMagicConstants.PAGE_SIZE_FAVORITE
|
||||
return JmModuleConfig.PAGE_SIZE_FAVORITE
|
||||
|
||||
def iter_folder_id_name(self) -> Generator[Tuple[str, str], None, None]:
|
||||
"""
|
||||
|
@ -601,7 +601,7 @@ class FavoriteFolderExportPlugin(JmOptionPlugin):
|
||||
for page in page_data:
|
||||
for aid, extra in page.content:
|
||||
data.append(
|
||||
(aid, extra.get('author', '') or JmMagicConstants.DEFAULT_AUTHOR, extra['name'])
|
||||
(aid, extra.get('author', '') or JmModuleConfig.DEFAULT_AUTHOR, extra['name'])
|
||||
)
|
||||
|
||||
if len(data) == 0:
|
||||
|
Loading…
Reference in New Issue
Block a user