v2.4.3: 实现最新的禁漫APP接口加解密算法(1.6.3),优化代码结构,更新README (#167)

This commit is contained in:
hect0x7 2023-11-22 17:54:23 +08:00 committed by GitHub
parent 52298d0faf
commit f35cb12f49
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 214 additions and 144 deletions

View File

@ -1,11 +1,20 @@
# Python API For JMComic (禁漫天堂)
封装了一套可用于爬取JM的Python API.
本项目封装了一套可用于爬取JM的Python API.
简单来说,就是可以通过简单的几行Python代码实现下载JM上的本子到本地并且是处理好的图片.
可以通过简单的几行Python代码实现下载JM上的本子到本地并且是处理好的图片
**友情提示珍爱JM为了减轻JM的服务器压力请不要一次性爬取太多本子西门🙏🙏🙏**.
## 项目介绍
本项目的核心功能是下载本子,基于此,设计了一套方便使用、便于扩展,能满足一些特殊下载需求的框架。
除了下载功能以外,也实现了其他的一些禁漫接口,例如登录、搜索、收藏夹等等,按需实现。
目前核心功能实现较为稳定,项目也处于维护阶段(因为禁漫接口经常变动,需要经常维护)。
## 安装教程
* 通过pip官方源安装推荐并且更新也是这个命令
@ -38,6 +47,7 @@ $ jmcomic 422866
## 项目特点
- **绕过Cloudflare的反爬虫**
- **实现禁漫APP接口最新的加解密算法 (1.6.3)**
- 用法多样:
- GitHub Actions网页上直接输入本子id就能下载[教程使用GitHub Actions下载禁漫本子](./assets/docs/sources/tutorial/1_github_actions.md)

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.4.2'
__version__ = '2.4.3'
from .api import *
from .jm_plugin import *

View File

@ -392,7 +392,7 @@ class JmHtmlClient(AbstractJmClient):
)
resp = self.post('/ajax/album_comment',
headers=JmModuleConfig.album_comment_headers,
headers=self.album_comment_headers,
data=data,
)
@ -467,6 +467,26 @@ class JmHtmlClient(AbstractJmClient):
+ (f'URL=[{url}]' if url is not None else '')
)
album_comment_headers = {
'authority': '18comic.vip',
'accept': 'application/json, text/javascript, */*; q=0.01',
'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
'cache-control': 'no-cache',
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'origin': 'https://18comic.vip',
'pragma': 'no-cache',
'referer': 'https://18comic.vip/album/248965/',
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/114.0.0.0 Safari/537.36',
'x-requested-with': 'XMLHttpRequest',
}
# 基于禁漫移动端APP实现的JmClient
class JmApiClient(AbstractJmClient):
@ -556,7 +576,7 @@ class JmApiClient(AbstractJmClient):
url,
params={
'id': apid,
}
},
)
self.require_resp_success(resp, url)
@ -571,11 +591,13 @@ class JmApiClient(AbstractJmClient):
resp = self.req_api(
self.API_SCRAMBLE,
params={
"id": photo_id,
"mode": "vertical",
"page": "0",
"app_img_shunt": "1",
}
'id': photo_id,
'mode': 'vertical',
'page': '0',
'app_img_shunt': '1',
'express': 'off',
'v': time_stamp(),
},
)
scramble_id = PatternTool.match_or_default(resp.text,
@ -713,21 +735,41 @@ class JmApiClient(AbstractJmClient):
return JmPageTool.parse_api_to_favorite_page(resp.model_data)
def req_api(self, url, get=True, **kwargs) -> JmApiResp:
# set headers
headers, key_ts = self.headers_key_ts
kwargs['headers'] = headers
ts = self.decide_headers_and_ts(kwargs, url)
if get:
resp = self.get(url, **kwargs)
else:
resp = self.post(url, **kwargs)
return JmApiResp.wrap(resp, key_ts)
return JmApiResp(resp, ts)
@property
def headers_key_ts(self):
key_ts = time_stamp()
return JmModuleConfig.new_api_headers(key_ts), key_ts
# noinspection PyMethodMayBeStatic
def decide_headers_and_ts(self, kwargs, url):
# 获取时间戳
if url == self.API_SCRAMBLE:
# /chapter_view_template
# 这个接口很特殊,用的密钥 18comicAPPContent 而不是 18comicAPP
# 如果用后者则会返回403信息
ts = time_stamp()
token, tokenparam = JmCryptoTool.token_and_tokenparam(ts, secret=JmMagicConstants.APP_TOKEN_SECRET_2)
elif JmModuleConfig.use_fix_timestamp:
ts, token, tokenparam = JmModuleConfig.get_fix_ts_token_tokenparam()
else:
ts = time_stamp()
token, tokenparam = JmCryptoTool.token_and_tokenparam(ts)
# 计算tokentokenparam
headers = kwargs.get('headers', JmMagicConstants.APP_HEADERS_TEMPLATE.copy())
headers.update({
'token': token,
'tokenparam': tokenparam,
})
kwargs['headers'] = headers
return ts
@classmethod
def require_resp_success(cls, resp: JmApiResp, orig_req_url: str):
@ -743,11 +785,22 @@ class JmApiClient(AbstractJmClient):
# 暂无
def after_init(self):
# cookies = self.__class__.fetch_init_cookies(self)
# self.get_root_postman().get_meta_data()['cookies'] = cookies
# 保证拥有cookies因为移动端要求必须携带cookies否则会直接跳转同一本子【禁漫娘】
if JmModuleConfig.api_client_require_cookies:
self.ensure_have_cookies()
self.get_root_postman().get_meta_data()['cookies'] = JmModuleConfig.get_cookies(self)
pass
from threading import Lock
client_init_cookies_lock = Lock()
def ensure_have_cookies(self):
if self.get_meta_data('cookies'):
return
with self.client_init_cookies_lock:
if self.get_meta_data('cookies'):
return
self['cookies'] = JmModuleConfig.get_cookies(self)
class FutureClientProxy(JmcomicClient):

View File

@ -68,45 +68,21 @@ class JmImageResp(JmResp):
class JmApiResp(JmResp):
@classmethod
def wrap(cls, resp, key_ts):
def __init__(self, resp, ts: str):
ExceptionTool.require_true(not isinstance(resp, JmApiResp), f'重复包装: {resp}')
return cls(resp, key_ts)
def __init__(self, resp, key_ts):
super().__init__(resp)
self.key_ts = key_ts
self.ts = ts
self.cache_decode_data = None
@property
def is_success(self) -> bool:
return super().is_success and self.json()['code'] == 200
@staticmethod
def parse_data(text, time) -> str:
# 1. base64解码
import base64
data = base64.b64decode(text)
# 2. AES-ECB解密
# key = 时间戳拼接 '18comicAPPContent' 的md5
import hashlib
key = hashlib.md5(f"{time}18comicAPPContent".encode("utf-8")).hexdigest().encode("utf-8")
from Crypto.Cipher import AES
data = AES.new(key, AES.MODE_ECB).decrypt(data)
# 3. 移除末尾的一些特殊字符
data = data[:-data[-1]]
# 4. 解码为字符串 (json)
res = data.decode('utf-8')
return res
@property
@field_cache('__cache_decoded_data__')
def decoded_data(self) -> str:
return self.parse_data(self.encoded_data, self.key_ts)
return JmCryptoTool.decode_resp_data(self.encoded_data, self.ts)
@property
def encoded_data(self) -> str:

View File

@ -1,6 +1,4 @@
def field_cache(*args, **kwargs):
from common import field_cache
return field_cache(*args, **kwargs)
from common import time_stamp, field_cache, str_to_list
def default_jm_logging(topic: str, msg: str):
@ -8,15 +6,6 @@ def default_jm_logging(topic: str, msg: str):
print(f'{format_ts()}:【{topic}{msg}')
def default_postman_constructor(session, **kwargs):
from common import Postmans
if session is True:
return Postmans.new_session(**kwargs)
return Postmans.new_postman(**kwargs)
def default_raise_exception_executor(msg, _extra):
raise JmModuleConfig.CLASS_EXCEPTION(msg)
@ -26,15 +15,11 @@ def system_proxy():
return ProxyBuilder.system_proxy()
def str_to_list(text):
from common import str_to_list
return str_to_list(text)
class JmcomicException(Exception):
pass
# 禁漫常量
class JmMagicConstants:
ORDER_BY_LATEST = 'mr'
ORDER_BY_VIEW = 'mv'
@ -55,9 +40,34 @@ class JmMagicConstants:
SCRAMBLE_421926 = 421926 # 2023-02-08后改了图片切割算法
# 移动端API密钥
APP_SECRET = '18comicAPPContent'
APP_TOKEN_SECRET = '18comicAPP'
APP_TOKEN_SECRET_2 = '18comicAPPContent'
APP_DATA_SECRET = '185Hcomic3PAPP7R'
APP_VERSION = '1.6.3'
APP_HEADERS_TEMPLATE = {
'Accept-Encoding': 'gzip',
'user-agent': 'Mozilla/5.0 (Linux; Android 9; V1938CT Build/PQ3A.190705.09211555; wv) AppleWebKit/537.36 (KHTML, '
'like Gecko) Version/4.0 Chrome/91.0.4472.114 Safari/537.36',
}
# 网页端headers
HTML_HEADERS_TEMPLATE = {
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
'application/signed-exchange;v=b3;q=0.7',
'accept-language': 'zh-CN,zh;q=0.9',
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 '
'Safari/537.36',
}
# 模块级别共用配置
class JmModuleConfig:
# 网站相关
PROT = "https://"
@ -128,11 +138,14 @@ class JmModuleConfig:
# 执行log的函数
log_executor = default_jm_logging
# postman构造函数
postman_constructor = default_postman_constructor
# 网页正则表达式解析失败时执行抛出异常的函数可以替换掉用于log
raise_exception_executor = default_raise_exception_executor
# 使用固定时间戳
use_fix_timestamp = True
# 移动端Client初始化cookies
api_client_require_cookies = True
# log开关标记
enable_jm_log = True
# log时解码url
@ -235,67 +248,32 @@ class JmModuleConfig:
cls.jm_log('module.html_domain_all', f'获取禁漫网页全部域名: [{resp.url}] → {domain_list}')
return domain_list
@classmethod
@field_cache("APP_COOKIES")
def get_cookies(cls, postman=None):
from .jm_toolkit import JmcomicText
url = JmcomicText.format_url('/setting', cls.DOMAIN_API_LIST[0])
postman = postman or cls.new_postman()
resp = postman.get(url)
cookies = dict(resp.cookies)
cls.jm_log('module.cookies', f'获取cookies: [{url}] → {cookies}')
return cookies
@classmethod
def new_html_headers(cls, domain='18comic.vip'):
"""
网页端的headers
"""
return {
headers = JmMagicConstants.HTML_HEADERS_TEMPLATE.copy()
headers.update({
'authority': domain,
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
'application/signed-exchange;v=b3;q=0.7',
'accept-language': 'zh-CN,zh;q=0.9',
'referer': f'https://{domain}',
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'document',
'sec-fetch-mode': 'navigate',
'sec-fetch-site': 'none',
'sec-fetch-user': '?1',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 '
'Safari/537.36',
}
})
return headers
@classmethod
def new_api_headers(cls, key_ts):
"""
根据key_ts生成移动端的headers
"""
if key_ts is None:
from common import time_stamp
key_ts = time_stamp()
@field_cache("APP_COOKIES")
def get_cookies(cls, client):
resp = client.setting()
cookies = dict(resp.resp.cookies)
return cookies
import hashlib
token = hashlib.md5(f"{key_ts}{JmMagicConstants.APP_SECRET}".encode("utf-8")).hexdigest()
return {
'token': token,
'tokenparam': f"{key_ts},1.6.0",
'User-Agent': 'Mozilla/5.0 (Linux; Android 9; V1938CT Build/PQ3A.190705.09211555; wv) AppleWebKit/537.36 (KHTML, '
'like Gecko) Version/4.0 Chrome/91.0.4472.114 Safari/537.36',
'X-Requested-With': 'com.jiaohua_browser',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
'application/signed-exchange;v=b3;q=0.9',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-User': '?1',
'Sec-Fetch-Dest': 'document',
'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
}
@classmethod
@field_cache("__fix_ts_token_tokenparam__")
def get_fix_ts_token_tokenparam(cls):
ts = time_stamp()
from .jm_toolkit import JmCryptoTool
token, tokenparam = JmCryptoTool.token_and_tokenparam(ts)
return ts, token, tokenparam
# noinspection PyUnusedLocal
@classmethod
@ -312,27 +290,13 @@ class JmModuleConfig:
kwargs.setdefault('impersonate', 'chrome110')
kwargs.setdefault('headers', JmModuleConfig.new_html_headers())
kwargs.setdefault('proxies', JmModuleConfig.DEFAULT_PROXIES)
return cls.postman_constructor(session, **kwargs)
album_comment_headers = {
'authority': '18comic.vip',
'accept': 'application/json, text/javascript, */*; q=0.01',
'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
'cache-control': 'no-cache',
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'origin': 'https://18comic.vip',
'pragma': 'no-cache',
'referer': 'https://18comic.vip/album/248965/',
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/114.0.0.0 Safari/537.36',
'x-requested-with': 'XMLHttpRequest',
}
from common import Postmans
if session is True:
return Postmans.new_session(**kwargs)
return Postmans.new_postman(**kwargs)
# option 相关的默认配置
JM_OPTION_VER = '2.1'

View File

@ -46,7 +46,7 @@ class CacheRegistry:
elif isinstance(cache, str):
func = getattr(cls, cache, None)
assert func is not None, f'未实现的cache配置名: {cache}'
ExceptionTool.require_true(func is not None, f'未实现的cache配置名: {cache}')
cache = func
cache: Callable

View File

@ -177,7 +177,7 @@ class JmcomicText:
@classmethod
def format_url(cls, path, domain):
assert isinstance(domain, str) and len(domain) != 0
ExceptionTool.require_true(isinstance(domain, str) and len(domain) != 0, '域名为空')
if domain.startswith(JmModuleConfig.PROT):
return f'{domain}{path}'
@ -202,7 +202,7 @@ class JmcomicText:
def match_os_env(cls, match: Match) -> str:
name = match[1]
value = os.getenv(name, None)
assert value is not None, f"未配置环境变量: {name}"
ExceptionTool.require_true(value is not None, f'未配置环境变量: {name}')
return value
dsl_replacer = DSLReplacer()
@ -778,3 +778,70 @@ class ExceptionTool:
raises(old, msg, extra)
JmModuleConfig.raise_exception_executor = new
class JmCryptoTool:
"""
禁漫加解密相关逻辑
"""
@classmethod
def token_and_tokenparam(cls,
ts,
ver=JmMagicConstants.APP_VERSION,
secret=JmMagicConstants.APP_TOKEN_SECRET,
):
"""
计算禁漫接口的请求headers的token和tokenparam
:param ts: 时间戳
:param ver: app版本
:param secret: 密钥
:return (token, tokenparam)
"""
# tokenparam: 1700566805,1.6.3
tokenparam = '{},{}'.format(ts, ver)
# token: 81498a20feea7fbb7149c637e49702e3
token = cls.md5hex(f'{ts}{secret}')
return token, tokenparam
@classmethod
def decode_resp_data(cls,
data: str,
ts,
secret=JmMagicConstants.APP_DATA_SECRET,
) -> str:
"""
解密接口返回值
:param data: data = resp.json()['data]
:param ts: 时间戳
:param secret: 密钥
:return: json格式的字符串
"""
# 1. base64解码
import base64
data_b64 = base64.b64decode(data)
# 2. AES-ECB解密
key = cls.md5hex(f'{ts}{secret}').encode('utf-8')
from Crypto.Cipher import AES
data_aes = AES.new(key, AES.MODE_ECB).decrypt(data_b64)
# 3. 移除末尾的一些特殊字符
data = data_aes[:-data_aes[-1]]
# 4. 解码为字符串 (json)
res = data.decode('utf-8')
return res
@classmethod
def md5hex(cls, key: str):
ExceptionTool.require_true(isinstance(key, str), 'key参数需为字符串')
from hashlib import md5
return md5(key.encode("utf-8")).hexdigest()

View File

@ -181,7 +181,7 @@ class Test_Client(JmTestConfigurable):
try:
page = self.client.search_site(**params)
print(page)
assert int(page[0][0]) == expected_result
self.assertEqual(int(page[0][0]), expected_result)
except Exception as e:
elist.append(e)