mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.3.16: 修复zip插件在album模式下的一些bug; 调整获取APP_COOKIES的位置; 一些代码优化。 (#160)
This commit is contained in:
parent
87ed6bce83
commit
9e3fa7f17a
2
.github/release.py
vendored
2
.github/release.py
vendored
@ -18,7 +18,7 @@ def parse_body(body):
|
|||||||
e: str = e.strip()
|
e: str = e.strip()
|
||||||
if e == '':
|
if e == '':
|
||||||
continue
|
continue
|
||||||
points.append(f'{i}. {e}')
|
points.append(f'{i + 1}. {e}')
|
||||||
|
|
||||||
return '\n'.join(points)
|
return '\n'.join(points)
|
||||||
|
|
||||||
|
@ -2,7 +2,7 @@
|
|||||||
# 被依赖方 <--- 使用方
|
# 被依赖方 <--- 使用方
|
||||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||||
|
|
||||||
__version__ = '2.3.15'
|
__version__ = '2.3.16'
|
||||||
|
|
||||||
from .api import *
|
from .api import *
|
||||||
from .jm_plugin import *
|
from .jm_plugin import *
|
||||||
|
@ -493,13 +493,13 @@ class JmApiClient(AbstractJmClient):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
match = JmcomicText.pattern_html_album_scramble_id.search(resp.text)
|
scramble_id = PatternTool.match_or_default(resp.text,
|
||||||
|
JmcomicText.pattern_html_album_scramble_id,
|
||||||
if match is not None:
|
None,
|
||||||
scramble_id = match[1]
|
)
|
||||||
else:
|
if scramble_id is None:
|
||||||
jm_debug('api.scramble', '未从响应中匹配到scramble_id,返回默认值220980')
|
jm_debug('api.scramble', f'未匹配到scramble_id,响应文本:{resp.text}')
|
||||||
scramble_id = '220980'
|
scramble_id = str(JmModuleConfig.SCRAMBLE_220980)
|
||||||
|
|
||||||
return scramble_id
|
return scramble_id
|
||||||
|
|
||||||
@ -610,15 +610,12 @@ class JmApiClient(AbstractJmClient):
|
|||||||
# 2. 是否是特殊的内容
|
# 2. 是否是特殊的内容
|
||||||
# 暂无
|
# 暂无
|
||||||
|
|
||||||
@classmethod
|
|
||||||
@field_cache('__init_cookies__')
|
|
||||||
def fetch_init_cookies(cls, client: 'JmApiClient'):
|
|
||||||
resp = client.setting()
|
|
||||||
return dict(resp.resp.cookies)
|
|
||||||
|
|
||||||
def after_init(self):
|
def after_init(self):
|
||||||
cookies = self.__class__.fetch_init_cookies(self)
|
# cookies = self.__class__.fetch_init_cookies(self)
|
||||||
self.get_root_postman().get_meta_data()['cookies'] = cookies
|
# self.get_root_postman().get_meta_data()['cookies'] = cookies
|
||||||
|
|
||||||
|
self.get_root_postman().get_meta_data()['cookies'] = JmModuleConfig.get_cookies(self)
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class FutureClientProxy(JmcomicClient):
|
class FutureClientProxy(JmcomicClient):
|
||||||
|
@ -3,7 +3,7 @@ def field_cache(*args, **kwargs):
|
|||||||
return field_cache(*args, **kwargs)
|
return field_cache(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
def default_jm_debug(topic: str, msg: str):
|
def default_jm_debug_logging(topic: str, msg: str):
|
||||||
from common import format_ts
|
from common import format_ts
|
||||||
print(f'{format_ts()}:【{topic}】{msg}')
|
print(f'{format_ts()}:【{topic}】{msg}')
|
||||||
|
|
||||||
@ -62,13 +62,14 @@ class JmModuleConfig:
|
|||||||
SCRAMBLE_421926 = 421926 # 2023-02-08后改了图片切割算法
|
SCRAMBLE_421926 = 421926 # 2023-02-08后改了图片切割算法
|
||||||
SCRAMBLE_CACHE = {}
|
SCRAMBLE_CACHE = {}
|
||||||
|
|
||||||
# 移动端API的相关配置
|
# 移动端API密钥
|
||||||
# API密钥
|
APP_SECRET = '18comicAPPContent'
|
||||||
APP_SECRET = '18comicAPP'
|
|
||||||
|
|
||||||
# 域名配置 - 移动端
|
# cookies,目前只在移动端使用,因为移动端请求接口须携带,但不会校验cookies的内容。
|
||||||
# 图片域名
|
APP_COOKIES = None
|
||||||
DOMAIN_API_IMAGE_LIST = str_to_list('''
|
|
||||||
|
# 移动端图片域名
|
||||||
|
DOMAIN_IMAGE_LIST = str_to_list('''
|
||||||
cdn-msp.jmapiproxy1.monster
|
cdn-msp.jmapiproxy1.monster
|
||||||
cdn-msp2.jmapiproxy1.monster
|
cdn-msp2.jmapiproxy1.monster
|
||||||
cdn-msp.jmapiproxy1.cc
|
cdn-msp.jmapiproxy1.cc
|
||||||
@ -78,7 +79,7 @@ class JmModuleConfig:
|
|||||||
|
|
||||||
''')
|
''')
|
||||||
|
|
||||||
# API域名
|
# 移动端API域名
|
||||||
DOMAIN_API_LIST = str_to_list('''
|
DOMAIN_API_LIST = str_to_list('''
|
||||||
www.jmapinode1.top
|
www.jmapinode1.top
|
||||||
www.jmapinode2.top
|
www.jmapinode2.top
|
||||||
@ -88,8 +89,11 @@ class JmModuleConfig:
|
|||||||
|
|
||||||
''')
|
''')
|
||||||
|
|
||||||
# 域名配置 - 网页端
|
# 网页端域名配置
|
||||||
# 无需配置,默认为None,需要的时候会发起请求获得
|
# 无需配置,默认为None,需要的时候会发起请求获得
|
||||||
|
# 使用优先级:
|
||||||
|
# 1. DOMAIN_HTML_LIST
|
||||||
|
# 2. [DOMAIN_HTML]
|
||||||
DOMAIN_HTML = None
|
DOMAIN_HTML = None
|
||||||
DOMAIN_HTML_LIST = None
|
DOMAIN_HTML_LIST = None
|
||||||
|
|
||||||
@ -106,7 +110,7 @@ class JmModuleConfig:
|
|||||||
REGISTRY_PLUGIN = {}
|
REGISTRY_PLUGIN = {}
|
||||||
|
|
||||||
# 执行debug的函数
|
# 执行debug的函数
|
||||||
debug_executor = default_jm_debug
|
debug_executor = default_jm_debug_logging
|
||||||
# postman构造函数
|
# postman构造函数
|
||||||
postman_constructor = default_postman_constructor
|
postman_constructor = default_postman_constructor
|
||||||
# 网页正则表达式解析失败时,执行抛出异常的函数,可以替换掉用于debug
|
# 网页正则表达式解析失败时,执行抛出异常的函数,可以替换掉用于debug
|
||||||
@ -190,7 +194,7 @@ class JmModuleConfig:
|
|||||||
postman = postman or cls.new_postman(session=True)
|
postman = postman or cls.new_postman(session=True)
|
||||||
|
|
||||||
url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL)
|
url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL)
|
||||||
cls.jm_debug('获取禁漫网页URL', f'[{cls.JM_REDIRECT_URL}] → [{url}]')
|
cls.jm_debug('module.html_url', f'获取禁漫网页URL: [{cls.JM_REDIRECT_URL}] → [{url}]')
|
||||||
return url
|
return url
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -211,9 +215,22 @@ class JmModuleConfig:
|
|||||||
from .jm_toolkit import JmcomicText
|
from .jm_toolkit import JmcomicText
|
||||||
domain_list = JmcomicText.analyse_jm_pub_html(resp.text)
|
domain_list = JmcomicText.analyse_jm_pub_html(resp.text)
|
||||||
|
|
||||||
cls.jm_debug('获取禁漫网页全部域名', f'[{resp.url}] → {domain_list}')
|
cls.jm_debug('module.html_domain_all', f'获取禁漫网页全部域名: [{resp.url}] → {domain_list}')
|
||||||
return domain_list
|
return domain_list
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@field_cache("APP_COOKIES")
|
||||||
|
def get_cookies(cls, postman=None):
|
||||||
|
from .jm_toolkit import JmcomicText
|
||||||
|
url = JmcomicText.format_url('/setting', cls.DOMAIN_API_LIST[0])
|
||||||
|
postman = postman or cls.new_postman()
|
||||||
|
|
||||||
|
resp = postman.get(url)
|
||||||
|
cookies = dict(resp.cookies)
|
||||||
|
|
||||||
|
cls.jm_debug('module.cookies', f'获取cookies: [{url}] → {cookies}')
|
||||||
|
return cookies
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def new_html_headers(cls, domain='18comic.vip'):
|
def new_html_headers(cls, domain='18comic.vip'):
|
||||||
"""
|
"""
|
||||||
@ -247,7 +264,7 @@ class JmModuleConfig:
|
|||||||
key_ts = time_stamp()
|
key_ts = time_stamp()
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode()).hexdigest()
|
token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'token': token,
|
'token': token,
|
||||||
|
@ -314,6 +314,7 @@ class JmOption:
|
|||||||
def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient:
|
def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient:
|
||||||
# 所有需要用到的 self.client 配置项如下
|
# 所有需要用到的 self.client 配置项如下
|
||||||
postman_conf: dict = self.client.postman.src_dict # postman dsl 配置
|
postman_conf: dict = self.client.postman.src_dict # postman dsl 配置
|
||||||
|
meta_data: dict = postman_conf['meta_data'] # 请求元信息
|
||||||
impl: str = impl or self.client.impl # client_key
|
impl: str = impl or self.client.impl # client_key
|
||||||
retry_times: int = self.client.retry_times # 重试次数
|
retry_times: int = self.client.retry_times # 重试次数
|
||||||
cache: str = cache or self.client.cache # 启用缓存
|
cache: str = cache or self.client.cache # 启用缓存
|
||||||
@ -335,15 +336,11 @@ class JmOption:
|
|||||||
|
|
||||||
# support kwargs overwrite meta_data
|
# support kwargs overwrite meta_data
|
||||||
if len(kwargs) != 0:
|
if len(kwargs) != 0:
|
||||||
postman_conf['meta_data'].update(kwargs)
|
meta_data.update(kwargs)
|
||||||
|
|
||||||
# headers
|
# headers
|
||||||
meta_data = postman_conf['meta_data']
|
|
||||||
if meta_data['headers'] is None:
|
if meta_data['headers'] is None:
|
||||||
headers = self.decide_postman_headers(impl, domain[0])
|
meta_data['headers'] = self.decide_postman_headers(impl, domain[0])
|
||||||
# if headers is None:
|
|
||||||
# postman_conf['type'] = 'requests'
|
|
||||||
meta_data['headers'] = headers
|
|
||||||
|
|
||||||
# postman
|
# postman
|
||||||
postman = Postmans.create(data=postman_conf)
|
postman = Postmans.create(data=postman_conf)
|
||||||
@ -375,6 +372,9 @@ class JmOption:
|
|||||||
|
|
||||||
if is_client_type(JmHtmlClient):
|
if is_client_type(JmHtmlClient):
|
||||||
# 网页端
|
# 网页端
|
||||||
|
domain_list = JmModuleConfig.DOMAIN_HTML_LIST
|
||||||
|
if domain_list is not None:
|
||||||
|
return domain_list
|
||||||
return [JmModuleConfig.get_html_domain()]
|
return [JmModuleConfig.get_html_domain()]
|
||||||
|
|
||||||
ExceptionTool.raises(f'没有配置域名,且是无法识别的client类型: {client_key}')
|
ExceptionTool.raises(f'没有配置域名,且是无法识别的client类型: {client_key}')
|
||||||
|
@ -249,26 +249,32 @@ class ZipPlugin(JmOptionPlugin):
|
|||||||
mkdir_if_not_exists(zip_dir)
|
mkdir_if_not_exists(zip_dir)
|
||||||
|
|
||||||
# 原文件夹 -> zip文件
|
# 原文件夹 -> zip文件
|
||||||
dir_zip_dict = {}
|
dir_zip_dict: Dict[str, Optional[str]] = {}
|
||||||
photo_dict = downloader.all_downloaded[album]
|
photo_dict = downloader.all_downloaded[album]
|
||||||
|
|
||||||
if level == 'album':
|
if level == 'album':
|
||||||
zip_path = self.get_zip_path(album, None, filename_rule, suffix, zip_dir)
|
zip_path = self.get_zip_path(album, None, filename_rule, suffix, zip_dir)
|
||||||
dir_path = self.zip_album(album, photo_dict, zip_path)
|
dir_path = self.zip_album(album, photo_dict, zip_path)
|
||||||
dir_zip_dict[dir_path] = zip_path
|
if dir_path is not None:
|
||||||
|
# 要删除这个album文件夹
|
||||||
|
dir_zip_dict[dir_path] = zip_path
|
||||||
|
# 也要删除album下的photo文件夹
|
||||||
|
for d in files_of_dir(dir_path):
|
||||||
|
dir_zip_dict[d] = None
|
||||||
|
|
||||||
elif level == 'photo':
|
elif level == 'photo':
|
||||||
for photo, image_list in photo_dict.items():
|
for photo, image_list in photo_dict.items():
|
||||||
zip_path = self.get_zip_path(None, photo, filename_rule, suffix, zip_dir)
|
zip_path = self.get_zip_path(None, photo, filename_rule, suffix, zip_dir)
|
||||||
dir_path = self.zip_photo(photo, image_list, zip_path)
|
dir_path = self.zip_photo(photo, image_list, zip_path)
|
||||||
dir_zip_dict[dir_path] = zip_path
|
if dir_path is not None:
|
||||||
|
dir_zip_dict[dir_path] = zip_path
|
||||||
|
|
||||||
else:
|
else:
|
||||||
ExceptionTool.raises(f'Not Implemented Zip Level: {level}')
|
ExceptionTool.raises(f'Not Implemented Zip Level: {level}')
|
||||||
|
|
||||||
self.after_zip(dir_zip_dict)
|
self.after_zip(dir_zip_dict)
|
||||||
|
|
||||||
def zip_photo(self, photo, image_list: list, zip_path: str):
|
def zip_photo(self, photo, image_list: list, zip_path: str) -> Optional[str]:
|
||||||
"""
|
"""
|
||||||
压缩photo文件夹
|
压缩photo文件夹
|
||||||
:returns: photo文件夹路径
|
:returns: photo文件夹路径
|
||||||
@ -277,46 +283,54 @@ class ZipPlugin(JmOptionPlugin):
|
|||||||
if len(image_list) == 0 \
|
if len(image_list) == 0 \
|
||||||
else os.path.dirname(image_list[0][0])
|
else os.path.dirname(image_list[0][0])
|
||||||
|
|
||||||
all_filepath = set(map(lambda t: t[0], image_list))
|
all_filepath = set(map(lambda t: self.unified_path(t[0]), image_list))
|
||||||
|
|
||||||
if len(all_filepath) == 0:
|
return self.do_zip(photo_dir,
|
||||||
self.debug('无下载文件,无需压缩', 'skip')
|
zip_path,
|
||||||
return
|
all_filepath,
|
||||||
|
f'压缩章节[{photo.photo_id}]成功 → {zip_path}',
|
||||||
|
)
|
||||||
|
|
||||||
from common import backup_dir_to_zip
|
@staticmethod
|
||||||
backup_dir_to_zip(photo_dir, zip_path, acceptor=lambda f: f in all_filepath)
|
def unified_path(f):
|
||||||
self.debug(f'压缩章节[{photo.photo_id}]成功 → {zip_path}', 'finish')
|
return fix_filepath(f, os.path.isdir(f))
|
||||||
|
|
||||||
return photo_dir
|
def zip_album(self, album, photo_dict: dict, zip_path) -> Optional[str]:
|
||||||
|
|
||||||
def zip_album(self, album, photo_dict: dict, zip_path):
|
|
||||||
"""
|
"""
|
||||||
压缩album文件夹
|
压缩album文件夹
|
||||||
:returns: album文件夹路径
|
:returns: album文件夹路径
|
||||||
"""
|
"""
|
||||||
album_dir = self.option.decide_album_dir(album)
|
|
||||||
all_filepath: Set[str] = set()
|
all_filepath: Set[str] = set()
|
||||||
|
|
||||||
for image_list in photo_dict.values():
|
def addpath(f):
|
||||||
image_list: List[Tuple[str, JmImageDetail]]
|
all_filepath.update(set(f))
|
||||||
for path, _ in image_list:
|
|
||||||
all_filepath.add(path)
|
|
||||||
|
|
||||||
|
album_dir = self.option.decide_album_dir(album)
|
||||||
|
# addpath(self.option.decide_image_save_dir(photo) for photo in photo_dict.keys())
|
||||||
|
addpath(path for ls in photo_dict.values() for path, _ in ls)
|
||||||
|
|
||||||
|
return self.do_zip(album_dir,
|
||||||
|
zip_path,
|
||||||
|
all_filepath,
|
||||||
|
msg=f'压缩本子[{album.album_id}]成功 → {zip_path}',
|
||||||
|
)
|
||||||
|
|
||||||
|
def do_zip(self, source_dir, zip_path, all_filepath, msg):
|
||||||
if len(all_filepath) == 0:
|
if len(all_filepath) == 0:
|
||||||
self.debug('无下载文件,无需压缩', 'skip')
|
self.debug('无下载文件,无需压缩', 'skip')
|
||||||
return
|
return None
|
||||||
|
|
||||||
from common import backup_dir_to_zip
|
from common import backup_dir_to_zip
|
||||||
backup_dir_to_zip(
|
backup_dir_to_zip(
|
||||||
album_dir,
|
source_dir,
|
||||||
zip_path,
|
zip_path,
|
||||||
acceptor=lambda f: f in all_filepath
|
acceptor=lambda f: os.path.isdir(f) or self.unified_path(f) in all_filepath
|
||||||
)
|
).close()
|
||||||
|
|
||||||
self.debug(f'压缩本子[{album.album_id}]成功 → {zip_path}', 'finish')
|
self.debug(msg, 'finish')
|
||||||
return album_dir
|
return self.unified_path(source_dir)
|
||||||
|
|
||||||
def after_zip(self, dir_zip_dict: Dict[str, str]):
|
def after_zip(self, dir_zip_dict: Dict[str, Optional[str]]):
|
||||||
# 是否要删除所有原文件
|
# 是否要删除所有原文件
|
||||||
if self.delete_original_file is True:
|
if self.delete_original_file is True:
|
||||||
self.delete_all_files_and_empty_dir(
|
self.delete_all_files_and_empty_dir(
|
||||||
@ -352,10 +366,10 @@ class ZipPlugin(JmOptionPlugin):
|
|||||||
os.remove(f)
|
os.remove(f)
|
||||||
self.debug(f'删除原文件: {f}', 'remove')
|
self.debug(f'删除原文件: {f}', 'remove')
|
||||||
|
|
||||||
for d in dir_list:
|
for d in sorted(dir_list, reverse=True):
|
||||||
# check exist
|
# check exist
|
||||||
if file_exists(d) and len(os.listdir(d)) == 0:
|
if file_exists(d):
|
||||||
os.removedirs(d)
|
os.rmdir(d)
|
||||||
self.debug(f'删除文件夹: {d}', 'remove')
|
self.debug(f'删除文件夹: {d}', 'remove')
|
||||||
|
|
||||||
|
|
||||||
|
@ -489,7 +489,7 @@ class JmApiAdaptTool:
|
|||||||
|
|
||||||
fields['sort'] = sort
|
fields['sort'] = sort
|
||||||
import random
|
import random
|
||||||
fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_API_IMAGE_LIST)
|
fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_IMAGE_LIST)
|
||||||
|
|
||||||
|
|
||||||
class JmImageTool:
|
class JmImageTool:
|
||||||
|
@ -55,8 +55,10 @@ class Test_Custom(JmTestConfigurable):
|
|||||||
JmModuleConfig.register_client(MyClient)
|
JmModuleConfig.register_client(MyClient)
|
||||||
|
|
||||||
html_domain = self.client.get_html_domain()
|
html_domain = self.client.get_html_domain()
|
||||||
|
JmModuleConfig.DOMAIN_HTML_LIST = [html_domain]
|
||||||
|
|
||||||
self.assertListEqual(
|
self.assertListEqual(
|
||||||
[html_domain],
|
JmModuleConfig.DOMAIN_HTML_LIST,
|
||||||
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list()
|
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user