v2.3.16: 修复zip插件在album模式下的一些bug; 调整获取APP_COOKIES的位置; 一些代码优化。 (#160)

This commit is contained in:
hect0x7 2023-10-31 02:55:33 +08:00 committed by GitHub
parent 87ed6bce83
commit 9e3fa7f17a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 97 additions and 67 deletions

2
.github/release.py vendored
View File

@ -18,7 +18,7 @@ def parse_body(body):
e: str = e.strip()
if e == '':
continue
points.append(f'{i}. {e}')
points.append(f'{i + 1}. {e}')
return '\n'.join(points)

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.3.15'
__version__ = '2.3.16'
from .api import *
from .jm_plugin import *

View File

@ -493,13 +493,13 @@ class JmApiClient(AbstractJmClient):
}
)
match = JmcomicText.pattern_html_album_scramble_id.search(resp.text)
if match is not None:
scramble_id = match[1]
else:
jm_debug('api.scramble', '未从响应中匹配到scramble_id返回默认值220980')
scramble_id = '220980'
scramble_id = PatternTool.match_or_default(resp.text,
JmcomicText.pattern_html_album_scramble_id,
None,
)
if scramble_id is None:
jm_debug('api.scramble', f'未匹配到scramble_id响应文本{resp.text}')
scramble_id = str(JmModuleConfig.SCRAMBLE_220980)
return scramble_id
@ -610,15 +610,12 @@ class JmApiClient(AbstractJmClient):
# 2. 是否是特殊的内容
# 暂无
@classmethod
@field_cache('__init_cookies__')
def fetch_init_cookies(cls, client: 'JmApiClient'):
resp = client.setting()
return dict(resp.resp.cookies)
def after_init(self):
cookies = self.__class__.fetch_init_cookies(self)
self.get_root_postman().get_meta_data()['cookies'] = cookies
# cookies = self.__class__.fetch_init_cookies(self)
# self.get_root_postman().get_meta_data()['cookies'] = cookies
self.get_root_postman().get_meta_data()['cookies'] = JmModuleConfig.get_cookies(self)
pass
class FutureClientProxy(JmcomicClient):

View File

@ -3,7 +3,7 @@ def field_cache(*args, **kwargs):
return field_cache(*args, **kwargs)
def default_jm_debug(topic: str, msg: str):
def default_jm_debug_logging(topic: str, msg: str):
from common import format_ts
print(f'{format_ts()}:【{topic}{msg}')
@ -62,13 +62,14 @@ class JmModuleConfig:
SCRAMBLE_421926 = 421926 # 2023-02-08后改了图片切割算法
SCRAMBLE_CACHE = {}
# 移动端API的相关配置
# API密钥
APP_SECRET = '18comicAPP'
# 移动端API密钥
APP_SECRET = '18comicAPPContent'
# 域名配置 - 移动端
# 图片域名
DOMAIN_API_IMAGE_LIST = str_to_list('''
# cookies目前只在移动端使用因为移动端请求接口须携带但不会校验cookies的内容。
APP_COOKIES = None
# 移动端图片域名
DOMAIN_IMAGE_LIST = str_to_list('''
cdn-msp.jmapiproxy1.monster
cdn-msp2.jmapiproxy1.monster
cdn-msp.jmapiproxy1.cc
@ -78,7 +79,7 @@ class JmModuleConfig:
''')
# API域名
# 移动端API域名
DOMAIN_API_LIST = str_to_list('''
www.jmapinode1.top
www.jmapinode2.top
@ -88,8 +89,11 @@ class JmModuleConfig:
''')
# 域名配置 - 网页端
# 网页端域名配置
# 无需配置默认为None需要的时候会发起请求获得
# 使用优先级:
# 1. DOMAIN_HTML_LIST
# 2. [DOMAIN_HTML]
DOMAIN_HTML = None
DOMAIN_HTML_LIST = None
@ -106,7 +110,7 @@ class JmModuleConfig:
REGISTRY_PLUGIN = {}
# 执行debug的函数
debug_executor = default_jm_debug
debug_executor = default_jm_debug_logging
# postman构造函数
postman_constructor = default_postman_constructor
# 网页正则表达式解析失败时执行抛出异常的函数可以替换掉用于debug
@ -190,7 +194,7 @@ class JmModuleConfig:
postman = postman or cls.new_postman(session=True)
url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL)
cls.jm_debug('获取禁漫网页URL', f'[{cls.JM_REDIRECT_URL}] → [{url}]')
cls.jm_debug('module.html_url', f'获取禁漫网页URL: [{cls.JM_REDIRECT_URL}] → [{url}]')
return url
@classmethod
@ -211,9 +215,22 @@ class JmModuleConfig:
from .jm_toolkit import JmcomicText
domain_list = JmcomicText.analyse_jm_pub_html(resp.text)
cls.jm_debug('获取禁漫网页全部域名', f'[{resp.url}] → {domain_list}')
cls.jm_debug('module.html_domain_all', f'获取禁漫网页全部域名: [{resp.url}] → {domain_list}')
return domain_list
@classmethod
@field_cache("APP_COOKIES")
def get_cookies(cls, postman=None):
from .jm_toolkit import JmcomicText
url = JmcomicText.format_url('/setting', cls.DOMAIN_API_LIST[0])
postman = postman or cls.new_postman()
resp = postman.get(url)
cookies = dict(resp.cookies)
cls.jm_debug('module.cookies', f'获取cookies: [{url}] → {cookies}')
return cookies
@classmethod
def new_html_headers(cls, domain='18comic.vip'):
"""
@ -247,7 +264,7 @@ class JmModuleConfig:
key_ts = time_stamp()
import hashlib
token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode()).hexdigest()
token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode("utf-8")).hexdigest()
return {
'token': token,

View File

@ -314,6 +314,7 @@ class JmOption:
def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient:
# 所有需要用到的 self.client 配置项如下
postman_conf: dict = self.client.postman.src_dict # postman dsl 配置
meta_data: dict = postman_conf['meta_data'] # 请求元信息
impl: str = impl or self.client.impl # client_key
retry_times: int = self.client.retry_times # 重试次数
cache: str = cache or self.client.cache # 启用缓存
@ -335,15 +336,11 @@ class JmOption:
# support kwargs overwrite meta_data
if len(kwargs) != 0:
postman_conf['meta_data'].update(kwargs)
meta_data.update(kwargs)
# headers
meta_data = postman_conf['meta_data']
if meta_data['headers'] is None:
headers = self.decide_postman_headers(impl, domain[0])
# if headers is None:
# postman_conf['type'] = 'requests'
meta_data['headers'] = headers
meta_data['headers'] = self.decide_postman_headers(impl, domain[0])
# postman
postman = Postmans.create(data=postman_conf)
@ -375,6 +372,9 @@ class JmOption:
if is_client_type(JmHtmlClient):
# 网页端
domain_list = JmModuleConfig.DOMAIN_HTML_LIST
if domain_list is not None:
return domain_list
return [JmModuleConfig.get_html_domain()]
ExceptionTool.raises(f'没有配置域名且是无法识别的client类型: {client_key}')

View File

@ -249,26 +249,32 @@ class ZipPlugin(JmOptionPlugin):
mkdir_if_not_exists(zip_dir)
# 原文件夹 -> zip文件
dir_zip_dict = {}
dir_zip_dict: Dict[str, Optional[str]] = {}
photo_dict = downloader.all_downloaded[album]
if level == 'album':
zip_path = self.get_zip_path(album, None, filename_rule, suffix, zip_dir)
dir_path = self.zip_album(album, photo_dict, zip_path)
dir_zip_dict[dir_path] = zip_path
if dir_path is not None:
# 要删除这个album文件夹
dir_zip_dict[dir_path] = zip_path
# 也要删除album下的photo文件夹
for d in files_of_dir(dir_path):
dir_zip_dict[d] = None
elif level == 'photo':
for photo, image_list in photo_dict.items():
zip_path = self.get_zip_path(None, photo, filename_rule, suffix, zip_dir)
dir_path = self.zip_photo(photo, image_list, zip_path)
dir_zip_dict[dir_path] = zip_path
if dir_path is not None:
dir_zip_dict[dir_path] = zip_path
else:
ExceptionTool.raises(f'Not Implemented Zip Level: {level}')
self.after_zip(dir_zip_dict)
def zip_photo(self, photo, image_list: list, zip_path: str):
def zip_photo(self, photo, image_list: list, zip_path: str) -> Optional[str]:
"""
压缩photo文件夹
:returns: photo文件夹路径
@ -277,46 +283,54 @@ class ZipPlugin(JmOptionPlugin):
if len(image_list) == 0 \
else os.path.dirname(image_list[0][0])
all_filepath = set(map(lambda t: t[0], image_list))
all_filepath = set(map(lambda t: self.unified_path(t[0]), image_list))
if len(all_filepath) == 0:
self.debug('无下载文件,无需压缩', 'skip')
return
return self.do_zip(photo_dir,
zip_path,
all_filepath,
f'压缩章节[{photo.photo_id}]成功 → {zip_path}',
)
from common import backup_dir_to_zip
backup_dir_to_zip(photo_dir, zip_path, acceptor=lambda f: f in all_filepath)
self.debug(f'压缩章节[{photo.photo_id}]成功 → {zip_path}', 'finish')
@staticmethod
def unified_path(f):
return fix_filepath(f, os.path.isdir(f))
return photo_dir
def zip_album(self, album, photo_dict: dict, zip_path):
def zip_album(self, album, photo_dict: dict, zip_path) -> Optional[str]:
"""
压缩album文件夹
:returns: album文件夹路径
"""
album_dir = self.option.decide_album_dir(album)
all_filepath: Set[str] = set()
for image_list in photo_dict.values():
image_list: List[Tuple[str, JmImageDetail]]
for path, _ in image_list:
all_filepath.add(path)
def addpath(f):
all_filepath.update(set(f))
album_dir = self.option.decide_album_dir(album)
# addpath(self.option.decide_image_save_dir(photo) for photo in photo_dict.keys())
addpath(path for ls in photo_dict.values() for path, _ in ls)
return self.do_zip(album_dir,
zip_path,
all_filepath,
msg=f'压缩本子[{album.album_id}]成功 → {zip_path}',
)
def do_zip(self, source_dir, zip_path, all_filepath, msg):
if len(all_filepath) == 0:
self.debug('无下载文件,无需压缩', 'skip')
return
return None
from common import backup_dir_to_zip
backup_dir_to_zip(
album_dir,
source_dir,
zip_path,
acceptor=lambda f: f in all_filepath
)
acceptor=lambda f: os.path.isdir(f) or self.unified_path(f) in all_filepath
).close()
self.debug(f'压缩本子[{album.album_id}]成功 → {zip_path}', 'finish')
return album_dir
self.debug(msg, 'finish')
return self.unified_path(source_dir)
def after_zip(self, dir_zip_dict: Dict[str, str]):
def after_zip(self, dir_zip_dict: Dict[str, Optional[str]]):
# 是否要删除所有原文件
if self.delete_original_file is True:
self.delete_all_files_and_empty_dir(
@ -352,10 +366,10 @@ class ZipPlugin(JmOptionPlugin):
os.remove(f)
self.debug(f'删除原文件: {f}', 'remove')
for d in dir_list:
for d in sorted(dir_list, reverse=True):
# check exist
if file_exists(d) and len(os.listdir(d)) == 0:
os.removedirs(d)
if file_exists(d):
os.rmdir(d)
self.debug(f'删除文件夹: {d}', 'remove')

View File

@ -489,7 +489,7 @@ class JmApiAdaptTool:
fields['sort'] = sort
import random
fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_API_IMAGE_LIST)
fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_IMAGE_LIST)
class JmImageTool:

View File

@ -55,8 +55,10 @@ class Test_Custom(JmTestConfigurable):
JmModuleConfig.register_client(MyClient)
html_domain = self.client.get_html_domain()
JmModuleConfig.DOMAIN_HTML_LIST = [html_domain]
self.assertListEqual(
[html_domain],
JmModuleConfig.DOMAIN_HTML_LIST,
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list()
)