v2.3.16: 修复zip插件在album模式下的一些bug; 调整获取APP_COOKIES的位置; 一些代码优化。 (#160)

This commit is contained in:
hect0x7 2023-10-31 02:55:33 +08:00 committed by GitHub
parent 87ed6bce83
commit 9e3fa7f17a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 97 additions and 67 deletions

2
.github/release.py vendored
View File

@ -18,7 +18,7 @@ def parse_body(body):
e: str = e.strip() e: str = e.strip()
if e == '': if e == '':
continue continue
points.append(f'{i}. {e}') points.append(f'{i + 1}. {e}')
return '\n'.join(points) return '\n'.join(points)

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方 # 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader # config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.3.15' __version__ = '2.3.16'
from .api import * from .api import *
from .jm_plugin import * from .jm_plugin import *

View File

@ -493,13 +493,13 @@ class JmApiClient(AbstractJmClient):
} }
) )
match = JmcomicText.pattern_html_album_scramble_id.search(resp.text) scramble_id = PatternTool.match_or_default(resp.text,
JmcomicText.pattern_html_album_scramble_id,
if match is not None: None,
scramble_id = match[1] )
else: if scramble_id is None:
jm_debug('api.scramble', '未从响应中匹配到scramble_id返回默认值220980') jm_debug('api.scramble', f'未匹配到scramble_id响应文本{resp.text}')
scramble_id = '220980' scramble_id = str(JmModuleConfig.SCRAMBLE_220980)
return scramble_id return scramble_id
@ -610,15 +610,12 @@ class JmApiClient(AbstractJmClient):
# 2. 是否是特殊的内容 # 2. 是否是特殊的内容
# 暂无 # 暂无
@classmethod
@field_cache('__init_cookies__')
def fetch_init_cookies(cls, client: 'JmApiClient'):
resp = client.setting()
return dict(resp.resp.cookies)
def after_init(self): def after_init(self):
cookies = self.__class__.fetch_init_cookies(self) # cookies = self.__class__.fetch_init_cookies(self)
self.get_root_postman().get_meta_data()['cookies'] = cookies # self.get_root_postman().get_meta_data()['cookies'] = cookies
self.get_root_postman().get_meta_data()['cookies'] = JmModuleConfig.get_cookies(self)
pass
class FutureClientProxy(JmcomicClient): class FutureClientProxy(JmcomicClient):

View File

@ -3,7 +3,7 @@ def field_cache(*args, **kwargs):
return field_cache(*args, **kwargs) return field_cache(*args, **kwargs)
def default_jm_debug(topic: str, msg: str): def default_jm_debug_logging(topic: str, msg: str):
from common import format_ts from common import format_ts
print(f'{format_ts()}:【{topic}{msg}') print(f'{format_ts()}:【{topic}{msg}')
@ -62,13 +62,14 @@ class JmModuleConfig:
SCRAMBLE_421926 = 421926 # 2023-02-08后改了图片切割算法 SCRAMBLE_421926 = 421926 # 2023-02-08后改了图片切割算法
SCRAMBLE_CACHE = {} SCRAMBLE_CACHE = {}
# 移动端API的相关配置 # 移动端API密钥
# API密钥 APP_SECRET = '18comicAPPContent'
APP_SECRET = '18comicAPP'
# 域名配置 - 移动端 # cookies目前只在移动端使用因为移动端请求接口须携带但不会校验cookies的内容。
# 图片域名 APP_COOKIES = None
DOMAIN_API_IMAGE_LIST = str_to_list('''
# 移动端图片域名
DOMAIN_IMAGE_LIST = str_to_list('''
cdn-msp.jmapiproxy1.monster cdn-msp.jmapiproxy1.monster
cdn-msp2.jmapiproxy1.monster cdn-msp2.jmapiproxy1.monster
cdn-msp.jmapiproxy1.cc cdn-msp.jmapiproxy1.cc
@ -78,7 +79,7 @@ class JmModuleConfig:
''') ''')
# API域名 # 移动端API域名
DOMAIN_API_LIST = str_to_list(''' DOMAIN_API_LIST = str_to_list('''
www.jmapinode1.top www.jmapinode1.top
www.jmapinode2.top www.jmapinode2.top
@ -88,8 +89,11 @@ class JmModuleConfig:
''') ''')
# 域名配置 - 网页端 # 网页端域名配置
# 无需配置默认为None需要的时候会发起请求获得 # 无需配置默认为None需要的时候会发起请求获得
# 使用优先级:
# 1. DOMAIN_HTML_LIST
# 2. [DOMAIN_HTML]
DOMAIN_HTML = None DOMAIN_HTML = None
DOMAIN_HTML_LIST = None DOMAIN_HTML_LIST = None
@ -106,7 +110,7 @@ class JmModuleConfig:
REGISTRY_PLUGIN = {} REGISTRY_PLUGIN = {}
# 执行debug的函数 # 执行debug的函数
debug_executor = default_jm_debug debug_executor = default_jm_debug_logging
# postman构造函数 # postman构造函数
postman_constructor = default_postman_constructor postman_constructor = default_postman_constructor
# 网页正则表达式解析失败时执行抛出异常的函数可以替换掉用于debug # 网页正则表达式解析失败时执行抛出异常的函数可以替换掉用于debug
@ -190,7 +194,7 @@ class JmModuleConfig:
postman = postman or cls.new_postman(session=True) postman = postman or cls.new_postman(session=True)
url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL) url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL)
cls.jm_debug('获取禁漫网页URL', f'[{cls.JM_REDIRECT_URL}] → [{url}]') cls.jm_debug('module.html_url', f'获取禁漫网页URL: [{cls.JM_REDIRECT_URL}] → [{url}]')
return url return url
@classmethod @classmethod
@ -211,9 +215,22 @@ class JmModuleConfig:
from .jm_toolkit import JmcomicText from .jm_toolkit import JmcomicText
domain_list = JmcomicText.analyse_jm_pub_html(resp.text) domain_list = JmcomicText.analyse_jm_pub_html(resp.text)
cls.jm_debug('获取禁漫网页全部域名', f'[{resp.url}] → {domain_list}') cls.jm_debug('module.html_domain_all', f'获取禁漫网页全部域名: [{resp.url}] → {domain_list}')
return domain_list return domain_list
@classmethod
@field_cache("APP_COOKIES")
def get_cookies(cls, postman=None):
from .jm_toolkit import JmcomicText
url = JmcomicText.format_url('/setting', cls.DOMAIN_API_LIST[0])
postman = postman or cls.new_postman()
resp = postman.get(url)
cookies = dict(resp.cookies)
cls.jm_debug('module.cookies', f'获取cookies: [{url}] → {cookies}')
return cookies
@classmethod @classmethod
def new_html_headers(cls, domain='18comic.vip'): def new_html_headers(cls, domain='18comic.vip'):
""" """
@ -247,7 +264,7 @@ class JmModuleConfig:
key_ts = time_stamp() key_ts = time_stamp()
import hashlib import hashlib
token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode()).hexdigest() token = hashlib.md5(f"{key_ts}{cls.APP_SECRET}".encode("utf-8")).hexdigest()
return { return {
'token': token, 'token': token,

View File

@ -314,6 +314,7 @@ class JmOption:
def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient: def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient:
# 所有需要用到的 self.client 配置项如下 # 所有需要用到的 self.client 配置项如下
postman_conf: dict = self.client.postman.src_dict # postman dsl 配置 postman_conf: dict = self.client.postman.src_dict # postman dsl 配置
meta_data: dict = postman_conf['meta_data'] # 请求元信息
impl: str = impl or self.client.impl # client_key impl: str = impl or self.client.impl # client_key
retry_times: int = self.client.retry_times # 重试次数 retry_times: int = self.client.retry_times # 重试次数
cache: str = cache or self.client.cache # 启用缓存 cache: str = cache or self.client.cache # 启用缓存
@ -335,15 +336,11 @@ class JmOption:
# support kwargs overwrite meta_data # support kwargs overwrite meta_data
if len(kwargs) != 0: if len(kwargs) != 0:
postman_conf['meta_data'].update(kwargs) meta_data.update(kwargs)
# headers # headers
meta_data = postman_conf['meta_data']
if meta_data['headers'] is None: if meta_data['headers'] is None:
headers = self.decide_postman_headers(impl, domain[0]) meta_data['headers'] = self.decide_postman_headers(impl, domain[0])
# if headers is None:
# postman_conf['type'] = 'requests'
meta_data['headers'] = headers
# postman # postman
postman = Postmans.create(data=postman_conf) postman = Postmans.create(data=postman_conf)
@ -375,6 +372,9 @@ class JmOption:
if is_client_type(JmHtmlClient): if is_client_type(JmHtmlClient):
# 网页端 # 网页端
domain_list = JmModuleConfig.DOMAIN_HTML_LIST
if domain_list is not None:
return domain_list
return [JmModuleConfig.get_html_domain()] return [JmModuleConfig.get_html_domain()]
ExceptionTool.raises(f'没有配置域名且是无法识别的client类型: {client_key}') ExceptionTool.raises(f'没有配置域名且是无法识别的client类型: {client_key}')

View File

@ -249,18 +249,24 @@ class ZipPlugin(JmOptionPlugin):
mkdir_if_not_exists(zip_dir) mkdir_if_not_exists(zip_dir)
# 原文件夹 -> zip文件 # 原文件夹 -> zip文件
dir_zip_dict = {} dir_zip_dict: Dict[str, Optional[str]] = {}
photo_dict = downloader.all_downloaded[album] photo_dict = downloader.all_downloaded[album]
if level == 'album': if level == 'album':
zip_path = self.get_zip_path(album, None, filename_rule, suffix, zip_dir) zip_path = self.get_zip_path(album, None, filename_rule, suffix, zip_dir)
dir_path = self.zip_album(album, photo_dict, zip_path) dir_path = self.zip_album(album, photo_dict, zip_path)
if dir_path is not None:
# 要删除这个album文件夹
dir_zip_dict[dir_path] = zip_path dir_zip_dict[dir_path] = zip_path
# 也要删除album下的photo文件夹
for d in files_of_dir(dir_path):
dir_zip_dict[d] = None
elif level == 'photo': elif level == 'photo':
for photo, image_list in photo_dict.items(): for photo, image_list in photo_dict.items():
zip_path = self.get_zip_path(None, photo, filename_rule, suffix, zip_dir) zip_path = self.get_zip_path(None, photo, filename_rule, suffix, zip_dir)
dir_path = self.zip_photo(photo, image_list, zip_path) dir_path = self.zip_photo(photo, image_list, zip_path)
if dir_path is not None:
dir_zip_dict[dir_path] = zip_path dir_zip_dict[dir_path] = zip_path
else: else:
@ -268,7 +274,7 @@ class ZipPlugin(JmOptionPlugin):
self.after_zip(dir_zip_dict) self.after_zip(dir_zip_dict)
def zip_photo(self, photo, image_list: list, zip_path: str): def zip_photo(self, photo, image_list: list, zip_path: str) -> Optional[str]:
""" """
压缩photo文件夹 压缩photo文件夹
:returns: photo文件夹路径 :returns: photo文件夹路径
@ -277,46 +283,54 @@ class ZipPlugin(JmOptionPlugin):
if len(image_list) == 0 \ if len(image_list) == 0 \
else os.path.dirname(image_list[0][0]) else os.path.dirname(image_list[0][0])
all_filepath = set(map(lambda t: t[0], image_list)) all_filepath = set(map(lambda t: self.unified_path(t[0]), image_list))
if len(all_filepath) == 0: return self.do_zip(photo_dir,
self.debug('无下载文件,无需压缩', 'skip') zip_path,
return all_filepath,
f'压缩章节[{photo.photo_id}]成功 → {zip_path}',
)
from common import backup_dir_to_zip @staticmethod
backup_dir_to_zip(photo_dir, zip_path, acceptor=lambda f: f in all_filepath) def unified_path(f):
self.debug(f'压缩章节[{photo.photo_id}]成功 → {zip_path}', 'finish') return fix_filepath(f, os.path.isdir(f))
return photo_dir def zip_album(self, album, photo_dict: dict, zip_path) -> Optional[str]:
def zip_album(self, album, photo_dict: dict, zip_path):
""" """
压缩album文件夹 压缩album文件夹
:returns: album文件夹路径 :returns: album文件夹路径
""" """
album_dir = self.option.decide_album_dir(album)
all_filepath: Set[str] = set() all_filepath: Set[str] = set()
for image_list in photo_dict.values(): def addpath(f):
image_list: List[Tuple[str, JmImageDetail]] all_filepath.update(set(f))
for path, _ in image_list:
all_filepath.add(path)
album_dir = self.option.decide_album_dir(album)
# addpath(self.option.decide_image_save_dir(photo) for photo in photo_dict.keys())
addpath(path for ls in photo_dict.values() for path, _ in ls)
return self.do_zip(album_dir,
zip_path,
all_filepath,
msg=f'压缩本子[{album.album_id}]成功 → {zip_path}',
)
def do_zip(self, source_dir, zip_path, all_filepath, msg):
if len(all_filepath) == 0: if len(all_filepath) == 0:
self.debug('无下载文件,无需压缩', 'skip') self.debug('无下载文件,无需压缩', 'skip')
return return None
from common import backup_dir_to_zip from common import backup_dir_to_zip
backup_dir_to_zip( backup_dir_to_zip(
album_dir, source_dir,
zip_path, zip_path,
acceptor=lambda f: f in all_filepath acceptor=lambda f: os.path.isdir(f) or self.unified_path(f) in all_filepath
) ).close()
self.debug(f'压缩本子[{album.album_id}]成功 → {zip_path}', 'finish') self.debug(msg, 'finish')
return album_dir return self.unified_path(source_dir)
def after_zip(self, dir_zip_dict: Dict[str, str]): def after_zip(self, dir_zip_dict: Dict[str, Optional[str]]):
# 是否要删除所有原文件 # 是否要删除所有原文件
if self.delete_original_file is True: if self.delete_original_file is True:
self.delete_all_files_and_empty_dir( self.delete_all_files_and_empty_dir(
@ -352,10 +366,10 @@ class ZipPlugin(JmOptionPlugin):
os.remove(f) os.remove(f)
self.debug(f'删除原文件: {f}', 'remove') self.debug(f'删除原文件: {f}', 'remove')
for d in dir_list: for d in sorted(dir_list, reverse=True):
# check exist # check exist
if file_exists(d) and len(os.listdir(d)) == 0: if file_exists(d):
os.removedirs(d) os.rmdir(d)
self.debug(f'删除文件夹: {d}', 'remove') self.debug(f'删除文件夹: {d}', 'remove')

View File

@ -489,7 +489,7 @@ class JmApiAdaptTool:
fields['sort'] = sort fields['sort'] = sort
import random import random
fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_API_IMAGE_LIST) fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_IMAGE_LIST)
class JmImageTool: class JmImageTool:

View File

@ -55,8 +55,10 @@ class Test_Custom(JmTestConfigurable):
JmModuleConfig.register_client(MyClient) JmModuleConfig.register_client(MyClient)
html_domain = self.client.get_html_domain() html_domain = self.client.get_html_domain()
JmModuleConfig.DOMAIN_HTML_LIST = [html_domain]
self.assertListEqual( self.assertListEqual(
[html_domain], JmModuleConfig.DOMAIN_HTML_LIST,
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list() self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list()
) )