mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.4.7: 实现收藏本子功能,优化获取收藏夹功能传参,优化Headers和域名的处理; 更新文档、dispatch工作流文案+脚本. (#176)
This commit is contained in:
parent
a7e8ea271a
commit
ae96e20cc0
6
.github/workflows/close_specific_pr.yml
vendored
6
.github/workflows/close_specific_pr.yml
vendored
@ -2,10 +2,12 @@ name: Close specific PR
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, ]
|
||||
# Note: If you use both the branches filter and the paths filter, the workflow will only run when both filters are satisfied.
|
||||
branches:
|
||||
- master
|
||||
paths:
|
||||
- 'usage/workflow_download.py'
|
||||
types: [opened, ]
|
||||
|
||||
jobs:
|
||||
close_pr:
|
||||
env:
|
||||
|
6
.github/workflows/download_dispatch.yml
vendored
6
.github/workflows/download_dispatch.yml
vendored
@ -15,19 +15,19 @@ on:
|
||||
|
||||
CLIENT_IMPL:
|
||||
type: string
|
||||
description: 客户端类型(client.impl),[api]=移动端,[html]=网页端。下载失败时,你可以尝试填入此项重试。
|
||||
description: 客户端类型(client.impl),下载失败时,你可以尝试填入此项重试。'api' 表示移动端,'html' 表示网页端。
|
||||
default: ''
|
||||
required: false
|
||||
|
||||
IMAGE_SUFFIX:
|
||||
type: string
|
||||
description: 图片后缀(download.cache.suffix),默认为空,表示不做图片格式转换。可填入例如 "png" "jpg"。
|
||||
description: 图片后缀(download.cache.suffix),默认为空,表示不做图片格式转换。可填入例如 'png' 'jpg'
|
||||
default: ''
|
||||
required: false
|
||||
|
||||
DIR_RULE:
|
||||
type: string
|
||||
description: 下载文件夹规则(dir_rule.rule)。此处可以不填,默认使用配置文件的'Bd_Aauthor_Atitle_Pindex'。
|
||||
description: 下载文件夹规则(dir_rule.rule)。默认使用配置文件的 'Bd_Aauthor_Atitle_Pindex'。
|
||||
default: ''
|
||||
required: false
|
||||
|
||||
|
@ -6,10 +6,10 @@ filter(过滤器)是v2.1.12新引入的机制,
|
||||
使用filter的步骤如下:
|
||||
|
||||
```
|
||||
1. 自定义class,继承JmDownloader,重写filter_iter_objs方法,即:
|
||||
1. 自定义class,继承JmDownloader,重写do_filter方法,即:
|
||||
class MyDownloader(JmDownloader):
|
||||
def filter_iter_objs(self, iter_objs: DownloadIterObjs):
|
||||
# 如何重写?参考JmDownloader.filter_iter_objs和下面的示例
|
||||
def do_filter(self, detail):
|
||||
# 如何重写?参考JmDownloader.do_filter和下面的示例
|
||||
...
|
||||
|
||||
2. 让你的class生效,使用如下代码:
|
||||
@ -30,8 +30,8 @@ from jmcomic import *
|
||||
|
||||
class First3ImageDownloader(JmDownloader):
|
||||
|
||||
def filter_iter_objs(self, iter_objs: DownloadIterObjs):
|
||||
if isinstance(iter_objs, JmPhotoDetail):
|
||||
def do_filter(self, detail):
|
||||
if detail.is_photo():
|
||||
photo: JmPhotoDetail = iter_objs
|
||||
# 支持[start,end,step]
|
||||
return photo[:3]
|
||||
@ -52,8 +52,8 @@ class FindUpdateDownloader(JmDownloader):
|
||||
'xxx': 'yyy'
|
||||
}
|
||||
|
||||
def filter_iter_objs(self, iter_objs: DownloadIterObjs):
|
||||
if not isinstance(iter_objs, JmAlbumDetail):
|
||||
def do_filter(self, detail):
|
||||
if not detail.is_album():
|
||||
return iter_objs
|
||||
|
||||
return self.find_update(iter_objs)
|
||||
|
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.4.6'
|
||||
__version__ = '2.4.7'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
@ -1,3 +1,5 @@
|
||||
from threading import Lock
|
||||
|
||||
from .jm_client_interface import *
|
||||
|
||||
|
||||
@ -25,6 +27,7 @@ class AbstractJmClient(
|
||||
self.retry_times = retry_times
|
||||
self.domain_list = domain_list
|
||||
self.CLIENT_CACHE = None
|
||||
self.__username = None # help for favorite_folder method
|
||||
self.enable_cache()
|
||||
self.after_init()
|
||||
|
||||
@ -50,7 +53,7 @@ class AbstractJmClient(
|
||||
resp.require_success()
|
||||
return resp
|
||||
|
||||
return self.get(img_url, judge=judge)
|
||||
return self.get(img_url, judge=judge, headers=JmModuleConfig.new_html_headers())
|
||||
|
||||
def request_with_retry(self,
|
||||
request,
|
||||
@ -61,7 +64,12 @@ class AbstractJmClient(
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
统一请求,支持重试
|
||||
支持重试和切换域名的机制
|
||||
|
||||
如果url包含了指定域名,则不会切换域名,例如图片URL。
|
||||
|
||||
如果需要拿到域名进行回调处理,可以重写 self.update_request_with_specify_domain 方法,例如更新headers
|
||||
|
||||
:param request: 请求方法
|
||||
:param url: 图片url / path (/album/xxx)
|
||||
:param domain_index: 域名下标
|
||||
@ -74,10 +82,11 @@ class AbstractJmClient(
|
||||
|
||||
if url.startswith('/'):
|
||||
# path → url
|
||||
url = self.of_api_url(
|
||||
api_path=url,
|
||||
domain=self.domain_list[domain_index],
|
||||
)
|
||||
domain = self.domain_list[domain_index]
|
||||
url = self.of_api_url(url, domain)
|
||||
|
||||
self.update_request_with_specify_domain(kwargs, domain)
|
||||
|
||||
jm_log(self.log_topic(), self.decode(url))
|
||||
else:
|
||||
# 图片url
|
||||
@ -96,6 +105,8 @@ class AbstractJmClient(
|
||||
try:
|
||||
resp = request(url, **kwargs)
|
||||
return judge(resp)
|
||||
except KeyboardInterrupt as e:
|
||||
raise e
|
||||
except Exception as e:
|
||||
if self.retry_times == 0:
|
||||
raise e
|
||||
@ -107,6 +118,12 @@ class AbstractJmClient(
|
||||
else:
|
||||
return self.request_with_retry(request, url, domain_index + 1, 0, judge, **kwargs)
|
||||
|
||||
def update_request_with_specify_domain(self, kwargs: dict, domain: str):
|
||||
"""
|
||||
域名自动切换时,用于更新请求参数的回调
|
||||
"""
|
||||
pass
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def log_topic(self):
|
||||
return self.client_key
|
||||
@ -205,6 +222,34 @@ class JmHtmlClient(AbstractJmClient):
|
||||
|
||||
func_to_cache = ['search', 'fetch_detail_entity']
|
||||
|
||||
def add_favorite_album(self,
|
||||
album_id,
|
||||
folder_id='0',
|
||||
):
|
||||
data = {
|
||||
'album_id': album_id,
|
||||
'fid': folder_id,
|
||||
}
|
||||
|
||||
resp = self.get_jm_html(
|
||||
'/ajax/favorite_album',
|
||||
data=data,
|
||||
)
|
||||
|
||||
res = resp.json()
|
||||
|
||||
if res['status'] != 1:
|
||||
msg = parse_unicode_escape_text(res['msg'])
|
||||
error_msg = PatternTool.match_or_default(msg, JmcomicText.pattern_ajax_favorite_msg, msg)
|
||||
# 此圖片已經在您最喜愛的清單!
|
||||
|
||||
self.raise_request_error(
|
||||
resp,
|
||||
error_msg
|
||||
)
|
||||
|
||||
return resp
|
||||
|
||||
def get_album_detail(self, album_id) -> JmAlbumDetail:
|
||||
return self.fetch_detail_entity(album_id, 'album')
|
||||
|
||||
@ -301,6 +346,7 @@ class JmHtmlClient(AbstractJmClient):
|
||||
return resp
|
||||
|
||||
self['cookies'] = new_cookies
|
||||
self.__username = username
|
||||
|
||||
return resp
|
||||
|
||||
@ -311,7 +357,8 @@ class JmHtmlClient(AbstractJmClient):
|
||||
username='',
|
||||
) -> JmFavoritePage:
|
||||
if username == '':
|
||||
username = self.get_username_or_raise()
|
||||
ExceptionTool.require_true(self.__username is not None, 'favorite_folder方法需要传username参数')
|
||||
username = self.__username
|
||||
|
||||
resp = self.get_jm_html(
|
||||
f'/user/{username}/favorite/albums',
|
||||
@ -325,13 +372,12 @@ class JmHtmlClient(AbstractJmClient):
|
||||
return JmPageTool.parse_html_to_favorite_page(resp.text)
|
||||
|
||||
# noinspection PyTypeChecker
|
||||
def get_username_or_raise(self) -> str:
|
||||
cookies = self.get_meta_data('cookies', None)
|
||||
if not cookies:
|
||||
ExceptionTool.raises('未登录,无法获取到对应的用户名,需要传username参数')
|
||||
|
||||
def get_username_from_cookies(self) -> str:
|
||||
# cookies = self.get_meta_data('cookies', None)
|
||||
# if not cookies:
|
||||
# ExceptionTool.raises('未登录,无法获取到对应的用户名,请给favorite方法传入username参数')
|
||||
# 解析cookies,可能需要用到 phpserialize,比较麻烦,暂不实现
|
||||
ExceptionTool.raises('需要传username参数')
|
||||
pass
|
||||
|
||||
def get_jm_html(self, url, require_200=True, **kwargs):
|
||||
"""
|
||||
@ -351,6 +397,12 @@ class JmHtmlClient(AbstractJmClient):
|
||||
|
||||
return resp
|
||||
|
||||
def update_request_with_specify_domain(self, kwargs: dict, domain: Optional[str]):
|
||||
latest_headers = kwargs.get('headers', None)
|
||||
base_headers = self.get_meta_data('headers', None) or JmModuleConfig.new_html_headers(domain)
|
||||
base_headers.update(latest_headers or {})
|
||||
kwargs['headers'] = base_headers
|
||||
|
||||
@classmethod
|
||||
def raise_request_error(cls, resp, msg: Optional[str] = None):
|
||||
"""
|
||||
@ -393,10 +445,7 @@ class JmHtmlClient(AbstractJmClient):
|
||||
(f' to ({comment_id})' if comment_id is not None else '')
|
||||
)
|
||||
|
||||
resp = self.post('/ajax/album_comment',
|
||||
headers=self.album_comment_headers,
|
||||
data=data,
|
||||
)
|
||||
resp = self.post('/ajax/album_comment', data=data)
|
||||
|
||||
ret = JmAlbumCommentResp(resp)
|
||||
jm_log('album.comment', f'{video_id}: [{comment}] ← ({ret.model().cid})')
|
||||
@ -469,26 +518,6 @@ class JmHtmlClient(AbstractJmClient):
|
||||
+ (f'URL=[{url}]' if url is not None else '')
|
||||
)
|
||||
|
||||
album_comment_headers = {
|
||||
'authority': '18comic.vip',
|
||||
'accept': 'application/json, text/javascript, */*; q=0.01',
|
||||
'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
|
||||
'cache-control': 'no-cache',
|
||||
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
|
||||
'origin': 'https://18comic.vip',
|
||||
'pragma': 'no-cache',
|
||||
'referer': 'https://18comic.vip/album/248965/',
|
||||
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'sec-ch-ua-platform': '"Windows"',
|
||||
'sec-fetch-dest': 'empty',
|
||||
'sec-fetch-mode': 'cors',
|
||||
'sec-fetch-site': 'same-origin',
|
||||
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
|
||||
'Chrome/114.0.0.0 Safari/537.36',
|
||||
'x-requested-with': 'XMLHttpRequest',
|
||||
}
|
||||
|
||||
|
||||
# 基于禁漫移动端(APP)实现的JmClient
|
||||
class JmApiClient(AbstractJmClient):
|
||||
@ -581,8 +610,6 @@ class JmApiClient(AbstractJmClient):
|
||||
},
|
||||
)
|
||||
|
||||
self.require_resp_success(resp, url)
|
||||
|
||||
return JmApiAdaptTool.parse_entity(resp.res_data, clazz)
|
||||
|
||||
def fetch_scramble_id(self, photo_id):
|
||||
@ -600,6 +627,7 @@ class JmApiClient(AbstractJmClient):
|
||||
'express': 'off',
|
||||
'v': time_stamp(),
|
||||
},
|
||||
require_success=False,
|
||||
)
|
||||
|
||||
scramble_id = PatternTool.match_or_default(resp.text,
|
||||
@ -712,7 +740,6 @@ class JmApiClient(AbstractJmClient):
|
||||
'password': password,
|
||||
})
|
||||
|
||||
resp.require_success()
|
||||
cookies = dict(resp.resp.cookies)
|
||||
cookies.update({'AVS': resp.res_data['s']})
|
||||
self['cookies'] = cookies
|
||||
@ -736,7 +763,34 @@ class JmApiClient(AbstractJmClient):
|
||||
|
||||
return JmPageTool.parse_api_to_favorite_page(resp.model_data)
|
||||
|
||||
def req_api(self, url, get=True, **kwargs) -> JmApiResp:
|
||||
def add_favorite_album(self,
|
||||
album_id,
|
||||
folder_id='0',
|
||||
):
|
||||
"""
|
||||
移动端没有提供folder_id参数
|
||||
"""
|
||||
resp = self.req_api(
|
||||
'/favorite',
|
||||
data={
|
||||
'aid': album_id,
|
||||
},
|
||||
)
|
||||
|
||||
self.require_resp_status_ok(resp)
|
||||
|
||||
return resp
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def require_resp_status_ok(self, resp: JmApiResp):
|
||||
"""
|
||||
检查返回数据中的status字段是否为ok
|
||||
"""
|
||||
data = resp.model_data
|
||||
if data.status == 'ok':
|
||||
ExceptionTool.raises_resp(data.msg, resp)
|
||||
|
||||
def req_api(self, url, get=True, require_success=True, **kwargs) -> JmApiResp:
|
||||
ts = self.decide_headers_and_ts(kwargs, url)
|
||||
|
||||
if get:
|
||||
@ -744,7 +798,15 @@ class JmApiClient(AbstractJmClient):
|
||||
else:
|
||||
resp = self.post(url, **kwargs)
|
||||
|
||||
return JmApiResp(resp, ts)
|
||||
resp = JmApiResp(resp, ts)
|
||||
|
||||
if require_success:
|
||||
self.require_resp_success(resp, url)
|
||||
|
||||
return resp
|
||||
|
||||
def update_request_with_specify_domain(self, kwargs: dict, domain: str):
|
||||
pass
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def decide_headers_and_ts(self, kwargs, url):
|
||||
@ -791,7 +853,6 @@ class JmApiClient(AbstractJmClient):
|
||||
if JmModuleConfig.flag_api_client_require_cookies:
|
||||
self.ensure_have_cookies()
|
||||
|
||||
from threading import Lock
|
||||
client_init_cookies_lock = Lock()
|
||||
|
||||
def ensure_have_cookies(self):
|
||||
@ -826,9 +887,6 @@ class FutureClientProxy(JmcomicClient):
|
||||
```
|
||||
"""
|
||||
client_key = 'cl_proxy_future'
|
||||
proxy_methods = ['album_comment', 'enable_cache', 'get_domain_list',
|
||||
'get_html_domain', 'get_html_domain_all', 'get_jm_image',
|
||||
'set_cache_dict', 'get_cache_dict', 'set_domain_list', ]
|
||||
|
||||
class FutureWrapper:
|
||||
def __init__(self, future, after_done_callback):
|
||||
@ -855,8 +913,7 @@ class FutureClientProxy(JmcomicClient):
|
||||
executors=None,
|
||||
):
|
||||
self.client = client
|
||||
for method in self.proxy_methods:
|
||||
setattr(self, method, getattr(client, method))
|
||||
self.route_notimpl_method_to_internal_client(client)
|
||||
|
||||
if executors is None:
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
@ -867,6 +924,25 @@ class FutureClientProxy(JmcomicClient):
|
||||
from threading import Lock
|
||||
self.lock = Lock()
|
||||
|
||||
def route_notimpl_method_to_internal_client(self, client):
|
||||
|
||||
impl_methods = str_to_set('''
|
||||
get_album_detail
|
||||
get_photo_detail
|
||||
search
|
||||
''')
|
||||
|
||||
# 获取对象的所有属性和方法的名称列表
|
||||
attributes_and_methods = dir(client)
|
||||
# 遍历属性和方法列表,并访问每个方法
|
||||
for method in attributes_and_methods:
|
||||
# 判断是否为方法(可调用对象)
|
||||
if (not method.startswith('_')
|
||||
and callable(getattr(client, method))
|
||||
and method not in impl_methods
|
||||
):
|
||||
setattr(self, method, getattr(client, method))
|
||||
|
||||
def get_album_detail(self, album_id) -> JmAlbumDetail:
|
||||
album_id = JmcomicText.parse_to_jm_id(album_id)
|
||||
cache_key = f'album_{album_id}'
|
||||
|
@ -229,6 +229,15 @@ class JmUserClient:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def add_favorite_album(self,
|
||||
album_id,
|
||||
folder_id='0',
|
||||
):
|
||||
"""
|
||||
把漫画加入收藏夹
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class JmImageClient:
|
||||
|
||||
|
@ -260,6 +260,7 @@ class JmModuleConfig:
|
||||
headers = JmMagicConstants.HTML_HEADERS_TEMPLATE.copy()
|
||||
headers.update({
|
||||
'authority': domain,
|
||||
'origin': f'https://{domain}',
|
||||
'referer': f'https://{domain}',
|
||||
})
|
||||
return headers
|
||||
@ -296,9 +297,12 @@ class JmModuleConfig:
|
||||
return Postmans.new_postman(**kwargs)
|
||||
|
||||
# option 相关的默认配置
|
||||
# 一般情况下,建议使用option配置文件来定制配置
|
||||
# 而如果只想修改几个简单常用的配置,也可以下方的DEFAULT_XXX属性
|
||||
JM_OPTION_VER = '2.1'
|
||||
DEFAULT_CLIENT_IMPL = 'html'
|
||||
DEFAULT_PROXIES = ProxyBuilder.system_proxy() # use system proxy by default
|
||||
DEFAULT_CLIENT_IMPL = 'html' # 默认Client实现类型为网页端
|
||||
DEFAULT_CLIENT_CACHE = True # 默认开启Client缓存,缓存级别是level_option,详见CacheRegistry
|
||||
DEFAULT_PROXIES = ProxyBuilder.system_proxy() # 默认使用系统代理
|
||||
|
||||
default_option_dict: dict = {
|
||||
'log': None,
|
||||
@ -355,7 +359,7 @@ class JmModuleConfig:
|
||||
# client cache
|
||||
client = option_dict['client']
|
||||
if client['cache'] is None:
|
||||
client['cache'] = True
|
||||
client['cache'] = cls.DEFAULT_CLIENT_CACHE
|
||||
|
||||
# client impl
|
||||
if client['impl'] is None:
|
||||
|
@ -115,7 +115,7 @@ class JmDownloader(DownloadCallback):
|
||||
"""
|
||||
调度本子/章节的下载
|
||||
"""
|
||||
iter_objs = self.filter_iter_objs(iter_objs)
|
||||
iter_objs = self.do_filter(iter_objs)
|
||||
count_real = len(iter_objs)
|
||||
|
||||
if count_real == 0:
|
||||
@ -136,14 +136,14 @@ class JmDownloader(DownloadCallback):
|
||||
)
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def filter_iter_objs(self, detail: DetailEntity):
|
||||
def do_filter(self, detail: DetailEntity):
|
||||
"""
|
||||
该方法可用于过滤本子/章节,默认不会做过滤。
|
||||
例如:
|
||||
只想下载 本子的最新一章,返回 [album[-1]]
|
||||
只想下载 章节的前10张图片,返回 [photo[:10]]
|
||||
|
||||
:param detail: 可能是本子或者章节,需要自行使用 isinstance / is_xxx 判断
|
||||
:param detail: 可能是本子或者章节,需要自行使用 isinstance / detail.is_xxx 判断
|
||||
:returns: 只想要下载的 本子的章节 或 章节的图片
|
||||
"""
|
||||
return detail
|
||||
@ -198,3 +198,18 @@ class JmDownloader(DownloadCallback):
|
||||
jm_log('dler.exception',
|
||||
f'{self.__class__.__name__} Exit with exception: {exc_type, exc_val}'
|
||||
)
|
||||
|
||||
|
||||
class DoNotDownloadImage(JmDownloader):
|
||||
"""
|
||||
本类仅用于测试
|
||||
|
||||
用法:
|
||||
|
||||
JmModuleConfig.CLASS_DOWNLOADER = DoNotDownloadImage
|
||||
"""
|
||||
|
||||
def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
|
||||
# ensure make dir
|
||||
self.option.decide_image_filepath(image)
|
||||
pass
|
||||
|
@ -175,8 +175,8 @@ class JmImageDetail(JmBaseEntity):
|
||||
self.img_file_suffix: str = img_file_suffix
|
||||
|
||||
self.from_photo: Optional[JmPhotoDetail] = from_photo
|
||||
self.query_params: StrNone = query_params
|
||||
self.index = index # 从1开始
|
||||
self.query_params: Optional[str] = query_params
|
||||
self.index = index # 从1开始
|
||||
|
||||
# temp fields, in order to simplify passing parameter
|
||||
self.save_path: str = ''
|
||||
@ -266,7 +266,7 @@ class JmPhotoDetail(DetailEntity):
|
||||
self._tags: str = tags
|
||||
self._series_id: int = int(series_id)
|
||||
|
||||
self._author: StrNone = author
|
||||
self._author: Optional[str] = author
|
||||
self.from_album: Optional[JmAlbumDetail] = from_album
|
||||
self.index = self.album_index
|
||||
|
||||
@ -278,7 +278,7 @@ class JmPhotoDetail(DetailEntity):
|
||||
# page_arr存放了该photo的所有图片文件名 img_name
|
||||
self.page_arr: List[str] = page_arr
|
||||
# 图片的cdn域名
|
||||
self.data_original_domain: StrNone = data_original_domain
|
||||
self.data_original_domain: Optional[str] = data_original_domain
|
||||
# 第一张图的URL
|
||||
self.data_original_0 = data_original_0
|
||||
|
||||
@ -372,7 +372,7 @@ class JmPhotoDetail(DetailEntity):
|
||||
return f'{JmModuleConfig.PROT}{domain}/media/photos/{self.photo_id}/{img_name}'
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def get_data_original_query_params(self, data_original_0: StrNone) -> str:
|
||||
def get_data_original_query_params(self, data_original_0: Optional[str]) -> str:
|
||||
if data_original_0 is None:
|
||||
return f'v={time_stamp()}'
|
||||
|
||||
@ -534,12 +534,18 @@ class JmPageContent(JmBaseEntity, IndexedEntity):
|
||||
|
||||
@property
|
||||
def page_count(self) -> int:
|
||||
"""
|
||||
页数
|
||||
"""
|
||||
page_size = self.page_size
|
||||
import math
|
||||
return math.ceil(int(self.total) / page_size)
|
||||
|
||||
@property
|
||||
def page_size(self) -> int:
|
||||
"""
|
||||
页大小
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def iter_id(self) -> Generator[str, None, None]:
|
||||
|
@ -276,22 +276,10 @@ class JmOption:
|
||||
return JmModuleConfig.option_default_dict()
|
||||
|
||||
@classmethod
|
||||
def default(cls, proxies=None, domain=None) -> 'JmOption':
|
||||
def default(cls) -> 'JmOption':
|
||||
"""
|
||||
使用默认的 JmOption
|
||||
proxies, domain 为常用配置项,为了方便起见直接支持参数配置。
|
||||
其他配置项建议还是使用配置文件
|
||||
:param proxies: clash; 127.0.0.1:7890; v2ray
|
||||
:param domain: 18comic.vip; ["18comic.vip"]
|
||||
"""
|
||||
if proxies is not None or domain is not None:
|
||||
return cls.construct({
|
||||
'client': {
|
||||
'domain': [domain] if isinstance(domain, str) else domain,
|
||||
'postman': {'meta_data': {'proxies': ProxyBuilder.build_by_str(proxies)}},
|
||||
},
|
||||
})
|
||||
|
||||
return cls.construct({})
|
||||
|
||||
@classmethod
|
||||
@ -372,7 +360,7 @@ class JmOption:
|
||||
"""
|
||||
return self.new_jm_client(**kwargs)
|
||||
|
||||
def new_jm_client(self, domain=None, impl=None, cache=None, **kwargs) -> JmcomicClient:
|
||||
def new_jm_client(self, domain_list=None, impl=None, cache=None, **kwargs) -> JmcomicClient:
|
||||
"""
|
||||
创建新的Client(客户端),不同Client之间的元数据不共享
|
||||
"""
|
||||
@ -380,10 +368,15 @@ class JmOption:
|
||||
|
||||
# 所有需要用到的 self.client 配置项如下
|
||||
postman_conf: dict = deepcopy(self.client.postman.src_dict) # postman dsl 配置
|
||||
|
||||
meta_data: dict = postman_conf['meta_data'] # 元数据
|
||||
|
||||
retry_times: int = self.client.retry_times # 重试次数
|
||||
|
||||
cache: str = cache if cache is not None else self.client.cache # 启用缓存
|
||||
|
||||
impl: str = impl or self.client.impl # client_key
|
||||
|
||||
if isinstance(impl, type):
|
||||
# eg: impl = JmHtmlClient
|
||||
# noinspection PyUnresolvedReferences
|
||||
@ -392,28 +385,30 @@ class JmOption:
|
||||
# start construct client
|
||||
|
||||
# domain
|
||||
def decide_domain():
|
||||
domain_list: Union[List[str], DictModel, dict] = domain if domain is not None \
|
||||
else self.client.domain # 域名
|
||||
def decide_domain_list():
|
||||
nonlocal domain_list
|
||||
|
||||
if not isinstance(domain_list, list):
|
||||
if domain_list is None:
|
||||
domain_list = self.client.domain
|
||||
|
||||
if not isinstance(domain_list, (list, str)):
|
||||
# dict
|
||||
domain_list = domain_list.get(impl, [])
|
||||
|
||||
if isinstance(domain_list, str):
|
||||
# multi-lines text
|
||||
domain_list = str_to_list(domain_list)
|
||||
|
||||
# list or str
|
||||
if len(domain_list) == 0:
|
||||
domain_list = self.decide_client_domain(impl)
|
||||
|
||||
return domain_list
|
||||
|
||||
domain: List[str] = decide_domain()
|
||||
|
||||
# support kwargs overwrite meta_data
|
||||
if len(kwargs) != 0:
|
||||
meta_data.update(kwargs)
|
||||
|
||||
# headers
|
||||
if meta_data['headers'] is None:
|
||||
meta_data['headers'] = self.decide_postman_headers(impl, domain[0])
|
||||
|
||||
# postman
|
||||
postman = Postmans.create(data=postman_conf)
|
||||
|
||||
@ -424,7 +419,7 @@ class JmOption:
|
||||
|
||||
client: AbstractJmClient = clazz(
|
||||
postman=postman,
|
||||
domain_list=domain,
|
||||
domain_list=decide_domain_list(),
|
||||
retry_times=retry_times,
|
||||
)
|
||||
|
||||
@ -459,20 +454,6 @@ class JmOption:
|
||||
|
||||
ExceptionTool.raises(f'没有配置域名,且是无法识别的client类型: {client_key}')
|
||||
|
||||
def decide_postman_headers(self, client_key, domain):
|
||||
is_client_type = lambda ctype: self.client_key_is_given_type(client_key, ctype)
|
||||
|
||||
if is_client_type(JmApiClient):
|
||||
# 移动端
|
||||
# 不配置headers,由client每次请求前创建headers
|
||||
return None
|
||||
|
||||
if is_client_type(JmHtmlClient):
|
||||
# 网页端
|
||||
return JmModuleConfig.new_html_headers(domain)
|
||||
|
||||
ExceptionTool.raises(f'没有配置域名,且是无法识别的client类型: {client_key}')
|
||||
|
||||
@classmethod
|
||||
def client_key_is_given_type(cls, client_key, ctype: Type[JmcomicClient]):
|
||||
if client_key == ctype.client_key:
|
||||
|
@ -208,7 +208,7 @@ class FindUpdatePlugin(JmOptionPlugin):
|
||||
return photo_ls
|
||||
|
||||
class FindUpdateDownloader(JmDownloader):
|
||||
def filter_iter_objs(self, detail):
|
||||
def do_filter(self, detail):
|
||||
if not detail.is_album():
|
||||
return detail
|
||||
|
||||
|
@ -55,6 +55,9 @@ class JmcomicText:
|
||||
# 評論(div)
|
||||
pattern_html_album_comment_count = compile(r'<div class="badge"[^>]*?id="total_video_comments">(\d+)</div>'), 0
|
||||
|
||||
# 提取接口返回值信息
|
||||
pattern_ajax_favorite_msg = compile(r'</button>(.*?)</div>')
|
||||
|
||||
@classmethod
|
||||
def parse_to_jm_domain(cls, text: str):
|
||||
if text.startswith(JmModuleConfig.PROT):
|
||||
@ -308,7 +311,7 @@ class PatternTool:
|
||||
def require_match(cls, html: str, pattern: Pattern, msg, rindex=1):
|
||||
match = pattern.search(html)
|
||||
if match is not None:
|
||||
return match[rindex]
|
||||
return match[rindex] if rindex is not None else match
|
||||
|
||||
ExceptionTool.raises_regex(
|
||||
msg,
|
||||
|
@ -45,7 +45,7 @@ class Test_Custom(JmTestConfigurable):
|
||||
|
||||
self.assertListEqual(
|
||||
JmModuleConfig.DOMAIN_API_LIST,
|
||||
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list()
|
||||
self.option.new_jm_client(domain_list=[], impl=MyClient.client_key).get_domain_list()
|
||||
)
|
||||
|
||||
def test_extends_html_client(self):
|
||||
@ -59,7 +59,7 @@ class Test_Custom(JmTestConfigurable):
|
||||
|
||||
self.assertListEqual(
|
||||
JmModuleConfig.DOMAIN_HTML_LIST,
|
||||
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list()
|
||||
self.option.new_jm_client(domain_list=[], impl=MyClient.client_key).get_domain_list()
|
||||
)
|
||||
|
||||
def test_client_key_missing(self):
|
||||
@ -74,7 +74,7 @@ class Test_Custom(JmTestConfigurable):
|
||||
)
|
||||
|
||||
def test_custom_client_empty_domain(self):
|
||||
class MyClient(JmcomicClient):
|
||||
class MyClient(AbstractJmClient):
|
||||
client_key = 'myclient'
|
||||
pass
|
||||
|
||||
@ -95,6 +95,6 @@ class Test_Custom(JmTestConfigurable):
|
||||
JmModuleConfig.register_client(MyClient)
|
||||
self.assertListEqual(
|
||||
JmModuleConfig.DOMAIN_API_LIST,
|
||||
self.option.new_jm_client(domain=[], impl=MyClient.client_key).get_domain_list(),
|
||||
self.option.new_jm_client(domain_list=[], impl=MyClient.client_key).get_domain_list(),
|
||||
msg='继承client,不配置域名',
|
||||
)
|
||||
|
@ -1,5 +1,5 @@
|
||||
from jmcomic import *
|
||||
from jmcomic.cl import get_env, JmcomicUI
|
||||
from jmcomic.cl import JmcomicUI
|
||||
|
||||
# 下方填入你要下载的本子的id,一行一个,每行的首尾可以有空白字符
|
||||
jm_albums = '''
|
||||
@ -16,11 +16,24 @@ jm_photos = '''
|
||||
'''
|
||||
|
||||
|
||||
def env(name, default, trim=('[]', '""', "''")):
|
||||
import os
|
||||
value = os.getenv(name, None)
|
||||
if value is None or value == '':
|
||||
return default
|
||||
|
||||
for pair in trim:
|
||||
if value.startswith(pair[0]) and value.endswith(pair[1]):
|
||||
value = value[1:-1]
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def get_id_set(env_name):
|
||||
aid_set = set()
|
||||
for text in [
|
||||
jm_albums,
|
||||
(get_env(env_name, '')).replace('-', '\n'),
|
||||
(env(env_name, '')).replace('-', '\n'),
|
||||
]:
|
||||
aid_set.update(str_to_set(text))
|
||||
|
||||
@ -39,6 +52,7 @@ def main():
|
||||
helper.run(option)
|
||||
option.call_all_plugin('after_download')
|
||||
|
||||
|
||||
def get_option():
|
||||
# 读取 option 配置文件
|
||||
option = create_option('../assets/option/option_workflow_download.yml')
|
||||
@ -53,23 +67,23 @@ def get_option():
|
||||
|
||||
|
||||
def cover_option_config(option: JmOption):
|
||||
dir_rule = get_env('DIR_RULE', None)
|
||||
dir_rule = env('DIR_RULE', None)
|
||||
if dir_rule is not None:
|
||||
the_old = option.dir_rule
|
||||
the_new = DirRule(dir_rule, base_dir=the_old.base_dir)
|
||||
option.dir_rule = the_new
|
||||
|
||||
impl = get_env('CLIENT_IMPL', None)
|
||||
impl = env('CLIENT_IMPL', None)
|
||||
if impl is not None:
|
||||
option.client.impl = impl
|
||||
|
||||
suffix = get_env('IMAGE_SUFFIX', None)
|
||||
suffix = env('IMAGE_SUFFIX', None)
|
||||
if suffix is not None:
|
||||
option.download.image.suffix = fix_suffix(suffix)
|
||||
|
||||
|
||||
def log_before_raise():
|
||||
jm_download_dir = get_env('JM_DOWNLOAD_DIR', workspace())
|
||||
jm_download_dir = env('JM_DOWNLOAD_DIR', workspace())
|
||||
mkdir_if_not_exists(jm_download_dir)
|
||||
|
||||
# 自定义异常抛出函数,在抛出前把HTML响应数据写到下载文件夹(日志留痕)
|
||||
|
Loading…
Reference in New Issue
Block a user