mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v2.3.6: 新增基于线程池实现的异步客户端代理及其对应的启用插件,用于解决移动端Client一次多个请求要等待的问题,优化重试机制和缓存机制 (#147)
This commit is contained in:
parent
df4111295f
commit
8070951feb
@ -6,7 +6,7 @@ client:
|
||||
retry_times: 3
|
||||
postman:
|
||||
meta_data:
|
||||
timeout: 5
|
||||
timeout: 7
|
||||
|
||||
# 插件配置
|
||||
plugin:
|
||||
@ -15,3 +15,8 @@ plugin:
|
||||
kwargs:
|
||||
interval: 0.5 # 间隔时间
|
||||
enable_warning: false # 不告警
|
||||
|
||||
- plugin: client_proxy
|
||||
kwargs:
|
||||
proxy_client_key: cl_proxy_future
|
||||
whitelist: [ api, ]
|
||||
@ -5,8 +5,7 @@ dir_rule:
|
||||
|
||||
client:
|
||||
domain:
|
||||
- jmcomic1.me
|
||||
- jmcomic.me
|
||||
html: [jmcomic1.me, jmcomic.me]
|
||||
|
||||
# 插件配置
|
||||
plugin:
|
||||
@ -15,3 +14,8 @@ plugin:
|
||||
kwargs:
|
||||
interval: 0.5 # 间隔时间
|
||||
enable_warning: false # 不告警
|
||||
|
||||
- plugin: client_proxy
|
||||
kwargs:
|
||||
proxy_client_key: cl_proxy_future
|
||||
whitelist: [ api, ]
|
||||
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.3.5'
|
||||
__version__ = '2.3.6'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
||||
@ -22,7 +22,7 @@ def download_batch(download_api,
|
||||
|
||||
return multi_thread_launcher(
|
||||
iter_objs=set(
|
||||
JmcomicText.parse_to_album_id(jmid)
|
||||
JmcomicText.parse_to_jm_id(jmid)
|
||||
for jmid in jm_id_iter
|
||||
),
|
||||
apply_each_obj_func=lambda aid: download_api(aid, option, downloader),
|
||||
|
||||
@ -62,7 +62,7 @@ class JmcomicUI:
|
||||
from .jm_toolkit import JmcomicText
|
||||
|
||||
try:
|
||||
return JmcomicText.parse_to_album_id(text)
|
||||
return JmcomicText.parse_to_jm_id(text)
|
||||
except Exception as e:
|
||||
print(e.args[0])
|
||||
exit(1)
|
||||
|
||||
@ -7,6 +7,7 @@ class AbstractJmClient(
|
||||
PostmanProxy,
|
||||
):
|
||||
client_key = None
|
||||
func_to_cache = []
|
||||
|
||||
def __init__(self,
|
||||
postman: Postman,
|
||||
@ -40,21 +41,22 @@ class AbstractJmClient(
|
||||
|
||||
def get_jm_image(self, img_url) -> JmImageResp:
|
||||
|
||||
def get_if_fail_raise(url):
|
||||
def judge(resp):
|
||||
"""
|
||||
使用此方法包装 self.get,使得图片数据为空时,判定为请求失败时,走重试逻辑
|
||||
"""
|
||||
resp = JmImageResp(self.get(url))
|
||||
resp = JmImageResp(resp)
|
||||
resp.require_success()
|
||||
return resp
|
||||
|
||||
return self.request_with_retry(get_if_fail_raise, img_url)
|
||||
return self.get(img_url, judge=judge)
|
||||
|
||||
def request_with_retry(self,
|
||||
request,
|
||||
url,
|
||||
domain_index=0,
|
||||
retry_count=0,
|
||||
judge=lambda resp: resp,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
@ -63,6 +65,7 @@ class AbstractJmClient(
|
||||
@param url: 图片url / path (/album/xxx)
|
||||
@param domain_index: 域名下标
|
||||
@param retry_count: 重试次数
|
||||
@param judge: 判定响应是否成功
|
||||
@param kwargs: 请求方法的kwargs
|
||||
"""
|
||||
if domain_index >= len(self.domain_list):
|
||||
@ -90,14 +93,15 @@ class AbstractJmClient(
|
||||
)
|
||||
|
||||
try:
|
||||
return request(url, **kwargs)
|
||||
resp = request(url, **kwargs)
|
||||
return judge(resp)
|
||||
except Exception as e:
|
||||
self.before_retry(e, kwargs, retry_count, url)
|
||||
|
||||
if retry_count < self.retry_times:
|
||||
return self.request_with_retry(request, url, domain_index, retry_count + 1, **kwargs)
|
||||
return self.request_with_retry(request, url, domain_index, retry_count + 1, judge, **kwargs)
|
||||
else:
|
||||
return self.request_with_retry(request, url, domain_index + 1, 0, **kwargs)
|
||||
return self.request_with_retry(request, url, domain_index + 1, 0, judge, **kwargs)
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def debug_topic_request(self):
|
||||
@ -111,55 +115,28 @@ class AbstractJmClient(
|
||||
if self.is_cache_enabled():
|
||||
return
|
||||
|
||||
def wrap_func_cache(func_name, cache_dict_name):
|
||||
if hasattr(self, cache_dict_name):
|
||||
def wrap_func_with_cache(func_name, cache_field_name):
|
||||
if hasattr(self, cache_field_name):
|
||||
return
|
||||
|
||||
if sys.version_info > (3, 9):
|
||||
import functools
|
||||
cache = functools.cache
|
||||
else:
|
||||
import common
|
||||
if common.VERSION > '0.4.8':
|
||||
cache = common.cache
|
||||
cache_dict = {}
|
||||
cache_hit_msg = (f'【缓存命中】{cache_dict_name} ' + '→ [{}]') if debug is True else None
|
||||
cache_miss_msg = (f'【缓存缺失】{cache_dict_name} ' + '← [{}]') if debug is True else None
|
||||
cache = cache(
|
||||
cache_dict=cache_dict,
|
||||
cache_hit_msg=cache_hit_msg,
|
||||
cache_miss_msg=cache_miss_msg,
|
||||
)
|
||||
setattr(self, cache_dict_name, cache_dict)
|
||||
else:
|
||||
ExceptionTool.raises('不支持启用JmcomicClient缓存。\n'
|
||||
'请更新python版本到3.9以上,'
|
||||
'或更新commonX: `pip install commonX --upgrade`')
|
||||
return
|
||||
from functools import lru_cache
|
||||
cache = lru_cache()
|
||||
|
||||
func = getattr(self, func_name)
|
||||
wrap_func = cache(func)
|
||||
setattr(self, func_name, cache(func))
|
||||
|
||||
setattr(self, func_name, wrap_func)
|
||||
|
||||
for func in {
|
||||
'get_photo_detail',
|
||||
'get_album_detail',
|
||||
'search',
|
||||
}:
|
||||
wrap_func_cache(func, func + '.cache.dict')
|
||||
for func_name in self.func_to_cache:
|
||||
wrap_func_with_cache(func_name, f'__{func_name}.cache.dict__')
|
||||
|
||||
setattr(self, '__enable_cache__', True)
|
||||
|
||||
def is_cache_enabled(self) -> bool:
|
||||
return getattr(self, '__enable_cache__', False)
|
||||
|
||||
def get_html_domain(self, postman=None):
|
||||
return JmModuleConfig.get_html_domain(postman or self.get_root_postman())
|
||||
|
||||
def get_html_domain_all(self, postman=None):
|
||||
return JmModuleConfig.get_html_domain_all(postman or self.get_root_postman())
|
||||
|
||||
def get_domain_list(self):
|
||||
return self.domain_list
|
||||
|
||||
@ -194,32 +171,40 @@ class AbstractJmClient(
|
||||
class JmHtmlClient(AbstractJmClient):
|
||||
client_key = 'html'
|
||||
|
||||
func_to_cache = ['search', 'fetch_detail_entity']
|
||||
|
||||
def get_album_detail(self, album_id) -> JmAlbumDetail:
|
||||
# 参数校验
|
||||
album_id = JmcomicText.parse_to_photo_id(album_id)
|
||||
return self.fetch_detail_entity(album_id, 'album')
|
||||
|
||||
# 请求
|
||||
resp = self.get_jm_html(f"/album/{album_id}")
|
||||
|
||||
# 用 JmcomicText 解析 html,返回实体类
|
||||
return JmcomicText.analyse_jm_album_html(resp.text)
|
||||
|
||||
def get_photo_detail(self, photo_id, fetch_album=True) -> JmPhotoDetail:
|
||||
# 参数校验
|
||||
photo_id = JmcomicText.parse_to_photo_id(photo_id)
|
||||
|
||||
# 请求
|
||||
resp = self.get_jm_html(f"/photo/{photo_id}")
|
||||
|
||||
# 用 JmcomicText 解析 html,返回实体类
|
||||
photo = JmcomicText.analyse_jm_photo_html(resp.text)
|
||||
def get_photo_detail(self,
|
||||
photo_id,
|
||||
fetch_album=True,
|
||||
fetch_scramble_id=True,
|
||||
) -> JmPhotoDetail:
|
||||
photo = self.fetch_detail_entity(photo_id, 'photo')
|
||||
|
||||
# 一并获取该章节的所处本子
|
||||
# todo: 可优化,获取章节所在本子,其实不需要等待章节获取完毕后。
|
||||
# 可以直接调用 self.get_album_detail(photo_id),会重定向返回本子的HTML
|
||||
if fetch_album is True:
|
||||
photo.from_album = self.get_album_detail(photo.album_id)
|
||||
|
||||
return photo
|
||||
|
||||
def fetch_detail_entity(self, apid, prefix):
|
||||
# 参数校验
|
||||
apid = JmcomicText.parse_to_jm_id(apid)
|
||||
|
||||
# 请求
|
||||
resp = self.get_jm_html(f"/{prefix}/{apid}")
|
||||
|
||||
# 用 JmcomicText 解析 html,返回实体类
|
||||
if prefix == 'album':
|
||||
return JmcomicText.analyse_jm_album_html(resp.text)
|
||||
|
||||
if prefix == 'photo':
|
||||
return JmcomicText.analyse_jm_photo_html(resp.text)
|
||||
|
||||
def search(self,
|
||||
search_query: str,
|
||||
page: int,
|
||||
@ -402,6 +387,8 @@ class JmHtmlClient(AbstractJmClient):
|
||||
# 基于禁漫移动端(APP)实现的JmClient
|
||||
class JmApiClient(AbstractJmClient):
|
||||
client_key = 'api'
|
||||
func_to_cache = ['search', 'fetch_detail_entity']
|
||||
|
||||
API_SEARCH = '/search'
|
||||
API_ALBUM = '/album'
|
||||
API_CHAPTER = '/chapter'
|
||||
@ -444,11 +431,17 @@ class JmApiClient(AbstractJmClient):
|
||||
JmModuleConfig.album_class(),
|
||||
)
|
||||
|
||||
def get_photo_detail(self, photo_id, fetch_album=True) -> JmPhotoDetail:
|
||||
def get_photo_detail(self,
|
||||
photo_id,
|
||||
fetch_album=True,
|
||||
fetch_scramble_id=True,
|
||||
) -> JmPhotoDetail:
|
||||
photo: JmPhotoDetail = self.fetch_detail_entity(photo_id,
|
||||
JmModuleConfig.photo_class(),
|
||||
)
|
||||
self.fetch_photo_additional_field(photo, fetch_album)
|
||||
if fetch_album or fetch_scramble_id:
|
||||
self.fetch_photo_additional_field(photo, fetch_album, fetch_scramble_id)
|
||||
|
||||
return photo
|
||||
|
||||
def get_scramble_id(self, photo_id):
|
||||
@ -463,17 +456,16 @@ class JmApiClient(AbstractJmClient):
|
||||
cache[photo_id] = scramble_id
|
||||
return scramble_id
|
||||
|
||||
def fetch_detail_entity(self, apid, clazz, **kwargs):
|
||||
def fetch_detail_entity(self, apid, clazz):
|
||||
"""
|
||||
请求实体类
|
||||
"""
|
||||
apid = JmcomicText.parse_to_album_id(apid)
|
||||
apid = JmcomicText.parse_to_jm_id(apid)
|
||||
url = self.API_ALBUM if issubclass(clazz, JmAlbumDetail) else self.API_CHAPTER
|
||||
resp = self.get_decode(
|
||||
url,
|
||||
params={
|
||||
'id': apid,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
@ -485,7 +477,7 @@ class JmApiClient(AbstractJmClient):
|
||||
"""
|
||||
请求scramble_id
|
||||
"""
|
||||
photo_id: str = JmcomicText.parse_to_photo_id(photo_id)
|
||||
photo_id: str = JmcomicText.parse_to_jm_id(photo_id)
|
||||
resp = self.get_decode(
|
||||
self.API_SCRAMBLE,
|
||||
params={
|
||||
@ -506,45 +498,31 @@ class JmApiClient(AbstractJmClient):
|
||||
|
||||
return scramble_id
|
||||
|
||||
def fetch_photo_additional_field(self, photo: JmPhotoDetail, fetch_album: bool):
|
||||
def fetch_photo_additional_field(self, photo: JmPhotoDetail, fetch_album: bool, fetch_scramble_id: bool):
|
||||
"""
|
||||
获取章节的额外信息
|
||||
1. scramble_id
|
||||
2. album
|
||||
如果都需要获取,会排队,效率低
|
||||
|
||||
这里的难点是,是否要采用异步的方式并发请求。
|
||||
todo: 改进实现
|
||||
1. 直接开两个线程跑
|
||||
2. 开两个线程,但是开之前检查重复性
|
||||
3. 线程池,也要检查重复性
|
||||
23做法要改不止一处地方
|
||||
"""
|
||||
aid = photo.album_id
|
||||
pid = photo.photo_id
|
||||
scramble_cache = JmModuleConfig.SCRAMBLE_CACHE
|
||||
if fetch_album:
|
||||
photo.from_album = self.get_album_detail(photo.photo_id)
|
||||
|
||||
if fetch_album is False and pid in scramble_cache:
|
||||
# 不用发请求,直接返回
|
||||
photo.scramble_id = scramble_cache[pid]
|
||||
return
|
||||
|
||||
if fetch_album is True and pid not in scramble_cache:
|
||||
# 要发起两个请求,这里实现很简易,直接排队请求
|
||||
# todo: 改进实现
|
||||
# 1. 直接开两个线程跑
|
||||
# 2. 开两个线程,但是开之前检查重复性
|
||||
# 3. 线程池,也要检查重复性
|
||||
# 23做法要改不止一处地方
|
||||
photo.from_album = self.get_scramble_id(pid)
|
||||
photo.scramble_id = self.get_album_detail(aid)
|
||||
return
|
||||
|
||||
if fetch_album is True:
|
||||
photo.from_album = self.get_album_detail(aid)
|
||||
else:
|
||||
photo.scramble_id = self.get_scramble_id(pid)
|
||||
if fetch_scramble_id:
|
||||
photo.scramble_id = self.get_scramble_id(photo.album_id)
|
||||
|
||||
def get_decode(self, url, **kwargs) -> JmApiResp:
|
||||
# set headers
|
||||
headers, key_ts = self.headers_key_ts
|
||||
kwargs.setdefault('headers', headers)
|
||||
|
||||
resp = super().get(url, **kwargs)
|
||||
resp = self.get(url, **kwargs)
|
||||
return JmApiResp.wrap(resp, key_ts)
|
||||
|
||||
@property
|
||||
@ -576,5 +554,133 @@ class JmApiClient(AbstractJmClient):
|
||||
# 暂无
|
||||
|
||||
|
||||
class FutureClientProxy(JmcomicClient):
|
||||
"""
|
||||
在Client上做了一层线程池封装来实现异步,对外仍然暴露JmcomicClient的接口,可以看作Client的代理。
|
||||
除了使用线程池做异步,还通过加锁和缓存结果,实现同一个请求不会被多个线程发出,减少开销
|
||||
|
||||
可通过插件 ClientProxyPlugin 启用本类,配置如下:
|
||||
```yml
|
||||
plugin:
|
||||
after_init:
|
||||
- plugin: client_proxy
|
||||
kwargs:
|
||||
proxy_client_key: cl_proxy_future
|
||||
```
|
||||
"""
|
||||
client_key = 'cl_proxy_future'
|
||||
proxy_methods = ['album_comment', 'enable_cache', 'get_domain_list',
|
||||
'get_html_domain', 'get_html_domain_all', 'get_jm_image',
|
||||
'is_cache_enabled', 'set_domain_list', ]
|
||||
|
||||
class FutureWrapper:
|
||||
def __init__(self, future):
|
||||
from concurrent.futures import Future
|
||||
future: Future
|
||||
self.future = future
|
||||
self.done = False
|
||||
self._result = None
|
||||
|
||||
def result(self):
|
||||
if not self.done:
|
||||
result = self.future.result()
|
||||
self._result = result
|
||||
self.done = True
|
||||
self.future = None # help gc
|
||||
|
||||
return self._result
|
||||
|
||||
def __init__(self,
|
||||
client: JmcomicClient,
|
||||
max_workers=None,
|
||||
executors=None,
|
||||
):
|
||||
self.client = client
|
||||
for method in self.proxy_methods:
|
||||
setattr(self, method, getattr(client, method))
|
||||
|
||||
if executors is None:
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
executors = ThreadPoolExecutor(max_workers)
|
||||
|
||||
self.executors = executors
|
||||
self.future_dict: Dict[str, FutureClientProxy.FutureWrapper] = {}
|
||||
from threading import Lock
|
||||
self.lock = Lock()
|
||||
|
||||
def get_album_detail(self, album_id) -> JmAlbumDetail:
|
||||
album_id = JmcomicText.parse_to_jm_id(album_id)
|
||||
cache_key = f'album_{album_id}'
|
||||
future = self.get_future(cache_key, task=lambda: self.client.get_album_detail(album_id))
|
||||
return future.result()
|
||||
|
||||
def get_future(self, cache_key, task):
|
||||
if cache_key in self.future_dict:
|
||||
return self.future_dict[cache_key]
|
||||
|
||||
with self.lock:
|
||||
if cache_key in self.future_dict:
|
||||
return self.future_dict[cache_key]
|
||||
|
||||
future = self.FutureWrapper(self.executors.submit(task))
|
||||
self.future_dict[cache_key] = future
|
||||
return future
|
||||
|
||||
def get_photo_detail(self, photo_id, fetch_album=True, fetch_scramble_id=True) -> JmPhotoDetail:
|
||||
photo_id = JmcomicText.parse_to_jm_id(photo_id)
|
||||
client: JmcomicClient = self.client
|
||||
futures = [None, None, None]
|
||||
results = [None, None, None]
|
||||
|
||||
# photo_detail
|
||||
photo_future = self.get_future(f'photo_{photo_id}',
|
||||
lambda: client.get_photo_detail(photo_id,
|
||||
False,
|
||||
False)
|
||||
)
|
||||
futures[0] = photo_future
|
||||
|
||||
# fetch_album
|
||||
if fetch_album:
|
||||
album_future = self.get_future(f'album_{photo_id}',
|
||||
lambda: client.get_album_detail(photo_id))
|
||||
futures[1] = album_future
|
||||
else:
|
||||
results[1] = None
|
||||
|
||||
# fetch_scramble_id
|
||||
if fetch_scramble_id and isinstance(client, JmApiClient):
|
||||
client: JmApiClient
|
||||
scramble_future = self.get_future(f'scramble_id_{photo_id}',
|
||||
lambda: client.get_scramble_id(photo_id))
|
||||
futures[2] = scramble_future
|
||||
else:
|
||||
results[2] = ''
|
||||
|
||||
# wait finish
|
||||
for i, f in enumerate(futures):
|
||||
if f is None:
|
||||
continue
|
||||
results[i] = f.result()
|
||||
|
||||
# compose
|
||||
photo: JmPhotoDetail = results[0]
|
||||
album = results[1]
|
||||
scramble_id = results[2]
|
||||
|
||||
if album is not None:
|
||||
photo.from_album = album
|
||||
if scramble_id != '':
|
||||
photo.scramble_id = scramble_id
|
||||
|
||||
return photo
|
||||
|
||||
def search(self, search_query: str, page: int, main_tag: int, order_by: str, time: str) -> JmSearchPage:
|
||||
cache_key = f'search_query_{search_query}_page_{page}_main_tag_{main_tag}_order_by_{order_by}_time_{time}'
|
||||
future = self.get_future(cache_key, task=lambda: self.client.search(search_query, page, main_tag, order_by, time))
|
||||
return future.result()
|
||||
|
||||
|
||||
JmModuleConfig.register_client(JmHtmlClient)
|
||||
JmModuleConfig.register_client(JmApiClient)
|
||||
JmModuleConfig.register_client(FutureClientProxy)
|
||||
|
||||
@ -150,7 +150,11 @@ class JmDetailClient:
|
||||
def get_album_detail(self, album_id) -> JmAlbumDetail:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_photo_detail(self, photo_id, fetch_album=True) -> JmPhotoDetail:
|
||||
def get_photo_detail(self,
|
||||
photo_id,
|
||||
fetch_album=True,
|
||||
fetch_scramble_id=True,
|
||||
) -> JmPhotoDetail:
|
||||
raise NotImplementedError
|
||||
|
||||
def of_api_url(self, api_path, domain):
|
||||
@ -234,7 +238,7 @@ class JmImageClient:
|
||||
@param decode_image: 要保存的是解密后的图还是原图
|
||||
"""
|
||||
if scramble_id is None:
|
||||
scramble_id = JmModuleConfig.SCRAMBLE_0
|
||||
scramble_id = JmModuleConfig.SCRAMBLE_220980
|
||||
|
||||
# 请求图片
|
||||
resp = self.get_jm_image(img_url)
|
||||
@ -387,8 +391,8 @@ class JmcomicClient(
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_html_domain(self):
|
||||
return JmModuleConfig.get_html_domain()
|
||||
def get_html_domain(self, postman=None):
|
||||
return JmModuleConfig.get_html_domain(postman or self.get_root_postman())
|
||||
|
||||
def get_html_domain_all(self):
|
||||
return JmModuleConfig.get_html_domain_all()
|
||||
def get_html_domain_all(self, postman=None):
|
||||
return JmModuleConfig.get_html_domain_all(postman or self.get_root_postman())
|
||||
|
||||
@ -47,9 +47,9 @@ class JmModuleConfig:
|
||||
}
|
||||
|
||||
# 图片分隔相关
|
||||
SCRAMBLE_0 = 220980
|
||||
SCRAMBLE_10 = 268850
|
||||
SCRAMBLE_NUM_8 = 421926 # 2023-02-08后改了图片切割算法
|
||||
SCRAMBLE_220980 = 220980
|
||||
SCRAMBLE_268850 = 268850
|
||||
SCRAMBLE_421926 = 421926 # 2023-02-08后改了图片切割算法
|
||||
SCRAMBLE_CACHE = {}
|
||||
|
||||
# 移动端API的相关配置
|
||||
|
||||
@ -82,8 +82,12 @@ class JmImageDetail(JmBaseEntity):
|
||||
query_params=None,
|
||||
index=-1,
|
||||
) -> None:
|
||||
self.aid: str = aid
|
||||
self.scramble_id: str = scramble_id
|
||||
if scramble_id is None or (isinstance(scramble_id, str) and scramble_id == ''):
|
||||
from .jm_toolkit import ExceptionTool
|
||||
ExceptionTool.raises(f'图片的scramble_id不能为空D')
|
||||
|
||||
self.aid: str = str(aid)
|
||||
self.scramble_id: str = str(scramble_id)
|
||||
self.img_url: str = img_url
|
||||
self.img_file_name: str = img_file_name # without suffix
|
||||
self.img_file_suffix: str = img_file_suffix
|
||||
@ -152,11 +156,11 @@ class JmPhotoDetail(DetailEntity):
|
||||
|
||||
def __init__(self,
|
||||
photo_id,
|
||||
scramble_id,
|
||||
name,
|
||||
series_id,
|
||||
sort,
|
||||
tags='',
|
||||
scramble_id='',
|
||||
page_arr=None,
|
||||
data_original_domain=None,
|
||||
data_original_0=None,
|
||||
|
||||
@ -16,7 +16,7 @@ class DirRule:
|
||||
|
||||
Detail = Union[JmAlbumDetail, JmPhotoDetail, None]
|
||||
RuleFunc = Callable[[Detail], str]
|
||||
RuleSolver = Tuple[int, RuleFunc]
|
||||
RuleSolver = Tuple[int, RuleFunc, str]
|
||||
RuleSolverList = List[RuleSolver]
|
||||
|
||||
rule_solver_cache: Dict[str, RuleSolver] = {}
|
||||
@ -37,7 +37,7 @@ class DirRule:
|
||||
ret = self.apply_rule_solver(album, photo, solver)
|
||||
except BaseException as e:
|
||||
# noinspection PyUnboundLocalVariable
|
||||
jm_debug('dir_rule', f'路径规则"{self.rule_dsl}"的解析出错: {e},')
|
||||
jm_debug('dir_rule', f'路径规则"{solver[2]}"的解析出错: {e}, album={album}, photo={photo}')
|
||||
raise e
|
||||
|
||||
path_ls.append(str(ret))
|
||||
@ -52,12 +52,12 @@ class DirRule:
|
||||
if '_' not in rule_dsl and rule_dsl != 'Bd':
|
||||
ExceptionTool.raises(f'不支持的dsl: "{rule_dsl}"')
|
||||
|
||||
rule_ls = rule_dsl.split('_')
|
||||
solver_ls = []
|
||||
rule_list = rule_dsl.split('_')
|
||||
solver_ls: List[DirRule.RuleSolver] = []
|
||||
|
||||
for rule in rule_ls:
|
||||
for rule in rule_list:
|
||||
if rule == 'Bd':
|
||||
solver_ls.append((0, lambda _: base_dir))
|
||||
solver_ls.append((0, lambda _: base_dir, 'Bd'))
|
||||
continue
|
||||
|
||||
rule_solver = self.get_rule_solver(rule)
|
||||
@ -83,7 +83,7 @@ class DirRule:
|
||||
solve_func = lambda detail, ref=rule[1:]: fix_windir_name(str(detail.get_dirname(ref)))
|
||||
|
||||
# 保存缓存
|
||||
rule_solver = (key, solve_func)
|
||||
rule_solver = (key, solve_func, rule)
|
||||
cls.rule_solver_cache[rule] = rule_solver
|
||||
return rule_solver
|
||||
|
||||
@ -106,7 +106,7 @@ class DirRule:
|
||||
if key == 2:
|
||||
return photo
|
||||
|
||||
key, func = rule_solver
|
||||
key, func, _ = rule_solver
|
||||
detail = choose_detail(key)
|
||||
return func(detail)
|
||||
|
||||
@ -301,28 +301,31 @@ class JmOption:
|
||||
def new_jm_client(self, domain_list=None, impl=None, **kwargs) -> JmcomicClient:
|
||||
postman_conf: dict = self.client.postman.src_dict
|
||||
impl = impl or self.client.impl
|
||||
|
||||
# domain_list
|
||||
if domain_list is None:
|
||||
domain_list = self.client.domain
|
||||
domain_list: List[str]
|
||||
|
||||
domain_list: Union[List[str], DictModel]
|
||||
if not isinstance(domain_list, list):
|
||||
domain_list_dict: DictModel = domain_list
|
||||
domain_list = domain_list_dict.get(impl, [])
|
||||
|
||||
if len(domain_list) == 0:
|
||||
domain_list = self.decide_client_domain(impl)
|
||||
|
||||
# support kwargs overwrite meta_data
|
||||
if len(kwargs) != 0:
|
||||
meta_data = postman_conf.get('meta_data', {})
|
||||
meta_data.update(kwargs)
|
||||
postman_conf['meta_data'] = meta_data
|
||||
|
||||
# postman
|
||||
postman = Postmans.create(data=postman_conf)
|
||||
|
||||
# domain_list
|
||||
if len(domain_list) == 0:
|
||||
domain_list = self.decide_client_domain(impl)
|
||||
postman_conf['meta_data'].update(kwargs)
|
||||
|
||||
# headers
|
||||
meta_data = postman_conf['meta_data']
|
||||
if meta_data['headers'] is None:
|
||||
meta_data['headers'] = JmModuleConfig.headers(domain_list[0])
|
||||
|
||||
# postman
|
||||
postman = Postmans.create(data=postman_conf)
|
||||
|
||||
# client
|
||||
client = JmModuleConfig.client_impl_class(impl)(
|
||||
postman,
|
||||
|
||||
@ -327,7 +327,36 @@ class ZipPlugin(JmOptionPlugin):
|
||||
jm_debug('plugin.zip.remove', f'删除文件夹: {d}')
|
||||
|
||||
|
||||
class ClientProxyPlugin(JmOptionPlugin):
|
||||
plugin_key = 'client_proxy'
|
||||
|
||||
def invoke(self,
|
||||
proxy_client_key,
|
||||
whitelist=None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
if whitelist is None:
|
||||
whitelist = set()
|
||||
else:
|
||||
whitelist = set(whitelist)
|
||||
|
||||
clazz = JmModuleConfig.client_impl_class(proxy_client_key)
|
||||
clazz_init_kwargs = kwargs
|
||||
new_jm_client = self.option.new_jm_client
|
||||
|
||||
def hook_new_jm_client(*args, **kwargs):
|
||||
client = new_jm_client(*args, **kwargs)
|
||||
if client.client_key not in whitelist:
|
||||
return client
|
||||
|
||||
jm_debug('plugin.client_proxy', f'proxy client {client} with {clazz}')
|
||||
return clazz(client, **clazz_init_kwargs)
|
||||
|
||||
self.option.new_jm_client = hook_new_jm_client
|
||||
|
||||
|
||||
JmModuleConfig.register_plugin(JmLoginPlugin)
|
||||
JmModuleConfig.register_plugin(UsageLogPlugin)
|
||||
JmModuleConfig.register_plugin(FindUpdatePlugin)
|
||||
JmModuleConfig.register_plugin(ZipPlugin)
|
||||
JmModuleConfig.register_plugin(ClientProxyPlugin)
|
||||
|
||||
@ -57,13 +57,13 @@ class JmcomicText:
|
||||
|
||||
@classmethod
|
||||
def parse_to_jm_domain(cls, text: str):
|
||||
if text.startswith("https"):
|
||||
if text.startswith(JmModuleConfig.PROT):
|
||||
return cls.pattern_jm_domain.search(text)[1]
|
||||
|
||||
return text
|
||||
|
||||
@classmethod
|
||||
def parse_to_photo_id(cls, text) -> str:
|
||||
def parse_to_jm_id(cls, text) -> str:
|
||||
if isinstance(text, int):
|
||||
return str(text)
|
||||
|
||||
@ -88,10 +88,6 @@ class JmcomicText:
|
||||
ExceptionTool.require_true(match is not None, f"无法解析jm车号, 文本为: {text}")
|
||||
return match[2]
|
||||
|
||||
@classmethod
|
||||
def parse_to_album_id(cls, text) -> str:
|
||||
return cls.parse_to_photo_id(text)
|
||||
|
||||
@classmethod
|
||||
def analyse_jm_pub_html(cls, html: str, domain_keyword=('jm', 'comic')) -> List[str]:
|
||||
domain_ls = cls.pattern_html_jm_pub_domain.findall(html)
|
||||
@ -179,14 +175,6 @@ class JmcomicText:
|
||||
|
||||
return clazz(**field_dict)
|
||||
|
||||
@classmethod
|
||||
def format_photo_url(cls, photo_id, domain=None):
|
||||
return cls.format_url(f'/photo/{cls.parse_to_photo_id(photo_id)}', domain)
|
||||
|
||||
@classmethod
|
||||
def format_album_url(cls, album_id, domain=None):
|
||||
return cls.format_url(f'/album/{cls.parse_to_album_id(album_id)}', domain)
|
||||
|
||||
@classmethod
|
||||
def format_url(cls, path, domain):
|
||||
assert isinstance(domain, str) and len(domain) != 0
|
||||
@ -328,7 +316,7 @@ class JmcomicSearchTool:
|
||||
|
||||
class JmApiAdaptTool:
|
||||
"""
|
||||
本类复杂把移动端的api返回值,适配为标准的实体类
|
||||
本类负责把移动端的api返回值,适配为标准的实体类
|
||||
|
||||
# album
|
||||
{
|
||||
@ -463,7 +451,6 @@ class JmApiAdaptTool:
|
||||
break
|
||||
|
||||
fields['sort'] = sort
|
||||
fields['scramble_id'] = '0'
|
||||
import random
|
||||
fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_API_IMAGE_LIST)
|
||||
|
||||
@ -561,11 +548,11 @@ class JmImageTool:
|
||||
|
||||
if aid < scramble_id:
|
||||
return 0
|
||||
elif aid < JmModuleConfig.SCRAMBLE_10:
|
||||
elif aid < JmModuleConfig.SCRAMBLE_268850:
|
||||
return 10
|
||||
else:
|
||||
import hashlib
|
||||
x = 10 if aid < JmModuleConfig.SCRAMBLE_NUM_8 else 8
|
||||
x = 10 if aid < JmModuleConfig.SCRAMBLE_421926 else 8
|
||||
s = f"{aid}{filename}" # 拼接
|
||||
s = s.encode()
|
||||
s = hashlib.md5(s).hexdigest()
|
||||
@ -581,7 +568,7 @@ class JmImageTool:
|
||||
"""
|
||||
return cls.get_num(
|
||||
scramble_id,
|
||||
aid=JmcomicText.parse_to_photo_id(url),
|
||||
aid=JmcomicText.parse_to_jm_id(url),
|
||||
filename=of_file_name(url, True),
|
||||
)
|
||||
|
||||
@ -653,7 +640,7 @@ class ExceptionTool:
|
||||
f'请求的{req_type}不存在!({org_req_url})\n'
|
||||
'原因可能为:\n'
|
||||
f'1. id有误,检查你的{req_type}id\n'
|
||||
'2. 该漫画只对登录用户可见,请配置你的cookies\n'
|
||||
'2. 该漫画只对登录用户可见,请配置你的cookies,或者使用移动端Client(api)\n'
|
||||
), resp)
|
||||
|
||||
@classmethod
|
||||
|
||||
@ -5,7 +5,7 @@ class Test_Client(JmTestConfigurable):
|
||||
|
||||
def test_download_image(self):
|
||||
jm_photo_id = 'JM438516'
|
||||
photo = self.client.get_photo_detail(jm_photo_id, False)
|
||||
photo = self.client.get_photo_detail(jm_photo_id)
|
||||
image = photo[0]
|
||||
filepath = self.option.decide_image_filepath(image)
|
||||
self.client.download_by_image_detail(image, filepath)
|
||||
@ -32,7 +32,7 @@ class Test_Client(JmTestConfigurable):
|
||||
|
||||
def test_gt_300_photo(self):
|
||||
photo_id = '147643'
|
||||
photo: JmPhotoDetail = self.client.get_photo_detail(photo_id, False)
|
||||
photo: JmPhotoDetail = self.client.get_photo_detail(photo_id)
|
||||
image = photo[3000]
|
||||
print(image.img_url)
|
||||
self.client.download_by_image_detail(image, self.option.decide_image_filepath(image))
|
||||
@ -59,7 +59,7 @@ class Test_Client(JmTestConfigurable):
|
||||
|
||||
JmModuleConfig.raise_exception_executor = default_raise_exception_executor
|
||||
ExceptionTool.replace_old_exception_executor(raises)
|
||||
self.assertRaises(B, JmcomicText.parse_to_album_id, 'asdhasjhkd')
|
||||
self.assertRaises(B, JmcomicText.parse_to_jm_id, 'asdhasjhkd')
|
||||
# 还原
|
||||
JmModuleConfig.raise_exception_executor = default_raise_exception_executor
|
||||
|
||||
@ -123,9 +123,13 @@ class Test_Client(JmTestConfigurable):
|
||||
)
|
||||
|
||||
for album, photo_ls in multi_photo_album_dict.items():
|
||||
ls1 = sorted([each.sort for each in album])
|
||||
ls2 = sorted([ans.sort for ans in photo_ls])
|
||||
print(ls1)
|
||||
print(ls2)
|
||||
self.assertListEqual(
|
||||
sorted([each.sort for each in album]),
|
||||
sorted([ans.sort for ans in photo_ls]),
|
||||
ls1,
|
||||
ls2,
|
||||
album.album_id
|
||||
)
|
||||
|
||||
@ -228,3 +232,20 @@ class Test_Client(JmTestConfigurable):
|
||||
for photo in album[0:3]:
|
||||
photo = client.get_photo_detail(photo.photo_id)
|
||||
print(photo.id, photo.name)
|
||||
|
||||
def test_cache_result_equal(self):
|
||||
cl = self.client
|
||||
cases = [
|
||||
(123, False, False),
|
||||
(123,),
|
||||
(123, False, True),
|
||||
(123, True, False),
|
||||
]
|
||||
|
||||
ans = None
|
||||
for args in cases:
|
||||
photo = cl.get_photo_detail(*args)
|
||||
if ans is None:
|
||||
ans = id(photo)
|
||||
else:
|
||||
self.assertEqual(ans, id(photo))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user