v2.2.8: 优化统一禁漫网页端和移动端的搜索返回类,增加移动端的测试,跟进文档 (#132)

This commit is contained in:
hect0x7 2023-09-13 17:14:26 +08:00 committed by GitHub
parent e0cc0b0d40
commit 4646146011
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 233 additions and 133 deletions

View File

@ -2,7 +2,7 @@ name: 跑测试
on:
schedule:
- cron: "0 8 * * *"
- cron: "0 0 * * *"
workflow_dispatch:
push:
branches: [ "dev" ]

View File

@ -72,8 +72,8 @@ jmcomic.download_album('422866') # 传入要下载的album的id即可下载
- **支持Plugin插件可以方便地扩展功能以及使用别人的插件**
- 目前内置支持的插件有:`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件`
- 支持自定义本子/章节/图片下载前后的回调函数
- 支持自定义debug日志的开关/格式
- 支持自定义Downloader/Option/Client/实体类
- 支持自定义debug日志
- 支持自定义类:Downloader(负责调度)/Option(负责配置)/Client(负责请求)/实体类 等等
- ......
- 支持**自动重试和域名切换**机制
- **多线程下载**(可细化到一图一线程,效率极高)

View File

@ -3,3 +3,4 @@ curl_cffi
PyYAML
Pillow
psutil
pycryptodome

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.2.7'
__version__ = '2.2.8'
from .api import *
from .jm_plugin import *

View File

@ -225,7 +225,7 @@ class JmHtmlClient(AbstractJmClient):
album = JmcomicText.analyse_jm_album_html(resp.text)
return JmSearchPage.wrap_single_album(album)
else:
return JmSearchSupport.analyse_jm_search_html(resp.text)
return JmcomicText.analyse_jm_search_html(resp.text)
# -- 帐号管理 --
@ -405,31 +405,8 @@ class JmApiClient(AbstractJmClient):
main_tag: int,
order_by: str,
time: str,
) -> JmApiResp:
"""
model_data: {
"search_query": "MANA",
"total": "177",
"content": [
{
"id": "441923",
"author": "MANA",
"description": "",
"name": "[MANA] 神里绫华5",
"image": "",
"category": {
"id": "1",
"title": "同人"
},
"category_sub": {
"id": "1",
"title": "同人"
}
}
]
}
"""
return self.get(
) -> JmSearchPage:
resp = self.get(
self.API_SEARCH,
params={
'search_query': search_query,
@ -440,6 +417,20 @@ class JmApiClient(AbstractJmClient):
}
)
# 直接搜索禁漫车号,发生重定向的响应数据 resp.model_data
# {
# "search_query": "310311",
# "total": 1,
# "redirect_aid": "310311",
# "content": []
# }
data = resp.model_data
if data.get('redirect_aid', None) is not None:
aid = data.redirect_aid
return JmSearchPage.wrap_single_album(self.get_album_detail(aid))
return JmcomicSearchTool.parse_api_resp_to_page(data)
def get(self, url, **kwargs) -> JmApiResp:
# set headers
headers, key_ts = self.headers_key_ts
@ -464,17 +455,5 @@ class JmApiClient(AbstractJmClient):
return 'api'
class AsyncSaveImageClient(JmImageClient):
def __init__(self, workers=None) -> None:
from concurrent.futures import ThreadPoolExecutor, Future
self.executor = ThreadPoolExecutor(max_workers=workers)
self.future_list: List[Future] = []
def save_image_resp(self, *args, **kwargs):
future = self.executor.submit(lambda: super().save_image_resp(*args, **kwargs))
self.future_list.append(future)
JmModuleConfig.register_client(JmHtmlClient)
JmModuleConfig.register_client(JmApiClient)

View File

@ -6,6 +6,7 @@ Response Entity
"""
DictModel = AdvancedEasyAccessDict
class JmResp(CommonResp):
@ -19,6 +20,9 @@ class JmResp(CommonResp):
def model(self) -> DictModel:
return DictModel(self.json())
def require_success(self):
if self.is_not_success:
raise JmModuleConfig.exception(self.resp.text)
class JmImageResp(JmResp):
@ -312,7 +316,7 @@ class JmSearchAlbumClient:
page: int = 1,
order_by: str = ORDER_BY_LATEST,
time: str = TIME_ALL,
) -> JmSearchPage:
):
"""
对应禁漫的站内搜索
"""
@ -323,7 +327,7 @@ class JmSearchAlbumClient:
page: int = 1,
order_by: str = ORDER_BY_LATEST,
time: str = TIME_ALL,
) -> JmSearchPage:
):
"""
搜索album的作品 work
"""
@ -334,7 +338,7 @@ class JmSearchAlbumClient:
page: int = 1,
order_by: str = ORDER_BY_LATEST,
time: str = TIME_ALL,
) -> JmSearchPage:
):
"""
搜索album的作者 author
"""
@ -345,7 +349,7 @@ class JmSearchAlbumClient:
page: int = 1,
order_by: str = ORDER_BY_LATEST,
time: str = TIME_ALL,
) -> JmSearchPage:
):
"""
搜索album的标签 tag
"""
@ -356,7 +360,7 @@ class JmSearchAlbumClient:
page: int = 1,
order_by: str = ORDER_BY_LATEST,
time: str = TIME_ALL,
) -> JmSearchPage:
):
"""
搜索album的登场角色 actor
"""

View File

@ -5,49 +5,19 @@ from .jm_config import *
class JmBaseEntity:
@staticmethod
def fix_title(title: str, limit=50):
"""
一些过长的标题可能含有 \n例如album: 360537
该方法会把 \n 去除
"""
if len(title) > limit and '\n' in title:
title = title.replace('\n', '')
return title.strip()
def save_to_file(self, filepath):
from common import PackerUtil
PackerUtil.pack(self, filepath)
class DetailEntity(JmBaseEntity):
@property
def id(self) -> str:
raise NotImplementedError
@property
def name(self) -> str:
return getattr(self, 'title')
# help for typing
JMPI = Union['JmPhotoDetail', 'JmImageDetail']
def getindex(self, index: int) -> JMPI:
class IndexedEntity:
def getindex(self, index: int):
raise NotImplementedError
def __len__(self):
raise NotImplementedError
def __iter__(self) -> Generator[JMPI, Any, None]:
for index in range(len(self)):
yield self.getindex(index)
def __str__(self):
return f'{self.__class__.__name__}({self.id}-{self.name})'
def __getitem__(self, item) -> Union[JMPI, List[JMPI]]:
def __getitem__(self, item) -> Any:
if isinstance(item, slice):
start = item.start or 0
stop = item.stop or len(self)
@ -60,6 +30,24 @@ class DetailEntity(JmBaseEntity):
else:
raise TypeError(f"Invalid item type for {self.__class__}")
def __iter__(self):
for index in range(len(self)):
yield self.getindex(index)
class DetailEntity(JmBaseEntity, IndexedEntity):
@property
def id(self) -> str:
raise NotImplementedError
@property
def name(self) -> str:
return getattr(self, 'title')
def __str__(self):
return f'{self.__class__.__name__}({self.id}-{self.name})'
@classmethod
def __alias__(cls):
# "JmAlbumDetail" -> "album" (本子)
@ -163,7 +151,7 @@ class JmPhotoDetail(DetailEntity):
):
self.photo_id: str = photo_id
self.scramble_id: str = scramble_id
self.title: str = self.fix_title(str(title))
self.title: str = str(title).strip()
self.sort: int = int(sort)
self._keywords: str = keywords
self._series_id: int = int(series_id)
@ -290,7 +278,7 @@ class JmPhotoDetail(DetailEntity):
def __len__(self):
return len(self.page_arr)
def __iter__(self) -> Generator[JmImageDetail, Any, None]:
def __iter__(self) -> Generator[JmImageDetail, None, None]:
return super().__iter__()
@ -418,21 +406,45 @@ class JmAlbumDetail(DetailEntity):
def __len__(self):
return len(self.episode_list)
def __iter__(self) -> Generator[JmPhotoDetail, Any, None]:
def __iter__(self) -> Generator[JmPhotoDetail, None, None]:
return super().__iter__()
class JmSearchPage(JmBaseEntity):
class JmSearchPage(JmBaseEntity, IndexedEntity):
ContentItem = Tuple[str, Dict[str, Any]]
def __init__(self, album_info_list: List[Tuple[str, str, StrNone, StrNone, List[str]]]):
# (album_id, title, category_none, label_sub_none, tag_list)
self.album_info_list = album_info_list
def __init__(self, content: List[ContentItem]):
# [
# album_id, {title, tag_list, ...}
# ]
self.content = content
def __len__(self):
return len(self.album_info_list)
def iter_id(self) -> Generator[str, None, None]:
"""
返回 album_id 的迭代器
"""
for aid, ainfo in self.content:
yield aid
def __getitem__(self, item) -> Tuple[str, str]:
return self.album_info_list[item][0:2]
def iter_id_title(self) -> Generator[Tuple[str, str], None, None]:
"""
返回 album_id, album_title 的迭代器
"""
for aid, ainfo in self.content:
yield aid, ainfo['name']
def iter_id_title_tag(self) -> Generator[Tuple[str, str, List[str]], None, None]:
"""
返回 album_id, album_title, album_tag_list 的迭代器
"""
for aid, ainfo in self.content:
yield aid, ainfo['name'], ainfo['tag_list']
# 下面的方法是对单个album的包装
@property
def is_single_album(self):
return hasattr(self, 'album')
@property
def single_album(self) -> JmAlbumDetail:
@ -440,17 +452,25 @@ class JmSearchPage(JmBaseEntity):
@classmethod
def wrap_single_album(cls, album: JmAlbumDetail) -> 'JmSearchPage':
# ('462257', '[無邪気漢化組] [きょくちょ] 楓と鈴 4.5', '短篇', '漢化', [])
# (album_id, title, category_none, label_sub_none, tag_list)
album_info = (
album.album_id,
album.title,
None,
None,
album.tag_list,
)
obj = JmSearchPage([album_info])
obj = JmSearchPage([(
album.album_id, {
'name': album.title,
'tag_list': album.tag_list,
}
)])
setattr(obj, 'album', album)
return obj
# 下面的方法实现方便的元素访问
def __len__(self):
return len(self.content)
def __iter__(self):
return self.iter_id_title()
def __getitem__(self, item) -> Union[ContentItem, List[ContentItem]]:
return super().__getitem__(item)
def getindex(self, index: int):
return self.content[index]

View File

@ -130,11 +130,11 @@ class JmOption:
# 路径规则配置
self.dir_rule = DirRule(**dir_rule)
# 请求配置
self.client = DictModel(client)
self.client = AdvancedEasyAccessDict(client)
# 下载配置
self.download = DictModel(download)
self.download = AdvancedEasyAccessDict(download)
# 插件配置
self.plugin = DictModel(plugin)
self.plugin = AdvancedEasyAccessDict(plugin)
# 其他配置
self.filepath = filepath

View File

@ -55,11 +55,26 @@ class UsageLogPlugin(JmOptionPlugin):
def invoke(self, **kwargs) -> None:
import threading
threading.Thread(
t = threading.Thread(
target=self.monitor_resource_usage,
kwargs=kwargs,
daemon=True,
).start()
)
t.start()
self.set_thread_as_option_attr(t)
def set_thread_as_option_attr(self, t):
"""
线程留痕
"""
name = f'thread_{self.plugin_key}'
thread_ls: Optional[list] = getattr(self.option, name, None)
if thread_ls is None:
setattr(self.option, name, [t])
else:
thread_ls.append(t)
def monitor_resource_usage(
self,

View File

@ -125,6 +125,10 @@ class JmcomicText:
JmModuleConfig.album_class()
)
@classmethod
def analyse_jm_search_html(cls, html: str) -> JmSearchPage:
return JmcomicSearchTool.parse_html_to_page(html)
@classmethod
def reflect_new_instance(cls, html: str, cls_field_prefix: str, clazz: type):
@ -185,6 +189,10 @@ class JmcomicText:
@classmethod
def format_url(cls, path, domain):
assert isinstance(domain, str) and len(domain) != 0
if domain.startswith(JmModuleConfig.PROT):
return f'{domain}{path}'
return f'{JmModuleConfig.PROT}{domain}{path}'
class DSLReplacer:
@ -220,7 +228,7 @@ class JmcomicText:
JmcomicText.dsl_replacer.add_dsl_and_replacer('\$\{(.*?)\}', JmcomicText.match_os_env)
class JmSearchSupport:
class JmcomicSearchTool:
# 用来缩减html的长度
pattern_html_search_shorten_for = compile('<div class="well well-sm">([\s\S]*)<div class="row">')
@ -242,7 +250,7 @@ class JmSearchSupport:
pattern_html_search_error = compile('<fieldset>\n<legend>(.*?)</legend>\n<div class=.*?>\n(.*?)\n</div>\n</fieldset>')
@classmethod
def analyse_jm_search_html(cls, html: str) -> JmSearchPage:
def parse_html_to_page(cls, html: str) -> JmSearchPage:
# 检查是否失败
match = cls.pattern_html_search_error.search(html)
if match is not None:
@ -256,14 +264,57 @@ class JmSearchSupport:
html = match[0]
# 提取结果
content = [] # content这个名字来源于是api版搜索返回值
album_info_list = cls.pattern_html_search_album_info_list.findall(html)
for i, (album_id, title, *args) in enumerate(album_info_list):
_, category_none, label_sub_none, tag_text = args
for (album_id, title, _, label_category, label_sub, tag_text) in album_info_list:
tag_list = cls.pattern_html_search_tag_list.findall(tag_text)
album_info_list[i] = (album_id, title, category_none, label_sub_none, tag_list)
content.append((
album_id, {
'name': title, # 改成name是为了兼容 parse_api_resp_to_page
'tag_list': tag_list
}
))
return JmSearchPage(album_info_list)
return JmSearchPage(content)
@classmethod
def parse_api_resp_to_page(cls, data: DictModel) -> JmSearchPage:
"""
model_data: {
"search_query": "MANA",
"total": "177",
"content": [
{
"id": "441923",
"author": "MANA",
"description": "",
"name": "[MANA] 神里绫华5",
"image": "",
"category": {
"id": "1",
"title": "同人"
},
"category_sub": {
"id": "1",
"title": "同人"
}
}
]
}
"""
def adapt_item(item: DictModel):
item: dict = item.src_dict
item.setdefault('tag_list', [])
return item
content = [
(item.id, adapt_item(item))
for item in data.content
]
return JmSearchPage(content)
class JmImageSupport:

View File

@ -12,12 +12,16 @@ class Test_Client(JmTestConfigurable):
def test_fetch_album(self):
album_id = "JM438516"
self.client.get_album_detail(album_id)
self.client.get_photo_detail(album_id)
def test_search(self):
jm_search_page: JmSearchPage = self.client.search_tag('+无修正 +中文 -全彩')
for album_id, title in reversed(jm_search_page):
print(album_id, title)
page: JmSearchPage = self.client.search_tag('+无修正 +中文 -全彩')
for album_id, title, tag_list in page.iter_id_title_tag():
print(album_id, title, tag_list)
aid = '438516'
page = self.client.search_site(aid)
search_aid, ainfo = page[0]
self.assertEqual(search_aid, aid)
def test_gt_300_photo(self):
photo_id = '147643'

View File

@ -0,0 +1,27 @@
from test_jmcomic import *
# 移动端专用的禁漫域名
domain_list = [
"https://www.jmapinode1.cc",
"https://www.jmapinode2.cc",
"https://www.jmapinode3.cc",
"https://www.jmapibranch2.cc"
]
class Test_MobileClient(JmTestConfigurable):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.client = cls.option.new_jm_client(domain_list, impl='api')
def test_search(self):
page = self.client.search_site('MANA')
if len(page) >= 1:
for aid, ainfo in page[0:1:1]:
print(aid, ainfo['description'], ainfo['category'])
for aid, atitle, tag_list in page.iter_id_title_tag():
print(aid, atitle, tag_list)

View File

@ -23,8 +23,7 @@ def get_domain_ls():
domain_set: Set[str] = set()
def fetch_domain(url):
# from curl_cffi import requests as postman
postman = CurlCffiPostman.create()
from curl_cffi import requests as postman
text = postman.get(url, allow_redirects=False, **meta_data).text
for domain in JmcomicText.analyse_jm_pub_html(text):
if domain.startswith('jm365.work'):

View File

@ -52,13 +52,14 @@ def get_album_photo_detail():
@timeit('搜索本子: ')
def search_jm_album():
# 分页查询search_site就是禁漫网页上的【站内搜索】
search_page: JmSearchPage = client.search_site(search_query='+MANA +无修正', page=1)
for album_id, title in search_page:
page: JmSearchPage = client.search_site(search_query='+MANA +无修正', page=1)
# page默认的迭代方式是page.iter_id_title(),每次迭代返回 albun_id, title
for album_id, title in page:
print(f'[{album_id}]: {title}')
# 直接搜索禁漫车号
search_page = client.search_site(search_query='427413')
album: JmAlbumDetail = search_page.single_album
page = client.search_site(search_query='427413')
album: JmAlbumDetail = page.single_album
print(album.keywords)
@ -67,17 +68,16 @@ def search_and_download():
tag = '無修正'
# 搜索标签可以使用search_tag。
# 搜索第一页。
search_page: JmSearchPage = client.search_tag(tag, page=1)
page: JmSearchPage = client.search_tag(tag, page=1)
id_list = []
aid_list = []
for arg in search_page.album_info_list:
(album_id, title, category_none, label_sub_none, tag_list) = arg
for aid, atitle, tag_list in page.iter_id_title_tag(): # 使用page的iter_id_title_tag迭代器
if tag in tag_list:
print(f'[标签/{tag}] 发现目标: [{album_id}]: [{title}]')
id_list.append(album_id)
print(f'[标签/{tag}] 发现目标: [{aid}]: [{atitle}]')
aid_list.append(aid)
download_album(id_list, option)
download_album(aid_list, option)
def main():