v2.3.12: 增强搜索功能,支持获取搜索结果的总页数,支持基于生成器调用搜索API (#154)

This commit is contained in:
hect0x7 2023-10-26 00:16:31 +08:00 committed by GitHub
parent a0e8af9fe5
commit f4493cb5cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 238 additions and 44 deletions

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.3.11'
__version__ = '2.3.12'
from .api import *
from .jm_plugin import *

View File

@ -210,14 +210,14 @@ class JmHtmlClient(AbstractJmClient):
page: int,
main_tag: int,
order_by: str,
date: str,
time: str,
) -> JmSearchPage:
params = {
'main_tag': main_tag,
'search_query': search_query,
'page': page,
'o': order_by,
't': date,
't': time,
}
resp = self.get_jm_html(
@ -401,16 +401,15 @@ class JmApiClient(AbstractJmClient):
order_by: str,
time: str,
) -> JmSearchPage:
resp = self.get_decode(
self.API_SEARCH,
params={
'search_query': search_query,
'main_tag': main_tag,
'page': page,
'o': order_by,
't': time,
}
)
params = {
'main_tag': main_tag,
'search_query': search_query,
'page': page,
'o': order_by,
't': time,
}
resp = self.get_decode(self.append_params_to_url(self.API_SEARCH, params))
# 直接搜索禁漫车号,发生重定向的响应数据 resp.model_data
# {
@ -424,7 +423,7 @@ class JmApiClient(AbstractJmClient):
aid = data.redirect_aid
return JmSearchPage.wrap_single_album(self.get_album_detail(aid))
return JmcomicSearchTool.parse_api_resp_to_page(data)
return JmSearchTool.parse_api_resp_to_page(data)
def get_album_detail(self, album_id) -> JmAlbumDetail:
return self.fetch_detail_entity(album_id,
@ -517,6 +516,64 @@ class JmApiClient(AbstractJmClient):
if fetch_scramble_id:
photo.scramble_id = self.get_scramble_id(photo.album_id)
def setting(self) -> JmApiResp:
"""
禁漫app的setting请求返回如下内容resp.res_data
{
"logo_path": "https://cdn-msp.jmapiproxy1.monster/media/logo/new_logo.png",
"main_web_host": "18-comic.work",
"img_host": "https://cdn-msp.jmapiproxy1.monster",
"base_url": "https://www.jmapinode.biz",
"is_cn": 0,
"cn_base_url": "https://www.jmapinode.biz",
"version": "1.6.0",
"test_version": "1.6.1",
"store_link": "https://play.google.com/store/apps/details?id=com.jiaohua_browser",
"ios_version": "1.6.0",
"ios_test_version": "1.6.1",
"ios_store_link": "https://18comic.vip/stray/",
"ad_cache_version": 1698140798,
"bundle_url": "https://18-comic.work/static/apk/patches1.6.0.zip",
"is_hot_update": true,
"api_banner_path": "https://cdn-msp.jmapiproxy1.monster/media/logo/channel_log.png?v=",
"version_info": "\nAPP & IOS更新\nV1.6.0\n#禁漫 APK 更新拉!!\n更新調整以下項目\n1. 系統優化\n\nV1.5.9\n1. 跳錯誤新增 重試 網頁 按鈕\n2. 圖片讀取優化\n3.
線路調整優化\n\n無法順利更新或是系統題是有風險請使用下方\n下載點2\n有問題可以到DC群反饋\nhttps://discord.gg/V74p7HM\n",
"app_shunts": [
{
"title": "圖源1",
"key": 1
},
{
"title": "圖源2",
"key": 2
},
{
"title": "圖源3",
"key": 3
},
{
"title": "圖源4",
"key": 4
}
],
"download_url": "https://18-comic.work/static/apk/1.6.0.apk",
"app_landing_page": "https://jm365.work/pXYbfA",
"float_ad": true
}
"""
resp = self.get_decode('/setting')
return resp
def login(self,
username,
password,
refresh_client_cookies=True,
id_remember='on',
login_remember='on',
):
jm_debug('api.login', '禁漫移动端无需登录调用login不会做任何操作')
pass
def get_decode(self, url, **kwargs) -> JmApiResp:
# set headers
headers, key_ts = self.headers_key_ts

View File

@ -371,6 +371,69 @@ class JmSearchAlbumClient:
"""
return self.search(search_query, page, 4, order_by, time)
def search_gen(self,
search_query: str,
main_tag=0,
page: int = 1,
order_by: str = ORDER_BY_LATEST,
time: str = TIME_ALL,
):
"""
搜索结果的生成器支持下面这种调用方式
```
for page in self.search_gen('无修正'):
# 每次循环page为新页的结果
pass
```
同时支持外界send参数可以改变搜索的设定例如
```
gen = client.search_gen('MANA')
for i, page in enumerate(gen):
print(page.page_count)
page = gen.send({
'search_query': '+MANA +无修正',
'page': 1
})
print(page.page_count)
break
```
"""
params = {
'search_query': search_query,
'main_tag': main_tag,
'order_by': order_by,
'time': time,
}
def search(page):
params['page'] = page
return self.search(**params)
from math import inf
def update(value: Union[Dict], page: int, search_page: JmSearchPage):
if value is None:
return page + 1, search_page.page_count
ExceptionTool.require_true(isinstance(value, dict), 'require dict params')
# 根据外界传递的参数更新params和page
page = value.get('page', page)
params.update(value)
return page, inf
total = inf
while page <= total:
search_page = search(page)
value = yield search_page
page, total = update(value, page, search_page)
# noinspection PyAbstractClass
class JmcomicClient(

View File

@ -26,6 +26,11 @@ def system_proxy():
return ProxyBuilder.system_proxy()
def str_to_list(text):
from common import str_to_list
return str_to_list(text)
class JmcomicException(Exception):
pass
@ -63,9 +68,25 @@ class JmModuleConfig:
# 域名配置 - 移动端
# 图片域名
DOMAIN_API_IMAGE_LIST = [f"cdn-msp.jmapiproxy{i}.cc" for i in range(1, 4)]
DOMAIN_API_IMAGE_LIST = str_to_list('''
cdn-msp.jmapiproxy1.monster
cdn-msp2.jmapiproxy1.monster
cdn-msp.jmapiproxy1.cc
cdn-msp.jmapiproxy2.cc
cdn-msp.jmapiproxy3.cc
cdn-msp.jmapiproxy4.cc
''')
# API域名
DOMAIN_API_LIST = [f'www.jmapinode{i}.top' for i in range(1, 4)]
DOMAIN_API_LIST = str_to_list('''
www.jmapinode1.top
www.jmapinode2.top
www.jmapinode3.top
www.jmapinode.biz
www.jmapinode.top
''')
# 域名配置 - 网页端
# 无需配置默认为None需要的时候会发起请求获得

View File

@ -425,11 +425,17 @@ class JmAlbumDetail(DetailEntity):
class JmSearchPage(JmBaseEntity, IndexedEntity):
ContentItem = Tuple[str, Dict[str, Any]]
def __init__(self, content: List[ContentItem]):
# [
# album_id, {title, tag_list, ...}
# ]
def __init__(self, content: List[ContentItem], page_count):
"""
[
album_id, {title, tag_list, ...}
]
:param content: 搜索结果移动端和网页端都一次返回80个
:param page_count: 总页数登录和不登录能看到的总页数不一样
"""
self.content = content
self.page_count = page_count
def iter_id(self) -> Generator[str, None, None]:
"""
@ -469,7 +475,7 @@ class JmSearchPage(JmBaseEntity, IndexedEntity):
'name': album.name,
'tag_list': album.tags,
}
)])
)], -1)
setattr(page, 'album', album)
return page

View File

@ -115,7 +115,7 @@ class JmcomicText:
@classmethod
def analyse_jm_search_html(cls, html: str) -> JmSearchPage:
return JmcomicSearchTool.parse_html_to_page(html)
return JmSearchTool.parse_html_to_page(html)
@classmethod
def reflect_new_instance(cls, html: str, cls_field_prefix: str, clazz: type):
@ -217,7 +217,39 @@ class JmcomicText:
JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env)
class JmcomicSearchTool:
class PatternTool:
@classmethod
def match_or_default(cls, html: str, pattern: Pattern, default):
match = pattern.search(html)
return default if match is None else match[1]
@classmethod
def require_match(cls, html: str, pattern: Pattern, msg, rindex=1):
match = pattern.search(html)
if match is not None:
return match[rindex]
ExceptionTool.raises_regex(
msg,
html=html,
pattern=pattern,
)
@classmethod
def require_not_match(cls, html: str, pattern: Pattern, *, msg_func):
match = pattern.search(html)
if match is None:
return
ExceptionTool.raises_regex(
msg_func(match),
html=html,
pattern=pattern,
)
class JmSearchTool:
# 用来缩减html的长度
pattern_html_search_shorten_for = compile(r'<div class="well well-sm">([\s\S]*)<div class="row">')
@ -238,30 +270,31 @@ class JmcomicSearchTool:
# 查找错误,例如 [错误,關鍵字過短,請至少輸入兩個字以上。]
pattern_html_search_error = compile(r'<fieldset>\n<legend>(.*?)</legend>\n<div class=.*?>\n(.*?)\n</div>\n</fieldset>')
pattern_html_search_total_count = compile(r'<span class="text-white">(\d+)</span> A漫.'), 0
@classmethod
def parse_html_to_page(cls, html: str) -> JmSearchPage:
# 检查是否失败
match = cls.pattern_html_search_error.search(html)
if match is not None:
topic, reason = match[1], match[2]
ExceptionTool.raises_regex(
f'{topic}: {reason}',
html=html,
pattern=cls.pattern_html_search_error,
)
# 1. 检查是否失败
PatternTool.require_not_match(
html,
cls.pattern_html_search_error,
msg_func=lambda match: '{}: {}'.format(match[1], match[2])
)
# 缩小文本范围
match = cls.pattern_html_search_shorten_for.search(html)
if match is None:
ExceptionTool.raises_regex(
'未匹配到搜索结果',
html=html,
pattern=cls.pattern_html_search_shorten_for,
)
html = match[0]
# 2. 缩小文本范围
html = PatternTool.require_match(
html,
cls.pattern_html_search_shorten_for,
msg='未匹配到搜索结果',
)
# 3. 提取结果
import math
# 提取结果
content = [] # content这个名字来源于api版搜索返回值
total_count = PatternTool.match_or_default(html, *cls.pattern_html_search_total_count) # 总结果数
page_count = math.ceil(int(total_count) / 80)
album_info_list = cls.pattern_html_search_album_info_list.findall(html)
for (album_id, title, _, label_category, label_sub, tag_text) in album_info_list:
@ -273,7 +306,7 @@ class JmcomicSearchTool:
}
))
return JmSearchPage(content)
return JmSearchPage(content, page_count)
@classmethod
def parse_api_resp_to_page(cls, data: DictModel) -> JmSearchPage:
@ -300,6 +333,7 @@ class JmcomicSearchTool:
]
}
"""
total: int = int(data.total)
def adapt_item(item: DictModel):
item: dict = item.src_dict
@ -311,7 +345,7 @@ class JmcomicSearchTool:
for item in data.content
]
return JmSearchPage(content)
return JmSearchPage(content, total)
class JmApiAdaptTool:

View File

@ -251,3 +251,16 @@ class Test_Client(JmTestConfigurable):
ans = id(photo)
else:
self.assertEqual(ans, id(photo))
def test_search_generator(self):
JmModuleConfig.decode_url_when_debug = False
gen = self.client.search_gen('MANA')
for i, page in enumerate(gen):
print(page.page_count)
page = gen.send({
'search_query': 'MANA +无修正',
'page': 1
})
print(page.page_count)
break