mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v2.3.12: 增强搜索功能,支持获取搜索结果的总页数,支持基于生成器调用搜索API (#154)
This commit is contained in:
parent
a0e8af9fe5
commit
f4493cb5cd
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.3.11'
|
||||
__version__ = '2.3.12'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
||||
@ -210,14 +210,14 @@ class JmHtmlClient(AbstractJmClient):
|
||||
page: int,
|
||||
main_tag: int,
|
||||
order_by: str,
|
||||
date: str,
|
||||
time: str,
|
||||
) -> JmSearchPage:
|
||||
params = {
|
||||
'main_tag': main_tag,
|
||||
'search_query': search_query,
|
||||
'page': page,
|
||||
'o': order_by,
|
||||
't': date,
|
||||
't': time,
|
||||
}
|
||||
|
||||
resp = self.get_jm_html(
|
||||
@ -401,16 +401,15 @@ class JmApiClient(AbstractJmClient):
|
||||
order_by: str,
|
||||
time: str,
|
||||
) -> JmSearchPage:
|
||||
resp = self.get_decode(
|
||||
self.API_SEARCH,
|
||||
params={
|
||||
'search_query': search_query,
|
||||
'main_tag': main_tag,
|
||||
'page': page,
|
||||
'o': order_by,
|
||||
't': time,
|
||||
}
|
||||
)
|
||||
params = {
|
||||
'main_tag': main_tag,
|
||||
'search_query': search_query,
|
||||
'page': page,
|
||||
'o': order_by,
|
||||
't': time,
|
||||
}
|
||||
|
||||
resp = self.get_decode(self.append_params_to_url(self.API_SEARCH, params))
|
||||
|
||||
# 直接搜索禁漫车号,发生重定向的响应数据 resp.model_data
|
||||
# {
|
||||
@ -424,7 +423,7 @@ class JmApiClient(AbstractJmClient):
|
||||
aid = data.redirect_aid
|
||||
return JmSearchPage.wrap_single_album(self.get_album_detail(aid))
|
||||
|
||||
return JmcomicSearchTool.parse_api_resp_to_page(data)
|
||||
return JmSearchTool.parse_api_resp_to_page(data)
|
||||
|
||||
def get_album_detail(self, album_id) -> JmAlbumDetail:
|
||||
return self.fetch_detail_entity(album_id,
|
||||
@ -517,6 +516,64 @@ class JmApiClient(AbstractJmClient):
|
||||
if fetch_scramble_id:
|
||||
photo.scramble_id = self.get_scramble_id(photo.album_id)
|
||||
|
||||
def setting(self) -> JmApiResp:
|
||||
"""
|
||||
禁漫app的setting请求,返回如下内容(resp.res_data)
|
||||
{
|
||||
"logo_path": "https://cdn-msp.jmapiproxy1.monster/media/logo/new_logo.png",
|
||||
"main_web_host": "18-comic.work",
|
||||
"img_host": "https://cdn-msp.jmapiproxy1.monster",
|
||||
"base_url": "https://www.jmapinode.biz",
|
||||
"is_cn": 0,
|
||||
"cn_base_url": "https://www.jmapinode.biz",
|
||||
"version": "1.6.0",
|
||||
"test_version": "1.6.1",
|
||||
"store_link": "https://play.google.com/store/apps/details?id=com.jiaohua_browser",
|
||||
"ios_version": "1.6.0",
|
||||
"ios_test_version": "1.6.1",
|
||||
"ios_store_link": "https://18comic.vip/stray/",
|
||||
"ad_cache_version": 1698140798,
|
||||
"bundle_url": "https://18-comic.work/static/apk/patches1.6.0.zip",
|
||||
"is_hot_update": true,
|
||||
"api_banner_path": "https://cdn-msp.jmapiproxy1.monster/media/logo/channel_log.png?v=",
|
||||
"version_info": "\nAPP & IOS更新\nV1.6.0\n#禁漫 APK 更新拉!!\n更新調整以下項目\n1. 系統優化\n\nV1.5.9\n1. 跳錯誤新增 重試 網頁 按鈕\n2. 圖片讀取優化\n3.
|
||||
線路調整優化\n\n無法順利更新或是系統題是有風險請使用下方\n下載點2\n有問題可以到DC群反饋\nhttps://discord.gg/V74p7HM\n",
|
||||
"app_shunts": [
|
||||
{
|
||||
"title": "圖源1",
|
||||
"key": 1
|
||||
},
|
||||
{
|
||||
"title": "圖源2",
|
||||
"key": 2
|
||||
},
|
||||
{
|
||||
"title": "圖源3",
|
||||
"key": 3
|
||||
},
|
||||
{
|
||||
"title": "圖源4",
|
||||
"key": 4
|
||||
}
|
||||
],
|
||||
"download_url": "https://18-comic.work/static/apk/1.6.0.apk",
|
||||
"app_landing_page": "https://jm365.work/pXYbfA",
|
||||
"float_ad": true
|
||||
}
|
||||
"""
|
||||
resp = self.get_decode('/setting')
|
||||
return resp
|
||||
|
||||
def login(self,
|
||||
username,
|
||||
password,
|
||||
refresh_client_cookies=True,
|
||||
id_remember='on',
|
||||
login_remember='on',
|
||||
):
|
||||
jm_debug('api.login', '禁漫移动端无需登录,调用login不会做任何操作')
|
||||
pass
|
||||
|
||||
def get_decode(self, url, **kwargs) -> JmApiResp:
|
||||
# set headers
|
||||
headers, key_ts = self.headers_key_ts
|
||||
|
||||
@ -371,6 +371,69 @@ class JmSearchAlbumClient:
|
||||
"""
|
||||
return self.search(search_query, page, 4, order_by, time)
|
||||
|
||||
def search_gen(self,
|
||||
search_query: str,
|
||||
main_tag=0,
|
||||
page: int = 1,
|
||||
order_by: str = ORDER_BY_LATEST,
|
||||
time: str = TIME_ALL,
|
||||
):
|
||||
"""
|
||||
搜索结果的生成器,支持下面这种调用方式:
|
||||
|
||||
```
|
||||
for page in self.search_gen('无修正'):
|
||||
# 每次循环,page为新页的结果
|
||||
pass
|
||||
```
|
||||
|
||||
同时支持外界send参数,可以改变搜索的设定,例如:
|
||||
|
||||
```
|
||||
gen = client.search_gen('MANA')
|
||||
for i, page in enumerate(gen):
|
||||
print(page.page_count)
|
||||
page = gen.send({
|
||||
'search_query': '+MANA +无修正',
|
||||
'page': 1
|
||||
})
|
||||
print(page.page_count)
|
||||
break
|
||||
```
|
||||
|
||||
"""
|
||||
params = {
|
||||
'search_query': search_query,
|
||||
'main_tag': main_tag,
|
||||
'order_by': order_by,
|
||||
'time': time,
|
||||
}
|
||||
|
||||
def search(page):
|
||||
params['page'] = page
|
||||
return self.search(**params)
|
||||
|
||||
from math import inf
|
||||
|
||||
def update(value: Union[Dict], page: int, search_page: JmSearchPage):
|
||||
if value is None:
|
||||
return page + 1, search_page.page_count
|
||||
|
||||
ExceptionTool.require_true(isinstance(value, dict), 'require dict params')
|
||||
|
||||
# 根据外界传递的参数,更新params和page
|
||||
page = value.get('page', page)
|
||||
params.update(value)
|
||||
|
||||
return page, inf
|
||||
|
||||
total = inf
|
||||
|
||||
while page <= total:
|
||||
search_page = search(page)
|
||||
value = yield search_page
|
||||
page, total = update(value, page, search_page)
|
||||
|
||||
|
||||
# noinspection PyAbstractClass
|
||||
class JmcomicClient(
|
||||
|
||||
@ -26,6 +26,11 @@ def system_proxy():
|
||||
return ProxyBuilder.system_proxy()
|
||||
|
||||
|
||||
def str_to_list(text):
|
||||
from common import str_to_list
|
||||
return str_to_list(text)
|
||||
|
||||
|
||||
class JmcomicException(Exception):
|
||||
pass
|
||||
|
||||
@ -63,9 +68,25 @@ class JmModuleConfig:
|
||||
|
||||
# 域名配置 - 移动端
|
||||
# 图片域名
|
||||
DOMAIN_API_IMAGE_LIST = [f"cdn-msp.jmapiproxy{i}.cc" for i in range(1, 4)]
|
||||
DOMAIN_API_IMAGE_LIST = str_to_list('''
|
||||
cdn-msp.jmapiproxy1.monster
|
||||
cdn-msp2.jmapiproxy1.monster
|
||||
cdn-msp.jmapiproxy1.cc
|
||||
cdn-msp.jmapiproxy2.cc
|
||||
cdn-msp.jmapiproxy3.cc
|
||||
cdn-msp.jmapiproxy4.cc
|
||||
|
||||
''')
|
||||
|
||||
# API域名
|
||||
DOMAIN_API_LIST = [f'www.jmapinode{i}.top' for i in range(1, 4)]
|
||||
DOMAIN_API_LIST = str_to_list('''
|
||||
www.jmapinode1.top
|
||||
www.jmapinode2.top
|
||||
www.jmapinode3.top
|
||||
www.jmapinode.biz
|
||||
www.jmapinode.top
|
||||
|
||||
''')
|
||||
|
||||
# 域名配置 - 网页端
|
||||
# 无需配置,默认为None,需要的时候会发起请求获得
|
||||
|
||||
@ -425,11 +425,17 @@ class JmAlbumDetail(DetailEntity):
|
||||
class JmSearchPage(JmBaseEntity, IndexedEntity):
|
||||
ContentItem = Tuple[str, Dict[str, Any]]
|
||||
|
||||
def __init__(self, content: List[ContentItem]):
|
||||
# [
|
||||
# album_id, {title, tag_list, ...}
|
||||
# ]
|
||||
def __init__(self, content: List[ContentItem], page_count):
|
||||
|
||||
"""
|
||||
[
|
||||
album_id, {title, tag_list, ...}
|
||||
]
|
||||
:param content: 搜索结果,移动端和网页端都一次返回80个
|
||||
:param page_count: 总页数,登录和不登录能看到的总页数不一样
|
||||
"""
|
||||
self.content = content
|
||||
self.page_count = page_count
|
||||
|
||||
def iter_id(self) -> Generator[str, None, None]:
|
||||
"""
|
||||
@ -469,7 +475,7 @@ class JmSearchPage(JmBaseEntity, IndexedEntity):
|
||||
'name': album.name,
|
||||
'tag_list': album.tags,
|
||||
}
|
||||
)])
|
||||
)], -1)
|
||||
setattr(page, 'album', album)
|
||||
return page
|
||||
|
||||
|
||||
@ -115,7 +115,7 @@ class JmcomicText:
|
||||
|
||||
@classmethod
|
||||
def analyse_jm_search_html(cls, html: str) -> JmSearchPage:
|
||||
return JmcomicSearchTool.parse_html_to_page(html)
|
||||
return JmSearchTool.parse_html_to_page(html)
|
||||
|
||||
@classmethod
|
||||
def reflect_new_instance(cls, html: str, cls_field_prefix: str, clazz: type):
|
||||
@ -217,7 +217,39 @@ class JmcomicText:
|
||||
JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env)
|
||||
|
||||
|
||||
class JmcomicSearchTool:
|
||||
class PatternTool:
|
||||
|
||||
@classmethod
|
||||
def match_or_default(cls, html: str, pattern: Pattern, default):
|
||||
match = pattern.search(html)
|
||||
return default if match is None else match[1]
|
||||
|
||||
@classmethod
|
||||
def require_match(cls, html: str, pattern: Pattern, msg, rindex=1):
|
||||
match = pattern.search(html)
|
||||
if match is not None:
|
||||
return match[rindex]
|
||||
|
||||
ExceptionTool.raises_regex(
|
||||
msg,
|
||||
html=html,
|
||||
pattern=pattern,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def require_not_match(cls, html: str, pattern: Pattern, *, msg_func):
|
||||
match = pattern.search(html)
|
||||
if match is None:
|
||||
return
|
||||
|
||||
ExceptionTool.raises_regex(
|
||||
msg_func(match),
|
||||
html=html,
|
||||
pattern=pattern,
|
||||
)
|
||||
|
||||
|
||||
class JmSearchTool:
|
||||
# 用来缩减html的长度
|
||||
pattern_html_search_shorten_for = compile(r'<div class="well well-sm">([\s\S]*)<div class="row">')
|
||||
|
||||
@ -238,30 +270,31 @@ class JmcomicSearchTool:
|
||||
# 查找错误,例如 [错误,關鍵字過短,請至少輸入兩個字以上。]
|
||||
pattern_html_search_error = compile(r'<fieldset>\n<legend>(.*?)</legend>\n<div class=.*?>\n(.*?)\n</div>\n</fieldset>')
|
||||
|
||||
pattern_html_search_total_count = compile(r'<span class="text-white">(\d+)</span> A漫.'), 0
|
||||
|
||||
@classmethod
|
||||
def parse_html_to_page(cls, html: str) -> JmSearchPage:
|
||||
# 检查是否失败
|
||||
match = cls.pattern_html_search_error.search(html)
|
||||
if match is not None:
|
||||
topic, reason = match[1], match[2]
|
||||
ExceptionTool.raises_regex(
|
||||
f'{topic}: {reason}',
|
||||
html=html,
|
||||
pattern=cls.pattern_html_search_error,
|
||||
)
|
||||
# 1. 检查是否失败
|
||||
PatternTool.require_not_match(
|
||||
html,
|
||||
cls.pattern_html_search_error,
|
||||
msg_func=lambda match: '{}: {}'.format(match[1], match[2])
|
||||
)
|
||||
|
||||
# 缩小文本范围
|
||||
match = cls.pattern_html_search_shorten_for.search(html)
|
||||
if match is None:
|
||||
ExceptionTool.raises_regex(
|
||||
'未匹配到搜索结果',
|
||||
html=html,
|
||||
pattern=cls.pattern_html_search_shorten_for,
|
||||
)
|
||||
html = match[0]
|
||||
# 2. 缩小文本范围
|
||||
html = PatternTool.require_match(
|
||||
html,
|
||||
cls.pattern_html_search_shorten_for,
|
||||
msg='未匹配到搜索结果',
|
||||
)
|
||||
|
||||
# 3. 提取结果
|
||||
import math
|
||||
|
||||
# 提取结果
|
||||
content = [] # content这个名字来源于api版搜索返回值
|
||||
total_count = PatternTool.match_or_default(html, *cls.pattern_html_search_total_count) # 总结果数
|
||||
page_count = math.ceil(int(total_count) / 80)
|
||||
|
||||
album_info_list = cls.pattern_html_search_album_info_list.findall(html)
|
||||
|
||||
for (album_id, title, _, label_category, label_sub, tag_text) in album_info_list:
|
||||
@ -273,7 +306,7 @@ class JmcomicSearchTool:
|
||||
}
|
||||
))
|
||||
|
||||
return JmSearchPage(content)
|
||||
return JmSearchPage(content, page_count)
|
||||
|
||||
@classmethod
|
||||
def parse_api_resp_to_page(cls, data: DictModel) -> JmSearchPage:
|
||||
@ -300,6 +333,7 @@ class JmcomicSearchTool:
|
||||
]
|
||||
}
|
||||
"""
|
||||
total: int = int(data.total)
|
||||
|
||||
def adapt_item(item: DictModel):
|
||||
item: dict = item.src_dict
|
||||
@ -311,7 +345,7 @@ class JmcomicSearchTool:
|
||||
for item in data.content
|
||||
]
|
||||
|
||||
return JmSearchPage(content)
|
||||
return JmSearchPage(content, total)
|
||||
|
||||
|
||||
class JmApiAdaptTool:
|
||||
|
||||
@ -251,3 +251,16 @@ class Test_Client(JmTestConfigurable):
|
||||
ans = id(photo)
|
||||
else:
|
||||
self.assertEqual(ans, id(photo))
|
||||
|
||||
def test_search_generator(self):
|
||||
JmModuleConfig.decode_url_when_debug = False
|
||||
|
||||
gen = self.client.search_gen('MANA')
|
||||
for i, page in enumerate(gen):
|
||||
print(page.page_count)
|
||||
page = gen.send({
|
||||
'search_query': 'MANA +无修正',
|
||||
'page': 1
|
||||
})
|
||||
print(page.page_count)
|
||||
break
|
||||
|
||||
Loading…
Reference in New Issue
Block a user