v2.2.4: 优化搜本API,增加本子类的字段和对应的正则表达式(喜欢数、观看数、评论数、作品、登场人物) (#121)

This commit is contained in:
hect0x7 2023-09-02 14:17:23 +08:00 committed by GitHub
parent a2c0a86646
commit dbd3f8ce66
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 175 additions and 60 deletions

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.2.3'
__version__ = '2.2.4'
from .api import *
from .jm_plugin import *

View File

@ -57,7 +57,7 @@ class AbstractJmClient(
api_path=url,
domain=self.domain_list[domain_index],
)
jm_debug(self.debug_topic_request(), url)
jm_debug(self.debug_topic_request(), self.decode(url))
else:
# 图片url
pass
@ -124,7 +124,7 @@ class AbstractJmClient(
for func in {
'get_photo_detail',
'get_album_detail',
'search_album',
'search',
}:
wrap_func_cache(func, func + '.cache.dict')
@ -151,6 +151,23 @@ class AbstractJmClient(
jm_debug('req.fallback', msg)
raise JmModuleConfig.exception(msg)
# noinspection PyMethodMayBeStatic
def append_params_to_url(self, url, params):
from urllib.parse import urlencode
# 将参数字典编码为查询字符串
query_string = urlencode(params)
url = f"{url}?{query_string}"
return url
# noinspection PyMethodMayBeStatic
def decode(self, url: str):
if not JmModuleConfig.decode_url_when_debug or '/search/' not in url:
return url
from urllib.parse import unquote
return unquote(url.replace('+', ' '))
# 基于网页实现的JmClient
class JmHtmlClient(AbstractJmClient):
@ -182,14 +199,17 @@ class JmHtmlClient(AbstractJmClient):
return photo
def search_album(self, search_query, main_tag=0, page=1) -> JmSearchPage:
def search(self, search_query, page, main_tag) -> JmSearchPage:
params = {
'main_tag': main_tag,
'search_query': search_query,
'page': page,
}
resp = self.get_jm_html('/search/photos', params=params, allow_redirects=True)
resp = self.get_jm_html(
self.append_params_to_url('/search/photos', params),
allow_redirects=True,
)
# 检查是否发生了重定向
# 因为如果搜索的是禁漫车号,会直接跳转到本子详情页面
@ -371,7 +391,7 @@ class JmApiClient(AbstractJmClient):
client_key = 'api'
API_SEARCH = '/search'
def search_album(self, search_query, main_tag=0, page=1) -> JmApiResp:
def search(self, search_query, main_tag=0, page=1) -> JmApiResp:
"""
model_data: {
"search_query": "MANA",

View File

@ -151,9 +151,6 @@ class JmDetailClient:
def get_photo_detail(self, photo_id, fetch_album=True) -> JmPhotoDetail:
raise NotImplementedError
def search_album(self, search_query: str, main_tag: int = 0, page: int = 1) -> JmSearchPage:
raise NotImplementedError
def of_api_url(self, api_path, domain):
raise NotImplementedError
@ -272,11 +269,65 @@ class JmImageClient:
return data_original.endswith('.gif')
class JmSearchAlbumClient:
"""
搜尋的最佳姿勢
包含搜尋
搜尋[+]全彩[空格][+]人妻,僅顯示全彩且是人妻的本本
範例:+全彩 +人妻
排除搜尋
搜尋全彩[空格][-]人妻,顯示全彩並排除人妻的本本
範例:全彩 -人妻
我都要搜尋
搜尋全彩[空格]人妻,會顯示所有包含全彩及人妻的本本
範例:全彩 人妻
"""
def search(self, search_query: str, page: int, main_tag: int) -> JmSearchPage:
"""
搜索成人A漫
"""
raise NotImplementedError
def search_site(self, search_query: str, page: int = 1) -> JmSearchPage:
"""
对应禁漫的站内搜索
"""
return self.search(search_query, page, 0)
def search_work(self, search_query: str, page: int = 1) -> JmSearchPage:
"""
搜索album的作品 work
"""
return self.search(search_query, page, 1)
def search_author(self, search_query: str, page: int = 1) -> JmSearchPage:
"""
搜索album的作者 author
"""
return self.search(search_query, page, 2)
def search_tag(self, search_query: str, page: int = 1) -> JmSearchPage:
"""
搜索album的标签 tag
"""
return self.search(search_query, page, 3)
def search_actor(self, search_query: str, page: int = 1) -> JmSearchPage:
"""
搜索album的登场角色 actor
"""
return self.search(search_query, page, 4)
# noinspection PyAbstractClass
class JmcomicClient(
JmImageClient,
JmDetailClient,
JmUserClient,
JmSearchAlbumClient,
Postman,
):
def get_jmcomic_url(self):

View File

@ -77,6 +77,8 @@ class JmModuleConfig:
# debug开关标记
enable_jm_debug = True
# debug时解码url
decode_url_when_debug = True
# 插件注册表
plugin_registry = {}

View File

@ -23,7 +23,7 @@ class DownloadCallback:
f'章节数: [{len(album)}], '
f'总页数: [{album.page_count}], '
f'标题: [{album.title}], '
f'关键词: [{album.keywords}]'
f'关键词: [{album.tag_list}]'
)
def after_album(self, album: JmAlbumDetail):

View File

@ -198,9 +198,9 @@ class JmPhotoDetail(DetailEntity):
return self._series_id == 0
@property
def keywords(self) -> List[str]:
def tags(self) -> List[str]:
if self.from_album is not None:
return self.from_album.keywords
return self.from_album.tag_list
return self._keywords.split(',')
@ -303,19 +303,30 @@ class JmAlbumDetail(DetailEntity):
episode_list,
page_count,
author_list,
keywords_list,
tag_list,
pub_date,
update_date,
likes,
views,
comment_count,
work_list,
actor_list,
):
self.album_id: str = album_id
self.scramble_id: str = scramble_id
self.title: str = title
self.page_count = int(page_count)
self._author_list: List[str] = author_list
self._keywords_list: List[str] = keywords_list
self.page_count = int(page_count) # 总页数
self.pub_date: str = pub_date # 发布日期
self.update_date: str = update_date # 更新日期
self.likes: str = likes # [1K] 點擊喜歡
self.views: str = views # [40K] 次觀看
self.comment_count = int(comment_count)
self.work_list: List[str] = work_list # 作品
self.actor_list: List[str] = actor_list # 登場人物
self.tag_list: List[str] = tag_list # 標籤
self.author_list: List[str] = author_list # 作者
# 有的 album 没有章节,则自成一章。
if len(episode_list) == 0:
# photo_id, photo_index, photo_title, photo_pub_date
@ -325,6 +336,38 @@ class JmAlbumDetail(DetailEntity):
self.episode_list: List[Tuple] = episode_list
@property
def author(self):
"""
作者
禁漫本子的作者标签可能有多个全部作者请使用字段 self.author_list
"""
if len(self.author_list) >= 1:
return self.author_list[0]
return JmModuleConfig.default_author
@property
def id(self):
return self.album_id
@staticmethod
def distinct_episode(episode_list):
ret = []
def not_exist(episode):
photo_id = episode[0]
for each in ret:
if each[0] == photo_id:
return False
return True
for episode in episode_list:
if not_exist(episode):
ret.append(episode)
return ret
def create_photo_detail(self, index) -> Tuple[JmPhotoDetail, Tuple]:
# 校验参数
length = len(self.episode_list)
@ -351,37 +394,6 @@ class JmAlbumDetail(DetailEntity):
return photo, episode_info
@property
def author(self):
if len(self._author_list) >= 1:
return self._author_list[0]
return JmModuleConfig.default_author
@property
def keywords(self) -> List[str]:
return self._keywords_list
@property
def id(self):
return self.album_id
@staticmethod
def distinct_episode(episode_list):
ret = []
def not_exist(episode):
photo_id = episode[0]
for each in ret:
if each[0] == photo_id:
return False
return True
for episode in episode_list:
if not_exist(episode):
ret.append(episode)
return ret
def getindex(self, item) -> JmPhotoDetail:
return self.create_photo_detail(item)[0]
@ -421,7 +433,7 @@ class JmSearchPage(JmBaseEntity, IterableEntity):
album.title,
None,
None,
album.keywords,
album.tag_list,
)
obj = JmSearchPage([album_info])

View File

@ -28,16 +28,32 @@ class JmcomicText:
pattern_html_album_page_count = compile('<span class="pagecount">.*?:(\d+)</span>')
pattern_html_album_pub_date = compile('>上架日期 : (.*?)</span>')
pattern_html_album_update_date = compile('>更新日期 : (.*?)</span>')
pattern_html_album_keywords_list = [
# 作品
pattern_html_album_work_list = [
compile('<span itemprop="author" data-type="works">([\s\S]*?)</span>'),
compile('<a[\s\S]*?>(.*?)</a>')
]
# 登場人物
pattern_html_album_actor_list = [
compile('<span itemprop="author" data-type="actor">([\s\S]*?)</span>'),
compile('<a[\s\S]*?>(.*?)</a>')
]
# 标签
pattern_html_album_tag_list = [
compile('<span itemprop="genre" data-type="tags">([\s\S]*?)</span>'),
compile('<a[\s\S]*?>(.*?)</a>')
]
# album 作者
# 作者
pattern_html_album_author_list = [
compile('作者: *<span itemprop="author" data-type="author">([\s\S]*?)</span>'),
compile("<a[\s\S]*?>(.*?)</a>"),
]
# 點擊喜歡
pattern_html_album_likes = compile('<span id="albim_likes_\d+">(.*?)</span>')
# 觀看
pattern_html_album_views = compile('<span>(.*?)</span> 次觀看')
# 評論
pattern_html_album_comment_count = compile('<div class="badge" id="total_video_comments">(\d+)</div></a></li>')
@classmethod
def parse_to_jm_domain(cls, text: str):
@ -142,7 +158,7 @@ class JmcomicText:
if field_value is None:
JmModuleConfig.raise_regex_error_executor(
f"文本没有匹配上字段:字段名为'{field_name}'pattern: [{pattern_value.pattern}]",
f"文本没有匹配上字段:字段名为'{field_name}'pattern: [{pattern_value}]",
html,
field_name,
pattern_value

View File

@ -99,8 +99,8 @@ class Test_Api(JmTestConfigurable):
def test_get_jmcomic_url(self):
func_list = {
self.client.get_jmcomic_url,
# self.client.get_jmcomic_domain_all,
JmModuleConfig.get_jmcomic_url,
self.client.get_jmcomic_domain_all,
# JmModuleConfig.get_jmcomic_url,
# JmModuleConfig.get_jmcomic_domain_all,
}

View File

@ -33,7 +33,7 @@ class Test_Client(JmTestConfigurable):
print(f'本子: [{album.title}] 一共有{album.page_count}页图')
def test_search(self):
jm_search_page: JmSearchPage = self.client.search_album('+无修正 +中文 -全彩')
jm_search_page: JmSearchPage = self.client.search_tag('+无修正 +中文 -全彩')
for album_id, title in reversed(jm_search_page):
print(album_id, title)
@ -51,3 +51,16 @@ class Test_Client(JmTestConfigurable):
self.client.get_album_detail,
'332583'
)
def test_entity(self):
album = self.client.get_album_detail(410090)
ans = [
(album.work_list, ['原神', 'Genshin']),
(album.actor_list, ['申鶴', '神里綾華', '甘雨']),
(album.tag_list, ['C101', '巨乳', '校服', '口交', '乳交', '群交', '連褲襪', '中文', '禁漫漢化組', '纯爱']),
(album.author_list, ['うぱ西']),
]
for pair in ans:
self.assertListEqual(pair[0], pair[1])

View File

@ -51,13 +51,13 @@ def get_album_photo_detail():
@timeit('搜索本子: ')
def search_jm_album():
# 分页查询
search_page: JmSearchPage = client.search_album(search_query='+MANA +无修正', page=1)
# 分页查询search_site就是禁漫网页上的【站内搜索】
search_page: JmSearchPage = client.search_site(search_query='+MANA +无修正', page=1)
for album_id, title in search_page:
print(f'[{album_id}]: {title}')
# 直接搜索禁漫车号
search_page = client.search_album(search_query='427413')
search_page = client.search_site(search_query='427413')
album: JmAlbumDetail = search_page.single_album
print(album.keywords)
@ -65,8 +65,9 @@ def search_jm_album():
@timeit('搜索并下载本子: ')
def search_and_download():
tag = '無修正'
# 搜索第一页
search_page: JmSearchPage = client.search_album(tag, main_tag=3)
# 搜索标签可以使用search_tag。
# 搜索第一页。
search_page: JmSearchPage = client.search_tag(tag, page=1)
id_list = []