mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.2.4: 优化搜本API,增加本子类的字段和对应的正则表达式(喜欢数、观看数、评论数、作品、登场人物) (#121)
This commit is contained in:
parent
a2c0a86646
commit
dbd3f8ce66
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.2.3'
|
||||
__version__ = '2.2.4'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
@ -57,7 +57,7 @@ class AbstractJmClient(
|
||||
api_path=url,
|
||||
domain=self.domain_list[domain_index],
|
||||
)
|
||||
jm_debug(self.debug_topic_request(), url)
|
||||
jm_debug(self.debug_topic_request(), self.decode(url))
|
||||
else:
|
||||
# 图片url
|
||||
pass
|
||||
@ -124,7 +124,7 @@ class AbstractJmClient(
|
||||
for func in {
|
||||
'get_photo_detail',
|
||||
'get_album_detail',
|
||||
'search_album',
|
||||
'search',
|
||||
}:
|
||||
wrap_func_cache(func, func + '.cache.dict')
|
||||
|
||||
@ -151,6 +151,23 @@ class AbstractJmClient(
|
||||
jm_debug('req.fallback', msg)
|
||||
raise JmModuleConfig.exception(msg)
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def append_params_to_url(self, url, params):
|
||||
from urllib.parse import urlencode
|
||||
|
||||
# 将参数字典编码为查询字符串
|
||||
query_string = urlencode(params)
|
||||
url = f"{url}?{query_string}"
|
||||
return url
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def decode(self, url: str):
|
||||
if not JmModuleConfig.decode_url_when_debug or '/search/' not in url:
|
||||
return url
|
||||
|
||||
from urllib.parse import unquote
|
||||
return unquote(url.replace('+', ' '))
|
||||
|
||||
|
||||
# 基于网页实现的JmClient
|
||||
class JmHtmlClient(AbstractJmClient):
|
||||
@ -182,14 +199,17 @@ class JmHtmlClient(AbstractJmClient):
|
||||
|
||||
return photo
|
||||
|
||||
def search_album(self, search_query, main_tag=0, page=1) -> JmSearchPage:
|
||||
def search(self, search_query, page, main_tag) -> JmSearchPage:
|
||||
params = {
|
||||
'main_tag': main_tag,
|
||||
'search_query': search_query,
|
||||
'page': page,
|
||||
}
|
||||
|
||||
resp = self.get_jm_html('/search/photos', params=params, allow_redirects=True)
|
||||
resp = self.get_jm_html(
|
||||
self.append_params_to_url('/search/photos', params),
|
||||
allow_redirects=True,
|
||||
)
|
||||
|
||||
# 检查是否发生了重定向
|
||||
# 因为如果搜索的是禁漫车号,会直接跳转到本子详情页面
|
||||
@ -371,7 +391,7 @@ class JmApiClient(AbstractJmClient):
|
||||
client_key = 'api'
|
||||
API_SEARCH = '/search'
|
||||
|
||||
def search_album(self, search_query, main_tag=0, page=1) -> JmApiResp:
|
||||
def search(self, search_query, main_tag=0, page=1) -> JmApiResp:
|
||||
"""
|
||||
model_data: {
|
||||
"search_query": "MANA",
|
||||
|
@ -151,9 +151,6 @@ class JmDetailClient:
|
||||
def get_photo_detail(self, photo_id, fetch_album=True) -> JmPhotoDetail:
|
||||
raise NotImplementedError
|
||||
|
||||
def search_album(self, search_query: str, main_tag: int = 0, page: int = 1) -> JmSearchPage:
|
||||
raise NotImplementedError
|
||||
|
||||
def of_api_url(self, api_path, domain):
|
||||
raise NotImplementedError
|
||||
|
||||
@ -272,11 +269,65 @@ class JmImageClient:
|
||||
return data_original.endswith('.gif')
|
||||
|
||||
|
||||
class JmSearchAlbumClient:
|
||||
"""
|
||||
搜尋的最佳姿勢?
|
||||
【包含搜尋】
|
||||
搜尋[+]全彩[空格][+]人妻,僅顯示全彩且是人妻的本本
|
||||
範例:+全彩 +人妻
|
||||
|
||||
【排除搜尋】
|
||||
搜尋全彩[空格][-]人妻,顯示全彩並排除人妻的本本
|
||||
範例:全彩 -人妻
|
||||
|
||||
【我都要搜尋】
|
||||
搜尋全彩[空格]人妻,會顯示所有包含全彩及人妻的本本
|
||||
範例:全彩 人妻
|
||||
"""
|
||||
|
||||
def search(self, search_query: str, page: int, main_tag: int) -> JmSearchPage:
|
||||
"""
|
||||
搜索【成人A漫】
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def search_site(self, search_query: str, page: int = 1) -> JmSearchPage:
|
||||
"""
|
||||
对应禁漫的站内搜索
|
||||
"""
|
||||
return self.search(search_query, page, 0)
|
||||
|
||||
def search_work(self, search_query: str, page: int = 1) -> JmSearchPage:
|
||||
"""
|
||||
搜索album的作品 work
|
||||
"""
|
||||
return self.search(search_query, page, 1)
|
||||
|
||||
def search_author(self, search_query: str, page: int = 1) -> JmSearchPage:
|
||||
"""
|
||||
搜索album的作者 author
|
||||
"""
|
||||
return self.search(search_query, page, 2)
|
||||
|
||||
def search_tag(self, search_query: str, page: int = 1) -> JmSearchPage:
|
||||
"""
|
||||
搜索album的标签 tag
|
||||
"""
|
||||
return self.search(search_query, page, 3)
|
||||
|
||||
def search_actor(self, search_query: str, page: int = 1) -> JmSearchPage:
|
||||
"""
|
||||
搜索album的登场角色 actor
|
||||
"""
|
||||
return self.search(search_query, page, 4)
|
||||
|
||||
|
||||
# noinspection PyAbstractClass
|
||||
class JmcomicClient(
|
||||
JmImageClient,
|
||||
JmDetailClient,
|
||||
JmUserClient,
|
||||
JmSearchAlbumClient,
|
||||
Postman,
|
||||
):
|
||||
def get_jmcomic_url(self):
|
||||
|
@ -77,6 +77,8 @@ class JmModuleConfig:
|
||||
|
||||
# debug开关标记
|
||||
enable_jm_debug = True
|
||||
# debug时解码url
|
||||
decode_url_when_debug = True
|
||||
|
||||
# 插件注册表
|
||||
plugin_registry = {}
|
||||
|
@ -23,7 +23,7 @@ class DownloadCallback:
|
||||
f'章节数: [{len(album)}], '
|
||||
f'总页数: [{album.page_count}], '
|
||||
f'标题: [{album.title}], '
|
||||
f'关键词: [{album.keywords}]'
|
||||
f'关键词: [{album.tag_list}]'
|
||||
)
|
||||
|
||||
def after_album(self, album: JmAlbumDetail):
|
||||
|
@ -198,9 +198,9 @@ class JmPhotoDetail(DetailEntity):
|
||||
return self._series_id == 0
|
||||
|
||||
@property
|
||||
def keywords(self) -> List[str]:
|
||||
def tags(self) -> List[str]:
|
||||
if self.from_album is not None:
|
||||
return self.from_album.keywords
|
||||
return self.from_album.tag_list
|
||||
|
||||
return self._keywords.split(',')
|
||||
|
||||
@ -303,19 +303,30 @@ class JmAlbumDetail(DetailEntity):
|
||||
episode_list,
|
||||
page_count,
|
||||
author_list,
|
||||
keywords_list,
|
||||
tag_list,
|
||||
pub_date,
|
||||
update_date,
|
||||
likes,
|
||||
views,
|
||||
comment_count,
|
||||
work_list,
|
||||
actor_list,
|
||||
):
|
||||
self.album_id: str = album_id
|
||||
self.scramble_id: str = scramble_id
|
||||
self.title: str = title
|
||||
self.page_count = int(page_count)
|
||||
self._author_list: List[str] = author_list
|
||||
self._keywords_list: List[str] = keywords_list
|
||||
self.page_count = int(page_count) # 总页数
|
||||
self.pub_date: str = pub_date # 发布日期
|
||||
self.update_date: str = update_date # 更新日期
|
||||
|
||||
self.likes: str = likes # [1K] 點擊喜歡
|
||||
self.views: str = views # [40K] 次觀看
|
||||
self.comment_count = int(comment_count)
|
||||
self.work_list: List[str] = work_list # 作品
|
||||
self.actor_list: List[str] = actor_list # 登場人物
|
||||
self.tag_list: List[str] = tag_list # 標籤
|
||||
self.author_list: List[str] = author_list # 作者
|
||||
|
||||
# 有的 album 没有章节,则自成一章。
|
||||
if len(episode_list) == 0:
|
||||
# photo_id, photo_index, photo_title, photo_pub_date
|
||||
@ -325,6 +336,38 @@ class JmAlbumDetail(DetailEntity):
|
||||
|
||||
self.episode_list: List[Tuple] = episode_list
|
||||
|
||||
@property
|
||||
def author(self):
|
||||
"""
|
||||
作者
|
||||
禁漫本子的作者标签可能有多个,全部作者请使用字段 self.author_list
|
||||
"""
|
||||
if len(self.author_list) >= 1:
|
||||
return self.author_list[0]
|
||||
|
||||
return JmModuleConfig.default_author
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self.album_id
|
||||
|
||||
@staticmethod
|
||||
def distinct_episode(episode_list):
|
||||
ret = []
|
||||
|
||||
def not_exist(episode):
|
||||
photo_id = episode[0]
|
||||
for each in ret:
|
||||
if each[0] == photo_id:
|
||||
return False
|
||||
return True
|
||||
|
||||
for episode in episode_list:
|
||||
if not_exist(episode):
|
||||
ret.append(episode)
|
||||
|
||||
return ret
|
||||
|
||||
def create_photo_detail(self, index) -> Tuple[JmPhotoDetail, Tuple]:
|
||||
# 校验参数
|
||||
length = len(self.episode_list)
|
||||
@ -351,37 +394,6 @@ class JmAlbumDetail(DetailEntity):
|
||||
|
||||
return photo, episode_info
|
||||
|
||||
@property
|
||||
def author(self):
|
||||
if len(self._author_list) >= 1:
|
||||
return self._author_list[0]
|
||||
return JmModuleConfig.default_author
|
||||
|
||||
@property
|
||||
def keywords(self) -> List[str]:
|
||||
return self._keywords_list
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
return self.album_id
|
||||
|
||||
@staticmethod
|
||||
def distinct_episode(episode_list):
|
||||
ret = []
|
||||
|
||||
def not_exist(episode):
|
||||
photo_id = episode[0]
|
||||
for each in ret:
|
||||
if each[0] == photo_id:
|
||||
return False
|
||||
return True
|
||||
|
||||
for episode in episode_list:
|
||||
if not_exist(episode):
|
||||
ret.append(episode)
|
||||
|
||||
return ret
|
||||
|
||||
def getindex(self, item) -> JmPhotoDetail:
|
||||
return self.create_photo_detail(item)[0]
|
||||
|
||||
@ -421,7 +433,7 @@ class JmSearchPage(JmBaseEntity, IterableEntity):
|
||||
album.title,
|
||||
None,
|
||||
None,
|
||||
album.keywords,
|
||||
album.tag_list,
|
||||
)
|
||||
obj = JmSearchPage([album_info])
|
||||
|
||||
|
@ -28,16 +28,32 @@ class JmcomicText:
|
||||
pattern_html_album_page_count = compile('<span class="pagecount">.*?:(\d+)</span>')
|
||||
pattern_html_album_pub_date = compile('>上架日期 : (.*?)</span>')
|
||||
pattern_html_album_update_date = compile('>更新日期 : (.*?)</span>')
|
||||
pattern_html_album_keywords_list = [
|
||||
# 作品
|
||||
pattern_html_album_work_list = [
|
||||
compile('<span itemprop="author" data-type="works">([\s\S]*?)</span>'),
|
||||
compile('<a[\s\S]*?>(.*?)</a>')
|
||||
]
|
||||
# 登場人物
|
||||
pattern_html_album_actor_list = [
|
||||
compile('<span itemprop="author" data-type="actor">([\s\S]*?)</span>'),
|
||||
compile('<a[\s\S]*?>(.*?)</a>')
|
||||
]
|
||||
# 标签
|
||||
pattern_html_album_tag_list = [
|
||||
compile('<span itemprop="genre" data-type="tags">([\s\S]*?)</span>'),
|
||||
compile('<a[\s\S]*?>(.*?)</a>')
|
||||
]
|
||||
|
||||
# album 作者
|
||||
# 作者
|
||||
pattern_html_album_author_list = [
|
||||
compile('作者: *<span itemprop="author" data-type="author">([\s\S]*?)</span>'),
|
||||
compile("<a[\s\S]*?>(.*?)</a>"),
|
||||
]
|
||||
# 點擊喜歡
|
||||
pattern_html_album_likes = compile('<span id="albim_likes_\d+">(.*?)</span>')
|
||||
# 觀看
|
||||
pattern_html_album_views = compile('<span>(.*?)</span> 次觀看')
|
||||
# 評論
|
||||
pattern_html_album_comment_count = compile('<div class="badge" id="total_video_comments">(\d+)</div></a></li>')
|
||||
|
||||
@classmethod
|
||||
def parse_to_jm_domain(cls, text: str):
|
||||
@ -142,7 +158,7 @@ class JmcomicText:
|
||||
|
||||
if field_value is None:
|
||||
JmModuleConfig.raise_regex_error_executor(
|
||||
f"文本没有匹配上字段:字段名为'{field_name}',pattern: [{pattern_value.pattern}]",
|
||||
f"文本没有匹配上字段:字段名为'{field_name}',pattern: [{pattern_value}]",
|
||||
html,
|
||||
field_name,
|
||||
pattern_value
|
||||
|
@ -99,8 +99,8 @@ class Test_Api(JmTestConfigurable):
|
||||
def test_get_jmcomic_url(self):
|
||||
func_list = {
|
||||
self.client.get_jmcomic_url,
|
||||
# self.client.get_jmcomic_domain_all,
|
||||
JmModuleConfig.get_jmcomic_url,
|
||||
self.client.get_jmcomic_domain_all,
|
||||
# JmModuleConfig.get_jmcomic_url,
|
||||
# JmModuleConfig.get_jmcomic_domain_all,
|
||||
}
|
||||
|
||||
|
@ -33,7 +33,7 @@ class Test_Client(JmTestConfigurable):
|
||||
print(f'本子: [{album.title}] 一共有{album.page_count}页图')
|
||||
|
||||
def test_search(self):
|
||||
jm_search_page: JmSearchPage = self.client.search_album('+无修正 +中文 -全彩')
|
||||
jm_search_page: JmSearchPage = self.client.search_tag('+无修正 +中文 -全彩')
|
||||
for album_id, title in reversed(jm_search_page):
|
||||
print(album_id, title)
|
||||
|
||||
@ -51,3 +51,16 @@ class Test_Client(JmTestConfigurable):
|
||||
self.client.get_album_detail,
|
||||
'332583'
|
||||
)
|
||||
|
||||
def test_entity(self):
|
||||
album = self.client.get_album_detail(410090)
|
||||
|
||||
ans = [
|
||||
(album.work_list, ['原神', 'Genshin']),
|
||||
(album.actor_list, ['申鶴', '神里綾華', '甘雨']),
|
||||
(album.tag_list, ['C101', '巨乳', '校服', '口交', '乳交', '群交', '連褲襪', '中文', '禁漫漢化組', '纯爱']),
|
||||
(album.author_list, ['うぱ西']),
|
||||
]
|
||||
|
||||
for pair in ans:
|
||||
self.assertListEqual(pair[0], pair[1])
|
||||
|
@ -51,13 +51,13 @@ def get_album_photo_detail():
|
||||
|
||||
@timeit('搜索本子: ')
|
||||
def search_jm_album():
|
||||
# 分页查询
|
||||
search_page: JmSearchPage = client.search_album(search_query='+MANA +无修正', page=1)
|
||||
# 分页查询,search_site就是禁漫网页上的【站内搜索】
|
||||
search_page: JmSearchPage = client.search_site(search_query='+MANA +无修正', page=1)
|
||||
for album_id, title in search_page:
|
||||
print(f'[{album_id}]: {title}')
|
||||
|
||||
# 直接搜索禁漫车号
|
||||
search_page = client.search_album(search_query='427413')
|
||||
search_page = client.search_site(search_query='427413')
|
||||
album: JmAlbumDetail = search_page.single_album
|
||||
print(album.keywords)
|
||||
|
||||
@ -65,8 +65,9 @@ def search_jm_album():
|
||||
@timeit('搜索并下载本子: ')
|
||||
def search_and_download():
|
||||
tag = '無修正'
|
||||
# 搜索第一页
|
||||
search_page: JmSearchPage = client.search_album(tag, main_tag=3)
|
||||
# 搜索标签,可以使用search_tag。
|
||||
# 搜索第一页。
|
||||
search_page: JmSearchPage = client.search_tag(tag, page=1)
|
||||
|
||||
id_list = []
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user