mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.0.7: 优化搜索功能、实体类、缓存、请求头,更新代码示例 (#63)
This commit is contained in:
parent
e7e85a1034
commit
9f732da485
@ -2,6 +2,6 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option
|
||||
|
||||
__version__ = '2.0.6'
|
||||
__version__ = '2.0.7'
|
||||
|
||||
from .api import *
|
||||
|
@ -75,23 +75,32 @@ class AbstractJmClient(
|
||||
|
||||
def enable_cache(self, debug=False):
|
||||
def wrap_func_cache(func_name, cache_dict_name):
|
||||
import common
|
||||
if common.VERSION > '0.4.8':
|
||||
cache = common.cache
|
||||
cache_dict = {}
|
||||
cache_hit_msg = (f'【缓存命中】{cache_dict_name} ' + '→ [{}]') if debug is True else None
|
||||
cache_miss_msg = (f'【缓存缺失】{cache_dict_name} ' + '← [{}]') if debug is True else None
|
||||
cache = cache(
|
||||
cache_dict=cache_dict,
|
||||
cache_hit_msg=cache_hit_msg,
|
||||
cache_miss_msg=cache_miss_msg,
|
||||
)
|
||||
setattr(self, cache_dict_name, cache_dict)
|
||||
else:
|
||||
if sys.version_info < (3, 9):
|
||||
raise NotImplementedError('不支持启用JmcomicClient缓存。\n'
|
||||
'请更新python版本到3.9以上,'
|
||||
'或更新commonX: `pip install commonX --upgrade`')
|
||||
import functools
|
||||
cache = functools.cache
|
||||
|
||||
if hasattr(self, cache_dict_name):
|
||||
return
|
||||
|
||||
cache_dict = {}
|
||||
setattr(self, cache_dict_name, cache_dict)
|
||||
|
||||
# 重载本对象的方法
|
||||
func = getattr(self, func_name)
|
||||
|
||||
cache_hit_msg = f'【缓存命中】{cache_dict_name} ' + '→ [{}]' if debug is True else None
|
||||
cache_miss_msg = f'【缓存缺失】{cache_dict_name} ' + '← [{}]' if debug is True else None
|
||||
|
||||
wrap_func = enable_cache(
|
||||
cache_dict=cache_dict,
|
||||
cache_hit_msg=cache_hit_msg,
|
||||
cache_miss_msg=cache_miss_msg,
|
||||
)(func)
|
||||
wrap_func = cache(func)
|
||||
|
||||
setattr(self, func_name, wrap_func)
|
||||
|
||||
@ -149,10 +158,11 @@ class JmHtmlClient(AbstractJmClient):
|
||||
new.from_album = photo_detail.from_album
|
||||
photo_detail.__dict__.update(new.__dict__)
|
||||
|
||||
def search_album(self, search_query, main_tag=0):
|
||||
def search_album(self, search_query, main_tag=0, page=1) -> JmSearchPage:
|
||||
params = {
|
||||
'main_tag': main_tag,
|
||||
'search_query': search_query,
|
||||
'page': page,
|
||||
}
|
||||
|
||||
resp = self.get_jm_html('/search/photos', params=params, allow_redirects=True)
|
||||
@ -160,7 +170,8 @@ class JmHtmlClient(AbstractJmClient):
|
||||
# 检查是否发生了重定向
|
||||
# 因为如果搜索的是禁漫车号,会直接跳转到本子详情页面
|
||||
if resp.redirect_count != 0 and '/album/' in resp.url:
|
||||
return JmcomicText.analyse_jm_album_html(resp.text)
|
||||
album = JmcomicText.analyse_jm_album_html(resp.text)
|
||||
return JmSearchPage.wrap_single_album(album)
|
||||
else:
|
||||
return JmSearchSupport.analyse_jm_search_html(resp.text)
|
||||
|
||||
@ -284,7 +295,7 @@ class JmHtmlClient(AbstractJmClient):
|
||||
class JmApiClient(AbstractJmClient):
|
||||
API_SEARCH = '/search'
|
||||
|
||||
def search_album(self, search_query: str, main_tag=0) -> JmApiResp:
|
||||
def search_album(self, search_query, main_tag=0, page=1) -> JmApiResp:
|
||||
"""
|
||||
model_data: {
|
||||
"search_query": "MANA",
|
||||
@ -312,6 +323,8 @@ class JmApiClient(AbstractJmClient):
|
||||
self.API_SEARCH,
|
||||
params={
|
||||
'search_query': search_query,
|
||||
'main_tag': main_tag,
|
||||
'page': page,
|
||||
}
|
||||
)
|
||||
|
||||
|
@ -133,7 +133,7 @@ class JmDetailClient:
|
||||
def ensure_photo_can_use(self, photo_detail: JmPhotoDetail):
|
||||
raise NotImplementedError
|
||||
|
||||
def search_album(self, search_query, main_tag=0) -> Union[JmSearchPage, JmAlbumDetail]:
|
||||
def search_album(self, search_query: str, main_tag: int = 0, page: int = 1) -> JmSearchPage:
|
||||
raise NotImplementedError
|
||||
|
||||
def of_api_url(self, api_path, domain):
|
||||
|
@ -74,16 +74,18 @@ class JmModuleConfig:
|
||||
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
|
||||
'application/signed-exchange;v=b3;q=0.7',
|
||||
'accept-language': 'zh-CN,zh;q=0.9',
|
||||
'cache-control': 'no-cache',
|
||||
'referer': 'https://18comic.vip',
|
||||
'sec-ch-ua': '"Google Chrome";v="111", "Not(A:Brand";v="8", "Chromium";v="111"',
|
||||
'pragma': 'no-cache',
|
||||
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'sec-ch-ua-platform': '"Windows"',
|
||||
'sec-fetch-dest': 'document',
|
||||
'sec-fetch-mode': 'navigate',
|
||||
'sec-fetch-site': 'same-origin',
|
||||
'sec-fetch-site': 'none',
|
||||
'sec-fetch-user': '?1',
|
||||
'upgrade-insecure-requests': '1',
|
||||
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 '
|
||||
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 '
|
||||
'Safari/537.36',
|
||||
}
|
||||
|
||||
|
@ -306,12 +306,33 @@ class JmAlbumDetail(WorkEntity):
|
||||
|
||||
class JmSearchPage(IterableEntity):
|
||||
|
||||
def __init__(self, album_info_list):
|
||||
def __init__(self, album_info_list: List[Tuple[str, str, StrNone, StrNone, List[str]]]):
|
||||
# (album_id, title, category_none, label_sub_none, tag_list)
|
||||
self.album_info_list: List[Tuple[str, str, StrNone, StrNone, List[str]]] = album_info_list
|
||||
self.album_info_list = album_info_list
|
||||
|
||||
def __len__(self):
|
||||
return len(self.album_info_list)
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self.album_info_list[item][0:2]
|
||||
|
||||
@property
|
||||
def single_album(self) -> JmAlbumDetail:
|
||||
return getattr(self, 'album')
|
||||
|
||||
@classmethod
|
||||
def wrap_single_album(cls, album: JmAlbumDetail) -> 'JmSearchPage':
|
||||
# ('462257', '[無邪気漢化組] [きょくちょ] 楓と鈴 4.5', '短篇', '漢化', [])
|
||||
# (album_id, title, category_none, label_sub_none, tag_list)
|
||||
|
||||
album_info = (
|
||||
album.album_id,
|
||||
album.title,
|
||||
None,
|
||||
None,
|
||||
album.keywords,
|
||||
)
|
||||
obj = JmSearchPage([album_info])
|
||||
|
||||
setattr(obj, 'album', album)
|
||||
return obj
|
||||
|
@ -74,4 +74,4 @@ class JmTestConfigurable(unittest.TestCase):
|
||||
|
||||
@classmethod
|
||||
def enable_client_cache(cls):
|
||||
cls.client.enable_cache()
|
||||
cls.client.enable_cache(debug=True)
|
||||
|
@ -20,10 +20,13 @@ def download_jm_album():
|
||||
@timeit('获取实体类: ')
|
||||
def get_album_photo_detail():
|
||||
client = jm_option.build_jm_client()
|
||||
# 启用缓存,会缓存id → album和photo的实体类
|
||||
client.enable_cache(debug=True)
|
||||
|
||||
album: JmAlbumDetail = client.get_album_detail('427413')
|
||||
|
||||
def show(p):
|
||||
p: JmPhotoDetail = client.get_photo_detail(p.photo_id)
|
||||
p: JmPhotoDetail = client.get_photo_detail(p.photo_id, False)
|
||||
for img in p:
|
||||
img: JmImageDetail
|
||||
print(img.img_url)
|
||||
@ -37,10 +40,31 @@ def get_album_photo_detail():
|
||||
@timeit('搜索本子: ')
|
||||
def search_jm_album():
|
||||
client = jm_option.build_jm_client()
|
||||
search_album: JmSearchPage = client.search_album(search_query='+MANA +无修正')
|
||||
for album_id, title, *_args in search_album:
|
||||
print(f'[{album_id}]:{title}')
|
||||
|
||||
# 分页查询
|
||||
search_page: JmSearchPage = client.search_album(search_query='+MANA +无修正', page=1)
|
||||
for album_id, title in search_page:
|
||||
print(f'[{album_id}]: {title}')
|
||||
|
||||
# 直接搜索禁漫车号
|
||||
search_page = client.search_album(search_query='427413')
|
||||
album: JmAlbumDetail = search_page.single_album
|
||||
print(album.keywords)
|
||||
|
||||
@timeit('搜索并下载本子: ')
|
||||
def search_and_download():
|
||||
tag = '無修正'
|
||||
search_album: JmSearchPage = cl.search_album(tag, main_tag=3)
|
||||
|
||||
id_list = []
|
||||
|
||||
for arg in search_album.album_info_list:
|
||||
(album_id, title, category_none, label_sub_none, tag_list) = arg
|
||||
if tag in tag_list:
|
||||
print(f'[标签/{tag}] 发现目标: [{album_id}]: [{title}]')
|
||||
id_list.append(album_id)
|
||||
|
||||
download_album(id_list, op)
|
||||
|
||||
def main():
|
||||
search_jm_album()
|
||||
|
Loading…
Reference in New Issue
Block a user