mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.4.9: 实现分类、热门排行榜功能 (#180)
This commit is contained in:
parent
f6692eb185
commit
dfd3fd6753
2
.github/workflows/close_specific_pr.yml
vendored
2
.github/workflows/close_specific_pr.yml
vendored
@ -1,4 +1,4 @@
|
||||
name: Close specific PR
|
||||
name: Close Specific PR
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
@ -10,7 +10,7 @@
|
||||
|
||||
本项目的核心功能是下载本子,基于此,设计了一套方便使用、便于扩展,能满足一些特殊下载需求的框架。
|
||||
|
||||
除了下载功能以外,也实现了其他的一些禁漫接口,例如登录、搜索、收藏夹等等,按需实现。
|
||||
除了下载功能以外,也实现了其他的一些禁漫接口,例如登录、搜索、收藏夹、分类、排行榜等等,按需实现。
|
||||
|
||||
目前核心功能实现较为稳定,项目也处于维护阶段(因为禁漫接口经常变动,需要经常维护)。
|
||||
|
||||
|
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.4.7'
|
||||
__version__ = '2.4.9'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
@ -309,7 +309,28 @@ class JmHtmlClient(AbstractJmClient):
|
||||
album = JmcomicText.analyse_jm_album_html(resp.text)
|
||||
return JmSearchPage.wrap_single_album(album)
|
||||
else:
|
||||
return JmcomicText.analyse_jm_search_html(resp.text)
|
||||
return JmPageTool.parse_html_to_search_page(resp.text)
|
||||
|
||||
def categories_filter(self,
|
||||
page: int,
|
||||
time: str,
|
||||
category: str,
|
||||
order_by: str,
|
||||
) -> JmCategoryPage:
|
||||
params = {
|
||||
'page': page,
|
||||
'o': order_by,
|
||||
't': time,
|
||||
}
|
||||
|
||||
url = f'/albums/' + (category if category != JmMagicConstants.CATEGORY_ALL else '')
|
||||
|
||||
resp = self.get_jm_html(
|
||||
self.append_params_to_url(url, params),
|
||||
allow_redirects=True,
|
||||
)
|
||||
|
||||
return JmPageTool.parse_html_to_category_page(resp.text)
|
||||
|
||||
# -- 帐号管理 --
|
||||
|
||||
@ -525,6 +546,7 @@ class JmApiClient(AbstractJmClient):
|
||||
func_to_cache = ['search', 'fetch_detail_entity']
|
||||
|
||||
API_SEARCH = '/search'
|
||||
API_CATEGORIES_FILTER = '/categories/filter'
|
||||
API_ALBUM = '/album'
|
||||
API_CHAPTER = '/chapter'
|
||||
API_SCRAMBLE = '/chapter_view_template'
|
||||
@ -561,6 +583,26 @@ class JmApiClient(AbstractJmClient):
|
||||
|
||||
return JmPageTool.parse_api_to_search_page(data)
|
||||
|
||||
def categories_filter(self,
|
||||
page: int,
|
||||
time: str,
|
||||
category: str,
|
||||
order_by: str,
|
||||
):
|
||||
# o: mv, mv_m, mv_w, mv_t
|
||||
o = f'{order_by}_{time}' if time != JmMagicConstants.TIME_ALL else order_by
|
||||
|
||||
params = {
|
||||
'page': page,
|
||||
'order': '', # 该参数为空
|
||||
'c': category,
|
||||
'o': o,
|
||||
}
|
||||
|
||||
resp = self.req_api(self.append_params_to_url(self.API_CATEGORIES_FILTER, params))
|
||||
|
||||
return JmPageTool.parse_api_to_search_page(resp.model_data)
|
||||
|
||||
def get_album_detail(self, album_id) -> JmAlbumDetail:
|
||||
return self.fetch_detail_entity(album_id,
|
||||
JmModuleConfig.album_class(),
|
||||
|
@ -84,6 +84,7 @@ class JmImageResp(JmResp):
|
||||
|
||||
class JmJsonResp(JmResp):
|
||||
|
||||
@field_cache()
|
||||
def json(self) -> Dict:
|
||||
return self.resp.json()
|
||||
|
||||
@ -128,9 +129,6 @@ class JmAlbumCommentResp(JmJsonResp):
|
||||
def is_success(self) -> bool:
|
||||
return super().is_success and self.json()['err'] is False
|
||||
|
||||
def json(self) -> Dict:
|
||||
return self.resp.json()
|
||||
|
||||
|
||||
"""
|
||||
|
||||
@ -151,15 +149,6 @@ class JmDetailClient:
|
||||
) -> JmPhotoDetail:
|
||||
raise NotImplementedError
|
||||
|
||||
def of_api_url(self, api_path, domain):
|
||||
raise NotImplementedError
|
||||
|
||||
def set_cache_dict(self, cache_dict: Optional[Dict]):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_cache_dict(self) -> Optional[Dict]:
|
||||
raise NotImplementedError
|
||||
|
||||
def check_photo(self, photo: JmPhotoDetail):
|
||||
"""
|
||||
photo来源有两种:
|
||||
@ -381,12 +370,78 @@ class JmSearchAlbumClient:
|
||||
return self.search(search_query, page, 4, order_by, time)
|
||||
|
||||
|
||||
class JmCategoryClient:
|
||||
"""
|
||||
该接口可以看作是对全体禁漫本子的排行,热门排行的功能也派生于此
|
||||
|
||||
月排行 = 分类【时间=月,排序=观看】
|
||||
周排行 = 分类【时间=周,排序=观看】
|
||||
日排行 = 分类【时间=周,排序=观看】
|
||||
"""
|
||||
|
||||
def categories_filter(self,
|
||||
page: int,
|
||||
time: str,
|
||||
category: str,
|
||||
order_by: str,
|
||||
) -> JmCategoryPage:
|
||||
"""
|
||||
分类
|
||||
|
||||
:param page: 页码
|
||||
:param time: 时间范围,默认是全部时间
|
||||
:param category: 类别,默认是最新,即显示最新的禁漫本子
|
||||
:param order_by: 排序方式,默认是观看数
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def month_ranking(self,
|
||||
page: int,
|
||||
category: str = JmMagicConstants.CATEGORY_ALL,
|
||||
):
|
||||
"""
|
||||
月排行 = 分类【时间=月,排序=观看】
|
||||
"""
|
||||
return self.categories_filter(page,
|
||||
JmMagicConstants.TIME_MONTH,
|
||||
category,
|
||||
JmMagicConstants.ORDER_BY_VIEW,
|
||||
)
|
||||
|
||||
def week_ranking(self,
|
||||
page: int,
|
||||
category: str = JmMagicConstants.CATEGORY_ALL,
|
||||
):
|
||||
"""
|
||||
周排行 = 分类【时间=周,排序=观看】
|
||||
"""
|
||||
return self.categories_filter(page,
|
||||
JmMagicConstants.TIME_WEEK,
|
||||
category,
|
||||
JmMagicConstants.ORDER_BY_VIEW,
|
||||
)
|
||||
|
||||
def day_ranking(self,
|
||||
page: int,
|
||||
category: str = JmMagicConstants.CATEGORY_ALL,
|
||||
):
|
||||
"""
|
||||
日排行 = 分类【时间=日,排序=观看】
|
||||
"""
|
||||
return self.categories_filter(page,
|
||||
JmMagicConstants.TIME_TODAY,
|
||||
category,
|
||||
JmMagicConstants.ORDER_BY_VIEW,
|
||||
)
|
||||
|
||||
|
||||
# noinspection PyAbstractClass
|
||||
class JmcomicClient(
|
||||
JmImageClient,
|
||||
JmDetailClient,
|
||||
JmUserClient,
|
||||
JmSearchAlbumClient,
|
||||
JmCategoryClient,
|
||||
Postman,
|
||||
):
|
||||
client_key: None
|
||||
@ -403,6 +458,15 @@ class JmcomicClient(
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def set_cache_dict(self, cache_dict: Optional[Dict]):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_cache_dict(self) -> Optional[Dict]:
|
||||
raise NotImplementedError
|
||||
|
||||
def of_api_url(self, api_path, domain):
|
||||
raise NotImplementedError
|
||||
|
||||
def get_html_domain(self, postman=None):
|
||||
return JmModuleConfig.get_html_domain(postman or self.get_root_postman())
|
||||
|
||||
@ -487,3 +551,20 @@ class JmcomicClient(
|
||||
}
|
||||
|
||||
yield from self.do_page_iter(params, page, self.search)
|
||||
|
||||
def categories_filter_gen(self,
|
||||
page: int = 1,
|
||||
time: str = JmMagicConstants.TIME_ALL,
|
||||
category: str = JmMagicConstants.CATEGORY_ALL,
|
||||
order_by: str = JmMagicConstants.ORDER_BY_LATEST,
|
||||
) -> Generator[JmCategoryPage, Dict, None]:
|
||||
"""
|
||||
见 search_gen
|
||||
"""
|
||||
params = {
|
||||
'time': time,
|
||||
'category': category,
|
||||
'order_by': order_by,
|
||||
}
|
||||
|
||||
yield from self.do_page_iter(params, page, self.categories_filter)
|
||||
|
@ -22,12 +22,28 @@ class JmMagicConstants:
|
||||
ORDER_BY_PICTURE = 'mp'
|
||||
ORDER_BY_LIKE = 'tf'
|
||||
|
||||
ORDER_MONTH_RANKING = 'mv_m'
|
||||
ORDER_WEEK_RANKING = 'mv_w'
|
||||
ORDER_DAY_RANKING = 'mv_t'
|
||||
|
||||
# 搜索参数-时间段
|
||||
TIME_TODAY = 't'
|
||||
TIME_WEEK = 'w'
|
||||
TIME_MONTH = 'm'
|
||||
TIME_ALL = 'a'
|
||||
|
||||
# 全部, 同人, 单本, 短篇, 其他, 韩漫, 美漫, cosplay, 3D
|
||||
# category = ["0", "doujin", "single", "short", "another", "hanman", "meiman", "doujin_cosplay", "3D"]
|
||||
CATEGORY_ALL = '0'
|
||||
CATEGORY_DOUJIN = 'doujin'
|
||||
CATEGORY_SINGLE = 'single'
|
||||
CATEGORY_SHORT = 'short'
|
||||
CATEGORY_ANOTHER = 'another'
|
||||
CATEGORY_HANMAN = 'hanman'
|
||||
CATEGORY_MEIMAN = 'meiman'
|
||||
CATEGORY_DOUJIN_COSPLAY = 'doujin_cosplay'
|
||||
CATEGORY_3D = '3D'
|
||||
|
||||
# 分页大小
|
||||
PAGE_SIZE_SEARCH = 80
|
||||
PAGE_SIZE_FAVORITE = 20
|
||||
|
@ -617,6 +617,9 @@ class JmSearchPage(JmPageContent):
|
||||
return page
|
||||
|
||||
|
||||
JmCategoryPage = JmSearchPage
|
||||
|
||||
|
||||
class JmFavoritePage(JmPageContent):
|
||||
|
||||
def __init__(self, content, folder_list, total):
|
||||
|
@ -220,6 +220,7 @@ class JmOption:
|
||||
break
|
||||
|
||||
from os.path import join
|
||||
# noinspection PyTypeChecker
|
||||
return join(*dir_layer)
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
@ -293,6 +294,7 @@ class JmOption:
|
||||
|
||||
# version
|
||||
version = dic.pop('version', None)
|
||||
# noinspection PyTypeChecker
|
||||
if version is not None and float(version) >= float(JmModuleConfig.JM_OPTION_VER):
|
||||
# 版本号更高,跳过兼容代码
|
||||
return cls(**dic)
|
||||
|
@ -116,10 +116,6 @@ class JmcomicText:
|
||||
JmModuleConfig.album_class()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def analyse_jm_search_html(cls, html: str) -> JmSearchPage:
|
||||
return JmPageTool.parse_html_to_search_page(html)
|
||||
|
||||
@classmethod
|
||||
def reflect_new_instance(cls, html: str, cls_field_prefix: str, clazz: type):
|
||||
|
||||
@ -347,6 +343,15 @@ class JmPageTool:
|
||||
r'(<a[\s\S]*?) </div>'
|
||||
)
|
||||
|
||||
# 用来提取分类页面的的album的信息
|
||||
pattern_html_category_album_info_list = compile(
|
||||
r'<a href="/album/(\d+)/[^>]*>[\s\S]*?title="(.*?)"[^>]*>'
|
||||
r'\n</a>\n'
|
||||
r'<div class="label-loveicon">'
|
||||
r'([\s\S]*?)'
|
||||
r'<div class="clearfix">'
|
||||
)
|
||||
|
||||
# 用来查找tag列表
|
||||
pattern_html_search_tag_list = compile(r'<a[^>]*?>(.*?)</a>')
|
||||
|
||||
@ -405,6 +410,26 @@ class JmPageTool:
|
||||
|
||||
return JmSearchPage(content, total)
|
||||
|
||||
@classmethod
|
||||
def parse_html_to_category_page(cls, html: str) -> JmSearchPage:
|
||||
# 3. 提取结果
|
||||
content = []
|
||||
total = int(PatternTool.match_or_default(html, *cls.pattern_html_search_total))
|
||||
|
||||
album_info_list = cls.pattern_html_category_album_info_list.findall(html)
|
||||
|
||||
for (album_id, title, tag_text) in album_info_list:
|
||||
tag_list = cls.pattern_html_search_tag_list.findall(tag_text)
|
||||
content.append((
|
||||
album_id, {
|
||||
'name': title, # 改成name是为了兼容 parse_api_resp_to_page
|
||||
'tag_list': tag_list
|
||||
}
|
||||
))
|
||||
|
||||
return JmSearchPage(content, total)
|
||||
|
||||
|
||||
@classmethod
|
||||
def parse_html_to_favorite_page(cls, html: str) -> JmFavoritePage:
|
||||
total = int(PatternTool.require_match(
|
||||
|
Loading…
Reference in New Issue
Block a user