mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v2.3.5: 优化章节去重算法,优化实体类字段,增加dispatch工作流的可输入配置 (#146)
This commit is contained in:
parent
fc3842f2ba
commit
df4111295f
11
.github/workflows/download_dispatch.yml
vendored
11
.github/workflows/download_dispatch.yml
vendored
@ -10,12 +10,18 @@ on:
|
||||
|
||||
JM_PHOTO_IDS:
|
||||
type: string
|
||||
description: 章节id(可以单独下载某个本子的某个章节,多个id用-隔开(同上))
|
||||
description: 章节id(单独下载章节,多个id同上)
|
||||
required: false
|
||||
|
||||
DIR_RULE:
|
||||
type: string
|
||||
description: 下载文件夹规则,对应option配置文件里的dir_rule.rule。此处可以不填,默认使用repo里的配置文件的`Bd_Aauthor_Atitle_Pindex`。
|
||||
description: 下载文件夹规则(dir_rule.rule)。此处可以不填,默认使用配置文件的'Bd_Aauthor_Atitle_Pindex'。
|
||||
default: ''
|
||||
required: false
|
||||
|
||||
CLIENT_IMPL:
|
||||
type: string
|
||||
description: 客户端类型(client.impl),[api]=移动端,[html]=网页端,此处可以不填,默认使用'html'。如果你发现默认的下载不了,可以填api试试。
|
||||
default: ''
|
||||
required: false
|
||||
|
||||
@ -52,6 +58,7 @@ jobs:
|
||||
JM_ALBUM_IDS: ${{ github.event.inputs.JM_ALBUM_IDS }}
|
||||
JM_PHOTO_IDS: ${{ github.event.inputs.JM_PHOTO_IDS }}
|
||||
DIR_RULE: ${{ github.event.inputs.DIR_RULE }}
|
||||
CLIENT_IMPL: ${{ github.event.inputs.CLIENT_IMPL }}
|
||||
ZIP_NAME: ${{ github.event.inputs.ZIP_NAME }}
|
||||
UPLOAD_NAME: ${{ github.event.inputs.UPLOAD_NAME }}
|
||||
# secrets
|
||||
|
||||
@ -3,15 +3,10 @@
|
||||
|
||||
client:
|
||||
impl: api
|
||||
domain: [
|
||||
"www.jmapinode1.cc",
|
||||
"www.jmapinode2.cc",
|
||||
"www.jmapinode3.cc",
|
||||
"www.jmapibranch2.cc"
|
||||
]
|
||||
retry_times: 3
|
||||
postman:
|
||||
meta_data:
|
||||
timeout: 10
|
||||
timeout: 5
|
||||
|
||||
# 插件配置
|
||||
plugin:
|
||||
|
||||
@ -2,12 +2,14 @@
|
||||
# 网页端配置
|
||||
|
||||
client:
|
||||
impl: html
|
||||
retry_times: 3
|
||||
postman:
|
||||
meta_data:
|
||||
timeout: 5
|
||||
domain:
|
||||
- jmcomic1.me
|
||||
- jmcomic.me
|
||||
postman:
|
||||
meta_data:
|
||||
timeout: 10
|
||||
|
||||
# 插件配置
|
||||
plugin:
|
||||
|
||||
@ -4,13 +4,9 @@ dir_rule:
|
||||
rule: Bd_Aauthor_Atitle_Pindex
|
||||
|
||||
client:
|
||||
impl: api
|
||||
domain: [
|
||||
"www.jmapinode1.cc",
|
||||
"www.jmapinode2.cc",
|
||||
"www.jmapinode3.cc",
|
||||
"www.jmapibranch2.cc",
|
||||
]
|
||||
domain:
|
||||
- jmcomic1.me
|
||||
- jmcomic.me
|
||||
|
||||
# 插件配置
|
||||
plugin:
|
||||
|
||||
@ -1,4 +1,7 @@
|
||||
client:
|
||||
# impl: 客户端实现类,不配默认是html,表示网页端
|
||||
impl: html
|
||||
|
||||
# domain: 域名配置,默认是 [],表示运行时自动获取域名。
|
||||
# 可配置特定域名,如下:
|
||||
# 程序会先用第一个域名,如果第一个域名重试n次失败,则换下一个域名重试,以此类推。
|
||||
|
||||
@ -16,6 +16,8 @@
|
||||
|
||||
直接拉到页面最底部,如下所示:
|
||||
|
||||
(最新提示,下图的1可以不做,即直接点绿色的Create fork按钮)
|
||||
|
||||

|
||||
|
||||
## 2. 填写你需要下载的本子id
|
||||
@ -30,8 +32,6 @@
|
||||
|
||||

|
||||
|
||||
|
||||
|
||||
### 2.2. 方式二
|
||||
|
||||
访问下面这个网址:
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.3.4'
|
||||
__version__ = '2.3.5'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
||||
@ -24,6 +24,10 @@ class AbstractJmClient(
|
||||
fallback_domain_list.insert(0, domain)
|
||||
|
||||
self.domain_list = fallback_domain_list
|
||||
self.after_init()
|
||||
|
||||
def after_init(self):
|
||||
pass
|
||||
|
||||
def get(self, url, **kwargs):
|
||||
return self.request_with_retry(self.postman.get, url, **kwargs)
|
||||
@ -104,31 +108,35 @@ class AbstractJmClient(
|
||||
jm_debug('req.error', str(e))
|
||||
|
||||
def enable_cache(self, debug=False):
|
||||
def wrap_func_cache(func_name, cache_dict_name):
|
||||
import common
|
||||
if common.VERSION > '0.4.8':
|
||||
if hasattr(self, cache_dict_name):
|
||||
return
|
||||
if self.is_cache_enabled():
|
||||
return
|
||||
|
||||
cache = common.cache
|
||||
cache_dict = {}
|
||||
cache_hit_msg = (f'【缓存命中】{cache_dict_name} ' + '→ [{}]') if debug is True else None
|
||||
cache_miss_msg = (f'【缓存缺失】{cache_dict_name} ' + '← [{}]') if debug is True else None
|
||||
cache = cache(
|
||||
cache_dict=cache_dict,
|
||||
cache_hit_msg=cache_hit_msg,
|
||||
cache_miss_msg=cache_miss_msg,
|
||||
)
|
||||
setattr(self, cache_dict_name, cache_dict)
|
||||
def wrap_func_cache(func_name, cache_dict_name):
|
||||
if hasattr(self, cache_dict_name):
|
||||
return
|
||||
|
||||
if sys.version_info > (3, 9):
|
||||
import functools
|
||||
cache = functools.cache
|
||||
else:
|
||||
if sys.version_info < (3, 9):
|
||||
import common
|
||||
if common.VERSION > '0.4.8':
|
||||
cache = common.cache
|
||||
cache_dict = {}
|
||||
cache_hit_msg = (f'【缓存命中】{cache_dict_name} ' + '→ [{}]') if debug is True else None
|
||||
cache_miss_msg = (f'【缓存缺失】{cache_dict_name} ' + '← [{}]') if debug is True else None
|
||||
cache = cache(
|
||||
cache_dict=cache_dict,
|
||||
cache_hit_msg=cache_hit_msg,
|
||||
cache_miss_msg=cache_miss_msg,
|
||||
)
|
||||
setattr(self, cache_dict_name, cache_dict)
|
||||
else:
|
||||
ExceptionTool.raises('不支持启用JmcomicClient缓存。\n'
|
||||
'请更新python版本到3.9以上,'
|
||||
'或更新commonX: `pip install commonX --upgrade`')
|
||||
import functools
|
||||
cache = functools.cache
|
||||
return
|
||||
|
||||
# 重载本对象的方法
|
||||
func = getattr(self, func_name)
|
||||
wrap_func = cache(func)
|
||||
|
||||
@ -399,14 +407,6 @@ class JmApiClient(AbstractJmClient):
|
||||
API_CHAPTER = '/chapter'
|
||||
API_SCRAMBLE = '/chapter_view_template'
|
||||
|
||||
def __init__(self,
|
||||
postman: Postman,
|
||||
retry_times: int,
|
||||
domain=None,
|
||||
fallback_domain_list=None,
|
||||
):
|
||||
super().__init__(postman, retry_times, domain, fallback_domain_list)
|
||||
|
||||
def search(self,
|
||||
search_query: str,
|
||||
page: int,
|
||||
@ -445,19 +445,34 @@ class JmApiClient(AbstractJmClient):
|
||||
)
|
||||
|
||||
def get_photo_detail(self, photo_id, fetch_album=True) -> JmPhotoDetail:
|
||||
photo_id = JmcomicText.parse_to_photo_id(photo_id)
|
||||
photo: JmPhotoDetail = self.fetch_detail_entity(photo_id,
|
||||
JmModuleConfig.photo_class(),
|
||||
)
|
||||
photo.scramble_id = self.get_scramble_id(photo.photo_id)
|
||||
self.fetch_photo_additional_field(photo, fetch_album)
|
||||
return photo
|
||||
|
||||
def get_scramble_id(self, photo_id):
|
||||
"""
|
||||
带有缓存的fetch_scramble_id,缓存位于JmModuleConfig.SCRAMBLE_CACHE
|
||||
"""
|
||||
cache = JmModuleConfig.SCRAMBLE_CACHE
|
||||
if photo_id in cache:
|
||||
return cache[photo_id]
|
||||
|
||||
scramble_id = self.fetch_scramble_id(photo_id)
|
||||
cache[photo_id] = scramble_id
|
||||
return scramble_id
|
||||
|
||||
def fetch_detail_entity(self, apid, clazz, **kwargs):
|
||||
"""
|
||||
请求实体类
|
||||
"""
|
||||
apid = JmcomicText.parse_to_album_id(apid)
|
||||
url = self.API_ALBUM if issubclass(clazz, JmAlbumDetail) else self.API_CHAPTER
|
||||
resp = self.get_decode(
|
||||
url,
|
||||
params={
|
||||
'id': JmcomicText.parse_to_album_id(apid),
|
||||
'id': apid,
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
@ -466,11 +481,11 @@ class JmApiClient(AbstractJmClient):
|
||||
|
||||
return JmApiAdaptTool.parse_entity(resp.res_data, clazz)
|
||||
|
||||
def get_scramble_id(self, photo_id):
|
||||
cache = JmModuleConfig.SCRAMBLE_CACHE
|
||||
if photo_id in cache:
|
||||
return cache[photo_id]
|
||||
|
||||
def fetch_scramble_id(self, photo_id):
|
||||
"""
|
||||
请求scramble_id
|
||||
"""
|
||||
photo_id: str = JmcomicText.parse_to_photo_id(photo_id)
|
||||
resp = self.get_decode(
|
||||
self.API_SCRAMBLE,
|
||||
params={
|
||||
@ -487,11 +502,43 @@ class JmApiClient(AbstractJmClient):
|
||||
scramble_id = match[1]
|
||||
else:
|
||||
jm_debug('api.scramble', '未从响应中匹配到scramble_id,返回默认值220980')
|
||||
scramble_id = 220980
|
||||
scramble_id = '220980'
|
||||
|
||||
cache[photo_id] = scramble_id
|
||||
return scramble_id
|
||||
|
||||
def fetch_photo_additional_field(self, photo: JmPhotoDetail, fetch_album: bool):
|
||||
"""
|
||||
获取章节的额外信息
|
||||
1. scramble_id
|
||||
2. album
|
||||
|
||||
这里的难点是,是否要采用异步的方式并发请求。
|
||||
"""
|
||||
aid = photo.album_id
|
||||
pid = photo.photo_id
|
||||
scramble_cache = JmModuleConfig.SCRAMBLE_CACHE
|
||||
|
||||
if fetch_album is False and pid in scramble_cache:
|
||||
# 不用发请求,直接返回
|
||||
photo.scramble_id = scramble_cache[pid]
|
||||
return
|
||||
|
||||
if fetch_album is True and pid not in scramble_cache:
|
||||
# 要发起两个请求,这里实现很简易,直接排队请求
|
||||
# todo: 改进实现
|
||||
# 1. 直接开两个线程跑
|
||||
# 2. 开两个线程,但是开之前检查重复性
|
||||
# 3. 线程池,也要检查重复性
|
||||
# 23做法要改不止一处地方
|
||||
photo.from_album = self.get_scramble_id(pid)
|
||||
photo.scramble_id = self.get_album_detail(aid)
|
||||
return
|
||||
|
||||
if fetch_album is True:
|
||||
photo.from_album = self.get_album_detail(aid)
|
||||
else:
|
||||
photo.scramble_id = self.get_scramble_id(pid)
|
||||
|
||||
def get_decode(self, url, **kwargs) -> JmApiResp:
|
||||
# set headers
|
||||
headers, key_ts = self.headers_key_ts
|
||||
|
||||
@ -154,20 +154,20 @@ class JmPhotoDetail(DetailEntity):
|
||||
photo_id,
|
||||
scramble_id,
|
||||
name,
|
||||
keywords,
|
||||
series_id,
|
||||
sort,
|
||||
tags='',
|
||||
page_arr=None,
|
||||
data_original_domain=None,
|
||||
data_original_0=None,
|
||||
author=None,
|
||||
from_album=None,
|
||||
):
|
||||
self.photo_id: str = photo_id
|
||||
self.scramble_id: str = scramble_id
|
||||
self.photo_id: str = str(photo_id)
|
||||
self.scramble_id: str = str(scramble_id)
|
||||
self.name: str = str(name).strip()
|
||||
self.sort: int = int(sort)
|
||||
self._keywords: str = keywords
|
||||
self._tags: str = tags
|
||||
self._series_id: int = int(series_id)
|
||||
|
||||
self._author: StrNone = author
|
||||
@ -204,7 +204,13 @@ class JmPhotoDetail(DetailEntity):
|
||||
if self.from_album is not None:
|
||||
return self.from_album.tags
|
||||
|
||||
return self._keywords.split(',')
|
||||
tag_str = self._tags
|
||||
if ',' in tag_str:
|
||||
# html
|
||||
return tag_str.split(',')
|
||||
else:
|
||||
# api
|
||||
return tag_str.split()
|
||||
|
||||
@property
|
||||
def indextitle(self):
|
||||
@ -262,7 +268,7 @@ class JmPhotoDetail(DetailEntity):
|
||||
例如:img_name = 01111.webp
|
||||
返回:https://cdn-msp2.18comic.org/media/photos/147643/01111.webp
|
||||
"""
|
||||
domain = self.data_original_domain or None
|
||||
domain = self.data_original_domain
|
||||
|
||||
from .jm_toolkit import ExceptionTool
|
||||
ExceptionTool.require_true(domain is not None, f'图片域名为空: {domain}')
|
||||
@ -319,26 +325,27 @@ class JmAlbumDetail(DetailEntity):
|
||||
self.album_id: str = str(album_id)
|
||||
self.scramble_id: str = str(scramble_id)
|
||||
self.name: str = name
|
||||
self.page_count = int(page_count) # 总页数
|
||||
self.page_count: int = int(page_count) # 总页数
|
||||
self.pub_date: str = pub_date # 发布日期
|
||||
self.update_date: str = update_date # 更新日期
|
||||
|
||||
self.likes: str = likes # [1K] 點擊喜歡
|
||||
self.views: str = views # [40K] 次觀看
|
||||
self.comment_count: int = self.__parse_comment_count(comment_count) # 评论数
|
||||
self.comment_count: int = int(comment_count) # 评论数
|
||||
self.works: List[str] = works # 作品
|
||||
self.actors: List[str] = actors # 登場人物
|
||||
self.tags: List[str] = tags # 標籤
|
||||
self.authors: List[str] = authors # 作者
|
||||
|
||||
# 有的 album 没有章节,则自成一章。
|
||||
episode_list: List[Tuple[str, str, str, str]]
|
||||
if len(episode_list) == 0:
|
||||
# photo_id, photo_index, photo_title, photo_pub_date
|
||||
episode_list = [(album_id, 1, name, pub_date)]
|
||||
episode_list = [(album_id, "1", name, pub_date)]
|
||||
else:
|
||||
episode_list = self.distinct_episode(episode_list)
|
||||
|
||||
self.episode_list: List[Tuple] = episode_list
|
||||
self.episode_list = episode_list
|
||||
self.related_list = related_list
|
||||
|
||||
@property
|
||||
@ -357,26 +364,20 @@ class JmAlbumDetail(DetailEntity):
|
||||
return self.album_id
|
||||
|
||||
@staticmethod
|
||||
def distinct_episode(episode_list):
|
||||
ret = []
|
||||
def distinct_episode(episode_list: list):
|
||||
"""
|
||||
去重章节
|
||||
photo_id, photo_index, photo_title, photo_pub_date
|
||||
"""
|
||||
episode_list.sort(key=lambda e: int(e[1])) # 按照photo_index排序
|
||||
ret = [episode_list[0]]
|
||||
|
||||
def not_exist(episode):
|
||||
photo_id = episode[0]
|
||||
for each in ret:
|
||||
if each[0] == photo_id:
|
||||
return False
|
||||
return True
|
||||
|
||||
for episode in episode_list:
|
||||
if not_exist(episode):
|
||||
ret.append(episode)
|
||||
for i in range(1, len(episode_list)):
|
||||
if ret[-1][1] != episode_list[i][1]:
|
||||
ret.append(episode_list[i])
|
||||
|
||||
return ret
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def __parse_comment_count(self, comment_count: str) -> int:
|
||||
return int(comment_count)
|
||||
|
||||
def create_photo_detail(self, index) -> Tuple[JmPhotoDetail, Tuple]:
|
||||
# 校验参数
|
||||
length = len(self.episode_list)
|
||||
@ -391,13 +392,9 @@ class JmAlbumDetail(DetailEntity):
|
||||
photo_id=pid,
|
||||
scramble_id=self.scramble_id,
|
||||
name=pname,
|
||||
keywords='',
|
||||
series_id=self.album_id,
|
||||
sort=pindex,
|
||||
author=self.author,
|
||||
from_album=self,
|
||||
page_arr=None,
|
||||
data_original_domain=None
|
||||
)
|
||||
|
||||
return photo, (self.episode_list[index])
|
||||
|
||||
@ -27,12 +27,10 @@ class JmOptionPlugin:
|
||||
return cls(option)
|
||||
|
||||
|
||||
"""
|
||||
扩展功能:登录禁漫,并保存登录后的cookies,让所有client都带上此cookies
|
||||
"""
|
||||
|
||||
|
||||
class JmLoginPlugin(JmOptionPlugin):
|
||||
"""
|
||||
功能:登录禁漫,并保存登录后的cookies,让所有client都带上此cookies
|
||||
"""
|
||||
plugin_key = 'login'
|
||||
|
||||
def invoke(self, username, password) -> None:
|
||||
@ -152,8 +150,10 @@ class UsageLogPlugin(JmOptionPlugin):
|
||||
sleep(interval)
|
||||
|
||||
|
||||
# 参考: https://github.com/hect0x7/JMComic-Crawler-Python/issues/95
|
||||
class FindUpdatePlugin(JmOptionPlugin):
|
||||
"""
|
||||
参考: https://github.com/hect0x7/JMComic-Crawler-Python/issues/95
|
||||
"""
|
||||
plugin_key = 'find_update'
|
||||
|
||||
def invoke(self, **kwargs) -> None:
|
||||
|
||||
@ -14,7 +14,7 @@ class JmcomicText:
|
||||
# pattern_html_photo_data_original_list = compile('data-original="(.*?)" id="album_photo_.+?"')
|
||||
pattern_html_photo_data_original_domain = compile('src="https://(.*?)/media/albums/blank')
|
||||
pattern_html_photo_data_original_0 = compile('data-original="(.*?)"[^>]*?id="album_photo[^>]*?data-page="0"')
|
||||
pattern_html_photo_keywords = compile('<meta name="keywords"[\s\S]*?content="(.*?)"')
|
||||
pattern_html_photo_tags = compile('<meta name="keywords"[\s\S]*?content="(.*?)"')
|
||||
pattern_html_photo_series_id = compile('var series_id = (\d+);')
|
||||
pattern_html_photo_sort = compile('var sort = (\d+);')
|
||||
pattern_html_photo_page_arr = compile('var page_arr = (.*?);')
|
||||
@ -401,7 +401,7 @@ class JmApiAdaptTool:
|
||||
JmPhotoDetail: [
|
||||
'name',
|
||||
'series_id',
|
||||
('tags', 'keywords'),
|
||||
'tags',
|
||||
('id', 'photo_id'),
|
||||
('images', 'page_arr'),
|
||||
|
||||
|
||||
@ -1,11 +0,0 @@
|
||||
from test_jmcomic import *
|
||||
|
||||
|
||||
class Test_MobileClient(JmTestConfigurable):
|
||||
|
||||
def test_cl(self):
|
||||
for cmd in str_to_list('''
|
||||
jmcomic 438516
|
||||
jmcomic 438516 p438516
|
||||
'''):
|
||||
self.assertEqual(os.system(cmd), 0)
|
||||
@ -28,7 +28,7 @@ class Test_Custom(JmTestConfigurable):
|
||||
dir_rule = DirRule('Bd_Aaname_Ppname', base_dir)
|
||||
# noinspection PyTypeChecker
|
||||
save_dir = dir_rule.deside_image_save_dir(
|
||||
MyAlbum('1', *['0'] * 13),
|
||||
MyAlbum('1', '0', '0', [], *['0'] * 10),
|
||||
MyPhoto('2', *['0'] * 7)
|
||||
)
|
||||
|
||||
|
||||
@ -65,6 +65,15 @@ def cover_option_config(option: JmOption):
|
||||
the_new = DirRule(dir_rule, base_dir=the_old.base_dir)
|
||||
option.dir_rule = the_new
|
||||
|
||||
impl = get_env('CLIENT_IMPL', None)
|
||||
if impl is not None:
|
||||
option.client.impl = impl
|
||||
else:
|
||||
impl = option.client.impl
|
||||
|
||||
if impl == 'api':
|
||||
option.client.domain = JmModuleConfig.DOMAIN_API_LIST
|
||||
|
||||
|
||||
def login_if_configured(option):
|
||||
# 检查环境变量中是否有禁漫的用户名和密码,如果有则登录
|
||||
|
||||
Loading…
Reference in New Issue
Block a user