v2.3.5: 优化章节去重算法,优化实体类字段,增加dispatch工作流的可输入配置 (#146)

This commit is contained in:
hect0x7 2023-10-02 03:36:40 +08:00 committed by GitHub
parent fc3842f2ba
commit df4111295f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 154 additions and 109 deletions

View File

@ -10,12 +10,18 @@ on:
JM_PHOTO_IDS:
type: string
description: 章节id可以单独下载某个本子的某个章节多个id用-隔开(同上
description: 章节id单独下载章节多个id同上
required: false
DIR_RULE:
type: string
description: 下载文件夹规则对应option配置文件里的dir_rule.rule。此处可以不填默认使用repo里的配置文件的`Bd_Aauthor_Atitle_Pindex`。
description: 下载文件夹规则dir_rule.rule。此处可以不填默认使用配置文件的'Bd_Aauthor_Atitle_Pindex'。
default: ''
required: false
CLIENT_IMPL:
type: string
description: 客户端类型client.impl[api]=移动端,[html]=网页端,此处可以不填,默认使用'html'。如果你发现默认的下载不了可以填api试试。
default: ''
required: false
@ -52,6 +58,7 @@ jobs:
JM_ALBUM_IDS: ${{ github.event.inputs.JM_ALBUM_IDS }}
JM_PHOTO_IDS: ${{ github.event.inputs.JM_PHOTO_IDS }}
DIR_RULE: ${{ github.event.inputs.DIR_RULE }}
CLIENT_IMPL: ${{ github.event.inputs.CLIENT_IMPL }}
ZIP_NAME: ${{ github.event.inputs.ZIP_NAME }}
UPLOAD_NAME: ${{ github.event.inputs.UPLOAD_NAME }}
# secrets

View File

@ -3,15 +3,10 @@
client:
impl: api
domain: [
"www.jmapinode1.cc",
"www.jmapinode2.cc",
"www.jmapinode3.cc",
"www.jmapibranch2.cc"
]
retry_times: 3
postman:
meta_data:
timeout: 10
timeout: 5
# 插件配置
plugin:

View File

@ -2,12 +2,14 @@
# 网页端配置
client:
impl: html
retry_times: 3
postman:
meta_data:
timeout: 5
domain:
- jmcomic1.me
- jmcomic.me
postman:
meta_data:
timeout: 10
# 插件配置
plugin:

View File

@ -4,13 +4,9 @@ dir_rule:
rule: Bd_Aauthor_Atitle_Pindex
client:
impl: api
domain: [
"www.jmapinode1.cc",
"www.jmapinode2.cc",
"www.jmapinode3.cc",
"www.jmapibranch2.cc",
]
domain:
- jmcomic1.me
- jmcomic.me
# 插件配置
plugin:

View File

@ -1,4 +1,7 @@
client:
# impl: 客户端实现类不配默认是html表示网页端
impl: html
# domain: 域名配置,默认是 [],表示运行时自动获取域名。
# 可配置特定域名,如下:
# 程序会先用第一个域名如果第一个域名重试n次失败则换下一个域名重试以此类推。

View File

@ -16,6 +16,8 @@
直接拉到页面最底部,如下所示:
最新提示下图的1可以不做即直接点绿色的Create fork按钮
![1](./images/1.png)
## 2. 填写你需要下载的本子id
@ -30,8 +32,6 @@
![5](./images/5.png)
### 2.2. 方式二
访问下面这个网址:

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.3.4'
__version__ = '2.3.5'
from .api import *
from .jm_plugin import *

View File

@ -24,6 +24,10 @@ class AbstractJmClient(
fallback_domain_list.insert(0, domain)
self.domain_list = fallback_domain_list
self.after_init()
def after_init(self):
pass
def get(self, url, **kwargs):
return self.request_with_retry(self.postman.get, url, **kwargs)
@ -104,31 +108,35 @@ class AbstractJmClient(
jm_debug('req.error', str(e))
def enable_cache(self, debug=False):
def wrap_func_cache(func_name, cache_dict_name):
import common
if common.VERSION > '0.4.8':
if hasattr(self, cache_dict_name):
return
if self.is_cache_enabled():
return
cache = common.cache
cache_dict = {}
cache_hit_msg = (f'【缓存命中】{cache_dict_name} ' + '→ [{}]') if debug is True else None
cache_miss_msg = (f'【缓存缺失】{cache_dict_name} ' + '← [{}]') if debug is True else None
cache = cache(
cache_dict=cache_dict,
cache_hit_msg=cache_hit_msg,
cache_miss_msg=cache_miss_msg,
)
setattr(self, cache_dict_name, cache_dict)
def wrap_func_cache(func_name, cache_dict_name):
if hasattr(self, cache_dict_name):
return
if sys.version_info > (3, 9):
import functools
cache = functools.cache
else:
if sys.version_info < (3, 9):
import common
if common.VERSION > '0.4.8':
cache = common.cache
cache_dict = {}
cache_hit_msg = (f'【缓存命中】{cache_dict_name} ' + '→ [{}]') if debug is True else None
cache_miss_msg = (f'【缓存缺失】{cache_dict_name} ' + '← [{}]') if debug is True else None
cache = cache(
cache_dict=cache_dict,
cache_hit_msg=cache_hit_msg,
cache_miss_msg=cache_miss_msg,
)
setattr(self, cache_dict_name, cache_dict)
else:
ExceptionTool.raises('不支持启用JmcomicClient缓存。\n'
'请更新python版本到3.9以上,'
'或更新commonX: `pip install commonX --upgrade`')
import functools
cache = functools.cache
return
# 重载本对象的方法
func = getattr(self, func_name)
wrap_func = cache(func)
@ -399,14 +407,6 @@ class JmApiClient(AbstractJmClient):
API_CHAPTER = '/chapter'
API_SCRAMBLE = '/chapter_view_template'
def __init__(self,
postman: Postman,
retry_times: int,
domain=None,
fallback_domain_list=None,
):
super().__init__(postman, retry_times, domain, fallback_domain_list)
def search(self,
search_query: str,
page: int,
@ -445,19 +445,34 @@ class JmApiClient(AbstractJmClient):
)
def get_photo_detail(self, photo_id, fetch_album=True) -> JmPhotoDetail:
photo_id = JmcomicText.parse_to_photo_id(photo_id)
photo: JmPhotoDetail = self.fetch_detail_entity(photo_id,
JmModuleConfig.photo_class(),
)
photo.scramble_id = self.get_scramble_id(photo.photo_id)
self.fetch_photo_additional_field(photo, fetch_album)
return photo
def get_scramble_id(self, photo_id):
"""
带有缓存的fetch_scramble_id缓存位于JmModuleConfig.SCRAMBLE_CACHE
"""
cache = JmModuleConfig.SCRAMBLE_CACHE
if photo_id in cache:
return cache[photo_id]
scramble_id = self.fetch_scramble_id(photo_id)
cache[photo_id] = scramble_id
return scramble_id
def fetch_detail_entity(self, apid, clazz, **kwargs):
"""
请求实体类
"""
apid = JmcomicText.parse_to_album_id(apid)
url = self.API_ALBUM if issubclass(clazz, JmAlbumDetail) else self.API_CHAPTER
resp = self.get_decode(
url,
params={
'id': JmcomicText.parse_to_album_id(apid),
'id': apid,
**kwargs,
}
)
@ -466,11 +481,11 @@ class JmApiClient(AbstractJmClient):
return JmApiAdaptTool.parse_entity(resp.res_data, clazz)
def get_scramble_id(self, photo_id):
cache = JmModuleConfig.SCRAMBLE_CACHE
if photo_id in cache:
return cache[photo_id]
def fetch_scramble_id(self, photo_id):
"""
请求scramble_id
"""
photo_id: str = JmcomicText.parse_to_photo_id(photo_id)
resp = self.get_decode(
self.API_SCRAMBLE,
params={
@ -487,11 +502,43 @@ class JmApiClient(AbstractJmClient):
scramble_id = match[1]
else:
jm_debug('api.scramble', '未从响应中匹配到scramble_id返回默认值220980')
scramble_id = 220980
scramble_id = '220980'
cache[photo_id] = scramble_id
return scramble_id
def fetch_photo_additional_field(self, photo: JmPhotoDetail, fetch_album: bool):
"""
获取章节的额外信息
1. scramble_id
2. album
这里的难点是是否要采用异步的方式并发请求
"""
aid = photo.album_id
pid = photo.photo_id
scramble_cache = JmModuleConfig.SCRAMBLE_CACHE
if fetch_album is False and pid in scramble_cache:
# 不用发请求,直接返回
photo.scramble_id = scramble_cache[pid]
return
if fetch_album is True and pid not in scramble_cache:
# 要发起两个请求,这里实现很简易,直接排队请求
# todo: 改进实现
# 1. 直接开两个线程跑
# 2. 开两个线程,但是开之前检查重复性
# 3. 线程池,也要检查重复性
# 23做法要改不止一处地方
photo.from_album = self.get_scramble_id(pid)
photo.scramble_id = self.get_album_detail(aid)
return
if fetch_album is True:
photo.from_album = self.get_album_detail(aid)
else:
photo.scramble_id = self.get_scramble_id(pid)
def get_decode(self, url, **kwargs) -> JmApiResp:
# set headers
headers, key_ts = self.headers_key_ts

View File

@ -154,20 +154,20 @@ class JmPhotoDetail(DetailEntity):
photo_id,
scramble_id,
name,
keywords,
series_id,
sort,
tags='',
page_arr=None,
data_original_domain=None,
data_original_0=None,
author=None,
from_album=None,
):
self.photo_id: str = photo_id
self.scramble_id: str = scramble_id
self.photo_id: str = str(photo_id)
self.scramble_id: str = str(scramble_id)
self.name: str = str(name).strip()
self.sort: int = int(sort)
self._keywords: str = keywords
self._tags: str = tags
self._series_id: int = int(series_id)
self._author: StrNone = author
@ -204,7 +204,13 @@ class JmPhotoDetail(DetailEntity):
if self.from_album is not None:
return self.from_album.tags
return self._keywords.split(',')
tag_str = self._tags
if ',' in tag_str:
# html
return tag_str.split(',')
else:
# api
return tag_str.split()
@property
def indextitle(self):
@ -262,7 +268,7 @@ class JmPhotoDetail(DetailEntity):
例如img_name = 01111.webp
返回https://cdn-msp2.18comic.org/media/photos/147643/01111.webp
"""
domain = self.data_original_domain or None
domain = self.data_original_domain
from .jm_toolkit import ExceptionTool
ExceptionTool.require_true(domain is not None, f'图片域名为空: {domain}')
@ -319,26 +325,27 @@ class JmAlbumDetail(DetailEntity):
self.album_id: str = str(album_id)
self.scramble_id: str = str(scramble_id)
self.name: str = name
self.page_count = int(page_count) # 总页数
self.page_count: int = int(page_count) # 总页数
self.pub_date: str = pub_date # 发布日期
self.update_date: str = update_date # 更新日期
self.likes: str = likes # [1K] 點擊喜歡
self.views: str = views # [40K] 次觀看
self.comment_count: int = self.__parse_comment_count(comment_count) # 评论数
self.comment_count: int = int(comment_count) # 评论数
self.works: List[str] = works # 作品
self.actors: List[str] = actors # 登場人物
self.tags: List[str] = tags # 標籤
self.authors: List[str] = authors # 作者
# 有的 album 没有章节,则自成一章。
episode_list: List[Tuple[str, str, str, str]]
if len(episode_list) == 0:
# photo_id, photo_index, photo_title, photo_pub_date
episode_list = [(album_id, 1, name, pub_date)]
episode_list = [(album_id, "1", name, pub_date)]
else:
episode_list = self.distinct_episode(episode_list)
self.episode_list: List[Tuple] = episode_list
self.episode_list = episode_list
self.related_list = related_list
@property
@ -357,26 +364,20 @@ class JmAlbumDetail(DetailEntity):
return self.album_id
@staticmethod
def distinct_episode(episode_list):
ret = []
def distinct_episode(episode_list: list):
"""
去重章节
photo_id, photo_index, photo_title, photo_pub_date
"""
episode_list.sort(key=lambda e: int(e[1])) # 按照photo_index排序
ret = [episode_list[0]]
def not_exist(episode):
photo_id = episode[0]
for each in ret:
if each[0] == photo_id:
return False
return True
for episode in episode_list:
if not_exist(episode):
ret.append(episode)
for i in range(1, len(episode_list)):
if ret[-1][1] != episode_list[i][1]:
ret.append(episode_list[i])
return ret
# noinspection PyMethodMayBeStatic
def __parse_comment_count(self, comment_count: str) -> int:
return int(comment_count)
def create_photo_detail(self, index) -> Tuple[JmPhotoDetail, Tuple]:
# 校验参数
length = len(self.episode_list)
@ -391,13 +392,9 @@ class JmAlbumDetail(DetailEntity):
photo_id=pid,
scramble_id=self.scramble_id,
name=pname,
keywords='',
series_id=self.album_id,
sort=pindex,
author=self.author,
from_album=self,
page_arr=None,
data_original_domain=None
)
return photo, (self.episode_list[index])

View File

@ -27,12 +27,10 @@ class JmOptionPlugin:
return cls(option)
"""
扩展功能登录禁漫并保存登录后的cookies让所有client都带上此cookies
"""
class JmLoginPlugin(JmOptionPlugin):
"""
功能登录禁漫并保存登录后的cookies让所有client都带上此cookies
"""
plugin_key = 'login'
def invoke(self, username, password) -> None:
@ -152,8 +150,10 @@ class UsageLogPlugin(JmOptionPlugin):
sleep(interval)
# 参考: https://github.com/hect0x7/JMComic-Crawler-Python/issues/95
class FindUpdatePlugin(JmOptionPlugin):
"""
参考: https://github.com/hect0x7/JMComic-Crawler-Python/issues/95
"""
plugin_key = 'find_update'
def invoke(self, **kwargs) -> None:

View File

@ -14,7 +14,7 @@ class JmcomicText:
# pattern_html_photo_data_original_list = compile('data-original="(.*?)" id="album_photo_.+?"')
pattern_html_photo_data_original_domain = compile('src="https://(.*?)/media/albums/blank')
pattern_html_photo_data_original_0 = compile('data-original="(.*?)"[^>]*?id="album_photo[^>]*?data-page="0"')
pattern_html_photo_keywords = compile('<meta name="keywords"[\s\S]*?content="(.*?)"')
pattern_html_photo_tags = compile('<meta name="keywords"[\s\S]*?content="(.*?)"')
pattern_html_photo_series_id = compile('var series_id = (\d+);')
pattern_html_photo_sort = compile('var sort = (\d+);')
pattern_html_photo_page_arr = compile('var page_arr = (.*?);')
@ -401,7 +401,7 @@ class JmApiAdaptTool:
JmPhotoDetail: [
'name',
'series_id',
('tags', 'keywords'),
'tags',
('id', 'photo_id'),
('images', 'page_arr'),

View File

@ -1,11 +0,0 @@
from test_jmcomic import *
class Test_MobileClient(JmTestConfigurable):
def test_cl(self):
for cmd in str_to_list('''
jmcomic 438516
jmcomic 438516 p438516
'''):
self.assertEqual(os.system(cmd), 0)

View File

@ -28,7 +28,7 @@ class Test_Custom(JmTestConfigurable):
dir_rule = DirRule('Bd_Aaname_Ppname', base_dir)
# noinspection PyTypeChecker
save_dir = dir_rule.deside_image_save_dir(
MyAlbum('1', *['0'] * 13),
MyAlbum('1', '0', '0', [], *['0'] * 10),
MyPhoto('2', *['0'] * 7)
)

View File

@ -65,6 +65,15 @@ def cover_option_config(option: JmOption):
the_new = DirRule(dir_rule, base_dir=the_old.base_dir)
option.dir_rule = the_new
impl = get_env('CLIENT_IMPL', None)
if impl is not None:
option.client.impl = impl
else:
impl = option.client.impl
if impl == 'api':
option.client.domain = JmModuleConfig.DOMAIN_API_LIST
def login_if_configured(option):
# 检查环境变量中是否有禁漫的用户名和密码,如果有则登录