mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.6.9: 增加下载本子封面图的函数以及download_cover插件; 修复client.after_init时advance_retry插件没有生效的问题(#472) (#482)
Some checks failed
Auto Release & Publish / release (push) Has been cancelled
Some checks failed
Auto Release & Publish / release (push) Has been cancelled
This commit is contained in:
parent
7943b9b66a
commit
f2adb33788
@ -109,12 +109,12 @@ dir_rule:
|
||||
# 规则: 根目录 / 本子id / 章节序号 / 图片文件
|
||||
# rule: 'Bd / Aid / Pindex'
|
||||
# rule: 'Bd_Aid_Pindex'
|
||||
|
||||
# 默认规则是: 根目录 / 章节标题 / 图片文件
|
||||
rule: Bd_Ptitle
|
||||
rule: Bd / Ptitle
|
||||
# jmcomic v2.5.36 以后,支持使用python的f-string的语法组合文件夹名,下为示例
|
||||
# rule: Bd / Aauthor / (JM{Aid}-{Pindex})-{Pname}
|
||||
# {}大括号里的内容同样是写 Axxx 或 Pxxx,其他语法自行参考python f-string的语法
|
||||
# 另外,rule开头的Bd可忽略不写,因为程序会自动插入Bd
|
||||
```
|
||||
|
||||
## 3. option插件配置项
|
||||
@ -194,6 +194,15 @@ plugins:
|
||||
album_photo_dict:
|
||||
324930: 424507
|
||||
|
||||
before_album:
|
||||
- plugin: download_cover # 额外下载本子封面的插件
|
||||
kwargs:
|
||||
size: '_3x4' # 可选项,禁漫搜索页的封面图尺寸是 4x3,和详情页不一样,想下搜索页的封面就设置此项
|
||||
dir_rule: # 封面图存放路径规则,写法同上
|
||||
base_dir: D:/a/b/c/
|
||||
rule: '{Atitle}/{Aid}_cover.jpg'
|
||||
|
||||
|
||||
after_album:
|
||||
- plugin: zip # 压缩文件插件
|
||||
kwargs:
|
||||
|
@ -38,7 +38,7 @@ download_album(123, option)
|
||||
option.download_album(123)
|
||||
```
|
||||
|
||||
## 获取本子/章节/图片的实体类,下载图片
|
||||
## 获取本子/章节/图片的实体类,下载图片/封面图
|
||||
|
||||
```python
|
||||
from jmcomic import *
|
||||
@ -49,23 +49,26 @@ client = JmOption.default().new_jm_client()
|
||||
# 本子实体类
|
||||
album: JmAlbumDetail = client.get_album_detail('427413')
|
||||
|
||||
# 下载本子封面图,保存为 cover.png (图片后缀可指定为jpg、webp等)
|
||||
client.download_album_cover('427413', './cover.png')
|
||||
|
||||
|
||||
def fetch(photo: JmPhotoDetail):
|
||||
# 章节实体类
|
||||
photo = client.get_photo_detail(photo.photo_id, False)
|
||||
print(f'章节id: {photo.photo_id}')
|
||||
|
||||
|
||||
# 图片实体类
|
||||
image: JmImageDetail
|
||||
for image in photo:
|
||||
print(f'图片url: {image.img_url}')
|
||||
|
||||
|
||||
# 下载单个图片
|
||||
client.download_by_image_detail(image, './a.jpg')
|
||||
# 如果是已知未混淆的图片,也可以直接使用url来下载
|
||||
random_image_domain = JmModuleConfig.DOMAIN_IMAGE_LIST[0]
|
||||
client.download_image(f'https://{random_image_domain}/media/albums/416130.jpg', './a.jpg')
|
||||
|
||||
|
||||
|
||||
# 多线程发起请求
|
||||
multi_thread_launcher(
|
||||
|
@ -27,7 +27,7 @@ plugins:
|
||||
proxy_client_key: photo_concurrent_fetcher_proxy
|
||||
whitelist: [ api, ]
|
||||
|
||||
- plugin: advanced-retry
|
||||
- plugin: advanced_retry
|
||||
kwargs:
|
||||
retry_config:
|
||||
retry_rounds: 3 # 一共对域名列表重试3轮
|
||||
|
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.6.7'
|
||||
__version__ = '2.6.9'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
@ -15,6 +15,7 @@ class AbstractJmClient(
|
||||
postman: Postman,
|
||||
domain_list: List[str],
|
||||
retry_times=0,
|
||||
domain_retry_strategy=None,
|
||||
):
|
||||
"""
|
||||
创建JM客户端
|
||||
@ -26,9 +27,11 @@ class AbstractJmClient(
|
||||
super().__init__(postman)
|
||||
self.retry_times = retry_times
|
||||
self.domain_list = domain_list
|
||||
self.domain_retry_strategy = None
|
||||
self.domain_retry_strategy = domain_retry_strategy
|
||||
self.CLIENT_CACHE = None
|
||||
self._username = None # help for favorite_folder method
|
||||
if domain_retry_strategy:
|
||||
domain_retry_strategy(self)
|
||||
self.enable_cache()
|
||||
self.after_init()
|
||||
|
||||
|
@ -293,6 +293,14 @@ class JmImageClient:
|
||||
# https://cdn-msp2.18comic.vip/media/photos/498976/00027.gif
|
||||
return data_original.endswith('.gif')
|
||||
|
||||
def download_album_cover(self, album_id, save_path: str, size: str = ''):
|
||||
self.download_image(
|
||||
img_url=JmcomicText.get_album_cover_url(album_id, size=size),
|
||||
img_save_path=save_path,
|
||||
scramble_id=None,
|
||||
decode_image=False,
|
||||
)
|
||||
|
||||
|
||||
class JmSearchAlbumClient:
|
||||
"""
|
||||
|
@ -158,7 +158,8 @@ class DirRule:
|
||||
if rule.startswith(('A', 'P')):
|
||||
return cls.parse_detail_rule
|
||||
|
||||
ExceptionTool.raises(f'不支持的rule配置: "{rule}"')
|
||||
return cls.parse_f_string_rule
|
||||
# ExceptionTool.raises(f'不支持的rule配置: "{rule}"')
|
||||
|
||||
@classmethod
|
||||
def apply_rule_to_filename(cls, album, photo, rule: str) -> str:
|
||||
@ -362,7 +363,13 @@ class JmOption:
|
||||
"""
|
||||
return self.new_jm_client(**kwargs)
|
||||
|
||||
def new_jm_client(self, domain_list=None, impl=None, cache=None, **kwargs) -> Union[JmHtmlClient, JmApiClient]:
|
||||
def new_jm_client(self,
|
||||
domain_list=None,
|
||||
impl=None,
|
||||
cache=None,
|
||||
domain_retry_strategy=None,
|
||||
**kwargs
|
||||
) -> Union[JmHtmlClient, JmApiClient]:
|
||||
"""
|
||||
创建新的Client(客户端),不同Client之间的元数据不共享
|
||||
"""
|
||||
@ -423,6 +430,7 @@ class JmOption:
|
||||
postman=postman,
|
||||
domain_list=decide_domain_list(),
|
||||
retry_times=retry_times,
|
||||
domain_retry_strategy=domain_retry_strategy,
|
||||
)
|
||||
|
||||
# enable cache
|
||||
|
@ -111,7 +111,7 @@ class JmOptionPlugin:
|
||||
def decide_filepath(self,
|
||||
album: Optional[JmAlbumDetail],
|
||||
photo: Optional[JmPhotoDetail],
|
||||
filename_rule: str, suffix: str, base_dir: Optional[str],
|
||||
filename_rule: Optional[str], suffix: Optional[str], base_dir: Optional[str],
|
||||
dir_rule_dict: Optional[dict]
|
||||
):
|
||||
"""
|
||||
@ -1219,7 +1219,7 @@ class ReplacePathStringPlugin(JmOptionPlugin):
|
||||
|
||||
|
||||
class AdvancedRetryPlugin(JmOptionPlugin):
|
||||
plugin_key = 'advanced-retry'
|
||||
plugin_key = 'advanced_retry'
|
||||
|
||||
def __init__(self, option: JmOption):
|
||||
super().__init__(option)
|
||||
@ -1234,15 +1234,18 @@ class AdvancedRetryPlugin(JmOptionPlugin):
|
||||
new_jm_client: Callable = self.option.new_jm_client
|
||||
|
||||
def hook_new_jm_client(*args, **kwargs):
|
||||
client: JmcomicClient = new_jm_client(*args, **kwargs)
|
||||
client.domain_retry_strategy = self.request_with_retry
|
||||
client.domain_req_failed_counter = {}
|
||||
from threading import Lock
|
||||
client.domain_counter_lock = Lock()
|
||||
return client
|
||||
return new_jm_client(*args, **kwargs, domain_retry_strategy=self)
|
||||
|
||||
self.option.new_jm_client = hook_new_jm_client
|
||||
|
||||
def __call__(self, client: AbstractJmClient, *args, **kwargs):
|
||||
if args:
|
||||
return self.request_with_retry(client, *args, **kwargs)
|
||||
# init
|
||||
from threading import Lock
|
||||
client.domain_req_failed_counter = {}
|
||||
client.domain_counter_lock = Lock()
|
||||
|
||||
def request_with_retry(self,
|
||||
client: AbstractJmClient,
|
||||
request: Callable,
|
||||
@ -1306,3 +1309,25 @@ class AdvancedRetryPlugin(JmOptionPlugin):
|
||||
def failed_count(client: JmcomicClient, domain: str) -> int:
|
||||
# noinspection PyUnresolvedReferences
|
||||
return client.domain_req_failed_counter.get(domain, 0)
|
||||
|
||||
|
||||
class DownloadCoverPlugin(JmOptionPlugin):
|
||||
plugin_key = 'download_cover'
|
||||
|
||||
def invoke(self,
|
||||
dir_rule: dict,
|
||||
size='',
|
||||
photo: JmPhotoDetail = None,
|
||||
album: JmAlbumDetail = None,
|
||||
downloader=None,
|
||||
**kwargs) -> None:
|
||||
album_id = album.id if album else photo.album_id
|
||||
save_path = self.decide_filepath(
|
||||
album, photo,
|
||||
None, None, None,
|
||||
dir_rule
|
||||
)
|
||||
if self.option.download.cache and os.path.exists(save_path):
|
||||
self.log(f'album-{album_id}的封面已存在,跳过下载: [{save_path}]', 'skip')
|
||||
return
|
||||
downloader.client.download_album_cover(album_id, save_path, size)
|
||||
|
@ -368,6 +368,26 @@ class JmcomicText:
|
||||
length = len(text)
|
||||
return text if length <= limit else (text[:limit] + f'...({length - limit}')
|
||||
|
||||
@classmethod
|
||||
def get_album_cover_url(cls,
|
||||
album_id: Union[str, int],
|
||||
image_domain: Optional[str] = None,
|
||||
size: str = '',
|
||||
) -> str:
|
||||
"""
|
||||
根据本子id生成封面url
|
||||
|
||||
:param album_id: 本子id
|
||||
:param image_domain: 图片cdn域名(可传入裸域或含协议的域名)
|
||||
:param size: 尺寸后缀,例如搜索列表页会使用 size="_3x4" 的封面图
|
||||
"""
|
||||
if image_domain is None:
|
||||
import random
|
||||
image_domain = random.choice(JmModuleConfig.DOMAIN_IMAGE_LIST)
|
||||
|
||||
path = f'/media/albums/{cls.parse_to_jm_id(album_id)}{size}.jpg'
|
||||
return cls.format_url(path, image_domain)
|
||||
|
||||
|
||||
# 支持dsl: #{???} -> os.getenv(???)
|
||||
JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env)
|
||||
|
@ -332,3 +332,8 @@ class Test_Client(JmTestConfigurable):
|
||||
# 打印page内容
|
||||
for aid, atitle in page:
|
||||
print(aid, atitle)
|
||||
|
||||
def test_download_cover(self):
|
||||
album_id = 123
|
||||
self.client.download_album_cover(album_id, f'{self.option.dir_rule.base_dir}/{album_id}.webp')
|
||||
self.client.download_album_cover(album_id, f'{self.option.dir_rule.base_dir}/{album_id}_3x4.webp', '_3x4')
|
||||
|
Loading…
Reference in New Issue
Block a user