mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.6.9: 增加下载本子封面图的函数以及download_cover插件; 修复client.after_init时advance_retry插件没有生效的问题(#472) (#482)
Some checks failed
Auto Release & Publish / release (push) Has been cancelled
Some checks failed
Auto Release & Publish / release (push) Has been cancelled
This commit is contained in:
parent
7943b9b66a
commit
f2adb33788
@ -109,12 +109,12 @@ dir_rule:
|
|||||||
# 规则: 根目录 / 本子id / 章节序号 / 图片文件
|
# 规则: 根目录 / 本子id / 章节序号 / 图片文件
|
||||||
# rule: 'Bd / Aid / Pindex'
|
# rule: 'Bd / Aid / Pindex'
|
||||||
# rule: 'Bd_Aid_Pindex'
|
# rule: 'Bd_Aid_Pindex'
|
||||||
|
|
||||||
# 默认规则是: 根目录 / 章节标题 / 图片文件
|
# 默认规则是: 根目录 / 章节标题 / 图片文件
|
||||||
rule: Bd_Ptitle
|
rule: Bd / Ptitle
|
||||||
# jmcomic v2.5.36 以后,支持使用python的f-string的语法组合文件夹名,下为示例
|
# jmcomic v2.5.36 以后,支持使用python的f-string的语法组合文件夹名,下为示例
|
||||||
# rule: Bd / Aauthor / (JM{Aid}-{Pindex})-{Pname}
|
# rule: Bd / Aauthor / (JM{Aid}-{Pindex})-{Pname}
|
||||||
# {}大括号里的内容同样是写 Axxx 或 Pxxx,其他语法自行参考python f-string的语法
|
# {}大括号里的内容同样是写 Axxx 或 Pxxx,其他语法自行参考python f-string的语法
|
||||||
|
# 另外,rule开头的Bd可忽略不写,因为程序会自动插入Bd
|
||||||
```
|
```
|
||||||
|
|
||||||
## 3. option插件配置项
|
## 3. option插件配置项
|
||||||
@ -194,6 +194,15 @@ plugins:
|
|||||||
album_photo_dict:
|
album_photo_dict:
|
||||||
324930: 424507
|
324930: 424507
|
||||||
|
|
||||||
|
before_album:
|
||||||
|
- plugin: download_cover # 额外下载本子封面的插件
|
||||||
|
kwargs:
|
||||||
|
size: '_3x4' # 可选项,禁漫搜索页的封面图尺寸是 4x3,和详情页不一样,想下搜索页的封面就设置此项
|
||||||
|
dir_rule: # 封面图存放路径规则,写法同上
|
||||||
|
base_dir: D:/a/b/c/
|
||||||
|
rule: '{Atitle}/{Aid}_cover.jpg'
|
||||||
|
|
||||||
|
|
||||||
after_album:
|
after_album:
|
||||||
- plugin: zip # 压缩文件插件
|
- plugin: zip # 压缩文件插件
|
||||||
kwargs:
|
kwargs:
|
||||||
|
@ -38,7 +38,7 @@ download_album(123, option)
|
|||||||
option.download_album(123)
|
option.download_album(123)
|
||||||
```
|
```
|
||||||
|
|
||||||
## 获取本子/章节/图片的实体类,下载图片
|
## 获取本子/章节/图片的实体类,下载图片/封面图
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from jmcomic import *
|
from jmcomic import *
|
||||||
@ -49,6 +49,9 @@ client = JmOption.default().new_jm_client()
|
|||||||
# 本子实体类
|
# 本子实体类
|
||||||
album: JmAlbumDetail = client.get_album_detail('427413')
|
album: JmAlbumDetail = client.get_album_detail('427413')
|
||||||
|
|
||||||
|
# 下载本子封面图,保存为 cover.png (图片后缀可指定为jpg、webp等)
|
||||||
|
client.download_album_cover('427413', './cover.png')
|
||||||
|
|
||||||
|
|
||||||
def fetch(photo: JmPhotoDetail):
|
def fetch(photo: JmPhotoDetail):
|
||||||
# 章节实体类
|
# 章节实体类
|
||||||
|
@ -27,7 +27,7 @@ plugins:
|
|||||||
proxy_client_key: photo_concurrent_fetcher_proxy
|
proxy_client_key: photo_concurrent_fetcher_proxy
|
||||||
whitelist: [ api, ]
|
whitelist: [ api, ]
|
||||||
|
|
||||||
- plugin: advanced-retry
|
- plugin: advanced_retry
|
||||||
kwargs:
|
kwargs:
|
||||||
retry_config:
|
retry_config:
|
||||||
retry_rounds: 3 # 一共对域名列表重试3轮
|
retry_rounds: 3 # 一共对域名列表重试3轮
|
||||||
|
@ -2,7 +2,7 @@
|
|||||||
# 被依赖方 <--- 使用方
|
# 被依赖方 <--- 使用方
|
||||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||||
|
|
||||||
__version__ = '2.6.7'
|
__version__ = '2.6.9'
|
||||||
|
|
||||||
from .api import *
|
from .api import *
|
||||||
from .jm_plugin import *
|
from .jm_plugin import *
|
||||||
|
@ -15,6 +15,7 @@ class AbstractJmClient(
|
|||||||
postman: Postman,
|
postman: Postman,
|
||||||
domain_list: List[str],
|
domain_list: List[str],
|
||||||
retry_times=0,
|
retry_times=0,
|
||||||
|
domain_retry_strategy=None,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
创建JM客户端
|
创建JM客户端
|
||||||
@ -26,9 +27,11 @@ class AbstractJmClient(
|
|||||||
super().__init__(postman)
|
super().__init__(postman)
|
||||||
self.retry_times = retry_times
|
self.retry_times = retry_times
|
||||||
self.domain_list = domain_list
|
self.domain_list = domain_list
|
||||||
self.domain_retry_strategy = None
|
self.domain_retry_strategy = domain_retry_strategy
|
||||||
self.CLIENT_CACHE = None
|
self.CLIENT_CACHE = None
|
||||||
self._username = None # help for favorite_folder method
|
self._username = None # help for favorite_folder method
|
||||||
|
if domain_retry_strategy:
|
||||||
|
domain_retry_strategy(self)
|
||||||
self.enable_cache()
|
self.enable_cache()
|
||||||
self.after_init()
|
self.after_init()
|
||||||
|
|
||||||
|
@ -293,6 +293,14 @@ class JmImageClient:
|
|||||||
# https://cdn-msp2.18comic.vip/media/photos/498976/00027.gif
|
# https://cdn-msp2.18comic.vip/media/photos/498976/00027.gif
|
||||||
return data_original.endswith('.gif')
|
return data_original.endswith('.gif')
|
||||||
|
|
||||||
|
def download_album_cover(self, album_id, save_path: str, size: str = ''):
|
||||||
|
self.download_image(
|
||||||
|
img_url=JmcomicText.get_album_cover_url(album_id, size=size),
|
||||||
|
img_save_path=save_path,
|
||||||
|
scramble_id=None,
|
||||||
|
decode_image=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class JmSearchAlbumClient:
|
class JmSearchAlbumClient:
|
||||||
"""
|
"""
|
||||||
|
@ -158,7 +158,8 @@ class DirRule:
|
|||||||
if rule.startswith(('A', 'P')):
|
if rule.startswith(('A', 'P')):
|
||||||
return cls.parse_detail_rule
|
return cls.parse_detail_rule
|
||||||
|
|
||||||
ExceptionTool.raises(f'不支持的rule配置: "{rule}"')
|
return cls.parse_f_string_rule
|
||||||
|
# ExceptionTool.raises(f'不支持的rule配置: "{rule}"')
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def apply_rule_to_filename(cls, album, photo, rule: str) -> str:
|
def apply_rule_to_filename(cls, album, photo, rule: str) -> str:
|
||||||
@ -362,7 +363,13 @@ class JmOption:
|
|||||||
"""
|
"""
|
||||||
return self.new_jm_client(**kwargs)
|
return self.new_jm_client(**kwargs)
|
||||||
|
|
||||||
def new_jm_client(self, domain_list=None, impl=None, cache=None, **kwargs) -> Union[JmHtmlClient, JmApiClient]:
|
def new_jm_client(self,
|
||||||
|
domain_list=None,
|
||||||
|
impl=None,
|
||||||
|
cache=None,
|
||||||
|
domain_retry_strategy=None,
|
||||||
|
**kwargs
|
||||||
|
) -> Union[JmHtmlClient, JmApiClient]:
|
||||||
"""
|
"""
|
||||||
创建新的Client(客户端),不同Client之间的元数据不共享
|
创建新的Client(客户端),不同Client之间的元数据不共享
|
||||||
"""
|
"""
|
||||||
@ -423,6 +430,7 @@ class JmOption:
|
|||||||
postman=postman,
|
postman=postman,
|
||||||
domain_list=decide_domain_list(),
|
domain_list=decide_domain_list(),
|
||||||
retry_times=retry_times,
|
retry_times=retry_times,
|
||||||
|
domain_retry_strategy=domain_retry_strategy,
|
||||||
)
|
)
|
||||||
|
|
||||||
# enable cache
|
# enable cache
|
||||||
|
@ -111,7 +111,7 @@ class JmOptionPlugin:
|
|||||||
def decide_filepath(self,
|
def decide_filepath(self,
|
||||||
album: Optional[JmAlbumDetail],
|
album: Optional[JmAlbumDetail],
|
||||||
photo: Optional[JmPhotoDetail],
|
photo: Optional[JmPhotoDetail],
|
||||||
filename_rule: str, suffix: str, base_dir: Optional[str],
|
filename_rule: Optional[str], suffix: Optional[str], base_dir: Optional[str],
|
||||||
dir_rule_dict: Optional[dict]
|
dir_rule_dict: Optional[dict]
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
@ -1219,7 +1219,7 @@ class ReplacePathStringPlugin(JmOptionPlugin):
|
|||||||
|
|
||||||
|
|
||||||
class AdvancedRetryPlugin(JmOptionPlugin):
|
class AdvancedRetryPlugin(JmOptionPlugin):
|
||||||
plugin_key = 'advanced-retry'
|
plugin_key = 'advanced_retry'
|
||||||
|
|
||||||
def __init__(self, option: JmOption):
|
def __init__(self, option: JmOption):
|
||||||
super().__init__(option)
|
super().__init__(option)
|
||||||
@ -1234,15 +1234,18 @@ class AdvancedRetryPlugin(JmOptionPlugin):
|
|||||||
new_jm_client: Callable = self.option.new_jm_client
|
new_jm_client: Callable = self.option.new_jm_client
|
||||||
|
|
||||||
def hook_new_jm_client(*args, **kwargs):
|
def hook_new_jm_client(*args, **kwargs):
|
||||||
client: JmcomicClient = new_jm_client(*args, **kwargs)
|
return new_jm_client(*args, **kwargs, domain_retry_strategy=self)
|
||||||
client.domain_retry_strategy = self.request_with_retry
|
|
||||||
client.domain_req_failed_counter = {}
|
|
||||||
from threading import Lock
|
|
||||||
client.domain_counter_lock = Lock()
|
|
||||||
return client
|
|
||||||
|
|
||||||
self.option.new_jm_client = hook_new_jm_client
|
self.option.new_jm_client = hook_new_jm_client
|
||||||
|
|
||||||
|
def __call__(self, client: AbstractJmClient, *args, **kwargs):
|
||||||
|
if args:
|
||||||
|
return self.request_with_retry(client, *args, **kwargs)
|
||||||
|
# init
|
||||||
|
from threading import Lock
|
||||||
|
client.domain_req_failed_counter = {}
|
||||||
|
client.domain_counter_lock = Lock()
|
||||||
|
|
||||||
def request_with_retry(self,
|
def request_with_retry(self,
|
||||||
client: AbstractJmClient,
|
client: AbstractJmClient,
|
||||||
request: Callable,
|
request: Callable,
|
||||||
@ -1306,3 +1309,25 @@ class AdvancedRetryPlugin(JmOptionPlugin):
|
|||||||
def failed_count(client: JmcomicClient, domain: str) -> int:
|
def failed_count(client: JmcomicClient, domain: str) -> int:
|
||||||
# noinspection PyUnresolvedReferences
|
# noinspection PyUnresolvedReferences
|
||||||
return client.domain_req_failed_counter.get(domain, 0)
|
return client.domain_req_failed_counter.get(domain, 0)
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadCoverPlugin(JmOptionPlugin):
|
||||||
|
plugin_key = 'download_cover'
|
||||||
|
|
||||||
|
def invoke(self,
|
||||||
|
dir_rule: dict,
|
||||||
|
size='',
|
||||||
|
photo: JmPhotoDetail = None,
|
||||||
|
album: JmAlbumDetail = None,
|
||||||
|
downloader=None,
|
||||||
|
**kwargs) -> None:
|
||||||
|
album_id = album.id if album else photo.album_id
|
||||||
|
save_path = self.decide_filepath(
|
||||||
|
album, photo,
|
||||||
|
None, None, None,
|
||||||
|
dir_rule
|
||||||
|
)
|
||||||
|
if self.option.download.cache and os.path.exists(save_path):
|
||||||
|
self.log(f'album-{album_id}的封面已存在,跳过下载: [{save_path}]', 'skip')
|
||||||
|
return
|
||||||
|
downloader.client.download_album_cover(album_id, save_path, size)
|
||||||
|
@ -368,6 +368,26 @@ class JmcomicText:
|
|||||||
length = len(text)
|
length = len(text)
|
||||||
return text if length <= limit else (text[:limit] + f'...({length - limit}')
|
return text if length <= limit else (text[:limit] + f'...({length - limit}')
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def get_album_cover_url(cls,
|
||||||
|
album_id: Union[str, int],
|
||||||
|
image_domain: Optional[str] = None,
|
||||||
|
size: str = '',
|
||||||
|
) -> str:
|
||||||
|
"""
|
||||||
|
根据本子id生成封面url
|
||||||
|
|
||||||
|
:param album_id: 本子id
|
||||||
|
:param image_domain: 图片cdn域名(可传入裸域或含协议的域名)
|
||||||
|
:param size: 尺寸后缀,例如搜索列表页会使用 size="_3x4" 的封面图
|
||||||
|
"""
|
||||||
|
if image_domain is None:
|
||||||
|
import random
|
||||||
|
image_domain = random.choice(JmModuleConfig.DOMAIN_IMAGE_LIST)
|
||||||
|
|
||||||
|
path = f'/media/albums/{cls.parse_to_jm_id(album_id)}{size}.jpg'
|
||||||
|
return cls.format_url(path, image_domain)
|
||||||
|
|
||||||
|
|
||||||
# 支持dsl: #{???} -> os.getenv(???)
|
# 支持dsl: #{???} -> os.getenv(???)
|
||||||
JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env)
|
JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env)
|
||||||
|
@ -332,3 +332,8 @@ class Test_Client(JmTestConfigurable):
|
|||||||
# 打印page内容
|
# 打印page内容
|
||||||
for aid, atitle in page:
|
for aid, atitle in page:
|
||||||
print(aid, atitle)
|
print(aid, atitle)
|
||||||
|
|
||||||
|
def test_download_cover(self):
|
||||||
|
album_id = 123
|
||||||
|
self.client.download_album_cover(album_id, f'{self.option.dir_rule.base_dir}/{album_id}.webp')
|
||||||
|
self.client.download_album_cover(album_id, f'{self.option.dir_rule.base_dir}/{album_id}_3x4.webp', '_3x4')
|
||||||
|
Loading…
Reference in New Issue
Block a user