mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.5.16: 新增插件【replace_path_string】可直接对下载路径做文本替换; 【img2pdf】插件支持对整个本子合并为一个pdf; 增加禁漫API HTTP状态码的检查; 优化文档. (#261)
This commit is contained in:
parent
e3ac598d1e
commit
c856b49c49
@ -35,6 +35,6 @@
|
||||
|
||||
## 自定义
|
||||
- [下载文件夹名](tutorial/9_custom_download_dir_name.md)
|
||||
- [日志](tutorial/9_custom_download_dir_name.md)
|
||||
- [日志](tutorial/11_log_custom.md)
|
||||
- [模块](tutorial/4_module_custom.md)
|
||||
|
||||
|
@ -130,7 +130,13 @@ plugins:
|
||||
kwargs:
|
||||
allowed_orig_suffix: # 后缀列表,表示只想下载以.gif结尾的图片
|
||||
- .gif
|
||||
|
||||
- plugin: replace_path_string # 字符串替换插件,直接对下载文件夹的路径进行文本替换
|
||||
kwargs:
|
||||
replace:
|
||||
# {左边写你要替换的原文}: {右边写替换成什么文本}
|
||||
aaa: bbb
|
||||
kyockcho: きょくちょ
|
||||
|
||||
- plugin: client_proxy # 客户端实现类代理插件,不建议非开发人员使用
|
||||
kwargs:
|
||||
proxy_client_key: photo_concurrent_fetcher_proxy # 代理类的client_key
|
||||
@ -228,7 +234,15 @@ plugins:
|
||||
- plugin: img2pdf
|
||||
kwargs:
|
||||
pdf_dir: D:/pdf/ # pdf存放文件夹
|
||||
filename_rule: Pid # pdf命名规则
|
||||
filename_rule: Pid # pdf命名规则,P代表photo, id代表使用photo.id也就是章节id
|
||||
|
||||
# img2pdf也支持合并整个本子,把上方的after_photo改为after_album即可。
|
||||
# https://github.com/hect0x7/JMComic-Crawler-Python/discussions/258
|
||||
# 配置到after_album时,需要修改filename_rule参数,不能写Pxx只能写Axx示例如下
|
||||
- plugin: img2pdf
|
||||
kwargs:
|
||||
pdf_dir: D:/pdf/ # pdf存放文件夹
|
||||
filename_rule: Aname # pdf命名规则,A代表album, name代表使用album.name也就是本子名称
|
||||
|
||||
# 请注意⚠
|
||||
# 下方的j2p插件的功能不如img2pdf插件,不建议使用。
|
||||
|
@ -53,11 +53,12 @@ album: JmAlbumDetail = client.get_album_detail('427413')
|
||||
def fetch(photo: JmPhotoDetail):
|
||||
# 章节实体类
|
||||
photo = client.get_photo_detail(photo.photo_id, False)
|
||||
|
||||
print(f'章节id: {photo.photo_id}')
|
||||
|
||||
# 图片实体类
|
||||
image: JmImageDetail
|
||||
for image in photo:
|
||||
print(image.img_url)
|
||||
print(f'图片url: {image.img_url}')
|
||||
|
||||
|
||||
# 多线程发起请求
|
||||
@ -67,6 +68,36 @@ multi_thread_launcher(
|
||||
)
|
||||
```
|
||||
|
||||
## jmcomic异常处理示例
|
||||
|
||||
```python
|
||||
from jmcomic import *
|
||||
|
||||
# 客户端
|
||||
client = JmOption.default().new_jm_client()
|
||||
|
||||
# 捕获jmcomic可能出现的异常
|
||||
try:
|
||||
# 请求本子实体类
|
||||
album: JmAlbumDetail = client.get_album_detail('427413')
|
||||
except MissingAlbumPhotoException as e:
|
||||
print(f'id={e.error_jmid}的本子不存在')
|
||||
|
||||
except JsonResolveFailException as e:
|
||||
print(f'解析json失败')
|
||||
# 响应对象
|
||||
resp = e.resp
|
||||
print(f'resp.text: {resp.text}, resp.status_code: {resp.status_code}')
|
||||
|
||||
except RequestRetryAllFailException as e:
|
||||
print(f'请求失败,重试次数耗尽')
|
||||
|
||||
except JmcomicException as e:
|
||||
# 捕获所有异常,用作兜底
|
||||
print(f'jmcomic遇到异常: {e}')
|
||||
```
|
||||
|
||||
|
||||
## 搜索本子
|
||||
|
||||
```python
|
||||
|
@ -1,21 +1,46 @@
|
||||
# 自定义下载文件夹名
|
||||
|
||||
## 0. 最简单直接粗暴有效的方式
|
||||
|
||||
使用插件`replace_path_string`:
|
||||
|
||||
这个插件可以直接替换下载文件夹路径,配置示例如下(把如下配置放入option配置文件即可):
|
||||
|
||||
```yml
|
||||
plugins:
|
||||
after_init:
|
||||
- plugin: replace_path_string
|
||||
kwargs:
|
||||
replace:
|
||||
# {左边写你要替换的原文}: {右边写替换成什么文本}
|
||||
kyockcho: きょくちょ
|
||||
```
|
||||
该示例会把文件夹路径中所有`kyockcho`都变为`きょくちょ`,例如:
|
||||
|
||||
`D:/a/[kyockcho]本子名称 - kyockcho/` 改为↓
|
||||
|
||||
`D:/a/[きょくちょ]本子名称 - きょくちょ/`
|
||||
|
||||
---------------
|
||||
**_如果上述简单的文本替换无法满足你,或者你需要更多上下文写逻辑代码,那么下面的内容正适合你阅读。_**
|
||||
|
||||
## 1. DirRule机制简介
|
||||
|
||||
## 1. DirRule简介
|
||||
|
||||
当你使用download_album下载本子时,本子会以一定的路径规则(DirRule)下载到你的磁盘上。
|
||||
|
||||
你可以使用配置文件定制DirRule,例如下面的例子
|
||||
你可以使用配置文件定制DirRule,例如下面的例子:
|
||||
|
||||
```yaml
|
||||
dir_rule:
|
||||
# 设定根目录 base_dir
|
||||
base_dir: D:/a/b/c/
|
||||
# 规则含义: 根目录 / 章节标题 / 图片文件
|
||||
rule: Bd_Ptitle # P表示章节,title表示使用章节的title字段
|
||||
rule: Bd / Ptitle # P表示章节,title表示使用章节的title字段
|
||||
# 这个规则的含义是,把图片下载到路径 {base_dir}/{Ptitle}/ 下
|
||||
# 即:根目录 / 章节标题 / 图片文件
|
||||
```
|
||||
|
||||
如果一个章节的名称(title)是ddd,则最后的下载文件夹结构为:
|
||||
例如,假设一个章节的名称(Ptitle)是ddd,则最后的下载文件夹结构为 `D:/a/b/c/ddd/`:
|
||||
|
||||
```
|
||||
D:/a/b/c/ddd/00001.webp
|
||||
@ -24,6 +49,13 @@ D:/a/b/c/ddd/00003.webp
|
||||
...
|
||||
```
|
||||
|
||||
上述的Ptitle,P表示章节,title表示使用章节的title字段。
|
||||
|
||||
除了title,你还可以写什么?其实Ptitle表示的是jmcomic里的章节实体类 JmPhotoDetail 的属性。
|
||||
|
||||
最终能写什么,取决于JmPhotoDetail有哪些属性,建议使用IDE来获知这些属性,不过这需要你懂一些python基础。
|
||||
|
||||
除了Pxxx,你还可以写Axxx,表示这个章节所在的本子的属性xxx,详见本子实体类 JmAlbumDetail。
|
||||
|
||||
|
||||
## 2. 自定义字段名
|
||||
|
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.5.15'
|
||||
__version__ = '2.5.16'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
@ -2,6 +2,7 @@ from .jm_downloader import *
|
||||
|
||||
__DOWNLOAD_API_RET = Tuple[JmAlbumDetail, JmDownloader]
|
||||
|
||||
|
||||
def download_batch(download_api,
|
||||
jm_id_iter: Union[Iterable, Generator],
|
||||
option=None,
|
||||
|
@ -27,7 +27,7 @@ class AbstractJmClient(
|
||||
self.retry_times = retry_times
|
||||
self.domain_list = domain_list
|
||||
self.CLIENT_CACHE = None
|
||||
self.__username = None # help for favorite_folder method
|
||||
self._username = None # help for favorite_folder method
|
||||
self.enable_cache()
|
||||
self.after_init()
|
||||
|
||||
@ -412,7 +412,7 @@ class JmHtmlClient(AbstractJmClient):
|
||||
return resp
|
||||
|
||||
self['cookies'] = new_cookies
|
||||
self.__username = username
|
||||
self._username = username
|
||||
|
||||
return resp
|
||||
|
||||
@ -423,8 +423,8 @@ class JmHtmlClient(AbstractJmClient):
|
||||
username='',
|
||||
) -> JmFavoritePage:
|
||||
if username == '':
|
||||
ExceptionTool.require_true(self.__username is not None, 'favorite_folder方法需要传username参数')
|
||||
username = self.__username
|
||||
ExceptionTool.require_true(self._username is not None, 'favorite_folder方法需要传username参数')
|
||||
username = self._username
|
||||
|
||||
resp = self.get_jm_html(
|
||||
f'/user/{username}/favorite/albums',
|
||||
@ -973,6 +973,11 @@ class JmApiClient(AbstractJmClient):
|
||||
# 例如图片请求
|
||||
return resp
|
||||
|
||||
code = resp.status_code
|
||||
if code >= 500:
|
||||
msg = JmModuleConfig.JM_ERROR_STATUS_CODE.get(code, f'HTTP状态码: {code}')
|
||||
ExceptionTool.raises_resp(f"禁漫API异常响应, {msg}", resp)
|
||||
|
||||
url = resp.request.url
|
||||
|
||||
if self.API_SCRAMBLE in url:
|
||||
|
@ -123,6 +123,7 @@ class JmModuleConfig:
|
||||
# JM的异常网页code
|
||||
JM_ERROR_STATUS_CODE = {
|
||||
403: 'ip地区禁止访问/爬虫被识别',
|
||||
500: '500: 禁漫服务器内部异常(可能是服务器过载,可以换个ip或稍后重试)',
|
||||
520: '520: Web server is returning an unknown error (禁漫服务器内部报错)',
|
||||
524: '524: The origin web server timed out responding to this request. (禁漫服务器处理超时)',
|
||||
}
|
||||
|
@ -1,3 +1,5 @@
|
||||
from functools import lru_cache
|
||||
|
||||
from common import *
|
||||
|
||||
from .jm_config import *
|
||||
@ -400,6 +402,7 @@ class JmPhotoDetail(DetailEntity, Downloadable):
|
||||
def id(self):
|
||||
return self.photo_id
|
||||
|
||||
@lru_cache(None)
|
||||
def getindex(self, index) -> JmImageDetail:
|
||||
return self.create_image_detail(index)
|
||||
|
||||
@ -514,6 +517,7 @@ class JmAlbumDetail(DetailEntity, Downloadable):
|
||||
|
||||
return photo
|
||||
|
||||
@lru_cache(None)
|
||||
def getindex(self, item) -> JmPhotoDetail:
|
||||
return self.create_photo_detail(item)
|
||||
|
||||
|
@ -270,15 +270,7 @@ class JmOption:
|
||||
)
|
||||
|
||||
if ensure_exists:
|
||||
try:
|
||||
mkdir_if_not_exists(save_dir)
|
||||
except OSError as e:
|
||||
if e.errno == 36:
|
||||
# 目录名过长
|
||||
limit = JmModuleConfig.VAR_FILE_NAME_LENGTH_LIMIT
|
||||
jm_log('error', f'目录名过长,无法创建目录,强制缩短到{limit}个字符并重试')
|
||||
save_dir = save_dir[0:limit]
|
||||
mkdir_if_not_exists(save_dir)
|
||||
save_dir = JmcomicText.try_mkdir(save_dir)
|
||||
|
||||
return save_dir
|
||||
|
||||
@ -517,13 +509,21 @@ class JmOption:
|
||||
|
||||
# 下面的方法提供面向对象的调用风格
|
||||
|
||||
def download_album(self, album_id):
|
||||
def download_album(self,
|
||||
album_id,
|
||||
downloader=None,
|
||||
callback=None,
|
||||
):
|
||||
from .api import download_album
|
||||
download_album(album_id, self)
|
||||
download_album(album_id, self, downloader, callback)
|
||||
|
||||
def download_photo(self, photo_id):
|
||||
def download_photo(self,
|
||||
photo_id,
|
||||
downloader=None,
|
||||
callback=None
|
||||
):
|
||||
from .api import download_photo
|
||||
download_photo(photo_id, self)
|
||||
download_photo(photo_id, self, downloader, callback)
|
||||
|
||||
# 下面的方法为调用插件提供支持
|
||||
|
||||
|
@ -734,13 +734,17 @@ class Img2pdfPlugin(JmOptionPlugin):
|
||||
plugin_key = 'img2pdf'
|
||||
|
||||
def invoke(self,
|
||||
photo: JmPhotoDetail,
|
||||
photo: JmPhotoDetail = None,
|
||||
album: JmAlbumDetail = None,
|
||||
downloader=None,
|
||||
pdf_dir=None,
|
||||
filename_rule='Pid',
|
||||
delete_original_file=False,
|
||||
**kwargs,
|
||||
):
|
||||
if photo is None and album is None:
|
||||
jm_log('wrong_usage', 'img2pdf必须运行在after_photo或after_album时')
|
||||
|
||||
try:
|
||||
import img2pdf
|
||||
except ImportError:
|
||||
@ -749,28 +753,50 @@ class Img2pdfPlugin(JmOptionPlugin):
|
||||
|
||||
self.delete_original_file = delete_original_file
|
||||
|
||||
# 处理文件夹配置
|
||||
filename = DirRule.apply_rule_directly(None, photo, filename_rule)
|
||||
photo_dir = self.option.decide_image_save_dir(photo)
|
||||
|
||||
# 处理生成的pdf文件的路径
|
||||
if pdf_dir is None:
|
||||
pdf_dir = photo_dir
|
||||
else:
|
||||
pdf_dir = fix_filepath(pdf_dir, True)
|
||||
mkdir_if_not_exists(pdf_dir)
|
||||
pdf_dir = self.ensure_make_pdf_dir(pdf_dir)
|
||||
|
||||
# 处理pdf文件名
|
||||
filename = DirRule.apply_rule_directly(album, photo, filename_rule)
|
||||
|
||||
# pdf路径
|
||||
pdf_filepath = os.path.join(pdf_dir, f'{filename}.pdf')
|
||||
|
||||
# 调用 img2pdf 把 photo_dir 下的所有图片转为pdf
|
||||
all_img = files_of_dir(photo_dir)
|
||||
with open(pdf_filepath, 'wb') as f:
|
||||
f.write(img2pdf.convert(all_img))
|
||||
img_path_ls, img_dir_ls = self.write_img_2_pdf(pdf_filepath, album, photo)
|
||||
self.log(f'Convert Successfully: JM{album or photo} → {pdf_filepath}')
|
||||
|
||||
# 执行删除
|
||||
self.log(f'Convert Successfully: JM{photo.id} → {pdf_filepath}')
|
||||
all_img.append(self.option.decide_image_save_dir(photo, ensure_exists=False))
|
||||
self.execute_deletion(all_img)
|
||||
img_path_ls += img_dir_ls
|
||||
self.execute_deletion(img_path_ls)
|
||||
|
||||
def write_img_2_pdf(self, pdf_filepath, album: JmAlbumDetail, photo: JmPhotoDetail):
|
||||
import img2pdf
|
||||
|
||||
if album is None:
|
||||
img_dir_ls = [self.option.decide_image_save_dir(photo)]
|
||||
else:
|
||||
img_dir_ls = [self.option.decide_image_save_dir(photo) for photo in album]
|
||||
|
||||
img_path_ls = []
|
||||
|
||||
for img_dir in img_dir_ls:
|
||||
imgs = files_of_dir(img_dir)
|
||||
if not imgs:
|
||||
continue
|
||||
img_path_ls += imgs
|
||||
|
||||
with open(pdf_filepath, 'wb') as f:
|
||||
f.write(img2pdf.convert(img_path_ls))
|
||||
|
||||
return img_path_ls, img_dir_ls
|
||||
|
||||
@staticmethod
|
||||
def ensure_make_pdf_dir(pdf_dir: str):
|
||||
pdf_dir = pdf_dir or os.getcwd()
|
||||
pdf_dir = fix_filepath(pdf_dir, True)
|
||||
mkdir_if_not_exists(pdf_dir)
|
||||
return pdf_dir
|
||||
|
||||
|
||||
class JmServerPlugin(JmOptionPlugin):
|
||||
@ -1067,3 +1093,27 @@ class DeleteDuplicatedFilesPlugin(JmOptionPlugin):
|
||||
[f' {path}' for path in paths]
|
||||
self.log('\n'.join(message))
|
||||
self.execute_deletion(paths)
|
||||
|
||||
|
||||
class ReplacePathStringPlugin(JmOptionPlugin):
|
||||
plugin_key = 'replace_path_string'
|
||||
|
||||
def invoke(self,
|
||||
replace: Dict[str, str],
|
||||
):
|
||||
if not replace:
|
||||
return
|
||||
|
||||
old_decide_dir = self.option.decide_image_save_dir
|
||||
|
||||
def new_decide_dir(photo, ensure_exists=True) -> str:
|
||||
original_path: str = old_decide_dir(photo, False)
|
||||
for k, v in replace.items():
|
||||
original_path = original_path.replace(k, v)
|
||||
|
||||
if ensure_exists:
|
||||
JmcomicText.try_mkdir(original_path)
|
||||
|
||||
return original_path
|
||||
|
||||
self.option.decide_image_save_dir = new_decide_dir
|
||||
|
@ -316,6 +316,19 @@ class JmcomicText:
|
||||
import zhconv
|
||||
return zhconv.convert(s, 'zh_cn')
|
||||
|
||||
@classmethod
|
||||
def try_mkdir(cls, save_dir: str):
|
||||
try:
|
||||
mkdir_if_not_exists(save_dir)
|
||||
except OSError as e:
|
||||
if e.errno == 36:
|
||||
# 目录名过长
|
||||
limit = JmModuleConfig.VAR_FILE_NAME_LENGTH_LIMIT
|
||||
jm_log('error', f'目录名过长,无法创建目录,强制缩短到{limit}个字符并重试')
|
||||
save_dir = save_dir[0:limit]
|
||||
mkdir_if_not_exists(save_dir)
|
||||
return save_dir
|
||||
|
||||
|
||||
# 支持dsl: #{???} -> os.getenv(???)
|
||||
JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env)
|
||||
|
Loading…
Reference in New Issue
Block a user