v2.1.6: 简化字段缓存的代码,调整下载章节并发数,更新文档 (#83)

This commit is contained in:
hect0x7 2023-07-31 13:51:03 +08:00 committed by GitHub
parent 9a5c5d12cd
commit 5e013e1818
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 70 additions and 61 deletions

View File

@ -21,15 +21,16 @@
## 快速上手
使用下面的两行代码,即可实现功能:把某个本子album里的所有本子photo下载到本地
使用下面的两行代码,即可实现功能:把某个本子album里的所有章节photo下载到本地
```python
import jmcomic # 导入此模块,需要先安装.
jmcomic.download_album('422866') # 传入要下载的album的id即可下载整个album到本地.
# 上面的这行代码还有一个可选参数option: JmOption表示配置项
# 配置项的作用是告诉程序下载时候的一些选择,
# 比如,要下载到哪个文件夹,使用怎样的路径组织方式(比如[/作者/本子id/图片] 或者 [/作者/本子名称/图片].
# 比如,要下载到哪个文件夹,使用怎样的路径组织规则(比如[/作者/本子id/图片] 或者 [/作者/本子名称/图片].
# 如果没有配置,则会使用 JmOption.default(),下载的路径是[当前工作文件夹/本子名称/图片].
# 如果你想要配置请参考assets/config/和usgae/下的文档和示例.
```
进一步的使用可以参考usage文件夹下的示例代码: `getting_started.py` `sample_usage.py`
@ -39,17 +40,21 @@ jmcomic.download_album('422866') # 传入要下载的album的id即可下载
- **绕过Cloudflare的反爬虫**
- 支持使用**Github Action**下载漫画,不会编程都能用([教程使用Github Actions下载禁漫本子](./assets/docs/教程使用Github%20Actions下载禁漫本子.md)
- 可配置性强
- 不配置也能使用,十分方便
- 配置可以从**配置文件**生成无需写Python代码
- 配置点有:`是否使用磁盘缓存` `是否使用代理` `图片类型转换` `本子下载路径` `请求元信息headers,cookies,重试次数)等 `
- 配置可以从**配置文件**生成支持多种文件格式无需写Python代码
- 配置点有:`是否使用磁盘缓存` `图片类型转换` `下载路径` `请求元信息headers,cookies,代理)等 `
- 可扩展性强
- 支持自定义本子/章节/图片下载前后的回调函数
- 支持自定义debug日志的开关/格式
- 支持自定义Option/Client/实体类
- 支持重试和域名切换机制
- 多线程下载(可细化到一图一线程,效率极高)
- 跟进了JM最新的图片分割算法2023-02-08
## 使用小说明
* Python >= 3.7
* 项目只有代码注释没有API文档。因此想深入高级地使用自行看源码和改造代码叭 ^^_
* 个人项目文档和示例会有不及时之处可以Issue提问
## 项目文件夹介绍

View File

@ -2,6 +2,6 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option
__version__ = '2.1.5'
__version__ = '2.1.6'
from .api import *

View File

@ -17,7 +17,7 @@ def download_album(jm_album_id, option=None):
option.before_album(album)
execute_by_condition(
iter_obj=album,
iter_objs=album,
apply=lambda photo: download_by_photo_detail(photo, option),
count_batch=option.decide_photo_batch_count(album)
)
@ -49,13 +49,11 @@ def download_by_photo_detail(photo: JmPhotoDetail, option=None):
# 下载每个图片的函数
def download_image(image: JmImageDetail):
img_save_path = option.decide_image_filepath(image)
# 已下载过,缓存命中
if use_cache is True and file_exists(img_save_path):
image.is_exists = True
return
image.is_exists = file_exists(img_save_path)
option.before_image(image, img_save_path)
if use_cache is True and image.is_exists:
return
jm_client.download_by_image_detail(
image,
img_save_path,
@ -65,7 +63,7 @@ def download_by_photo_detail(photo: JmPhotoDetail, option=None):
option.before_photo(photo)
execute_by_condition(
iter_obj=photo,
iter_objs=photo,
apply=download_image,
count_batch=option.decide_image_batch_count(photo)
)
@ -88,28 +86,28 @@ def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
option = JmOption.default()
return thread_pool_executor(
iter_objs=((album_id, option) for album_id in jm_album_id_iter),
apply_each_obj_func=download_album,
iter_objs=set(JmcomicText.parse_to_album_id(album_id) for album_id in jm_album_id_iter),
apply_each_obj_func=lambda album_id: download_album(album_id, option),
wait_finish=wait_finish,
)
def execute_by_condition(iter_obj, apply: Callable, count_batch: int):
def execute_by_condition(iter_objs, apply: Callable, count_batch: int):
"""
章节/图片的下载调度逻辑
"""
count_real = len(iter_obj)
count_real = len(iter_objs)
if count_batch >= count_real:
# 一图一线程
# 一/章节 对应 线程
multi_thread_launcher(
iter_objs=iter_obj,
iter_objs=iter_objs,
apply_each_obj_func=apply,
)
else:
# 创建batch个线程的线程池,当图片数>batch时要等待。
# 创建batch个线程的线程池
thread_pool_executor(
iter_objs=iter_obj,
iter_objs=iter_objs,
apply_each_obj_func=apply,
max_workers=count_batch,
)

View File

@ -1,3 +1,8 @@
def field_cache(*args, **kwargs):
from common import field_cache
return field_cache(*args, **kwargs)
def default_jm_debug(topic: str, msg: str):
from common import format_ts
print(f'{format_ts()}:【{topic}{msg}')
@ -15,11 +20,13 @@ def default_postman_constructor(session, **kwargs):
class JmModuleConfig:
# 网站相关
PROT = "https://"
DOMAIN = None
JM_REDIRECT_URL = f'{PROT}jm365.work/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来
JM_PUB_URL = f'{PROT}jmcomic2.bet'
JM_CDN_IMAGE_URL_TEMPLATE = PROT + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始
JM_IMAGE_SUFFIX = ['.jpg', '.webp', '.png', '.gif']
# 缓存字段
DOMAIN = None
DOMAIN_LIST = None
# 访问JM可能会遇到的异常网页
JM_ERROR_RESPONSE_TEXT = {
@ -52,17 +59,15 @@ class JmModuleConfig:
postman_constructor = default_postman_constructor
@classmethod
@field_cache("DOMAIN")
def domain(cls, postman=None):
"""
由于禁漫的域名经常变化调用此方法可以获取一个当前可用的最新的域名 domain
并且设置把 domain 设置为禁漫模块的默认域名
这样一来配置文件也不用配置域名了一切都在运行时动态获取
"""
if cls.DOMAIN is None:
from .jm_toolkit import JmcomicText
cls.DOMAIN = JmcomicText.parse_to_jm_domain(cls.get_jmcomic_url(postman))
return cls.DOMAIN # jmcomic默认域名
return JmcomicText.parse_to_jm_domain(cls.get_jmcomic_url(postman))
@classmethod
def headers(cls, domain='18comic.vip'):
@ -116,6 +121,7 @@ class JmModuleConfig:
return url
@classmethod
@field_cache("DOMAIN_LIST")
def get_jmcomic_domain_all(cls, postman=None):
"""
访问禁漫发布页得到所有禁漫的域名

View File

@ -32,7 +32,7 @@ class DownloadCallback:
f'图片已存在: {image.tag} ← [{img_save_path}]'
)
else:
jm_debug('image_before',
jm_debug('image-before',
f'图片准备下载: {image.tag}, [{image.img_url}] → [{img_save_path}]'
)
@ -159,9 +159,6 @@ class JmOption(DownloadCallback):
# 其他配置
self.filepath = filepath
# 字段
self.jm_client_cache = None
@property
def download_cache(self):
return self.download.cache
@ -186,9 +183,9 @@ class JmOption(DownloadCallback):
def decide_image_batch_count(self, photo: JmPhotoDetail):
return self.download_threading_batch_count
# noinspection PyMethodMayBeStatic
# noinspection PyMethodMayBeStatic,PyUnusedLocal
def decide_photo_batch_count(self, album: JmAlbumDetail):
return len(album)
return os.cpu_count()
def decide_image_save_dir(self, photo) -> str:
# 使用 self.dir_rule 决定 save_dir
@ -266,23 +263,15 @@ class JmOption(DownloadCallback):
"""
# 缓存
cache_jm_client = True
jm_client_impl_mapping: Dict[str, Type[AbstractJmClient]] = {
'html': JmHtmlClient,
'api': JmApiClient,
}
@field_cache("__jm_client_cache__")
def build_jm_client(self, **kwargs) -> JmcomicClient:
if self.cache_jm_client is not True:
return self.new_jm_client(**kwargs)
client = self.jm_client_cache
if client is None:
client = self.new_jm_client(**kwargs)
self.jm_client_cache = client
return client
def new_jm_client(self, **kwargs) -> JmcomicClient:
postman_conf: dict = self.client.postman.src_dict

View File

@ -30,7 +30,6 @@ jmcomic.download_album(aid for aid in ('422866', '1', '2', '3')) # 生成器
获取域名介绍
--------------------
"""
# 方式1: 访问禁漫发布页
url_ls = jmcomic.JmModuleConfig.get_jmcomic_url_all()
print(url_ls)
@ -55,3 +54,16 @@ jm_option.to_file('./你的配置文件路名称.json') # json格式
# 如果你修改了默认配置,现在想用你修改后的配置来下载,使用如下代码
jm_option = jmcomic.create_option('./你的配置文件路名称.yml')
jmcomic.download_album('23333', jm_option)
# 如果你只想做简单的配置,也可以使用如下形式
# 具体可以写什么,请参考 JmOption.default_dict你只需要覆盖里面的键值即可
# 配置代理
jm_option = JmOption.construct({
'client': {
'postman': {
'meta_data': {
'proxies': ProxyBuilder.clash_proxy(),
}
}
}
})

View File

@ -1,8 +1,10 @@
from jmcomic import *
jm_option = create_option(
option = create_option(
f'你的配置文件路径,例如: D:/a/b/c/jmcomic/config.yml'
)
client = option.build_jm_client()
client.enable_cache(debug=True)
@timeit('下载本子集: ')
@ -13,21 +15,18 @@ def download_jm_album():
''')
download_album(ls, jm_option) # 效果同下面的代码
# download_album_batch(ls, jm_option)
download_album(ls, option) # 效果同下面的代码
# download_album_batch(ls, op)
@timeit('获取实体类: ')
def get_album_photo_detail():
client = jm_option.build_jm_client()
# 启用缓存会缓存id → album和photo的实体类
client.enable_cache(debug=True)
album: JmAlbumDetail = client.get_album_detail('427413')
def show(p):
p: JmPhotoDetail = client.get_photo_detail(p.photo_id, False)
for img in p:
def show(photo):
photo: JmPhotoDetail = client.get_photo_detail(photo.photo_id, False)
for img in photo:
img: JmImageDetail
print(img.img_url)
@ -39,8 +38,6 @@ def get_album_photo_detail():
@timeit('搜索本子: ')
def search_jm_album():
client = jm_option.build_jm_client()
# 分页查询
search_page: JmSearchPage = client.search_album(search_query='+MANA +无修正', page=1)
for album_id, title in search_page:
@ -51,20 +48,22 @@ def search_jm_album():
album: JmAlbumDetail = search_page.single_album
print(album.keywords)
@timeit('搜索并下载本子: ')
def search_and_download():
tag = '無修正'
search_album: JmSearchPage = cl.search_album(tag, main_tag=3)
search_page: JmSearchPage = client.search_album(tag, main_tag=3)
id_list = []
for arg in search_album.album_info_list:
for arg in search_page.album_info_list:
(album_id, title, category_none, label_sub_none, tag_list) = arg
if tag in tag_list:
print(f'[标签/{tag}] 发现目标: [{album_id}]: [{title}]')
id_list.append(album_id)
download_album(id_list, op)
download_album(id_list, option)
def main():
search_jm_album()

View File

@ -2,8 +2,8 @@
# 每行的首尾可以有空白字符
jm_albums = '''
452859
https://18comic.vip/photo/452859/mana-ディシア-1-原神-中国語-無修正
JM452859
'''