mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.1.6: 简化字段缓存的代码,调整下载章节并发数,更新文档 (#83)
This commit is contained in:
parent
9a5c5d12cd
commit
5e013e1818
17
README.md
17
README.md
@ -21,15 +21,16 @@
|
||||
|
||||
## 快速上手
|
||||
|
||||
使用下面的两行代码,即可实现功能:把某个本子集(album)里的所有本子(photo)下载到本地
|
||||
使用下面的两行代码,即可实现功能:把某个本子(album)里的所有章节(photo)下载到本地
|
||||
|
||||
```python
|
||||
import jmcomic # 导入此模块,需要先安装.
|
||||
jmcomic.download_album('422866') # 传入要下载的album的id,即可下载整个album到本地.
|
||||
# 上面的这行代码,还有一个可选参数option: JmOption,表示配置项,
|
||||
# 配置项的作用是告诉程序下载时候的一些选择,
|
||||
# 比如,要下载到哪个文件夹,使用怎样的路径组织方式(比如[/作者/本子id/图片] 或者 [/作者/本子名称/图片]).
|
||||
# 比如,要下载到哪个文件夹,使用怎样的路径组织规则(比如[/作者/本子id/图片] 或者 [/作者/本子名称/图片]).
|
||||
# 如果没有配置,则会使用 JmOption.default(),下载的路径是[当前工作文件夹/本子名称/图片].
|
||||
# 如果你想要配置,请参考assets/config/和usgae/下的文档和示例.
|
||||
```
|
||||
|
||||
进一步的使用可以参考usage文件夹下的示例代码: `getting_started.py` `sample_usage.py`
|
||||
@ -39,17 +40,21 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
||||
- **绕过Cloudflare的反爬虫**
|
||||
- 支持使用**Github Action**下载漫画,不会编程都能用([教程:使用Github Actions下载禁漫本子](./assets/docs/教程:使用Github%20Actions下载禁漫本子.md))
|
||||
- 可配置性强
|
||||
|
||||
- 不配置也能使用,十分方便
|
||||
- 配置可以从**配置文件**生成,无需写Python代码
|
||||
- 配置点有:`是否使用磁盘缓存` `是否使用代理` `图片类型转换` `本子下载路径` `请求元信息(headers,cookies,重试次数)等 `
|
||||
- 配置可以从**配置文件**生成,支持多种文件格式,无需写Python代码
|
||||
- 配置点有:`是否使用磁盘缓存` `图片类型转换` `下载路径` `请求元信息(headers,cookies,代理)等 `
|
||||
- 可扩展性强
|
||||
- 支持自定义本子/章节/图片下载前后的回调函数
|
||||
- 支持自定义debug日志的开关/格式
|
||||
- 支持自定义Option/Client/实体类
|
||||
- 支持重试和域名切换机制
|
||||
- 多线程下载(可细化到一图一线程,效率极高)
|
||||
- 跟进了JM最新的图片分割算法(2023-02-08)
|
||||
|
||||
## 使用小说明
|
||||
|
||||
* Python >= 3.7
|
||||
* 项目只有代码注释,没有API文档。因此想深入高级地使用,自行看源码和改造代码叭 ^^_
|
||||
* 个人项目,文档和示例会有不及时之处,可以Issue提问
|
||||
|
||||
## 项目文件夹介绍
|
||||
|
||||
|
@ -2,6 +2,6 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option
|
||||
|
||||
__version__ = '2.1.5'
|
||||
__version__ = '2.1.6'
|
||||
|
||||
from .api import *
|
||||
|
@ -17,7 +17,7 @@ def download_album(jm_album_id, option=None):
|
||||
|
||||
option.before_album(album)
|
||||
execute_by_condition(
|
||||
iter_obj=album,
|
||||
iter_objs=album,
|
||||
apply=lambda photo: download_by_photo_detail(photo, option),
|
||||
count_batch=option.decide_photo_batch_count(album)
|
||||
)
|
||||
@ -49,13 +49,11 @@ def download_by_photo_detail(photo: JmPhotoDetail, option=None):
|
||||
# 下载每个图片的函数
|
||||
def download_image(image: JmImageDetail):
|
||||
img_save_path = option.decide_image_filepath(image)
|
||||
|
||||
# 已下载过,缓存命中
|
||||
if use_cache is True and file_exists(img_save_path):
|
||||
image.is_exists = True
|
||||
return
|
||||
image.is_exists = file_exists(img_save_path)
|
||||
|
||||
option.before_image(image, img_save_path)
|
||||
if use_cache is True and image.is_exists:
|
||||
return
|
||||
jm_client.download_by_image_detail(
|
||||
image,
|
||||
img_save_path,
|
||||
@ -65,7 +63,7 @@ def download_by_photo_detail(photo: JmPhotoDetail, option=None):
|
||||
|
||||
option.before_photo(photo)
|
||||
execute_by_condition(
|
||||
iter_obj=photo,
|
||||
iter_objs=photo,
|
||||
apply=download_image,
|
||||
count_batch=option.decide_image_batch_count(photo)
|
||||
)
|
||||
@ -88,28 +86,28 @@ def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
|
||||
option = JmOption.default()
|
||||
|
||||
return thread_pool_executor(
|
||||
iter_objs=((album_id, option) for album_id in jm_album_id_iter),
|
||||
apply_each_obj_func=download_album,
|
||||
iter_objs=set(JmcomicText.parse_to_album_id(album_id) for album_id in jm_album_id_iter),
|
||||
apply_each_obj_func=lambda album_id: download_album(album_id, option),
|
||||
wait_finish=wait_finish,
|
||||
)
|
||||
|
||||
|
||||
def execute_by_condition(iter_obj, apply: Callable, count_batch: int):
|
||||
def execute_by_condition(iter_objs, apply: Callable, count_batch: int):
|
||||
"""
|
||||
章节/图片的下载调度逻辑
|
||||
"""
|
||||
count_real = len(iter_obj)
|
||||
count_real = len(iter_objs)
|
||||
|
||||
if count_batch >= count_real:
|
||||
# 一图一线程
|
||||
# 一个图/章节 对应 一个线程
|
||||
multi_thread_launcher(
|
||||
iter_objs=iter_obj,
|
||||
iter_objs=iter_objs,
|
||||
apply_each_obj_func=apply,
|
||||
)
|
||||
else:
|
||||
# 创建batch个线程的线程池,当图片数>batch时要等待。
|
||||
# 创建batch个线程的线程池
|
||||
thread_pool_executor(
|
||||
iter_objs=iter_obj,
|
||||
iter_objs=iter_objs,
|
||||
apply_each_obj_func=apply,
|
||||
max_workers=count_batch,
|
||||
)
|
||||
|
@ -1,3 +1,8 @@
|
||||
def field_cache(*args, **kwargs):
|
||||
from common import field_cache
|
||||
return field_cache(*args, **kwargs)
|
||||
|
||||
|
||||
def default_jm_debug(topic: str, msg: str):
|
||||
from common import format_ts
|
||||
print(f'{format_ts()}:【{topic}】{msg}')
|
||||
@ -15,11 +20,13 @@ def default_postman_constructor(session, **kwargs):
|
||||
class JmModuleConfig:
|
||||
# 网站相关
|
||||
PROT = "https://"
|
||||
DOMAIN = None
|
||||
JM_REDIRECT_URL = f'{PROT}jm365.work/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来
|
||||
JM_PUB_URL = f'{PROT}jmcomic2.bet'
|
||||
JM_CDN_IMAGE_URL_TEMPLATE = PROT + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始
|
||||
JM_IMAGE_SUFFIX = ['.jpg', '.webp', '.png', '.gif']
|
||||
# 缓存字段
|
||||
DOMAIN = None
|
||||
DOMAIN_LIST = None
|
||||
|
||||
# 访问JM可能会遇到的异常网页
|
||||
JM_ERROR_RESPONSE_TEXT = {
|
||||
@ -52,17 +59,15 @@ class JmModuleConfig:
|
||||
postman_constructor = default_postman_constructor
|
||||
|
||||
@classmethod
|
||||
@field_cache("DOMAIN")
|
||||
def domain(cls, postman=None):
|
||||
"""
|
||||
由于禁漫的域名经常变化,调用此方法可以获取一个当前可用的最新的域名 domain,
|
||||
并且设置把 domain 设置为禁漫模块的默认域名。
|
||||
这样一来,配置文件也不用配置域名了,一切都在运行时动态获取。
|
||||
"""
|
||||
if cls.DOMAIN is None:
|
||||
from .jm_toolkit import JmcomicText
|
||||
cls.DOMAIN = JmcomicText.parse_to_jm_domain(cls.get_jmcomic_url(postman))
|
||||
|
||||
return cls.DOMAIN # jmcomic默认域名
|
||||
return JmcomicText.parse_to_jm_domain(cls.get_jmcomic_url(postman))
|
||||
|
||||
@classmethod
|
||||
def headers(cls, domain='18comic.vip'):
|
||||
@ -116,6 +121,7 @@ class JmModuleConfig:
|
||||
return url
|
||||
|
||||
@classmethod
|
||||
@field_cache("DOMAIN_LIST")
|
||||
def get_jmcomic_domain_all(cls, postman=None):
|
||||
"""
|
||||
访问禁漫发布页,得到所有禁漫的域名
|
||||
|
@ -32,7 +32,7 @@ class DownloadCallback:
|
||||
f'图片已存在: {image.tag} ← [{img_save_path}]'
|
||||
)
|
||||
else:
|
||||
jm_debug('image_before',
|
||||
jm_debug('image-before',
|
||||
f'图片准备下载: {image.tag}, [{image.img_url}] → [{img_save_path}]'
|
||||
)
|
||||
|
||||
@ -159,9 +159,6 @@ class JmOption(DownloadCallback):
|
||||
# 其他配置
|
||||
self.filepath = filepath
|
||||
|
||||
# 字段
|
||||
self.jm_client_cache = None
|
||||
|
||||
@property
|
||||
def download_cache(self):
|
||||
return self.download.cache
|
||||
@ -186,9 +183,9 @@ class JmOption(DownloadCallback):
|
||||
def decide_image_batch_count(self, photo: JmPhotoDetail):
|
||||
return self.download_threading_batch_count
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
# noinspection PyMethodMayBeStatic,PyUnusedLocal
|
||||
def decide_photo_batch_count(self, album: JmAlbumDetail):
|
||||
return len(album)
|
||||
return os.cpu_count()
|
||||
|
||||
def decide_image_save_dir(self, photo) -> str:
|
||||
# 使用 self.dir_rule 决定 save_dir
|
||||
@ -266,23 +263,15 @@ class JmOption(DownloadCallback):
|
||||
"""
|
||||
|
||||
# 缓存
|
||||
cache_jm_client = True
|
||||
jm_client_impl_mapping: Dict[str, Type[AbstractJmClient]] = {
|
||||
'html': JmHtmlClient,
|
||||
'api': JmApiClient,
|
||||
}
|
||||
|
||||
@field_cache("__jm_client_cache__")
|
||||
def build_jm_client(self, **kwargs) -> JmcomicClient:
|
||||
if self.cache_jm_client is not True:
|
||||
return self.new_jm_client(**kwargs)
|
||||
|
||||
client = self.jm_client_cache
|
||||
if client is None:
|
||||
client = self.new_jm_client(**kwargs)
|
||||
self.jm_client_cache = client
|
||||
|
||||
return client
|
||||
|
||||
def new_jm_client(self, **kwargs) -> JmcomicClient:
|
||||
postman_conf: dict = self.client.postman.src_dict
|
||||
|
||||
|
@ -30,7 +30,6 @@ jmcomic.download_album(aid for aid in ('422866', '1', '2', '3')) # 生成器
|
||||
获取域名介绍
|
||||
--------------------
|
||||
"""
|
||||
|
||||
# 方式1: 访问禁漫发布页
|
||||
url_ls = jmcomic.JmModuleConfig.get_jmcomic_url_all()
|
||||
print(url_ls)
|
||||
@ -55,3 +54,16 @@ jm_option.to_file('./你的配置文件路名称.json') # json格式
|
||||
# 如果你修改了默认配置,现在想用你修改后的配置来下载,使用如下代码
|
||||
jm_option = jmcomic.create_option('./你的配置文件路名称.yml')
|
||||
jmcomic.download_album('23333', jm_option)
|
||||
|
||||
# 如果你只想做简单的配置,也可以使用如下形式
|
||||
# 具体可以写什么,请参考 JmOption.default_dict,你只需要覆盖里面的键值即可
|
||||
# 配置代理
|
||||
jm_option = JmOption.construct({
|
||||
'client': {
|
||||
'postman': {
|
||||
'meta_data': {
|
||||
'proxies': ProxyBuilder.clash_proxy(),
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
@ -1,8 +1,10 @@
|
||||
from jmcomic import *
|
||||
|
||||
jm_option = create_option(
|
||||
option = create_option(
|
||||
f'你的配置文件路径,例如: D:/a/b/c/jmcomic/config.yml'
|
||||
)
|
||||
client = option.build_jm_client()
|
||||
client.enable_cache(debug=True)
|
||||
|
||||
|
||||
@timeit('下载本子集: ')
|
||||
@ -13,21 +15,18 @@ def download_jm_album():
|
||||
|
||||
''')
|
||||
|
||||
download_album(ls, jm_option) # 效果同下面的代码
|
||||
# download_album_batch(ls, jm_option)
|
||||
download_album(ls, option) # 效果同下面的代码
|
||||
# download_album_batch(ls, op)
|
||||
|
||||
|
||||
@timeit('获取实体类: ')
|
||||
def get_album_photo_detail():
|
||||
client = jm_option.build_jm_client()
|
||||
# 启用缓存,会缓存id → album和photo的实体类
|
||||
client.enable_cache(debug=True)
|
||||
|
||||
album: JmAlbumDetail = client.get_album_detail('427413')
|
||||
|
||||
def show(p):
|
||||
p: JmPhotoDetail = client.get_photo_detail(p.photo_id, False)
|
||||
for img in p:
|
||||
def show(photo):
|
||||
photo: JmPhotoDetail = client.get_photo_detail(photo.photo_id, False)
|
||||
for img in photo:
|
||||
img: JmImageDetail
|
||||
print(img.img_url)
|
||||
|
||||
@ -39,8 +38,6 @@ def get_album_photo_detail():
|
||||
|
||||
@timeit('搜索本子: ')
|
||||
def search_jm_album():
|
||||
client = jm_option.build_jm_client()
|
||||
|
||||
# 分页查询
|
||||
search_page: JmSearchPage = client.search_album(search_query='+MANA +无修正', page=1)
|
||||
for album_id, title in search_page:
|
||||
@ -51,20 +48,22 @@ def search_jm_album():
|
||||
album: JmAlbumDetail = search_page.single_album
|
||||
print(album.keywords)
|
||||
|
||||
|
||||
@timeit('搜索并下载本子: ')
|
||||
def search_and_download():
|
||||
tag = '無修正'
|
||||
search_album: JmSearchPage = cl.search_album(tag, main_tag=3)
|
||||
search_page: JmSearchPage = client.search_album(tag, main_tag=3)
|
||||
|
||||
id_list = []
|
||||
|
||||
for arg in search_album.album_info_list:
|
||||
for arg in search_page.album_info_list:
|
||||
(album_id, title, category_none, label_sub_none, tag_list) = arg
|
||||
if tag in tag_list:
|
||||
print(f'[标签/{tag}] 发现目标: [{album_id}]: [{title}]')
|
||||
id_list.append(album_id)
|
||||
|
||||
download_album(id_list, op)
|
||||
download_album(id_list, option)
|
||||
|
||||
|
||||
def main():
|
||||
search_jm_album()
|
||||
|
@ -2,8 +2,8 @@
|
||||
# 每行的首尾可以有空白字符
|
||||
jm_albums = '''
|
||||
452859
|
||||
|
||||
|
||||
https://18comic.vip/photo/452859/mana-ディシア-1-原神-中国語-無修正
|
||||
JM452859
|
||||
|
||||
|
||||
'''
|
||||
|
Loading…
Reference in New Issue
Block a user