mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v2.2.9: 实现基于命令行的调用方式,更加简单易用,更新文档 (#133)
This commit is contained in:
parent
05dcb7e41b
commit
f6f2d273a7
15
README.md
15
README.md
@ -37,6 +37,14 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
|||||||
# 如果你想要配置,请参考文件 assets/config/常用配置介绍.yml
|
# 如果你想要配置,请参考文件 assets/config/常用配置介绍.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
|
* v2.2.9: 新增命令行调用方式,上述的代码可以转为一行命令
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 下载album_id为422866的本子
|
||||||
|
$ jmcomic 422866
|
||||||
|
# 更多用法请参考文件 usage/usage_cl.py (命令行使用介绍)
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 进阶使用
|
## 进阶使用
|
||||||
@ -44,6 +52,7 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
|||||||
进阶使用可以参考本repo下usage文件夹内的示例代码文件,下面是各个文件的作用,你可以挑感兴趣的阅读:
|
进阶使用可以参考本repo下usage文件夹内的示例代码文件,下面是各个文件的作用,你可以挑感兴趣的阅读:
|
||||||
|
|
||||||
- API上手介绍: `getting_started.py`
|
- API上手介绍: `getting_started.py`
|
||||||
|
- 命令行使用介绍: `usage_cl.py`
|
||||||
- 使用API实现简单功能: `usage_simple.py`
|
- 使用API实现简单功能: `usage_simple.py`
|
||||||
- 演示jmcomic模块的可自定义功能点: `usage_custom.py`
|
- 演示jmcomic模块的可自定义功能点: `usage_custom.py`
|
||||||
- 使用API的Filter过滤功能: `usage_feature_filter.py`
|
- 使用API的Filter过滤功能: `usage_feature_filter.py`
|
||||||
@ -52,7 +61,6 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
|||||||
- 包括6个功能需求的介绍、实现方案和完整运行日志
|
- 包括6个功能需求的介绍、实现方案和完整运行日志
|
||||||
- 实现方案非常简洁,充分jmcomic的便利性,以及强大的插件扩展机制
|
- 实现方案非常简洁,充分jmcomic的便利性,以及强大的插件扩展机制
|
||||||
|
|
||||||
|
|
||||||
以及一些趣味用法:
|
以及一些趣味用法:
|
||||||
|
|
||||||
- 测试你的ip可以访问哪些禁漫域名: `pick_domain.py`
|
- 测试你的ip可以访问哪些禁漫域名: `pick_domain.py`
|
||||||
@ -63,10 +71,11 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
|||||||
## 项目特点
|
## 项目特点
|
||||||
|
|
||||||
- **绕过Cloudflare的反爬虫**
|
- **绕过Cloudflare的反爬虫**
|
||||||
|
- 支持使用**命令行**下载本子,无需写Python代码,简单易用
|
||||||
- 支持使用**GitHub Actions**下载本子,网页上直接输入本子id就能下载([教程:使用GitHub Actions下载禁漫本子](./assets/docs/教程:使用GitHub%20Actions下载禁漫本子.md))
|
- 支持使用**GitHub Actions**下载本子,网页上直接输入本子id就能下载([教程:使用GitHub Actions下载禁漫本子](./assets/docs/教程:使用GitHub%20Actions下载禁漫本子.md))
|
||||||
- **可配置性强**
|
- **可配置性强**
|
||||||
- 不配置也能使用,十分方便
|
- 不配置也能使用,十分方便
|
||||||
- 配置可以从**配置文件**生成,支持多种文件格式,无需写Python代码
|
- 配置可以从**配置文件**生成,支持多种文件格式
|
||||||
- 配置点有:`是否使用磁盘缓存` `并发下载图片数` `图片类型转换` `下载路径` `请求元信息(headers,cookies,proxies)`等
|
- 配置点有:`是否使用磁盘缓存` `并发下载图片数` `图片类型转换` `下载路径` `请求元信息(headers,cookies,proxies)`等
|
||||||
- **可扩展性强**
|
- **可扩展性强**
|
||||||
- **支持Plugin插件,可以方便地扩展功能,以及使用别人的插件**
|
- **支持Plugin插件,可以方便地扩展功能,以及使用别人的插件**
|
||||||
@ -81,6 +90,7 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 使用小说明
|
## 使用小说明
|
||||||
|
|
||||||
* Python >= 3.7
|
* Python >= 3.7
|
||||||
@ -88,6 +98,7 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
## 项目文件夹介绍
|
## 项目文件夹介绍
|
||||||
|
|
||||||
* assets:存放一些非代码的资源文件
|
* assets:存放一些非代码的资源文件
|
||||||
|
|||||||
5
setup.py
5
setup.py
@ -45,4 +45,9 @@ setup(
|
|||||||
"Operating System :: POSIX :: Linux",
|
"Operating System :: POSIX :: Linux",
|
||||||
"Operating System :: Microsoft :: Windows",
|
"Operating System :: Microsoft :: Windows",
|
||||||
],
|
],
|
||||||
|
entry_points={
|
||||||
|
'console_scripts': [
|
||||||
|
'jmcomic = jmcomic:main'
|
||||||
|
]
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|||||||
@ -2,7 +2,8 @@
|
|||||||
# 被依赖方 <--- 使用方
|
# 被依赖方 <--- 使用方
|
||||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||||
|
|
||||||
__version__ = '2.2.8'
|
__version__ = '2.2.9'
|
||||||
|
|
||||||
from .api import *
|
from .api import *
|
||||||
from .jm_plugin import *
|
from .jm_plugin import *
|
||||||
|
from .cl import main
|
||||||
|
|||||||
111
src/jmcomic/cl.py
Normal file
111
src/jmcomic/cl.py
Normal file
@ -0,0 +1,111 @@
|
|||||||
|
"""
|
||||||
|
command-line usage
|
||||||
|
|
||||||
|
for example, download album 123 456, photo 333:
|
||||||
|
|
||||||
|
$ jmcomic 123 456 p333 --option="D:/option.yml"
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
|
import os.path
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
|
||||||
|
def get_env(name, default):
|
||||||
|
import os
|
||||||
|
value = os.getenv(name, None)
|
||||||
|
if value is None or value == '':
|
||||||
|
return default
|
||||||
|
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class JmcomicUI:
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.option_path = None
|
||||||
|
self.raw_id_list: List[str] = []
|
||||||
|
self.album_id_list: List[str] = []
|
||||||
|
self.photo_id_list: List[str] = []
|
||||||
|
|
||||||
|
def parse_arg(self):
|
||||||
|
import argparse
|
||||||
|
parser = argparse.ArgumentParser(prog='python -m jmcomic', description='JMComic Command Line Downloader')
|
||||||
|
parser.add_argument(
|
||||||
|
'id_list',
|
||||||
|
nargs='*',
|
||||||
|
help='input all album/photo ids that you want to download, separating them by spaces. '
|
||||||
|
'Need add a "p" prefix to indicate a photo id, such as `123 456 p333`.',
|
||||||
|
default=[],
|
||||||
|
)
|
||||||
|
|
||||||
|
parser.add_argument(
|
||||||
|
'--option',
|
||||||
|
help='path to the option file, you can also specify it by env `JM_OPTION_PATH`',
|
||||||
|
default=get_env('JM_OPTION_PATH', './option.yml'),
|
||||||
|
)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
self.option_path = os.path.abspath(args.option)
|
||||||
|
self.raw_id_list = args.id_list
|
||||||
|
|
||||||
|
self.parse_raw_id()
|
||||||
|
|
||||||
|
def parse_raw_id(self):
|
||||||
|
|
||||||
|
def parse(text):
|
||||||
|
from .jm_toolkit import JmcomicText
|
||||||
|
|
||||||
|
try:
|
||||||
|
return JmcomicText.parse_to_album_id(text)
|
||||||
|
except Exception as e:
|
||||||
|
print(e.args[0])
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
for raw_id in self.raw_id_list:
|
||||||
|
if raw_id.startswith('p'):
|
||||||
|
self.photo_id_list.append(parse(raw_id[1:]))
|
||||||
|
elif raw_id.startswith('a'):
|
||||||
|
self.album_id_list.append(parse(raw_id[1:]))
|
||||||
|
else:
|
||||||
|
self.album_id_list.append(parse(raw_id))
|
||||||
|
|
||||||
|
def main(self):
|
||||||
|
self.parse_arg()
|
||||||
|
from .api import jm_debug
|
||||||
|
jm_debug('command_line',
|
||||||
|
f'start downloading...\n'
|
||||||
|
f'- using option: [{self.option_path}]\n'
|
||||||
|
f'to be downloaded: \n'
|
||||||
|
f'- album: {self.album_id_list}\n'
|
||||||
|
f'- photo: {self.photo_id_list}')
|
||||||
|
self.run()
|
||||||
|
|
||||||
|
def run(self):
|
||||||
|
from .api import download_album, download_photo, create_option
|
||||||
|
from common import MultiTaskLauncher
|
||||||
|
|
||||||
|
option = create_option(self.option_path)
|
||||||
|
|
||||||
|
if len(self.album_id_list) == 0:
|
||||||
|
download_photo(self.photo_id_list, option)
|
||||||
|
elif len(self.photo_id_list) == 0:
|
||||||
|
download_album(self.album_id_list, option)
|
||||||
|
else:
|
||||||
|
# 同时下载album和photo
|
||||||
|
launcher = MultiTaskLauncher()
|
||||||
|
|
||||||
|
launcher.create_task(
|
||||||
|
target=download_album,
|
||||||
|
args=(self.album_id_list, option)
|
||||||
|
)
|
||||||
|
launcher.create_task(
|
||||||
|
target=download_photo,
|
||||||
|
args=(self.photo_id_list, option)
|
||||||
|
)
|
||||||
|
|
||||||
|
launcher.wait_finish()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
JmcomicUI().main()
|
||||||
@ -271,7 +271,9 @@ class JmModuleConfig:
|
|||||||
返回JmOption.default()的默认配置字典。
|
返回JmOption.default()的默认配置字典。
|
||||||
这样做是为了支持外界自行覆盖option默认配置字典
|
这样做是为了支持外界自行覆盖option默认配置字典
|
||||||
"""
|
"""
|
||||||
option_dict = cls.default_option_dict.copy()
|
from copy import deepcopy
|
||||||
|
|
||||||
|
option_dict = deepcopy(cls.default_option_dict)
|
||||||
|
|
||||||
# debug
|
# debug
|
||||||
if option_dict['debug'] is None:
|
if option_dict['debug'] is None:
|
||||||
|
|||||||
@ -452,14 +452,14 @@ class JmSearchPage(JmBaseEntity, IndexedEntity):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def wrap_single_album(cls, album: JmAlbumDetail) -> 'JmSearchPage':
|
def wrap_single_album(cls, album: JmAlbumDetail) -> 'JmSearchPage':
|
||||||
obj = JmSearchPage([(
|
page = JmSearchPage([(
|
||||||
album.album_id, {
|
album.album_id, {
|
||||||
'name': album.title,
|
'name': album.title,
|
||||||
'tag_list': album.tag_list,
|
'tag_list': album.tag_list,
|
||||||
}
|
}
|
||||||
)])
|
)])
|
||||||
setattr(obj, 'album', album)
|
setattr(page, 'album', album)
|
||||||
return obj
|
return page
|
||||||
|
|
||||||
# 下面的方法实现方便的元素访问
|
# 下面的方法实现方便的元素访问
|
||||||
|
|
||||||
|
|||||||
@ -5,7 +5,7 @@ from .jm_entity import *
|
|||||||
|
|
||||||
class JmcomicText:
|
class JmcomicText:
|
||||||
pattern_jm_domain = compile('https://([\w.-]+)')
|
pattern_jm_domain = compile('https://([\w.-]+)')
|
||||||
pattern_jm_pa_id = compile('/(photos?|album)/(\d+)')
|
pattern_jm_pa_id = compile('(photos?|album)/(\d+)')
|
||||||
pattern_html_jm_pub_domain = compile('[\w-]+\.\w+/?\w+')
|
pattern_html_jm_pub_domain = compile('[\w-]+\.\w+/?\w+')
|
||||||
|
|
||||||
pattern_html_photo_photo_id = compile('<meta property="og:url" content=".*?/photo/(\d+)/?.*?">')
|
pattern_html_photo_photo_id = compile('<meta property="og:url" content=".*?/photo/(\d+)/?.*?">')
|
||||||
@ -264,7 +264,7 @@ class JmcomicSearchTool:
|
|||||||
html = match[0]
|
html = match[0]
|
||||||
|
|
||||||
# 提取结果
|
# 提取结果
|
||||||
content = [] # content这个名字来源于是api版搜索返回值
|
content = [] # content这个名字来源于api版搜索返回值
|
||||||
album_info_list = cls.pattern_html_search_album_info_list.findall(html)
|
album_info_list = cls.pattern_html_search_album_info_list.findall(html)
|
||||||
|
|
||||||
for (album_id, title, _, label_category, label_sub, tag_text) in album_info_list:
|
for (album_id, title, _, label_category, label_sub, tag_text) in album_info_list:
|
||||||
|
|||||||
27
usage/usage_cl.py
Normal file
27
usage/usage_cl.py
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
"""
|
||||||
|
|
||||||
|
使用命令行下载禁漫本子
|
||||||
|
|
||||||
|
1. 基本用法
|
||||||
|
# 下载album 123 456,下载photo 333。彼此之间使用空格间隔
|
||||||
|
```
|
||||||
|
jmcomic 123 456 p333
|
||||||
|
```
|
||||||
|
|
||||||
|
2. 自定义option
|
||||||
|
|
||||||
|
2.1. 通过命令行
|
||||||
|
```
|
||||||
|
jmcomic 123 --option="D:/a.yml"
|
||||||
|
```
|
||||||
|
|
||||||
|
2.2. 通过环境变量
|
||||||
|
设置环境: JM_OPTION_PATH = D:/a.yml
|
||||||
|
命令行的命令不变
|
||||||
|
```
|
||||||
|
jmcomic 123
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
Loading…
Reference in New Issue
Block a user