mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
【项目文档】更新README,增加代码示例,支持GitHub Actions输入DIR_RULE (#115)
This commit is contained in:
parent
2fa094cdc0
commit
b537a0e683
14
.github/workflows/download_dispatch.yml
vendored
14
.github/workflows/download_dispatch.yml
vendored
@ -7,7 +7,12 @@ on:
|
|||||||
type: string
|
type: string
|
||||||
description: 本子id(多个id用-隔开,如 '123-456-789')
|
description: 本子id(多个id用-隔开,如 '123-456-789')
|
||||||
required: true
|
required: true
|
||||||
|
|
||||||
|
DIR_RULE:
|
||||||
|
type: string
|
||||||
|
description: 下载文件夹规则,对应option配置文件里的dir_rule.rule。此处可以不填,默认使用repo里的配置文件的`Bd_Aauthor_Atitle_Pindex`。
|
||||||
default: ''
|
default: ''
|
||||||
|
required: false
|
||||||
|
|
||||||
ZIP_NAME:
|
ZIP_NAME:
|
||||||
type: string
|
type: string
|
||||||
@ -40,13 +45,12 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
# 工作流输入
|
# 工作流输入
|
||||||
JM_ALBUM_IDS: ${{ github.event.inputs.JM_ALBUM_IDS }}
|
JM_ALBUM_IDS: ${{ github.event.inputs.JM_ALBUM_IDS }}
|
||||||
JM_USERNAME: ${{ secrets.JM_USERNAME }}
|
DIR_RULE: ${{ github.event.inputs.DIR_RULE }}
|
||||||
JM_PASSWORD: ${{ secrets.JM_PASSWORD }}
|
|
||||||
ZIP_NAME: ${{ github.event.inputs.ZIP_NAME }}
|
ZIP_NAME: ${{ github.event.inputs.ZIP_NAME }}
|
||||||
UPLOAD_NAME: ${{ github.event.inputs.UPLOAD_NAME }}
|
UPLOAD_NAME: ${{ github.event.inputs.UPLOAD_NAME }}
|
||||||
# JM_USERNAME: ${{ github.event.inputs.JM_USERNAME }}
|
# sccrets
|
||||||
# JM_PASSWORD: ${{ github.event.inputs.JM_PASSWORD }}
|
JM_USERNAME: ${{ secrets.JM_USERNAME }}
|
||||||
|
JM_PASSWORD: ${{ secrets.JM_PASSWORD }}
|
||||||
# 固定值
|
# 固定值
|
||||||
JM_DOWNLOAD_DIR: /home/runner/work/jmcomic/download/
|
JM_DOWNLOAD_DIR: /home/runner/work/jmcomic/download/
|
||||||
|
|
||||||
|
18
README.md
18
README.md
@ -8,7 +8,7 @@
|
|||||||
|
|
||||||
## 安装教程
|
## 安装教程
|
||||||
|
|
||||||
* 通过pip官方源安装(推荐)
|
* 通过pip官方源安装(推荐,并且更新也是这个命令)
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
pip install jmcomic -i https://pypi.org/project --upgrade
|
pip install jmcomic -i https://pypi.org/project --upgrade
|
||||||
@ -30,10 +30,16 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
|||||||
# 配置项的作用是告诉程序下载时候的一些选择,
|
# 配置项的作用是告诉程序下载时候的一些选择,
|
||||||
# 比如,要下载到哪个文件夹,使用怎样的路径组织规则(比如[/作者/本子id/图片] 或者 [/作者/本子名称/图片]).
|
# 比如,要下载到哪个文件夹,使用怎样的路径组织规则(比如[/作者/本子id/图片] 或者 [/作者/本子名称/图片]).
|
||||||
# 如果没有配置,则会使用 JmOption.default(),下载的路径是[当前工作文件夹/本子名称/图片].
|
# 如果没有配置,则会使用 JmOption.default(),下载的路径是[当前工作文件夹/本子名称/图片].
|
||||||
# 如果你想要配置,请参考assets/config/和usgae/下的文档和示例.
|
# 如果你想要配置,请参考文件 assets/config/常用配置介绍.yml
|
||||||
```
|
```
|
||||||
|
|
||||||
进一步的使用可以参考usage文件夹下的示例代码: `getting_started.py` `usage_simple.py` `usage_feature_filter`
|
进阶使用可以参考本repo下usage文件夹内的示例代码文件:
|
||||||
|
|
||||||
|
- API上手介绍: `getting_started.py`
|
||||||
|
- 使用API实现简单功能: `usage_simple.py`
|
||||||
|
- 使用API的Filter过滤功能: `usage_feature_filter.py`
|
||||||
|
- 测试你的ip可以访问哪些禁漫域名: `pick_domain.py`
|
||||||
|
- 基于GitHub Actions下载本子: `workflow_download.py`
|
||||||
|
|
||||||
## 项目特点
|
## 项目特点
|
||||||
|
|
||||||
@ -42,13 +48,13 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
|||||||
- **可配置性强**
|
- **可配置性强**
|
||||||
- 不配置也能使用,十分方便
|
- 不配置也能使用,十分方便
|
||||||
- 配置可以从**配置文件**生成,支持多种文件格式,无需写Python代码
|
- 配置可以从**配置文件**生成,支持多种文件格式,无需写Python代码
|
||||||
- 配置点有:`是否使用磁盘缓存` `图片类型转换` `下载路径` `请求元信息(headers,cookies,代理)`等
|
- 配置点有:`是否使用磁盘缓存` `并发下载图片数` `图片类型转换` `下载路径` `请求元信息(headers,cookies,proxies)`等
|
||||||
- **可扩展性强**
|
- **可扩展性强**
|
||||||
- 支持自定义本子/章节/图片下载前后的回调函数
|
- 支持自定义本子/章节/图片下载前后的回调函数
|
||||||
- 支持自定义debug日志的开关/格式
|
- 支持自定义debug日志的开关/格式
|
||||||
- 支持自定义Option/Client/实体类
|
- 支持自定义Downloader/Option/Client/实体类
|
||||||
- ...
|
- ...
|
||||||
- 支持重试和域名切换机制
|
- 支持自动重试和域名切换机制
|
||||||
- **多线程下载**(可细化到一图一线程,效率极高)
|
- **多线程下载**(可细化到一图一线程,效率极高)
|
||||||
- 跟进了JM最新的图片分割算法(2023-02-08)
|
- 跟进了JM最新的图片分割算法(2023-02-08)
|
||||||
|
|
||||||
|
@ -8,5 +8,5 @@ dir_rule:
|
|||||||
|
|
||||||
client:
|
client:
|
||||||
domain:
|
domain:
|
||||||
|
- jmcomic1.me
|
||||||
- jmcomic.me
|
- jmcomic.me
|
||||||
- jmcomic1.me
|
|
55
usage/pick_domain.py
Normal file
55
usage/pick_domain.py
Normal file
@ -0,0 +1,55 @@
|
|||||||
|
"""
|
||||||
|
该脚本的作用:测试使用当前ip可以访问哪些禁漫域名
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
from jmcomic import *
|
||||||
|
|
||||||
|
option = JmOption.default()
|
||||||
|
|
||||||
|
|
||||||
|
def get_domain_ls():
|
||||||
|
template = 'https://jmcmomic.github.io/go/{}.html'
|
||||||
|
url_ls = [
|
||||||
|
template.format(i)
|
||||||
|
for i in range(300, 309)
|
||||||
|
]
|
||||||
|
domain_set: Set[str] = set()
|
||||||
|
|
||||||
|
def fetch_domain(url):
|
||||||
|
postman = JmModuleConfig.new_postman()
|
||||||
|
text = postman.get(url, allow_redirects=False).text
|
||||||
|
for domain in JmcomicText.analyse_jm_pub_html(text):
|
||||||
|
domain_set.add(domain)
|
||||||
|
|
||||||
|
multi_thread_launcher(
|
||||||
|
iter_objs=url_ls,
|
||||||
|
apply_each_obj_func=fetch_domain,
|
||||||
|
)
|
||||||
|
return domain_set
|
||||||
|
|
||||||
|
|
||||||
|
domain_set = get_domain_ls()
|
||||||
|
domain_status_dict = {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_domain(domain: str):
|
||||||
|
client = option.new_jm_client(domain_list=[domain])
|
||||||
|
status = 'ok'
|
||||||
|
|
||||||
|
try:
|
||||||
|
client.get_album_detail('123456')
|
||||||
|
except Exception as e:
|
||||||
|
status = str(e.args)
|
||||||
|
pass
|
||||||
|
|
||||||
|
domain_status_dict[domain] = status
|
||||||
|
|
||||||
|
|
||||||
|
multi_thread_launcher(
|
||||||
|
iter_objs=domain_set,
|
||||||
|
apply_each_obj_func=test_domain,
|
||||||
|
)
|
||||||
|
|
||||||
|
for domain, status in domain_status_dict.items():
|
||||||
|
print(f'{domain}: {status}')
|
@ -1,17 +1,19 @@
|
|||||||
|
from jmcomic import *
|
||||||
|
|
||||||
# 下方填入你要下载的本子的id,一行一个。
|
# 下方填入你要下载的本子的id,一行一个。
|
||||||
# 每行的首尾可以有空白字符
|
# 每行的首尾可以有空白字符
|
||||||
|
# 你也可以填入本子网址,程序会识别出本子id
|
||||||
|
# 例如:
|
||||||
|
# [https://18comic.vip/album/452859/mana-ディシア-1-原神-中国語-無修正] -> [452859]
|
||||||
|
#
|
||||||
jm_albums = '''
|
jm_albums = '''
|
||||||
452859
|
|
||||||
https://18comic.vip/photo/452859/mana-ディシア-1-原神-中国語-無修正
|
|
||||||
JM452859
|
|
||||||
|
|
||||||
|
|
||||||
'''
|
'''
|
||||||
|
|
||||||
|
|
||||||
def get_jm_album_ids():
|
def get_jm_album_ids():
|
||||||
from common import str_to_set
|
|
||||||
|
|
||||||
aid_set = set()
|
aid_set = set()
|
||||||
for text in [
|
for text in [
|
||||||
jm_albums,
|
jm_albums,
|
||||||
@ -23,43 +25,54 @@ def get_jm_album_ids():
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
from jmcomic import download_album
|
|
||||||
# 下载漫画
|
# 下载漫画
|
||||||
download_album(get_jm_album_ids(), option=get_option())
|
download_album(get_jm_album_ids(), option=get_option())
|
||||||
|
|
||||||
|
|
||||||
def get_option():
|
def get_option():
|
||||||
from jmcomic import create_option, print_eye_catching
|
|
||||||
|
|
||||||
# 读取 option 配置文件
|
# 读取 option 配置文件
|
||||||
option = create_option('../assets/config/option_workflow_download.yml')
|
option = create_option('../assets/config/option_workflow_download.yml')
|
||||||
|
|
||||||
|
# 支持工作流覆盖配置文件的配置
|
||||||
|
cover_option_config(option)
|
||||||
|
|
||||||
|
# 覆盖client实现类,实现把请求错误的html下载到文件,方便GitHub Actions下载查看日志
|
||||||
hook_debug(option)
|
hook_debug(option)
|
||||||
|
|
||||||
# 启用 client 的缓存
|
# 启用 client 的缓存
|
||||||
client = option.build_jm_client()
|
client = option.build_jm_client()
|
||||||
client.enable_cache()
|
client.enable_cache()
|
||||||
|
|
||||||
|
# 登录,如果有配置的话
|
||||||
|
login_if_configured(client)
|
||||||
|
|
||||||
|
return option
|
||||||
|
|
||||||
|
|
||||||
|
def cover_option_config(option: JmOption):
|
||||||
|
dir_rule = get_env('DIR_RULE')
|
||||||
|
if dir_rule is not None:
|
||||||
|
the_old = option.dir_rule
|
||||||
|
the_new = DirRule(dir_rule, base_dir=the_old.base_dir)
|
||||||
|
option.dir_rule = the_new
|
||||||
|
|
||||||
|
|
||||||
|
def login_if_configured(client):
|
||||||
# 检查环境变量中是否有禁漫的用户名和密码,如果有则登录
|
# 检查环境变量中是否有禁漫的用户名和密码,如果有则登录
|
||||||
# 禁漫的大部分本子,下载是不需要登录的,少部分敏感题材需要登录
|
# 禁漫的大部分本子,下载是不需要登录的,少部分敏感题材需要登录
|
||||||
# 如果你希望以登录状态下载本子,你需要自己配置一下GitHub Actions的 `secrets`
|
# 如果你希望以登录状态下载本子,你需要自己配置一下GitHub Actions的 `secrets`
|
||||||
# 配置的方式很简单,网页上点一点就可以了
|
# 配置的方式很简单,网页上点一点就可以了
|
||||||
# 具体做法请去看官方教程:https://docs.github.com/en/actions/security-guides/encrypted-secrets
|
# 具体做法请去看官方教程:https://docs.github.com/en/actions/security-guides/encrypted-secrets
|
||||||
|
|
||||||
# 萌新注意!!!如果你想 `开源` 你的禁漫帐号,你也可以直接把账号密码写到下面的代码😅
|
# 萌新注意!!!如果你想 `开源` 你的禁漫帐号,你也可以直接把账号密码写到下面的代码😅
|
||||||
|
|
||||||
username = get_env('JM_USERNAME')
|
username = get_env('JM_USERNAME')
|
||||||
password = get_env('JM_PASSWORD')
|
password = get_env('JM_PASSWORD')
|
||||||
|
|
||||||
if username is not None and password is not None:
|
if username is not None and password is not None:
|
||||||
client.login(username, password, True)
|
client.login(username, password, True)
|
||||||
print_eye_catching(f'登录禁漫成功')
|
print_eye_catching(f'登录禁漫成功')
|
||||||
|
|
||||||
return option
|
|
||||||
|
|
||||||
|
|
||||||
|
# noinspection PyUnusedLocal
|
||||||
def hook_debug(option):
|
def hook_debug(option):
|
||||||
from jmcomic import JmHtmlClient, workspace, mkdir_if_not_exists, JmModuleConfig
|
|
||||||
|
|
||||||
jm_download_dir = get_env('JM_DOWNLOAD_DIR') or workspace()
|
jm_download_dir = get_env('JM_DOWNLOAD_DIR') or workspace()
|
||||||
mkdir_if_not_exists(jm_download_dir)
|
mkdir_if_not_exists(jm_download_dir)
|
||||||
|
|
||||||
@ -67,7 +80,7 @@ def hook_debug(option):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def raise_request_error(cls, resp, msg=None):
|
def raise_request_error(cls, resp, msg=None):
|
||||||
from common import write_text, fix_windir_name, format_ts
|
from common import write_text, fix_windir_name
|
||||||
write_text(
|
write_text(
|
||||||
f'{jm_download_dir}/{fix_windir_name(resp.url)}',
|
f'{jm_download_dir}/{fix_windir_name(resp.url)}',
|
||||||
resp.text
|
resp.text
|
||||||
|
Loading…
Reference in New Issue
Block a user