mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
【项目文档】更新README,增加代码示例,支持GitHub Actions输入DIR_RULE (#115)
This commit is contained in:
parent
2fa094cdc0
commit
b537a0e683
14
.github/workflows/download_dispatch.yml
vendored
14
.github/workflows/download_dispatch.yml
vendored
@ -7,7 +7,12 @@ on:
|
||||
type: string
|
||||
description: 本子id(多个id用-隔开,如 '123-456-789')
|
||||
required: true
|
||||
|
||||
DIR_RULE:
|
||||
type: string
|
||||
description: 下载文件夹规则,对应option配置文件里的dir_rule.rule。此处可以不填,默认使用repo里的配置文件的`Bd_Aauthor_Atitle_Pindex`。
|
||||
default: ''
|
||||
required: false
|
||||
|
||||
ZIP_NAME:
|
||||
type: string
|
||||
@ -40,13 +45,12 @@ jobs:
|
||||
env:
|
||||
# 工作流输入
|
||||
JM_ALBUM_IDS: ${{ github.event.inputs.JM_ALBUM_IDS }}
|
||||
JM_USERNAME: ${{ secrets.JM_USERNAME }}
|
||||
JM_PASSWORD: ${{ secrets.JM_PASSWORD }}
|
||||
DIR_RULE: ${{ github.event.inputs.DIR_RULE }}
|
||||
ZIP_NAME: ${{ github.event.inputs.ZIP_NAME }}
|
||||
UPLOAD_NAME: ${{ github.event.inputs.UPLOAD_NAME }}
|
||||
# JM_USERNAME: ${{ github.event.inputs.JM_USERNAME }}
|
||||
# JM_PASSWORD: ${{ github.event.inputs.JM_PASSWORD }}
|
||||
|
||||
# sccrets
|
||||
JM_USERNAME: ${{ secrets.JM_USERNAME }}
|
||||
JM_PASSWORD: ${{ secrets.JM_PASSWORD }}
|
||||
# 固定值
|
||||
JM_DOWNLOAD_DIR: /home/runner/work/jmcomic/download/
|
||||
|
||||
|
18
README.md
18
README.md
@ -8,7 +8,7 @@
|
||||
|
||||
## 安装教程
|
||||
|
||||
* 通过pip官方源安装(推荐)
|
||||
* 通过pip官方源安装(推荐,并且更新也是这个命令)
|
||||
|
||||
```shell
|
||||
pip install jmcomic -i https://pypi.org/project --upgrade
|
||||
@ -30,10 +30,16 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
||||
# 配置项的作用是告诉程序下载时候的一些选择,
|
||||
# 比如,要下载到哪个文件夹,使用怎样的路径组织规则(比如[/作者/本子id/图片] 或者 [/作者/本子名称/图片]).
|
||||
# 如果没有配置,则会使用 JmOption.default(),下载的路径是[当前工作文件夹/本子名称/图片].
|
||||
# 如果你想要配置,请参考assets/config/和usgae/下的文档和示例.
|
||||
# 如果你想要配置,请参考文件 assets/config/常用配置介绍.yml
|
||||
```
|
||||
|
||||
进一步的使用可以参考usage文件夹下的示例代码: `getting_started.py` `usage_simple.py` `usage_feature_filter`
|
||||
进阶使用可以参考本repo下usage文件夹内的示例代码文件:
|
||||
|
||||
- API上手介绍: `getting_started.py`
|
||||
- 使用API实现简单功能: `usage_simple.py`
|
||||
- 使用API的Filter过滤功能: `usage_feature_filter.py`
|
||||
- 测试你的ip可以访问哪些禁漫域名: `pick_domain.py`
|
||||
- 基于GitHub Actions下载本子: `workflow_download.py`
|
||||
|
||||
## 项目特点
|
||||
|
||||
@ -42,13 +48,13 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载
|
||||
- **可配置性强**
|
||||
- 不配置也能使用,十分方便
|
||||
- 配置可以从**配置文件**生成,支持多种文件格式,无需写Python代码
|
||||
- 配置点有:`是否使用磁盘缓存` `图片类型转换` `下载路径` `请求元信息(headers,cookies,代理)`等
|
||||
- 配置点有:`是否使用磁盘缓存` `并发下载图片数` `图片类型转换` `下载路径` `请求元信息(headers,cookies,proxies)`等
|
||||
- **可扩展性强**
|
||||
- 支持自定义本子/章节/图片下载前后的回调函数
|
||||
- 支持自定义debug日志的开关/格式
|
||||
- 支持自定义Option/Client/实体类
|
||||
- 支持自定义Downloader/Option/Client/实体类
|
||||
- ...
|
||||
- 支持重试和域名切换机制
|
||||
- 支持自动重试和域名切换机制
|
||||
- **多线程下载**(可细化到一图一线程,效率极高)
|
||||
- 跟进了JM最新的图片分割算法(2023-02-08)
|
||||
|
||||
|
@ -8,5 +8,5 @@ dir_rule:
|
||||
|
||||
client:
|
||||
domain:
|
||||
- jmcomic1.me
|
||||
- jmcomic.me
|
||||
- jmcomic1.me
|
55
usage/pick_domain.py
Normal file
55
usage/pick_domain.py
Normal file
@ -0,0 +1,55 @@
|
||||
"""
|
||||
该脚本的作用:测试使用当前ip可以访问哪些禁漫域名
|
||||
|
||||
"""
|
||||
|
||||
from jmcomic import *
|
||||
|
||||
option = JmOption.default()
|
||||
|
||||
|
||||
def get_domain_ls():
|
||||
template = 'https://jmcmomic.github.io/go/{}.html'
|
||||
url_ls = [
|
||||
template.format(i)
|
||||
for i in range(300, 309)
|
||||
]
|
||||
domain_set: Set[str] = set()
|
||||
|
||||
def fetch_domain(url):
|
||||
postman = JmModuleConfig.new_postman()
|
||||
text = postman.get(url, allow_redirects=False).text
|
||||
for domain in JmcomicText.analyse_jm_pub_html(text):
|
||||
domain_set.add(domain)
|
||||
|
||||
multi_thread_launcher(
|
||||
iter_objs=url_ls,
|
||||
apply_each_obj_func=fetch_domain,
|
||||
)
|
||||
return domain_set
|
||||
|
||||
|
||||
domain_set = get_domain_ls()
|
||||
domain_status_dict = {}
|
||||
|
||||
|
||||
def test_domain(domain: str):
|
||||
client = option.new_jm_client(domain_list=[domain])
|
||||
status = 'ok'
|
||||
|
||||
try:
|
||||
client.get_album_detail('123456')
|
||||
except Exception as e:
|
||||
status = str(e.args)
|
||||
pass
|
||||
|
||||
domain_status_dict[domain] = status
|
||||
|
||||
|
||||
multi_thread_launcher(
|
||||
iter_objs=domain_set,
|
||||
apply_each_obj_func=test_domain,
|
||||
)
|
||||
|
||||
for domain, status in domain_status_dict.items():
|
||||
print(f'{domain}: {status}')
|
@ -1,17 +1,19 @@
|
||||
from jmcomic import *
|
||||
|
||||
# 下方填入你要下载的本子的id,一行一个。
|
||||
# 每行的首尾可以有空白字符
|
||||
# 你也可以填入本子网址,程序会识别出本子id
|
||||
# 例如:
|
||||
# [https://18comic.vip/album/452859/mana-ディシア-1-原神-中国語-無修正] -> [452859]
|
||||
#
|
||||
jm_albums = '''
|
||||
452859
|
||||
https://18comic.vip/photo/452859/mana-ディシア-1-原神-中国語-無修正
|
||||
JM452859
|
||||
|
||||
|
||||
|
||||
'''
|
||||
|
||||
|
||||
def get_jm_album_ids():
|
||||
from common import str_to_set
|
||||
|
||||
aid_set = set()
|
||||
for text in [
|
||||
jm_albums,
|
||||
@ -23,43 +25,54 @@ def get_jm_album_ids():
|
||||
|
||||
|
||||
def main():
|
||||
from jmcomic import download_album
|
||||
# 下载漫画
|
||||
download_album(get_jm_album_ids(), option=get_option())
|
||||
|
||||
|
||||
def get_option():
|
||||
from jmcomic import create_option, print_eye_catching
|
||||
|
||||
# 读取 option 配置文件
|
||||
option = create_option('../assets/config/option_workflow_download.yml')
|
||||
|
||||
# 支持工作流覆盖配置文件的配置
|
||||
cover_option_config(option)
|
||||
|
||||
# 覆盖client实现类,实现把请求错误的html下载到文件,方便GitHub Actions下载查看日志
|
||||
hook_debug(option)
|
||||
|
||||
# 启用 client 的缓存
|
||||
client = option.build_jm_client()
|
||||
client.enable_cache()
|
||||
|
||||
# 登录,如果有配置的话
|
||||
login_if_configured(client)
|
||||
|
||||
return option
|
||||
|
||||
|
||||
def cover_option_config(option: JmOption):
|
||||
dir_rule = get_env('DIR_RULE')
|
||||
if dir_rule is not None:
|
||||
the_old = option.dir_rule
|
||||
the_new = DirRule(dir_rule, base_dir=the_old.base_dir)
|
||||
option.dir_rule = the_new
|
||||
|
||||
|
||||
def login_if_configured(client):
|
||||
# 检查环境变量中是否有禁漫的用户名和密码,如果有则登录
|
||||
# 禁漫的大部分本子,下载是不需要登录的,少部分敏感题材需要登录
|
||||
# 如果你希望以登录状态下载本子,你需要自己配置一下GitHub Actions的 `secrets`
|
||||
# 配置的方式很简单,网页上点一点就可以了
|
||||
# 具体做法请去看官方教程:https://docs.github.com/en/actions/security-guides/encrypted-secrets
|
||||
|
||||
# 萌新注意!!!如果你想 `开源` 你的禁漫帐号,你也可以直接把账号密码写到下面的代码😅
|
||||
|
||||
username = get_env('JM_USERNAME')
|
||||
password = get_env('JM_PASSWORD')
|
||||
|
||||
if username is not None and password is not None:
|
||||
client.login(username, password, True)
|
||||
print_eye_catching(f'登录禁漫成功')
|
||||
|
||||
return option
|
||||
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
def hook_debug(option):
|
||||
from jmcomic import JmHtmlClient, workspace, mkdir_if_not_exists, JmModuleConfig
|
||||
|
||||
jm_download_dir = get_env('JM_DOWNLOAD_DIR') or workspace()
|
||||
mkdir_if_not_exists(jm_download_dir)
|
||||
|
||||
@ -67,7 +80,7 @@ def hook_debug(option):
|
||||
|
||||
@classmethod
|
||||
def raise_request_error(cls, resp, msg=None):
|
||||
from common import write_text, fix_windir_name, format_ts
|
||||
from common import write_text, fix_windir_name
|
||||
write_text(
|
||||
f'{jm_download_dir}/{fix_windir_name(resp.url)}',
|
||||
resp.text
|
||||
|
Loading…
Reference in New Issue
Block a user