mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.3.3: 实现移动端API的图片下载功能,重构异常机制,优化工作流 (#143)
This commit is contained in:
parent
ed83be0629
commit
49e6d26aaf
2
.github/workflows/download.yml
vendored
2
.github/workflows/download.yml
vendored
@ -1,4 +1,4 @@
|
||||
name: 下载JM本子
|
||||
name: 下载JM本子 (commit)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
4
.github/workflows/download_dispatch.yml
vendored
4
.github/workflows/download_dispatch.yml
vendored
@ -1,4 +1,4 @@
|
||||
name: 下载JM本子(dispatch)
|
||||
name: 下载JM本子 (dispatch)
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@ -54,7 +54,7 @@ jobs:
|
||||
DIR_RULE: ${{ github.event.inputs.DIR_RULE }}
|
||||
ZIP_NAME: ${{ github.event.inputs.ZIP_NAME }}
|
||||
UPLOAD_NAME: ${{ github.event.inputs.UPLOAD_NAME }}
|
||||
# sccrets
|
||||
# secrets
|
||||
JM_USERNAME: ${{ secrets.JM_USERNAME }}
|
||||
JM_PASSWORD: ${{ secrets.JM_PASSWORD }}
|
||||
# 固定值
|
||||
|
9
.github/workflows/release.yml
vendored
9
.github/workflows/release.yml
vendored
@ -1,4 +1,5 @@
|
||||
name: 发布PYPI
|
||||
# publish when new release
|
||||
name: Publish If Release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@ -7,7 +8,7 @@ on:
|
||||
|
||||
jobs:
|
||||
release:
|
||||
name: 构建并发布 jmcomic 到 PyPI
|
||||
name: Publish `jmcomic` to PYPI
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
@ -17,12 +18,12 @@ jobs:
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
- name: 构建模块
|
||||
- name: Build
|
||||
run: |
|
||||
python -m pip install build
|
||||
python -m build
|
||||
|
||||
- name: 发布PYPI
|
||||
- name: Publish PYPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
password: ${{ secrets.PYPI_JMCOMIC }}
|
||||
|
6
.github/workflows/release_auto.yml
vendored
6
.github/workflows/release_auto.yml
vendored
@ -1,5 +1,5 @@
|
||||
name: Release (auto)
|
||||
|
||||
# auto release and publish
|
||||
name: Auto Release & Publish
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
@ -30,7 +30,7 @@ jobs:
|
||||
tag_name: ${{ steps.tb.outputs.tag }}
|
||||
body: ${{ steps.tb.outputs.body }}
|
||||
|
||||
- name: Build Module
|
||||
- name: Build
|
||||
run: |
|
||||
python -m pip install build
|
||||
python -m build
|
||||
|
@ -1,4 +1,4 @@
|
||||
name: 跑测试
|
||||
name: Run Test (API)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
@ -9,8 +9,8 @@ on:
|
||||
paths:
|
||||
- 'src/**/*.py'
|
||||
- 'tests/**/*.py'
|
||||
- '.github/workflows/test.yml'
|
||||
- 'assets/config/option_test.yml'
|
||||
- '.github/workflows/test_api.yml'
|
||||
- 'assets/config/option_test_api.yml'
|
||||
|
||||
jobs:
|
||||
test: # This code is based on https://github.com/gaogaotiantian/viztracer/blob/master/.github/workflows/python-package.yml
|
||||
@ -20,6 +20,9 @@ jobs:
|
||||
os: [ ubuntu-latest ]
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
# 配置文件路径
|
||||
JM_OPTION_PATH_TEST: ./assets/config/option_test_api.yml
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
@ -31,28 +34,15 @@ jobs:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
if: matrix.os != 'windows-latest'
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
|
||||
|
||||
- name: Install dependencies on Windows
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
if (Test-Path -Path '.\requirements-dev.txt' -PathType Leaf) {pip install -r requirements-dev.txt}
|
||||
|
||||
- name: Install local
|
||||
run: |
|
||||
pip install -e ./
|
||||
|
||||
- name: 跑测试
|
||||
if: matrix.os != 'windows-latest'
|
||||
- name: Run Test
|
||||
run: |
|
||||
cd ./tests/
|
||||
python -m unittest
|
||||
|
||||
- name: 跑测试(Windows系统)
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: |
|
||||
python -m unittest discover -s ./tests -p "test_*.py"
|
48
.github/workflows/test_html.yml
vendored
Normal file
48
.github/workflows/test_html.yml
vendored
Normal file
@ -0,0 +1,48 @@
|
||||
name: Run Test (HTML)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * *"
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches: [ "dev" ]
|
||||
paths:
|
||||
- 'src/**/*.py'
|
||||
- 'tests/**/*.py'
|
||||
- '.github/workflows/test_html.yml'
|
||||
- 'assets/config/option_test_html.yml'
|
||||
|
||||
jobs:
|
||||
test: # This code is based on https://github.com/gaogaotiantian/viztracer/blob/master/.github/workflows/python-package.yml
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [ 3.7, 3.8, 3.9, "3.10", "3.11" ]
|
||||
os: [ ubuntu-latest ]
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 5
|
||||
env:
|
||||
# 配置文件路径
|
||||
JM_OPTION_PATH_TEST: ./assets/config/option_test_html.yml
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
|
||||
|
||||
- name: Install local
|
||||
run: |
|
||||
pip install -e ./
|
||||
|
||||
- name: Run Test
|
||||
run: |
|
||||
cd ./tests/
|
||||
python -m unittest
|
22
assets/config/option_test_api.yml
Normal file
22
assets/config/option_test_api.yml
Normal file
@ -0,0 +1,22 @@
|
||||
# GitHub Actions 测试用
|
||||
# 移动端配置
|
||||
|
||||
client:
|
||||
impl: api
|
||||
domain: [
|
||||
"www.jmapinode1.cc",
|
||||
"www.jmapinode2.cc",
|
||||
"www.jmapinode3.cc",
|
||||
"www.jmapibranch2.cc"
|
||||
]
|
||||
postman:
|
||||
meta_data:
|
||||
timeout: 10
|
||||
|
||||
# 插件配置
|
||||
plugin:
|
||||
after_init:
|
||||
- plugin: usage_log # 实时打印硬件占用率的插件
|
||||
kwargs:
|
||||
interval: 0.5 # 间隔时间
|
||||
enable_warning: false # 不告警
|
@ -1,10 +1,13 @@
|
||||
# GitHub Actions 测试用
|
||||
version: '2.0'
|
||||
# 网页端配置
|
||||
|
||||
client:
|
||||
domain:
|
||||
- jmcomic1.me
|
||||
- jmcomic.me
|
||||
postman:
|
||||
meta_data:
|
||||
timeout: 10
|
||||
|
||||
# 插件配置
|
||||
plugin:
|
@ -38,11 +38,13 @@ download:
|
||||
decode: true # JM的原图是混淆过的,要不要还原?
|
||||
suffix: .jpg # 把图片都转为.jpg格式
|
||||
threading:
|
||||
# batch_count: 章节的批量下载图片线程数
|
||||
# image: 同时下载的图片数,默认是30张图
|
||||
# 数值大,下得快,配置要求高,对禁漫压力大
|
||||
# 数值小,下得慢,配置要求低,对禁漫压力小
|
||||
# PS: 禁漫网页一般是一次请求50张图
|
||||
batch_count: 30
|
||||
# PS: 禁漫网页一次最多请求50张图
|
||||
image: 30
|
||||
# photo: 同时下载的章节数,不配置默认是cpu核心数
|
||||
photo: 16
|
||||
|
||||
|
||||
|
||||
|
@ -2,7 +2,7 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.3.2'
|
||||
__version__ = '2.3.3'
|
||||
|
||||
from .api import *
|
||||
from .jm_plugin import *
|
||||
|
@ -68,3 +68,12 @@ def new_downloader(option=None, downloader=None) -> JmDownloader:
|
||||
|
||||
def create_option(filepath):
|
||||
return JmModuleConfig.option_class().from_file(filepath)
|
||||
|
||||
|
||||
def create_option_by_env(env_name='JM_OPTION_PATH'):
|
||||
from .cl import get_env
|
||||
|
||||
filepath = get_env(env_name, None)
|
||||
ExceptionTool.require_true(filepath is not None,
|
||||
f'未配置环境变量: {env_name},请配置为option的文件路径')
|
||||
return create_option(filepath)
|
||||
|
@ -33,6 +33,18 @@ class AbstractJmClient(
|
||||
def of_api_url(self, api_path, domain):
|
||||
return JmcomicText.format_url(api_path, domain)
|
||||
|
||||
def get_jm_image(self, img_url) -> JmImageResp:
|
||||
|
||||
def get_if_fail_raise(url):
|
||||
"""
|
||||
使用此方法包装 self.get,使得图片数据为空时,判定为请求失败时,走重试逻辑
|
||||
"""
|
||||
resp = JmImageResp(self.get(url))
|
||||
resp.require_success()
|
||||
return resp
|
||||
|
||||
return self.request_with_retry(get_if_fail_raise, img_url)
|
||||
|
||||
def request_with_retry(self,
|
||||
request,
|
||||
url,
|
||||
@ -84,7 +96,7 @@ class AbstractJmClient(
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def debug_topic_request(self):
|
||||
return 'html'
|
||||
return self.client_key
|
||||
|
||||
# noinspection PyMethodMayBeStatic, PyUnusedLocal
|
||||
def before_retry(self, e, kwargs, retry_count, url):
|
||||
@ -109,9 +121,9 @@ class AbstractJmClient(
|
||||
setattr(self, cache_dict_name, cache_dict)
|
||||
else:
|
||||
if sys.version_info < (3, 9):
|
||||
raise NotImplementedError('不支持启用JmcomicClient缓存。\n'
|
||||
'请更新python版本到3.9以上,'
|
||||
'或更新commonX: `pip install commonX --upgrade`')
|
||||
ExceptionTool.raises('不支持启用JmcomicClient缓存。\n'
|
||||
'请更新python版本到3.9以上,'
|
||||
'或更新commonX: `pip install commonX --upgrade`')
|
||||
import functools
|
||||
cache = functools.cache
|
||||
|
||||
@ -149,7 +161,7 @@ class AbstractJmClient(
|
||||
def fallback(self, request, url, domain_index, retry_count, **kwargs):
|
||||
msg = f"请求重试全部失败: [{url}], {self.domain_list}"
|
||||
jm_debug('req.fallback', msg)
|
||||
raise JmModuleConfig.exception(msg)
|
||||
ExceptionTool.raises(msg)
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def append_params_to_url(self, url, params):
|
||||
@ -251,7 +263,7 @@ class JmHtmlClient(AbstractJmClient):
|
||||
)
|
||||
|
||||
if resp.status_code != 301:
|
||||
raise JmModuleConfig.exception(f'登录失败,状态码为{resp.status_code}')
|
||||
ExceptionTool.raises_resp(f'登录失败,状态码为{resp.status_code}', resp)
|
||||
|
||||
if refresh_client_cookies is True:
|
||||
self['cookies'] = resp.cookies
|
||||
@ -265,7 +277,10 @@ class JmHtmlClient(AbstractJmClient):
|
||||
resp = self.get(url, **kwargs)
|
||||
|
||||
if require_200 is True and resp.status_code != 200:
|
||||
# 检查是否是特殊的状态码(JmModuleConfig.JM_ERROR_STATUS_CODE)
|
||||
# 如果是,直接抛出异常
|
||||
self.check_special_http_code(resp)
|
||||
# 运行到这里说明上一步没有抛异常,说明是未知状态码,抛异常兜底处理
|
||||
self.raise_request_error(resp)
|
||||
|
||||
# 检查请求是否成功
|
||||
@ -285,26 +300,8 @@ class JmHtmlClient(AbstractJmClient):
|
||||
+ (f"响应文本=[{resp.text}]" if len(resp.text) < 200 else
|
||||
f'响应文本过长(len={len(resp.text)}),不打印'
|
||||
)
|
||||
raise JmModuleConfig.exception(msg)
|
||||
|
||||
def get_jm_image(self, img_url) -> JmImageResp:
|
||||
|
||||
def get_if_fail_raise(url):
|
||||
"""
|
||||
使用此方法包装 self.get
|
||||
"""
|
||||
resp = JmImageResp(self.get(url))
|
||||
|
||||
if resp.is_success:
|
||||
return resp
|
||||
|
||||
self.raise_request_error(
|
||||
resp.resp, resp.get_error_msg()
|
||||
)
|
||||
|
||||
return resp
|
||||
|
||||
return self.request_with_retry(get_if_fail_raise, img_url)
|
||||
ExceptionTool.raises_resp(msg, resp)
|
||||
|
||||
def album_comment(self,
|
||||
video_id,
|
||||
@ -344,17 +341,15 @@ class JmHtmlClient(AbstractJmClient):
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def require_resp_success_else_raise(cls, resp, req_url):
|
||||
def require_resp_success_else_raise(cls, resp, org_req_url: str):
|
||||
"""
|
||||
@param resp: 响应对象
|
||||
@param org_req_url: /photo/12412312
|
||||
"""
|
||||
# 1. 检查是否 album_missing
|
||||
error_album_missing = '/error/album_missing'
|
||||
if resp.url.endswith(error_album_missing) and not req_url.endswith(error_album_missing):
|
||||
cls.raise_request_error(
|
||||
resp,
|
||||
f'请求的本子不存在!({req_url})\n'
|
||||
'原因可能为:\n'
|
||||
'1. id有误,检查你的本子/章节id\n'
|
||||
'2. 该漫画只对登录用户可见,请配置你的cookies\n'
|
||||
)
|
||||
if resp.url.endswith(error_album_missing) and not org_req_url.endswith(error_album_missing):
|
||||
ExceptionTool.raise_missing(resp, org_req_url)
|
||||
|
||||
# 2. 是否是特殊的内容
|
||||
cls.check_special_text(resp)
|
||||
@ -400,6 +395,7 @@ class JmApiClient(AbstractJmClient):
|
||||
API_SEARCH = '/search'
|
||||
API_ALBUM = '/album'
|
||||
API_CHAPTER = '/chapter'
|
||||
API_SCRAMBLE = '/chapter_view_template'
|
||||
|
||||
def __init__(self,
|
||||
postman: Postman,
|
||||
@ -416,7 +412,7 @@ class JmApiClient(AbstractJmClient):
|
||||
order_by: str,
|
||||
time: str,
|
||||
) -> JmSearchPage:
|
||||
resp = self.get(
|
||||
resp = self.get_decode(
|
||||
self.API_SEARCH,
|
||||
params={
|
||||
'search_query': search_query,
|
||||
@ -447,13 +443,16 @@ class JmApiClient(AbstractJmClient):
|
||||
)
|
||||
|
||||
def get_photo_detail(self, photo_id, fetch_album=True) -> JmPhotoDetail:
|
||||
return self.fetch_detail_entity(photo_id,
|
||||
JmModuleConfig.photo_class(),
|
||||
)
|
||||
photo_id = JmcomicText.parse_to_photo_id(photo_id)
|
||||
photo: JmPhotoDetail = self.fetch_detail_entity(photo_id,
|
||||
JmModuleConfig.photo_class(),
|
||||
)
|
||||
photo.scramble_id = self.get_scramble_id(photo.photo_id)
|
||||
return photo
|
||||
|
||||
def fetch_detail_entity(self, apid, clazz, **kwargs):
|
||||
url = self.API_ALBUM if issubclass(clazz, JmAlbumDetail) else self.API_CHAPTER
|
||||
resp = self.get(
|
||||
resp = self.get_decode(
|
||||
url,
|
||||
params={
|
||||
'id': JmcomicText.parse_to_album_id(apid),
|
||||
@ -461,11 +460,37 @@ class JmApiClient(AbstractJmClient):
|
||||
}
|
||||
)
|
||||
|
||||
self.require_resp_success(resp)
|
||||
self.require_resp_success(resp, url)
|
||||
|
||||
return JmApiAdaptTool.parse_entity(resp.res_data, clazz)
|
||||
|
||||
def get(self, url, **kwargs) -> JmApiResp:
|
||||
def get_scramble_id(self, photo_id):
|
||||
cache = JmModuleConfig.SCRAMBLE_CACHE
|
||||
if photo_id in cache:
|
||||
return cache[photo_id]
|
||||
|
||||
resp = self.get_decode(
|
||||
self.API_SCRAMBLE,
|
||||
params={
|
||||
"id": photo_id,
|
||||
"mode": "vertical",
|
||||
"page": "0",
|
||||
"app_img_shunt": "NaN",
|
||||
}
|
||||
)
|
||||
|
||||
match = JmcomicText.pattern_html_album_scramble_id.search(resp.text)
|
||||
|
||||
if match is not None:
|
||||
scramble_id = match[1]
|
||||
else:
|
||||
jm_debug('api.scramble', '未从响应中匹配到scramble_id,返回默认值220980')
|
||||
scramble_id = 220980
|
||||
|
||||
cache[photo_id] = scramble_id
|
||||
return scramble_id
|
||||
|
||||
def get_decode(self, url, **kwargs) -> JmApiResp:
|
||||
# set headers
|
||||
headers, key_ts = self.headers_key_ts
|
||||
kwargs.setdefault('headers', headers)
|
||||
@ -488,10 +513,19 @@ class JmApiClient(AbstractJmClient):
|
||||
def debug_topic_request(self):
|
||||
return 'api'
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def require_resp_success(self, resp: JmApiResp):
|
||||
@classmethod
|
||||
def require_resp_success(cls, resp: JmApiResp, org_req_url: str):
|
||||
resp.require_success()
|
||||
|
||||
# 1. 检查是否 album_missing
|
||||
# json: {'code': 200, 'data': []}
|
||||
data = resp.model().data
|
||||
if isinstance(data, list) and len(data) == 0:
|
||||
ExceptionTool.raise_missing(resp, org_req_url)
|
||||
|
||||
# 2. 是否是特殊的内容
|
||||
# 暂无
|
||||
|
||||
|
||||
JmModuleConfig.register_client(JmHtmlClient)
|
||||
JmModuleConfig.register_client(JmApiClient)
|
||||
|
@ -8,6 +8,7 @@ Response Entity
|
||||
|
||||
DictModel = AdvancedEasyAccessDict
|
||||
|
||||
|
||||
class JmResp(CommonResp):
|
||||
|
||||
@property
|
||||
@ -22,7 +23,8 @@ class JmResp(CommonResp):
|
||||
|
||||
def require_success(self):
|
||||
if self.is_not_success:
|
||||
raise JmModuleConfig.exception(self.resp.text)
|
||||
ExceptionTool.raises_resp(self.text, self.resp)
|
||||
|
||||
|
||||
class JmImageResp(JmResp):
|
||||
|
||||
@ -30,10 +32,7 @@ class JmImageResp(JmResp):
|
||||
raise NotImplementedError
|
||||
|
||||
def require_success(self):
|
||||
if self.is_success:
|
||||
return
|
||||
|
||||
raise JmModuleConfig.exception(self.get_error_msg())
|
||||
ExceptionTool.require_true(self.is_success, self.get_error_msg())
|
||||
|
||||
def get_error_msg(self):
|
||||
msg = f'禁漫图片获取失败: [{self.url}]'
|
||||
@ -53,16 +52,16 @@ class JmImageResp(JmResp):
|
||||
|
||||
if decode_image is False:
|
||||
# 不解密图片,直接保存文件
|
||||
JmImageSupport.save_resp_img(
|
||||
JmImageTool.save_resp_img(
|
||||
self,
|
||||
path,
|
||||
need_convert=suffix_not_equal(img_url, path),
|
||||
)
|
||||
else:
|
||||
# 解密图片并保存文件
|
||||
JmImageSupport.decode_and_save(
|
||||
JmImageSupport.get_num_by_url(scramble_id, img_url),
|
||||
JmImageSupport.open_Image(self.content),
|
||||
JmImageTool.decode_and_save(
|
||||
JmImageTool.get_num_by_url(scramble_id, img_url),
|
||||
JmImageTool.open_Image(self.content),
|
||||
path,
|
||||
)
|
||||
|
||||
@ -71,8 +70,7 @@ class JmApiResp(JmResp):
|
||||
|
||||
@classmethod
|
||||
def wrap(cls, resp, key_ts):
|
||||
if isinstance(resp, JmApiResp):
|
||||
raise JmModuleConfig.exception('重复包装')
|
||||
ExceptionTool.require_true(not isinstance(resp, JmApiResp), f'重复包装: {resp}')
|
||||
|
||||
return cls(resp, key_ts)
|
||||
|
||||
@ -249,7 +247,7 @@ class JmImageClient:
|
||||
# gif图无需加解密,需要最先判断
|
||||
if self.img_is_not_need_to_decode(img_url, resp):
|
||||
# 相当于调用save_directly,但使用save_resp_img可以统一调用入口
|
||||
JmImageSupport.save_resp_img(resp, img_save_path, False)
|
||||
JmImageTool.save_resp_img(resp, img_save_path, False)
|
||||
else:
|
||||
resp.transfer_to(img_save_path, scramble_id, decode_image, img_url)
|
||||
|
||||
|
@ -17,8 +17,8 @@ def default_postman_constructor(session, **kwargs):
|
||||
return Postmans.new_postman(**kwargs)
|
||||
|
||||
|
||||
def default_raise_exception_executor(msg, **_kwargs):
|
||||
raise JmModuleConfig.exception(msg)
|
||||
def default_raise_exception_executor(msg, _extra):
|
||||
raise JmModuleConfig.CLASS_EXCEPTION(msg)
|
||||
|
||||
|
||||
class JmcomicException(Exception):
|
||||
@ -50,9 +50,13 @@ class JmModuleConfig:
|
||||
SCRAMBLE_0 = 220980
|
||||
SCRAMBLE_10 = 268850
|
||||
SCRAMBLE_NUM_8 = 421926 # 2023-02-08后改了图片切割算法
|
||||
SCRAMBLE_CACHE = {}
|
||||
|
||||
# API的相关配置
|
||||
# 移动端api密钥
|
||||
MAGIC_18COMICAPPCONTENT = '18comicAPPContent'
|
||||
# 移动端的图片域名
|
||||
DOMAIN_IMAGE_LIST = [f"cdn-msp.jmapiproxy{i}.cc" for i in range(1, 4)]
|
||||
|
||||
# 下载时的一些默认值配置
|
||||
default_author = 'default-author'
|
||||
@ -66,7 +70,7 @@ class JmModuleConfig:
|
||||
CLASS_PHOTO = None
|
||||
CLASS_IMAGE = None
|
||||
CLASS_CLIENT_IMPL = {}
|
||||
CLASS_EXCEPTION = None
|
||||
CLASS_EXCEPTION = JmcomicException
|
||||
|
||||
# 插件注册表
|
||||
PLUGIN_REGISTRY = {}
|
||||
@ -125,34 +129,14 @@ class JmModuleConfig:
|
||||
|
||||
@classmethod
|
||||
def client_impl_class(cls, client_key: str):
|
||||
client_impl_dict = cls.CLASS_CLIENT_IMPL
|
||||
clazz_dict = cls.CLASS_CLIENT_IMPL
|
||||
|
||||
impl_class = client_impl_dict.get(client_key, None)
|
||||
if impl_class is None:
|
||||
raise NotImplementedError(f'not found client impl class for key: "{client_key}"')
|
||||
clazz = clazz_dict.get(client_key, None)
|
||||
if clazz is None:
|
||||
from .jm_toolkit import ExceptionTool
|
||||
ExceptionTool.raises(f'not found client impl class for key: "{client_key}"')
|
||||
|
||||
return impl_class
|
||||
|
||||
@classmethod
|
||||
def exception(cls, msg: str):
|
||||
"""
|
||||
获取jmcomic模块的异常类
|
||||
"""
|
||||
if cls.CLASS_EXCEPTION is not None:
|
||||
return cls.CLASS_EXCEPTION(msg)
|
||||
|
||||
return JmcomicException(msg)
|
||||
|
||||
@classmethod
|
||||
def raises(cls, msg: str, **kwargs):
|
||||
"""
|
||||
抛出异常,支持把一些上下文参数传递为kwargs
|
||||
真正抛出异常的是函数 cls.raise_exception_executor,用户可自定义此字段
|
||||
|
||||
如果只想抛异常,不想支持一些扩展处理,使用 raise cls.exception(msg)
|
||||
如果想支持一些扩展处理,使用 cls.raises(msg, context=context)
|
||||
"""
|
||||
cls.raise_exception_executor(msg, **kwargs)
|
||||
return clazz
|
||||
|
||||
@classmethod
|
||||
@field_cache("DOMAIN")
|
||||
@ -189,7 +173,8 @@ class JmModuleConfig:
|
||||
|
||||
resp = postman.get(cls.JM_PUB_URL)
|
||||
if resp.status_code != 200:
|
||||
raise JmModuleConfig.exception(resp.text)
|
||||
from .jm_toolkit import ExceptionTool
|
||||
ExceptionTool.raises_resp(f'请求失败,访问禁漫发布页获取所有域名,HTTP状态码为: {resp.status_code}', resp)
|
||||
|
||||
from .jm_toolkit import JmcomicText
|
||||
domain_list = JmcomicText.analyse_jm_pub_html(resp.text)
|
||||
@ -254,14 +239,18 @@ class JmModuleConfig:
|
||||
}
|
||||
|
||||
# option 默认配置字典
|
||||
JM_OPTION_VER = '2.1'
|
||||
default_option_dict: dict = {
|
||||
'version': '2.0',
|
||||
'version': JM_OPTION_VER,
|
||||
'debug': None,
|
||||
'dir_rule': {'rule': 'Bd_Pname', 'base_dir': None},
|
||||
'download': {
|
||||
'cache': True,
|
||||
'image': {'decode': True, 'suffix': None},
|
||||
'threading': {'batch_count': 30},
|
||||
'threading': {
|
||||
'image': 30,
|
||||
'photo': None,
|
||||
},
|
||||
},
|
||||
'client': {
|
||||
'cache': None,
|
||||
@ -309,6 +298,12 @@ class JmModuleConfig:
|
||||
if meta_data['headers'] is None:
|
||||
meta_data['headers'] = cls.headers()
|
||||
|
||||
# threading photo
|
||||
dt = option_dict['download']['threading']
|
||||
if dt['photo'] is None:
|
||||
import os
|
||||
dt['photo'] = os.cpu_count()
|
||||
|
||||
return option_dict
|
||||
|
||||
@classmethod
|
||||
|
@ -55,6 +55,20 @@ class DetailEntity(JmBaseEntity, IndexedEntity):
|
||||
cls_name = cls.__name__
|
||||
return cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")].lower()
|
||||
|
||||
def get_dirname(self, ref: str) -> str:
|
||||
"""
|
||||
该方法被 DirDule 调用,用于生成特定层次的文件夹
|
||||
通常调用方式如下:
|
||||
Atitle -> ref = 'title' -> album.get_dirname(ref)
|
||||
该方法需要返回 ref 对应的文件夹名,默认实现直接返回 getattr(self, ref)
|
||||
|
||||
用户可重写此方法,来实现自定义文件夹名
|
||||
|
||||
@param ref: 字段名
|
||||
@return: 文件夹名
|
||||
"""
|
||||
return getattr(self, ref)
|
||||
|
||||
|
||||
class JmImageDetail(JmBaseEntity):
|
||||
|
||||
@ -71,7 +85,7 @@ class JmImageDetail(JmBaseEntity):
|
||||
self.aid: str = aid
|
||||
self.scramble_id: str = scramble_id
|
||||
self.img_url: str = img_url
|
||||
self.img_file_name: str = img_file_name
|
||||
self.img_file_name: str = img_file_name # without suffix
|
||||
self.img_file_suffix: str = img_file_suffix
|
||||
|
||||
self.from_photo: Optional[JmPhotoDetail] = from_photo
|
||||
@ -80,15 +94,15 @@ class JmImageDetail(JmBaseEntity):
|
||||
self.index = index
|
||||
|
||||
@property
|
||||
def filename(self) -> str:
|
||||
return self.img_file_name + self.img_file_suffix
|
||||
def filename_without_suffix(self):
|
||||
return self.img_file_name
|
||||
|
||||
@property
|
||||
def download_url(self) -> str:
|
||||
"""
|
||||
图片的下载路径
|
||||
与 self.img_url 的唯一不同是,在最后会带上 ?{self.query_params}
|
||||
@return:
|
||||
@return: 图片的下载路径
|
||||
"""
|
||||
if self.query_params is None:
|
||||
return self.img_url
|
||||
@ -131,7 +145,7 @@ class JmImageDetail(JmBaseEntity):
|
||||
|
||||
@property
|
||||
def tag(self) -> str:
|
||||
return f'{self.aid}/{self.filename} [{self.index + 1}/{len(self.from_photo)}]'
|
||||
return f'{self.aid}/{self.img_file_name}{self.img_file_suffix} [{self.index + 1}/{len(self.from_photo)}]'
|
||||
|
||||
|
||||
class JmPhotoDetail(DetailEntity):
|
||||
@ -229,7 +243,7 @@ class JmPhotoDetail(DetailEntity):
|
||||
# 校验参数
|
||||
length = len(self.page_arr)
|
||||
if index >= length:
|
||||
raise JmModuleConfig.exception(f'创建JmImageDetail失败,{index} >= {length}')
|
||||
raise IndexError(f'image index out of range for photo-{self.photo_id}: {index} >= {length}')
|
||||
|
||||
data_original = self.get_img_data_original(self.page_arr[index])
|
||||
|
||||
@ -248,11 +262,12 @@ class JmPhotoDetail(DetailEntity):
|
||||
例如:img_name = 01111.webp
|
||||
返回:https://cdn-msp2.18comic.org/media/photos/147643/01111.webp
|
||||
"""
|
||||
data_original_domain = self.data_original_domain
|
||||
if data_original_domain is None:
|
||||
raise JmModuleConfig.exception(f'图片域名为空: {self.__dict__}')
|
||||
domain = self.data_original_domain or None
|
||||
|
||||
return f'https://{data_original_domain}/media/photos/{self.photo_id}/{img_name}'
|
||||
from .jm_toolkit import ExceptionTool
|
||||
ExceptionTool.require_true(domain is not None, f'图片域名为空: {domain}')
|
||||
|
||||
return f'{JmModuleConfig.PROT}{domain}/media/photos/{self.photo_id}/{img_name}'
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def get_data_original_query_params(self, data_original_0: StrNone) -> str:
|
||||
@ -301,8 +316,8 @@ class JmAlbumDetail(DetailEntity):
|
||||
tags,
|
||||
related_list=None,
|
||||
):
|
||||
self.album_id: str = album_id
|
||||
self.scramble_id: str = scramble_id
|
||||
self.album_id: str = str(album_id)
|
||||
self.scramble_id: str = str(scramble_id)
|
||||
self.name: str = name
|
||||
self.page_count = int(page_count) # 总页数
|
||||
self.pub_date: str = pub_date # 发布日期
|
||||
@ -367,7 +382,7 @@ class JmAlbumDetail(DetailEntity):
|
||||
length = len(self.episode_list)
|
||||
|
||||
if index >= length:
|
||||
raise JmModuleConfig.exception(f'创建JmPhotoDetail失败,{index} >= {length}')
|
||||
raise IndexError(f'photo index out of range for album-{self.album_id}: {index} >= {length}')
|
||||
|
||||
# ('212214', '81', '94 突然打來', '2020-08-29')
|
||||
pid, pindex, pname, _pub_date = self.episode_list[index]
|
||||
|
@ -50,7 +50,7 @@ class DirRule:
|
||||
"""
|
||||
|
||||
if '_' not in rule_dsl and rule_dsl != 'Bd':
|
||||
raise NotImplementedError(f'不支持的dsl: "{rule_dsl}"')
|
||||
ExceptionTool.raises(f'不支持的dsl: "{rule_dsl}"')
|
||||
|
||||
rule_ls = rule_dsl.split('_')
|
||||
solver_ls = []
|
||||
@ -62,7 +62,7 @@ class DirRule:
|
||||
|
||||
rule_solver = self.get_rule_solver(rule)
|
||||
if rule_solver is None:
|
||||
raise NotImplementedError(f'不支持的dsl: "{rule}" in "{rule_dsl}"')
|
||||
ExceptionTool.raises(f'不支持的dsl: "{rule}" in "{rule_dsl}"')
|
||||
|
||||
solver_ls.append(rule_solver)
|
||||
|
||||
@ -80,7 +80,7 @@ class DirRule:
|
||||
|
||||
# Axxx or Pyyy
|
||||
key = 1 if rule[0] == 'A' else 2
|
||||
solve_func = lambda entity, ref=rule[1:]: fix_windir_name(str(getattr(entity, ref)))
|
||||
solve_func = lambda detail, ref=rule[1:]: fix_windir_name(str(detail.get_dirname(ref)))
|
||||
|
||||
# 保存缓存
|
||||
rule_solver = (key, solve_func)
|
||||
@ -116,7 +116,6 @@ class DirRule:
|
||||
|
||||
|
||||
class JmOption:
|
||||
JM_OP_VER = '2.0'
|
||||
|
||||
def __init__(self,
|
||||
dir_rule: Dict,
|
||||
@ -126,7 +125,7 @@ class JmOption:
|
||||
filepath=None,
|
||||
):
|
||||
# 版本号
|
||||
self.version = self.JM_OP_VER
|
||||
self.version = JmModuleConfig.JM_OPTION_VER
|
||||
# 路径规则配置
|
||||
self.dir_rule = DirRule(**dir_rule)
|
||||
# 请求配置
|
||||
@ -148,10 +147,6 @@ class JmOption:
|
||||
def download_image_decode(self):
|
||||
return self.download.image.decode
|
||||
|
||||
@property
|
||||
def download_threading_batch_count(self):
|
||||
return self.download.threading.batch_count
|
||||
|
||||
@property
|
||||
def download_image_suffix(self):
|
||||
return self.download.image.suffix
|
||||
@ -162,11 +157,11 @@ class JmOption:
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
def decide_image_batch_count(self, photo: JmPhotoDetail):
|
||||
return self.download_threading_batch_count
|
||||
return self.download.threading.image
|
||||
|
||||
# noinspection PyMethodMayBeStatic,PyUnusedLocal
|
||||
def decide_photo_batch_count(self, album: JmAlbumDetail):
|
||||
return os.cpu_count()
|
||||
return self.download.threading.photo
|
||||
|
||||
def decide_image_save_dir(self, photo) -> str:
|
||||
# 使用 self.dir_rule 决定 save_dir
|
||||
@ -212,7 +207,7 @@ class JmOption:
|
||||
# 通过拼接生成绝对路径
|
||||
save_dir = self.decide_image_save_dir(image.from_photo)
|
||||
suffix = self.decide_image_suffix(image)
|
||||
return save_dir + image.img_file_name + suffix
|
||||
return os.path.join(save_dir, image.filename_without_suffix + suffix)
|
||||
|
||||
"""
|
||||
下面是创建对象相关方法
|
||||
@ -242,19 +237,27 @@ class JmOption:
|
||||
return cls.construct({})
|
||||
|
||||
@classmethod
|
||||
def construct(cls, dic: Dict, cover_default=True) -> 'JmOption':
|
||||
if cover_default:
|
||||
dic = cls.merge_default_dict(dic)
|
||||
|
||||
version = dic.pop('version', None)
|
||||
if float(version) != float(cls.JM_OP_VER):
|
||||
# 版本兼容
|
||||
raise NotImplementedError('不支持的option版本')
|
||||
def construct(cls, orgdic: Dict, cover_default=True) -> 'JmOption':
|
||||
dic = cls.merge_default_dict(orgdic) if cover_default else orgdic
|
||||
|
||||
# debug
|
||||
debug = dic.pop('debug', True)
|
||||
if debug is False:
|
||||
disable_jm_debug()
|
||||
|
||||
# version
|
||||
version = dic.pop('version', None)
|
||||
if version is None or float(version) >= float(JmModuleConfig.JM_OPTION_VER):
|
||||
return cls(**dic)
|
||||
|
||||
# 旧版本option,做兼容
|
||||
|
||||
# 1) 2.0 -> 2.1,并发配置的键名更改了
|
||||
dt: dict = dic['download']['threading']
|
||||
if 'batch_count' in dt:
|
||||
batch_count = dt.pop('batch_count')
|
||||
dt['image'] = batch_count
|
||||
|
||||
return cls(**dic)
|
||||
|
||||
def deconstruct(self) -> Dict:
|
||||
@ -278,8 +281,7 @@ class JmOption:
|
||||
if filepath is None:
|
||||
filepath = self.filepath
|
||||
|
||||
if filepath is None:
|
||||
raise JmModuleConfig.exception("未指定JmOption的保存路径")
|
||||
ExceptionTool.require_true(filepath is not None, "未指定JmOption的保存路径")
|
||||
|
||||
PackerUtil.pack(self.deconstruct(), filepath)
|
||||
|
||||
@ -369,8 +371,7 @@ class JmOption:
|
||||
key, kwargs = pinfo['plugin'], pinfo['kwargs']
|
||||
plugin_class: Optional[Type[JmOptionPlugin]] = plugin_registry.get(key, None)
|
||||
|
||||
if plugin_class is None:
|
||||
raise JmModuleConfig.exception(f'[{group}] 未注册的plugin: {key}')
|
||||
ExceptionTool.require_true(plugin_class is not None, f'[{group}] 未注册的plugin: {key}')
|
||||
|
||||
self.invoke_plugin(plugin_class, kwargs, extra)
|
||||
|
||||
@ -408,8 +409,10 @@ class JmOption:
|
||||
kwargs将来要传给方法参数,这要求kwargs的key是str类型,
|
||||
该方法检查kwargs的key的类型,如果不是str,尝试转为str,不行则抛异常。
|
||||
"""
|
||||
if not isinstance(kwargs, dict):
|
||||
raise JmModuleConfig.exception(f'插件的kwargs参数必须为dict类型,而不能是类型: {type(kwargs)}')
|
||||
ExceptionTool.require_true(
|
||||
isinstance(kwargs, dict),
|
||||
f'插件的kwargs参数必须为dict类型,而不能是类型: {type(kwargs)}'
|
||||
)
|
||||
|
||||
kwargs: dict
|
||||
new_kwargs: Dict[str, Any] = {}
|
||||
@ -425,7 +428,7 @@ class JmOption:
|
||||
new_kwargs[newk] = v
|
||||
continue
|
||||
|
||||
raise JmModuleConfig.exception(
|
||||
ExceptionTool.raises(
|
||||
f'插件kwargs参数类型有误,'
|
||||
f'字段: {k},预期类型为str,实际类型为{type(k)}'
|
||||
)
|
||||
|
@ -238,7 +238,7 @@ class ZipPlugin(JmOptionPlugin):
|
||||
dir_zip_dict[dir_path] = zip_path
|
||||
|
||||
else:
|
||||
raise NotImplementedError(f'level: {level}')
|
||||
ExceptionTool.raises(f'Not Implemented Zip Level: {level}')
|
||||
|
||||
self.after_zip(dir_zip_dict)
|
||||
|
||||
|
@ -67,16 +67,14 @@ class JmcomicText:
|
||||
if isinstance(text, int):
|
||||
return str(text)
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise JmModuleConfig.exception(f"无法解析jm车号, 参数类型为: {type(text)}")
|
||||
ExceptionTool.require_true(isinstance(text, str), f"无法解析jm车号, 参数类型为: {type(text)}")
|
||||
|
||||
# 43210
|
||||
if text.isdigit():
|
||||
return text
|
||||
|
||||
# Jm43210
|
||||
if len(text) <= 2:
|
||||
raise JmModuleConfig.exception(f"无法解析jm车号, 文本为: {text}")
|
||||
ExceptionTool.require_true(len(text) >= 2, f"无法解析jm车号, 文本太短: {text}")
|
||||
|
||||
# text: JM12341
|
||||
c0 = text[0]
|
||||
@ -87,8 +85,7 @@ class JmcomicText:
|
||||
else:
|
||||
# https://xxx/photo/412038
|
||||
match = cls.pattern_jm_pa_id.search(text)
|
||||
if match is None:
|
||||
raise JmModuleConfig.exception(f"无法解析jm车号, 文本为: {text}")
|
||||
ExceptionTool.require_true(match is not None, f"无法解析jm车号, 文本为: {text}")
|
||||
return match[2]
|
||||
|
||||
@classmethod
|
||||
@ -169,11 +166,10 @@ class JmcomicText:
|
||||
|
||||
if field_value is None:
|
||||
if default is None:
|
||||
JmModuleConfig.raises(
|
||||
ExceptionTool.raises_regex(
|
||||
f"文本没有匹配上字段:字段名为'{field_name}',pattern: [{pattern}]",
|
||||
re_match_html=html,
|
||||
re_match_field_name=field_name,
|
||||
re_match_pattern=pattern,
|
||||
html=html,
|
||||
pattern=pattern,
|
||||
)
|
||||
else:
|
||||
field_value = default
|
||||
@ -260,12 +256,20 @@ class JmcomicSearchTool:
|
||||
match = cls.pattern_html_search_error.search(html)
|
||||
if match is not None:
|
||||
topic, reason = match[1], match[2]
|
||||
JmModuleConfig.raises(f'{topic}: {reason}', re_search_html=html)
|
||||
ExceptionTool.raises_regex(
|
||||
f'{topic}: {reason}',
|
||||
html=html,
|
||||
pattern=cls.pattern_html_search_error,
|
||||
)
|
||||
|
||||
# 缩小文本范围
|
||||
match = cls.pattern_html_search_shorten_for.search(html)
|
||||
if match is None:
|
||||
JmModuleConfig.raises('未匹配到搜索结果', re_shorten_html=html)
|
||||
ExceptionTool.raises_regex(
|
||||
'未匹配到搜索结果',
|
||||
html=html,
|
||||
pattern=cls.pattern_html_search_shorten_for,
|
||||
)
|
||||
html = match[0]
|
||||
|
||||
# 提取结果
|
||||
@ -324,6 +328,8 @@ class JmcomicSearchTool:
|
||||
|
||||
class JmApiAdaptTool:
|
||||
"""
|
||||
本类复杂把移动端的api返回值,适配为标准的实体类
|
||||
|
||||
# album
|
||||
{
|
||||
"id": 123,
|
||||
@ -397,7 +403,7 @@ class JmApiAdaptTool:
|
||||
'series_id',
|
||||
('tags', 'keywords'),
|
||||
('id', 'photo_id'),
|
||||
('images', 'page_arr')
|
||||
('images', 'page_arr'),
|
||||
|
||||
]
|
||||
}
|
||||
@ -429,7 +435,7 @@ class JmApiAdaptTool:
|
||||
if issubclass(clazz, k):
|
||||
return v
|
||||
|
||||
raise AssertionError(clazz)
|
||||
ExceptionTool.raises(f'不支持的类型: {clazz}')
|
||||
|
||||
@classmethod
|
||||
def post_adapt_album(cls, data: dict, _clazz: type, fields: dict):
|
||||
@ -447,16 +453,22 @@ class JmApiAdaptTool:
|
||||
|
||||
@classmethod
|
||||
def post_adapt_photo(cls, data: dict, _clazz: type, fields: dict):
|
||||
for chapter in data['series']:
|
||||
# 1. 获取sort字段,如果data['series']中没有,使用默认值1
|
||||
sort = 1
|
||||
series: list = data['series'] # series中的sort从1开始
|
||||
for chapter in series:
|
||||
chapter = DictModel(chapter)
|
||||
if int(chapter.id) == int(data['id']):
|
||||
fields['sort'] = chapter.sort
|
||||
sort = chapter.sort
|
||||
break
|
||||
|
||||
fields['sort'] = sort
|
||||
fields['scramble_id'] = '0'
|
||||
import random
|
||||
fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_IMAGE_LIST)
|
||||
|
||||
|
||||
class JmImageSupport:
|
||||
class JmImageTool:
|
||||
|
||||
@classmethod
|
||||
def save_resp_img(cls, resp: Any, filepath: str, need_convert=True):
|
||||
@ -579,3 +591,83 @@ class JmImageSupport:
|
||||
获得图片分割数
|
||||
"""
|
||||
return cls.get_num(detail.scramble_id, detail.aid, detail.img_file_name)
|
||||
|
||||
|
||||
class ExceptionTool:
|
||||
"""
|
||||
抛异常的工具
|
||||
1: 能简化 if-raise 语句的编写
|
||||
2: 有更好的上下文信息传递方式
|
||||
"""
|
||||
|
||||
EXTRA_KEY_RESP = 'resp'
|
||||
EXTRA_KEY_HTML = 'html'
|
||||
EXTRA_KEY_RE_PATTERN = 'pattern'
|
||||
|
||||
@classmethod
|
||||
def raises(cls, msg: str, extra: dict = None):
|
||||
if extra is None:
|
||||
extra = {}
|
||||
|
||||
JmModuleConfig.raise_exception_executor(msg, extra)
|
||||
|
||||
@classmethod
|
||||
def raises_regex(cls,
|
||||
msg: str,
|
||||
html: str,
|
||||
pattern: Pattern,
|
||||
):
|
||||
cls.raises(
|
||||
msg, {
|
||||
cls.EXTRA_KEY_HTML: html,
|
||||
cls.EXTRA_KEY_RE_PATTERN: pattern,
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def raises_resp(cls,
|
||||
msg: str,
|
||||
resp,
|
||||
):
|
||||
cls.raises(
|
||||
msg, {
|
||||
cls.EXTRA_KEY_RESP: resp
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def raise_missing(cls,
|
||||
resp,
|
||||
org_req_url=None,
|
||||
):
|
||||
"""
|
||||
抛出本子/章节的异常
|
||||
@param resp: 响应对象
|
||||
@param org_req_url: 原始请求url,可不传
|
||||
"""
|
||||
if org_req_url is None:
|
||||
org_req_url = resp.url
|
||||
|
||||
req_type = "本子" if "album" in org_req_url else "章节"
|
||||
cls.raises_resp((
|
||||
f'请求的{req_type}不存在!({org_req_url})\n'
|
||||
'原因可能为:\n'
|
||||
f'1. id有误,检查你的{req_type}id\n'
|
||||
'2. 该漫画只对登录用户可见,请配置你的cookies\n'
|
||||
), resp)
|
||||
|
||||
@classmethod
|
||||
def require_true(cls, case: bool, msg: str):
|
||||
if case:
|
||||
return
|
||||
|
||||
cls.raises(msg)
|
||||
|
||||
@classmethod
|
||||
def replace_old_exception_executor(cls, raises: Callable[[Callable, str, dict], None]):
|
||||
old = JmModuleConfig.raise_exception_executor
|
||||
|
||||
def new(msg, extra):
|
||||
raises(old, msg, extra)
|
||||
|
||||
JmModuleConfig.raise_exception_executor = new
|
||||
|
@ -29,7 +29,7 @@ class JmTestConfigurable(unittest.TestCase):
|
||||
os.chdir(cls.project_dir)
|
||||
|
||||
# 设置 JmOption,JmcomicClient
|
||||
option = cls.use_option('option_test.yml')
|
||||
option = create_option_by_env('JM_OPTION_PATH_TEST')
|
||||
cls.option = option
|
||||
cls.client = option.build_jm_client()
|
||||
|
||||
|
@ -6,7 +6,6 @@ class Test_MobileClient(JmTestConfigurable):
|
||||
def test_cl(self):
|
||||
for cmd in str_to_list('''
|
||||
jmcomic 438516
|
||||
jmcomic 438516 --option=''
|
||||
jmcomic 438516 p438516
|
||||
'''):
|
||||
self.assertEqual(os.system(cmd), 0)
|
||||
|
@ -7,7 +7,9 @@ class Test_Client(JmTestConfigurable):
|
||||
jm_photo_id = 'JM438516'
|
||||
photo = self.client.get_photo_detail(jm_photo_id, False)
|
||||
image = photo[0]
|
||||
self.client.download_by_image_detail(image, self.option.decide_image_filepath(image))
|
||||
filepath = self.option.decide_image_filepath(image)
|
||||
self.client.download_by_image_detail(image, filepath)
|
||||
print(filepath)
|
||||
|
||||
def test_fetch_album(self):
|
||||
album_id = "JM438516"
|
||||
@ -15,8 +17,13 @@ class Test_Client(JmTestConfigurable):
|
||||
|
||||
def test_search(self):
|
||||
page: JmSearchPage = self.client.search_tag('+无修正 +中文 -全彩')
|
||||
for album_id, title, tag_list in page.iter_id_title_tag():
|
||||
print(album_id, title, tag_list)
|
||||
|
||||
if len(page) >= 1:
|
||||
for aid, ainfo in page[0:1:1]:
|
||||
print(aid, ainfo)
|
||||
|
||||
for aid, atitle, tag_list in page.iter_id_title_tag():
|
||||
print(aid, atitle, tag_list)
|
||||
|
||||
aid = '438516'
|
||||
page = self.client.search_site(aid)
|
||||
@ -31,13 +38,29 @@ class Test_Client(JmTestConfigurable):
|
||||
self.client.download_by_image_detail(image, self.option.decide_image_filepath(image))
|
||||
|
||||
def test_album_missing(self):
|
||||
JmModuleConfig.CLASS_EXCEPTION = JmcomicException
|
||||
class A(BaseException):
|
||||
pass
|
||||
|
||||
JmModuleConfig.CLASS_EXCEPTION = A
|
||||
self.assertRaises(
|
||||
JmcomicException,
|
||||
A,
|
||||
self.client.get_album_detail,
|
||||
'332583'
|
||||
'0'
|
||||
)
|
||||
|
||||
def test_raise_exception(self):
|
||||
|
||||
class B(BaseException):
|
||||
pass
|
||||
|
||||
def raises(old, _msg, _extra):
|
||||
self.assertEqual(old, default_raise_exception_executor)
|
||||
raise B()
|
||||
|
||||
JmModuleConfig.raise_exception_executor = default_raise_exception_executor
|
||||
ExceptionTool.replace_old_exception_executor(raises)
|
||||
self.assertRaises(B, JmcomicText.parse_to_album_id, 'asdhasjhkd')
|
||||
|
||||
def test_detail_property_list(self):
|
||||
album = self.client.get_album_detail(410090)
|
||||
|
||||
@ -49,7 +72,7 @@ class Test_Client(JmTestConfigurable):
|
||||
]
|
||||
|
||||
for pair in ans:
|
||||
self.assertListEqual(pair[0], pair[1])
|
||||
self.assertListEqual(pair[0][0:9], pair[1][0:9])
|
||||
|
||||
def test_photo_sort(self):
|
||||
client = self.option.build_jm_client()
|
||||
@ -193,3 +216,13 @@ class Test_Client(JmTestConfigurable):
|
||||
0,
|
||||
aid,
|
||||
)
|
||||
|
||||
def test_get_detail(self):
|
||||
client = self.client
|
||||
|
||||
album = client.get_album_detail(400222)
|
||||
print(album.id, album.name, album.tags)
|
||||
|
||||
for photo in album[0:3]:
|
||||
photo = client.get_photo_detail(photo.photo_id)
|
||||
print(photo.id, photo.name)
|
||||
|
@ -1,37 +0,0 @@
|
||||
from test_jmcomic import *
|
||||
|
||||
# 移动端专用的禁漫域名
|
||||
domain_list = [
|
||||
"https://www.jmapinode1.cc",
|
||||
"https://www.jmapinode2.cc",
|
||||
"https://www.jmapinode3.cc",
|
||||
"https://www.jmapibranch2.cc"
|
||||
]
|
||||
|
||||
|
||||
class Test_MobileClient(JmTestConfigurable):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
cls.client = cls.option.new_jm_client(domain_list, impl='api')
|
||||
|
||||
def test_search(self):
|
||||
page = self.client.search_site('MANA')
|
||||
|
||||
if len(page) >= 1:
|
||||
for aid, ainfo in page[0:1:1]:
|
||||
print(aid, ainfo['description'], ainfo['category'])
|
||||
|
||||
for aid, atitle, tag_list in page.iter_id_title_tag():
|
||||
print(aid, atitle, tag_list)
|
||||
|
||||
def test_get_detail(self):
|
||||
client = self.client
|
||||
|
||||
album = client.get_album_detail(400222)
|
||||
print(album.id, album.name, album.tags)
|
||||
|
||||
for photo in album[0:3]:
|
||||
photo = client.get_photo_detail(photo.photo_id)
|
||||
print(photo.id, photo.name)
|
@ -77,17 +77,15 @@ def custom_client_class():
|
||||
def custom_album_photo_image_detail_class():
|
||||
"""
|
||||
该函数演示替换实体类(本子/章节/图片)
|
||||
"""
|
||||
|
||||
# 在使用路径规则 DirRule 时,可能会遇到需要自定义实体类属性的情况,例如:
|
||||
"""
|
||||
在使用路径规则 DirRule 时,可能会遇到需要自定义实体类属性的情况,例如:
|
||||
dir_rule:
|
||||
base_dir: ${workspace}
|
||||
rule: Bd_Acustom_Pcustom
|
||||
"""
|
||||
|
||||
# 上面的Acustom,Pcustom都是自定义字段
|
||||
# 如果你想要使用这种自定义字段,你就需要替换默认的实体类,例如
|
||||
上面的Acustom,Pcustom都是自定义字段
|
||||
如果你想要使用这种自定义字段,你就需要替换默认的实体类,方式如下
|
||||
"""
|
||||
|
||||
# 自定义本子实体类
|
||||
class MyAlbum(JmAlbumDetail):
|
||||
@ -103,14 +101,21 @@ def custom_album_photo_image_detail_class():
|
||||
def custom(self):
|
||||
return f'custom_{self.title}'
|
||||
|
||||
# 自定义图片实体类
|
||||
class MyImage(JmImageDetail):
|
||||
pass
|
||||
"""
|
||||
v2.3.3: 支持更灵活的自定义方式,可以使用函数,效果同上,示例见下
|
||||
"""
|
||||
|
||||
class MyAlbum2(JmAlbumDetail):
|
||||
|
||||
def get_dirname(self, ref: str) -> str:
|
||||
if ref == 'custom':
|
||||
return f'custom_{self.name}'
|
||||
|
||||
return super().get_dirname(ref)
|
||||
|
||||
# 最后,替换默认实体类来让你的自定义类生效
|
||||
JmModuleConfig.CLASS_ALBUM = MyAlbum
|
||||
JmModuleConfig.CLASS_PHOTO = MyPhoto
|
||||
JmModuleConfig.CLASS_IMAGE = MyImage
|
||||
|
||||
|
||||
def custom_jm_debug():
|
||||
@ -133,30 +138,3 @@ def custom_jm_debug():
|
||||
|
||||
# 2. 让my_debug生效
|
||||
JmModuleConfig.debug_executor = my_debug
|
||||
|
||||
|
||||
def custom_exception_raise():
|
||||
"""
|
||||
该函数演示jmcomic的异常机制
|
||||
"""
|
||||
|
||||
# jmcomic 代码在运行过程中可能抛出异常,以获取album实体类为例:
|
||||
album = client.get_album_detail('999999')
|
||||
|
||||
# 上面这行代码用于获取本子id为 999999 的JmAlbumDetail
|
||||
# 如果本子不存在,则会抛出异常,异常类默认是 JmcomicException
|
||||
|
||||
# 你可以自定义抛出的异常类,做法如下:
|
||||
# 1. 自定义异常类
|
||||
class MyExceptionClass(Exception):
|
||||
pass
|
||||
|
||||
# 2. 替换默认异常类
|
||||
JmModuleConfig.CLASS_EXCEPTION = MyExceptionClass
|
||||
|
||||
# 这样一来,抛出的异常类就是 MyExceptionClass
|
||||
try:
|
||||
album = client.get_album_detail('999999')
|
||||
except MyExceptionClass as e:
|
||||
print('捕获MyExceptionClass异常')
|
||||
pass
|
||||
|
@ -49,11 +49,11 @@ def get_option():
|
||||
# 支持工作流覆盖配置文件的配置
|
||||
cover_option_config(option)
|
||||
|
||||
# 覆盖client实现类,实现把请求错误的html下载到文件,方便GitHub Actions下载查看日志
|
||||
hook_debug(option)
|
||||
# 把请求错误的html下载到文件,方便GitHub Actions下载查看日志
|
||||
log_before_raise()
|
||||
|
||||
# 登录,如果有配置的话
|
||||
login_if_configured(option.build_jm_client())
|
||||
login_if_configured(option)
|
||||
|
||||
return option
|
||||
|
||||
@ -66,7 +66,7 @@ def cover_option_config(option: JmOption):
|
||||
option.dir_rule = the_new
|
||||
|
||||
|
||||
def login_if_configured(client):
|
||||
def login_if_configured(option):
|
||||
# 检查环境变量中是否有禁漫的用户名和密码,如果有则登录
|
||||
# 禁漫的大部分本子,下载是不需要登录的,少部分敏感题材需要登录
|
||||
# 如果你希望以登录状态下载本子,你需要自己配置一下GitHub Actions的 `secrets`
|
||||
@ -76,28 +76,28 @@ def login_if_configured(client):
|
||||
username = get_env('JM_USERNAME', None)
|
||||
password = get_env('JM_PASSWORD', None)
|
||||
if username is not None and password is not None:
|
||||
client.login(username, password, True)
|
||||
print_eye_catching(f'登录禁漫成功')
|
||||
# 调用login插件
|
||||
JmLoginPlugin(option).invoke(username, password)
|
||||
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
def hook_debug(option):
|
||||
def log_before_raise():
|
||||
jm_download_dir = get_env('JM_DOWNLOAD_DIR', workspace())
|
||||
mkdir_if_not_exists(jm_download_dir)
|
||||
|
||||
class RaiseErrorAwareClient(JmHtmlClient):
|
||||
# 自定义异常抛出函数,在抛出前把HTML响应数据写到下载文件夹(日志留痕)
|
||||
def raises(old, msg, extra: dict):
|
||||
if ExceptionTool.EXTRA_KEY_RESP not in extra:
|
||||
return old(msg, extra)
|
||||
|
||||
@classmethod
|
||||
def raise_request_error(cls, resp, msg=None):
|
||||
from common import write_text, fix_windir_name
|
||||
write_text(
|
||||
f'{jm_download_dir}/{fix_windir_name(resp.url)}',
|
||||
resp.text
|
||||
)
|
||||
resp = extra[ExceptionTool.EXTRA_KEY_RESP]
|
||||
# 写文件
|
||||
from common import write_text, fix_windir_name
|
||||
write_text(f'{jm_download_dir}/{fix_windir_name(resp.url)}', resp.text)
|
||||
|
||||
return super().raise_request_error(resp, msg)
|
||||
return old(msg, extra)
|
||||
|
||||
JmModuleConfig.CLASS_CLIENT_IMPL['html'] = RaiseErrorAwareClient
|
||||
# 应用函数
|
||||
ExceptionTool.replace_old_exception_executor(raises)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
Loading…
Reference in New Issue
Block a user