v2.3.3: 实现移动端API的图片下载功能,重构异常机制,优化工作流 (#143)

This commit is contained in:
hect0x7 2023-09-29 21:54:12 +08:00 committed by GitHub
parent ed83be0629
commit 49e6d26aaf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 463 additions and 278 deletions

View File

@ -1,4 +1,4 @@
name: 下载JM本子
name: 下载JM本子 (commit)
on:
workflow_dispatch:

View File

@ -1,4 +1,4 @@
name: 下载JM本子(dispatch)
name: 下载JM本子 (dispatch)
on:
workflow_dispatch:
@ -54,7 +54,7 @@ jobs:
DIR_RULE: ${{ github.event.inputs.DIR_RULE }}
ZIP_NAME: ${{ github.event.inputs.ZIP_NAME }}
UPLOAD_NAME: ${{ github.event.inputs.UPLOAD_NAME }}
# sccrets
# secrets
JM_USERNAME: ${{ secrets.JM_USERNAME }}
JM_PASSWORD: ${{ secrets.JM_PASSWORD }}
# 固定值

View File

@ -1,4 +1,5 @@
name: 发布PYPI
# publish when new release
name: Publish If Release
on:
workflow_dispatch:
@ -7,7 +8,7 @@ on:
jobs:
release:
name: 构建并发布 jmcomic 到 PyPI
name: Publish `jmcomic` to PYPI
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
@ -17,12 +18,12 @@ jobs:
with:
python-version: "3.11"
- name: 构建模块
- name: Build
run: |
python -m pip install build
python -m build
- name: 发布PYPI
- name: Publish PYPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets.PYPI_JMCOMIC }}

View File

@ -1,5 +1,5 @@
name: Release (auto)
# auto release and publish
name: Auto Release & Publish
on:
workflow_dispatch:
@ -30,7 +30,7 @@ jobs:
tag_name: ${{ steps.tb.outputs.tag }}
body: ${{ steps.tb.outputs.body }}
- name: Build Module
- name: Build
run: |
python -m pip install build
python -m build

View File

@ -1,4 +1,4 @@
name: 跑测试
name: Run Test (API)
on:
schedule:
@ -9,8 +9,8 @@ on:
paths:
- 'src/**/*.py'
- 'tests/**/*.py'
- '.github/workflows/test.yml'
- 'assets/config/option_test.yml'
- '.github/workflows/test_api.yml'
- 'assets/config/option_test_api.yml'
jobs:
test: # This code is based on https://github.com/gaogaotiantian/viztracer/blob/master/.github/workflows/python-package.yml
@ -20,6 +20,9 @@ jobs:
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
timeout-minutes: 5
env:
# 配置文件路径
JM_OPTION_PATH_TEST: ./assets/config/option_test_api.yml
steps:
- name: Checkout repository
@ -31,28 +34,15 @@ jobs:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: matrix.os != 'windows-latest'
run: |
python -m pip install --upgrade pip
if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
- name: Install dependencies on Windows
if: matrix.os == 'windows-latest'
run: |
python -m pip install --upgrade pip
if (Test-Path -Path '.\requirements-dev.txt' -PathType Leaf) {pip install -r requirements-dev.txt}
- name: Install local
run: |
pip install -e ./
- name: 跑测试
if: matrix.os != 'windows-latest'
- name: Run Test
run: |
cd ./tests/
python -m unittest
- name: 跑测试Windows系统
if: matrix.os == 'windows-latest'
run: |
python -m unittest discover -s ./tests -p "test_*.py"

48
.github/workflows/test_html.yml vendored Normal file
View File

@ -0,0 +1,48 @@
name: Run Test (HTML)
on:
schedule:
- cron: "0 0 * * *"
workflow_dispatch:
push:
branches: [ "dev" ]
paths:
- 'src/**/*.py'
- 'tests/**/*.py'
- '.github/workflows/test_html.yml'
- 'assets/config/option_test_html.yml'
jobs:
test: # This code is based on https://github.com/gaogaotiantian/viztracer/blob/master/.github/workflows/python-package.yml
strategy:
matrix:
python-version: [ 3.7, 3.8, 3.9, "3.10", "3.11" ]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
timeout-minutes: 5
env:
# 配置文件路径
JM_OPTION_PATH_TEST: ./assets/config/option_test_html.yml
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
- name: Install local
run: |
pip install -e ./
- name: Run Test
run: |
cd ./tests/
python -m unittest

View File

@ -0,0 +1,22 @@
# GitHub Actions 测试用
# 移动端配置
client:
impl: api
domain: [
"www.jmapinode1.cc",
"www.jmapinode2.cc",
"www.jmapinode3.cc",
"www.jmapibranch2.cc"
]
postman:
meta_data:
timeout: 10
# 插件配置
plugin:
after_init:
- plugin: usage_log # 实时打印硬件占用率的插件
kwargs:
interval: 0.5 # 间隔时间
enable_warning: false # 不告警

View File

@ -1,10 +1,13 @@
# GitHub Actions 测试用
version: '2.0'
# 网页端配置
client:
domain:
- jmcomic1.me
- jmcomic.me
postman:
meta_data:
timeout: 10
# 插件配置
plugin:

View File

@ -38,11 +38,13 @@ download:
decode: true # JM的原图是混淆过的要不要还原
suffix: .jpg # 把图片都转为.jpg格式
threading:
# batch_count: 章节的批量下载图片线程数
# image: 同时下载的图片数默认是30张图
# 数值大,下得快,配置要求高,对禁漫压力大
# 数值小,下得慢,配置要求低,对禁漫压力小
# PS: 禁漫网页一般是一次请求50张图
batch_count: 30
# PS: 禁漫网页一次最多请求50张图
image: 30
# photo: 同时下载的章节数不配置默认是cpu核心数
photo: 16

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.3.2'
__version__ = '2.3.3'
from .api import *
from .jm_plugin import *

View File

@ -68,3 +68,12 @@ def new_downloader(option=None, downloader=None) -> JmDownloader:
def create_option(filepath):
return JmModuleConfig.option_class().from_file(filepath)
def create_option_by_env(env_name='JM_OPTION_PATH'):
from .cl import get_env
filepath = get_env(env_name, None)
ExceptionTool.require_true(filepath is not None,
f'未配置环境变量: {env_name}请配置为option的文件路径')
return create_option(filepath)

View File

@ -33,6 +33,18 @@ class AbstractJmClient(
def of_api_url(self, api_path, domain):
return JmcomicText.format_url(api_path, domain)
def get_jm_image(self, img_url) -> JmImageResp:
def get_if_fail_raise(url):
"""
使用此方法包装 self.get使得图片数据为空时判定为请求失败时走重试逻辑
"""
resp = JmImageResp(self.get(url))
resp.require_success()
return resp
return self.request_with_retry(get_if_fail_raise, img_url)
def request_with_retry(self,
request,
url,
@ -84,7 +96,7 @@ class AbstractJmClient(
# noinspection PyMethodMayBeStatic
def debug_topic_request(self):
return 'html'
return self.client_key
# noinspection PyMethodMayBeStatic, PyUnusedLocal
def before_retry(self, e, kwargs, retry_count, url):
@ -109,9 +121,9 @@ class AbstractJmClient(
setattr(self, cache_dict_name, cache_dict)
else:
if sys.version_info < (3, 9):
raise NotImplementedError('不支持启用JmcomicClient缓存。\n'
'请更新python版本到3.9以上,'
'或更新commonX: `pip install commonX --upgrade`')
ExceptionTool.raises('不支持启用JmcomicClient缓存。\n'
'请更新python版本到3.9以上,'
'或更新commonX: `pip install commonX --upgrade`')
import functools
cache = functools.cache
@ -149,7 +161,7 @@ class AbstractJmClient(
def fallback(self, request, url, domain_index, retry_count, **kwargs):
msg = f"请求重试全部失败: [{url}], {self.domain_list}"
jm_debug('req.fallback', msg)
raise JmModuleConfig.exception(msg)
ExceptionTool.raises(msg)
# noinspection PyMethodMayBeStatic
def append_params_to_url(self, url, params):
@ -251,7 +263,7 @@ class JmHtmlClient(AbstractJmClient):
)
if resp.status_code != 301:
raise JmModuleConfig.exception(f'登录失败,状态码为{resp.status_code}')
ExceptionTool.raises_resp(f'登录失败,状态码为{resp.status_code}', resp)
if refresh_client_cookies is True:
self['cookies'] = resp.cookies
@ -265,7 +277,10 @@ class JmHtmlClient(AbstractJmClient):
resp = self.get(url, **kwargs)
if require_200 is True and resp.status_code != 200:
# 检查是否是特殊的状态码JmModuleConfig.JM_ERROR_STATUS_CODE
# 如果是,直接抛出异常
self.check_special_http_code(resp)
# 运行到这里说明上一步没有抛异常,说明是未知状态码,抛异常兜底处理
self.raise_request_error(resp)
# 检查请求是否成功
@ -285,26 +300,8 @@ class JmHtmlClient(AbstractJmClient):
+ (f"响应文本=[{resp.text}]" if len(resp.text) < 200 else
f'响应文本过长(len={len(resp.text)}),不打印'
)
raise JmModuleConfig.exception(msg)
def get_jm_image(self, img_url) -> JmImageResp:
def get_if_fail_raise(url):
"""
使用此方法包装 self.get
"""
resp = JmImageResp(self.get(url))
if resp.is_success:
return resp
self.raise_request_error(
resp.resp, resp.get_error_msg()
)
return resp
return self.request_with_retry(get_if_fail_raise, img_url)
ExceptionTool.raises_resp(msg, resp)
def album_comment(self,
video_id,
@ -344,17 +341,15 @@ class JmHtmlClient(AbstractJmClient):
return ret
@classmethod
def require_resp_success_else_raise(cls, resp, req_url):
def require_resp_success_else_raise(cls, resp, org_req_url: str):
"""
@param resp: 响应对象
@param org_req_url: /photo/12412312
"""
# 1. 检查是否 album_missing
error_album_missing = '/error/album_missing'
if resp.url.endswith(error_album_missing) and not req_url.endswith(error_album_missing):
cls.raise_request_error(
resp,
f'请求的本子不存在!({req_url})\n'
'原因可能为:\n'
'1. id有误检查你的本子/章节id\n'
'2. 该漫画只对登录用户可见请配置你的cookies\n'
)
if resp.url.endswith(error_album_missing) and not org_req_url.endswith(error_album_missing):
ExceptionTool.raise_missing(resp, org_req_url)
# 2. 是否是特殊的内容
cls.check_special_text(resp)
@ -400,6 +395,7 @@ class JmApiClient(AbstractJmClient):
API_SEARCH = '/search'
API_ALBUM = '/album'
API_CHAPTER = '/chapter'
API_SCRAMBLE = '/chapter_view_template'
def __init__(self,
postman: Postman,
@ -416,7 +412,7 @@ class JmApiClient(AbstractJmClient):
order_by: str,
time: str,
) -> JmSearchPage:
resp = self.get(
resp = self.get_decode(
self.API_SEARCH,
params={
'search_query': search_query,
@ -447,13 +443,16 @@ class JmApiClient(AbstractJmClient):
)
def get_photo_detail(self, photo_id, fetch_album=True) -> JmPhotoDetail:
return self.fetch_detail_entity(photo_id,
JmModuleConfig.photo_class(),
)
photo_id = JmcomicText.parse_to_photo_id(photo_id)
photo: JmPhotoDetail = self.fetch_detail_entity(photo_id,
JmModuleConfig.photo_class(),
)
photo.scramble_id = self.get_scramble_id(photo.photo_id)
return photo
def fetch_detail_entity(self, apid, clazz, **kwargs):
url = self.API_ALBUM if issubclass(clazz, JmAlbumDetail) else self.API_CHAPTER
resp = self.get(
resp = self.get_decode(
url,
params={
'id': JmcomicText.parse_to_album_id(apid),
@ -461,11 +460,37 @@ class JmApiClient(AbstractJmClient):
}
)
self.require_resp_success(resp)
self.require_resp_success(resp, url)
return JmApiAdaptTool.parse_entity(resp.res_data, clazz)
def get(self, url, **kwargs) -> JmApiResp:
def get_scramble_id(self, photo_id):
cache = JmModuleConfig.SCRAMBLE_CACHE
if photo_id in cache:
return cache[photo_id]
resp = self.get_decode(
self.API_SCRAMBLE,
params={
"id": photo_id,
"mode": "vertical",
"page": "0",
"app_img_shunt": "NaN",
}
)
match = JmcomicText.pattern_html_album_scramble_id.search(resp.text)
if match is not None:
scramble_id = match[1]
else:
jm_debug('api.scramble', '未从响应中匹配到scramble_id返回默认值220980')
scramble_id = 220980
cache[photo_id] = scramble_id
return scramble_id
def get_decode(self, url, **kwargs) -> JmApiResp:
# set headers
headers, key_ts = self.headers_key_ts
kwargs.setdefault('headers', headers)
@ -488,10 +513,19 @@ class JmApiClient(AbstractJmClient):
def debug_topic_request(self):
return 'api'
# noinspection PyMethodMayBeStatic
def require_resp_success(self, resp: JmApiResp):
@classmethod
def require_resp_success(cls, resp: JmApiResp, org_req_url: str):
resp.require_success()
# 1. 检查是否 album_missing
# json: {'code': 200, 'data': []}
data = resp.model().data
if isinstance(data, list) and len(data) == 0:
ExceptionTool.raise_missing(resp, org_req_url)
# 2. 是否是特殊的内容
# 暂无
JmModuleConfig.register_client(JmHtmlClient)
JmModuleConfig.register_client(JmApiClient)

View File

@ -8,6 +8,7 @@ Response Entity
DictModel = AdvancedEasyAccessDict
class JmResp(CommonResp):
@property
@ -22,7 +23,8 @@ class JmResp(CommonResp):
def require_success(self):
if self.is_not_success:
raise JmModuleConfig.exception(self.resp.text)
ExceptionTool.raises_resp(self.text, self.resp)
class JmImageResp(JmResp):
@ -30,10 +32,7 @@ class JmImageResp(JmResp):
raise NotImplementedError
def require_success(self):
if self.is_success:
return
raise JmModuleConfig.exception(self.get_error_msg())
ExceptionTool.require_true(self.is_success, self.get_error_msg())
def get_error_msg(self):
msg = f'禁漫图片获取失败: [{self.url}]'
@ -53,16 +52,16 @@ class JmImageResp(JmResp):
if decode_image is False:
# 不解密图片,直接保存文件
JmImageSupport.save_resp_img(
JmImageTool.save_resp_img(
self,
path,
need_convert=suffix_not_equal(img_url, path),
)
else:
# 解密图片并保存文件
JmImageSupport.decode_and_save(
JmImageSupport.get_num_by_url(scramble_id, img_url),
JmImageSupport.open_Image(self.content),
JmImageTool.decode_and_save(
JmImageTool.get_num_by_url(scramble_id, img_url),
JmImageTool.open_Image(self.content),
path,
)
@ -71,8 +70,7 @@ class JmApiResp(JmResp):
@classmethod
def wrap(cls, resp, key_ts):
if isinstance(resp, JmApiResp):
raise JmModuleConfig.exception('重复包装')
ExceptionTool.require_true(not isinstance(resp, JmApiResp), f'重复包装: {resp}')
return cls(resp, key_ts)
@ -249,7 +247,7 @@ class JmImageClient:
# gif图无需加解密需要最先判断
if self.img_is_not_need_to_decode(img_url, resp):
# 相当于调用save_directly但使用save_resp_img可以统一调用入口
JmImageSupport.save_resp_img(resp, img_save_path, False)
JmImageTool.save_resp_img(resp, img_save_path, False)
else:
resp.transfer_to(img_save_path, scramble_id, decode_image, img_url)

View File

@ -17,8 +17,8 @@ def default_postman_constructor(session, **kwargs):
return Postmans.new_postman(**kwargs)
def default_raise_exception_executor(msg, **_kwargs):
raise JmModuleConfig.exception(msg)
def default_raise_exception_executor(msg, _extra):
raise JmModuleConfig.CLASS_EXCEPTION(msg)
class JmcomicException(Exception):
@ -50,9 +50,13 @@ class JmModuleConfig:
SCRAMBLE_0 = 220980
SCRAMBLE_10 = 268850
SCRAMBLE_NUM_8 = 421926 # 2023-02-08后改了图片切割算法
SCRAMBLE_CACHE = {}
# API的相关配置
# 移动端api密钥
MAGIC_18COMICAPPCONTENT = '18comicAPPContent'
# 移动端的图片域名
DOMAIN_IMAGE_LIST = [f"cdn-msp.jmapiproxy{i}.cc" for i in range(1, 4)]
# 下载时的一些默认值配置
default_author = 'default-author'
@ -66,7 +70,7 @@ class JmModuleConfig:
CLASS_PHOTO = None
CLASS_IMAGE = None
CLASS_CLIENT_IMPL = {}
CLASS_EXCEPTION = None
CLASS_EXCEPTION = JmcomicException
# 插件注册表
PLUGIN_REGISTRY = {}
@ -125,34 +129,14 @@ class JmModuleConfig:
@classmethod
def client_impl_class(cls, client_key: str):
client_impl_dict = cls.CLASS_CLIENT_IMPL
clazz_dict = cls.CLASS_CLIENT_IMPL
impl_class = client_impl_dict.get(client_key, None)
if impl_class is None:
raise NotImplementedError(f'not found client impl class for key: "{client_key}"')
clazz = clazz_dict.get(client_key, None)
if clazz is None:
from .jm_toolkit import ExceptionTool
ExceptionTool.raises(f'not found client impl class for key: "{client_key}"')
return impl_class
@classmethod
def exception(cls, msg: str):
"""
获取jmcomic模块的异常类
"""
if cls.CLASS_EXCEPTION is not None:
return cls.CLASS_EXCEPTION(msg)
return JmcomicException(msg)
@classmethod
def raises(cls, msg: str, **kwargs):
"""
抛出异常支持把一些上下文参数传递为kwargs
真正抛出异常的是函数 cls.raise_exception_executor用户可自定义此字段
如果只想抛异常不想支持一些扩展处理使用 raise cls.exception(msg)
如果想支持一些扩展处理使用 cls.raises(msg, context=context)
"""
cls.raise_exception_executor(msg, **kwargs)
return clazz
@classmethod
@field_cache("DOMAIN")
@ -189,7 +173,8 @@ class JmModuleConfig:
resp = postman.get(cls.JM_PUB_URL)
if resp.status_code != 200:
raise JmModuleConfig.exception(resp.text)
from .jm_toolkit import ExceptionTool
ExceptionTool.raises_resp(f'请求失败访问禁漫发布页获取所有域名HTTP状态码为: {resp.status_code}', resp)
from .jm_toolkit import JmcomicText
domain_list = JmcomicText.analyse_jm_pub_html(resp.text)
@ -254,14 +239,18 @@ class JmModuleConfig:
}
# option 默认配置字典
JM_OPTION_VER = '2.1'
default_option_dict: dict = {
'version': '2.0',
'version': JM_OPTION_VER,
'debug': None,
'dir_rule': {'rule': 'Bd_Pname', 'base_dir': None},
'download': {
'cache': True,
'image': {'decode': True, 'suffix': None},
'threading': {'batch_count': 30},
'threading': {
'image': 30,
'photo': None,
},
},
'client': {
'cache': None,
@ -309,6 +298,12 @@ class JmModuleConfig:
if meta_data['headers'] is None:
meta_data['headers'] = cls.headers()
# threading photo
dt = option_dict['download']['threading']
if dt['photo'] is None:
import os
dt['photo'] = os.cpu_count()
return option_dict
@classmethod

View File

@ -55,6 +55,20 @@ class DetailEntity(JmBaseEntity, IndexedEntity):
cls_name = cls.__name__
return cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")].lower()
def get_dirname(self, ref: str) -> str:
"""
该方法被 DirDule 调用用于生成特定层次的文件夹
通常调用方式如下:
Atitle -> ref = 'title' -> album.get_dirname(ref)
该方法需要返回 ref 对应的文件夹名默认实现直接返回 getattr(self, ref)
用户可重写此方法来实现自定义文件夹名
@param ref: 字段名
@return: 文件夹名
"""
return getattr(self, ref)
class JmImageDetail(JmBaseEntity):
@ -71,7 +85,7 @@ class JmImageDetail(JmBaseEntity):
self.aid: str = aid
self.scramble_id: str = scramble_id
self.img_url: str = img_url
self.img_file_name: str = img_file_name
self.img_file_name: str = img_file_name # without suffix
self.img_file_suffix: str = img_file_suffix
self.from_photo: Optional[JmPhotoDetail] = from_photo
@ -80,15 +94,15 @@ class JmImageDetail(JmBaseEntity):
self.index = index
@property
def filename(self) -> str:
return self.img_file_name + self.img_file_suffix
def filename_without_suffix(self):
return self.img_file_name
@property
def download_url(self) -> str:
"""
图片的下载路径
self.img_url 的唯一不同是在最后会带上 ?{self.query_params}
@return:
@return: 图片的下载路径
"""
if self.query_params is None:
return self.img_url
@ -131,7 +145,7 @@ class JmImageDetail(JmBaseEntity):
@property
def tag(self) -> str:
return f'{self.aid}/{self.filename} [{self.index + 1}/{len(self.from_photo)}]'
return f'{self.aid}/{self.img_file_name}{self.img_file_suffix} [{self.index + 1}/{len(self.from_photo)}]'
class JmPhotoDetail(DetailEntity):
@ -229,7 +243,7 @@ class JmPhotoDetail(DetailEntity):
# 校验参数
length = len(self.page_arr)
if index >= length:
raise JmModuleConfig.exception(f'创建JmImageDetail失败{index} >= {length}')
raise IndexError(f'image index out of range for photo-{self.photo_id}: {index} >= {length}')
data_original = self.get_img_data_original(self.page_arr[index])
@ -248,11 +262,12 @@ class JmPhotoDetail(DetailEntity):
例如img_name = 01111.webp
返回https://cdn-msp2.18comic.org/media/photos/147643/01111.webp
"""
data_original_domain = self.data_original_domain
if data_original_domain is None:
raise JmModuleConfig.exception(f'图片域名为空: {self.__dict__}')
domain = self.data_original_domain or None
return f'https://{data_original_domain}/media/photos/{self.photo_id}/{img_name}'
from .jm_toolkit import ExceptionTool
ExceptionTool.require_true(domain is not None, f'图片域名为空: {domain}')
return f'{JmModuleConfig.PROT}{domain}/media/photos/{self.photo_id}/{img_name}'
# noinspection PyMethodMayBeStatic
def get_data_original_query_params(self, data_original_0: StrNone) -> str:
@ -301,8 +316,8 @@ class JmAlbumDetail(DetailEntity):
tags,
related_list=None,
):
self.album_id: str = album_id
self.scramble_id: str = scramble_id
self.album_id: str = str(album_id)
self.scramble_id: str = str(scramble_id)
self.name: str = name
self.page_count = int(page_count) # 总页数
self.pub_date: str = pub_date # 发布日期
@ -367,7 +382,7 @@ class JmAlbumDetail(DetailEntity):
length = len(self.episode_list)
if index >= length:
raise JmModuleConfig.exception(f'创建JmPhotoDetail失败{index} >= {length}')
raise IndexError(f'photo index out of range for album-{self.album_id}: {index} >= {length}')
# ('212214', '81', '94 突然打來', '2020-08-29')
pid, pindex, pname, _pub_date = self.episode_list[index]

View File

@ -50,7 +50,7 @@ class DirRule:
"""
if '_' not in rule_dsl and rule_dsl != 'Bd':
raise NotImplementedError(f'不支持的dsl: "{rule_dsl}"')
ExceptionTool.raises(f'不支持的dsl: "{rule_dsl}"')
rule_ls = rule_dsl.split('_')
solver_ls = []
@ -62,7 +62,7 @@ class DirRule:
rule_solver = self.get_rule_solver(rule)
if rule_solver is None:
raise NotImplementedError(f'不支持的dsl: "{rule}" in "{rule_dsl}"')
ExceptionTool.raises(f'不支持的dsl: "{rule}" in "{rule_dsl}"')
solver_ls.append(rule_solver)
@ -80,7 +80,7 @@ class DirRule:
# Axxx or Pyyy
key = 1 if rule[0] == 'A' else 2
solve_func = lambda entity, ref=rule[1:]: fix_windir_name(str(getattr(entity, ref)))
solve_func = lambda detail, ref=rule[1:]: fix_windir_name(str(detail.get_dirname(ref)))
# 保存缓存
rule_solver = (key, solve_func)
@ -116,7 +116,6 @@ class DirRule:
class JmOption:
JM_OP_VER = '2.0'
def __init__(self,
dir_rule: Dict,
@ -126,7 +125,7 @@ class JmOption:
filepath=None,
):
# 版本号
self.version = self.JM_OP_VER
self.version = JmModuleConfig.JM_OPTION_VER
# 路径规则配置
self.dir_rule = DirRule(**dir_rule)
# 请求配置
@ -148,10 +147,6 @@ class JmOption:
def download_image_decode(self):
return self.download.image.decode
@property
def download_threading_batch_count(self):
return self.download.threading.batch_count
@property
def download_image_suffix(self):
return self.download.image.suffix
@ -162,11 +157,11 @@ class JmOption:
# noinspection PyUnusedLocal
def decide_image_batch_count(self, photo: JmPhotoDetail):
return self.download_threading_batch_count
return self.download.threading.image
# noinspection PyMethodMayBeStatic,PyUnusedLocal
def decide_photo_batch_count(self, album: JmAlbumDetail):
return os.cpu_count()
return self.download.threading.photo
def decide_image_save_dir(self, photo) -> str:
# 使用 self.dir_rule 决定 save_dir
@ -212,7 +207,7 @@ class JmOption:
# 通过拼接生成绝对路径
save_dir = self.decide_image_save_dir(image.from_photo)
suffix = self.decide_image_suffix(image)
return save_dir + image.img_file_name + suffix
return os.path.join(save_dir, image.filename_without_suffix + suffix)
"""
下面是创建对象相关方法
@ -242,19 +237,27 @@ class JmOption:
return cls.construct({})
@classmethod
def construct(cls, dic: Dict, cover_default=True) -> 'JmOption':
if cover_default:
dic = cls.merge_default_dict(dic)
version = dic.pop('version', None)
if float(version) != float(cls.JM_OP_VER):
# 版本兼容
raise NotImplementedError('不支持的option版本')
def construct(cls, orgdic: Dict, cover_default=True) -> 'JmOption':
dic = cls.merge_default_dict(orgdic) if cover_default else orgdic
# debug
debug = dic.pop('debug', True)
if debug is False:
disable_jm_debug()
# version
version = dic.pop('version', None)
if version is None or float(version) >= float(JmModuleConfig.JM_OPTION_VER):
return cls(**dic)
# 旧版本option做兼容
# 1) 2.0 -> 2.1,并发配置的键名更改了
dt: dict = dic['download']['threading']
if 'batch_count' in dt:
batch_count = dt.pop('batch_count')
dt['image'] = batch_count
return cls(**dic)
def deconstruct(self) -> Dict:
@ -278,8 +281,7 @@ class JmOption:
if filepath is None:
filepath = self.filepath
if filepath is None:
raise JmModuleConfig.exception("未指定JmOption的保存路径")
ExceptionTool.require_true(filepath is not None, "未指定JmOption的保存路径")
PackerUtil.pack(self.deconstruct(), filepath)
@ -369,8 +371,7 @@ class JmOption:
key, kwargs = pinfo['plugin'], pinfo['kwargs']
plugin_class: Optional[Type[JmOptionPlugin]] = plugin_registry.get(key, None)
if plugin_class is None:
raise JmModuleConfig.exception(f'[{group}] 未注册的plugin: {key}')
ExceptionTool.require_true(plugin_class is not None, f'[{group}] 未注册的plugin: {key}')
self.invoke_plugin(plugin_class, kwargs, extra)
@ -408,8 +409,10 @@ class JmOption:
kwargs将来要传给方法参数这要求kwargs的key是str类型
该方法检查kwargs的key的类型如果不是str尝试转为str不行则抛异常
"""
if not isinstance(kwargs, dict):
raise JmModuleConfig.exception(f'插件的kwargs参数必须为dict类型而不能是类型: {type(kwargs)}')
ExceptionTool.require_true(
isinstance(kwargs, dict),
f'插件的kwargs参数必须为dict类型而不能是类型: {type(kwargs)}'
)
kwargs: dict
new_kwargs: Dict[str, Any] = {}
@ -425,7 +428,7 @@ class JmOption:
new_kwargs[newk] = v
continue
raise JmModuleConfig.exception(
ExceptionTool.raises(
f'插件kwargs参数类型有误'
f'字段: {k}预期类型为str实际类型为{type(k)}'
)

View File

@ -238,7 +238,7 @@ class ZipPlugin(JmOptionPlugin):
dir_zip_dict[dir_path] = zip_path
else:
raise NotImplementedError(f'level: {level}')
ExceptionTool.raises(f'Not Implemented Zip Level: {level}')
self.after_zip(dir_zip_dict)

View File

@ -67,16 +67,14 @@ class JmcomicText:
if isinstance(text, int):
return str(text)
if not isinstance(text, str):
raise JmModuleConfig.exception(f"无法解析jm车号, 参数类型为: {type(text)}")
ExceptionTool.require_true(isinstance(text, str), f"无法解析jm车号, 参数类型为: {type(text)}")
# 43210
if text.isdigit():
return text
# Jm43210
if len(text) <= 2:
raise JmModuleConfig.exception(f"无法解析jm车号, 文本为: {text}")
ExceptionTool.require_true(len(text) >= 2, f"无法解析jm车号, 文本太短: {text}")
# text: JM12341
c0 = text[0]
@ -87,8 +85,7 @@ class JmcomicText:
else:
# https://xxx/photo/412038
match = cls.pattern_jm_pa_id.search(text)
if match is None:
raise JmModuleConfig.exception(f"无法解析jm车号, 文本为: {text}")
ExceptionTool.require_true(match is not None, f"无法解析jm车号, 文本为: {text}")
return match[2]
@classmethod
@ -169,11 +166,10 @@ class JmcomicText:
if field_value is None:
if default is None:
JmModuleConfig.raises(
ExceptionTool.raises_regex(
f"文本没有匹配上字段:字段名为'{field_name}'pattern: [{pattern}]",
re_match_html=html,
re_match_field_name=field_name,
re_match_pattern=pattern,
html=html,
pattern=pattern,
)
else:
field_value = default
@ -260,12 +256,20 @@ class JmcomicSearchTool:
match = cls.pattern_html_search_error.search(html)
if match is not None:
topic, reason = match[1], match[2]
JmModuleConfig.raises(f'{topic}: {reason}', re_search_html=html)
ExceptionTool.raises_regex(
f'{topic}: {reason}',
html=html,
pattern=cls.pattern_html_search_error,
)
# 缩小文本范围
match = cls.pattern_html_search_shorten_for.search(html)
if match is None:
JmModuleConfig.raises('未匹配到搜索结果', re_shorten_html=html)
ExceptionTool.raises_regex(
'未匹配到搜索结果',
html=html,
pattern=cls.pattern_html_search_shorten_for,
)
html = match[0]
# 提取结果
@ -324,6 +328,8 @@ class JmcomicSearchTool:
class JmApiAdaptTool:
"""
本类复杂把移动端的api返回值适配为标准的实体类
# album
{
"id": 123,
@ -397,7 +403,7 @@ class JmApiAdaptTool:
'series_id',
('tags', 'keywords'),
('id', 'photo_id'),
('images', 'page_arr')
('images', 'page_arr'),
]
}
@ -429,7 +435,7 @@ class JmApiAdaptTool:
if issubclass(clazz, k):
return v
raise AssertionError(clazz)
ExceptionTool.raises(f'不支持的类型: {clazz}')
@classmethod
def post_adapt_album(cls, data: dict, _clazz: type, fields: dict):
@ -447,16 +453,22 @@ class JmApiAdaptTool:
@classmethod
def post_adapt_photo(cls, data: dict, _clazz: type, fields: dict):
for chapter in data['series']:
# 1. 获取sort字段如果data['series']中没有使用默认值1
sort = 1
series: list = data['series'] # series中的sort从1开始
for chapter in series:
chapter = DictModel(chapter)
if int(chapter.id) == int(data['id']):
fields['sort'] = chapter.sort
sort = chapter.sort
break
fields['sort'] = sort
fields['scramble_id'] = '0'
import random
fields['data_original_domain'] = random.choice(JmModuleConfig.DOMAIN_IMAGE_LIST)
class JmImageSupport:
class JmImageTool:
@classmethod
def save_resp_img(cls, resp: Any, filepath: str, need_convert=True):
@ -579,3 +591,83 @@ class JmImageSupport:
获得图片分割数
"""
return cls.get_num(detail.scramble_id, detail.aid, detail.img_file_name)
class ExceptionTool:
"""
抛异常的工具
1: 能简化 if-raise 语句的编写
2: 有更好的上下文信息传递方式
"""
EXTRA_KEY_RESP = 'resp'
EXTRA_KEY_HTML = 'html'
EXTRA_KEY_RE_PATTERN = 'pattern'
@classmethod
def raises(cls, msg: str, extra: dict = None):
if extra is None:
extra = {}
JmModuleConfig.raise_exception_executor(msg, extra)
@classmethod
def raises_regex(cls,
msg: str,
html: str,
pattern: Pattern,
):
cls.raises(
msg, {
cls.EXTRA_KEY_HTML: html,
cls.EXTRA_KEY_RE_PATTERN: pattern,
}
)
@classmethod
def raises_resp(cls,
msg: str,
resp,
):
cls.raises(
msg, {
cls.EXTRA_KEY_RESP: resp
}
)
@classmethod
def raise_missing(cls,
resp,
org_req_url=None,
):
"""
抛出本子/章节的异常
@param resp: 响应对象
@param org_req_url: 原始请求url可不传
"""
if org_req_url is None:
org_req_url = resp.url
req_type = "本子" if "album" in org_req_url else "章节"
cls.raises_resp((
f'请求的{req_type}不存在!({org_req_url})\n'
'原因可能为:\n'
f'1. id有误检查你的{req_type}id\n'
'2. 该漫画只对登录用户可见请配置你的cookies\n'
), resp)
@classmethod
def require_true(cls, case: bool, msg: str):
if case:
return
cls.raises(msg)
@classmethod
def replace_old_exception_executor(cls, raises: Callable[[Callable, str, dict], None]):
old = JmModuleConfig.raise_exception_executor
def new(msg, extra):
raises(old, msg, extra)
JmModuleConfig.raise_exception_executor = new

View File

@ -29,7 +29,7 @@ class JmTestConfigurable(unittest.TestCase):
os.chdir(cls.project_dir)
# 设置 JmOptionJmcomicClient
option = cls.use_option('option_test.yml')
option = create_option_by_env('JM_OPTION_PATH_TEST')
cls.option = option
cls.client = option.build_jm_client()

View File

@ -6,7 +6,6 @@ class Test_MobileClient(JmTestConfigurable):
def test_cl(self):
for cmd in str_to_list('''
jmcomic 438516
jmcomic 438516 --option=''
jmcomic 438516 p438516
'''):
self.assertEqual(os.system(cmd), 0)

View File

@ -7,7 +7,9 @@ class Test_Client(JmTestConfigurable):
jm_photo_id = 'JM438516'
photo = self.client.get_photo_detail(jm_photo_id, False)
image = photo[0]
self.client.download_by_image_detail(image, self.option.decide_image_filepath(image))
filepath = self.option.decide_image_filepath(image)
self.client.download_by_image_detail(image, filepath)
print(filepath)
def test_fetch_album(self):
album_id = "JM438516"
@ -15,8 +17,13 @@ class Test_Client(JmTestConfigurable):
def test_search(self):
page: JmSearchPage = self.client.search_tag('+无修正 +中文 -全彩')
for album_id, title, tag_list in page.iter_id_title_tag():
print(album_id, title, tag_list)
if len(page) >= 1:
for aid, ainfo in page[0:1:1]:
print(aid, ainfo)
for aid, atitle, tag_list in page.iter_id_title_tag():
print(aid, atitle, tag_list)
aid = '438516'
page = self.client.search_site(aid)
@ -31,13 +38,29 @@ class Test_Client(JmTestConfigurable):
self.client.download_by_image_detail(image, self.option.decide_image_filepath(image))
def test_album_missing(self):
JmModuleConfig.CLASS_EXCEPTION = JmcomicException
class A(BaseException):
pass
JmModuleConfig.CLASS_EXCEPTION = A
self.assertRaises(
JmcomicException,
A,
self.client.get_album_detail,
'332583'
'0'
)
def test_raise_exception(self):
class B(BaseException):
pass
def raises(old, _msg, _extra):
self.assertEqual(old, default_raise_exception_executor)
raise B()
JmModuleConfig.raise_exception_executor = default_raise_exception_executor
ExceptionTool.replace_old_exception_executor(raises)
self.assertRaises(B, JmcomicText.parse_to_album_id, 'asdhasjhkd')
def test_detail_property_list(self):
album = self.client.get_album_detail(410090)
@ -49,7 +72,7 @@ class Test_Client(JmTestConfigurable):
]
for pair in ans:
self.assertListEqual(pair[0], pair[1])
self.assertListEqual(pair[0][0:9], pair[1][0:9])
def test_photo_sort(self):
client = self.option.build_jm_client()
@ -193,3 +216,13 @@ class Test_Client(JmTestConfigurable):
0,
aid,
)
def test_get_detail(self):
client = self.client
album = client.get_album_detail(400222)
print(album.id, album.name, album.tags)
for photo in album[0:3]:
photo = client.get_photo_detail(photo.photo_id)
print(photo.id, photo.name)

View File

@ -1,37 +0,0 @@
from test_jmcomic import *
# 移动端专用的禁漫域名
domain_list = [
"https://www.jmapinode1.cc",
"https://www.jmapinode2.cc",
"https://www.jmapinode3.cc",
"https://www.jmapibranch2.cc"
]
class Test_MobileClient(JmTestConfigurable):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.client = cls.option.new_jm_client(domain_list, impl='api')
def test_search(self):
page = self.client.search_site('MANA')
if len(page) >= 1:
for aid, ainfo in page[0:1:1]:
print(aid, ainfo['description'], ainfo['category'])
for aid, atitle, tag_list in page.iter_id_title_tag():
print(aid, atitle, tag_list)
def test_get_detail(self):
client = self.client
album = client.get_album_detail(400222)
print(album.id, album.name, album.tags)
for photo in album[0:3]:
photo = client.get_photo_detail(photo.photo_id)
print(photo.id, photo.name)

View File

@ -77,17 +77,15 @@ def custom_client_class():
def custom_album_photo_image_detail_class():
"""
该函数演示替换实体类本子/章节/图片
"""
# 在使用路径规则 DirRule 时,可能会遇到需要自定义实体类属性的情况,例如:
"""
在使用路径规则 DirRule 可能会遇到需要自定义实体类属性的情况例如
dir_rule:
base_dir: ${workspace}
rule: Bd_Acustom_Pcustom
"""
# 上面的AcustomPcustom都是自定义字段
# 如果你想要使用这种自定义字段,你就需要替换默认的实体类,例如
上面的AcustomPcustom都是自定义字段
如果你想要使用这种自定义字段你就需要替换默认的实体类方式如下
"""
# 自定义本子实体类
class MyAlbum(JmAlbumDetail):
@ -103,14 +101,21 @@ def custom_album_photo_image_detail_class():
def custom(self):
return f'custom_{self.title}'
# 自定义图片实体类
class MyImage(JmImageDetail):
pass
"""
v2.3.3: 支持更灵活的自定义方式可以使用函数效果同上示例见下
"""
class MyAlbum2(JmAlbumDetail):
def get_dirname(self, ref: str) -> str:
if ref == 'custom':
return f'custom_{self.name}'
return super().get_dirname(ref)
# 最后,替换默认实体类来让你的自定义类生效
JmModuleConfig.CLASS_ALBUM = MyAlbum
JmModuleConfig.CLASS_PHOTO = MyPhoto
JmModuleConfig.CLASS_IMAGE = MyImage
def custom_jm_debug():
@ -133,30 +138,3 @@ def custom_jm_debug():
# 2. 让my_debug生效
JmModuleConfig.debug_executor = my_debug
def custom_exception_raise():
"""
该函数演示jmcomic的异常机制
"""
# jmcomic 代码在运行过程中可能抛出异常以获取album实体类为例
album = client.get_album_detail('999999')
# 上面这行代码用于获取本子id为 999999 的JmAlbumDetail
# 如果本子不存在,则会抛出异常,异常类默认是 JmcomicException
# 你可以自定义抛出的异常类,做法如下:
# 1. 自定义异常类
class MyExceptionClass(Exception):
pass
# 2. 替换默认异常类
JmModuleConfig.CLASS_EXCEPTION = MyExceptionClass
# 这样一来,抛出的异常类就是 MyExceptionClass
try:
album = client.get_album_detail('999999')
except MyExceptionClass as e:
print('捕获MyExceptionClass异常')
pass

View File

@ -49,11 +49,11 @@ def get_option():
# 支持工作流覆盖配置文件的配置
cover_option_config(option)
# 覆盖client实现类实现把请求错误的html下载到文件方便GitHub Actions下载查看日志
hook_debug(option)
# 把请求错误的html下载到文件方便GitHub Actions下载查看日志
log_before_raise()
# 登录,如果有配置的话
login_if_configured(option.build_jm_client())
login_if_configured(option)
return option
@ -66,7 +66,7 @@ def cover_option_config(option: JmOption):
option.dir_rule = the_new
def login_if_configured(client):
def login_if_configured(option):
# 检查环境变量中是否有禁漫的用户名和密码,如果有则登录
# 禁漫的大部分本子,下载是不需要登录的,少部分敏感题材需要登录
# 如果你希望以登录状态下载本子你需要自己配置一下GitHub Actions的 `secrets`
@ -76,28 +76,28 @@ def login_if_configured(client):
username = get_env('JM_USERNAME', None)
password = get_env('JM_PASSWORD', None)
if username is not None and password is not None:
client.login(username, password, True)
print_eye_catching(f'登录禁漫成功')
# 调用login插件
JmLoginPlugin(option).invoke(username, password)
# noinspection PyUnusedLocal
def hook_debug(option):
def log_before_raise():
jm_download_dir = get_env('JM_DOWNLOAD_DIR', workspace())
mkdir_if_not_exists(jm_download_dir)
class RaiseErrorAwareClient(JmHtmlClient):
# 自定义异常抛出函数在抛出前把HTML响应数据写到下载文件夹日志留痕
def raises(old, msg, extra: dict):
if ExceptionTool.EXTRA_KEY_RESP not in extra:
return old(msg, extra)
@classmethod
def raise_request_error(cls, resp, msg=None):
from common import write_text, fix_windir_name
write_text(
f'{jm_download_dir}/{fix_windir_name(resp.url)}',
resp.text
)
resp = extra[ExceptionTool.EXTRA_KEY_RESP]
# 写文件
from common import write_text, fix_windir_name
write_text(f'{jm_download_dir}/{fix_windir_name(resp.url)}', resp.text)
return super().raise_request_error(resp, msg)
return old(msg, extra)
JmModuleConfig.CLASS_CLIENT_IMPL['html'] = RaiseErrorAwareClient
# 应用函数
ExceptionTool.replace_old_exception_executor(raises)
if __name__ == '__main__':