v2.3.0: 初步实现移动端API(JmApiClient),优化正则表达式并适配大陆直连域名,增加测试,优化工作流 (#137)

This commit is contained in:
hect0x7 2023-09-15 17:51:33 +08:00 committed by GitHub
parent c5810d9535
commit 89deeda02e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 347 additions and 88 deletions

41
.github/workflows/release_auto.yml vendored Normal file
View File

@ -0,0 +1,41 @@
name: Release (auto)
on:
workflow_dispatch:
jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Parse Tag & Body
id: tb
run: |
commit_message=$(git log --format=%B -n 1 ${{ github.sha }})
python release.py "$commit_message"
- name: Create Release
uses: softprops/action-gh-release@v1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ steps.tb.outputs.tag }}
body: ${{ steps.tb.outputs.body }}
- name: Build Module
run: |
python -m pip install build
python -m build
- name: Release PYPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
password: ${{ secrets.PYPI_JMCOMIC }}

9
TODO.md Normal file
View File

@ -0,0 +1,9 @@
# 大版本更新内容计划
| 版本范围 | 更新内容 |
|:--------:|:--------------------------------------:|
| v2.3.* | 实现移动端API的基础功能统一HTML和API的实现 |
| v2.2.* | 新的插件体系,新的命令行调用,完善搜索功能。 |
| v2.1.* | 拆分Downloader抽象调度优化可扩展性、代码复用性、模块级别自定义。 |
| v2.0.* | 重新设计合理的抽象层次实现请求重试切换域名机制新的option配置设计。 |
| v1.\*.\* | 基于HTML实现基础功能。 |

20
release.py Normal file
View File

@ -0,0 +1,20 @@
import os
import sys
import re
def add_output(k, v):
print(f'set {k} = {v}')
print(os.system(f'echo {k}={v} >> $GITHUB_OUTPUT'))
msg = sys.argv[1]
print(f'msg: {msg}')
p = re.compile('(.*?): ?(.*)')
match = p.search(msg)
assert match is not None, f'commit message format is wrong: {msg}'
tag, body = match[1], match[2]
add_output('tag', tag)
add_output('body', body)

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.2.11'
__version__ = '2.3.0'
from .api import *
from .jm_plugin import *

View File

@ -42,11 +42,11 @@ class JmcomicUI:
parser.add_argument(
'--option',
help='path to the option file, you can also specify it by env `JM_OPTION_PATH`',
default=get_env('JM_OPTION_PATH', None),
default=get_env('JM_OPTION_PATH', ''),
)
args = parser.parse_args()
if args.option is not None:
if len(args.option) != 0:
self.option_path = os.path.abspath(args.option)
else:
self.option_path = None
@ -88,7 +88,7 @@ class JmcomicUI:
option = create_option(self.option_path)
else:
option = JmOption.default()
self.run(option)
def run(self, option):

View File

@ -398,6 +398,16 @@ class JmHtmlClient(AbstractJmClient):
class JmApiClient(AbstractJmClient):
client_key = 'api'
API_SEARCH = '/search'
API_ALBUM = '/album'
API_CHAPTER = '/chapter'
def __init__(self,
postman: Postman,
retry_times: int,
domain=None,
fallback_domain_list=None,
):
super().__init__(postman, retry_times, domain, fallback_domain_list)
def search(self,
search_query: str,
@ -431,6 +441,30 @@ class JmApiClient(AbstractJmClient):
return JmcomicSearchTool.parse_api_resp_to_page(data)
def get_album_detail(self, album_id) -> JmAlbumDetail:
return self.fetch_detail_entity(album_id,
JmModuleConfig.album_class(),
)
def get_photo_detail(self, photo_id, fetch_album=True) -> JmPhotoDetail:
return self.fetch_detail_entity(photo_id,
JmModuleConfig.photo_class(),
)
def fetch_detail_entity(self, apid, clazz, **kwargs):
url = self.API_ALBUM if issubclass(clazz, JmAlbumDetail) else self.API_CHAPTER
resp = self.get(
url,
params={
'id': JmcomicText.parse_to_album_id(apid),
**kwargs,
}
)
self.require_resp_success(resp)
return JmApiAdaptTool.parse_entity(resp.res_data, clazz)
def get(self, url, **kwargs) -> JmApiResp:
# set headers
headers, key_ts = self.headers_key_ts
@ -454,6 +488,10 @@ class JmApiClient(AbstractJmClient):
def debug_topic_request(self):
return 'api'
# noinspection PyMethodMayBeStatic
def require_resp_success(self, resp: JmApiResp):
resp.require_success()
JmModuleConfig.register_client(JmHtmlClient)
JmModuleConfig.register_client(JmApiClient)

View File

@ -81,6 +81,10 @@ class JmApiResp(JmResp):
self.key_ts = key_ts
self.cache_decode_data = None
@property
def is_success(self) -> bool:
return super().is_success and self.json()['code'] == 200
@staticmethod
def parse_data(text, time) -> str:
# 1. base64解码
@ -102,11 +106,9 @@ class JmApiResp(JmResp):
return res
@property
@field_cache('__cache_decoded_data__')
def decoded_data(self) -> str:
if self.cache_decode_data is None:
self.cache_decode_data = self.parse_data(self.encoded_data, self.key_ts)
return self.cache_decode_data
return self.parse_data(self.encoded_data, self.key_ts)
@property
def encoded_data(self) -> str:
@ -118,6 +120,7 @@ class JmApiResp(JmResp):
from json import loads
return loads(self.decoded_data)
@field_cache('__cache_json__')
def json(self, **kwargs) -> Dict:
return self.resp.json()
@ -136,9 +139,6 @@ class JmAcResp(JmResp):
def json(self, **kwargs) -> Dict:
return self.resp.json()
def model(self) -> DictModel:
return DictModel(self.json())
"""

View File

@ -68,6 +68,9 @@ class JmModuleConfig:
CLASS_CLIENT_IMPL = {}
CLASS_EXCEPTION = None
# 插件注册表
PLUGIN_REGISTRY = {}
# 执行debug的函数
debug_executor = default_jm_debug
# postman构造函数
@ -80,9 +83,6 @@ class JmModuleConfig:
# debug时解码url
decode_url_when_debug = True
# 插件注册表
plugin_registry = {}
@classmethod
def downloader_class(cls):
if cls.CLASS_DOWNLOADER is not None:
@ -243,7 +243,7 @@ class JmModuleConfig:
default_option_dict: dict = {
'version': '2.0',
'debug': None,
'dir_rule': {'rule': 'Bd_Ptitle', 'base_dir': None},
'dir_rule': {'rule': 'Bd_Pname', 'base_dir': None},
'download': {
'cache': True,
'image': {'decode': True, 'suffix': None},
@ -299,7 +299,7 @@ class JmModuleConfig:
@classmethod
def register_plugin(cls, plugin_class):
cls.plugin_registry[plugin_class.plugin_key] = plugin_class
cls.PLUGIN_REGISTRY[plugin_class.plugin_key] = plugin_class
@classmethod
def register_client(cls, client_class):

View File

@ -22,8 +22,8 @@ class DownloadCallback:
f'作者: [{album.author}], '
f'章节数: [{len(album)}], '
f'总页数: [{album.page_count}], '
f'标题: [{album.title}], '
f'关键词: [{album.tag_list}]'
f'标题: [{album.name}], '
f'关键词: [{album.tags}]'
)
def after_album(self, album: JmAlbumDetail):
@ -32,7 +32,7 @@ class DownloadCallback:
def before_photo(self, photo: JmPhotoDetail):
jm_debug('photo.before',
f'开始下载章节: {photo.id} ({photo.album_id}[{photo.index}/{len(photo.from_album)}]), '
f'标题: [{photo.title}], '
f'标题: [{photo.name}], '
f'图片数为[{len(photo)}]'
)

View File

@ -42,11 +42,11 @@ class DetailEntity(JmBaseEntity, IndexedEntity):
raise NotImplementedError
@property
def name(self) -> str:
return getattr(self, 'title')
def title(self) -> str:
return getattr(self, 'name')
def __str__(self):
return f'{self.__class__.__name__}({self.id}-{self.name})'
return f'{self.__class__.__name__}({self.id}-{self.title})'
@classmethod
def __alias__(cls):
@ -139,7 +139,7 @@ class JmPhotoDetail(DetailEntity):
def __init__(self,
photo_id,
scramble_id,
title,
name,
keywords,
series_id,
sort,
@ -151,7 +151,7 @@ class JmPhotoDetail(DetailEntity):
):
self.photo_id: str = photo_id
self.scramble_id: str = scramble_id
self.title: str = str(title).strip()
self.name: str = str(name).strip()
self.sort: int = int(sort)
self._keywords: str = keywords
self._series_id: int = int(series_id)
@ -188,13 +188,13 @@ class JmPhotoDetail(DetailEntity):
@property
def tags(self) -> List[str]:
if self.from_album is not None:
return self.from_album.tag_list
return self.from_album.tags
return self._keywords.split(',')
@property
def indextitle(self):
return f'{self.album_index}{self.title}'
return f'{self.album_index}{self.name}'
@property
def album_id(self) -> str:
@ -287,22 +287,23 @@ class JmAlbumDetail(DetailEntity):
def __init__(self,
album_id,
scramble_id,
title,
name,
episode_list,
page_count,
author_list,
tag_list,
pub_date,
update_date,
likes,
views,
comment_count,
work_list,
actor_list,
works,
actors,
authors,
tags,
related_list=None,
):
self.album_id: str = album_id
self.scramble_id: str = scramble_id
self.title: str = title
self.name: str = name
self.page_count = int(page_count) # 总页数
self.pub_date: str = pub_date # 发布日期
self.update_date: str = update_date # 更新日期
@ -310,19 +311,20 @@ class JmAlbumDetail(DetailEntity):
self.likes: str = likes # [1K] 點擊喜歡
self.views: str = views # [40K] 次觀看
self.comment_count: int = self.__parse_comment_count(comment_count) # 评论数
self.work_list: List[str] = work_list # 作品
self.actor_list: List[str] = actor_list # 登場人物
self.tag_list: List[str] = tag_list # 標籤
self.author_list: List[str] = author_list # 作者
self.works: List[str] = works # 作品
self.actors: List[str] = actors # 登場人物
self.tags: List[str] = tags # 標籤
self.authors: List[str] = authors # 作者
# 有的 album 没有章节,则自成一章。
if len(episode_list) == 0:
# photo_id, photo_index, photo_title, photo_pub_date
episode_list = [(album_id, 1, title, pub_date)]
episode_list = [(album_id, 1, name, pub_date)]
else:
episode_list = self.distinct_episode(episode_list)
self.episode_list: List[Tuple] = episode_list
self.related_list = related_list
@property
def author(self):
@ -330,8 +332,8 @@ class JmAlbumDetail(DetailEntity):
作者
禁漫本子的作者标签可能有多个全部作者请使用字段 self.author_list
"""
if len(self.author_list) >= 1:
return self.author_list[0]
if len(self.authors) >= 1:
return self.authors[0]
return JmModuleConfig.default_author
@ -358,18 +360,7 @@ class JmAlbumDetail(DetailEntity):
# noinspection PyMethodMayBeStatic
def __parse_comment_count(self, comment_count: str) -> int:
if comment_count == '':
return 0
try:
from .jm_toolkit import JmcomicText
match = JmcomicText.pattern_total_video_comments.search(comment_count)
if match is None:
return 0
return int(match[1])
except ValueError:
jm_debug('regular.error', f'评论数匹配失败: {comment_count}')
return 0
return int(comment_count)
def create_photo_detail(self, index) -> Tuple[JmPhotoDetail, Tuple]:
# 校验参数
@ -378,24 +369,23 @@ class JmAlbumDetail(DetailEntity):
if index >= length:
raise JmModuleConfig.exception(f'创建JmPhotoDetail失败{index} >= {length}')
# episode_info: ('212214', '81', '94 突然打來', '2020-08-29')
episode_info: tuple = self.episode_list[index]
photo_id, photo_index, photo_title, photo_pub_date = episode_info
# ('212214', '81', '94 突然打來', '2020-08-29')
pid, pindex, pname, _pub_date = self.episode_list[index]
photo = JmModuleConfig.photo_class()(
photo_id=photo_id,
photo_id=pid,
scramble_id=self.scramble_id,
title=photo_title,
name=pname,
keywords='',
series_id=self.album_id,
sort=photo_index,
sort=pindex,
author=self.author,
from_album=self,
page_arr=None,
data_original_domain=None
)
return photo, episode_info
return photo, (self.episode_list[index])
def getindex(self, item) -> JmPhotoDetail:
return self.create_photo_detail(item)[0]
@ -454,8 +444,8 @@ class JmSearchPage(JmBaseEntity, IndexedEntity):
def wrap_single_album(cls, album: JmAlbumDetail) -> 'JmSearchPage':
page = JmSearchPage([(
album.album_id, {
'name': album.title,
'tag_list': album.tag_list,
'name': album.name,
'tag_list': album.tags,
}
)])
setattr(page, 'album', album)

View File

@ -346,7 +346,7 @@ class JmOption:
# 保证 jm_plugin.py 被加载
from .jm_plugin import JmOptionPlugin
plugin_registry = JmModuleConfig.plugin_registry
plugin_registry = JmModuleConfig.PLUGIN_REGISTRY
for pinfo in plugin_list:
key, kwargs = pinfo['plugin'], pinfo['kwargs']
plugin_class: Optional[Type[JmOptionPlugin]] = plugin_registry.get(key, None)

View File

@ -81,7 +81,7 @@ class UsageLogPlugin(JmOptionPlugin):
interval=1,
enable_warning=True,
warning_cpu_percent=70,
warning_mem_percent=50,
warning_mem_percent=70,
warning_thread_count=100,
):
try:

View File

@ -10,7 +10,7 @@ class JmcomicText:
pattern_html_photo_photo_id = compile('<meta property="og:url" content=".*?/photo/(\d+)/?.*?">')
pattern_html_photo_scramble_id = compile('var scramble_id = (\d+);')
pattern_html_photo_title = compile('<title>([\s\S]*?)\|.*</title>')
pattern_html_photo_name = compile('<title>([\s\S]*?)\|.*</title>')
# pattern_html_photo_data_original_list = compile('data-original="(.*?)" id="album_photo_.+?"')
pattern_html_photo_data_original_domain = compile('src="https://(.*?)/media/albums/blank')
pattern_html_photo_data_original_0 = compile('data-original="(.*?)"[ \n]*?id="album_photo')
@ -21,7 +21,7 @@ class JmcomicText:
pattern_html_album_album_id = compile('<span class="number">.*?JM(\d+)</span>')
pattern_html_album_scramble_id = compile('var scramble_id = (\d+);')
pattern_html_album_title = compile('<h1 class="book-name" id="book-name">([\s\S]*?)</h1>')
pattern_html_album_name = compile('<h1 class="book-name" id="book-name">([\s\S]*?)</h1>')
pattern_html_album_episode_list = compile('data-album="(\d+)">\n *?<li.*?>\n *'
'第(\d+)話\n([\s\S]*?)\n *'
'<[\s\S]*?>(\d+-\d+-\d+).*?')
@ -29,36 +29,31 @@ class JmcomicText:
pattern_html_album_pub_date = compile('>上架日期 : (.*?)</span>')
pattern_html_album_update_date = compile('>更新日期 : (.*?)</span>')
# 作品
pattern_html_album_work_list = [
pattern_html_album_works = [
compile('<span itemprop="author" data-type="works">([\s\S]*?)</span>'),
compile('<a[\s\S]*?>(.*?)</a>')
]
# 登場人物
pattern_html_album_actor_list = [
pattern_html_album_actors = [
compile('<span itemprop="author" data-type="actor">([\s\S]*?)</span>'),
compile('<a[\s\S]*?>(.*?)</a>')
]
# 标签
pattern_html_album_tag_list = [
pattern_html_album_tags = [
compile('<span itemprop="genre" data-type="tags">([\s\S]*?)</span>'),
compile('<a[\s\S]*?>(.*?)</a>')
]
# 作者
pattern_html_album_author_list = [
pattern_html_album_authors = [
compile('作者: *<span itemprop="author" data-type="author">([\s\S]*?)</span>'),
compile("<a[\s\S]*?>(.*?)</a>"),
]
# 點擊喜歡
pattern_html_album_likes = compile('<span id="albim_likes_\d+">(.*?)</span>')
# 觀看
pattern_html_album_views = compile('<span>(.*?)</span> 次觀看')
pattern_html_album_views = compile('<span>(.*?)</span> (次觀看|观看次数)')
# 評論(div)
pattern_html_album_comment_count = compile(
'forum-open btn btn-primary" .*>\w{2}\n'
'(<div class="badge" id="total_video_comments">(\d+)</div>|)?'
)
# 评论(number)
pattern_total_video_comments = compile('<div class="badge" id="total_video_comments">(\d+)</div>')
pattern_html_album_comment_count = compile('<div class="badge"\n? *id="total_video_comments">(\d+)</div>'), 0
@classmethod
def parse_to_jm_domain(cls, text: str):
@ -145,7 +140,8 @@ class JmcomicText:
if match is None:
return None
text = match[0]
pattern = last_pattern
return last_pattern.findall(text)
if field_key.endswith("_list"):
return pattern.findall(text)
@ -157,21 +153,30 @@ class JmcomicText:
field_dict = {}
pattern_name: str
for pattern_name, pattern_value in cls.__dict__.items():
for pattern_name, pattern in cls.__dict__.items():
if not pattern_name.startswith(cls_field_prefix):
continue
# 支持如果不匹配,使用默认值
if isinstance(pattern, tuple):
pattern, default = pattern
else:
default = None
# 获取字段名和值
field_name = pattern_name[pattern_name.index(cls_field_prefix) + len(cls_field_prefix):]
field_value = match_field(field_name, pattern_value, html)
field_value = match_field(field_name, pattern, html)
if field_value is None:
JmModuleConfig.raise_regex_error_executor(
f"文本没有匹配上字段:字段名为'{field_name}'pattern: [{pattern_value}]",
html,
field_name,
pattern_value
)
if default is None:
JmModuleConfig.raise_regex_error_executor(
f"文本没有匹配上字段:字段名为'{field_name}'pattern: [{pattern}]",
html,
field_name,
pattern
)
else:
field_value = default
# 保存字段
field_dict[field_name] = field_value
@ -317,6 +322,140 @@ class JmcomicSearchTool:
return JmSearchPage(content)
class JmApiAdaptTool:
"""
# album
{
"id": 123,
"name": "[狗野叉漢化]",
"author": [
"AREA188"
],
"images": [
"00004.webp"
],
"description": null,
"total_views": "41314",
"likes": "918",
"series": [],
"series_id": "0",
"comment_total": "5",
"tags": [
"全彩",
"中文"
],
"works": [],
"actors": [],
"related_list": [
{
"id": "333718",
"author": "been",
"description": "",
"name": "[been]The illusion of lies1[中國語][無修正][全彩]",
"image": ""
}
],
"liked": false,
"is_favorite": false
}
# photo
{
"id": 413446,
"series": [
{
"id": "487043",
"name": "第48話",
"sort": "48"
}
],
"tags": "慾望 調教 NTL 地鐵 戲劇",
"name": "癡漢成癮-第2話",
"images": [
"00047.webp"
],
"series_id": "400222",
"is_favorite": false,
"liked": false
}
"""
field_adapter = {
JmAlbumDetail: [
'likes',
'tags',
'works',
'actors',
'related_list',
'name',
('id', 'album_id'),
('author', 'authors'),
('total_views', 'views'),
('comment_total', 'comment_count'),
],
JmPhotoDetail: [
'name',
'series_id',
('tags', 'keywords'),
('id', 'photo_id'),
('images', 'page_arr')
]
}
@classmethod
def parse_entity(cls, data: dict, clazz: type):
adapter = cls.get_adapter(clazz)
fields = {}
for k in adapter:
if isinstance(k, str):
v = data[k]
fields[k] = v
elif isinstance(k, tuple):
k, rename_k = k
v = data[k]
fields[rename_k] = v
if issubclass(clazz, JmAlbumDetail):
cls.post_adapt_album(data, clazz, fields)
else:
cls.post_adapt_photo(data, clazz, fields)
return clazz(**fields)
@classmethod
def get_adapter(cls, clazz: type):
for k, v in cls.field_adapter.items():
if issubclass(clazz, k):
return v
raise AssertionError(clazz)
@classmethod
def post_adapt_album(cls, data: dict, _clazz: type, fields: dict):
series = data['series']
episode_list = []
for chapter in series:
chapter = DictModel(chapter)
# photo_id, photo_index, photo_title, photo_pub_date
episode_list.append(
(chapter.id, chapter.sort, chapter.name, None)
)
fields['episode_list'] = episode_list
for it in 'scramble_id', 'page_count', 'pub_date', 'update_date':
fields[it] = '0'
@classmethod
def post_adapt_photo(cls, data: dict, _clazz: type, fields: dict):
for chapter in data['series']:
chapter = DictModel(chapter)
if int(chapter.id) == int(data['id']):
fields['sort'] = chapter.sort
break
fields['scramble_id'] = '0'
class JmImageSupport:
@classmethod

View File

@ -0,0 +1,12 @@
from test_jmcomic import *
class Test_MobileClient(JmTestConfigurable):
def test_cl(self):
for cmd in str_to_list('''
jmcomic 438516
jmcomic 438516 --option=''
jmcomic 438516 p438516
'''):
self.assertEqual(os.system(cmd), 0)

View File

@ -42,10 +42,10 @@ class Test_Client(JmTestConfigurable):
album = self.client.get_album_detail(410090)
ans = [
(album.work_list, ['原神', 'Genshin']),
(album.actor_list, ['申鶴', '神里綾華', '甘雨']),
(album.tag_list, ['C101', '巨乳', '校服', '口交', '乳交', '群交', '連褲襪', '中文', '禁漫漢化組', '纯爱']),
(album.author_list, ['うぱ西']),
(album.works, ['原神', 'Genshin']),
(album.actors, ['申鶴', '神里綾華', '甘雨']),
(album.tags, ['C101', '巨乳', '校服', '口交', '乳交', '群交', '連褲襪', '中文', '禁漫漢化組', '纯爱']),
(album.authors, ['うぱ西']),
]
for pair in ans:

View File

@ -25,3 +25,13 @@ class Test_MobileClient(JmTestConfigurable):
for aid, atitle, tag_list in page.iter_id_title_tag():
print(aid, atitle, tag_list)
def test_get_detail(self):
client = self.client
album = client.get_album_detail(400222)
print(album.id, album.name, album.tags)
for photo in album[0:3]:
photo = client.get_photo_detail(photo.photo_id)
print(photo.id, photo.name)