mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v1.1.0: 优化debug日志,使用线程池调整下载调度。 (#5)
This commit is contained in:
parent
5a370e36cf
commit
2e29d53631
53
.github/workflows/action_run_tests.yml
vendored
Normal file
53
.github/workflows/action_run_tests.yml
vendored
Normal file
@ -0,0 +1,53 @@
|
||||
name: 跑测试
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "dev", "master" ]
|
||||
paths:
|
||||
- 'src/**/*.py' # 源码
|
||||
- 'tests/**/*.py' # 测试代码
|
||||
|
||||
jobs:
|
||||
test: # This code is based on https://github.com/gaogaotiantian/viztracer/blob/master/.github/workflows/python-package.yml
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [ 3.7, 3.8, 3.9, "3.10", "3.11" ]
|
||||
os: [ ubuntu-latest, macos-latest ]
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 5
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Install dependencies
|
||||
if: matrix.os != 'windows-latest'
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
|
||||
|
||||
- name: Install dependencies on Windows
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
if (Test-Path -Path '.\requirements-dev.txt' -PathType Leaf) {pip install -r requirements-dev.txt}
|
||||
|
||||
- name: Install local
|
||||
run: |
|
||||
pip install -e ./
|
||||
|
||||
- name: 跑测试
|
||||
if: matrix.os != 'windows-latest'
|
||||
run: |
|
||||
cd ./tests/
|
||||
python -m unittest
|
||||
|
||||
- name: 跑测试(Windows系统)
|
||||
if: matrix.os == 'windows-latest'
|
||||
run: |
|
||||
python -m unittest discover -s ./tests -p "test_*.py"
|
2
.github/workflows/action_workflow.yml
vendored
2
.github/workflows/action_workflow.yml
vendored
@ -19,7 +19,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python 3.11
|
||||
uses: actions/setup-python@v3
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
|
2
.github/workflows/action_workflow_local.yml
vendored
2
.github/workflows/action_workflow_local.yml
vendored
@ -25,7 +25,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python 3.11
|
||||
uses: actions/setup-python@v3
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
|
@ -1,25 +1,27 @@
|
||||
name: 发布PYPI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "releases/**" ]
|
||||
release:
|
||||
types: [ published ]
|
||||
|
||||
jobs:
|
||||
build-and-publish:
|
||||
release:
|
||||
name: 构建并发布 jmcomic 到 PyPI
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@main
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python 3.7
|
||||
uses: actions/setup-python@v3
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.7"
|
||||
|
||||
- name: 构建模块
|
||||
run: |
|
||||
python -m pip install build
|
||||
python -m build
|
||||
- name: 发布上传
|
||||
|
||||
- name: 发布PYPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
with:
|
||||
password:
|
||||
${{ secrets.PYPI_JMCOMIC }}
|
||||
password: ${{ secrets.PYPI_JMCOMIC }}
|
@ -28,7 +28,7 @@ client_config:
|
||||
headers:
|
||||
accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
|
||||
accept-language: zh-CN,zh;q=0.9
|
||||
authority: jmcomic1.rocks
|
||||
authority: jmcomic1.group
|
||||
sec-ch-ua: '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"'
|
||||
sec-ch-ua-mobile: ?0
|
||||
sec-ch-ua-platform: '"Windows"'
|
||||
|
41
assets/config/option_test.yml
Normal file
41
assets/config/option_test.yml
Normal file
@ -0,0 +1,41 @@
|
||||
!!python/object:jmcomic.jm_option.JmOption
|
||||
dir_tree: !!python/object:jmcomic.jm_option.DownloadDirTree
|
||||
Bd: $(workspace}/download/
|
||||
flag: Bd_Id_Image
|
||||
|
||||
disable_jm_module_debug: false
|
||||
download_convert_image_suffix: null
|
||||
download_image_then_decode: true
|
||||
download_multi_thread_photo_batch_count: 3
|
||||
download_multi_thread_photo_len_limit: 5
|
||||
download_use_disk_cache: true
|
||||
|
||||
|
||||
client_config:
|
||||
# 请求失败时,重试的次数,配个正数就行
|
||||
retry_times: 10
|
||||
domain: jmcomic1.group
|
||||
postman_type: cffi
|
||||
postman_type_list:
|
||||
- requests
|
||||
- requests_Session
|
||||
- cffi
|
||||
- cffi_Session
|
||||
meta_data:
|
||||
proxies: null
|
||||
allow_redirects: true
|
||||
cookies: null
|
||||
headers:
|
||||
accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
|
||||
accept-language: zh-CN,zh;q=0.9
|
||||
authority: jmcomic1.group
|
||||
sec-ch-ua: '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"'
|
||||
sec-ch-ua-mobile: ?0
|
||||
sec-ch-ua-platform: '"Windows"'
|
||||
sec-fetch-dest: document
|
||||
sec-fetch-mode: navigate
|
||||
sec-fetch-site: none
|
||||
sec-fetch-user: ?1
|
||||
upgrade-insecure-requests: '1'
|
||||
user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML,
|
||||
like Gecko) Chrome/109.0.0.0 Safari/537.36
|
@ -2,3 +2,6 @@ commonX
|
||||
curl_cffi
|
||||
PyYAML
|
||||
Pillow
|
||||
|
||||
# for test
|
||||
pyperclip
|
@ -2,6 +2,6 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- service <--- option
|
||||
|
||||
__version__ = '1.0.0'
|
||||
__version__ = '1.1.0'
|
||||
|
||||
from .api import *
|
||||
|
@ -19,8 +19,11 @@ def download_album(jm_album_id, option=None):
|
||||
f'获得album_detail成功,准备下载。'
|
||||
f'本子作者是【{album_detail.author}】,一共有{len(album_detail)}集本子')
|
||||
|
||||
def download_photo(index, photo_detail: JmPhotoDetail, debug_topic='download_album_photo'):
|
||||
jm_client.fill_page_arr(photo_detail)
|
||||
def download_photo(index: int,
|
||||
photo_detail: JmPhotoDetail,
|
||||
debug_topic='download_album_photo',
|
||||
):
|
||||
jm_client.update(photo_detail)
|
||||
|
||||
jm_debug(debug_topic,
|
||||
f"下载第[{index + 1}]集: "
|
||||
@ -37,16 +40,16 @@ def download_album(jm_album_id, option=None):
|
||||
f"下载完成:({photo_detail.title}) "
|
||||
f"-- photo {photo_detail.photo_id}")
|
||||
|
||||
multi_thread_launcher(
|
||||
thread_pool_executor(
|
||||
iter_objs=enumerate(album_detail),
|
||||
apply_each_obj_func=download_photo,
|
||||
wait_finish=True,
|
||||
)
|
||||
|
||||
|
||||
def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
|
||||
option=None,
|
||||
wait_finish=True) -> List[Thread]:
|
||||
wait_finish=True,
|
||||
) -> List[Thread]:
|
||||
"""
|
||||
批量下载album,每个album一个线程,使用的是同一个option。
|
||||
|
||||
@ -58,7 +61,7 @@ def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
|
||||
if option is None:
|
||||
option = JmOption.default()
|
||||
|
||||
return multi_thread_launcher(
|
||||
return thread_pool_executor(
|
||||
iter_objs=((album_id, option) for album_id in jm_album_id_iter),
|
||||
apply_each_obj_func=download_album,
|
||||
wait_finish=wait_finish,
|
||||
@ -92,18 +95,17 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
|
||||
jm_client.fill_from_album(photo_detail)
|
||||
|
||||
if photo_detail.page_arr is None:
|
||||
jm_client.fill_page_arr(photo_detail)
|
||||
jm_client.update(photo_detail)
|
||||
|
||||
# 下载每个图片的函数
|
||||
def download_image(index, debug_topic='download_images_of_photo'):
|
||||
img_detail = photo_detail[index]
|
||||
def download_image(index,img_detail, debug_topic='download_images_of_photo'):
|
||||
img_save_path = option.decide_image_filepath(photo_detail, index)
|
||||
|
||||
# 已下载过,缓存命中
|
||||
if use_cache is True and file_exists(img_save_path):
|
||||
jm_debug(debug_topic, f'photo-{img_detail.aid}: '
|
||||
f'图片{img_detail.filename}已下载过,'
|
||||
f'命中磁盘缓存({img_detail.img_url})')
|
||||
f'命中磁盘缓存({img_save_path})')
|
||||
return
|
||||
|
||||
# 开始下载
|
||||
@ -121,26 +123,15 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
|
||||
if length <= option.download_multi_thread_photo_len_limit:
|
||||
# 如果图片数小的话,直接使用多线程下载,一张图一个线程。
|
||||
multi_thread_launcher(
|
||||
iter_objs=range(len(photo_detail)),
|
||||
iter_objs=enumerate(photo_detail),
|
||||
apply_each_obj_func=download_image,
|
||||
wait_finish=True
|
||||
)
|
||||
else:
|
||||
# 如果图片数多的话,还是分批下载。
|
||||
batch_count = option.download_multi_thread_photo_batch_count
|
||||
batch_times = length // batch_count
|
||||
|
||||
for i in range(batch_times):
|
||||
begin = i * batch_count
|
||||
multi_thread_launcher(
|
||||
iter_objs=range(begin, begin + batch_count),
|
||||
apply_each_obj_func=download_image,
|
||||
)
|
||||
|
||||
multi_thread_launcher(
|
||||
iter_objs=range(batch_times * batch_count,
|
||||
length),
|
||||
multi_task_launcher_batch(
|
||||
iter_objs=enumerate(photo_detail),
|
||||
apply_each_obj_func=download_image,
|
||||
batch_size=option.download_multi_thread_photo_batch_count
|
||||
)
|
||||
|
||||
|
||||
|
@ -2,7 +2,6 @@ from .jm_toolkit import *
|
||||
|
||||
|
||||
class JmcomicClient(PostmanProxy):
|
||||
debug_from_class = 'jm-client'
|
||||
retry_postman_type = RetryPostman
|
||||
|
||||
def __init__(self, postman: Postman, domain, retry_times=None):
|
||||
@ -98,11 +97,9 @@ class JmcomicClient(PostmanProxy):
|
||||
photo_detail.from_album = album_detail
|
||||
return album_detail
|
||||
|
||||
def fill_page_arr(self, photo_detail: JmPhotoDetail):
|
||||
def update(self, photo_detail: JmPhotoDetail):
|
||||
new = self.get_photo_detail(photo_detail.photo_id)
|
||||
photo_detail.page_arr = new.page_arr
|
||||
photo_detail.data_original_domain = new.data_original_domain
|
||||
return new
|
||||
photo_detail.__dict__.update(new.__dict__)
|
||||
|
||||
# -- search --
|
||||
|
||||
@ -127,7 +124,7 @@ class JmcomicClient(PostmanProxy):
|
||||
"""
|
||||
url = self.of_api_url(url) if is_api is True else url
|
||||
if is_api is True:
|
||||
self.debug("api", url)
|
||||
jm_debug("api", url)
|
||||
|
||||
resp = self.get(url, **kwargs)
|
||||
|
||||
@ -152,10 +149,6 @@ class JmcomicClient(PostmanProxy):
|
||||
def is_empty_image(cls, resp):
|
||||
return resp.status_code != 200 or len(resp.content) == 0
|
||||
|
||||
@staticmethod
|
||||
def debug(topic: str, *args, sep='', end='\n', file=None, from_class=debug_from_class):
|
||||
jm_debug(topic, *args, sep=sep, end=end, file=file, from_class=from_class)
|
||||
|
||||
@classmethod
|
||||
def img_is_not_need_to_decode(cls, data_original: str, _resp):
|
||||
return data_original.endswith('.gif')
|
||||
|
@ -46,9 +46,9 @@ class JmModuleConfig:
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
@classmethod
|
||||
def jm_debug(cls, topic: str, *args, sep='', end='\n', file=None, from_class='api'):
|
||||
def jm_debug(cls, topic: str, msg: str, from_class=None):
|
||||
if cls.enable_jm_debug is True:
|
||||
cls.debug_printer(f'【{topic}】', *args, sep=sep, end=end, file=file)
|
||||
cls.debug_printer(f'【{topic}】{msg}')
|
||||
|
||||
@classmethod
|
||||
def disable_jm_debug(cls):
|
||||
|
@ -85,11 +85,21 @@ class DownloadDirTree:
|
||||
AdditionalHandler = Callable[[Optional[JmAlbumDetail], JmPhotoDetail], str]
|
||||
additional_tree_flag_handler_mapping: Dict[int, AdditionalHandler] = {}
|
||||
|
||||
dsl_support = {
|
||||
'${workspace}': lambda text: workspace(),
|
||||
}
|
||||
|
||||
def fix_bd(self, Bd):
|
||||
for dsl_k, func in self.dsl_support.items():
|
||||
if dsl_k in Bd:
|
||||
Bd = Bd.replace(dsl_k, func(Bd))
|
||||
return fix_filepath(os.path.abspath(Bd), True)
|
||||
|
||||
def __init__(self,
|
||||
Bd: str,
|
||||
flag: Union[str, int],
|
||||
):
|
||||
self.Bd = fix_filepath(Bd)
|
||||
self.Bd = self.fix_bd(Bd)
|
||||
self.flag = self.get_flag_enum(flag)
|
||||
|
||||
def get_flag_enum(self, flag):
|
||||
@ -119,12 +129,12 @@ class DownloadDirTree:
|
||||
@param photo_dir: 章节文件夹名
|
||||
"""
|
||||
from common import fix_windir_name
|
||||
photo_dir = fix_windir_name(photo_dir)
|
||||
photo_dir = fix_windir_name(photo_dir.strip())
|
||||
|
||||
if album_dir is None:
|
||||
return f'{self.Bd}{photo_dir}/'
|
||||
|
||||
album_dir = fix_windir_name(album_dir)
|
||||
album_dir = fix_windir_name(album_dir.strip())
|
||||
return f'{self.Bd}{album_dir}/{photo_dir}/'
|
||||
|
||||
def photo_dir(flag_for_title):
|
||||
|
@ -131,7 +131,6 @@ class CdnFetchService:
|
||||
):
|
||||
self.client = client
|
||||
self.config = config
|
||||
self.debug = self.client.debug
|
||||
|
||||
def download_photo_from_cdn_directly(self,
|
||||
req: CdnRequest,
|
||||
@ -172,16 +171,16 @@ class CdnFetchService:
|
||||
if resp_info[0] is None:
|
||||
# 是,表示命中缓存,直接返回
|
||||
save_path = resp_info[2]
|
||||
self.debug('图片下载成功',
|
||||
f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})')
|
||||
jm_debug('图片下载成功',
|
||||
f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})')
|
||||
return
|
||||
|
||||
# 2. 判断是否命中缓存
|
||||
save_path = req.save_path_provider(img_url, suffix, index, decode_image)
|
||||
if use_cache is True and file_exists(save_path):
|
||||
# 命中,直接返回
|
||||
self.debug('图片下载成功',
|
||||
f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})')
|
||||
jm_debug('图片下载成功',
|
||||
f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})')
|
||||
return
|
||||
|
||||
# 3. 保存图片
|
||||
@ -195,10 +194,10 @@ class CdnFetchService:
|
||||
)
|
||||
|
||||
# 4. debug 消息
|
||||
self.debug('图片下载成功',
|
||||
f'photo-{photo_id}: {index}{suffix}下载完成 ('
|
||||
+ ('已解码' if decode_image is True else '未解码') +
|
||||
f') [{img_url}] → [{save_path}]')
|
||||
jm_debug('图片下载成功',
|
||||
f'photo-{photo_id}: {index}{suffix}下载完成 ('
|
||||
+ ('已解码' if decode_image is True else '未解码') +
|
||||
f') [{img_url}] → [{save_path}]')
|
||||
|
||||
# 调用爬虫策略
|
||||
self.config.fetch_strategy(
|
||||
@ -223,7 +222,7 @@ class CdnFetchService:
|
||||
return resp, suffix, url
|
||||
|
||||
# 下面进行重试
|
||||
client.debug(
|
||||
jm_debug(
|
||||
'图片获取重试',
|
||||
f'photo-{photo_id},图片序号为{index},url=[{url}]'
|
||||
)
|
||||
@ -233,14 +232,14 @@ class CdnFetchService:
|
||||
url = change_file_suffix(url, alter_suffix)
|
||||
resp = client.jm_get(url, False, False)
|
||||
if not client.is_empty_image(resp):
|
||||
client.debug(
|
||||
jm_debug(
|
||||
'图片获取重试 → 成功√',
|
||||
f'更改请求后缀({suffix} -> {alter_suffix}),url={url}'
|
||||
)
|
||||
return resp, alter_suffix, url
|
||||
|
||||
# 结论:可能是图片到头了
|
||||
client.debug(
|
||||
jm_debug(
|
||||
'图片获取重试 ← 失败×',
|
||||
f'更换后缀名不成功,停止爬取。'
|
||||
f'(推断本子长度<={index - 1}。当前图片序号为{index},已经到达尽头,'
|
||||
|
76
tests/test_jmcomic/__init__.py
Normal file
76
tests/test_jmcomic/__init__.py
Normal file
@ -0,0 +1,76 @@
|
||||
import sys
|
||||
import io
|
||||
import platform
|
||||
import unittest
|
||||
|
||||
# noinspection PyUnresolvedReferences
|
||||
import jmcomic
|
||||
from jmcomic import *
|
||||
|
||||
# set encoding
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, 'utf-8')
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, 'utf-8')
|
||||
|
||||
|
||||
class JmTestConfigurable(unittest.TestCase):
|
||||
option: JmOption = None
|
||||
client: JmcomicClient = None
|
||||
|
||||
def setUp(self) -> None:
|
||||
print_sep('>')
|
||||
|
||||
def tearDown(self) -> None:
|
||||
print_sep('<')
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
# 获取项目根目录
|
||||
application_workspace = os.path.abspath(os.path.dirname(__file__) + '/../..')
|
||||
|
||||
# 设置 workspace → assets/
|
||||
set_application_workspace(f'{application_workspace}/assets/')
|
||||
# 设置 实体类的save_dir → assets/download
|
||||
WorkEntity.detail_save_base_dir = workspace("/download/", is_dir=True)
|
||||
|
||||
# 设置 JmOption,JmcomicClient
|
||||
option = cls.use_option('option_test.yml')
|
||||
cls.option = option
|
||||
cls.client = option.build_jm_client()
|
||||
|
||||
# 跨平台设置
|
||||
cls.adapt_os()
|
||||
|
||||
@staticmethod
|
||||
def use_option(op_filename: str) -> JmOption:
|
||||
return create_option(workspace(f"/config/{op_filename}"))
|
||||
|
||||
@staticmethod
|
||||
def move_workspace(new_dir: str):
|
||||
set_application_workspace(workspace(f"/{new_dir}/", is_dir=True))
|
||||
|
||||
@classmethod
|
||||
def adapt_os(cls):
|
||||
adapt_func_dict = {
|
||||
'Windows': cls.adapt_win,
|
||||
'Darwin': cls.adapt_macos,
|
||||
'Linux': cls.adapt_linux,
|
||||
}
|
||||
|
||||
adapt_func_dict.get(platform.system(), lambda *args, **kwargs: None)()
|
||||
|
||||
@classmethod
|
||||
def adapt_win(cls):
|
||||
# 尝试更新 cookies
|
||||
cookies = ChromePluginCookieParser({'remember', 'comic'}) \
|
||||
.apply(when_valid_message="更新jmcomic-option成功!!!!")
|
||||
if cookies is not None:
|
||||
cls.option.client_config['meta_data']['cookies'] = cookies
|
||||
cls.option.save_to_file()
|
||||
|
||||
@classmethod
|
||||
def adapt_linux(cls):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def adapt_macos(cls):
|
||||
pass
|
29
tests/test_jmcomic/test_dir_tree.py
Normal file
29
tests/test_jmcomic/test_dir_tree.py
Normal file
@ -0,0 +1,29 @@
|
||||
from test_jmcomic import *
|
||||
|
||||
|
||||
class Test_DirTree(JmTestConfigurable):
|
||||
|
||||
def test_dir_tree(self):
|
||||
album_detail = JmAlbumDetail(None, None, 'album-title', [], 0, ['album-author'], '')
|
||||
photo_detail = JmPhotoDetail('6666', None, 'photo-title', [], '0', [], 'domain', 'photo-author', None)
|
||||
|
||||
ans = {
|
||||
'Bd_Author_Title_Image': '/album-author/photo-title/',
|
||||
'Bd_Author_Id_Image': '/album-author/6666/',
|
||||
'Bd_Title_Title_Image': '/album-title/photo-title/',
|
||||
'Bd_Title_Id_Image': '/album-title/6666/',
|
||||
'Bd_Title_Image': '/photo-title/',
|
||||
'Bd_Id_Image': '/6666/'
|
||||
}
|
||||
|
||||
Bd = workspace()
|
||||
dic = {}
|
||||
for name, flag in DownloadDirTree.accept_flag_dict().items():
|
||||
image_save_dir = DownloadDirTree.of(
|
||||
Bd, flag
|
||||
).deside_image_save_dir(album_detail, photo_detail)
|
||||
|
||||
dic[name] = image_save_dir.replace(Bd, '/')
|
||||
print(f"{name:20s}\t|\t{image_save_dir.replace(Bd, '/')}")
|
||||
|
||||
self.assertDictEqual(ans, dic)
|
62
tests/test_jmcomic/test_jm_api.py
Normal file
62
tests/test_jmcomic/test_jm_api.py
Normal file
@ -0,0 +1,62 @@
|
||||
from test_jmcomic import *
|
||||
|
||||
|
||||
class Test_Api(JmTestConfigurable):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
cls.move_workspace('download')
|
||||
|
||||
def test_download_photo_by_id(self):
|
||||
"""
|
||||
测试jmcomic模块的api的使用
|
||||
"""
|
||||
photo_id = "438516"
|
||||
jmcomic.download_photo(photo_id, self.option)
|
||||
|
||||
def test_download_album_by_id(self):
|
||||
"""
|
||||
测试jmcomic模块的api的使用
|
||||
"""
|
||||
album_id = '438516'
|
||||
jmcomic.download_album(album_id, self.option)
|
||||
|
||||
def test_batch(self):
|
||||
album_ls = str_to_list('''
|
||||
326361
|
||||
366867
|
||||
438516
|
||||
''')
|
||||
|
||||
test_cases: Iterable = [
|
||||
{e: None for e in album_ls}.keys(),
|
||||
{i: e for i, e in enumerate(album_ls)}.values(),
|
||||
set(album_ls),
|
||||
tuple(album_ls),
|
||||
album_ls,
|
||||
]
|
||||
|
||||
for case in test_cases:
|
||||
ret1 = jmcomic.download_album(case, self.option)
|
||||
self.assertEqual(len(ret1), len(album_ls), str(case))
|
||||
|
||||
ret2 = jmcomic.download_album_batch(case, self.option)
|
||||
self.assertEqual(len(ret2), len(album_ls), str(case))
|
||||
|
||||
# 测试 Generator
|
||||
ret2 = jmcomic.download_album_batch((e for e in album_ls), self.option)
|
||||
self.assertEqual(len(ret2), len(album_ls), 'Generator')
|
||||
|
||||
def test_jm_option_advice(self):
|
||||
class MyAdvice(JmOptionAdvice):
|
||||
def decide_image_filepath(self,
|
||||
option: 'JmOption',
|
||||
photo_detail: JmPhotoDetail,
|
||||
index: int,
|
||||
) -> StrNone:
|
||||
return workspace(f'{time_stamp()}_{photo_detail[index].img_file_name}.test.png')
|
||||
|
||||
option = JmOption.default()
|
||||
option.register_advice(MyAdvice())
|
||||
jmcomic.download_album('366867', option)
|
60
tests/test_jmcomic/test_jm_client.py
Normal file
60
tests/test_jmcomic/test_jm_client.py
Normal file
@ -0,0 +1,60 @@
|
||||
from test_jmcomic import *
|
||||
|
||||
|
||||
class Test_Client(JmTestConfigurable):
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
super().setUpClass()
|
||||
cls.move_workspace('download')
|
||||
|
||||
def test_download_from_cdn_directly(self):
|
||||
photo_id = 'JM438516'
|
||||
option = self.option
|
||||
cdn_crawler = option.build_cdn_crawler()
|
||||
cdn_crawler.download_photo_from_cdn_directly(
|
||||
option.build_cdn_request(photo_id),
|
||||
)
|
||||
|
||||
def test_download_image(self):
|
||||
jm_photo_id = 'JM438516'
|
||||
photo_detail = self.client.get_photo_detail(jm_photo_id)
|
||||
self.client.download_by_image_detail(
|
||||
photo_detail[0],
|
||||
img_save_path=workspace('test_download_image.png')
|
||||
)
|
||||
|
||||
def test_get_album_detail_by_jm_photo_id(self):
|
||||
album_id = "JM438516"
|
||||
print_obj_dict(self.client.get_album_detail(album_id))
|
||||
|
||||
def test_get_photo_detail_by_jm_photo_id(self):
|
||||
"""
|
||||
测试通过 JmcomicClient 和 jm_photo_id 获取 JmPhotoDetail对象
|
||||
"""
|
||||
jm_photo_id = 'JM438516'
|
||||
photo_detail = self.client.get_photo_detail(jm_photo_id)
|
||||
photo_detail.when_del_save_file = True
|
||||
photo_detail.after_save_print_info = True
|
||||
del photo_detail
|
||||
|
||||
def test_multi_album_and_single_album(self):
|
||||
multi_photo_album_id = [
|
||||
"195822",
|
||||
]
|
||||
|
||||
for album_id in multi_photo_album_id:
|
||||
album_detail: JmAlbumDetail = self.client.get_album_detail(album_id)
|
||||
print(f'本子: [{album_detail.title}] 一共有{album_detail.page_count}页图')
|
||||
|
||||
def test_search(self):
|
||||
jm_search_page = self.client.search_album('MANA')
|
||||
for album_id in jm_search_page.album_id_iter():
|
||||
print(album_id)
|
||||
|
||||
def test_gt_300_photo(self):
|
||||
photo_id = '147643'
|
||||
photo_detail: JmPhotoDetail = self.client.get_photo_detail(photo_id)
|
||||
image = photo_detail[3000]
|
||||
print(image.img_url)
|
||||
self.client.download_by_image_detail(image, workspace('3000.png'))
|
16
tests/test_jmcomic/test_jm_search.py
Normal file
16
tests/test_jmcomic/test_jm_search.py
Normal file
@ -0,0 +1,16 @@
|
||||
from test_jmcomic import *
|
||||
|
||||
|
||||
class Test_Search(JmTestConfigurable):
|
||||
|
||||
def test_analyse_search_html(self):
|
||||
search_album: JmSearchPage = self.client.search_album('MANA')
|
||||
for i, info in enumerate(search_album):
|
||||
# info: (album_id, title, category_none, label_sub_none, tag_list)
|
||||
print(f'index: {i}--------------------------------------------------------\n'
|
||||
f'id: {info[0]}\n'
|
||||
f'title: {info[1]}\n'
|
||||
f'category_none: {info[2]}\n'
|
||||
f'label_sub_none: {info[3]}\n'
|
||||
f'tag_list: {info[4]}\n\n'
|
||||
)
|
Loading…
Reference in New Issue
Block a user