v1.1.0: 优化debug日志,使用线程池调整下载调度。 (#5)

This commit is contained in:
hect0x7 2023-04-04 00:13:49 +08:00 committed by GitHub
parent 5a370e36cf
commit 2e29d53631
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 399 additions and 64 deletions

53
.github/workflows/action_run_tests.yml vendored Normal file
View File

@ -0,0 +1,53 @@
name: 跑测试
on:
push:
branches: [ "dev", "master" ]
paths:
- 'src/**/*.py' # 源码
- 'tests/**/*.py' # 测试代码
jobs:
test: # This code is based on https://github.com/gaogaotiantian/viztracer/blob/master/.github/workflows/python-package.yml
strategy:
matrix:
python-version: [ 3.7, 3.8, 3.9, "3.10", "3.11" ]
os: [ ubuntu-latest, macos-latest ]
runs-on: ${{ matrix.os }}
timeout-minutes: 5
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: matrix.os != 'windows-latest'
run: |
python -m pip install --upgrade pip
if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
- name: Install dependencies on Windows
if: matrix.os == 'windows-latest'
run: |
python -m pip install --upgrade pip
if (Test-Path -Path '.\requirements-dev.txt' -PathType Leaf) {pip install -r requirements-dev.txt}
- name: Install local
run: |
pip install -e ./
- name: 跑测试
if: matrix.os != 'windows-latest'
run: |
cd ./tests/
python -m unittest
- name: 跑测试Windows系统
if: matrix.os == 'windows-latest'
run: |
python -m unittest discover -s ./tests -p "test_*.py"

View File

@ -19,7 +19,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.11
uses: actions/setup-python@v3
uses: actions/setup-python@v4
with:
python-version: "3.11"

View File

@ -25,7 +25,7 @@ jobs:
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.11
uses: actions/setup-python@v3
uses: actions/setup-python@v4
with:
python-version: "3.11"

View File

@ -1,25 +1,27 @@
name: 发布PYPI
on:
push:
branches: [ "releases/**" ]
release:
types: [ published ]
jobs:
build-and-publish:
release:
name: 构建并发布 jmcomic 到 PyPI
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@main
- uses: actions/checkout@v3
- name: Set up Python 3.7
uses: actions/setup-python@v3
uses: actions/setup-python@v4
with:
python-version: "3.7"
- name: 构建模块
run: |
python -m pip install build
python -m build
- name: 发布上传
- name: 发布PYPI
uses: pypa/gh-action-pypi-publish@release/v1
with:
password:
${{ secrets.PYPI_JMCOMIC }}
password: ${{ secrets.PYPI_JMCOMIC }}

View File

@ -28,7 +28,7 @@ client_config:
headers:
accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
accept-language: zh-CN,zh;q=0.9
authority: jmcomic1.rocks
authority: jmcomic1.group
sec-ch-ua: '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"'
sec-ch-ua-mobile: ?0
sec-ch-ua-platform: '"Windows"'

View File

@ -0,0 +1,41 @@
!!python/object:jmcomic.jm_option.JmOption
dir_tree: !!python/object:jmcomic.jm_option.DownloadDirTree
Bd: $(workspace}/download/
flag: Bd_Id_Image
disable_jm_module_debug: false
download_convert_image_suffix: null
download_image_then_decode: true
download_multi_thread_photo_batch_count: 3
download_multi_thread_photo_len_limit: 5
download_use_disk_cache: true
client_config:
# 请求失败时,重试的次数,配个正数就行
retry_times: 10
domain: jmcomic1.group
postman_type: cffi
postman_type_list:
- requests
- requests_Session
- cffi
- cffi_Session
meta_data:
proxies: null
allow_redirects: true
cookies: null
headers:
accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9
accept-language: zh-CN,zh;q=0.9
authority: jmcomic1.group
sec-ch-ua: '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"'
sec-ch-ua-mobile: ?0
sec-ch-ua-platform: '"Windows"'
sec-fetch-dest: document
sec-fetch-mode: navigate
sec-fetch-site: none
sec-fetch-user: ?1
upgrade-insecure-requests: '1'
user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML,
like Gecko) Chrome/109.0.0.0 Safari/537.36

View File

@ -2,3 +2,6 @@ commonX
curl_cffi
PyYAML
Pillow
# for test
pyperclip

View File

@ -2,6 +2,6 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- service <--- option
__version__ = '1.0.0'
__version__ = '1.1.0'
from .api import *

View File

@ -19,8 +19,11 @@ def download_album(jm_album_id, option=None):
f'获得album_detail成功准备下载。'
f'本子作者是【{album_detail.author}】,一共有{len(album_detail)}集本子')
def download_photo(index, photo_detail: JmPhotoDetail, debug_topic='download_album_photo'):
jm_client.fill_page_arr(photo_detail)
def download_photo(index: int,
photo_detail: JmPhotoDetail,
debug_topic='download_album_photo',
):
jm_client.update(photo_detail)
jm_debug(debug_topic,
f"下载第[{index + 1}]集: "
@ -37,16 +40,16 @@ def download_album(jm_album_id, option=None):
f"下载完成:({photo_detail.title}) "
f"-- photo {photo_detail.photo_id}")
multi_thread_launcher(
thread_pool_executor(
iter_objs=enumerate(album_detail),
apply_each_obj_func=download_photo,
wait_finish=True,
)
def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
option=None,
wait_finish=True) -> List[Thread]:
wait_finish=True,
) -> List[Thread]:
"""
批量下载album每个album一个线程使用的是同一个option
@ -58,7 +61,7 @@ def download_album_batch(jm_album_id_iter: Union[Iterable, Generator],
if option is None:
option = JmOption.default()
return multi_thread_launcher(
return thread_pool_executor(
iter_objs=((album_id, option) for album_id in jm_album_id_iter),
apply_each_obj_func=download_album,
wait_finish=wait_finish,
@ -92,18 +95,17 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
jm_client.fill_from_album(photo_detail)
if photo_detail.page_arr is None:
jm_client.fill_page_arr(photo_detail)
jm_client.update(photo_detail)
# 下载每个图片的函数
def download_image(index, debug_topic='download_images_of_photo'):
img_detail = photo_detail[index]
def download_image(index,img_detail, debug_topic='download_images_of_photo'):
img_save_path = option.decide_image_filepath(photo_detail, index)
# 已下载过,缓存命中
if use_cache is True and file_exists(img_save_path):
jm_debug(debug_topic, f'photo-{img_detail.aid}: '
f'图片{img_detail.filename}已下载过,'
f'命中磁盘缓存({img_detail.img_url}')
f'命中磁盘缓存({img_save_path}')
return
# 开始下载
@ -121,26 +123,15 @@ def download_by_photo_detail(photo_detail: JmPhotoDetail,
if length <= option.download_multi_thread_photo_len_limit:
# 如果图片数小的话,直接使用多线程下载,一张图一个线程。
multi_thread_launcher(
iter_objs=range(len(photo_detail)),
iter_objs=enumerate(photo_detail),
apply_each_obj_func=download_image,
wait_finish=True
)
else:
# 如果图片数多的话,还是分批下载。
batch_count = option.download_multi_thread_photo_batch_count
batch_times = length // batch_count
for i in range(batch_times):
begin = i * batch_count
multi_thread_launcher(
iter_objs=range(begin, begin + batch_count),
apply_each_obj_func=download_image,
)
multi_thread_launcher(
iter_objs=range(batch_times * batch_count,
length),
multi_task_launcher_batch(
iter_objs=enumerate(photo_detail),
apply_each_obj_func=download_image,
batch_size=option.download_multi_thread_photo_batch_count
)

View File

@ -2,7 +2,6 @@ from .jm_toolkit import *
class JmcomicClient(PostmanProxy):
debug_from_class = 'jm-client'
retry_postman_type = RetryPostman
def __init__(self, postman: Postman, domain, retry_times=None):
@ -98,11 +97,9 @@ class JmcomicClient(PostmanProxy):
photo_detail.from_album = album_detail
return album_detail
def fill_page_arr(self, photo_detail: JmPhotoDetail):
def update(self, photo_detail: JmPhotoDetail):
new = self.get_photo_detail(photo_detail.photo_id)
photo_detail.page_arr = new.page_arr
photo_detail.data_original_domain = new.data_original_domain
return new
photo_detail.__dict__.update(new.__dict__)
# -- search --
@ -127,7 +124,7 @@ class JmcomicClient(PostmanProxy):
"""
url = self.of_api_url(url) if is_api is True else url
if is_api is True:
self.debug("api", url)
jm_debug("api", url)
resp = self.get(url, **kwargs)
@ -152,10 +149,6 @@ class JmcomicClient(PostmanProxy):
def is_empty_image(cls, resp):
return resp.status_code != 200 or len(resp.content) == 0
@staticmethod
def debug(topic: str, *args, sep='', end='\n', file=None, from_class=debug_from_class):
jm_debug(topic, *args, sep=sep, end=end, file=file, from_class=from_class)
@classmethod
def img_is_not_need_to_decode(cls, data_original: str, _resp):
return data_original.endswith('.gif')

View File

@ -46,9 +46,9 @@ class JmModuleConfig:
# noinspection PyUnusedLocal
@classmethod
def jm_debug(cls, topic: str, *args, sep='', end='\n', file=None, from_class='api'):
def jm_debug(cls, topic: str, msg: str, from_class=None):
if cls.enable_jm_debug is True:
cls.debug_printer(f'{topic}', *args, sep=sep, end=end, file=file)
cls.debug_printer(f'{topic}{msg}')
@classmethod
def disable_jm_debug(cls):

View File

@ -85,11 +85,21 @@ class DownloadDirTree:
AdditionalHandler = Callable[[Optional[JmAlbumDetail], JmPhotoDetail], str]
additional_tree_flag_handler_mapping: Dict[int, AdditionalHandler] = {}
dsl_support = {
'${workspace}': lambda text: workspace(),
}
def fix_bd(self, Bd):
for dsl_k, func in self.dsl_support.items():
if dsl_k in Bd:
Bd = Bd.replace(dsl_k, func(Bd))
return fix_filepath(os.path.abspath(Bd), True)
def __init__(self,
Bd: str,
flag: Union[str, int],
):
self.Bd = fix_filepath(Bd)
self.Bd = self.fix_bd(Bd)
self.flag = self.get_flag_enum(flag)
def get_flag_enum(self, flag):
@ -119,12 +129,12 @@ class DownloadDirTree:
@param photo_dir: 章节文件夹名
"""
from common import fix_windir_name
photo_dir = fix_windir_name(photo_dir)
photo_dir = fix_windir_name(photo_dir.strip())
if album_dir is None:
return f'{self.Bd}{photo_dir}/'
album_dir = fix_windir_name(album_dir)
album_dir = fix_windir_name(album_dir.strip())
return f'{self.Bd}{album_dir}/{photo_dir}/'
def photo_dir(flag_for_title):

View File

@ -131,7 +131,6 @@ class CdnFetchService:
):
self.client = client
self.config = config
self.debug = self.client.debug
def download_photo_from_cdn_directly(self,
req: CdnRequest,
@ -172,16 +171,16 @@ class CdnFetchService:
if resp_info[0] is None:
# 是,表示命中缓存,直接返回
save_path = resp_info[2]
self.debug('图片下载成功',
f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})')
jm_debug('图片下载成功',
f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})')
return
# 2. 判断是否命中缓存
save_path = req.save_path_provider(img_url, suffix, index, decode_image)
if use_cache is True and file_exists(save_path):
# 命中,直接返回
self.debug('图片下载成功',
f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})')
jm_debug('图片下载成功',
f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})')
return
# 3. 保存图片
@ -195,10 +194,10 @@ class CdnFetchService:
)
# 4. debug 消息
self.debug('图片下载成功',
f'photo-{photo_id}: {index}{suffix}下载完成 ('
+ ('已解码' if decode_image is True else '未解码') +
f') [{img_url}] → [{save_path}]')
jm_debug('图片下载成功',
f'photo-{photo_id}: {index}{suffix}下载完成 ('
+ ('已解码' if decode_image is True else '未解码') +
f') [{img_url}] → [{save_path}]')
# 调用爬虫策略
self.config.fetch_strategy(
@ -223,7 +222,7 @@ class CdnFetchService:
return resp, suffix, url
# 下面进行重试
client.debug(
jm_debug(
'图片获取重试',
f'photo-{photo_id},图片序号为{index}url=[{url}]'
)
@ -233,14 +232,14 @@ class CdnFetchService:
url = change_file_suffix(url, alter_suffix)
resp = client.jm_get(url, False, False)
if not client.is_empty_image(resp):
client.debug(
jm_debug(
'图片获取重试 → 成功√',
f'更改请求后缀({suffix} -> {alter_suffix}url={url}'
)
return resp, alter_suffix, url
# 结论:可能是图片到头了
client.debug(
jm_debug(
'图片获取重试 ← 失败×',
f'更换后缀名不成功,停止爬取。'
f'(推断本子长度<={index - 1}。当前图片序号为{index},已经到达尽头,'

View File

@ -0,0 +1,76 @@
import sys
import io
import platform
import unittest
# noinspection PyUnresolvedReferences
import jmcomic
from jmcomic import *
# set encoding
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, 'utf-8')
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, 'utf-8')
class JmTestConfigurable(unittest.TestCase):
option: JmOption = None
client: JmcomicClient = None
def setUp(self) -> None:
print_sep('>')
def tearDown(self) -> None:
print_sep('<')
@classmethod
def setUpClass(cls):
# 获取项目根目录
application_workspace = os.path.abspath(os.path.dirname(__file__) + '/../..')
# 设置 workspace → assets/
set_application_workspace(f'{application_workspace}/assets/')
# 设置 实体类的save_dir → assets/download
WorkEntity.detail_save_base_dir = workspace("/download/", is_dir=True)
# 设置 JmOptionJmcomicClient
option = cls.use_option('option_test.yml')
cls.option = option
cls.client = option.build_jm_client()
# 跨平台设置
cls.adapt_os()
@staticmethod
def use_option(op_filename: str) -> JmOption:
return create_option(workspace(f"/config/{op_filename}"))
@staticmethod
def move_workspace(new_dir: str):
set_application_workspace(workspace(f"/{new_dir}/", is_dir=True))
@classmethod
def adapt_os(cls):
adapt_func_dict = {
'Windows': cls.adapt_win,
'Darwin': cls.adapt_macos,
'Linux': cls.adapt_linux,
}
adapt_func_dict.get(platform.system(), lambda *args, **kwargs: None)()
@classmethod
def adapt_win(cls):
# 尝试更新 cookies
cookies = ChromePluginCookieParser({'remember', 'comic'}) \
.apply(when_valid_message="更新jmcomic-option成功")
if cookies is not None:
cls.option.client_config['meta_data']['cookies'] = cookies
cls.option.save_to_file()
@classmethod
def adapt_linux(cls):
pass
@classmethod
def adapt_macos(cls):
pass

View File

@ -0,0 +1,29 @@
from test_jmcomic import *
class Test_DirTree(JmTestConfigurable):
def test_dir_tree(self):
album_detail = JmAlbumDetail(None, None, 'album-title', [], 0, ['album-author'], '')
photo_detail = JmPhotoDetail('6666', None, 'photo-title', [], '0', [], 'domain', 'photo-author', None)
ans = {
'Bd_Author_Title_Image': '/album-author/photo-title/',
'Bd_Author_Id_Image': '/album-author/6666/',
'Bd_Title_Title_Image': '/album-title/photo-title/',
'Bd_Title_Id_Image': '/album-title/6666/',
'Bd_Title_Image': '/photo-title/',
'Bd_Id_Image': '/6666/'
}
Bd = workspace()
dic = {}
for name, flag in DownloadDirTree.accept_flag_dict().items():
image_save_dir = DownloadDirTree.of(
Bd, flag
).deside_image_save_dir(album_detail, photo_detail)
dic[name] = image_save_dir.replace(Bd, '/')
print(f"{name:20s}\t|\t{image_save_dir.replace(Bd, '/')}")
self.assertDictEqual(ans, dic)

View File

@ -0,0 +1,62 @@
from test_jmcomic import *
class Test_Api(JmTestConfigurable):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.move_workspace('download')
def test_download_photo_by_id(self):
"""
测试jmcomic模块的api的使用
"""
photo_id = "438516"
jmcomic.download_photo(photo_id, self.option)
def test_download_album_by_id(self):
"""
测试jmcomic模块的api的使用
"""
album_id = '438516'
jmcomic.download_album(album_id, self.option)
def test_batch(self):
album_ls = str_to_list('''
326361
366867
438516
''')
test_cases: Iterable = [
{e: None for e in album_ls}.keys(),
{i: e for i, e in enumerate(album_ls)}.values(),
set(album_ls),
tuple(album_ls),
album_ls,
]
for case in test_cases:
ret1 = jmcomic.download_album(case, self.option)
self.assertEqual(len(ret1), len(album_ls), str(case))
ret2 = jmcomic.download_album_batch(case, self.option)
self.assertEqual(len(ret2), len(album_ls), str(case))
# 测试 Generator
ret2 = jmcomic.download_album_batch((e for e in album_ls), self.option)
self.assertEqual(len(ret2), len(album_ls), 'Generator')
def test_jm_option_advice(self):
class MyAdvice(JmOptionAdvice):
def decide_image_filepath(self,
option: 'JmOption',
photo_detail: JmPhotoDetail,
index: int,
) -> StrNone:
return workspace(f'{time_stamp()}_{photo_detail[index].img_file_name}.test.png')
option = JmOption.default()
option.register_advice(MyAdvice())
jmcomic.download_album('366867', option)

View File

@ -0,0 +1,60 @@
from test_jmcomic import *
class Test_Client(JmTestConfigurable):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.move_workspace('download')
def test_download_from_cdn_directly(self):
photo_id = 'JM438516'
option = self.option
cdn_crawler = option.build_cdn_crawler()
cdn_crawler.download_photo_from_cdn_directly(
option.build_cdn_request(photo_id),
)
def test_download_image(self):
jm_photo_id = 'JM438516'
photo_detail = self.client.get_photo_detail(jm_photo_id)
self.client.download_by_image_detail(
photo_detail[0],
img_save_path=workspace('test_download_image.png')
)
def test_get_album_detail_by_jm_photo_id(self):
album_id = "JM438516"
print_obj_dict(self.client.get_album_detail(album_id))
def test_get_photo_detail_by_jm_photo_id(self):
"""
测试通过 JmcomicClient jm_photo_id 获取 JmPhotoDetail对象
"""
jm_photo_id = 'JM438516'
photo_detail = self.client.get_photo_detail(jm_photo_id)
photo_detail.when_del_save_file = True
photo_detail.after_save_print_info = True
del photo_detail
def test_multi_album_and_single_album(self):
multi_photo_album_id = [
"195822",
]
for album_id in multi_photo_album_id:
album_detail: JmAlbumDetail = self.client.get_album_detail(album_id)
print(f'本子: [{album_detail.title}] 一共有{album_detail.page_count}页图')
def test_search(self):
jm_search_page = self.client.search_album('MANA')
for album_id in jm_search_page.album_id_iter():
print(album_id)
def test_gt_300_photo(self):
photo_id = '147643'
photo_detail: JmPhotoDetail = self.client.get_photo_detail(photo_id)
image = photo_detail[3000]
print(image.img_url)
self.client.download_by_image_detail(image, workspace('3000.png'))

View File

@ -0,0 +1,16 @@
from test_jmcomic import *
class Test_Search(JmTestConfigurable):
def test_analyse_search_html(self):
search_album: JmSearchPage = self.client.search_album('MANA')
for i, info in enumerate(search_album):
# info: (album_id, title, category_none, label_sub_none, tag_list)
print(f'index: {i}--------------------------------------------------------\n'
f'id: {info[0]}\n'
f'title: {info[1]}\n'
f'category_none: {info[2]}\n'
f'label_sub_none: {info[3]}\n'
f'tag_list: {info[4]}\n\n'
)