commit 1df7e64f8c68dffb8fb39f813a0defdb0240c8b1 Author: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Sun Apr 2 22:19:44 2023 +0800 init commit diff --git a/.github/workflows/action_workflow.yml b/.github/workflows/action_workflow.yml new file mode 100644 index 0000000..156fc3f --- /dev/null +++ b/.github/workflows/action_workflow.yml @@ -0,0 +1,45 @@ +name: 下载JM本子 + +on: + push: + branches: [ "workflow" ] + paths: + - 'usage/**/*.py' # 工作流脚本 + - 'assets/config/*.yml' # 工作流配置 + pull_request: + branches: [ "workflow" ] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: 安装依赖项 + run: | + python -m pip install --upgrade pip + pip install jmcomic -i https://pypi.org/project --upgrade + + - name: 运行下载脚本 + run: | + cd ./usage/ + python jmcomic_workflows.py + + - name: 上传结果 + uses: actions/upload-artifact@v3.1.2 + with: + # Artifact name + name: 下载完成的本子 + # A file, directory or wildcard pattern that describes what to upload + path: /home/runner/work/jmcomic/download/ + # The desired behavior if no files are found using the provided path. + if-no-files-found: warn + # Duration after which artifact will expire in days. 0 means using default retention.Minimum 1 day. Maximum 90 days unless changed from the repository settings page. + retention-days: 90 \ No newline at end of file diff --git a/.github/workflows/action_workflow_local.yml b/.github/workflows/action_workflow_local.yml new file mode 100644 index 0000000..da27a21 --- /dev/null +++ b/.github/workflows/action_workflow_local.yml @@ -0,0 +1,51 @@ +# 这个工作流使用workflow_local分支的jmcomic模块。 +# 你可以在workflow_local分支修改jmcomic的源码, +# 这样工作流就会使用你修改后的代码来下载禁漫本子。 + +name: 下载JM本子 (Local) + +on: + push: + branches: [ "workflow_local" ] + paths: + - 'src/**/*.py' # 源码 + - 'usage/**/*.py' # 工作流脚本 + - 'assets/config/*.yml' # 工作流配置 + pull_request: + branches: [ "workflow_local" ] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + + - name: 安装依赖项 + run: | + python -m pip install --upgrade pip + pip install commonX -i https://pypi.org/project --upgrade + pip install -e ./ + + - name: 运行下载脚本 + run: | + cd ./usage/ + python jmcomic_workflows.py + + - name: 上传结果 + uses: actions/upload-artifact@v3.1.2 + with: + # Artifact name + name: 下载完成的本子 + # A file, directory or wildcard pattern that describes what to upload + path: /home/runner/work/jmcomic/download/ + # The desired behavior if no files are found using the provided path. + if-no-files-found: warn + # Duration after which artifact will expire in days. 0 means using default retention.Minimum 1 day. Maximum 90 days unless changed from the repository settings page. + retention-days: 90 diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml new file mode 100644 index 0000000..ad71de2 --- /dev/null +++ b/.github/workflows/publish_pypi.yml @@ -0,0 +1,25 @@ +name: 发布PYPI + +on: + push: + branches: [ "releases/**" ] + +jobs: + build-and-publish: + name: 构建并发布 jmcomic 到 PyPI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@main + - name: Set up Python 3.7 + uses: actions/setup-python@v3 + with: + python-version: "3.7" + - name: 构建模块 + run: | + python -m pip install build + python -m build + - name: 发布上传 + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: + ${{ secrets.PYPI_JMCOMIC }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5990fab --- /dev/null +++ b/.gitignore @@ -0,0 +1,164 @@ +# this repo +/assets/download/ + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..158a232 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 hect0x7 + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..0876477 --- /dev/null +++ b/README.md @@ -0,0 +1,72 @@ +# Python API For JMComic (禁漫天堂) + +封装了一套可用于爬取JM的Python API. + +简单来说,就是可以通过简单的几行Python代码,实现下载JM上的本子到本地,并且是处理好的图片. + +**友情提示:珍爱JM,为了减轻JM的服务器压力,请不要一次性爬取太多本子,西门🙏🙏🙏**. + +## 安装教程 + +* 通过pip官方源安装(推荐) + + ```shell + pip install jmcomic -i https://pypi.org/project --upgrade + ``` +* 本地安装 + + ```shell + cd ./modules/core/ + pip install -e ./ + ``` + +## 快速上手 + +使用下面的两行代码,即可实现功能:把某个本子集(album)里的所有本子(photo)下载到本地 + +```python +import jmcomic # 导入此模块,需要先安装. +jmcomic.download_album('422866') # 传入要下载的album的id,即可下载整个album到本地. +# 上面的这行代码,还有一个可选参数option: JmOption,表示配置项, +# 配置项的作用是告诉程序下载时候的一些选择, +# 比如,要下载到哪个文件夹,使用怎样的路径组织方式(比如[/作者/本子id/图片] 或者 [/作者/本子名称/图片]). +# 如果没有配置,则会使用 JmOption.default(),下载的路径是[当前工作文件夹/本子名称/图片]. +``` + +进一步的使用可以参考usage文件夹下的示例代码: `jmcomic_getting_started.py` `jmcomic_usage.py` + +## 项目特点 + +- **绕过Cloudflare的反爬虫** +- 支持使用**Github Action**下载漫画,不会编程都能用([教程:使用Github Actions下载禁漫本子](./assets/docs/教程:使用Github%20Actions下载禁漫本子.md)) +- 可配置性强 + + - 不配置也能使用,十分方便 + - 配置可以从**配置文件**生成,无需写Python代码 + - 配置点有:`是否使用磁盘缓存` `是否使用代理` `图片类型转换` `本子下载路径` `请求元信息(headers,cookies,重试次数)等 ` +- 多线程下载(可细化到一图一线程,效率极高) +- 跟进了JM最新的图片分割算法(2023-02-08) + +## 使用小说明 + +* Python >= 3.7 +* 项目只有代码注释,没有API文档。因此想深入高级地使用,自行看源码和改造代码叭 ^^_ +* JM不是前后端分离的网站,因此本api是通过正则表达式解析HTML网页的信息(详见`JmcomicText`),进而实现的下载图片。 + +## 项目文件夹介绍 + +* assets:存放一些非代码的资源文件 + + * config:存放配置文件 + * docs:项目文档 +* src:存放源代码 + + * jmcomic:`jmcomic`模块 +* tests:测试目录,存放测试代码,使用unittest +* usage:用法目录,存放示例/使用代码 + +## 感谢以下项目 + +### 图片分割算法的来源 + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=tonquer&repo=JMComic-qt)](https://github.com/tonquer/JMComic-qt) diff --git a/assets/config/option.yml b/assets/config/option.yml new file mode 100644 index 0000000..cf01acf --- /dev/null +++ b/assets/config/option.yml @@ -0,0 +1,41 @@ +!!python/object:jmcomic.jm_option.JmOption +dir_tree: !!python/object:jmcomic.jm_option.DownloadDirTree + Bd: D:/jmcomic/download/ + flag: Bd_Title_Image + +disable_jm_module_debug: false +download_convert_image_suffix: .jpg +download_image_then_decode: true +download_multi_thread_photo_batch_count: 10 # 一次多线程批量下载的数量,太多了禁漫服务器可能顶不住 +download_multi_thread_photo_len_limit: 30 +download_use_disk_cache: true +filepath: null + +client_config: + domain: jmcomic1.group + # 请求失败时,重试的次数,配个正数就行,不配默认是5 + retry_times: 5 + postman_type: cffi + postman_type_list: + - requests + - requests_Session + - cffi + - cffi_Session + meta_data: + proxies: null + allow_redirects: true + cookies: null + headers: + accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 + accept-language: zh-CN,zh;q=0.9 + authority: jmcomic1.rocks + sec-ch-ua: '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"' + sec-ch-ua-mobile: ?0 + sec-ch-ua-platform: '"Windows"' + sec-fetch-dest: document + sec-fetch-mode: navigate + sec-fetch-site: none + sec-fetch-user: ?1 + upgrade-insecure-requests: '1' + user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, + like Gecko) Chrome/109.0.0.0 Safari/537.36 diff --git a/assets/config/option_use_proxy.yml b/assets/config/option_use_proxy.yml new file mode 100644 index 0000000..7994c66 --- /dev/null +++ b/assets/config/option_use_proxy.yml @@ -0,0 +1,56 @@ +!!python/object:jmcomic.jm_option.JmOption +dir_tree: !!python/object:jmcomic.jm_option.DownloadDirTree + Bd: D:/jmcomic/download/ + flag: Bd_Title_Image + +disable_jm_module_debug: false +download_convert_image_suffix: .png +download_image_then_decode: true +download_multi_thread_photo_batch_count: 10 # 一次多线程批量下载的数量,太多了禁漫服务器可能顶不住 +download_multi_thread_photo_len_limit: 30 +download_use_disk_cache: true +filepath: null + +client_config: + domain: 18comic.vip + # 请求失败时,重试的次数,配个正数就行,不配默认是5 + retry_times: 5 + postman_type: cffi + postman_type_list: + - requests + - requests_Session + - cffi + - cffi_Session + meta_data: + # 以下proxies的写法都可以: + # proxies: clash + # proxies: v2ray + # proxies: 127.0.0.1:7890 + # proxies: + # http: http://127.0.0.1:7890 + # https: https://127.0.0.1:7890 + proxies: clash + # 允许重定向 + allow_redirects: true + headers: + accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7 + accept-language: zh-CN,zh;q=0.9 + authority: 18comic.vip + cache-control: no-cache + pragma: no-cache + sec-ch-ua: '"Google Chrome";v="111", "Not(A:Brand";v="8", "Chromium";v="111"' + sec-ch-ua-mobile: ?0 + sec-ch-ua-platform: '"Windows"' + sec-fetch-dest: document + sec-fetch-mode: navigate + sec-fetch-site: none + sec-fetch-user: ?1 + upgrade-insecure-requests: '1' + user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, + like Gecko) Chrome/111.0.0.0 Safari/537.36 + # cookies为null,作用是以非登录状态访问JM + cookies: null + # 但如果你要下载的是需要登录才能看的漫画,你需要配置cookies的AVS的值,如下。 +# cookies: +# AVS: 你的AVS + diff --git a/assets/config/workflow_option.yml b/assets/config/workflow_option.yml new file mode 100644 index 0000000..d7d0732 --- /dev/null +++ b/assets/config/workflow_option.yml @@ -0,0 +1,39 @@ +!!python/object:jmcomic.jm_option.JmOption +dir_tree: !!python/object:jmcomic.jm_option.DownloadDirTree + Bd: /home/runner/work/jmcomic/download/ + flag: Bd_Author_Title_Image + +disable_jm_module_debug: false +download_convert_image_suffix: null +download_image_then_decode: true +download_multi_thread_photo_batch_count: 10 # 一次多线程批量下载的数量,太多了禁漫服务器可能顶不住 +download_multi_thread_photo_len_limit: 30 +download_use_disk_cache: true + +client_config: + domain: null # null表明动态获取最新可用的禁漫域名 + retry_times: 5 # 请求失败时,重试的次数,配个正数就行,不配默认是5 + postman_type: cffi + postman_type_list: + - requests + - requests_Session + - cffi + - cffi_Session + meta_data: + proxies: null + allow_redirects: true + cookies: null + headers: + accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 + accept-language: zh-CN,zh;q=0.9 + authority: jmcomic1.rocks + sec-ch-ua: '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"' + sec-ch-ua-mobile: ?0 + sec-ch-ua-platform: '"Windows"' + sec-fetch-dest: document + sec-fetch-mode: navigate + sec-fetch-site: none + sec-fetch-user: ?1 + upgrade-insecure-requests: '1' + user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, + like Gecko) Chrome/109.0.0.0 Safari/537.36 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..7ab1dd4 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +commonX +curl_cffi +PyYAML +Pillow diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..0c9a99f --- /dev/null +++ b/setup.py @@ -0,0 +1,48 @@ +from setuptools import setup, find_packages + +with open("README.md", encoding='utf-8') as f: + long_description = f.read() + +version = None +with open('./src/jmcomic/__init__.py', encoding='utf-8') as f: + for line in f: + if '__version__' in line: + version = line[line.index("'") + 1: line.rindex("'")] + break + +if version is None: + print('Set version first!') + exit(1) + +setup( + name='jmcomic', + version=version, + description='Python API For JMComic (禁漫天堂)', + long_description_content_type="text/markdown", + long_description=long_description, + url='https://github.com/hect0x7/JMComic-Crawler-Python', + author='hect0x7', + author_email='93357912+hect0x7@users.noreply.github.com', + packages=find_packages("src"), + package_dir={"": "src"}, + python_requires=">=3.7", + install_requires=[ + 'commonX', + 'curl_cffi', + 'PyYAML', + 'Pillow', + ], + keywords=['python', 'jmcomic', '18comic', '禁漫天堂', 'NSFW'], + classifiers=[ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Operating System :: MacOS", + "Operating System :: POSIX :: Linux", + "Operating System :: Microsoft :: Windows", + ], +) diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py new file mode 100644 index 0000000..71f3045 --- /dev/null +++ b/src/jmcomic/__init__.py @@ -0,0 +1,7 @@ +# 模块依赖关系如下: +# 被依赖方 <--- 使用方 +# config <--- entity <--- toolkit <--- client <--- service <--- option + +__version__ = '1.0.0' + +from .api import * diff --git a/src/jmcomic/api.py b/src/jmcomic/api.py new file mode 100644 index 0000000..509f911 --- /dev/null +++ b/src/jmcomic/api.py @@ -0,0 +1,180 @@ +from .jm_option import * + + +def download_album(jm_album_id, option=None): + """ + 下载一个本子集,入口api + @param jm_album_id: 禁漫的本子的id,类型可以是str/int/iterable[str]。 + 如果是iterable[str],则会调用批量下载方法 download_album_batch + @param option: 下载选项,为空默认是 JmOption.default() + """ + + if not isinstance(jm_album_id, (str, int)): + return download_album_batch(jm_album_id, option) + + option, jm_client = build_client(option) + album_detail: JmAlbumDetail = jm_client.get_album_detail(jm_album_id) + + jm_debug('download_album', + f'获得album_detail成功,准备下载。' + f'本子作者是【{album_detail.author}】,一共有{len(album_detail)}集本子') + + def download_photo(index, photo_detail: JmPhotoDetail, debug_topic='download_album_photo'): + jm_client.fill_page_arr(photo_detail) + + jm_debug(debug_topic, + f"下载第[{index + 1}]集: " + f"图片数为[{len(photo_detail)}]," + f"标题为:({photo_detail.title}) " + f"-- photo {photo_detail.photo_id}") + + download_by_photo_detail( + photo_detail, + option, + ) + + jm_debug(debug_topic, + f"下载完成:({photo_detail.title}) " + f"-- photo {photo_detail.photo_id}") + + multi_thread_launcher( + iter_objs=enumerate(album_detail), + apply_each_obj_func=download_photo, + wait_finish=True, + ) + + +def download_album_batch(jm_album_id_iter: Union[Iterable, Generator], + option=None, + wait_finish=True) -> List[Thread]: + """ + 批量下载album,每个album一个线程,使用的是同一个option。 + + @param jm_album_id_iter: album_id的可迭代对象 + @param option: 下载选项,为空默认是 JmOption.default() + @param wait_finish: 是否要等待这些下载线程全部完成 + @return 返回值是List[Thread],里面是每个下载漫画的线程。 + """ + if option is None: + option = JmOption.default() + + return multi_thread_launcher( + iter_objs=((album_id, option) for album_id in jm_album_id_iter), + apply_each_obj_func=download_album, + wait_finish=wait_finish, + ) + + +def download_photo(jm_photo_id, option=None): + """ + 下载一个本子的一章,入口api + """ + option, jm_client = build_client(option) + photo_detail = jm_client.get_photo_detail(jm_photo_id) + download_by_photo_detail(photo_detail, option) + + +def download_by_photo_detail(photo_detail: JmPhotoDetail, + option=None, + ): + """ + 下载一个本子的一章,根据 photo_detail + @param photo_detail: 本子章节信息 + @param option: 选项 + """ + option, jm_client = build_client(option) + + # 下载准备 + use_cache = option.download_use_disk_cache + decode_image = option.download_image_then_decode + + if photo_detail.from_album is None: + jm_client.fill_from_album(photo_detail) + + if photo_detail.page_arr is None: + jm_client.fill_page_arr(photo_detail) + + # 下载每个图片的函数 + def download_image(index, debug_topic='download_images_of_photo'): + img_detail = photo_detail[index] + img_save_path = option.decide_image_filepath(photo_detail, index) + + # 已下载过,缓存命中 + if use_cache is True and file_exists(img_save_path): + jm_debug(debug_topic, f'photo-{img_detail.aid}: ' + f'图片{img_detail.filename}已下载过,' + f'命中磁盘缓存({img_detail.img_url})') + return + + # 开始下载 + jm_client.download_by_image_detail( + img_detail, + img_save_path, + decode_image=decode_image, + ) + jm_debug(debug_topic, f'photo-{img_detail.aid}: ' + f'图片{img_detail.filename}下载完成:' + f'[{img_detail.img_url}] → [{img_save_path}]') + + length = len(photo_detail) + # 根据图片数,决定下载策略 + if length <= option.download_multi_thread_photo_len_limit: + # 如果图片数小的话,直接使用多线程下载,一张图一个线程。 + multi_thread_launcher( + iter_objs=range(len(photo_detail)), + apply_each_obj_func=download_image, + wait_finish=True + ) + else: + # 如果图片数多的话,还是分批下载。 + batch_count = option.download_multi_thread_photo_batch_count + batch_times = length // batch_count + + for i in range(batch_times): + begin = i * batch_count + multi_thread_launcher( + iter_objs=range(begin, begin + batch_count), + apply_each_obj_func=download_image, + ) + + multi_thread_launcher( + iter_objs=range(batch_times * batch_count, + length), + apply_each_obj_func=download_image, + ) + + +def renew_jm_default_domain(): + """ + 由于禁漫的域名经常变化,调用此方法可以获取一个当前可用的最新的域名 domain, + 并且设置把 domain 设置为禁漫模块的默认域名。 + 这样一来,配置文件也不用配置域名了,一切都在运行时动态获取。 + """ + domain = JmcomicText.parse_to_jm_domain(JmModuleConfig.get_jmcomic_url()) + JmModuleConfig.DOMAIN = domain + return domain + + +def build_client(option: Optional[JmOption]) -> Tuple[JmOption, JmcomicClient]: + """ + 处理option的判空,并且创建jm_client + """ + if option is None: + option = JmOption.default() + option.client_config['domain'] = renew_jm_default_domain() + jm_client = option.build_jm_client() + return option, jm_client + + +def create_option(filepath: str) -> JmOption: + """ + 创建 JmOption,同时检查域名是否配置,未配置则补上配置 + @param filepath: + @return: + """ + option = JmOption.create_from_file(filepath) + client_config = option.client_config + key = 'domain' + if client_config.get(key, None) is None or client_config[key] is None: + client_config[key] = renew_jm_default_domain() + return option diff --git a/src/jmcomic/jm_client.py b/src/jmcomic/jm_client.py new file mode 100644 index 0000000..7898a8f --- /dev/null +++ b/src/jmcomic/jm_client.py @@ -0,0 +1,186 @@ +from .jm_toolkit import * + + +class JmcomicClient(PostmanProxy): + debug_from_class = 'jm-client' + retry_postman_type = RetryPostman + + def __init__(self, postman: Postman, domain, retry_times=None): + super().__init__(postman.with_retry(retry_times or 5, self.retry_postman_type)) + self.domain = domain + + def download_image(self, + data_original: str, + img_save_path: str, + scramble_id: str, + photo_id: str = None, + decode_image=True, + ): + """ + 下载JM的图片 + @param data_original: 图片url + @param img_save_path: 图片保存位置 + @param scramble_id: 图片所在photo的scramble_id + @param photo_id: 图片所在photo的photo_id,可为空 + @param decode_image: 要保存的是解密后的图还是原图 + """ + # 请求图片 + resp = self.jm_get(data_original, is_api=False) + + if self.is_empty_image(resp): + raise AssertionError(f'接收到的图片数据为空: {resp.url}') + + # gif图无需加解密,需要最先判断 + if self.img_is_not_need_to_decode(data_original, resp): + JmImageSupport.save_resp_img(resp, img_save_path, False) + return + + # 不解密图片,直接返回 + if decode_image is False: + # 保存图片 + JmImageSupport.save_resp_img( + resp, + img_save_path, + need_convert=suffix_not_equal(data_original, img_save_path), + ) + + JmImageSupport.save_resp_decoded_img( + resp=resp, + img_detail=JmImageDetail.of( + photo_id or JmcomicText.parse_to_photo_id(data_original), + scramble_id, + data_original, + ), + filepath=img_save_path + ) + + def download_by_image_detail(self, + img_detail: JmImageDetail, + img_save_path, + decode_image=True, + ): + self.download_image( + img_detail.img_url, + img_save_path, + img_detail.scramble_id, + photo_id=img_detail.aid, + decode_image=decode_image, + ) + + # -- get detail(返回值是实体类) -- + + def get_album_detail(self, album_id) -> JmAlbumDetail: + # 参数校验 + album_id = JmcomicText.parse_to_photo_id(album_id) + + # 请求 + resp = self.jm_get(f"/album/{album_id}") + + # 用 JmcomicText 解析 html,返回实体类 + return JmcomicText.analyse_jm_album_html(resp.text) + + def get_photo_detail(self, jm_photo_id: str) -> JmPhotoDetail: + # 参数校验 + photo_id = JmcomicText.parse_to_photo_id(jm_photo_id) + + # 请求 + resp = self.jm_get(f"/photo/{photo_id}") + + # 用 JmcomicText 解析 html,返回实体类 + return JmcomicText.analyse_jm_photo_html(resp.text) + + def fill_from_album(self, photo_detail: JmPhotoDetail) -> JmAlbumDetail: + """ + 获取 photo_detail 所在的 album。 + 并把 album_detail 赋值给 photo_detail.from_album + """ + album_detail = self.get_album_detail(photo_detail.album_id) + photo_detail.from_album = album_detail + return album_detail + + def fill_page_arr(self, photo_detail: JmPhotoDetail): + new = self.get_photo_detail(photo_detail.photo_id) + photo_detail.page_arr = new.page_arr + photo_detail.data_original_domain = new.data_original_domain + return new + + # -- search -- + + def search_album(self, search_query, main_tag=0) -> JmSearchPage: + params = { + 'main_tag': main_tag, + 'search_query': search_query, + } + + resp = self.jm_get('/search/photos', params=params) + + return JmSearchSupport.analyse_jm_search_html(resp.text) + + # -- 对象方法 -- + + def of_api_url(self, api_path): + return f"{JmModuleConfig.HTTP}{self.domain}{api_path}" + + def jm_get(self, url, is_api=True, require_200=True, **kwargs): + """ + 向禁漫发请求的统一入口 + """ + url = self.of_api_url(url) if is_api is True else url + if is_api is True: + self.debug("api", url) + + resp = self.get(url, **kwargs) + + if require_200 is True and resp.status_code != 200: + write_text('./resp.html', resp.text) + raise AssertionError(f"请求失败," + f"响应状态码为{resp.status_code}," + f"URL=[{resp.url}]," + + + (f"响应文本=[{resp.text}]" if len(resp.text) < 50 else + f'响应文本过长(len={len(resp.text)}),不打印') + ) + + if is_api is True and resp.text.strip() == JmModuleConfig.JM_SERVER_ERROR_HTML: + raise AssertionError("【JM异常】Could not connect to mysql! Please check your database settings!") + + return resp + + # -- 类方法 -- + + @classmethod + def is_empty_image(cls, resp): + return resp.status_code != 200 or len(resp.content) == 0 + + @staticmethod + def debug(topic: str, *args, sep='', end='\n', file=None, from_class=debug_from_class): + jm_debug(topic, *args, sep=sep, end=end, file=file, from_class=from_class) + + @classmethod + def img_is_not_need_to_decode(cls, data_original: str, _resp): + return data_original.endswith('.gif') + + +# 爬取策略 +class FetchStrategy: + + def __init__(self, + from_index, + photo_len, + resp_getter, + resp_consumer, + ): + self.from_index = from_index + self.photo_len = photo_len + self.resp_getter = resp_getter + self.resp_consumer = resp_consumer + + def do_fetch(self): + raise NotImplementedError + + def args(self): + return (self.from_index, + self.photo_len, + self.resp_getter, + self.resp_consumer, + ) diff --git a/src/jmcomic/jm_config.py b/src/jmcomic/jm_config.py new file mode 100644 index 0000000..1b25644 --- /dev/null +++ b/src/jmcomic/jm_config.py @@ -0,0 +1,76 @@ +class JmModuleConfig: + # 网站相关 + HTTP = "https://" + DOMAIN = "jmcomic1.group" # jmcomic默认域名 + JM_REDIRECT_URL = f'{HTTP}jm365.xyz/3YeBdF' # 永久網域,怕走失的小伙伴收藏起来↓ + JM_PUB_URL = f'{HTTP}jmcomic1.bet' + JM_CDN_IMAGE_URL_TEMPLATE = HTTP + 'cdn-msp.{domain}/media/photos/{photo_id}/{index:05}{suffix}' # index 从1开始 + JM_SERVER_ERROR_HTML = "Could not connect to mysql! Please check your database settings!" + JM_IMAGE_SUFFIX = ['.jpg', '.webp', '.png', '.gif'] + + # 图片分隔相关 + SCRAMBLE_0 = 220980 + SCRAMBLE_10 = 268850 + SCRAMBLE_NUM_8 = 421926 # 2023-02-08后改了图片切割算法 + + # 下载时的一些默认值 + default_author = 'default-author' + default_photo_title = 'default-photo-title' + default_photo_id = 'default-photo-id' + + # debug + enable_jm_debug = True + debug_printer = print + + # 缓存 + jm_client_caches = {} + + @classmethod + def default_headers(cls): + return { + 'authority': cls.DOMAIN, + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,' + 'application/signed-exchange;v=b3;q=0.9', + 'accept-language': 'zh-CN,zh;q=0.9', + 'sec-ch-ua': '"Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'document', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-user': '?1', + 'upgrade-insecure-requests': '1', + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 ' + 'Safari/537.36', + } + + # noinspection PyUnusedLocal + @classmethod + def jm_debug(cls, topic: str, *args, sep='', end='\n', file=None, from_class='api'): + if cls.enable_jm_debug is True: + cls.debug_printer(f'【{topic}】', *args, sep=sep, end=end, file=file) + + @classmethod + def disable_jm_debug(cls): + cls.enable_jm_debug = False + + @classmethod + def get_jmcomic_url(cls): + """ + 访问禁漫的永久网域,从而得到一个可用的禁漫网址, + """ + from common import Postmans + + domain = Postmans \ + .get_impl_clazz('cffi') \ + .create(headers=cls.default_headers()) \ + .with_wrap_resp() \ + .get(cls.JM_REDIRECT_URL, allow_redirects=False) \ + .redirect_url + + cls.jm_debug('获取禁漫地址', f'获取成功,最新可用的禁漫地址: {domain}') + return domain + + +jm_debug = JmModuleConfig.jm_debug +disable_jm_debug = JmModuleConfig.disable_jm_debug diff --git a/src/jmcomic/jm_entity.py b/src/jmcomic/jm_entity.py new file mode 100644 index 0000000..cdd8c36 --- /dev/null +++ b/src/jmcomic/jm_entity.py @@ -0,0 +1,319 @@ +from common import * + +from .jm_config import * + + +class JmBaseEntity: + pass + + +class WorkEntity(JmBaseEntity, SaveableEntity, IterableEntity): + when_del_save_file = False + after_save_print_info = True + attr_char = '_' + + cache_getitem_result = True + cache_field_name = '__cache_items_dict__' + detail_save_base_dir = workspace() + detail_save_file_suffix = '.yml' + + def save_base_dir(self): + return self.detail_save_base_dir + + def save_file_name(self) -> str: + return f"【{self.get_id_prefix_of_filename()}{self.get_id()}】{self.get_title()}{self.detail_save_file_suffix}" + + def get_id_prefix_of_filename(self): + # "JmAlbumDetail" -> "album" + cls_name = self.__class__.__name__ + id_prefix = cls_name[cls_name.index("m") + 1: cls_name.rfind("Detail")] + return f'{id_prefix}-' + + def get_id(self) -> str: + raise NotImplementedError + + def get_title(self) -> str: + raise NotImplementedError + + def __len__(self): + raise NotImplementedError + + def __getitem__(self, item): + raise NotImplementedError + + +class JmImageDetail(JmBaseEntity): + + def __init__(self, + aid, + scramble_id, + img_url, + img_file_name, + img_file_suffix, + from_photo=None, + ) -> None: + self.aid: str = aid + self.scramble_id: str = scramble_id + self.img_url: str = img_url + self.img_file_name: str = img_file_name + self.img_file_suffix: str = img_file_suffix + + self.from_photo: Optional[JmPhotoDetail] = from_photo + + @property + def filename(self): + return self.img_file_name + self.img_file_suffix + + @classmethod + def of(cls, + photo_id: str, + scramble_id: str, + data_original: str, + from_photo=None, + ) -> 'JmImageDetail': + """ + 该方法用于创建 JmImageDetail 对象 + """ + + # /xxx.yyy + # ↑ ↑ + # x y + x = data_original.rfind('/') + y = data_original.rfind('.') + + return JmImageDetail( + aid=photo_id, + scramble_id=scramble_id, + img_url=data_original, + img_file_name=data_original[x + 1:y], + img_file_suffix=data_original[y:], + from_photo=from_photo, + ) + + +class JmPhotoDetail(WorkEntity): + + def __init__(self, + photo_id, + scramble_id, + title, + keywords, + series_id, + page_arr=None, + data_original_domain=None, + author=None, + from_album=None, + ): + self.photo_id: str = photo_id + self.scramble_id: str = scramble_id + self.title: str = title + self._keywords: str = keywords + self._series_id: int = int(series_id) + + self._author: StrNone = author + self.from_album: Optional[JmAlbumDetail] = from_album + + # 下面的属性和图片url有关 + if isinstance(page_arr, str): + import json + page_arr = json.loads(page_arr) + + # 该photo的所有图片名 img_name + self.page_arr: List[str] = page_arr + self.data_original_domain = data_original_domain + + def is_single_album(self) -> bool: + return self._series_id == 0 + + @property + def keyword_list(self) -> List[str]: + return self._keywords.split(',') + + @property + def album_id(self) -> str: + return self.photo_id if self.is_single_album() else self._series_id + + @property + def author(self) -> str: + # self._author 不为空字符串 + if self._author is not None and self._author != '': + return self._author.strip() + + # self._author 为空,先向上找 + if self.from_album is not None: + return self.from_album.author + + # 无向上元素,使用默认 + return JmModuleConfig.default_author + + def create_image_detail(self, index) -> JmImageDetail: + # 校验参数 + length = len(self.page_arr) + if index >= length: + raise AssertionError(f'创建JmImageDetail失败,{index} >= {length}') + + data_original = self.get_img_data_original(self.page_arr[index]) + + return JmImageDetail.of( + self.photo_id, + self.scramble_id, + data_original, + from_photo=self + ) + + def get_img_data_original(self, img_name: str) -> str: + """ + 根据图片名,生成图片的完整请求路径 URL + 例如:img_name = 01111.webp + 返回:https://cdn-msp2.18comic.org/media/photos/147643/01111.webp + """ + return f'https://{self.data_original_domain}/media/photos/{self.photo_id}/{img_name}' + + def __getitem__(self, item) -> JmImageDetail: + return self.create_image_detail(item) + + def get_id(self): + return self.photo_id + + def get_title(self): + return self.title + + def __len__(self): + return len(self.page_arr) + + +class JmAlbumDetail(WorkEntity): + + def __init__(self, + album_id, + scramble_id, + title, + episode_list, + page_count, + author_list, + pub_date, + ): + self.album_id: str = album_id + self.scramble_id: str = scramble_id + self.title: str = title + self.page_count = int(page_count) + self._author_list: List[str] = author_list + self.pub_date = pub_date + + # 有的 album 没有章节,则自成一章。 + if len(episode_list) == 0: + # photo_id, photo_index_of_album, photo_title, photo_pub_date + episode_list = [(album_id, 0, title, pub_date)] + + self.episode_list: List[Tuple] = self.distinct_episode(episode_list) + + def create_photo_detail(self, index) -> Tuple[JmPhotoDetail, Tuple]: + # 校验参数 + length = len(self.episode_list) + + if index >= length: + raise AssertionError(f'创建JmPhotoDetail失败,{index} >= {length}') + + # episode_info: ('212214', '81', '94 突然打來', '2020-08-29') + episode_info: tuple = self.episode_list[index] + photo_id, photo_index_of_album, photo_title, photo_pub_date = episode_info + + photo_detail = JmPhotoDetail( + photo_id=photo_id, + scramble_id=self.scramble_id, + title=photo_title, + keywords='', + series_id=self.album_id, + author=self.author, + from_album=self, + page_arr=None, + data_original_domain=None + ) + + return photo_detail, episode_info + + @property + def author(self): + if len(self._author_list) >= 1: + return self._author_list[0] + return JmModuleConfig.default_author + + def get_id(self): + return self.album_id + + def get_title(self): + return self.title + + def __len__(self): + return len(self.episode_list) + + def __getitem__(self, item) -> JmPhotoDetail: + return self.create_photo_detail(item)[0] + + @staticmethod + def distinct_episode(episode_list): + ret = [] + + def not_exist(episode): + photo_id = episode[0] + for each in ret: + if each[0] == photo_id: + return False + return True + + for episode in episode_list: + if not_exist(episode): + ret.append(episode) + + return ret + + +class JmSearchPage(IterableEntity): + + def __init__(self, album_info_list) -> None: + # (album_id, title, category_none, label_sub_none, tag_list) + self.data: List[Tuple[str, str, StrNone, StrNone, List[str]]] = album_info_list + + def album_id_iter(self): + for album_info in self.data: + yield album_info[0] + + def __len__(self): + return len(self.data) + + def __getitem__(self, item): + return self.data[item] + + +# cdn爬取请求 +class CdnRequest: + SavePathProvider = Callable[[str, str, int, bool], str] + + def __init__(self, + photo_id, + scramble_id, + from_index: int, + photo_len: Optional[int], + save_path_provider: SavePathProvider, + ): + self.photo_id = photo_id + self.scramble_id = scramble_id + self.from_index = from_index + self.photo_len = photo_len + self.save_path_provider = save_path_provider + + @classmethod + def create(cls, + photo_id, + save_path_provider: SavePathProvider, + scramble_id, + from_index=1, + photo_len=None, + ): + return CdnRequest( + photo_id, + scramble_id, + from_index, + photo_len, + save_path_provider, + ) diff --git a/src/jmcomic/jm_option.py b/src/jmcomic/jm_option.py new file mode 100644 index 0000000..2773379 --- /dev/null +++ b/src/jmcomic/jm_option.py @@ -0,0 +1,480 @@ +from .jm_service import * + + +class JmOptionAdvice: + + def decide_image_save_dir(self, + option: 'JmOption', + album_detail: Optional[JmAlbumDetail], + photo_detail: JmPhotoDetail, + ) -> StrNone: + """ + 决定一个本子图片的下载文件夹 + @param option: JmOption对象 + @param album_detail: 本子集实体类 + @param photo_detail: 本子章节实体类 + @return: 下载文件夹,为空表示不处理 + """ + pass + + def decide_image_filepath(self, + option: 'JmOption', + photo_detail: JmPhotoDetail, + index: int, + ) -> StrNone: + """ + 决定一个本子图片的绝对路径 + @param option: JmOption对象 + @param photo_detail: 本子章节实体类 + @param index: 本子章节里的第几章图片 + @return: 下载绝对路径,为空表示不处理 + """ + pass + + def decide_image_suffix(self, + option: 'JmOption', + img_detail: JmImageDetail, + ) -> StrNone: + """ + 决定一个图片的保存后缀名 + @param option: JmOption对象 + @param img_detail: 禁漫图片实体类 + @return: 保存后缀名,为空表示不处理 + """ + pass + + +class JmAdviceRegistry: + advice_registration: Dict[Any, List[JmOptionAdvice]] = {} + + @classmethod + def register_advice(cls, base, *advice): + advice_ls = cls.advice_registration.get(base, None) + + if advice_ls is None: + advice_ls = list(advice) + cls.advice_registration[base] = advice_ls + else: + for e in advice: + advice_ls.append(e) + + return advice + + @classmethod + def get_advice(cls, base) -> List[JmOptionAdvice]: + return cls.advice_registration.setdefault(base, []) + + +# 目录树的组成 +class DownloadDirTree: + # 根目录 / Album作者 / Photo标题 / 图片文件 + Bd_Author_Title_Image = 0 + # 根目录 / Album作者 / Photo号 / 图片文件 + Bd_Author_Id_Image = 1 + + # 根目录 / Album标题 / Photo标题 / 图片文件 + Bd_Title_Title_Image = 2 + # 根目录 / Album标题 / Photo号 / 图片文件 + Bd_Title_Id_Image = 3 + + # 根目录 / Photo标题 / 图片文件 + Bd_Title_Image = 4 + # 根目录 / Photo号 / 图片文件 + Bd_Id_Image = 5 + + AdditionalHandler = Callable[[Optional[JmAlbumDetail], JmPhotoDetail], str] + additional_tree_flag_handler_mapping: Dict[int, AdditionalHandler] = {} + + def __init__(self, + Bd: str, + flag: Union[str, int], + ): + self.Bd = fix_filepath(Bd) + self.flag = self.get_flag_enum(flag) + + def get_flag_enum(self, flag): + if isinstance(flag, int): + return flag + + if not isinstance(flag, str): + raise NotImplementedError(flag) + + ret = self.__class__.__dict__.get(flag, None) + if ret is None: + raise NotImplementedError(flag) + + return ret + + def deside_image_save_dir(self, + album: Optional[JmAlbumDetail], + photo: JmPhotoDetail, + ) -> str: + + if photo is None: + raise AssertionError('章节信息不能为None') + + def dirpath(album_dir: Optional[str], photo_dir: str): + """ + @param album_dir: 相册文件夹名 + @param photo_dir: 章节文件夹名 + """ + from common import fix_windir_name + photo_dir = fix_windir_name(photo_dir) + + if album_dir is None: + return f'{self.Bd}{photo_dir}/' + + album_dir = fix_windir_name(album_dir) + return f'{self.Bd}{album_dir}/{photo_dir}/' + + def photo_dir(flag_for_title): + return photo.title if flag == flag_for_title else photo.photo_id + + flag = self.flag + + if flag in (0, 1): + # 根目录 / Album作者 / Photo标题 / 图片文件 + # 根目录 / Album作者 / Photo号 / 图片文件 + + author = album.author if album is not None else JmModuleConfig.default_author + return dirpath(author, photo_dir(0)) + + elif flag in (2, 3): + # 根目录 / Album标题 / Photo标题 / 图片文件 + # 根目录 / Album标题 / Photo号 / 图片文件 + + album_title = album.title if album is not None else photo.title + return dirpath(album_title, photo_dir(2)) + + elif flag in (4, 5): + # 根目录 / Photo标题 / 图片文件 + # 根目录 / Photo号 / 图片文件 + + return dirpath(None, photo_dir(4)) + + else: + if flag in self.additional_tree_flag_handler_mapping: + return self.additional_tree_flag_handler_mapping[flag](album, photo) + else: + raise NotImplementedError + + @staticmethod + def random_id(): + import common + return common.time_stamp() + + @staticmethod + def random_title(): + import common + return common.format_ts(f_time="%Y-%m-%d-%H-%M-%S") + + @classmethod + def accept_flag_dict(cls): + return dict(filter(lambda item: item[0].startswith('Bd_'), + cls.__dict__.items())) + + @classmethod + def of(cls, + tree_base_dir: str, + tree_flag: int, + ): + for accpet_flag in cls.accept_flag_dict().values(): + if accpet_flag is tree_flag: + return DownloadDirTree(tree_base_dir, tree_flag) + + raise AssertionError(f'不支持的flag: {tree_flag},' + f'请使用DownloadDirTree类字段中的值,或直接实例化DownloadDirTree对象') + + +class JmOption(SaveableEntity): + _proxies_mapping = { + 'clash': ProxyBuilder.clash_proxy, + 'v2Ray': ProxyBuilder.v2Ray_proxy + } + + when_del_save_file = False + cache_jm_client = True + + def __init__(self, + dir_tree: DownloadDirTree, + client_config: dict, + filepath: StrNone = None, + disable_jm_module_debug=False, + download_use_disk_cache=True, + download_convert_image_suffix: StrNone = None, + download_image_then_decode=True, + download_multi_thread_photo_len_limit=30, + download_multi_thread_photo_batch_count=10, + ): + + # 请求配置 + self.client_config = client_config + self.client_config['postman_type_list'] = list(Postmans.postman_impl_class_dict.keys()) + + # 下载配置 + self.dir_tree = dir_tree + self.download_use_disk_cache = download_use_disk_cache + self.download_image_then_decode = download_image_then_decode + self.download_multi_thread_photo_len_limit = download_multi_thread_photo_len_limit + self.download_multi_thread_photo_batch_count = download_multi_thread_photo_batch_count + # suffix的标准形式是 ".xxx"。如果传入的是"xxx",要补成 ".xxx" + if download_convert_image_suffix is not None: + download_convert_image_suffix = fix_suffix(download_convert_image_suffix) + self.download_convert_image_suffix = download_convert_image_suffix + + # 其他配置 + self.filepath = filepath + self.disable_jm_module_debug = disable_jm_module_debug + if disable_jm_module_debug: + disable_jm_debug() + + """ + 下面是决定图片保存路径的方法 + """ + + def decide_image_save_dir(self, album_detail, photo_detail) -> str: + if album_detail is None and photo_detail is None: + raise AssertionError('album和photo不能同时为None') + + # 先检查advice的回调,如果回调支持,则优先使用回调 + for advice in JmAdviceRegistry.get_advice(self): + save_dir = advice.decide_image_save_dir(self, album_detail, photo_detail) + if save_dir is not None: + return save_dir + + # 使用 self.dir_tree 决定 save_dir + save_dir = self.dir_tree.deside_image_save_dir( + album_detail or photo_detail.from_album, + photo_detail + ) + + mkdir_if_not_exists(save_dir) + return save_dir + + def decide_image_suffix(self, img_detail: JmImageDetail): + # 先检查advice的回调,如果回调支持,则优先使用回调 + for advice in JmAdviceRegistry.get_advice(self): + suffix = advice.decide_image_suffix(self, img_detail) + if suffix is not None: + return suffix + + # 动图则使用原后缀 + suffix = img_detail.img_file_suffix + if suffix.endswith("gif"): + return suffix + + # 非动图,以配置为先 + return self.download_convert_image_suffix or suffix + + def decide_image_filepath(self, photo_detail: JmPhotoDetail, index: int) -> str: + # 先检查advice的回调,如果回调支持,则优先使用回调 + for advice in JmAdviceRegistry.get_advice(self): + filepath = advice.decide_image_filepath(self, photo_detail, index) + if filepath is not None: + return filepath + + # 通过拼接生成绝对路径 + save_dir = self.decide_image_save_dir(photo_detail.from_album, photo_detail) + image: JmImageDetail = photo_detail[index] + suffix = self.decide_image_suffix(image) + return save_dir + image.img_file_name + suffix + + """ + 下面是对Advice的支持 + """ + + def register_advice(self, *advice): + JmAdviceRegistry.register_advice(self, *advice) + + """ + 下面是创建对象相关方法 + """ + + @classmethod + def default(cls) -> 'JmOption': + return JmOption( + DownloadDirTree( + workspace(), + DownloadDirTree.Bd_Title_Image, + ), + cls.default_client_config(), + ) + + @classmethod + def create_from_file(cls, filepath: str) -> 'JmOption': + jm_option: JmOption = PackerUtil.unpack(filepath, JmOption)[0] + jm_option.filepath = filepath + return jm_option + + @classmethod + def default_client_config(cls): + return { + 'domain': JmModuleConfig.DOMAIN, + 'meta_data': { + 'cookies': None, + 'headers': JmModuleConfig.default_headers(), + 'allow_redirects': True, + }, + 'postman_type_list': [ + 'requests', + 'requests_Session', + 'cffi', + 'cffi_Session', + ] + } + + def save_base_dir(self): + return of_dir_path(self.filepath) + + def save_file_name(self) -> str: + return of_file_name(self.filepath) + + def save_to_file(self, filepath=None): + if filepath is None: + filepath = self.filepath + + if filepath is None: + raise AssertionError("未指定JmOption的保存路径") + + super().save_to_file(filepath) + + """ + 下面是 build 方法 + """ + + def build_jm_client(self) -> JmcomicClient: + if self.cache_jm_client is not True: + client = self.new_jm_client() + else: + key = self + client = JmModuleConfig.jm_client_caches.get(key, None) + if client is None: + client = self.new_jm_client() + JmModuleConfig.jm_client_caches.setdefault(key, client) + + return client + + def new_jm_client(self) -> JmcomicClient: + meta_data = self.client_config['meta_data'] + + # 处理代理 + def handle_proxies(key='proxies'): + proxies = meta_data.get(key, None) + + # 无代理,或代理已配置好好的 + if proxies is None or isinstance(proxies, dict): + return + + # 有代理 + if proxies in self._proxies_mapping: + proxies = self._proxies_mapping[proxies]() + else: + proxies = ProxyBuilder.build_proxy(proxies) + + meta_data[key] = proxies + + # 处理 headers + def handle_headers(key='headers'): + headers = meta_data.get(key, None) + if headers is None or (not isinstance(headers, dict)) or len(headers) == 0: + meta_data[key] = JmModuleConfig.default_headers() + + # 处理【特殊配置项】 + handle_proxies() + handle_headers() + + # 决定Postman的实现类,根据配置项 client_config.postman + postman_clazz = Postmans.get_impl_clazz(self.client_config.get('postman_type', 'cffi')) + # 决定域名 + domain = self.client_config.get('domain', JmModuleConfig.DOMAIN) + + jm_debug('创建JmcomicClient', f'使用域名: {domain},使用Postman实现: {postman_clazz}') + # 创建 JmcomicClient 实例 + client = JmcomicClient( + postman=postman_clazz(meta_data), + domain=domain, + retry_times=self.client_config.get('retry_times', None) + ) + + return client + + """ + 下面的方法是对【CdnOption】的支持,已弃用 + """ + + def build_cdn_option(self, use_multi_thread_strategy=True): + + return CdnConfig.create( + cdn_domain=self.client_config.get('domain', JmModuleConfig.DOMAIN), + fetch_strategy=MultiThreadFetch if use_multi_thread_strategy else InOrderFetch, + cdn_image_suffix=None, + use_cache=self.download_use_disk_cache, + decode_image=self.download_image_then_decode + ) + + def build_cdn_crawler(self, use_multi_thread_strategy=True): + return CdnFetchService(self.build_cdn_option(use_multi_thread_strategy), + self.build_jm_client()) + + def build_cdn_request(self, + photo_id: str, + scramble_id=str(JmModuleConfig.SCRAMBLE_10), + from_index=1, + photo_len=None, + use_default=False + ) -> CdnRequest: + return CdnRequest.create( + photo_id, + self.build_save_path_provider(use_default, None if use_default else photo_id), + scramble_id, + from_index, + photo_len, + ) + + def build_save_path_provider(self, + use_all_default_save_path, + photo_id=None, + ) -> CdnRequest.SavePathProvider: + + if use_all_default_save_path is True: + # 不通过请求获取 photo 的信息,相当于使用【空本子】和【空集】 + photo_detail, album_detail = None, None + else: + # 通过请求获得 photo 的本子信息 + client = self.build_jm_client() + photo_detail = client.get_photo_detail(photo_id) + album_detail = client.fill_from_album(photo_detail) + + def save_path_provider(url, _suffix: str, _index, _is_decode): + return '{0}{1}{2}'.format(self.decide_image_save_dir(album_detail, photo_detail), + of_file_name(url, trim_suffix=True), + self.download_convert_image_suffix or _suffix) + + return save_path_provider + + +def _register_yaml_constructor(): + from yaml import add_constructor, Loader, Node + + tag_mapping = { + 'tag:yaml.org,2002:python/object:jmcomic.jm_option.JmOption': JmOption, + 'tag:yaml.org,2002:python/object:jmcomic.jm_option.DownloadDirTree': DownloadDirTree, + } + + def constructor(loader: Loader, node: Node): + for tag, clazz in tag_mapping.items(): + if node.tag == tag: + state = loader.construct_mapping(node) + try: + obj = clazz(**state) + except TypeError as e: + raise AssertionError(f"构造函数不匹配: {clazz.__name__}\nTypeError: {e}") + + # obj.__dict__.update(state) + return obj + + for tag in tag_mapping.keys(): + add_constructor(tag, constructor) + + +_register_yaml_constructor() diff --git a/src/jmcomic/jm_service.py b/src/jmcomic/jm_service.py new file mode 100644 index 0000000..1249735 --- /dev/null +++ b/src/jmcomic/jm_service.py @@ -0,0 +1,248 @@ +from .jm_client import * + + +class CdnConfig: + + def __init__(self, + cdn_domain: str, + cdn_image_suffix: str, + fetch_strategy: Type[FetchStrategy], + use_cache, + decode_image, + ): + self.cdn_domain = cdn_domain + self.cdn_image_suffix = cdn_image_suffix + self.fetch_strategy = fetch_strategy + self.use_cache = use_cache + self.decode_image = decode_image + + def get_cdn_image_url(self, photo_id: str, index: int) -> str: + return JmModuleConfig.JM_CDN_IMAGE_URL_TEMPLATE.format( + domain=self.cdn_domain, + photo_id=photo_id, + index=index, + suffix=self.cdn_image_suffix + ) + + def check_request_is_valid(self, req: CdnRequest): + from common import is_function + + if req.photo_len is not None and not isinstance(req.photo_len, int): + raise AssertionError('传参错误,photo_len要么给整数,要么给None') + if not is_function(req.save_path_provider): + raise AssertionError('传参错误,save_path_provider应该是函数') + if self.decode_image is True and req.scramble_id is None: + raise AssertionError('传参缺失,指定decode_image=True,就必须提供scramble_id') + if req.from_index <= 0: + raise AssertionError(f'传参错误,from_index必须大于1: {req.from_index} < 1') + + @classmethod + def create(cls, + cdn_domain, + fetch_strategy, + cdn_image_suffix=None, + use_cache=False, + decode_image=True, + ): + + return CdnConfig( + cdn_domain, + cdn_image_suffix or '.webp', + fetch_strategy, + use_cache, + decode_image, + ) + + +class InOrderFetch(FetchStrategy): + + def do_fetch(self): + index, photo_len, resp_getter, resp_consumer = self.args() + + while True: + if photo_len is not None and index > photo_len: + break + + resp_info = resp_getter(index) + if resp_info[0] is None: + break + resp_consumer(resp_info, index) + index += 1 + + +class MultiThreadFetch(FetchStrategy): + fetch_batch_count = 5 + + def do_fetch(self): + from common import multi_thread_launcher + begin, photo_len, resp_getter, resp_consumer = self.args() + + if photo_len is not None: + # 确定要爬多少页 + + def accpet_index(index): + resp_info = resp_getter(index) + if resp_info[0] is None: + return + resp_consumer(resp_info, index) + + multi_thread_launcher( + iter_objs=range(begin, photo_len + 1), + apply_each_obj_func=accpet_index, + wait_finish=True, + ) + else: + # 不确定要爬多少页 + + # 批量单位 + batch_count = self.fetch_batch_count + # 标记位 + fetch_suffer_none_resp = False + + def accpet_index(index): + resp_info = resp_getter(index) + if resp_info is None: + nonlocal fetch_suffer_none_resp + fetch_suffer_none_resp = True + return + resp_consumer(resp_info, index) + + while True: + # 5次5次爬,每5次中,但凡有1次出现了失败,就停止 + + multi_thread_launcher( + iter_objs=range(begin, begin + batch_count), + apply_each_obj_func=accpet_index, + wait_finish=True + ) + + # 检查标记 + if fetch_suffer_none_resp is False: + begin += batch_count + else: + break + + +class CdnFetchService: + + def __init__(self, + config: CdnConfig, + client: 'JmcomicClient', + ): + self.client = client + self.config = config + self.debug = self.client.debug + + def download_photo_from_cdn_directly(self, + req: CdnRequest, + ): + # 校验参数 + self.config.check_request_is_valid(req) + + # 基本信息 + photo_id = req.photo_id + scramble_id = req.scramble_id + use_cache = self.config.use_cache + decode_image = self.config.decode_image + + # 获得响应 + def get_resp(index: int) -> Tuple[Optional[Any], str, str]: + url = self.config.get_cdn_image_url(photo_id, index) + suffix = self.config.cdn_image_suffix + pre_try_save_path = req.save_path_provider(url, suffix, index, decode_image) + + # 判断是否命中缓存(预先尝试) + if use_cache is True and file_exists(pre_try_save_path): + # 命中,返回特殊值 + return None, suffix, pre_try_save_path + else: + return self.try_get_cdn_image_resp( + self.client, + url, + suffix, + photo_id, + index, + ) + + # 保存响应 + def save_resp(resp_info: Tuple[Optional[Any], str, str], index: int): + resp, suffix, img_url = resp_info + + # 1. 判断是不是特殊值 + if resp_info[0] is None: + # 是,表示命中缓存,直接返回 + save_path = resp_info[2] + self.debug('图片下载成功', + f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})') + return + + # 2. 判断是否命中缓存 + save_path = req.save_path_provider(img_url, suffix, index, decode_image) + if use_cache is True and file_exists(save_path): + # 命中,直接返回 + self.debug('图片下载成功', + f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})') + return + + # 3. 保存图片 + if decode_image is False: + JmImageSupport.save_resp_img(resp, save_path) + else: + JmImageSupport.save_resp_decoded_img( + resp, + JmImageDetail.of(photo_id, scramble_id, data_original=img_url), + save_path, + ) + + # 4. debug 消息 + self.debug('图片下载成功', + f'photo-{photo_id}: {index}{suffix}下载完成 (' + + ('已解码' if decode_image is True else '未解码') + + f') [{img_url}] → [{save_path}]') + + # 调用爬虫策略 + self.config.fetch_strategy( + req.from_index, + req.photo_len, + resp_getter=get_resp, + resp_consumer=save_resp, + ).do_fetch() + + # 准备提供给爬虫策略的函数 + @staticmethod + def try_get_cdn_image_resp(client: JmcomicClient, + url: str, + suffix: str, + photo_id, + index, + ) -> Optional[Tuple[Any, str, str]]: + resp = client.jm_get(url, False, False) + + # 第一次校验,不空则直接返回 + if not client.is_empty_image(resp): + return resp, suffix, url + + # 下面进行重试 + client.debug( + '图片获取重试', + f'photo-{photo_id},图片序号为{index},url=[{url}]' + ) + + # 重试点1:是否文件后缀名不对? + for alter_suffix in JmModuleConfig.JM_IMAGE_SUFFIX: + url = change_file_suffix(url, alter_suffix) + resp = client.jm_get(url, False, False) + if not client.is_empty_image(resp): + client.debug( + '图片获取重试 → 成功√', + f'更改请求后缀({suffix} -> {alter_suffix}),url={url}' + ) + return resp, alter_suffix, url + + # 结论:可能是图片到头了 + client.debug( + '图片获取重试 ← 失败×', + f'更换后缀名不成功,停止爬取。' + f'(推断本子长度<={index - 1}。当前图片序号为{index},已经到达尽头,' + f'photo-{photo_id})' + ) diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py new file mode 100644 index 0000000..3209c9d --- /dev/null +++ b/src/jmcomic/jm_toolkit.py @@ -0,0 +1,281 @@ +from PIL import Image + +from .jm_entity import * + + +class JmcomicText: + pattern_jm_domain = compile('https://([\w.]+)') + pattern_jm_pa_id = compile('/(photos?|album)/(\d+)') + + pattern_html_photo_photo_id = compile('') + pattern_html_photo_scramble_id = compile('var scramble_id = (\d+);') + pattern_html_photo_title = compile('(.*?)\|.*') + # pattern_html_photo_data_original_list = compile('data-original="(.*?)" id="album_photo_.+?"') + pattern_html_photo_data_original_domain = compile('src="https://(.*?)/media/albums/blank') + pattern_html_photo_keywords = compile('') + pattern_html_photo_series_id = compile('var series_id = (\d+);') + pattern_html_photo_page_arr = compile('var page_arr = (.*?);') + + pattern_html_album_album_id = compile('.*?:JM(\d+)') + pattern_html_album_scramble_id = compile('var scramble_id = (\d+);') + pattern_html_album_title = compile('panel-heading[\s\S]*?

(.*?)

') + pattern_html_album_episode_list = compile('\n' + '\n第(\d+)話\n(.*)\n' + '<[\s\S]*?>(\d+-\d+-\d+).*?') + pattern_html_album_page_count = compile('.*?:(\d+)') + pattern_html_album_pub_date = compile('itemprop="datePublished" content=".*?">更新日期 : (.*?)') + + # album 作者 + pattern_html_album_author_list = [ + compile('作者: )*\s*'), + compile("(.*?)"), + ] + + @classmethod + def parse_to_jm_domain(cls, text: str): + if text.startswith("https"): + return cls.pattern_jm_domain.search(text)[1] + + return text + + @classmethod + def parse_to_photo_id(cls, text) -> str: + if isinstance(text, int): + return str(text) + + if not isinstance(text, str): + raise AssertionError(f"无法解析jm车号, 参数类型为: {type(text)}") + + if len(text) <= 2: + raise AssertionError(f"无法解析jm车号, 文本为: {text}") + + # text: JM12341 + c0 = text[0] + c1 = text[1] + if (c0 == 'J' or c0 == 'j') and (c1 == 'M' or c1 == 'm'): + # JM123456 + return text[2:] + + elif text.isdigit(): + # 123456 + return str(text) + + else: + # https://xxx/photo/412038 + match = cls.pattern_jm_pa_id.search(text) + if match is None: + raise AssertionError(f"无法解析jm车号, 文本为: {text}") + return match[2] + + @classmethod + def parse_to_album_id(cls, text) -> str: + return cls.parse_to_photo_id(text) + + @classmethod + def analyse_jm_photo_html(cls, html: str) -> JmPhotoDetail: + return cls.reflect_new_instance( + html, + "pattern_html_photo_", + JmPhotoDetail + ) + + @classmethod + def analyse_jm_album_html(cls, html: str) -> JmAlbumDetail: + return cls.reflect_new_instance( + html, + "pattern_html_album_", + JmAlbumDetail + ) + + @classmethod + def reflect_new_instance(cls, html: str, cls_field_prefix: str, clazz: type): + + def match_field(field_key: str, pattern: Union[Pattern, List[Pattern]], text): + + if isinstance(pattern, list): + # 如果是 pattern 是 List[re.Pattern], + # 取最后一个 pattern 用于 match field, + # 其他的 pattern 用来给文本缩小范围(相当于多次正则匹配) + last_pattern = pattern[len(pattern) - 1] + # 缩小文本 + for i in range(0, len(pattern) - 1): + match = pattern[i].search(text) + if match is None: + return None + text = match[0] + pattern = last_pattern + + if field_key.endswith("_list"): + return pattern.findall(text) + else: + match = pattern.search(text) + if match is not None: + return match[1] + return None + + field_dict = {} + pattern_name: str + + for pattern_name, pattern_value in cls.__dict__.items(): + if not pattern_name.startswith(cls_field_prefix): + continue + + # 获取字段名和值 + field_name = pattern_name[pattern_name.index(cls_field_prefix) + len(cls_field_prefix):] + field_value = match_field(field_name, pattern_value, html) + + if field_value is None: + raise AssertionError(f"文本没有匹配上字段:字段名为{field_name},pattern为cls.{pattern_name}") + + # 保存字段 + field_dict[field_name] = field_value + + return clazz(**field_dict) + + +class JmSearchSupport: + # 用来缩减html的长度 + pattern_html_search_shorten_for = compile('
([\s\S]*)' + '
[\s\S]*' + '
') + + # 用来提取搜索页面的的album的信息 + pattern_html_search_album_info_list = compile( + '' + '\n(.*)\n
\n
' + '(.*?)\n<[\s\S]*?)?' + '
(.*?)') + + @classmethod + def analyse_jm_search_html(cls, html: str) -> JmSearchPage: + html = cls.pattern_html_search_shorten_for.search(html)[0] + album_info_list = cls.pattern_html_search_album_info_list.findall(html) + + for i, (album_id, title, *args) in enumerate(album_info_list): + _, category_none, label_sub_none, tag_text = args + tag_list = cls.pattern_html_search_tag_list.findall(tag_text) + album_info_list[i] = (album_id, title, category_none, label_sub_none, tag_list) + + return JmSearchPage(album_info_list) + + +class JmImageSupport: + + @classmethod + def save_resp_img(cls, resp: Any, filepath: str, need_convert=True): + """ + 保存图片的响应数据 + @param resp: Response对象 + @param filepath: 响应数据保存的绝对路径 + @param need_convert: True 使用PIL打开图片再保存; False 直接保存resp.content; + 如果需要改变图片的格式,比如 .jpg → .png,则需要neet_convert=True。 + 如果不需要改变文件的格式,使用need_convert=False可以跳过PIL解析图片,效率更高。 + """ + if need_convert is True: + cls.open_Image(resp.content).save(filepath) + else: + save_resp_content(resp, filepath) + + @classmethod + def save_resp_decoded_img(cls, + resp: Any, + img_detail: JmImageDetail, + filepath: str + ) -> None: + cls.decode_and_save( + cls.calculate_segmentation_num(img_detail), + cls.open_Image(resp.content), + filepath + ) + + @classmethod + def decode_disk_img(cls, + img_detail: JmImageDetail, + img_filepath: str, + decoded_save_path: str + ) -> None: + cls.decode_and_save( + cls.calculate_segmentation_num(img_detail), + cls.open_Image(img_filepath), + decoded_save_path + ) + + @classmethod + def decode_and_save(cls, + num: int, + img_src: Image, + decoded_save_path: str + ) -> None: + """ + 解密图片并保存 + @param num: 分割数,可以用 cls.calculate_segmentation_num 计算 + @param img_src: 原始图片 + @param decoded_save_path: 解密图片的保存路径 + """ + + # 无需解密,直接保存 + if num == 0: + img_src.save(decoded_save_path) + return + + import math + w, h = img_src.size + + # 创建新的解密图片 + img_decode = Image.new("RGB", (w, h)) + remainder = h % num + copyW = w + for i in range(num): + copyH = math.floor(h / num) + py = copyH * i + y = h - (copyH * (i + 1)) - remainder + + if i == 0: + copyH += remainder + else: + py += remainder + + img_decode.paste( + img_src.crop((0, y, copyW, y + copyH)), + (0, py, copyW, py + copyH) + ) + + # 保存到新的解密文件 + img_decode.save(decoded_save_path) + + @classmethod + def open_Image(cls, fp: Union[str, bytes]): + from PIL import Image + from io import BytesIO + fp = fp if isinstance(fp, str) else BytesIO(fp) + return Image.open(fp) + + @classmethod + def calculate_segmentation_num(cls, detail: JmImageDetail) -> int: + """ + 获得图片分割数 + """ + scramble_id = int(detail.scramble_id) + aid = int(detail.aid) + + if aid < scramble_id: + return 0 + elif aid < JmModuleConfig.SCRAMBLE_10: + return 10 + else: + import hashlib + x = 10 if aid < JmModuleConfig.SCRAMBLE_NUM_8 else 8 + s = f"{aid}{detail.img_file_name}" # 拼接 + s = s.encode() + s = hashlib.md5(s).hexdigest() + num = ord(s[-1]) + num %= x + num = num * 2 + 2 + return num diff --git a/usage/jmcomic_getting_started.py b/usage/jmcomic_getting_started.py new file mode 100644 index 0000000..3d37691 --- /dev/null +++ b/usage/jmcomic_getting_started.py @@ -0,0 +1,40 @@ +""" +-------------------- + API快速上手 +-------------------- +""" +import jmcomic # 导入此模块,需要先安装. +jmcomic.download_album('422866') # 传入要下载的album的id,即可下载整个album到本地. +# 上面的这行代码,还有一个可选参数option: JmOption,表示配置项, +# 配置项的作用是告诉程序下载时候的一些选择, +# 比如,要下载到哪个文件夹,使用怎样的路径组织方式(比如[/作者/本子id/图片] 或者 [/作者/本子名称/图片]). +# 如果没有配置,则会使用 JmOption.default(),下载的路径是[当前工作文件夹/本子名称/图片]. + + +""" +-------------------- + 批量下载介绍 +-------------------- +""" +# 如果你想要批量下载,可以使用 list/set/tuple/生成器 作为第一个参数。 +# 第二个参数依然是可选的JmOption对象 +jmcomic.download_album(['422866', '1', '2', '3']) # list +jmcomic.download_album({'422866', '1', '2', '3'}) # set +jmcomic.download_album(('422866', '1', '2', '3')) # tuple +jmcomic.download_album(aid for aid in ('422866', '1', '2', '3')) # 生成器 + +""" +-------------------- + 配置文件介绍 +-------------------- +""" +# 先获取默认的JmOption对象 +jm_option = jmcomic.JmOption.default() + +# 可以把对象保存为yml文件,方便日后修改。保存格式会根据后缀名智能转换。 +jm_option.save_to_file('./禁漫下载默认配置.yml') # yml格式,推荐 +jm_option.save_to_file('./禁漫下载默认配置.json') # json格式 + +# 如果你修改了默认配置,现在想用你修改后的配置来下载,使用如下代码 +jm_option = jmcomic.create_option('./禁漫下载默认配置.yml') +jmcomic.download_album('23333', jm_option) \ No newline at end of file diff --git a/usage/jmcomic_usage.py b/usage/jmcomic_usage.py new file mode 100644 index 0000000..2f114ac --- /dev/null +++ b/usage/jmcomic_usage.py @@ -0,0 +1,52 @@ +from jmcomic import * + +jm_option = create_option( + f'你的配置文件路径,例如: D:/a/b/c/jmcomic/config.yml' +) + + +@timeit('下载本子集: ') +def download_jm_album(): + ls = str_to_list(''' + JM435413 + + + ''') + + download_album(ls, jm_option) # 效果同下面的代码 + # download_album_batch(ls, jm_option) + + +@timeit('获取实体类: ') +def get_album_photo_detail(): + client = jm_option.build_jm_client() + album: JmAlbumDetail = client.get_album_detail('427413') + + def show(p): + p: JmPhotoDetail = client.get_photo_detail(p.photo_id) + for img in p: + img: JmImageDetail + print(img.img_url) + + multi_thread_launcher( + iter_objs=album, + apply_each_obj_func=show + ) + + +@timeit('搜索本子: ') +def search_jm_album(): + client = jm_option.build_jm_client() + search_album: JmSearchPage = client.search_album(search_query='+MANA +无修正') + for album_id, title, *_args in search_album: + print(f'[{album_id}]:{title}') + + +def main(): + search_jm_album() + download_jm_album() + get_album_photo_detail() + + +if __name__ == '__main__': + main() diff --git a/usage/jmcomic_workflows.py b/usage/jmcomic_workflows.py new file mode 100644 index 0000000..b2e84cb --- /dev/null +++ b/usage/jmcomic_workflows.py @@ -0,0 +1,17 @@ +from jmcomic import * + +def get_option(): + option = create_option('../assets/config/workflow_option.yml') + return option + + +# 下方填入你要下载的本子的id,一行一个。 +# 每行的首尾可以有空白字符 +jm_albums = str_to_list(''' +422866 + + +''') + +# 调用jmcomic的download_album方法,下载漫画 +download_album(jm_albums, option=get_option())