v1.9.0: 支持登录禁漫、优化正则表达式 (#20)

This commit is contained in:
hect0x7 2023-04-27 20:07:40 +08:00 committed by GitHub
parent 0c8b3f4cb2
commit 5f0243f369
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 99 additions and 355 deletions

View File

@ -12,7 +12,7 @@ jobs:
strategy:
matrix:
python-version: [ 3.7, 3.8, 3.9, "3.10", "3.11" ]
os: [ ubuntu-latest, macos-latest ]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
timeout-minutes: 5

View File

@ -36,6 +36,9 @@ jobs:
pip install -e ./
- name: 运行下载脚本
env:
JM_USERNAME: ${{ secrets.JM_USERNAME }}
JM_PASSWORD: ${{ secrets.JM_PASSWORD }}
run: |
cd ./usage/
python jmcomic_workflows.py

View File

@ -20,6 +20,4 @@ client_config:
- cffi
- cffi_Session
meta_data:
proxies: null
allow_redirects: true
cookies: null

View File

@ -1,7 +1,7 @@
# 模块依赖关系如下:
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- service <--- option
# config <--- entity <--- toolkit <--- client <--- option
__version__ = '1.8.0'
__version__ = '1.9.0'
from .api import *

View File

@ -192,6 +192,37 @@ class JmcomicClient(PostmanProxy):
def get_jmcomic_url_all(self, postman=None):
return JmModuleConfig.get_jmcomic_url_all(postman or self)
def login(self,
username,
password,
refresh_client_cookies=True,
id_remember='on',
login_remember='on',
):
data = {
'username': username,
'password': password,
'id_remember': id_remember,
'login_remember': login_remember,
'submit_login': '',
}
resp = self \
.get_root_postman() \
.post(self.of_api_url('/login'),
data=data,
allow_redirects=False,
)
if resp.status_code != 301:
raise AssertionError(f'登录失败,状态码为{resp.status_code}')
if refresh_client_cookies is True:
self['cookies'] = resp.cookies
return resp
@classmethod
def require_resp_success_else_raise(cls, resp, url):
# 1. 是否 album_missing
@ -212,28 +243,3 @@ class JmcomicClient(PostmanProxy):
return
raise AssertionError(f'{error_msg}' + f': {url}' if url is not None else '')
# 爬取策略
class FetchStrategy:
def __init__(self,
from_index,
photo_len,
resp_getter,
resp_consumer,
):
self.from_index = from_index
self.photo_len = photo_len
self.resp_getter = resp_getter
self.resp_consumer = resp_consumer
def do_fetch(self):
raise NotImplementedError
def args(self):
return (self.from_index,
self.photo_len,
self.resp_getter,
self.resp_consumer,
)

View File

@ -38,7 +38,8 @@ class JmModuleConfig:
这样一来配置文件也不用配置域名了一切都在运行时动态获取
"""
if cls._DOMAIN is None:
cls._DOMAIN = cls.get_jmcomic_url(postman).replace(cls.PROT, '')
from .jm_toolkit import JmcomicText
cls._DOMAIN = JmcomicText.parse_to_jm_domain(cls.get_jmcomic_url(postman))
return cls._DOMAIN # jmcomic默认域名
@ -81,7 +82,8 @@ class JmModuleConfig:
from common import Postmans
postman = Postmans.get_impl_clazz('cffi_Session').create()
url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL)
resp = postman.get(cls.JM_REDIRECT_URL)
url = resp.url
cls.jm_debug('获取禁漫地址', f'[{cls.JM_REDIRECT_URL}] → [{url}]')
return url

View File

@ -126,7 +126,10 @@ class JmPhotoDetail(WorkEntity):
return self._series_id == 0
@property
def keyword_list(self) -> List[str]:
def keywords(self) -> List[str]:
if self.from_album is not None:
return self.from_album.keywords
return self._keywords.split(',')
@property
@ -204,6 +207,7 @@ class JmAlbumDetail(WorkEntity):
episode_list,
page_count,
author_list,
keywords_list,
pub_date,
update_date,
):
@ -212,6 +216,7 @@ class JmAlbumDetail(WorkEntity):
self.title: str = title
self.page_count = int(page_count)
self._author_list: List[str] = author_list
self._keywords_list: List[str] = keywords_list
self.pub_date: str = pub_date # 发布日期
self.update_date: str = update_date # 更新日期
@ -254,6 +259,10 @@ class JmAlbumDetail(WorkEntity):
return self._author_list[0]
return JmModuleConfig.default_author
@property
def keywords(self) -> List[str]:
return self._keywords_list
def get_id(self):
return self.album_id

View File

@ -1,4 +1,4 @@
from .jm_service import *
from .jm_client import *
class JmOptionAdvice:
@ -431,58 +431,6 @@ class JmOption(SaveableEntity):
return client
"""
下面的方法是对CdnOption的支持已弃用
"""
def build_cdn_option(self, use_multi_thread_strategy=True):
return CdnConfig.create(
cdn_domain=self.client_config.get('domain', JmModuleConfig.domain()),
fetch_strategy=MultiThreadFetch if use_multi_thread_strategy else InOrderFetch,
cdn_image_suffix=None,
use_cache=self.download_use_disk_cache,
decode_image=self.download_image_then_decode
)
def build_cdn_crawler(self, use_multi_thread_strategy=True):
return CdnFetchService(self.build_cdn_option(use_multi_thread_strategy),
self.build_jm_client())
def build_cdn_request(self,
photo_id: str,
scramble_id=str(JmModuleConfig.SCRAMBLE_10),
from_index=1,
photo_len=None,
use_default=False
) -> CdnRequest:
return CdnRequest.create(
photo_id,
self.build_save_path_provider(use_default, None if use_default else photo_id),
scramble_id,
from_index,
photo_len,
)
def build_save_path_provider(self,
use_all_default_save_path,
photo_id=None,
) -> CdnRequest.SavePathProvider:
if use_all_default_save_path is True:
# 不通过请求获取 photo 的信息,相当于使用【空本子】和【空集】
photo_detail = None
else:
# 通过请求获得 photo 的本子信息
photo_detail = self.build_jm_client().get_photo_detail(photo_id)
def save_path_provider(url, _suffix: str, _index, _is_decode):
return '{0}{1}{2}'.format(self.decide_image_save_dir(photo_detail),
of_file_name(url, trim_suffix=True),
self.download_convert_image_suffix or _suffix)
return save_path_provider
def _register_yaml_constructor():
from yaml import add_constructor, Loader, Node

View File

@ -1,247 +0,0 @@
from .jm_client import *
class CdnConfig:
def __init__(self,
cdn_domain: str,
cdn_image_suffix: str,
fetch_strategy: Type[FetchStrategy],
use_cache,
decode_image,
):
self.cdn_domain = cdn_domain
self.cdn_image_suffix = cdn_image_suffix
self.fetch_strategy = fetch_strategy
self.use_cache = use_cache
self.decode_image = decode_image
def get_cdn_image_url(self, photo_id: str, index: int) -> str:
return JmModuleConfig.JM_CDN_IMAGE_URL_TEMPLATE.format(
domain=self.cdn_domain,
photo_id=photo_id,
index=index,
suffix=self.cdn_image_suffix
)
def check_request_is_valid(self, req: CdnRequest):
from common import is_function
if req.photo_len is not None and not isinstance(req.photo_len, int):
raise AssertionError('传参错误photo_len要么给整数要么给None')
if not is_function(req.save_path_provider):
raise AssertionError('传参错误save_path_provider应该是函数')
if self.decode_image is True and req.scramble_id is None:
raise AssertionError('传参缺失指定decode_image=True就必须提供scramble_id')
if req.from_index <= 0:
raise AssertionError(f'传参错误from_index必须大于1: {req.from_index} < 1')
@classmethod
def create(cls,
cdn_domain,
fetch_strategy,
cdn_image_suffix=None,
use_cache=False,
decode_image=True,
):
return CdnConfig(
cdn_domain,
cdn_image_suffix or '.webp',
fetch_strategy,
use_cache,
decode_image,
)
class InOrderFetch(FetchStrategy):
def do_fetch(self):
index, photo_len, resp_getter, resp_consumer = self.args()
while True:
if photo_len is not None and index > photo_len:
break
resp_info = resp_getter(index)
if resp_info[0] is None:
break
resp_consumer(resp_info, index)
index += 1
class MultiThreadFetch(FetchStrategy):
fetch_batch_count = 5
def do_fetch(self):
from common import multi_thread_launcher
begin, photo_len, resp_getter, resp_consumer = self.args()
if photo_len is not None:
# 确定要爬多少页
def accpet_index(index):
resp_info = resp_getter(index)
if resp_info[0] is None:
return
resp_consumer(resp_info, index)
multi_thread_launcher(
iter_objs=range(begin, photo_len + 1),
apply_each_obj_func=accpet_index,
wait_finish=True,
)
else:
# 不确定要爬多少页
# 批量单位
batch_count = self.fetch_batch_count
# 标记位
fetch_suffer_none_resp = False
def accpet_index(index):
resp_info = resp_getter(index)
if resp_info is None:
nonlocal fetch_suffer_none_resp
fetch_suffer_none_resp = True
return
resp_consumer(resp_info, index)
while True:
# 5次5次爬每5次中但凡有1次出现了失败就停止
multi_thread_launcher(
iter_objs=range(begin, begin + batch_count),
apply_each_obj_func=accpet_index,
wait_finish=True
)
# 检查标记
if fetch_suffer_none_resp is False:
begin += batch_count
else:
break
class CdnFetchService:
def __init__(self,
config: CdnConfig,
client: 'JmcomicClient',
):
self.client = client
self.config = config
def download_photo_from_cdn_directly(self,
req: CdnRequest,
):
# 校验参数
self.config.check_request_is_valid(req)
# 基本信息
photo_id = req.photo_id
scramble_id = req.scramble_id
use_cache = self.config.use_cache
decode_image = self.config.decode_image
# 获得响应
def get_resp(index: int) -> Tuple[Optional[Any], str, str]:
url = self.config.get_cdn_image_url(photo_id, index)
suffix = self.config.cdn_image_suffix
pre_try_save_path = req.save_path_provider(url, suffix, index, decode_image)
# 判断是否命中缓存(预先尝试)
if use_cache is True and file_exists(pre_try_save_path):
# 命中,返回特殊值
return None, suffix, pre_try_save_path
else:
return self.try_get_cdn_image_resp(
self.client,
url,
suffix,
photo_id,
index,
)
# 保存响应
def save_resp(resp_info: Tuple[Optional[Any], str, str], index: int):
resp, suffix, img_url = resp_info
# 1. 判断是不是特殊值
if resp_info[0] is None:
# 是,表示命中缓存,直接返回
save_path = resp_info[2]
jm_debug('图片下载成功',
f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})')
return
# 2. 判断是否命中缓存
save_path = req.save_path_provider(img_url, suffix, index, decode_image)
if use_cache is True and file_exists(save_path):
# 命中,直接返回
jm_debug('图片下载成功',
f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})')
return
# 3. 保存图片
if decode_image is False:
JmImageSupport.save_resp_img(resp, save_path)
else:
JmImageSupport.save_resp_decoded_img(
resp,
JmImageDetail.of(photo_id, scramble_id, data_original=img_url),
save_path,
)
# 4. debug 消息
jm_debug('图片下载成功',
f'photo-{photo_id}: {index}{suffix}下载完成 ('
+ ('已解码' if decode_image is True else '未解码') +
f') [{img_url}] → [{save_path}]')
# 调用爬虫策略
self.config.fetch_strategy(
req.from_index,
req.photo_len,
resp_getter=get_resp,
resp_consumer=save_resp,
).do_fetch()
# 准备提供给爬虫策略的函数
@staticmethod
def try_get_cdn_image_resp(client: JmcomicClient,
url: str,
suffix: str,
photo_id,
index,
) -> Optional[Tuple[Any, str, str]]:
resp = client.jm_get(url, False, False)
# 第一次校验,不空则直接返回
if not client.is_empty_image(resp):
return resp, suffix, url
# 下面进行重试
jm_debug(
'图片获取重试',
f'photo-{photo_id},图片序号为{index}url=[{url}]'
)
# 重试点1是否文件后缀名不对
for alter_suffix in JmModuleConfig.JM_IMAGE_SUFFIX:
url = change_file_suffix(url, alter_suffix)
resp = client.jm_get(url, False, False)
if not client.is_empty_image(resp):
jm_debug(
'图片获取重试 → 成功√',
f'更改请求后缀({suffix} -> {alter_suffix}url={url}'
)
return resp, alter_suffix, url
# 结论:可能是图片到头了
jm_debug(
'图片获取重试 ← 失败×',
f'更换后缀名不成功,停止爬取。'
f'(推断本子长度<={index - 1}。当前图片序号为{index},已经到达尽头,'
f'photo-{photo_id})'
)

View File

@ -13,7 +13,7 @@ class JmcomicText:
pattern_html_photo_title = compile('<title>(.*?)\|.*</title>')
# pattern_html_photo_data_original_list = compile('data-original="(.*?)" id="album_photo_.+?"')
pattern_html_photo_data_original_domain = compile('src="https://(.*?)/media/albums/blank')
pattern_html_photo_keywords = compile('<meta name="keywords" content="(.*?)" />')
pattern_html_photo_keywords = compile('<meta name="keywords"[\s\S]*?content="(.*?)"')
pattern_html_photo_series_id = compile('var series_id = (\d+);')
pattern_html_photo_sort = compile('var sort = (\d+);')
pattern_html_photo_page_arr = compile('var page_arr = (.*?);')
@ -27,6 +27,10 @@ class JmcomicText:
pattern_html_album_page_count = compile('<span class="pagecount">.*?:(\d+)</span>')
pattern_html_album_pub_date = compile('>上架日期 : (.*?)</span>')
pattern_html_album_update_date = compile('>更新日期 : (.*?)</span>')
pattern_html_album_keywords_list = [
compile('<span itemprop="genre" data-type="tags">([\s\S]*?)</span>'),
compile('<a[\s\S]*?>(.*?)</a>')
]
# album 作者
pattern_html_album_author_list = [

View File

@ -8,14 +8,6 @@ class Test_Client(JmTestConfigurable):
super().setUpClass()
cls.move_workspace('download')
def test_download_from_cdn_directly(self):
photo_id = 'JM438516'
option = self.option
cdn_crawler = option.build_cdn_crawler()
cdn_crawler.download_photo_from_cdn_directly(
option.build_cdn_request(photo_id),
)
def test_download_image(self):
jm_photo_id = 'JM438516'
photo_detail = self.client.get_photo_detail(jm_photo_id)

View File

@ -8,7 +8,7 @@ jm_option = create_option(
@timeit('下载本子集: ')
def download_jm_album():
ls = str_to_list('''
JM435413
438696
''')

View File

@ -1,18 +1,47 @@
from jmcomic import *
def get_option():
option = create_option('../assets/config/workflow_option.yml')
return option
# 下方填入你要下载的本子的id一行一个。
# 每行的首尾可以有空白字符
jm_albums = str_to_list('''
422866
jm_albums = '''
380460
''')
'''
# 调用jmcomic的download_album方法下载漫画
download_album(jm_albums, option=get_option())
def main():
from jmcomic import create_option, str_to_list, download_album, print_eye_catching
def get_option():
# 读取 option 配置文件
option = create_option('../assets/config/workflow_option.yml')
# 启用 client 的缓存
client = option.build_jm_client()
client.enable_cache()
# 检查环境变量中是否有禁漫的用户名和密码,如果有则登录
def get_env(name):
import os
value = os.getenv(name, None)
if value is None or value == '':
return None
return value
username = get_env('JM_USERNAME')
password = get_env('JM_PASSWORD')
if username is not None and password is not None:
client.login(username, password, True)
print_eye_catching(f'登录禁漫成功')
return option
# 调用jmcomic的download_album方法下载漫画
download_album(str_to_list(jm_albums), option=get_option())
if __name__ == '__main__':
main()