mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v1.9.0: 支持登录禁漫、优化正则表达式 (#20)
This commit is contained in:
parent
0c8b3f4cb2
commit
5f0243f369
2
.github/workflows/action_run_tests.yml
vendored
2
.github/workflows/action_run_tests.yml
vendored
@ -12,7 +12,7 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: [ 3.7, 3.8, 3.9, "3.10", "3.11" ]
|
||||
os: [ ubuntu-latest, macos-latest ]
|
||||
os: [ ubuntu-latest ]
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 5
|
||||
|
||||
|
3
.github/workflows/action_workflow_local.yml
vendored
3
.github/workflows/action_workflow_local.yml
vendored
@ -36,6 +36,9 @@ jobs:
|
||||
pip install -e ./
|
||||
|
||||
- name: 运行下载脚本
|
||||
env:
|
||||
JM_USERNAME: ${{ secrets.JM_USERNAME }}
|
||||
JM_PASSWORD: ${{ secrets.JM_PASSWORD }}
|
||||
run: |
|
||||
cd ./usage/
|
||||
python jmcomic_workflows.py
|
||||
|
@ -20,6 +20,4 @@ client_config:
|
||||
- cffi
|
||||
- cffi_Session
|
||||
meta_data:
|
||||
proxies: null
|
||||
allow_redirects: true
|
||||
cookies: null
|
||||
|
@ -1,7 +1,7 @@
|
||||
# 模块依赖关系如下:
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- service <--- option
|
||||
# config <--- entity <--- toolkit <--- client <--- option
|
||||
|
||||
__version__ = '1.8.0'
|
||||
__version__ = '1.9.0'
|
||||
|
||||
from .api import *
|
||||
|
@ -192,6 +192,37 @@ class JmcomicClient(PostmanProxy):
|
||||
def get_jmcomic_url_all(self, postman=None):
|
||||
return JmModuleConfig.get_jmcomic_url_all(postman or self)
|
||||
|
||||
def login(self,
|
||||
username,
|
||||
password,
|
||||
refresh_client_cookies=True,
|
||||
id_remember='on',
|
||||
login_remember='on',
|
||||
):
|
||||
|
||||
data = {
|
||||
'username': username,
|
||||
'password': password,
|
||||
'id_remember': id_remember,
|
||||
'login_remember': login_remember,
|
||||
'submit_login': '',
|
||||
}
|
||||
|
||||
resp = self \
|
||||
.get_root_postman() \
|
||||
.post(self.of_api_url('/login'),
|
||||
data=data,
|
||||
allow_redirects=False,
|
||||
)
|
||||
|
||||
if resp.status_code != 301:
|
||||
raise AssertionError(f'登录失败,状态码为{resp.status_code}')
|
||||
|
||||
if refresh_client_cookies is True:
|
||||
self['cookies'] = resp.cookies
|
||||
|
||||
return resp
|
||||
|
||||
@classmethod
|
||||
def require_resp_success_else_raise(cls, resp, url):
|
||||
# 1. 是否 album_missing
|
||||
@ -212,28 +243,3 @@ class JmcomicClient(PostmanProxy):
|
||||
return
|
||||
|
||||
raise AssertionError(f'{error_msg}' + f': {url}' if url is not None else '')
|
||||
|
||||
|
||||
# 爬取策略
|
||||
class FetchStrategy:
|
||||
|
||||
def __init__(self,
|
||||
from_index,
|
||||
photo_len,
|
||||
resp_getter,
|
||||
resp_consumer,
|
||||
):
|
||||
self.from_index = from_index
|
||||
self.photo_len = photo_len
|
||||
self.resp_getter = resp_getter
|
||||
self.resp_consumer = resp_consumer
|
||||
|
||||
def do_fetch(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def args(self):
|
||||
return (self.from_index,
|
||||
self.photo_len,
|
||||
self.resp_getter,
|
||||
self.resp_consumer,
|
||||
)
|
||||
|
@ -38,7 +38,8 @@ class JmModuleConfig:
|
||||
这样一来,配置文件也不用配置域名了,一切都在运行时动态获取。
|
||||
"""
|
||||
if cls._DOMAIN is None:
|
||||
cls._DOMAIN = cls.get_jmcomic_url(postman).replace(cls.PROT, '')
|
||||
from .jm_toolkit import JmcomicText
|
||||
cls._DOMAIN = JmcomicText.parse_to_jm_domain(cls.get_jmcomic_url(postman))
|
||||
|
||||
return cls._DOMAIN # jmcomic默认域名
|
||||
|
||||
@ -81,7 +82,8 @@ class JmModuleConfig:
|
||||
from common import Postmans
|
||||
postman = Postmans.get_impl_clazz('cffi_Session').create()
|
||||
|
||||
url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL)
|
||||
resp = postman.get(cls.JM_REDIRECT_URL)
|
||||
url = resp.url
|
||||
cls.jm_debug('获取禁漫地址', f'[{cls.JM_REDIRECT_URL}] → [{url}]')
|
||||
return url
|
||||
|
||||
|
@ -126,7 +126,10 @@ class JmPhotoDetail(WorkEntity):
|
||||
return self._series_id == 0
|
||||
|
||||
@property
|
||||
def keyword_list(self) -> List[str]:
|
||||
def keywords(self) -> List[str]:
|
||||
if self.from_album is not None:
|
||||
return self.from_album.keywords
|
||||
|
||||
return self._keywords.split(',')
|
||||
|
||||
@property
|
||||
@ -204,6 +207,7 @@ class JmAlbumDetail(WorkEntity):
|
||||
episode_list,
|
||||
page_count,
|
||||
author_list,
|
||||
keywords_list,
|
||||
pub_date,
|
||||
update_date,
|
||||
):
|
||||
@ -212,6 +216,7 @@ class JmAlbumDetail(WorkEntity):
|
||||
self.title: str = title
|
||||
self.page_count = int(page_count)
|
||||
self._author_list: List[str] = author_list
|
||||
self._keywords_list: List[str] = keywords_list
|
||||
self.pub_date: str = pub_date # 发布日期
|
||||
self.update_date: str = update_date # 更新日期
|
||||
|
||||
@ -254,6 +259,10 @@ class JmAlbumDetail(WorkEntity):
|
||||
return self._author_list[0]
|
||||
return JmModuleConfig.default_author
|
||||
|
||||
@property
|
||||
def keywords(self) -> List[str]:
|
||||
return self._keywords_list
|
||||
|
||||
def get_id(self):
|
||||
return self.album_id
|
||||
|
||||
|
@ -1,4 +1,4 @@
|
||||
from .jm_service import *
|
||||
from .jm_client import *
|
||||
|
||||
|
||||
class JmOptionAdvice:
|
||||
@ -431,58 +431,6 @@ class JmOption(SaveableEntity):
|
||||
|
||||
return client
|
||||
|
||||
"""
|
||||
下面的方法是对【CdnOption】的支持,已弃用
|
||||
"""
|
||||
|
||||
def build_cdn_option(self, use_multi_thread_strategy=True):
|
||||
|
||||
return CdnConfig.create(
|
||||
cdn_domain=self.client_config.get('domain', JmModuleConfig.domain()),
|
||||
fetch_strategy=MultiThreadFetch if use_multi_thread_strategy else InOrderFetch,
|
||||
cdn_image_suffix=None,
|
||||
use_cache=self.download_use_disk_cache,
|
||||
decode_image=self.download_image_then_decode
|
||||
)
|
||||
|
||||
def build_cdn_crawler(self, use_multi_thread_strategy=True):
|
||||
return CdnFetchService(self.build_cdn_option(use_multi_thread_strategy),
|
||||
self.build_jm_client())
|
||||
|
||||
def build_cdn_request(self,
|
||||
photo_id: str,
|
||||
scramble_id=str(JmModuleConfig.SCRAMBLE_10),
|
||||
from_index=1,
|
||||
photo_len=None,
|
||||
use_default=False
|
||||
) -> CdnRequest:
|
||||
return CdnRequest.create(
|
||||
photo_id,
|
||||
self.build_save_path_provider(use_default, None if use_default else photo_id),
|
||||
scramble_id,
|
||||
from_index,
|
||||
photo_len,
|
||||
)
|
||||
|
||||
def build_save_path_provider(self,
|
||||
use_all_default_save_path,
|
||||
photo_id=None,
|
||||
) -> CdnRequest.SavePathProvider:
|
||||
|
||||
if use_all_default_save_path is True:
|
||||
# 不通过请求获取 photo 的信息,相当于使用【空本子】和【空集】
|
||||
photo_detail = None
|
||||
else:
|
||||
# 通过请求获得 photo 的本子信息
|
||||
photo_detail = self.build_jm_client().get_photo_detail(photo_id)
|
||||
|
||||
def save_path_provider(url, _suffix: str, _index, _is_decode):
|
||||
return '{0}{1}{2}'.format(self.decide_image_save_dir(photo_detail),
|
||||
of_file_name(url, trim_suffix=True),
|
||||
self.download_convert_image_suffix or _suffix)
|
||||
|
||||
return save_path_provider
|
||||
|
||||
|
||||
def _register_yaml_constructor():
|
||||
from yaml import add_constructor, Loader, Node
|
||||
|
@ -1,247 +0,0 @@
|
||||
from .jm_client import *
|
||||
|
||||
|
||||
class CdnConfig:
|
||||
|
||||
def __init__(self,
|
||||
cdn_domain: str,
|
||||
cdn_image_suffix: str,
|
||||
fetch_strategy: Type[FetchStrategy],
|
||||
use_cache,
|
||||
decode_image,
|
||||
):
|
||||
self.cdn_domain = cdn_domain
|
||||
self.cdn_image_suffix = cdn_image_suffix
|
||||
self.fetch_strategy = fetch_strategy
|
||||
self.use_cache = use_cache
|
||||
self.decode_image = decode_image
|
||||
|
||||
def get_cdn_image_url(self, photo_id: str, index: int) -> str:
|
||||
return JmModuleConfig.JM_CDN_IMAGE_URL_TEMPLATE.format(
|
||||
domain=self.cdn_domain,
|
||||
photo_id=photo_id,
|
||||
index=index,
|
||||
suffix=self.cdn_image_suffix
|
||||
)
|
||||
|
||||
def check_request_is_valid(self, req: CdnRequest):
|
||||
from common import is_function
|
||||
|
||||
if req.photo_len is not None and not isinstance(req.photo_len, int):
|
||||
raise AssertionError('传参错误,photo_len要么给整数,要么给None')
|
||||
if not is_function(req.save_path_provider):
|
||||
raise AssertionError('传参错误,save_path_provider应该是函数')
|
||||
if self.decode_image is True and req.scramble_id is None:
|
||||
raise AssertionError('传参缺失,指定decode_image=True,就必须提供scramble_id')
|
||||
if req.from_index <= 0:
|
||||
raise AssertionError(f'传参错误,from_index必须大于1: {req.from_index} < 1')
|
||||
|
||||
@classmethod
|
||||
def create(cls,
|
||||
cdn_domain,
|
||||
fetch_strategy,
|
||||
cdn_image_suffix=None,
|
||||
use_cache=False,
|
||||
decode_image=True,
|
||||
):
|
||||
|
||||
return CdnConfig(
|
||||
cdn_domain,
|
||||
cdn_image_suffix or '.webp',
|
||||
fetch_strategy,
|
||||
use_cache,
|
||||
decode_image,
|
||||
)
|
||||
|
||||
|
||||
class InOrderFetch(FetchStrategy):
|
||||
|
||||
def do_fetch(self):
|
||||
index, photo_len, resp_getter, resp_consumer = self.args()
|
||||
|
||||
while True:
|
||||
if photo_len is not None and index > photo_len:
|
||||
break
|
||||
|
||||
resp_info = resp_getter(index)
|
||||
if resp_info[0] is None:
|
||||
break
|
||||
resp_consumer(resp_info, index)
|
||||
index += 1
|
||||
|
||||
|
||||
class MultiThreadFetch(FetchStrategy):
|
||||
fetch_batch_count = 5
|
||||
|
||||
def do_fetch(self):
|
||||
from common import multi_thread_launcher
|
||||
begin, photo_len, resp_getter, resp_consumer = self.args()
|
||||
|
||||
if photo_len is not None:
|
||||
# 确定要爬多少页
|
||||
|
||||
def accpet_index(index):
|
||||
resp_info = resp_getter(index)
|
||||
if resp_info[0] is None:
|
||||
return
|
||||
resp_consumer(resp_info, index)
|
||||
|
||||
multi_thread_launcher(
|
||||
iter_objs=range(begin, photo_len + 1),
|
||||
apply_each_obj_func=accpet_index,
|
||||
wait_finish=True,
|
||||
)
|
||||
else:
|
||||
# 不确定要爬多少页
|
||||
|
||||
# 批量单位
|
||||
batch_count = self.fetch_batch_count
|
||||
# 标记位
|
||||
fetch_suffer_none_resp = False
|
||||
|
||||
def accpet_index(index):
|
||||
resp_info = resp_getter(index)
|
||||
if resp_info is None:
|
||||
nonlocal fetch_suffer_none_resp
|
||||
fetch_suffer_none_resp = True
|
||||
return
|
||||
resp_consumer(resp_info, index)
|
||||
|
||||
while True:
|
||||
# 5次5次爬,每5次中,但凡有1次出现了失败,就停止
|
||||
|
||||
multi_thread_launcher(
|
||||
iter_objs=range(begin, begin + batch_count),
|
||||
apply_each_obj_func=accpet_index,
|
||||
wait_finish=True
|
||||
)
|
||||
|
||||
# 检查标记
|
||||
if fetch_suffer_none_resp is False:
|
||||
begin += batch_count
|
||||
else:
|
||||
break
|
||||
|
||||
|
||||
class CdnFetchService:
|
||||
|
||||
def __init__(self,
|
||||
config: CdnConfig,
|
||||
client: 'JmcomicClient',
|
||||
):
|
||||
self.client = client
|
||||
self.config = config
|
||||
|
||||
def download_photo_from_cdn_directly(self,
|
||||
req: CdnRequest,
|
||||
):
|
||||
# 校验参数
|
||||
self.config.check_request_is_valid(req)
|
||||
|
||||
# 基本信息
|
||||
photo_id = req.photo_id
|
||||
scramble_id = req.scramble_id
|
||||
use_cache = self.config.use_cache
|
||||
decode_image = self.config.decode_image
|
||||
|
||||
# 获得响应
|
||||
def get_resp(index: int) -> Tuple[Optional[Any], str, str]:
|
||||
url = self.config.get_cdn_image_url(photo_id, index)
|
||||
suffix = self.config.cdn_image_suffix
|
||||
pre_try_save_path = req.save_path_provider(url, suffix, index, decode_image)
|
||||
|
||||
# 判断是否命中缓存(预先尝试)
|
||||
if use_cache is True and file_exists(pre_try_save_path):
|
||||
# 命中,返回特殊值
|
||||
return None, suffix, pre_try_save_path
|
||||
else:
|
||||
return self.try_get_cdn_image_resp(
|
||||
self.client,
|
||||
url,
|
||||
suffix,
|
||||
photo_id,
|
||||
index,
|
||||
)
|
||||
|
||||
# 保存响应
|
||||
def save_resp(resp_info: Tuple[Optional[Any], str, str], index: int):
|
||||
resp, suffix, img_url = resp_info
|
||||
|
||||
# 1. 判断是不是特殊值
|
||||
if resp_info[0] is None:
|
||||
# 是,表示命中缓存,直接返回
|
||||
save_path = resp_info[2]
|
||||
jm_debug('图片下载成功',
|
||||
f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})')
|
||||
return
|
||||
|
||||
# 2. 判断是否命中缓存
|
||||
save_path = req.save_path_provider(img_url, suffix, index, decode_image)
|
||||
if use_cache is True and file_exists(save_path):
|
||||
# 命中,直接返回
|
||||
jm_debug('图片下载成功',
|
||||
f'photo-{photo_id}: {index}{suffix}命中磁盘缓存 ({save_path})')
|
||||
return
|
||||
|
||||
# 3. 保存图片
|
||||
if decode_image is False:
|
||||
JmImageSupport.save_resp_img(resp, save_path)
|
||||
else:
|
||||
JmImageSupport.save_resp_decoded_img(
|
||||
resp,
|
||||
JmImageDetail.of(photo_id, scramble_id, data_original=img_url),
|
||||
save_path,
|
||||
)
|
||||
|
||||
# 4. debug 消息
|
||||
jm_debug('图片下载成功',
|
||||
f'photo-{photo_id}: {index}{suffix}下载完成 ('
|
||||
+ ('已解码' if decode_image is True else '未解码') +
|
||||
f') [{img_url}] → [{save_path}]')
|
||||
|
||||
# 调用爬虫策略
|
||||
self.config.fetch_strategy(
|
||||
req.from_index,
|
||||
req.photo_len,
|
||||
resp_getter=get_resp,
|
||||
resp_consumer=save_resp,
|
||||
).do_fetch()
|
||||
|
||||
# 准备提供给爬虫策略的函数
|
||||
@staticmethod
|
||||
def try_get_cdn_image_resp(client: JmcomicClient,
|
||||
url: str,
|
||||
suffix: str,
|
||||
photo_id,
|
||||
index,
|
||||
) -> Optional[Tuple[Any, str, str]]:
|
||||
resp = client.jm_get(url, False, False)
|
||||
|
||||
# 第一次校验,不空则直接返回
|
||||
if not client.is_empty_image(resp):
|
||||
return resp, suffix, url
|
||||
|
||||
# 下面进行重试
|
||||
jm_debug(
|
||||
'图片获取重试',
|
||||
f'photo-{photo_id},图片序号为{index},url=[{url}]'
|
||||
)
|
||||
|
||||
# 重试点1:是否文件后缀名不对?
|
||||
for alter_suffix in JmModuleConfig.JM_IMAGE_SUFFIX:
|
||||
url = change_file_suffix(url, alter_suffix)
|
||||
resp = client.jm_get(url, False, False)
|
||||
if not client.is_empty_image(resp):
|
||||
jm_debug(
|
||||
'图片获取重试 → 成功√',
|
||||
f'更改请求后缀({suffix} -> {alter_suffix}),url={url}'
|
||||
)
|
||||
return resp, alter_suffix, url
|
||||
|
||||
# 结论:可能是图片到头了
|
||||
jm_debug(
|
||||
'图片获取重试 ← 失败×',
|
||||
f'更换后缀名不成功,停止爬取。'
|
||||
f'(推断本子长度<={index - 1}。当前图片序号为{index},已经到达尽头,'
|
||||
f'photo-{photo_id})'
|
||||
)
|
@ -13,7 +13,7 @@ class JmcomicText:
|
||||
pattern_html_photo_title = compile('<title>(.*?)\|.*</title>')
|
||||
# pattern_html_photo_data_original_list = compile('data-original="(.*?)" id="album_photo_.+?"')
|
||||
pattern_html_photo_data_original_domain = compile('src="https://(.*?)/media/albums/blank')
|
||||
pattern_html_photo_keywords = compile('<meta name="keywords" content="(.*?)" />')
|
||||
pattern_html_photo_keywords = compile('<meta name="keywords"[\s\S]*?content="(.*?)"')
|
||||
pattern_html_photo_series_id = compile('var series_id = (\d+);')
|
||||
pattern_html_photo_sort = compile('var sort = (\d+);')
|
||||
pattern_html_photo_page_arr = compile('var page_arr = (.*?);')
|
||||
@ -27,6 +27,10 @@ class JmcomicText:
|
||||
pattern_html_album_page_count = compile('<span class="pagecount">.*?:(\d+)</span>')
|
||||
pattern_html_album_pub_date = compile('>上架日期 : (.*?)</span>')
|
||||
pattern_html_album_update_date = compile('>更新日期 : (.*?)</span>')
|
||||
pattern_html_album_keywords_list = [
|
||||
compile('<span itemprop="genre" data-type="tags">([\s\S]*?)</span>'),
|
||||
compile('<a[\s\S]*?>(.*?)</a>')
|
||||
]
|
||||
|
||||
# album 作者
|
||||
pattern_html_album_author_list = [
|
||||
|
@ -8,14 +8,6 @@ class Test_Client(JmTestConfigurable):
|
||||
super().setUpClass()
|
||||
cls.move_workspace('download')
|
||||
|
||||
def test_download_from_cdn_directly(self):
|
||||
photo_id = 'JM438516'
|
||||
option = self.option
|
||||
cdn_crawler = option.build_cdn_crawler()
|
||||
cdn_crawler.download_photo_from_cdn_directly(
|
||||
option.build_cdn_request(photo_id),
|
||||
)
|
||||
|
||||
def test_download_image(self):
|
||||
jm_photo_id = 'JM438516'
|
||||
photo_detail = self.client.get_photo_detail(jm_photo_id)
|
||||
|
@ -8,7 +8,7 @@ jm_option = create_option(
|
||||
@timeit('下载本子集: ')
|
||||
def download_jm_album():
|
||||
ls = str_to_list('''
|
||||
JM435413
|
||||
438696
|
||||
|
||||
|
||||
''')
|
||||
|
@ -1,18 +1,47 @@
|
||||
from jmcomic import *
|
||||
|
||||
def get_option():
|
||||
option = create_option('../assets/config/workflow_option.yml')
|
||||
return option
|
||||
|
||||
|
||||
# 下方填入你要下载的本子的id,一行一个。
|
||||
# 每行的首尾可以有空白字符
|
||||
jm_albums = str_to_list('''
|
||||
422866
|
||||
jm_albums = '''
|
||||
380460
|
||||
|
||||
|
||||
|
||||
''')
|
||||
'''
|
||||
|
||||
# 调用jmcomic的download_album方法,下载漫画
|
||||
download_album(jm_albums, option=get_option())
|
||||
|
||||
def main():
|
||||
from jmcomic import create_option, str_to_list, download_album, print_eye_catching
|
||||
|
||||
def get_option():
|
||||
# 读取 option 配置文件
|
||||
option = create_option('../assets/config/workflow_option.yml')
|
||||
|
||||
# 启用 client 的缓存
|
||||
client = option.build_jm_client()
|
||||
client.enable_cache()
|
||||
|
||||
# 检查环境变量中是否有禁漫的用户名和密码,如果有则登录
|
||||
|
||||
def get_env(name):
|
||||
import os
|
||||
value = os.getenv(name, None)
|
||||
|
||||
if value is None or value == '':
|
||||
return None
|
||||
|
||||
return value
|
||||
|
||||
username = get_env('JM_USERNAME')
|
||||
password = get_env('JM_PASSWORD')
|
||||
|
||||
if username is not None and password is not None:
|
||||
client.login(username, password, True)
|
||||
print_eye_catching(f'登录禁漫成功')
|
||||
|
||||
return option
|
||||
|
||||
# 调用jmcomic的download_album方法,下载漫画
|
||||
download_album(str_to_list(jm_albums), option=get_option())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
Loading…
Reference in New Issue
Block a user