mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.1.8: 默认启用client-cache、所有Downloader共用一个client,优化代码 (#86)
This commit is contained in:
parent
929dd46552
commit
ec0822478a
@ -2,8 +2,6 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||
|
||||
__version__ = '2.1.7'
|
||||
__version__ = '2.1.8'
|
||||
|
||||
# noinspection PyUnresolvedReferences
|
||||
from common import *
|
||||
from .api import *
|
||||
|
@ -142,7 +142,8 @@ class JmModuleConfig:
|
||||
def get_jmcomic_domain_all(cls, postman=None):
|
||||
"""
|
||||
访问禁漫发布页,得到所有禁漫的域名
|
||||
@return:['18comic.vip', ..., 'jm365.xyz/ZNPJam'], 最后一个是【APP軟件下載】
|
||||
|
||||
@return: ['18comic.vip', ..., 'jm365.xyz/ZNPJam'], 最后一个是【APP軟件下載】
|
||||
"""
|
||||
postman = postman or cls.new_postman(session=True)
|
||||
|
||||
|
@ -50,7 +50,7 @@ class JmDownloader(DownloadCallback):
|
||||
JmDownloader = JmOption + 调度逻辑
|
||||
"""
|
||||
|
||||
def __init__(self, option) -> None:
|
||||
def __init__(self, option: JmOption) -> None:
|
||||
self.option = option
|
||||
self.use_cache = self.option.download_cache
|
||||
self.decode_image = self.option.download_image_decode
|
||||
@ -58,34 +58,32 @@ class JmDownloader(DownloadCallback):
|
||||
def download_album(self, album_id):
|
||||
client = self.client_for_album(album_id)
|
||||
album = client.get_album_detail(album_id)
|
||||
|
||||
self.before_album(album)
|
||||
self.download_by_album_detail(album, client)
|
||||
self.after_album(album)
|
||||
|
||||
def download_by_album_detail(self, album: JmAlbumDetail, client: JmcomicClient):
|
||||
self.before_album(album)
|
||||
self.execute_by_condition(
|
||||
iter_objs=album,
|
||||
apply=lambda photo: self.download_by_photo_detail(photo, client),
|
||||
count_batch=self.option.decide_photo_batch_count(album)
|
||||
)
|
||||
self.after_album(album)
|
||||
|
||||
def download_photo(self, photo_id):
|
||||
client = self.client_for_photo(photo_id)
|
||||
photo = client.get_photo_detail(photo_id)
|
||||
|
||||
self.before_photo(photo)
|
||||
self.download_by_photo_detail(photo, client)
|
||||
self.after_photo(photo)
|
||||
|
||||
def download_by_photo_detail(self, photo: JmPhotoDetail, client: JmcomicClient):
|
||||
client.check_photo(photo)
|
||||
|
||||
self.before_photo(photo)
|
||||
self.execute_by_condition(
|
||||
iter_objs=photo,
|
||||
apply=lambda image: self.download_by_image_detail(image, client),
|
||||
count_batch=self.option.decide_image_batch_count(photo)
|
||||
)
|
||||
self.after_photo(photo)
|
||||
|
||||
def download_by_image_detail(self, image: JmImageDetail, client: JmcomicClient):
|
||||
img_save_path = self.option.decide_image_filepath(image)
|
||||
@ -123,20 +121,18 @@ class JmDownloader(DownloadCallback):
|
||||
)
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
def client_for_album(self, jm_album_id):
|
||||
def client_for_album(self, jm_album_id) -> JmcomicClient:
|
||||
"""
|
||||
默认情况下,每次调用JmDownloader的download_album或download_photo,
|
||||
都会使用一个新的 JmcomicClient
|
||||
默认情况下,所有的JmDownloader共用一个JmcomicClient
|
||||
"""
|
||||
return self.option.new_jm_client()
|
||||
return self.option.build_jm_client()
|
||||
|
||||
# noinspection PyUnusedLocal
|
||||
def client_for_photo(self, jm_photo_id):
|
||||
def client_for_photo(self, jm_photo_id) -> JmcomicClient:
|
||||
"""
|
||||
默认情况下,每次调用JmDownloader的download_album或download_photo,
|
||||
都会使用一个新的 JmcomicClient
|
||||
默认情况下,所有的JmDownloader共用一个JmcomicClient
|
||||
"""
|
||||
return self.option.new_jm_client()
|
||||
return self.option.build_jm_client()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
@ -230,12 +230,17 @@ class JmOption:
|
||||
|
||||
@field_cache("__jm_client_cache__")
|
||||
def build_jm_client(self, **kwargs) -> JmcomicClient:
|
||||
"""
|
||||
该方法会首次调用会创建JmcomicClient对象,
|
||||
然后保存在self.__jm_client_cache__中,
|
||||
多次调用`不会`创建新的JmcomicClient对象
|
||||
"""
|
||||
return self.new_jm_client(**kwargs)
|
||||
|
||||
def new_jm_client(self, **kwargs) -> JmcomicClient:
|
||||
postman_conf: dict = self.client.postman.src_dict
|
||||
|
||||
# support overwrite meta_data
|
||||
# support kwargs overwrite meta_data
|
||||
if len(kwargs) != 0:
|
||||
meta_data = postman_conf.get('meta_data', {})
|
||||
meta_data.update(kwargs)
|
||||
@ -256,6 +261,10 @@ class JmOption:
|
||||
fallback_domain_list=domain_list,
|
||||
)
|
||||
|
||||
# enable cache
|
||||
if self.client.cache is True:
|
||||
client.enable_cache()
|
||||
|
||||
return client
|
||||
|
||||
@classmethod
|
||||
@ -270,6 +279,7 @@ class JmOption:
|
||||
'threading': {'batch_count': 30},
|
||||
},
|
||||
'client': {
|
||||
'cache': True,
|
||||
'domain': [],
|
||||
'postman': {
|
||||
'type': 'cffi',
|
||||
|
@ -34,9 +34,6 @@ class JmTestConfigurable(unittest.TestCase):
|
||||
cls.option = option
|
||||
cls.client = option.build_jm_client()
|
||||
|
||||
# 启用 JmClientClient 缓存
|
||||
cls.enable_client_cache()
|
||||
|
||||
# 跨平台设置
|
||||
cls.adapt_os()
|
||||
|
||||
@ -69,7 +66,3 @@ class JmTestConfigurable(unittest.TestCase):
|
||||
@classmethod
|
||||
def adapt_macos(cls):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def enable_client_cache(cls):
|
||||
cls.client.enable_cache(debug=True)
|
||||
|
Loading…
Reference in New Issue
Block a user