mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v2.1.0: 优化请求重试机制 (#74)
This commit is contained in:
parent
9ab4de6313
commit
58f1b8bb36
@ -2,6 +2,6 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option
|
||||
|
||||
__version__ = '2.0.9'
|
||||
__version__ = '2.1.0'
|
||||
|
||||
from .api import *
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
from .jm_client_interface import *
|
||||
|
||||
|
||||
# noinspection PyAbstractClass
|
||||
class AbstractJmClient(
|
||||
JmcomicClient,
|
||||
PostmanProxy,
|
||||
@ -39,21 +40,32 @@ class AbstractJmClient(
|
||||
retry_count=0,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
统一请求,支持重试
|
||||
@param request: 请求方法
|
||||
@param url: 图片url / path (/album/xxx)
|
||||
@param domain_index: 域名下标
|
||||
@param retry_count: 重试次数
|
||||
@param kwargs: 请求方法的kwargs
|
||||
"""
|
||||
if domain_index >= len(self.domain_list):
|
||||
raise AssertionError("All domains failed.")
|
||||
raise AssertionError(f"请求重试全部失败: [{url}], {self.domain_list}")
|
||||
|
||||
domain = self.domain_list[domain_index]
|
||||
|
||||
if not url.startswith(JmModuleConfig.PROT):
|
||||
if url.startswith('/'):
|
||||
# path
|
||||
domain = self.domain_list[domain_index]
|
||||
url = self.of_api_url(url, domain)
|
||||
jm_debug('api', url)
|
||||
else:
|
||||
# 图片url
|
||||
pass
|
||||
|
||||
if domain_index != 0 and retry_count != 0:
|
||||
if domain_index != 0 or retry_count != 0:
|
||||
jm_debug(
|
||||
f'请求重试',
|
||||
f'request_retry',
|
||||
', '.join([
|
||||
f'次数: [{retry_count}/{self.retry_times}]',
|
||||
f'域名: [{domain} ({domain_index}/{len(self.domain_list)})]',
|
||||
f'域名: [{domain_index} of {self.domain_list}]',
|
||||
f'路径: [{url}]',
|
||||
f'参数: [{kwargs if "login" not in url else "#login_form#"}]'
|
||||
])
|
||||
@ -64,10 +76,11 @@ class AbstractJmClient(
|
||||
except Exception as e:
|
||||
self.before_retry(e, kwargs, retry_count, url)
|
||||
|
||||
if retry_count < self.retry_times:
|
||||
return self.request_with_retry(request, url, domain_index, retry_count + 1, **kwargs)
|
||||
else:
|
||||
return self.request_with_retry(request, url, domain_index + 1, 0, **kwargs)
|
||||
if retry_count < self.retry_times:
|
||||
return self.request_with_retry(request, url, domain_index, retry_count + 1, **kwargs)
|
||||
else:
|
||||
return self.request_with_retry(request, url, domain_index + 1, 0, **kwargs)
|
||||
|
||||
|
||||
# noinspection PyMethodMayBeStatic, PyUnusedLocal
|
||||
def before_retry(self, e, kwargs, retry_count, url):
|
||||
@ -237,7 +250,60 @@ class JmHtmlClient(AbstractJmClient):
|
||||
raise AssertionError(msg)
|
||||
|
||||
def get_jm_image(self, img_url) -> JmImageResp:
|
||||
return JmImageResp(self.get(img_url))
|
||||
|
||||
def get_if_fail_raise(url):
|
||||
"""
|
||||
使用此方法包装 self.get
|
||||
"""
|
||||
resp = JmImageResp(self.get(url))
|
||||
|
||||
if resp.is_success:
|
||||
return resp
|
||||
|
||||
self.raise_request_error(
|
||||
resp.resp, resp.get_error_msg()
|
||||
)
|
||||
|
||||
return resp
|
||||
|
||||
return self.request_with_retry(get_if_fail_raise, img_url)
|
||||
|
||||
def album_comment(self,
|
||||
video_id,
|
||||
comment,
|
||||
originator='',
|
||||
status='true',
|
||||
comment_id=None,
|
||||
**kwargs,
|
||||
) -> JmAcResp:
|
||||
data = {
|
||||
'video_id': video_id,
|
||||
'comment': comment,
|
||||
'originator': originator,
|
||||
'status': status,
|
||||
}
|
||||
|
||||
# 处理回复评论
|
||||
if comment_id is not None:
|
||||
data.pop('status')
|
||||
data['comment_id'] = comment_id
|
||||
data['is_reply'] = 1
|
||||
data['forum_subject'] = 1
|
||||
|
||||
jm_debug('album_comment',
|
||||
f'{video_id}: [{comment}]' +
|
||||
(f' to ({comment_id})' if comment_id is not None else '')
|
||||
)
|
||||
|
||||
resp = self.post('https://18comic.vip/ajax/album_comment',
|
||||
headers=JmModuleConfig.album_comment_headers,
|
||||
data=data,
|
||||
)
|
||||
|
||||
ret = JmAcResp(resp)
|
||||
jm_debug('album_comment', f'{video_id}: [{comment}] ← ({ret.model().cid})')
|
||||
|
||||
return ret
|
||||
|
||||
@classmethod
|
||||
def require_resp_success_else_raise(cls, resp, req_url):
|
||||
|
||||
@ -13,20 +13,31 @@ class JmResp(CommonResp):
|
||||
def is_success(self) -> bool:
|
||||
return self.http_code == 200 and len(self.content) != 0
|
||||
|
||||
def json(self, **kwargs) -> Dict:
|
||||
raise NotImplementedError
|
||||
|
||||
def model(self) -> DictModel:
|
||||
return DictModel(self.json())
|
||||
|
||||
|
||||
class JmImageResp(JmResp):
|
||||
|
||||
def json(self, **kwargs) -> Dict:
|
||||
raise AssertionError
|
||||
|
||||
def require_success(self):
|
||||
if self.is_success:
|
||||
return
|
||||
|
||||
raise AssertionError(self.get_error_msg())
|
||||
|
||||
def get_error_msg(self):
|
||||
msg = f'禁漫图片获取失败: [{self.url}]'
|
||||
if self.http_code != 200:
|
||||
msg += f',http状态码={self.http_code}'
|
||||
if len(self.content) == 0:
|
||||
msg += f',响应数据为空'
|
||||
|
||||
raise AssertionError(msg)
|
||||
return msg
|
||||
|
||||
def transfer_to(self,
|
||||
path,
|
||||
@ -106,15 +117,25 @@ class JmApiResp(JmResp):
|
||||
def json(self, **kwargs) -> Dict:
|
||||
return self.resp.json()
|
||||
|
||||
def model(self) -> DictModel:
|
||||
return DictModel(self.json())
|
||||
|
||||
@property
|
||||
def model_data(self) -> DictModel:
|
||||
self.require_success()
|
||||
return DictModel(self.res_data)
|
||||
|
||||
|
||||
# album-comment
|
||||
class JmAcResp(JmResp):
|
||||
|
||||
def is_success(self) -> bool:
|
||||
return super().is_success and self.json()['err'] is False
|
||||
|
||||
def json(self, **kwargs) -> Dict:
|
||||
return self.resp.json()
|
||||
|
||||
def model(self) -> DictModel:
|
||||
return DictModel(self.json())
|
||||
|
||||
|
||||
"""
|
||||
|
||||
Client Interface
|
||||
@ -154,6 +175,25 @@ class JmUserClient:
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
def album_comment(self,
|
||||
video_id,
|
||||
comment,
|
||||
originator='',
|
||||
status='true',
|
||||
comment_id=None,
|
||||
**kwargs,
|
||||
) -> JmAcResp:
|
||||
"""
|
||||
评论漫画/评论回复
|
||||
@param video_id: album_id/photo_id
|
||||
@param comment: 评论内容
|
||||
@param status: 是否 "有劇透"
|
||||
@param comment_id: 被回复评论的id
|
||||
@param originator:
|
||||
@return: JmAcResp 对象
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class JmImageClient:
|
||||
|
||||
@ -204,6 +244,7 @@ class JmImageClient:
|
||||
return data_original.endswith('.gif')
|
||||
|
||||
|
||||
# noinspection PyAbstractClass
|
||||
class JmcomicClient(
|
||||
JmImageClient,
|
||||
JmDetailClient,
|
||||
|
||||
@ -65,14 +65,14 @@ class JmModuleConfig:
|
||||
return cls.DOMAIN # jmcomic默认域名
|
||||
|
||||
@classmethod
|
||||
def headers(cls, authority=None):
|
||||
def headers(cls, domain='18comic.vip'):
|
||||
return {
|
||||
'authority': authority or '18comic.vip',
|
||||
'authority': domain,
|
||||
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
|
||||
'application/signed-exchange;v=b3;q=0.7',
|
||||
'accept-language': 'zh-CN,zh;q=0.9',
|
||||
'cache-control': 'no-cache',
|
||||
'referer': 'https://18comic.vip',
|
||||
'referer': f'https://{domain}',
|
||||
'pragma': 'no-cache',
|
||||
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
@ -130,6 +130,26 @@ class JmModuleConfig:
|
||||
from .jm_toolkit import JmcomicText
|
||||
return JmcomicText.analyse_jm_pub_html(resp.text)
|
||||
|
||||
album_comment_headers = {
|
||||
'authority': '18comic.vip',
|
||||
'accept': 'application/json, text/javascript, */*; q=0.01',
|
||||
'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
|
||||
'cache-control': 'no-cache',
|
||||
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
|
||||
'origin': 'https://18comic.vip',
|
||||
'pragma': 'no-cache',
|
||||
'referer': 'https://18comic.vip/album/248965/',
|
||||
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'sec-ch-ua-platform': '"Windows"',
|
||||
'sec-fetch-dest': 'empty',
|
||||
'sec-fetch-mode': 'cors',
|
||||
'sec-fetch-site': 'same-origin',
|
||||
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
|
||||
'Chrome/114.0.0.0 Safari/537.36',
|
||||
'x-requested-with': 'XMLHttpRequest',
|
||||
}
|
||||
|
||||
|
||||
jm_debug = JmModuleConfig.jm_debug
|
||||
disable_jm_debug = JmModuleConfig.disable_jm_debug
|
||||
|
||||
@ -10,7 +10,7 @@ class Test_Client(JmTestConfigurable):
|
||||
|
||||
def test_download_image(self):
|
||||
jm_photo_id = 'JM438516'
|
||||
photo_detail = self.client.get_photo_detail(jm_photo_id)
|
||||
photo_detail = self.client.get_photo_detail(jm_photo_id, False)
|
||||
self.client.download_by_image_detail(
|
||||
photo_detail[0],
|
||||
img_save_path=workspace('test_download_image.png')
|
||||
@ -25,7 +25,7 @@ class Test_Client(JmTestConfigurable):
|
||||
测试通过 JmcomicClient 和 jm_photo_id 获取 JmPhotoDetail对象
|
||||
"""
|
||||
jm_photo_id = 'JM438516'
|
||||
photo_detail = self.client.get_photo_detail(jm_photo_id)
|
||||
photo_detail = self.client.get_photo_detail(jm_photo_id, False)
|
||||
photo_detail.when_del_save_file = True
|
||||
photo_detail.after_save_print_info = True
|
||||
del photo_detail
|
||||
@ -46,7 +46,7 @@ class Test_Client(JmTestConfigurable):
|
||||
|
||||
def test_gt_300_photo(self):
|
||||
photo_id = '147643'
|
||||
photo_detail: JmPhotoDetail = self.client.get_photo_detail(photo_id)
|
||||
photo_detail: JmPhotoDetail = self.client.get_photo_detail(photo_id, False)
|
||||
image = photo_detail[3000]
|
||||
print(image.img_url)
|
||||
self.client.download_by_image_detail(image, workspace('3000.png'))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user