v2.1.0: 优化请求重试机制 (#74)

This commit is contained in:
hect0x7 2023-07-13 17:13:37 +08:00 committed by GitHub
parent 9ab4de6313
commit 58f1b8bb36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 151 additions and 24 deletions

View File

@ -2,6 +2,6 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option
__version__ = '2.0.9'
__version__ = '2.1.0'
from .api import *

View File

@ -1,6 +1,7 @@
from .jm_client_interface import *
# noinspection PyAbstractClass
class AbstractJmClient(
JmcomicClient,
PostmanProxy,
@ -39,21 +40,32 @@ class AbstractJmClient(
retry_count=0,
**kwargs,
):
"""
统一请求支持重试
@param request: 请求方法
@param url: 图片url / path (/album/xxx)
@param domain_index: 域名下标
@param retry_count: 重试次数
@param kwargs: 请求方法的kwargs
"""
if domain_index >= len(self.domain_list):
raise AssertionError("All domains failed.")
raise AssertionError(f"请求重试全部失败: [{url}], {self.domain_list}")
domain = self.domain_list[domain_index]
if not url.startswith(JmModuleConfig.PROT):
if url.startswith('/'):
# path
domain = self.domain_list[domain_index]
url = self.of_api_url(url, domain)
jm_debug('api', url)
else:
# 图片url
pass
if domain_index != 0 and retry_count != 0:
if domain_index != 0 or retry_count != 0:
jm_debug(
f'请求重试',
f'request_retry',
', '.join([
f'次数: [{retry_count}/{self.retry_times}]',
f'域名: [{domain} ({domain_index}/{len(self.domain_list)})]',
f'域名: [{domain_index} of {self.domain_list}]',
f'路径: [{url}]',
f'参数: [{kwargs if "login" not in url else "#login_form#"}]'
])
@ -64,10 +76,11 @@ class AbstractJmClient(
except Exception as e:
self.before_retry(e, kwargs, retry_count, url)
if retry_count < self.retry_times:
return self.request_with_retry(request, url, domain_index, retry_count + 1, **kwargs)
else:
return self.request_with_retry(request, url, domain_index + 1, 0, **kwargs)
if retry_count < self.retry_times:
return self.request_with_retry(request, url, domain_index, retry_count + 1, **kwargs)
else:
return self.request_with_retry(request, url, domain_index + 1, 0, **kwargs)
# noinspection PyMethodMayBeStatic, PyUnusedLocal
def before_retry(self, e, kwargs, retry_count, url):
@ -237,7 +250,60 @@ class JmHtmlClient(AbstractJmClient):
raise AssertionError(msg)
def get_jm_image(self, img_url) -> JmImageResp:
return JmImageResp(self.get(img_url))
def get_if_fail_raise(url):
"""
使用此方法包装 self.get
"""
resp = JmImageResp(self.get(url))
if resp.is_success:
return resp
self.raise_request_error(
resp.resp, resp.get_error_msg()
)
return resp
return self.request_with_retry(get_if_fail_raise, img_url)
def album_comment(self,
video_id,
comment,
originator='',
status='true',
comment_id=None,
**kwargs,
) -> JmAcResp:
data = {
'video_id': video_id,
'comment': comment,
'originator': originator,
'status': status,
}
# 处理回复评论
if comment_id is not None:
data.pop('status')
data['comment_id'] = comment_id
data['is_reply'] = 1
data['forum_subject'] = 1
jm_debug('album_comment',
f'{video_id}: [{comment}]' +
(f' to ({comment_id})' if comment_id is not None else '')
)
resp = self.post('https://18comic.vip/ajax/album_comment',
headers=JmModuleConfig.album_comment_headers,
data=data,
)
ret = JmAcResp(resp)
jm_debug('album_comment', f'{video_id}: [{comment}] ← ({ret.model().cid})')
return ret
@classmethod
def require_resp_success_else_raise(cls, resp, req_url):

View File

@ -13,20 +13,31 @@ class JmResp(CommonResp):
def is_success(self) -> bool:
return self.http_code == 200 and len(self.content) != 0
def json(self, **kwargs) -> Dict:
raise NotImplementedError
def model(self) -> DictModel:
return DictModel(self.json())
class JmImageResp(JmResp):
def json(self, **kwargs) -> Dict:
raise AssertionError
def require_success(self):
if self.is_success:
return
raise AssertionError(self.get_error_msg())
def get_error_msg(self):
msg = f'禁漫图片获取失败: [{self.url}]'
if self.http_code != 200:
msg += f'http状态码={self.http_code}'
if len(self.content) == 0:
msg += f',响应数据为空'
raise AssertionError(msg)
return msg
def transfer_to(self,
path,
@ -106,15 +117,25 @@ class JmApiResp(JmResp):
def json(self, **kwargs) -> Dict:
return self.resp.json()
def model(self) -> DictModel:
return DictModel(self.json())
@property
def model_data(self) -> DictModel:
self.require_success()
return DictModel(self.res_data)
# album-comment
class JmAcResp(JmResp):
def is_success(self) -> bool:
return super().is_success and self.json()['err'] is False
def json(self, **kwargs) -> Dict:
return self.resp.json()
def model(self) -> DictModel:
return DictModel(self.json())
"""
Client Interface
@ -154,6 +175,25 @@ class JmUserClient:
):
raise NotImplementedError
def album_comment(self,
video_id,
comment,
originator='',
status='true',
comment_id=None,
**kwargs,
) -> JmAcResp:
"""
评论漫画/评论回复
@param video_id: album_id/photo_id
@param comment: 评论内容
@param status: 是否 "有劇透"
@param comment_id: 被回复评论的id
@param originator:
@return: JmAcResp 对象
"""
raise NotImplementedError
class JmImageClient:
@ -204,6 +244,7 @@ class JmImageClient:
return data_original.endswith('.gif')
# noinspection PyAbstractClass
class JmcomicClient(
JmImageClient,
JmDetailClient,

View File

@ -65,14 +65,14 @@ class JmModuleConfig:
return cls.DOMAIN # jmcomic默认域名
@classmethod
def headers(cls, authority=None):
def headers(cls, domain='18comic.vip'):
return {
'authority': authority or '18comic.vip',
'authority': domain,
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
'application/signed-exchange;v=b3;q=0.7',
'accept-language': 'zh-CN,zh;q=0.9',
'cache-control': 'no-cache',
'referer': 'https://18comic.vip',
'referer': f'https://{domain}',
'pragma': 'no-cache',
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
'sec-ch-ua-mobile': '?0',
@ -130,6 +130,26 @@ class JmModuleConfig:
from .jm_toolkit import JmcomicText
return JmcomicText.analyse_jm_pub_html(resp.text)
album_comment_headers = {
'authority': '18comic.vip',
'accept': 'application/json, text/javascript, */*; q=0.01',
'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
'cache-control': 'no-cache',
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'origin': 'https://18comic.vip',
'pragma': 'no-cache',
'referer': 'https://18comic.vip/album/248965/',
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/114.0.0.0 Safari/537.36',
'x-requested-with': 'XMLHttpRequest',
}
jm_debug = JmModuleConfig.jm_debug
disable_jm_debug = JmModuleConfig.disable_jm_debug

View File

@ -10,7 +10,7 @@ class Test_Client(JmTestConfigurable):
def test_download_image(self):
jm_photo_id = 'JM438516'
photo_detail = self.client.get_photo_detail(jm_photo_id)
photo_detail = self.client.get_photo_detail(jm_photo_id, False)
self.client.download_by_image_detail(
photo_detail[0],
img_save_path=workspace('test_download_image.png')
@ -25,7 +25,7 @@ class Test_Client(JmTestConfigurable):
测试通过 JmcomicClient jm_photo_id 获取 JmPhotoDetail对象
"""
jm_photo_id = 'JM438516'
photo_detail = self.client.get_photo_detail(jm_photo_id)
photo_detail = self.client.get_photo_detail(jm_photo_id, False)
photo_detail.when_del_save_file = True
photo_detail.after_save_print_info = True
del photo_detail
@ -46,7 +46,7 @@ class Test_Client(JmTestConfigurable):
def test_gt_300_photo(self):
photo_id = '147643'
photo_detail: JmPhotoDetail = self.client.get_photo_detail(photo_id)
photo_detail: JmPhotoDetail = self.client.get_photo_detail(photo_id, False)
image = photo_detail[3000]
print(image.img_url)
self.client.download_by_image_detail(image, workspace('3000.png'))