v2.1.0: 优化请求重试机制 (#74)

This commit is contained in:
hect0x7 2023-07-13 17:13:37 +08:00 committed by GitHub
parent 9ab4de6313
commit 58f1b8bb36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 151 additions and 24 deletions

View File

@ -2,6 +2,6 @@
# 被依赖方 <--- 使用方 # 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option # config <--- entity <--- toolkit <--- client <--- option
__version__ = '2.0.9' __version__ = '2.1.0'
from .api import * from .api import *

View File

@ -1,6 +1,7 @@
from .jm_client_interface import * from .jm_client_interface import *
# noinspection PyAbstractClass
class AbstractJmClient( class AbstractJmClient(
JmcomicClient, JmcomicClient,
PostmanProxy, PostmanProxy,
@ -39,21 +40,32 @@ class AbstractJmClient(
retry_count=0, retry_count=0,
**kwargs, **kwargs,
): ):
"""
统一请求支持重试
@param request: 请求方法
@param url: 图片url / path (/album/xxx)
@param domain_index: 域名下标
@param retry_count: 重试次数
@param kwargs: 请求方法的kwargs
"""
if domain_index >= len(self.domain_list): if domain_index >= len(self.domain_list):
raise AssertionError("All domains failed.") raise AssertionError(f"请求重试全部失败: [{url}], {self.domain_list}")
if url.startswith('/'):
# path
domain = self.domain_list[domain_index] domain = self.domain_list[domain_index]
if not url.startswith(JmModuleConfig.PROT):
url = self.of_api_url(url, domain) url = self.of_api_url(url, domain)
jm_debug('api', url) jm_debug('api', url)
else:
# 图片url
pass
if domain_index != 0 and retry_count != 0: if domain_index != 0 or retry_count != 0:
jm_debug( jm_debug(
f'请求重试', f'request_retry',
', '.join([ ', '.join([
f'次数: [{retry_count}/{self.retry_times}]', f'次数: [{retry_count}/{self.retry_times}]',
f'域名: [{domain} ({domain_index}/{len(self.domain_list)})]', f'域名: [{domain_index} of {self.domain_list}]',
f'路径: [{url}]', f'路径: [{url}]',
f'参数: [{kwargs if "login" not in url else "#login_form#"}]' f'参数: [{kwargs if "login" not in url else "#login_form#"}]'
]) ])
@ -69,6 +81,7 @@ class AbstractJmClient(
else: else:
return self.request_with_retry(request, url, domain_index + 1, 0, **kwargs) return self.request_with_retry(request, url, domain_index + 1, 0, **kwargs)
# noinspection PyMethodMayBeStatic, PyUnusedLocal # noinspection PyMethodMayBeStatic, PyUnusedLocal
def before_retry(self, e, kwargs, retry_count, url): def before_retry(self, e, kwargs, retry_count, url):
jm_debug('error', str(e)) jm_debug('error', str(e))
@ -237,7 +250,60 @@ class JmHtmlClient(AbstractJmClient):
raise AssertionError(msg) raise AssertionError(msg)
def get_jm_image(self, img_url) -> JmImageResp: def get_jm_image(self, img_url) -> JmImageResp:
return JmImageResp(self.get(img_url))
def get_if_fail_raise(url):
"""
使用此方法包装 self.get
"""
resp = JmImageResp(self.get(url))
if resp.is_success:
return resp
self.raise_request_error(
resp.resp, resp.get_error_msg()
)
return resp
return self.request_with_retry(get_if_fail_raise, img_url)
def album_comment(self,
video_id,
comment,
originator='',
status='true',
comment_id=None,
**kwargs,
) -> JmAcResp:
data = {
'video_id': video_id,
'comment': comment,
'originator': originator,
'status': status,
}
# 处理回复评论
if comment_id is not None:
data.pop('status')
data['comment_id'] = comment_id
data['is_reply'] = 1
data['forum_subject'] = 1
jm_debug('album_comment',
f'{video_id}: [{comment}]' +
(f' to ({comment_id})' if comment_id is not None else '')
)
resp = self.post('https://18comic.vip/ajax/album_comment',
headers=JmModuleConfig.album_comment_headers,
data=data,
)
ret = JmAcResp(resp)
jm_debug('album_comment', f'{video_id}: [{comment}] ← ({ret.model().cid})')
return ret
@classmethod @classmethod
def require_resp_success_else_raise(cls, resp, req_url): def require_resp_success_else_raise(cls, resp, req_url):

View File

@ -13,20 +13,31 @@ class JmResp(CommonResp):
def is_success(self) -> bool: def is_success(self) -> bool:
return self.http_code == 200 and len(self.content) != 0 return self.http_code == 200 and len(self.content) != 0
def json(self, **kwargs) -> Dict:
raise NotImplementedError
def model(self) -> DictModel:
return DictModel(self.json())
class JmImageResp(JmResp): class JmImageResp(JmResp):
def json(self, **kwargs) -> Dict:
raise AssertionError
def require_success(self): def require_success(self):
if self.is_success: if self.is_success:
return return
raise AssertionError(self.get_error_msg())
def get_error_msg(self):
msg = f'禁漫图片获取失败: [{self.url}]' msg = f'禁漫图片获取失败: [{self.url}]'
if self.http_code != 200: if self.http_code != 200:
msg += f'http状态码={self.http_code}' msg += f'http状态码={self.http_code}'
if len(self.content) == 0: if len(self.content) == 0:
msg += f',响应数据为空' msg += f',响应数据为空'
return msg
raise AssertionError(msg)
def transfer_to(self, def transfer_to(self,
path, path,
@ -106,15 +117,25 @@ class JmApiResp(JmResp):
def json(self, **kwargs) -> Dict: def json(self, **kwargs) -> Dict:
return self.resp.json() return self.resp.json()
def model(self) -> DictModel:
return DictModel(self.json())
@property @property
def model_data(self) -> DictModel: def model_data(self) -> DictModel:
self.require_success() self.require_success()
return DictModel(self.res_data) return DictModel(self.res_data)
# album-comment
class JmAcResp(JmResp):
def is_success(self) -> bool:
return super().is_success and self.json()['err'] is False
def json(self, **kwargs) -> Dict:
return self.resp.json()
def model(self) -> DictModel:
return DictModel(self.json())
""" """
Client Interface Client Interface
@ -154,6 +175,25 @@ class JmUserClient:
): ):
raise NotImplementedError raise NotImplementedError
def album_comment(self,
video_id,
comment,
originator='',
status='true',
comment_id=None,
**kwargs,
) -> JmAcResp:
"""
评论漫画/评论回复
@param video_id: album_id/photo_id
@param comment: 评论内容
@param status: 是否 "有劇透"
@param comment_id: 被回复评论的id
@param originator:
@return: JmAcResp 对象
"""
raise NotImplementedError
class JmImageClient: class JmImageClient:
@ -204,6 +244,7 @@ class JmImageClient:
return data_original.endswith('.gif') return data_original.endswith('.gif')
# noinspection PyAbstractClass
class JmcomicClient( class JmcomicClient(
JmImageClient, JmImageClient,
JmDetailClient, JmDetailClient,

View File

@ -65,14 +65,14 @@ class JmModuleConfig:
return cls.DOMAIN # jmcomic默认域名 return cls.DOMAIN # jmcomic默认域名
@classmethod @classmethod
def headers(cls, authority=None): def headers(cls, domain='18comic.vip'):
return { return {
'authority': authority or '18comic.vip', 'authority': domain,
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,' 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,'
'application/signed-exchange;v=b3;q=0.7', 'application/signed-exchange;v=b3;q=0.7',
'accept-language': 'zh-CN,zh;q=0.9', 'accept-language': 'zh-CN,zh;q=0.9',
'cache-control': 'no-cache', 'cache-control': 'no-cache',
'referer': 'https://18comic.vip', 'referer': f'https://{domain}',
'pragma': 'no-cache', 'pragma': 'no-cache',
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"', 'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
'sec-ch-ua-mobile': '?0', 'sec-ch-ua-mobile': '?0',
@ -130,6 +130,26 @@ class JmModuleConfig:
from .jm_toolkit import JmcomicText from .jm_toolkit import JmcomicText
return JmcomicText.analyse_jm_pub_html(resp.text) return JmcomicText.analyse_jm_pub_html(resp.text)
album_comment_headers = {
'authority': '18comic.vip',
'accept': 'application/json, text/javascript, */*; q=0.01',
'accept-language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
'cache-control': 'no-cache',
'content-type': 'application/x-www-form-urlencoded; charset=UTF-8',
'origin': 'https://18comic.vip',
'pragma': 'no-cache',
'referer': 'https://18comic.vip/album/248965/',
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Google Chrome";v="114"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/114.0.0.0 Safari/537.36',
'x-requested-with': 'XMLHttpRequest',
}
jm_debug = JmModuleConfig.jm_debug jm_debug = JmModuleConfig.jm_debug
disable_jm_debug = JmModuleConfig.disable_jm_debug disable_jm_debug = JmModuleConfig.disable_jm_debug

View File

@ -10,7 +10,7 @@ class Test_Client(JmTestConfigurable):
def test_download_image(self): def test_download_image(self):
jm_photo_id = 'JM438516' jm_photo_id = 'JM438516'
photo_detail = self.client.get_photo_detail(jm_photo_id) photo_detail = self.client.get_photo_detail(jm_photo_id, False)
self.client.download_by_image_detail( self.client.download_by_image_detail(
photo_detail[0], photo_detail[0],
img_save_path=workspace('test_download_image.png') img_save_path=workspace('test_download_image.png')
@ -25,7 +25,7 @@ class Test_Client(JmTestConfigurable):
测试通过 JmcomicClient jm_photo_id 获取 JmPhotoDetail对象 测试通过 JmcomicClient jm_photo_id 获取 JmPhotoDetail对象
""" """
jm_photo_id = 'JM438516' jm_photo_id = 'JM438516'
photo_detail = self.client.get_photo_detail(jm_photo_id) photo_detail = self.client.get_photo_detail(jm_photo_id, False)
photo_detail.when_del_save_file = True photo_detail.when_del_save_file = True
photo_detail.after_save_print_info = True photo_detail.after_save_print_info = True
del photo_detail del photo_detail
@ -46,7 +46,7 @@ class Test_Client(JmTestConfigurable):
def test_gt_300_photo(self): def test_gt_300_photo(self):
photo_id = '147643' photo_id = '147643'
photo_detail: JmPhotoDetail = self.client.get_photo_detail(photo_id) photo_detail: JmPhotoDetail = self.client.get_photo_detail(photo_id, False)
image = photo_detail[3000] image = photo_detail[3000]
print(image.img_url) print(image.img_url)
self.client.download_by_image_detail(image, workspace('3000.png')) self.client.download_by_image_detail(image, workspace('3000.png'))