v2.6.7: 优化域名重试策略; 优化禁漫api返回值的json解析 (#472) (#474)
Some checks failed
Auto Release & Publish / release (push) Has been cancelled

---------

Co-authored-by: RSLN-creator <3316399314@qq.com>
This commit is contained in:
hect0x7 2025-09-09 23:57:48 +08:00 committed by GitHub
parent 49c489195b
commit 7943b9b66a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 139 additions and 20 deletions

21
.github/release.yml vendored Normal file
View File

@ -0,0 +1,21 @@
# .github/release.yml
changelog:
exclude:
labels:
- ignore-for-release
authors:
- octocat
categories:
- title: 🏕 Features
labels:
- '*'
exclude:
labels:
- dependencies
- title: 👒 Dependencies
labels:
- dependencies
- title: Other Changes
labels:
- "*"

View File

@ -29,12 +29,13 @@ jobs:
python .github/release.py "$commit_message"
- name: Create Release
uses: softprops/action-gh-release@v1
uses: softprops/action-gh-release@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ steps.tb.outputs.tag }}
body_path: release_body.txt
generate_release_notes: true
- name: Build
run: |

View File

@ -25,4 +25,10 @@ plugins:
- plugin: client_proxy
kwargs:
proxy_client_key: photo_concurrent_fetcher_proxy
whitelist: [ api, ]
whitelist: [ api, ]
- plugin: advanced-retry
kwargs:
retry_config:
retry_rounds: 3 # 一共对域名列表重试3轮
retry_domain_max_times: 5 # 当一个域名重试次数超过5次忽略该域名不再重试

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.6.6'
__version__ = '2.6.7'
from .api import *
from .jm_plugin import *

View File

@ -895,7 +895,7 @@ class JmApiClient(AbstractJmClient):
检查返回数据中的status字段是否为ok
"""
data = resp.model_data
if data.status == 'ok':
if data.status != 'ok':
ExceptionTool.raises_resp(data.msg, resp)
def req_api(self, url, get=True, require_success=True, **kwargs) -> JmApiResp:
@ -995,7 +995,7 @@ class JmApiClient(AbstractJmClient):
# 找到第一个有效字符
ExceptionTool.require_true(
char == '{',
f'请求不是json格式强制重试响应文本: [{resp.text}]'
f'请求不是json格式强制重试响应文本: [{JmcomicText.limit_text(text, 200)}]'
)
return resp

View File

@ -101,6 +101,14 @@ class JmApiResp(JmJsonResp):
super().__init__(resp)
self.ts = ts
# 重写json()方法可以忽略一些非json格式的脏数据
@field_cache()
def json(self) -> Dict:
try:
return JmcomicText.try_parse_json_object(self.resp.text)
except Exception as e:
ExceptionTool.raises_resp(f'json解析失败: {e}', self, JsonResolveFailException)
@property
def is_success(self) -> bool:
return super().is_success and self.json()['code'] == 200

View File

@ -419,7 +419,7 @@ class JmOption:
if clazz == AbstractJmClient or not issubclass(clazz, AbstractJmClient):
raise NotImplementedError(clazz)
client: AbstractJmClient = clazz(
client: JmcomicClient = clazz(
postman=postman,
domain_list=decide_domain_list(),
retry_times=retry_times,

View File

@ -1221,23 +1221,88 @@ class ReplacePathStringPlugin(JmOptionPlugin):
class AdvancedRetryPlugin(JmOptionPlugin):
plugin_key = 'advanced-retry'
def __init__(self, option: JmOption):
super().__init__(option)
self.retry_config = None
def invoke(self,
retry_config,
**kwargs):
self.require_param(isinstance(retry_config, dict), '必须配置retry_config为dict')
self.retry_config = retry_config
new_jm_client: Callable = self.option.new_jm_client
def hook_new_jm_client(*args, **kwargs):
client: AbstractJmClient = new_jm_client(*args, **kwargs)
client: JmcomicClient = new_jm_client(*args, **kwargs)
client.domain_retry_strategy = self.request_with_retry
client.domain_req_failed_counter = {}
from threading import Lock
client.domain_counter_lock = Lock()
return client
self.option.new_jm_client = hook_new_jm_client
def request_with_retry(self,
client,
request,
url,
is_image,
client: AbstractJmClient,
request: Callable,
url: str,
is_image: bool,
**kwargs,
):
pass
"""
实现如下域名重试机制
- 对域名列表轮询请求配置retry_rounds
- 限制单个域名最大失败次数配置retry_domain_max_times
- 轮询域名列表前根据历史失败次数对域名列表排序失败多的后置
"""
def do_request(domain):
url_to_use = url
if url_to_use.startswith('/'):
# path → url
url_to_use = client.of_api_url(url, domain)
client.update_request_with_specify_domain(kwargs, domain, is_image)
jm_log(client.log_topic(), client.decode(url_to_use))
elif is_image:
# 图片url
client.update_request_with_specify_domain(kwargs, None, is_image)
resp = request(url_to_use, **kwargs)
resp = client.raise_if_resp_should_retry(resp, is_image)
return resp
retry_domain_max_times: int = self.retry_config['retry_domain_max_times']
retry_rounds: int = self.retry_config['retry_rounds']
for rindex in range(retry_rounds):
domain_list = self.get_sorted_domain(client, retry_domain_max_times)
for i, domain in enumerate(domain_list):
if self.failed_count(client, domain) >= retry_domain_max_times:
continue
try:
return do_request(domain)
except Exception as e:
from common import traceback_print_exec
traceback_print_exec()
jm_log('req.error', str(e))
self.update_failed_count(client, domain)
return client.fallback(request, url, 0, 0, is_image, **kwargs)
def get_sorted_domain(self, client: JmcomicClient, times):
domain_list = client.get_domain_list()
return sorted(
filter(lambda d: self.failed_count(client, d) < times, domain_list),
key=lambda d: self.failed_count(client, d)
)
# noinspection PyUnresolvedReferences
def update_failed_count(self, client: AbstractJmClient, domain: str):
with client.domain_counter_lock:
client.domain_req_failed_counter[domain] = self.failed_count(client, domain) + 1
@staticmethod
def failed_count(client: JmcomicClient, domain: str) -> int:
# noinspection PyUnresolvedReferences
return client.domain_req_failed_counter.get(domain, 0)

View File

@ -61,6 +61,8 @@ class JmcomicText:
# 提取接口返回值信息
pattern_ajax_favorite_msg = compile(r'</button>(.*?)</div>')
# 提取api接口返回值里的json防止返回值里有无关日志导致json解析报错
pattern_api_response_json_object = compile(r'\{[\s\S]*?}')
@classmethod
def parse_to_jm_domain(cls, text: str):
@ -344,6 +346,28 @@ class JmcomicText:
raise e
return save_dir
# noinspection PyTypeChecker
@classmethod
def try_parse_json_object(cls, resp_text: str) -> dict:
import json
text = resp_text.strip()
if text.startswith('{') and text.endswith('}'):
# fast case
return json.loads(text)
for match in cls.pattern_api_response_json_object.finditer(text):
try:
return json.loads(match.group(0))
except Exception as e:
jm_log('parse_json_object.error', e)
raise AssertionError(f'未解析出json数据: {cls.limit_text(resp_text, 200)}')
@classmethod
def limit_text(cls, text: str, limit: int) -> str:
length = len(text)
return text if length <= limit else (text[:limit] + f'...({length - limit}')
# 支持dsl: #{???} -> os.getenv(???)
JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env)
@ -450,10 +474,7 @@ class JmPageTool:
# 这里不作解析,因为没什么用...
tags = cls.pattern_html_search_tags.findall(tag_text)
content.append((
album_id, {
'name': title, # 改成name是为了兼容 parse_api_resp_to_page
'tags': tags
}
album_id, dict(name=title, tags=tags) # 改成name是为了兼容 parse_api_resp_to_page
))
return JmSearchPage(content, total)
@ -468,10 +489,7 @@ class JmPageTool:
for (album_id, title, tag_text) in album_info_list:
tags = cls.pattern_html_search_tags.findall(tag_text)
content.append((
album_id, {
'name': title, # 改成name是为了兼容 parse_api_resp_to_page
'tags': tags
}
album_id, dict(name=title, tags=tags) # 改成name是为了兼容 parse_api_resp_to_page
))
return JmSearchPage(content, total)