mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-09-26 22:31:30 +08:00
v2.6.7: 优化域名重试策略; 优化禁漫api返回值的json解析 (#472) (#474)
Some checks failed
Auto Release & Publish / release (push) Has been cancelled
Some checks failed
Auto Release & Publish / release (push) Has been cancelled
--------- Co-authored-by: RSLN-creator <3316399314@qq.com>
This commit is contained in:
parent
49c489195b
commit
7943b9b66a
21
.github/release.yml
vendored
Normal file
21
.github/release.yml
vendored
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
# .github/release.yml
|
||||||
|
|
||||||
|
changelog:
|
||||||
|
exclude:
|
||||||
|
labels:
|
||||||
|
- ignore-for-release
|
||||||
|
authors:
|
||||||
|
- octocat
|
||||||
|
categories:
|
||||||
|
- title: 🏕 Features
|
||||||
|
labels:
|
||||||
|
- '*'
|
||||||
|
exclude:
|
||||||
|
labels:
|
||||||
|
- dependencies
|
||||||
|
- title: 👒 Dependencies
|
||||||
|
labels:
|
||||||
|
- dependencies
|
||||||
|
- title: Other Changes
|
||||||
|
labels:
|
||||||
|
- "*"
|
3
.github/workflows/release_auto.yml
vendored
3
.github/workflows/release_auto.yml
vendored
@ -29,12 +29,13 @@ jobs:
|
|||||||
python .github/release.py "$commit_message"
|
python .github/release.py "$commit_message"
|
||||||
|
|
||||||
- name: Create Release
|
- name: Create Release
|
||||||
uses: softprops/action-gh-release@v1
|
uses: softprops/action-gh-release@v2
|
||||||
env:
|
env:
|
||||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||||
with:
|
with:
|
||||||
tag_name: ${{ steps.tb.outputs.tag }}
|
tag_name: ${{ steps.tb.outputs.tag }}
|
||||||
body_path: release_body.txt
|
body_path: release_body.txt
|
||||||
|
generate_release_notes: true
|
||||||
|
|
||||||
- name: Build
|
- name: Build
|
||||||
run: |
|
run: |
|
||||||
|
@ -26,3 +26,9 @@ plugins:
|
|||||||
kwargs:
|
kwargs:
|
||||||
proxy_client_key: photo_concurrent_fetcher_proxy
|
proxy_client_key: photo_concurrent_fetcher_proxy
|
||||||
whitelist: [ api, ]
|
whitelist: [ api, ]
|
||||||
|
|
||||||
|
- plugin: advanced-retry
|
||||||
|
kwargs:
|
||||||
|
retry_config:
|
||||||
|
retry_rounds: 3 # 一共对域名列表重试3轮
|
||||||
|
retry_domain_max_times: 5 # 当一个域名重试次数超过5次,忽略该域名,不再重试
|
@ -2,7 +2,7 @@
|
|||||||
# 被依赖方 <--- 使用方
|
# 被依赖方 <--- 使用方
|
||||||
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
|
||||||
|
|
||||||
__version__ = '2.6.6'
|
__version__ = '2.6.7'
|
||||||
|
|
||||||
from .api import *
|
from .api import *
|
||||||
from .jm_plugin import *
|
from .jm_plugin import *
|
||||||
|
@ -895,7 +895,7 @@ class JmApiClient(AbstractJmClient):
|
|||||||
检查返回数据中的status字段是否为ok
|
检查返回数据中的status字段是否为ok
|
||||||
"""
|
"""
|
||||||
data = resp.model_data
|
data = resp.model_data
|
||||||
if data.status == 'ok':
|
if data.status != 'ok':
|
||||||
ExceptionTool.raises_resp(data.msg, resp)
|
ExceptionTool.raises_resp(data.msg, resp)
|
||||||
|
|
||||||
def req_api(self, url, get=True, require_success=True, **kwargs) -> JmApiResp:
|
def req_api(self, url, get=True, require_success=True, **kwargs) -> JmApiResp:
|
||||||
@ -995,7 +995,7 @@ class JmApiClient(AbstractJmClient):
|
|||||||
# 找到第一个有效字符
|
# 找到第一个有效字符
|
||||||
ExceptionTool.require_true(
|
ExceptionTool.require_true(
|
||||||
char == '{',
|
char == '{',
|
||||||
f'请求不是json格式,强制重试!响应文本: [{resp.text}]'
|
f'请求不是json格式,强制重试!响应文本: [{JmcomicText.limit_text(text, 200)}]'
|
||||||
)
|
)
|
||||||
return resp
|
return resp
|
||||||
|
|
||||||
|
@ -101,6 +101,14 @@ class JmApiResp(JmJsonResp):
|
|||||||
super().__init__(resp)
|
super().__init__(resp)
|
||||||
self.ts = ts
|
self.ts = ts
|
||||||
|
|
||||||
|
# 重写json()方法,可以忽略一些非json格式的脏数据
|
||||||
|
@field_cache()
|
||||||
|
def json(self) -> Dict:
|
||||||
|
try:
|
||||||
|
return JmcomicText.try_parse_json_object(self.resp.text)
|
||||||
|
except Exception as e:
|
||||||
|
ExceptionTool.raises_resp(f'json解析失败: {e}', self, JsonResolveFailException)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_success(self) -> bool:
|
def is_success(self) -> bool:
|
||||||
return super().is_success and self.json()['code'] == 200
|
return super().is_success and self.json()['code'] == 200
|
||||||
|
@ -419,7 +419,7 @@ class JmOption:
|
|||||||
if clazz == AbstractJmClient or not issubclass(clazz, AbstractJmClient):
|
if clazz == AbstractJmClient or not issubclass(clazz, AbstractJmClient):
|
||||||
raise NotImplementedError(clazz)
|
raise NotImplementedError(clazz)
|
||||||
|
|
||||||
client: AbstractJmClient = clazz(
|
client: JmcomicClient = clazz(
|
||||||
postman=postman,
|
postman=postman,
|
||||||
domain_list=decide_domain_list(),
|
domain_list=decide_domain_list(),
|
||||||
retry_times=retry_times,
|
retry_times=retry_times,
|
||||||
|
@ -1221,23 +1221,88 @@ class ReplacePathStringPlugin(JmOptionPlugin):
|
|||||||
class AdvancedRetryPlugin(JmOptionPlugin):
|
class AdvancedRetryPlugin(JmOptionPlugin):
|
||||||
plugin_key = 'advanced-retry'
|
plugin_key = 'advanced-retry'
|
||||||
|
|
||||||
|
def __init__(self, option: JmOption):
|
||||||
|
super().__init__(option)
|
||||||
|
self.retry_config = None
|
||||||
|
|
||||||
def invoke(self,
|
def invoke(self,
|
||||||
retry_config,
|
retry_config,
|
||||||
**kwargs):
|
**kwargs):
|
||||||
|
self.require_param(isinstance(retry_config, dict), '必须配置retry_config为dict')
|
||||||
|
self.retry_config = retry_config
|
||||||
|
|
||||||
new_jm_client: Callable = self.option.new_jm_client
|
new_jm_client: Callable = self.option.new_jm_client
|
||||||
|
|
||||||
def hook_new_jm_client(*args, **kwargs):
|
def hook_new_jm_client(*args, **kwargs):
|
||||||
client: AbstractJmClient = new_jm_client(*args, **kwargs)
|
client: JmcomicClient = new_jm_client(*args, **kwargs)
|
||||||
client.domain_retry_strategy = self.request_with_retry
|
client.domain_retry_strategy = self.request_with_retry
|
||||||
|
client.domain_req_failed_counter = {}
|
||||||
|
from threading import Lock
|
||||||
|
client.domain_counter_lock = Lock()
|
||||||
return client
|
return client
|
||||||
|
|
||||||
self.option.new_jm_client = hook_new_jm_client
|
self.option.new_jm_client = hook_new_jm_client
|
||||||
|
|
||||||
def request_with_retry(self,
|
def request_with_retry(self,
|
||||||
client,
|
client: AbstractJmClient,
|
||||||
request,
|
request: Callable,
|
||||||
url,
|
url: str,
|
||||||
is_image,
|
is_image: bool,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
pass
|
"""
|
||||||
|
实现如下域名重试机制:
|
||||||
|
- 对域名列表轮询请求,配置:retry_rounds
|
||||||
|
- 限制单个域名最大失败次数,配置:retry_domain_max_times
|
||||||
|
- 轮询域名列表前,根据历史失败次数对域名列表排序,失败多的后置
|
||||||
|
"""
|
||||||
|
|
||||||
|
def do_request(domain):
|
||||||
|
url_to_use = url
|
||||||
|
if url_to_use.startswith('/'):
|
||||||
|
# path → url
|
||||||
|
url_to_use = client.of_api_url(url, domain)
|
||||||
|
client.update_request_with_specify_domain(kwargs, domain, is_image)
|
||||||
|
jm_log(client.log_topic(), client.decode(url_to_use))
|
||||||
|
elif is_image:
|
||||||
|
# 图片url
|
||||||
|
client.update_request_with_specify_domain(kwargs, None, is_image)
|
||||||
|
|
||||||
|
resp = request(url_to_use, **kwargs)
|
||||||
|
resp = client.raise_if_resp_should_retry(resp, is_image)
|
||||||
|
return resp
|
||||||
|
|
||||||
|
retry_domain_max_times: int = self.retry_config['retry_domain_max_times']
|
||||||
|
retry_rounds: int = self.retry_config['retry_rounds']
|
||||||
|
for rindex in range(retry_rounds):
|
||||||
|
domain_list = self.get_sorted_domain(client, retry_domain_max_times)
|
||||||
|
for i, domain in enumerate(domain_list):
|
||||||
|
if self.failed_count(client, domain) >= retry_domain_max_times:
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
return do_request(domain)
|
||||||
|
except Exception as e:
|
||||||
|
from common import traceback_print_exec
|
||||||
|
traceback_print_exec()
|
||||||
|
jm_log('req.error', str(e))
|
||||||
|
self.update_failed_count(client, domain)
|
||||||
|
|
||||||
|
return client.fallback(request, url, 0, 0, is_image, **kwargs)
|
||||||
|
|
||||||
|
def get_sorted_domain(self, client: JmcomicClient, times):
|
||||||
|
domain_list = client.get_domain_list()
|
||||||
|
return sorted(
|
||||||
|
filter(lambda d: self.failed_count(client, d) < times, domain_list),
|
||||||
|
key=lambda d: self.failed_count(client, d)
|
||||||
|
)
|
||||||
|
|
||||||
|
# noinspection PyUnresolvedReferences
|
||||||
|
def update_failed_count(self, client: AbstractJmClient, domain: str):
|
||||||
|
with client.domain_counter_lock:
|
||||||
|
client.domain_req_failed_counter[domain] = self.failed_count(client, domain) + 1
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def failed_count(client: JmcomicClient, domain: str) -> int:
|
||||||
|
# noinspection PyUnresolvedReferences
|
||||||
|
return client.domain_req_failed_counter.get(domain, 0)
|
||||||
|
@ -61,6 +61,8 @@ class JmcomicText:
|
|||||||
|
|
||||||
# 提取接口返回值信息
|
# 提取接口返回值信息
|
||||||
pattern_ajax_favorite_msg = compile(r'</button>(.*?)</div>')
|
pattern_ajax_favorite_msg = compile(r'</button>(.*?)</div>')
|
||||||
|
# 提取api接口返回值里的json,防止返回值里有无关日志导致json解析报错
|
||||||
|
pattern_api_response_json_object = compile(r'\{[\s\S]*?}')
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def parse_to_jm_domain(cls, text: str):
|
def parse_to_jm_domain(cls, text: str):
|
||||||
@ -344,6 +346,28 @@ class JmcomicText:
|
|||||||
raise e
|
raise e
|
||||||
return save_dir
|
return save_dir
|
||||||
|
|
||||||
|
# noinspection PyTypeChecker
|
||||||
|
@classmethod
|
||||||
|
def try_parse_json_object(cls, resp_text: str) -> dict:
|
||||||
|
import json
|
||||||
|
text = resp_text.strip()
|
||||||
|
if text.startswith('{') and text.endswith('}'):
|
||||||
|
# fast case
|
||||||
|
return json.loads(text)
|
||||||
|
|
||||||
|
for match in cls.pattern_api_response_json_object.finditer(text):
|
||||||
|
try:
|
||||||
|
return json.loads(match.group(0))
|
||||||
|
except Exception as e:
|
||||||
|
jm_log('parse_json_object.error', e)
|
||||||
|
|
||||||
|
raise AssertionError(f'未解析出json数据: {cls.limit_text(resp_text, 200)}')
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def limit_text(cls, text: str, limit: int) -> str:
|
||||||
|
length = len(text)
|
||||||
|
return text if length <= limit else (text[:limit] + f'...({length - limit}')
|
||||||
|
|
||||||
|
|
||||||
# 支持dsl: #{???} -> os.getenv(???)
|
# 支持dsl: #{???} -> os.getenv(???)
|
||||||
JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env)
|
JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env)
|
||||||
@ -450,10 +474,7 @@ class JmPageTool:
|
|||||||
# 这里不作解析,因为没什么用...
|
# 这里不作解析,因为没什么用...
|
||||||
tags = cls.pattern_html_search_tags.findall(tag_text)
|
tags = cls.pattern_html_search_tags.findall(tag_text)
|
||||||
content.append((
|
content.append((
|
||||||
album_id, {
|
album_id, dict(name=title, tags=tags) # 改成name是为了兼容 parse_api_resp_to_page
|
||||||
'name': title, # 改成name是为了兼容 parse_api_resp_to_page
|
|
||||||
'tags': tags
|
|
||||||
}
|
|
||||||
))
|
))
|
||||||
|
|
||||||
return JmSearchPage(content, total)
|
return JmSearchPage(content, total)
|
||||||
@ -468,10 +489,7 @@ class JmPageTool:
|
|||||||
for (album_id, title, tag_text) in album_info_list:
|
for (album_id, title, tag_text) in album_info_list:
|
||||||
tags = cls.pattern_html_search_tags.findall(tag_text)
|
tags = cls.pattern_html_search_tags.findall(tag_text)
|
||||||
content.append((
|
content.append((
|
||||||
album_id, {
|
album_id, dict(name=title, tags=tags) # 改成name是为了兼容 parse_api_resp_to_page
|
||||||
'name': title, # 改成name是为了兼容 parse_api_resp_to_page
|
|
||||||
'tags': tags
|
|
||||||
}
|
|
||||||
))
|
))
|
||||||
|
|
||||||
return JmSearchPage(content, total)
|
return JmSearchPage(content, total)
|
||||||
|
Loading…
Reference in New Issue
Block a user