From 7943b9b66ae3063000768eef5869eb3f33dadf4e Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Tue, 9 Sep 2025 23:57:48 +0800 Subject: [PATCH] =?UTF-8?q?v2.6.7:=20=E4=BC=98=E5=8C=96=E5=9F=9F=E5=90=8D?= =?UTF-8?q?=E9=87=8D=E8=AF=95=E7=AD=96=E7=95=A5;=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E7=A6=81=E6=BC=ABapi=E8=BF=94=E5=9B=9E=E5=80=BC=E7=9A=84json?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=20(#472)=20(#474)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --------- Co-authored-by: RSLN-creator <3316399314@qq.com> --- .github/release.yml | 21 ++++++++ .github/workflows/release_auto.yml | 3 +- assets/option/option_test_api.yml | 8 +++- src/jmcomic/__init__.py | 2 +- src/jmcomic/jm_client_impl.py | 4 +- src/jmcomic/jm_client_interface.py | 8 ++++ src/jmcomic/jm_option.py | 2 +- src/jmcomic/jm_plugin.py | 77 +++++++++++++++++++++++++++--- src/jmcomic/jm_toolkit.py | 34 +++++++++---- 9 files changed, 139 insertions(+), 20 deletions(-) create mode 100644 .github/release.yml diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..d2e4719 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,21 @@ +# .github/release.yml + +changelog: + exclude: + labels: + - ignore-for-release + authors: + - octocat + categories: + - title: 🏕 Features + labels: + - '*' + exclude: + labels: + - dependencies + - title: 👒 Dependencies + labels: + - dependencies + - title: Other Changes + labels: + - "*" diff --git a/.github/workflows/release_auto.yml b/.github/workflows/release_auto.yml index e6a5b32..3b85c38 100644 --- a/.github/workflows/release_auto.yml +++ b/.github/workflows/release_auto.yml @@ -29,12 +29,13 @@ jobs: python .github/release.py "$commit_message" - name: Create Release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ steps.tb.outputs.tag }} body_path: release_body.txt + generate_release_notes: true - name: Build run: | diff --git a/assets/option/option_test_api.yml b/assets/option/option_test_api.yml index 29efdaa..f4c0b7b 100644 --- a/assets/option/option_test_api.yml +++ b/assets/option/option_test_api.yml @@ -25,4 +25,10 @@ plugins: - plugin: client_proxy kwargs: proxy_client_key: photo_concurrent_fetcher_proxy - whitelist: [ api, ] \ No newline at end of file + whitelist: [ api, ] + + - plugin: advanced-retry + kwargs: + retry_config: + retry_rounds: 3 # 一共对域名列表重试3轮 + retry_domain_max_times: 5 # 当一个域名重试次数超过5次,忽略该域名,不再重试 \ No newline at end of file diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index a4e9c97..5aff691 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,7 +2,7 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.6.6' +__version__ = '2.6.7' from .api import * from .jm_plugin import * diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py index be29b9a..0a801f3 100644 --- a/src/jmcomic/jm_client_impl.py +++ b/src/jmcomic/jm_client_impl.py @@ -895,7 +895,7 @@ class JmApiClient(AbstractJmClient): 检查返回数据中的status字段是否为ok """ data = resp.model_data - if data.status == 'ok': + if data.status != 'ok': ExceptionTool.raises_resp(data.msg, resp) def req_api(self, url, get=True, require_success=True, **kwargs) -> JmApiResp: @@ -995,7 +995,7 @@ class JmApiClient(AbstractJmClient): # 找到第一个有效字符 ExceptionTool.require_true( char == '{', - f'请求不是json格式,强制重试!响应文本: [{resp.text}]' + f'请求不是json格式,强制重试!响应文本: [{JmcomicText.limit_text(text, 200)}]' ) return resp diff --git a/src/jmcomic/jm_client_interface.py b/src/jmcomic/jm_client_interface.py index f522694..5980282 100644 --- a/src/jmcomic/jm_client_interface.py +++ b/src/jmcomic/jm_client_interface.py @@ -101,6 +101,14 @@ class JmApiResp(JmJsonResp): super().__init__(resp) self.ts = ts + # 重写json()方法,可以忽略一些非json格式的脏数据 + @field_cache() + def json(self) -> Dict: + try: + return JmcomicText.try_parse_json_object(self.resp.text) + except Exception as e: + ExceptionTool.raises_resp(f'json解析失败: {e}', self, JsonResolveFailException) + @property def is_success(self) -> bool: return super().is_success and self.json()['code'] == 200 diff --git a/src/jmcomic/jm_option.py b/src/jmcomic/jm_option.py index 32b4b36..f71b6c8 100644 --- a/src/jmcomic/jm_option.py +++ b/src/jmcomic/jm_option.py @@ -419,7 +419,7 @@ class JmOption: if clazz == AbstractJmClient or not issubclass(clazz, AbstractJmClient): raise NotImplementedError(clazz) - client: AbstractJmClient = clazz( + client: JmcomicClient = clazz( postman=postman, domain_list=decide_domain_list(), retry_times=retry_times, diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py index 27a7b8f..37a3ac4 100644 --- a/src/jmcomic/jm_plugin.py +++ b/src/jmcomic/jm_plugin.py @@ -1221,23 +1221,88 @@ class ReplacePathStringPlugin(JmOptionPlugin): class AdvancedRetryPlugin(JmOptionPlugin): plugin_key = 'advanced-retry' + def __init__(self, option: JmOption): + super().__init__(option) + self.retry_config = None + def invoke(self, retry_config, **kwargs): + self.require_param(isinstance(retry_config, dict), '必须配置retry_config为dict') + self.retry_config = retry_config + new_jm_client: Callable = self.option.new_jm_client def hook_new_jm_client(*args, **kwargs): - client: AbstractJmClient = new_jm_client(*args, **kwargs) + client: JmcomicClient = new_jm_client(*args, **kwargs) client.domain_retry_strategy = self.request_with_retry + client.domain_req_failed_counter = {} + from threading import Lock + client.domain_counter_lock = Lock() return client self.option.new_jm_client = hook_new_jm_client def request_with_retry(self, - client, - request, - url, - is_image, + client: AbstractJmClient, + request: Callable, + url: str, + is_image: bool, **kwargs, ): - pass + """ + 实现如下域名重试机制: + - 对域名列表轮询请求,配置:retry_rounds + - 限制单个域名最大失败次数,配置:retry_domain_max_times + - 轮询域名列表前,根据历史失败次数对域名列表排序,失败多的后置 + """ + + def do_request(domain): + url_to_use = url + if url_to_use.startswith('/'): + # path → url + url_to_use = client.of_api_url(url, domain) + client.update_request_with_specify_domain(kwargs, domain, is_image) + jm_log(client.log_topic(), client.decode(url_to_use)) + elif is_image: + # 图片url + client.update_request_with_specify_domain(kwargs, None, is_image) + + resp = request(url_to_use, **kwargs) + resp = client.raise_if_resp_should_retry(resp, is_image) + return resp + + retry_domain_max_times: int = self.retry_config['retry_domain_max_times'] + retry_rounds: int = self.retry_config['retry_rounds'] + for rindex in range(retry_rounds): + domain_list = self.get_sorted_domain(client, retry_domain_max_times) + for i, domain in enumerate(domain_list): + if self.failed_count(client, domain) >= retry_domain_max_times: + continue + + try: + return do_request(domain) + except Exception as e: + from common import traceback_print_exec + traceback_print_exec() + jm_log('req.error', str(e)) + self.update_failed_count(client, domain) + + return client.fallback(request, url, 0, 0, is_image, **kwargs) + + def get_sorted_domain(self, client: JmcomicClient, times): + domain_list = client.get_domain_list() + return sorted( + filter(lambda d: self.failed_count(client, d) < times, domain_list), + key=lambda d: self.failed_count(client, d) + ) + + # noinspection PyUnresolvedReferences + def update_failed_count(self, client: AbstractJmClient, domain: str): + with client.domain_counter_lock: + client.domain_req_failed_counter[domain] = self.failed_count(client, domain) + 1 + + @staticmethod + def failed_count(client: JmcomicClient, domain: str) -> int: + # noinspection PyUnresolvedReferences + return client.domain_req_failed_counter.get(domain, 0) diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py index e5eada1..7ec84d5 100644 --- a/src/jmcomic/jm_toolkit.py +++ b/src/jmcomic/jm_toolkit.py @@ -61,6 +61,8 @@ class JmcomicText: # 提取接口返回值信息 pattern_ajax_favorite_msg = compile(r'(.*?)') + # 提取api接口返回值里的json,防止返回值里有无关日志导致json解析报错 + pattern_api_response_json_object = compile(r'\{[\s\S]*?}') @classmethod def parse_to_jm_domain(cls, text: str): @@ -344,6 +346,28 @@ class JmcomicText: raise e return save_dir + # noinspection PyTypeChecker + @classmethod + def try_parse_json_object(cls, resp_text: str) -> dict: + import json + text = resp_text.strip() + if text.startswith('{') and text.endswith('}'): + # fast case + return json.loads(text) + + for match in cls.pattern_api_response_json_object.finditer(text): + try: + return json.loads(match.group(0)) + except Exception as e: + jm_log('parse_json_object.error', e) + + raise AssertionError(f'未解析出json数据: {cls.limit_text(resp_text, 200)}') + + @classmethod + def limit_text(cls, text: str, limit: int) -> str: + length = len(text) + return text if length <= limit else (text[:limit] + f'...({length - limit}') + # 支持dsl: #{???} -> os.getenv(???) JmcomicText.dsl_replacer.add_dsl_and_replacer(r'\$\{(.*?)\}', JmcomicText.match_os_env) @@ -450,10 +474,7 @@ class JmPageTool: # 这里不作解析,因为没什么用... tags = cls.pattern_html_search_tags.findall(tag_text) content.append(( - album_id, { - 'name': title, # 改成name是为了兼容 parse_api_resp_to_page - 'tags': tags - } + album_id, dict(name=title, tags=tags) # 改成name是为了兼容 parse_api_resp_to_page )) return JmSearchPage(content, total) @@ -468,10 +489,7 @@ class JmPageTool: for (album_id, title, tag_text) in album_info_list: tags = cls.pattern_html_search_tags.findall(tag_text) content.append(( - album_id, { - 'name': title, # 改成name是为了兼容 parse_api_resp_to_page - 'tags': tags - } + album_id, dict(name=title, tags=tags) # 改成name是为了兼容 parse_api_resp_to_page )) return JmSearchPage(content, total)