From 4646146011a42863a890d2f1f7cdc673b6a33564 Mon Sep 17 00:00:00 2001 From: hect0x7 <93357912+hect0x7@users.noreply.github.com> Date: Wed, 13 Sep 2023 17:14:26 +0800 Subject: [PATCH] =?UTF-8?q?v2.2.8:=20=E4=BC=98=E5=8C=96=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E7=A6=81=E6=BC=AB=E7=BD=91=E9=A1=B5=E7=AB=AF=E5=92=8C=E7=A7=BB?= =?UTF-8?q?=E5=8A=A8=E7=AB=AF=E7=9A=84=E6=90=9C=E7=B4=A2=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E7=B1=BB=EF=BC=8C=E5=A2=9E=E5=8A=A0=E7=A7=BB=E5=8A=A8=E7=AB=AF?= =?UTF-8?q?=E7=9A=84=E6=B5=8B=E8=AF=95=EF=BC=8C=E8=B7=9F=E8=BF=9B=E6=96=87?= =?UTF-8?q?=E6=A1=A3=20(#132)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yml | 2 +- README.md | 6 +- requirements-dev.txt | 3 +- src/jmcomic/__init__.py | 2 +- src/jmcomic/jm_client_impl.py | 55 +++----- src/jmcomic/jm_client_interface.py | 14 ++- src/jmcomic/jm_entity.py | 132 +++++++++++--------- src/jmcomic/jm_option.py | 6 +- src/jmcomic/jm_plugin.py | 19 ++- src/jmcomic/jm_toolkit.py | 63 +++++++++- tests/test_jmcomic/test_jm_client.py | 12 +- tests/test_jmcomic/test_jm_mobile_client.py | 27 ++++ usage/pick_domain.py | 3 +- usage/usage_simple.py | 22 ++-- 14 files changed, 233 insertions(+), 133 deletions(-) create mode 100644 tests/test_jmcomic/test_jm_mobile_client.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b72ce00..d8f9c53 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: 跑测试 on: schedule: - - cron: "0 8 * * *" + - cron: "0 0 * * *" workflow_dispatch: push: branches: [ "dev" ] diff --git a/README.md b/README.md index 79b7e2f..84a4d5d 100644 --- a/README.md +++ b/README.md @@ -70,10 +70,10 @@ jmcomic.download_album('422866') # 传入要下载的album的id,即可下载 - 配置点有:`是否使用磁盘缓存` `并发下载图片数` `图片类型转换` `下载路径` `请求元信息(headers,cookies,proxies)`等 - **可扩展性强** - **支持Plugin插件,可以方便地扩展功能,以及使用别人的插件** - - 目前内置支持的插件有:`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` + - 目前内置支持的插件有:`登录插件` `硬件占用监控插件` `只下载新章插件` `压缩文件插件` - 支持自定义本子/章节/图片下载前后的回调函数 - - 支持自定义debug日志的开关/格式 - - 支持自定义Downloader/Option/Client/实体类 + - 支持自定义debug日志 + - 支持自定义类:Downloader(负责调度)/Option(负责配置)/Client(负责请求)/实体类 等等 - ...... - 支持**自动重试和域名切换**机制 - **多线程下载**(可细化到一图一线程,效率极高) diff --git a/requirements-dev.txt b/requirements-dev.txt index bba8b7d..a3fbf81 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,4 +2,5 @@ commonX curl_cffi PyYAML Pillow -psutil \ No newline at end of file +psutil +pycryptodome \ No newline at end of file diff --git a/src/jmcomic/__init__.py b/src/jmcomic/__init__.py index 623bbd0..ee10607 100644 --- a/src/jmcomic/__init__.py +++ b/src/jmcomic/__init__.py @@ -2,7 +2,7 @@ # 被依赖方 <--- 使用方 # config <--- entity <--- toolkit <--- client <--- option <--- downloader -__version__ = '2.2.7' +__version__ = '2.2.8' from .api import * from .jm_plugin import * diff --git a/src/jmcomic/jm_client_impl.py b/src/jmcomic/jm_client_impl.py index 402b5ed..fa63a2e 100644 --- a/src/jmcomic/jm_client_impl.py +++ b/src/jmcomic/jm_client_impl.py @@ -225,7 +225,7 @@ class JmHtmlClient(AbstractJmClient): album = JmcomicText.analyse_jm_album_html(resp.text) return JmSearchPage.wrap_single_album(album) else: - return JmSearchSupport.analyse_jm_search_html(resp.text) + return JmcomicText.analyse_jm_search_html(resp.text) # -- 帐号管理 -- @@ -405,31 +405,8 @@ class JmApiClient(AbstractJmClient): main_tag: int, order_by: str, time: str, - ) -> JmApiResp: - """ - model_data: { - "search_query": "MANA", - "total": "177", - "content": [ - { - "id": "441923", - "author": "MANA", - "description": "", - "name": "[MANA] 神里绫华5", - "image": "", - "category": { - "id": "1", - "title": "同人" - }, - "category_sub": { - "id": "1", - "title": "同人" - } - } - ] - } - """ - return self.get( + ) -> JmSearchPage: + resp = self.get( self.API_SEARCH, params={ 'search_query': search_query, @@ -440,6 +417,20 @@ class JmApiClient(AbstractJmClient): } ) + # 直接搜索禁漫车号,发生重定向的响应数据 resp.model_data + # { + # "search_query": "310311", + # "total": 1, + # "redirect_aid": "310311", + # "content": [] + # } + data = resp.model_data + if data.get('redirect_aid', None) is not None: + aid = data.redirect_aid + return JmSearchPage.wrap_single_album(self.get_album_detail(aid)) + + return JmcomicSearchTool.parse_api_resp_to_page(data) + def get(self, url, **kwargs) -> JmApiResp: # set headers headers, key_ts = self.headers_key_ts @@ -464,17 +455,5 @@ class JmApiClient(AbstractJmClient): return 'api' -class AsyncSaveImageClient(JmImageClient): - - def __init__(self, workers=None) -> None: - from concurrent.futures import ThreadPoolExecutor, Future - self.executor = ThreadPoolExecutor(max_workers=workers) - self.future_list: List[Future] = [] - - def save_image_resp(self, *args, **kwargs): - future = self.executor.submit(lambda: super().save_image_resp(*args, **kwargs)) - self.future_list.append(future) - - JmModuleConfig.register_client(JmHtmlClient) JmModuleConfig.register_client(JmApiClient) diff --git a/src/jmcomic/jm_client_interface.py b/src/jmcomic/jm_client_interface.py index c39a3a1..ccb73e8 100644 --- a/src/jmcomic/jm_client_interface.py +++ b/src/jmcomic/jm_client_interface.py @@ -6,6 +6,7 @@ Response Entity """ +DictModel = AdvancedEasyAccessDict class JmResp(CommonResp): @@ -19,6 +20,9 @@ class JmResp(CommonResp): def model(self) -> DictModel: return DictModel(self.json()) + def require_success(self): + if self.is_not_success: + raise JmModuleConfig.exception(self.resp.text) class JmImageResp(JmResp): @@ -312,7 +316,7 @@ class JmSearchAlbumClient: page: int = 1, order_by: str = ORDER_BY_LATEST, time: str = TIME_ALL, - ) -> JmSearchPage: + ): """ 对应禁漫的站内搜索 """ @@ -323,7 +327,7 @@ class JmSearchAlbumClient: page: int = 1, order_by: str = ORDER_BY_LATEST, time: str = TIME_ALL, - ) -> JmSearchPage: + ): """ 搜索album的作品 work """ @@ -334,7 +338,7 @@ class JmSearchAlbumClient: page: int = 1, order_by: str = ORDER_BY_LATEST, time: str = TIME_ALL, - ) -> JmSearchPage: + ): """ 搜索album的作者 author """ @@ -345,7 +349,7 @@ class JmSearchAlbumClient: page: int = 1, order_by: str = ORDER_BY_LATEST, time: str = TIME_ALL, - ) -> JmSearchPage: + ): """ 搜索album的标签 tag """ @@ -356,7 +360,7 @@ class JmSearchAlbumClient: page: int = 1, order_by: str = ORDER_BY_LATEST, time: str = TIME_ALL, - ) -> JmSearchPage: + ): """ 搜索album的登场角色 actor """ diff --git a/src/jmcomic/jm_entity.py b/src/jmcomic/jm_entity.py index 1c60359..ddcd5f7 100644 --- a/src/jmcomic/jm_entity.py +++ b/src/jmcomic/jm_entity.py @@ -5,49 +5,19 @@ from .jm_config import * class JmBaseEntity: - @staticmethod - def fix_title(title: str, limit=50): - """ - 一些过长的标题可能含有 \n,例如album: 360537 - 该方法会把 \n 去除 - """ - if len(title) > limit and '\n' in title: - title = title.replace('\n', '') - - return title.strip() - def save_to_file(self, filepath): from common import PackerUtil PackerUtil.pack(self, filepath) -class DetailEntity(JmBaseEntity): - - @property - def id(self) -> str: - raise NotImplementedError - - @property - def name(self) -> str: - return getattr(self, 'title') - - # help for typing - JMPI = Union['JmPhotoDetail', 'JmImageDetail'] - - def getindex(self, index: int) -> JMPI: +class IndexedEntity: + def getindex(self, index: int): raise NotImplementedError def __len__(self): raise NotImplementedError - def __iter__(self) -> Generator[JMPI, Any, None]: - for index in range(len(self)): - yield self.getindex(index) - - def __str__(self): - return f'{self.__class__.__name__}({self.id}-{self.name})' - - def __getitem__(self, item) -> Union[JMPI, List[JMPI]]: + def __getitem__(self, item) -> Any: if isinstance(item, slice): start = item.start or 0 stop = item.stop or len(self) @@ -60,6 +30,24 @@ class DetailEntity(JmBaseEntity): else: raise TypeError(f"Invalid item type for {self.__class__}") + def __iter__(self): + for index in range(len(self)): + yield self.getindex(index) + + +class DetailEntity(JmBaseEntity, IndexedEntity): + + @property + def id(self) -> str: + raise NotImplementedError + + @property + def name(self) -> str: + return getattr(self, 'title') + + def __str__(self): + return f'{self.__class__.__name__}({self.id}-{self.name})' + @classmethod def __alias__(cls): # "JmAlbumDetail" -> "album" (本子) @@ -163,7 +151,7 @@ class JmPhotoDetail(DetailEntity): ): self.photo_id: str = photo_id self.scramble_id: str = scramble_id - self.title: str = self.fix_title(str(title)) + self.title: str = str(title).strip() self.sort: int = int(sort) self._keywords: str = keywords self._series_id: int = int(series_id) @@ -290,7 +278,7 @@ class JmPhotoDetail(DetailEntity): def __len__(self): return len(self.page_arr) - def __iter__(self) -> Generator[JmImageDetail, Any, None]: + def __iter__(self) -> Generator[JmImageDetail, None, None]: return super().__iter__() @@ -418,21 +406,45 @@ class JmAlbumDetail(DetailEntity): def __len__(self): return len(self.episode_list) - def __iter__(self) -> Generator[JmPhotoDetail, Any, None]: + def __iter__(self) -> Generator[JmPhotoDetail, None, None]: return super().__iter__() -class JmSearchPage(JmBaseEntity): +class JmSearchPage(JmBaseEntity, IndexedEntity): + ContentItem = Tuple[str, Dict[str, Any]] - def __init__(self, album_info_list: List[Tuple[str, str, StrNone, StrNone, List[str]]]): - # (album_id, title, category_none, label_sub_none, tag_list) - self.album_info_list = album_info_list + def __init__(self, content: List[ContentItem]): + # [ + # album_id, {title, tag_list, ...} + # ] + self.content = content - def __len__(self): - return len(self.album_info_list) + def iter_id(self) -> Generator[str, None, None]: + """ + 返回 album_id 的迭代器 + """ + for aid, ainfo in self.content: + yield aid - def __getitem__(self, item) -> Tuple[str, str]: - return self.album_info_list[item][0:2] + def iter_id_title(self) -> Generator[Tuple[str, str], None, None]: + """ + 返回 album_id, album_title 的迭代器 + """ + for aid, ainfo in self.content: + yield aid, ainfo['name'] + + def iter_id_title_tag(self) -> Generator[Tuple[str, str, List[str]], None, None]: + """ + 返回 album_id, album_title, album_tag_list 的迭代器 + """ + for aid, ainfo in self.content: + yield aid, ainfo['name'], ainfo['tag_list'] + + # 下面的方法是对单个album的包装 + + @property + def is_single_album(self): + return hasattr(self, 'album') @property def single_album(self) -> JmAlbumDetail: @@ -440,17 +452,25 @@ class JmSearchPage(JmBaseEntity): @classmethod def wrap_single_album(cls, album: JmAlbumDetail) -> 'JmSearchPage': - # ('462257', '[無邪気漢化組] [きょくちょ] 楓と鈴 4.5', '短篇', '漢化', []) - # (album_id, title, category_none, label_sub_none, tag_list) - - album_info = ( - album.album_id, - album.title, - None, - None, - album.tag_list, - ) - obj = JmSearchPage([album_info]) - + obj = JmSearchPage([( + album.album_id, { + 'name': album.title, + 'tag_list': album.tag_list, + } + )]) setattr(obj, 'album', album) return obj + + # 下面的方法实现方便的元素访问 + + def __len__(self): + return len(self.content) + + def __iter__(self): + return self.iter_id_title() + + def __getitem__(self, item) -> Union[ContentItem, List[ContentItem]]: + return super().__getitem__(item) + + def getindex(self, index: int): + return self.content[index] diff --git a/src/jmcomic/jm_option.py b/src/jmcomic/jm_option.py index ef8eca3..70f42f1 100644 --- a/src/jmcomic/jm_option.py +++ b/src/jmcomic/jm_option.py @@ -130,11 +130,11 @@ class JmOption: # 路径规则配置 self.dir_rule = DirRule(**dir_rule) # 请求配置 - self.client = DictModel(client) + self.client = AdvancedEasyAccessDict(client) # 下载配置 - self.download = DictModel(download) + self.download = AdvancedEasyAccessDict(download) # 插件配置 - self.plugin = DictModel(plugin) + self.plugin = AdvancedEasyAccessDict(plugin) # 其他配置 self.filepath = filepath diff --git a/src/jmcomic/jm_plugin.py b/src/jmcomic/jm_plugin.py index 9676d63..9fcc312 100644 --- a/src/jmcomic/jm_plugin.py +++ b/src/jmcomic/jm_plugin.py @@ -55,11 +55,26 @@ class UsageLogPlugin(JmOptionPlugin): def invoke(self, **kwargs) -> None: import threading - threading.Thread( + t = threading.Thread( target=self.monitor_resource_usage, kwargs=kwargs, daemon=True, - ).start() + ) + t.start() + + self.set_thread_as_option_attr(t) + + def set_thread_as_option_attr(self, t): + """ + 线程留痕 + """ + name = f'thread_{self.plugin_key}' + + thread_ls: Optional[list] = getattr(self.option, name, None) + if thread_ls is None: + setattr(self.option, name, [t]) + else: + thread_ls.append(t) def monitor_resource_usage( self, diff --git a/src/jmcomic/jm_toolkit.py b/src/jmcomic/jm_toolkit.py index 52dbe75..04a3fa0 100644 --- a/src/jmcomic/jm_toolkit.py +++ b/src/jmcomic/jm_toolkit.py @@ -125,6 +125,10 @@ class JmcomicText: JmModuleConfig.album_class() ) + @classmethod + def analyse_jm_search_html(cls, html: str) -> JmSearchPage: + return JmcomicSearchTool.parse_html_to_page(html) + @classmethod def reflect_new_instance(cls, html: str, cls_field_prefix: str, clazz: type): @@ -185,6 +189,10 @@ class JmcomicText: @classmethod def format_url(cls, path, domain): assert isinstance(domain, str) and len(domain) != 0 + + if domain.startswith(JmModuleConfig.PROT): + return f'{domain}{path}' + return f'{JmModuleConfig.PROT}{domain}{path}' class DSLReplacer: @@ -220,7 +228,7 @@ class JmcomicText: JmcomicText.dsl_replacer.add_dsl_and_replacer('\$\{(.*?)\}', JmcomicText.match_os_env) -class JmSearchSupport: +class JmcomicSearchTool: # 用来缩减html的长度 pattern_html_search_shorten_for = compile('