v2.4.4: 优化GitHub Actions、文档、注释、代码结构 (#168)

This commit is contained in:
hect0x7 2023-11-24 15:37:18 +08:00 committed by GitHub
parent f35cb12f49
commit 6ab7456f99
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 157 additions and 95 deletions

View File

@ -15,8 +15,8 @@ on:
CLIENT_IMPL:
type: string
description: 客户端类型client.impl[api]=移动端,[html]=网页端。
default: 'api'
description: 客户端类型client.impl[api]=移动端,[html]=网页端。下载失败时,你可以尝试填入此项重试。
default: ''
required: false
IMAGE_SUFFIX:

Binary file not shown.

After

Width:  |  Height:  |  Size: 58 KiB

View File

@ -22,11 +22,15 @@
## 2. 填写你需要下载的本子id
在开始下面的步骤之前你需要先启用你的repo的Actions开启方式如下
![6](../images/6.png)
### 2.1. 方式一(最新、简单、推荐)
访问下面这个网址:
`https://github.com/你的用户名/JMComic-Crawler-Python/master/workflows/download_dispatch.yml`
`https://github.com/你的用户名/JMComic-Crawler-Python/actions/workflows/download_dispatch.yml`
按下图步骤进行操作:

View File

@ -81,4 +81,37 @@ for aid, atitle, tag_list in page.iter_id_title_tag(): # 使用page的iter_id_t
aid_list.append(aid)
download_album(aid_list, option)
```
```
## 手动创建Client
```python
# 默认的使用方式是先创建optionoption封装了所有配置然后由option.new_jm_client() 创建客户端client使用client可以访问禁漫接口
# 下面演示直接构造client的方式
from jmcomic import *
"""
创建JM客户端
:param postman: 负责实现HTTP请求的对象持有cookies、headers、proxies等信息
:param domain_list: 禁漫域名
:param retry_times: 重试次数
"""
# 网页端
cl = JmHtmlClient(
postman=JmModuleConfig.new_postman(),
domain_list=['18comic.vip'],
retry_times=1
)
# API端APP
cl = JmApiClient(
postman=JmModuleConfig.new_postman(),
domain_list=JmModuleConfig.DOMAIN_API_LIST,
retry_times=1
)
```

View File

@ -4,6 +4,7 @@ dir_rule:
rule: Bd_Aauthor_Atitle_Pindex
client:
impl: api # 使用api可免登录下载本子
domain:
html: [ jmcomic1.me, jmcomic.me ]

View File

@ -27,7 +27,7 @@ setup(
package_dir={"": "src"},
python_requires=">=3.7",
install_requires=[
'commonX>=0.6.3',
'commonX>=0.6.4',
'curl_cffi',
'PyYAML',
'Pillow',

View File

@ -2,7 +2,7 @@
# 被依赖方 <--- 使用方
# config <--- entity <--- toolkit <--- client <--- option <--- downloader
__version__ = '2.4.3'
__version__ = '2.4.4'
from .api import *
from .jm_plugin import *

View File

@ -11,20 +11,19 @@ class AbstractJmClient(
def __init__(self,
postman: Postman,
retry_times: int,
domain=None,
fallback_domain_list=None,
domain_list: List[str],
retry_times=0,
):
"""
创建JM客户端
:param postman: 负责实现HTTP请求的对象持有cookiesheadersproxies等信息
:param domain_list: 禁漫域名
:param retry_times: 重试次数
"""
super().__init__(postman)
self.retry_times = retry_times
if fallback_domain_list is None:
fallback_domain_list = []
if domain is not None:
fallback_domain_list.insert(0, domain)
self.domain_list = fallback_domain_list
self.domain_list = domain_list
self.CLIENT_CACHE = None
self.enable_cache()
self.after_init()
@ -98,6 +97,9 @@ class AbstractJmClient(
resp = request(url, **kwargs)
return judge(resp)
except Exception as e:
if self.retry_times == 0:
raise e
self.before_retry(e, kwargs, retry_count, url)
if retry_count < self.retry_times:
@ -190,7 +192,7 @@ class AbstractJmClient(
# noinspection PyMethodMayBeStatic
def decode(self, url: str):
if not JmModuleConfig.decode_url_when_logging or '/search/' not in url:
if not JmModuleConfig.flag_decode_url_when_logging or '/search/' not in url:
return url
from urllib.parse import unquote
@ -371,7 +373,7 @@ class JmHtmlClient(AbstractJmClient):
status='true',
comment_id=None,
**kwargs,
) -> JmAcResp:
) -> JmAlbumCommentResp:
data = {
'video_id': video_id,
'comment': comment,
@ -396,7 +398,7 @@ class JmHtmlClient(AbstractJmClient):
data=data,
)
ret = JmAcResp(resp)
ret = JmAlbumCommentResp(resp)
jm_log('album.comment', f'{video_id}: [{comment}] ← ({ret.model().cid})')
return ret
@ -754,15 +756,15 @@ class JmApiClient(AbstractJmClient):
ts = time_stamp()
token, tokenparam = JmCryptoTool.token_and_tokenparam(ts, secret=JmMagicConstants.APP_TOKEN_SECRET_2)
elif JmModuleConfig.use_fix_timestamp:
elif JmModuleConfig.flag_use_fix_timestamp:
ts, token, tokenparam = JmModuleConfig.get_fix_ts_token_tokenparam()
else:
ts = time_stamp()
token, tokenparam = JmCryptoTool.token_and_tokenparam(ts)
# 计算tokentokenparam
headers = kwargs.get('headers', JmMagicConstants.APP_HEADERS_TEMPLATE.copy())
# 设置headers
headers = kwargs.get('headers', None) or JmMagicConstants.APP_HEADERS_TEMPLATE.copy()
headers.update({
'token': token,
'tokenparam': tokenparam,
@ -786,7 +788,7 @@ class JmApiClient(AbstractJmClient):
def after_init(self):
# 保证拥有cookies因为移动端要求必须携带cookies否则会直接跳转同一本子【禁漫娘】
if JmModuleConfig.api_client_require_cookies:
if JmModuleConfig.flag_api_client_require_cookies:
self.ensure_have_cookies()
from threading import Lock
@ -800,7 +802,13 @@ class JmApiClient(AbstractJmClient):
if self.get_meta_data('cookies'):
return
self['cookies'] = JmModuleConfig.get_cookies(self)
self['cookies'] = self.get_cookies()
@field_cache("APP_COOKIES", obj=JmModuleConfig)
def get_cookies(self):
resp = self.setting()
cookies = dict(resp.resp.cookies)
return cookies
class FutureClientProxy(JmcomicClient):
@ -823,12 +831,13 @@ class FutureClientProxy(JmcomicClient):
'set_cache_dict', 'get_cache_dict', 'set_domain_list', ]
class FutureWrapper:
def __init__(self, future):
def __init__(self, future, after_done_callback):
from concurrent.futures import Future
future: Future
self.future = future
self.done = False
self._result = None
self.after_done_callback = after_done_callback
def result(self):
if not self.done:
@ -836,6 +845,7 @@ class FutureClientProxy(JmcomicClient):
self._result = result
self.done = True
self.future = None # help gc
self.after_done_callback()
return self._result
@ -871,7 +881,9 @@ class FutureClientProxy(JmcomicClient):
if cache_key in self.future_dict:
return self.future_dict[cache_key]
future = self.FutureWrapper(self.executors.submit(task))
future = self.FutureWrapper(self.executors.submit(task),
after_done_callback=lambda: self.future_dict.pop(cache_key, None)
)
self.future_dict[cache_key] = future
return future

View File

@ -9,30 +9,46 @@ Response Entity
DictModel = AdvancedEasyAccessDict
class JmResp(CommonResp):
class JmResp:
def __init__(self, resp):
ExceptionTool.require_true(not isinstance(resp, JmResp), f'重复包装: {resp}')
self.resp = resp
@property
def is_success(self) -> bool:
return self.http_code == 200 and len(self.content) != 0
def json(self, **kwargs) -> Dict:
raise NotImplementedError
@property
def is_not_success(self) -> bool:
return not self.is_success
def model(self) -> DictModel:
return DictModel(self.json())
@property
def content(self):
return self.resp.content
@property
def http_code(self):
return self.resp.status_code
@property
def text(self) -> str:
return self.resp.text
@property
def url(self) -> str:
return self.resp.url
def require_success(self):
if self.is_not_success:
ExceptionTool.raises_resp(self.text, self.resp)
ExceptionTool.raises_resp(self.text, self)
class JmImageResp(JmResp):
def json(self, **kwargs) -> Dict:
raise NotImplementedError
def require_success(self):
ExceptionTool.require_true(self.is_success, self.get_error_msg())
if self.is_not_success:
ExceptionTool.raises_resp(self.get_error_msg(), self)
def get_error_msg(self):
msg = f'禁漫图片获取失败: [{self.url}]'
@ -66,21 +82,27 @@ class JmImageResp(JmResp):
)
class JmApiResp(JmResp):
class JmJsonResp(JmResp):
def json(self) -> Dict:
return self.resp.json()
def model(self) -> DictModel:
return DictModel(self.json())
class JmApiResp(JmJsonResp):
def __init__(self, resp, ts: str):
ExceptionTool.require_true(not isinstance(resp, JmApiResp), f'重复包装: {resp}')
super().__init__(resp)
self.ts = ts
self.cache_decode_data = None
@property
def is_success(self) -> bool:
return super().is_success and self.json()['code'] == 200
@property
@field_cache('__cache_decoded_data__')
@field_cache()
def decoded_data(self) -> str:
return JmCryptoTool.decode_resp_data(self.encoded_data, self.ts)
@ -94,10 +116,6 @@ class JmApiResp(JmResp):
from json import loads
return loads(self.decoded_data)
@field_cache('__cache_json__')
def json(self, **kwargs) -> Dict:
return self.resp.json()
@property
def model_data(self) -> DictModel:
self.require_success()
@ -105,12 +123,12 @@ class JmApiResp(JmResp):
# album-comment
class JmAcResp(JmResp):
class JmAlbumCommentResp(JmJsonResp):
def is_success(self) -> bool:
return super().is_success and self.json()['err'] is False
def json(self, **kwargs) -> Dict:
def json(self) -> Dict:
return self.resp.json()
@ -184,7 +202,7 @@ class JmUserClient:
status='true',
comment_id=None,
**kwargs,
) -> JmAcResp:
) -> JmAlbumCommentResp:
"""
评论漫画/评论回复
:param video_id: album_id/photo_id

View File

@ -1,4 +1,4 @@
from common import time_stamp, field_cache, str_to_list
from common import time_stamp, str_to_list, field_cache, ProxyBuilder
def default_jm_logging(topic: str, msg: str):
@ -10,22 +10,19 @@ def default_raise_exception_executor(msg, _extra):
raise JmModuleConfig.CLASS_EXCEPTION(msg)
def system_proxy():
from common import ProxyBuilder
return ProxyBuilder.system_proxy()
class JmcomicException(Exception):
pass
# 禁漫常量
class JmMagicConstants:
# 搜索参数-排序
ORDER_BY_LATEST = 'mr'
ORDER_BY_VIEW = 'mv'
ORDER_BY_PICTURE = 'mp'
ORDER_BY_LIKE = 'tf'
# 搜索参数-时间段
TIME_TODAY = 't'
TIME_WEEK = 'w'
TIME_MONTH = 'm'
@ -35,10 +32,14 @@ class JmMagicConstants:
PAGE_SIZE_SEARCH = 80
PAGE_SIZE_FAVORITE = 20
# 图片分割参数
SCRAMBLE_220980 = 220980
SCRAMBLE_268850 = 268850
SCRAMBLE_421926 = 421926 # 2023-02-08后改了图片切割算法
# 当本子没有作者名字时,顶替作者名字
DEFAULT_AUTHOR = 'default_author'
# 移动端API密钥
APP_TOKEN_SECRET = '18comicAPP'
APP_TOKEN_SECRET_2 = '18comicAPPContent'
@ -137,21 +138,18 @@ class JmModuleConfig:
REGISTRY_PLUGIN = {}
# 执行log的函数
log_executor = default_jm_logging
executor_log = default_jm_logging
# 网页正则表达式解析失败时执行抛出异常的函数可以替换掉用于log
raise_exception_executor = default_raise_exception_executor
executor_raise_exception = default_raise_exception_executor
# 使用固定时间戳
use_fix_timestamp = True
flag_use_fix_timestamp = True
# 移动端Client初始化cookies
api_client_require_cookies = True
flag_api_client_require_cookies = True
# log开关标记
enable_jm_log = True
flag_enable_jm_log = True
# log时解码url
decode_url_when_logging = True
# 下载时的一些默认值配置
DEFAULT_AUTHOR = 'default-author'
flag_decode_url_when_logging = True
@classmethod
def downloader_class(cls):
@ -260,13 +258,6 @@ class JmModuleConfig:
})
return headers
@classmethod
@field_cache("APP_COOKIES")
def get_cookies(cls, client):
resp = client.setting()
cookies = dict(resp.resp.cookies)
return cookies
@classmethod
@field_cache("__fix_ts_token_tokenparam__")
def get_fix_ts_token_tokenparam(cls):
@ -278,12 +269,12 @@ class JmModuleConfig:
# noinspection PyUnusedLocal
@classmethod
def jm_log(cls, topic: str, msg: str):
if cls.enable_jm_log is True:
cls.log_executor(topic, msg)
if cls.flag_enable_jm_log is True:
cls.executor_log(topic, msg)
@classmethod
def disable_jm_log(cls):
cls.enable_jm_log = False
cls.flag_enable_jm_log = False
@classmethod
def new_postman(cls, session=False, **kwargs):
@ -301,7 +292,7 @@ class JmModuleConfig:
# option 相关的默认配置
JM_OPTION_VER = '2.1'
DEFAULT_CLIENT_IMPL = 'html'
DEFAULT_PROXIES = system_proxy() # use system proxy by default
DEFAULT_PROXIES = ProxyBuilder.system_proxy() # use system proxy by default
default_option_dict: dict = {
'log': None,
@ -347,7 +338,7 @@ class JmModuleConfig:
# log
if option_dict['log'] is None:
option_dict['log'] = cls.enable_jm_log
option_dict['log'] = cls.flag_enable_jm_log
# dir_rule.base_dir
dir_rule = option_dict['dir_rule']

View File

@ -11,7 +11,7 @@ class DownloadCallback:
f'章节数: [{len(album)}], '
f'总页数: [{album.page_count}], '
f'标题: [{album.name}], '
f'关键词: [{album.tags}]'
f'关键词: {album.tags}'
)
def after_album(self, album: JmAlbumDetail):

View File

@ -269,7 +269,7 @@ class JmPhotoDetail(DetailEntity):
return self._author.strip()
# 使用默认
return JmModuleConfig.DEFAULT_AUTHOR
return JmMagicConstants.DEFAULT_AUTHOR
def create_image_detail(self, index) -> JmImageDetail:
# 校验参数
@ -387,7 +387,7 @@ class JmAlbumDetail(DetailEntity):
if len(self.authors) >= 1:
return self.authors[0]
return JmModuleConfig.DEFAULT_AUTHOR
return JmMagicConstants.DEFAULT_AUTHOR
@property
def id(self):

View File

@ -323,7 +323,7 @@ class JmOption:
def deconstruct(self) -> Dict:
return {
'version': self.version,
'log': JmModuleConfig.enable_jm_log,
'log': JmModuleConfig.flag_enable_jm_log,
'dir_rule': {
'rule': self.dir_rule.rule_dsl,
'base_dir': self.dir_rule.base_dir,
@ -354,11 +354,11 @@ class JmOption:
下面是创建客户端的相关方法
"""
@field_cache("__jm_client_cache__")
@field_cache()
def build_jm_client(self, **kwargs):
"""
该方法会首次调用会创建JmcomicClient对象
然后保存在self.__jm_client_cache__
然后保存在self
多次调用`不会`创建新的JmcomicClient对象
"""
return self.new_jm_client(**kwargs)
@ -414,9 +414,9 @@ class JmOption:
raise NotImplementedError(clazz)
client: AbstractJmClient = clazz(
postman,
retry_times,
fallback_domain_list=decide_domain(),
postman=postman,
domain_list=decide_domain(),
retry_times=retry_times,
)
# enable cache

View File

@ -454,7 +454,7 @@ class LogTopicFilterPlugin(JmOptionPlugin):
if whitelist is not None:
whitelist = set(whitelist)
old_jm_log = JmModuleConfig.log_executor
old_jm_log = JmModuleConfig.executor_log
def new_jm_log(topic, msg):
if whitelist is not None and topic not in whitelist:
@ -462,7 +462,7 @@ class LogTopicFilterPlugin(JmOptionPlugin):
old_jm_log(topic, msg)
JmModuleConfig.log_executor = new_jm_log
JmModuleConfig.executor_log = new_jm_log
class AutoSetBrowserCookiesPlugin(JmOptionPlugin):

View File

@ -163,7 +163,10 @@ class JmcomicText:
if field_value is None:
if default is None:
ExceptionTool.raises_regex(
f"文本没有匹配上字段:字段名为'{field_name}'pattern: [{pattern}]",
f"文本没有匹配上字段:字段名为'{field_name}'pattern: [{pattern}]" \
+ (f"\n响应文本=[{html}]" if len(html) < 200 else
f'响应文本过长(len={len(html)}),不打印'
),
html=html,
pattern=pattern,
)
@ -716,7 +719,7 @@ class ExceptionTool:
if extra is None:
extra = {}
JmModuleConfig.raise_exception_executor(msg, extra)
JmModuleConfig.executor_raise_exception(msg, extra)
@classmethod
def raises_regex(cls,
@ -772,12 +775,12 @@ class ExceptionTool:
@classmethod
def replace_old_exception_executor(cls, raises: Callable[[Callable, str, dict], None]):
old = JmModuleConfig.raise_exception_executor
old = JmModuleConfig.executor_raise_exception
def new(msg, extra):
raises(old, msg, extra)
JmModuleConfig.raise_exception_executor = new
JmModuleConfig.executor_raise_exception = new
class JmCryptoTool:
@ -831,7 +834,7 @@ class JmCryptoTool:
from Crypto.Cipher import AES
data_aes = AES.new(key, AES.MODE_ECB).decrypt(data_b64)
# 3. 移除末尾的一些特殊字符
# 3. 移除末尾的padding
data = data_aes[:-data_aes[-1]]
# 4. 解码为字符串 (json)

View File

@ -57,11 +57,11 @@ class Test_Client(JmTestConfigurable):
self.assertEqual(old, default_raise_exception_executor)
raise B()
JmModuleConfig.raise_exception_executor = default_raise_exception_executor
JmModuleConfig.executor_raise_exception = default_raise_exception_executor
ExceptionTool.replace_old_exception_executor(raises)
self.assertRaises(B, JmcomicText.parse_to_jm_id, 'asdhasjhkd')
# 还原
JmModuleConfig.raise_exception_executor = default_raise_exception_executor
JmModuleConfig.executor_raise_exception = default_raise_exception_executor
def test_detail_property_list(self):
album = self.client.get_album_detail(410090)
@ -253,7 +253,7 @@ class Test_Client(JmTestConfigurable):
self.assertEqual(ans, id(photo))
def test_search_generator(self):
JmModuleConfig.decode_url_when_logging = False
JmModuleConfig.flag_decode_url_when_logging = False
gen = self.client.search_gen('MANA')
for i, page in enumerate(gen):