mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v2.1.3: 实现对禁漫图片URL中的v参数的模拟 (#78)
This commit is contained in:
parent
41b3c0cd3c
commit
36f02c4382
10
.github/workflows/test.yml
vendored
10
.github/workflows/test.yml
vendored
@ -3,10 +3,12 @@ name: 跑测试
|
||||
on:
|
||||
push:
|
||||
branches: [ "dev" ]
|
||||
paths:
|
||||
- 'assets/config/*.yml' # option配置文件
|
||||
- 'src/**/*.py' # 源码
|
||||
- 'tests/**/*.py' # 测试代码
|
||||
paths-ignore:
|
||||
- 'src/**/__init__.py'
|
||||
- '.github/workflows/*'
|
||||
- 'usage/workflow_download.py'
|
||||
- 'assets/config/option_workflow_download.yml'
|
||||
- 'assets/docs/*'
|
||||
|
||||
jobs:
|
||||
test: # This code is based on https://github.com/gaogaotiantian/viztracer/blob/master/.github/workflows/python-package.yml
|
||||
|
||||
@ -2,6 +2,6 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- option
|
||||
|
||||
__version__ = '2.1.2'
|
||||
__version__ = '2.1.3'
|
||||
|
||||
from .api import *
|
||||
|
||||
@ -246,7 +246,7 @@ class JmHtmlClient(AbstractJmClient):
|
||||
"""
|
||||
使用此方法包装 self.get
|
||||
"""
|
||||
resp = JmImageResp(self.get(url, params={'v': format_ts()}))
|
||||
resp = JmImageResp(self.get(url))
|
||||
|
||||
if resp.is_success:
|
||||
return resp
|
||||
|
||||
@ -237,7 +237,7 @@ class JmImageClient:
|
||||
decode_image=True,
|
||||
):
|
||||
self.download_image(
|
||||
img_detail.img_url,
|
||||
img_detail.download_url,
|
||||
img_save_path,
|
||||
img_detail.scramble_id,
|
||||
decode_image=decode_image,
|
||||
|
||||
@ -48,6 +48,7 @@ class JmImageDetail(JmBaseEntity):
|
||||
img_file_name,
|
||||
img_file_suffix,
|
||||
from_photo=None,
|
||||
query_params=None,
|
||||
) -> None:
|
||||
self.aid: str = aid
|
||||
self.scramble_id: str = scramble_id
|
||||
@ -56,17 +57,31 @@ class JmImageDetail(JmBaseEntity):
|
||||
self.img_file_suffix: str = img_file_suffix
|
||||
|
||||
self.from_photo: Optional[JmPhotoDetail] = from_photo
|
||||
self.query_params: StrNone = query_params
|
||||
|
||||
@property
|
||||
def filename(self):
|
||||
def filename(self) -> str:
|
||||
return self.img_file_name + self.img_file_suffix
|
||||
|
||||
@property
|
||||
def download_url(self) -> str:
|
||||
"""
|
||||
图片的下载路径
|
||||
与 self.img_url 的唯一不同是,在最后会带上 ?{self.query_params}
|
||||
@return:
|
||||
"""
|
||||
if self.query_params is None:
|
||||
return self.img_url
|
||||
|
||||
return f'{self.img_url}?{self.query_params}'
|
||||
|
||||
@classmethod
|
||||
def of(cls,
|
||||
photo_id: str,
|
||||
scramble_id: str,
|
||||
data_original: str,
|
||||
from_photo=None,
|
||||
query_params=None,
|
||||
) -> 'JmImageDetail':
|
||||
"""
|
||||
该方法用于创建 JmImageDetail 对象
|
||||
@ -85,6 +100,7 @@ class JmImageDetail(JmBaseEntity):
|
||||
img_file_name=data_original[x + 1:y],
|
||||
img_file_suffix=data_original[y:],
|
||||
from_photo=from_photo,
|
||||
query_params=query_params,
|
||||
)
|
||||
|
||||
|
||||
@ -99,6 +115,7 @@ class JmPhotoDetail(WorkEntity):
|
||||
sort,
|
||||
page_arr=None,
|
||||
data_original_domain=None,
|
||||
data_original_0=None,
|
||||
author=None,
|
||||
from_album=None,
|
||||
):
|
||||
@ -111,17 +128,28 @@ class JmPhotoDetail(WorkEntity):
|
||||
|
||||
self._author: StrNone = author
|
||||
self.from_album: Optional[JmAlbumDetail] = from_album
|
||||
self.index = self.album_index
|
||||
|
||||
# 下面的属性和图片url有关
|
||||
if isinstance(page_arr, str):
|
||||
import json
|
||||
page_arr = json.loads(page_arr)
|
||||
|
||||
# 该photo的所有图片名 img_name
|
||||
# page_arr存放了该photo的所有图片文件名 img_name
|
||||
self.page_arr: List[str] = page_arr
|
||||
# 图片域名
|
||||
# 图片的cdn域名
|
||||
self.data_original_domain: StrNone = data_original_domain
|
||||
self.index = self.album_index
|
||||
# 第一张图的URL
|
||||
self.data_original_0 = data_original_0
|
||||
|
||||
# 2023-07-14
|
||||
# 禁漫的图片url加上了一个参数v,如果没有带上这个参数v,图片会返回空数据
|
||||
# 参数v的特点:
|
||||
# 1. 值似乎是该photo的更新时间的时间戳,因此所有图片都使用同一个值
|
||||
# 2. 值目前在网页端只在photo页面的图片标签的data-original属性出现
|
||||
# 这里的模拟思路是,获取到第一个图片标签的data-original,
|
||||
# 取出其query参数 → self.data_original_query_params, 该值未来会传递给 JmImageDetail
|
||||
self.data_original_query_params = self.get_data_original_query_params(data_original_0)
|
||||
|
||||
@property
|
||||
def is_single_album(self) -> bool:
|
||||
@ -179,7 +207,8 @@ class JmPhotoDetail(WorkEntity):
|
||||
self.photo_id,
|
||||
self.scramble_id,
|
||||
data_original,
|
||||
from_photo=self
|
||||
from_photo=self,
|
||||
query_params=self.data_original_query_params,
|
||||
)
|
||||
|
||||
def get_img_data_original(self, img_name: str) -> str:
|
||||
@ -194,6 +223,17 @@ class JmPhotoDetail(WorkEntity):
|
||||
|
||||
return f'https://{data_original_domain}/media/photos/{self.photo_id}/{img_name}'
|
||||
|
||||
# noinspection PyMethodMayBeStatic
|
||||
def get_data_original_query_params(self, data_original_0: StrNone) -> str:
|
||||
if data_original_0 is None:
|
||||
return f'v={time_stamp()}'
|
||||
|
||||
index = data_original_0.rfind('?')
|
||||
if index == -1:
|
||||
return f'v={time_stamp()}'
|
||||
|
||||
return data_original_0[index + 1:]
|
||||
|
||||
def __getitem__(self, item) -> JmImageDetail:
|
||||
return self.create_image_detail(item)
|
||||
|
||||
|
||||
@ -11,6 +11,7 @@ class JmcomicText:
|
||||
pattern_html_photo_title = compile('<title>(.*?)\|.*</title>')
|
||||
# pattern_html_photo_data_original_list = compile('data-original="(.*?)" id="album_photo_.+?"')
|
||||
pattern_html_photo_data_original_domain = compile('src="https://(.*?)/media/albums/blank')
|
||||
pattern_html_photo_data_original_0 = compile('data-original="(.*?)" id="album_photo')
|
||||
pattern_html_photo_keywords = compile('<meta name="keywords"[\s\S]*?content="(.*?)"')
|
||||
pattern_html_photo_series_id = compile('var series_id = (\d+);')
|
||||
pattern_html_photo_sort = compile('var sort = (\d+);')
|
||||
|
||||
@ -102,7 +102,31 @@ class Test_Api(JmTestConfigurable):
|
||||
)
|
||||
|
||||
def test_get_jmcomic_url(self):
|
||||
print(self.client.get_jmcomic_url())
|
||||
print(self.client.get_jmcomic_domain_all())
|
||||
print(JmModuleConfig.get_jmcomic_url())
|
||||
print(JmModuleConfig.get_jmcomic_domain_all())
|
||||
func_list = {
|
||||
self.client.get_jmcomic_url,
|
||||
self.client.get_jmcomic_domain_all,
|
||||
JmModuleConfig.get_jmcomic_url,
|
||||
JmModuleConfig.get_jmcomic_domain_all,
|
||||
}
|
||||
|
||||
exception_list = []
|
||||
|
||||
def run_func_async(func):
|
||||
try:
|
||||
func()
|
||||
except BaseException as e:
|
||||
exception_list.append(e)
|
||||
traceback_print_exec()
|
||||
|
||||
multi_thread_launcher(
|
||||
iter_objs=func_list,
|
||||
apply_each_obj_func=run_func_async,
|
||||
)
|
||||
|
||||
if len(exception_list) == 0:
|
||||
return
|
||||
|
||||
for e in exception_list:
|
||||
print(e)
|
||||
|
||||
raise AssertionError(exception_list)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user