mirror of
https://github.com/hect0x7/JMComic-Crawler-Python.git
synced 2025-11-04 14:49:43 +08:00
v1.7.0: 增加对禁漫发布页的解析,可拿到所有禁漫网址 (#17)
This commit is contained in:
parent
0227da2394
commit
56c72b96e3
@ -2,6 +2,6 @@
|
||||
# 被依赖方 <--- 使用方
|
||||
# config <--- entity <--- toolkit <--- client <--- service <--- option
|
||||
|
||||
__version__ = '1.6.0'
|
||||
__version__ = '1.7.0'
|
||||
|
||||
from .api import *
|
||||
|
||||
@ -183,6 +183,12 @@ class JmcomicClient(PostmanProxy):
|
||||
wrap_func_cache('get_photo_detail', 'album_cache_dict')
|
||||
wrap_func_cache('get_album_detail', 'photo_cache_dict')
|
||||
|
||||
def get_jmcomic_url(self, postman=None):
|
||||
return JmModuleConfig.get_jmcomic_url(postman or self)
|
||||
|
||||
def get_jmcomic_url_all(self, postman=None):
|
||||
return JmModuleConfig.get_jmcomic_url_all(postman or self)
|
||||
|
||||
|
||||
# 爬取策略
|
||||
class FetchStrategy:
|
||||
|
||||
@ -74,15 +74,33 @@ class JmModuleConfig:
|
||||
@classmethod
|
||||
def get_jmcomic_url(cls, postman=None):
|
||||
"""
|
||||
访问禁漫的永久网域,从而得到一个可用的禁漫网址,
|
||||
访问禁漫的永久网域,从而得到一个可用的禁漫网址
|
||||
@return: https://jm-comic2.cc
|
||||
"""
|
||||
if postman is None:
|
||||
from common import Postmans
|
||||
postman = Postmans.get_impl_clazz('cffi_Session').create()
|
||||
|
||||
domain = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL)
|
||||
cls.jm_debug('获取禁漫地址', f'[{cls.JM_REDIRECT_URL}] → [{domain}]')
|
||||
return domain
|
||||
url = postman.with_redirect_catching().get(cls.JM_REDIRECT_URL)
|
||||
cls.jm_debug('获取禁漫地址', f'[{cls.JM_REDIRECT_URL}] → [{url}]')
|
||||
return url
|
||||
|
||||
@classmethod
|
||||
def get_jmcomic_url_all(cls, postman=None):
|
||||
"""
|
||||
访问禁漫发布页,得到所有禁漫的域名
|
||||
@return:['18comic.vip', ..., 'jm365.xyz/ZNPJam'], 最后一个是【APP軟件下載】
|
||||
"""
|
||||
if postman is None:
|
||||
from common import Postmans
|
||||
postman = Postmans.get_impl_clazz('cffi').create()
|
||||
|
||||
resp = postman.get(cls.JM_PUB_URL)
|
||||
if resp.status_code != 200:
|
||||
raise AssertionError(resp.text)
|
||||
|
||||
from .jm_toolkit import JmcomicText
|
||||
return JmcomicText.analyse_jm_pub_html(resp.text)
|
||||
|
||||
@classmethod
|
||||
def check_html(cls, html: str, url=None):
|
||||
|
||||
@ -6,6 +6,7 @@ from .jm_entity import *
|
||||
class JmcomicText:
|
||||
pattern_jm_domain = compile('https://([\w.-]+)')
|
||||
pattern_jm_pa_id = compile('/(photos?|album)/(\d+)')
|
||||
pattern_html_jm_pub_domain = compile('[\w-]+\.\w+/?\w+')
|
||||
|
||||
pattern_html_photo_photo_id = compile('<meta property="og:url" content=".*?/photo/(\d+)/?.*?">')
|
||||
pattern_html_photo_scramble_id = compile('var scramble_id = (\d+);')
|
||||
@ -73,6 +74,15 @@ class JmcomicText:
|
||||
def parse_to_album_id(cls, text) -> str:
|
||||
return cls.parse_to_photo_id(text)
|
||||
|
||||
@classmethod
|
||||
def analyse_jm_pub_html(cls, html: str, domain_keyword=('jm', 'comic')) -> List[str]:
|
||||
domain_ls = cls.pattern_html_jm_pub_domain.findall(html)
|
||||
|
||||
return list(filter(
|
||||
lambda domain: any(kw in domain for kw in domain_keyword),
|
||||
domain_ls
|
||||
))
|
||||
|
||||
@classmethod
|
||||
def analyse_jm_photo_html(cls, html: str) -> JmPhotoDetail:
|
||||
return cls.reflect_new_instance(
|
||||
|
||||
@ -113,3 +113,8 @@ class Test_Api(JmTestConfigurable):
|
||||
sorted([ans.sort for ans in photo_ls]),
|
||||
album.album_id
|
||||
)
|
||||
|
||||
def test_get_jmcomic_url(self):
|
||||
print(JmModuleConfig.get_jmcomic_url_all())
|
||||
print(self.client.get_jmcomic_url())
|
||||
print(self.client.get_jmcomic_url_all())
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
--------------------
|
||||
"""
|
||||
import jmcomic # 导入此模块,需要先安装.
|
||||
|
||||
jmcomic.download_album('422866') # 传入要下载的album的id,即可下载整个album到本地.
|
||||
# 上面的这行代码,还有一个可选参数option: JmOption,表示配置项,
|
||||
# 配置项的作用是告诉程序下载时候的一些选择,
|
||||
@ -23,6 +24,22 @@ jmcomic.download_album({'422866', '1', '2', '3'}) # set
|
||||
jmcomic.download_album(('422866', '1', '2', '3')) # tuple
|
||||
jmcomic.download_album(aid for aid in ('422866', '1', '2', '3')) # 生成器
|
||||
|
||||
|
||||
"""
|
||||
--------------------
|
||||
获取域名介绍
|
||||
--------------------
|
||||
"""
|
||||
|
||||
# 方式1: 访问禁漫发布页
|
||||
url_ls = jmcomic.JmModuleConfig.get_jmcomic_url_all()
|
||||
print(url_ls)
|
||||
|
||||
# 方式2(可能会报错,需要你自己配置梯子)
|
||||
url = jmcomic.JmModuleConfig.get_jmcomic_url()
|
||||
print(url)
|
||||
|
||||
|
||||
"""
|
||||
--------------------
|
||||
配置文件介绍
|
||||
@ -37,4 +54,4 @@ jm_option.save_to_file('./禁漫下载默认配置.json') # json格式
|
||||
|
||||
# 如果你修改了默认配置,现在想用你修改后的配置来下载,使用如下代码
|
||||
jm_option = jmcomic.create_option('./禁漫下载默认配置.yml')
|
||||
jmcomic.download_album('23333', jm_option)
|
||||
jmcomic.download_album('23333', jm_option)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user