Enhancements

Fix when subscription download timeout before complete, the parser will still try to parse downloaded content.
Change default regex library to PCRE2.
Change Quantumult X "url-test" equivalent to "available".
Optimize codes.
Update build scripts.
This commit is contained in:
Tindy X 2020-02-02 23:18:20 +08:00
parent fd5a74ec93
commit 00976c9eeb
No known key found for this signature in database
GPG Key ID: C6AD413169968D58
12 changed files with 164 additions and 133 deletions

View File

@ -7,7 +7,7 @@ ENDIF()
SET(CMAKE_CXX_STANDARD 17)
ADD_DEFINITIONS(-Wall -Wextra -Wno-unused-parameter -Wno-unused-result)
OPTION(USING_STD_REGEX "Use std::regex from C++ library instead of PCRECPP." OFF)
OPTION(USING_STD_REGEX "Use std::regex from C++ library instead of PCRE2." OFF)
OPTION(USING_MALLOC_TRIM "Call malloc_trim after processing request to lower memory usage (Your system must support malloc_trim)." OFF)
INCLUDE(CheckCXXSourceCompiles)
@ -77,10 +77,13 @@ TARGET_LINK_LIBRARIES(subconverter ${YAML_CPP_LIBRARY})
IF(USING_STD_REGEX STREQUAL "ON")
ADD_DEFINITIONS(-DUSE_STD_REGEX)
ELSE()
FIND_PACKAGE(PCRECPP REQUIRED)
INCLUDE_DIRECTORIES(${PCRECPP_INCLUDE_DIRS})
TARGET_LINK_LIBRARIES(subconverter ${PCRECPP_LIBRARIES})
ADD_DEFINITIONS(-DPCRE_STATIC)
PKG_CHECK_MODULES(PCRE2 libpcre2-8 REQUIRED)
INCLUDE_DIRECTORIES(${PCRE2_INCLUDE_DIRS})
TARGET_LINK_LIBRARIES(subconverter ${PCRE2_LIBRARIES})
ADD_DEFINITIONS(-DPCRE2_STATIC)
FIND_PACKAGE(JPCRE2 REQUIRED)
INCLUDE_DIRECTORIES(${JPCRE2_INCLUDE_DIRS})
ENDIF()
IF(WIN32)

27
cmake/FindJPCRE2.cmake Normal file
View File

@ -0,0 +1,27 @@
# - Find JPCRE2
# Find the JPCRE2 headers.
#
# JPCRE2_INCLUDE_DIRS - where to find jpcre2.h.
# JPCRE2_FOUND - True if JPCRE2 found.
# Look for the header file.
FIND_PATH(JPCRE2_INCLUDE_DIR NAMES jpcre2.hpp)
MARK_AS_ADVANCED(JPCRE2_INCLUDE_DIR)
if(JPCRE2_INCLUDE_DIR)
set(JPCRE2_FOUND TRUE)
endif()
IF(JPCRE2_FOUND)
SET(JPCRE2_INCLUDE_DIRS ${JPCRE2_INCLUDE_DIR})
ENDIF(JPCRE2_FOUND)
if(JPCRE2_FOUND)
if(NOT JPCRE2_FIND_QUIETLY)
message(STATUS "Found JPCRE2 header files in ${JPCRE2_INCLUDE_DIRS}")
endif()
elseif(JPCRE2_FIND_REQUIRED)
message(FATAL_ERROR "Could not find JPCRE2")
else()
message(STATUS "Optional package JPCRE2 was not found")
endif()

View File

@ -3,7 +3,7 @@ MAINTAINER Tindy X <tindy.it@gmail.com>
# build minimized
RUN apk add git g++ build-base linux-headers cmake && \
apk add libressl-dev curl-dev rapidjson-dev libevent-dev pcre-dev yaml-cpp-dev && \
apk add libressl-dev curl-dev rapidjson-dev libevent-dev pcre2-dev yaml-cpp-dev && \
git clone https://github.com/tindy2013/subconverter && \
cd subconverter && \
cmake . && \
@ -12,8 +12,8 @@ RUN apk add git g++ build-base linux-headers cmake && \
mv base ../ && \
cd .. && \
rm -rf subconverter && \
apk add libcurl yaml-cpp libevent libpcrecpp && \
apk del git gcc g++ build-base linux-headers cmake libressl-dev curl-dev rapidjson-dev libevent-dev pcre-dev yaml-cpp-dev
apk add pcre2 libcurl yaml-cpp libevent libpcrecpp && \
apk del git gcc g++ build-base linux-headers cmake libressl-dev curl-dev rapidjson-dev libevent-dev pcre2-dev yaml-cpp-dev
# set entry
CMD /base/subconverter

View File

@ -3,7 +3,7 @@ mkdir obj
set -xe
apk add gcc g++ build-base linux-headers cmake make autoconf automake libtool
apk add openssl-dev openssl-libs-static curl-dev curl-static nghttp2-static zlib-dev rapidjson-dev libevent-dev libevent-static zlib-static pcre-dev bzip2-static
apk add openssl-dev openssl-libs-static curl-dev curl-static nghttp2-static zlib-dev rapidjson-dev libevent-dev libevent-static zlib-static pcre2-dev bzip2-static
git clone https://github.com/jbeder/yaml-cpp
cd yaml-cpp
@ -11,9 +11,12 @@ cmake -DYAML_CPP_BUILD_TESTS=OFF -DYAML_CPP_BUILD_TOOLS=OFF . > /dev/null
make install -j4 > /dev/null
cd ..
curl -LO https://raw.githubusercontent.com/jpcre2/jpcre2/release/src/jpcre2.hpp
cp jpcre2.hpp /usr/local/include/
cmake .
make -j4
g++ -o base/subconverter CMakeFiles/subconverter.dir/src/*.o -static -lpcrecpp -lpcre -levent -lyaml-cpp -lcurl -lnghttp2 -lssl -lcrypto -lz -lbz2 -ldl -lpthread -O3 -s
g++ -o base/subconverter CMakeFiles/subconverter.dir/src/*.o -static -lpcre2-8 -levent -lyaml-cpp -lcurl -lnghttp2 -lssl -lcrypto -lz -lbz2 -ldl -lpthread -O3 -s
cd base
chmod +rx subconverter

View File

@ -3,16 +3,16 @@ mkdir obj
set -xe
c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/logger.cpp -o obj/logger.o
c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/main.cpp -o obj/main.o
c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/misc.cpp -o obj/misc.o
c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/multithread.cpp -o obj/multithread.o
c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/nodemanip.cpp -o obj/nodemanip.o
c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/rapidjson_extra.cpp -o obj/rapidjson_extra.o
c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/speedtestutil.cpp -o obj/speedtestutil.o
c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/subexport.cpp -o obj/subexport.o
c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/webget.cpp -o obj/webget.o
c++ -std=c++17 -D_MACOS -Wall -fexceptions -c src/webserver_libevent.cpp -o obj/webserver_libevent.o
c++ -o subconverter obj/logger.o obj/main.o obj/misc.o obj/multithread.o obj/nodemanip.o obj/rapidjson_extra.o obj/speedtestutil.o obj/subexport.o obj/webget.o obj/webserver_libevent.o -lpcrecpp -lpcre -levent -lpthread -lyaml-cpp -lcurl -lssl -lcrypto -lz -O3 -s
c++ -std=c++17 -Wall -fexceptions -c src/logger.cpp -o obj/logger.o
c++ -std=c++17 -Wall -fexceptions -c src/main.cpp -o obj/main.o
c++ -std=c++17 -Wall -fexceptions -c src/misc.cpp -o obj/misc.o
c++ -std=c++17 -Wall -fexceptions -c src/multithread.cpp -o obj/multithread.o
c++ -std=c++17 -Wall -fexceptions -c src/nodemanip.cpp -o obj/nodemanip.o
c++ -std=c++17 -Wall -fexceptions -c src/rapidjson_extra.cpp -o obj/rapidjson_extra.o
c++ -std=c++17 -Wall -fexceptions -c src/speedtestutil.cpp -o obj/speedtestutil.o
c++ -std=c++17 -Wall -fexceptions -c src/subexport.cpp -o obj/subexport.o
c++ -std=c++17 -Wall -fexceptions -c src/webget.cpp -o obj/webget.o
c++ -std=c++17 -Wall -fexceptions -c src/webserver_libevent.cpp -o obj/webserver_libevent.o
c++ -o subconverter obj/*.o -lpcre2-8 -levent -lpthread -lyaml-cpp -lcurl -lssl -lcrypto -lz -O3 -s
chmod +x subconverter

View File

@ -2,7 +2,7 @@
mkdir obj
set -xe
brew reinstall yaml-cpp rapidjson libevent zlib pcre bzip2 pkgconfig
brew reinstall yaml-cpp rapidjson libevent zlib pcre2 bzip2 pkgconfig
git clone https://github.com/curl/curl
cd curl
@ -11,20 +11,22 @@ cd curl
make -j8 > /dev/null
cd ..
curl -LO https://raw.githubusercontent.com/jpcre2/jpcre2/release/src/jpcre2.hpp
cp jpcre2.hpp /usr/local/include/
cp curl/lib/.libs/libcurl.a .
cp /usr/local/lib/libevent.a .
cp /usr/local/opt/zlib/lib/libz.a .
cp /usr/local/opt/openssl@1.1/lib/libssl.a .
cp /usr/local/opt/openssl@1.1/lib/libcrypto.a .
cp /usr/local/lib/libyaml-cpp.a .
cp /usr/local/lib/libpcre.a .
cp /usr/local/lib/libpcrecpp.a .
cp /usr/local/lib/libpcre2-8.a .
cp /usr/local/opt/bzip2/lib/libbz2.a .
export CMAKE_CXX_FLAGS="-I/usr/local/include -I/usr/local/opt/openssl@1.1/include -I/usr/local/opt/curl/include"
cmake -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@1.1 -DMACOS=on .
cmake -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl@1.1 .
make -j8
c++ -Xlinker -unexported_symbol -Xlinker "*" -o base/subconverter CMakeFiles/subconverter.dir/src/*.o libpcrecpp.a libpcre.a libevent.a libcurl.a libz.a libssl.a libcrypto.a libyaml-cpp.a libbz2.a -ldl -lpthread -O3
c++ -Xlinker -unexported_symbol -Xlinker "*" -o base/subconverter CMakeFiles/subconverter.dir/src/*.o libpcre2-8.a libevent.a libcurl.a libz.a libssl.a libcrypto.a libyaml-cpp.a libbz2.a -ldl -lpthread -O3
cd base
chmod +rx subconverter

View File

@ -3,7 +3,7 @@ set -xe
apt update
apt install -y git cmake clang pkg-config
apt install -y libevent libcurl openssl pcre
apt install -y libevent libcurl openssl pcre2
git clone https://github.com/jbeder/yaml-cpp
cd yaml-cpp
@ -15,3 +15,6 @@ git clone https://github.com/tencent/rapidjson
cd rapidjson
cp -r include/* /data/data/com.termux/files/usr/include/
cd ..
curl -LO https://raw.githubusercontent.com/jpcre2/jpcre2/release/src/jpcre2.hpp
cp jpcre2.hpp /data/data/com.termux/files/usr/include/

View File

@ -11,7 +11,8 @@
#ifdef USE_STD_REGEX
#include <regex>
#else
#include <pcrecpp.h>
#include <jpcre2.hpp>
typedef jpcre2::select<char> jp;
#endif // USE_STD_REGEX
#include <rapidjson/document.h>
@ -489,7 +490,7 @@ bool regValid(std::string &reg)
{
try
{
std::regex r(reg);
std::regex r(reg, std::regex::ECMAScript);
return true;
}
catch (std::regex_error &e)
@ -502,7 +503,13 @@ bool regFind(std::string src, std::string target)
{
try
{
std::regex reg(target);
std::regex::flag_type flags = std::regex::extended | std::regex::ECMAScript;
if(target.find("(?i)") == 0)
{
target.erase(0, 4);
flags |= std::regex::icase;
}
std::regex reg(target, flags);
return regex_search(src, reg);
}
catch (std::regex_error &e)
@ -516,7 +523,13 @@ std::string regReplace(std::string src, std::string match, std::string rep)
std::string result = "";
try
{
std::regex reg(match);
std::regex::flag_type flags = std::regex::extended | std::regex::ECMAScript;
if(match.find("(?i)") == 0)
{
match.erase(0, 4);
flags |= std::regex::icase;
}
std::regex reg(match, flags);
regex_replace(back_inserter(result), src.begin(), src.end(), reg, rep);
}
catch (std::regex_error &e)
@ -530,7 +543,13 @@ bool regMatch(std::string src, std::string match)
{
try
{
std::regex reg(match);
std::regex::flag_type flags = std::regex::extended | std::regex::ECMAScript;
if(match.find("(?i)") == 0)
{
match.erase(0, 4);
flags |= std::regex::icase;
}
std::regex reg(match, flags);
return regex_match(src, reg);
}
catch (std::regex_error &e)
@ -541,62 +560,37 @@ bool regMatch(std::string src, std::string match)
#else
bool regValid(std::string &reg)
{
try
{
pcrecpp::RE r(reg);
return true;
}
catch (std::exception &e)
bool regMatch(std::string src, std::string target, bool partial)
{
jp::Regex reg;
reg.setPattern(target).addModifier("gm").compile();
if(!reg)
return false;
}
return reg.replace(src, "$0") == src;
}
bool regFind(std::string src, std::string target)
{
try
{
pcrecpp::RE reg(target);
return reg.PartialMatch(src);
}
catch (std::exception &e)
{
jp::Regex reg;
reg.setPattern(target).addModifier("gm").compile();
if(!reg)
return false;
}
return reg.match(src);
}
std::string regReplace(std::string src, std::string match, std::string rep)
std::string regReplace(std::string src, std::string target, std::string rep)
{
std::string result = src;
try
{
if(rep.find("$") != rep.npos)
rep = replace_all_distinct(rep, "$", "\\");
pcrecpp::RE reg(match);
if(reg.GlobalReplace(rep, &result))
return result;
else
jp::Regex reg;
reg.setPattern(target).addModifier("gm").compile();
if(!reg)
return src;
}
catch (std::exception &e)
{
return src;
}
return reg.replace(src, rep);
}
bool regMatch(std::string src, std::string match)
bool regValid(std::string &target)
{
try
{
pcrecpp::RE reg(match);
return reg.FullMatch(src);
}
catch (std::exception &e)
{
return false;
}
jp::Regex reg(target);
return !!reg;
}
#endif // USE_STD_REGEX

View File

@ -40,10 +40,10 @@ bool is_str_utf8(std::string data);
std::string getFormData(const std::string &raw_data);
void sleep(int interval);
bool regValid(std::string &reg);
bool regValid(std::string &target);
bool regFind(std::string src, std::string target);
std::string regReplace(std::string src, std::string match, std::string rep);
bool regMatch(std::string src, std::string match);
bool regMatch(std::string src, std::string match, bool partial = false);
std::string speedCalc(double speed);
std::string getMD5(std::string data);
bool isIPv4(std::string &address);

View File

@ -1593,22 +1593,22 @@ time_t dateStringToTimestamp(std::string date)
bool getSubInfoFromHeader(std::string &header, std::string &result)
{
bool ret = false;
pcrecpp::RE reg("^(?i:Subscription-UserInfo): (.*?)\\r$", pcrecpp::MULTILINE());
try
std::string pattern = "(?:[\\s\\S]*?)^(?i:Subscription-UserInfo): (.*?)\\s$(?:[\\s\\S]*)", retStr;
if(regFind(header, pattern))
{
ret = reg.PartialMatch(header, &result);
}
catch (std::exception &e)
retStr = regReplace(header, pattern, "$1");
if(retStr != header)
{
//ignore
result = retStr;
return true;
}
return ret;
}
return false;
}
bool getSubInfoFromNodes(std::vector<nodeInfo> &nodes, string_array &stream_rules, string_array &time_rules, std::string &result)
{
std::string remarks, pattern, target, stream_info, time_info;
std::string remarks, pattern, target, stream_info, time_info, retStr;
string_array vArray;
for(nodeInfo &x : nodes)
@ -1623,13 +1623,14 @@ bool getSubInfoFromNodes(std::vector<nodeInfo> &nodes, string_array &stream_rule
continue;
pattern = vArray[0];
target = vArray[1];
pcrecpp::RE reg(pattern, pcrecpp::UTF8());
if(reg.FullMatch(remarks))
if(regMatch(remarks, pattern))
{
if(target.find("$") != target.npos)
target = replace_all_distinct(target, "$", "\\");
if(reg.GlobalReplace(target, &remarks))
stream_info = remarks;
retStr = regReplace(remarks, pattern, target);
if(retStr != remarks)
{
stream_info = retStr;
break;
}
}
else
continue;
@ -1646,13 +1647,14 @@ bool getSubInfoFromNodes(std::vector<nodeInfo> &nodes, string_array &stream_rule
continue;
pattern = vArray[0];
target = vArray[1];
pcrecpp::RE reg(pattern, pcrecpp::UTF8());
if(reg.FullMatch(remarks))
if(regMatch(remarks, pattern))
{
if(target.find("$") != target.npos)
target = replace_all_distinct(target, "$", "\\");
if(reg.GlobalReplace(target, &remarks))
time_info = remarks;
retStr = regReplace(remarks, pattern, target);
if(retStr != remarks)
{
time_info = retStr;
break;
}
}
else
continue;

View File

@ -1625,14 +1625,7 @@ void netchToQuanX(std::vector<nodeInfo> &nodes, INIReader &ini, std::vector<rule
type = "static";
rules_upper_bound = vArray.size();
}
else if(vArray[1] == "url-test")
{
if(vArray.size() < 5)
continue;
type = "static";
rules_upper_bound = vArray.size() - 2;
}
else if(vArray[1] == "fallback")
else if(vArray[1] == "url-test" || vArray[1] == "fallback")
{
if(vArray.size() < 5)
continue;

View File

@ -23,6 +23,7 @@ static int writer(char *data, size_t size, size_t nmemb, std::string *writerData
std::string curlGet(std::string url, std::string proxy, std::string &response_headers)
{
CURL *curl_handle;
CURLcode res;
std::string data;
curl_global_init(CURL_GLOBAL_ALL);
@ -43,9 +44,12 @@ std::string curlGet(std::string url, std::string proxy, std::string &response_he
if(proxy != "")
curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.data());
curl_easy_perform(curl_handle);
res = curl_easy_perform(curl_handle);
curl_easy_cleanup(curl_handle);
if(res != CURLE_OK)
data.clear();
return data;
}