Breaking changes

Fix failed to download subscriptions, configs and rulesets due to bad implementation of download size checker.
Remove base pre-generation function.
Refactor code base.
This commit is contained in:
Tindy X 2020-08-14 01:52:53 +08:00
parent 0c8913f87d
commit 97c89150a5
No known key found for this signature in database
GPG Key ID: C6AD413169968D58
12 changed files with 544 additions and 672 deletions

View File

@ -135,7 +135,6 @@ advanced:
max_allowed_rulesets: 0
max_allowed_rules: 0
max_allowed_download_size: 0
enable_base_gen: false
enable_cache: false
cache_subscription: 60
cache_config: 300

View File

@ -259,7 +259,6 @@ max_concurrent_threads=2
max_allowed_rulesets=0
max_allowed_rules=0
max_allowed_download_size=0
enable_base_gen=false
enable_cache=false
cache_subscription=60
cache_config=300

File diff suppressed because it is too large Load Diff

View File

@ -10,7 +10,6 @@
void refreshRulesets(string_array &ruleset_list, std::vector<ruleset_content> &rca);
void readConf();
void generateBase();
int simpleGenerator();
std::string convertRuleset(const std::string &content, int type);

View File

@ -7,8 +7,8 @@
#include "logger.h"
extern bool print_debug_info;
int global_log_level = LOG_LEVEL_INFO;
extern bool gPrintDbgInfo;
int gLogLevel = LOG_LEVEL_INFO;
std::string getTime(int type)
{
@ -38,7 +38,7 @@ std::string getTime(int type)
void writeLog(int type, const std::string &content, int level)
{
if(level > global_log_level)
if(level > gLogLevel)
return;
const char *levels[] = {"[FATL]", "[ERRO]", "[WARN]", "[INFO]", "[DEBG]", "[VERB]"};
std::cerr<<getTime(2)<<" ["<<getpid()<<" "<<std::this_thread::get_id()<<"]"<<levels[level % 6];

View File

@ -10,11 +10,11 @@
#include "webget.h"
#include "logger.h"
extern std::string pref_path, access_token, listen_address, gen_profile, managed_config_prefix;
extern bool api_mode, generator_mode, cfw_child_process, update_ruleset_on_request;
extern int listen_port, max_concurrent_threads, max_pending_connections;
extern string_array rulesets;
extern std::vector<ruleset_content> ruleset_content_array;
extern std::string gPrefPath, gAccessToken, gListenAddress, gGenerateProfiles, gManagedConfigPrefix;
extern bool gAPIMode, gGeneratorMode, gCFWChildProcess, gUpdateRulesetOnRequest;
extern int gListenPort, gMaxConcurThreads, gMaxPendingConns;
extern string_array gCustomRulesets;
extern std::vector<ruleset_content> gRulesetContent;
#ifndef _WIN32
void SetConsoleTitle(const std::string &title)
@ -54,22 +54,22 @@ void chkArg(int argc, char *argv[])
{
if(strcmp(argv[i], "-cfw") == 0)
{
cfw_child_process = true;
update_ruleset_on_request = true;
gCFWChildProcess = true;
gUpdateRulesetOnRequest = true;
}
else if(strcmp(argv[i], "-f") == 0 || strcmp(argv[i], "--file") == 0)
{
if(i < argc - 1)
pref_path.assign(argv[++i]);
gPrefPath.assign(argv[++i]);
}
else if(strcmp(argv[i], "-g") == 0 || strcmp(argv[i], "--gen") == 0)
{
generator_mode = true;
gGeneratorMode = true;
}
else if(strcmp(argv[i], "--artifact") == 0)
{
if(i < argc - 1)
gen_profile.assign(argv[++i]);
gGenerateProfiles.assign(argv[++i]);
}
else if(strcmp(argv[i], "-l") == 0 || strcmp(argv[i], "--log") == 0)
{
@ -104,9 +104,9 @@ int main(int argc, char *argv[])
setcd(prgpath); //first switch to program directory
#endif // _DEBUG
if(fileExist("pref.yml"))
pref_path = "pref.yml";
gPrefPath = "pref.yml";
chkArg(argc, argv);
setcd(pref_path); //then switch to pref directory
setcd(gPrefPath); //then switch to pref directory
writeLog(0, "SubConverter " VERSION " starting up..", LOG_LEVEL_INFO);
#ifdef _WIN32
WSADATA wsaData;
@ -130,18 +130,17 @@ int main(int argc, char *argv[])
SetConsoleTitle("SubConverter " VERSION);
readConf();
if(!update_ruleset_on_request)
refreshRulesets(rulesets, ruleset_content_array);
generateBase();
if(!gUpdateRulesetOnRequest)
refreshRulesets(gCustomRulesets, gRulesetContent);
std::string env_api_mode = GetEnv("API_MODE"), env_managed_prefix = GetEnv("MANAGED_PREFIX"), env_token = GetEnv("API_TOKEN");
api_mode = tribool().parse(toLower(env_api_mode)).get(api_mode);
gAPIMode = tribool().parse(toLower(env_api_mode)).get(gAPIMode);
if(env_managed_prefix.size())
managed_config_prefix = env_managed_prefix;
gManagedConfigPrefix = env_managed_prefix;
if(env_token.size())
access_token = env_token;
gAccessToken = env_token;
if(generator_mode)
if(gGeneratorMode)
return simpleGenerator();
append_response("GET", "/", "text/plain", [](RESPONSE_CALLBACK_ARGS) -> std::string
@ -156,42 +155,40 @@ int main(int argc, char *argv[])
append_response("GET", "/refreshrules", "text/plain", [](RESPONSE_CALLBACK_ARGS) -> std::string
{
if(access_token.size())
if(gAccessToken.size())
{
std::string token = getUrlArg(request.argument, "token");
if(token != access_token)
if(token != gAccessToken)
{
response.status_code = 403;
return "Forbidden\n";
}
}
refreshRulesets(rulesets, ruleset_content_array);
generateBase();
refreshRulesets(gCustomRulesets, gRulesetContent);
return "done\n";
});
append_response("GET", "/readconf", "text/plain", [](RESPONSE_CALLBACK_ARGS) -> std::string
{
if(access_token.size())
if(gAccessToken.size())
{
std::string token = getUrlArg(request.argument, "token");
if(token != access_token)
if(token != gAccessToken)
{
response.status_code = 403;
return "Forbidden\n";
}
}
readConf();
generateBase();
return "done\n";
});
append_response("POST", "/updateconf", "text/plain", [](RESPONSE_CALLBACK_ARGS) -> std::string
{
if(access_token.size())
if(gAccessToken.size())
{
std::string token = getUrlArg(request.argument, "token");
if(token != access_token)
if(token != gAccessToken)
{
response.status_code = 403;
return "Forbidden\n";
@ -199,9 +196,9 @@ int main(int argc, char *argv[])
}
std::string type = getUrlArg(request.argument, "type");
if(type == "form")
fileWrite(pref_path, getFormData(request.postdata), true);
fileWrite(gPrefPath, getFormData(request.postdata), true);
else if(type == "direct")
fileWrite(pref_path, request.postdata, true);
fileWrite(gPrefPath, request.postdata, true);
else
{
response.status_code = 501;
@ -209,9 +206,8 @@ int main(int argc, char *argv[])
}
readConf();
if(!update_ruleset_on_request)
refreshRulesets(rulesets, ruleset_content_array);
generateBase();
if(!gUpdateRulesetOnRequest)
refreshRulesets(gCustomRulesets, gRulesetContent);
return "done\n";
});
@ -233,7 +229,7 @@ int main(int argc, char *argv[])
append_response("GET", "/convert", "text/plain;charset=utf-8", getConvertedRuleset);
if(!api_mode)
if(!gAPIMode)
{
append_response("GET", "/get", "text/plain;charset=utf-8", [](RESPONSE_CALLBACK_ARGS) -> std::string
{
@ -249,10 +245,10 @@ int main(int argc, char *argv[])
std::string env_port = GetEnv("PORT");
if(env_port.size())
listen_port = to_int(env_port, listen_port);
listener_args args = {listen_address, listen_port, max_pending_connections, max_concurrent_threads};
gListenPort = to_int(env_port, gListenPort);
listener_args args = {gListenAddress, gListenPort, gMaxPendingConns, gMaxConcurThreads};
//std::cout<<"Serving HTTP @ http://"<<listen_address<<":"<<listen_port<<std::endl;
writeLog(0, "Startup completed. Serving HTTP @ http://" + listen_address + ":" + std::to_string(listen_port), LOG_LEVEL_INFO);
writeLog(0, "Startup completed. Serving HTTP @ http://" + gListenAddress + ":" + std::to_string(gListenPort), LOG_LEVEL_INFO);
start_web_server_multi(&args);
#ifdef _WIN32

View File

@ -4,71 +4,57 @@
#include "multithread.h"
//safety lock for multi-thread
std::mutex on_emoji, on_rename, on_stream, on_time, clash_base_mutex, mellow_base_mutex;
std::mutex on_emoji, on_rename, on_stream, on_time;
extern string_array emojis, renames;
extern string_array stream_rules, time_rules;
extern YAML::Node clash_base;
extern INIReader mellow_base;
extern string_array gEmojis, gRenames;
extern string_array gStreamNodeRules, gTimeNodeRules;
string_array safe_get_emojis()
{
guarded_mutex guard(on_emoji);
return emojis;
return gEmojis;
}
string_array safe_get_renames()
{
guarded_mutex guard(on_rename);
return renames;
return gRenames;
}
string_array safe_get_streams()
{
guarded_mutex guard(on_stream);
return stream_rules;
return gStreamNodeRules;
}
string_array safe_get_times()
{
guarded_mutex guard(on_time);
return time_rules;
}
YAML::Node safe_get_clash_base()
{
guarded_mutex guard(clash_base_mutex);
return YAML::Clone(clash_base);
}
INIReader safe_get_mellow_base()
{
guarded_mutex guard(mellow_base_mutex);
return mellow_base;
return gTimeNodeRules;
}
void safe_set_emojis(string_array &data)
{
guarded_mutex guard(on_emoji);
emojis.swap(data);
gEmojis.swap(data);
}
void safe_set_renames(string_array &data)
{
guarded_mutex guard(on_rename);
renames.swap(data);
gRenames.swap(data);
}
void safe_set_streams(string_array &data)
{
guarded_mutex guard(on_stream);
stream_rules.swap(data);
gStreamNodeRules.swap(data);
}
void safe_set_times(string_array &data)
{
guarded_mutex guard(on_time);
time_rules.swap(data);
gTimeNodeRules.swap(data);
}
std::shared_future<std::string> fetchFileAsync(const std::string &path, const std::string &proxy, int cache_ttl, bool async)

View File

@ -12,7 +12,7 @@
std::string override_conf_port;
bool ss_libev, ssr_libev;
extern int cache_subscription;
extern int gCacheSubscription;
void copyNodes(std::vector<nodeInfo> &source, std::vector<nodeInfo> &dest)
{
@ -90,7 +90,7 @@ int addNodes(std::string link, std::vector<nodeInfo> &allNodes, int groupID, con
writeLog(LOG_TYPE_INFO, "Downloading subscription data...");
if(startsWith(link, "surge:///install-config")) //surge config link
link = UrlDecode(getUrlArg(link, "url"));
strSub = webGet(link, proxy, cache_subscription, &extra_headers, &request_headers);
strSub = webGet(link, proxy, gCacheSubscription, &extra_headers, &request_headers);
/*
if(strSub.size() == 0)
{

View File

@ -9,8 +9,8 @@
#include "socket.h"
#include "webget.h"
extern int cache_config;
extern std::string proxy_config;
extern int gCacheConfig;
extern std::string gProxyConfig;
std::string parseProxy(const std::string &source);
@ -109,7 +109,7 @@ static duk_ret_t fetch(duk_context *ctx)
{
std::string filepath, proxy;
duktape_get_arguments_str(ctx, 1, 2, &filepath, &proxy);
std::string content = fetchFile(filepath, proxy, cache_config);
std::string content = fetchFile(filepath, proxy, gCacheConfig);
duk_push_lstring(ctx, content.c_str(), content.size());
return 1;
}
@ -138,7 +138,7 @@ static duk_ret_t getGeoIP(duk_context *ctx)
if(address.empty())
duk_push_undefined(ctx);
else
duk_push_string(ctx, fetchFile("https://api.ip.sb/geoip/" + address, parseProxy(proxy), cache_config).c_str());
duk_push_string(ctx, fetchFile("https://api.ip.sb/geoip/" + address, parseProxy(proxy), gCacheConfig).c_str());
return 1;
}

View File

@ -24,9 +24,9 @@
#include "yamlcpp_extra.h"
#include "interfaces.h"
extern bool api_mode, surge_ssr_resolve;
extern bool gAPIMode, gSurgeResolveHostname;
extern string_array ss_ciphers, ssr_ciphers;
extern size_t max_allowed_rules;
extern size_t gMaxAllowedRules;
const string_array clashr_protocols = {"origin", "auth_sha1_v4", "auth_aes128_md5", "auth_aes128_sha1", "auth_chain_a", "auth_chain_b"};
const string_array clashr_obfs = {"plain", "http_simple", "http_post", "random_head", "tls1.2_ticket_auth", "tls1.2_ticket_fastauth"};
@ -34,11 +34,11 @@ const string_array clash_ssr_ciphers = {"rc4-md5", "aes-128-ctr", "aes-192-ctr",
/// rule type lists
#define basic_types "DOMAIN", "DOMAIN-SUFFIX", "DOMAIN-KEYWORD", "IP-CIDR", "SRC-IP-CIDR", "GEOIP", "MATCH", "FINAL"
string_array clash_rule_type = {basic_types, "IP-CIDR6", "SRC-PORT", "DST-PORT"};
string_array surge2_rule_type = {basic_types, "IP-CIDR6", "USER-AGENT", "URL-REGEX", "PROCESS-NAME", "IN-PORT", "DEST-PORT", "SRC-IP"};
string_array surge_rule_type = {basic_types, "IP-CIDR6", "USER-AGENT", "URL-REGEX", "AND", "OR", "NOT", "PROCESS-NAME", "IN-PORT", "DEST-PORT", "SRC-IP"};
string_array quanx_rule_type = {basic_types, "USER-AGENT", "HOST", "HOST-SUFFIX", "HOST-KEYWORD"};
string_array surfb_rule_type = {basic_types, "IP-CIDR6", "PROCESS-NAME", "IN-PORT", "DEST-PORT", "SRC-IP"};
string_array ClashRuleTypes = {basic_types, "IP-CIDR6", "SRC-PORT", "DST-PORT"};
string_array Surge2RuleTypes = {basic_types, "IP-CIDR6", "USER-AGENT", "URL-REGEX", "PROCESS-NAME", "IN-PORT", "DEST-PORT", "SRC-IP"};
string_array SurgeRuleTypes = {basic_types, "IP-CIDR6", "USER-AGENT", "URL-REGEX", "AND", "OR", "NOT", "PROCESS-NAME", "IN-PORT", "DEST-PORT", "SRC-IP"};
string_array QuanXRuleTypes = {basic_types, "USER-AGENT", "HOST", "HOST-SUFFIX", "HOST-KEYWORD"};
string_array SurfRuleTypes = {basic_types, "IP-CIDR6", "PROCESS-NAME", "IN-PORT", "DEST-PORT", "SRC-IP"};
std::string hostnameToIPAddr(const std::string &host)
{
@ -649,7 +649,7 @@ void rulesetToClash(YAML::Node &base_rule, std::vector<ruleset_content> &ruleset
for(ruleset_content &x : ruleset_content_array)
{
if(max_allowed_rules && total_rules > max_allowed_rules)
if(gMaxAllowedRules && total_rules > gMaxAllowedRules)
break;
rule_group = x.rule_group;
retrieved_rules = x.rule_content.get();
@ -678,7 +678,7 @@ void rulesetToClash(YAML::Node &base_rule, std::vector<ruleset_content> &ruleset
std::string::size_type lineSize;
while(getline(strStrm, strLine, delimiter))
{
if(max_allowed_rules && total_rules > max_allowed_rules)
if(gMaxAllowedRules && total_rules > gMaxAllowedRules)
break;
lineSize = strLine.size();
/*
@ -696,7 +696,7 @@ void rulesetToClash(YAML::Node &base_rule, std::vector<ruleset_content> &ruleset
if(strLine.find("USER-AGENT") == 0 || strLine.find("URL-REGEX") == 0 || strLine.find("PROCESS-NAME") == 0 || strLine.find("AND") == 0 || strLine.find("OR") == 0) //remove unsupported types
continue;
*/
if(!std::any_of(clash_rule_type.begin(), clash_rule_type.end(), [strLine](std::string type){return startsWith(strLine, type);}))
if(!std::any_of(ClashRuleTypes.begin(), ClashRuleTypes.end(), [strLine](std::string type){return startsWith(strLine, type);}))
continue;
/*
if(strLine.find("IP-CIDR") == 0)
@ -737,7 +737,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector<ruleset_content
for(ruleset_content &x : ruleset_content_array)
{
if(max_allowed_rules && total_rules > max_allowed_rules)
if(gMaxAllowedRules && total_rules > gMaxAllowedRules)
break;
rule_group = x.rule_group;
retrieved_rules = x.rule_content.get();
@ -766,7 +766,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector<ruleset_content
std::string::size_type lineSize;
while(getline(strStrm, strLine, delimiter))
{
if(max_allowed_rules && total_rules > max_allowed_rules)
if(gMaxAllowedRules && total_rules > gMaxAllowedRules)
break;
lineSize = strLine.size();
if(lineSize && strLine[lineSize - 1] == '\r')
@ -776,7 +776,7 @@ std::string rulesetToClashStr(YAML::Node &base_rule, std::vector<ruleset_content
}
if(!lineSize || strLine[0] == ';' || strLine[0] == '#' || (lineSize >= 2 && strLine[0] == '/' && strLine[1] == '/')) //empty lines and comments are ignored
continue;
if(std::none_of(clash_rule_type.begin(), clash_rule_type.end(), [strLine](std::string type){return startsWith(strLine, type);}))
if(std::none_of(ClashRuleTypes.begin(), ClashRuleTypes.end(), [strLine](std::string type){return startsWith(strLine, type);}))
continue;
strLine += "," + rule_group;
if(count_least(strLine, ',', 3))
@ -828,7 +828,7 @@ void rulesetToSurge(INIReader &base_rule, std::vector<ruleset_content> &ruleset_
for(ruleset_content &x : ruleset_content_array)
{
if(max_allowed_rules && total_rules > max_allowed_rules)
if(gMaxAllowedRules && total_rules > gMaxAllowedRules)
break;
rule_group = x.rule_group;
rule_path = x.rule_path;
@ -939,7 +939,7 @@ void rulesetToSurge(INIReader &base_rule, std::vector<ruleset_content> &ruleset_
std::string::size_type lineSize;
while(getline(strStrm, strLine, delimiter))
{
if(max_allowed_rules && total_rules > max_allowed_rules)
if(gMaxAllowedRules && total_rules > gMaxAllowedRules)
break;
lineSize = strLine.size();
if(lineSize && strLine[lineSize - 1] == '\r') //remove line break
@ -955,22 +955,22 @@ void rulesetToSurge(INIReader &base_rule, std::vector<ruleset_content> &ruleset_
continue;
[[fallthrough]];
case -1:
if(!std::any_of(quanx_rule_type.begin(), quanx_rule_type.end(), [strLine](std::string type){return startsWith(strLine, type);}))
if(!std::any_of(QuanXRuleTypes.begin(), QuanXRuleTypes.end(), [strLine](std::string type){return startsWith(strLine, type);}))
continue;
break;
case -3:
if(!std::any_of(surfb_rule_type.begin(), surfb_rule_type.end(), [strLine](std::string type){return startsWith(strLine, type);}))
if(!std::any_of(SurfRuleTypes.begin(), SurfRuleTypes.end(), [strLine](std::string type){return startsWith(strLine, type);}))
continue;
break;
default:
if(surge_ver > 2)
{
if(!std::any_of(surge_rule_type.begin(), surge_rule_type.end(), [strLine](std::string type){return startsWith(strLine, type);}))
if(!std::any_of(SurgeRuleTypes.begin(), SurgeRuleTypes.end(), [strLine](std::string type){return startsWith(strLine, type);}))
continue;
}
else
{
if(!std::any_of(surge2_rule_type.begin(), surge2_rule_type.end(), [strLine](std::string type){return startsWith(strLine, type);}))
if(!std::any_of(Surge2RuleTypes.begin(), Surge2RuleTypes.end(), [strLine](std::string type){return startsWith(strLine, type);}))
continue;
}
}
@ -1645,7 +1645,7 @@ std::string netchToSurge(std::vector<nodeInfo> &nodes, const std::string &base_c
proxy += "\", local-port=" + std::to_string(local_port);
if(isIPv4(hostname) || isIPv6(hostname))
proxy += ", addresses=" + hostname;
else if(surge_ssr_resolve)
else if(gSurgeResolveHostname)
proxy += ", addresses=" + hostnameToIPAddr(hostname);
local_port++;
break;

View File

@ -11,7 +11,7 @@
#include "misc.h"
#include "webget.h"
extern std::string managed_config_prefix;
extern std::string gManagedConfigPrefix;
namespace inja
{
@ -133,7 +133,7 @@ int render_template(const std::string &content, const template_args &vars, std::
});
m_callbacks.add_callback("getLink", 1, [](inja::Arguments &args)
{
return managed_config_prefix + args.at(0)->get<std::string>();
return gManagedConfigPrefix + args.at(0)->get<std::string>();
});
m_callbacks.add_callback("startsWith", 2, [](inja::Arguments &args)
{

View File

@ -16,13 +16,13 @@
#endif // _stat
#endif // _WIN32
extern bool print_debug_info, serve_cache_on_fetch_fail;
extern int global_log_level;
extern bool gPrintDbgInfo, gServeCacheOnFetchFail;
extern int gLogLevel;
typedef std::lock_guard<std::mutex> guarded_mutex;
std::mutex cache_rw_lock;
long max_allowed_download_size = 1048576L;
long gMaxAllowedDownloadSize = 1048576L;
//std::string user_agent_str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36";
std::string user_agent_str = "subconverter/" VERSION " cURL/" LIBCURL_VERSION;
@ -60,7 +60,7 @@ static int dummy_writer(char *data, size_t size, size_t nmemb, void *writerData)
return size * nmemb;
}
static int size_checker(void *clientp, double dltotal, double dlnow, double ultotal, double ulnow)
static int size_checker(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
{
if(clientp)
{
@ -77,7 +77,7 @@ static int size_checker(void *clientp, double dltotal, double dlnow, double ulto
static inline void curl_set_common_options(CURL *curl_handle, const char *url, curl_progress_data *data)
{
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, global_log_level == LOG_LEVEL_VERBOSE ? 1L : 0L);
curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, gLogLevel == LOG_LEVEL_VERBOSE ? 1L : 0L);
curl_easy_setopt(curl_handle, CURLOPT_NOPROGRESS, 0L);
curl_easy_setopt(curl_handle, CURLOPT_NOSIGNAL, 1L);
curl_easy_setopt(curl_handle, CURLOPT_FOLLOWLOCATION, 1L);
@ -117,7 +117,8 @@ static int curlGet(const FetchArgument &argument, FetchResult &result)
else
curl_easy_setopt(curl_handle, CURLOPT_PROXY, argument.proxy.data());
}
curl_progress_data limit = {max_allowed_download_size};
curl_progress_data limit;
limit.size_limit = gMaxAllowedDownloadSize;
curl_set_common_options(curl_handle, new_url.data(), &limit);
if(argument.request_headers)
@ -234,7 +235,7 @@ std::string webGet(const std::string &url, const std::string &proxy, unsigned in
}
else
{
if(fileExist(path) && serve_cache_on_fetch_fail) // failed, check if cache exist
if(fileExist(path) && gServeCacheOnFetchFail) // failed, check if cache exist
{
writeLog(0, "Fetch failed. Serving cached content."); // cache exist, serving cache
guarded_mutex guard(cache_rw_lock);