From 91e7c9817fc6ff2487838d2f0f45d886bce411ec Mon Sep 17 00:00:00 2001 From: Elysia <118418714+elysias123@users.noreply.github.com> Date: Sat, 2 Aug 2025 20:35:17 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9Edomain=5Fsuffix=E5=92=8Cdomai?= =?UTF-8?q?n=5Fkeyword=E5=8C=B9=E9=85=8D&=E5=88=A0=E9=99=A4singbox-regex.p?= =?UTF-8?q?y?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yaml | 6 ++-- README.md | 11 ++++-- config.py | 67 ++++++++++++++++++++++++++---------- main.py | 49 ++------------------------ requirements.txt | 1 - rule/domain_keyword.txt | 5 +++ rule/domain_suffix.txt | 13 +++++++ script/Adblock.py | 17 ++++++++- script/Adguard.py | 18 +++++++++- script/Clash-Classical.py | 25 +++++++++++++- script/Clash.py | 16 +++++---- script/Dnsmasq.py | 2 +- script/QuantumultX.py | 18 ++++++++-- script/Singbox-regex.py | 33 ------------------ script/Singbox.py | 42 ++++++++++++++++++++-- script/Surge-RULE-SET.py | 16 ++++++++- script/Surge-module.py | 16 ++++++++- script/Surge.py | 2 +- 18 files changed, 232 insertions(+), 125 deletions(-) delete mode 100644 requirements.txt create mode 100644 rule/domain_keyword.txt create mode 100644 rule/domain_suffix.txt delete mode 100755 script/Singbox-regex.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 821b8c6..beda3e2 100755 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -24,7 +24,6 @@ jobs: git clone "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}" -b build build cd build mkdir out - pip install -r requirements.txt python main.py cd out mkdir Filters @@ -34,17 +33,16 @@ jobs: - name: build singbox mrs run: | cd build/out/Filters - wget -O ./sing-box.tar.gz https://github.com/SagerNet/sing-box/releases/download/v1.11.4/sing-box-1.11.4-linux-amd64.tar.gz + wget -O ./sing-box.tar.gz https://github.com/SagerNet/sing-box/releases/download/v1.11.15/sing-box-1.11.15-linux-amd64.tar.gz tar -xzvf sing-box.tar.gz chmod +x */sing-box ./*/sing-box rule-set compile AWAvenue-Ads-Rule-Singbox.json - ./*/sing-box rule-set compile AWAvenue-Ads-Rule-Singbox-regex.json rm -rf *sing-box* - name: build mihomo mrs run: | cd build/out/Filters - wget -O ./mihomo.gz https://github.com/MetaCubeX/mihomo/releases/download/v1.19.3/mihomo-linux-amd64-v1.19.3.gz + wget -O ./mihomo.gz https://github.com/MetaCubeX/mihomo/releases/download/v1.19.12/mihomo-linux-amd64-v1.19.12.gz gzip -d mihomo.gz chmod +x mihomo ./mihomo convert-ruleset domain yaml AWAvenue-Ads-Rule-Clash.yaml AWAvenue-Ads-Rule-Clash.mrs diff --git a/README.md b/README.md index 06c6100..3edc470 100755 --- a/README.md +++ b/README.md @@ -20,9 +20,16 @@ python main.py ## 须知 -`domain.txt` 文件为域名列表 一行一个 不会匹配子域名 +> 一行一个 不会匹配子域名 -`domain_regex.txt` 文件为`domain.txt`的正则表达式版本(并不是所有插件都会调用此规则) + +`domain.txt` 文件为域名列表 + +`domain_regex.txt` 匹配域名正则表达式(并不是所有插件都会调用此规则) + +`domain_suffix.txt` 匹配域名后缀(并不是所有插件都会调用此规则) + +`domain_keyword.txt` 匹配域名的关键字(并不是所有插件都会调用此规则) `ip.txt` 文件为ipv4列表(并不是所有插件都会调用此规则) diff --git a/config.py b/config.py index 56c1519..1e4819d 100644 --- a/config.py +++ b/config.py @@ -1,32 +1,61 @@ import os -import dns.resolver - +import re SCRIPT_PATH = os.path.join(os.getcwd(), "script") # 插件文件夹 RULE_PATH = os.path.join(os.getcwd(), "rule") # 规则文件夹 OUT_PATH = os.getcwd() + "/out" # 输出文件夹 domain_file=RULE_PATH + "/domain.txt" # 规则文件 regex_file=RULE_PATH + "/domain_regex.txt" # 正则规则文件 +suffix_file=RULE_PATH + "/domain_suffix.txt" # 后缀规则文件 +keyword_file=RULE_PATH + "/domain_keyword.txt" # 关键词规则文件 ip_file=RULE_PATH + "/ip.txt" # IP规则文件 ip6_file=RULE_PATH + "/ip6.txt" # IPv6规则文件 +class RuleList: + def __init__(self): + self.domain_list = self.domain_file(domain_file) + self.regex_list = self.regex_file(regex_file) + self.suffix_list = self.suffix_file(suffix_file) + self.keyword_list = self.keyword_file(keyword_file) + self.ip_list = self.ip_file(ip_file) + self.ip6_list = self.ip6_file(ip6_file) + ## 以下内容由deepseek提供技术支持(bushi -def check_domain(domain): - resolver = dns.resolver.Resolver(configure=False) - resolver.nameservers = [ '223.5.5.5', '119.29.29.29', '8.8.8.8', '1.1.1.1', '2400:3200:baba::1'] - resolver.timeout = 3 - resolver.lifetime = 5 + def domain_file(self, filename): + with open(filename, 'r') as file: + return sorted({line.strip() for line in file}) - result = {"A": False, "AAAA": False} - try: - A = resolver.resolve(domain, "A", raise_on_no_answer=False) - result["A"] = bool(A and A.rrset) - except Exception as e: - print(f"{domain} 解析 A 记录时出错 从domain_list中去除 {e}") - try: - AAAA = resolver.resolve(domain, "AAAA", raise_on_no_answer=False) - result["AAAA"] = bool(AAAA and AAAA.rrset) - except Exception as e: - print(f"{domain} 解析 AAAA 记录时出错 从domainv6_list中去除 {e}") - return result \ No newline at end of file + def suffix_file(self, filename): + with open(filename, 'r') as file: + return sorted({line.strip() for line in file}) + + def keyword_file(self, filename): + with open(filename, 'r') as file: + return sorted({line.strip() for line in file}) + + def regex_file(self, filename): + with open(filename, 'r') as file: + return sorted({line.strip() for line in file}) + + def ip_file(self, filename): + ipv4_pattern = re.compile(r'^(\d{1,3}\.){3}\d{1,3}$') + ips = set() + with open(filename, 'r') as file: + for line in file: + ip = line.strip() + if ipv4_pattern.match(ip): + parts = ip.split('.') + if all(0 <= int(part) <= 255 for part in parts): + ips.add(ip) + return sorted(ips) + + def ip6_file(self, filename): + ipv6_pattern = re.compile(r'^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$') + ips = set() + with open(filename, 'r') as file: + for line in file: + ip = line.strip() + if ipv6_pattern.match(ip): + ips.add(ip) + return sorted(ips) \ No newline at end of file diff --git a/main.py b/main.py index 00b3555..e6512ab 100755 --- a/main.py +++ b/main.py @@ -1,12 +1,12 @@ -import re import os import config import importlib import subprocess -import concurrent.futures from datetime import datetime +rule = config.RuleList() + def get_latest_git_tag(): # 获取最新的git tag process = subprocess.Popen('git tag', stdout=subprocess.PIPE, shell=True) @@ -17,49 +17,6 @@ def get_latest_git_tag(): # 获取最新的git tag else: return None - -class RuleList: - def __init__(self, domain_file, regex_file, ip_file, ip6_file): - self.domain_list = self.domain_file(domain_file) - self.regex_list = self.regex_file(regex_file) - self.ip_list = self.ip_file(ip_file) - self.ip6_list = self.ip6_file(ip6_file) - - - ## 以下内容由deepseek提供技术支持(bushi - - def domain_file(self, filename): - with open(filename, 'r') as file: - return sorted({line.strip() for line in file}) - - def regex_file(self, filename): - with open(filename, 'r') as file: - return sorted({line.strip() for line in file}) - - def ip_file(self, filename): - ipv4_pattern = re.compile(r'^(\d{1,3}\.){3}\d{1,3}$') - ips = set() - with open(filename, 'r') as file: - for line in file: - ip = line.strip() - if ipv4_pattern.match(ip): - parts = ip.split('.') - if all(0 <= int(part) <= 255 for part in parts): - ips.add(ip) - return sorted(ips) - - def ip6_file(self, filename): - ipv6_pattern = re.compile(r'^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$') - ips = set() - with open(filename, 'r') as file: - for line in file: - ip = line.strip() - if ipv6_pattern.match(ip): - ips.add(ip) - return sorted(ips) - - - def WriteFile(name, text, suffix, comment, module_total): # 写入文件 try: with open(config.OUT_PATH + "/AWAvenue-Ads-Rule-" + name + suffix, 'w', encoding="utf-8") as file: @@ -88,8 +45,6 @@ def WriteFile(name, text, suffix, comment, module_total): # 写入文件 def RunScript(): - rule = RuleList(config.domain_file, config.regex_file, config.ip_file, config.ip6_file) - config.rule = rule for filename in os.listdir(config.SCRIPT_PATH): # 遍历script目录下的所有文件 if filename.endswith(".py"): diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c0ba5c3..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -dnspython \ No newline at end of file diff --git a/rule/domain_keyword.txt b/rule/domain_keyword.txt new file mode 100644 index 0000000..bfa3e9d --- /dev/null +++ b/rule/domain_keyword.txt @@ -0,0 +1,5 @@ +-ad.sm.cn +-ad.video.yximgs.com +-ad.wtzw.com +-be-pack-sign.pglstatp-toutiao.com +-normal.zijieapi.com \ No newline at end of file diff --git a/rule/domain_suffix.txt b/rule/domain_suffix.txt new file mode 100644 index 0000000..366d76d --- /dev/null +++ b/rule/domain_suffix.txt @@ -0,0 +1,13 @@ +1rtb.com +a.market.xiaomi.com +ad.gameley.com +ad.gameley.com +adintl.cn +adx.adwangmai.com +adx.adwangmai.com +data.hicloud.com +log.aliyuncs.com +ubixioe.com +vlion.cn +yfanads.com +zhangyuyidong.cn \ No newline at end of file diff --git a/script/Adblock.py b/script/Adblock.py index ed4c2ab..d0524dc 100755 --- a/script/Adblock.py +++ b/script/Adblock.py @@ -1,3 +1,5 @@ +# https://help.adblockplus.org/hc/en-us/articles/360062733293-How-to-write-filters + def format_domain(List): domain = [] for line in List: @@ -5,6 +7,19 @@ def format_domain(List): domain.append(domain_lines) return domain +def format_suffix(List): + suffix = [] + for line in List: + suffix_lines = f"||{line.strip()}^" + suffix.append(suffix_lines) + return suffix + +def format_keyword(List): + keyword = [] + for line in List: + keyword_lines = f"*{line.strip()}*" + keyword.append(keyword_lines) + return keyword def format_regex(List): regex = [] @@ -14,5 +29,5 @@ def format_regex(List): return regex def build(rule): - list = format_domain(rule.domain_list) + format_regex(rule.regex_list) + list = (format_domain(rule.domain_list) + format_suffix(rule.suffix_list) + format_keyword(rule.keyword_list) + format_regex(rule.regex_list)) return {'list': list, 'suffix': '.txt', 'comment': '!', 'total': len(list)} \ No newline at end of file diff --git a/script/Adguard.py b/script/Adguard.py index ed4c2ab..667003a 100755 --- a/script/Adguard.py +++ b/script/Adguard.py @@ -1,3 +1,6 @@ +# https://adguard.com/kb/zh-CN/general/ad-filtering/create-own-filters +# # https://help.adblockplus.org/hc/en-us/articles/360062733293-How-to-write-filters + def format_domain(List): domain = [] for line in List: @@ -5,6 +8,19 @@ def format_domain(List): domain.append(domain_lines) return domain +def format_suffix(List): + suffix = [] + for line in List: + suffix_lines = f"||{line.strip()}^" + suffix.append(suffix_lines) + return suffix + +def format_keyword(List): + keyword = [] + for line in List: + keyword_lines = f"*{line.strip()}*" + keyword.append(keyword_lines) + return keyword def format_regex(List): regex = [] @@ -14,5 +30,5 @@ def format_regex(List): return regex def build(rule): - list = format_domain(rule.domain_list) + format_regex(rule.regex_list) + list = (format_domain(rule.domain_list) + format_suffix(rule.suffix_list) + format_keyword(rule.keyword_list) + format_regex(rule.regex_list)) return {'list': list, 'suffix': '.txt', 'comment': '!', 'total': len(list)} \ No newline at end of file diff --git a/script/Clash-Classical.py b/script/Clash-Classical.py index 740598b..f1ed0dd 100755 --- a/script/Clash-Classical.py +++ b/script/Clash-Classical.py @@ -1,3 +1,5 @@ +# https://wiki.metacubex.one/config/rules/#_1 +# https://wiki.metacubex.one/config/rule-providers/content/#classical def format_domain(List): domain = [] @@ -6,6 +8,20 @@ def format_domain(List): domain.append(domain_lines) return domain +def format_suffix(List): + suffix = [] + for line in List: + suffix_lines = f" - DOMAIN-SUFFIX,{line.strip()}" + suffix.append(suffix_lines) + return suffix + +def format_keyword(List): + keyword = [] + for line in List: + keyword_lines = f" - DOMAIN-KEYWORD,{line.strip()}" + keyword.append(keyword_lines) + return keyword + def format_regex(List): regex = [] for line in List: @@ -20,6 +36,13 @@ def format_ip(List): ip.append(ip_lines) return ip +def format_ip6(List): + ip6 = [] + for line in List: + ip6_lines = f" - IP-CIDR,{line.strip()},no-resolve" + ip6.append(ip6_lines) + return ip6 + def build(rule): - list = format_ip(rule.ip_list) + format_domain(rule.domain_list) + format_regex(rule.regex_list) + list = format_ip(rule.ip_list) + format_ip6(rule.ip6_list) + format_domain(rule.domain_list) + format_suffix(rule.suffix_list) + format_keyword(rule.keyword_list) + format_regex(rule.regex_list) return {'list': ["payload:"] + list, 'suffix': '.yaml', 'comment': '#', 'total': len(list)} diff --git a/script/Clash.py b/script/Clash.py index cd2d054..887fd22 100755 --- a/script/Clash.py +++ b/script/Clash.py @@ -1,3 +1,5 @@ +# https://wiki.metacubex.one/handbook/syntax/#_8 +# https://wiki.metacubex.one/config/rule-providers/content/#domain def format_domain(List): domain = [] @@ -6,14 +8,14 @@ def format_domain(List): domain.append(domain_lines) return domain -def format_regex(List): - regex = [] +def format_suffix(List): + suffix = [] for line in List: - regex_lines = f" - '{line.strip()}'".replace("$","").replace("^","") - regex.append(regex_lines) - return regex + suffix_lines = f" - '+.{line.strip()}'" + suffix.append(suffix_lines) + return suffix def build(rule): - list =format_domain(rule.domain_list) + format_regex(rule.regex_list) - return {'list': ["payload:"] + list, 'suffix': '.yaml', 'comment': '#', 'total': len(list)} + list = format_domain(rule.domain_list) + format_suffix(rule.suffix_list) + return {'list': ["payload:"] + list, 'suffix': '.yaml', 'comment': '#', 'total': len(list)} \ No newline at end of file diff --git a/script/Dnsmasq.py b/script/Dnsmasq.py index 61447e5..5c66c33 100644 --- a/script/Dnsmasq.py +++ b/script/Dnsmasq.py @@ -1,4 +1,4 @@ - +# https://wiki.archlinuxcn.org/wiki/Dnsmasq#%E5%9F%9F%E5%90%8D%E9%98%BB%E6%AD%A2%E5%88%97%E8%A1%A8 def format_domain(List): domain = [] for line in List: diff --git a/script/QuantumultX.py b/script/QuantumultX.py index bd7cbb1..f724a35 100755 --- a/script/QuantumultX.py +++ b/script/QuantumultX.py @@ -6,10 +6,24 @@ def format_domain(List): domain.append(domain_lines) return domain +def format_suffix(List): + suffix = [] + for line in List: + suffix_lines = f"DOMAIN-SUFFIX,{line.strip()},reject" + suffix.append(suffix_lines) + return suffix + +def format_keyword(List): + keyword = [] + for line in List: + keyword_lines = f"DOMAIN-KEYWORD,{line.strip()},reject" + keyword.append(keyword_lines) + return keyword + def format_regex(List): regex = [] for line in List: - regex_lines = f"DOMAIN-KEYWORD,{line.strip()},reject".replace("$","").replace("^","").replace("*","").replace("\\.",".") + regex_lines = f"DOMAIN-REGEX,{line.strip()},reject".replace("$","").replace("^","").replace("*","").replace("\\.",".") regex.append(regex_lines) return regex @@ -21,5 +35,5 @@ def format_ip(List): return ip def build(rule): - list = format_ip(rule.ip_list) + format_domain(rule.domain_list) + format_regex(rule.regex_list) + list = format_ip(rule.ip_list) + format_domain(rule.domain_list) + format_suffix(rule.suffix_list) + format_keyword(rule.keyword_list) + format_regex(rule.regex_list) return {'list': list, 'suffix': '.list', 'comment': '#', 'total': len(list)} \ No newline at end of file diff --git a/script/Singbox-regex.py b/script/Singbox-regex.py deleted file mode 100755 index c8920ed..0000000 --- a/script/Singbox-regex.py +++ /dev/null @@ -1,33 +0,0 @@ -import json - -def format_domain(List): - domain = [] - for line in List: - domain_lines = f"{line.strip()}" - domain.append(domain_lines) - return domain - - -def format_regex(List): - regex = [] - for line in List: - regex_lines = f"{line.strip()}" - regex.append(regex_lines) - return regex - -def build(rule): - domain_list = format_domain(rule.domain_list) - regex_list = format_regex(rule.regex_list) - List = { - "version": 1, - "rules": [ - { - "domain": domain_list, - "domain_regex": regex_list, - "domain_suffix": [] - } - ] - } - - json_data = [json.dumps(List, indent=2)] - return {'list': json_data, 'suffix': '.json', 'comment': '', 'total': len(json_data)} \ No newline at end of file diff --git a/script/Singbox.py b/script/Singbox.py index 18cbe09..1e8cda6 100755 --- a/script/Singbox.py +++ b/script/Singbox.py @@ -1,5 +1,6 @@ import json +# https://sing-box.sagernet.org/zh/configuration/rule-set/headless-rule/#_1 def format_domain(List): domain = [] for line in List: @@ -7,15 +8,50 @@ def format_domain(List): domain.append(domain_lines) return domain +def format_keyword(List): + keyword = [] + for line in List: + keyword_lines = f"{line.strip()}" + keyword.append(keyword_lines) + return keyword + +def format_suffix(List): + suffix = [] + for line in List: + suffix_lines = f"{line.strip()}" + suffix.append(suffix_lines) + return suffix + +def format_regex(List): + regex = [] + for line in List: + regex_lines = f"{line.strip()}" + regex.append(regex_lines) + return regex + +def format_ip(List): + ip = [] + for line in List: + ip_lines = f"{line.strip()}" + ip.append(ip_lines) + return ip + def build(rule): domain_list = format_domain(rule.domain_list) + keyword_list = format_keyword(rule.keyword_list) + suffix_list = format_suffix(rule.suffix_list) + ip_list = format_ip(rule.ip_list + rule.ip6_list) + regex_list = format_regex(rule.regex_list) + rule = { - "version": 1, + "version": 3, "rules": [ { + "ip_cidr": ip_list, "domain": domain_list, - "domain_regex": [], - "domain_suffix": [] + "domain_keyword": keyword_list, + "domain_suffix": suffix_list, + "domain_regex": regex_list } ] } diff --git a/script/Surge-RULE-SET.py b/script/Surge-RULE-SET.py index a22c53b..651df10 100755 --- a/script/Surge-RULE-SET.py +++ b/script/Surge-RULE-SET.py @@ -5,6 +5,20 @@ def format_domain(List): domain.append(domain_lines) return domain +def format_suffix(List): + suffix = [] + for line in List: + suffix_lines = f"DOMAIN-SUFFIX,{line.strip()}" + suffix.append(suffix_lines) + return suffix + +def format_keyword(List): + keyword = [] + for line in List: + keyword_lines = f"DOMAIN-KEYWORD,{line.strip()}" + keyword.append(keyword_lines) + return keyword + def format_regex(List): regex = [] for line in List: @@ -20,5 +34,5 @@ def format_ip(List): return ip def build(rule): - list = format_ip(rule.ip_list) + format_domain(rule.domain_list) + format_regex(rule.regex_list) + list = format_ip(rule.ip_list) + format_domain(rule.domain_list) + format_suffix(rule.suffix_list) + format_keyword(rule.keyword_list) + format_regex(rule.regex_list) return {'list': list, 'suffix': '.list', 'comment': '#', 'total': len(list)} \ No newline at end of file diff --git a/script/Surge-module.py b/script/Surge-module.py index 50b163f..adcd72d 100755 --- a/script/Surge-module.py +++ b/script/Surge-module.py @@ -19,10 +19,24 @@ def format_domain(List): domain.append(domain_lines) return domain +def format_suffix(List): + suffix = [] + for line in List: + suffix_lines = f"DOMAIN-SUFFIX,{line.strip()},REJECT,extended-matching,pre-matching" + suffix.append(suffix_lines) + return suffix + +def format_keyword(List): + keyword = [] + for line in List: + keyword_lines = f"DOMAIN-KEYWORD,{line.strip()},REJECT,extended-matching,pre-matching" + keyword.append(keyword_lines) + return keyword + def format_regex(List): regex = [] for line in List: - regex_lines = f"DOMAIN-KEYWORD,{line.strip()},REJECT,extended-matching,pre-matching" + regex_lines = f"DOMAIN-REGEX,{line.strip()},REJECT,extended-matching,pre-matching" regex.append(regex_lines) return regex diff --git a/script/Surge.py b/script/Surge.py index b057ca8..8407bb5 100755 --- a/script/Surge.py +++ b/script/Surge.py @@ -6,5 +6,5 @@ def format_domain(List): return domain def build(rule): - list = format_domain(rule.domain_list) + list = format_domain(rule.domain_list) + format_domain(rule.suffix_list) return {'list': list, 'suffix': '.list', 'comment': '#', 'total': len(list)} \ No newline at end of file