feat: import adblock filter domains
This commit is contained in:
ngfchl
+8
-3
@@ -29,7 +29,7 @@ const spamTesting = ref(false);
|
||||
const spamTestResult = ref<Record<string, any> | null>(null);
|
||||
const ruleForm = reactive({ type: 'keyword', pattern: '', score: 75 });
|
||||
const ruleSaving = ref(false);
|
||||
const spamRules = reactive({ keywords: [] as Record<string, any>[], regex: [] as Record<string, any>[], counts: { keywords: 0, regex: 0 } });
|
||||
const spamRules = reactive({ keywords: [] as Record<string, any>[], regex: [] as Record<string, any>[], adblock_domains: [] as Record<string, any>[], counts: { keywords: 0, regex: 0, adblock_domains: 0 } });
|
||||
const rulesLoading = ref(false);
|
||||
const ruleKeyword = ref('');
|
||||
const shopForm = reactive({ item_key: '', name: '', cost: 0, stock: -1, desc: '', enabled: true });
|
||||
@@ -425,6 +425,11 @@ const filteredRegexRules = computed(() => {
|
||||
const rows = [...spamRules.regex].reverse();
|
||||
return key ? rows.filter(r => String(r.pattern || '').toLowerCase().includes(key)).slice(0, 300) : rows.slice(0, 120);
|
||||
});
|
||||
const filteredAdblockRules = computed(() => {
|
||||
const key = ruleKeyword.value.trim().toLowerCase();
|
||||
const rows = [...spamRules.adblock_domains];
|
||||
return key ? rows.filter(r => String(r.pattern || '').toLowerCase().includes(key)).slice(0, 300) : rows.slice(0, 120);
|
||||
});
|
||||
|
||||
async function saveSpamRule() {
|
||||
if (!ruleForm.pattern.trim()) return showToast('请输入规则内容');
|
||||
@@ -865,7 +870,7 @@ onBeforeUnmount(() => {
|
||||
<div class="rule-result-head"><STag :color="spamTestResult.final_is_spam ? 'destructive' : spamTestResult.final_is_spam === null ? 'warning' : 'success'" variant="soft">{{ spamTestResult.final_is_spam ? '广告/垃圾/灰产' : spamTestResult.final_is_spam === null ? '需 AI 判断' : '正常' }}</STag><strong>评分：{{ spamTestResult.score }}</strong></div>
|
||||
<div class="rule-reason"><b>本地命中</b><span>{{ spamTestResult.reason }}</span></div>
|
||||
<div v-if="spamTestResult.ai_reason" class="rule-reason"><b>AI 结果</b><span>{{ spamTestResult.ai_reason }}</span></div>
|
||||
<div class="rule-counts">规则库：关键词 {{ spamTestResult.rule_counts?.keywords }} 条 / 正则 {{ spamTestResult.rule_counts?.regex }} 条</div>
|
||||
<div class="rule-counts">规则库：关键词 {{ spamTestResult.rule_counts?.keywords }} 条 / 正则 {{ spamTestResult.rule_counts?.regex }} 条 / Adblock 域名 {{ spamTestResult.rule_counts?.adblock_domains || spamRules.counts.adblock_domains || 0 }} 条</div>
|
||||
</template>
|
||||
</div>
|
||||
</div>
|
||||
@@ -877,7 +882,7 @@ onBeforeUnmount(() => {
|
||||
</div>
|
||||
</SCard>
|
||||
|
||||
<SCard class="panel page-panel rule-library-card" title="📚 规则库" :description="`当前拦截规则：关键词 ${spamRules.counts.keywords} 条 / 正则 ${spamRules.counts.regex} 条`">
|
||||
<SCard class="panel page-panel rule-library-card" title="📚 规则库" :description="`关键词 ${spamRules.counts.keywords} 条 / 正则 ${spamRules.counts.regex} 条 / Adblock 域名 ${spamRules.counts.adblock_domains || 0} 条`">
|
||||
<template #extra><SButtonLoading size="xs" variant="outline" :loading="rulesLoading" :disabled="rulesLoading" @click="loadSpamRules">刷新</SButtonLoading></template>
|
||||
<div class="rule-library-toolbar"><SInput v-model="ruleKeyword" placeholder="搜索规则条目" /></div>
|
||||
<div class="rule-library-grid">
|
||||
|
||||
@@ -488,3 +488,21 @@ body { background: #f3f6fb; }
|
||||
@media (max-width: 1180px) {
|
||||
.rule-test-grid, .inline-rule-writer.rule-write-grid, .rule-library-grid { grid-template-columns: 1fr; }
|
||||
}
|
||||
|
||||
|
||||
/* Adblock rule library polish */
|
||||
.rule-library-grid.three { grid-template-columns: repeat(3, minmax(0, 1fr)); }
|
||||
.rule-library-card .s-card__description { word-break: break-word; }
|
||||
.rule-result-card { justify-content: flex-start; }
|
||||
.rule-result-head { padding-bottom: 8px; border-bottom: 1px solid #e2e8f0; }
|
||||
.rule-counts { padding: 8px 10px; border-radius: 12px; background: #eef2ff; color: #475569; }
|
||||
@media (max-width: 1280px) { .rule-library-grid.three { grid-template-columns: 1fr; } }
|
||||
|
||||
|
||||
/* Keep rule management compact even with huge imported lists */
|
||||
.rule-library-card { max-height: 58vh; }
|
||||
.rule-library-card .s-card__content { min-height: 0; overflow: hidden; }
|
||||
.rule-library-grid, .rule-library-grid.three { min-height: 0; overflow: hidden; }
|
||||
.rule-library-grid > div { min-height: 0; display: grid; grid-template-rows: auto minmax(0, 1fr); }
|
||||
.rule-list { height: min(28vh, 300px); max-height: min(28vh, 300px); overflow-y: auto; }
|
||||
.rule-manage-page { padding-bottom: 24px; }
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
from pathlib import Path
|
||||
import re
|
||||
from datetime import datetime
|
||||
|
||||
SRC = Path('/tmp/adblock_import')
|
||||
OUT = Path('rules/adblock_domains.txt')
|
||||
files = [
|
||||
SRC / 'easylistchina.txt',
|
||||
SRC / 'gitee_adblock/adblock.txt',
|
||||
SRC / 'gitee_adblock/adblock_lite.txt',
|
||||
SRC / 'gitee_adblock/adblock_plus.txt',
|
||||
SRC / 'gitee_adblock/adblock_privacy.txt',
|
||||
SRC / 'github_adblock/ADBLOCK_RULE_COLLECTION_DOMAIN.txt',
|
||||
SRC / 'github_adblock/ADBLOCK_RULE_COLLECTION_DOMAIN_Lite.txt',
|
||||
]
|
||||
# 宽松域名提取，后面再过滤明显非域名/静态资源片段
|
||||
DOMAIN_RE = re.compile(r'(?i)(?:\|\|)?\b([a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+)\b')
|
||||
BAD_SUFFIX = {'.js', '.css', '.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico', '.mp4', '.m3u8', '.json', '.xml', '.php', '.html'}
|
||||
COMMON_ALLOW = {
|
||||
'github.com','gitee.com','google.com','microsoft.com','apple.com','qq.com','weixin.qq.com','telegram.org',
|
||||
't.me','youtube.com','bilibili.com','baidu.com','cloudflare.com','jquery.com','vuejs.org','npmjs.com'
|
||||
}
|
||||
|
||||
def valid_domain(d: str) -> bool:
|
||||
d = d.lower().strip('.-_^*/')
|
||||
if not d or d in COMMON_ALLOW:
|
||||
return False
|
||||
if any(d.endswith(s) for s in BAD_SUFFIX):
|
||||
return False
|
||||
parts = d.split('.')
|
||||
if len(parts) < 2 or any(not p or len(p) > 63 for p in parts):
|
||||
return False
|
||||
tld = parts[-1]
|
||||
if len(tld) < 2 or not re.fullmatch(r'[a-z][a-z0-9-]{1,23}', tld):
|
||||
return False
|
||||
# 排除纯数字/IP
|
||||
if all(p.isdigit() for p in parts):
|
||||
return False
|
||||
return True
|
||||
|
||||
domains = set()
|
||||
source_counts = {}
|
||||
for path in files:
|
||||
if not path.exists():
|
||||
continue
|
||||
count_before = len(domains)
|
||||
with path.open('r', encoding='utf-8', errors='ignore') as f:
|
||||
for raw in f:
|
||||
line = raw.strip()
|
||||
if not line or line.startswith(('!', '#', '[')) or line.startswith('@@'):
|
||||
continue
|
||||
# 跳过元素隐藏/CSS 规则
|
||||
if '##' in line or '#?#' in line or '#@#' in line:
|
||||
continue
|
||||
# 去掉 options，减少误提取
|
||||
line = line.split('$', 1)[0]
|
||||
for m in DOMAIN_RE.finditer(line):
|
||||
d = m.group(1).lower().strip('.-_^*/')
|
||||
if valid_domain(d):
|
||||
domains.add(d)
|
||||
source_counts[str(path.relative_to(SRC))] = len(domains) - count_before
|
||||
|
||||
OUT.parent.mkdir(parents=True, exist_ok=True)
|
||||
existing_header = ''
|
||||
if OUT.exists():
|
||||
backup = OUT.with_suffix(f'.txt.bak-{datetime.now().strftime("%Y%m%d%H%M%S")}')
|
||||
OUT.replace(backup)
|
||||
lines = [
|
||||
'# Adblock 域名过滤库（自动导入）',
|
||||
'# 来源：gitee.com/uniartisan2018/adblock_list、github.com/REIJI007/Adblock-Rule-Collection、EasyList China',
|
||||
'# 用途：仅当聊天消息中出现 URL/域名时匹配，不作为普通关键词扫描，避免误杀与性能问题。',
|
||||
f'# 更新时间：{datetime.now().isoformat(timespec="seconds")}',
|
||||
f'# 有效域名：{len(domains)}',
|
||||
]
|
||||
for k, v in source_counts.items():
|
||||
lines.append(f'# source {k}: +{v}')
|
||||
lines.append('')
|
||||
lines.extend(sorted(domains))
|
||||
OUT.write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||||
print({'domains': len(domains), 'sources': source_counts, 'output': str(OUT)})
|
||||
+437451
File diff suppressed because it is too large
Load Diff
@@ -79,18 +79,32 @@ def load_regex_rules(path: Path, default_score: int = 70) -> list[Rule]:
|
||||
return rules
|
||||
|
||||
|
||||
def load_adblock_domains(path: Path) -> set[str]:
|
||||
"""加载 Adblock 域名过滤库。仅用于消息中 URL/域名匹配，不参与普通关键词扫描。"""
|
||||
if not path.exists():
|
||||
return set()
|
||||
domains: set[str] = set()
|
||||
for line in path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
raw = line.strip().lower()
|
||||
if not raw or raw.startswith("#"):
|
||||
continue
|
||||
domains.add(raw.lstrip("."))
|
||||
return domains
|
||||
|
||||
EXTERNAL_KEYWORD_RULES = load_keyword_rules(RULES_DIR / "spam_keywords.txt")
|
||||
EXTERNAL_REGEX_RULES = load_regex_rules(RULES_DIR / "spam_regex.txt")
|
||||
logger.info(f"🧩 已加载外部垃圾规则：关键词 {len(EXTERNAL_KEYWORD_RULES)} 条，正则 {len(EXTERNAL_REGEX_RULES)} 条")
|
||||
ADBLOCK_DOMAINS = load_adblock_domains(RULES_DIR / "adblock_domains.txt")
|
||||
logger.info(f"🧩 已加载外部垃圾规则：关键词 {len(EXTERNAL_KEYWORD_RULES)} 条，正则 {len(EXTERNAL_REGEX_RULES)} 条，Adblock 域名 {len(ADBLOCK_DOMAINS)} 条")
|
||||
|
||||
|
||||
def reload_external_rules() -> dict:
|
||||
"""重新加载外部关键词/正则规则，用于 Web 写规则后热更新。"""
|
||||
global EXTERNAL_KEYWORD_RULES, EXTERNAL_REGEX_RULES
|
||||
"""重新加载外部关键词/正则/Adblock 域名规则，用于 Web 写规则或导入后热更新。"""
|
||||
global EXTERNAL_KEYWORD_RULES, EXTERNAL_REGEX_RULES, ADBLOCK_DOMAINS
|
||||
EXTERNAL_KEYWORD_RULES = load_keyword_rules(RULES_DIR / "spam_keywords.txt")
|
||||
EXTERNAL_REGEX_RULES = load_regex_rules(RULES_DIR / "spam_regex.txt")
|
||||
logger.info(f"🧩 已热加载外部垃圾规则：关键词 {len(EXTERNAL_KEYWORD_RULES)} 条，正则 {len(EXTERNAL_REGEX_RULES)} 条")
|
||||
return {"keywords": len(EXTERNAL_KEYWORD_RULES), "regex": len(EXTERNAL_REGEX_RULES)}
|
||||
ADBLOCK_DOMAINS = load_adblock_domains(RULES_DIR / "adblock_domains.txt")
|
||||
logger.info(f"🧩 已热加载外部垃圾规则：关键词 {len(EXTERNAL_KEYWORD_RULES)} 条，正则 {len(EXTERNAL_REGEX_RULES)} 条，Adblock 域名 {len(ADBLOCK_DOMAINS)} 条")
|
||||
return {"keywords": len(EXTERNAL_KEYWORD_RULES), "regex": len(EXTERNAL_REGEX_RULES), "adblock_domains": len(ADBLOCK_DOMAINS)}
|
||||
|
||||
|
||||
# 高危黑产：单项命中即可高度可疑或直接垃圾
|
||||
@@ -103,6 +117,32 @@ HARD_RULES = [
|
||||
]
|
||||
|
||||
# 广告意图/黑灰产常见话术：评分组合
|
||||
DOMAIN_TEXT_RE = re.compile(r"(?i)(?:https?://|www\.)?([a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+)")
|
||||
COMMON_TEXT_DOMAINS = {"ptools.fun", "github.com", "gitee.com", "telegram.org", "t.me", "qq.com", "weixin.qq.com"}
|
||||
|
||||
def extract_domains_from_text(text: str) -> set[str]:
|
||||
domains: set[str] = set()
|
||||
for m in DOMAIN_TEXT_RE.finditer(text or ""):
|
||||
d = m.group(1).lower().strip(".-_")
|
||||
if d and d not in COMMON_TEXT_DOMAINS:
|
||||
domains.add(d)
|
||||
return domains
|
||||
|
||||
def match_adblock_domains(text: str) -> list[Rule]:
|
||||
"""只在文本出现 URL/域名时匹配广告域名，降低普通聊天误杀。"""
|
||||
if not ADBLOCK_DOMAINS or not re.search(r"(?i)(https?://|www\.|[a-z0-9-]+\.[a-z]{2,})", text or ""):
|
||||
return []
|
||||
hits: list[Rule] = []
|
||||
for domain in extract_domains_from_text(text):
|
||||
parts = domain.split(".")
|
||||
candidates = [".".join(parts[i:]) for i in range(max(0, len(parts) - 4), len(parts) - 1)]
|
||||
candidates.insert(0, domain)
|
||||
for c in candidates:
|
||||
if c in ADBLOCK_DOMAINS:
|
||||
hits.append(Rule(f"Adblock域名:{c}", re.escape(c), 80))
|
||||
break
|
||||
return hits
|
||||
|
||||
SCORE_RULES = [
|
||||
# 收益承诺
|
||||
Rule("收益承诺-日赚", r"(?:日赚|日入|每天赚|一天赚|轻松赚|稳赚|躺赚|副业收入).{0,10}(?:\d{2,}|几百|上千|过千|一千|几千|上万|万元?|k|K|w|W)", 75),
|
||||
@@ -234,6 +274,7 @@ def local_classify(text: str) -> tuple[bool | None, str, int]:
|
||||
external_hits.extend(hit_rules(target, EXTERNAL_KEYWORD_RULES))
|
||||
external_hits.extend(hit_rules(target, EXTERNAL_REGEX_RULES))
|
||||
normal_hits.extend(hit_rules(target, NORMAL_RULES))
|
||||
external_hits.extend(match_adblock_domains(norm))
|
||||
|
||||
# 去重
|
||||
dedup = lambda xs: list({x.name: x for x in xs}.values())
|
||||
|
||||
+13
-2
@@ -633,7 +633,7 @@ async def api_list_spam_rules(limit: int = Query(default=500, ge=1, le=2000)):
|
||||
if not path.exists():
|
||||
return items
|
||||
for i, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
line = raw.strip()
|
||||
line = raw.strip().lstrip("\ufeff")
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "|" in line:
|
||||
@@ -648,7 +648,18 @@ async def api_list_spam_rules(limit: int = Query(default=500, ge=1, le=2000)):
|
||||
return items
|
||||
keywords = read_rules(spam_detector.RULES_DIR / "spam_keywords.txt", "keyword")
|
||||
regex = read_rules(spam_detector.RULES_DIR / "spam_regex.txt", "regex")
|
||||
return JSONResponse({"ok": True, "keywords": keywords[-limit:], "regex": regex[-limit:], "counts": {"keywords": len(keywords), "regex": len(regex)}})
|
||||
adblock_path = spam_detector.RULES_DIR / "adblock_domains.txt"
|
||||
adblock_preview = []
|
||||
adblock_count = 0
|
||||
if adblock_path.exists():
|
||||
for line in adblock_path.read_text(encoding="utf-8", errors="ignore").splitlines():
|
||||
raw = line.strip()
|
||||
if not raw or raw.startswith("#"):
|
||||
continue
|
||||
adblock_count += 1
|
||||
if len(adblock_preview) < min(limit, 200):
|
||||
adblock_preview.append({"type": "adblock_domain", "line": adblock_count, "pattern": raw, "score": 80})
|
||||
return JSONResponse({"ok": True, "keywords": keywords[-limit:], "regex": regex[-limit:], "adblock_domains": adblock_preview, "counts": {"keywords": len(keywords), "regex": len(regex), "adblock_domains": adblock_count}})
|
||||
|
||||
|
||||
@app.post("/api/spam-test")
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -5,8 +5,8 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<link rel="icon" type="image/svg+xml" href="/static/icon.svg" />
|
||||
<title>TG Spam Guard</title>
|
||||
<script type="module" crossorigin src="/static/assets/index-BF4yz--N.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/static/assets/index-DRJmg6LH.css">
|
||||
<script type="module" crossorigin src="/static/assets/index-DmKpLuqC.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/static/assets/index-BxBv2zp1.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
|
||||
Reference in New Issue
Block a user