feat: load external chinese spam rule lists

Dockerfile

@@ -21,6 +21,7 @@ COPY config.py .
 COPY db/ db/
 COPY handlers/ handlers/
 COPY services/ services/
+COPY rules/ rules/
 COPY web/ web/
 COPY knowledge/ knowledge/
 COPY main.py .

rules/spam_keywords.txt

@@ -0,0 +1,114 @@
+# 中文垃圾广告/黑灰产关键词词库（可追加网上词库，一行一个词）
+# 格式：词条 或 词条|分数
+
+# 赌博/博彩
+菠菜|100
+赌博|100
+博彩|100
+赌场|100
+棋牌|100
+彩票|100
+开奖|90
+中奖|80
+下注|100
+百家乐|100
+时时彩|100
+六合彩|100
+赌球|100
+盘口|90
+
+# 色情/擦边
+约炮|100
+外围|100
+楼凤|100
+裸聊|100
+成人视频|100
+一夜情|100
+上门服务|100
+
+# 诈骗/灰产
+刷单|100
+返佣|70
+传销|100
+资金盘|100
+杀猪盘|100
+跑分|100
+洗钱|100
+代收款|100
+接码|90
+猫池|90
+办证|100
+假证|100
+代开发票|100
+
+# 赚钱/兼职广告
+日赚|75
+日入|75
+每天赚|75
+一天赚|75
+轻松赚|75
+稳赚|75
+躺赚|70
+月入|70
+月赚|70
+副业|35
+兼职日结|60
+手机操作|45
+无需经验|45
+宝妈|35
+学生党|35
+小白可带|65
+手把手带|65
+带你赚|70
+带你做|65
+
+# 引流/联系方式
+加微信|50
+微信号|50
+vx|45
+v信|45
+薇信|45
+加我|40
+私聊|45
+私信|45
+联系我|45
+看主页|45
+看签名|45
+主页有|45
+点头像|45
+进群|45
+拉群|45
+频道|35
+
+# 币圈/投资/贷款
+USDT|50
+泰达币|50
+虚拟币|50
+币圈|45
+搬砖|50
+套利|50
+贷款|45
+套现|55
+黑户|60
+秒批|55
+包过|55
+下款|55
+投资带单|70
+股票老师|60
+期货老师|60
+外汇老师|60
+量化跟单|65
+
+# 营销诱导
+免费领|45
+免费送|45
+限时领|45
+点击领取|45
+扫码领取|45
+红包|30
+福利|30
+返现|40
+代理|35
+加盟|35
+招商|35
+招代理|45

rules/spam_regex.txt

@@ -0,0 +1,10 @@
+# 中文垃圾广告正则库（可追加网上规则，一行一个正则）
+# 格式：正则 或 正则|分数
+(?:日赚|日入|每天赚|一天赚|轻松赚|稳赚|躺赚).{0,10}(?:\d{2,}|几百|上千|过千|一千|几千|上万|万元?|k|K|w|W)|90
+(?:月入|月赚|月收益|月收入).{0,10}(?:\d{3,}|过万|上万|几万|万元?|w|W)|85
+(?:赚钱|收益|收入|副业|项目|路子).{0,20}(?:加我|私聊|看主页|看签名|进群|联系我|微信|vx|QQ)|85
+(?:\d{3,}|几百|上千|过千|几千|上万|万元?|k|K|w|W).{0,20}(?:加我|私聊|看主页|看签名|进群|联系我|微信|vx|QQ)|80
+(?:兼职|副业|日结|周结|在家做|手机操作|无需经验).{0,18}(?:赚钱|收入|结算|工资|项目|单)|75
+(?:USDT|U\s?S\s?D\s?T|泰达币|虚拟币|币圈|搬砖|套利).{0,18}(?:赚钱|收益|项目|加我|看主页|进群)|85
+(?:贷款|借款|套现|信用卡|花呗|白条|提额|下款|黑户|征信).{0,18}(?:秒批|当天|无视|包过|渠道)|85
+(?:t\.me|telegram\.me|tg://|飞机群).{0,40}(?:join|group|channel|\+|进群|频道)|95

services/spam_detector.py

@@ -11,6 +11,7 @@
 import logging
 import re
 from dataclasses import dataclass
+from pathlib import Path
 
 import httpx
 
@@ -26,6 +27,63 @@ class Rule:
     score: int
 
 
+ROOT_DIR = Path(__file__).resolve().parent.parent
+RULES_DIR = ROOT_DIR / "rules"
+
+
+def load_keyword_rules(path: Path, default_score: int = 45) -> list[Rule]:
+    """加载外部关键词词库：词条 或 词条|分数。"""
+    rules: list[Rule] = []
+    if not path.exists():
+        return rules
+    for line in path.read_text(encoding="utf-8").splitlines():
+        raw = line.strip()
+        if not raw or raw.startswith("#"):
+            continue
+        if "|" in raw:
+            word, score_text = raw.rsplit("|", 1)
+            try:
+                score = int(score_text.strip())
+            except ValueError:
+                word, score = raw, default_score
+        else:
+            word, score = raw, default_score
+        word = word.strip()
+        if word:
+            rules.append(Rule(f"外部关键词:{word}", re.escape(word), score))
+    return rules
+
+
+def load_regex_rules(path: Path, default_score: int = 70) -> list[Rule]:
+    """加载外部正则库：正则 或 正则|分数。"""
+    rules: list[Rule] = []
+    if not path.exists():
+        return rules
+    for i, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+        raw = line.strip()
+        if not raw or raw.startswith("#"):
+            continue
+        if "|" in raw:
+            pattern, score_text = raw.rsplit("|", 1)
+            try:
+                score = int(score_text.strip())
+            except ValueError:
+                pattern, score = raw, default_score
+        else:
+            pattern, score = raw, default_score
+        try:
+            re.compile(pattern)
+            rules.append(Rule(f"外部正则:{i}", pattern, score))
+        except re.error as e:
+            logger.warning(f"⚠️ 跳过无效外部正则 {path}:{i}: {e}")
+    return rules
+
+
+EXTERNAL_KEYWORD_RULES = load_keyword_rules(RULES_DIR / "spam_keywords.txt")
+EXTERNAL_REGEX_RULES = load_regex_rules(RULES_DIR / "spam_regex.txt")
+logger.info(f"🧩 已加载外部垃圾规则：关键词 {len(EXTERNAL_KEYWORD_RULES)} 条，正则 {len(EXTERNAL_REGEX_RULES)} 条")
+
+
 # 高危黑产：单项命中即可高度可疑或直接垃圾
 HARD_RULES = [
     Rule("博彩赌博", r"菠菜|赌博|博彩|赌场|棋牌|彩票|开奖|中奖|赔率|下注|百家乐|时时彩|六合彩", 100),
@@ -114,28 +172,31 @@ def local_classify(text: str) -> tuple[bool | None, str, int]:
     hard_hits = []
     score_hits = []
     combo_hits = []
+    external_hits = []
     normal_hits = []
 
     for target in targets:
         hard_hits.extend(hit_rules(target, HARD_RULES))
         score_hits.extend(hit_rules(target, SCORE_RULES))
         combo_hits.extend(hit_rules(target, COMBO_RULES))
+        external_hits.extend(hit_rules(target, EXTERNAL_KEYWORD_RULES))
+        external_hits.extend(hit_rules(target, EXTERNAL_REGEX_RULES))
         normal_hits.extend(hit_rules(target, NORMAL_RULES))
 
     # 去重
     dedup = lambda xs: list({x.name: x for x in xs}.values())
-    hard_hits, score_hits, combo_hits, normal_hits = map(dedup, [hard_hits, score_hits, combo_hits, normal_hits])
+    hard_hits, score_hits, combo_hits, external_hits, normal_hits = map(dedup, [hard_hits, score_hits, combo_hits, external_hits, normal_hits])
 
     if hard_hits:
         names = ",".join(r.name for r in hard_hits[:3])
         return True, f"硬规则命中: {names}", 100
 
-    spam_score = sum(r.score for r in score_hits) + sum(r.score for r in combo_hits)
+    spam_score = sum(r.score for r in score_hits) + sum(r.score for r in combo_hits) + sum(r.score for r in external_hits)
     normal_score = sum(r.score for r in normal_hits)
 
     # 广告组合是核心：即使很短也要先判垃圾
     if spam_score >= 75:
-        names = ",".join(r.name for r in (score_hits + combo_hits)[:4])
+        names = ",".join(r.name for r in (score_hits + combo_hits + external_hits)[:5])
         return True, f"广告规则评分 {spam_score}: {names}", spam_score
 
     # 纯短消息只有在没有广告分时才放行
@@ -152,7 +213,7 @@ def local_classify(text: str) -> tuple[bool | None, str, int]:
         return False, "无广告信号", 0
 
     # 模糊样本交给 AI
-    names = ",".join(r.name for r in (score_hits + combo_hits)[:4])
+    names = ",".join(r.name for r in (score_hits + combo_hits + external_hits)[:5])
     return None, f"疑似广告评分 {spam_score}: {names}", spam_score