diff --git a/Dockerfile b/Dockerfile index 2a4181f..3bb9c75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ COPY config.py . COPY db/ db/ COPY handlers/ handlers/ COPY services/ services/ +COPY rules/ rules/ COPY web/ web/ COPY knowledge/ knowledge/ COPY main.py . diff --git a/rules/spam_keywords.txt b/rules/spam_keywords.txt new file mode 100644 index 0000000..bee5b9e --- /dev/null +++ b/rules/spam_keywords.txt @@ -0,0 +1,114 @@ +# 中文垃圾广告/黑灰产关键词词库（可追加网上词库，一行一个词） +# 格式：词条 或 词条|分数 + +# 赌博/博彩 +菠菜|100 +赌博|100 +博彩|100 +赌场|100 +棋牌|100 +彩票|100 +开奖|90 +中奖|80 +下注|100 +百家乐|100 +时时彩|100 +六合彩|100 +赌球|100 +盘口|90 + +# 色情/擦边 +约炮|100 +外围|100 +楼凤|100 +裸聊|100 +成人视频|100 +一夜情|100 +上门服务|100 + +# 诈骗/灰产 +刷单|100 +返佣|70 +传销|100 +资金盘|100 +杀猪盘|100 +跑分|100 +洗钱|100 +代收款|100 +接码|90 +猫池|90 +办证|100 +假证|100 +代开发票|100 + +# 赚钱/兼职广告 +日赚|75 +日入|75 +每天赚|75 +一天赚|75 +轻松赚|75 +稳赚|75 +躺赚|70 +月入|70 +月赚|70 +副业|35 +兼职日结|60 +手机操作|45 +无需经验|45 +宝妈|35 +学生党|35 +小白可带|65 +手把手带|65 +带你赚|70 +带你做|65 + +# 引流/联系方式 +加微信|50 +微信号|50 +vx|45 +v信|45 +薇信|45 +加我|40 +私聊|45 +私信|45 +联系我|45 +看主页|45 +看签名|45 +主页有|45 +点头像|45 +进群|45 +拉群|45 +频道|35 + +# 币圈/投资/贷款 +USDT|50 +泰达币|50 +虚拟币|50 +币圈|45 +搬砖|50 +套利|50 +贷款|45 +套现|55 +黑户|60 +秒批|55 +包过|55 +下款|55 +投资带单|70 +股票老师|60 +期货老师|60 +外汇老师|60 +量化跟单|65 + +# 营销诱导 +免费领|45 +免费送|45 +限时领|45 +点击领取|45 +扫码领取|45 +红包|30 +福利|30 +返现|40 +代理|35 +加盟|35 +招商|35 +招代理|45 diff --git a/rules/spam_regex.txt b/rules/spam_regex.txt new file mode 100644 index 0000000..a187978 --- /dev/null +++ b/rules/spam_regex.txt @@ -0,0 +1,10 @@ +# 中文垃圾广告正则库（可追加网上规则，一行一个正则） +# 格式：正则 或 正则|分数 +(?:日赚|日入|每天赚|一天赚|轻松赚|稳赚|躺赚).{0,10}(?:\d{2,}|几百|上千|过千|一千|几千|上万|万元?|k|K|w|W)|90 +(?:月入|月赚|月收益|月收入).{0,10}(?:\d{3,}|过万|上万|几万|万元?|w|W)|85 +(?:赚钱|收益|收入|副业|项目|路子).{0,20}(?:加我|私聊|看主页|看签名|进群|联系我|微信|vx|QQ)|85 +(?:\d{3,}|几百|上千|过千|几千|上万|万元?|k|K|w|W).{0,20}(?:加我|私聊|看主页|看签名|进群|联系我|微信|vx|QQ)|80 +(?:兼职|副业|日结|周结|在家做|手机操作|无需经验).{0,18}(?:赚钱|收入|结算|工资|项目|单)|75 +(?:USDT|U\s?S\s?D\s?T|泰达币|虚拟币|币圈|搬砖|套利).{0,18}(?:赚钱|收益|项目|加我|看主页|进群)|85 +(?:贷款|借款|套现|信用卡|花呗|白条|提额|下款|黑户|征信).{0,18}(?:秒批|当天|无视|包过|渠道)|85 +(?:t\.me|telegram\.me|tg://|飞机群).{0,40}(?:join|group|channel|\+|进群|频道)|95 diff --git a/services/spam_detector.py b/services/spam_detector.py index cfa29c6..f02f6ae 100644 --- a/services/spam_detector.py +++ b/services/spam_detector.py @@ -11,6 +11,7 @@ import logging import re from dataclasses import dataclass +from pathlib import Path import httpx @@ -26,6 +27,63 @@ class Rule: score: int +ROOT_DIR = Path(__file__).resolve().parent.parent +RULES_DIR = ROOT_DIR / "rules" + + +def load_keyword_rules(path: Path, default_score: int = 45) -> list[Rule]: + """加载外部关键词词库：词条 或 词条|分数。""" + rules: list[Rule] = [] + if not path.exists(): + return rules + for line in path.read_text(encoding="utf-8").splitlines(): + raw = line.strip() + if not raw or raw.startswith("#"): + continue + if "|" in raw: + word, score_text = raw.rsplit("|", 1) + try: + score = int(score_text.strip()) + except ValueError: + word, score = raw, default_score + else: + word, score = raw, default_score + word = word.strip() + if word: + rules.append(Rule(f"外部关键词:{word}", re.escape(word), score)) + return rules + + +def load_regex_rules(path: Path, default_score: int = 70) -> list[Rule]: + """加载外部正则库：正则 或 正则|分数。""" + rules: list[Rule] = [] + if not path.exists(): + return rules + for i, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + raw = line.strip() + if not raw or raw.startswith("#"): + continue + if "|" in raw: + pattern, score_text = raw.rsplit("|", 1) + try: + score = int(score_text.strip()) + except ValueError: + pattern, score = raw, default_score + else: + pattern, score = raw, default_score + try: + re.compile(pattern) + rules.append(Rule(f"外部正则:{i}", pattern, score)) + except re.error as e: + logger.warning(f"⚠️ 跳过无效外部正则 {path}:{i}: {e}") + return rules + + +EXTERNAL_KEYWORD_RULES = load_keyword_rules(RULES_DIR / "spam_keywords.txt") +EXTERNAL_REGEX_RULES = load_regex_rules(RULES_DIR / "spam_regex.txt") +logger.info(f"🧩 已加载外部垃圾规则：关键词 {len(EXTERNAL_KEYWORD_RULES)} 条，正则 {len(EXTERNAL_REGEX_RULES)} 条") + + # 高危黑产：单项命中即可高度可疑或直接垃圾 HARD_RULES = [ Rule("博彩赌博", r"菠菜|赌博|博彩|赌场|棋牌|彩票|开奖|中奖|赔率|下注|百家乐|时时彩|六合彩", 100), @@ -114,28 +172,31 @@ def local_classify(text: str) -> tuple[bool | None, str, int]: hard_hits = [] score_hits = [] combo_hits = [] + external_hits = [] normal_hits = [] for target in targets: hard_hits.extend(hit_rules(target, HARD_RULES)) score_hits.extend(hit_rules(target, SCORE_RULES)) combo_hits.extend(hit_rules(target, COMBO_RULES)) + external_hits.extend(hit_rules(target, EXTERNAL_KEYWORD_RULES)) + external_hits.extend(hit_rules(target, EXTERNAL_REGEX_RULES)) normal_hits.extend(hit_rules(target, NORMAL_RULES)) # 去重 dedup = lambda xs: list({x.name: x for x in xs}.values()) - hard_hits, score_hits, combo_hits, normal_hits = map(dedup, [hard_hits, score_hits, combo_hits, normal_hits]) + hard_hits, score_hits, combo_hits, external_hits, normal_hits = map(dedup, [hard_hits, score_hits, combo_hits, external_hits, normal_hits]) if hard_hits: names = ",".join(r.name for r in hard_hits[:3]) return True, f"硬规则命中: {names}", 100 - spam_score = sum(r.score for r in score_hits) + sum(r.score for r in combo_hits) + spam_score = sum(r.score for r in score_hits) + sum(r.score for r in combo_hits) + sum(r.score for r in external_hits) normal_score = sum(r.score for r in normal_hits) # 广告组合是核心：即使很短也要先判垃圾 if spam_score >= 75: - names = ",".join(r.name for r in (score_hits + combo_hits)[:4]) + names = ",".join(r.name for r in (score_hits + combo_hits + external_hits)[:5]) return True, f"广告规则评分 {spam_score}: {names}", spam_score # 纯短消息只有在没有广告分时才放行 @@ -152,7 +213,7 @@ def local_classify(text: str) -> tuple[bool | None, str, int]: return False, "无广告信号", 0 # 模糊样本交给 AI - names = ",".join(r.name for r in (score_hits + combo_hits)[:4]) + names = ",".join(r.name for r in (score_hits + combo_hits + external_hits)[:5]) return None, f"疑似广告评分 {spam_score}: {names}", spam_score