ngfchl
28 lines
934 B
Python
28 lines
934 B
Python
#!/usr/bin/env python3
|
|
"""构建轻量知识库 metadata，不依赖 torch/sentence-transformers。"""
|
|
import json
|
|
import sys
|
|
|
|
sys.path.insert(0, "/vol3/1000/Code/tg-spam-guard")
|
|
|
|
from knowledge import INDEX_DIR, load_all_docs, save_qa_cache, get_stats
|
|
|
|
|
|
if __name__ == "__main__":
|
|
print("📚 开始构建轻量知识库 metadata...")
|
|
INDEX_DIR.mkdir(parents=True, exist_ok=True)
|
|
docs = load_all_docs()
|
|
if not docs:
|
|
print("❌ 没有找到文档")
|
|
sys.exit(1)
|
|
|
|
(INDEX_DIR / "metadata.json").write_text(
|
|
json.dumps(docs, ensure_ascii=False, indent=2), encoding="utf-8"
|
|
)
|
|
save_qa_cache({})
|
|
|
|
lengths = [len(d.get("text", "")) for d in docs]
|
|
print(f"✅ metadata 构建完成：{len(docs)} 个文档块")
|
|
print(f"📊 chunk 长度：avg={sum(lengths)//len(lengths)}, min={min(lengths)}, max={max(lengths)}")
|
|
print(json.dumps(get_stats(), ensure_ascii=False, indent=2))
|