273 lines
10 KiB
Python
273 lines
10 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
新闻智能跟踪系统 - 智能分析引擎
|
|
两级分析：
|
|
1. 规则打分（快、无需外部依赖）：兴趣相关度 + 重要度启发式 → total_score
|
|
2. LLM 深度分析（准、逐条）：对候选重要资讯调用 DeepSeek 输出重要度/分类/结论
|
|
"""
|
|
import json
|
|
import threading
|
|
|
|
import requests
|
|
|
|
import config
|
|
import db
|
|
|
|
# 重要度启发式信号词（命中越多分越高）
|
|
_STRONG_SIGNAL = [
|
|
"发布", "宣布", "推出", "开源", "突破", "融资", "IPO", "上市", "收购", "并购",
|
|
"合并", "裁员", "禁令", "制裁", "管制", "监管", "起诉", "反垄断", "罚款",
|
|
"重大", "首个", "最强", "新规", "停止", "暂停", "安全事件", "数据泄露",
|
|
"里程碑", "量产", "实施细则", "生效", "世界第一",
|
|
]
|
|
_MID_SIGNAL = [
|
|
"升级", "更新", "合作", "投资", "签署", "获批", "中标", "测试", "开放", "公测",
|
|
"翻倍", "增长", "新高", "新纪录", "接入", "部署", "扩大", "提升",
|
|
]
|
|
_WEAK_SIGNAL = ["讨论", "传闻", "预计", "可能", "或将", "消息人士", "知情人士"]
|
|
|
|
# 大额资金/规模信号：出现强金额词且伴随大数字 → 重要度加成
|
|
_MONEY_WORDS = ["亿美元", "亿元", "万亿", "估值", "融资", "收购", "IPO", "罚款", "大单", "募资", "基金"]
|
|
_MONEY_BIG = ["10亿", "20亿", "50亿", "百亿", "千亿", "万亿", "估值突破", "估值超", "亿美元",
|
|
"10 亿", "20 亿", "50 亿", "规模突破", "市值"]
|
|
|
|
_DOMAIN_RULES = {
|
|
"AI模型与算法": ["模型", "GPT", "DeepSeek", "大模型", "推理", "多模态", "智能体", "Agent",
|
|
"Scaling", "思维链", "视频生成", "扩散", "检索增强", "端侧模型", "开源模型"],
|
|
"芯片与硬件": ["芯片", "GPU", "半导体", "晶圆", "制程", "数据中心", "服务器", "AI芯片", "光刻机"],
|
|
"云计算与算力": ["云计算", "算力", "数据中心", "集群", "云厂商", "推理集群", "超算"],
|
|
"政策与监管": ["监管", "政策", "法案", "法规", "管制", "禁令", "制裁", "新规", "合规", "备案", "诉讼", "审查", "治理框架", "罚款"],
|
|
"投融资": ["融资", "IPO", "投资", "收购", "并购", "估值", "募资", "上市", "风投", "D轮", "大单"],
|
|
"企业动态": ["重组", "架构", "高管", "任命", "裁员", "人事", "部门", "收购"],
|
|
"学术研究": ["论文", "arXiv", "研究", "实验", "学者", "基准", "团队发布"],
|
|
"开源生态": ["开源", "GitHub", "权重", "社区", "代码", "star", "周榜"],
|
|
}
|
|
|
|
_IMPORTANCE_SCALE = {"strong": 32, "mid": 16, "weak": 6}
|
|
|
|
|
|
def _text_of(a):
|
|
return (a.get("title", "") + " " + a.get("content", "") + " " + a.get("summary", ""))
|
|
|
|
|
|
def classify_domain(a):
|
|
text = _text_of(a)
|
|
best, best_hits = "", 0
|
|
for dom, kws in _DOMAIN_RULES.items():
|
|
hits = sum(1 for kw in kws if kw.lower() in text.lower())
|
|
if hits > best_hits:
|
|
best, best_hits = dom, hits
|
|
return best
|
|
|
|
|
|
def extract_entities(a):
|
|
"""返回 (全部实体, 真实关注公司)"""
|
|
text = _text_of(a)
|
|
found, real = [], []
|
|
for c in db.list_companies():
|
|
name = c["name"]
|
|
if name and name.lower() in text.lower():
|
|
if name not in found:
|
|
found.append(name)
|
|
if name not in real:
|
|
real.append(name)
|
|
for e in a.get("entities") or []:
|
|
if e not in found:
|
|
found.append(e)
|
|
return found, real
|
|
|
|
|
|
def _keyword_hits(text):
|
|
hits, score = [], 0
|
|
for kw in db.list_keywords():
|
|
if not kw["enabled"]:
|
|
continue
|
|
if kw["keyword"].lower() in text.lower():
|
|
hits.append(kw["keyword"])
|
|
score += kw["weight"]
|
|
return hits, min(50, score)
|
|
|
|
|
|
def _domain_score(domain):
|
|
for dom in db.list_domains():
|
|
if dom["enabled"] and dom["name"] == domain:
|
|
return min(30, dom["weight"] * 4)
|
|
return 0
|
|
|
|
|
|
def _company_match(real_companies):
|
|
"""返回 (相关度加分, 重要度impact)"""
|
|
n = len(real_companies)
|
|
if n == 0:
|
|
return 0, 0
|
|
return min(20, 10 + (n - 1) * 3), min(14, 8 + (n - 1) * 4)
|
|
|
|
|
|
def _signal_score(text):
|
|
strong = [w for w in _STRONG_SIGNAL if w in text]
|
|
if strong:
|
|
return min(38, _IMPORTANCE_SCALE["strong"] + 6 * (len(strong) - 1))
|
|
mid = [w for w in _MID_SIGNAL if w in text]
|
|
if mid:
|
|
return min(22, _IMPORTANCE_SCALE["mid"] + 4 * (len(mid) - 1))
|
|
return _IMPORTANCE_SCALE["weak"] if any(w in text for w in _WEAK_SIGNAL) else 0
|
|
|
|
|
|
def _money_magnitude(text):
|
|
if any(k in text for k in _MONEY_WORDS):
|
|
if any(m in text for m in _MONEY_BIG):
|
|
return 12
|
|
return 6
|
|
return 0
|
|
|
|
|
|
def _recency_bonus(a):
|
|
try:
|
|
from datetime import datetime
|
|
pub = datetime.strptime(a.get("published_at", ""), "%Y-%m-%d %H:%M:%S")
|
|
hours = (datetime.now() - pub).total_seconds() / 3600
|
|
except Exception:
|
|
return 5
|
|
if hours <= 6:
|
|
return 12
|
|
if hours <= 24:
|
|
return 8
|
|
if hours <= 48:
|
|
return 4
|
|
return 0
|
|
|
|
|
|
def _source_weight(a):
|
|
sid = a.get("source_id") or 0
|
|
s = db.get_source(sid)
|
|
if not s:
|
|
return 0
|
|
return int(round(s["weight"] * 10))
|
|
|
|
|
|
def analyze_article(aid):
|
|
"""规则打分（立即生效）"""
|
|
a = db.get_article(aid)
|
|
if not a:
|
|
return None
|
|
text = _text_of(a)
|
|
domain = classify_domain(a)
|
|
entities, real = extract_entities(a)
|
|
|
|
_, kw_score = _keyword_hits(text)
|
|
dom_score = _domain_score(domain)
|
|
comp_score, comp_impact = _company_match(real)
|
|
rel = min(100, kw_score + dom_score + comp_score)
|
|
|
|
sig = _signal_score(text)
|
|
money = _money_magnitude(text)
|
|
rec = _recency_bonus(a)
|
|
src = _source_weight(a)
|
|
imp = min(100, sig + money + rec + src + comp_impact + int(rel * 0.2))
|
|
|
|
total = min(100, int(round(0.4 * rel + 0.6 * imp)))
|
|
|
|
threshold = int(db.get_setting("realtime_threshold", config.AUTO_DEFAULTS["realtime_threshold"]))
|
|
is_important = 1 if (total >= threshold or (rel >= 65 and imp >= 70)) else 0
|
|
|
|
db.update_article(
|
|
aid, domain=domain, entities=entities, relevance=rel, total_score=total,
|
|
is_important=is_important,
|
|
)
|
|
return {"id": aid, "domain": domain, "entities": entities, "relevance": rel,
|
|
"importance_rule": imp, "total_score": total, "is_important": is_important}
|
|
|
|
|
|
def llm_analyze(aid):
|
|
"""LLM 深度分析单条：重要度 1-10 + 相关度 + 结论。失败则标记 error 不阻塞。"""
|
|
a = db.get_article(aid)
|
|
if not a:
|
|
return None
|
|
profile = _profile_text()
|
|
prompt = (
|
|
"你是一位资深科技资讯分析师，专注AI领域。\n"
|
|
f"用户兴趣画像：\n{profile}\n\n"
|
|
f"资讯标题：{a['title']}\n"
|
|
f"资讯内容：{a.get('content') or a.get('summary')}\n\n"
|
|
"请只输出一个 JSON 对象（不要任何其他文字），格式：\n"
|
|
'{"importance": 1-10的整数, "relevance": 0-100的整数, '
|
|
'"is_important": true或false, "category": "分类名", "reason": "为什么对用户重要（40字内中文）"}'
|
|
)
|
|
try:
|
|
resp = requests.post(
|
|
f"{config.LLM_BASE_URL}/chat/completions",
|
|
headers={"Authorization": f"Bearer {config.LLM_API_KEY}",
|
|
"Content-Type": "application/json"},
|
|
json={
|
|
"model": config.LLM_MODEL,
|
|
"messages": [{"role": "user", "content": prompt}],
|
|
"temperature": config.LLM_TEMPERATURE,
|
|
"max_tokens": config.LLM_MAX_TOKENS,
|
|
"response_format": {"type": "json_object"},
|
|
},
|
|
timeout=config.LLM_TIMEOUT,
|
|
)
|
|
data = resp.json()
|
|
content = data["choices"][0]["message"]["content"]
|
|
parsed = json.loads(content)
|
|
importance = max(1, min(10, int(parsed.get("importance", 5))))
|
|
relevance = max(0, min(100, int(parsed.get("relevance", 50))))
|
|
is_important = 1 if parsed.get("is_important") else 0
|
|
category = parsed.get("category", a.get("domain", ""))
|
|
reason = parsed.get("reason", "")
|
|
|
|
# LLM 结论与规则分融合
|
|
total = a.get("total_score", 0)
|
|
llm_component = int(round(importance * 10 * 0.5 + relevance * 0.2))
|
|
total = min(100, int(round(0.6 * total + 0.4 * llm_component)))
|
|
threshold = int(db.get_setting("realtime_threshold", config.AUTO_DEFAULTS["realtime_threshold"]))
|
|
if is_important == 0 and total >= threshold:
|
|
is_important = 1
|
|
|
|
db.update_article(
|
|
aid, importance=importance, relevance=max(relevance, a.get("relevance", 0)),
|
|
total_score=total, is_important=is_important, analysis=reason,
|
|
domain=category, llm_status="done",
|
|
)
|
|
return {"id": aid, "importance": importance, "relevance": relevance,
|
|
"total_score": total, "is_important": is_important, "reason": reason}
|
|
except Exception as e:
|
|
db.update_article(aid, llm_status="error")
|
|
return {"id": aid, "error": str(e)}
|
|
|
|
|
|
def _profile_text():
|
|
kws = "、".join(k["keyword"] for k in db.list_keywords() if k["enabled"])
|
|
comps = "、".join(c["name"] for c in db.list_companies() if c["enabled"])
|
|
doms = "、".join(d["name"] for d in db.list_domains() if d["enabled"])
|
|
return f"关注关键词：{kws}\n关注公司：{comps}\n关注领域：{doms}"
|
|
|
|
|
|
def batch_llm_analyze(limit=10):
|
|
"""后台线程：对 pending 且规则分达标的资讯做 LLM 深度分析"""
|
|
threshold = int(db.get_setting("llm_threshold", config.AUTO_DEFAULTS["llm_threshold"]))
|
|
arts = db.pending_llm_articles(limit=limit)
|
|
results = {"done": 0, "error": 0, "skipped": 0}
|
|
for a in arts:
|
|
if a.get("total_score", 0) < threshold:
|
|
db.update_article(a["id"], llm_status="skipped")
|
|
results["skipped"] += 1
|
|
continue
|
|
r = llm_analyze(a["id"])
|
|
if r and "error" not in r:
|
|
results["done"] += 1
|
|
else:
|
|
results["error"] += 1
|
|
return results
|
|
|
|
|
|
def run_llm_background(limit=8):
|
|
def _job():
|
|
try:
|
|
batch_llm_analyze(limit=limit)
|
|
except Exception as e:
|
|
db.add_log("realtime", "LLM分析异常", 0, [], status="error", detail=str(e))
|
|
t = threading.Thread(target=_job, daemon=True)
|
|
t.start()
|
|
return t
|