hz4th_coder
e4efd24e1c
- 新增 crawler.py: requests+bs4 readability风格清洗, 提取候选链接按文章相似度排序, 前5条抓全文存 articles.full_text, 详情页展示; example.com占位源走模拟, 真实源失败标记error不造假 - 大模型: 新增 llm_providers 表(预置 SiliconFlow/DeepSeek官方/Autodl/Local Qwen), 设置页增删改/测试/一键切换, 激活接口失败自动切换备用 - 数据源: 前端补编辑按钮+弹窗(后端 update 已支持), 列表显示URL与采集状态 - 数据库迁移: articles.full_text 列 + llm_providers 表
315 lines
12 KiB
Python
315 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
新闻智能跟踪系统 - Flask 主应用
|
|
网页：仪表盘 / 资讯列表 / 资讯详情 / 数据源 / 兴趣画像 / 通知日志 / 设置
|
|
API：采集 / LLM分析 / 汇总 / 画像维护 / 设置维护
|
|
"""
|
|
from datetime import datetime, timedelta
|
|
|
|
from flask import Flask, render_template, request, jsonify, redirect, url_for
|
|
|
|
import requests
|
|
|
|
import config
|
|
import db
|
|
import simulate
|
|
import analysis
|
|
import notifier
|
|
import scheduler
|
|
|
|
app = Flask(__name__, static_folder=config.STATIC_DIR, template_folder=config.TEMPLATE_DIR)
|
|
|
|
|
|
# ---------------- 页面 ----------------
|
|
@app.route("/")
|
|
def index():
|
|
return redirect(url_for("dashboard"))
|
|
|
|
|
|
@app.route("/dashboard")
|
|
def dashboard():
|
|
stats = db.article_stats()
|
|
latest = db.list_articles(limit=12, order="a.collected_at DESC")
|
|
important = db.list_articles(is_important=1, limit=12, order="a.total_score DESC")
|
|
dom_rows = db.get_conn().execute(
|
|
"SELECT domain, COUNT(*) c FROM articles GROUP BY domain ORDER BY c DESC").fetchall()
|
|
domain_stats = [{"name": r["domain"] or "未分类", "count": r["c"]} for r in dom_rows]
|
|
return render_template("dashboard.html", stats=stats, latest=latest,
|
|
important=important, domain_stats=domain_stats)
|
|
|
|
|
|
@app.route("/news")
|
|
def news():
|
|
domain = request.args.get("domain", "")
|
|
important = request.args.get("important", "")
|
|
q = request.args.get("q", "")
|
|
page = max(1, int(request.args.get("page", 1)))
|
|
per = 20
|
|
filters = {}
|
|
if domain:
|
|
filters["domain"] = domain
|
|
if important == "1":
|
|
filters["is_important"] = 1
|
|
if q:
|
|
filters["q"] = q
|
|
total = db.count_articles(domain=filters.get("domain"), is_important=filters.get("is_important"))
|
|
if q:
|
|
total = len(db.list_articles(q=q))
|
|
pages = max(1, (total + per - 1) // per)
|
|
articles = db.list_articles(offset=(page - 1) * per, limit=per, **filters)
|
|
domains = db.get_conn().execute(
|
|
"SELECT DISTINCT domain FROM articles WHERE domain<>'' ORDER BY domain").fetchall()
|
|
return render_template("news.html", articles=articles, domains=[d["domain"] for d in domains],
|
|
domain=domain, important=important, q=q, page=page, pages=pages, total=total)
|
|
|
|
|
|
@app.route("/news/<int:aid>")
|
|
def news_detail(aid):
|
|
art = db.get_article(aid)
|
|
if not art:
|
|
return "not found", 404
|
|
return render_template("detail.html", art=art)
|
|
|
|
|
|
@app.route("/sources")
|
|
def sources():
|
|
return render_template("sources.html", sources=db.list_sources())
|
|
|
|
|
|
@app.route("/profile")
|
|
def profile():
|
|
return render_template("profile.html", keywords=db.list_keywords(),
|
|
domains=db.list_domains(), companies=db.list_companies())
|
|
|
|
|
|
@app.route("/logs")
|
|
def logs():
|
|
return render_template("logs.html", logs=db.list_logs(limit=100))
|
|
|
|
|
|
@app.route("/settings")
|
|
def settings_page():
|
|
return render_template("settings.html", auto=db.get_all_settings(),
|
|
providers=db.list_providers())
|
|
|
|
|
|
# ---------------- API ----------------
|
|
@app.route("/api/stats")
|
|
def api_stats():
|
|
stats = db.article_stats()
|
|
dom_rows = db.get_conn().execute(
|
|
"SELECT domain, COUNT(*) c FROM articles GROUP BY domain ORDER BY c DESC").fetchall()
|
|
stats["domains"] = [{"name": r["domain"] or "未分类", "count": r["c"]} for r in dom_rows]
|
|
trend_rows = db.get_conn().execute(
|
|
"SELECT substr(collected_at,1,10) d, COUNT(*) c FROM articles "
|
|
"GROUP BY d ORDER BY d DESC LIMIT 7").fetchall()
|
|
stats["trend"] = [{"date": r["d"], "count": r["c"]} for r in reversed(trend_rows)]
|
|
return jsonify(stats)
|
|
|
|
|
|
@app.route("/api/sources", methods=["POST"])
|
|
def api_sources():
|
|
data = request.get_json(force=True) or {}
|
|
action = data.get("action")
|
|
if action == "add":
|
|
sid = db.add_source(data.get("name", ""), data.get("type", ""), data.get("url", ""),
|
|
data.get("description", ""), float(data.get("weight", 1.0)))
|
|
return jsonify({"ok": True, "id": sid})
|
|
if action == "update":
|
|
db.update_source(data["id"], name=data.get("name"), type=data.get("type"),
|
|
url=data.get("url"), description=data.get("description"),
|
|
weight=float(data.get("weight", 1.0)),
|
|
enabled=1 if data.get("enabled") else 0)
|
|
return jsonify({"ok": True})
|
|
if action == "delete":
|
|
db.delete_source(data["id"])
|
|
return jsonify({"ok": True})
|
|
if action == "toggle":
|
|
s = db.get_source(data["id"])
|
|
db.update_source(data["id"], enabled=0 if s["enabled"] else 1)
|
|
return jsonify({"ok": True})
|
|
return jsonify({"ok": False, "error": "unknown action"})
|
|
|
|
|
|
@app.route("/api/profile", methods=["POST"])
|
|
def api_profile():
|
|
data = request.get_json(force=True) or {}
|
|
action = data.get("action")
|
|
kind = data.get("kind")
|
|
if action == "add":
|
|
if kind == "keyword":
|
|
db.add_keyword(data.get("name", ""), int(data.get("weight", 5)))
|
|
elif kind == "domain":
|
|
db.add_domain(data.get("name", ""), int(data.get("weight", 5)))
|
|
elif kind == "company":
|
|
db.add_company(data.get("name", ""))
|
|
return jsonify({"ok": True})
|
|
if action == "delete":
|
|
if kind == "keyword":
|
|
db.delete_keyword(data["id"])
|
|
elif kind == "domain":
|
|
db.delete_domain(data["id"])
|
|
elif kind == "company":
|
|
db.delete_company(data["id"])
|
|
return jsonify({"ok": True})
|
|
return jsonify({"ok": False, "error": "unknown action"})
|
|
|
|
|
|
@app.route("/api/settings", methods=["POST"])
|
|
def api_settings():
|
|
data = request.get_json(force=True) or {}
|
|
auto = {}
|
|
for k in config.AUTO_DEFAULTS:
|
|
if k in data:
|
|
auto[k] = data[k]
|
|
if auto:
|
|
cur = db.get_all_settings()
|
|
cur.update(auto)
|
|
db.set_setting("auto", auto)
|
|
if "mail" in data and isinstance(data["mail"], dict):
|
|
cur = db.get_all_settings().get("mail", {})
|
|
cur.update(data["mail"])
|
|
db.set_setting("mail", cur)
|
|
return jsonify({"ok": True})
|
|
|
|
|
|
@app.route("/api/llm", methods=["POST"])
|
|
def api_llm():
|
|
"""大模型接口管理：增删改 / 一键切换 / 测试"""
|
|
data = request.get_json(force=True) or {}
|
|
action = data.get("action")
|
|
if action == "add":
|
|
pid = db.add_provider(
|
|
data.get("name", ""), data.get("base_url", ""), data.get("api_key", ""),
|
|
data.get("model", ""),
|
|
active=1 if data.get("active") else 0,
|
|
enabled=1 if data.get("enabled", 1) else 0,
|
|
)
|
|
if data.get("active"):
|
|
db.set_active_provider(pid)
|
|
return jsonify({"ok": True, "id": pid})
|
|
if action == "update":
|
|
db.update_provider(data["id"], name=data.get("name"), base_url=data.get("base_url"),
|
|
api_key=data.get("api_key"), model=data.get("model"),
|
|
enabled=1 if data.get("enabled", 1) else 0)
|
|
return jsonify({"ok": True})
|
|
if action == "delete":
|
|
db.delete_provider(data["id"])
|
|
return jsonify({"ok": True})
|
|
if action == "switch":
|
|
db.set_active_provider(data["id"])
|
|
p = db.get_provider(data["id"])
|
|
return jsonify({"ok": True, "name": p["name"] if p else ""})
|
|
if action == "toggle":
|
|
p = db.get_provider(data["id"])
|
|
if not p:
|
|
return jsonify({"ok": False, "error": "not found"})
|
|
db.update_provider(data["id"], enabled=0 if p["enabled"] else 1)
|
|
return jsonify({"ok": True})
|
|
if action == "test":
|
|
# 用指定接口（或当前激活接口）发一条测试消息
|
|
cfg = None
|
|
if data.get("id"):
|
|
p = db.get_provider(data["id"])
|
|
if p and p.get("base_url"):
|
|
cfg = {"name": p["name"], "base_url": p["base_url"].rstrip("/"),
|
|
"api_key": p.get("api_key", ""), "model": p.get("model", "")}
|
|
try:
|
|
if cfg is None:
|
|
import analysis
|
|
cfg = analysis.get_llm_cfg()
|
|
r = requests.post(
|
|
f"{cfg['base_url']}/chat/completions",
|
|
headers={"Authorization": f"Bearer {cfg['api_key']}",
|
|
"Content-Type": "application/json"},
|
|
json={"model": cfg["model"],
|
|
"messages": [{"role": "user", "content": "请回复：连接正常"}],
|
|
"max_tokens": 60, "temperature": 0.3},
|
|
timeout=60,
|
|
)
|
|
r.raise_for_status()
|
|
content = r.json()["choices"][0]["message"]["content"]
|
|
return jsonify({"ok": True, "name": cfg["name"], "model": cfg["model"],
|
|
"reply": content})
|
|
except Exception as e:
|
|
return jsonify({"ok": False, "error": str(e)})
|
|
return jsonify({"ok": False, "error": "unknown action"})
|
|
|
|
|
|
@app.route("/api/actions", methods=["POST"])
|
|
def api_actions():
|
|
data = request.get_json(force=True) or {}
|
|
action = data.get("action")
|
|
if action == "collect":
|
|
n = scheduler.collect_once()
|
|
return jsonify({"ok": True, "added": n})
|
|
if action == "llm":
|
|
r = analysis.batch_llm_analyze(limit=int(data.get("limit", 10)))
|
|
return jsonify({"ok": True, **r})
|
|
if action == "summary":
|
|
n = scheduler.send_daily_summary()
|
|
return jsonify({"ok": True, "sent": n})
|
|
if action == "seed":
|
|
n = simulate.seed_all()
|
|
return jsonify({"ok": True, "added": n})
|
|
if action == "reanalyze":
|
|
# 重新跑规则打分（如改了兴趣画像后）
|
|
conn = db.get_conn()
|
|
ids = [r["id"] for r in conn.execute("SELECT id FROM articles").fetchall()]
|
|
conn.close()
|
|
cnt = 0
|
|
for aid in ids:
|
|
analysis.analyze_article(aid)
|
|
cnt += 1
|
|
return jsonify({"ok": True, "count": cnt})
|
|
if action == "test_mail":
|
|
try:
|
|
notifier.send_email("📮 新闻智能跟踪系统测试", "<h3>测试成功</h3><p>邮件通知链路正常。</p>")
|
|
return jsonify({"ok": True, "msg": "测试邮件已发送"})
|
|
except Exception as e:
|
|
return jsonify({"ok": False, "error": str(e)})
|
|
return jsonify({"ok": False, "error": "unknown action"})
|
|
|
|
|
|
@app.errorhandler(404)
|
|
def not_found(e):
|
|
return render_template("404.html"), 404
|
|
|
|
|
|
# ---------------- 启动 ----------------
|
|
def main():
|
|
db.init_db()
|
|
# 首次初始化：写入默认数据源 / 兴趣画像 / 默认设置 / 模拟数据
|
|
if db.get_setting("initialized") != 1:
|
|
for s in config.DEFAULT_SOURCES:
|
|
db.add_source(s["name"], s["type"], s["url"], s["description"], s["weight"])
|
|
for kw, w in config.DEFAULT_KEYWORDS:
|
|
db.add_keyword(kw, w)
|
|
for d, w in config.DEFAULT_DOMAINS:
|
|
db.add_domain(d, w)
|
|
for c in config.DEFAULT_COMPANIES:
|
|
db.add_company(c)
|
|
# 预置大模型接口（可一键切换）
|
|
if not db.list_providers():
|
|
for p in config.LLM_PROVIDERS_DEFAULT:
|
|
pid = db.add_provider(p["name"], p["base_url"], p["api_key"], p["model"],
|
|
active=1 if p.get("active") else 0)
|
|
if p.get("active"):
|
|
db.set_active_provider(pid)
|
|
db.set_setting("initialized", 1)
|
|
db.set_setting("auto", dict(config.AUTO_DEFAULTS))
|
|
db.set_setting("mail", dict(config.MAIL_DEFAULTS))
|
|
simulate.seed_all()
|
|
analysis.run_llm_background()
|
|
stop_event, t = scheduler.start_scheduler()
|
|
print(f"✅ {config.SERVICE_NAME} 启动完成")
|
|
print(f" Web: http://0.0.0.0:{config.SERVICE_PORT}/")
|
|
print(f" 每日汇总: {db.get_setting('summary_time', config.AUTO_DEFAULTS['summary_time'])}"
|
|
f" | 采集间隔: {db.get_setting('scan_interval_min', 30)}分钟"
|
|
f" | 实时阈值: {db.get_setting('realtime_threshold', 80)}")
|
|
app.run(host=config.SERVICE_HOST, port=config.SERVICE_PORT, threaded=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|