hz4th_coder
e4efd24e1c
- 新增 crawler.py: requests+bs4 readability风格清洗, 提取候选链接按文章相似度排序, 前5条抓全文存 articles.full_text, 详情页展示; example.com占位源走模拟, 真实源失败标记error不造假 - 大模型: 新增 llm_providers 表(预置 SiliconFlow/DeepSeek官方/Autodl/Local Qwen), 设置页增删改/测试/一键切换, 激活接口失败自动切换备用 - 数据源: 前端补编辑按钮+弹窗(后端 update 已支持), 列表显示URL与采集状态 - 数据库迁移: articles.full_text 列 + llm_providers 表
574 lines
17 KiB
Python
574 lines
17 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
新闻智能跟踪系统 - SQLite 数据访问层
|
||
原生 sqlite3，无 ORM，结构与其它项目保持一致。
|
||
"""
|
||
import sqlite3
|
||
import json
|
||
import os
|
||
from datetime import datetime
|
||
|
||
import config
|
||
|
||
_SCHEMA = """
|
||
CREATE TABLE IF NOT EXISTS sources (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
name TEXT NOT NULL,
|
||
type TEXT DEFAULT '',
|
||
url TEXT DEFAULT '',
|
||
description TEXT DEFAULT '',
|
||
weight REAL DEFAULT 1.0,
|
||
enabled INTEGER DEFAULT 1,
|
||
status TEXT DEFAULT 'ok', -- ok / error
|
||
last_fetch TEXT DEFAULT '',
|
||
last_count INTEGER DEFAULT 0,
|
||
created_at TEXT DEFAULT ''
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS articles (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
source_id INTEGER DEFAULT 0,
|
||
title TEXT NOT NULL,
|
||
url TEXT DEFAULT '',
|
||
author TEXT DEFAULT '',
|
||
content TEXT DEFAULT '',
|
||
summary TEXT DEFAULT '',
|
||
domain TEXT DEFAULT '', -- 分类：AI模型与算法 / 芯片与硬件 ...
|
||
entities TEXT DEFAULT '[]', -- JSON 数组：涉及的公司/实体
|
||
importance INTEGER DEFAULT 0, -- LLM 深度评分 1-10
|
||
relevance INTEGER DEFAULT 0, -- 兴趣相关度 0-100
|
||
total_score INTEGER DEFAULT 0, -- 综合分 0-100
|
||
is_important INTEGER DEFAULT 0,
|
||
analysis TEXT DEFAULT '', -- LLM 分析结论(为什么重要)
|
||
llm_status TEXT DEFAULT 'pending', -- pending / done / skipped / error
|
||
notified INTEGER DEFAULT 0, -- 是否已实时邮件通知
|
||
status TEXT DEFAULT 'new', -- new / summarized
|
||
published_at TEXT DEFAULT '',
|
||
collected_at TEXT DEFAULT ''
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS keywords (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
keyword TEXT NOT NULL,
|
||
weight INTEGER DEFAULT 5,
|
||
enabled INTEGER DEFAULT 1
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS domains (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
name TEXT NOT NULL,
|
||
weight INTEGER DEFAULT 5,
|
||
enabled INTEGER DEFAULT 1
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS companies (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
name TEXT NOT NULL,
|
||
enabled INTEGER DEFAULT 1
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS notification_log (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
type TEXT DEFAULT 'realtime', -- realtime / summary
|
||
subject TEXT DEFAULT '',
|
||
count INTEGER DEFAULT 0,
|
||
article_ids TEXT DEFAULT '[]',
|
||
status TEXT DEFAULT 'ok',
|
||
detail TEXT DEFAULT '',
|
||
sent_at TEXT DEFAULT ''
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS settings (
|
||
key TEXT PRIMARY KEY,
|
||
value TEXT DEFAULT ''
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS llm_providers (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
name TEXT NOT NULL,
|
||
base_url TEXT DEFAULT '',
|
||
api_key TEXT DEFAULT '',
|
||
model TEXT DEFAULT '',
|
||
active INTEGER DEFAULT 0, -- 1=当前激活（网页一键切换）
|
||
enabled INTEGER DEFAULT 1,
|
||
created_at TEXT DEFAULT ''
|
||
);
|
||
|
||
CREATE INDEX IF NOT EXISTS idx_articles_collected ON articles(collected_at);
|
||
CREATE INDEX IF NOT EXISTS idx_articles_score ON articles(total_score);
|
||
CREATE INDEX IF NOT EXISTS idx_articles_status ON articles(llm_status);
|
||
"""
|
||
|
||
|
||
def get_conn():
|
||
conn = sqlite3.connect(config.DB_PATH)
|
||
conn.row_factory = sqlite3.Row
|
||
conn.execute("PRAGMA journal_mode=WAL")
|
||
return conn
|
||
|
||
|
||
def init_db():
|
||
os.makedirs(config.DATA_DIR, exist_ok=True)
|
||
conn = get_conn()
|
||
conn.executescript(_SCHEMA)
|
||
# 迁移：旧库补充 full_text 列（存页面可读正文）
|
||
cols = [r["name"] for r in conn.execute("PRAGMA table_info(articles)").fetchall()]
|
||
if "full_text" not in cols:
|
||
conn.execute("ALTER TABLE articles ADD COLUMN full_text TEXT DEFAULT ''")
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def now_str():
|
||
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||
|
||
|
||
# ---------------- settings ----------------
|
||
def get_setting(key, default=None):
|
||
conn = get_conn()
|
||
row = conn.execute("SELECT value FROM settings WHERE key=?", (key,)).fetchone()
|
||
conn.close()
|
||
if row is None:
|
||
return default
|
||
try:
|
||
return json.loads(row["value"])
|
||
except Exception:
|
||
return row["value"]
|
||
|
||
|
||
def set_setting(key, value):
|
||
conn = get_conn()
|
||
conn.execute(
|
||
"INSERT INTO settings(key,value) VALUES(?,?) "
|
||
"ON CONFLICT(key) DO UPDATE SET value=excluded.value",
|
||
(key, json.dumps(value, ensure_ascii=False)),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def get_all_settings():
|
||
conn = get_conn()
|
||
rows = conn.execute("SELECT key,value FROM settings").fetchall()
|
||
conn.close()
|
||
out = {}
|
||
for r in rows:
|
||
try:
|
||
out[r["key"]] = json.loads(r["value"])
|
||
except Exception:
|
||
out[r["key"]] = r["value"]
|
||
return out
|
||
|
||
|
||
# ---------------- sources ----------------
|
||
def list_sources(only_enabled=False):
|
||
conn = get_conn()
|
||
sql = "SELECT * FROM sources"
|
||
if only_enabled:
|
||
sql += " WHERE enabled=1"
|
||
sql += " ORDER BY id"
|
||
rows = conn.execute(sql).fetchall()
|
||
conn.close()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
def get_source(sid):
|
||
conn = get_conn()
|
||
row = conn.execute("SELECT * FROM sources WHERE id=?", (sid,)).fetchone()
|
||
conn.close()
|
||
return dict(row) if row else None
|
||
|
||
|
||
def add_source(name, type_, url, desc, weight=1.0):
|
||
conn = get_conn()
|
||
cur = conn.execute(
|
||
"INSERT INTO sources(name,type,url,description,weight,created_at) VALUES(?,?,?,?,?,?)",
|
||
(name, type_, url, desc, weight, now_str()),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
return cur.lastrowid
|
||
|
||
|
||
def update_source(sid, **fields):
|
||
allowed = {"name", "type", "url", "description", "weight", "enabled"}
|
||
sets, vals = [], []
|
||
for k, v in fields.items():
|
||
if k in allowed:
|
||
sets.append(f"{k}=?")
|
||
vals.append(v)
|
||
if not sets:
|
||
return
|
||
vals.append(sid)
|
||
conn = get_conn()
|
||
conn.execute(f"UPDATE sources SET {','.join(sets)} WHERE id=?", vals)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def delete_source(sid):
|
||
conn = get_conn()
|
||
conn.execute("DELETE FROM sources WHERE id=?", (sid,))
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def update_source_fetch(sid, status="ok", count=0):
|
||
conn = get_conn()
|
||
conn.execute(
|
||
"UPDATE sources SET status=?, last_fetch=?, last_count=? WHERE id=?",
|
||
(status, now_str(), count, sid),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
# ---------------- articles ----------------
|
||
def article_exists(url):
|
||
conn = get_conn()
|
||
row = conn.execute("SELECT id FROM articles WHERE url=?", (url,)).fetchone()
|
||
conn.close()
|
||
return row is not None
|
||
|
||
|
||
def add_article(a):
|
||
conn = get_conn()
|
||
cur = conn.execute(
|
||
"""INSERT INTO articles(source_id,title,url,author,content,summary,domain,entities,
|
||
importance,relevance,total_score,is_important,analysis,llm_status,status,
|
||
published_at,collected_at,full_text)
|
||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) """,
|
||
(a.get("source_id", 0), a["title"], a.get("url", ""), a.get("author", ""),
|
||
a.get("content", ""), a.get("summary", ""), a.get("domain", ""),
|
||
json.dumps(a.get("entities", []), ensure_ascii=False),
|
||
a.get("importance", 0), a.get("relevance", 0), a.get("total_score", 0),
|
||
a.get("is_important", 0), a.get("analysis", ""), a.get("llm_status", "pending"),
|
||
a.get("status", "new"), a.get("published_at", ""), now_str(),
|
||
a.get("full_text", "")),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
return cur.lastrowid
|
||
|
||
|
||
def update_article(aid, **fields):
|
||
allowed = {"summary", "domain", "entities", "importance", "relevance",
|
||
"total_score", "is_important", "analysis", "llm_status", "notified", "status",
|
||
"full_text", "content", "title", "url"}
|
||
sets, vals = [], []
|
||
for k, v in fields.items():
|
||
if k in allowed:
|
||
if k == "entities":
|
||
v = json.dumps(v, ensure_ascii=False)
|
||
sets.append(f"{k}=?")
|
||
vals.append(v)
|
||
if not sets:
|
||
return
|
||
vals.append(aid)
|
||
conn = get_conn()
|
||
conn.execute(f"UPDATE articles SET {','.join(sets)} WHERE id=?", vals)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def get_article(aid):
|
||
conn = get_conn()
|
||
row = conn.execute(
|
||
"SELECT a.*, s.name AS source_name FROM articles a "
|
||
"LEFT JOIN sources s ON a.source_id=s.id WHERE a.id=?", (aid,)).fetchone()
|
||
conn.close()
|
||
if not row:
|
||
return None
|
||
d = dict(row)
|
||
try:
|
||
d["entities"] = json.loads(d["entities"] or "[]")
|
||
except Exception:
|
||
d["entities"] = []
|
||
return d
|
||
|
||
|
||
def list_articles(**filters):
|
||
"""支持 source_id/domain/is_important/llm_status/min_score/q/date_from/order/limit/offset"""
|
||
sql = ("SELECT a.*, s.name AS source_name FROM articles a "
|
||
"LEFT JOIN sources s ON a.source_id=s.id WHERE 1=1")
|
||
args = []
|
||
if filters.get("source_id"):
|
||
sql += " AND a.source_id=?"
|
||
args.append(filters["source_id"])
|
||
if filters.get("domain"):
|
||
sql += " AND a.domain=?"
|
||
args.append(filters["domain"])
|
||
if filters.get("is_important") is not None:
|
||
sql += " AND a.is_important=?"
|
||
args.append(1 if filters["is_important"] else 0)
|
||
if filters.get("llm_status"):
|
||
sql += " AND a.llm_status=?"
|
||
args.append(filters["llm_status"])
|
||
if filters.get("min_score"):
|
||
sql += " AND a.total_score>=?"
|
||
args.append(filters["min_score"])
|
||
if filters.get("q"):
|
||
sql += " AND (a.title LIKE ? OR a.content LIKE ?)"
|
||
args += [f"%{filters['q']}%", f"%{filters['q']}%"]
|
||
if filters.get("date_from"):
|
||
sql += " AND a.published_at>=?"
|
||
args.append(filters["date_from"])
|
||
order = filters.get("order", "a.collected_at DESC")
|
||
sql += f" ORDER BY {order}"
|
||
if filters.get("limit"):
|
||
sql += " LIMIT ?"
|
||
args.append(filters["limit"])
|
||
if filters.get("offset"):
|
||
sql += " OFFSET ?"
|
||
args.append(filters["offset"])
|
||
conn = get_conn()
|
||
rows = conn.execute(sql, args).fetchall()
|
||
conn.close()
|
||
out = []
|
||
for r in rows:
|
||
d = dict(r)
|
||
try:
|
||
d["entities"] = json.loads(d["entities"] or "[]")
|
||
except Exception:
|
||
d["entities"] = []
|
||
out.append(d)
|
||
return out
|
||
|
||
|
||
def count_articles(**filters):
|
||
sql = "SELECT COUNT(*) c FROM articles a WHERE 1=1"
|
||
args = []
|
||
if filters.get("domain"):
|
||
sql += " AND a.domain=?"
|
||
args.append(filters["domain"])
|
||
if filters.get("is_important") is not None:
|
||
sql += " AND a.is_important=?"
|
||
args.append(1 if filters["is_important"] else 0)
|
||
conn = get_conn()
|
||
c = conn.execute(sql, args).fetchone()["c"]
|
||
conn.close()
|
||
return c
|
||
|
||
|
||
def article_stats():
|
||
conn = get_conn()
|
||
total = conn.execute("SELECT COUNT(*) c FROM articles").fetchone()["c"]
|
||
important = conn.execute("SELECT COUNT(*) c FROM articles WHERE is_important=1").fetchone()["c"]
|
||
pending_llm = conn.execute("SELECT COUNT(*) c FROM articles WHERE llm_status='pending'").fetchone()["c"]
|
||
notified = conn.execute("SELECT COUNT(*) c FROM articles WHERE notified=1").fetchone()["c"]
|
||
conn.close()
|
||
return {"total": total, "important": important, "pending_llm": pending_llm, "notified": notified}
|
||
|
||
|
||
def latest_articles_for_summary(window_hours=24):
|
||
"""汇总窗口内（含未通知、未汇总的）资讯"""
|
||
conn = get_conn()
|
||
rows = conn.execute(
|
||
"SELECT a.*, s.name AS source_name FROM articles a "
|
||
"LEFT JOIN sources s ON a.source_id=s.id "
|
||
"WHERE a.published_at >= datetime('now','localtime','-%d hours') "
|
||
"AND a.total_score>=50 ORDER BY a.total_score DESC LIMIT 200" % int(window_hours)
|
||
).fetchall()
|
||
conn.close()
|
||
out = []
|
||
for r in rows:
|
||
d = dict(r)
|
||
try:
|
||
d["entities"] = json.loads(d["entities"] or "[]")
|
||
except Exception:
|
||
d["entities"] = []
|
||
out.append(d)
|
||
return out
|
||
|
||
|
||
def pending_llm_articles(limit=10):
|
||
conn = get_conn()
|
||
rows = conn.execute(
|
||
"SELECT a.*, s.name AS source_name FROM articles a "
|
||
"LEFT JOIN sources s ON a.source_id=s.id "
|
||
"WHERE a.llm_status='pending' ORDER BY a.total_score DESC LIMIT ?", (limit,)
|
||
).fetchall()
|
||
conn.close()
|
||
out = []
|
||
for r in rows:
|
||
d = dict(r)
|
||
try:
|
||
d["entities"] = json.loads(d["entities"] or "[]")
|
||
except Exception:
|
||
d["entities"] = []
|
||
out.append(d)
|
||
return out
|
||
|
||
|
||
# ---------------- 兴趣画像 ----------------
|
||
def list_keywords():
|
||
conn = get_conn()
|
||
rows = conn.execute("SELECT * FROM keywords ORDER BY weight DESC").fetchall()
|
||
conn.close()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
def add_keyword(kw, weight=5):
|
||
conn = get_conn()
|
||
conn.execute("INSERT OR IGNORE INTO keywords(keyword,weight) VALUES(?,?)", (kw, weight))
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def delete_keyword(kid):
|
||
conn = get_conn()
|
||
conn.execute("DELETE FROM keywords WHERE id=?", (kid,))
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def list_domains():
|
||
conn = get_conn()
|
||
rows = conn.execute("SELECT * FROM domains ORDER BY weight DESC").fetchall()
|
||
conn.close()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
def add_domain(name, weight=5):
|
||
conn = get_conn()
|
||
conn.execute("INSERT OR IGNORE INTO domains(name,weight) VALUES(?,?)", (name, weight))
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def delete_domain(did):
|
||
conn = get_conn()
|
||
conn.execute("DELETE FROM domains WHERE id=?", (did,))
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def list_companies():
|
||
conn = get_conn()
|
||
rows = conn.execute("SELECT * FROM companies ORDER BY id").fetchall()
|
||
conn.close()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
def add_company(name):
|
||
conn = get_conn()
|
||
conn.execute("INSERT OR IGNORE INTO companies(name) VALUES(?)", (name,))
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def delete_company(cid):
|
||
conn = get_conn()
|
||
conn.execute("DELETE FROM companies WHERE id=?", (cid,))
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
# ---------------- 大模型接口（llm_providers） ----------------
|
||
def list_providers():
|
||
conn = get_conn()
|
||
rows = conn.execute("SELECT * FROM llm_providers ORDER BY active DESC, id").fetchall()
|
||
conn.close()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
def get_provider(pid):
|
||
conn = get_conn()
|
||
row = conn.execute("SELECT * FROM llm_providers WHERE id=?", (pid,)).fetchone()
|
||
conn.close()
|
||
return dict(row) if row else None
|
||
|
||
|
||
def add_provider(name, base_url, api_key, model, active=0, enabled=1):
|
||
conn = get_conn()
|
||
cur = conn.execute(
|
||
"INSERT INTO llm_providers(name,base_url,api_key,model,active,enabled,created_at) "
|
||
"VALUES(?,?,?,?,?,?,?)",
|
||
(name, base_url, api_key, model, active, enabled, now_str()),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
return cur.lastrowid
|
||
|
||
|
||
def update_provider(pid, **fields):
|
||
allowed = {"name", "base_url", "api_key", "model", "active", "enabled"}
|
||
sets, vals = [], []
|
||
for k, v in fields.items():
|
||
if k in allowed:
|
||
sets.append(f"{k}=?")
|
||
vals.append(v)
|
||
if not sets:
|
||
return
|
||
vals.append(pid)
|
||
conn = get_conn()
|
||
conn.execute(f"UPDATE llm_providers SET {','.join(sets)} WHERE id=?", vals)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def delete_provider(pid):
|
||
conn = get_conn()
|
||
conn.execute("DELETE FROM llm_providers WHERE id=?", (pid,))
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def get_active_provider():
|
||
conn = get_conn()
|
||
row = conn.execute("SELECT * FROM llm_providers WHERE active=1 AND enabled=1 LIMIT 1").fetchone()
|
||
conn.close()
|
||
return dict(row) if row else None
|
||
|
||
|
||
def set_active_provider(pid):
|
||
conn = get_conn()
|
||
conn.execute("UPDATE llm_providers SET active=0")
|
||
conn.execute("UPDATE llm_providers SET active=1, enabled=1 WHERE id=?", (pid,))
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def enabled_providers_except(pid):
|
||
conn = get_conn()
|
||
rows = conn.execute(
|
||
"SELECT * FROM llm_providers WHERE enabled=1 AND id!=? ORDER BY id", (pid,)).fetchall()
|
||
conn.close()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
# ---------------- 通知日志 ----------------
|
||
def add_log(type_, subject, count, article_ids, status="ok", detail=""):
|
||
conn = get_conn()
|
||
conn.execute(
|
||
"INSERT INTO notification_log(type,subject,count,article_ids,status,detail,sent_at) "
|
||
"VALUES(?,?,?,?,?,?,?)",
|
||
(type_, subject, count, json.dumps(article_ids, ensure_ascii=False), status, detail, now_str()),
|
||
)
|
||
conn.commit()
|
||
conn.close()
|
||
|
||
|
||
def list_logs(limit=50):
|
||
conn = get_conn()
|
||
rows = conn.execute(
|
||
"SELECT * FROM notification_log ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
|
||
conn.close()
|
||
out = []
|
||
for r in rows:
|
||
d = dict(r)
|
||
try:
|
||
d["article_ids"] = json.loads(d["article_ids"] or "[]")
|
||
except Exception:
|
||
d["article_ids"] = []
|
||
out.append(d)
|
||
return out
|
||
|
||
|
||
def clear_old_articles(days=30):
|
||
conn = get_conn()
|
||
conn.execute(
|
||
"DELETE FROM articles WHERE collected_at < datetime('now','localtime','-%d days')" % days)
|
||
conn.commit()
|
||
conn.close()
|