hz4th_coder/news-tracker
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v1.3.0: 每源独立采集周期 + 定制监控与新闻监控分离(独立配置/独立汇总) + 历史采样留档与提取API

Browse Source
This commit is contained in:
hz4th_coder
2026-08-29 19:29:28 +08:00
parent 2cad7a2976
commit de08530958
12 changed files with 489 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db.py
+80 -11
View File
@@ -20,6 +20,7 @@ CREATE TABLE IF NOT EXISTS sources (
weight REAL DEFAULT 1.0,
kind TEXT DEFAULT 'normal', -- normal 普通(按权重打分) / custom 定制监控(按推送标准LLM判断)
monitor_standard TEXT DEFAULT '', -- 定制监控：推送标准说明
scan_interval_min INTEGER DEFAULT 0, -- 本源采集间隔(分钟)，0=跟随所属机制的全局值
enabled INTEGER DEFAULT 1,
status TEXT DEFAULT 'ok', -- ok / error
last_fetch TEXT DEFAULT '',
@@ -27,6 +28,17 @@ CREATE TABLE IF NOT EXISTS sources (
created_at TEXT DEFAULT ''
);
CREATE TABLE IF NOT EXISTS source_snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_id INTEGER NOT NULL,
fetched_at TEXT DEFAULT '', -- 本次采样时间
count INTEGER DEFAULT 0, -- 本次采到条数
status TEXT DEFAULT 'ok', -- ok / error
detail TEXT DEFAULT '', -- 错误信息 / 备注
created_at TEXT DEFAULT ''
);
CREATE INDEX IF NOT EXISTS idx_snapshots_source ON source_snapshots(source_id, id);
CREATE TABLE IF NOT EXISTS articles (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_id INTEGER DEFAULT 0,
@@ -113,12 +125,25 @@ def init_db():
os.makedirs(config.DATA_DIR, exist_ok=True)
conn = get_conn()
conn.executescript(_SCHEMA)
# 迁移：旧库补充 sources.kind / monitor_standard（定制监控类型）
# 迁移：旧库补充 sources.kind / monitor_standard / scan_interval_min
cols = [r["name"] for r in conn.execute("PRAGMA table_info(sources)").fetchall()]
if "kind" not in cols:
conn.execute("ALTER TABLE sources ADD COLUMN kind TEXT DEFAULT 'normal'")
if "monitor_standard" not in cols:
conn.execute("ALTER TABLE sources ADD COLUMN monitor_standard TEXT DEFAULT ''")
if "scan_interval_min" not in cols:
conn.execute("ALTER TABLE sources ADD COLUMN scan_interval_min INTEGER DEFAULT 0")
# 迁移：历史采样表（每次采集留档）
conn.execute("""CREATE TABLE IF NOT EXISTS source_snapshots (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source_id INTEGER NOT NULL,
fetched_at TEXT DEFAULT '',
count INTEGER DEFAULT 0,
status TEXT DEFAULT 'ok',
detail TEXT DEFAULT '',
created_at TEXT DEFAULT ''
)""")
conn.execute("CREATE INDEX IF NOT EXISTS idx_snapshots_source ON source_snapshots(source_id, id)")
# 迁移：旧库补充 full_text 列（存页面可读正文）
cols = [r["name"] for r in conn.execute("PRAGMA table_info(articles)").fetchall()]
if "full_text" not in cols:
@@ -187,12 +212,12 @@ def get_source(sid):
return dict(row) if row else None
def add_source(name, type_, url, desc, weight=1.0, kind="normal", monitor_standard=""):
def add_source(name, type_, url, desc, weight=1.0, kind="normal", monitor_standard="", scan_interval_min=0):
conn = get_conn()
cur = conn.execute(
"INSERT INTO sources(name,type,url,description,weight,kind,monitor_standard,created_at) "
"VALUES(?,?,?,?,?,?,?,?)",
(name, type_, url, desc, weight, kind, monitor_standard, now_str()),
"INSERT INTO sources(name,type,url,description,weight,kind,monitor_standard,scan_interval_min,created_at) "
"VALUES(?,?,?,?,?,?,?,?,?)",
(name, type_, url, desc, weight, kind, monitor_standard, scan_interval_min, now_str()),
)
conn.commit()
conn.close()
@@ -200,7 +225,8 @@ def add_source(name, type_, url, desc, weight=1.0, kind="normal", monitor_standa
def update_source(sid, **fields):
allowed = {"name", "type", "url", "description", "weight", "kind", "monitor_standard", "enabled"}
allowed = {"name", "type", "url", "description", "weight", "kind", "monitor_standard",
"scan_interval_min", "enabled"}
sets, vals = [], []
for k, v in fields.items():
if k in allowed:
@@ -232,6 +258,28 @@ def update_source_fetch(sid, status="ok", count=0):
conn.close()
# ---------------- 历史采样（每次采集留档，可查看/自动流程提取） ----------------
def add_source_snapshot(source_id, count=0, status="ok", detail=""):
conn = get_conn()
conn.execute(
"INSERT INTO source_snapshots(source_id,fetched_at,count,status,detail,created_at) "
"VALUES(?,?,?,?,?,?)",
(source_id, now_str(), count, status, detail, now_str()),
)
conn.commit()
conn.close()
def list_source_snapshots(source_id, limit=50):
conn = get_conn()
rows = conn.execute(
"SELECT * FROM source_snapshots WHERE source_id=? ORDER BY id DESC LIMIT ?",
(source_id, limit),
).fetchall()
conn.close()
return [dict(r) for r in rows]
# ---------------- articles ----------------
def article_exists(url):
conn = get_conn()
@@ -370,15 +418,36 @@ def article_stats():
def latest_articles_for_summary(window_hours=24):
"""汇总窗口内（含未通知、未汇总的）资讯
普通源按 total_score>=50；定制监控命中推送标准(is_important=1)的也纳入，不看分数。"""
"""新闻机制汇总：普通源(normal)，窗口内 total_score>=50"""
conn = get_conn()
rows = conn.execute(
"SELECT a.*, s.name AS source_name FROM articles a "
"SELECT a.*, s.name AS source_name, s.kind AS source_kind FROM articles a "
"LEFT JOIN sources s ON a.source_id=s.id "
"WHERE a.published_at >= datetime('now','localtime','-%d hours') "
"AND (a.total_score>=50 OR (s.kind='custom' AND a.is_important=1)) "
"ORDER BY a.total_score DESC LIMIT 200" % int(window_hours)
"AND (s.kind IS NULL OR s.kind != 'custom') "
"AND a.total_score>=50 ORDER BY a.total_score DESC LIMIT 200" % int(window_hours)
).fetchall()
conn.close()
out = []
for r in rows:
d = dict(r)
try:
d["entities"] = json.loads(d["entities"] or "[]")
except Exception:
d["entities"] = []
out.append(d)
return out
def custom_articles_for_summary(window_hours=24):
"""定制监控汇总：定制源(custom)，窗口内命中推送标准(is_important=1)的资讯"""
conn = get_conn()
rows = conn.execute(
"SELECT a.*, s.name AS source_name, s.kind AS source_kind FROM articles a "
"LEFT JOIN sources s ON a.source_id=s.id "
"WHERE a.published_at >= datetime('now','localtime','-%d hours') "
"AND s.kind='custom' AND a.is_important=1 "
"ORDER BY a.published_at DESC, a.id DESC LIMIT 200" % int(window_hours)
).fetchall()
conn.close()
out = []