v1.0.11: 新增历史记录回填脚本 backfill.py(幂等, 支持指定任务), 补齐数据库功能上线前的爬取记录

This commit is contained in:
2026-08-11 19:29:57 +08:00
parent abde505d32
commit bf6be47c01
2 changed files with 53 additions and 0 deletions
+27
View File
@@ -262,3 +262,30 @@ def sync_run(run, persist):
insert_results(run, results[synced:])
run["_db_count"] = len(results)
persist(run.get("task_id"), run)
# ---------------- 历史数据回填 (幂等) ----------------
def sync_all_history(task_ids=None):
"""把本地 JSON 中的历史任务/运行/结果全量回填数据库
可重复执行 (INSERT IGNORE + 唯一键去重)
task_ids: 指定只回填的任务ID列表, 默认全部
返回统计 dict
"""
import store as _store
stats = {"tasks": 0, "runs": 0, "results": 0}
for task in _store.load_tasks():
if task_ids and task["id"] not in task_ids:
continue
upsert_task(task)
stats["tasks"] += 1
for run in _store.get_runs(task["id"]):
upsert_run(run)
stats["runs"] += 1
results = run.get("results", [])
if results:
insert_results(run, results)
stats["results"] += len(results)
run["_db_count"] = len(results)
_store.save_run(task["id"], run) # 记录已同步标记, 避免运行中重复插入
return stats