v1.0.11: 新增历史记录回填脚本 backfill.py(幂等, 支持指定任务), 补齐数据库功能上线前的爬取记录
This commit is contained in:
@@ -262,3 +262,30 @@ def sync_run(run, persist):
|
||||
insert_results(run, results[synced:])
|
||||
run["_db_count"] = len(results)
|
||||
persist(run.get("task_id"), run)
|
||||
|
||||
|
||||
# ---------------- 历史数据回填 (幂等) ----------------
|
||||
|
||||
def sync_all_history(task_ids=None):
|
||||
"""把本地 JSON 中的历史任务/运行/结果全量回填数据库
|
||||
可重复执行 (INSERT IGNORE + 唯一键去重)
|
||||
task_ids: 指定只回填的任务ID列表, 默认全部
|
||||
返回统计 dict
|
||||
"""
|
||||
import store as _store
|
||||
stats = {"tasks": 0, "runs": 0, "results": 0}
|
||||
for task in _store.load_tasks():
|
||||
if task_ids and task["id"] not in task_ids:
|
||||
continue
|
||||
upsert_task(task)
|
||||
stats["tasks"] += 1
|
||||
for run in _store.get_runs(task["id"]):
|
||||
upsert_run(run)
|
||||
stats["runs"] += 1
|
||||
results = run.get("results", [])
|
||||
if results:
|
||||
insert_results(run, results)
|
||||
stats["results"] += len(results)
|
||||
run["_db_count"] = len(results)
|
||||
_store.save_run(task["id"], run) # 记录已同步标记, 避免运行中重复插入
|
||||
return stats
|
||||
Reference in New Issue
Block a user