Compare commits

..
1 Commits
+6 -1
View File
@@ -778,10 +778,15 @@ class CrawlJob:
finally:
self._close_browser()
# 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
# 保存前应用当前排除规则过滤, 避免运行中配置的 exclude 被内存快照覆盖
try:
auto_cfg = self.task.get("auto") or {}
keep = [(u, d, s) for u, d, s in queue if not url_excluded(u, auto_cfg)]
if len(keep) != len(queue):
self._log("info", f"保存状态时按排除规则过滤 {len(queue) - len(keep)}")
store.save_auto_state(self.task["id"], {
"pending": [
{"url": u, "depth": d, "source": s} for u, d, s in queue],
{"url": u, "depth": d, "source": s} for u, d, s in keep],
"visited": list(visited),
})
db.upsert_task_async(self.task)