Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
143a3d5c89 |
@@ -778,10 +778,15 @@ class CrawlJob:
|
||||
finally:
|
||||
self._close_browser()
|
||||
# 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
|
||||
# 保存前应用当前排除规则过滤, 避免运行中配置的 exclude 被内存快照覆盖
|
||||
try:
|
||||
auto_cfg = self.task.get("auto") or {}
|
||||
keep = [(u, d, s) for u, d, s in queue if not url_excluded(u, auto_cfg)]
|
||||
if len(keep) != len(queue):
|
||||
self._log("info", f"保存状态时按排除规则过滤 {len(queue) - len(keep)} 条")
|
||||
store.save_auto_state(self.task["id"], {
|
||||
"pending": [
|
||||
{"url": u, "depth": d, "source": s} for u, d, s in queue],
|
||||
{"url": u, "depth": d, "source": s} for u, d, s in keep],
|
||||
"visited": list(visited),
|
||||
})
|
||||
db.upsert_task_async(self.task)
|
||||
|
||||
Reference in New Issue
Block a user