Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
143a3d5c89 |
@@ -778,10 +778,15 @@ class CrawlJob:
|
|||||||
finally:
|
finally:
|
||||||
self._close_browser()
|
self._close_browser()
|
||||||
# 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
|
# 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
|
||||||
|
# 保存前应用当前排除规则过滤, 避免运行中配置的 exclude 被内存快照覆盖
|
||||||
try:
|
try:
|
||||||
|
auto_cfg = self.task.get("auto") or {}
|
||||||
|
keep = [(u, d, s) for u, d, s in queue if not url_excluded(u, auto_cfg)]
|
||||||
|
if len(keep) != len(queue):
|
||||||
|
self._log("info", f"保存状态时按排除规则过滤 {len(queue) - len(keep)} 条")
|
||||||
store.save_auto_state(self.task["id"], {
|
store.save_auto_state(self.task["id"], {
|
||||||
"pending": [
|
"pending": [
|
||||||
{"url": u, "depth": d, "source": s} for u, d, s in queue],
|
{"url": u, "depth": d, "source": s} for u, d, s in keep],
|
||||||
"visited": list(visited),
|
"visited": list(visited),
|
||||||
})
|
})
|
||||||
db.upsert_task_async(self.task)
|
db.upsert_task_async(self.task)
|
||||||
|
|||||||
Reference in New Issue
Block a user