auto 任务运行中实时显示已爬/待爬缓存数; 人工停止后继续爬取验证

- engine: CrawlJob 增加 _auto_visited/_auto_pending 运行中实时状态 (含发现新链接入队后刷新)
- app.py 详情接口: 任务运行中优先读内存实时值, 不再显示上次运行结束的旧数字
- 端到端验证: 人工停止(已爬3,缓存18) → 继续爬取(跳过seed,爬完18页,缓存0) 
- 运行中采样: visited 1→2 实时增长, pending 15→14 同步递减 
This commit is contained in:
2026-08-12 10:19:19 +08:00
parent d9c9f0c633
commit 32eebf3dd1
2 changed files with 19 additions and 3 deletions
+8
View File
@@ -226,6 +226,9 @@ class CrawlJob:
self.persist = persist # callable(task_id, run)
self._stop = threading.Event()
self._pause = threading.Event()
# 运行中的 auto 实时状态 (供详情接口读取; 任务结束时由状态文件兜底)
self._auto_visited = None
self._auto_pending = None
self._cfg_lock = threading.RLock()
self.thread = None
@@ -630,6 +633,8 @@ class CrawlJob:
queued.add(normalize_url(u))
run["progress"]["total"] = len(queue)
self._auto_visited = len(visited)
self._auto_pending = len(queue)
self._persist()
p, browser, ctx, page, cookie_file = self._open_browser()
@@ -650,6 +655,8 @@ class CrawlJob:
if key in visited and url != seed: # 起始网址不去重, 其余已爬跳过
continue
visited.add(key)
self._auto_visited = len(visited)
self._auto_pending = len(queue)
idx = len(visited)
run["progress"]["current_url"] = url
run["progress"]["done"] = len(visited)
@@ -666,6 +673,7 @@ class CrawlJob:
if lk not in visited and lk not in queued:
queued.add(lk)
queue.append((link, depth + 1, url))
self._auto_pending = len(queue) # 新链接入队后实时刷新
if entry["status"] == "OK":
self._delay()
run["progress"]["total"] = len(visited)