auto 任务运行中实时显示已爬/待爬缓存数; 人工停止后继续爬取验证
- engine: CrawlJob 增加 _auto_visited/_auto_pending 运行中实时状态 (含发现新链接入队后刷新) - app.py 详情接口: 任务运行中优先读内存实时值, 不再显示上次运行结束的旧数字 - 端到端验证: 人工停止(已爬3,缓存18) → 继续爬取(跳过seed,爬完18页,缓存0) ✅ - 运行中采样: visited 1→2 实时增长, pending 15→14 同步递减 ✅
This commit is contained in:
@@ -226,6 +226,9 @@ class CrawlJob:
|
||||
self.persist = persist # callable(task_id, run)
|
||||
self._stop = threading.Event()
|
||||
self._pause = threading.Event()
|
||||
# 运行中的 auto 实时状态 (供详情接口读取; 任务结束时由状态文件兜底)
|
||||
self._auto_visited = None
|
||||
self._auto_pending = None
|
||||
self._cfg_lock = threading.RLock()
|
||||
self.thread = None
|
||||
|
||||
@@ -630,6 +633,8 @@ class CrawlJob:
|
||||
queued.add(normalize_url(u))
|
||||
|
||||
run["progress"]["total"] = len(queue)
|
||||
self._auto_visited = len(visited)
|
||||
self._auto_pending = len(queue)
|
||||
self._persist()
|
||||
|
||||
p, browser, ctx, page, cookie_file = self._open_browser()
|
||||
@@ -650,6 +655,8 @@ class CrawlJob:
|
||||
if key in visited and url != seed: # 起始网址不去重, 其余已爬跳过
|
||||
continue
|
||||
visited.add(key)
|
||||
self._auto_visited = len(visited)
|
||||
self._auto_pending = len(queue)
|
||||
idx = len(visited)
|
||||
run["progress"]["current_url"] = url
|
||||
run["progress"]["done"] = len(visited)
|
||||
@@ -666,6 +673,7 @@ class CrawlJob:
|
||||
if lk not in visited and lk not in queued:
|
||||
queued.add(lk)
|
||||
queue.append((link, depth + 1, url))
|
||||
self._auto_pending = len(queue) # 新链接入队后实时刷新
|
||||
if entry["status"] == "OK":
|
||||
self._delay()
|
||||
run["progress"]["total"] = len(visited)
|
||||
|
||||
Reference in New Issue
Block a user