新增继续爬取: auto 任务达 max_pages 上限停止后, 有待爬缓存链接时可在详情页一键继续
- POST /api/tasks/<tid>/continue: 校验 auto 模式 + pending 非空, skip_seed 启动 - engine skip_seed: 跳过起始网址直接从缓存队列消费 (不重爬 seed, 不重复入 visited) - 关键修复: max_pages 原为累计上限(visited>=max_pages), 已达标任务无法继续; 继续模式改为按本次新增页数重新计算上限, 可反复继续直到缓存耗尽 - 详情弹窗 auto 任务且有缓存时显示 [⏩ 继续爬取（缓存 N 条）] 按钮 - 端到端测试: max_pages=2 链路 执行(seed+p1,缓存4) → 继续(p2,p3,缓存2) → 继续(p4,p5,缓存0) → 无缓存报错 ✅
This commit is contained in:
@@ -104,13 +104,16 @@ def persist_cb(task_id, run):
|
||||
print(f"[db] 同步失败: {e}", flush=True)
|
||||
|
||||
|
||||
def start_run(task):
|
||||
"""为任务启动一次爬取, 返回 (run, error)"""
|
||||
def start_run(task, skip_seed=False):
|
||||
"""为任务启动一次爬取, 返回 (run, error)
|
||||
skip_seed: auto 任务继续爬取模式, 跳过起始网址直接从缓存队列消费
|
||||
"""
|
||||
with JOBS_LOCK:
|
||||
job = JOBS.get(task["id"])
|
||||
if job and job.is_running():
|
||||
return None, "该任务已有正在运行的爬取"
|
||||
run = make_run(task)
|
||||
run["skip_seed"] = bool(skip_seed)
|
||||
store.add_run(task["id"], run)
|
||||
job = CrawlJob(task, run, persist_cb)
|
||||
JOBS[task["id"]] = job
|
||||
@@ -424,6 +427,23 @@ def api_trash_clear():
|
||||
|
||||
# ---------------- API: 运行控制 ----------------
|
||||
|
||||
@app.route("/api/tasks/<tid>/continue", methods=["POST"])
|
||||
def api_continue(tid):
|
||||
"""继续爬取: auto 任务从待爬缓存队列接着爬 (跳过起始网址, 保留已爬集合)"""
|
||||
task = store.get_task(tid)
|
||||
if not task:
|
||||
return jsonify({"error": "任务不存在"}), 404
|
||||
if task.get("mode") != "auto":
|
||||
return jsonify({"error": "仅自动爬取任务支持继续爬取"}), 400
|
||||
st = store.load_auto_state(tid, task)
|
||||
if not st.get("pending"):
|
||||
return jsonify({"error": "没有待爬缓存链接，无需继续"}), 400
|
||||
run, err = start_run(task, skip_seed=True)
|
||||
if err:
|
||||
return jsonify({"error": err}), 409
|
||||
return jsonify(run)
|
||||
|
||||
|
||||
@app.route("/api/tasks/<tid>/start", methods=["POST"])
|
||||
def api_start(tid):
|
||||
task = store.get_task(tid)
|
||||
|
||||
+501
-496
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"pending": [],
|
||||
"visited": [
|
||||
"http://127.0.0.1:38081/p2",
|
||||
"http://127.0.0.1:38081/p5",
|
||||
"http://127.0.0.1:38081/p3",
|
||||
"http://127.0.0.1:38081/",
|
||||
"http://127.0.0.1:38081/p1",
|
||||
"http://127.0.0.1:38081/p4"
|
||||
]
|
||||
}
|
||||
@@ -598,9 +598,10 @@ class CrawlJob:
|
||||
def _crawl_auto(self):
|
||||
"""自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到上限
|
||||
- max_depth: 0=无限制, N=只爬 N 层
|
||||
- max_pages: 0=无限制, N=安全上限
|
||||
- max_pages: 0=无限制, N=安全上限 (继续爬取模式按本次新增页数重新计算)
|
||||
- 提取但未爬取的链接持久化到 auto.pending, 已爬集合持久化到 auto.visited
|
||||
- 停止后再次运行从缓存队列继续爬; 起始网址每次运行都重新爬(不去重)
|
||||
- skip_seed(继续爬取): 跳过起始网址直接消费缓存队列, 页数上限按本次新增重新计算
|
||||
"""
|
||||
run = self.run
|
||||
auto = self.task.get("auto", {})
|
||||
@@ -615,8 +616,13 @@ class CrawlJob:
|
||||
state = store.load_auto_state(self.task["id"], self.task)
|
||||
visited = set(state.get("visited", []) or [])
|
||||
pending = state.get("pending", []) or []
|
||||
visited_base = len(visited)
|
||||
# 起始网址每次运行都爬(不做去重), 缓存队列继续消费
|
||||
queue = [(seed, 0, "")]
|
||||
# 继续爬取模式(skip_seed): 有缓存队列时跳过起始网址, 直接从待爬队列接着爬
|
||||
skip_seed = bool(self.run.get("skip_seed"))
|
||||
queue = []
|
||||
if not (skip_seed and pending):
|
||||
queue.append((seed, 0, ""))
|
||||
if pending:
|
||||
queue.extend((p["url"], p.get("depth", 0), p.get("source", "")) for p in pending)
|
||||
queued = set(visited)
|
||||
@@ -630,9 +636,15 @@ class CrawlJob:
|
||||
try:
|
||||
while queue and not self._stop.is_set():
|
||||
self._wait_if_paused()
|
||||
if max_pages > 0 and len(visited) >= max_pages:
|
||||
self._log("info", f"达到最大页数上限 {max_pages}, 停止")
|
||||
break
|
||||
if max_pages > 0:
|
||||
if skip_seed:
|
||||
# 继续爬取模式: 页数上限按本次新增页数重新计算 (累计 visited 不阻塞继续)
|
||||
if len(visited) - visited_base >= max_pages:
|
||||
self._log("info", f"本次继续爬取达到页数上限 {max_pages}, 停止")
|
||||
break
|
||||
elif len(visited) >= max_pages:
|
||||
self._log("info", f"达到最大页数上限 {max_pages}, 停止")
|
||||
break
|
||||
url, depth, src = queue.pop(0)
|
||||
key = normalize_url(url)
|
||||
if key in visited and url != seed: # 起始网址不去重, 其余已爬跳过
|
||||
|
||||
+13
-1
@@ -602,7 +602,10 @@ function renderDetail() {
|
||||
${t.running ? `
|
||||
<button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
|
||||
<button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
|
||||
: `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`}
|
||||
: `
|
||||
<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>
|
||||
${t.mode === "auto" && (t.auto_pending_count || 0) > 0 ? `
|
||||
<button class="btn" onclick="contTask('${t.id}')">⏩ 继续爬取（缓存 ${t.auto_pending_count} 条）</button>` : ""}`}
|
||||
${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>
|
||||
<button class="btn" ${t.running ? "disabled" : ""} onclick="clearCache('${t.id}')">🧹 清空缓存</button>` : ""}
|
||||
<button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
|
||||
@@ -734,6 +737,15 @@ function stopLogPoll() {
|
||||
if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; }
|
||||
}
|
||||
|
||||
async function contTask(tid) {
|
||||
try {
|
||||
const r = await api(`/api/tasks/${tid}/continue`, { method: "POST" });
|
||||
toast("已开始继续爬取缓存队列");
|
||||
loadTasks();
|
||||
if (state.detail.task && state.detail.task.id === tid) openDetail(tid);
|
||||
} catch (e) { toast(e.message, true); }
|
||||
}
|
||||
|
||||
/* ---------------- 打包导出 ---------------- */
|
||||
async function exportZip(tid) {
|
||||
const btn = window.event && window.event.target;
|
||||
|
||||
Reference in New Issue
Block a user