新增继续爬取: auto 任务达 max_pages 上限停止后, 有待爬缓存链接时可在详情页一键继续

- POST /api/tasks/<tid>/continue: 校验 auto 模式 + pending 非空, skip_seed 启动
- engine skip_seed: 跳过起始网址直接从缓存队列消费 (不重爬 seed, 不重复入 visited)
- 关键修复: max_pages 原为累计上限(visited>=max_pages), 已达标任务无法继续; 继续模式改为按本次新增页数重新计算上限, 可反复继续直到缓存耗尽
- 详情弹窗 auto 任务且有缓存时显示 [⏩ 继续爬取（缓存 N 条）] 按钮
- 端到端测试: max_pages=2 链路 执行(seed+p1,缓存4) → 继续(p2,p3,缓存2) → 继续(p4,p5,缓存0) → 无缓存报错 ✅

.gitignore

@@ -4,3 +4,4 @@ data/*.json
 data/cookies_*.json
 logs/
 out/
+data/exports/

app.py

@@ -104,13 +104,16 @@ def persist_cb(task_id, run):
         print(f"[db] 同步失败: {e}", flush=True)
 
 
-def start_run(task):
+def start_run(task, skip_seed=False):
-    """为任务启动一次爬取, 返回 (run, error)"""
+    """为任务启动一次爬取, 返回 (run, error)
+    skip_seed: auto 任务继续爬取模式, 跳过起始网址直接从缓存队列消费
+    """
     with JOBS_LOCK:
         job = JOBS.get(task["id"])
         if job and job.is_running():
             return None, "该任务已有正在运行的爬取"
         run = make_run(task)
+        run["skip_seed"] = bool(skip_seed)
         store.add_run(task["id"], run)
         job = CrawlJob(task, run, persist_cb)
         JOBS[task["id"]] = job
@@ -424,6 +427,23 @@ def api_trash_clear():
 
 # ---------------- API: 运行控制 ----------------
 
+@app.route("/api/tasks/<tid>/continue", methods=["POST"])
+def api_continue(tid):
+    """继续爬取: auto 任务从待爬缓存队列接着爬 (跳过起始网址, 保留已爬集合)"""
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    if task.get("mode") != "auto":
+        return jsonify({"error": "仅自动爬取任务支持继续爬取"}), 400
+    st = store.load_auto_state(tid, task)
+    if not st.get("pending"):
+        return jsonify({"error": "没有待爬缓存链接，无需继续"}), 400
+    run, err = start_run(task, skip_seed=True)
+    if err:
+        return jsonify({"error": err}), 409
+    return jsonify(run)
+
+
 @app.route("/api/tasks/<tid>/start", methods=["POST"])
 def api_start(tid):
     task = store.get_task(tid)

data/auto_state/t_99e73a95c81f.json

@@ -0,0 +1,11 @@
+{
+  "pending": [],
+  "visited": [
+    "http://127.0.0.1:38081/p2",
+    "http://127.0.0.1:38081/p5",
+    "http://127.0.0.1:38081/p3",
+    "http://127.0.0.1:38081/",
+    "http://127.0.0.1:38081/p1",
+    "http://127.0.0.1:38081/p4"
+  ]
+}

engine.py

@@ -598,9 +598,10 @@ class CrawlJob:
     def _crawl_auto(self):
         """自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到上限
         - max_depth: 0=无限制, N=只爬 N 层
-        - max_pages: 0=无限制, N=安全上限
+        - max_pages: 0=无限制, N=安全上限 (继续爬取模式按本次新增页数重新计算)
         - 提取但未爬取的链接持久化到 auto.pending, 已爬集合持久化到 auto.visited
         - 停止后再次运行从缓存队列继续爬; 起始网址每次运行都重新爬(不去重)
+        - skip_seed(继续爬取): 跳过起始网址直接消费缓存队列, 页数上限按本次新增重新计算
         """
         run = self.run
         auto = self.task.get("auto", {})
@@ -615,8 +616,13 @@ class CrawlJob:
         state = store.load_auto_state(self.task["id"], self.task)
         visited = set(state.get("visited", []) or [])
         pending = state.get("pending", []) or []
+        visited_base = len(visited)
         # 起始网址每次运行都爬(不做去重), 缓存队列继续消费
-        queue = [(seed, 0, "")]
+        # 继续爬取模式(skip_seed): 有缓存队列时跳过起始网址, 直接从待爬队列接着爬
+        skip_seed = bool(self.run.get("skip_seed"))
+        queue = []
+        if not (skip_seed and pending):
+            queue.append((seed, 0, ""))
         if pending:
             queue.extend((p["url"], p.get("depth", 0), p.get("source", "")) for p in pending)
         queued = set(visited)
@@ -630,9 +636,15 @@ class CrawlJob:
         try:
             while queue and not self._stop.is_set():
                 self._wait_if_paused()
-                if max_pages > 0 and len(visited) >= max_pages:
+                if max_pages > 0:
-                    self._log("info", f"达到最大页数上限 {max_pages}, 停止")
+                    if skip_seed:
-                    break
+                        # 继续爬取模式: 页数上限按本次新增页数重新计算 (累计 visited 不阻塞继续)
+                        if len(visited) - visited_base >= max_pages:
+                            self._log("info", f"本次继续爬取达到页数上限 {max_pages}, 停止")
+                            break
+                    elif len(visited) >= max_pages:
+                        self._log("info", f"达到最大页数上限 {max_pages}, 停止")
+                        break
                 url, depth, src = queue.pop(0)
                 key = normalize_url(url)
                 if key in visited and url != seed:     # 起始网址不去重, 其余已爬跳过

static/app.js

@@ -602,7 +602,10 @@ function renderDetail() {
       ${t.running ? `
         <button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
         <button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
-      : `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`}
+      : `
+        <button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>
+        ${t.mode === "auto" && (t.auto_pending_count || 0) > 0 ? `
+          <button class="btn" onclick="contTask('${t.id}')">⏩ 继续爬取（缓存 ${t.auto_pending_count} 条）</button>` : ""}`}
       ${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>
         <button class="btn" ${t.running ? "disabled" : ""} onclick="clearCache('${t.id}')">🧹 清空缓存</button>` : ""}
       <button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
@@ -734,6 +737,15 @@ function stopLogPoll() {
   if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; }
 }
 
+async function contTask(tid) {
+  try {
+    const r = await api(`/api/tasks/${tid}/continue`, { method: "POST" });
+    toast("已开始继续爬取缓存队列");
+    loadTasks();
+    if (state.detail.task && state.detail.task.id === tid) openDetail(tid);
+  } catch (e) { toast(e.message, true); }
+}
+
 /* ---------------- 打包导出 ---------------- */
 async function exportZip(tid) {
   const btn = window.event && window.event.target;