v1.4.3 任务结束保存状态前应用排除规则, 防止覆盖运行中配置的队列清理

- engine.url_excluded: 与 filter_links 同逻辑的排除判定辅助函数
- 保存 auto 配置时, 若 exclude 规则变化且任务未运行, 自动从 pending 队列
  剔除命中规则的 URL (visited 已爬标记保留)
- 排除输入框文案优化
- cnblogs-auto 已配置 exclude=[MyComments.html, OtherPosts.html, /comments], 清理 546 条用户中心页

app.py

@@ -15,7 +15,7 @@ from flask import Flask, jsonify, request, send_file, send_from_directory
 import store
 import db
 import notify
-from engine import CrawlJob, probe_links
+from engine import CrawlJob, probe_links, normalize_url, url_excluded
 from scheduler import Scheduler, cron_next, interval_delta
 
 HERE = os.path.dirname(os.path.abspath(__file__))
@@ -345,7 +345,22 @@ def api_update_task(tid):
             if running:
                 job.update_config(body["config"])       # 运行中热更新
         if "auto" in body and task.get("mode") == "auto":
+            old_exclude = task.get("auto", {}).get("exclude") or []
             task["auto"] = {**task.get("auto", {}), **body["auto"]}
+            new_exclude = task["auto"].get("exclude") or []
+            # 排除规则发生变化时, 同步清理待爬队列中已命中的 URL (已爬 visited 保留)
+            if new_exclude and new_exclude != old_exclude and not running:
+                st = store.load_auto_state(tid, task)
+                kept = [p for p in st.get("pending", [])
+                        if not url_excluded(p.get("url", ""), task["auto"])]
+                removed = len(st.get("pending", [])) - len(kept)
+                if removed:
+                    store.save_auto_state(tid, {
+                        "pending": kept,
+                        "visited": st.get("visited", []),
+                    })
+                    if "exclude" in body.get("auto", {}):
+                        task["auto"]["_cleaned"] = removed
         if "schedule" in body and task.get("mode") == "scheduled":
             sch = {**task.get("schedule", {}), **body["schedule"]}
             try:
@@ -435,6 +450,54 @@ def api_trash_clear():
 
 # ---------------- API: 运行控制 ----------------
 
+@app.route("/api/tasks/<tid>/retry-failed", methods=["POST"])
+def api_retry_failed(tid):
+    """重爬失败页: 提取指定 run (默认最新) 中失败的 URL, 从已爬集合解除标记并注入
+    待爬队列头部; 返回注入数量, 调用方可随后点「继续爬取」重爬这些页面"""
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    if task.get("mode") != "auto":
+        return jsonify({"error": "仅自动模式任务支持重爬失败页"}), 400
+    try:
+        body = request.get_json(force=True) or {}
+    except Exception:
+        body = {}
+    rid = request.args.get("run", "") or body.get("run", "")
+    runs = store.get_runs(tid)
+    cur = None
+    if rid:
+        cur = next((r for r in runs if r["id"] == rid), None)
+    else:
+        cur = runs[-1] if runs else None
+    if not cur:
+        return jsonify({"error": "运行记录不存在"}), 404
+    failed = [res.get("url") for res in cur.get("results", [])
+              if res.get("status") == "FAIL" and res.get("url")]
+    if not failed:
+        return jsonify({"error": "该运行记录没有失败页面", "injected": 0})
+    st = store.load_auto_state(tid, task)
+    visited = set(st.get("visited", []))
+    pending = st.get("pending", [])
+    pending_urls = {normalize_url(p.get("url", "")) for p in pending}
+    # 待重爬: 已爬过且不在待爬队列中的失败 URL (去重)
+    to_inject, seen = [], set()
+    for u in failed:
+        key = normalize_url(u)
+        if key in visited and key not in pending_urls and key not in seen:
+            seen.add(key)
+            to_inject.append({"url": u, "depth": 0, "source": "retry-failed"})
+    if not to_inject:
+        return jsonify({"error": "失败页面均已爬或已在待爬队列中", "injected": 0})
+    # 解除已爬标记
+    remove_keys = {normalize_url(u) for u in failed}
+    visited = {v for v in visited if normalize_url(v) not in remove_keys}
+    # 注入队列头部, 优先重爬
+    pending = to_inject + pending
+    store.save_auto_state(tid, {"pending": pending, "visited": sorted(visited)})
+    return jsonify({"injected": len(to_inject), "pending_total": len(pending)})
+
+
 @app.route("/api/tasks/<tid>/continue", methods=["POST"])
 def api_continue(tid):
     """继续爬取: auto 任务从待爬缓存队列接着爬 (跳过起始网址, 保留已爬集合)"""

engine.py

@@ -151,6 +151,21 @@ def normalize_url(url):
         return str(url)
 
 
+def url_excluded(url, auto=None):
+    """判断 URL 是否命中排除规则 (与 filter_links 的 exclude 判定逻辑一致, 供队列清理使用)"""
+    if not auto:
+        return False
+    exclude = auto.get("exclude") or []
+    if not exclude:
+        return False
+    use_regex = bool(auto.get("use_regex"))
+    s = str(url or "")
+    if use_regex:
+        return any(re.search(p, s) for p in exclude)
+    low = s.lower()
+    return any(p.lower() in low for p in exclude)
+
+
 def filter_links(hrefs, seed_url, include=None, exclude=None,
                  same_domain=True, use_regex=False):
     """按规则过滤链接, 返回 (included, excluded); excluded 含排除原因"""
@@ -763,10 +778,15 @@ class CrawlJob:
         finally:
             self._close_browser()
             # 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
+            # 保存前应用当前排除规则过滤, 避免运行中配置的 exclude 被内存快照覆盖
             try:
+                auto_cfg = self.task.get("auto") or {}
+                keep = [(u, d, s) for u, d, s in queue if not url_excluded(u, auto_cfg)]
+                if len(keep) != len(queue):
+                    self._log("info", f"保存状态时按排除规则过滤 {len(queue) - len(keep)} 条")
                 store.save_auto_state(self.task["id"], {
                     "pending": [
-                        {"url": u, "depth": d, "source": s} for u, d, s in queue],
+                        {"url": u, "depth": d, "source": s} for u, d, s in keep],
                     "visited": list(visited),
                 })
                 db.upsert_task_async(self.task)

static/app.js

@@ -618,6 +618,26 @@ function pageWindow(cur, pages, width) {
   return win;
 }
 
+/* 重爬失败页: 注入待爬队列并自动继续爬取 */
+async function retryFailed() {
+  const t = state.detail.task;
+  if (!t || !state.detail.runId) return;
+  if (!confirm("将把本次运行失败的页面重新加入待爬队列并立即继续爬取，确定？")) return;
+  try {
+    const d = await api(`/api/tasks/${t.id}/retry-failed`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ run: state.detail.runId }),
+    });
+    toast(`已注入 ${d.injected} 个失败页，开始继续爬取…`);
+    if ((d.injected || 0) > 0) {
+      await api(`/api/tasks/${t.id}/continue`, { method: "POST" });
+    }
+    loadTasks();
+    openDetail(t.id);
+  } catch (e) { toast(e.message, true); }
+}
+
 function renderDetail() {
   const t = state.detail.task;
   const runs = t.runs || [];
@@ -752,6 +772,8 @@ function renderRunPanel(t, run) {
       <span class="ok">✅ ${s.ok}</span>
       <span class="fail">❌ ${s.fail}</span>
       <span class="img">🖼️ ${s.images}</span>
+      ${t.mode === "auto" && (s.fail || 0) > 0 && run.status !== "running" && run.status !== "paused" ? `
+        <button class="btn sm" title="将本次运行失败的页面重新加入待爬队列并继续爬取" onclick="retryFailed()">🔄 重爬失败页 (${s.fail})</button>` : ""}
     </div>
     ${prog}
     <div class="card-line">当前: <b>${esc(run.progress.current_url || "")}</b></div>

static/index.html

@@ -126,7 +126,7 @@
             <div class="field"><label>包含规则（每行一个，子串或正则）</label>
               <textarea name="include" rows="3" placeholder="techpowerup.com/review&#10;/news/"></textarea></div>
             <div class="field"><label>排除规则</label>
-              <textarea name="exclude" rows="3" placeholder="login, signup, /tag/, /forum/"></textarea></div>
+              <textarea name="exclude" rows="3" placeholder="每行一个关键词，URL 含任一关键词即不爬取&#10;如: MyComments.html、OtherPosts.html、/comments&#10;勾选正则后按正则匹配"></textarea></div>
           </div>
           <div class="row2">
             <div class="field check"><label><input name="same_domain" type="checkbox" checked> 仅爬同域名</label></div>