v1.4.2 屏蔽排除规则自动清理队列: 设置 exclude 后待爬队列同步移除命中 URL

- engine.url_excluded: 与 filter_links 同逻辑的排除判定辅助函数
- 保存 auto 配置时, 若 exclude 规则变化且任务未运行, 自动从 pending 队列
  剔除命中规则的 URL (visited 已爬标记保留)
- 排除输入框文案优化
- cnblogs-auto 已配置 exclude=[MyComments.html, OtherPosts.html, /comments], 清理 546 条用户中心页

.gitignore

@@ -4,3 +4,5 @@ data/*.json
 data/cookies_*.json
 logs/
 out/
+data/exports/
+data/auto_state/

app.py

@@ -4,15 +4,18 @@
 启动: /home/hz1/miniconda3/envs/openclaw/bin/python app.py  (默认端口 16062)
 """
 import os
+import re
 import threading
 import time
+import zipfile
 from datetime import datetime
 
 from flask import Flask, jsonify, request, send_file, send_from_directory
 
 import store
 import db
-from engine import CrawlJob, probe_links
+import notify
+from engine import CrawlJob, probe_links, normalize_url, url_excluded
 from scheduler import Scheduler, cron_next, interval_delta
 
 HERE = os.path.dirname(os.path.abspath(__file__))
@@ -101,13 +104,16 @@ def persist_cb(task_id, run):
         print(f"[db] 同步失败: {e}", flush=True)
 
 
-def start_run(task):
+def start_run(task, skip_seed=False):
-    """为任务启动一次爬取, 返回 (run, error)"""
+    """为任务启动一次爬取, 返回 (run, error)
+    skip_seed: auto 任务继续爬取模式, 跳过起始网址直接从缓存队列消费
+    """
     with JOBS_LOCK:
         job = JOBS.get(task["id"])
         if job and job.is_running():
             return None, "该任务已有正在运行的爬取"
         run = make_run(task)
+        run["skip_seed"] = bool(skip_seed)
         store.add_run(task["id"], run)
         job = CrawlJob(task, run, persist_cb)
         JOBS[task["id"]] = job
@@ -302,11 +308,19 @@ def api_task_detail(tid):
         result["run_page"] = page
         result["run_pages"] = pages
         result["run_page_size"] = page_size
-    # auto 状态数量 (独立文件)
+    # auto 状态数量: 运行中优先读内存实时值, 否则读状态文件
     if task.get("mode") == "auto":
+        live_v = live_p = None
+        with JOBS_LOCK:
+            job = JOBS.get(tid)
+            if job:
+                live_v = getattr(job, "_auto_visited", None)
+                live_p = getattr(job, "_auto_pending", None)
         st = store.load_auto_state(tid, task)
-        result["auto_pending_count"] = len(st.get("pending", []))
+        result["auto_pending_count"] = (
-        result["auto_visited_count"] = len(st.get("visited", []))
+            live_p if live_p is not None else len(st.get("pending", [])))
+        result["auto_visited_count"] = (
+            live_v if live_v is not None else len(st.get("visited", [])))
     return jsonify(result)
 
 
@@ -331,7 +345,22 @@ def api_update_task(tid):
             if running:
                 job.update_config(body["config"])       # 运行中热更新
         if "auto" in body and task.get("mode") == "auto":
+            old_exclude = task.get("auto", {}).get("exclude") or []
             task["auto"] = {**task.get("auto", {}), **body["auto"]}
+            new_exclude = task["auto"].get("exclude") or []
+            # 排除规则发生变化时, 同步清理待爬队列中已命中的 URL (已爬 visited 保留)
+            if new_exclude and new_exclude != old_exclude and not running:
+                st = store.load_auto_state(tid, task)
+                kept = [p for p in st.get("pending", [])
+                        if not url_excluded(p.get("url", ""), task["auto"])]
+                removed = len(st.get("pending", [])) - len(kept)
+                if removed:
+                    store.save_auto_state(tid, {
+                        "pending": kept,
+                        "visited": st.get("visited", []),
+                    })
+                    if "exclude" in body.get("auto", {}):
+                        task["auto"]["_cleaned"] = removed
         if "schedule" in body and task.get("mode") == "scheduled":
             sch = {**task.get("schedule", {}), **body["schedule"]}
             try:
@@ -421,6 +450,71 @@ def api_trash_clear():
 
 # ---------------- API: 运行控制 ----------------
 
+@app.route("/api/tasks/<tid>/retry-failed", methods=["POST"])
+def api_retry_failed(tid):
+    """重爬失败页: 提取指定 run (默认最新) 中失败的 URL, 从已爬集合解除标记并注入
+    待爬队列头部; 返回注入数量, 调用方可随后点「继续爬取」重爬这些页面"""
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    if task.get("mode") != "auto":
+        return jsonify({"error": "仅自动模式任务支持重爬失败页"}), 400
+    try:
+        body = request.get_json(force=True) or {}
+    except Exception:
+        body = {}
+    rid = request.args.get("run", "") or body.get("run", "")
+    runs = store.get_runs(tid)
+    cur = None
+    if rid:
+        cur = next((r for r in runs if r["id"] == rid), None)
+    else:
+        cur = runs[-1] if runs else None
+    if not cur:
+        return jsonify({"error": "运行记录不存在"}), 404
+    failed = [res.get("url") for res in cur.get("results", [])
+              if res.get("status") == "FAIL" and res.get("url")]
+    if not failed:
+        return jsonify({"error": "该运行记录没有失败页面", "injected": 0})
+    st = store.load_auto_state(tid, task)
+    visited = set(st.get("visited", []))
+    pending = st.get("pending", [])
+    pending_urls = {normalize_url(p.get("url", "")) for p in pending}
+    # 待重爬: 已爬过且不在待爬队列中的失败 URL (去重)
+    to_inject, seen = [], set()
+    for u in failed:
+        key = normalize_url(u)
+        if key in visited and key not in pending_urls and key not in seen:
+            seen.add(key)
+            to_inject.append({"url": u, "depth": 0, "source": "retry-failed"})
+    if not to_inject:
+        return jsonify({"error": "失败页面均已爬或已在待爬队列中", "injected": 0})
+    # 解除已爬标记
+    remove_keys = {normalize_url(u) for u in failed}
+    visited = {v for v in visited if normalize_url(v) not in remove_keys}
+    # 注入队列头部, 优先重爬
+    pending = to_inject + pending
+    store.save_auto_state(tid, {"pending": pending, "visited": sorted(visited)})
+    return jsonify({"injected": len(to_inject), "pending_total": len(pending)})
+
+
+@app.route("/api/tasks/<tid>/continue", methods=["POST"])
+def api_continue(tid):
+    """继续爬取: auto 任务从待爬缓存队列接着爬 (跳过起始网址, 保留已爬集合)"""
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    if task.get("mode") != "auto":
+        return jsonify({"error": "仅自动爬取任务支持继续爬取"}), 400
+    st = store.load_auto_state(tid, task)
+    if not st.get("pending"):
+        return jsonify({"error": "没有待爬缓存链接，无需继续"}), 400
+    run, err = start_run(task, skip_seed=True)
+    if err:
+        return jsonify({"error": err}), 409
+    return jsonify(run)
+
+
 @app.route("/api/tasks/<tid>/start", methods=["POST"])
 def api_start(tid):
     task = store.get_task(tid)
@@ -674,6 +768,103 @@ def api_file():
     return send_file(full)
 
 
+# ---------------- API: 打包导出 ----------------
+
+EXPORT_DIR = os.path.join(HERE, "data", "exports")
+EXPORT_KEEP = 10          # 最多保留的打包文件数
+
+
+def _safe_zip_name(task):
+    """任务名安全化为文件名 (保留中文, 去非法字符)"""
+    name = re.sub(r'[\\/:*?"<>|\s]+', "_", task.get("name", "")).strip("_")
+    return (name or task["id"])[:60]
+
+
+def _make_zip(task):
+    """把任务输出目录打包为 zip, 返回 (zip_path, err) 或 (None, 错误信息)"""
+    out_dir = os.path.realpath(resolve_out_dir(task))
+    if not os.path.isdir(out_dir):
+        return None, "输出目录不存在"
+    os.makedirs(EXPORT_DIR, exist_ok=True)
+    zip_path = os.path.join(EXPORT_DIR, f"{task['id']}_{int(time.time())}.zip")
+    prefix = _safe_zip_name(task) + "/"
+    count = 0
+    try:
+        with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+            for root, _dirs, files in os.walk(out_dir):
+                for f in files:
+                    full = os.path.join(root, f)
+                    rel = os.path.relpath(full, out_dir)
+                    zf.write(full, prefix + rel)
+                    count += 1
+    except Exception as e:
+        try:
+            os.remove(zip_path)
+        except OSError:
+            pass
+        return None, f"打包失败: {e}"
+    _cleanup_exports()
+    return zip_path, None
+
+
+def _cleanup_exports():
+    """清理旧的打包文件, 只保留最近 EXPORT_KEEP 个"""
+    try:
+        files = sorted(
+            (os.path.join(EXPORT_DIR, f) for f in os.listdir(EXPORT_DIR)
+             if f.endswith(".zip")),
+            key=os.path.getmtime, reverse=True,
+        )
+        for f in files[EXPORT_KEEP:]:
+            os.remove(f)
+    except Exception:
+        pass
+
+
+@app.route("/api/export")
+def api_export():
+    """打包任务输出目录为 zip 并提供下载"""
+    tid = request.args.get("task_id", "")
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    zip_path, err = _make_zip(task)
+    if err:
+        return jsonify({"error": err}), 400
+    return send_file(
+        zip_path, as_attachment=True,
+        download_name=_safe_zip_name(task) + ".zip",
+        mimetype="application/zip",
+    )
+
+
+@app.route("/api/export/email", methods=["POST"])
+def api_export_email():
+    """打包任务输出目录为 zip 并发送到指定邮箱 (默认 wlq@tphai.com)"""
+    body = request.get_json(force=True) or {}
+    tid = body.get("task_id", "")
+    email = (body.get("email") or "").strip() or "wlq@tphai.com"
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    if "@" not in email:
+        return jsonify({"error": "邮箱格式不正确"}), 400
+    zip_path, err = _make_zip(task)
+    if err:
+        return jsonify({"error": err}), 400
+    size_mb = round(os.path.getsize(zip_path) / 1048576, 1)
+    body_text = (f"项目: {task['name']}\n"
+                 f"输出目录: {resolve_out_dir(task)}\n"
+                 f"打包文件: {os.path.basename(zip_path)}\n"
+                 f"压缩包大小: {size_mb} MB\n\n"
+                 f"打包时间: {now_str()}")
+    ok, msg = notify.send_attachment(
+        f"[爬虫打包] {task['name']}", body_text, email, zip_path)
+    if not ok:
+        return jsonify({"error": f"邮件发送失败: {msg}"}), 500
+    return jsonify({"ok": True, "msg": f"已发送到 {email} (zip {size_mb} MB)"})
+
+
 # ---------------- 启动 ----------------
 
 scheduler = Scheduler(start_run)

engine.py

@@ -30,21 +30,60 @@ IMG_EXTS = (".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp")
 DEFAULT_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
               "(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36")
 
-_CHALLENGE_MARKS = [
+# title 中出现任一标记即判定反爬 (title 短小可靠, 不会误伤正文)
+_CHALLENGE_TITLE_MARKS = [
     "access denied", "403 forbidden", "just a moment", "attention required",
     "captcha", "bot check", "cf-challenge", "verify you are human",
     "checking your browser", "enable javascript and cookies",
 ]
 
+# html 中仅匹配强特征标记, 且只在 head 区域(前 20KB)搜索。
+# 不能全文匹配 "captcha"/"challenge" 等词——正常文章正文提到这些词会被误判为反爬。
+_CHALLENGE_HTML_MARKS = [
+    "cf-challenge", "challenge-platform", "cf-browser-verification",
+    "verify you are human", "checking your browser",
+    "enable javascript and cookies", "just a moment",
+]
+
+# 反爬拦截类错误信号: 命中后不再重试 (重试无意义且拖慢任务)
+_ANTI_CRAWL_MARKS = [
+    "403", "429", "503", "forbidden", "access denied", "too many requests",
+    "captcha", "cloudflare", "challenge", "verify you are human",
+    "just a moment", "blocked", "被反爬拦截",
+]
+
+
+def is_anti_crawl_error(err):
+    """判断错误是否属于反爬拦截 (此类失败重试也无法通过, 直接放弃该页)"""
+    s = str(err or "").lower()
+    return any(m in s for m in _ANTI_CRAWL_MARKS)
+
+
+_BROWSER_DEAD_MARKS = [
+    "browser has been closed", "target page, context or browser has been closed",
+    "has been disposed", "execution context was destroyed", "browser closed",
+    "page closed", "target closed", "crash",
+]
+
+
+def is_browser_dead_error(err):
+    """判断错误是否属于浏览器/页面失效 (需重启浏览器后重试, 重试同一 URL 才有意义)"""
+    s = str(err or "").lower()
+    return any(m in s for m in _BROWSER_DEAD_MARKS)
+
 
 def is_challenge_page(title, html):
-    """判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)"""
+    """判断是否仍在反爬验证页
-    low = html.lower()
+    - title 出现反爬关键词: 判定 (可靠, 反爬页 title 基本都会变)
+    - html 仅在前 20KB(head 区域)出现强特征标记时判定,
+      避免正文含 captcha/challenge 等词的正常页面被误杀
+    """
     t = (title or "").lower()
-    for mark in _CHALLENGE_MARKS:
+    for mark in _CHALLENGE_TITLE_MARKS:
-        if mark in t or mark in low:
+        if mark in t:
             return True
-    return False
+    low = (html or "").lower()[:20000]
+    return any(m in low for m in _CHALLENGE_HTML_MARKS)
 
 
 def safe_name(url, idx):
@@ -56,6 +95,7 @@ def safe_name(url, idx):
 def _settle_wait(page, timeout_s):
     """等待页面稳定(无 stop/pause 检查, 供试爬取等独立流程使用)"""
     last_title, stable = "", 0
+    challenge_hits = 0
     start = time.time()
     while time.time() - start < timeout_s:
         time.sleep(1)
@@ -65,8 +105,12 @@ def _settle_wait(page, timeout_s):
         except Exception:
             continue
         if is_challenge_page(title, html):
+            challenge_hits += 1
+            if challenge_hits >= 8:        # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
+                return True, title, html
             stable = 0
             continue
+        challenge_hits = 0
         if title == last_title:
             stable += 1
             if stable >= 2 and len(html) > 1000:
@@ -107,6 +151,21 @@ def normalize_url(url):
         return str(url)
 
 
+def url_excluded(url, auto=None):
+    """判断 URL 是否命中排除规则 (与 filter_links 的 exclude 判定逻辑一致, 供队列清理使用)"""
+    if not auto:
+        return False
+    exclude = auto.get("exclude") or []
+    if not exclude:
+        return False
+    use_regex = bool(auto.get("use_regex"))
+    s = str(url or "")
+    if use_regex:
+        return any(re.search(p, s) for p in exclude)
+    low = s.lower()
+    return any(p.lower() in low for p in exclude)
+
+
 def filter_links(hrefs, seed_url, include=None, exclude=None,
                  same_domain=True, use_regex=False):
     """按规则过滤链接, 返回 (included, excluded); excluded 含排除原因"""
@@ -208,8 +267,19 @@ class CrawlJob:
         self.persist = persist          # callable(task_id, run)
         self._stop = threading.Event()
         self._pause = threading.Event()
+        # 运行中的 auto 实时状态 (供详情接口读取; 任务结束时由状态文件兜底)
+        self._auto_visited = None
+        self._auto_pending = None
         self._cfg_lock = threading.RLock()
         self.thread = None
+        # 浏览器句柄 (p, browser, ctx, page, cookie_file); 崩溃后重建
+        self._browser = None
+
+    def _page(self):
+        return self._browser[3] if self._browser else None
+
+    def _ctx(self):
+        return self._browser[2] if self._browser else None
 
     # ---------------- 控制接口 ----------------
     def start(self):
@@ -288,9 +358,13 @@ class CrawlJob:
                 pass
         Stealth().apply_stealth_sync(ctx)
         page = ctx.new_page()
-        return p, browser, ctx, page, cookie_file
+        self._browser = (p, browser, ctx, page, cookie_file)
+        return self._browser
 
-    def _close_browser(self, p, browser, ctx, cookie_file):
+    def _close_browser(self):
+        if not self._browser:
+            return
+        p, browser, ctx, _page, cookie_file = self._browser
         try:
             json.dump(ctx.cookies(), open(cookie_file, "w"))
         except Exception:
@@ -303,9 +377,24 @@ class CrawlJob:
             p.stop()
         except Exception:
             pass
+        self._browser = None
+
+    def _reopen_browser(self):
+        """浏览器失效后重建: 保存 cookie -> 关闭旧实例 -> 启动新实例"""
+        self._close_browser()
+        time.sleep(1)
+        self._open_browser()
+        self._log("info", "浏览器已重启")
+
+    def _need_browser_restart(self, done_count):
+        """每爬 N 页主动重启一次浏览器, 防止长时间运行内存膨胀导致崩溃 (browser_max_pages=0 关闭)"""
+        limit = int(self._cfg("browser_max_pages", 0) or 0)
+        return limit > 0 and done_count > 1 and (done_count - 1) % limit == 0
 
     def _wait_page_settle(self, page, timeout_s):
         last_title, stable = "", 0
+        challenge_hits = 0
+        err_streak = 0                        # 连续读取失败计数
         start = time.time()
         while time.time() - start < timeout_s:
             if self._stop.is_set():
@@ -316,10 +405,18 @@ class CrawlJob:
                 title = page.title()
                 html = page.content()
             except Exception:
+                err_streak += 1
+                if err_streak >= 3:           # 页面/浏览器已失效, 提前失败而非干等到超时
+                    raise RuntimeError("Target page, context or browser has been closed")
                 continue                      # 正在跳转
+            err_streak = 0
             if is_challenge_page(title, html):
+                challenge_hits += 1
+                if challenge_hits >= 8:       # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
+                    return True, title, html
                 stable = 0
                 continue
+            challenge_hits = 0
             if title == last_title:
                 stable += 1
                 if stable >= 2 and len(html) > 1000:
@@ -435,7 +532,9 @@ class CrawlJob:
             "source_url": source_url, "depth": depth,
             "images": [], "attempts": 0,
         }
-        for attempt in range(retries + 1):
+        dead_strikes = 0                      # 浏览器失效重建次数 (防止无限重建)
+        attempt = 0
+        while attempt <= retries:
             if self._stop.is_set():
                 entry["error"] = "任务已终止"
                 break
@@ -459,7 +558,28 @@ class CrawlJob:
             except Exception as e:
                 entry["error"] = str(e)
                 entry["crawl_time"] = store.now_str()
+                if is_browser_dead_error(e):
+                    # 浏览器/页面失效: 重启浏览器后重试同一 URL, 不消耗重试次数
+                    dead_strikes += 1
+                    if dead_strikes > 3:
+                        entry["error"] = f"浏览器多次重启仍失效, 放弃: {e}"
+                        self._log("error", f"浏览器多次重启仍失效, 放弃 {url}")
+                        break
+                    self._log("warn", f"浏览器已失效({e}), 重启后重试: {url}")
+                    try:
+                        self._reopen_browser()
+                        page, ctx = self._page(), self._ctx()
+                    except Exception as re_err:
+                        entry["error"] = f"浏览器重启失败: {re_err}"
+                        self._log("error", f"浏览器重启失败, 放弃 {url}: {re_err}")
+                        break
+                    continue
                 self._log("warn", f"第{attempt + 1}次失败 {url}: {e}")
+                if is_anti_crawl_error(e):
+                    # 反爬拦截: 重试也过不去, 直接放弃, 不再消耗重试次数
+                    entry["error"] = f"反爬拦截, 跳过重试: {e}"
+                    self._log("warn", f"判定为反爬拦截, 放弃重试: {url}")
+                    break
                 if attempt < retries:
                     self._wait_if_paused()
                     t0 = time.time()
@@ -468,6 +588,7 @@ class CrawlJob:
                             break
                         self._wait_if_paused()
                         time.sleep(0.3)
+                attempt += 1
         self._write_page_meta(out_dir, base, entry)
         return entry
 
@@ -531,17 +652,20 @@ class CrawlJob:
         os.makedirs(out_dir, exist_ok=True)
         self._persist()
 
-        p, browser, ctx, page, cookie_file = self._open_browser()
+        self._open_browser()
         try:
             for i, url in enumerate(urls, 1):
                 if self._stop.is_set():
                     self._log("info", "收到终止信号, 停止爬取")
                     break
                 self._wait_if_paused()
+                if self._need_browser_restart(i):
+                    self._log("info", f"已爬 {i - 1} 页, 主动重启浏览器")
+                    self._reopen_browser()
                 run["progress"]["current_url"] = url
                 run["progress"]["done"] = i - 1
                 self._persist()
-                entry = self._retry_crawl(page, ctx, url, i, out_dir)
+                entry = self._retry_crawl(self._page(), self._ctx(), url, i, out_dir)
                 run["results"].append(entry)
                 self._bump_stats(entry)
                 run["progress"]["done"] = i
@@ -549,7 +673,7 @@ class CrawlJob:
                 if entry["status"] == "OK":
                     self._delay()
         finally:
-            self._close_browser(p, browser, ctx, cookie_file)
+            self._close_browser()
 
     def _discover_links(self, page):
         """从当前页面提取符合规则的链接"""
@@ -570,9 +694,10 @@ class CrawlJob:
     def _crawl_auto(self):
         """自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到上限
         - max_depth: 0=无限制, N=只爬 N 层
-        - max_pages: 0=无限制, N=安全上限
+        - max_pages: 0=无限制, N=安全上限 (继续爬取模式按本次新增页数重新计算)
         - 提取但未爬取的链接持久化到 auto.pending, 已爬集合持久化到 auto.visited
         - 停止后再次运行从缓存队列继续爬; 起始网址每次运行都重新爬(不去重)
+        - skip_seed(继续爬取): 跳过起始网址直接消费缓存队列, 页数上限按本次新增重新计算
         """
         run = self.run
         auto = self.task.get("auto", {})
@@ -587,8 +712,13 @@ class CrawlJob:
         state = store.load_auto_state(self.task["id"], self.task)
         visited = set(state.get("visited", []) or [])
         pending = state.get("pending", []) or []
+        visited_base = len(visited)
         # 起始网址每次运行都爬(不做去重), 缓存队列继续消费
-        queue = [(seed, 0, "")]
+        # 继续爬取模式(skip_seed): 有缓存队列时跳过起始网址, 直接从待爬队列接着爬
+        skip_seed = bool(self.run.get("skip_seed"))
+        queue = []
+        if not (skip_seed and pending):
+            queue.append((seed, 0, ""))
         if pending:
             queue.extend((p["url"], p.get("depth", 0), p.get("source", "")) for p in pending)
         queued = set(visited)
@@ -596,43 +726,57 @@ class CrawlJob:
             queued.add(normalize_url(u))
 
         run["progress"]["total"] = len(queue)
+        self._auto_visited = len(visited)
+        self._auto_pending = len(queue)
         self._persist()
 
-        p, browser, ctx, page, cookie_file = self._open_browser()
+        self._open_browser()
         try:
             while queue and not self._stop.is_set():
                 self._wait_if_paused()
-                if max_pages > 0 and len(visited) >= max_pages:
+                if max_pages > 0:
-                    self._log("info", f"达到最大页数上限 {max_pages}, 停止")
+                    if skip_seed:
-                    break
+                        # 继续爬取模式: 页数上限按本次新增页数重新计算 (累计 visited 不阻塞继续)
+                        if len(visited) - visited_base >= max_pages:
+                            self._log("info", f"本次继续爬取达到页数上限 {max_pages}, 停止")
+                            break
+                    elif len(visited) >= max_pages:
+                        self._log("info", f"达到最大页数上限 {max_pages}, 停止")
+                        break
+                if self._need_browser_restart(len(visited)):
+                    self._log("info", f"已爬 {len(visited) - 1} 页, 主动重启浏览器")
+                    self._reopen_browser()
                 url, depth, src = queue.pop(0)
                 key = normalize_url(url)
                 if key in visited and url != seed:     # 起始网址不去重, 其余已爬跳过
                     continue
                 visited.add(key)
+                self._auto_visited = len(visited)
+                self._auto_pending = len(queue)
                 idx = len(visited)
                 run["progress"]["current_url"] = url
                 run["progress"]["done"] = len(visited)
                 self._persist()
-                entry = self._retry_crawl(page, ctx, url, idx, out_dir,
+                entry = self._retry_crawl(self._page(), self._ctx(), url, idx, out_dir,
                                           source_url=src, depth=depth)
                 run["results"].append(entry)
                 self._bump_stats(entry)
                 self._persist()
                 # 无深度限制或未达深度限制时持续发现链接
                 if entry["status"] == "OK" and (max_depth == 0 or depth < max_depth):
-                    for link in self._discover_links(page):
+                    for link in self._discover_links(self._page()):
                         lk = normalize_url(link)
                         if lk not in visited and lk not in queued:
                             queued.add(lk)
                             queue.append((link, depth + 1, url))
+                    self._auto_pending = len(queue)   # 新链接入队后实时刷新
                 if entry["status"] == "OK":
                     self._delay()
             run["progress"]["total"] = len(visited)
             if not self._stop.is_set() and len(queue) == 0:
                 self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)")
         finally:
-            self._close_browser(p, browser, ctx, cookie_file)
+            self._close_browser()
             # 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
             try:
                 store.save_auto_state(self.task["id"], {

notify.py

@@ -33,11 +33,24 @@ def notify_email(task, run):
     if len(results) > 50:
         lines.append(f"  ... 共 {len(results)} 条")
     body = "\n".join(lines)
+    return send_attachment(f"[爬虫完成] {task['name']}", body, to)
+
+
+def send_attachment(subject, body, to, attach_path=None):
+    """发送带附件的邮件 (复用 send_email.py), 返回 (bool, msg)
+    attach_path: 附件文件路径, 可多个
+    """
+    if not os.path.exists(SEND_EMAIL):
+        return False, "send_email.py 不存在"
+    cmd = [sys.executable, SEND_EMAIL, subject, body, "--to", to]
+    if attach_path:
+        if isinstance(attach_path, str):
+            attach_path = [attach_path]
+        for p in attach_path:
+            if os.path.isfile(p):
+                cmd += ["--attach", p]
     try:
-        r = subprocess.run(
+        r = subprocess.run(cmd, timeout=600, capture_output=True)
-            [sys.executable, SEND_EMAIL, f"[爬虫完成] {task['name']}", body, "--to", to],
-            timeout=60, capture_output=True,
-        )
         return r.returncode == 0, (r.stderr or b"").decode(errors="ignore")[:200]
     except Exception as e:
         return False, str(e)

static/app.js

@@ -3,12 +3,13 @@
 
 const $ = (id) => document.getElementById(id);
 const MODE_LABEL = { batch: "批量", scheduled: "定时", auto: "自动" };
+const DETAIL_PAGE_SIZE = 100;   // 详情结果每页条数 (默认)
 
 const state = {
   tasks: [],
   editTask: null,          // 正在编辑的任务
   mode: "batch",           // 当前表单模式
-  detail: { task: null, runId: null, logOffset: 0, timer: null },
+  detail: { task: null, runId: null, logOffset: 0, timer: null, pageSize: DETAIL_PAGE_SIZE },
   logTimer: null,
 };
 let formDirty = false;      // 新建/编辑表单是否有未保存修改
@@ -522,7 +523,6 @@ async function submitForm(e) {
 }
 
 /* ---------------- 详情 ---------------- */
-const DETAIL_PAGE_SIZE = 100;   // 详情结果每页条数
 
 async function openDetail(tid) {
   try {
@@ -531,6 +531,7 @@ async function openDetail(tid) {
     const runs = t.runs || [];
     const cur = t.run;
     state.detail.runId = cur ? cur.id : (runs[0] ? runs[0].id : null);
+    state.detail.pageSize = (t.run && t.run.run_page_size) || DETAIL_PAGE_SIZE;
     state.detail.logOffset = 0;
     $("detailTitle").textContent = `任务详情 · ${t.name}`;
     renderDetail();
@@ -539,8 +540,9 @@ async function openDetail(tid) {
   } catch (e) { toast(e.message, true); }
 }
 
-async function loadRunPage(tid, rid, page) {
+async function loadRunPage(tid, rid, page, size) {
-  return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${DETAIL_PAGE_SIZE}`);
+  const ps = size || state.detail.pageSize || DETAIL_PAGE_SIZE;
+  return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${ps}`);
 }
 
 async function selectRun(rid) {
@@ -550,6 +552,7 @@ async function selectRun(rid) {
     const d = await loadRunPage(t.id, rid, 1);
     state.detail.task = d;
     state.detail.runId = d.run ? d.run.id : null;
+    state.detail.pageSize = (d.run && d.run.run_page_size) || DETAIL_PAGE_SIZE;
     state.detail.logOffset = 0;
     renderDetail();
     startLogPoll();
@@ -559,13 +562,82 @@ async function selectRun(rid) {
 async function goRunPage(p) {
   const t = state.detail.task;
   if (!t || !state.detail.runId) return;
+  const ps = (t.run && t.run.run_page_size) || state.detail.pageSize || DETAIL_PAGE_SIZE;
   try {
-    const d = await loadRunPage(t.id, state.detail.runId, p);
+    const d = await loadRunPage(t.id, state.detail.runId, p, ps);
     state.detail.task = d;
+    state.detail.pageSize = (d.run && d.run.run_page_size) || ps;
     renderDetail();
   } catch (e) { toast(e.message, true); }
 }
 
+/* 跳转到指定页码 (输入框回车/点击 GO) */
+function jumpRunPage() {
+  const t = state.detail.task;
+  if (!t || !state.detail.runId || !t.run) return;
+  const inp = $("pagerJump");
+  const pages = t.run.run_pages || 1;
+  let p = parseInt(inp.value, 10);
+  if (!p || isNaN(p)) p = 1;
+  p = Math.min(Math.max(p, 1), pages);
+  inp.value = p;
+  goRunPage(p);
+}
+
+/* 切换每页条数 */
+async function changePageSize(size) {
+  const t = state.detail.task;
+  if (!t || !state.detail.runId) return;
+  try {
+    const d = await loadRunPage(t.id, state.detail.runId, 1, parseInt(size, 10) || 100);
+    state.detail.task = d;
+    state.detail.pageSize = (d.run && d.run.run_page_size) || 100;
+    renderDetail();
+  } catch (e) { toast(e.message, true); }
+}
+
+/* 生成页码窗口列表, 0 表示省略号: 1 2 3 … 97 98 99 100 */
+function pageWindow(cur, pages, width) {
+  width = width || 5;
+  const win = [];
+  if (pages <= width + 2) {
+    for (let i = 1; i <= pages; i++) win.push(i);
+    return win;
+  }
+  win.push(1);
+  let lo = Math.max(2, cur - Math.floor(width / 2));
+  let hi = Math.min(pages - 1, cur + Math.floor(width / 2));
+  if (hi - lo < width - 1) {
+    if (lo <= 2) hi = lo + width - 1;
+    else lo = hi - width + 1;
+  }
+  if (lo > 2) win.push(0);
+  for (let i = lo; i <= hi; i++) win.push(i);
+  if (hi < pages - 1) win.push(0);
+  win.push(pages);
+  return win;
+}
+
+/* 重爬失败页: 注入待爬队列并自动继续爬取 */
+async function retryFailed() {
+  const t = state.detail.task;
+  if (!t || !state.detail.runId) return;
+  if (!confirm("将把本次运行失败的页面重新加入待爬队列并立即继续爬取，确定？")) return;
+  try {
+    const d = await api(`/api/tasks/${t.id}/retry-failed`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ run: state.detail.runId }),
+    });
+    toast(`已注入 ${d.injected} 个失败页，开始继续爬取…`);
+    if ((d.injected || 0) > 0) {
+      await api(`/api/tasks/${t.id}/continue`, { method: "POST" });
+    }
+    loadTasks();
+    openDetail(t.id);
+  } catch (e) { toast(e.message, true); }
+}
+
 function renderDetail() {
   const t = state.detail.task;
   const runs = t.runs || [];
@@ -602,10 +674,15 @@ function renderDetail() {
       ${t.running ? `
         <button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
         <button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
-      : `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`}
+      : `
+        <button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>
+        ${t.mode === "auto" && (t.auto_pending_count || 0) > 0 ? `
+          <button class="btn" onclick="contTask('${t.id}')">⏩ 继续爬取（缓存 ${t.auto_pending_count} 条）</button>` : ""}`}
       ${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>
         <button class="btn" ${t.running ? "disabled" : ""} onclick="clearCache('${t.id}')">🧹 清空缓存</button>` : ""}
       <button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
+      <button class="btn" onclick="exportZip('${t.id}')">📦 打包下载</button>
+      <button class="btn" onclick="exportEmail('${t.id}')">📧 发邮箱</button>
     </div>`;
 
   const chips = runs.map((r) => `
@@ -659,13 +736,30 @@ function renderRunPanel(t, run) {
   const total = run.results_total || results.length;
   const pages = run.run_pages || 1;
   const page = run.run_page || 1;
+  const pageSize = run.run_page_size || DETAIL_PAGE_SIZE;
+  const multi = pages > 1;
   const pager = total > 0 ? `
     <div class="pager">
-      ${total > results.length ? `
+      ${multi ? `
-        <button class="btn sm" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(${page - 1})">◀ 上一页</button>
+        <button class="btn sm" title="首页" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(1)">⏮</button>
+        <button class="btn sm" title="上一页" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(${page - 1})">◀</button>
+        <span class="pager-pages">${pageWindow(page, pages).map((p) => p === 0
+          ? '<span class="pager-ellipsis">…</span>'
+          : `<button class="btn sm page-btn ${p === page ? "active" : ""}" onclick="goRunPage(${p})">${p}</button>`).join("")}</span>
+        <button class="btn sm" title="下一页" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${page + 1})">▶</button>
+        <button class="btn sm" title="末页" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${pages})">⏭</button>
         <span class="pager-info">第 <b>${page}</b> / ${pages} 页 · 共 ${total} 条</span>
-        <button class="btn sm" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${page + 1})">下一页 ▶</button>`
+        <span class="pager-jump">跳至
+          <input type="number" id="pagerJump" min="1" max="${pages}" value="${page}"
+                 onkeydown="if(event.key==='Enter')jumpRunPage()"> 页
+          <button class="btn sm" onclick="jumpRunPage()">GO</button>
+        </span>`
       : `<span class="pager-info">共 ${total} 条</span>`}
+      <span class="pager-size">每页
+        <select id="pagerSize" onchange="changePageSize(this.value)">
+          ${[50, 100, 200, 500].map((s) => `<option value="${s}" ${s === pageSize ? "selected" : ""}>${s}</option>`).join("")}
+        </select> 条
+      </span>
     </div>` : "";
 
   return `
@@ -678,6 +772,8 @@ function renderRunPanel(t, run) {
       <span class="ok">✅ ${s.ok}</span>
       <span class="fail">❌ ${s.fail}</span>
       <span class="img">🖼️ ${s.images}</span>
+      ${t.mode === "auto" && (s.fail || 0) > 0 && run.status !== "running" && run.status !== "paused" ? `
+        <button class="btn sm" title="将本次运行失败的页面重新加入待爬队列并继续爬取" onclick="retryFailed()">🔄 重爬失败页 (${s.fail})</button>` : ""}
     </div>
     ${prog}
     <div class="card-line">当前: <b>${esc(run.progress.current_url || "")}</b></div>
@@ -732,6 +828,68 @@ function stopLogPoll() {
   if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; }
 }
 
+async function contTask(tid) {
+  try {
+    const r = await api(`/api/tasks/${tid}/continue`, { method: "POST" });
+    toast("已开始继续爬取缓存队列");
+    loadTasks();
+    if (state.detail.task && state.detail.task.id === tid) openDetail(tid);
+  } catch (e) { toast(e.message, true); }
+}
+
+/* ---------------- 打包导出 ---------------- */
+async function exportZip(tid) {
+  const btn = window.event && window.event.target;
+  if (btn) { btn.disabled = true; btn.textContent = "⏳ 打包中..."; }
+  try {
+    const res = await fetch(`/api/export?task_id=${tid}`);
+    if (!res.ok) {
+      let d = null;
+      try { d = await res.json(); } catch (e) { /* ignore */ }
+      throw new Error((d && d.error) || `HTTP ${res.status}`);
+    }
+    let fname = "export.zip";
+    const disp = res.headers.get("Content-Disposition") || "";
+    const m = disp.match(/filename\*?=(?:UTF-8'')?"?([^";]+)"?/i);
+    if (m) { try { fname = decodeURIComponent(m[1]); } catch (e) { fname = m[1]; } }
+    const blob = await res.blob();
+    const a = document.createElement("a");
+    a.href = URL.createObjectURL(blob);
+    a.download = fname;
+    document.body.appendChild(a);
+    a.click();
+    a.remove();
+    setTimeout(() => URL.revokeObjectURL(a.href), 5000);
+    toast(`打包下载完成 ${fname} (${(blob.size / 1048576).toFixed(1)} MB)`);
+  } catch (e) {
+    toast("打包失败: " + e.message, true);
+  } finally {
+    if (btn) { btn.disabled = false; btn.textContent = "📦 打包下载"; }
+  }
+}
+
+async function exportEmail(tid) {
+  const t = (state.detail.task && state.detail.task.id === tid)
+    ? state.detail.task : state.tasks.find((x) => x.id === tid);
+  const def = (t && t.config && t.config.notify_email) || "wlq@tphai.com";
+  const email = prompt("发送到邮箱（留空默认 wlq@tphai.com）:", def);
+  if (email === null) return;   // 用户取消
+  const btn = window.event && window.event.target;
+  if (btn) { btn.disabled = true; btn.textContent = "⏳ 打包发送中..."; }
+  try {
+    const r = await api("/api/export/email", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ task_id: tid, email: email.trim() || "wlq@tphai.com" }),
+    });
+    toast(r.msg || "已发送");
+  } catch (e) {
+    toast("发送失败: " + e.message, true);
+  } finally {
+    if (btn) { btn.disabled = false; btn.textContent = "📧 发邮箱"; }
+  }
+}
+
 /* ---------------- 试爬取 ---------------- */
 function collectProbeFromForm() {
   const f = $("taskForm");

static/index.html

@@ -126,7 +126,7 @@
             <div class="field"><label>包含规则（每行一个，子串或正则）</label>
               <textarea name="include" rows="3" placeholder="techpowerup.com/review&#10;/news/"></textarea></div>
             <div class="field"><label>排除规则</label>
-              <textarea name="exclude" rows="3" placeholder="login, signup, /tag/, /forum/"></textarea></div>
+              <textarea name="exclude" rows="3" placeholder="每行一个关键词，URL 含任一关键词即不爬取&#10;如: MyComments.html、OtherPosts.html、/comments&#10;勾选正则后按正则匹配"></textarea></div>
           </div>
           <div class="row2">
             <div class="field check"><label><input name="same_domain" type="checkbox" checked> 仅爬同域名</label></div>

static/style.css

@@ -287,3 +287,20 @@ td.title-cell { max-width: 220px; overflow: hidden; text-overflow: ellipsis; }
 }
 .pager-info { color: var(--text-dim, #999); font-size: 13px; }
 .pager .btn:disabled { opacity: 0.4; cursor: not-allowed; }
+.pager-pages { display: flex; align-items: center; gap: 4px; flex-wrap: wrap; }
+.pager-pages .page-btn { min-width: 30px; padding: 4px 6px; }
+.pager-pages .page-btn.active {
+  background: var(--accent, #2d6cdf);
+  color: #fff;
+  border-color: var(--accent, #2d6cdf);
+  font-weight: 600;
+}
+.pager-ellipsis { color: var(--text-dim, #999); padding: 0 2px; user-select: none; }
+.pager-jump { display: inline-flex; align-items: center; gap: 4px; color: var(--text-dim, #999); font-size: 13px; }
+.pager-jump input {
+  width: 56px;
+  padding: 3px 6px;
+  text-align: center;
+}
+.pager-size { display: inline-flex; align-items: center; gap: 4px; color: var(--text-dim, #999); font-size: 13px; }
+.pager-size select { padding: 3px 4px; width: auto; }