反爬拦截不再重试: 判定为反爬类错误(403/429/503/Cloudflare/captcha/验证页等)直接放弃该页, 节省重试时间; 验证页持续8秒即判定反爬, 不再干等满超时(默认60s)

- engine.is_anti_crawl_error: 反爬错误信号关键词判定
- _retry_crawl: 反爬错误直接 break, entry.error 标注'反爬拦截, 跳过重试'
- _wait_page_settle/_settle_wait: 连续8秒检测到验证页即判定反爬尽早返回
- 集成测试: 反爬页 attempts=1 且 8.1s 放弃; 非反爬错误仍 attempts=retry_count+1 正常重试

app.py

@@ -5,6 +5,7 @@
 """
 import os
 import threading
+import time
 from datetime import datetime
 
 from flask import Flask, jsonify, request, send_file, send_from_directory
@@ -95,7 +96,7 @@ def persist_cb(task_id, run):
     run["progress"]["percent"] = round(done * 100 / total) if total else 0
     store.save_run(task_id, run)
     try:
-        db.sync_run(run, lambda tid, r: store.save_run(tid, r))
+        db.sync_run_async(run)      # 后台异步同步, 不再阻塞爬虫线程
     except Exception as e:
         print(f"[db] 同步失败: {e}", flush=True)
 
@@ -111,7 +112,7 @@ def start_run(task):
         job = CrawlJob(task, run, persist_cb)
         JOBS[task["id"]] = job
         job.start()
-        db.upsert_task(task)              # 确保任务在库中
+        db.upsert_task_async(task)        # 确保任务在库中 (异步)
         return run, None
 
 
@@ -147,12 +148,37 @@ def api_status():
 
 # ---------------- API: 任务 ----------------
 
+def _strip_auto_state(task):
+    """剥离任务对象中的待爬/已爬队列 (已独立存储, 避免大 JSON 拖慢接口)"""
+    if task.get("mode") == "auto" and task.get("auto"):
+        task["auto"] = {k: v for k, v in task["auto"].items()
+                         if k not in ("pending", "visited")}
+    return task
+
+
+def _run_summary(run):
+    """运行记录摘要 (不含 results/logs), 供列表/详情页使用"""
+    return {
+        "id": run["id"],
+        "task_id": run.get("task_id", ""),
+        "mode": run.get("mode", ""),
+        "status": run.get("status", ""),
+        "progress": run.get("progress", {}),
+        "started_at": run.get("started_at", ""),
+        "finished_at": run.get("finished_at", ""),
+        "stats": run.get("stats", {}),
+        "out_dir": run.get("out_dir", ""),
+    }
+
+
 @app.route("/api/tasks", methods=["GET"])
 def api_tasks():
     tasks = [t for t in store.load_tasks() if not t.get("deleted_at")]
+    runs_map = store.load_runs_map()      # 只全量读一次
     for t in tasks:
-        runs = store.get_runs(t["id"])
-        t["latest_run"] = runs[-1] if runs else None
+        _strip_auto_state(t)
+        runs = runs_map.get(t["id"], [])
+        t["latest_run"] = _run_summary(runs[-1]) if runs else None
         with JOBS_LOCK:
             job = JOBS.get(t["id"])
             t["running"] = bool(job and job.is_running())
@@ -220,7 +246,7 @@ def api_create_task():
             return jsonify({"error": "请至少填写一个网址"}), 400
 
     store.upsert_task(task)
-    db.upsert_task(task)
+    db.upsert_task_async(task)
     return jsonify(task), 201
 
 
@@ -229,11 +255,59 @@ def api_task_detail(tid):
     task = store.get_task(tid)
     if not task:
         return jsonify({"error": "任务不存在"}), 404
-    task["runs"] = list(reversed(store.get_runs(tid)))
+    # 分页参数: 指定 run + 页码 (默认最新 run 第 1 页)
+    rid = request.args.get("run", "")
+    try:
+        page = max(int(request.args.get("page", 1)), 1)
+    except ValueError:
+        page = 1
+    try:
+        page_size = min(max(int(request.args.get("page_size", 100)), 10), 500)
+    except ValueError:
+        page_size = 100
+
+    runs = store.get_runs(tid)
+    if rid:
+        cur = next((r for r in runs if r["id"] == rid), None)
+        if cur is None:
+            cur = runs[-1] if runs else None   # 指定的 run 不存在时回退最新
+    else:
+        cur = runs[-1] if runs else None
     with JOBS_LOCK:
         job = JOBS.get(tid)
         task["running"] = bool(job and job.is_running())
-    return jsonify(task)
+
+    result = _strip_auto_state(dict(task))
+    result["runs"] = [_run_summary(r) for r in reversed(runs)]
+    result["run"] = None
+    result["run_id"] = ""
+    result["run_total"] = 0
+    result["run_page"] = 1
+    result["run_pages"] = 1
+    if cur:
+        results = cur.get("results", [])
+        total = len(results)
+        pages = max((total + page_size - 1) // page_size, 1)
+        page = min(page, pages)
+        start = (page - 1) * page_size
+        snap = {k: v for k, v in cur.items() if k != "logs"}   # logs 走独立轮询接口
+        snap["results"] = results[start:start + page_size]
+        snap["results_total"] = total
+        snap["run_page"] = page
+        snap["run_pages"] = pages
+        snap["run_page_size"] = page_size
+        result["run"] = snap
+        result["run_id"] = cur["id"]
+        result["run_total"] = total
+        result["run_page"] = page
+        result["run_pages"] = pages
+        result["run_page_size"] = page_size
+    # auto 状态数量 (独立文件)
+    if task.get("mode") == "auto":
+        st = store.load_auto_state(tid, task)
+        result["auto_pending_count"] = len(st.get("pending", []))
+        result["auto_visited_count"] = len(st.get("visited", []))
+    return jsonify(result)
 
 
 @app.route("/api/tasks/<tid>", methods=["PUT"])
@@ -267,7 +341,7 @@ def api_update_task(tid):
             task["schedule"] = sch
         task["updated_at"] = now_str()
         store.upsert_task(task)
-        db.upsert_task(task)
+        db.upsert_task_async(task)
     return jsonify(task)
 
 
@@ -283,7 +357,7 @@ def api_delete_task(tid):
     with JOBS_LOCK:
         JOBS.pop(tid, None)
     store.soft_delete_task(tid, now_str())
-    db.upsert_task(store.get_task(tid))
+    db.upsert_task_async(store.get_task(tid))
     return jsonify({"ok": True, "msg": "已移入回收站"})
 
 
@@ -306,8 +380,9 @@ def _purge_out_dir(out_dir):
 @app.route("/api/trash", methods=["GET"])
 def api_trash_list():
     items = store.list_trash()
+    runs_map = store.load_runs_map()
     for t in items:
-        t["runs_count"] = len(store.get_runs(t["id"]))
+        t["runs_count"] = len(runs_map.get(t["id"], []))
         t["out_dir"] = resolve_out_dir(t)
     return jsonify(items)
 
@@ -318,7 +393,7 @@ def api_trash_restore(tid):
     if not task or not task.get("deleted_at"):
         return jsonify({"error": "任务不在回收站中"}), 404
     store.restore_task(tid)
-    db.upsert_task(store.get_task(tid))
+    db.upsert_task_async(store.get_task(tid))
     return jsonify({"ok": True, "msg": "已恢复"})
 
 
@@ -386,13 +461,20 @@ def api_resume(tid):
 
 # ---------------- API: 统计 ----------------
 
+# 磁盘占用统计缓存 (避免每次轮询都逐个 stat 文件)
+_DISK_CACHE = {"ts": 0.0, "mb": 0.0}
+DISK_CACHE_TTL = 30   # 秒
+
+
 @app.route("/api/stats")
 def api_stats():
-    tasks = [t for t in store.load_tasks() if not t.get("deleted_at")]
-    trash_count = sum(1 for t in store.load_tasks() if t.get("deleted_at"))
+    tasks = store.load_tasks()
+    active = [t for t in tasks if not t.get("deleted_at")]
+    trash_count = len(tasks) - len(active)
+    runs_map = store.load_runs_map()      # 只全量读一次
     total_runs = ok = fail = imgs = 0
-    for t in tasks:
-        for r in store.get_runs(t["id"]):
+    for t in active:
+        for r in runs_map.get(t["id"], []):
             total_runs += 1
             st = r.get("stats") or {}
             ok += st.get("ok", 0)
@@ -400,28 +482,30 @@ def api_stats():
             imgs += st.get("images", 0)
     with JOBS_LOCK:
         running = sum(1 for j in JOBS.values() if j.is_running())
-    # 统计各任务输出目录的磁盘占用
-    size = 0
-    seen = set()
-    for t in tasks:
-        d = os.path.realpath(resolve_out_dir(t))
-        if d in seen or not os.path.isdir(d):
-            continue
-        seen.add(d)
-        for root, _dirs, files in os.walk(d):
-            for f in files:
-                try:
-                    size += os.path.getsize(os.path.join(root, f))
-                except OSError:
-                    pass
+    now = time.time()
+    if now - _DISK_CACHE["ts"] > DISK_CACHE_TTL:
+        size = 0
+        seen = set()
+        for t in active:
+            d = os.path.realpath(resolve_out_dir(t))
+            if d in seen or not os.path.isdir(d):
+                continue
+            seen.add(d)
+            for root, _dirs, files in os.walk(d):
+                for f in files:
+                    try:
+                        size += os.path.getsize(os.path.join(root, f))
+                    except OSError:
+                        pass
+        _DISK_CACHE.update(ts=now, mb=round(size / 1048576, 1))
     return jsonify({
-        "tasks": len(tasks),
+        "tasks": len(active),
         "running": running,
         "runs": total_runs,
         "ok": ok,
         "fail": fail,
         "images": imgs,
-        "disk_mb": round(size / 1048576, 1),
+        "disk_mb": _DISK_CACHE["mb"],
         "trash": trash_count,
     })
 
@@ -463,9 +547,10 @@ def api_clear_cache(tid):
     auto = task.setdefault("auto", {})
     auto["pending"] = []
     auto["visited"] = []
+    store.clear_auto_state(tid)          # 同时清空独立状态文件
     task["updated_at"] = now_str()
     store.upsert_task(task)
-    db.upsert_task(task)
+    db.upsert_task_async(task)
     return jsonify({"ok": True, "msg": "缓存队列已清空"})
 
 
@@ -500,6 +585,7 @@ def api_search():
     ql = q.lower()
     results = []
     seen = set()
+    runs_map = store.load_runs_map()      # 只全量读一次
     for task in store.load_tasks():
         if task.get("deleted_at"):
             continue                       # 回收站任务不参与搜索
@@ -509,7 +595,7 @@ def api_search():
             or any(ql in u.lower() for u in task.get("urls", []))
             or ql in ((auto.get("seed_url") or "").lower())
         )
-        for run in store.get_runs(task["id"]):
+        for run in runs_map.get(task["id"], []):
             for r in run.get("results", []):
                 url = r.get("url") or ""
                 title = r.get("title") or ""
@@ -593,13 +679,35 @@ def api_file():
 scheduler = Scheduler(start_run)
 
 
+def migrate_auto_state():
+    """启动时把旧任务对象里的 auto pending/visited 迁移到独立文件, 给 tasks.json 瘦身"""
+    migrated = 0
+    for t in store.load_tasks():
+        if t.get("mode") != "auto":
+            continue
+        auto = t.get("auto") or {}
+        if auto.get("pending") or auto.get("visited"):
+            store.save_auto_state(t["id"], {
+                "pending": auto.get("pending", []),
+                "visited": auto.get("visited", []),
+            })
+            auto.pop("pending", None)
+            auto.pop("visited", None)
+            t["updated_at"] = now_str()
+            store.upsert_task(t)
+            migrated += 1
+    if migrated:
+        print(f"[universal-crawler] 已迁移 {migrated} 个 auto 任务的队列状态到独立文件", flush=True)
+
+
 def main():
     os.makedirs(os.path.join(HERE, "data"), exist_ok=True)
     os.makedirs(os.path.join(HERE, "out"), exist_ok=True)
     db.init_db()
-    # 全量同步存量任务到数据库
+    migrate_auto_state()
+    # 全量同步存量任务到数据库 (异步, 不阻塞启动)
     for t in store.load_tasks():
-        db.upsert_task(t)
+        db.upsert_task_async(t)
     scheduler.start()
     print(f"[universal-crawler] 启动完成, 管理界面: http://0.0.0.0:{PORT}/")
     app.run(host="0.0.0.0", port=PORT, threaded=True, debug=False)

db.py

@@ -5,9 +5,11 @@ MySQL 记录层: 任务 / 运行 / 爬取结果 写入数据库
 - 爬取成功/失败均有 status 标记 (OK / FAIL), 运行记录有 run 状态标记
 - 所有操作容错: 数据库不可用时不影响爬取主流程 (仅打印日志)
 """
+import copy
 import json
 import os
 import time
+from concurrent.futures import ThreadPoolExecutor
 
 import pymysql
 
@@ -101,6 +103,39 @@ def _safe(fn, *args, **kwargs):
         print(f"[db] 操作失败: {e}", flush=True)
 
 
+# ---------------- 异步同步 (避免远程 MySQL 阻塞爬虫/API 主流程) ----------------
+
+_db_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="db-sync")
+
+
+def upsert_task_async(task):
+    """异步写入/更新任务 (不阻塞调用方)"""
+    try:
+        snap = copy.deepcopy(task)
+        _db_executor.submit(upsert_task, snap)
+    except Exception as e:
+        print(f"[db] 异步任务同步失败: {e}", flush=True)
+
+
+def sync_run_async(run):
+    """异步同步运行记录+爬取结果; 完成后把 _db_count 回写到主线程 run 对象"""
+    try:
+        snap = copy.deepcopy(run)
+
+        def _work():
+            upsert_run(snap)
+            results = snap.get("results", [])
+            synced = snap.get("_db_count", 0)
+            if len(results) > synced:
+                insert_results(snap, results[synced:])
+                snap["_db_count"] = len(results)
+                run["_db_count"] = snap["_db_count"]
+
+        _db_executor.submit(_work)
+    except Exception as e:
+        print(f"[db] 异步运行同步失败: {e}", flush=True)
+
+
 def init_db():
     """建库建表 (幂等)"""
     try:

engine.py

@@ -36,6 +36,19 @@ _CHALLENGE_MARKS = [
     "checking your browser", "enable javascript and cookies",
 ]
 
+# 反爬拦截类错误信号: 命中后不再重试 (重试无意义且拖慢任务)
+_ANTI_CRAWL_MARKS = [
+    "403", "429", "503", "forbidden", "access denied", "too many requests",
+    "captcha", "cloudflare", "challenge", "verify you are human",
+    "just a moment", "blocked", "被反爬拦截",
+]
+
+
+def is_anti_crawl_error(err):
+    """判断错误是否属于反爬拦截 (此类失败重试也无法通过, 直接放弃该页)"""
+    s = str(err or "").lower()
+    return any(m in s for m in _ANTI_CRAWL_MARKS)
+
 
 def is_challenge_page(title, html):
     """判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)"""
@@ -56,6 +69,7 @@ def safe_name(url, idx):
 def _settle_wait(page, timeout_s):
     """等待页面稳定(无 stop/pause 检查, 供试爬取等独立流程使用)"""
     last_title, stable = "", 0
+    challenge_hits = 0
     start = time.time()
     while time.time() - start < timeout_s:
         time.sleep(1)
@@ -65,8 +79,12 @@ def _settle_wait(page, timeout_s):
         except Exception:
             continue
         if is_challenge_page(title, html):
+            challenge_hits += 1
+            if challenge_hits >= 8:        # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
+                return True, title, html
             stable = 0
             continue
+        challenge_hits = 0
         if title == last_title:
             stable += 1
             if stable >= 2 and len(html) > 1000:
@@ -306,6 +324,7 @@ class CrawlJob:
 
     def _wait_page_settle(self, page, timeout_s):
         last_title, stable = "", 0
+        challenge_hits = 0
         start = time.time()
         while time.time() - start < timeout_s:
             if self._stop.is_set():
@@ -318,8 +337,12 @@ class CrawlJob:
             except Exception:
                 continue                      # 正在跳转
             if is_challenge_page(title, html):
+                challenge_hits += 1
+                if challenge_hits >= 8:       # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
+                    return True, title, html
                 stable = 0
                 continue
+            challenge_hits = 0
             if title == last_title:
                 stable += 1
                 if stable >= 2 and len(html) > 1000:
@@ -460,6 +483,11 @@ class CrawlJob:
                 entry["error"] = str(e)
                 entry["crawl_time"] = store.now_str()
                 self._log("warn", f"第{attempt + 1}次失败 {url}: {e}")
+                if is_anti_crawl_error(e):
+                    # 反爬拦截: 重试也过不去, 直接放弃, 不再消耗重试次数
+                    entry["error"] = f"反爬拦截, 跳过重试: {e}"
+                    self._log("warn", f"判定为反爬拦截, 放弃重试: {url}")
+                    break
                 if attempt < retries:
                     self._wait_if_paused()
                     t0 = time.time()
@@ -583,9 +611,10 @@ class CrawlJob:
         run["out_dir"] = out_dir
         os.makedirs(out_dir, exist_ok=True)
 
-        # 恢复持久化状态: 已爬集合 + 上次未爬完的缓存队列
-        visited = set(auto.get("visited", []) or [])
-        pending = auto.get("pending", []) or []
+        # 恢复持久化状态: 已爬集合 + 上次未爬完的缓存队列 (独立状态文件, 不撑大 tasks.json)
+        state = store.load_auto_state(self.task["id"], self.task)
+        visited = set(state.get("visited", []) or [])
+        pending = state.get("pending", []) or []
         # 起始网址每次运行都爬(不做去重), 缓存队列继续消费
         queue = [(seed, 0, "")]
         if pending:
@@ -634,11 +663,12 @@ class CrawlJob:
             self._close_browser(p, browser, ctx, cookie_file)
             # 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
             try:
-                auto["pending"] = [
-                    {"url": u, "depth": d, "source": s} for u, d, s in queue]
-                auto["visited"] = list(visited)
-                store.upsert_task(self.task)
-                db.upsert_task(self.task)
+                store.save_auto_state(self.task["id"], {
+                    "pending": [
+                        {"url": u, "depth": d, "source": s} for u, d, s in queue],
+                    "visited": list(visited),
+                })
+                db.upsert_task_async(self.task)
             except Exception:
                 pass
         self._persist()

static/app.js

@@ -45,26 +45,32 @@ async function loadTasks() {
     pill.textContent = `运行中任务: ${running}`;
     pill.classList.toggle("active", running > 0);
     const st = await api("/api/status");
-    $("version").textContent = `v${st.version}`;
+    $("version").textContent = st && st.version ? `v${st.version}` : "v-";
   } catch (e) {
     toast("加载任务失败: " + e.message, true);
   }
   loadStats();
 }
 
+const STAT_IDS = ["stTasks", "stRunning", "stRuns", "stOk", "stFail", "stImgs", "stDisk"];
+
 async function loadStats() {
+  // 未读到真实数据前, 一律显示 "-"
+  STAT_IDS.forEach((id) => { $(id).textContent = "-"; });
+  $("trashCount").textContent = "-";
   try {
     const s = await api("/api/stats");
-    $("stTasks").textContent = s.tasks;
-    $("stRunning").textContent = s.running;
-    $("stRuns").textContent = s.runs;
-    $("stOk").textContent = s.ok;
-    $("stFail").textContent = s.fail;
-    $("stImgs").textContent = s.images;
-    $("stDisk").textContent = s.disk_mb >= 1024
-      ? (s.disk_mb / 1024).toFixed(1) + " GB" : s.disk_mb + " MB";
-    $("trashCount").textContent = s.trash || 0;
-  } catch (e) { /* 统计失败忽略 */ }
+    $("stTasks").textContent = s.tasks ?? "-";
+    $("stRunning").textContent = s.running ?? "-";
+    $("stRuns").textContent = s.runs ?? "-";
+    $("stOk").textContent = s.ok ?? "-";
+    $("stFail").textContent = s.fail ?? "-";
+    $("stImgs").textContent = s.images ?? "-";
+    const mb = Number(s.disk_mb);
+    $("stDisk").textContent = Number.isFinite(mb) && s.disk_mb != null
+      ? (mb >= 1024 ? (mb / 1024).toFixed(1) + " GB" : mb + " MB") : "-";
+    $("trashCount").textContent = s.trash ?? "-";
+  } catch (e) { /* 统计失败保持 "-" */ }
 }
 
 function taskStatusBadge(t) {
@@ -516,13 +522,15 @@ async function submitForm(e) {
 }
 
 /* ---------------- 详情 ---------------- */
+const DETAIL_PAGE_SIZE = 100;   // 详情结果每页条数
+
 async function openDetail(tid) {
   try {
-    const t = await api(`/api/tasks/${tid}`);
+    const t = await api(`/api/tasks/${tid}?page_size=${DETAIL_PAGE_SIZE}`);
     state.detail.task = t;
     const runs = t.runs || [];
-    state.detail.runId = runs.find((r) => r.status === "running")
-      ? runs[0].id : (runs[0] ? runs[0].id : null);
+    const cur = t.run;
+    state.detail.runId = cur ? cur.id : (runs[0] ? runs[0].id : null);
     state.detail.logOffset = 0;
     $("detailTitle").textContent = `任务详情 · ${t.name}`;
     renderDetail();
@@ -531,11 +539,38 @@ async function openDetail(tid) {
   } catch (e) { toast(e.message, true); }
 }
 
+async function loadRunPage(tid, rid, page) {
+  return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${DETAIL_PAGE_SIZE}`);
+}
+
+async function selectRun(rid) {
+  const t = state.detail.task;
+  if (!t) return;
+  try {
+    const d = await loadRunPage(t.id, rid, 1);
+    state.detail.task = d;
+    state.detail.runId = d.run ? d.run.id : null;
+    state.detail.logOffset = 0;
+    renderDetail();
+    startLogPoll();
+  } catch (e) { toast(e.message, true); }
+}
+
+async function goRunPage(p) {
+  const t = state.detail.task;
+  if (!t || !state.detail.runId) return;
+  try {
+    const d = await loadRunPage(t.id, state.detail.runId, p);
+    state.detail.task = d;
+    renderDetail();
+  } catch (e) { toast(e.message, true); }
+}
+
 function renderDetail() {
   const t = state.detail.task;
   const runs = t.runs || [];
-  const run = runs.find((r) => r.id === state.detail.runId) || runs[0] || null;
-  state.detail.runId = run ? run.id : null;
+  const run = t.run || null;
+  state.detail.runId = run ? run.id : (runs[0] ? runs[0].id : null);
   const cfg = t.config || {};
 
   const meta = `
@@ -546,8 +581,8 @@ function renderDetail() {
       <span>输出: <b>${esc(cfg.out_dir || "out/" + t.id)}</b></span>
       ${t.mode === "auto" ? `
         <span>起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b></span>
-        <span>待爬缓存: <b>${(t.auto && (t.auto.pending || []).length) || 0}</b> 条（停止后可继续爬取）</span>
-        <span>已爬: <b>${(t.auto && (t.auto.visited || []).length) || 0}</b> 页</span>` : ""}
+        <span>待爬缓存: <b>${t.auto_pending_count ?? "-"}</b> 条（停止后可继续爬取）</span>
+        <span>已爬: <b>${t.auto_visited_count ?? "-"}</b> 页</span>` : ""}
     </div>`;
 
   let sched = "";
@@ -604,7 +639,7 @@ function renderRunPanel(t, run) {
     const meta = r.meta_file
       ? `<span class="file-link" onclick="showMeta('${t.id}','${esc(r.meta_file)}')">📋 元数据</span>` : "—";
     return `<tr>
-      <td>${i + 1}</td>
+      <td>${(run.results_total ? (run.run_page - 1) * DETAIL_PAGE_SIZE : 0) + i + 1}</td>
       <td class="${statusCls}">${r.status}</td>
       <td class="title-cell" title="${esc(r.title)}">${esc(r.title)}</td>
       <td class="url-cell" title="${esc(r.url)}">${esc(r.url)}</td>
@@ -621,6 +656,18 @@ function renderRunPanel(t, run) {
         onclick="previewFile('${t.id}','${esc(im.file)}','图片预览')">`).join("")}
     </div>` : "";
 
+  const total = run.results_total || results.length;
+  const pages = run.run_pages || 1;
+  const page = run.run_page || 1;
+  const pager = total > 0 ? `
+    <div class="pager">
+      ${total > results.length ? `
+        <button class="btn sm" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(${page - 1})">◀ 上一页</button>
+        <span class="pager-info">第 <b>${page}</b> / ${pages} 页 · 共 ${total} 条</span>
+        <button class="btn sm" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${page + 1})">下一页 ▶</button>`
+      : `<span class="pager-info">共 ${total} 条</span>`}
+    </div>` : "";
+
   return `
   <div class="run-panel">
     <div class="run-stats">
@@ -640,7 +687,8 @@ function renderRunPanel(t, run) {
           <thead><tr><th>#</th><th>状态</th><th>标题</th><th>网址</th><th>爬取时间</th><th>HTML</th><th>TXT</th><th>图片</th><th>元数据</th><th>错误</th></tr></thead>
           <tbody>${rows}</tbody>
         </table>
-      </div>` : '<div class="card-line">暂无结果</div>'}
+      </div>
+      ${pager}` : '<div class="card-line">暂无结果</div>'}
     ${thumbHtml}
     <div class="logs" id="logBox"></div>
   </div>`;
@@ -651,13 +699,6 @@ function scrollThumbs() {
   if (box) box.scrollIntoView({ behavior: "smooth", block: "center" });
 }
 
-async function selectRun(rid) {
-  state.detail.runId = rid;
-  state.detail.logOffset = 0;
-  renderDetail();
-  startLogPoll();
-}
-
 async function startLogPoll() {
   stopLogPoll();
   const t = state.detail.task;

static/index.html

@@ -14,9 +14,9 @@
       <input id="searchInput" placeholder="搜索网址 / 标题 / 任务名..." maxlength="100">
       <button id="btnSearch" class="btn sm primary" title="搜索">🔍</button>
     </div>
-    <span class="status-pill" id="statusPill">运行中任务: 0</span>
+    <span class="status-pill" id="statusPill">运行中任务: -</span>
     <button id="btnTheme" class="btn ghost" title="切换日间/夜间主题">☀️</button>
-    <button id="btnTrash" class="btn ghost" title="回收站">🗑️ 回收站 <span id="trashCount" class="trash-count">0</span></button>
+    <button id="btnTrash" class="btn ghost" title="回收站">🗑️ 回收站 <span id="trashCount" class="trash-count">-</span></button>
     <button id="btnRefresh" class="btn ghost">⟳ 刷新</button>
     <button id="btnNew" class="btn primary">＋ 新建任务</button>
   </div>
@@ -24,13 +24,13 @@
 
 <main>
   <div class="stats-bar" id="statsBar">
-    <div class="stat-card"><div class="stat-num" id="stTasks">0</div><div class="stat-label">📋 任务总数</div></div>
-    <div class="stat-card"><div class="stat-num num-run" id="stRunning">0</div><div class="stat-label">🔄 正在运行</div></div>
-    <div class="stat-card"><div class="stat-num" id="stRuns">0</div><div class="stat-label">📊 累计运行次数</div></div>
-    <div class="stat-card"><div class="stat-num num-ok" id="stOk">0</div><div class="stat-label">✅ 成功页面</div></div>
-    <div class="stat-card"><div class="stat-num num-fail" id="stFail">0</div><div class="stat-label">❌ 失败页面</div></div>
-    <div class="stat-card"><div class="stat-num" id="stImgs">0</div><div class="stat-label">🖼️ 已爬图片</div></div>
-    <div class="stat-card"><div class="stat-num" id="stDisk">0</div><div class="stat-label">💾 磁盘占用</div></div>
+    <div class="stat-card"><div class="stat-num" id="stTasks">-</div><div class="stat-label">📋 任务总数</div></div>
+    <div class="stat-card"><div class="stat-num num-run" id="stRunning">-</div><div class="stat-label">🔄 正在运行</div></div>
+    <div class="stat-card"><div class="stat-num" id="stRuns">-</div><div class="stat-label">📊 累计运行次数</div></div>
+    <div class="stat-card"><div class="stat-num num-ok" id="stOk">-</div><div class="stat-label">✅ 成功页面</div></div>
+    <div class="stat-card"><div class="stat-num num-fail" id="stFail">-</div><div class="stat-label">❌ 失败页面</div></div>
+    <div class="stat-card"><div class="stat-num" id="stImgs">-</div><div class="stat-label">🖼️ 已爬图片</div></div>
+    <div class="stat-card"><div class="stat-num" id="stDisk">-</div><div class="stat-label">💾 磁盘占用</div></div>
   </div>
   <div id="taskList" class="task-grid"></div>
   <div id="emptyState" class="empty hidden">

static/style.css

@@ -275,3 +275,15 @@ td.title-cell { max-width: 220px; overflow: hidden; text-overflow: ellipsis; }
 @keyframes fadein { from { opacity: 0; transform: translate(-50%, -8px); } }
 ::-webkit-scrollbar { width: 8px; height: 8px; }
 ::-webkit-scrollbar-thumb { background: var(--thumb); border-radius: 4px; }
+
+/* 结果分页控件 */
+.pager {
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  gap: 12px;
+  padding: 10px 0;
+  flex-wrap: wrap;
+}
+.pager-info { color: var(--text-dim, #999); font-size: 13px; }
+.pager .btn:disabled { opacity: 0.4; cursor: not-allowed; }

store.py

@@ -10,6 +10,7 @@ HERE = os.path.dirname(os.path.abspath(__file__))
 DATA_DIR = os.path.join(HERE, "data")
 TASKS_FILE = os.path.join(DATA_DIR, "tasks.json")
 RUNS_FILE = os.path.join(DATA_DIR, "runs.json")   # {task_id: [run, ...]}
+AUTO_STATE_DIR = os.path.join(DATA_DIR, "auto_state")  # auto 任务待爬/已爬队列独立存储
 
 _lock = threading.RLock()
 
@@ -140,6 +141,46 @@ def purge_trash():
         return purged
 
 
+# ---------------- auto 任务状态 (待爬队列/已爬集合) ----------------
+# 独立于 tasks.json 存储: 避免上万条 URL 撑大任务文件导致全量读写变慢
+
+def auto_state_path(task_id):
+    return os.path.join(AUTO_STATE_DIR, f"{task_id}.json")
+
+
+def load_auto_state(task_id, fallback_task=None):
+    """读取 auto 任务的 pending/visited 状态; 无独立文件时从旧任务对象迁移"""
+    with _lock:
+        path = auto_state_path(task_id)
+        if os.path.exists(path):
+            st = _load(path, {}) or {}
+            return {"pending": st.get("pending", []), "visited": st.get("visited", [])}
+        if fallback_task:
+            auto = fallback_task.get("auto") or {}
+            st = {"pending": auto.get("pending", []), "visited": auto.get("visited", [])}
+            if st["pending"] or st["visited"]:
+                _save(path, st)   # 自动迁移旧数据
+                return st
+    return {"pending": [], "visited": []}
+
+
+def save_auto_state(task_id, state):
+    with _lock:
+        _save(auto_state_path(task_id), {
+            "pending": state.get("pending", []),
+            "visited": state.get("visited", []),
+        })
+
+
+def clear_auto_state(task_id):
+    """删除 auto 状态文件"""
+    with _lock:
+        try:
+            os.remove(auto_state_path(task_id))
+        except OSError:
+            pass
+
+
 # ---------------- 运行记录 ----------------
 
 def load_runs_map():