v1.0.14: 自动爬取支持深度/页数无限制(默认0/1000), 待爬链接持久化缓存队列(停止后续爬), 起始网址不去重, 清空缓存API

README.md

@@ -49,12 +49,14 @@
 - 可启用/停用调度，自动计算下次执行时间；到点自动开跑，跑完自动计算下一次
 
 ### 4. 自动爬取模式
-给定一个起始网址，系统自动从页面里发现链接、按规则筛选后 BFS 爬取：
+给定一个起始网址，系统**持续递归**爬取：爬取页面 → 自动发现符合规则的链接 → 继续爬取 → 继续发现……直到**无新链接可爬**时自动结束。
 - **🧪 试爬取**：先用起始网址试跑一次，展示规则筛选后的链接清单（将爬取哪些、排除哪些及原因），确认规则符合预期后再正式开爬
+- **最大爬取深度**：0=无限制（默认），N=只爬 N 层
+- **最大页数**：0=无限制，默认 1000 作安全上限（防止动态无限链接的站点失控）
+- **🔄 缓存续爬**：提取但未爬取的链接自动存入缓存队列（连同已爬集合一并持久化），任务停止/中断后再次运行，从缓存队列**继续爬取**，已爬过的不会重复；起始网址每次运行都重新爬取（不去重）；规则变更后可点「🧹 清空缓存」重新开始
 - **包含规则**：只爬包含指定子串（或正则）的链接
 - **排除规则**：跳过匹配的链接（如 login、/tag/）
 - **仅同域名**：限制在起始网站内
-- **最大页数 / 最大深度**：控制爬取规模
 - 其余参数（间隔、重试、图片、通知）同批量模式
 
 ### 5. 资源操作信息（元数据）

app.py

@@ -448,6 +448,27 @@ def api_probe():
     return jsonify(result)
 
 
+@app.route("/api/tasks/<tid>/clear_cache", methods=["POST"])
+def api_clear_cache(tid):
+    """清空自动任务的待爬缓存队列与已爬集合 (规则变更后重新开始用)"""
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    if task.get("mode") != "auto":
+        return jsonify({"error": "仅自动爬取任务支持清空缓存"}), 400
+    with JOBS_LOCK:
+        job = JOBS.get(tid)
+        if job and job.is_running():
+            return jsonify({"error": "任务正在运行，无法清空缓存"}), 409
+    auto = task.setdefault("auto", {})
+    auto["pending"] = []
+    auto["visited"] = []
+    task["updated_at"] = now_str()
+    store.upsert_task(task)
+    db.upsert_task(task)
+    return jsonify({"ok": True, "msg": "缓存队列已清空"})
+
+
 @app.route("/api/tasks/<tid>/probe", methods=["POST"])
 def api_task_probe(tid):
     """对已保存的自动任务执行试爬取 (使用保存的规则)"""

backfill.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+历史爬取记录回填数据库 (幂等, 可重复执行)
+
+用法:
+    python backfill.py                  # 回填所有任务的历史记录
+    python backfill.py <task_id> ...    # 只回填指定任务
+
+说明:
+    - 将本地 data/ 中的任务/运行/爬取结果(含成功与失败)全量写入 MySQL
+    - 网页内容不入库, 只写元数据; 重复执行不会产生重复记录
+"""
+import sys
+
+import db
+import store
+
+if __name__ == "__main__":
+    ids = [a for a in sys.argv[1:] if a.strip()] or None
+    db.init_db()
+    stats = db.sync_all_history(ids)
+    if ids:
+        print(f"[backfill] 已回填 {len(ids)} 个任务: {stats}")
+    else:
+        print(f"[backfill] 已回填全部任务: {stats}")

db.py

@@ -67,9 +67,7 @@ DDL = [
       source_url VARCHAR(2000),
       depth INT,
       attempts INT DEFAULT 1,
-      html_file VARCHAR(500),
+      base_file VARCHAR(500),
-      txt_file VARCHAR(500),
-      meta_file VARCHAR(500),
       image_count INT DEFAULT 0,
       image_files TEXT,
       UNIQUE KEY uk_run_url (run_id, url(500))
@@ -78,6 +76,17 @@ DDL = [
 ]
 
 
+def _migrate(conn):
+    """存量表结构迁移: 合并 html_file/txt_file/meta_file 为 base_file"""
+    with conn.cursor() as cur:
+        cur.execute("SHOW COLUMNS FROM crawl_results LIKE 'html_file'")
+        if cur.fetchone():
+            cur.execute("ALTER TABLE crawl_results ADD COLUMN base_file VARCHAR(500) NULL AFTER meta_file")
+            cur.execute("UPDATE crawl_results SET base_file = REPLACE(html_file, '.html', '') WHERE base_file IS NULL")
+            cur.execute("ALTER TABLE crawl_results DROP COLUMN html_file, DROP COLUMN txt_file, DROP COLUMN meta_file")
+            print("[db] 表结构迁移完成: html_file/txt_file/meta_file -> base_file")
+
+
 def _conn():
     cfg = dict(DB_CONFIG)
     cfg["database"] = DB_NAME
@@ -100,6 +109,7 @@ def init_db():
             cur.execute(f"CREATE DATABASE IF NOT EXISTS `{DB_NAME}` DEFAULT CHARACTER SET utf8mb4")
         conn.close()
         conn = _conn()
+        _migrate(conn)
         with conn.cursor() as cur:
             for ddl in DDL:
                 cur.execute(ddl)
@@ -219,8 +229,21 @@ def upsert_run(run):
 
 # ---------------- 爬取结果 (每页一条, 成功失败均记录) ----------------
 
+def _base_of(entry):
+    """从结果条目提取基础文件名 (html/txt/meta 三个后缀共用同一前缀)"""
+    for f in (entry.get("meta_file"), entry.get("html_file"), entry.get("txt_file")):
+        f = f or ""
+        if f.endswith(".meta.json"):
+            return f[:-10]
+        if f.endswith(".html"):
+            return f[:-5]
+        if f.endswith(".txt"):
+            return f[:-4]
+    return ""
+
+
 def insert_results(run, results):
-    """批量插入爬取结果 (增量)"""
+    """批量插入爬取结果 (增量); 文件只记基础名 base_file"""
     if not results:
         return
 
@@ -234,7 +257,7 @@ def insert_results(run, results):
                 r.get("status", "FAIL"), r.get("error"),
                 _dt(r.get("crawl_time")), (r.get("source_url") or "")[:2000],
                 r.get("depth"), r.get("attempts", 1),
-                r.get("html_file", ""), r.get("txt_file", ""), r.get("meta_file", ""),
+                _base_of(r),
                 len(r.get("images", []) or []),
                 _j([im.get("file") for im in (r.get("images") or [])]),
             ))
@@ -243,8 +266,8 @@ def insert_results(run, results):
                 """INSERT IGNORE INTO crawl_results
                    (run_id, task_id, mode, url, title, status, error,
                     crawl_time, source_url, depth, attempts,
-                    html_file, txt_file, meta_file, image_count, image_files)
+                    base_file, image_count, image_files)
-                   VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
+                   VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
                 rows,
             )
         conn.close()
@@ -262,3 +285,30 @@ def sync_run(run, persist):
         insert_results(run, results[synced:])
         run["_db_count"] = len(results)
         persist(run.get("task_id"), run)
+
+
+# ---------------- 历史数据回填 (幂等) ----------------
+
+def sync_all_history(task_ids=None):
+    """把本地 JSON 中的历史任务/运行/结果全量回填数据库
+    可重复执行 (INSERT IGNORE + 唯一键去重)
+    task_ids: 指定只回填的任务ID列表, 默认全部
+    返回统计 dict
+    """
+    import store as _store
+    stats = {"tasks": 0, "runs": 0, "results": 0}
+    for task in _store.load_tasks():
+        if task_ids and task["id"] not in task_ids:
+            continue
+        upsert_task(task)
+        stats["tasks"] += 1
+        for run in _store.get_runs(task["id"]):
+            upsert_run(run)
+            stats["runs"] += 1
+            results = run.get("results", [])
+            if results:
+                insert_results(run, results)
+                stats["results"] += len(results)
+                run["_db_count"] = len(results)
+                _store.save_run(task["id"], run)   # 记录已同步标记, 避免运行中重复插入
+    return stats

engine.py

@@ -21,6 +21,7 @@ from playwright_stealth import Stealth
 
 import notify
 import store
+import db
 
 HERE = os.path.dirname(os.path.abspath(__file__))
 DATA_DIR = os.path.join(HERE, "data")
@@ -567,33 +568,48 @@ class CrawlJob:
         return included
 
     def _crawl_auto(self):
+        """自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到上限
+        - max_depth: 0=无限制, N=只爬 N 层
+        - max_pages: 0=无限制, N=安全上限
+        - 提取但未爬取的链接持久化到 auto.pending, 已爬集合持久化到 auto.visited
+        - 停止后再次运行从缓存队列继续爬; 起始网址每次运行都重新爬(不去重)
+        """
         run = self.run
         auto = self.task.get("auto", {})
         seed = auto.get("seed_url", "")
-        max_pages = int(auto.get("max_pages", 50) or 50)
+        max_pages = int(auto.get("max_pages", 1000) or 0)     # 0 = 无限制
-        max_depth = int(auto.get("max_depth", 2) or 2)
+        max_depth = int(auto.get("max_depth", 0) or 0)        # 0 = 无限制
-        run["progress"]["total"] = max_pages
         out_dir = self._resolve_out_dir()
         run["out_dir"] = out_dir
         os.makedirs(out_dir, exist_ok=True)
+
+        # 恢复持久化状态: 已爬集合 + 上次未爬完的缓存队列
+        visited = set(auto.get("visited", []) or [])
+        pending = auto.get("pending", []) or []
+        # 起始网址每次运行都爬(不做去重), 缓存队列继续消费
+        queue = [(seed, 0, "")]
+        if pending:
+            queue.extend((p["url"], p.get("depth", 0), p.get("source", "")) for p in pending)
+        queued = set(visited)
+        for u, _d, _s in queue:
+            queued.add(normalize_url(u))
+
+        run["progress"]["total"] = len(queue)
         self._persist()
 
         p, browser, ctx, page, cookie_file = self._open_browser()
-        queue = [(seed, 0, "")]               # (url, depth, 来源链接)
-        visited = set()                        # 规范化 URL 去重
-        queued = set([normalize_url(seed)])
-        idx = 0
         try:
             while queue and not self._stop.is_set():
                 self._wait_if_paused()
+                if max_pages > 0 and len(visited) >= max_pages:
+                    self._log("info", f"达到最大页数上限 {max_pages}, 停止")
+                    break
                 url, depth, src = queue.pop(0)
                 key = normalize_url(url)
-                if key in visited:
+                if key in visited and url != seed:     # 起始网址不去重, 其余已爬跳过
                     continue
-                if len(visited) >= max_pages:
-                    break
                 visited.add(key)
-                idx += 1
+                idx = len(visited)
                 run["progress"]["current_url"] = url
                 run["progress"]["done"] = len(visited)
                 self._persist()
@@ -602,7 +618,8 @@ class CrawlJob:
                 run["results"].append(entry)
                 self._bump_stats(entry)
                 self._persist()
-                if entry["status"] == "OK" and depth < max_depth:
+                # 无深度限制或未达深度限制时持续发现链接
+                if entry["status"] == "OK" and (max_depth == 0 or depth < max_depth):
                     for link in self._discover_links(page):
                         lk = normalize_url(link)
                         if lk not in visited and lk not in queued:
@@ -610,7 +627,18 @@ class CrawlJob:
                             queue.append((link, depth + 1, url))
                 if entry["status"] == "OK":
                     self._delay()
+            run["progress"]["total"] = len(visited)
+            if not self._stop.is_set() and len(queue) == 0:
+                self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)")
         finally:
             self._close_browser(p, browser, ctx, cookie_file)
-        run["progress"]["total"] = len(visited)
+            # 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
+            try:
+                auto["pending"] = [
+                    {"url": u, "depth": d, "source": s} for u, d, s in queue]
+                auto["visited"] = list(visited)
+                store.upsert_task(self.task)
+                db.upsert_task(self.task)
+            except Exception:
+                pass
         self._persist()

static/app.js

@@ -82,7 +82,7 @@ function taskStatusBadge(t) {
 const GROUPS = [
   { key: "batch", label: "📄 批量爬取", desc: "一次性爬取指定网址列表" },
   { key: "scheduled", label: "⏰ 定时爬取", desc: "按间隔或 cron 表达式定时执行" },
-  { key: "auto", label: "🤖 自动爬取", desc: "从起始网址自动发现链接并爬取" },
+  { key: "auto", label: "🤖 自动爬取", desc: "从起始网址自动发现链接，持续递归爬取到无新链接为止" },
 ];
 
 function renderTasks() {
@@ -434,8 +434,8 @@ async function openEdit(tid) {
     f.elements["exclude"].value = (t.auto.exclude || []).join("\n");
     f.elements["same_domain"].checked = t.auto.same_domain !== false;
     f.elements["use_regex"].checked = !!t.auto.use_regex;
-    f.elements["max_pages"].value = t.auto.max_pages ?? 50;
+    f.elements["max_pages"].value = t.auto.max_pages ?? 1000;
-    f.elements["max_depth"].value = t.auto.max_depth ?? 2;
+    f.elements["max_depth"].value = t.auto.max_depth ?? 0;
   }
   syncScheduleUI();
   $("formHint").textContent = t.running
@@ -481,8 +481,8 @@ async function submitForm(e) {
       exclude: splitLines(f.elements["exclude"].value),
       same_domain: f.elements["same_domain"].checked,
       use_regex: f.elements["use_regex"].checked,
-      max_pages: parseInt(f.elements["max_pages"].value) || 50,
+      max_pages: parseInt(f.elements["max_pages"].value) || 0,
-      max_depth: parseInt(f.elements["max_depth"].value) || 2,
+      max_depth: parseInt(f.elements["max_depth"].value) || 0,
     };
   }
   if (mode === "scheduled") {
@@ -544,7 +544,10 @@ function renderDetail() {
       <span>任务ID: <b>${t.id}</b></span>
       <span>创建: <b>${fmtTime(t.created_at)}</b></span>
       <span>输出: <b>${esc(cfg.out_dir || "out/" + t.id)}</b></span>
-      ${t.mode === "auto" ? `<span>起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b></span>` : ""}
+      ${t.mode === "auto" ? `
+        <span>起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b></span>
+        <span>待爬缓存: <b>${(t.auto && (t.auto.pending || []).length) || 0}</b> 条（停止后可继续爬取）</span>
+        <span>已爬: <b>${(t.auto && (t.auto.visited || []).length) || 0}</b> 页</span>` : ""}
     </div>`;
 
   let sched = "";
@@ -565,7 +568,8 @@ function renderDetail() {
         <button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
         <button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
       : `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`}
-      ${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>` : ""}
+      ${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>
+        <button class="btn" ${t.running ? "disabled" : ""} onclick="clearCache('${t.id}')">🧹 清空缓存</button>` : ""}
       <button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
     </div>`;
 
@@ -721,6 +725,15 @@ async function probeTask(tid) {
   } catch (e) { toast(e.message, true); }
 }
 
+async function clearCache(tid) {
+  if (!confirm("清空待爬缓存与已爬记录？\n下次运行将从起始网址重新开始爬取。")) return;
+  try {
+    const r = await api(`/api/tasks/${tid}/clear_cache`, { method: "POST" });
+    toast(r.msg || "已清空");
+    openDetail(tid);
+  } catch (e) { toast(e.message, true); }
+}
+
 function renderProbe(r) {
   const body = $("probeBody");
   if (!r.ok) {

static/index.html

@@ -116,7 +116,7 @@
         </div>
 
         <div id="autoBox" class="hidden box">
-          <div class="field"><label>起始网址 *（系统将自动发现符合规则的链接并爬取）</label>
+          <div class="field"><label>起始网址 *（自动发现符合规则的链接，持续递归爬取直到无新链接可爬）</label>
             <div class="row2">
               <input name="seed_url" placeholder="https://example.com/news" style="flex:1">
               <button type="button" class="btn sm" id="btnProbe">🧪 试爬取</button>
@@ -133,8 +133,10 @@
             <div class="field check"><label><input name="use_regex" type="checkbox"> 规则按正则匹配</label></div>
           </div>
           <div class="row2">
-            <div class="field"><label>最大页数</label><input name="max_pages" type="number" value="50"></div>
+            <div class="field"><label>最大爬取深度（0=无限制，N=只爬 N 层）</label>
-            <div class="field"><label>最大爬取深度</label><input name="max_depth" type="number" value="2"></div>
+              <input name="max_depth" type="number" min="0" value="0"></div>
+            <div class="field"><label>最大页数（0=无限制，默认 1000 作安全上限）</label>
+              <input name="max_pages" type="number" min="0" value="1000"></div>
           </div>
         </div>
       </div>