v1.0.2: 自动爬取试爬取功能 + 每个页面/图片生成操作信息元数据(模式/时间/网址/来源链接/深度等)

app.py

@@ -10,7 +10,7 @@ from datetime import datetime
 from flask import Flask, jsonify, request, send_file, send_from_directory
 
 import store
-from engine import CrawlJob
+from engine import CrawlJob, probe_links
 from scheduler import Scheduler, cron_next, interval_delta
 
 HERE = os.path.dirname(os.path.abspath(__file__))
@@ -337,6 +337,48 @@ def api_stats():
     })
 
 
+# ---------------- API: 试爬取 ----------------
+
+@app.route("/api/probe", methods=["POST"])
+def api_probe():
+    """试爬取: 按表单给出的规则探测起始页, 返回将爬取的链接清单"""
+    body = request.get_json(force=True) or {}
+    seed = (body.get("seed_url") or "").strip()
+    if not seed:
+        return jsonify({"error": "请填写起始网址"}), 400
+    if not seed.startswith("http"):
+        seed = "https://" + seed
+    result = probe_links(
+        seed,
+        include=[x.strip() for x in (body.get("include") or []) if x.strip()],
+        exclude=[x.strip() for x in (body.get("exclude") or []) if x.strip()],
+        same_domain=body.get("same_domain", True),
+        use_regex=bool(body.get("use_regex", False)),
+        timeout=int(body.get("timeout") or 45),
+    )
+    return jsonify(result)
+
+
+@app.route("/api/tasks/<tid>/probe", methods=["POST"])
+def api_task_probe(tid):
+    """对已保存的自动任务执行试爬取 (使用保存的规则)"""
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    if task.get("mode") != "auto":
+        return jsonify({"error": "仅自动爬取任务支持试爬取"}), 400
+    auto = task.get("auto", {})
+    result = probe_links(
+        auto.get("seed_url", ""),
+        include=auto.get("include", []),
+        exclude=auto.get("exclude", []),
+        same_domain=auto.get("same_domain", True),
+        use_regex=bool(auto.get("use_regex", False)),
+        timeout=int((task.get("config") or {}).get("timeout", 60)),
+    )
+    return jsonify(result)
+
+
 # ---------------- API: 运行记录与文件 ----------------
 
 @app.route("/api/runs/<rid>")