新增打包导出: 任务输出目录打包为 zip, 支持网页下载或发送到指定邮箱(默认 wlq@tphai.com)

- GET /api/export?task_id= 打包输出目录为 zip 并下载 (zip 内按任务名建顶层目录)
- POST /api/export/email 打包后通过 send_email.py 发送附件到邮箱, 邮箱可指定, 默认 wlq@tphai.com
- 详情弹窗新增 [📦 打包下载] [📧 发邮箱] 按钮, 打包中按钮禁用防重复点击
- notify.py 重构: send_attachment(subject, body, to, attach) 通用附件发送
- 打包文件存 data/exports/, 自动清理只保留最近 10 个
- 实测: cnblogs-auto 66MB/1606文件 -> 11.9MB zip 仅1.5s; 邮件发送成功 2.2s

app.py

@@ -4,14 +4,17 @@
 启动: /home/hz1/miniconda3/envs/openclaw/bin/python app.py  (默认端口 16062)
 """
 import os
+import re
 import threading
 import time
+import zipfile
 from datetime import datetime
 
 from flask import Flask, jsonify, request, send_file, send_from_directory
 
 import store
 import db
+import notify
 from engine import CrawlJob, probe_links
 from scheduler import Scheduler, cron_next, interval_delta
 
@@ -674,6 +677,103 @@ def api_file():
     return send_file(full)
 
 
+# ---------------- API: 打包导出 ----------------
+
+EXPORT_DIR = os.path.join(HERE, "data", "exports")
+EXPORT_KEEP = 10          # 最多保留的打包文件数
+
+
+def _safe_zip_name(task):
+    """任务名安全化为文件名 (保留中文, 去非法字符)"""
+    name = re.sub(r'[\\/:*?"<>|\s]+', "_", task.get("name", "")).strip("_")
+    return (name or task["id"])[:60]
+
+
+def _make_zip(task):
+    """把任务输出目录打包为 zip, 返回 (zip_path, err) 或 (None, 错误信息)"""
+    out_dir = os.path.realpath(resolve_out_dir(task))
+    if not os.path.isdir(out_dir):
+        return None, "输出目录不存在"
+    os.makedirs(EXPORT_DIR, exist_ok=True)
+    zip_path = os.path.join(EXPORT_DIR, f"{task['id']}_{int(time.time())}.zip")
+    prefix = _safe_zip_name(task) + "/"
+    count = 0
+    try:
+        with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+            for root, _dirs, files in os.walk(out_dir):
+                for f in files:
+                    full = os.path.join(root, f)
+                    rel = os.path.relpath(full, out_dir)
+                    zf.write(full, prefix + rel)
+                    count += 1
+    except Exception as e:
+        try:
+            os.remove(zip_path)
+        except OSError:
+            pass
+        return None, f"打包失败: {e}"
+    _cleanup_exports()
+    return zip_path, None
+
+
+def _cleanup_exports():
+    """清理旧的打包文件, 只保留最近 EXPORT_KEEP 个"""
+    try:
+        files = sorted(
+            (os.path.join(EXPORT_DIR, f) for f in os.listdir(EXPORT_DIR)
+             if f.endswith(".zip")),
+            key=os.path.getmtime, reverse=True,
+        )
+        for f in files[EXPORT_KEEP:]:
+            os.remove(f)
+    except Exception:
+        pass
+
+
+@app.route("/api/export")
+def api_export():
+    """打包任务输出目录为 zip 并提供下载"""
+    tid = request.args.get("task_id", "")
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    zip_path, err = _make_zip(task)
+    if err:
+        return jsonify({"error": err}), 400
+    return send_file(
+        zip_path, as_attachment=True,
+        download_name=_safe_zip_name(task) + ".zip",
+        mimetype="application/zip",
+    )
+
+
+@app.route("/api/export/email", methods=["POST"])
+def api_export_email():
+    """打包任务输出目录为 zip 并发送到指定邮箱 (默认 wlq@tphai.com)"""
+    body = request.get_json(force=True) or {}
+    tid = body.get("task_id", "")
+    email = (body.get("email") or "").strip() or "wlq@tphai.com"
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    if "@" not in email:
+        return jsonify({"error": "邮箱格式不正确"}), 400
+    zip_path, err = _make_zip(task)
+    if err:
+        return jsonify({"error": err}), 400
+    size_mb = round(os.path.getsize(zip_path) / 1048576, 1)
+    body_text = (f"项目: {task['name']}\n"
+                 f"输出目录: {resolve_out_dir(task)}\n"
+                 f"打包文件: {os.path.basename(zip_path)}\n"
+                 f"压缩包大小: {size_mb} MB\n\n"
+                 f"打包时间: {now_str()}")
+    ok, msg = notify.send_attachment(
+        f"[爬虫打包] {task['name']}", body_text, email, zip_path)
+    if not ok:
+        return jsonify({"error": f"邮件发送失败: {msg}"}), 500
+    return jsonify({"ok": True, "msg": f"已发送到 {email} (zip {size_mb} MB)"})
+
+
 # ---------------- 启动 ----------------
 
 scheduler = Scheduler(start_run)

engine.py

@@ -36,6 +36,19 @@ _CHALLENGE_MARKS = [
     "checking your browser", "enable javascript and cookies",
 ]
 
+# 反爬拦截类错误信号: 命中后不再重试 (重试无意义且拖慢任务)
+_ANTI_CRAWL_MARKS = [
+    "403", "429", "503", "forbidden", "access denied", "too many requests",
+    "captcha", "cloudflare", "challenge", "verify you are human",
+    "just a moment", "blocked", "被反爬拦截",
+]
+
+
+def is_anti_crawl_error(err):
+    """判断错误是否属于反爬拦截 (此类失败重试也无法通过, 直接放弃该页)"""
+    s = str(err or "").lower()
+    return any(m in s for m in _ANTI_CRAWL_MARKS)
+
 
 def is_challenge_page(title, html):
     """判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)"""
@@ -56,6 +69,7 @@ def safe_name(url, idx):
 def _settle_wait(page, timeout_s):
     """等待页面稳定(无 stop/pause 检查, 供试爬取等独立流程使用)"""
     last_title, stable = "", 0
+    challenge_hits = 0
     start = time.time()
     while time.time() - start < timeout_s:
         time.sleep(1)
@@ -65,8 +79,12 @@ def _settle_wait(page, timeout_s):
         except Exception:
             continue
         if is_challenge_page(title, html):
+            challenge_hits += 1
+            if challenge_hits >= 8:        # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
+                return True, title, html
             stable = 0
             continue
+        challenge_hits = 0
         if title == last_title:
             stable += 1
             if stable >= 2 and len(html) > 1000:
@@ -306,6 +324,7 @@ class CrawlJob:
 
     def _wait_page_settle(self, page, timeout_s):
         last_title, stable = "", 0
+        challenge_hits = 0
         start = time.time()
         while time.time() - start < timeout_s:
             if self._stop.is_set():
@@ -318,8 +337,12 @@ class CrawlJob:
             except Exception:
                 continue                      # 正在跳转
             if is_challenge_page(title, html):
+                challenge_hits += 1
+                if challenge_hits >= 8:       # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
+                    return True, title, html
                 stable = 0
                 continue
+            challenge_hits = 0
             if title == last_title:
                 stable += 1
                 if stable >= 2 and len(html) > 1000:
@@ -460,6 +483,11 @@ class CrawlJob:
                 entry["error"] = str(e)
                 entry["crawl_time"] = store.now_str()
                 self._log("warn", f"第{attempt + 1}次失败 {url}: {e}")
+                if is_anti_crawl_error(e):
+                    # 反爬拦截: 重试也过不去, 直接放弃, 不再消耗重试次数
+                    entry["error"] = f"反爬拦截, 跳过重试: {e}"
+                    self._log("warn", f"判定为反爬拦截, 放弃重试: {url}")
+                    break
                 if attempt < retries:
                     self._wait_if_paused()
                     t0 = time.time()

notify.py

@@ -33,11 +33,24 @@ def notify_email(task, run):
     if len(results) > 50:
         lines.append(f"  ... 共 {len(results)} 条")
     body = "\n".join(lines)
+    return send_attachment(f"[爬虫完成] {task['name']}", body, to)
+
+
+def send_attachment(subject, body, to, attach_path=None):
+    """发送带附件的邮件 (复用 send_email.py), 返回 (bool, msg)
+    attach_path: 附件文件路径, 可多个
+    """
+    if not os.path.exists(SEND_EMAIL):
+        return False, "send_email.py 不存在"
+    cmd = [sys.executable, SEND_EMAIL, subject, body, "--to", to]
+    if attach_path:
+        if isinstance(attach_path, str):
+            attach_path = [attach_path]
+        for p in attach_path:
+            if os.path.isfile(p):
+                cmd += ["--attach", p]
     try:
-        r = subprocess.run(
-            [sys.executable, SEND_EMAIL, f"[爬虫完成] {task['name']}", body, "--to", to],
-            timeout=60, capture_output=True,
-        )
+        r = subprocess.run(cmd, timeout=600, capture_output=True)
         return r.returncode == 0, (r.stderr or b"").decode(errors="ignore")[:200]
     except Exception as e:
         return False, str(e)

static/app.js

@@ -606,6 +606,8 @@ function renderDetail() {
       ${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>
         <button class="btn" ${t.running ? "disabled" : ""} onclick="clearCache('${t.id}')">🧹 清空缓存</button>` : ""}
       <button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
+      <button class="btn" onclick="exportZip('${t.id}')">📦 打包下载</button>
+      <button class="btn" onclick="exportEmail('${t.id}')">📧 发邮箱</button>
     </div>`;
 
   const chips = runs.map((r) => `
@@ -732,6 +734,59 @@ function stopLogPoll() {
   if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; }
 }
 
+/* ---------------- 打包导出 ---------------- */
+async function exportZip(tid) {
+  const btn = window.event && window.event.target;
+  if (btn) { btn.disabled = true; btn.textContent = "⏳ 打包中..."; }
+  try {
+    const res = await fetch(`/api/export?task_id=${tid}`);
+    if (!res.ok) {
+      let d = null;
+      try { d = await res.json(); } catch (e) { /* ignore */ }
+      throw new Error((d && d.error) || `HTTP ${res.status}`);
+    }
+    let fname = "export.zip";
+    const disp = res.headers.get("Content-Disposition") || "";
+    const m = disp.match(/filename\*?=(?:UTF-8'')?"?([^";]+)"?/i);
+    if (m) { try { fname = decodeURIComponent(m[1]); } catch (e) { fname = m[1]; } }
+    const blob = await res.blob();
+    const a = document.createElement("a");
+    a.href = URL.createObjectURL(blob);
+    a.download = fname;
+    document.body.appendChild(a);
+    a.click();
+    a.remove();
+    setTimeout(() => URL.revokeObjectURL(a.href), 5000);
+    toast(`打包下载完成 ${fname} (${(blob.size / 1048576).toFixed(1)} MB)`);
+  } catch (e) {
+    toast("打包失败: " + e.message, true);
+  } finally {
+    if (btn) { btn.disabled = false; btn.textContent = "📦 打包下载"; }
+  }
+}
+
+async function exportEmail(tid) {
+  const t = (state.detail.task && state.detail.task.id === tid)
+    ? state.detail.task : state.tasks.find((x) => x.id === tid);
+  const def = (t && t.config && t.config.notify_email) || "wlq@tphai.com";
+  const email = prompt("发送到邮箱（留空默认 wlq@tphai.com）:", def);
+  if (email === null) return;   // 用户取消
+  const btn = window.event && window.event.target;
+  if (btn) { btn.disabled = true; btn.textContent = "⏳ 打包发送中..."; }
+  try {
+    const r = await api("/api/export/email", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ task_id: tid, email: email.trim() || "wlq@tphai.com" }),
+    });
+    toast(r.msg || "已发送");
+  } catch (e) {
+    toast("发送失败: " + e.message, true);
+  } finally {
+    if (btn) { btn.disabled = false; btn.textContent = "📧 发邮箱"; }
+  }
+}
+
 /* ---------------- 试爬取 ---------------- */
 function collectProbeFromForm() {
   const f = $("taskForm");