新增打包导出: 任务输出目录打包为 zip, 支持网页下载或发送到指定邮箱(默认 wlq@tphai.com)

- GET /api/export?task_id= 打包输出目录为 zip 并下载 (zip 内按任务名建顶层目录) - POST /api/export/email 打包后通过 send_email.py 发送附件到邮箱, 邮箱可指定, 默认 wlq@tphai.com - 详情弹窗新增 [📦 打包下载] [📧 发邮箱] 按钮, 打包中按钮禁用防重复点击 - notify.py 重构: send_attachment(subject, body, to, attach) 通用附件发送 - 打包文件存 data/exports/, 自动清理只保留最近 10 个 - 实测: cnblogs-auto 66MB/1606文件 -> 11.9MB zip 仅1.5s; 邮件发送成功 2.2s
2026-08-12 09:41:59 +08:00
parent 0482284cc0
commit 56f30f78c4
8 changed files with 172 additions and 4 deletions
@@ -4,14 +4,17 @@
 启动: /home/hz1/miniconda3/envs/openclaw/bin/python app.py  (默认端口 16062)
 """
 import os
+import re
 import threading
 import time
+import zipfile
 from datetime import datetime

 from flask import Flask, jsonify, request, send_file, send_from_directory

 import store
 import db
+import notify
 from engine import CrawlJob, probe_links
 from scheduler import Scheduler, cron_next, interval_delta

@@ -674,6 +677,103 @@ def api_file():
    return send_file(full)


+# ---------------- API: 打包导出 ----------------
+
+EXPORT_DIR = os.path.join(HERE, "data", "exports")
+EXPORT_KEEP = 10          # 最多保留的打包文件数
+
+
+def _safe_zip_name(task):
+    """任务名安全化为文件名 (保留中文, 去非法字符)"""
+    name = re.sub(r'[\\/:*?"<>|\s]+', "_", task.get("name", "")).strip("_")
+    return (name or task["id"])[:60]
+
+
+def _make_zip(task):
+    """把任务输出目录打包为 zip, 返回 (zip_path, err) 或 (None, 错误信息)"""
+    out_dir = os.path.realpath(resolve_out_dir(task))
+    if not os.path.isdir(out_dir):
+        return None, "输出目录不存在"
+    os.makedirs(EXPORT_DIR, exist_ok=True)
+    zip_path = os.path.join(EXPORT_DIR, f"{task['id']}_{int(time.time())}.zip")
+    prefix = _safe_zip_name(task) + "/"
+    count = 0
+    try:
+        with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+            for root, _dirs, files in os.walk(out_dir):
+                for f in files:
+                    full = os.path.join(root, f)
+                    rel = os.path.relpath(full, out_dir)
+                    zf.write(full, prefix + rel)
+                    count += 1
+    except Exception as e:
+        try:
+            os.remove(zip_path)
+        except OSError:
+            pass
+        return None, f"打包失败: {e}"
+    _cleanup_exports()
+    return zip_path, None
+
+
+def _cleanup_exports():
+    """清理旧的打包文件, 只保留最近 EXPORT_KEEP 个"""
+    try:
+        files = sorted(
+            (os.path.join(EXPORT_DIR, f) for f in os.listdir(EXPORT_DIR)
+             if f.endswith(".zip")),
+            key=os.path.getmtime, reverse=True,
+        )
+        for f in files[EXPORT_KEEP:]:
+            os.remove(f)
+    except Exception:
+        pass
+
+
+@app.route("/api/export")
+def api_export():
+    """打包任务输出目录为 zip 并提供下载"""
+    tid = request.args.get("task_id", "")
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    zip_path, err = _make_zip(task)
+    if err:
+        return jsonify({"error": err}), 400
+    return send_file(
+        zip_path, as_attachment=True,
+        download_name=_safe_zip_name(task) + ".zip",
+        mimetype="application/zip",
+    )
+
+
+@app.route("/api/export/email", methods=["POST"])
+def api_export_email():
+    """打包任务输出目录为 zip 并发送到指定邮箱 (默认 wlq@tphai.com)"""
+    body = request.get_json(force=True) or {}
+    tid = body.get("task_id", "")
+    email = (body.get("email") or "").strip() or "wlq@tphai.com"
+    task = store.get_task(tid)
+    if not task:
+        return jsonify({"error": "任务不存在"}), 404
+    if "@" not in email:
+        return jsonify({"error": "邮箱格式不正确"}), 400
+    zip_path, err = _make_zip(task)
+    if err:
+        return jsonify({"error": err}), 400
+    size_mb = round(os.path.getsize(zip_path) / 1048576, 1)
+    body_text = (f"项目: {task['name']}\n"
+                 f"输出目录: {resolve_out_dir(task)}\n"
+                 f"打包文件: {os.path.basename(zip_path)}\n"
+                 f"压缩包大小: {size_mb} MB\n\n"
+                 f"打包时间: {now_str()}")
+    ok, msg = notify.send_attachment(
+        f"[爬虫打包] {task['name']}", body_text, email, zip_path)
+    if not ok:
+        return jsonify({"error": f"邮件发送失败: {msg}"}), 500
+    return jsonify({"ok": True, "msg": f"已发送到 {email} (zip {size_mb} MB)"})
+
+
 # ---------------- 启动 ----------------

 scheduler = Scheduler(start_run)
@@ -33,11 +33,24 @@ def notify_email(task, run):
    if len(results) > 50:
        lines.append(f"  ... 共 {len(results)} 条")
    body = "\n".join(lines)
+    return send_attachment(f"[爬虫完成] {task['name']}", body, to)
+
+
+def send_attachment(subject, body, to, attach_path=None):
+    """发送带附件的邮件 (复用 send_email.py), 返回 (bool, msg)
+    attach_path: 附件文件路径, 可多个
+    """
+    if not os.path.exists(SEND_EMAIL):
+        return False, "send_email.py 不存在"
+    cmd = [sys.executable, SEND_EMAIL, subject, body, "--to", to]
+    if attach_path:
+        if isinstance(attach_path, str):
+            attach_path = [attach_path]
+        for p in attach_path:
+            if os.path.isfile(p):
+                cmd += ["--attach", p]
    try:
-        r = subprocess.run(
-            [sys.executable, SEND_EMAIL, f"[爬虫完成] {task['name']}", body, "--to", to],
-            timeout=60, capture_output=True,
-        )
+        r = subprocess.run(cmd, timeout=600, capture_output=True)
        return r.returncode == 0, (r.stderr or b"").decode(errors="ignore")[:200]
    except Exception as e:
        return False, str(e)
@@ -606,6 +606,8 @@ function renderDetail() {
      ${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>
        <button class="btn" ${t.running ? "disabled" : ""} onclick="clearCache('${t.id}')">🧹 清空缓存</button>` : ""}
      <button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
+      <button class="btn" onclick="exportZip('${t.id}')">📦 打包下载</button>
+      <button class="btn" onclick="exportEmail('${t.id}')">📧 发邮箱</button>
    </div>`;

  const chips = runs.map((r) => `
@@ -732,6 +734,59 @@ function stopLogPoll() {
  if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; }
 }

+/* ---------------- 打包导出 ---------------- */
+async function exportZip(tid) {
+  const btn = window.event && window.event.target;
+  if (btn) { btn.disabled = true; btn.textContent = "⏳ 打包中..."; }
+  try {
+    const res = await fetch(`/api/export?task_id=${tid}`);
+    if (!res.ok) {
+      let d = null;
+      try { d = await res.json(); } catch (e) { /* ignore */ }
+      throw new Error((d && d.error) || `HTTP ${res.status}`);
+    }
+    let fname = "export.zip";
+    const disp = res.headers.get("Content-Disposition") || "";
+    const m = disp.match(/filename\*?=(?:UTF-8'')?"?([^";]+)"?/i);
+    if (m) { try { fname = decodeURIComponent(m[1]); } catch (e) { fname = m[1]; } }
+    const blob = await res.blob();
+    const a = document.createElement("a");
+    a.href = URL.createObjectURL(blob);
+    a.download = fname;
+    document.body.appendChild(a);
+    a.click();
+    a.remove();
+    setTimeout(() => URL.revokeObjectURL(a.href), 5000);
+    toast(`打包下载完成 ${fname} (${(blob.size / 1048576).toFixed(1)} MB)`);
+  } catch (e) {
+    toast("打包失败: " + e.message, true);
+  } finally {
+    if (btn) { btn.disabled = false; btn.textContent = "📦 打包下载"; }
+  }
+}
+
+async function exportEmail(tid) {
+  const t = (state.detail.task && state.detail.task.id === tid)
+    ? state.detail.task : state.tasks.find((x) => x.id === tid);
+  const def = (t && t.config && t.config.notify_email) || "wlq@tphai.com";
+  const email = prompt("发送到邮箱（留空默认 wlq@tphai.com）:", def);
+  if (email === null) return;   // 用户取消
+  const btn = window.event && window.event.target;
+  if (btn) { btn.disabled = true; btn.textContent = "⏳ 打包发送中..."; }
+  try {
+    const r = await api("/api/export/email", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ task_id: tid, email: email.trim() || "wlq@tphai.com" }),
+    });
+    toast(r.msg || "已发送");
+  } catch (e) {
+    toast("发送失败: " + e.message, true);
+  } finally {
+    if (btn) { btn.disabled = false; btn.textContent = "📧 发邮箱"; }
+  }
+}
+
 /* ---------------- 试爬取 ---------------- */
 function collectProbeFromForm() {
  const f = $("taskForm");