Compare commits

..
7 Commits
Author SHA1 Message Date
hz4th_coder ec053266a7 v1.4.0 修复反爬误判+浏览器崩溃空转, 详情页多元翻页
- 修复反爬误判: is_challenge_page 仅匹配 title 关键词 + html 前 20KB 强特征标记,
  不再全文匹配 captcha/challenge 等词 (博客园正文提 captcha 的页面被误杀)
- 修复浏览器崩溃后空转: 检测 Target page/context/browser closed,
  自动重启浏览器并重试当前 URL (不消耗重试配额, 单页最多重建3次);
  页面读取连续异常3次快速失败; 新增 browser_max_pages 配置定期重启浏览器防内存膨胀
- 详情页多元翻页: 首页/末页/页码列表(带省略号)/页码跳转输入框(回车/GO)/每页条数选择(50/100/200/500)
2026-08-14 10:28:25 +08:00
hz4th_coder 32eebf3dd1 auto 任务运行中实时显示已爬/待爬缓存数; 人工停止后继续爬取验证
- engine: CrawlJob 增加 _auto_visited/_auto_pending 运行中实时状态 (含发现新链接入队后刷新)
- app.py 详情接口: 任务运行中优先读内存实时值, 不再显示上次运行结束的旧数字
- 端到端验证: 人工停止(已爬3,缓存18) → 继续爬取(跳过seed,爬完18页,缓存0) 
- 运行中采样: visited 1→2 实时增长, pending 15→14 同步递减 
2026-08-12 10:19:19 +08:00
hz4th_coder d9c9f0c633 chore: data/auto_state 为运行时爬取状态, 移出版本控制 (.gitignore) 2026-08-12 09:57:53 +08:00
hz4th_coder 909e8e01b5 新增继续爬取: auto 任务达 max_pages 上限停止后, 有待爬缓存链接时可在详情页一键继续
- POST /api/tasks/<tid>/continue: 校验 auto 模式 + pending 非空, skip_seed 启动
- engine skip_seed: 跳过起始网址直接从缓存队列消费 (不重爬 seed, 不重复入 visited)
- 关键修复: max_pages 原为累计上限(visited>=max_pages), 已达标任务无法继续; 继续模式改为按本次新增页数重新计算上限, 可反复继续直到缓存耗尽
- 详情弹窗 auto 任务且有缓存时显示 [ 继续爬取(缓存 N 条)] 按钮
- 端到端测试: max_pages=2 链路 执行(seed+p1,缓存4) → 继续(p2,p3,缓存2) → 继续(p4,p5,缓存0) → 无缓存报错 
2026-08-12 09:57:46 +08:00
hz4th_coder e749d70a43 chore: data/exports 为运行时打包产物, 移出版本控制 (.gitignore) 2026-08-12 09:43:32 +08:00
hz4th_coder 56f30f78c4 新增打包导出: 任务输出目录打包为 zip, 支持网页下载或发送到指定邮箱(默认 wlq@tphai.com)
- GET /api/export?task_id= 打包输出目录为 zip 并下载 (zip 内按任务名建顶层目录)
- POST /api/export/email 打包后通过 send_email.py 发送附件到邮箱, 邮箱可指定, 默认 wlq@tphai.com
- 详情弹窗新增 [📦 打包下载] [📧 发邮箱] 按钮, 打包中按钮禁用防重复点击
- notify.py 重构: send_attachment(subject, body, to, attach) 通用附件发送
- 打包文件存 data/exports/, 自动清理只保留最近 10 个
- 实测: cnblogs-auto 66MB/1606文件 -> 11.9MB zip 仅1.5s; 邮件发送成功 2.2s
2026-08-12 09:41:59 +08:00
hz4th_coder 0482284cc0 反爬拦截不再重试: 判定为反爬类错误(403/429/503/Cloudflare/captcha/验证页等)直接放弃该页, 节省重试时间; 验证页持续8秒即判定反爬, 不再干等满超时(默认60s)
- engine.is_anti_crawl_error: 反爬错误信号关键词判定
- _retry_crawl: 反爬错误直接 break, entry.error 标注'反爬拦截, 跳过重试'
- _wait_page_settle/_settle_wait: 连续8秒检测到验证页即判定反爬尽早返回
- 集成测试: 反爬页 attempts=1 且 8.1s 放弃; 非反爬错误仍 attempts=retry_count+1 正常重试
2026-08-12 09:37:23 +08:00
7 changed files with 464 additions and 46925 deletions
+2
View File
@@ -4,3 +4,5 @@ data/*.json
data/cookies_*.json
logs/
out/
data/exports/
data/auto_state/
+133 -5
View File
@@ -4,14 +4,17 @@
启动: /home/hz1/miniconda3/envs/openclaw/bin/python app.py (默认端口 16062)
"""
import os
import re
import threading
import time
import zipfile
from datetime import datetime
from flask import Flask, jsonify, request, send_file, send_from_directory
import store
import db
import notify
from engine import CrawlJob, probe_links
from scheduler import Scheduler, cron_next, interval_delta
@@ -101,13 +104,16 @@ def persist_cb(task_id, run):
print(f"[db] 同步失败: {e}", flush=True)
def start_run(task):
"""为任务启动一次爬取, 返回 (run, error)"""
def start_run(task, skip_seed=False):
"""为任务启动一次爬取, 返回 (run, error)
skip_seed: auto 任务继续爬取模式, 跳过起始网址直接从缓存队列消费
"""
with JOBS_LOCK:
job = JOBS.get(task["id"])
if job and job.is_running():
return None, "该任务已有正在运行的爬取"
run = make_run(task)
run["skip_seed"] = bool(skip_seed)
store.add_run(task["id"], run)
job = CrawlJob(task, run, persist_cb)
JOBS[task["id"]] = job
@@ -302,11 +308,19 @@ def api_task_detail(tid):
result["run_page"] = page
result["run_pages"] = pages
result["run_page_size"] = page_size
# auto 状态数量 (独立文件)
# auto 状态数量: 运行中优先读内存实时值, 否则读状态文件
if task.get("mode") == "auto":
live_v = live_p = None
with JOBS_LOCK:
job = JOBS.get(tid)
if job:
live_v = getattr(job, "_auto_visited", None)
live_p = getattr(job, "_auto_pending", None)
st = store.load_auto_state(tid, task)
result["auto_pending_count"] = len(st.get("pending", []))
result["auto_visited_count"] = len(st.get("visited", []))
result["auto_pending_count"] = (
live_p if live_p is not None else len(st.get("pending", [])))
result["auto_visited_count"] = (
live_v if live_v is not None else len(st.get("visited", [])))
return jsonify(result)
@@ -421,6 +435,23 @@ def api_trash_clear():
# ---------------- API: 运行控制 ----------------
@app.route("/api/tasks/<tid>/continue", methods=["POST"])
def api_continue(tid):
"""继续爬取: auto 任务从待爬缓存队列接着爬 (跳过起始网址, 保留已爬集合)"""
task = store.get_task(tid)
if not task:
return jsonify({"error": "任务不存在"}), 404
if task.get("mode") != "auto":
return jsonify({"error": "仅自动爬取任务支持继续爬取"}), 400
st = store.load_auto_state(tid, task)
if not st.get("pending"):
return jsonify({"error": "没有待爬缓存链接,无需继续"}), 400
run, err = start_run(task, skip_seed=True)
if err:
return jsonify({"error": err}), 409
return jsonify(run)
@app.route("/api/tasks/<tid>/start", methods=["POST"])
def api_start(tid):
task = store.get_task(tid)
@@ -674,6 +705,103 @@ def api_file():
return send_file(full)
# ---------------- API: 打包导出 ----------------
EXPORT_DIR = os.path.join(HERE, "data", "exports")
EXPORT_KEEP = 10 # 最多保留的打包文件数
def _safe_zip_name(task):
"""任务名安全化为文件名 (保留中文, 去非法字符)"""
name = re.sub(r'[\\/:*?"<>|\s]+', "_", task.get("name", "")).strip("_")
return (name or task["id"])[:60]
def _make_zip(task):
"""把任务输出目录打包为 zip, 返回 (zip_path, err) 或 (None, 错误信息)"""
out_dir = os.path.realpath(resolve_out_dir(task))
if not os.path.isdir(out_dir):
return None, "输出目录不存在"
os.makedirs(EXPORT_DIR, exist_ok=True)
zip_path = os.path.join(EXPORT_DIR, f"{task['id']}_{int(time.time())}.zip")
prefix = _safe_zip_name(task) + "/"
count = 0
try:
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
for root, _dirs, files in os.walk(out_dir):
for f in files:
full = os.path.join(root, f)
rel = os.path.relpath(full, out_dir)
zf.write(full, prefix + rel)
count += 1
except Exception as e:
try:
os.remove(zip_path)
except OSError:
pass
return None, f"打包失败: {e}"
_cleanup_exports()
return zip_path, None
def _cleanup_exports():
"""清理旧的打包文件, 只保留最近 EXPORT_KEEP 个"""
try:
files = sorted(
(os.path.join(EXPORT_DIR, f) for f in os.listdir(EXPORT_DIR)
if f.endswith(".zip")),
key=os.path.getmtime, reverse=True,
)
for f in files[EXPORT_KEEP:]:
os.remove(f)
except Exception:
pass
@app.route("/api/export")
def api_export():
"""打包任务输出目录为 zip 并提供下载"""
tid = request.args.get("task_id", "")
task = store.get_task(tid)
if not task:
return jsonify({"error": "任务不存在"}), 404
zip_path, err = _make_zip(task)
if err:
return jsonify({"error": err}), 400
return send_file(
zip_path, as_attachment=True,
download_name=_safe_zip_name(task) + ".zip",
mimetype="application/zip",
)
@app.route("/api/export/email", methods=["POST"])
def api_export_email():
"""打包任务输出目录为 zip 并发送到指定邮箱 (默认 wlq@tphai.com)"""
body = request.get_json(force=True) or {}
tid = body.get("task_id", "")
email = (body.get("email") or "").strip() or "wlq@tphai.com"
task = store.get_task(tid)
if not task:
return jsonify({"error": "任务不存在"}), 404
if "@" not in email:
return jsonify({"error": "邮箱格式不正确"}), 400
zip_path, err = _make_zip(task)
if err:
return jsonify({"error": err}), 400
size_mb = round(os.path.getsize(zip_path) / 1048576, 1)
body_text = (f"项目: {task['name']}\n"
f"输出目录: {resolve_out_dir(task)}\n"
f"打包文件: {os.path.basename(zip_path)}\n"
f"压缩包大小: {size_mb} MB\n\n"
f"打包时间: {now_str()}")
ok, msg = notify.send_attachment(
f"[爬虫打包] {task['name']}", body_text, email, zip_path)
if not ok:
return jsonify({"error": f"邮件发送失败: {msg}"}), 500
return jsonify({"ok": True, "msg": f"已发送到 {email} (zip {size_mb} MB)"})
# ---------------- 启动 ----------------
scheduler = Scheduler(start_run)
File diff suppressed because it is too large Load Diff
+150 -21
View File
@@ -30,21 +30,60 @@ IMG_EXTS = (".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp")
DEFAULT_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36")
_CHALLENGE_MARKS = [
# title 中出现任一标记即判定反爬 (title 短小可靠, 不会误伤正文)
_CHALLENGE_TITLE_MARKS = [
"access denied", "403 forbidden", "just a moment", "attention required",
"captcha", "bot check", "cf-challenge", "verify you are human",
"checking your browser", "enable javascript and cookies",
]
# html 中仅匹配强特征标记, 且只在 head 区域(前 20KB)搜索。
# 不能全文匹配 "captcha"/"challenge" 等词——正常文章正文提到这些词会被误判为反爬。
_CHALLENGE_HTML_MARKS = [
"cf-challenge", "challenge-platform", "cf-browser-verification",
"verify you are human", "checking your browser",
"enable javascript and cookies", "just a moment",
]
# 反爬拦截类错误信号: 命中后不再重试 (重试无意义且拖慢任务)
_ANTI_CRAWL_MARKS = [
"403", "429", "503", "forbidden", "access denied", "too many requests",
"captcha", "cloudflare", "challenge", "verify you are human",
"just a moment", "blocked", "被反爬拦截",
]
def is_anti_crawl_error(err):
"""判断错误是否属于反爬拦截 (此类失败重试也无法通过, 直接放弃该页)"""
s = str(err or "").lower()
return any(m in s for m in _ANTI_CRAWL_MARKS)
_BROWSER_DEAD_MARKS = [
"browser has been closed", "target page, context or browser has been closed",
"has been disposed", "execution context was destroyed", "browser closed",
"page closed", "target closed", "crash",
]
def is_browser_dead_error(err):
"""判断错误是否属于浏览器/页面失效 (需重启浏览器后重试, 重试同一 URL 才有意义)"""
s = str(err or "").lower()
return any(m in s for m in _BROWSER_DEAD_MARKS)
def is_challenge_page(title, html):
"""判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)"""
low = html.lower()
"""判断是否仍在反爬验证页
- title 出现反爬关键词: 判定 (可靠, 反爬页 title 基本都会变)
- html 仅在前 20KB(head 区域)出现强特征标记时判定,
避免正文含 captcha/challenge 等词的正常页面被误杀
"""
t = (title or "").lower()
for mark in _CHALLENGE_MARKS:
if mark in t or mark in low:
for mark in _CHALLENGE_TITLE_MARKS:
if mark in t:
return True
return False
low = (html or "").lower()[:20000]
return any(m in low for m in _CHALLENGE_HTML_MARKS)
def safe_name(url, idx):
@@ -56,6 +95,7 @@ def safe_name(url, idx):
def _settle_wait(page, timeout_s):
"""等待页面稳定(无 stop/pause 检查, 供试爬取等独立流程使用)"""
last_title, stable = "", 0
challenge_hits = 0
start = time.time()
while time.time() - start < timeout_s:
time.sleep(1)
@@ -65,8 +105,12 @@ def _settle_wait(page, timeout_s):
except Exception:
continue
if is_challenge_page(title, html):
challenge_hits += 1
if challenge_hits >= 8: # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
return True, title, html
stable = 0
continue
challenge_hits = 0
if title == last_title:
stable += 1
if stable >= 2 and len(html) > 1000:
@@ -208,8 +252,19 @@ class CrawlJob:
self.persist = persist # callable(task_id, run)
self._stop = threading.Event()
self._pause = threading.Event()
# 运行中的 auto 实时状态 (供详情接口读取; 任务结束时由状态文件兜底)
self._auto_visited = None
self._auto_pending = None
self._cfg_lock = threading.RLock()
self.thread = None
# 浏览器句柄 (p, browser, ctx, page, cookie_file); 崩溃后重建
self._browser = None
def _page(self):
return self._browser[3] if self._browser else None
def _ctx(self):
return self._browser[2] if self._browser else None
# ---------------- 控制接口 ----------------
def start(self):
@@ -288,9 +343,13 @@ class CrawlJob:
pass
Stealth().apply_stealth_sync(ctx)
page = ctx.new_page()
return p, browser, ctx, page, cookie_file
self._browser = (p, browser, ctx, page, cookie_file)
return self._browser
def _close_browser(self, p, browser, ctx, cookie_file):
def _close_browser(self):
if not self._browser:
return
p, browser, ctx, _page, cookie_file = self._browser
try:
json.dump(ctx.cookies(), open(cookie_file, "w"))
except Exception:
@@ -303,9 +362,24 @@ class CrawlJob:
p.stop()
except Exception:
pass
self._browser = None
def _reopen_browser(self):
"""浏览器失效后重建: 保存 cookie -> 关闭旧实例 -> 启动新实例"""
self._close_browser()
time.sleep(1)
self._open_browser()
self._log("info", "浏览器已重启")
def _need_browser_restart(self, done_count):
"""每爬 N 页主动重启一次浏览器, 防止长时间运行内存膨胀导致崩溃 (browser_max_pages=0 关闭)"""
limit = int(self._cfg("browser_max_pages", 0) or 0)
return limit > 0 and done_count > 1 and (done_count - 1) % limit == 0
def _wait_page_settle(self, page, timeout_s):
last_title, stable = "", 0
challenge_hits = 0
err_streak = 0 # 连续读取失败计数
start = time.time()
while time.time() - start < timeout_s:
if self._stop.is_set():
@@ -316,10 +390,18 @@ class CrawlJob:
title = page.title()
html = page.content()
except Exception:
err_streak += 1
if err_streak >= 3: # 页面/浏览器已失效, 提前失败而非干等到超时
raise RuntimeError("Target page, context or browser has been closed")
continue # 正在跳转
err_streak = 0
if is_challenge_page(title, html):
challenge_hits += 1
if challenge_hits >= 8: # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
return True, title, html
stable = 0
continue
challenge_hits = 0
if title == last_title:
stable += 1
if stable >= 2 and len(html) > 1000:
@@ -435,7 +517,9 @@ class CrawlJob:
"source_url": source_url, "depth": depth,
"images": [], "attempts": 0,
}
for attempt in range(retries + 1):
dead_strikes = 0 # 浏览器失效重建次数 (防止无限重建)
attempt = 0
while attempt <= retries:
if self._stop.is_set():
entry["error"] = "任务已终止"
break
@@ -459,7 +543,28 @@ class CrawlJob:
except Exception as e:
entry["error"] = str(e)
entry["crawl_time"] = store.now_str()
if is_browser_dead_error(e):
# 浏览器/页面失效: 重启浏览器后重试同一 URL, 不消耗重试次数
dead_strikes += 1
if dead_strikes > 3:
entry["error"] = f"浏览器多次重启仍失效, 放弃: {e}"
self._log("error", f"浏览器多次重启仍失效, 放弃 {url}")
break
self._log("warn", f"浏览器已失效({e}), 重启后重试: {url}")
try:
self._reopen_browser()
page, ctx = self._page(), self._ctx()
except Exception as re_err:
entry["error"] = f"浏览器重启失败: {re_err}"
self._log("error", f"浏览器重启失败, 放弃 {url}: {re_err}")
break
continue
self._log("warn", f"{attempt + 1}次失败 {url}: {e}")
if is_anti_crawl_error(e):
# 反爬拦截: 重试也过不去, 直接放弃, 不再消耗重试次数
entry["error"] = f"反爬拦截, 跳过重试: {e}"
self._log("warn", f"判定为反爬拦截, 放弃重试: {url}")
break
if attempt < retries:
self._wait_if_paused()
t0 = time.time()
@@ -468,6 +573,7 @@ class CrawlJob:
break
self._wait_if_paused()
time.sleep(0.3)
attempt += 1
self._write_page_meta(out_dir, base, entry)
return entry
@@ -531,17 +637,20 @@ class CrawlJob:
os.makedirs(out_dir, exist_ok=True)
self._persist()
p, browser, ctx, page, cookie_file = self._open_browser()
self._open_browser()
try:
for i, url in enumerate(urls, 1):
if self._stop.is_set():
self._log("info", "收到终止信号, 停止爬取")
break
self._wait_if_paused()
if self._need_browser_restart(i):
self._log("info", f"已爬 {i - 1} 页, 主动重启浏览器")
self._reopen_browser()
run["progress"]["current_url"] = url
run["progress"]["done"] = i - 1
self._persist()
entry = self._retry_crawl(page, ctx, url, i, out_dir)
entry = self._retry_crawl(self._page(), self._ctx(), url, i, out_dir)
run["results"].append(entry)
self._bump_stats(entry)
run["progress"]["done"] = i
@@ -549,7 +658,7 @@ class CrawlJob:
if entry["status"] == "OK":
self._delay()
finally:
self._close_browser(p, browser, ctx, cookie_file)
self._close_browser()
def _discover_links(self, page):
"""从当前页面提取符合规则的链接"""
@@ -570,9 +679,10 @@ class CrawlJob:
def _crawl_auto(self):
"""自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到上限
- max_depth: 0=无限制, N=只爬 N 层
- max_pages: 0=无限制, N=安全上限
- max_pages: 0=无限制, N=安全上限 (继续爬取模式按本次新增页数重新计算)
- 提取但未爬取的链接持久化到 auto.pending, 已爬集合持久化到 auto.visited
- 停止后再次运行从缓存队列继续爬; 起始网址每次运行都重新爬(不去重)
- skip_seed(继续爬取): 跳过起始网址直接消费缓存队列, 页数上限按本次新增重新计算
"""
run = self.run
auto = self.task.get("auto", {})
@@ -587,8 +697,13 @@ class CrawlJob:
state = store.load_auto_state(self.task["id"], self.task)
visited = set(state.get("visited", []) or [])
pending = state.get("pending", []) or []
visited_base = len(visited)
# 起始网址每次运行都爬(不做去重), 缓存队列继续消费
queue = [(seed, 0, "")]
# 继续爬取模式(skip_seed): 有缓存队列时跳过起始网址, 直接从待爬队列接着爬
skip_seed = bool(self.run.get("skip_seed"))
queue = []
if not (skip_seed and pending):
queue.append((seed, 0, ""))
if pending:
queue.extend((p["url"], p.get("depth", 0), p.get("source", "")) for p in pending)
queued = set(visited)
@@ -596,43 +711,57 @@ class CrawlJob:
queued.add(normalize_url(u))
run["progress"]["total"] = len(queue)
self._auto_visited = len(visited)
self._auto_pending = len(queue)
self._persist()
p, browser, ctx, page, cookie_file = self._open_browser()
self._open_browser()
try:
while queue and not self._stop.is_set():
self._wait_if_paused()
if max_pages > 0 and len(visited) >= max_pages:
self._log("info", f"达到最大页数上限 {max_pages}, 停止")
break
if max_pages > 0:
if skip_seed:
# 继续爬取模式: 页数上限按本次新增页数重新计算 (累计 visited 不阻塞继续)
if len(visited) - visited_base >= max_pages:
self._log("info", f"本次继续爬取达到页数上限 {max_pages}, 停止")
break
elif len(visited) >= max_pages:
self._log("info", f"达到最大页数上限 {max_pages}, 停止")
break
if self._need_browser_restart(len(visited)):
self._log("info", f"已爬 {len(visited) - 1} 页, 主动重启浏览器")
self._reopen_browser()
url, depth, src = queue.pop(0)
key = normalize_url(url)
if key in visited and url != seed: # 起始网址不去重, 其余已爬跳过
continue
visited.add(key)
self._auto_visited = len(visited)
self._auto_pending = len(queue)
idx = len(visited)
run["progress"]["current_url"] = url
run["progress"]["done"] = len(visited)
self._persist()
entry = self._retry_crawl(page, ctx, url, idx, out_dir,
entry = self._retry_crawl(self._page(), self._ctx(), url, idx, out_dir,
source_url=src, depth=depth)
run["results"].append(entry)
self._bump_stats(entry)
self._persist()
# 无深度限制或未达深度限制时持续发现链接
if entry["status"] == "OK" and (max_depth == 0 or depth < max_depth):
for link in self._discover_links(page):
for link in self._discover_links(self._page()):
lk = normalize_url(link)
if lk not in visited and lk not in queued:
queued.add(lk)
queue.append((link, depth + 1, url))
self._auto_pending = len(queue) # 新链接入队后实时刷新
if entry["status"] == "OK":
self._delay()
run["progress"]["total"] = len(visited)
if not self._stop.is_set() and len(queue) == 0:
self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)")
finally:
self._close_browser(p, browser, ctx, cookie_file)
self._close_browser()
# 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
try:
store.save_auto_state(self.task["id"], {
+17 -4
View File
@@ -33,11 +33,24 @@ def notify_email(task, run):
if len(results) > 50:
lines.append(f" ... 共 {len(results)}")
body = "\n".join(lines)
return send_attachment(f"[爬虫完成] {task['name']}", body, to)
def send_attachment(subject, body, to, attach_path=None):
"""发送带附件的邮件 (复用 send_email.py), 返回 (bool, msg)
attach_path: 附件文件路径, 可多个
"""
if not os.path.exists(SEND_EMAIL):
return False, "send_email.py 不存在"
cmd = [sys.executable, SEND_EMAIL, subject, body, "--to", to]
if attach_path:
if isinstance(attach_path, str):
attach_path = [attach_path]
for p in attach_path:
if os.path.isfile(p):
cmd += ["--attach", p]
try:
r = subprocess.run(
[sys.executable, SEND_EMAIL, f"[爬虫完成] {task['name']}", body, "--to", to],
timeout=60, capture_output=True,
)
r = subprocess.run(cmd, timeout=600, capture_output=True)
return r.returncode == 0, (r.stderr or b"").decode(errors="ignore")[:200]
except Exception as e:
return False, str(e)
+145 -9
View File
@@ -3,12 +3,13 @@
const $ = (id) => document.getElementById(id);
const MODE_LABEL = { batch: "批量", scheduled: "定时", auto: "自动" };
const DETAIL_PAGE_SIZE = 100; // 详情结果每页条数 (默认)
const state = {
tasks: [],
editTask: null, // 正在编辑的任务
mode: "batch", // 当前表单模式
detail: { task: null, runId: null, logOffset: 0, timer: null },
detail: { task: null, runId: null, logOffset: 0, timer: null, pageSize: DETAIL_PAGE_SIZE },
logTimer: null,
};
let formDirty = false; // 新建/编辑表单是否有未保存修改
@@ -522,7 +523,6 @@ async function submitForm(e) {
}
/* ---------------- 详情 ---------------- */
const DETAIL_PAGE_SIZE = 100; // 详情结果每页条数
async function openDetail(tid) {
try {
@@ -531,6 +531,7 @@ async function openDetail(tid) {
const runs = t.runs || [];
const cur = t.run;
state.detail.runId = cur ? cur.id : (runs[0] ? runs[0].id : null);
state.detail.pageSize = (t.run && t.run.run_page_size) || DETAIL_PAGE_SIZE;
state.detail.logOffset = 0;
$("detailTitle").textContent = `任务详情 · ${t.name}`;
renderDetail();
@@ -539,8 +540,9 @@ async function openDetail(tid) {
} catch (e) { toast(e.message, true); }
}
async function loadRunPage(tid, rid, page) {
return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${DETAIL_PAGE_SIZE}`);
async function loadRunPage(tid, rid, page, size) {
const ps = size || state.detail.pageSize || DETAIL_PAGE_SIZE;
return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${ps}`);
}
async function selectRun(rid) {
@@ -550,6 +552,7 @@ async function selectRun(rid) {
const d = await loadRunPage(t.id, rid, 1);
state.detail.task = d;
state.detail.runId = d.run ? d.run.id : null;
state.detail.pageSize = (d.run && d.run.run_page_size) || DETAIL_PAGE_SIZE;
state.detail.logOffset = 0;
renderDetail();
startLogPoll();
@@ -559,13 +562,62 @@ async function selectRun(rid) {
async function goRunPage(p) {
const t = state.detail.task;
if (!t || !state.detail.runId) return;
const ps = (t.run && t.run.run_page_size) || state.detail.pageSize || DETAIL_PAGE_SIZE;
try {
const d = await loadRunPage(t.id, state.detail.runId, p);
const d = await loadRunPage(t.id, state.detail.runId, p, ps);
state.detail.task = d;
state.detail.pageSize = (d.run && d.run.run_page_size) || ps;
renderDetail();
} catch (e) { toast(e.message, true); }
}
/* 跳转到指定页码 (输入框回车/点击 GO) */
function jumpRunPage() {
const t = state.detail.task;
if (!t || !state.detail.runId || !t.run) return;
const inp = $("pagerJump");
const pages = t.run.run_pages || 1;
let p = parseInt(inp.value, 10);
if (!p || isNaN(p)) p = 1;
p = Math.min(Math.max(p, 1), pages);
inp.value = p;
goRunPage(p);
}
/* 切换每页条数 */
async function changePageSize(size) {
const t = state.detail.task;
if (!t || !state.detail.runId) return;
try {
const d = await loadRunPage(t.id, state.detail.runId, 1, parseInt(size, 10) || 100);
state.detail.task = d;
state.detail.pageSize = (d.run && d.run.run_page_size) || 100;
renderDetail();
} catch (e) { toast(e.message, true); }
}
/* 生成页码窗口列表, 0 表示省略号: 1 2 3 … 97 98 99 100 */
function pageWindow(cur, pages, width) {
width = width || 5;
const win = [];
if (pages <= width + 2) {
for (let i = 1; i <= pages; i++) win.push(i);
return win;
}
win.push(1);
let lo = Math.max(2, cur - Math.floor(width / 2));
let hi = Math.min(pages - 1, cur + Math.floor(width / 2));
if (hi - lo < width - 1) {
if (lo <= 2) hi = lo + width - 1;
else lo = hi - width + 1;
}
if (lo > 2) win.push(0);
for (let i = lo; i <= hi; i++) win.push(i);
if (hi < pages - 1) win.push(0);
win.push(pages);
return win;
}
function renderDetail() {
const t = state.detail.task;
const runs = t.runs || [];
@@ -602,10 +654,15 @@ function renderDetail() {
${t.running ? `
<button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
<button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
: `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`}
: `
<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>
${t.mode === "auto" && (t.auto_pending_count || 0) > 0 ? `
<button class="btn" onclick="contTask('${t.id}')">⏩ 继续爬取(缓存 ${t.auto_pending_count} 条)</button>` : ""}`}
${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>
<button class="btn" ${t.running ? "disabled" : ""} onclick="clearCache('${t.id}')">🧹 清空缓存</button>` : ""}
<button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
<button class="btn" onclick="exportZip('${t.id}')">📦 打包下载</button>
<button class="btn" onclick="exportEmail('${t.id}')">📧 发邮箱</button>
</div>`;
const chips = runs.map((r) => `
@@ -659,13 +716,30 @@ function renderRunPanel(t, run) {
const total = run.results_total || results.length;
const pages = run.run_pages || 1;
const page = run.run_page || 1;
const pageSize = run.run_page_size || DETAIL_PAGE_SIZE;
const multi = pages > 1;
const pager = total > 0 ? `
<div class="pager">
${total > results.length ? `
<button class="btn sm" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(${page - 1})">◀ 上一页</button>
${multi ? `
<button class="btn sm" title="首页" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(1)">⏮</button>
<button class="btn sm" title="上一页" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(${page - 1})">◀</button>
<span class="pager-pages">${pageWindow(page, pages).map((p) => p === 0
? '<span class="pager-ellipsis">…</span>'
: `<button class="btn sm page-btn ${p === page ? "active" : ""}" onclick="goRunPage(${p})">${p}</button>`).join("")}</span>
<button class="btn sm" title="下一页" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${page + 1})">▶</button>
<button class="btn sm" title="末页" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${pages})">⏭</button>
<span class="pager-info">第 <b>${page}</b> / ${pages} 页 · 共 ${total} 条</span>
<button class="btn sm" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${page + 1})">下一页 ▶</button>`
<span class="pager-jump">跳至
<input type="number" id="pagerJump" min="1" max="${pages}" value="${page}"
onkeydown="if(event.key==='Enter')jumpRunPage()"> 页
<button class="btn sm" onclick="jumpRunPage()">GO</button>
</span>`
: `<span class="pager-info">共 ${total} 条</span>`}
<span class="pager-size">每页
<select id="pagerSize" onchange="changePageSize(this.value)">
${[50, 100, 200, 500].map((s) => `<option value="${s}" ${s === pageSize ? "selected" : ""}>${s}</option>`).join("")}
</select> 条
</span>
</div>` : "";
return `
@@ -732,6 +806,68 @@ function stopLogPoll() {
if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; }
}
async function contTask(tid) {
try {
const r = await api(`/api/tasks/${tid}/continue`, { method: "POST" });
toast("已开始继续爬取缓存队列");
loadTasks();
if (state.detail.task && state.detail.task.id === tid) openDetail(tid);
} catch (e) { toast(e.message, true); }
}
/* ---------------- 打包导出 ---------------- */
async function exportZip(tid) {
const btn = window.event && window.event.target;
if (btn) { btn.disabled = true; btn.textContent = "⏳ 打包中..."; }
try {
const res = await fetch(`/api/export?task_id=${tid}`);
if (!res.ok) {
let d = null;
try { d = await res.json(); } catch (e) { /* ignore */ }
throw new Error((d && d.error) || `HTTP ${res.status}`);
}
let fname = "export.zip";
const disp = res.headers.get("Content-Disposition") || "";
const m = disp.match(/filename\*?=(?:UTF-8'')?"?([^";]+)"?/i);
if (m) { try { fname = decodeURIComponent(m[1]); } catch (e) { fname = m[1]; } }
const blob = await res.blob();
const a = document.createElement("a");
a.href = URL.createObjectURL(blob);
a.download = fname;
document.body.appendChild(a);
a.click();
a.remove();
setTimeout(() => URL.revokeObjectURL(a.href), 5000);
toast(`打包下载完成 ${fname} (${(blob.size / 1048576).toFixed(1)} MB)`);
} catch (e) {
toast("打包失败: " + e.message, true);
} finally {
if (btn) { btn.disabled = false; btn.textContent = "📦 打包下载"; }
}
}
async function exportEmail(tid) {
const t = (state.detail.task && state.detail.task.id === tid)
? state.detail.task : state.tasks.find((x) => x.id === tid);
const def = (t && t.config && t.config.notify_email) || "wlq@tphai.com";
const email = prompt("发送到邮箱(留空默认 wlq@tphai.com:", def);
if (email === null) return; // 用户取消
const btn = window.event && window.event.target;
if (btn) { btn.disabled = true; btn.textContent = "⏳ 打包发送中..."; }
try {
const r = await api("/api/export/email", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ task_id: tid, email: email.trim() || "wlq@tphai.com" }),
});
toast(r.msg || "已发送");
} catch (e) {
toast("发送失败: " + e.message, true);
} finally {
if (btn) { btn.disabled = false; btn.textContent = "📧 发邮箱"; }
}
}
/* ---------------- 试爬取 ---------------- */
function collectProbeFromForm() {
const f = $("taskForm");
+17
View File
@@ -287,3 +287,20 @@ td.title-cell { max-width: 220px; overflow: hidden; text-overflow: ellipsis; }
}
.pager-info { color: var(--text-dim, #999); font-size: 13px; }
.pager .btn:disabled { opacity: 0.4; cursor: not-allowed; }
.pager-pages { display: flex; align-items: center; gap: 4px; flex-wrap: wrap; }
.pager-pages .page-btn { min-width: 30px; padding: 4px 6px; }
.pager-pages .page-btn.active {
background: var(--accent, #2d6cdf);
color: #fff;
border-color: var(--accent, #2d6cdf);
font-weight: 600;
}
.pager-ellipsis { color: var(--text-dim, #999); padding: 0 2px; user-select: none; }
.pager-jump { display: inline-flex; align-items: center; gap: 4px; color: var(--text-dim, #999); font-size: 13px; }
.pager-jump input {
width: 56px;
padding: 3px 6px;
text-align: center;
}
.pager-size { display: inline-flex; align-items: center; gap: 4px; color: var(--text-dim, #999); font-size: 13px; }
.pager-size select { padding: 3px 4px; width: auto; }