Compare commits

..
7 Commits
Author SHA1 Message Date
hz4th_coder ec053266a7 v1.4.0 修复反爬误判+浏览器崩溃空转, 详情页多元翻页
- 修复反爬误判: is_challenge_page 仅匹配 title 关键词 + html 前 20KB 强特征标记,
  不再全文匹配 captcha/challenge 等词 (博客园正文提 captcha 的页面被误杀)
- 修复浏览器崩溃后空转: 检测 Target page/context/browser closed,
  自动重启浏览器并重试当前 URL (不消耗重试配额, 单页最多重建3次);
  页面读取连续异常3次快速失败; 新增 browser_max_pages 配置定期重启浏览器防内存膨胀
- 详情页多元翻页: 首页/末页/页码列表(带省略号)/页码跳转输入框(回车/GO)/每页条数选择(50/100/200/500)
2026-08-14 10:28:25 +08:00
hz4th_coder 32eebf3dd1 auto 任务运行中实时显示已爬/待爬缓存数; 人工停止后继续爬取验证
- engine: CrawlJob 增加 _auto_visited/_auto_pending 运行中实时状态 (含发现新链接入队后刷新)
- app.py 详情接口: 任务运行中优先读内存实时值, 不再显示上次运行结束的旧数字
- 端到端验证: 人工停止(已爬3,缓存18) → 继续爬取(跳过seed,爬完18页,缓存0) 
- 运行中采样: visited 1→2 实时增长, pending 15→14 同步递减 
2026-08-12 10:19:19 +08:00
hz4th_coder d9c9f0c633 chore: data/auto_state 为运行时爬取状态, 移出版本控制 (.gitignore) 2026-08-12 09:57:53 +08:00
hz4th_coder 909e8e01b5 新增继续爬取: auto 任务达 max_pages 上限停止后, 有待爬缓存链接时可在详情页一键继续
- POST /api/tasks/<tid>/continue: 校验 auto 模式 + pending 非空, skip_seed 启动
- engine skip_seed: 跳过起始网址直接从缓存队列消费 (不重爬 seed, 不重复入 visited)
- 关键修复: max_pages 原为累计上限(visited>=max_pages), 已达标任务无法继续; 继续模式改为按本次新增页数重新计算上限, 可反复继续直到缓存耗尽
- 详情弹窗 auto 任务且有缓存时显示 [ 继续爬取(缓存 N 条)] 按钮
- 端到端测试: max_pages=2 链路 执行(seed+p1,缓存4) → 继续(p2,p3,缓存2) → 继续(p4,p5,缓存0) → 无缓存报错 
2026-08-12 09:57:46 +08:00
hz4th_coder e749d70a43 chore: data/exports 为运行时打包产物, 移出版本控制 (.gitignore) 2026-08-12 09:43:32 +08:00
hz4th_coder 56f30f78c4 新增打包导出: 任务输出目录打包为 zip, 支持网页下载或发送到指定邮箱(默认 wlq@tphai.com)
- GET /api/export?task_id= 打包输出目录为 zip 并下载 (zip 内按任务名建顶层目录)
- POST /api/export/email 打包后通过 send_email.py 发送附件到邮箱, 邮箱可指定, 默认 wlq@tphai.com
- 详情弹窗新增 [📦 打包下载] [📧 发邮箱] 按钮, 打包中按钮禁用防重复点击
- notify.py 重构: send_attachment(subject, body, to, attach) 通用附件发送
- 打包文件存 data/exports/, 自动清理只保留最近 10 个
- 实测: cnblogs-auto 66MB/1606文件 -> 11.9MB zip 仅1.5s; 邮件发送成功 2.2s
2026-08-12 09:41:59 +08:00
hz4th_coder 0482284cc0 反爬拦截不再重试: 判定为反爬类错误(403/429/503/Cloudflare/captcha/验证页等)直接放弃该页, 节省重试时间; 验证页持续8秒即判定反爬, 不再干等满超时(默认60s)
- engine.is_anti_crawl_error: 反爬错误信号关键词判定
- _retry_crawl: 反爬错误直接 break, entry.error 标注'反爬拦截, 跳过重试'
- _wait_page_settle/_settle_wait: 连续8秒检测到验证页即判定反爬尽早返回
- 集成测试: 反爬页 attempts=1 且 8.1s 放弃; 非反爬错误仍 attempts=retry_count+1 正常重试
2026-08-12 09:37:23 +08:00
7 changed files with 464 additions and 46925 deletions
+2
View File
@@ -4,3 +4,5 @@ data/*.json
data/cookies_*.json data/cookies_*.json
logs/ logs/
out/ out/
data/exports/
data/auto_state/
+133 -5
View File
@@ -4,14 +4,17 @@
启动: /home/hz1/miniconda3/envs/openclaw/bin/python app.py (默认端口 16062) 启动: /home/hz1/miniconda3/envs/openclaw/bin/python app.py (默认端口 16062)
""" """
import os import os
import re
import threading import threading
import time import time
import zipfile
from datetime import datetime from datetime import datetime
from flask import Flask, jsonify, request, send_file, send_from_directory from flask import Flask, jsonify, request, send_file, send_from_directory
import store import store
import db import db
import notify
from engine import CrawlJob, probe_links from engine import CrawlJob, probe_links
from scheduler import Scheduler, cron_next, interval_delta from scheduler import Scheduler, cron_next, interval_delta
@@ -101,13 +104,16 @@ def persist_cb(task_id, run):
print(f"[db] 同步失败: {e}", flush=True) print(f"[db] 同步失败: {e}", flush=True)
def start_run(task): def start_run(task, skip_seed=False):
"""为任务启动一次爬取, 返回 (run, error)""" """为任务启动一次爬取, 返回 (run, error)
skip_seed: auto 任务继续爬取模式, 跳过起始网址直接从缓存队列消费
"""
with JOBS_LOCK: with JOBS_LOCK:
job = JOBS.get(task["id"]) job = JOBS.get(task["id"])
if job and job.is_running(): if job and job.is_running():
return None, "该任务已有正在运行的爬取" return None, "该任务已有正在运行的爬取"
run = make_run(task) run = make_run(task)
run["skip_seed"] = bool(skip_seed)
store.add_run(task["id"], run) store.add_run(task["id"], run)
job = CrawlJob(task, run, persist_cb) job = CrawlJob(task, run, persist_cb)
JOBS[task["id"]] = job JOBS[task["id"]] = job
@@ -302,11 +308,19 @@ def api_task_detail(tid):
result["run_page"] = page result["run_page"] = page
result["run_pages"] = pages result["run_pages"] = pages
result["run_page_size"] = page_size result["run_page_size"] = page_size
# auto 状态数量 (独立文件) # auto 状态数量: 运行中优先读内存实时值, 否则读状态文件
if task.get("mode") == "auto": if task.get("mode") == "auto":
live_v = live_p = None
with JOBS_LOCK:
job = JOBS.get(tid)
if job:
live_v = getattr(job, "_auto_visited", None)
live_p = getattr(job, "_auto_pending", None)
st = store.load_auto_state(tid, task) st = store.load_auto_state(tid, task)
result["auto_pending_count"] = len(st.get("pending", [])) result["auto_pending_count"] = (
result["auto_visited_count"] = len(st.get("visited", [])) live_p if live_p is not None else len(st.get("pending", [])))
result["auto_visited_count"] = (
live_v if live_v is not None else len(st.get("visited", [])))
return jsonify(result) return jsonify(result)
@@ -421,6 +435,23 @@ def api_trash_clear():
# ---------------- API: 运行控制 ---------------- # ---------------- API: 运行控制 ----------------
@app.route("/api/tasks/<tid>/continue", methods=["POST"])
def api_continue(tid):
"""继续爬取: auto 任务从待爬缓存队列接着爬 (跳过起始网址, 保留已爬集合)"""
task = store.get_task(tid)
if not task:
return jsonify({"error": "任务不存在"}), 404
if task.get("mode") != "auto":
return jsonify({"error": "仅自动爬取任务支持继续爬取"}), 400
st = store.load_auto_state(tid, task)
if not st.get("pending"):
return jsonify({"error": "没有待爬缓存链接,无需继续"}), 400
run, err = start_run(task, skip_seed=True)
if err:
return jsonify({"error": err}), 409
return jsonify(run)
@app.route("/api/tasks/<tid>/start", methods=["POST"]) @app.route("/api/tasks/<tid>/start", methods=["POST"])
def api_start(tid): def api_start(tid):
task = store.get_task(tid) task = store.get_task(tid)
@@ -674,6 +705,103 @@ def api_file():
return send_file(full) return send_file(full)
# ---------------- API: 打包导出 ----------------
EXPORT_DIR = os.path.join(HERE, "data", "exports")
EXPORT_KEEP = 10 # 最多保留的打包文件数
def _safe_zip_name(task):
"""任务名安全化为文件名 (保留中文, 去非法字符)"""
name = re.sub(r'[\\/:*?"<>|\s]+', "_", task.get("name", "")).strip("_")
return (name or task["id"])[:60]
def _make_zip(task):
"""把任务输出目录打包为 zip, 返回 (zip_path, err) 或 (None, 错误信息)"""
out_dir = os.path.realpath(resolve_out_dir(task))
if not os.path.isdir(out_dir):
return None, "输出目录不存在"
os.makedirs(EXPORT_DIR, exist_ok=True)
zip_path = os.path.join(EXPORT_DIR, f"{task['id']}_{int(time.time())}.zip")
prefix = _safe_zip_name(task) + "/"
count = 0
try:
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
for root, _dirs, files in os.walk(out_dir):
for f in files:
full = os.path.join(root, f)
rel = os.path.relpath(full, out_dir)
zf.write(full, prefix + rel)
count += 1
except Exception as e:
try:
os.remove(zip_path)
except OSError:
pass
return None, f"打包失败: {e}"
_cleanup_exports()
return zip_path, None
def _cleanup_exports():
"""清理旧的打包文件, 只保留最近 EXPORT_KEEP 个"""
try:
files = sorted(
(os.path.join(EXPORT_DIR, f) for f in os.listdir(EXPORT_DIR)
if f.endswith(".zip")),
key=os.path.getmtime, reverse=True,
)
for f in files[EXPORT_KEEP:]:
os.remove(f)
except Exception:
pass
@app.route("/api/export")
def api_export():
"""打包任务输出目录为 zip 并提供下载"""
tid = request.args.get("task_id", "")
task = store.get_task(tid)
if not task:
return jsonify({"error": "任务不存在"}), 404
zip_path, err = _make_zip(task)
if err:
return jsonify({"error": err}), 400
return send_file(
zip_path, as_attachment=True,
download_name=_safe_zip_name(task) + ".zip",
mimetype="application/zip",
)
@app.route("/api/export/email", methods=["POST"])
def api_export_email():
"""打包任务输出目录为 zip 并发送到指定邮箱 (默认 wlq@tphai.com)"""
body = request.get_json(force=True) or {}
tid = body.get("task_id", "")
email = (body.get("email") or "").strip() or "wlq@tphai.com"
task = store.get_task(tid)
if not task:
return jsonify({"error": "任务不存在"}), 404
if "@" not in email:
return jsonify({"error": "邮箱格式不正确"}), 400
zip_path, err = _make_zip(task)
if err:
return jsonify({"error": err}), 400
size_mb = round(os.path.getsize(zip_path) / 1048576, 1)
body_text = (f"项目: {task['name']}\n"
f"输出目录: {resolve_out_dir(task)}\n"
f"打包文件: {os.path.basename(zip_path)}\n"
f"压缩包大小: {size_mb} MB\n\n"
f"打包时间: {now_str()}")
ok, msg = notify.send_attachment(
f"[爬虫打包] {task['name']}", body_text, email, zip_path)
if not ok:
return jsonify({"error": f"邮件发送失败: {msg}"}), 500
return jsonify({"ok": True, "msg": f"已发送到 {email} (zip {size_mb} MB)"})
# ---------------- 启动 ---------------- # ---------------- 启动 ----------------
scheduler = Scheduler(start_run) scheduler = Scheduler(start_run)
File diff suppressed because it is too large Load Diff
+150 -21
View File
@@ -30,21 +30,60 @@ IMG_EXTS = (".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp")
DEFAULT_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " DEFAULT_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36") "(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36")
_CHALLENGE_MARKS = [ # title 中出现任一标记即判定反爬 (title 短小可靠, 不会误伤正文)
_CHALLENGE_TITLE_MARKS = [
"access denied", "403 forbidden", "just a moment", "attention required", "access denied", "403 forbidden", "just a moment", "attention required",
"captcha", "bot check", "cf-challenge", "verify you are human", "captcha", "bot check", "cf-challenge", "verify you are human",
"checking your browser", "enable javascript and cookies", "checking your browser", "enable javascript and cookies",
] ]
# html 中仅匹配强特征标记, 且只在 head 区域(前 20KB)搜索。
# 不能全文匹配 "captcha"/"challenge" 等词——正常文章正文提到这些词会被误判为反爬。
_CHALLENGE_HTML_MARKS = [
"cf-challenge", "challenge-platform", "cf-browser-verification",
"verify you are human", "checking your browser",
"enable javascript and cookies", "just a moment",
]
# 反爬拦截类错误信号: 命中后不再重试 (重试无意义且拖慢任务)
_ANTI_CRAWL_MARKS = [
"403", "429", "503", "forbidden", "access denied", "too many requests",
"captcha", "cloudflare", "challenge", "verify you are human",
"just a moment", "blocked", "被反爬拦截",
]
def is_anti_crawl_error(err):
"""判断错误是否属于反爬拦截 (此类失败重试也无法通过, 直接放弃该页)"""
s = str(err or "").lower()
return any(m in s for m in _ANTI_CRAWL_MARKS)
_BROWSER_DEAD_MARKS = [
"browser has been closed", "target page, context or browser has been closed",
"has been disposed", "execution context was destroyed", "browser closed",
"page closed", "target closed", "crash",
]
def is_browser_dead_error(err):
"""判断错误是否属于浏览器/页面失效 (需重启浏览器后重试, 重试同一 URL 才有意义)"""
s = str(err or "").lower()
return any(m in s for m in _BROWSER_DEAD_MARKS)
def is_challenge_page(title, html): def is_challenge_page(title, html):
"""判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)""" """判断是否仍在反爬验证页
low = html.lower() - title 出现反爬关键词: 判定 (可靠, 反爬页 title 基本都会变)
- html 仅在前 20KB(head 区域)出现强特征标记时判定,
避免正文含 captcha/challenge 等词的正常页面被误杀
"""
t = (title or "").lower() t = (title or "").lower()
for mark in _CHALLENGE_MARKS: for mark in _CHALLENGE_TITLE_MARKS:
if mark in t or mark in low: if mark in t:
return True return True
return False low = (html or "").lower()[:20000]
return any(m in low for m in _CHALLENGE_HTML_MARKS)
def safe_name(url, idx): def safe_name(url, idx):
@@ -56,6 +95,7 @@ def safe_name(url, idx):
def _settle_wait(page, timeout_s): def _settle_wait(page, timeout_s):
"""等待页面稳定(无 stop/pause 检查, 供试爬取等独立流程使用)""" """等待页面稳定(无 stop/pause 检查, 供试爬取等独立流程使用)"""
last_title, stable = "", 0 last_title, stable = "", 0
challenge_hits = 0
start = time.time() start = time.time()
while time.time() - start < timeout_s: while time.time() - start < timeout_s:
time.sleep(1) time.sleep(1)
@@ -65,8 +105,12 @@ def _settle_wait(page, timeout_s):
except Exception: except Exception:
continue continue
if is_challenge_page(title, html): if is_challenge_page(title, html):
challenge_hits += 1
if challenge_hits >= 8: # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
return True, title, html
stable = 0 stable = 0
continue continue
challenge_hits = 0
if title == last_title: if title == last_title:
stable += 1 stable += 1
if stable >= 2 and len(html) > 1000: if stable >= 2 and len(html) > 1000:
@@ -208,8 +252,19 @@ class CrawlJob:
self.persist = persist # callable(task_id, run) self.persist = persist # callable(task_id, run)
self._stop = threading.Event() self._stop = threading.Event()
self._pause = threading.Event() self._pause = threading.Event()
# 运行中的 auto 实时状态 (供详情接口读取; 任务结束时由状态文件兜底)
self._auto_visited = None
self._auto_pending = None
self._cfg_lock = threading.RLock() self._cfg_lock = threading.RLock()
self.thread = None self.thread = None
# 浏览器句柄 (p, browser, ctx, page, cookie_file); 崩溃后重建
self._browser = None
def _page(self):
return self._browser[3] if self._browser else None
def _ctx(self):
return self._browser[2] if self._browser else None
# ---------------- 控制接口 ---------------- # ---------------- 控制接口 ----------------
def start(self): def start(self):
@@ -288,9 +343,13 @@ class CrawlJob:
pass pass
Stealth().apply_stealth_sync(ctx) Stealth().apply_stealth_sync(ctx)
page = ctx.new_page() page = ctx.new_page()
return p, browser, ctx, page, cookie_file self._browser = (p, browser, ctx, page, cookie_file)
return self._browser
def _close_browser(self, p, browser, ctx, cookie_file): def _close_browser(self):
if not self._browser:
return
p, browser, ctx, _page, cookie_file = self._browser
try: try:
json.dump(ctx.cookies(), open(cookie_file, "w")) json.dump(ctx.cookies(), open(cookie_file, "w"))
except Exception: except Exception:
@@ -303,9 +362,24 @@ class CrawlJob:
p.stop() p.stop()
except Exception: except Exception:
pass pass
self._browser = None
def _reopen_browser(self):
"""浏览器失效后重建: 保存 cookie -> 关闭旧实例 -> 启动新实例"""
self._close_browser()
time.sleep(1)
self._open_browser()
self._log("info", "浏览器已重启")
def _need_browser_restart(self, done_count):
"""每爬 N 页主动重启一次浏览器, 防止长时间运行内存膨胀导致崩溃 (browser_max_pages=0 关闭)"""
limit = int(self._cfg("browser_max_pages", 0) or 0)
return limit > 0 and done_count > 1 and (done_count - 1) % limit == 0
def _wait_page_settle(self, page, timeout_s): def _wait_page_settle(self, page, timeout_s):
last_title, stable = "", 0 last_title, stable = "", 0
challenge_hits = 0
err_streak = 0 # 连续读取失败计数
start = time.time() start = time.time()
while time.time() - start < timeout_s: while time.time() - start < timeout_s:
if self._stop.is_set(): if self._stop.is_set():
@@ -316,10 +390,18 @@ class CrawlJob:
title = page.title() title = page.title()
html = page.content() html = page.content()
except Exception: except Exception:
err_streak += 1
if err_streak >= 3: # 页面/浏览器已失效, 提前失败而非干等到超时
raise RuntimeError("Target page, context or browser has been closed")
continue # 正在跳转 continue # 正在跳转
err_streak = 0
if is_challenge_page(title, html): if is_challenge_page(title, html):
challenge_hits += 1
if challenge_hits >= 8: # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
return True, title, html
stable = 0 stable = 0
continue continue
challenge_hits = 0
if title == last_title: if title == last_title:
stable += 1 stable += 1
if stable >= 2 and len(html) > 1000: if stable >= 2 and len(html) > 1000:
@@ -435,7 +517,9 @@ class CrawlJob:
"source_url": source_url, "depth": depth, "source_url": source_url, "depth": depth,
"images": [], "attempts": 0, "images": [], "attempts": 0,
} }
for attempt in range(retries + 1): dead_strikes = 0 # 浏览器失效重建次数 (防止无限重建)
attempt = 0
while attempt <= retries:
if self._stop.is_set(): if self._stop.is_set():
entry["error"] = "任务已终止" entry["error"] = "任务已终止"
break break
@@ -459,7 +543,28 @@ class CrawlJob:
except Exception as e: except Exception as e:
entry["error"] = str(e) entry["error"] = str(e)
entry["crawl_time"] = store.now_str() entry["crawl_time"] = store.now_str()
if is_browser_dead_error(e):
# 浏览器/页面失效: 重启浏览器后重试同一 URL, 不消耗重试次数
dead_strikes += 1
if dead_strikes > 3:
entry["error"] = f"浏览器多次重启仍失效, 放弃: {e}"
self._log("error", f"浏览器多次重启仍失效, 放弃 {url}")
break
self._log("warn", f"浏览器已失效({e}), 重启后重试: {url}")
try:
self._reopen_browser()
page, ctx = self._page(), self._ctx()
except Exception as re_err:
entry["error"] = f"浏览器重启失败: {re_err}"
self._log("error", f"浏览器重启失败, 放弃 {url}: {re_err}")
break
continue
self._log("warn", f"{attempt + 1}次失败 {url}: {e}") self._log("warn", f"{attempt + 1}次失败 {url}: {e}")
if is_anti_crawl_error(e):
# 反爬拦截: 重试也过不去, 直接放弃, 不再消耗重试次数
entry["error"] = f"反爬拦截, 跳过重试: {e}"
self._log("warn", f"判定为反爬拦截, 放弃重试: {url}")
break
if attempt < retries: if attempt < retries:
self._wait_if_paused() self._wait_if_paused()
t0 = time.time() t0 = time.time()
@@ -468,6 +573,7 @@ class CrawlJob:
break break
self._wait_if_paused() self._wait_if_paused()
time.sleep(0.3) time.sleep(0.3)
attempt += 1
self._write_page_meta(out_dir, base, entry) self._write_page_meta(out_dir, base, entry)
return entry return entry
@@ -531,17 +637,20 @@ class CrawlJob:
os.makedirs(out_dir, exist_ok=True) os.makedirs(out_dir, exist_ok=True)
self._persist() self._persist()
p, browser, ctx, page, cookie_file = self._open_browser() self._open_browser()
try: try:
for i, url in enumerate(urls, 1): for i, url in enumerate(urls, 1):
if self._stop.is_set(): if self._stop.is_set():
self._log("info", "收到终止信号, 停止爬取") self._log("info", "收到终止信号, 停止爬取")
break break
self._wait_if_paused() self._wait_if_paused()
if self._need_browser_restart(i):
self._log("info", f"已爬 {i - 1} 页, 主动重启浏览器")
self._reopen_browser()
run["progress"]["current_url"] = url run["progress"]["current_url"] = url
run["progress"]["done"] = i - 1 run["progress"]["done"] = i - 1
self._persist() self._persist()
entry = self._retry_crawl(page, ctx, url, i, out_dir) entry = self._retry_crawl(self._page(), self._ctx(), url, i, out_dir)
run["results"].append(entry) run["results"].append(entry)
self._bump_stats(entry) self._bump_stats(entry)
run["progress"]["done"] = i run["progress"]["done"] = i
@@ -549,7 +658,7 @@ class CrawlJob:
if entry["status"] == "OK": if entry["status"] == "OK":
self._delay() self._delay()
finally: finally:
self._close_browser(p, browser, ctx, cookie_file) self._close_browser()
def _discover_links(self, page): def _discover_links(self, page):
"""从当前页面提取符合规则的链接""" """从当前页面提取符合规则的链接"""
@@ -570,9 +679,10 @@ class CrawlJob:
def _crawl_auto(self): def _crawl_auto(self):
"""自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到上限 """自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到上限
- max_depth: 0=无限制, N=只爬 N 层 - max_depth: 0=无限制, N=只爬 N 层
- max_pages: 0=无限制, N=安全上限 - max_pages: 0=无限制, N=安全上限 (继续爬取模式按本次新增页数重新计算)
- 提取但未爬取的链接持久化到 auto.pending, 已爬集合持久化到 auto.visited - 提取但未爬取的链接持久化到 auto.pending, 已爬集合持久化到 auto.visited
- 停止后再次运行从缓存队列继续爬; 起始网址每次运行都重新爬(不去重) - 停止后再次运行从缓存队列继续爬; 起始网址每次运行都重新爬(不去重)
- skip_seed(继续爬取): 跳过起始网址直接消费缓存队列, 页数上限按本次新增重新计算
""" """
run = self.run run = self.run
auto = self.task.get("auto", {}) auto = self.task.get("auto", {})
@@ -587,8 +697,13 @@ class CrawlJob:
state = store.load_auto_state(self.task["id"], self.task) state = store.load_auto_state(self.task["id"], self.task)
visited = set(state.get("visited", []) or []) visited = set(state.get("visited", []) or [])
pending = state.get("pending", []) or [] pending = state.get("pending", []) or []
visited_base = len(visited)
# 起始网址每次运行都爬(不做去重), 缓存队列继续消费 # 起始网址每次运行都爬(不做去重), 缓存队列继续消费
queue = [(seed, 0, "")] # 继续爬取模式(skip_seed): 有缓存队列时跳过起始网址, 直接从待爬队列接着爬
skip_seed = bool(self.run.get("skip_seed"))
queue = []
if not (skip_seed and pending):
queue.append((seed, 0, ""))
if pending: if pending:
queue.extend((p["url"], p.get("depth", 0), p.get("source", "")) for p in pending) queue.extend((p["url"], p.get("depth", 0), p.get("source", "")) for p in pending)
queued = set(visited) queued = set(visited)
@@ -596,43 +711,57 @@ class CrawlJob:
queued.add(normalize_url(u)) queued.add(normalize_url(u))
run["progress"]["total"] = len(queue) run["progress"]["total"] = len(queue)
self._auto_visited = len(visited)
self._auto_pending = len(queue)
self._persist() self._persist()
p, browser, ctx, page, cookie_file = self._open_browser() self._open_browser()
try: try:
while queue and not self._stop.is_set(): while queue and not self._stop.is_set():
self._wait_if_paused() self._wait_if_paused()
if max_pages > 0 and len(visited) >= max_pages: if max_pages > 0:
self._log("info", f"达到最大页数上限 {max_pages}, 停止") if skip_seed:
break # 继续爬取模式: 页数上限按本次新增页数重新计算 (累计 visited 不阻塞继续)
if len(visited) - visited_base >= max_pages:
self._log("info", f"本次继续爬取达到页数上限 {max_pages}, 停止")
break
elif len(visited) >= max_pages:
self._log("info", f"达到最大页数上限 {max_pages}, 停止")
break
if self._need_browser_restart(len(visited)):
self._log("info", f"已爬 {len(visited) - 1} 页, 主动重启浏览器")
self._reopen_browser()
url, depth, src = queue.pop(0) url, depth, src = queue.pop(0)
key = normalize_url(url) key = normalize_url(url)
if key in visited and url != seed: # 起始网址不去重, 其余已爬跳过 if key in visited and url != seed: # 起始网址不去重, 其余已爬跳过
continue continue
visited.add(key) visited.add(key)
self._auto_visited = len(visited)
self._auto_pending = len(queue)
idx = len(visited) idx = len(visited)
run["progress"]["current_url"] = url run["progress"]["current_url"] = url
run["progress"]["done"] = len(visited) run["progress"]["done"] = len(visited)
self._persist() self._persist()
entry = self._retry_crawl(page, ctx, url, idx, out_dir, entry = self._retry_crawl(self._page(), self._ctx(), url, idx, out_dir,
source_url=src, depth=depth) source_url=src, depth=depth)
run["results"].append(entry) run["results"].append(entry)
self._bump_stats(entry) self._bump_stats(entry)
self._persist() self._persist()
# 无深度限制或未达深度限制时持续发现链接 # 无深度限制或未达深度限制时持续发现链接
if entry["status"] == "OK" and (max_depth == 0 or depth < max_depth): if entry["status"] == "OK" and (max_depth == 0 or depth < max_depth):
for link in self._discover_links(page): for link in self._discover_links(self._page()):
lk = normalize_url(link) lk = normalize_url(link)
if lk not in visited and lk not in queued: if lk not in visited and lk not in queued:
queued.add(lk) queued.add(lk)
queue.append((link, depth + 1, url)) queue.append((link, depth + 1, url))
self._auto_pending = len(queue) # 新链接入队后实时刷新
if entry["status"] == "OK": if entry["status"] == "OK":
self._delay() self._delay()
run["progress"]["total"] = len(visited) run["progress"]["total"] = len(visited)
if not self._stop.is_set() and len(queue) == 0: if not self._stop.is_set() and len(queue) == 0:
self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)") self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)")
finally: finally:
self._close_browser(p, browser, ctx, cookie_file) self._close_browser()
# 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续 # 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
try: try:
store.save_auto_state(self.task["id"], { store.save_auto_state(self.task["id"], {
+17 -4
View File
@@ -33,11 +33,24 @@ def notify_email(task, run):
if len(results) > 50: if len(results) > 50:
lines.append(f" ... 共 {len(results)}") lines.append(f" ... 共 {len(results)}")
body = "\n".join(lines) body = "\n".join(lines)
return send_attachment(f"[爬虫完成] {task['name']}", body, to)
def send_attachment(subject, body, to, attach_path=None):
"""发送带附件的邮件 (复用 send_email.py), 返回 (bool, msg)
attach_path: 附件文件路径, 可多个
"""
if not os.path.exists(SEND_EMAIL):
return False, "send_email.py 不存在"
cmd = [sys.executable, SEND_EMAIL, subject, body, "--to", to]
if attach_path:
if isinstance(attach_path, str):
attach_path = [attach_path]
for p in attach_path:
if os.path.isfile(p):
cmd += ["--attach", p]
try: try:
r = subprocess.run( r = subprocess.run(cmd, timeout=600, capture_output=True)
[sys.executable, SEND_EMAIL, f"[爬虫完成] {task['name']}", body, "--to", to],
timeout=60, capture_output=True,
)
return r.returncode == 0, (r.stderr or b"").decode(errors="ignore")[:200] return r.returncode == 0, (r.stderr or b"").decode(errors="ignore")[:200]
except Exception as e: except Exception as e:
return False, str(e) return False, str(e)
+145 -9
View File
@@ -3,12 +3,13 @@
const $ = (id) => document.getElementById(id); const $ = (id) => document.getElementById(id);
const MODE_LABEL = { batch: "批量", scheduled: "定时", auto: "自动" }; const MODE_LABEL = { batch: "批量", scheduled: "定时", auto: "自动" };
const DETAIL_PAGE_SIZE = 100; // 详情结果每页条数 (默认)
const state = { const state = {
tasks: [], tasks: [],
editTask: null, // 正在编辑的任务 editTask: null, // 正在编辑的任务
mode: "batch", // 当前表单模式 mode: "batch", // 当前表单模式
detail: { task: null, runId: null, logOffset: 0, timer: null }, detail: { task: null, runId: null, logOffset: 0, timer: null, pageSize: DETAIL_PAGE_SIZE },
logTimer: null, logTimer: null,
}; };
let formDirty = false; // 新建/编辑表单是否有未保存修改 let formDirty = false; // 新建/编辑表单是否有未保存修改
@@ -522,7 +523,6 @@ async function submitForm(e) {
} }
/* ---------------- 详情 ---------------- */ /* ---------------- 详情 ---------------- */
const DETAIL_PAGE_SIZE = 100; // 详情结果每页条数
async function openDetail(tid) { async function openDetail(tid) {
try { try {
@@ -531,6 +531,7 @@ async function openDetail(tid) {
const runs = t.runs || []; const runs = t.runs || [];
const cur = t.run; const cur = t.run;
state.detail.runId = cur ? cur.id : (runs[0] ? runs[0].id : null); state.detail.runId = cur ? cur.id : (runs[0] ? runs[0].id : null);
state.detail.pageSize = (t.run && t.run.run_page_size) || DETAIL_PAGE_SIZE;
state.detail.logOffset = 0; state.detail.logOffset = 0;
$("detailTitle").textContent = `任务详情 · ${t.name}`; $("detailTitle").textContent = `任务详情 · ${t.name}`;
renderDetail(); renderDetail();
@@ -539,8 +540,9 @@ async function openDetail(tid) {
} catch (e) { toast(e.message, true); } } catch (e) { toast(e.message, true); }
} }
async function loadRunPage(tid, rid, page) { async function loadRunPage(tid, rid, page, size) {
return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${DETAIL_PAGE_SIZE}`); const ps = size || state.detail.pageSize || DETAIL_PAGE_SIZE;
return api(`/api/tasks/${tid}?run=${rid}&page=${page}&page_size=${ps}`);
} }
async function selectRun(rid) { async function selectRun(rid) {
@@ -550,6 +552,7 @@ async function selectRun(rid) {
const d = await loadRunPage(t.id, rid, 1); const d = await loadRunPage(t.id, rid, 1);
state.detail.task = d; state.detail.task = d;
state.detail.runId = d.run ? d.run.id : null; state.detail.runId = d.run ? d.run.id : null;
state.detail.pageSize = (d.run && d.run.run_page_size) || DETAIL_PAGE_SIZE;
state.detail.logOffset = 0; state.detail.logOffset = 0;
renderDetail(); renderDetail();
startLogPoll(); startLogPoll();
@@ -559,13 +562,62 @@ async function selectRun(rid) {
async function goRunPage(p) { async function goRunPage(p) {
const t = state.detail.task; const t = state.detail.task;
if (!t || !state.detail.runId) return; if (!t || !state.detail.runId) return;
const ps = (t.run && t.run.run_page_size) || state.detail.pageSize || DETAIL_PAGE_SIZE;
try { try {
const d = await loadRunPage(t.id, state.detail.runId, p); const d = await loadRunPage(t.id, state.detail.runId, p, ps);
state.detail.task = d; state.detail.task = d;
state.detail.pageSize = (d.run && d.run.run_page_size) || ps;
renderDetail(); renderDetail();
} catch (e) { toast(e.message, true); } } catch (e) { toast(e.message, true); }
} }
/* 跳转到指定页码 (输入框回车/点击 GO) */
function jumpRunPage() {
const t = state.detail.task;
if (!t || !state.detail.runId || !t.run) return;
const inp = $("pagerJump");
const pages = t.run.run_pages || 1;
let p = parseInt(inp.value, 10);
if (!p || isNaN(p)) p = 1;
p = Math.min(Math.max(p, 1), pages);
inp.value = p;
goRunPage(p);
}
/* 切换每页条数 */
async function changePageSize(size) {
const t = state.detail.task;
if (!t || !state.detail.runId) return;
try {
const d = await loadRunPage(t.id, state.detail.runId, 1, parseInt(size, 10) || 100);
state.detail.task = d;
state.detail.pageSize = (d.run && d.run.run_page_size) || 100;
renderDetail();
} catch (e) { toast(e.message, true); }
}
/* 生成页码窗口列表, 0 表示省略号: 1 2 3 … 97 98 99 100 */
function pageWindow(cur, pages, width) {
width = width || 5;
const win = [];
if (pages <= width + 2) {
for (let i = 1; i <= pages; i++) win.push(i);
return win;
}
win.push(1);
let lo = Math.max(2, cur - Math.floor(width / 2));
let hi = Math.min(pages - 1, cur + Math.floor(width / 2));
if (hi - lo < width - 1) {
if (lo <= 2) hi = lo + width - 1;
else lo = hi - width + 1;
}
if (lo > 2) win.push(0);
for (let i = lo; i <= hi; i++) win.push(i);
if (hi < pages - 1) win.push(0);
win.push(pages);
return win;
}
function renderDetail() { function renderDetail() {
const t = state.detail.task; const t = state.detail.task;
const runs = t.runs || []; const runs = t.runs || [];
@@ -602,10 +654,15 @@ function renderDetail() {
${t.running ? ` ${t.running ? `
<button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button> <button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
<button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>` <button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
: `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`} : `
<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>
${t.mode === "auto" && (t.auto_pending_count || 0) > 0 ? `
<button class="btn" onclick="contTask('${t.id}')">⏩ 继续爬取(缓存 ${t.auto_pending_count} 条)</button>` : ""}`}
${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button> ${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>
<button class="btn" ${t.running ? "disabled" : ""} onclick="clearCache('${t.id}')">🧹 清空缓存</button>` : ""} <button class="btn" ${t.running ? "disabled" : ""} onclick="clearCache('${t.id}')">🧹 清空缓存</button>` : ""}
<button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button> <button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
<button class="btn" onclick="exportZip('${t.id}')">📦 打包下载</button>
<button class="btn" onclick="exportEmail('${t.id}')">📧 发邮箱</button>
</div>`; </div>`;
const chips = runs.map((r) => ` const chips = runs.map((r) => `
@@ -659,13 +716,30 @@ function renderRunPanel(t, run) {
const total = run.results_total || results.length; const total = run.results_total || results.length;
const pages = run.run_pages || 1; const pages = run.run_pages || 1;
const page = run.run_page || 1; const page = run.run_page || 1;
const pageSize = run.run_page_size || DETAIL_PAGE_SIZE;
const multi = pages > 1;
const pager = total > 0 ? ` const pager = total > 0 ? `
<div class="pager"> <div class="pager">
${total > results.length ? ` ${multi ? `
<button class="btn sm" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(${page - 1})">◀ 上一页</button> <button class="btn sm" title="首页" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(1)">⏮</button>
<button class="btn sm" title="上一页" ${page <= 1 ? "disabled" : ""} onclick="goRunPage(${page - 1})">◀</button>
<span class="pager-pages">${pageWindow(page, pages).map((p) => p === 0
? '<span class="pager-ellipsis">…</span>'
: `<button class="btn sm page-btn ${p === page ? "active" : ""}" onclick="goRunPage(${p})">${p}</button>`).join("")}</span>
<button class="btn sm" title="下一页" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${page + 1})">▶</button>
<button class="btn sm" title="末页" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${pages})">⏭</button>
<span class="pager-info">第 <b>${page}</b> / ${pages} 页 · 共 ${total} 条</span> <span class="pager-info">第 <b>${page}</b> / ${pages} 页 · 共 ${total} 条</span>
<button class="btn sm" ${page >= pages ? "disabled" : ""} onclick="goRunPage(${page + 1})">下一页 ▶</button>` <span class="pager-jump">跳至
<input type="number" id="pagerJump" min="1" max="${pages}" value="${page}"
onkeydown="if(event.key==='Enter')jumpRunPage()"> 页
<button class="btn sm" onclick="jumpRunPage()">GO</button>
</span>`
: `<span class="pager-info">共 ${total} 条</span>`} : `<span class="pager-info">共 ${total} 条</span>`}
<span class="pager-size">每页
<select id="pagerSize" onchange="changePageSize(this.value)">
${[50, 100, 200, 500].map((s) => `<option value="${s}" ${s === pageSize ? "selected" : ""}>${s}</option>`).join("")}
</select> 条
</span>
</div>` : ""; </div>` : "";
return ` return `
@@ -732,6 +806,68 @@ function stopLogPoll() {
if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; } if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; }
} }
async function contTask(tid) {
try {
const r = await api(`/api/tasks/${tid}/continue`, { method: "POST" });
toast("已开始继续爬取缓存队列");
loadTasks();
if (state.detail.task && state.detail.task.id === tid) openDetail(tid);
} catch (e) { toast(e.message, true); }
}
/* ---------------- 打包导出 ---------------- */
async function exportZip(tid) {
const btn = window.event && window.event.target;
if (btn) { btn.disabled = true; btn.textContent = "⏳ 打包中..."; }
try {
const res = await fetch(`/api/export?task_id=${tid}`);
if (!res.ok) {
let d = null;
try { d = await res.json(); } catch (e) { /* ignore */ }
throw new Error((d && d.error) || `HTTP ${res.status}`);
}
let fname = "export.zip";
const disp = res.headers.get("Content-Disposition") || "";
const m = disp.match(/filename\*?=(?:UTF-8'')?"?([^";]+)"?/i);
if (m) { try { fname = decodeURIComponent(m[1]); } catch (e) { fname = m[1]; } }
const blob = await res.blob();
const a = document.createElement("a");
a.href = URL.createObjectURL(blob);
a.download = fname;
document.body.appendChild(a);
a.click();
a.remove();
setTimeout(() => URL.revokeObjectURL(a.href), 5000);
toast(`打包下载完成 ${fname} (${(blob.size / 1048576).toFixed(1)} MB)`);
} catch (e) {
toast("打包失败: " + e.message, true);
} finally {
if (btn) { btn.disabled = false; btn.textContent = "📦 打包下载"; }
}
}
async function exportEmail(tid) {
const t = (state.detail.task && state.detail.task.id === tid)
? state.detail.task : state.tasks.find((x) => x.id === tid);
const def = (t && t.config && t.config.notify_email) || "wlq@tphai.com";
const email = prompt("发送到邮箱(留空默认 wlq@tphai.com:", def);
if (email === null) return; // 用户取消
const btn = window.event && window.event.target;
if (btn) { btn.disabled = true; btn.textContent = "⏳ 打包发送中..."; }
try {
const r = await api("/api/export/email", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ task_id: tid, email: email.trim() || "wlq@tphai.com" }),
});
toast(r.msg || "已发送");
} catch (e) {
toast("发送失败: " + e.message, true);
} finally {
if (btn) { btn.disabled = false; btn.textContent = "📧 发邮箱"; }
}
}
/* ---------------- 试爬取 ---------------- */ /* ---------------- 试爬取 ---------------- */
function collectProbeFromForm() { function collectProbeFromForm() {
const f = $("taskForm"); const f = $("taskForm");
+17
View File
@@ -287,3 +287,20 @@ td.title-cell { max-width: 220px; overflow: hidden; text-overflow: ellipsis; }
} }
.pager-info { color: var(--text-dim, #999); font-size: 13px; } .pager-info { color: var(--text-dim, #999); font-size: 13px; }
.pager .btn:disabled { opacity: 0.4; cursor: not-allowed; } .pager .btn:disabled { opacity: 0.4; cursor: not-allowed; }
.pager-pages { display: flex; align-items: center; gap: 4px; flex-wrap: wrap; }
.pager-pages .page-btn { min-width: 30px; padding: 4px 6px; }
.pager-pages .page-btn.active {
background: var(--accent, #2d6cdf);
color: #fff;
border-color: var(--accent, #2d6cdf);
font-weight: 600;
}
.pager-ellipsis { color: var(--text-dim, #999); padding: 0 2px; user-select: none; }
.pager-jump { display: inline-flex; align-items: center; gap: 4px; color: var(--text-dim, #999); font-size: 13px; }
.pager-jump input {
width: 56px;
padding: 3px 6px;
text-align: center;
}
.pager-size { display: inline-flex; align-items: center; gap: 4px; color: var(--text-dim, #999); font-size: 13px; }
.pager-size select { padding: 3px 4px; width: auto; }