通用爬虫系统 v1.0.0: 批量/定时/自动爬取 + Web管理界面 + 图片/通知/热更新
This commit is contained in:
+497
@@ -0,0 +1,497 @@
|
||||
/* 通用爬虫系统 - 前端逻辑 */
|
||||
"use strict";
|
||||
|
||||
const $ = (id) => document.getElementById(id);
|
||||
const MODE_LABEL = { batch: "批量", scheduled: "定时", auto: "自动" };
|
||||
|
||||
const state = {
|
||||
tasks: [],
|
||||
editTask: null, // 正在编辑的任务
|
||||
mode: "batch", // 当前表单模式
|
||||
detail: { task: null, runId: null, logOffset: 0, timer: null },
|
||||
logTimer: null,
|
||||
};
|
||||
|
||||
/* ---------------- 工具 ---------------- */
|
||||
function toast(msg, isErr) {
|
||||
const t = document.createElement("div");
|
||||
t.className = "toast" + (isErr ? " error" : "");
|
||||
t.textContent = msg;
|
||||
document.body.appendChild(t);
|
||||
setTimeout(() => t.remove(), 2600);
|
||||
}
|
||||
|
||||
async function api(url, opts) {
|
||||
const res = await fetch(url, opts);
|
||||
let data = null;
|
||||
try { data = await res.json(); } catch (e) { /* ignore */ }
|
||||
if (!res.ok) throw new Error((data && data.error) || `HTTP ${res.status}`);
|
||||
return data;
|
||||
}
|
||||
|
||||
const esc = (s) => String(s == null ? "" : s).replace(/[&<>"']/g,
|
||||
(c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c]));
|
||||
|
||||
function fmtTime(iso) { return iso ? String(iso).replace("T", " ").slice(0, 19) : "—"; }
|
||||
|
||||
/* ---------------- 任务列表 ---------------- */
|
||||
async function loadTasks() {
|
||||
try {
|
||||
state.tasks = await api("/api/tasks");
|
||||
renderTasks();
|
||||
const running = state.tasks.filter((t) => t.running).length;
|
||||
const pill = $("statusPill");
|
||||
pill.textContent = `运行中任务: ${running}`;
|
||||
pill.classList.toggle("active", running > 0);
|
||||
const st = await api("/api/status");
|
||||
$("version").textContent = `v${st.version}`;
|
||||
} catch (e) {
|
||||
toast("加载任务失败: " + e.message, true);
|
||||
}
|
||||
}
|
||||
|
||||
function taskStatusBadge(t) {
|
||||
if (t.running) return '<span class="badge running">● 运行中</span>';
|
||||
const r = t.latest_run;
|
||||
if (r && r.status === "paused") return '<span class="badge paused">⏸ 已暂停</span>';
|
||||
if (t.mode === "scheduled" && t.schedule && t.schedule.enabled) {
|
||||
const next = t.schedule.next_run ? ` · 下次 ${t.schedule.next_run.slice(11)}` : "";
|
||||
return `<span class="badge scheduled-wait">⏰ 等待调度${esc(next)}</span>`;
|
||||
}
|
||||
if (r) return `<span class="badge ${r.status}">${({ completed: "✅ 已完成", failed: "❌ 失败", stopped: "⏹ 已停止" })[r.status] || esc(r.status)}</span>`;
|
||||
return '<span class="badge">未运行</span>';
|
||||
}
|
||||
|
||||
function renderTasks() {
|
||||
const box = $("taskList");
|
||||
$("emptyState").classList.toggle("hidden", state.tasks.length > 0);
|
||||
box.innerHTML = state.tasks.map(taskCard).join("");
|
||||
}
|
||||
|
||||
function taskCard(t) {
|
||||
const r = t.latest_run;
|
||||
const cfg = t.config || {};
|
||||
const prog = r && (r.status === "running" || r.status === "paused") ? `
|
||||
<div class="progress"><div style="width:${r.progress.percent || 0}%"></div></div>
|
||||
<div class="progress-label">
|
||||
<span>${r.progress.done || 0} / ${r.progress.total || "?"}</span>
|
||||
<span class="url-cell" style="max-width:220px">${esc(r.progress.current_url || "")}</span>
|
||||
</div>` : "";
|
||||
const stats = r ? `
|
||||
<div class="stats">
|
||||
<span class="ok">✅ ${r.stats.ok || 0}</span>
|
||||
<span class="fail">❌ ${r.stats.fail || 0}</span>
|
||||
<span class="img">🖼️ ${r.stats.images || 0}</span>
|
||||
</div>` : "";
|
||||
const urlCount = (t.urls || []).length;
|
||||
const line2 = t.mode === "auto"
|
||||
? `起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b>`
|
||||
: `网址: <b>${urlCount}</b> 个`;
|
||||
const line3 = t.mode === "scheduled"
|
||||
? `调度: <b>${t.schedule.type === "cron" ? esc(t.schedule.cron) : "每 " + (t.schedule.interval_value || "-") + " " + ({ minutes: "分钟", hours: "小时", days: "天" }[t.schedule.interval_unit] || "")}</b>`
|
||||
: "";
|
||||
const outDir = cfg.out_dir || "out/" + t.id;
|
||||
|
||||
let acts = "";
|
||||
if (t.running) {
|
||||
acts = `
|
||||
<button class="btn sm" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
|
||||
<button class="btn sm" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`;
|
||||
} else {
|
||||
const canStart = t.mode !== "scheduled" || (t.urls || []).length > 0;
|
||||
acts = `<button class="btn sm primary" ${canStart ? "" : "disabled"} onclick="actTask('${t.id}','start')">▶ 开始</button>`;
|
||||
}
|
||||
acts += `
|
||||
<button class="btn sm" onclick="openEdit('${t.id}')">✏️ 编辑</button>
|
||||
<button class="btn sm" onclick="openDetail('${t.id}')">📋 详情</button>
|
||||
<button class="btn sm danger" onclick="delTask('${t.id}')">🗑️</button>`;
|
||||
|
||||
return `
|
||||
<div class="card">
|
||||
<div class="card-head">
|
||||
<span class="card-name" title="${esc(t.name)}">${esc(t.name)}</span>
|
||||
${taskStatusBadge(t)}
|
||||
</div>
|
||||
<div class="card-meta">
|
||||
<span class="badge ${t.mode}">${MODE_LABEL[t.mode]}</span>
|
||||
<span class="card-line">${line2}</span>
|
||||
</div>
|
||||
${line3 ? `<div class="card-line">${line3}</div>` : ""}
|
||||
<div class="card-line">输出: <b>${esc(outDir)}</b></div>
|
||||
${prog}
|
||||
${stats}
|
||||
<div class="card-actions">${acts}</div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
/* ---------------- 任务操作 ---------------- */
|
||||
async function actTask(tid, act) {
|
||||
try {
|
||||
const r = await api(`/api/tasks/${tid}/${act}`, { method: "POST" });
|
||||
if (r.msg) toast(r.msg);
|
||||
loadTasks();
|
||||
if (state.detail.task && state.detail.task.id === tid) openDetail(tid);
|
||||
} catch (e) { toast(e.message, true); }
|
||||
}
|
||||
|
||||
async function delTask(tid) {
|
||||
const t = state.tasks.find((x) => x.id === tid);
|
||||
if (!confirm(`确定删除任务「${t ? t.name : tid}」?\n运行记录与任务配置将被删除(已爬取的文件保留在磁盘上)。`)) return;
|
||||
try {
|
||||
await api(`/api/tasks/${tid}`, { method: "DELETE" });
|
||||
toast("已删除");
|
||||
loadTasks();
|
||||
} catch (e) { toast(e.message, true); }
|
||||
}
|
||||
|
||||
/* ---------------- 新建 / 编辑表单 ---------------- */
|
||||
function switchMode(mode, lock) {
|
||||
state.mode = mode;
|
||||
document.querySelectorAll("#modeTabs .tab").forEach((b) => {
|
||||
b.classList.toggle("active", b.dataset.mode === mode);
|
||||
b.disabled = !!lock;
|
||||
});
|
||||
$("urlsField").classList.toggle("hidden", mode !== "batch" && mode !== "scheduled");
|
||||
$("scheduleBox").classList.toggle("hidden", mode !== "scheduled");
|
||||
$("autoBox").classList.toggle("hidden", mode !== "auto");
|
||||
}
|
||||
|
||||
function openCreate() {
|
||||
state.editTask = null;
|
||||
$("modalTitle").textContent = "新建爬取任务";
|
||||
$("taskForm").reset();
|
||||
$("taskForm").elements["delay_min"].value = 2;
|
||||
$("taskForm").elements["delay_max"].value = 5;
|
||||
$("taskForm").elements["timeout"].value = 60;
|
||||
$("taskForm").elements["retry_count"].value = 2;
|
||||
$("taskForm").elements["retry_interval"].value = 3;
|
||||
$("taskForm").elements["notify_email"].value = "wlq@tphai.com";
|
||||
$("formHint").textContent = "";
|
||||
switchMode("batch", false);
|
||||
showModal("taskModal");
|
||||
}
|
||||
|
||||
async function openEdit(tid) {
|
||||
const t = state.tasks.find((x) => x.id === tid) || await api(`/api/tasks/${tid}`);
|
||||
state.editTask = t;
|
||||
$("modalTitle").textContent = `编辑任务「${t.name}」`;
|
||||
const f = $("taskForm");
|
||||
f.reset();
|
||||
const cfg = t.config || {};
|
||||
f.elements["name"].value = t.name || "";
|
||||
f.elements["out_dir"].value = cfg.out_dir || "";
|
||||
f.elements["delay_min"].value = cfg.delay_min ?? 2;
|
||||
f.elements["delay_max"].value = cfg.delay_max ?? 5;
|
||||
f.elements["timeout"].value = cfg.timeout ?? 60;
|
||||
f.elements["retry_count"].value = cfg.retry_count ?? 2;
|
||||
f.elements["retry_interval"].value = cfg.retry_interval ?? 3;
|
||||
f.elements["crawl_images"].checked = !!cfg.crawl_images;
|
||||
f.elements["notify"].checked = !!cfg.notify;
|
||||
f.elements["notify_email"].value = cfg.notify_email || "wlq@tphai.com";
|
||||
f.elements["urls"].value = (t.urls || []).join("\n");
|
||||
if (t.mode === "scheduled" && t.schedule) {
|
||||
f.elements["schedule_type"].value = t.schedule.type || "interval";
|
||||
f.elements["schedule_enabled"].checked = t.schedule.enabled !== false;
|
||||
f.elements["interval_value"].value = t.schedule.interval_value ?? 24;
|
||||
f.elements["interval_unit"].value = t.schedule.interval_unit || "hours";
|
||||
f.elements["cron"].value = t.schedule.cron || "";
|
||||
}
|
||||
if (t.mode === "auto" && t.auto) {
|
||||
f.elements["seed_url"].value = t.auto.seed_url || "";
|
||||
f.elements["include"].value = (t.auto.include || []).join("\n");
|
||||
f.elements["exclude"].value = (t.auto.exclude || []).join("\n");
|
||||
f.elements["same_domain"].checked = t.auto.same_domain !== false;
|
||||
f.elements["use_regex"].checked = !!t.auto.use_regex;
|
||||
f.elements["max_pages"].value = t.auto.max_pages ?? 50;
|
||||
f.elements["max_depth"].value = t.auto.max_depth ?? 2;
|
||||
}
|
||||
syncScheduleUI();
|
||||
$("formHint").textContent = t.running
|
||||
? "⚠️ 任务运行中:参数修改将热更新(网址/规则改动下次运行生效)"
|
||||
: "";
|
||||
switchMode(t.mode, true);
|
||||
showModal("taskModal");
|
||||
}
|
||||
|
||||
function syncScheduleUI() {
|
||||
const f = $("taskForm");
|
||||
const t = f.elements["schedule_type"].value;
|
||||
$("intervalBox").classList.toggle("hidden", t !== "interval");
|
||||
$("cronBox").classList.toggle("hidden", t !== "cron");
|
||||
}
|
||||
|
||||
function splitLines(v) { return String(v || "").split("\n").map((s) => s.trim()).filter(Boolean); }
|
||||
|
||||
async function submitForm(e) {
|
||||
e.preventDefault();
|
||||
const f = $("taskForm");
|
||||
const name = f.elements["name"].value.trim();
|
||||
if (!name) { toast("请填写项目名称", true); return; }
|
||||
const mode = state.mode;
|
||||
const config = {
|
||||
out_dir: f.elements["out_dir"].value.trim(),
|
||||
delay_min: parseFloat(f.elements["delay_min"].value) || 2,
|
||||
delay_max: parseFloat(f.elements["delay_max"].value) || 5,
|
||||
timeout: parseInt(f.elements["timeout"].value) || 60,
|
||||
retry_count: parseInt(f.elements["retry_count"].value) || 0,
|
||||
retry_interval: parseFloat(f.elements["retry_interval"].value) || 3,
|
||||
crawl_images: f.elements["crawl_images"].checked,
|
||||
notify: f.elements["notify"].checked,
|
||||
notify_email: f.elements["notify_email"].value.trim() || "wlq@tphai.com",
|
||||
};
|
||||
const urls = splitLines(f.elements["urls"].value);
|
||||
const payload = { name, mode, config, urls };
|
||||
if (mode === "auto") {
|
||||
payload.auto = {
|
||||
seed_url: f.elements["seed_url"].value.trim(),
|
||||
include: splitLines(f.elements["include"].value),
|
||||
exclude: splitLines(f.elements["exclude"].value),
|
||||
same_domain: f.elements["same_domain"].checked,
|
||||
use_regex: f.elements["use_regex"].checked,
|
||||
max_pages: parseInt(f.elements["max_pages"].value) || 50,
|
||||
max_depth: parseInt(f.elements["max_depth"].value) || 2,
|
||||
};
|
||||
}
|
||||
if (mode === "scheduled") {
|
||||
payload.schedule = {
|
||||
type: f.elements["schedule_type"].value,
|
||||
enabled: f.elements["schedule_enabled"].checked,
|
||||
interval_unit: f.elements["interval_unit"].value,
|
||||
interval_value: parseInt(f.elements["interval_value"].value) || 1,
|
||||
cron: f.elements["cron"].value.trim(),
|
||||
};
|
||||
}
|
||||
try {
|
||||
if (state.editTask) {
|
||||
await api(`/api/tasks/${state.editTask.id}`, {
|
||||
method: "PUT", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
toast("任务已更新");
|
||||
} else {
|
||||
const t = await api("/api/tasks", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
toast(`任务「${t.name}」已创建`);
|
||||
}
|
||||
hideModal("taskModal");
|
||||
loadTasks();
|
||||
} catch (err) { toast(err.message, true); }
|
||||
}
|
||||
|
||||
/* ---------------- 详情 ---------------- */
|
||||
async function openDetail(tid) {
|
||||
try {
|
||||
const t = await api(`/api/tasks/${tid}`);
|
||||
state.detail.task = t;
|
||||
const runs = t.runs || [];
|
||||
state.detail.runId = runs.find((r) => r.status === "running")
|
||||
? runs[0].id : (runs[0] ? runs[0].id : null);
|
||||
state.detail.logOffset = 0;
|
||||
$("detailTitle").textContent = `任务详情 · ${t.name}`;
|
||||
renderDetail();
|
||||
showModal("detailModal");
|
||||
startLogPoll();
|
||||
} catch (e) { toast(e.message, true); }
|
||||
}
|
||||
|
||||
function renderDetail() {
|
||||
const t = state.detail.task;
|
||||
const runs = t.runs || [];
|
||||
const run = runs.find((r) => r.id === state.detail.runId) || runs[0] || null;
|
||||
state.detail.runId = run ? run.id : null;
|
||||
const cfg = t.config || {};
|
||||
|
||||
const meta = `
|
||||
<div class="detail-meta">
|
||||
<span>模式: <b>${MODE_LABEL[t.mode] || t.mode}</b></span>
|
||||
<span>任务ID: <b>${t.id}</b></span>
|
||||
<span>创建: <b>${fmtTime(t.created_at)}</b></span>
|
||||
<span>输出: <b>${esc(cfg.out_dir || "out/" + t.id)}</b></span>
|
||||
${t.mode === "auto" ? `<span>起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b></span>` : ""}
|
||||
</div>`;
|
||||
|
||||
let sched = "";
|
||||
if (t.mode === "scheduled" && t.schedule) {
|
||||
const s = t.schedule;
|
||||
sched = `<div class="detail-meta">
|
||||
<span>调度: <b>${s.type === "cron" ? esc(s.cron) : "每 " + (s.interval_value || "-") + " " + ({ minutes: "分钟", hours: "小时", days: "天" }[s.interval_unit] || "")}</b></span>
|
||||
<span>启用: <b>${s.enabled ? "是" : "否"}</b></span>
|
||||
<span>下次: <b>${esc(s.next_run || "—")}</b></span>
|
||||
<span>上次: <b>${esc(s.last_run || "—")}</b></span>
|
||||
<span>已运行: <b>${s.runs_count || 0}</b> 次</span>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
const acts = `
|
||||
<div class="detail-actions">
|
||||
${t.running ? `
|
||||
<button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
|
||||
<button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
|
||||
: `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`}
|
||||
<button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
|
||||
</div>`;
|
||||
|
||||
const chips = runs.map((r) => `
|
||||
<span class="run-chip ${r.id === state.detail.runId ? "active" : ""}"
|
||||
onclick="selectRun('${r.id}')">
|
||||
${fmtTime(r.started_at).slice(5)} · ${r.status} · ${r.progress.done || 0}/${r.progress.total || "?"}
|
||||
</span>`).join("");
|
||||
|
||||
const body = run ? renderRunPanel(t, run) : '<div class="card-line">暂无运行记录,点击「立即执行」开始第一次爬取。</div>';
|
||||
|
||||
$("detailBody").innerHTML = `
|
||||
${meta}
|
||||
${sched}
|
||||
${acts}
|
||||
<div class="runs-bar">${chips || '<span class="card-line">暂无运行记录</span>'}</div>
|
||||
${body}`;
|
||||
}
|
||||
|
||||
function renderRunPanel(t, run) {
|
||||
const s = run.stats || { ok: 0, fail: 0, images: 0 };
|
||||
const prog = run.status === "running" || run.status === "paused" ? `
|
||||
<div class="progress"><div style="width:${run.progress.percent || 0}%"></div></div>` : "";
|
||||
const results = run.results || [];
|
||||
const rows = results.map((r, i) => {
|
||||
const statusCls = r.status === "OK" ? "t-ok" : "t-fail";
|
||||
const html = r.html_file ? `<span class="file-link" onclick="previewFile('${t.id}','${esc(r.html_file)}','HTML: ${esc(r.url)}')">HTML</span>` : "—";
|
||||
const txt = r.txt_file ? `<span class="file-link" onclick="previewFile('${t.id}','${esc(r.txt_file)}','文本: ${esc(r.url)}')">TXT</span>` : "—";
|
||||
const imgs = (r.images || []).length
|
||||
? `<span class="file-link" onclick="scrollThumbs()">🖼️ ${(r.images || []).length}</span>` : "—";
|
||||
return `<tr>
|
||||
<td>${i + 1}</td>
|
||||
<td class="${statusCls}">${r.status}</td>
|
||||
<td class="title-cell" title="${esc(r.title)}">${esc(r.title)}</td>
|
||||
<td class="url-cell" title="${esc(r.url)}">${esc(r.url)}</td>
|
||||
<td>${html}</td><td>${txt}</td><td>${imgs}</td>
|
||||
<td title="${esc(r.error || "")}">${esc((r.error || "").slice(0, 60))}</td>
|
||||
</tr>`;
|
||||
}).join("");
|
||||
const thumbs = results.flatMap((r) => r.images || []).slice(0, 60);
|
||||
const thumbHtml = thumbs.length ? `
|
||||
<div id="thumbsBox" class="thumbs">
|
||||
${thumbs.map((im) => `<img src="/api/file?task_id=${t.id}&path=${esc(im.file)}" loading="lazy"
|
||||
onclick="previewFile('${t.id}','${esc(im.file)}','图片预览')">`).join("")}
|
||||
</div>` : "";
|
||||
|
||||
return `
|
||||
<div class="run-panel">
|
||||
<div class="run-stats">
|
||||
<span>运行ID: <b>${run.id}</b></span>
|
||||
<span>状态: <b class="${run.status === "running" ? "t-ok" : ""}">${run.status}</b></span>
|
||||
<span>开始: <b>${fmtTime(run.started_at)}</b></span>
|
||||
<span>结束: <b>${fmtTime(run.finished_at)}</b></span>
|
||||
<span class="ok">✅ ${s.ok}</span>
|
||||
<span class="fail">❌ ${s.fail}</span>
|
||||
<span class="img">🖼️ ${s.images}</span>
|
||||
</div>
|
||||
${prog}
|
||||
<div class="card-line">当前: <b>${esc(run.progress.current_url || "")}</b></div>
|
||||
${results.length ? `
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>#</th><th>状态</th><th>标题</th><th>网址</th><th>HTML</th><th>TXT</th><th>图片</th><th>错误</th></tr></thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
</div>` : '<div class="card-line">暂无结果</div>'}
|
||||
${thumbHtml}
|
||||
<div class="logs" id="logBox"></div>
|
||||
</div>`;
|
||||
}
|
||||
|
||||
function scrollThumbs() {
|
||||
const box = $("thumbsBox");
|
||||
if (box) box.scrollIntoView({ behavior: "smooth", block: "center" });
|
||||
}
|
||||
|
||||
async function selectRun(rid) {
|
||||
state.detail.runId = rid;
|
||||
state.detail.logOffset = 0;
|
||||
renderDetail();
|
||||
startLogPoll();
|
||||
}
|
||||
|
||||
async function startLogPoll() {
|
||||
stopLogPoll();
|
||||
const t = state.detail.task;
|
||||
if (!t || !state.detail.runId) return;
|
||||
const rid = state.detail.runId;
|
||||
const tick = async () => {
|
||||
try {
|
||||
const d = await api(`/api/runs/${rid}/logs?offset=${state.detail.logOffset}`);
|
||||
if (d.logs && d.logs.length) {
|
||||
const box = $("logBox");
|
||||
if (box) {
|
||||
const atBottom = box.scrollHeight - box.scrollTop - box.clientHeight < 40;
|
||||
d.logs.forEach((l) => {
|
||||
const div = document.createElement("div");
|
||||
div.className = l.level;
|
||||
div.textContent = `[${l.ts}] ${l.msg}`;
|
||||
box.appendChild(div);
|
||||
});
|
||||
if (atBottom) box.scrollTop = box.scrollHeight;
|
||||
state.detail.logOffset += d.logs.length;
|
||||
}
|
||||
}
|
||||
} catch (e) { /* ignore */ }
|
||||
loadTasks(); // 后台保持列表刷新
|
||||
};
|
||||
tick();
|
||||
state.logTimer = setInterval(tick, 2500);
|
||||
}
|
||||
|
||||
function stopLogPoll() {
|
||||
if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; }
|
||||
}
|
||||
|
||||
/* ---------------- 文件预览 ---------------- */
|
||||
function previewFile(tid, path, title) {
|
||||
$("previewTitle").textContent = title || "预览";
|
||||
$("previewFrame").src = `/api/file?task_id=${tid}&path=${encodeURIComponent(path)}`;
|
||||
showModal("previewModal");
|
||||
}
|
||||
|
||||
/* ---------------- 弹窗控制 ---------------- */
|
||||
function showModal(id) { $(id).classList.remove("hidden"); }
|
||||
function hideModal(id) { $(id).classList.add("hidden"); }
|
||||
|
||||
/* ---------------- 事件绑定 ---------------- */
|
||||
$("btnNew").onclick = openCreate;
|
||||
$("btnNew2").onclick = openCreate;
|
||||
$("btnRefresh").onclick = loadTasks;
|
||||
|
||||
document.querySelectorAll("#modeTabs .tab").forEach((b) => {
|
||||
b.onclick = () => { switchMode(b.dataset.mode, false); };
|
||||
});
|
||||
$("taskForm").onsubmit = submitForm;
|
||||
$("taskForm").elements["schedule_type"].onchange = syncScheduleUI;
|
||||
|
||||
document.querySelectorAll("[data-close]").forEach((b) => b.onclick = () => hideModal("taskModal"));
|
||||
document.querySelectorAll("[data-close-detail]").forEach((b) => b.onclick = () => { stopLogPoll(); hideModal("detailModal"); });
|
||||
document.querySelectorAll("[data-close-preview]").forEach((b) => b.onclick = () => { $("previewFrame").src = "about:blank"; hideModal("previewModal"); });
|
||||
|
||||
document.querySelectorAll(".modal-overlay").forEach((ov) => {
|
||||
ov.addEventListener("mousedown", (e) => {
|
||||
if (e.target === ov) {
|
||||
if (ov.id === "detailModal") stopLogPoll();
|
||||
if (ov.id === "previewModal") $("previewFrame").src = "about:blank";
|
||||
ov.classList.add("hidden");
|
||||
}
|
||||
});
|
||||
});
|
||||
document.addEventListener("keydown", (e) => {
|
||||
if (e.key === "Escape") {
|
||||
stopLogPoll();
|
||||
$("previewFrame").src = "about:blank";
|
||||
["taskModal", "detailModal", "previewModal"].forEach((id) => $(id).classList.add("hidden"));
|
||||
}
|
||||
});
|
||||
|
||||
/* ---------------- 启动 ---------------- */
|
||||
loadTasks();
|
||||
setInterval(() => {
|
||||
const detailOpen = !$("detailModal").classList.contains("hidden");
|
||||
if (!detailOpen) loadTasks();
|
||||
}, 4000);
|
||||
@@ -0,0 +1,143 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>通用爬虫系统</title>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<div class="logo">🕷️ 通用爬虫系统 <span id="version" class="version"></span></div>
|
||||
<div class="header-right">
|
||||
<span class="status-pill" id="statusPill">运行中任务: 0</span>
|
||||
<button id="btnRefresh" class="btn ghost">⟳ 刷新</button>
|
||||
<button id="btnNew" class="btn primary">+ 新建任务</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div id="taskList" class="task-grid"></div>
|
||||
<div id="emptyState" class="empty hidden">
|
||||
<div class="empty-icon">🕸️</div>
|
||||
<p>还没有爬取任务</p>
|
||||
<button id="btnNew2" class="btn primary">+ 创建第一个任务</button>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
<!-- 新建 / 编辑任务弹窗 -->
|
||||
<div id="taskModal" class="modal-overlay hidden">
|
||||
<div class="modal">
|
||||
<div class="modal-head">
|
||||
<span id="modalTitle">新建爬取任务</span>
|
||||
<button class="btn ghost sm" data-close>✕</button>
|
||||
</div>
|
||||
<div class="tabs" id="modeTabs">
|
||||
<button class="tab active" data-mode="batch">📄 批量爬取</button>
|
||||
<button class="tab" data-mode="scheduled">⏰ 定时爬取</button>
|
||||
<button class="tab" data-mode="auto">🤖 自动爬取</button>
|
||||
</div>
|
||||
<form id="taskForm">
|
||||
<div class="form-body">
|
||||
<div class="field"><label>项目名称 *</label><input name="name" required placeholder="例如: TPU 评测抓取"></div>
|
||||
<div class="field"><label>输出目录</label><input name="out_dir" placeholder="留空自动: out/任务ID;可填绝对路径"></div>
|
||||
<div class="row2">
|
||||
<div class="field"><label>爬取间隔(秒) 最小</label><input name="delay_min" type="number" min="0.1" step="0.1" value="2"></div>
|
||||
<div class="field"><label>最大</label><input name="delay_max" type="number" min="0.1" step="0.1" value="5"></div>
|
||||
</div>
|
||||
<div class="row2">
|
||||
<div class="field"><label>单页超时(秒)</label><input name="timeout" type="number" value="60"></div>
|
||||
<div class="field"><label>失败重试次数</label><input name="retry_count" type="number" value="2"></div>
|
||||
</div>
|
||||
<div class="row2">
|
||||
<div class="field"><label>重试间隔(秒)</label><input name="retry_interval" type="number" value="3"></div>
|
||||
<div class="field check"><label><input name="crawl_images" type="checkbox"> 爬取图片</label></div>
|
||||
</div>
|
||||
<div class="row2">
|
||||
<div class="field check"><label><input name="notify" type="checkbox"> 完成后邮件通知</label></div>
|
||||
<div class="field"><label>通知邮箱</label><input name="notify_email" value="wlq@tphai.com"></div>
|
||||
</div>
|
||||
|
||||
<div class="field" id="urlsField"><label>网址列表(每行一个,# 开头为注释)</label>
|
||||
<textarea name="urls" rows="5" placeholder="https://www.example.com/ https://www.example.com/page2"></textarea>
|
||||
</div>
|
||||
|
||||
<div id="scheduleBox" class="hidden box">
|
||||
<div class="row2">
|
||||
<div class="field"><label>调度类型</label>
|
||||
<select name="schedule_type">
|
||||
<option value="interval">间隔调度(每 N 时间)</option>
|
||||
<option value="cron">cron 表达式</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="field check"><label><input name="schedule_enabled" type="checkbox" checked> 启用调度</label></div>
|
||||
</div>
|
||||
<div id="intervalBox" class="row2">
|
||||
<div class="field"><label>每隔</label><input name="interval_value" type="number" value="24"></div>
|
||||
<div class="field"><label>单位</label>
|
||||
<select name="interval_unit">
|
||||
<option value="minutes">分钟</option>
|
||||
<option value="hours" selected>小时</option>
|
||||
<option value="days">天</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div id="cronBox" class="field hidden">
|
||||
<label>cron 表达式(分 时 日 月 周,周 0/7=周日)</label>
|
||||
<input name="cron" placeholder="0 3 * * * 每天凌晨3点">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="autoBox" class="hidden box">
|
||||
<div class="field"><label>起始网址 *(系统将自动发现符合规则的链接并爬取)</label>
|
||||
<input name="seed_url" placeholder="https://example.com/news"></div>
|
||||
<div class="row2">
|
||||
<div class="field"><label>包含规则(每行一个,子串或正则)</label>
|
||||
<textarea name="include" rows="3" placeholder="techpowerup.com/review /news/"></textarea></div>
|
||||
<div class="field"><label>排除规则</label>
|
||||
<textarea name="exclude" rows="3" placeholder="login, signup, /tag/, /forum/"></textarea></div>
|
||||
</div>
|
||||
<div class="row2">
|
||||
<div class="field check"><label><input name="same_domain" type="checkbox" checked> 仅爬同域名</label></div>
|
||||
<div class="field check"><label><input name="use_regex" type="checkbox"> 规则按正则匹配</label></div>
|
||||
</div>
|
||||
<div class="row2">
|
||||
<div class="field"><label>最大页数</label><input name="max_pages" type="number" value="50"></div>
|
||||
<div class="field"><label>最大爬取深度</label><input name="max_depth" type="number" value="2"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="modal-foot">
|
||||
<span id="formHint" class="hint"></span>
|
||||
<button type="button" class="btn ghost" data-close>取消</button>
|
||||
<button type="submit" class="btn primary">保存</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 任务详情弹窗 -->
|
||||
<div id="detailModal" class="modal-overlay hidden">
|
||||
<div class="modal wide">
|
||||
<div class="modal-head">
|
||||
<span id="detailTitle">任务详情</span>
|
||||
<button class="btn ghost sm" data-close-detail>✕</button>
|
||||
</div>
|
||||
<div id="detailBody"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 文件预览弹窗 -->
|
||||
<div id="previewModal" class="modal-overlay hidden">
|
||||
<div class="modal wide tall">
|
||||
<div class="modal-head">
|
||||
<span id="previewTitle">预览</span>
|
||||
<button class="btn ghost sm" data-close-preview>✕</button>
|
||||
</div>
|
||||
<iframe id="previewFrame" sandbox="allow-same-origin"></iframe>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,181 @@
|
||||
:root {
|
||||
--bg: #0f1420;
|
||||
--panel: #171e2e;
|
||||
--panel2: #1e2740;
|
||||
--border: #2a3550;
|
||||
--text: #dbe4f5;
|
||||
--muted: #8b97b5;
|
||||
--accent: #4f8cff;
|
||||
--green: #37d39a;
|
||||
--red: #ff5d6c;
|
||||
--yellow: #ffc857;
|
||||
--purple: #a78bfa;
|
||||
}
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body {
|
||||
background: var(--bg); color: var(--text);
|
||||
font-family: -apple-system, "PingFang SC", "Microsoft YaHei", "Segoe UI", sans-serif;
|
||||
font-size: 14px; min-height: 100vh;
|
||||
}
|
||||
header {
|
||||
display: flex; align-items: center; justify-content: space-between;
|
||||
padding: 14px 24px; background: var(--panel);
|
||||
border-bottom: 1px solid var(--border); position: sticky; top: 0; z-index: 10;
|
||||
}
|
||||
.logo { font-size: 18px; font-weight: 700; letter-spacing: .5px; }
|
||||
.version { font-size: 12px; color: var(--muted); font-weight: 400; margin-left: 8px; }
|
||||
.header-right { display: flex; align-items: center; gap: 10px; }
|
||||
.status-pill {
|
||||
background: var(--panel2); border: 1px solid var(--border);
|
||||
padding: 5px 12px; border-radius: 20px; font-size: 12px; color: var(--muted);
|
||||
}
|
||||
.status-pill.active { color: var(--green); border-color: var(--green); }
|
||||
main { padding: 20px 24px; max-width: 1500px; margin: 0 auto; }
|
||||
|
||||
/* ---------- buttons ---------- */
|
||||
.btn {
|
||||
background: var(--panel2); color: var(--text); border: 1px solid var(--border);
|
||||
padding: 7px 14px; border-radius: 8px; cursor: pointer; font-size: 13px;
|
||||
transition: .15s; white-space: nowrap;
|
||||
}
|
||||
.btn:hover { border-color: var(--accent); color: #fff; }
|
||||
.btn.primary { background: var(--accent); border-color: var(--accent); color: #fff; }
|
||||
.btn.primary:hover { filter: brightness(1.1); }
|
||||
.btn.ghost { background: transparent; }
|
||||
.btn.danger:hover { border-color: var(--red); color: var(--red); }
|
||||
.btn.sm { padding: 3px 8px; font-size: 12px; border-radius: 6px; }
|
||||
.btn:disabled { opacity: .4; cursor: not-allowed; }
|
||||
|
||||
/* ---------- task grid ---------- */
|
||||
.task-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(380px, 1fr)); gap: 16px; }
|
||||
.card {
|
||||
background: var(--panel); border: 1px solid var(--border);
|
||||
border-radius: 12px; padding: 16px; display: flex; flex-direction: column; gap: 10px;
|
||||
transition: .15s;
|
||||
}
|
||||
.card:hover { border-color: #3a4a75; transform: translateY(-1px); }
|
||||
.card-head { display: flex; align-items: center; justify-content: space-between; gap: 8px; }
|
||||
.card-name { font-size: 15px; font-weight: 600; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.card-meta { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; }
|
||||
.badge {
|
||||
font-size: 11px; padding: 2px 8px; border-radius: 10px;
|
||||
background: var(--panel2); border: 1px solid var(--border); color: var(--muted);
|
||||
}
|
||||
.badge.batch { color: #7cc4ff; border-color: #7cc4ff55; }
|
||||
.badge.scheduled { color: var(--purple); border-color: var(--purple); }
|
||||
.badge.auto { color: var(--yellow); border-color: var(--yellow); }
|
||||
.badge.running { color: var(--green); border-color: var(--green); animation: pulse 1.6s infinite; }
|
||||
.badge.paused { color: var(--yellow); border-color: var(--yellow); }
|
||||
.badge.completed { color: var(--green); border-color: var(--green); }
|
||||
.badge.failed { color: var(--red); border-color: var(--red); }
|
||||
.badge.stopped { color: var(--muted); border-color: var(--border); }
|
||||
.badge.scheduled-wait { color: var(--purple); border-color: var(--purple); }
|
||||
@keyframes pulse { 50% { opacity: .5; } }
|
||||
|
||||
.card-line { color: var(--muted); font-size: 12px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.card-line b { color: var(--text); font-weight: 500; }
|
||||
.progress { height: 6px; background: var(--panel2); border-radius: 4px; overflow: hidden; }
|
||||
.progress > div { height: 100%; background: linear-gradient(90deg, var(--accent), var(--green)); transition: width .5s; }
|
||||
.progress-label { display: flex; justify-content: space-between; font-size: 11px; color: var(--muted); }
|
||||
.stats { display: flex; gap: 14px; font-size: 12px; color: var(--muted); }
|
||||
.stats .ok { color: var(--green); } .stats .fail { color: var(--red); } .stats .img { color: var(--yellow); }
|
||||
.card-actions { display: flex; gap: 6px; flex-wrap: wrap; margin-top: auto; }
|
||||
|
||||
/* ---------- empty ---------- */
|
||||
.empty { text-align: center; padding: 80px 0; color: var(--muted); }
|
||||
.empty-icon { font-size: 48px; margin-bottom: 12px; }
|
||||
.empty button { margin-top: 14px; }
|
||||
|
||||
/* ---------- modal ---------- */
|
||||
.modal-overlay {
|
||||
position: fixed; inset: 0; background: rgba(5, 8, 15, .7);
|
||||
display: flex; align-items: center; justify-content: center; z-index: 100;
|
||||
backdrop-filter: blur(3px);
|
||||
}
|
||||
.modal {
|
||||
background: var(--panel); border: 1px solid var(--border); border-radius: 14px;
|
||||
width: 640px; max-width: 95vw; max-height: 92vh; display: flex; flex-direction: column;
|
||||
box-shadow: 0 20px 60px rgba(0,0,0,.5);
|
||||
}
|
||||
.modal.wide { width: 1080px; }
|
||||
.modal.tall { width: 1000px; height: 88vh; }
|
||||
.modal-head {
|
||||
display: flex; justify-content: space-between; align-items: center;
|
||||
padding: 14px 18px; border-bottom: 1px solid var(--border); font-size: 15px; font-weight: 600;
|
||||
}
|
||||
.modal-foot {
|
||||
display: flex; justify-content: flex-end; gap: 10px; align-items: center;
|
||||
padding: 12px 18px; border-top: 1px solid var(--border);
|
||||
}
|
||||
.hint { color: var(--yellow); font-size: 12px; margin-right: auto; }
|
||||
|
||||
.tabs { display: flex; gap: 6px; padding: 12px 18px 0; }
|
||||
.tab {
|
||||
padding: 8px 16px; background: transparent; border: 1px solid var(--border);
|
||||
border-bottom: none; border-radius: 10px 10px 0 0; color: var(--muted);
|
||||
cursor: pointer; font-size: 13px;
|
||||
}
|
||||
.tab.active { background: var(--panel2); color: #fff; border-color: var(--accent); }
|
||||
.tab:disabled { opacity: .4; cursor: not-allowed; }
|
||||
|
||||
.form-body { padding: 16px 18px; overflow-y: auto; display: flex; flex-direction: column; gap: 12px; }
|
||||
.field { display: flex; flex-direction: column; gap: 5px; flex: 1; min-width: 0; }
|
||||
.field label { font-size: 12px; color: var(--muted); }
|
||||
.field.check { flex-direction: row; align-items: center; gap: 8px; padding-top: 22px; }
|
||||
.field.check label { color: var(--text); font-size: 13px; display: flex; align-items: center; gap: 7px; cursor: pointer; }
|
||||
input, select, textarea {
|
||||
background: var(--panel2); border: 1px solid var(--border); color: var(--text);
|
||||
border-radius: 8px; padding: 8px 10px; font-size: 13px; font-family: inherit; width: 100%;
|
||||
}
|
||||
input:focus, select:focus, textarea:focus { outline: none; border-color: var(--accent); }
|
||||
textarea { resize: vertical; }
|
||||
input[type="checkbox"] { width: auto; accent-color: var(--accent); }
|
||||
.row2 { display: flex; gap: 12px; }
|
||||
.box { border: 1px dashed var(--border); border-radius: 10px; padding: 12px; display: flex; flex-direction: column; gap: 10px; background: #141b2b; }
|
||||
.hidden { display: none !important; }
|
||||
|
||||
/* ---------- detail ---------- */
|
||||
.detail-wrap { padding: 16px 18px; overflow-y: auto; display: flex; flex-direction: column; gap: 14px; }
|
||||
.detail-meta { display: flex; flex-wrap: wrap; gap: 14px; font-size: 12px; color: var(--muted); }
|
||||
.detail-meta b { color: var(--text); font-weight: 500; }
|
||||
.detail-actions { display: flex; gap: 8px; flex-wrap: wrap; }
|
||||
.runs-bar { display: flex; gap: 8px; flex-wrap: wrap; align-items: center; }
|
||||
.run-chip {
|
||||
padding: 5px 12px; border-radius: 8px; border: 1px solid var(--border);
|
||||
background: var(--panel2); cursor: pointer; font-size: 12px; color: var(--muted);
|
||||
}
|
||||
.run-chip.active { border-color: var(--accent); color: #fff; }
|
||||
.run-panel { border: 1px solid var(--border); border-radius: 10px; padding: 14px; display: flex; flex-direction: column; gap: 10px; background: var(--panel2); }
|
||||
.run-stats { display: flex; gap: 16px; font-size: 12px; flex-wrap: wrap; }
|
||||
.table-wrap { overflow: auto; max-height: 320px; border: 1px solid var(--border); border-radius: 8px; }
|
||||
table { width: 100%; border-collapse: collapse; font-size: 12px; }
|
||||
th, td { padding: 7px 10px; text-align: left; border-bottom: 1px solid var(--border); white-space: nowrap; }
|
||||
th { position: sticky; top: 0; background: var(--panel2); color: var(--muted); font-weight: 500; z-index: 1; }
|
||||
td.url-cell { max-width: 300px; overflow: hidden; text-overflow: ellipsis; }
|
||||
td.title-cell { max-width: 220px; overflow: hidden; text-overflow: ellipsis; }
|
||||
.t-ok { color: var(--green); } .t-fail { color: var(--red); }
|
||||
.file-link { color: var(--accent); cursor: pointer; text-decoration: underline; }
|
||||
.file-link:hover { color: #9ec0ff; }
|
||||
.thumbs { display: flex; flex-wrap: wrap; gap: 8px; }
|
||||
.thumbs img { width: 96px; height: 72px; object-fit: cover; border-radius: 6px; border: 1px solid var(--border); cursor: pointer; }
|
||||
.logs {
|
||||
background: #0b0f1a; border: 1px solid var(--border); border-radius: 8px;
|
||||
padding: 10px; font-family: ui-monospace, Consolas, monospace; font-size: 12px;
|
||||
max-height: 220px; overflow-y: auto; line-height: 1.7;
|
||||
}
|
||||
.logs .info { color: var(--text); } .logs .warn { color: var(--yellow); } .logs .error { color: var(--red); }
|
||||
|
||||
/* ---------- preview ---------- */
|
||||
#previewFrame { flex: 1; border: none; background: #fff; border-radius: 0 0 14px 14px; }
|
||||
|
||||
/* ---------- misc ---------- */
|
||||
.toast {
|
||||
position: fixed; top: 70px; left: 50%; transform: translateX(-50%);
|
||||
background: var(--panel2); border: 1px solid var(--accent); color: #fff;
|
||||
padding: 10px 20px; border-radius: 10px; z-index: 300; font-size: 13px;
|
||||
box-shadow: 0 8px 30px rgba(0,0,0,.4); animation: fadein .2s;
|
||||
}
|
||||
.toast.error { border-color: var(--red); }
|
||||
@keyframes fadein { from { opacity: 0; transform: translate(-50%, -8px); } }
|
||||
::-webkit-scrollbar { width: 8px; height: 8px; }
|
||||
::-webkit-scrollbar-thumb { background: #2a3550; border-radius: 4px; }
|
||||
Reference in New Issue
Block a user