hz4th_coder
796e667533
- /api/tasks 响应 1.57MB -> 12KB (latest_run 只带摘要, 一次读 runs.json) - /api/stats 磁盘占用加 30s 缓存, 去除重复全量读 - 详情接口分页返回 results (默认100条/页), 前端表格分页+页码跳转 - auto 任务 pending/visited 队列迁移到 data/auto_state/ 独立文件 (tasks.json 1.6MB -> 14KB) - MySQL 同步改后台线程异步执行 (db.sync_run_async/upsert_task_async), 不再阻塞爬虫和 API - 启动时自动迁移存量 auto 状态 - 统计/回收站/运行中数值在未读到真实数据前显示 '-'
221 lines
10 KiB
HTML
221 lines
10 KiB
HTML
<!DOCTYPE html>
|
||
<html lang="zh-CN">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<title>通用爬虫系统</title>
|
||
<link rel="stylesheet" href="style.css">
|
||
</head>
|
||
<body>
|
||
<header>
|
||
<div class="logo">🕷️ 通用爬虫系统 <span id="version" class="version"></span></div>
|
||
<div class="header-right">
|
||
<div class="search-box">
|
||
<input id="searchInput" placeholder="搜索网址 / 标题 / 任务名..." maxlength="100">
|
||
<button id="btnSearch" class="btn sm primary" title="搜索">🔍</button>
|
||
</div>
|
||
<span class="status-pill" id="statusPill">运行中任务: -</span>
|
||
<button id="btnTheme" class="btn ghost" title="切换日间/夜间主题">☀️</button>
|
||
<button id="btnTrash" class="btn ghost" title="回收站">🗑️ 回收站 <span id="trashCount" class="trash-count">-</span></button>
|
||
<button id="btnRefresh" class="btn ghost">⟳ 刷新</button>
|
||
<button id="btnNew" class="btn primary">＋ 新建任务</button>
|
||
</div>
|
||
</header>
|
||
|
||
<main>
|
||
<div class="stats-bar" id="statsBar">
|
||
<div class="stat-card"><div class="stat-num" id="stTasks">-</div><div class="stat-label">📋 任务总数</div></div>
|
||
<div class="stat-card"><div class="stat-num num-run" id="stRunning">-</div><div class="stat-label">🔄 正在运行</div></div>
|
||
<div class="stat-card"><div class="stat-num" id="stRuns">-</div><div class="stat-label">📊 累计运行次数</div></div>
|
||
<div class="stat-card"><div class="stat-num num-ok" id="stOk">-</div><div class="stat-label">✅ 成功页面</div></div>
|
||
<div class="stat-card"><div class="stat-num num-fail" id="stFail">-</div><div class="stat-label">❌ 失败页面</div></div>
|
||
<div class="stat-card"><div class="stat-num" id="stImgs">-</div><div class="stat-label">🖼️ 已爬图片</div></div>
|
||
<div class="stat-card"><div class="stat-num" id="stDisk">-</div><div class="stat-label">💾 磁盘占用</div></div>
|
||
</div>
|
||
<div id="taskList" class="task-grid"></div>
|
||
<div id="emptyState" class="empty hidden">
|
||
<div class="empty-icon">🕸️</div>
|
||
<p>还没有爬取任务</p>
|
||
<button id="btnNew2" class="btn primary">＋ 创建第一个任务</button>
|
||
</div>
|
||
</main>
|
||
|
||
<!-- 新建 / 编辑任务弹窗 -->
|
||
<div id="taskModal" class="modal-overlay hidden">
|
||
<div class="modal">
|
||
<div class="modal-head">
|
||
<span id="modalTitle">新建爬取任务</span>
|
||
<button class="btn ghost sm" data-close>✕</button>
|
||
</div>
|
||
<div class="tabs" id="modeTabs">
|
||
<button class="tab active" data-mode="batch">📄 批量爬取</button>
|
||
<button class="tab" data-mode="scheduled">⏰ 定时爬取</button>
|
||
<button class="tab" data-mode="auto">🤖 自动爬取</button>
|
||
</div>
|
||
<form id="taskForm">
|
||
<div class="form-body">
|
||
<div class="field"><label>项目名称 *</label><input name="name" required placeholder="例如: TPU 评测抓取"></div>
|
||
<div class="field"><label>输出目录</label><input name="out_dir" placeholder="留空自动: out/任务ID；可填绝对路径"></div>
|
||
<div class="row2">
|
||
<div class="field"><label>爬取间隔(秒) 最小</label><input name="delay_min" type="number" min="0.1" step="0.1" value="2"></div>
|
||
<div class="field"><label>最大</label><input name="delay_max" type="number" min="0.1" step="0.1" value="5"></div>
|
||
</div>
|
||
<div class="row2">
|
||
<div class="field"><label>单页超时(秒)</label><input name="timeout" type="number" value="60"></div>
|
||
<div class="field"><label>失败重试次数</label><input name="retry_count" type="number" value="2"></div>
|
||
</div>
|
||
<div class="row2">
|
||
<div class="field"><label>重试间隔(秒)</label><input name="retry_interval" type="number" value="3"></div>
|
||
<div class="field check"><label><input name="crawl_images" type="checkbox"> 爬取图片</label></div>
|
||
</div>
|
||
<div class="row2">
|
||
<div class="field check"><label><input name="notify" type="checkbox"> 完成后邮件通知</label></div>
|
||
<div class="field"><label>通知邮箱</label><input name="notify_email" value="wlq@tphai.com"></div>
|
||
</div>
|
||
|
||
<div class="field" id="urlsField">
|
||
<label>网址列表（每行一个，# 开头为注释）</label>
|
||
<div class="row2" style="align-items:center">
|
||
<button type="button" class="btn sm" id="btnImportUrls">📂 导入网址文件(.txt)</button>
|
||
<select id="importMode" title="对已有网址的处理方式">
|
||
<option value="append" selected>追加（保留已有）</option>
|
||
<option value="overwrite">覆盖（清空已有）</option>
|
||
</select>
|
||
<span class="card-line">支持 UTF-8 / GBK，自动去重</span>
|
||
<input type="file" id="urlFileInput" accept=".txt,.csv,.urls,text/plain" hidden>
|
||
</div>
|
||
<textarea name="urls" rows="5" placeholder="https://www.example.com/ https://www.example.com/page2"></textarea>
|
||
</div>
|
||
|
||
<div id="scheduleBox" class="hidden box">
|
||
<div class="field"><label>首次执行时间（可留空；留空 = 尽快执行；已过的时间将立即执行）</label>
|
||
<input name="first_run" type="datetime-local"></div>
|
||
<div class="row2">
|
||
<div class="field"><label>调度类型</label>
|
||
<select name="schedule_type">
|
||
<option value="interval">间隔调度（每 N 时间）</option>
|
||
<option value="cron">cron 表达式</option>
|
||
</select>
|
||
</div>
|
||
<div class="field check"><label><input name="schedule_enabled" type="checkbox" checked> 启用调度</label></div>
|
||
</div>
|
||
<div id="intervalBox" class="row2">
|
||
<div class="field"><label>每隔</label><input name="interval_value" type="number" value="24"></div>
|
||
<div class="field"><label>单位</label>
|
||
<select name="interval_unit">
|
||
<option value="minutes">分钟</option>
|
||
<option value="hours" selected>小时</option>
|
||
<option value="days">天</option>
|
||
</select>
|
||
</div>
|
||
</div>
|
||
<div id="cronBox" class="field hidden">
|
||
<label>cron 表达式（分 时 日 月 周，周 0/7=周日）</label>
|
||
<input name="cron" placeholder="0 3 * * * 每天凌晨3点">
|
||
</div>
|
||
</div>
|
||
|
||
<div id="autoBox" class="hidden box">
|
||
<div class="field"><label>起始网址 *（自动发现符合规则的链接，持续递归爬取直到无新链接可爬）</label>
|
||
<div class="row2">
|
||
<input name="seed_url" placeholder="https://example.com/news" style="flex:1">
|
||
<button type="button" class="btn sm" id="btnProbe">🧪 试爬取</button>
|
||
</div>
|
||
</div>
|
||
<div class="row2">
|
||
<div class="field"><label>包含规则（每行一个，子串或正则）</label>
|
||
<textarea name="include" rows="3" placeholder="techpowerup.com/review /news/"></textarea></div>
|
||
<div class="field"><label>排除规则</label>
|
||
<textarea name="exclude" rows="3" placeholder="login, signup, /tag/, /forum/"></textarea></div>
|
||
</div>
|
||
<div class="row2">
|
||
<div class="field check"><label><input name="same_domain" type="checkbox" checked> 仅爬同域名</label></div>
|
||
<div class="field check"><label><input name="use_regex" type="checkbox"> 规则按正则匹配</label></div>
|
||
</div>
|
||
<div class="row2">
|
||
<div class="field"><label>最大爬取深度（0=无限制，N=只爬 N 层）</label>
|
||
<input name="max_depth" type="number" min="0" value="0"></div>
|
||
<div class="field"><label>最大页数（0=无限制，默认 1000 作安全上限）</label>
|
||
<input name="max_pages" type="number" min="0" value="1000"></div>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div class="modal-foot">
|
||
<span id="formHint" class="hint"></span>
|
||
<button type="button" class="btn ghost" data-close>取消</button>
|
||
<button type="submit" class="btn primary">保存</button>
|
||
</div>
|
||
</form>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- 任务详情弹窗 -->
|
||
<div id="detailModal" class="modal-overlay hidden">
|
||
<div class="modal wide">
|
||
<div class="modal-head">
|
||
<span id="detailTitle">任务详情</span>
|
||
<button class="btn ghost sm" data-close-detail>✕</button>
|
||
</div>
|
||
<div id="detailBody"></div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- 搜索结果弹窗 -->
|
||
<div id="searchModal" class="modal-overlay hidden">
|
||
<div class="modal wide">
|
||
<div class="modal-head">
|
||
<span>🔍 搜索结果</span>
|
||
<button class="btn ghost sm" data-close-search>✕</button>
|
||
</div>
|
||
<div id="searchBody" class="detail-wrap"></div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- 回收站弹窗 -->
|
||
<div id="trashModal" class="modal-overlay hidden">
|
||
<div class="modal wide">
|
||
<div class="modal-head">
|
||
<span>🗑️ 回收站（删除的任务统一在这里，可恢复或彻底删除）</span>
|
||
<button class="btn ghost sm" data-close-trash>✕</button>
|
||
</div>
|
||
<div id="trashBody" class="detail-wrap"></div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- 试爬取结果弹窗 -->
|
||
<div id="probeModal" class="modal-overlay hidden">
|
||
<div class="modal wide">
|
||
<div class="modal-head">
|
||
<span>🧪 试爬取结果（规则筛选预览）</span>
|
||
<button class="btn ghost sm" data-close-probe>✕</button>
|
||
</div>
|
||
<div id="probeBody" class="detail-wrap"></div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- 元数据弹窗 -->
|
||
<div id="metaModal" class="modal-overlay hidden">
|
||
<div class="modal">
|
||
<div class="modal-head">
|
||
<span>📋 资源操作信息（元数据）</span>
|
||
<button class="btn ghost sm" data-close-meta>✕</button>
|
||
</div>
|
||
<div id="metaBody" class="detail-wrap"></div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- 文件预览弹窗 -->
|
||
<div id="previewModal" class="modal-overlay hidden">
|
||
<div class="modal wide tall">
|
||
<div class="modal-head">
|
||
<span id="previewTitle">预览</span>
|
||
<button class="btn ghost sm" data-close-preview>✕</button>
|
||
</div>
|
||
<iframe id="previewFrame" sandbox="allow-same-origin"></iframe>
|
||
</div>
|
||
</div>
|
||
|
||
<script src="app.js"></script>
|
||
</body>
|
||
</html>
|