Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 hz4th_coder
|
1de2a0af54 | ||
|
hz4th_coder
|
29176fc66b |
@@ -22,9 +22,12 @@
|
||||
## 功能总览
|
||||
|
||||
### 1. 前端管理界面
|
||||
- **总体统计区**：任务总数 / 运行中 / 累计运行次数 / 成功失败页面 / 图片数 / 磁盘占用
|
||||
- **日间/夜间双主题**：右上角按钮一键切换，自动记忆选择
|
||||
- 任务卡片总览：状态、进度、统计、下次调度时间一目了然
|
||||
- 一键操作：开始 / 暂停 / 恢复 / 终止 / 编辑 / 删除
|
||||
- 任务详情：历次运行记录、结果明细表（HTML/TXT 在线预览）、图片缩略图、实时日志
|
||||
- **防误关保护**：新建/编辑弹窗有未保存修改时，点窗口外 / ESC / 关闭会先确认
|
||||
- 运行中的任务参数支持**热更新**（修改后从下一页起生效）
|
||||
|
||||
### 2. 批量爬取模式
|
||||
@@ -36,18 +39,25 @@
|
||||
- 完成后邮件通知（复用 send_email.py，默认发到 wlq@tphai.com）
|
||||
|
||||
### 3. 定时爬取模式
|
||||
- **间隔调度**：每 N 分钟 / 小时 / 天
|
||||
- **cron 表达式**：5 段式（分 时 日 月 周，周 0/7=周日），如 `0 3 * * *` 每天凌晨 3 点
|
||||
- **间隔调度**：每 N 分钟 / 小时 / 天，可指定**首次执行时间**（留空=尽快，已过时间立即执行）
|
||||
- **cron 表达式**：5 段式（分 时 日 月 周，周 0/7=周日），如 `0 3 * * *` 每天凌晨 3 点，同样支持首次执行时间作为计算起点
|
||||
- 可启用/停用调度，自动计算下次执行时间；到点自动开跑，跑完自动计算下一次
|
||||
|
||||
### 4. 自动爬取模式
|
||||
给定一个起始网址，系统自动从页面里发现链接、按规则筛选后 BFS 爬取：
|
||||
- **🧪 试爬取**：先用起始网址试跑一次，展示规则筛选后的链接清单（将爬取哪些、排除哪些及原因），确认规则符合预期后再正式开爬
|
||||
- **包含规则**：只爬包含指定子串（或正则）的链接
|
||||
- **排除规则**：跳过匹配的链接（如 login、/tag/）
|
||||
- **仅同域名**：限制在起始网站内
|
||||
- **最大页数 / 最大深度**：控制爬取规模
|
||||
- 其余参数（间隔、重试、图片、通知）同批量模式
|
||||
|
||||
### 5. 资源操作信息（元数据）
|
||||
每个爬取的网页和图片都自动生成 `.meta.json` 操作信息文件：
|
||||
- **网页**（`<文件名>.meta.json`）：爬取模式（批量/定时/自动）、爬取时间、爬取网址、**来源链接**（自动模式下从哪个页面发现）、爬取深度、任务/运行 ID、页面标题、状态、尝试次数、文件列表、图片明细
|
||||
- **图片集**（`<文件名>_img/meta.json`）：所属页面、来源链接、每张图片的原始 URL / 大小 / 下载时间
|
||||
- 详情页结果表中点「📋 元数据」即可在线查看
|
||||
|
||||
## 输出文件
|
||||
|
||||
每个任务输出到独立目录（默认 `out/<任务ID>/`）：
|
||||
@@ -66,6 +76,8 @@
|
||||
| POST | `/api/tasks/<id>/pause` | 暂停 |
|
||||
| POST | `/api/tasks/<id>/resume` | 恢复 |
|
||||
| POST | `/api/tasks/<id>/stop` | 终止 |
|
||||
| POST | `/api/probe` | 试爬取（表单规则预览链接清单） |
|
||||
| POST | `/api/tasks/<id>/probe` | 对已保存的自动任务试爬取 |
|
||||
| GET | `/api/runs/<rid>` | 运行详情（结果+日志） |
|
||||
| GET | `/api/runs/<rid>/logs?offset=N` | 增量日志 |
|
||||
| GET | `/api/file?task_id=&path=` | 读取输出文件（HTML/TXT/图片） |
|
||||
|
||||
@@ -10,7 +10,7 @@ from datetime import datetime
|
||||
from flask import Flask, jsonify, request, send_file, send_from_directory
|
||||
|
||||
import store
|
||||
from engine import CrawlJob
|
||||
from engine import CrawlJob, probe_links
|
||||
from scheduler import Scheduler, cron_next, interval_delta
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
@@ -38,6 +38,33 @@ def now_str():
|
||||
return store.now_str()
|
||||
|
||||
|
||||
def _parse_first_run(s):
|
||||
"""解析表单提交的首次执行时间 (datetime-local 格式), 非法返回 None"""
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(str(s), "%Y-%m-%dT%H:%M")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _schedule_next_run(sch):
|
||||
"""根据调度配置 + 首次执行时间计算 next_run (str)"""
|
||||
first_run = _parse_first_run(sch.get("first_run"))
|
||||
base = first_run if (first_run and first_run > datetime.now()) else None
|
||||
if sch.get("type") == "cron":
|
||||
expr = sch.get("cron") or "0 * * * *"
|
||||
nn = cron_next(expr, base or datetime.now())
|
||||
if not nn:
|
||||
raise ValueError("cron 表达式在未来一年内无匹配时间")
|
||||
return nn.strftime("%Y-%m-%d %H:%M:%S")
|
||||
sch.setdefault("interval_unit", "hours")
|
||||
sch.setdefault("interval_value", 24)
|
||||
if base:
|
||||
return base.strftime("%Y-%m-%d %H:%M:%S")
|
||||
return (datetime.now() + interval_delta(sch)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def resolve_out_dir(task):
|
||||
cfg = task.get("config", {}) or {}
|
||||
if cfg.get("out_dir", "").strip():
|
||||
@@ -161,16 +188,7 @@ def api_create_task():
|
||||
sch.setdefault("enabled", True)
|
||||
sch.setdefault("type", "interval")
|
||||
try:
|
||||
if sch.get("type") == "cron":
|
||||
expr = sch.get("cron") or "0 * * * *"
|
||||
nn = cron_next(expr)
|
||||
if not nn:
|
||||
raise ValueError("cron 表达式在未来一年内无匹配时间")
|
||||
sch["next_run"] = nn.strftime("%Y-%m-%d %H:%M:%S")
|
||||
else:
|
||||
sch.setdefault("interval_unit", "hours")
|
||||
sch.setdefault("interval_value", 24)
|
||||
sch["next_run"] = (datetime.now() + interval_delta(sch)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
sch["next_run"] = _schedule_next_run(sch)
|
||||
except ValueError as e:
|
||||
return jsonify({"error": f"调度配置错误: {e}"}), 400
|
||||
sch.setdefault("last_run", "")
|
||||
@@ -221,13 +239,7 @@ def api_update_task(tid):
|
||||
if "schedule" in body and task.get("mode") == "scheduled":
|
||||
sch = {**task.get("schedule", {}), **body["schedule"]}
|
||||
try:
|
||||
if sch.get("type") == "cron":
|
||||
nn = cron_next(sch.get("cron") or "0 * * * *")
|
||||
if not nn:
|
||||
raise ValueError("cron 表达式在未来一年内无匹配时间")
|
||||
sch["next_run"] = nn.strftime("%Y-%m-%d %H:%M:%S")
|
||||
else:
|
||||
sch["next_run"] = (datetime.now() + interval_delta(sch)).strftime("%Y-%m-%d %H:%M:%S")
|
||||
sch["next_run"] = _schedule_next_run(sch)
|
||||
except ValueError as e:
|
||||
return jsonify({"error": f"调度配置错误: {e}"}), 400
|
||||
task["schedule"] = sch
|
||||
@@ -285,6 +297,88 @@ def api_resume(tid):
|
||||
return jsonify({"error": "任务未在运行"}), 409
|
||||
|
||||
|
||||
# ---------------- API: 统计 ----------------
|
||||
|
||||
@app.route("/api/stats")
|
||||
def api_stats():
|
||||
tasks = store.load_tasks()
|
||||
total_runs = ok = fail = imgs = 0
|
||||
for t in tasks:
|
||||
for r in store.get_runs(t["id"]):
|
||||
total_runs += 1
|
||||
st = r.get("stats") or {}
|
||||
ok += st.get("ok", 0)
|
||||
fail += st.get("fail", 0)
|
||||
imgs += st.get("images", 0)
|
||||
with JOBS_LOCK:
|
||||
running = sum(1 for j in JOBS.values() if j.is_running())
|
||||
# 统计各任务输出目录的磁盘占用
|
||||
size = 0
|
||||
seen = set()
|
||||
for t in tasks:
|
||||
d = os.path.realpath(resolve_out_dir(t))
|
||||
if d in seen or not os.path.isdir(d):
|
||||
continue
|
||||
seen.add(d)
|
||||
for root, _dirs, files in os.walk(d):
|
||||
for f in files:
|
||||
try:
|
||||
size += os.path.getsize(os.path.join(root, f))
|
||||
except OSError:
|
||||
pass
|
||||
return jsonify({
|
||||
"tasks": len(tasks),
|
||||
"running": running,
|
||||
"runs": total_runs,
|
||||
"ok": ok,
|
||||
"fail": fail,
|
||||
"images": imgs,
|
||||
"disk_mb": round(size / 1048576, 1),
|
||||
})
|
||||
|
||||
|
||||
# ---------------- API: 试爬取 ----------------
|
||||
|
||||
@app.route("/api/probe", methods=["POST"])
|
||||
def api_probe():
|
||||
"""试爬取: 按表单给出的规则探测起始页, 返回将爬取的链接清单"""
|
||||
body = request.get_json(force=True) or {}
|
||||
seed = (body.get("seed_url") or "").strip()
|
||||
if not seed:
|
||||
return jsonify({"error": "请填写起始网址"}), 400
|
||||
if not seed.startswith("http"):
|
||||
seed = "https://" + seed
|
||||
result = probe_links(
|
||||
seed,
|
||||
include=[x.strip() for x in (body.get("include") or []) if x.strip()],
|
||||
exclude=[x.strip() for x in (body.get("exclude") or []) if x.strip()],
|
||||
same_domain=body.get("same_domain", True),
|
||||
use_regex=bool(body.get("use_regex", False)),
|
||||
timeout=int(body.get("timeout") or 45),
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@app.route("/api/tasks/<tid>/probe", methods=["POST"])
|
||||
def api_task_probe(tid):
|
||||
"""对已保存的自动任务执行试爬取 (使用保存的规则)"""
|
||||
task = store.get_task(tid)
|
||||
if not task:
|
||||
return jsonify({"error": "任务不存在"}), 404
|
||||
if task.get("mode") != "auto":
|
||||
return jsonify({"error": "仅自动爬取任务支持试爬取"}), 400
|
||||
auto = task.get("auto", {})
|
||||
result = probe_links(
|
||||
auto.get("seed_url", ""),
|
||||
include=auto.get("include", []),
|
||||
exclude=auto.get("exclude", []),
|
||||
same_domain=auto.get("same_domain", True),
|
||||
use_regex=bool(auto.get("use_regex", False)),
|
||||
timeout=int((task.get("config") or {}).get("timeout", 60)),
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
# ---------------- API: 运行记录与文件 ----------------
|
||||
|
||||
@app.route("/api/runs/<rid>")
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
通用爬虫引擎 (Playwright + stealth + 系统 Chrome)
|
||||
- 批量模式: 逐条爬取网址列表
|
||||
- 自动模式: 从起始网址按匹配规则自动发现链接并 BFS 爬取
|
||||
- 试爬取: 仅抓取起始页, 列出按规则将爬取的链接(不保存文件)
|
||||
- 每个页面/图片生成 .meta.json 操作信息(模式/时间/网址/来源链接/深度等)
|
||||
- 支持: 随机延迟 / 重试 / 图片下载 / 暂停恢复 / 终止 / 配置热更新 / cookie 复用
|
||||
"""
|
||||
import json
|
||||
@@ -40,7 +42,6 @@ def is_challenge_page(title, html):
|
||||
for mark in _CHALLENGE_MARKS:
|
||||
if mark in t or mark in low:
|
||||
return True
|
||||
# 极小页面 + 无正文结构 -> 疑似验证壳
|
||||
if len(html) < 5000 and ("<article" not in low and "<main" not in low):
|
||||
return True
|
||||
return False
|
||||
@@ -52,6 +53,122 @@ def safe_name(url, idx):
|
||||
return f"{idx:04d}_{host}_{ts}"
|
||||
|
||||
|
||||
def _settle_wait(page, timeout_s):
|
||||
"""等待页面稳定(无 stop/pause 检查, 供试爬取等独立流程使用)"""
|
||||
last_title, stable = "", 0
|
||||
start = time.time()
|
||||
while time.time() - start < timeout_s:
|
||||
time.sleep(1)
|
||||
try:
|
||||
title = page.title()
|
||||
html = page.content()
|
||||
except Exception:
|
||||
continue
|
||||
if is_challenge_page(title, html):
|
||||
stable = 0
|
||||
continue
|
||||
if title == last_title:
|
||||
stable += 1
|
||||
if stable >= 2 and len(html) > 1000:
|
||||
return True, title, html
|
||||
else:
|
||||
stable = 0
|
||||
last_title = title
|
||||
return True, page.title(), page.content()
|
||||
|
||||
|
||||
def filter_links(hrefs, seed_url, include=None, exclude=None,
|
||||
same_domain=True, use_regex=False):
|
||||
"""按规则过滤链接, 返回 (included, excluded); excluded 含排除原因"""
|
||||
include = include or []
|
||||
exclude = exclude or []
|
||||
included, excluded = [], []
|
||||
seed_host = urllib.parse.urlparse(seed_url).hostname or ""
|
||||
for h in hrefs:
|
||||
if not str(h).startswith("http"):
|
||||
continue
|
||||
if same_domain and seed_host:
|
||||
host = urllib.parse.urlparse(h).hostname or ""
|
||||
if host != seed_host and not host.endswith("." + seed_host):
|
||||
excluded.append({"url": h, "reason": "不在同域名内"})
|
||||
continue
|
||||
if use_regex:
|
||||
if include and not any(re.search(p, h) for p in include):
|
||||
excluded.append({"url": h, "reason": "未匹配包含规则"})
|
||||
continue
|
||||
hit = next((p for p in exclude if re.search(p, h)), None)
|
||||
if hit:
|
||||
excluded.append({"url": h, "reason": f"命中排除规则: {hit}"})
|
||||
continue
|
||||
else:
|
||||
if include and not any(p.lower() in h.lower() for p in include):
|
||||
excluded.append({"url": h, "reason": "未匹配包含规则"})
|
||||
continue
|
||||
hit = next((p for p in exclude if p.lower() in h.lower()), None)
|
||||
if hit:
|
||||
excluded.append({"url": h, "reason": f"命中排除规则: {hit}"})
|
||||
continue
|
||||
included.append(h)
|
||||
return included, excluded
|
||||
|
||||
|
||||
def probe_links(seed_url, include=None, exclude=None,
|
||||
same_domain=True, use_regex=False, timeout=45):
|
||||
"""试爬取: 抓取起始页并列出按规则将爬取的链接 (不保存任何文件)"""
|
||||
result = {"ok": False, "error": "", "seed_url": seed_url, "title": "",
|
||||
"crawled_at": "", "total_links": 0,
|
||||
"total_included": 0, "total_excluded": 0,
|
||||
"included": [], "excluded": []}
|
||||
try:
|
||||
p = sync_playwright().start()
|
||||
browser = p.chromium.launch(
|
||||
headless=True, executable_path=CHROME,
|
||||
args=["--disable-blink-features=AutomationControlled",
|
||||
"--no-sandbox", "--disable-gpu", "--disable-dev-shm-usage"],
|
||||
)
|
||||
ctx = browser.new_context(
|
||||
user_agent=DEFAULT_UA,
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
locale="en-US",
|
||||
)
|
||||
Stealth().apply_stealth_sync(ctx)
|
||||
page = ctx.new_page()
|
||||
try:
|
||||
page.goto(seed_url, wait_until="domcontentloaded", timeout=timeout * 1000)
|
||||
_ok, title, html = _settle_wait(page, timeout)
|
||||
if is_challenge_page(title, html):
|
||||
result["error"] = f"起始页被反爬拦截: title={title!r}"
|
||||
else:
|
||||
try:
|
||||
hrefs = page.evaluate(
|
||||
"() => Array.from(document.querySelectorAll('a[href]')).map(a => a.href)"
|
||||
)
|
||||
except Exception:
|
||||
hrefs = []
|
||||
included, excluded = filter_links(
|
||||
hrefs, seed_url, include, exclude, same_domain, use_regex)
|
||||
result.update(
|
||||
ok=True, title=title, crawled_at=store.now_str(),
|
||||
total_links=len(hrefs),
|
||||
total_included=len(included), total_excluded=len(excluded),
|
||||
included=included[:200], excluded=excluded[:200],
|
||||
)
|
||||
except Exception as e:
|
||||
result["error"] = f"试爬取失败: {e}"
|
||||
finally:
|
||||
try:
|
||||
browser.close()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
p.stop()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
result["error"] = f"浏览器启动失败: {e}"
|
||||
return result
|
||||
|
||||
|
||||
class CrawlJob:
|
||||
"""一次爬取执行 (独立线程运行)"""
|
||||
|
||||
@@ -195,8 +312,8 @@ class CrawlJob:
|
||||
text = ""
|
||||
return title, html, text
|
||||
|
||||
def _crawl_images(self, ctx, page, out_dir, base):
|
||||
"""下载页面图片, 返回 [{file, url, size}]"""
|
||||
def _crawl_images(self, ctx, page, out_dir, base, page_url, source_url):
|
||||
"""下载页面图片并生成图片集 meta.json, 返回 [{file,url,size,download_time}]"""
|
||||
try:
|
||||
urls = page.evaluate(
|
||||
"() => Array.from(document.querySelectorAll('img'))"
|
||||
@@ -222,26 +339,78 @@ class CrawlJob:
|
||||
os.makedirs(img_dir, exist_ok=True)
|
||||
with open(os.path.join(img_dir, fname), "wb") as f:
|
||||
f.write(resp.body())
|
||||
saved.append({"file": f"{base}_img/{fname}", "url": u, "size": len(resp.body())})
|
||||
saved.append({
|
||||
"file": f"{base}_img/{fname}", "url": u,
|
||||
"size": len(resp.body()), "download_time": store.now_str(),
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
if saved:
|
||||
try:
|
||||
meta = {
|
||||
"type": "images",
|
||||
"mode": self.run.get("mode"),
|
||||
"task_id": self.task["id"],
|
||||
"task_name": self.task.get("name", ""),
|
||||
"run_id": self.run.get("id"),
|
||||
"crawl_time": store.now_str(),
|
||||
"page_url": page_url,
|
||||
"source_url": source_url,
|
||||
"images": saved,
|
||||
}
|
||||
with open(os.path.join(img_dir, "meta.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f, ensure_ascii=False, indent=2)
|
||||
except Exception:
|
||||
pass
|
||||
return saved
|
||||
|
||||
def _retry_crawl(self, page, ctx, url, idx, out_dir):
|
||||
"""带重试的单页爬取, 返回结果 entry"""
|
||||
def _write_page_meta(self, out_dir, base, entry):
|
||||
"""为每个爬取页面生成操作信息 meta.json"""
|
||||
meta = {
|
||||
"type": "page",
|
||||
"mode": self.run.get("mode"),
|
||||
"task_id": self.task["id"],
|
||||
"task_name": self.task.get("name", ""),
|
||||
"run_id": self.run.get("id"),
|
||||
"crawl_time": entry.get("crawl_time", ""),
|
||||
"url": entry.get("url", ""),
|
||||
"source_url": entry.get("source_url", ""),
|
||||
"depth": entry.get("depth"),
|
||||
"title": entry.get("title", ""),
|
||||
"status": entry.get("status", ""),
|
||||
"error": entry.get("error", ""),
|
||||
"attempts": entry.get("attempts", 1),
|
||||
"html_file": entry.get("html_file", ""),
|
||||
"txt_file": entry.get("txt_file", ""),
|
||||
"images": entry.get("images", []),
|
||||
}
|
||||
try:
|
||||
with open(os.path.join(out_dir, base + ".meta.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(meta, f, ensure_ascii=False, indent=2)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _retry_crawl(self, page, ctx, url, idx, out_dir, source_url="", depth=None):
|
||||
"""带重试的单页爬取, 返回结果 entry (含 meta 信息)"""
|
||||
timeout = int(self._cfg("timeout", 60))
|
||||
retries = int(self._cfg("retry_count", 2))
|
||||
retry_wait = float(self._cfg("retry_interval", 3))
|
||||
crawl_images = bool(self._cfg("crawl_images", False))
|
||||
|
||||
entry = {"url": url, "title": "", "status": "FAIL", "error": "",
|
||||
"html_file": "", "txt_file": "", "images": []}
|
||||
base = safe_name(url, idx)
|
||||
entry = {
|
||||
"url": url, "title": "", "status": "FAIL", "error": "",
|
||||
"html_file": "", "txt_file": "", "meta_file": base + ".meta.json",
|
||||
"crawl_time": store.now_str(),
|
||||
"source_url": source_url, "depth": depth,
|
||||
"images": [], "attempts": 0,
|
||||
}
|
||||
for attempt in range(retries + 1):
|
||||
if self._stop.is_set():
|
||||
entry["error"] = "任务已终止"
|
||||
break
|
||||
self._wait_if_paused()
|
||||
entry["attempts"] += 1
|
||||
try:
|
||||
title, html, text = self._crawl_one(page, url, timeout)
|
||||
html_path = os.path.join(out_dir, base + ".html")
|
||||
@@ -251,13 +420,15 @@ class CrawlJob:
|
||||
with open(txt_path, "w", encoding="utf-8") as f:
|
||||
f.write(text)
|
||||
entry.update(title=title, status="OK",
|
||||
html_file=base + ".html", txt_file=base + ".txt")
|
||||
html_file=base + ".html", txt_file=base + ".txt",
|
||||
error="", crawl_time=store.now_str())
|
||||
if crawl_images:
|
||||
entry["images"] = self._crawl_images(ctx, page, out_dir, base)
|
||||
entry["images"] = self._crawl_images(ctx, page, out_dir, base, url, source_url)
|
||||
self._log("info", f"OK {title[:50]!r} html={len(html)//1024}KB 图片={len(entry['images'])}")
|
||||
break
|
||||
except Exception as e:
|
||||
entry["error"] = str(e)
|
||||
entry["crawl_time"] = store.now_str()
|
||||
self._log("warn", f"第{attempt + 1}次失败 {url}: {e}")
|
||||
if attempt < retries:
|
||||
self._wait_if_paused()
|
||||
@@ -267,6 +438,7 @@ class CrawlJob:
|
||||
break
|
||||
self._wait_if_paused()
|
||||
time.sleep(0.3)
|
||||
self._write_page_meta(out_dir, base, entry)
|
||||
return entry
|
||||
|
||||
def _delay(self):
|
||||
@@ -317,7 +489,6 @@ class CrawlJob:
|
||||
self._log("error", f"邮件通知失败: {msg}")
|
||||
except Exception as e:
|
||||
self._log("error", f"邮件通知异常: {e}")
|
||||
self._persist()
|
||||
self._log("info", f"任务结束: {run['status']} 成功{run['stats'].get('ok', 0)} 失败{run['stats'].get('fail', 0)}")
|
||||
self._persist()
|
||||
|
||||
@@ -353,37 +524,18 @@ class CrawlJob:
|
||||
def _discover_links(self, page):
|
||||
"""从当前页面提取符合规则的链接"""
|
||||
auto = self.task.get("auto", {})
|
||||
include = [x.strip() for x in (auto.get("include") or []) if x.strip()]
|
||||
exclude = [x.strip() for x in (auto.get("exclude") or []) if x.strip()]
|
||||
use_regex = bool(auto.get("use_regex", False))
|
||||
same_domain = auto.get("same_domain", True)
|
||||
seed_host = urllib.parse.urlparse(auto.get("seed_url", "")).hostname or ""
|
||||
try:
|
||||
hrefs = page.evaluate(
|
||||
"() => Array.from(document.querySelectorAll('a[href]')).map(a => a.href)"
|
||||
)
|
||||
except Exception:
|
||||
return []
|
||||
out = []
|
||||
for h in hrefs:
|
||||
if not str(h).startswith("http"):
|
||||
continue
|
||||
if same_domain and seed_host:
|
||||
host = urllib.parse.urlparse(h).hostname or ""
|
||||
if host != seed_host and not host.endswith("." + seed_host):
|
||||
continue
|
||||
if use_regex:
|
||||
if include and not any(re.search(p, h) for p in include):
|
||||
continue
|
||||
if any(re.search(p, h) for p in exclude):
|
||||
continue
|
||||
else:
|
||||
if include and not any(p.lower() in h.lower() for p in include):
|
||||
continue
|
||||
if any(p.lower() in h.lower() for p in exclude):
|
||||
continue
|
||||
out.append(h)
|
||||
return out
|
||||
included, _excluded = filter_links(
|
||||
hrefs, auto.get("seed_url", ""),
|
||||
auto.get("include", []), auto.get("exclude", []),
|
||||
auto.get("same_domain", True), bool(auto.get("use_regex", False)),
|
||||
)
|
||||
return included
|
||||
|
||||
def _crawl_auto(self):
|
||||
run = self.run
|
||||
@@ -398,14 +550,14 @@ class CrawlJob:
|
||||
self._persist()
|
||||
|
||||
p, browser, ctx, page, cookie_file = self._open_browser()
|
||||
queue = [(seed, 0)]
|
||||
queue = [(seed, 0, "")] # (url, depth, 来源链接)
|
||||
visited = set()
|
||||
queued = set([seed])
|
||||
idx = 0
|
||||
try:
|
||||
while queue and not self._stop.is_set():
|
||||
self._wait_if_paused()
|
||||
url, depth = queue.pop(0)
|
||||
url, depth, src = queue.pop(0)
|
||||
if url in visited:
|
||||
continue
|
||||
if len(visited) >= max_pages:
|
||||
@@ -415,7 +567,8 @@ class CrawlJob:
|
||||
run["progress"]["current_url"] = url
|
||||
run["progress"]["done"] = len(visited)
|
||||
self._persist()
|
||||
entry = self._retry_crawl(page, ctx, url, idx, out_dir)
|
||||
entry = self._retry_crawl(page, ctx, url, idx, out_dir,
|
||||
source_url=src, depth=depth)
|
||||
run["results"].append(entry)
|
||||
self._bump_stats(entry)
|
||||
self._persist()
|
||||
@@ -423,7 +576,7 @@ class CrawlJob:
|
||||
for link in self._discover_links(page):
|
||||
if link not in visited and link not in queued:
|
||||
queued.add(link)
|
||||
queue.append((link, depth + 1))
|
||||
queue.append((link, depth + 1, url))
|
||||
if entry["status"] == "OK":
|
||||
self._delay()
|
||||
finally:
|
||||
|
||||
+198
-7
@@ -11,6 +11,7 @@ const state = {
|
||||
detail: { task: null, runId: null, logOffset: 0, timer: null },
|
||||
logTimer: null,
|
||||
};
|
||||
let formDirty = false; // 新建/编辑表单是否有未保存修改
|
||||
|
||||
/* ---------------- 工具 ---------------- */
|
||||
function toast(msg, isErr) {
|
||||
@@ -48,6 +49,21 @@ async function loadTasks() {
|
||||
} catch (e) {
|
||||
toast("加载任务失败: " + e.message, true);
|
||||
}
|
||||
loadStats();
|
||||
}
|
||||
|
||||
async function loadStats() {
|
||||
try {
|
||||
const s = await api("/api/stats");
|
||||
$("stTasks").textContent = s.tasks;
|
||||
$("stRunning").textContent = s.running;
|
||||
$("stRuns").textContent = s.runs;
|
||||
$("stOk").textContent = s.ok;
|
||||
$("stFail").textContent = s.fail;
|
||||
$("stImgs").textContent = s.images;
|
||||
$("stDisk").textContent = s.disk_mb >= 1024
|
||||
? (s.disk_mb / 1024).toFixed(1) + " GB" : s.disk_mb + " MB";
|
||||
} catch (e) { /* 统计失败忽略 */ }
|
||||
}
|
||||
|
||||
function taskStatusBadge(t) {
|
||||
@@ -156,8 +172,17 @@ function switchMode(mode, lock) {
|
||||
$("autoBox").classList.toggle("hidden", mode !== "auto");
|
||||
}
|
||||
|
||||
function toLocalInputVal(iso) {
|
||||
if (!iso) return "";
|
||||
const d = new Date(String(iso).replace(" ", "T"));
|
||||
if (isNaN(d.getTime())) return "";
|
||||
const p = (n) => String(n).padStart(2, "0");
|
||||
return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}T${p(d.getHours())}:${p(d.getMinutes())}`;
|
||||
}
|
||||
|
||||
function openCreate() {
|
||||
state.editTask = null;
|
||||
formDirty = false;
|
||||
$("modalTitle").textContent = "新建爬取任务";
|
||||
$("taskForm").reset();
|
||||
$("taskForm").elements["delay_min"].value = 2;
|
||||
@@ -195,6 +220,7 @@ async function openEdit(tid) {
|
||||
f.elements["interval_value"].value = t.schedule.interval_value ?? 24;
|
||||
f.elements["interval_unit"].value = t.schedule.interval_unit || "hours";
|
||||
f.elements["cron"].value = t.schedule.cron || "";
|
||||
f.elements["first_run"].value = toLocalInputVal(t.schedule.next_run || "");
|
||||
}
|
||||
if (t.mode === "auto" && t.auto) {
|
||||
f.elements["seed_url"].value = t.auto.seed_url || "";
|
||||
@@ -209,6 +235,7 @@ async function openEdit(tid) {
|
||||
$("formHint").textContent = t.running
|
||||
? "⚠️ 任务运行中：参数修改将热更新（网址/规则改动下次运行生效）"
|
||||
: "";
|
||||
formDirty = false;
|
||||
switchMode(t.mode, true);
|
||||
showModal("taskModal");
|
||||
}
|
||||
@@ -259,6 +286,7 @@ async function submitForm(e) {
|
||||
interval_unit: f.elements["interval_unit"].value,
|
||||
interval_value: parseInt(f.elements["interval_value"].value) || 1,
|
||||
cron: f.elements["cron"].value.trim(),
|
||||
first_run: f.elements["first_run"].value || "",
|
||||
};
|
||||
}
|
||||
try {
|
||||
@@ -275,6 +303,7 @@ async function submitForm(e) {
|
||||
});
|
||||
toast(`任务「${t.name}」已创建`);
|
||||
}
|
||||
formDirty = false;
|
||||
hideModal("taskModal");
|
||||
loadTasks();
|
||||
} catch (err) { toast(err.message, true); }
|
||||
@@ -330,6 +359,7 @@ function renderDetail() {
|
||||
<button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
|
||||
<button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
|
||||
: `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`}
|
||||
${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>` : ""}
|
||||
<button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
|
||||
</div>`;
|
||||
|
||||
@@ -360,13 +390,18 @@ function renderRunPanel(t, run) {
|
||||
const txt = r.txt_file ? `<span class="file-link" onclick="previewFile('${t.id}','${esc(r.txt_file)}','文本: ${esc(r.url)}')">TXT</span>` : "—";
|
||||
const imgs = (r.images || []).length
|
||||
? `<span class="file-link" onclick="scrollThumbs()">🖼️ ${(r.images || []).length}</span>` : "—";
|
||||
const ctime = r.crawl_time ? `<span title="${esc(r.crawl_time)}">${esc(r.crawl_time.slice(5, 19))}</span>` : "—";
|
||||
const meta = r.meta_file
|
||||
? `<span class="file-link" onclick="showMeta('${t.id}','${esc(r.meta_file)}')">📋 元数据</span>` : "—";
|
||||
return `<tr>
|
||||
<td>${i + 1}</td>
|
||||
<td class="${statusCls}">${r.status}</td>
|
||||
<td class="title-cell" title="${esc(r.title)}">${esc(r.title)}</td>
|
||||
<td class="url-cell" title="${esc(r.url)}">${esc(r.url)}</td>
|
||||
<td>${ctime}</td>
|
||||
<td>${html}</td><td>${txt}</td><td>${imgs}</td>
|
||||
<td title="${esc(r.error || "")}">${esc((r.error || "").slice(0, 60))}</td>
|
||||
<td>${meta}</td>
|
||||
<td title="${esc(r.error || "")}">${esc((r.error || "").slice(0, 50))}</td>
|
||||
</tr>`;
|
||||
}).join("");
|
||||
const thumbs = results.flatMap((r) => r.images || []).slice(0, 60);
|
||||
@@ -392,7 +427,7 @@ function renderRunPanel(t, run) {
|
||||
${results.length ? `
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>#</th><th>状态</th><th>标题</th><th>网址</th><th>HTML</th><th>TXT</th><th>图片</th><th>错误</th></tr></thead>
|
||||
<thead><tr><th>#</th><th>状态</th><th>标题</th><th>网址</th><th>爬取时间</th><th>HTML</th><th>TXT</th><th>图片</th><th>元数据</th><th>错误</th></tr></thead>
|
||||
<tbody>${rows}</tbody>
|
||||
</table>
|
||||
</div>` : '<div class="card-line">暂无结果</div>'}
|
||||
@@ -446,6 +481,120 @@ function stopLogPoll() {
|
||||
if (state.logTimer) { clearInterval(state.logTimer); state.logTimer = null; }
|
||||
}
|
||||
|
||||
/* ---------------- 试爬取 ---------------- */
|
||||
function collectProbeFromForm() {
|
||||
const f = $("taskForm");
|
||||
return {
|
||||
seed_url: f.elements["seed_url"].value.trim(),
|
||||
include: splitLines(f.elements["include"].value),
|
||||
exclude: splitLines(f.elements["exclude"].value),
|
||||
same_domain: f.elements["same_domain"].checked,
|
||||
use_regex: f.elements["use_regex"].checked,
|
||||
timeout: parseInt(f.elements["timeout"].value) || 60,
|
||||
};
|
||||
}
|
||||
|
||||
async function runProbe(params, btn) {
|
||||
if (btn) { btn.disabled = true; btn.textContent = "⏳ 探测中..."; }
|
||||
try {
|
||||
const r = await api("/api/probe", {
|
||||
method: "POST", headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(params),
|
||||
});
|
||||
renderProbe(r);
|
||||
} catch (e) { toast(e.message, true); }
|
||||
finally {
|
||||
if (btn) { btn.disabled = false; btn.textContent = "🧪 试爬取"; }
|
||||
}
|
||||
}
|
||||
|
||||
async function probeTask(tid) {
|
||||
try {
|
||||
const r = await api(`/api/tasks/${tid}/probe`, { method: "POST" });
|
||||
renderProbe(r);
|
||||
} catch (e) { toast(e.message, true); }
|
||||
}
|
||||
|
||||
function renderProbe(r) {
|
||||
const body = $("probeBody");
|
||||
if (!r.ok) {
|
||||
body.innerHTML = `<div class="card-line">❌ ${esc(r.error || "试爬取失败")}</div>`;
|
||||
showModal("probeModal");
|
||||
return;
|
||||
}
|
||||
const included = r.included || [];
|
||||
const excluded = r.excluded || [];
|
||||
const incHtml = included.length
|
||||
? included.map((u) => `<div class="probe-item included"><span class="u">${esc(u)}</span></div>`).join("")
|
||||
: '<div class="card-line">没有符合条件的链接，请调整规则</div>';
|
||||
const excHtml = excluded.length
|
||||
? excluded.map((x) => `<div class="probe-item excluded"><span class="u">${esc(x.url)}</span><span class="reason">${esc(x.reason)}</span></div>`).join("")
|
||||
: '<div class="card-line">无</div>';
|
||||
const moreInc = r.total_included > included.length
|
||||
? `<div class="card-line">... 还有 ${r.total_included - included.length} 条未显示</div>` : "";
|
||||
const moreExc = r.total_excluded > excluded.length
|
||||
? `<div class="card-line">... 还有 ${r.total_excluded - excluded.length} 条未显示</div>` : "";
|
||||
body.innerHTML = `
|
||||
<div class="probe-summary">
|
||||
<span>页面标题: <b>${esc(r.title)}</b></span>
|
||||
<span>抓取时间: <b>${esc(r.crawled_at)}</b></span>
|
||||
</div>
|
||||
<div class="probe-summary">
|
||||
<span>发现链接: <b>${r.total_links}</b> 条</span>
|
||||
<span class="t-ok">✅ 将爬取: <b>${r.total_included}</b> 条</span>
|
||||
<span class="t-fail">🚫 被排除: <b>${r.total_excluded}</b> 条</span>
|
||||
</div>
|
||||
<div class="card-line" style="font-weight:600">✅ 符合规则、将爬取的链接（显示前 ${included.length} 条）:</div>
|
||||
<div class="probe-list">${incHtml}${moreInc}</div>
|
||||
<div class="card-line" style="font-weight:600;margin-top:8px">🚫 被排除的链接及原因（显示前 ${excluded.length} 条）:</div>
|
||||
<div class="probe-list">${excHtml}${moreExc}</div>`;
|
||||
showModal("probeModal");
|
||||
}
|
||||
|
||||
/* ---------------- 元数据查看 ---------------- */
|
||||
const META_FIELDS = [
|
||||
["type", "资源类型"], ["mode", "爬取模式"], ["task_name", "任务名称"],
|
||||
["task_id", "任务ID"], ["run_id", "运行ID"], ["crawl_time", "爬取时间"],
|
||||
["url", "爬取网址"], ["source_url", "来源链接"], ["depth", "爬取深度"],
|
||||
["title", "页面标题"], ["status", "状态"], ["attempts", "尝试次数"],
|
||||
["error", "错误信息"], ["html_file", "HTML文件"], ["txt_file", "文本文件"],
|
||||
["page_url", "所属页面"],
|
||||
];
|
||||
|
||||
async function showMeta(taskId, metaFile) {
|
||||
try {
|
||||
const data = await api(`/api/file?task_id=${taskId}&path=${encodeURIComponent(metaFile)}`);
|
||||
const rows = META_FIELDS.filter(([k]) => data[k] !== undefined && data[k] !== "" && data[k] !== null)
|
||||
.map(([k, label]) => {
|
||||
let v = data[k];
|
||||
if (k === "status") v = `<span class="${v === "OK" ? "t-ok" : "t-fail"}">${esc(v)}</span>`;
|
||||
if (k === "mode") v = `<span class="badge ${esc(v)}">${MODE_LABEL[v] || esc(v)}</span>`;
|
||||
if (k === "type") v = v === "images" ? "🖼️ 图片集" : "📄 网页";
|
||||
if (k === "depth") v = v === 0 ? "0（起始页）" : esc(v);
|
||||
return `<tr><td>${label}</td><td>${v}</td></tr>`;
|
||||
}).join("");
|
||||
// 来源链接空值也显示(起始页无来源)
|
||||
const srcRow = `<tr><td>来源链接</td><td>${data.source_url ? esc(data.source_url) : "—（起始页，无来源）"}</td></tr>`;
|
||||
const finalRows = rows.includes(srcRow) ? rows : rows + srcRow;
|
||||
let imgs = "";
|
||||
if (data.images && data.images.length) {
|
||||
imgs = `
|
||||
<div class="card-line" style="font-weight:600;margin-top:8px">🖼️ 图片明细（${data.images.length} 张）:</div>
|
||||
<div class="table-wrap" style="max-height:200px">
|
||||
<table><thead><tr><th>文件</th><th>原始URL</th><th>大小</th><th>下载时间</th></tr></thead><tbody>
|
||||
${data.images.map((im) => `<tr>
|
||||
<td>${esc(im.file)}</td>
|
||||
<td class="url-cell" title="${esc(im.url)}">${esc(im.url)}</td>
|
||||
<td>${im.size ? Math.round(im.size / 1024) + " KB" : "—"}</td>
|
||||
<td>${esc(im.download_time || "")}</td></tr>`).join("")}
|
||||
</tbody></table>
|
||||
</div>`;
|
||||
}
|
||||
$("metaBody").innerHTML = `<table class="meta-table"><tbody>${finalRows}</tbody></table>${imgs}`;
|
||||
showModal("metaModal");
|
||||
} catch (e) { toast(e.message, true); }
|
||||
}
|
||||
|
||||
/* ---------------- 文件预览 ---------------- */
|
||||
function previewFile(tid, path, title) {
|
||||
$("previewTitle").textContent = title || "预览";
|
||||
@@ -457,10 +606,38 @@ function previewFile(tid, path, title) {
|
||||
function showModal(id) { $(id).classList.remove("hidden"); }
|
||||
function hideModal(id) { $(id).classList.add("hidden"); }
|
||||
|
||||
/* 关闭任务弹窗: 有未保存修改时先确认 */
|
||||
function safeCloseTaskModal() {
|
||||
if (formDirty && !confirm("有未保存的修改，确定要放弃吗？")) return false;
|
||||
formDirty = false;
|
||||
hideModal("taskModal");
|
||||
return true;
|
||||
}
|
||||
|
||||
/* ---------------- 主题切换 ---------------- */
|
||||
function applyTheme(theme) {
|
||||
document.body.dataset.theme = theme;
|
||||
$("btnTheme").textContent = theme === "light" ? "🌙" : "☀️";
|
||||
try { localStorage.setItem("crawler_theme", theme); } catch (e) { /* ignore */ }
|
||||
}
|
||||
$("btnTheme").onclick = () => {
|
||||
applyTheme(document.body.dataset.theme === "light" ? "dark" : "light");
|
||||
};
|
||||
try {
|
||||
applyTheme(localStorage.getItem("crawler_theme") || "dark");
|
||||
} catch (e) {
|
||||
applyTheme("dark");
|
||||
}
|
||||
|
||||
/* ---------------- 事件绑定 ---------------- */
|
||||
$("btnNew").onclick = openCreate;
|
||||
$("btnNew2").onclick = openCreate;
|
||||
$("btnRefresh").onclick = loadTasks;
|
||||
$("btnProbe").onclick = () => {
|
||||
const p = collectProbeFromForm();
|
||||
if (!p.seed_url) { toast("请先填写起始网址", true); return; }
|
||||
runProbe(p, $("btnProbe"));
|
||||
};
|
||||
|
||||
document.querySelectorAll("#modeTabs .tab").forEach((b) => {
|
||||
b.onclick = () => { switchMode(b.dataset.mode, false); };
|
||||
@@ -468,24 +645,38 @@ document.querySelectorAll("#modeTabs .tab").forEach((b) => {
|
||||
$("taskForm").onsubmit = submitForm;
|
||||
$("taskForm").elements["schedule_type"].onchange = syncScheduleUI;
|
||||
|
||||
document.querySelectorAll("[data-close]").forEach((b) => b.onclick = () => hideModal("taskModal"));
|
||||
document.querySelectorAll("[data-close]").forEach((b) => b.onclick = safeCloseTaskModal);
|
||||
document.querySelectorAll("[data-close-detail]").forEach((b) => b.onclick = () => { stopLogPoll(); hideModal("detailModal"); });
|
||||
document.querySelectorAll("[data-close-preview]").forEach((b) => b.onclick = () => { $("previewFrame").src = "about:blank"; hideModal("previewModal"); });
|
||||
document.querySelectorAll("[data-close-probe]").forEach((b) => b.onclick = () => hideModal("probeModal"));
|
||||
document.querySelectorAll("[data-close-meta]").forEach((b) => b.onclick = () => hideModal("metaModal"));
|
||||
|
||||
/* 表单改动监听 -> 脏标记 */
|
||||
$("taskForm").addEventListener("input", () => { formDirty = true; });
|
||||
$("taskForm").addEventListener("change", () => { formDirty = true; });
|
||||
|
||||
document.querySelectorAll(".modal-overlay").forEach((ov) => {
|
||||
ov.addEventListener("mousedown", (e) => {
|
||||
if (e.target === ov) {
|
||||
if (ov.id === "detailModal") stopLogPoll();
|
||||
if (ov.id === "previewModal") $("previewFrame").src = "about:blank";
|
||||
ov.classList.add("hidden");
|
||||
if (ov.id === "taskModal") {
|
||||
safeCloseTaskModal();
|
||||
} else {
|
||||
if (ov.id === "detailModal") stopLogPoll();
|
||||
if (ov.id === "previewModal") $("previewFrame").src = "about:blank";
|
||||
ov.classList.add("hidden");
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
document.addEventListener("keydown", (e) => {
|
||||
if (e.key === "Escape") {
|
||||
if (!$("taskModal").classList.contains("hidden")) {
|
||||
safeCloseTaskModal();
|
||||
return;
|
||||
}
|
||||
stopLogPoll();
|
||||
$("previewFrame").src = "about:blank";
|
||||
["taskModal", "detailModal", "previewModal"].forEach((id) => $(id).classList.add("hidden"));
|
||||
["detailModal", "probeModal", "metaModal", "previewModal"].forEach((id) => $(id).classList.add("hidden"));
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
+39
-1
@@ -11,12 +11,22 @@
|
||||
<div class="logo">🕷️ 通用爬虫系统 <span id="version" class="version"></span></div>
|
||||
<div class="header-right">
|
||||
<span class="status-pill" id="statusPill">运行中任务: 0</span>
|
||||
<button id="btnTheme" class="btn ghost" title="切换日间/夜间主题">☀️</button>
|
||||
<button id="btnRefresh" class="btn ghost">⟳ 刷新</button>
|
||||
<button id="btnNew" class="btn primary">＋ 新建任务</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<main>
|
||||
<div class="stats-bar" id="statsBar">
|
||||
<div class="stat-card"><div class="stat-num" id="stTasks">0</div><div class="stat-label">📋 任务总数</div></div>
|
||||
<div class="stat-card"><div class="stat-num num-run" id="stRunning">0</div><div class="stat-label">🔄 正在运行</div></div>
|
||||
<div class="stat-card"><div class="stat-num" id="stRuns">0</div><div class="stat-label">📊 累计运行次数</div></div>
|
||||
<div class="stat-card"><div class="stat-num num-ok" id="stOk">0</div><div class="stat-label">✅ 成功页面</div></div>
|
||||
<div class="stat-card"><div class="stat-num num-fail" id="stFail">0</div><div class="stat-label">❌ 失败页面</div></div>
|
||||
<div class="stat-card"><div class="stat-num" id="stImgs">0</div><div class="stat-label">🖼️ 已爬图片</div></div>
|
||||
<div class="stat-card"><div class="stat-num" id="stDisk">0</div><div class="stat-label">💾 磁盘占用</div></div>
|
||||
</div>
|
||||
<div id="taskList" class="task-grid"></div>
|
||||
<div id="emptyState" class="empty hidden">
|
||||
<div class="empty-icon">🕸️</div>
|
||||
@@ -63,6 +73,8 @@
|
||||
</div>
|
||||
|
||||
<div id="scheduleBox" class="hidden box">
|
||||
<div class="field"><label>首次执行时间（可留空；留空 = 尽快执行；已过的时间将立即执行）</label>
|
||||
<input name="first_run" type="datetime-local"></div>
|
||||
<div class="row2">
|
||||
<div class="field"><label>调度类型</label>
|
||||
<select name="schedule_type">
|
||||
@@ -90,7 +102,11 @@
|
||||
|
||||
<div id="autoBox" class="hidden box">
|
||||
<div class="field"><label>起始网址 *（系统将自动发现符合规则的链接并爬取）</label>
|
||||
<input name="seed_url" placeholder="https://example.com/news"></div>
|
||||
<div class="row2">
|
||||
<input name="seed_url" placeholder="https://example.com/news" style="flex:1">
|
||||
<button type="button" class="btn sm" id="btnProbe">🧪 试爬取</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="row2">
|
||||
<div class="field"><label>包含规则（每行一个，子串或正则）</label>
|
||||
<textarea name="include" rows="3" placeholder="techpowerup.com/review /news/"></textarea></div>
|
||||
@@ -127,6 +143,28 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 试爬取结果弹窗 -->
|
||||
<div id="probeModal" class="modal-overlay hidden">
|
||||
<div class="modal wide">
|
||||
<div class="modal-head">
|
||||
<span>🧪 试爬取结果（规则筛选预览）</span>
|
||||
<button class="btn ghost sm" data-close-probe>✕</button>
|
||||
</div>
|
||||
<div id="probeBody" class="detail-wrap"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 元数据弹窗 -->
|
||||
<div id="metaModal" class="modal-overlay hidden">
|
||||
<div class="modal">
|
||||
<div class="modal-head">
|
||||
<span>📋 资源操作信息（元数据）</span>
|
||||
<button class="btn ghost sm" data-close-meta>✕</button>
|
||||
</div>
|
||||
<div id="metaBody" class="detail-wrap"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 文件预览弹窗 -->
|
||||
<div id="previewModal" class="modal-overlay hidden">
|
||||
<div class="modal wide tall">
|
||||
|
||||
+78
-16
@@ -10,12 +10,36 @@
|
||||
--red: #ff5d6c;
|
||||
--yellow: #ffc857;
|
||||
--purple: #a78bfa;
|
||||
--overlay: rgba(5, 8, 15, .72);
|
||||
--box-bg: #141b2b;
|
||||
--log-bg: #0b0f1a;
|
||||
--thumb: #2a3550;
|
||||
--preview-bg: #fff;
|
||||
}
|
||||
body[data-theme="light"] {
|
||||
--bg: #f2f5fa;
|
||||
--panel: #ffffff;
|
||||
--panel2: #eef1f7;
|
||||
--border: #d8e0ee;
|
||||
--text: #1b2436;
|
||||
--muted: #5d6b86;
|
||||
--accent: #2f6bff;
|
||||
--green: #0e9f6e;
|
||||
--red: #e04a5a;
|
||||
--yellow: #b8860b;
|
||||
--purple: #7c5cf0;
|
||||
--overlay: rgba(15, 23, 42, .35);
|
||||
--box-bg: #f7f9fd;
|
||||
--log-bg: #f4f6fa;
|
||||
--thumb: #cdd7e6;
|
||||
--preview-bg: #fff;
|
||||
}
|
||||
* { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
body {
|
||||
background: var(--bg); color: var(--text);
|
||||
font-family: -apple-system, "PingFang SC", "Microsoft YaHei", "Segoe UI", sans-serif;
|
||||
font-size: 14px; min-height: 100vh;
|
||||
transition: background .25s, color .25s;
|
||||
}
|
||||
header {
|
||||
display: flex; align-items: center; justify-content: space-between;
|
||||
@@ -38,7 +62,7 @@ main { padding: 20px 24px; max-width: 1500px; margin: 0 auto; }
|
||||
padding: 7px 14px; border-radius: 8px; cursor: pointer; font-size: 13px;
|
||||
transition: .15s; white-space: nowrap;
|
||||
}
|
||||
.btn:hover { border-color: var(--accent); color: #fff; }
|
||||
.btn:hover { border-color: var(--accent); }
|
||||
.btn.primary { background: var(--accent); border-color: var(--accent); color: #fff; }
|
||||
.btn.primary:hover { filter: brightness(1.1); }
|
||||
.btn.ghost { background: transparent; }
|
||||
@@ -46,6 +70,21 @@ main { padding: 20px 24px; max-width: 1500px; margin: 0 auto; }
|
||||
.btn.sm { padding: 3px 8px; font-size: 12px; border-radius: 6px; }
|
||||
.btn:disabled { opacity: .4; cursor: not-allowed; }
|
||||
|
||||
/* ---------- 统计区 ---------- */
|
||||
.stats-bar {
|
||||
display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr));
|
||||
gap: 12px; margin-bottom: 20px;
|
||||
}
|
||||
.stat-card {
|
||||
background: var(--panel); border: 1px solid var(--border);
|
||||
border-radius: 12px; padding: 12px 16px;
|
||||
}
|
||||
.stat-num { font-size: 24px; font-weight: 700; line-height: 1.2; }
|
||||
.stat-num.num-ok { color: var(--green); }
|
||||
.stat-num.num-fail { color: var(--red); }
|
||||
.stat-num.num-run { color: var(--accent); }
|
||||
.stat-label { font-size: 12px; color: var(--muted); margin-top: 3px; }
|
||||
|
||||
/* ---------- task grid ---------- */
|
||||
.task-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(380px, 1fr)); gap: 16px; }
|
||||
.card {
|
||||
@@ -53,7 +92,7 @@ main { padding: 20px 24px; max-width: 1500px; margin: 0 auto; }
|
||||
border-radius: 12px; padding: 16px; display: flex; flex-direction: column; gap: 10px;
|
||||
transition: .15s;
|
||||
}
|
||||
.card:hover { border-color: #3a4a75; transform: translateY(-1px); }
|
||||
.card:hover { border-color: var(--accent); transform: translateY(-1px); }
|
||||
.card-head { display: flex; align-items: center; justify-content: space-between; gap: 8px; }
|
||||
.card-name { font-size: 15px; font-weight: 600; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
|
||||
.card-meta { display: flex; flex-wrap: wrap; gap: 6px; align-items: center; }
|
||||
@@ -61,7 +100,7 @@ main { padding: 20px 24px; max-width: 1500px; margin: 0 auto; }
|
||||
font-size: 11px; padding: 2px 8px; border-radius: 10px;
|
||||
background: var(--panel2); border: 1px solid var(--border); color: var(--muted);
|
||||
}
|
||||
.badge.batch { color: #7cc4ff; border-color: #7cc4ff55; }
|
||||
.badge.batch { color: var(--accent); border-color: var(--accent); }
|
||||
.badge.scheduled { color: var(--purple); border-color: var(--purple); }
|
||||
.badge.auto { color: var(--yellow); border-color: var(--yellow); }
|
||||
.badge.running { color: var(--green); border-color: var(--green); animation: pulse 1.6s infinite; }
|
||||
@@ -88,7 +127,7 @@ main { padding: 20px 24px; max-width: 1500px; margin: 0 auto; }
|
||||
|
||||
/* ---------- modal ---------- */
|
||||
.modal-overlay {
|
||||
position: fixed; inset: 0; background: rgba(5, 8, 15, .7);
|
||||
position: fixed; inset: 0; background: var(--overlay);
|
||||
display: flex; align-items: center; justify-content: center; z-index: 100;
|
||||
backdrop-filter: blur(3px);
|
||||
}
|
||||
@@ -101,24 +140,29 @@ main { padding: 20px 24px; max-width: 1500px; margin: 0 auto; }
|
||||
.modal.tall { width: 1000px; height: 88vh; }
|
||||
.modal-head {
|
||||
display: flex; justify-content: space-between; align-items: center;
|
||||
padding: 14px 18px; border-bottom: 1px solid var(--border); font-size: 15px; font-weight: 600;
|
||||
padding: 14px 18px; border-bottom: 1px solid var(--border);
|
||||
font-size: 15px; font-weight: 600; flex-shrink: 0;
|
||||
}
|
||||
.modal-foot {
|
||||
display: flex; justify-content: flex-end; gap: 10px; align-items: center;
|
||||
padding: 12px 18px; border-top: 1px solid var(--border);
|
||||
padding: 12px 18px; border-top: 1px solid var(--border); flex-shrink: 0;
|
||||
}
|
||||
.hint { color: var(--yellow); font-size: 12px; margin-right: auto; }
|
||||
|
||||
.tabs { display: flex; gap: 6px; padding: 12px 18px 0; }
|
||||
.tabs { display: flex; gap: 6px; padding: 12px 18px 0; flex-shrink: 0; }
|
||||
#taskForm { display: flex; flex-direction: column; flex: 1 1 auto; min-height: 0; }
|
||||
.tab {
|
||||
padding: 8px 16px; background: transparent; border: 1px solid var(--border);
|
||||
border-bottom: none; border-radius: 10px 10px 0 0; color: var(--muted);
|
||||
cursor: pointer; font-size: 13px;
|
||||
}
|
||||
.tab.active { background: var(--panel2); color: #fff; border-color: var(--accent); }
|
||||
.tab.active { background: var(--panel2); color: var(--text); border-color: var(--accent); }
|
||||
.tab:disabled { opacity: .4; cursor: not-allowed; }
|
||||
|
||||
.form-body { padding: 16px 18px; overflow-y: auto; display: flex; flex-direction: column; gap: 12px; }
|
||||
.form-body {
|
||||
padding: 16px 18px; overflow-y: auto; flex: 1 1 auto; min-height: 0;
|
||||
display: flex; flex-direction: column; gap: 12px;
|
||||
}
|
||||
.field { display: flex; flex-direction: column; gap: 5px; flex: 1; min-width: 0; }
|
||||
.field label { font-size: 12px; color: var(--muted); }
|
||||
.field.check { flex-direction: row; align-items: center; gap: 8px; padding-top: 22px; }
|
||||
@@ -130,8 +174,12 @@ input, select, textarea {
|
||||
input:focus, select:focus, textarea:focus { outline: none; border-color: var(--accent); }
|
||||
textarea { resize: vertical; }
|
||||
input[type="checkbox"] { width: auto; accent-color: var(--accent); }
|
||||
input[type="datetime-local"] { color-scheme: light dark; }
|
||||
.row2 { display: flex; gap: 12px; }
|
||||
.box { border: 1px dashed var(--border); border-radius: 10px; padding: 12px; display: flex; flex-direction: column; gap: 10px; background: #141b2b; }
|
||||
.box {
|
||||
border: 1px dashed var(--border); border-radius: 10px; padding: 12px;
|
||||
display: flex; flex-direction: column; gap: 10px; background: var(--box-bg);
|
||||
}
|
||||
.hidden { display: none !important; }
|
||||
|
||||
/* ---------- detail ---------- */
|
||||
@@ -144,7 +192,7 @@ input[type="checkbox"] { width: auto; accent-color: var(--accent); }
|
||||
padding: 5px 12px; border-radius: 8px; border: 1px solid var(--border);
|
||||
background: var(--panel2); cursor: pointer; font-size: 12px; color: var(--muted);
|
||||
}
|
||||
.run-chip.active { border-color: var(--accent); color: #fff; }
|
||||
.run-chip.active { border-color: var(--accent); color: var(--text); }
|
||||
.run-panel { border: 1px solid var(--border); border-radius: 10px; padding: 14px; display: flex; flex-direction: column; gap: 10px; background: var(--panel2); }
|
||||
.run-stats { display: flex; gap: 16px; font-size: 12px; flex-wrap: wrap; }
|
||||
.table-wrap { overflow: auto; max-height: 320px; border: 1px solid var(--border); border-radius: 8px; }
|
||||
@@ -155,27 +203,41 @@ td.url-cell { max-width: 300px; overflow: hidden; text-overflow: ellipsis; }
|
||||
td.title-cell { max-width: 220px; overflow: hidden; text-overflow: ellipsis; }
|
||||
.t-ok { color: var(--green); } .t-fail { color: var(--red); }
|
||||
.file-link { color: var(--accent); cursor: pointer; text-decoration: underline; }
|
||||
.file-link:hover { color: #9ec0ff; }
|
||||
.thumbs { display: flex; flex-wrap: wrap; gap: 8px; }
|
||||
.thumbs img { width: 96px; height: 72px; object-fit: cover; border-radius: 6px; border: 1px solid var(--border); cursor: pointer; }
|
||||
.logs {
|
||||
background: #0b0f1a; border: 1px solid var(--border); border-radius: 8px;
|
||||
background: var(--log-bg); border: 1px solid var(--border); border-radius: 8px;
|
||||
padding: 10px; font-family: ui-monospace, Consolas, monospace; font-size: 12px;
|
||||
max-height: 220px; overflow-y: auto; line-height: 1.7;
|
||||
}
|
||||
.logs .info { color: var(--text); } .logs .warn { color: var(--yellow); } .logs .error { color: var(--red); }
|
||||
|
||||
/* ---------- preview ---------- */
|
||||
#previewFrame { flex: 1; border: none; background: #fff; border-radius: 0 0 14px 14px; }
|
||||
#previewFrame { flex: 1; border: none; background: var(--preview-bg); border-radius: 0 0 14px 14px; }
|
||||
|
||||
/* ---------- 试爬取结果 / 元数据 ---------- */
|
||||
.probe-summary { display: flex; flex-wrap: wrap; gap: 16px; font-size: 13px; }
|
||||
.probe-list {
|
||||
border: 1px solid var(--border); border-radius: 8px; max-height: 260px;
|
||||
overflow: auto; padding: 8px 12px; display: flex; flex-direction: column;
|
||||
gap: 5px; font-size: 12px; background: var(--log-bg);
|
||||
}
|
||||
.probe-item { display: flex; gap: 10px; align-items: baseline; }
|
||||
.probe-item .u { color: var(--text); word-break: break-all; }
|
||||
.probe-item.included .u { color: var(--green); }
|
||||
.probe-item.excluded .u { color: var(--muted); text-decoration: line-through; }
|
||||
.probe-item .reason { color: var(--yellow); font-size: 11px; flex-shrink: 0; }
|
||||
.meta-table td { white-space: normal; word-break: break-all; }
|
||||
.meta-table td:first-child { width: 110px; color: var(--muted); }
|
||||
|
||||
/* ---------- misc ---------- */
|
||||
.toast {
|
||||
position: fixed; top: 70px; left: 50%; transform: translateX(-50%);
|
||||
background: var(--panel2); border: 1px solid var(--accent); color: #fff;
|
||||
background: var(--panel2); border: 1px solid var(--accent); color: var(--text);
|
||||
padding: 10px 20px; border-radius: 10px; z-index: 300; font-size: 13px;
|
||||
box-shadow: 0 8px 30px rgba(0,0,0,.4); animation: fadein .2s;
|
||||
}
|
||||
.toast.error { border-color: var(--red); }
|
||||
@keyframes fadein { from { opacity: 0; transform: translate(-50%, -8px); } }
|
||||
::-webkit-scrollbar { width: 8px; height: 8px; }
|
||||
::-webkit-scrollbar-thumb { background: #2a3550; border-radius: 4px; }
|
||||
::-webkit-scrollbar-thumb { background: var(--thumb); border-radius: 4px; }
|
||||
Reference in New Issue
Block a user