Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 hz4th_coder
|
66ba1a7cfc | ||
|
hz4th_coder
|
3ad3d00ca8 |
@@ -49,12 +49,14 @@
|
||||
- 可启用/停用调度，自动计算下次执行时间；到点自动开跑，跑完自动计算下一次
|
||||
|
||||
### 4. 自动爬取模式
|
||||
给定一个起始网址，系统自动从页面里发现链接、按规则筛选后 BFS 爬取：
|
||||
给定一个起始网址，系统**持续递归**爬取：爬取页面 → 自动发现符合规则的链接 → 继续爬取 → 继续发现……直到**无新链接可爬**时自动结束。
|
||||
- **🧪 试爬取**：先用起始网址试跑一次，展示规则筛选后的链接清单（将爬取哪些、排除哪些及原因），确认规则符合预期后再正式开爬
|
||||
- **最大爬取深度**：0=无限制（默认），N=只爬 N 层
|
||||
- **最大页数**：0=无限制，默认 1000 作安全上限（防止动态无限链接的站点失控）
|
||||
- **🔄 缓存续爬**：提取但未爬取的链接自动存入缓存队列（连同已爬集合一并持久化），任务停止/中断后再次运行，从缓存队列**继续爬取**，已爬过的不会重复；起始网址每次运行都重新爬取（不去重）；规则变更后可点「🧹 清空缓存」重新开始
|
||||
- **包含规则**：只爬包含指定子串（或正则）的链接
|
||||
- **排除规则**：跳过匹配的链接（如 login、/tag/）
|
||||
- **仅同域名**：限制在起始网站内
|
||||
- **最大页数 / 最大深度**：控制爬取规模
|
||||
- 其余参数（间隔、重试、图片、通知）同批量模式
|
||||
|
||||
### 5. 资源操作信息（元数据）
|
||||
|
||||
@@ -448,6 +448,27 @@ def api_probe():
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@app.route("/api/tasks/<tid>/clear_cache", methods=["POST"])
|
||||
def api_clear_cache(tid):
|
||||
"""清空自动任务的待爬缓存队列与已爬集合 (规则变更后重新开始用)"""
|
||||
task = store.get_task(tid)
|
||||
if not task:
|
||||
return jsonify({"error": "任务不存在"}), 404
|
||||
if task.get("mode") != "auto":
|
||||
return jsonify({"error": "仅自动爬取任务支持清空缓存"}), 400
|
||||
with JOBS_LOCK:
|
||||
job = JOBS.get(tid)
|
||||
if job and job.is_running():
|
||||
return jsonify({"error": "任务正在运行，无法清空缓存"}), 409
|
||||
auto = task.setdefault("auto", {})
|
||||
auto["pending"] = []
|
||||
auto["visited"] = []
|
||||
task["updated_at"] = now_str()
|
||||
store.upsert_task(task)
|
||||
db.upsert_task(task)
|
||||
return jsonify({"ok": True, "msg": "缓存队列已清空"})
|
||||
|
||||
|
||||
@app.route("/api/tasks/<tid>/probe", methods=["POST"])
|
||||
def api_task_probe(tid):
|
||||
"""对已保存的自动任务执行试爬取 (使用保存的规则)"""
|
||||
|
||||
@@ -21,6 +21,7 @@ from playwright_stealth import Stealth
|
||||
|
||||
import notify
|
||||
import store
|
||||
import db
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DATA_DIR = os.path.join(HERE, "data")
|
||||
@@ -567,33 +568,48 @@ class CrawlJob:
|
||||
return included
|
||||
|
||||
def _crawl_auto(self):
|
||||
"""自动爬取: 持续递归 爬取->发现链接->爬取... 直到无新链接可爬或达到上限
|
||||
- max_depth: 0=无限制, N=只爬 N 层
|
||||
- max_pages: 0=无限制, N=安全上限
|
||||
- 提取但未爬取的链接持久化到 auto.pending, 已爬集合持久化到 auto.visited
|
||||
- 停止后再次运行从缓存队列继续爬; 起始网址每次运行都重新爬(不去重)
|
||||
"""
|
||||
run = self.run
|
||||
auto = self.task.get("auto", {})
|
||||
seed = auto.get("seed_url", "")
|
||||
max_pages = int(auto.get("max_pages", 50) or 50)
|
||||
max_depth = int(auto.get("max_depth", 2) or 2)
|
||||
run["progress"]["total"] = max_pages
|
||||
max_pages = int(auto.get("max_pages", 1000) or 0) # 0 = 无限制
|
||||
max_depth = int(auto.get("max_depth", 0) or 0) # 0 = 无限制
|
||||
out_dir = self._resolve_out_dir()
|
||||
run["out_dir"] = out_dir
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
|
||||
# 恢复持久化状态: 已爬集合 + 上次未爬完的缓存队列
|
||||
visited = set(auto.get("visited", []) or [])
|
||||
pending = auto.get("pending", []) or []
|
||||
# 起始网址每次运行都爬(不做去重), 缓存队列继续消费
|
||||
queue = [(seed, 0, "")]
|
||||
if pending:
|
||||
queue.extend((p["url"], p.get("depth", 0), p.get("source", "")) for p in pending)
|
||||
queued = set(visited)
|
||||
for u, _d, _s in queue:
|
||||
queued.add(normalize_url(u))
|
||||
|
||||
run["progress"]["total"] = len(queue)
|
||||
self._persist()
|
||||
|
||||
p, browser, ctx, page, cookie_file = self._open_browser()
|
||||
queue = [(seed, 0, "")] # (url, depth, 来源链接)
|
||||
visited = set() # 规范化 URL 去重
|
||||
queued = set([normalize_url(seed)])
|
||||
idx = 0
|
||||
try:
|
||||
while queue and not self._stop.is_set():
|
||||
self._wait_if_paused()
|
||||
if max_pages > 0 and len(visited) >= max_pages:
|
||||
self._log("info", f"达到最大页数上限 {max_pages}, 停止")
|
||||
break
|
||||
url, depth, src = queue.pop(0)
|
||||
key = normalize_url(url)
|
||||
if key in visited:
|
||||
if key in visited and url != seed: # 起始网址不去重, 其余已爬跳过
|
||||
continue
|
||||
if len(visited) >= max_pages:
|
||||
break
|
||||
visited.add(key)
|
||||
idx += 1
|
||||
idx = len(visited)
|
||||
run["progress"]["current_url"] = url
|
||||
run["progress"]["done"] = len(visited)
|
||||
self._persist()
|
||||
@@ -602,7 +618,8 @@ class CrawlJob:
|
||||
run["results"].append(entry)
|
||||
self._bump_stats(entry)
|
||||
self._persist()
|
||||
if entry["status"] == "OK" and depth < max_depth:
|
||||
# 无深度限制或未达深度限制时持续发现链接
|
||||
if entry["status"] == "OK" and (max_depth == 0 or depth < max_depth):
|
||||
for link in self._discover_links(page):
|
||||
lk = normalize_url(link)
|
||||
if lk not in visited and lk not in queued:
|
||||
@@ -610,7 +627,18 @@ class CrawlJob:
|
||||
queue.append((link, depth + 1, url))
|
||||
if entry["status"] == "OK":
|
||||
self._delay()
|
||||
run["progress"]["total"] = len(visited)
|
||||
if not self._stop.is_set() and len(queue) == 0:
|
||||
self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)")
|
||||
finally:
|
||||
self._close_browser(p, browser, ctx, cookie_file)
|
||||
run["progress"]["total"] = len(visited)
|
||||
# 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
|
||||
try:
|
||||
auto["pending"] = [
|
||||
{"url": u, "depth": d, "source": s} for u, d, s in queue]
|
||||
auto["visited"] = list(visited)
|
||||
store.upsert_task(self.task)
|
||||
db.upsert_task(self.task)
|
||||
except Exception:
|
||||
pass
|
||||
self._persist()
|
||||
+20
-7
@@ -82,7 +82,7 @@ function taskStatusBadge(t) {
|
||||
const GROUPS = [
|
||||
{ key: "batch", label: "📄 批量爬取", desc: "一次性爬取指定网址列表" },
|
||||
{ key: "scheduled", label: "⏰ 定时爬取", desc: "按间隔或 cron 表达式定时执行" },
|
||||
{ key: "auto", label: "🤖 自动爬取", desc: "从起始网址自动发现链接并爬取" },
|
||||
{ key: "auto", label: "🤖 自动爬取", desc: "从起始网址自动发现链接，持续递归爬取到无新链接为止" },
|
||||
];
|
||||
|
||||
function renderTasks() {
|
||||
@@ -434,8 +434,8 @@ async function openEdit(tid) {
|
||||
f.elements["exclude"].value = (t.auto.exclude || []).join("\n");
|
||||
f.elements["same_domain"].checked = t.auto.same_domain !== false;
|
||||
f.elements["use_regex"].checked = !!t.auto.use_regex;
|
||||
f.elements["max_pages"].value = t.auto.max_pages ?? 50;
|
||||
f.elements["max_depth"].value = t.auto.max_depth ?? 2;
|
||||
f.elements["max_pages"].value = t.auto.max_pages ?? 1000;
|
||||
f.elements["max_depth"].value = t.auto.max_depth ?? 0;
|
||||
}
|
||||
syncScheduleUI();
|
||||
$("formHint").textContent = t.running
|
||||
@@ -481,8 +481,8 @@ async function submitForm(e) {
|
||||
exclude: splitLines(f.elements["exclude"].value),
|
||||
same_domain: f.elements["same_domain"].checked,
|
||||
use_regex: f.elements["use_regex"].checked,
|
||||
max_pages: parseInt(f.elements["max_pages"].value) || 50,
|
||||
max_depth: parseInt(f.elements["max_depth"].value) || 2,
|
||||
max_pages: parseInt(f.elements["max_pages"].value) || 0,
|
||||
max_depth: parseInt(f.elements["max_depth"].value) || 0,
|
||||
};
|
||||
}
|
||||
if (mode === "scheduled") {
|
||||
@@ -544,7 +544,10 @@ function renderDetail() {
|
||||
<span>任务ID: <b>${t.id}</b></span>
|
||||
<span>创建: <b>${fmtTime(t.created_at)}</b></span>
|
||||
<span>输出: <b>${esc(cfg.out_dir || "out/" + t.id)}</b></span>
|
||||
${t.mode === "auto" ? `<span>起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b></span>` : ""}
|
||||
${t.mode === "auto" ? `
|
||||
<span>起始: <b>${esc((t.auto && t.auto.seed_url) || "")}</b></span>
|
||||
<span>待爬缓存: <b>${(t.auto && (t.auto.pending || []).length) || 0}</b> 条（停止后可继续爬取）</span>
|
||||
<span>已爬: <b>${(t.auto && (t.auto.visited || []).length) || 0}</b> 页</span>` : ""}
|
||||
</div>`;
|
||||
|
||||
let sched = "";
|
||||
@@ -565,7 +568,8 @@ function renderDetail() {
|
||||
<button class="btn" onclick="actTask('${t.id}','pause')">⏸ 暂停</button>
|
||||
<button class="btn danger" onclick="actTask('${t.id}','stop')">⏹ 终止</button>`
|
||||
: `<button class="btn primary" onclick="actTask('${t.id}','start')">▶ 立即执行</button>`}
|
||||
${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>` : ""}
|
||||
${t.mode === "auto" ? `<button class="btn" onclick="probeTask('${t.id}')">🧪 试爬取</button>
|
||||
<button class="btn" ${t.running ? "disabled" : ""} onclick="clearCache('${t.id}')">🧹 清空缓存</button>` : ""}
|
||||
<button class="btn" onclick="openEdit('${t.id}')">✏️ 编辑配置</button>
|
||||
</div>`;
|
||||
|
||||
@@ -721,6 +725,15 @@ async function probeTask(tid) {
|
||||
} catch (e) { toast(e.message, true); }
|
||||
}
|
||||
|
||||
async function clearCache(tid) {
|
||||
if (!confirm("清空待爬缓存与已爬记录？\n下次运行将从起始网址重新开始爬取。")) return;
|
||||
try {
|
||||
const r = await api(`/api/tasks/${tid}/clear_cache`, { method: "POST" });
|
||||
toast(r.msg || "已清空");
|
||||
openDetail(tid);
|
||||
} catch (e) { toast(e.message, true); }
|
||||
}
|
||||
|
||||
function renderProbe(r) {
|
||||
const body = $("probeBody");
|
||||
if (!r.ok) {
|
||||
|
||||
+5
-3
@@ -116,7 +116,7 @@
|
||||
</div>
|
||||
|
||||
<div id="autoBox" class="hidden box">
|
||||
<div class="field"><label>起始网址 *（系统将自动发现符合规则的链接并爬取）</label>
|
||||
<div class="field"><label>起始网址 *（自动发现符合规则的链接，持续递归爬取直到无新链接可爬）</label>
|
||||
<div class="row2">
|
||||
<input name="seed_url" placeholder="https://example.com/news" style="flex:1">
|
||||
<button type="button" class="btn sm" id="btnProbe">🧪 试爬取</button>
|
||||
@@ -133,8 +133,10 @@
|
||||
<div class="field check"><label><input name="use_regex" type="checkbox"> 规则按正则匹配</label></div>
|
||||
</div>
|
||||
<div class="row2">
|
||||
<div class="field"><label>最大页数</label><input name="max_pages" type="number" value="50"></div>
|
||||
<div class="field"><label>最大爬取深度</label><input name="max_depth" type="number" value="2"></div>
|
||||
<div class="field"><label>最大爬取深度（0=无限制，N=只爬 N 层）</label>
|
||||
<input name="max_depth" type="number" min="0" value="0"></div>
|
||||
<div class="field"><label>最大页数（0=无限制，默认 1000 作安全上限）</label>
|
||||
<input name="max_pages" type="number" min="0" value="1000"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user