Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
13a760fa6f | ||
|
|
add2cea545 | ||
|
|
44852f7be7 |
@@ -33,7 +33,10 @@
|
||||
- 运行中的任务参数支持**热更新**(修改后从下一页起生效)
|
||||
|
||||
### 2. 批量爬取模式
|
||||
一次粘贴多个网址(每行一个,`#` 注释),可配置:
|
||||
一次粘贴多个网址(每行一个,`#` 注释),或点击「📂 导入网址文件」上传 .txt 文件批量导入:
|
||||
- **处理方式可选**:追加(保留已有,默认)或覆盖(清空已有)
|
||||
- 自动识别 UTF-8/GBK 编码、自动去重、自动清理行内注释
|
||||
可配置:
|
||||
- 项目名称、输出目录(默认 `out/<任务ID>`,可填绝对路径)
|
||||
- 爬取间隔(随机秒数区间,防封 IP)、单页超时
|
||||
- 失败重试次数 / 重试间隔
|
||||
|
||||
@@ -154,6 +154,19 @@ def api_tasks():
|
||||
return jsonify(tasks)
|
||||
|
||||
|
||||
def _clean_urls(lines):
|
||||
"""清洗网址行: 去空白, 去行内 # 注释 (URL 本身不含空格, 安全), 自动补 https 前缀"""
|
||||
out = []
|
||||
for u in lines or []:
|
||||
u = str(u).split(" #")[0].strip()
|
||||
if not u:
|
||||
continue
|
||||
if not u.startswith("http"):
|
||||
u = "https://" + u
|
||||
out.append(u)
|
||||
return out
|
||||
|
||||
|
||||
@app.route("/api/tasks", methods=["POST"])
|
||||
def api_create_task():
|
||||
body = request.get_json(force=True) or {}
|
||||
@@ -171,7 +184,7 @@ def api_create_task():
|
||||
"created_at": now_str(),
|
||||
"updated_at": now_str(),
|
||||
"config": {**DEFAULT_CONFIG, **(body.get("config") or {})},
|
||||
"urls": [u.strip() for u in (body.get("urls") or []) if u.strip()],
|
||||
"urls": _clean_urls(body.get("urls") or []),
|
||||
}
|
||||
|
||||
if mode == "auto":
|
||||
@@ -230,7 +243,7 @@ def api_update_task(tid):
|
||||
if "name" in body and str(body["name"]).strip():
|
||||
task["name"] = str(body["name"]).strip()
|
||||
if "urls" in body:
|
||||
task["urls"] = [u.strip() for u in body["urls"] if u.strip()]
|
||||
task["urls"] = _clean_urls(body["urls"])
|
||||
if "config" in body:
|
||||
merged = {**task.get("config", {}), **body["config"]}
|
||||
task["config"] = merged
|
||||
|
||||
+71
-1
@@ -79,10 +79,28 @@ function taskStatusBadge(t) {
|
||||
return '<span class="badge">未运行</span>';
|
||||
}
|
||||
|
||||
const GROUPS = [
|
||||
{ key: "batch", label: "📄 批量爬取", desc: "一次性爬取指定网址列表" },
|
||||
{ key: "scheduled", label: "⏰ 定时爬取", desc: "按间隔或 cron 表达式定时执行" },
|
||||
{ key: "auto", label: "🤖 自动爬取", desc: "从起始网址自动发现链接并爬取" },
|
||||
];
|
||||
|
||||
function renderTasks() {
|
||||
const box = $("taskList");
|
||||
$("emptyState").classList.toggle("hidden", state.tasks.length > 0);
|
||||
box.innerHTML = state.tasks.map(taskCard).join("");
|
||||
box.innerHTML = GROUPS.map((g) => {
|
||||
const items = state.tasks.filter((t) => t.mode === g.key);
|
||||
if (!items.length) return "";
|
||||
return `
|
||||
<div class="group">
|
||||
<div class="group-head">
|
||||
<span class="group-title">${g.label}</span>
|
||||
<span class="group-desc">${g.desc}</span>
|
||||
<span class="group-count">${items.length} 个任务</span>
|
||||
</div>
|
||||
<div class="task-grid">${items.map(taskCard).join("")}</div>
|
||||
</div>`;
|
||||
}).join("");
|
||||
}
|
||||
|
||||
function taskCard(t) {
|
||||
@@ -161,6 +179,58 @@ async function delTask(tid) {
|
||||
} catch (e) { toast(e.message, true); }
|
||||
}
|
||||
|
||||
/* ---------------- 网址文件导入 ---------------- */
|
||||
const MAX_IMPORT_SIZE = 2 * 1024 * 1024; // 2MB 上限
|
||||
|
||||
async function readFileSmart(file) {
|
||||
/* 自动识别 UTF-8 / GBK 编码 */
|
||||
const buf = await file.arrayBuffer();
|
||||
let text = new TextDecoder("utf-8").decode(buf);
|
||||
if (text.includes("\uFFFD")) {
|
||||
try { text = new TextDecoder("gbk").decode(buf); } catch (e) { /* 保留 utf-8 结果 */ }
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
function importUrlsText(text, mode) {
|
||||
const ta = $("taskForm").elements["urls"];
|
||||
const clean = (s) => s.split(" #")[0].trim(); // 去掉行内注释 (URL 不含空格, 安全)
|
||||
const existing = new Set(
|
||||
mode === "overwrite" ? [] : ta.value.split(/\r?\n/).map(clean).filter(Boolean)
|
||||
);
|
||||
const fresh = text.split(/\r?\n/).map(clean).filter(Boolean);
|
||||
let added = 0;
|
||||
for (const line of fresh) {
|
||||
if (!existing.has(line)) { existing.add(line); added++; }
|
||||
}
|
||||
ta.value = Array.from(existing).join("\n");
|
||||
return { total: fresh.length, added };
|
||||
}
|
||||
|
||||
$("btnImportUrls").onclick = () => $("urlFileInput").click();
|
||||
$("urlFileInput").onchange = async (e) => {
|
||||
const file = e.target.files && e.target.files[0];
|
||||
e.target.value = ""; // 允许重复选择同一文件
|
||||
if (!file) return;
|
||||
if (file.size > MAX_IMPORT_SIZE) {
|
||||
toast("文件过大(上限 2MB),请拆分后导入", true);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const text = await readFileSmart(file);
|
||||
const mode = $("importMode").value;
|
||||
const r = importUrlsText(text, mode);
|
||||
formDirty = true;
|
||||
if (mode === "overwrite") {
|
||||
toast(`已导入 ${file.name}:共 ${r.total} 行(覆盖原列表,新增 ${r.added} 条)`);
|
||||
} else {
|
||||
toast(`已导入 ${file.name}:共 ${r.total} 行,新增 ${r.added} 条网址(追加)`);
|
||||
}
|
||||
} catch (err) {
|
||||
toast("文件读取失败: " + err.message, true);
|
||||
}
|
||||
};
|
||||
|
||||
/* ---------------- 搜索 ---------------- */
|
||||
async function doSearch() {
|
||||
const q = $("searchInput").value.trim();
|
||||
|
||||
+11
-1
@@ -73,7 +73,17 @@
|
||||
<div class="field"><label>通知邮箱</label><input name="notify_email" value="wlq@tphai.com"></div>
|
||||
</div>
|
||||
|
||||
<div class="field" id="urlsField"><label>网址列表(每行一个,# 开头为注释)</label>
|
||||
<div class="field" id="urlsField">
|
||||
<label>网址列表(每行一个,# 开头为注释)</label>
|
||||
<div class="row2" style="align-items:center">
|
||||
<button type="button" class="btn sm" id="btnImportUrls">📂 导入网址文件(.txt)</button>
|
||||
<select id="importMode" title="对已有网址的处理方式">
|
||||
<option value="append" selected>追加(保留已有)</option>
|
||||
<option value="overwrite">覆盖(清空已有)</option>
|
||||
</select>
|
||||
<span class="card-line">支持 UTF-8 / GBK,自动去重</span>
|
||||
<input type="file" id="urlFileInput" accept=".txt,.csv,.urls,text/plain" hidden>
|
||||
</div>
|
||||
<textarea name="urls" rows="5" placeholder="https://www.example.com/ https://www.example.com/page2"></textarea>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -85,6 +85,16 @@ main { padding: 20px 24px; max-width: 1500px; margin: 0 auto; }
|
||||
.stat-num.num-run { color: var(--accent); }
|
||||
.stat-label { font-size: 12px; color: var(--muted); margin-top: 3px; }
|
||||
|
||||
/* ---------- 分组展示 ---------- */
|
||||
.group { display: flex; flex-direction: column; gap: 12px; margin-bottom: 24px; }
|
||||
.group-head { display: flex; align-items: baseline; gap: 10px; flex-wrap: wrap; }
|
||||
.group-title { font-size: 16px; font-weight: 700; }
|
||||
.group-desc { font-size: 12px; color: var(--muted); }
|
||||
.group-count {
|
||||
font-size: 11px; padding: 2px 10px; border-radius: 10px;
|
||||
background: var(--panel2); border: 1px solid var(--border); color: var(--muted);
|
||||
}
|
||||
|
||||
/* ---------- task grid ---------- */
|
||||
.task-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(380px, 1fr)); gap: 16px; }
|
||||
.card {
|
||||
|
||||
Reference in New Issue
Block a user