v1.0.8: 自动爬取URL规范化去重(锚点/跟踪参数/尾部斜杠/默认端口) + 修复反爬启发式误伤小页面

README.md

@@ -33,7 +33,10 @@
 - 运行中的任务参数支持**热更新**（修改后从下一页起生效）
 
 ### 2. 批量爬取模式
-一次粘贴多个网址（每行一个，`#` 注释），或点击「📂 导入网址文件」上传 .txt 文件批量导入（自动识别 UTF-8/GBK 编码、自动去重、自动清理行内注释）：
+一次粘贴多个网址（每行一个，`#` 注释），或点击「📂 导入网址文件」上传 .txt 文件批量导入：
+- **处理方式可选**：追加（保留已有，默认）或覆盖（清空已有）
+- 自动识别 UTF-8/GBK 编码、自动去重、自动清理行内注释
+可配置：
 - 项目名称、输出目录（默认 `out/<任务ID>`，可填绝对路径）
 - 爬取间隔（随机秒数区间，防封 IP）、单页超时
 - 失败重试次数 / 重试间隔

engine.py

@@ -37,13 +37,12 @@ _CHALLENGE_MARKS = [
 
 
 def is_challenge_page(title, html):
+    """判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)"""
     low = html.lower()
     t = (title or "").lower()
     for mark in _CHALLENGE_MARKS:
         if mark in t or mark in low:
             return True
-    if len(html) < 5000 and ("<article" not in low and "<main" not in low):
-        return True
     return False
 
 
@@ -77,6 +76,36 @@ def _settle_wait(page, timeout_s):
     return True, page.title(), page.content()
 
 
+_TRACKING_PARAMS = {
+    "utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content",
+    "fbclid", "gclid", "yclid", "mc_cid", "mc_eid", "ref", "ref_src",
+}
+
+
+def normalize_url(url):
+    """URL 规范化 (用于去重): 去锚点/跟踪参数/尾部斜杠/默认端口, host 小写"""
+    try:
+        p = urllib.parse.urlparse(str(url))
+        host = (p.hostname or "").lower()
+        if not host:
+            return str(url)
+        port = ""
+        if p.port and p.port not in (80, 443):
+            port = f":{p.port}"
+        path = p.path or "/"
+        if len(path) > 1 and path.endswith("/"):
+            path = path.rstrip("/")
+        query = ""
+        if p.query:
+            kept = [kv for kv in p.query.split("&")
+                    if kv.split("=", 1)[0].lower() not in _TRACKING_PARAMS]
+            if kept:
+                query = "?" + "&".join(kept)
+        return f"{p.scheme.lower()}://{host}{port}{path}{query}"
+    except Exception:
+        return str(url)
+
+
 def filter_links(hrefs, seed_url, include=None, exclude=None,
                  same_domain=True, use_regex=False):
     """按规则过滤链接, 返回 (included, excluded); excluded 含排除原因"""
@@ -551,18 +580,19 @@ class CrawlJob:
 
         p, browser, ctx, page, cookie_file = self._open_browser()
         queue = [(seed, 0, "")]               # (url, depth, 来源链接)
-        visited = set()
+        visited = set()                        # 规范化 URL 去重
-        queued = set([seed])
+        queued = set([normalize_url(seed)])
         idx = 0
         try:
             while queue and not self._stop.is_set():
                 self._wait_if_paused()
                 url, depth, src = queue.pop(0)
-                if url in visited:
+                key = normalize_url(url)
+                if key in visited:
                     continue
                 if len(visited) >= max_pages:
                     break
-                visited.add(url)
+                visited.add(key)
                 idx += 1
                 run["progress"]["current_url"] = url
                 run["progress"]["done"] = len(visited)
@@ -574,8 +604,9 @@ class CrawlJob:
                 self._persist()
                 if entry["status"] == "OK" and depth < max_depth:
                     for link in self._discover_links(page):
-                        if link not in visited and link not in queued:
+                        lk = normalize_url(link)
-                            queued.add(link)
+                        if lk not in visited and lk not in queued:
+                            queued.add(lk)
                             queue.append((link, depth + 1, url))
                 if entry["status"] == "OK":
                     self._delay()

static/app.js

@@ -79,10 +79,28 @@ function taskStatusBadge(t) {
   return '<span class="badge">未运行</span>';
 }
 
+const GROUPS = [
+  { key: "batch", label: "📄 批量爬取", desc: "一次性爬取指定网址列表" },
+  { key: "scheduled", label: "⏰ 定时爬取", desc: "按间隔或 cron 表达式定时执行" },
+  { key: "auto", label: "🤖 自动爬取", desc: "从起始网址自动发现链接并爬取" },
+];
+
 function renderTasks() {
   const box = $("taskList");
   $("emptyState").classList.toggle("hidden", state.tasks.length > 0);
-  box.innerHTML = state.tasks.map(taskCard).join("");
+  box.innerHTML = GROUPS.map((g) => {
+    const items = state.tasks.filter((t) => t.mode === g.key);
+    if (!items.length) return "";
+    return `
+      <div class="group">
+        <div class="group-head">
+          <span class="group-title">${g.label}</span>
+          <span class="group-desc">${g.desc}</span>
+          <span class="group-count">${items.length} 个任务</span>
+        </div>
+        <div class="task-grid">${items.map(taskCard).join("")}</div>
+      </div>`;
+  }).join("");
 }
 
 function taskCard(t) {
@@ -174,10 +192,12 @@ async function readFileSmart(file) {
   return text;
 }
 
-function importUrlsText(text) {
+function importUrlsText(text, mode) {
   const ta = $("taskForm").elements["urls"];
   const clean = (s) => s.split(" #")[0].trim();     // 去掉行内注释 (URL 不含空格, 安全)
-  const existing = new Set(ta.value.split(/\r?\n/).map(clean).filter(Boolean));
+  const existing = new Set(
+    mode === "overwrite" ? [] : ta.value.split(/\r?\n/).map(clean).filter(Boolean)
+  );
   const fresh = text.split(/\r?\n/).map(clean).filter(Boolean);
   let added = 0;
   for (const line of fresh) {
@@ -198,9 +218,14 @@ $("urlFileInput").onchange = async (e) => {
   }
   try {
     const text = await readFileSmart(file);
-    const r = importUrlsText(text);
+    const mode = $("importMode").value;
+    const r = importUrlsText(text, mode);
     formDirty = true;
-    toast(`已导入 ${file.name}：共 ${r.total} 行，新增 ${r.added} 条网址`);
+    if (mode === "overwrite") {
+      toast(`已导入 ${file.name}：共 ${r.total} 行（覆盖原列表，新增 ${r.added} 条）`);
+    } else {
+      toast(`已导入 ${file.name}：共 ${r.total} 行，新增 ${r.added} 条网址（追加）`);
+    }
   } catch (err) {
     toast("文件读取失败: " + err.message, true);
   }

static/index.html

@@ -77,7 +77,11 @@
           <label>网址列表（每行一个，# 开头为注释）</label>
           <div class="row2" style="align-items:center">
             <button type="button" class="btn sm" id="btnImportUrls">📂 导入网址文件(.txt)</button>
-            <span class="card-line">支持 UTF-8 / GBK 编码，自动去重追加</span>
+            <select id="importMode" title="对已有网址的处理方式">
+              <option value="append" selected>追加（保留已有）</option>
+              <option value="overwrite">覆盖（清空已有）</option>
+            </select>
+            <span class="card-line">支持 UTF-8 / GBK，自动去重</span>
             <input type="file" id="urlFileInput" accept=".txt,.csv,.urls,text/plain" hidden>
           </div>
           <textarea name="urls" rows="5" placeholder="https://www.example.com/&#10;https://www.example.com/page2"></textarea>

static/style.css

@@ -85,6 +85,16 @@ main { padding: 20px 24px; max-width: 1500px; margin: 0 auto; }
 .stat-num.num-run { color: var(--accent); }
 .stat-label { font-size: 12px; color: var(--muted); margin-top: 3px; }
 
+/* ---------- 分组展示 ---------- */
+.group { display: flex; flex-direction: column; gap: 12px; margin-bottom: 24px; }
+.group-head { display: flex; align-items: baseline; gap: 10px; flex-wrap: wrap; }
+.group-title { font-size: 16px; font-weight: 700; }
+.group-desc { font-size: 12px; color: var(--muted); }
+.group-count {
+  font-size: 11px; padding: 2px 10px; border-radius: 10px;
+  background: var(--panel2); border: 1px solid var(--border); color: var(--muted);
+}
+
 /* ---------- task grid ---------- */
 .task-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(380px, 1fr)); gap: 16px; }
 .card {