hz4th_coder/universal-crawler
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v1.4.0 修复反爬误判+浏览器崩溃空转, 详情页多元翻页

Browse Source 
- 修复反爬误判: is_challenge_page 仅匹配 title 关键词 + html 前 20KB 强特征标记,
  不再全文匹配 captcha/challenge 等词 (博客园正文提 captcha 的页面被误杀)
- 修复浏览器崩溃后空转: 检测 Target page/context/browser closed,
  自动重启浏览器并重试当前 URL (不消耗重试配额, 单页最多重建3次);
  页面读取连续异常3次快速失败; 新增 browser_max_pages 配置定期重启浏览器防内存膨胀
- 详情页多元翻页: 首页/末页/页码列表(带省略号)/页码跳转输入框(回车/GO)/每页条数选择(50/100/200/500)
This commit is contained in:
hz4th_coder
2026-08-14 10:28:25 +08:00
parent 32eebf3dd1
commit ec053266a7
3 changed files with 191 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
engine.py
+97 -16
View File
@@ -30,12 +30,21 @@ IMG_EXTS = (".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp")
DEFAULT_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36")
_CHALLENGE_MARKS = [
# title 中出现任一标记即判定反爬 (title 短小可靠, 不会误伤正文)
_CHALLENGE_TITLE_MARKS = [
"access denied", "403 forbidden", "just a moment", "attention required",
"captcha", "bot check", "cf-challenge", "verify you are human",
"checking your browser", "enable javascript and cookies",
]
# html 中仅匹配强特征标记, 且只在 head 区域(前 20KB)搜索。
# 不能全文匹配 "captcha"/"challenge" 等词——正常文章正文提到这些词会被误判为反爬。
_CHALLENGE_HTML_MARKS = [
"cf-challenge", "challenge-platform", "cf-browser-verification",
"verify you are human", "checking your browser",
"enable javascript and cookies", "just a moment",
]
# 反爬拦截类错误信号: 命中后不再重试 (重试无意义且拖慢任务)
_ANTI_CRAWL_MARKS = [
"403", "429", "503", "forbidden", "access denied", "too many requests",
@@ -50,14 +59,31 @@ def is_anti_crawl_error(err):
return any(m in s for m in _ANTI_CRAWL_MARKS)
_BROWSER_DEAD_MARKS = [
"browser has been closed", "target page, context or browser has been closed",
"has been disposed", "execution context was destroyed", "browser closed",
"page closed", "target closed", "crash",
]
def is_browser_dead_error(err):
"""判断错误是否属于浏览器/页面失效 (需重启浏览器后重试, 重试同一 URL 才有意义)"""
s = str(err or "").lower()
return any(m in s for m in _BROWSER_DEAD_MARKS)
def is_challenge_page(title, html):
"""判断是否仍在反爬验证页 (仅关键词启发, 避免误伤正常小页面)"""
low = html.lower()
"""判断是否仍在反爬验证页
- title 出现反爬关键词: 判定 (可靠, 反爬页 title 基本都会变)
- html 仅在前 20KB(head 区域)出现强特征标记时判定,
避免正文含 captcha/challenge 等词的正常页面被误杀
"""
t = (title or "").lower()
for mark in _CHALLENGE_MARKS:
if mark in t or mark in low:
for mark in _CHALLENGE_TITLE_MARKS:
if mark in t:
return True
return False
low = (html or "").lower()[:20000]
return any(m in low for m in _CHALLENGE_HTML_MARKS)
def safe_name(url, idx):
@@ -231,6 +257,14 @@ class CrawlJob:
self._auto_pending = None
self._cfg_lock = threading.RLock()
self.thread = None
# 浏览器句柄 (p, browser, ctx, page, cookie_file); 崩溃后重建
self._browser = None
def _page(self):
return self._browser[3] if self._browser else None
def _ctx(self):
return self._browser[2] if self._browser else None
# ---------------- 控制接口 ----------------
def start(self):
@@ -309,9 +343,13 @@ class CrawlJob:
pass
Stealth().apply_stealth_sync(ctx)
page = ctx.new_page()
return p, browser, ctx, page, cookie_file
self._browser = (p, browser, ctx, page, cookie_file)
return self._browser
def _close_browser(self, p, browser, ctx, cookie_file):
def _close_browser(self):
if not self._browser:
return
p, browser, ctx, _page, cookie_file = self._browser
try:
json.dump(ctx.cookies(), open(cookie_file, "w"))
except Exception:
@@ -324,10 +362,24 @@ class CrawlJob:
p.stop()
except Exception:
pass
self._browser = None
def _reopen_browser(self):
"""浏览器失效后重建: 保存 cookie -> 关闭旧实例 -> 启动新实例"""
self._close_browser()
time.sleep(1)
self._open_browser()
self._log("info", "浏览器已重启")
def _need_browser_restart(self, done_count):
"""每爬 N 页主动重启一次浏览器, 防止长时间运行内存膨胀导致崩溃 (browser_max_pages=0 关闭)"""
limit = int(self._cfg("browser_max_pages", 0) or 0)
return limit > 0 and done_count > 1 and (done_count - 1) % limit == 0
def _wait_page_settle(self, page, timeout_s):
last_title, stable = "", 0
challenge_hits = 0
err_streak = 0 # 连续读取失败计数
start = time.time()
while time.time() - start < timeout_s:
if self._stop.is_set():
@@ -338,7 +390,11 @@ class CrawlJob:
title = page.title()
html = page.content()
except Exception:
err_streak += 1
if err_streak >= 3: # 页面/浏览器已失效, 提前失败而非干等到超时
raise RuntimeError("Target page, context or browser has been closed")
continue # 正在跳转
err_streak = 0
if is_challenge_page(title, html):
challenge_hits += 1
if challenge_hits >= 8: # 持续 8 秒仍是验证页, 判定反爬, 尽早放弃
@@ -461,7 +517,9 @@ class CrawlJob:
"source_url": source_url, "depth": depth,
"images": [], "attempts": 0,
}
for attempt in range(retries + 1):
dead_strikes = 0 # 浏览器失效重建次数 (防止无限重建)
attempt = 0
while attempt <= retries:
if self._stop.is_set():
entry["error"] = "任务已终止"
break
@@ -485,6 +543,22 @@ class CrawlJob:
except Exception as e:
entry["error"] = str(e)
entry["crawl_time"] = store.now_str()
if is_browser_dead_error(e):
# 浏览器/页面失效: 重启浏览器后重试同一 URL, 不消耗重试次数
dead_strikes += 1
if dead_strikes > 3:
entry["error"] = f"浏览器多次重启仍失效, 放弃: {e}"
self._log("error", f"浏览器多次重启仍失效, 放弃 {url}")
break
self._log("warn", f"浏览器已失效({e}), 重启后重试: {url}")
try:
self._reopen_browser()
page, ctx = self._page(), self._ctx()
except Exception as re_err:
entry["error"] = f"浏览器重启失败: {re_err}"
self._log("error", f"浏览器重启失败, 放弃 {url}: {re_err}")
break
continue
self._log("warn", f"第{attempt + 1}次失败 {url}: {e}")
if is_anti_crawl_error(e):
# 反爬拦截: 重试也过不去, 直接放弃, 不再消耗重试次数
@@ -499,6 +573,7 @@ class CrawlJob:
break
self._wait_if_paused()
time.sleep(0.3)
attempt += 1
self._write_page_meta(out_dir, base, entry)
return entry
@@ -562,17 +637,20 @@ class CrawlJob:
os.makedirs(out_dir, exist_ok=True)
self._persist()
p, browser, ctx, page, cookie_file = self._open_browser()
self._open_browser()
try:
for i, url in enumerate(urls, 1):
if self._stop.is_set():
self._log("info", "收到终止信号, 停止爬取")
break
self._wait_if_paused()
if self._need_browser_restart(i):
self._log("info", f"已爬 {i - 1} 页, 主动重启浏览器")
self._reopen_browser()
run["progress"]["current_url"] = url
run["progress"]["done"] = i - 1
self._persist()
entry = self._retry_crawl(page, ctx, url, i, out_dir)
entry = self._retry_crawl(self._page(), self._ctx(), url, i, out_dir)
run["results"].append(entry)
self._bump_stats(entry)
run["progress"]["done"] = i
@@ -580,7 +658,7 @@ class CrawlJob:
if entry["status"] == "OK":
self._delay()
finally:
self._close_browser(p, browser, ctx, cookie_file)
self._close_browser()
def _discover_links(self, page):
"""从当前页面提取符合规则的链接"""
@@ -637,7 +715,7 @@ class CrawlJob:
self._auto_pending = len(queue)
self._persist()
p, browser, ctx, page, cookie_file = self._open_browser()
self._open_browser()
try:
while queue and not self._stop.is_set():
self._wait_if_paused()
@@ -650,6 +728,9 @@ class CrawlJob:
elif len(visited) >= max_pages:
self._log("info", f"达到最大页数上限 {max_pages}, 停止")
break
if self._need_browser_restart(len(visited)):
self._log("info", f"已爬 {len(visited) - 1} 页, 主动重启浏览器")
self._reopen_browser()
url, depth, src = queue.pop(0)
key = normalize_url(url)
if key in visited and url != seed: # 起始网址不去重, 其余已爬跳过
@@ -661,14 +742,14 @@ class CrawlJob:
run["progress"]["current_url"] = url
run["progress"]["done"] = len(visited)
self._persist()
entry = self._retry_crawl(page, ctx, url, idx, out_dir,
entry = self._retry_crawl(self._page(), self._ctx(), url, idx, out_dir,
source_url=src, depth=depth)
run["results"].append(entry)
self._bump_stats(entry)
self._persist()
# 无深度限制或未达深度限制时持续发现链接
if entry["status"] == "OK" and (max_depth == 0 or depth < max_depth):
for link in self._discover_links(page):
for link in self._discover_links(self._page()):
lk = normalize_url(link)
if lk not in visited and lk not in queued:
queued.add(lk)
@@ -680,7 +761,7 @@ class CrawlJob:
if not self._stop.is_set() and len(queue) == 0:
self._log("info", f"无新链接可爬, 任务结束 (共 {len(visited)} 页)")
finally:
self._close_browser(p, browser, ctx, cookie_file)
self._close_browser()
# 无论完成/停止/异常, 都保存缓存队列与已爬集合, 便于下次继续
try:
store.save_auto_state(self.task["id"], {