From 2a9f187ce80b449ce288bf2a0db5713eddd1e197 Mon Sep 17 00:00:00 2001 From: hz4th_coder Date: Sat, 29 Aug 2026 17:37:43 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=8F=AF=E8=AF=BB=E6=96=87?= =?UTF-8?q?=E6=9C=AC=E6=8F=90=E5=8F=96=20+=20=E5=8E=86=E5=8F=B2=E8=AE=B0?= =?UTF-8?q?=E5=BD=95=E5=88=86=E9=A1=B5=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 text 动作：提取页面可读文本（innerText + 空行/行尾清理，剔除脚本样式标签），agent-browser 与 playwright 双后端支持，自动提取页面标题 - 提取历史持久化(SQLite WAL)：每次捕获自动入库，截图文件存持久化目录 - 新增 API：GET /api/history(分页/类型筛选/搜索)、GET /api/history/(详情)、GET /api/history//file(截图)、DELETE /api/history/ - 前端：新增提取文本选项+干净文本展示；新增提取历史卡片（分页/筛选/搜索/查看弹窗/删除/下载/复制） --- .gitignore | 3 + app.py | 259 ++++++++++++++++-- templates/index.html | 605 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 837 insertions(+), 30 deletions(-) diff --git a/.gitignore b/.gitignore index 2ff8484..ff71ebb 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,9 @@ captures/ *.png *.html +# 运行时数据（历史记录数据库 + 截图文件） +data/ + # 环境变量 .env .env.local \ No newline at end of file diff --git a/app.py b/app.py index 493e8fb..e9044e5 100644 --- a/app.py +++ b/app.py @@ -5,8 +5,11 @@ Web Capture API - 网页截图与代码提取服务 """ import os +import re import json import asyncio +import sqlite3 +import shutil import subprocess import tempfile import uuid @@ -53,6 +56,80 @@ CORS(app) CAPTURE_DIR = Path(tempfile.gettempdir()) / "web_captures" CAPTURE_DIR.mkdir(exist_ok=True) +# 持久化存储（历史记录） +PROJECT_DIR = Path(__file__).parent +DATA_DIR = PROJECT_DIR / "data" +CAPTURE_DATA_DIR = DATA_DIR / "captures" +HISTORY_DB = DATA_DIR / "history.db" +DATA_DIR.mkdir(exist_ok=True) +CAPTURE_DATA_DIR.mkdir(exist_ok=True) + + +def get_db(): + """获取数据库连接（多进程安全：WAL + busy_timeout）""" + conn = sqlite3.connect(HISTORY_DB, timeout=15) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA busy_timeout=15000") + return conn + + +def init_db(): + """初始化历史记录表""" + conn = get_db() + conn.execute(""" + CREATE TABLE IF NOT EXISTS captures ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + url TEXT NOT NULL, + title TEXT DEFAULT '', + action TEXT NOT NULL, + backend TEXT DEFAULT '', + status TEXT NOT NULL DEFAULT 'success', + file_path TEXT DEFAULT '', + content TEXT DEFAULT '', + error TEXT DEFAULT '', + created_at TEXT NOT NULL + ) + """) + conn.commit() + conn.close() + + +def save_capture(url, title, action, backend, status, file_path='', content='', error=''): + """保存一条提取历史记录""" + conn = get_db() + cur = conn.execute( + "INSERT INTO captures (url, title, action, backend, status, file_path, content, error, created_at) " + "VALUES (?,?,?,?,?,?,?,?,?)", + (url, title, action, backend, status, file_path, content, error, + datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + ) + conn.commit() + rid = cur.lastrowid + conn.close() + return rid + + +def persist_screenshot(src_path): + """把临时截图复制到持久化目录，供历史记录长期访问""" + name = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}.png" + dest = CAPTURE_DATA_DIR / name + shutil.copy2(src_path, dest) + return str(dest) + + +def clean_text(raw): + """清理可读文本：去掉行尾空格、压缩多余空行、去掉首尾空白""" + if not raw: + return "" + lines = [ln.rstrip() for ln in raw.split("\n")] + text = "\n".join(lines) + text = re.sub(r"\n{3,}", "\n\n", text) + return text.strip() + + +init_db() + # agent-browser socket 目录 os.environ.setdefault("AGENT_BROWSER_SOCKET_DIR", "/tmp/agent-browser-sockets") Path(os.environ["AGENT_BROWSER_SOCKET_DIR"]).mkdir(exist_ok=True, parents=True) @@ -111,6 +188,38 @@ class AgentBrowserSession: timeout=10000 ) return success, output, error + + @staticmethod + def _decode_eval(output): + """agent-browser eval 结果按 JSON 编码返回，这里解码回原始字符串""" + if output is None: + return "" + out = output.strip() + try: + return json.loads(out) + except Exception: + # 非 JSON（如数字/布尔/纯文本），去掉可能的外层引号 + return out.strip('\"\'') + + def get_title(self): + """获取页面标题""" + success, output, error = self.run( + ["eval", "document.title"], + timeout=10000 + ) + if success: + return self._decode_eval(output) + return "" + + def get_text(self): + """提取页面可读文本（干净内容，剔除脚本/样式/标签）""" + success, output, error = self.run( + ["eval", "document.body.innerText"], + timeout=30000 + ) + if success: + return True, self._decode_eval(output), "" + return False, "", error def scroll_down(self, pixels=800): """向下滚动""" @@ -165,25 +274,35 @@ def capture_with_agent_browser( session.scroll_down(800) session.wait(scroll_delay) + # 提取页面标题 + title = session.get_title() + # 执行操作 if action == "screenshot": temp_file = CAPTURE_DIR / f"{session.session_id}.png" success, _, error = session.screenshot(temp_file, full_page=full_page) if success and temp_file.exists(): - return {"success": True, "file_path": str(temp_file)} + return {"success": True, "title": title, "file_path": str(temp_file)} else: - return {"success": False, "error": f"Screenshot failed: {error}"} + return {"success": False, "title": title, "error": f"Screenshot failed: {error}"} elif action == "html": success, html, error = session.get_html() if success: - return {"success": True, "html": html} + return {"success": True, "title": title, "html": html} else: - return {"success": False, "error": f"Get HTML failed: {error}"} + return {"success": False, "title": title, "error": f"Get HTML failed: {error}"} + + elif action == "text": + success, raw_text, error = session.get_text() + if success: + return {"success": True, "title": title, "text": clean_text(raw_text)} + else: + return {"success": False, "title": title, "error": f"Extract text failed: {error}"} else: - return {"success": False, "error": f"Unknown action: {action}"} + return {"success": False, "title": title, "error": f"Unknown action: {action}"} except Exception as e: return {"success": False, "error": str(e)} @@ -343,19 +462,29 @@ async def capture_with_playwright( await page.wait_for_timeout(scroll_delay) await page.wait_for_timeout(500) + # 提取页面标题 + try: + title = await page.title() + except: + title = "" + result = {} if action == "screenshot": temp_file = CAPTURE_DIR / f"{session_id}.png" await page.screenshot(path=str(temp_file), full_page=full_page) - result = {"success": True, "file_path": str(temp_file)} + result = {"success": True, "title": title, "file_path": str(temp_file)} elif action == "html": html = await page.content() - result = {"success": True, "html": html} + result = {"success": True, "title": title, "html": html} + + elif action == "text": + raw_text = await page.evaluate("document.body.innerText") + result = {"success": True, "title": title, "text": clean_text(raw_text)} else: - result = {"success": False, "error": f"Unknown action: {action}"} + result = {"success": False, "title": title, "error": f"Unknown action: {action}"} await browser.close() return result @@ -446,9 +575,11 @@ def capture_webpage( if backend == "agent-browser": if not AGENT_BROWSER_AVAILABLE: return {"success": False, "error": "agent-browser not available"} - return capture_with_agent_browser( + result = capture_with_agent_browser( url, action, scroll_times, scroll_delay, full_page, viewport, wait_time ) + result["backend"] = "agent-browser" + return result elif backend == "playwright": if not PLAYWRIGHT_AVAILABLE: @@ -463,6 +594,7 @@ def capture_webpage( ) ) loop.close() + result["backend"] = "playwright" return result except Exception as e: return {"success": False, "error": str(e)} @@ -482,18 +614,105 @@ def api_info(): """API信息""" return jsonify({ "service": "Web Capture API", - "version": "2.0.0", + "version": "2.1.0", "backends": { "agent-browser": "available" if AGENT_BROWSER_AVAILABLE else "unavailable", "playwright": "available" if PLAYWRIGHT_AVAILABLE else "unavailable" }, "endpoints": { - "/api/capture": "POST - Capture webpage screenshot or HTML", + "/api/capture": "POST - Capture webpage (screenshot/html/text) + 自动入库历史", + "/api/history": "GET - 历史记录分页列表 (?page&page_size&action&search)", + "/api/history/": "GET - 历史记录详情 / DELETE - 删除记录", + "/api/history//file": "GET - 读取历史截图文件", "/health": "GET - Health check" } }) +@app.route('/api/history', methods=['GET']) +def history_list(): + """ + 历史记录分页列表 + 参数: page(默认1), page_size(默认15, 最大100), action(筛选: all/screenshot/html/text), search(按url/title模糊搜索) + """ + page = max(1, int(request.args.get('page', 1))) + page_size = min(100, max(1, int(request.args.get('page_size', 15)))) + action = request.args.get('action', 'all') + search = request.args.get('search', '').strip() + + where = [] + params = [] + if action and action != 'all': + where.append("action = ?") + params.append(action) + if search: + where.append("(url LIKE ? OR title LIKE ?)") + params.append(f"%{search}%") + params.append(f"%{search}%") + where_sql = (" WHERE " + " AND ".join(where)) if where else "" + + conn = get_db() + total = conn.execute(f"SELECT COUNT(*) AS c FROM captures{where_sql}", params).fetchone()["c"] + rows = conn.execute( + f"SELECT id, url, title, action, backend, status, file_path, created_at, " + f"LENGTH(content) AS content_size, substr(content, 1, 200) AS content_preview, error " + f"FROM captures{where_sql} ORDER BY id DESC LIMIT ? OFFSET ?", + params + [page_size, (page - 1) * page_size] + ).fetchall() + conn.close() + + records = [dict(r) for r in rows] + return jsonify({ + "success": True, + "page": page, + "page_size": page_size, + "total": total, + "total_pages": (total + page_size - 1) // page_size if total else 1, + "records": records + }) + + +@app.route('/api/history/', methods=['GET']) +def history_detail(rid): + """历史记录详情（含完整内容）""" + conn = get_db() + r = conn.execute("SELECT * FROM captures WHERE id=?", (rid,)).fetchone() + conn.close() + if not r: + return jsonify({"success": False, "error": "记录不存在"}), 404 + return jsonify({"success": True, "record": dict(r)}) + + +@app.route('/api/history//file', methods=['GET']) +def history_file(rid): + """读取历史截图文件""" + conn = get_db() + r = conn.execute("SELECT file_path FROM captures WHERE id=?", (rid,)).fetchone() + conn.close() + if not r or not r["file_path"] or not Path(r["file_path"]).exists(): + return jsonify({"success": False, "error": "文件不存在或已删除"}), 404 + return send_file(r["file_path"], mimetype='image/png') + + +@app.route('/api/history/', methods=['DELETE']) +def history_delete(rid): + """删除历史记录（连带删除截图文件）""" + conn = get_db() + r = conn.execute("SELECT file_path FROM captures WHERE id=?", (rid,)).fetchone() + if r: + conn.execute("DELETE FROM captures WHERE id=?", (rid,)) + conn.commit() + conn.close() + if r and r["file_path"]: + fp = Path(r["file_path"]) + if fp.exists() and str(fp).startswith(str(CAPTURE_DATA_DIR)): + try: + fp.unlink() + except Exception: + pass + return jsonify({"success": True}) + + @app.route('/health') def health(): """健康检查""" @@ -555,19 +774,27 @@ def capture(): full_page=full_page, viewport=viewport, wait_time=wait_time, - backend=backend, - - + backend=backend ) + backend_used = result.get("backend", backend) + title = result.get("title", "") + if not result["success"]: + save_capture(url, title, action, backend_used, "failed", error=result.get("error", "")) return jsonify(result), 400 if action == "screenshot": - file_path = result["file_path"] - return send_file(file_path, mimetype='image/png') + persisted = persist_screenshot(result["file_path"]) + save_capture(url, title, action, backend_used, "success", file_path=persisted) + return send_file(persisted, mimetype='image/png') elif action == "html": + save_capture(url, title, action, backend_used, "success", content=result["html"]) + return jsonify(result) + + elif action == "text": + save_capture(url, title, action, backend_used, "success", content=result["text"]) return jsonify(result) diff --git a/templates/index.html b/templates/index.html index 680b78d..9d04f5b 100644 --- a/templates/index.html +++ b/templates/index.html @@ -3,7 +3,7 @@ - Web Capture - 网页截图与代码提取 + Web Capture - 网页截图/HTML/文本提取

🌐 Web Capture

-

网页截图与代码提取服务

+

网页截图 / HTML / 可读文本 提取服务

@@ -281,6 +587,10 @@ 📄 提取HTML +
@@ -350,25 +660,87 @@ +
+
+

📜 提取历史

+
+ + + + +
+
+ +
+ +
+ + + + + + + + + + + + +
时间类型网址标题状态操作
+
+ + + + +
+

📚 使用说明

重要提示：

    +
  • 📝 提取文本：自动剔除脚本/样式/标签，得到干净的前端可读文本，适合直接用于分析/喂给大模型
  • ⏱️ 页面加载等待：某些网站需要验证过程（如 Cloudflare），请设置较长时间（10000-30000ms）
  • 🔄 滚动次数：用于加载动态内容（如微博、推特等），建议 3-5 次
  • 🎯 全页截图：滚动加载后建议开启此选项
    • +
  • 📜 提取历史：每次提取自动入库，支持分页浏览、按类型筛选、搜索与删除

API 调用：

POST /api/capture
 {
   "url": "https://example.com",
-  "action": "screenshot",  // 或 "html"
-  "wait_time": 15000,     // 重要！验证网站需设置较长等待
-  "scroll_times": 3,      // 可选：加载动态内容
-  "scroll_delay": 1000,  // 可选：滚动间隔
-  "full_page": true,     // 可选：全页截图
-  "backend": "playwright" // 可选：playwright 或 agent-browser
-}
+ "action": "text", // "screenshot" | "html" | "text" + "wait_time": 15000, // 重要！验证网站需设置较长等待 + "scroll_times": 3, // 可选：加载动态内容 + "scroll_delay": 1000, // 可选：滚动间隔 + "full_page": true, // 可选：全页截图 + "backend": "playwright" // 可选：playwright / agent-browser +} + +GET /api/history?page=1&page_size=15&action=all&search=关键词 // 历史分页 +GET /api/history/<id> // 历史详情 +GET /api/history/<id>/file // 历史截图文件 +DELETE /api/history/<id> // 删除历史 +
+ + + + @@ -380,6 +752,12 @@ const submitBtn = document.getElementById('submitBtn'); let currentData = null; let currentBlob = null; + let currentPage = 1; + let currentHistory = { page: 1, total: 0, total_pages: 1 }; + let modalRecord = null; + + // 页面加载时读取历史 + window.addEventListener('load', () => loadHistory(1)); form.addEventListener('submit', async (e) => { e.preventDefault(); @@ -429,16 +807,26 @@ if (action === 'screenshot') { currentBlob = await response.blob(); const imageUrl = URL.createObjectURL(currentBlob); - document.getElementById('resultContent').innerHTML = + document.getElementById('resultContent').innerHTML = `截图结果`; } else { const data = await response.json(); - currentBlob = new Blob([data.html], { type: 'text/html' }); - document.getElementById('resultContent').innerHTML = - `
${escapeHtml(data.html)}
`; + if (action === 'html') { + currentBlob = new Blob([data.html], { type: 'text/html' }); + document.getElementById('resultContent').innerHTML = + `
${escapeHtml(data.html)}
`; + } else { + currentBlob = new Blob([data.text], { type: 'text/plain;charset=utf-8' }); + const lines = (data.text || '').split('\n').length; + document.getElementById('resultContent').innerHTML = + `
页面标题：${escapeHtml(data.title || '无')} ｜ 共 ${lines} 行，${(data.text || '').length} 字符（已剔除标签与无效字符）
` + + `
${escapeHtml(data.text || '')}
`; + } } result.classList.add('active'); + // 捕获成功后刷新历史（回到第一页看最新） + loadHistory(1); } catch (err) { error.textContent = '❌ ' + err.message; error.classList.add('active'); @@ -460,7 +848,9 @@ const url = URL.createObjectURL(currentBlob); const a = document.createElement('a'); a.href = url; - a.download = currentData.action === 'screenshot' ? 'screenshot.png' : 'page.html'; + const name = currentData.action === 'screenshot' ? 'screenshot.png' + : currentData.action === 'html' ? 'page.html' : 'page.txt'; + a.download = name; a.click(); URL.revokeObjectURL(url); } @@ -483,6 +873,193 @@ currentData = null; currentBlob = null; } + + /* ===== 历史记录 ===== */ + async function loadHistory(page) { + const action = document.getElementById('historyActionFilter').value; + const search = document.getElementById('historySearch').value.trim(); + const qs = new URLSearchParams({ page: page, page_size: 15, action, search }); + const res = await fetch('/api/history?' + qs.toString()); + const data = await res.json(); + + if (!data.success) { + document.getElementById('historyEmpty').style.display = 'block'; + document.getElementById('historyEmpty').textContent = '❌ 加载历史失败：' + (data.error || '未知错误'); + return; + } + + currentPage = page; + currentHistory = data; + renderHistory(data); + } + + function refreshHistory() { + loadHistory(currentPage); + } + + function renderHistory(data) { + const body = document.getElementById('historyBody'); + const empty = document.getElementById('historyEmpty'); + const count = document.getElementById('historyCount'); + + count.textContent = `共 ${data.total} 条记录`; + + if (!data.records.length) { + body.innerHTML = ''; + empty.style.display = 'block'; + empty.textContent = '暂无提取记录，先去上方提取一个网页吧 🚀'; + renderPagination(data); + return; + } + empty.style.display = 'none'; + + const badges = { + screenshot: '📸 截图', + html: '📄 HTML', + text: '📝 文本' + }; + + body.innerHTML = data.records.map(r => { + const badgeClass = r.status === 'failed' ? 'badge-failed' : ('badge-' + r.action); + const badgeText = r.status === 'failed' ? '❌ 失败' : (badges[r.action] || r.action); + const statusText = r.status === 'failed' + ? `失败` + : '成功'; + return ` + ${escapeHtml(r.created_at)} + ${badgeText} + ${escapeHtml(r.url)} + ${escapeHtml(r.title || '—')} + ${statusText} + + + + + `; + }).join(''); + + renderPagination(data); + } + + function renderPagination(data) { + const el = document.getElementById('pagination'); + if (!data.records.length) { + el.innerHTML = ''; + return; + } + + let html = ''; + html += ``; + + const total = data.total_pages; + const cur = data.page; + let start = Math.max(1, cur - 2); + let end = Math.min(total, cur + 2); + if (end - start < 4) { + start = Math.max(1, end - 4); + end = Math.min(total, start + 4); + } + + if (start > 1) html += ``; + if (start > 2) html += `…`; + for (let p = start; p <= end; p++) { + html += ``; + } + if (end < total - 1) html += `…`; + if (end < total) html += ``; + + html += ``; + html += `第 ${data.page} / ${total} 页`; + + el.innerHTML = html; + } + + async function viewRecord(id) { + const res = await fetch('/api/history/' + id); + const data = await res.json(); + if (!data.success) { + alert('加载失败：' + (data.error || '未知错误')); + return; + } + modalRecord = data.record; + const r = modalRecord; + + document.getElementById('modalTitle').textContent = (r.title || r.url || '记录详情'); + const badgeClass = r.status === 'failed' ? 'badge-failed' : ('badge-' + r.action); + const badgeText = r.status === 'failed' ? '❌ 失败' : ({screenshot: '📸 截图', html: '📄 HTML', text: '📝 文本'}[r.action] || r.action); + document.getElementById('modalMeta').innerHTML = + `类型：${badgeText}` + + `时间：${escapeHtml(r.created_at)}` + + `后端：${escapeHtml(r.backend || '—')}` + + `状态：${r.status === 'failed' ? '失败' : '成功'}`; + + const body = document.getElementById('modalBody'); + if (r.status === 'failed') { + body.innerHTML = `
❌ 提取失败：${escapeHtml(r.error || '')}
`; + } else if (r.action === 'screenshot') { + body.innerHTML = `历史截图`; + } else if (r.action === 'html') { + body.innerHTML = `
${escapeHtml(r.content || '')}
`; + } else { + body.innerHTML = `
${escapeHtml(r.content || '')}
`; + } + + document.getElementById('modalOverlay').classList.add('active'); + } + + function closeModal() { + document.getElementById('modalOverlay').classList.remove('active'); + modalRecord = null; + } + + async function deleteRecord(id) { + if (!confirm('确定删除这条记录吗？截图文件将一并删除。')) return; + const res = await fetch('/api/history/' + id, { method: 'DELETE' }); + const data = await res.json(); + if (data.success) { + // 若当前页删空了则回退一页 + if (currentHistory.records.length === 1 && currentPage > 1) { + loadHistory(currentPage - 1); + } else { + loadHistory(currentPage); + } + } else { + alert('删除失败：' + (data.error || '未知错误')); + } + } + + function currentModalBlob() { + if (!modalRecord) return null; + if (modalRecord.action === 'screenshot') return null; // 图片用链接下载 + const type = modalRecord.action === 'html' ? 'text/html' : 'text/plain;charset=utf-8'; + return new Blob([modalRecord.content || ''], { type }); + } + + function downloadHistoryContent() { + if (!modalRecord) return; + if (modalRecord.action === 'screenshot') { + const a = document.createElement('a'); + a.href = `/api/history/${modalRecord.id}/file`; + a.download = `capture_${modalRecord.id}.png`; + a.click(); + return; + } + const blob = currentModalBlob(); + if (!blob) return; + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = modalRecord.action === 'html' ? `page_${modalRecord.id}.html` : `page_${modalRecord.id}.txt`; + a.click(); + URL.revokeObjectURL(url); + } + + function copyHistoryContent() { + if (!modalRecord || !modalRecord.content) return; + navigator.clipboard.writeText(modalRecord.content).then(() => { + alert('已复制到剪贴板'); + }); + } \ No newline at end of file