新增可读文本提取 + 历史记录分页管理
- 新增 text 动作：提取页面可读文本（innerText + 空行/行尾清理，剔除脚本样式标签），agent-browser 与 playwright 双后端支持，自动提取页面标题 - 提取历史持久化(SQLite WAL)：每次捕获自动入库，截图文件存持久化目录 - 新增 API：GET /api/history(分页/类型筛选/搜索)、GET /api/history/<id>(详情)、GET /api/history/<id>/file(截图)、DELETE /api/history/<id> - 前端：新增提取文本选项+干净文本展示；新增提取历史卡片（分页/筛选/搜索/查看弹窗/删除/下载/复制）
This commit is contained in:
@@ -5,8 +5,11 @@ Web Capture API - 网页截图与代码提取服务
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import uuid
|
||||
@@ -53,6 +56,80 @@ CORS(app)
|
||||
CAPTURE_DIR = Path(tempfile.gettempdir()) / "web_captures"
|
||||
CAPTURE_DIR.mkdir(exist_ok=True)
|
||||
|
||||
# 持久化存储（历史记录）
|
||||
PROJECT_DIR = Path(__file__).parent
|
||||
DATA_DIR = PROJECT_DIR / "data"
|
||||
CAPTURE_DATA_DIR = DATA_DIR / "captures"
|
||||
HISTORY_DB = DATA_DIR / "history.db"
|
||||
DATA_DIR.mkdir(exist_ok=True)
|
||||
CAPTURE_DATA_DIR.mkdir(exist_ok=True)
|
||||
|
||||
|
||||
def get_db():
|
||||
"""获取数据库连接（多进程安全：WAL + busy_timeout）"""
|
||||
conn = sqlite3.connect(HISTORY_DB, timeout=15)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA busy_timeout=15000")
|
||||
return conn
|
||||
|
||||
|
||||
def init_db():
|
||||
"""初始化历史记录表"""
|
||||
conn = get_db()
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS captures (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
url TEXT NOT NULL,
|
||||
title TEXT DEFAULT '',
|
||||
action TEXT NOT NULL,
|
||||
backend TEXT DEFAULT '',
|
||||
status TEXT NOT NULL DEFAULT 'success',
|
||||
file_path TEXT DEFAULT '',
|
||||
content TEXT DEFAULT '',
|
||||
error TEXT DEFAULT '',
|
||||
created_at TEXT NOT NULL
|
||||
)
|
||||
""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def save_capture(url, title, action, backend, status, file_path='', content='', error=''):
|
||||
"""保存一条提取历史记录"""
|
||||
conn = get_db()
|
||||
cur = conn.execute(
|
||||
"INSERT INTO captures (url, title, action, backend, status, file_path, content, error, created_at) "
|
||||
"VALUES (?,?,?,?,?,?,?,?,?)",
|
||||
(url, title, action, backend, status, file_path, content, error,
|
||||
datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
|
||||
)
|
||||
conn.commit()
|
||||
rid = cur.lastrowid
|
||||
conn.close()
|
||||
return rid
|
||||
|
||||
|
||||
def persist_screenshot(src_path):
|
||||
"""把临时截图复制到持久化目录，供历史记录长期访问"""
|
||||
name = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}.png"
|
||||
dest = CAPTURE_DATA_DIR / name
|
||||
shutil.copy2(src_path, dest)
|
||||
return str(dest)
|
||||
|
||||
|
||||
def clean_text(raw):
|
||||
"""清理可读文本：去掉行尾空格、压缩多余空行、去掉首尾空白"""
|
||||
if not raw:
|
||||
return ""
|
||||
lines = [ln.rstrip() for ln in raw.split("\n")]
|
||||
text = "\n".join(lines)
|
||||
text = re.sub(r"\n{3,}", "\n\n", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
init_db()
|
||||
|
||||
# agent-browser socket 目录
|
||||
os.environ.setdefault("AGENT_BROWSER_SOCKET_DIR", "/tmp/agent-browser-sockets")
|
||||
Path(os.environ["AGENT_BROWSER_SOCKET_DIR"]).mkdir(exist_ok=True, parents=True)
|
||||
@@ -111,6 +188,38 @@ class AgentBrowserSession:
|
||||
timeout=10000
|
||||
)
|
||||
return success, output, error
|
||||
|
||||
@staticmethod
|
||||
def _decode_eval(output):
|
||||
"""agent-browser eval 结果按 JSON 编码返回，这里解码回原始字符串"""
|
||||
if output is None:
|
||||
return ""
|
||||
out = output.strip()
|
||||
try:
|
||||
return json.loads(out)
|
||||
except Exception:
|
||||
# 非 JSON（如数字/布尔/纯文本），去掉可能的外层引号
|
||||
return out.strip('\"\'')
|
||||
|
||||
def get_title(self):
|
||||
"""获取页面标题"""
|
||||
success, output, error = self.run(
|
||||
["eval", "document.title"],
|
||||
timeout=10000
|
||||
)
|
||||
if success:
|
||||
return self._decode_eval(output)
|
||||
return ""
|
||||
|
||||
def get_text(self):
|
||||
"""提取页面可读文本（干净内容，剔除脚本/样式/标签）"""
|
||||
success, output, error = self.run(
|
||||
["eval", "document.body.innerText"],
|
||||
timeout=30000
|
||||
)
|
||||
if success:
|
||||
return True, self._decode_eval(output), ""
|
||||
return False, "", error
|
||||
|
||||
def scroll_down(self, pixels=800):
|
||||
"""向下滚动"""
|
||||
@@ -165,25 +274,35 @@ def capture_with_agent_browser(
|
||||
session.scroll_down(800)
|
||||
session.wait(scroll_delay)
|
||||
|
||||
# 提取页面标题
|
||||
title = session.get_title()
|
||||
|
||||
# 执行操作
|
||||
if action == "screenshot":
|
||||
temp_file = CAPTURE_DIR / f"{session.session_id}.png"
|
||||
success, _, error = session.screenshot(temp_file, full_page=full_page)
|
||||
|
||||
if success and temp_file.exists():
|
||||
return {"success": True, "file_path": str(temp_file)}
|
||||
return {"success": True, "title": title, "file_path": str(temp_file)}
|
||||
else:
|
||||
return {"success": False, "error": f"Screenshot failed: {error}"}
|
||||
return {"success": False, "title": title, "error": f"Screenshot failed: {error}"}
|
||||
|
||||
elif action == "html":
|
||||
success, html, error = session.get_html()
|
||||
if success:
|
||||
return {"success": True, "html": html}
|
||||
return {"success": True, "title": title, "html": html}
|
||||
else:
|
||||
return {"success": False, "error": f"Get HTML failed: {error}"}
|
||||
return {"success": False, "title": title, "error": f"Get HTML failed: {error}"}
|
||||
|
||||
elif action == "text":
|
||||
success, raw_text, error = session.get_text()
|
||||
if success:
|
||||
return {"success": True, "title": title, "text": clean_text(raw_text)}
|
||||
else:
|
||||
return {"success": False, "title": title, "error": f"Extract text failed: {error}"}
|
||||
|
||||
else:
|
||||
return {"success": False, "error": f"Unknown action: {action}"}
|
||||
return {"success": False, "title": title, "error": f"Unknown action: {action}"}
|
||||
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
@@ -343,19 +462,29 @@ async def capture_with_playwright(
|
||||
await page.wait_for_timeout(scroll_delay)
|
||||
await page.wait_for_timeout(500)
|
||||
|
||||
# 提取页面标题
|
||||
try:
|
||||
title = await page.title()
|
||||
except:
|
||||
title = ""
|
||||
|
||||
result = {}
|
||||
|
||||
if action == "screenshot":
|
||||
temp_file = CAPTURE_DIR / f"{session_id}.png"
|
||||
await page.screenshot(path=str(temp_file), full_page=full_page)
|
||||
result = {"success": True, "file_path": str(temp_file)}
|
||||
result = {"success": True, "title": title, "file_path": str(temp_file)}
|
||||
|
||||
elif action == "html":
|
||||
html = await page.content()
|
||||
result = {"success": True, "html": html}
|
||||
result = {"success": True, "title": title, "html": html}
|
||||
|
||||
elif action == "text":
|
||||
raw_text = await page.evaluate("document.body.innerText")
|
||||
result = {"success": True, "title": title, "text": clean_text(raw_text)}
|
||||
|
||||
else:
|
||||
result = {"success": False, "error": f"Unknown action: {action}"}
|
||||
result = {"success": False, "title": title, "error": f"Unknown action: {action}"}
|
||||
|
||||
await browser.close()
|
||||
return result
|
||||
@@ -446,9 +575,11 @@ def capture_webpage(
|
||||
if backend == "agent-browser":
|
||||
if not AGENT_BROWSER_AVAILABLE:
|
||||
return {"success": False, "error": "agent-browser not available"}
|
||||
return capture_with_agent_browser(
|
||||
result = capture_with_agent_browser(
|
||||
url, action, scroll_times, scroll_delay, full_page, viewport, wait_time
|
||||
)
|
||||
result["backend"] = "agent-browser"
|
||||
return result
|
||||
|
||||
elif backend == "playwright":
|
||||
if not PLAYWRIGHT_AVAILABLE:
|
||||
@@ -463,6 +594,7 @@ def capture_webpage(
|
||||
)
|
||||
)
|
||||
loop.close()
|
||||
result["backend"] = "playwright"
|
||||
return result
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
@@ -482,18 +614,105 @@ def api_info():
|
||||
"""API信息"""
|
||||
return jsonify({
|
||||
"service": "Web Capture API",
|
||||
"version": "2.0.0",
|
||||
"version": "2.1.0",
|
||||
"backends": {
|
||||
"agent-browser": "available" if AGENT_BROWSER_AVAILABLE else "unavailable",
|
||||
"playwright": "available" if PLAYWRIGHT_AVAILABLE else "unavailable"
|
||||
},
|
||||
"endpoints": {
|
||||
"/api/capture": "POST - Capture webpage screenshot or HTML",
|
||||
"/api/capture": "POST - Capture webpage (screenshot/html/text) + 自动入库历史",
|
||||
"/api/history": "GET - 历史记录分页列表 (?page&page_size&action&search)",
|
||||
"/api/history/<id>": "GET - 历史记录详情 / DELETE - 删除记录",
|
||||
"/api/history/<id>/file": "GET - 读取历史截图文件",
|
||||
"/health": "GET - Health check"
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
@app.route('/api/history', methods=['GET'])
|
||||
def history_list():
|
||||
"""
|
||||
历史记录分页列表
|
||||
参数: page(默认1), page_size(默认15, 最大100), action(筛选: all/screenshot/html/text), search(按url/title模糊搜索)
|
||||
"""
|
||||
page = max(1, int(request.args.get('page', 1)))
|
||||
page_size = min(100, max(1, int(request.args.get('page_size', 15))))
|
||||
action = request.args.get('action', 'all')
|
||||
search = request.args.get('search', '').strip()
|
||||
|
||||
where = []
|
||||
params = []
|
||||
if action and action != 'all':
|
||||
where.append("action = ?")
|
||||
params.append(action)
|
||||
if search:
|
||||
where.append("(url LIKE ? OR title LIKE ?)")
|
||||
params.append(f"%{search}%")
|
||||
params.append(f"%{search}%")
|
||||
where_sql = (" WHERE " + " AND ".join(where)) if where else ""
|
||||
|
||||
conn = get_db()
|
||||
total = conn.execute(f"SELECT COUNT(*) AS c FROM captures{where_sql}", params).fetchone()["c"]
|
||||
rows = conn.execute(
|
||||
f"SELECT id, url, title, action, backend, status, file_path, created_at, "
|
||||
f"LENGTH(content) AS content_size, substr(content, 1, 200) AS content_preview, error "
|
||||
f"FROM captures{where_sql} ORDER BY id DESC LIMIT ? OFFSET ?",
|
||||
params + [page_size, (page - 1) * page_size]
|
||||
).fetchall()
|
||||
conn.close()
|
||||
|
||||
records = [dict(r) for r in rows]
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
"total": total,
|
||||
"total_pages": (total + page_size - 1) // page_size if total else 1,
|
||||
"records": records
|
||||
})
|
||||
|
||||
|
||||
@app.route('/api/history/<int:rid>', methods=['GET'])
|
||||
def history_detail(rid):
|
||||
"""历史记录详情（含完整内容）"""
|
||||
conn = get_db()
|
||||
r = conn.execute("SELECT * FROM captures WHERE id=?", (rid,)).fetchone()
|
||||
conn.close()
|
||||
if not r:
|
||||
return jsonify({"success": False, "error": "记录不存在"}), 404
|
||||
return jsonify({"success": True, "record": dict(r)})
|
||||
|
||||
|
||||
@app.route('/api/history/<int:rid>/file', methods=['GET'])
|
||||
def history_file(rid):
|
||||
"""读取历史截图文件"""
|
||||
conn = get_db()
|
||||
r = conn.execute("SELECT file_path FROM captures WHERE id=?", (rid,)).fetchone()
|
||||
conn.close()
|
||||
if not r or not r["file_path"] or not Path(r["file_path"]).exists():
|
||||
return jsonify({"success": False, "error": "文件不存在或已删除"}), 404
|
||||
return send_file(r["file_path"], mimetype='image/png')
|
||||
|
||||
|
||||
@app.route('/api/history/<int:rid>', methods=['DELETE'])
|
||||
def history_delete(rid):
|
||||
"""删除历史记录（连带删除截图文件）"""
|
||||
conn = get_db()
|
||||
r = conn.execute("SELECT file_path FROM captures WHERE id=?", (rid,)).fetchone()
|
||||
if r:
|
||||
conn.execute("DELETE FROM captures WHERE id=?", (rid,))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
if r and r["file_path"]:
|
||||
fp = Path(r["file_path"])
|
||||
if fp.exists() and str(fp).startswith(str(CAPTURE_DATA_DIR)):
|
||||
try:
|
||||
fp.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
return jsonify({"success": True})
|
||||
|
||||
|
||||
@app.route('/health')
|
||||
def health():
|
||||
"""健康检查"""
|
||||
@@ -555,19 +774,27 @@ def capture():
|
||||
full_page=full_page,
|
||||
viewport=viewport,
|
||||
wait_time=wait_time,
|
||||
backend=backend,
|
||||
|
||||
|
||||
backend=backend
|
||||
)
|
||||
|
||||
backend_used = result.get("backend", backend)
|
||||
title = result.get("title", "")
|
||||
|
||||
if not result["success"]:
|
||||
save_capture(url, title, action, backend_used, "failed", error=result.get("error", ""))
|
||||
return jsonify(result), 400
|
||||
|
||||
if action == "screenshot":
|
||||
file_path = result["file_path"]
|
||||
return send_file(file_path, mimetype='image/png')
|
||||
persisted = persist_screenshot(result["file_path"])
|
||||
save_capture(url, title, action, backend_used, "success", file_path=persisted)
|
||||
return send_file(persisted, mimetype='image/png')
|
||||
|
||||
elif action == "html":
|
||||
save_capture(url, title, action, backend_used, "success", content=result["html"])
|
||||
return jsonify(result)
|
||||
|
||||
elif action == "text":
|
||||
save_capture(url, title, action, backend_used, "success", content=result["text"])
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user