2 Commits
Author SHA1 Message Date
hz4th_coder 2a9f187ce8 新增可读文本提取 + 历史记录分页管理
- 新增 text 动作:提取页面可读文本(innerText + 空行/行尾清理,剔除脚本样式标签),agent-browser 与 playwright 双后端支持,自动提取页面标题
- 提取历史持久化(SQLite WAL):每次捕获自动入库,截图文件存持久化目录
- 新增 API:GET /api/history(分页/类型筛选/搜索)、GET /api/history/<id>(详情)、GET /api/history/<id>/file(截图)、DELETE /api/history/<id>
- 前端:新增提取文本选项+干净文本展示;新增提取历史卡片(分页/筛选/搜索/查看弹窗/删除/下载/复制)
2026-08-29 17:37:43 +08:00
hz4th_coder f7f14d7114 恢复端口为16025 2026-07-07 13:23:36 +08:00
11 changed files with 863 additions and 56 deletions
+3
View File
@@ -31,6 +31,9 @@ captures/
*.png
*.html
# 运行时数据(历史记录数据库 + 截图文件)
data/
# 环境变量
.env
.env.local
+3 -3
View File
@@ -49,10 +49,10 @@ COPY . .
# 创建临时目录
RUN mkdir -p /tmp/web_captures
EXPOSE 16026
EXPOSE 16025
# 健康检查
HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
CMD curl -f http://localhost:16026/health || exit 1
CMD curl -f http://localhost:16025/health || exit 1
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:16026", "--timeout", "120", "app:app"]
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:16025", "--timeout", "120", "app:app"]
+3 -3
View File
@@ -6,7 +6,7 @@
- **当前版本**: v1.0.2
- **提交历史**:
- v1.0.0: 初始化项目
- v1.0.1: 修改端口为 16026
- v1.0.1: 修改端口为 16025
- v1.0.2: 修复 agent-browser socket 权限问题
## 操作步骤
@@ -39,5 +39,5 @@ git push -u origin master --tags
## 服务地址
- Web 界面: http://192.168.0.101:16026
- API 接口: http://192.168.0.101:16026/api/capture
- Web 界面: http://192.168.0.101:16025
- API 接口: http://192.168.0.101:16025/api/capture
+11 -11
View File
@@ -104,18 +104,18 @@ API 信息接口
```bash
# 基础截图
curl -X POST http://localhost:16026/api/capture \
curl -X POST http://localhost:16025/api/capture \
-H "Content-Type: application/json" \
-d '{"url": "https://example.com", "action": "screenshot"}' \
--output screenshot.png
# 提取HTML
curl -X POST http://localhost:16026/api/capture \
curl -X POST http://localhost:16025/api/capture \
-H "Content-Type: application/json" \
-d '{"url": "https://example.com", "action": "html"}'
# 滚动加载后全页截图
curl -X POST http://localhost:16026/api/capture \
curl -X POST http://localhost:16025/api/capture \
-H "Content-Type: application/json" \
-d '{
"url": "https://news.ycombinator.com",
@@ -127,7 +127,7 @@ curl -X POST http://localhost:16026/api/capture \
--output full_page.png
# 自定义视口
curl -X POST http://localhost:16026/api/capture \
curl -X POST http://localhost:16025/api/capture \
-H "Content-Type: application/json" \
-d '{
"url": "https://example.com",
@@ -144,7 +144,7 @@ import requests
# 截图
response = requests.post(
'http://localhost:16026/api/capture',
'http://localhost:16025/api/capture',
json={
'url': 'https://example.com',
'action': 'screenshot'
@@ -155,7 +155,7 @@ with open('screenshot.png', 'wb') as f:
# 提取HTML
response = requests.post(
'http://localhost:16026/api/capture',
'http://localhost:16025/api/capture',
json={
'url': 'https://example.com',
'action': 'html'
@@ -169,7 +169,7 @@ print(html)
```javascript
// 截图
const response = await fetch('http://localhost:16026/api/capture', {
const response = await fetch('http://localhost:16025/api/capture', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
@@ -182,7 +182,7 @@ const response = await fetch('http://localhost:16026/api/capture', {
const blob = await response.blob();
// 提取HTML
const response = await fetch('http://localhost:16026/api/capture', {
const response = await fetch('http://localhost:16025/api/capture', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({
@@ -199,7 +199,7 @@ console.log(data.html);
### 使用 Gunicorn
```bash
gunicorn -w 4 -b 0.0.0.0:16026 app:app
gunicorn -w 4 -b 0.0.0.0:16025 app:app
```
### 使用 Systemd 服务
@@ -230,12 +230,12 @@ RUN playwright install chromium --with-deps
COPY . .
EXPOSE 5000
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:16026", "app:app"]
CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:16025", "app:app"]
```
```bash
docker build -t web-capture-api .
docker run -p 16026:16026 web-capture-api
docker run -p 16025:16025 web-capture-api
```
## 性能优化
+245 -18
View File
@@ -5,8 +5,11 @@ Web Capture API - 网页截图与代码提取服务
"""
import os
import re
import json
import asyncio
import sqlite3
import shutil
import subprocess
import tempfile
import uuid
@@ -53,6 +56,80 @@ CORS(app)
CAPTURE_DIR = Path(tempfile.gettempdir()) / "web_captures"
CAPTURE_DIR.mkdir(exist_ok=True)
# 持久化存储(历史记录)
PROJECT_DIR = Path(__file__).parent
DATA_DIR = PROJECT_DIR / "data"
CAPTURE_DATA_DIR = DATA_DIR / "captures"
HISTORY_DB = DATA_DIR / "history.db"
DATA_DIR.mkdir(exist_ok=True)
CAPTURE_DATA_DIR.mkdir(exist_ok=True)
def get_db():
"""获取数据库连接(多进程安全:WAL + busy_timeout"""
conn = sqlite3.connect(HISTORY_DB, timeout=15)
conn.row_factory = sqlite3.Row
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA busy_timeout=15000")
return conn
def init_db():
"""初始化历史记录表"""
conn = get_db()
conn.execute("""
CREATE TABLE IF NOT EXISTS captures (
id INTEGER PRIMARY KEY AUTOINCREMENT,
url TEXT NOT NULL,
title TEXT DEFAULT '',
action TEXT NOT NULL,
backend TEXT DEFAULT '',
status TEXT NOT NULL DEFAULT 'success',
file_path TEXT DEFAULT '',
content TEXT DEFAULT '',
error TEXT DEFAULT '',
created_at TEXT NOT NULL
)
""")
conn.commit()
conn.close()
def save_capture(url, title, action, backend, status, file_path='', content='', error=''):
"""保存一条提取历史记录"""
conn = get_db()
cur = conn.execute(
"INSERT INTO captures (url, title, action, backend, status, file_path, content, error, created_at) "
"VALUES (?,?,?,?,?,?,?,?,?)",
(url, title, action, backend, status, file_path, content, error,
datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
)
conn.commit()
rid = cur.lastrowid
conn.close()
return rid
def persist_screenshot(src_path):
"""把临时截图复制到持久化目录,供历史记录长期访问"""
name = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}.png"
dest = CAPTURE_DATA_DIR / name
shutil.copy2(src_path, dest)
return str(dest)
def clean_text(raw):
"""清理可读文本:去掉行尾空格、压缩多余空行、去掉首尾空白"""
if not raw:
return ""
lines = [ln.rstrip() for ln in raw.split("\n")]
text = "\n".join(lines)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
init_db()
# agent-browser socket 目录
os.environ.setdefault("AGENT_BROWSER_SOCKET_DIR", "/tmp/agent-browser-sockets")
Path(os.environ["AGENT_BROWSER_SOCKET_DIR"]).mkdir(exist_ok=True, parents=True)
@@ -111,6 +188,38 @@ class AgentBrowserSession:
timeout=10000
)
return success, output, error
@staticmethod
def _decode_eval(output):
"""agent-browser eval 结果按 JSON 编码返回,这里解码回原始字符串"""
if output is None:
return ""
out = output.strip()
try:
return json.loads(out)
except Exception:
# 非 JSON(如数字/布尔/纯文本),去掉可能的外层引号
return out.strip('\"\'')
def get_title(self):
"""获取页面标题"""
success, output, error = self.run(
["eval", "document.title"],
timeout=10000
)
if success:
return self._decode_eval(output)
return ""
def get_text(self):
"""提取页面可读文本(干净内容,剔除脚本/样式/标签)"""
success, output, error = self.run(
["eval", "document.body.innerText"],
timeout=30000
)
if success:
return True, self._decode_eval(output), ""
return False, "", error
def scroll_down(self, pixels=800):
"""向下滚动"""
@@ -165,25 +274,35 @@ def capture_with_agent_browser(
session.scroll_down(800)
session.wait(scroll_delay)
# 提取页面标题
title = session.get_title()
# 执行操作
if action == "screenshot":
temp_file = CAPTURE_DIR / f"{session.session_id}.png"
success, _, error = session.screenshot(temp_file, full_page=full_page)
if success and temp_file.exists():
return {"success": True, "file_path": str(temp_file)}
return {"success": True, "title": title, "file_path": str(temp_file)}
else:
return {"success": False, "error": f"Screenshot failed: {error}"}
return {"success": False, "title": title, "error": f"Screenshot failed: {error}"}
elif action == "html":
success, html, error = session.get_html()
if success:
return {"success": True, "html": html}
return {"success": True, "title": title, "html": html}
else:
return {"success": False, "error": f"Get HTML failed: {error}"}
return {"success": False, "title": title, "error": f"Get HTML failed: {error}"}
elif action == "text":
success, raw_text, error = session.get_text()
if success:
return {"success": True, "title": title, "text": clean_text(raw_text)}
else:
return {"success": False, "title": title, "error": f"Extract text failed: {error}"}
else:
return {"success": False, "error": f"Unknown action: {action}"}
return {"success": False, "title": title, "error": f"Unknown action: {action}"}
except Exception as e:
return {"success": False, "error": str(e)}
@@ -343,19 +462,29 @@ async def capture_with_playwright(
await page.wait_for_timeout(scroll_delay)
await page.wait_for_timeout(500)
# 提取页面标题
try:
title = await page.title()
except:
title = ""
result = {}
if action == "screenshot":
temp_file = CAPTURE_DIR / f"{session_id}.png"
await page.screenshot(path=str(temp_file), full_page=full_page)
result = {"success": True, "file_path": str(temp_file)}
result = {"success": True, "title": title, "file_path": str(temp_file)}
elif action == "html":
html = await page.content()
result = {"success": True, "html": html}
result = {"success": True, "title": title, "html": html}
elif action == "text":
raw_text = await page.evaluate("document.body.innerText")
result = {"success": True, "title": title, "text": clean_text(raw_text)}
else:
result = {"success": False, "error": f"Unknown action: {action}"}
result = {"success": False, "title": title, "error": f"Unknown action: {action}"}
await browser.close()
return result
@@ -446,9 +575,11 @@ def capture_webpage(
if backend == "agent-browser":
if not AGENT_BROWSER_AVAILABLE:
return {"success": False, "error": "agent-browser not available"}
return capture_with_agent_browser(
result = capture_with_agent_browser(
url, action, scroll_times, scroll_delay, full_page, viewport, wait_time
)
result["backend"] = "agent-browser"
return result
elif backend == "playwright":
if not PLAYWRIGHT_AVAILABLE:
@@ -463,6 +594,7 @@ def capture_webpage(
)
)
loop.close()
result["backend"] = "playwright"
return result
except Exception as e:
return {"success": False, "error": str(e)}
@@ -482,18 +614,105 @@ def api_info():
"""API信息"""
return jsonify({
"service": "Web Capture API",
"version": "2.0.0",
"version": "2.1.0",
"backends": {
"agent-browser": "available" if AGENT_BROWSER_AVAILABLE else "unavailable",
"playwright": "available" if PLAYWRIGHT_AVAILABLE else "unavailable"
},
"endpoints": {
"/api/capture": "POST - Capture webpage screenshot or HTML",
"/api/capture": "POST - Capture webpage (screenshot/html/text) + 自动入库历史",
"/api/history": "GET - 历史记录分页列表 (?page&page_size&action&search)",
"/api/history/<id>": "GET - 历史记录详情 / DELETE - 删除记录",
"/api/history/<id>/file": "GET - 读取历史截图文件",
"/health": "GET - Health check"
}
})
@app.route('/api/history', methods=['GET'])
def history_list():
"""
历史记录分页列表
参数: page(默认1), page_size(默认15, 最大100), action(筛选: all/screenshot/html/text), search(按url/title模糊搜索)
"""
page = max(1, int(request.args.get('page', 1)))
page_size = min(100, max(1, int(request.args.get('page_size', 15))))
action = request.args.get('action', 'all')
search = request.args.get('search', '').strip()
where = []
params = []
if action and action != 'all':
where.append("action = ?")
params.append(action)
if search:
where.append("(url LIKE ? OR title LIKE ?)")
params.append(f"%{search}%")
params.append(f"%{search}%")
where_sql = (" WHERE " + " AND ".join(where)) if where else ""
conn = get_db()
total = conn.execute(f"SELECT COUNT(*) AS c FROM captures{where_sql}", params).fetchone()["c"]
rows = conn.execute(
f"SELECT id, url, title, action, backend, status, file_path, created_at, "
f"LENGTH(content) AS content_size, substr(content, 1, 200) AS content_preview, error "
f"FROM captures{where_sql} ORDER BY id DESC LIMIT ? OFFSET ?",
params + [page_size, (page - 1) * page_size]
).fetchall()
conn.close()
records = [dict(r) for r in rows]
return jsonify({
"success": True,
"page": page,
"page_size": page_size,
"total": total,
"total_pages": (total + page_size - 1) // page_size if total else 1,
"records": records
})
@app.route('/api/history/<int:rid>', methods=['GET'])
def history_detail(rid):
"""历史记录详情(含完整内容)"""
conn = get_db()
r = conn.execute("SELECT * FROM captures WHERE id=?", (rid,)).fetchone()
conn.close()
if not r:
return jsonify({"success": False, "error": "记录不存在"}), 404
return jsonify({"success": True, "record": dict(r)})
@app.route('/api/history/<int:rid>/file', methods=['GET'])
def history_file(rid):
"""读取历史截图文件"""
conn = get_db()
r = conn.execute("SELECT file_path FROM captures WHERE id=?", (rid,)).fetchone()
conn.close()
if not r or not r["file_path"] or not Path(r["file_path"]).exists():
return jsonify({"success": False, "error": "文件不存在或已删除"}), 404
return send_file(r["file_path"], mimetype='image/png')
@app.route('/api/history/<int:rid>', methods=['DELETE'])
def history_delete(rid):
"""删除历史记录(连带删除截图文件)"""
conn = get_db()
r = conn.execute("SELECT file_path FROM captures WHERE id=?", (rid,)).fetchone()
if r:
conn.execute("DELETE FROM captures WHERE id=?", (rid,))
conn.commit()
conn.close()
if r and r["file_path"]:
fp = Path(r["file_path"])
if fp.exists() and str(fp).startswith(str(CAPTURE_DATA_DIR)):
try:
fp.unlink()
except Exception:
pass
return jsonify({"success": True})
@app.route('/health')
def health():
"""健康检查"""
@@ -555,19 +774,27 @@ def capture():
full_page=full_page,
viewport=viewport,
wait_time=wait_time,
backend=backend,
backend=backend
)
backend_used = result.get("backend", backend)
title = result.get("title", "")
if not result["success"]:
save_capture(url, title, action, backend_used, "failed", error=result.get("error", ""))
return jsonify(result), 400
if action == "screenshot":
file_path = result["file_path"]
return send_file(file_path, mimetype='image/png')
persisted = persist_screenshot(result["file_path"])
save_capture(url, title, action, backend_used, "success", file_path=persisted)
return send_file(persisted, mimetype='image/png')
elif action == "html":
save_capture(url, title, action, backend_used, "success", content=result["html"])
return jsonify(result)
elif action == "text":
save_capture(url, title, action, backend_used, "success", content=result["text"])
return jsonify(result)
@@ -583,5 +810,5 @@ if __name__ == '__main__':
print(" agent-browser: npm install -g agent-browser && agent-browser install")
print(" playwright: pip install playwright && playwright install chromium")
print("\n🚀 Server running on http://0.0.0.0:16026")
app.run(host='0.0.0.0', port=16026, debug=True)
print("\n🚀 Server running on http://0.0.0.0:16025")
app.run(host='0.0.0.0', port=16025, debug=True)
+1 -1
View File
@@ -23,4 +23,4 @@ echo "运行方式:"
echo " 开发模式: python3.12 app.py"
echo " 生产模式: gunicorn -w 4 -b 0.0.0.0:5000 app:app"
echo ""
echo "访问地址: http://localhost:16026"
echo "访问地址: http://localhost:16025"
+2 -2
View File
@@ -6,14 +6,14 @@ services:
image: web-capture-api:latest
container_name: web-capture-api
ports:
- "16026:16026"
- "16025:16025"
environment:
- TZ=Asia/Shanghai
volumes:
- ./captures:/tmp/web_captures
restart: unless-stopped
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:16026/health"]
test: ["CMD", "curl", "-f", "http://localhost:16025/health"]
interval: 30s
timeout: 10s
retries: 3
+2 -2
View File
@@ -110,7 +110,7 @@ echo " cd $(pwd)"
echo " $PYTHON app.py"
echo ""
echo "📖 访问地址:"
echo " http://localhost:16026"
echo " http://localhost:16025"
echo ""
echo "📚 API 文档:"
echo " http://localhost:16026/api"
echo " http://localhost:16025/api"
+591 -14
View File
@@ -3,7 +3,7 @@
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Web Capture - 网页截图与代码提取</title>
<title>Web Capture - 网页截图/HTML/文本提取</title>
<style>
* {
margin: 0;
@@ -77,6 +77,7 @@
.radio-group {
display: flex;
gap: 20px;
flex-wrap: wrap;
}
.radio-label {
@@ -201,16 +202,62 @@
word-wrap: break-word;
}
.result-text {
background: #f8f9fa;
border: 1px solid #e0e0e0;
color: #222;
padding: 20px;
border-radius: 8px;
overflow: auto;
max-height: 600px;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
font-size: 14px;
line-height: 1.8;
white-space: pre-wrap;
word-wrap: break-word;
}
.actions {
margin-top: 20px;
display: flex;
gap: 10px;
flex-wrap: wrap;
}
.btn-secondary {
background: #6c757d;
padding: 10px 20px;
font-size: 14px;
width: auto;
margin-top: 0;
}
.btn-small {
background: #667eea;
color: white;
border: none;
padding: 6px 14px;
font-size: 13px;
border-radius: 6px;
cursor: pointer;
transition: background 0.2s;
}
.btn-small:hover {
background: #5568d8;
}
.btn-small.danger {
background: #dc3545;
}
.btn-small.danger:hover {
background: #c82333;
}
.btn-small:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.error {
@@ -227,6 +274,256 @@
display: block;
}
/* ===== 历史记录区域 ===== */
.history-header {
display: flex;
align-items: center;
justify-content: space-between;
flex-wrap: wrap;
gap: 15px;
margin-bottom: 20px;
}
.history-header h3 {
font-size: 1.4em;
color: #333;
}
.history-toolbar {
display: flex;
gap: 10px;
flex-wrap: wrap;
align-items: center;
}
.history-toolbar select,
.history-toolbar input[type="text"] {
width: auto;
min-width: 120px;
padding: 8px 12px;
font-size: 14px;
}
.history-toolbar .search-input {
min-width: 220px;
}
.history-count {
color: #666;
font-size: 14px;
}
.history-table {
width: 100%;
border-collapse: collapse;
}
.history-table th,
.history-table td {
padding: 12px 10px;
text-align: left;
border-bottom: 1px solid #eee;
vertical-align: middle;
}
.history-table th {
font-size: 13px;
color: #666;
font-weight: 600;
white-space: nowrap;
}
.history-table td {
font-size: 14px;
}
.history-table tr:hover td {
background: #f8f9ff;
}
.badge {
display: inline-block;
padding: 3px 10px;
border-radius: 20px;
font-size: 12px;
font-weight: 600;
white-space: nowrap;
}
.badge-screenshot {
background: #e3f2fd;
color: #1565c0;
}
.badge-html {
background: #fff3e0;
color: #e65100;
}
.badge-text {
background: #e8f5e9;
color: #2e7d32;
}
.badge-failed {
background: #fdecea;
color: #c62828;
}
.history-url {
max-width: 280px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
color: #1565c0;
text-decoration: none;
display: inline-block;
vertical-align: bottom;
}
.history-title {
max-width: 220px;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
color: #333;
display: inline-block;
vertical-align: bottom;
}
.history-empty {
text-align: center;
padding: 40px;
color: #999;
font-size: 15px;
}
.pagination {
display: flex;
align-items: center;
justify-content: center;
gap: 8px;
margin-top: 20px;
flex-wrap: wrap;
}
.page-btn {
min-width: 36px;
padding: 7px 12px;
border: 1px solid #ddd;
background: white;
border-radius: 6px;
cursor: pointer;
font-size: 14px;
color: #333;
transition: all 0.2s;
}
.page-btn:hover:not(:disabled) {
border-color: #667eea;
color: #667eea;
}
.page-btn.active {
background: #667eea;
color: white;
border-color: #667eea;
}
.page-btn:disabled {
opacity: 0.4;
cursor: not-allowed;
}
.page-info {
color: #666;
font-size: 14px;
white-space: nowrap;
}
/* ===== 查看详情弹窗 ===== */
.modal-overlay {
display: none;
position: fixed;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: rgba(0, 0, 0, 0.6);
z-index: 1000;
padding: 30px;
overflow-y: auto;
}
.modal-overlay.active {
display: flex;
}
.modal {
background: white;
border-radius: 14px;
width: 100%;
max-width: 900px;
margin: auto;
box-shadow: 0 20px 60px rgba(0, 0, 0, 0.4);
overflow: hidden;
}
.modal-header {
display: flex;
align-items: center;
justify-content: space-between;
padding: 18px 24px;
background: #f8f9fa;
border-bottom: 1px solid #eee;
gap: 15px;
flex-wrap: wrap;
}
.modal-header h3 {
font-size: 16px;
color: #333;
word-break: break-all;
}
.modal-meta {
padding: 10px 24px;
background: #f8f9fa;
border-bottom: 1px solid #eee;
font-size: 13px;
color: #666;
display: flex;
gap: 20px;
flex-wrap: wrap;
}
.modal-body {
padding: 20px 24px;
max-height: 70vh;
overflow: auto;
}
.modal-body .result-image {
width: 100%;
}
.modal-close {
background: #6c757d;
color: white;
border: none;
width: 32px;
height: 32px;
border-radius: 50%;
font-size: 18px;
cursor: pointer;
line-height: 1;
flex-shrink: 0;
}
.modal-close:hover {
background: #545b62;
}
.api-docs {
margin-top: 30px;
padding: 20px;
@@ -254,13 +551,22 @@
overflow-x: auto;
margin-top: 10px;
}
@media (max-width: 700px) {
.history-table th:nth-child(2),
.history-table td:nth-child(2),
.history-table th:nth-child(3),
.history-table td:nth-child(3) {
display: none;
}
}
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>🌐 Web Capture</h1>
<p>网页截图与代码提取服务</p>
<p>网页截图 / HTML / 可读文本 提取服务</p>
</div>
<div class="card">
@@ -281,6 +587,10 @@
<input type="radio" name="action" value="html">
📄 提取HTML
</label>
<label class="radio-label">
<input type="radio" name="action" value="text">
📝 提取文本
</label>
</div>
</div>
@@ -350,25 +660,87 @@
</div>
</div>
<div class="card">
<div class="history-header">
<h3>📜 提取历史</h3>
<div class="history-toolbar">
<select id="historyActionFilter" onchange="loadHistory(1)">
<option value="all">全部类型</option>
<option value="screenshot">📸 截图</option>
<option value="html">📄 HTML</option>
<option value="text">📝 文本</option>
</select>
<input type="text" class="search-input" id="historySearch" placeholder="🔍 搜索网址/标题..." onkeydown="if(event.key==='Enter')loadHistory(1)">
<button class="btn-small" onclick="loadHistory(1)">搜索</button>
<button class="btn-small" onclick="refreshHistory()">🔄 刷新</button>
</div>
</div>
<div class="history-count" id="historyCount"></div>
<div style="overflow-x: auto; margin-top: 10px;">
<table class="history-table">
<thead>
<tr>
<th>时间</th>
<th>类型</th>
<th>网址</th>
<th>标题</th>
<th>状态</th>
<th style="text-align: right;">操作</th>
</tr>
</thead>
<tbody id="historyBody"></tbody>
</table>
</div>
<div class="history-empty" id="historyEmpty" style="display: none;">暂无提取记录,先去上方提取一个网页吧 🚀</div>
<div class="pagination" id="pagination"></div>
</div>
<div class="card api-docs">
<h3>📚 使用说明</h3>
<p><strong>重要提示:</strong></p>
<ul style="margin: 10px 0; line-height: 1.8;">
<li>📝 <strong>提取文本</strong>:自动剔除脚本/样式/标签,得到干净的前端可读文本,适合直接用于分析/喂给大模型</li>
<li>⏱️ <strong>页面加载等待</strong>:某些网站需要验证过程(如 Cloudflare),请设置较长时间(10000-30000ms</li>
<li>🔄 <strong>滚动次数</strong>:用于加载动态内容(如微博、推特等),建议 3-5 次</li>
<li>🎯 <strong>全页截图</strong>:滚动加载后建议开启此选项</li>
<li>📜 <strong>提取历史</strong>:每次提取自动入库,支持分页浏览、按类型筛选、搜索与删除</li>
</ul>
<p style="margin-top: 15px;"><strong>API 调用:</strong></p>
<pre><code>POST /api/capture
{
"url": "https://example.com",
"action": "screenshot", // 或 "html"
"wait_time": 15000, // 重要!验证网站需设置较长等待
"scroll_times": 3, // 可选:加载动态内容
"scroll_delay": 1000, // 可选:滚动间隔
"full_page": true, // 可选:全页截图
"backend": "playwright" // 可选:playwright agent-browser
}</code></pre>
"action": "text", // "screenshot" | "html" | "text"
"wait_time": 15000, // 重要!验证网站需设置较长等待
"scroll_times": 3, // 可选:加载动态内容
"scroll_delay": 1000, // 可选:滚动间隔
"full_page": true, // 可选:全页截图
"backend": "playwright" // 可选:playwright / agent-browser
}
GET /api/history?page=1&page_size=15&action=all&search=关键词 // 历史分页
GET /api/history/&lt;id&gt; // 历史详情
GET /api/history/&lt;id&gt;/file // 历史截图文件
DELETE /api/history/&lt;id&gt; // 删除历史</code></pre>
</div>
</div>
<!-- 历史详情弹窗 -->
<div class="modal-overlay" id="modalOverlay" onclick="if(event.target===this)closeModal()">
<div class="modal">
<div class="modal-header">
<h3 id="modalTitle">记录详情</h3>
<button class="modal-close" onclick="closeModal()"></button>
</div>
<div class="modal-meta" id="modalMeta"></div>
<div class="modal-body" id="modalBody"></div>
<div class="actions" style="padding: 0 24px 20px;">
<button class="btn btn-secondary" onclick="downloadHistoryContent()">💾 下载内容</button>
<button class="btn btn-secondary" onclick="copyHistoryContent()">📋 复制内容</button>
</div>
</div>
</div>
@@ -380,6 +752,12 @@
const submitBtn = document.getElementById('submitBtn');
let currentData = null;
let currentBlob = null;
let currentPage = 1;
let currentHistory = { page: 1, total: 0, total_pages: 1 };
let modalRecord = null;
// 页面加载时读取历史
window.addEventListener('load', () => loadHistory(1));
form.addEventListener('submit', async (e) => {
e.preventDefault();
@@ -429,16 +807,26 @@
if (action === 'screenshot') {
currentBlob = await response.blob();
const imageUrl = URL.createObjectURL(currentBlob);
document.getElementById('resultContent').innerHTML =
document.getElementById('resultContent').innerHTML =
`<img src="${imageUrl}" class="result-image" alt="截图结果">`;
} else {
const data = await response.json();
currentBlob = new Blob([data.html], { type: 'text/html' });
document.getElementById('resultContent').innerHTML =
`<div class="result-code"><pre>${escapeHtml(data.html)}</pre></div>`;
if (action === 'html') {
currentBlob = new Blob([data.html], { type: 'text/html' });
document.getElementById('resultContent').innerHTML =
`<div class="result-code"><pre>${escapeHtml(data.html)}</pre></div>`;
} else {
currentBlob = new Blob([data.text], { type: 'text/plain;charset=utf-8' });
const lines = (data.text || '').split('\n').length;
document.getElementById('resultContent').innerHTML =
`<div style="margin-bottom:10px;color:#666;font-size:13px;">页面标题:${escapeHtml(data.title || '无')} ${lines} 行,${(data.text || '').length} 字符(已剔除标签与无效字符)</div>` +
`<div class="result-text">${escapeHtml(data.text || '')}</div>`;
}
}
result.classList.add('active');
// 捕获成功后刷新历史(回到第一页看最新)
loadHistory(1);
} catch (err) {
error.textContent = '❌ ' + err.message;
error.classList.add('active');
@@ -460,7 +848,9 @@
const url = URL.createObjectURL(currentBlob);
const a = document.createElement('a');
a.href = url;
a.download = currentData.action === 'screenshot' ? 'screenshot.png' : 'page.html';
const name = currentData.action === 'screenshot' ? 'screenshot.png'
: currentData.action === 'html' ? 'page.html' : 'page.txt';
a.download = name;
a.click();
URL.revokeObjectURL(url);
}
@@ -483,6 +873,193 @@
currentData = null;
currentBlob = null;
}
/* ===== 历史记录 ===== */
async function loadHistory(page) {
const action = document.getElementById('historyActionFilter').value;
const search = document.getElementById('historySearch').value.trim();
const qs = new URLSearchParams({ page: page, page_size: 15, action, search });
const res = await fetch('/api/history?' + qs.toString());
const data = await res.json();
if (!data.success) {
document.getElementById('historyEmpty').style.display = 'block';
document.getElementById('historyEmpty').textContent = '❌ 加载历史失败:' + (data.error || '未知错误');
return;
}
currentPage = page;
currentHistory = data;
renderHistory(data);
}
function refreshHistory() {
loadHistory(currentPage);
}
function renderHistory(data) {
const body = document.getElementById('historyBody');
const empty = document.getElementById('historyEmpty');
const count = document.getElementById('historyCount');
count.textContent = `${data.total} 条记录`;
if (!data.records.length) {
body.innerHTML = '';
empty.style.display = 'block';
empty.textContent = '暂无提取记录,先去上方提取一个网页吧 🚀';
renderPagination(data);
return;
}
empty.style.display = 'none';
const badges = {
screenshot: '📸 截图',
html: '📄 HTML',
text: '📝 文本'
};
body.innerHTML = data.records.map(r => {
const badgeClass = r.status === 'failed' ? 'badge-failed' : ('badge-' + r.action);
const badgeText = r.status === 'failed' ? '❌ 失败' : (badges[r.action] || r.action);
const statusText = r.status === 'failed'
? `<span style="color:#c62828;" title="${escapeHtml(r.error || '')}">失败</span>`
: '<span style="color:#2e7d32;">成功</span>';
return `<tr>
<td style="white-space:nowrap;">${escapeHtml(r.created_at)}</td>
<td><span class="badge ${escapeHtml(badgeClass)}">${badgeText}</span></td>
<td><a class="history-url" href="${escapeHtml(r.url)}" target="_blank" rel="noopener">${escapeHtml(r.url)}</a></td>
<td><span class="history-title" title="${escapeHtml(r.title || '')}">${escapeHtml(r.title || '—')}</span></td>
<td>${statusText}</td>
<td style="text-align:right; white-space:nowrap;">
<button class="btn-small" onclick="viewRecord(${r.id})">👁 查看</button>
<button class="btn-small danger" onclick="deleteRecord(${r.id})">🗑</button>
</td>
</tr>`;
}).join('');
renderPagination(data);
}
function renderPagination(data) {
const el = document.getElementById('pagination');
if (!data.records.length) {
el.innerHTML = '';
return;
}
let html = '';
html += `<button class="page-btn" onclick="loadHistory(${data.page - 1})" ${data.page <= 1 ? 'disabled' : ''}> 上一页</button>`;
const total = data.total_pages;
const cur = data.page;
let start = Math.max(1, cur - 2);
let end = Math.min(total, cur + 2);
if (end - start < 4) {
start = Math.max(1, end - 4);
end = Math.min(total, start + 4);
}
if (start > 1) html += `<button class="page-btn" onclick="loadHistory(1)">1</button>`;
if (start > 2) html += `<span style="color:#999;">…</span>`;
for (let p = start; p <= end; p++) {
html += `<button class="page-btn ${p === cur ? 'active' : ''}" onclick="loadHistory(${p})">${p}</button>`;
}
if (end < total - 1) html += `<span style="color:#999;">…</span>`;
if (end < total) html += `<button class="page-btn" onclick="loadHistory(${total})">${total}</button>`;
html += `<button class="page-btn" onclick="loadHistory(${data.page + 1})" ${data.page >= total ? 'disabled' : ''}>下一页 </button>`;
html += `<span class="page-info">第 ${data.page} / ${total} 页</span>`;
el.innerHTML = html;
}
async function viewRecord(id) {
const res = await fetch('/api/history/' + id);
const data = await res.json();
if (!data.success) {
alert('加载失败:' + (data.error || '未知错误'));
return;
}
modalRecord = data.record;
const r = modalRecord;
document.getElementById('modalTitle').textContent = (r.title || r.url || '记录详情');
const badgeClass = r.status === 'failed' ? 'badge-failed' : ('badge-' + r.action);
const badgeText = r.status === 'failed' ? '❌ 失败' : ({screenshot: '📸 截图', html: '📄 HTML', text: '📝 文本'}[r.action] || r.action);
document.getElementById('modalMeta').innerHTML =
`<span>类型:<span class="badge ${badgeClass}">${badgeText}</span></span>` +
`<span>时间:${escapeHtml(r.created_at)}</span>` +
`<span>后端:${escapeHtml(r.backend || '—')}</span>` +
`<span>状态:${r.status === 'failed' ? '<span style="color:#c62828;">失败</span>' : '<span style="color:#2e7d32;">成功</span>'}</span>`;
const body = document.getElementById('modalBody');
if (r.status === 'failed') {
body.innerHTML = `<div class="result-text" style="color:#c62828;">❌ 提取失败:${escapeHtml(r.error || '')}</div>`;
} else if (r.action === 'screenshot') {
body.innerHTML = `<img src="/api/history/${r.id}/file" class="result-image" alt="历史截图" onerror="this.parentElement.innerHTML='&lt;p style=&quot;color:#c62828;&quot;&gt;截图文件不存在或已删除&lt;/p&gt;'">`;
} else if (r.action === 'html') {
body.innerHTML = `<div class="result-code"><pre>${escapeHtml(r.content || '')}</pre></div>`;
} else {
body.innerHTML = `<div class="result-text">${escapeHtml(r.content || '')}</div>`;
}
document.getElementById('modalOverlay').classList.add('active');
}
function closeModal() {
document.getElementById('modalOverlay').classList.remove('active');
modalRecord = null;
}
async function deleteRecord(id) {
if (!confirm('确定删除这条记录吗?截图文件将一并删除。')) return;
const res = await fetch('/api/history/' + id, { method: 'DELETE' });
const data = await res.json();
if (data.success) {
// 若当前页删空了则回退一页
if (currentHistory.records.length === 1 && currentPage > 1) {
loadHistory(currentPage - 1);
} else {
loadHistory(currentPage);
}
} else {
alert('删除失败:' + (data.error || '未知错误'));
}
}
function currentModalBlob() {
if (!modalRecord) return null;
if (modalRecord.action === 'screenshot') return null; // 图片用链接下载
const type = modalRecord.action === 'html' ? 'text/html' : 'text/plain;charset=utf-8';
return new Blob([modalRecord.content || ''], { type });
}
function downloadHistoryContent() {
if (!modalRecord) return;
if (modalRecord.action === 'screenshot') {
const a = document.createElement('a');
a.href = `/api/history/${modalRecord.id}/file`;
a.download = `capture_${modalRecord.id}.png`;
a.click();
return;
}
const blob = currentModalBlob();
if (!blob) return;
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = modalRecord.action === 'html' ? `page_${modalRecord.id}.html` : `page_${modalRecord.id}.txt`;
a.click();
URL.revokeObjectURL(url);
}
function copyHistoryContent() {
if (!modalRecord || !modalRecord.content) return;
navigator.clipboard.writeText(modalRecord.content).then(() => {
alert('已复制到剪贴板');
});
}
</script>
</body>
</html>
+1 -1
View File
@@ -7,7 +7,7 @@ import requests
import json
import sys
API_URL = "http://localhost:16026"
API_URL = "http://localhost:16025"
def test_health():
+1 -1
View File
@@ -6,7 +6,7 @@ After=network.target
Type=simple
User=openclaw
WorkingDirectory=/home/openclaw/.openclaw/workspace-hz4th_coder/works/web-capture-api
ExecStart=/home/openclaw/.openclaw/workspace-hz4th_coder/works/web-capture-api/venv/bin/gunicorn -w 4 -b 0.0.0.0:16026 app:app
ExecStart=/home/openclaw/.openclaw/workspace-hz4th_coder/works/web-capture-api/venv/bin/gunicorn -w 4 -b 0.0.0.0:16025 app:app
Restart=always
RestartSec=10