From 9c63887e109fa1ac867d1b9876d6ee501cef9b78 Mon Sep 17 00:00:00 2001 From: hz4th_coder Date: Mon, 31 Aug 2026 13:12:22 +0800 Subject: [PATCH] =?UTF-8?q?v1.7.0=20=E5=8E=86=E5=8F=B2=E8=AE=B0=E5=BD=95?= =?UTF-8?q?=E6=A0=87=E6=B3=A8=E8=B0=83=E7=94=A8=E8=80=85/=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E6=96=B9=E5=BC=8F=20+=20=E6=AF=8F=E6=AC=A1=E6=8F=90?= =?UTF-8?q?=E5=8F=96=E8=87=AA=E5=8A=A8=E6=8C=89=E6=9C=88=E5=BD=92=E6=A1=A3?= =?UTF-8?q?=E5=8E=9F=E5=A7=8BHTML=20+=20=E4=BF=AE=E5=A4=8Dagent-browser=20?= =?UTF-8?q?HTML=20JSON=E8=BD=AC=E4=B9=89bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 历史记录新增 caller(调用者: 游客/项目名) 与 call_method(调用方式: web/api) * 请求体 caller 或请求头 X-Caller/X-Project/X-App 标识调用者, 默认游客 * 请求体 call_method 或 X-From 标识方式, 缺省按 Referer 自动判定(本前端=web, 否则api) * 前端提取请求带 X-From:web / X-Caller:游客; news-tracker 调用带 caller=news-tracker - 每次提取后自动保存最原始 HTML 到 data/html// 按月目录归档 * screenshot/html/text/smart 四种动作均顺带抓取原始HTML并落盘 * 历史详情新增 📄原始HTML文件 链接(GET /api/history//html), 删除记录连带删文件 - 修复: agent-browser 后端 html 提取返回 JSON 转义文本(非最原始HTML)的bug, 现解码为干净HTML --- app.py | 159 +++++++++++++++++++++++++++++++++++-------- templates/index.html | 38 +++++++++-- 2 files changed, 163 insertions(+), 34 deletions(-) diff --git a/app.py b/app.py index d32e9b9..812d73c 100644 --- a/app.py +++ b/app.py @@ -61,9 +61,11 @@ CAPTURE_DIR.mkdir(exist_ok=True) PROJECT_DIR = Path(__file__).parent DATA_DIR = PROJECT_DIR / "data" CAPTURE_DATA_DIR = DATA_DIR / "captures" +HTML_DATA_DIR = DATA_DIR / "html" # 原始HTML按月归档目录 HISTORY_DB = DATA_DIR / "history.db" DATA_DIR.mkdir(exist_ok=True) CAPTURE_DATA_DIR.mkdir(exist_ok=True) +HTML_DATA_DIR.mkdir(exist_ok=True) def get_db(): @@ -89,20 +91,31 @@ def init_db(): file_path TEXT DEFAULT '', content TEXT DEFAULT '', error TEXT DEFAULT '', + caller TEXT DEFAULT '游客', + call_method TEXT DEFAULT 'api', + html_path TEXT DEFAULT '', created_at TEXT NOT NULL ) """) + # 兼容旧库：缺列则 ALTER TABLE 补充 + cols = [row[1] for row in conn.execute("PRAGMA table_info(captures)").fetchall()] + for col, ddl in (("caller", "TEXT DEFAULT '游客'"), + ("call_method", "TEXT DEFAULT 'api'"), + ("html_path", "TEXT DEFAULT ''")): + if col not in cols: + conn.execute(f"ALTER TABLE captures ADD COLUMN {col} {ddl}") conn.commit() conn.close() -def save_capture(url, title, action, backend, status, file_path='', content='', error=''): - """保存一条提取历史记录""" +def save_capture(url, title, action, backend, status, file_path='', content='', error='', + caller='游客', call_method='api', html_path=''): + """保存一条提取历史记录（含调用者/调用方式/原始HTML路径）""" conn = get_db() cur = conn.execute( - "INSERT INTO captures (url, title, action, backend, status, file_path, content, error, created_at) " - "VALUES (?,?,?,?,?,?,?,?,?)", - (url, title, action, backend, status, file_path, content, error, + "INSERT INTO captures (url, title, action, backend, status, file_path, content, error, caller, call_method, html_path, created_at) " + "VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + (url, title, action, backend, status, file_path, content, error, caller, call_method, html_path, datetime.now().strftime("%Y-%m-%d %H:%M:%S")) ) conn.commit() @@ -111,6 +124,22 @@ def save_capture(url, title, action, backend, status, file_path='', content='', return rid +def save_raw_html(html, url=''): + """把最原始 HTML 保存到 data/html// 按月目录归档；失败返回 ''""" + if not html: + return '' + try: + month = datetime.now().strftime("%Y-%m") + month_dir = HTML_DATA_DIR / month + month_dir.mkdir(parents=True, exist_ok=True) + name = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}.html" + dest = month_dir / name + dest.write_text(html if isinstance(html, str) else str(html), encoding="utf-8") + return str(dest) + except Exception: + return '' + + def persist_screenshot(src_path): """把临时截图复制到持久化目录，供历史记录长期访问""" name = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}.png" @@ -401,21 +430,32 @@ def capture_with_agent_browser( success, _, error = session.screenshot(temp_file, full_page=full_page) if success and temp_file.exists(): - return {"success": True, "title": title, "file_path": str(temp_file)} + res = {"success": True, "title": title, "file_path": str(temp_file)} + # 顺带抓取最原始 HTML 存档 + ok, html_out, _ = session.get_html() + if ok and html_out: + res["html"] = session._decode_eval(html_out) + return res else: return {"success": False, "title": title, "error": f"Screenshot failed: {error}"} elif action == "html": success, html, error = session.get_html() if success: - return {"success": True, "title": title, "html": html} + # 修复：eval 结果按 JSON 编码返回，需解码成最原始 HTML + return {"success": True, "title": title, "html": session._decode_eval(html)} else: return {"success": False, "title": title, "error": f"Get HTML failed: {error}"} elif action == "text": success, raw_text, error = session.get_text() if success: - return {"success": True, "title": title, "text": clean_text(raw_text)} + res = {"success": True, "title": title, "text": clean_text(raw_text)} + # 顺带抓取最原始 HTML 存档 + ok, html_out, _ = session.get_html() + if ok and html_out: + res["html"] = session._decode_eval(html_out) + return res else: return {"success": False, "title": title, "error": f"Extract text failed: {error}"} @@ -592,6 +632,11 @@ async def capture_with_playwright( temp_file = CAPTURE_DIR / f"{session_id}.png" await page.screenshot(path=str(temp_file), full_page=full_page) result = {"success": True, "title": title, "file_path": str(temp_file)} + # 顺带抓取最原始 HTML 存档 + try: + result["html"] = await page.content() + except Exception: + pass elif action == "html": html = await page.content() @@ -600,6 +645,11 @@ async def capture_with_playwright( elif action == "text": raw_text = await page.evaluate("document.body.innerText") result = {"success": True, "title": title, "text": clean_text(raw_text)} + # 顺带抓取最原始 HTML 存档 + try: + result["html"] = await page.content() + except Exception: + pass else: result = {"success": False, "title": title, "error": f"Unknown action: {action}"} @@ -738,7 +788,7 @@ def smart_capture_agent_browser(url, cfg, wait_time, viewport): stitched = stitch_images(shot_paths, out) title = session.get_title() - return { + res = { "success": True, "title": title, "file_path": stitched, @@ -747,6 +797,11 @@ def smart_capture_agent_browser(url, cfg, wait_time, viewport): "stop_reason": stop_reason, "backend": "agent-browser" } + # 顺带抓取最原始 HTML 存档 + ok, html_out, _ = session.get_html() + if ok and html_out: + res["html"] = session._decode_eval(html_out) + return res except Exception as e: return {"success": False, "error": str(e)} finally: @@ -864,9 +919,7 @@ async def smart_capture_playwright(url, cfg, wait_time, viewport): title = await page.title() except Exception: title = "" - await browser.close() - - return { + res = { "success": True, "title": title, "file_path": stitched, @@ -875,6 +928,13 @@ async def smart_capture_playwright(url, cfg, wait_time, viewport): "stop_reason": stop_reason, "backend": "playwright" } + # 顺带抓取最原始 HTML 存档 + try: + res["html"] = await page.content() + except Exception: + pass + await browser.close() + return res except Exception as e: return {"success": False, "error": str(e)} @@ -972,10 +1032,11 @@ def api_info(): "playwright": "available" if PLAYWRIGHT_AVAILABLE else "unavailable" }, "endpoints": { - "/api/capture": "POST - Capture webpage (screenshot/html/text) + 自动入库历史", + "/api/capture": "POST - Capture webpage (screenshot/html/text/smart) + 自动入库历史(记录调用者/方式/原始HTML)", "/api/history": "GET - 历史记录分页列表 (?page&page_size&action&search)", "/api/history/": "GET - 历史记录详情 / DELETE - 删除记录", "/api/history//file": "GET - 读取历史截图文件", + "/api/history//html": "GET - 读取保存的原始HTML文件", "/health": "GET - Health check" } }) @@ -1006,7 +1067,7 @@ def history_list(): conn = get_db() total = conn.execute(f"SELECT COUNT(*) AS c FROM captures{where_sql}", params).fetchone()["c"] rows = conn.execute( - f"SELECT id, url, title, action, backend, status, file_path, created_at, " + f"SELECT id, url, title, action, backend, status, file_path, caller, call_method, html_path, created_at, " f"LENGTH(content) AS content_size, substr(content, 1, 200) AS content_preview, error " f"FROM captures{where_sql} ORDER BY id DESC LIMIT ? OFFSET ?", params + [page_size, (page - 1) * page_size] @@ -1046,22 +1107,36 @@ def history_file(rid): return send_file(r["file_path"], mimetype='image/png') +@app.route('/api/history//html', methods=['GET']) +def history_html_file(rid): + """读取保存的原始 HTML 文件（data/html/<月份>/xxx.html）""" + conn = get_db() + r = conn.execute("SELECT html_path FROM captures WHERE id=?", (rid,)).fetchone() + conn.close() + if not r or not r["html_path"] or not Path(r["html_path"]).exists(): + return jsonify({"success": False, "error": "原始HTML文件不存在或已删除"}), 404 + return send_file(r["html_path"], mimetype='text/html; charset=utf-8') + + @app.route('/api/history/', methods=['DELETE']) def history_delete(rid): - """删除历史记录（连带删除截图文件）""" + """删除历史记录（连带删除截图文件 / 原始HTML文件）""" conn = get_db() - r = conn.execute("SELECT file_path FROM captures WHERE id=?", (rid,)).fetchone() + r = conn.execute("SELECT file_path, html_path FROM captures WHERE id=?", (rid,)).fetchone() if r: conn.execute("DELETE FROM captures WHERE id=?", (rid,)) conn.commit() conn.close() - if r and r["file_path"]: - fp = Path(r["file_path"]) - if fp.exists() and str(fp).startswith(str(CAPTURE_DATA_DIR)): - try: - fp.unlink() - except Exception: - pass + if r: + for fp_s in (r["file_path"], r["html_path"]): + if not fp_s: + continue + fp = Path(fp_s) + if fp.exists() and str(fp).startswith(str(DATA_DIR)): + try: + fp.unlink() + except Exception: + pass return jsonify({"success": True}) @@ -1166,6 +1241,19 @@ def capture(): cdp_port = int(data.get("cdp_port", 9222)) url_hint = data.get("url_hint", "") + # ---- 调用者与调用方式识别 ---- + # 调用者：优先请求体 caller，其次请求头 X-Caller/X-Project/X-App，默认游客 + caller = (data.get("caller") or request.headers.get("X-Caller") + or request.headers.get("X-Project") or request.headers.get("X-App") + or "游客") + # 调用方式：请求体 call_method 或请求头 X-From；缺省按 Referer 判断（本前端=web，否则 api） + call_method = data.get("call_method") or request.headers.get("X-From") or "" + if not call_method: + ref = request.headers.get("Referer") or "" + call_method = "web" if (ref and request.host in ref) else "api" + elif call_method not in ("web", "api"): + call_method = "api" + # 按需截图：滚动截图 + 视觉大模型实时判断 if action == "smart": smart_cfg = load_smart_config() @@ -1178,13 +1266,17 @@ def capture(): backend_used = result.get("backend", backend) title = result.get("title", "") if not result["success"]: - save_capture(url, title, "smart", backend_used, "failed", error=result.get("error", "")) + save_capture(url, title, "smart", backend_used, "failed", error=result.get("error", ""), + caller=caller, call_method=call_method) return jsonify(result), 400 - # 拼接长图已存持久化目录，直接入库 + # 拼接长图已存持久化目录，直接入库；原始HTML按月归档 steps_log = json.dumps(result.get("steps", []), ensure_ascii=False) + html_path = save_raw_html(result.get("html", "")) rid = save_capture(url, title, "smart", backend_used, "success", - file_path=result["file_path"], content=steps_log) + file_path=result["file_path"], content=steps_log, + html_path=html_path, caller=caller, call_method=call_method) result["history_id"] = rid + result["raw_html_path"] = html_path return jsonify(result) result = capture_webpage( @@ -1202,20 +1294,27 @@ def capture(): title = result.get("title", "") if not result["success"]: - save_capture(url, title, action, backend_used, "failed", error=result.get("error", "")) + save_capture(url, title, action, backend_used, "failed", error=result.get("error", ""), + caller=caller, call_method=call_method) return jsonify(result), 400 + + # 每次提取后都把最原始 HTML 按月归档到本地 + html_path = save_raw_html(result.get("html", "")) if action == "screenshot": persisted = persist_screenshot(result["file_path"]) - save_capture(url, title, action, backend_used, "success", file_path=persisted) + save_capture(url, title, action, backend_used, "success", file_path=persisted, + html_path=html_path, caller=caller, call_method=call_method) return send_file(persisted, mimetype='image/png') elif action == "html": - save_capture(url, title, action, backend_used, "success", content=result["html"]) + save_capture(url, title, action, backend_used, "success", content=result["html"], + html_path=html_path, caller=caller, call_method=call_method) return jsonify(result) elif action == "text": - save_capture(url, title, action, backend_used, "success", content=result["text"]) + save_capture(url, title, action, backend_used, "success", content=result["text"], + html_path=html_path, caller=caller, call_method=call_method) return jsonify(result) diff --git a/templates/index.html b/templates/index.html index feb95a1..da17141 100644 --- a/templates/index.html +++ b/templates/index.html @@ -370,6 +370,16 @@ color: #c62828; } + .badge-api { + background: #ede7f6; + color: #5e35b1; + } + + .badge-web { + background: #e0f7fa; + color: #00695c; + } + .history-url { max-width: 280px; overflow: hidden; @@ -556,7 +566,9 @@ .history-table th:nth-child(2), .history-table td:nth-child(2), .history-table th:nth-child(3), - .history-table td:nth-child(3) { + .history-table td:nth-child(3), + .history-table th:nth-child(4), + .history-table td:nth-child(4) { display: none; } } @@ -693,6 +705,8 @@ 时间 类型 + 调用者 + 方式 网址 标题 状态 @@ -717,7 +731,8 @@
  • ⏱️ 页面加载等待：某些网站需要验证过程（如 Cloudflare），请设置较长时间（10000-30000ms）
  • 🔄 滚动次数：用于加载动态内容（如微博、推特等），建议 3-5 次
  • 🎯 全页截图：滚动加载后建议开启此选项
    • -
  • 📜 提取历史：每次提取自动入库，支持分页浏览、按类型筛选、搜索与删除
    • +
  • 📜 提取历史：每次提取自动入库，支持分页浏览、按类型筛选、搜索与删除；每条记录会标注调用者（游客/项目名）与调用方式（web/api），并自动把最原始 HTML 按月归档到 data/html/<月份>/（详情里可点「📄 原始HTML文件」查看）
    • +
  • 👤 API 调用方标识：请求体带 caller（或请求头 X-Caller/X-Project）即可在历史里显示项目名；带 call_method 或 X-From 标注调用方式，缺省按 Referer 自动判定 web/api

    • API 调用：

    POST /api/capture
@@ -729,6 +744,8 @@
   "scroll_delay": 1000,       // 可选：滚动间隔
   "full_page": true,          // 可选：全页截图
   "backend": "playwright",    // 可选：playwright / agent-browser
+  "caller": "news-tracker",     // 可选：调用者标识（历史记录里显示，默认游客）
+  "call_method": "api",        // 可选：调用方式 web/api（缺省自动判定）
   "viewport": {"width":1280,"height":700},
   "smart_config": {           // 可选：临时覆盖按需截图 LLM 配置
     "base_url": "https://www.autodl.art/api/v1",
@@ -957,7 +974,9 @@ DELETE /api/history/<id>        // 删除历史
    const response = await fetch('/api/capture', { method: 'POST', headers: { - 'Content-Type': 'application/json' + 'Content-Type': 'application/json', + 'X-From': 'web', + 'X-Caller': '游客' }, body: JSON.stringify(currentData) }); @@ -1111,9 +1130,14 @@ DELETE /api/history/<id> // 删除历史 const statusText = r.status === 'failed' ? `失败` : '成功'; + const callerText = r.caller || '游客'; + const methodText = r.call_method || 'api'; + const methodBadge = methodText === 'web' ? 'badge-web' : 'badge-api'; return ` ${escapeHtml(r.created_at)} ${badgeText} + ${escapeHtml(callerText)} + ${escapeHtml(methodText)} ${escapeHtml(r.url)} ${escapeHtml(r.title || '—')} ${statusText} @@ -1173,11 +1197,17 @@ DELETE /api/history/<id> // 删除历史 document.getElementById('modalTitle').textContent = (r.title || r.url || '记录详情'); const badgeClass = r.status === 'failed' ? 'badge-failed' : ('badge-' + r.action); const badgeText = r.status === 'failed' ? '❌ 失败' : ({screenshot: '📸 截图', html: '📄 HTML', text: '📝 文本'}[r.action] || r.action); + const htmlLink = r.html_path + ? `📄 原始HTML文件` + : ''; document.getElementById('modalMeta').innerHTML = `类型：${badgeText}` + `时间：${escapeHtml(r.created_at)}` + + `调用者：${escapeHtml(r.caller || '游客')}` + + `方式：${escapeHtml(r.call_method || 'api')}` + `后端：${escapeHtml(r.backend || '—')}` + - `状态：${r.status === 'failed' ? '失败' : '成功'}`; + `状态：${r.status === 'failed' ? '失败' : '成功'}` + + htmlLink; const body = document.getElementById('modalBody'); if (r.status === 'failed') {