新增按需截图功能：滚动截图 + 视觉大模型实时判断
- 新 action=smart：每次截图后用视觉大模型判断主题内容是否已截全、是否还需向下滚动，自动滚动到判定完成，最后纵向拼接成长图
- 默认视觉模型 qwen3.6-plus（autodl），支持 agent-browser / playwright 双后端
- LLM 接口可网页配置：⚙️智能配置弹窗(base_url/api_key/model/提示词/最大滚动次数/滚动比例/超时) + 测试连接 + 持久化到 data/config.json
- 新增 API：GET/POST /api/smart/config、POST /api/smart/test
- 安全兜底：大模型判断失败保守停止、滚动位置不再变化判底部、最大滚动次数上限
- 历史新增 smart 类型：入库拼接长图+AI判断过程日志，详情弹窗可查看每一步截图判断
- 前端：智能模式提示条 + 结果展示拼接长图 + AI判断过程折叠查看 + 历史筛选/徽章/详情
This commit is contained in:
@@ -7,6 +7,7 @@ Web Capture API - 网页截图与代码提取服务
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import base64
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import shutil
|
||||
@@ -130,6 +131,123 @@ def clean_text(raw):
|
||||
|
||||
init_db()
|
||||
|
||||
# ===== 按需截图（AI智能判断滚动）配置 =====
|
||||
CONFIG_FILE = DATA_DIR / "config.json"
|
||||
|
||||
DEFAULT_SMART_CONFIG = {
|
||||
"base_url": "https://www.autodl.art/api/v1",
|
||||
"api_key": "F9MBfolzuapqTsD4KmUf9qen720rXvUZ3Sp3IrWiCTukqonx",
|
||||
"model": "qwen3.6-plus",
|
||||
"prompt": (
|
||||
"你是网页内容完整性判断助手。下面是一张网页滚动截图的当前视口画面。"
|
||||
"请判断：这个网页的主题内容（正文/主要内容）是否已经完整截取完成，是否还需要继续向下滚动？\n"
|
||||
"规则：\n"
|
||||
"1. 如果当前视口底部已经是页面底部（如页脚、版权信息、导航链接列表、没有更多正文），说明已完整 → complete=true\n"
|
||||
"2. 如果底部还有正文内容被截断、或还有更多正文需要滚动才能看到 → complete=false\n"
|
||||
"只输出JSON：{\"complete\": true/false, \"reason\": \"简短理由\"}"
|
||||
),
|
||||
"max_scrolls": 20,
|
||||
"scroll_ratio": 0.85, # 每次滚动视口高度的比例（重叠 15% 保证拼接连续）
|
||||
"timeout": 120
|
||||
}
|
||||
|
||||
|
||||
def load_smart_config():
|
||||
"""读取按需截图 LLM 配置（缺省回默认）"""
|
||||
cfg = json.loads(json.dumps(DEFAULT_SMART_CONFIG))
|
||||
if CONFIG_FILE.exists():
|
||||
try:
|
||||
saved = json.loads(CONFIG_FILE.read_text(encoding="utf-8"))
|
||||
if isinstance(saved, dict):
|
||||
for k in DEFAULT_SMART_CONFIG:
|
||||
if k in saved:
|
||||
cfg[k] = saved[k]
|
||||
except Exception:
|
||||
pass
|
||||
return cfg
|
||||
|
||||
|
||||
def save_smart_config(cfg):
|
||||
"""保存按需截图 LLM 配置"""
|
||||
merged = json.loads(json.dumps(DEFAULT_SMART_CONFIG))
|
||||
if isinstance(cfg, dict):
|
||||
for k in DEFAULT_SMART_CONFIG:
|
||||
if k in cfg and cfg[k] not in (None, ""):
|
||||
merged[k] = cfg[k]
|
||||
CONFIG_FILE.parent.mkdir(exist_ok=True)
|
||||
CONFIG_FILE.write_text(json.dumps(merged, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return merged
|
||||
|
||||
|
||||
def extract_json_from_content(content):
|
||||
"""从大模型输出中稳健提取 JSON 对象（兼容 ```json 代码块包裹、前后废话）"""
|
||||
if not content:
|
||||
return None
|
||||
text = str(content).strip()
|
||||
if text.startswith("```"):
|
||||
text = re.sub(r"^```[a-zA-Z]*\s*", "", text)
|
||||
text = re.sub(r"\s*```$", "", text)
|
||||
s, e = text.find("{"), text.rfind("}")
|
||||
if s != -1 and e > s:
|
||||
text = text[s:e + 1]
|
||||
try:
|
||||
return json.loads(text)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def llm_judge_screenshot(image_path, cfg):
|
||||
"""
|
||||
调用视觉大模型判断当前截图是否已覆盖主题内容、是否还需滚动
|
||||
返回: {"complete": bool, "reason": str}
|
||||
异常时抛 ValueError（调用方决定如何处置）
|
||||
"""
|
||||
import requests
|
||||
img_b64 = base64.b64encode(Path(image_path).read_bytes()).decode()
|
||||
url = cfg["base_url"].rstrip("/") + "/chat/completions"
|
||||
payload = {
|
||||
"model": cfg["model"],
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": cfg["prompt"]},
|
||||
{"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_b64}"}}
|
||||
]
|
||||
}],
|
||||
"max_tokens": 300
|
||||
}
|
||||
headers = {"Authorization": f"Bearer {cfg['api_key']}", "Content-Type": "application/json"}
|
||||
resp = requests.post(url, headers=headers, json=payload, timeout=int(cfg.get("timeout", 120)))
|
||||
if resp.status_code != 200:
|
||||
raise ValueError(f"LLM 接口返回 {resp.status_code}: {resp.text[:200]}")
|
||||
try:
|
||||
content = resp.json()["choices"][0]["message"]["content"]
|
||||
except Exception:
|
||||
raise ValueError(f"LLM 响应格式异常: {resp.text[:200]}")
|
||||
parsed = extract_json_from_content(content)
|
||||
if not parsed or "complete" not in parsed:
|
||||
raise ValueError(f"LLM 判断结果解析失败，原始输出: {content[:200]}")
|
||||
return {"complete": bool(parsed["complete"]), "reason": str(parsed.get("reason", ""))}
|
||||
|
||||
|
||||
def stitch_images(image_paths, out_path):
|
||||
"""把多张视口截图纵向拼接成一张长图（所有图对齐到最宽宽度）"""
|
||||
from PIL import Image
|
||||
images = [Image.open(p).convert("RGB") for p in image_paths]
|
||||
if not images:
|
||||
return None
|
||||
width = max(im.width for im in images)
|
||||
height = sum(im.height for im in images)
|
||||
canvas = Image.new("RGB", (width, height), "white")
|
||||
y = 0
|
||||
for im in images:
|
||||
canvas.paste(im, (0, y))
|
||||
y += im.height
|
||||
out_path.parent.mkdir(exist_ok=True, parents=True)
|
||||
canvas.save(out_path)
|
||||
return str(out_path)
|
||||
|
||||
|
||||
# agent-browser socket 目录
|
||||
os.environ.setdefault("AGENT_BROWSER_SOCKET_DIR", "/tmp/agent-browser-sockets")
|
||||
Path(os.environ["AGENT_BROWSER_SOCKET_DIR"]).mkdir(exist_ok=True, parents=True)
|
||||
@@ -546,6 +664,240 @@ async def capture_with_cdp(
|
||||
return {"success": False, "error": f"CDP connection failed: {str(e)}"}
|
||||
|
||||
|
||||
def _scroll_pixels(view_h, ratio):
|
||||
"""按视口高度与比例计算单次滚动像素（下限 50）"""
|
||||
return max(50, int(view_h * float(ratio)))
|
||||
|
||||
|
||||
def smart_capture_agent_browser(url, cfg, wait_time, viewport):
|
||||
"""agent-browser 后端：滚动截图 + 视觉大模型实时判断"""
|
||||
session = AgentBrowserSession()
|
||||
vw = viewport.get("width", 1280) if viewport else 1280
|
||||
vh = viewport.get("height", 700) if viewport else 700
|
||||
steps = []
|
||||
shot_paths = []
|
||||
max_scrolls = int(cfg.get("max_scrolls", 20))
|
||||
stop_reason = ""
|
||||
|
||||
try:
|
||||
session.set_viewport(vw, vh)
|
||||
success, _, err = session.open(url)
|
||||
if not success:
|
||||
return {"success": False, "error": f"打开网页失败: {err}"}
|
||||
session.wait(wait_time)
|
||||
|
||||
# 先回到顶部
|
||||
session.run(["eval", "window.scrollTo(0, 0)"], timeout=5000)
|
||||
session.wait(300)
|
||||
|
||||
for i in range(max_scrolls):
|
||||
step_no = i + 1
|
||||
shot = CAPTURE_DIR / f"smart_{session.session_id}_{step_no}.png"
|
||||
ok, _, serr = session.screenshot(shot)
|
||||
if not ok or not shot.exists():
|
||||
return {"success": False, "error": f"第 {step_no} 步截图失败: {serr}"}
|
||||
shot_paths.append(shot)
|
||||
|
||||
# 视觉大模型实时判断
|
||||
try:
|
||||
judge = llm_judge_screenshot(shot, cfg)
|
||||
steps.append({"step": step_no, "complete": judge["complete"], "reason": judge["reason"]})
|
||||
except ValueError as ve:
|
||||
# 判断失败：保守停止，保留已截内容
|
||||
steps.append({"step": step_no, "complete": True, "reason": f"⚠️ 大模型判断失败，停止滚动: {ve}"})
|
||||
stop_reason = str(ve)
|
||||
break
|
||||
|
||||
if judge["complete"]:
|
||||
break
|
||||
|
||||
# 记录滚动前位置，防止页面无法再滚动
|
||||
try:
|
||||
_, before, _ = session.run(["eval", "window.scrollY"], timeout=5000)
|
||||
before = float(session._decode_eval(before) or 0)
|
||||
except Exception:
|
||||
before = -1
|
||||
|
||||
session.scroll_down(_scroll_pixels(vh, cfg.get("scroll_ratio", 0.85)))
|
||||
session.wait(int(wait_time) if wait_time else 500)
|
||||
|
||||
if before >= 0:
|
||||
try:
|
||||
_, after_raw, _ = session.run(["eval", "window.scrollY"], timeout=5000)
|
||||
after = float(session._decode_eval(after_raw) or before)
|
||||
if after <= before:
|
||||
steps.append({"step": step_no, "complete": True, "reason": "已滚动到页面底部（滚动位置不再变化）"})
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
steps.append({"step": max_scrolls, "complete": True, "reason": "达到最大滚动次数上限，停止"})
|
||||
|
||||
# 拼接长图
|
||||
out = CAPTURE_DATA_DIR / f"smart_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}.png"
|
||||
stitched = stitch_images(shot_paths, out)
|
||||
title = session.get_title()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"title": title,
|
||||
"file_path": stitched,
|
||||
"steps": steps,
|
||||
"total_steps": len(shot_paths),
|
||||
"stop_reason": stop_reason,
|
||||
"backend": "agent-browser"
|
||||
}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
async def smart_capture_playwright(url, cfg, wait_time, viewport):
|
||||
"""playwright 后端：滚动截图 + 视觉大模型实时判断"""
|
||||
if not PLAYWRIGHT_AVAILABLE:
|
||||
return {"success": False, "error": "Playwright not installed"}
|
||||
vw = viewport.get("width", 1280) if viewport else 1280
|
||||
vh = viewport.get("height", 700) if viewport else 700
|
||||
steps = []
|
||||
shot_paths = []
|
||||
max_scrolls = int(cfg.get("max_scrolls", 20))
|
||||
stop_reason = ""
|
||||
|
||||
try:
|
||||
async with async_playwright() as p:
|
||||
try:
|
||||
browser = await p.chromium.launch(
|
||||
headless=True, channel='chrome',
|
||||
args=['--disable-blink-features=AutomationControlled', '--disable-dev-shm-usage',
|
||||
'--no-sandbox', '--ignore-certificate-errors']
|
||||
)
|
||||
except Exception:
|
||||
browser = await p.chromium.launch(
|
||||
headless=True,
|
||||
args=['--disable-blink-features=AutomationControlled', '--disable-dev-shm-usage',
|
||||
'--no-sandbox', '--ignore-certificate-errors']
|
||||
)
|
||||
context = await browser.new_context(
|
||||
viewport={"width": vw, "height": vh},
|
||||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.6099.109 Safari/537.36'
|
||||
)
|
||||
page = await context.new_page()
|
||||
if STEALTH_AVAILABLE:
|
||||
try:
|
||||
from playwright_stealth import stealth_async
|
||||
await stealth_async(page)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||
except Exception:
|
||||
try:
|
||||
await page.goto(url, wait_until="commit", timeout=30000)
|
||||
except Exception:
|
||||
pass
|
||||
if wait_time > 0:
|
||||
await page.wait_for_timeout(wait_time)
|
||||
try:
|
||||
await page.wait_for_load_state("networkidle", timeout=5000)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
await page.evaluate("window.scrollTo(0, 0)")
|
||||
await page.wait_for_timeout(300)
|
||||
except Exception:
|
||||
# 页面导航导致上下文销毁，等稳定后再试
|
||||
try:
|
||||
await page.wait_for_timeout(800)
|
||||
await page.evaluate("window.scrollTo(0, 0)")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def _safe_scroll_y():
|
||||
try:
|
||||
return await page.evaluate("window.scrollY")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
for i in range(max_scrolls):
|
||||
step_no = i + 1
|
||||
shot = CAPTURE_DIR / f"smart_pw_{uuid.uuid4().hex[:8]}_{step_no}.png"
|
||||
try:
|
||||
await page.screenshot(path=str(shot))
|
||||
except Exception:
|
||||
try:
|
||||
await page.wait_for_timeout(800)
|
||||
await page.screenshot(path=str(shot))
|
||||
except Exception as e:
|
||||
return {"success": False, "error": f"第 {step_no} 步截图失败: {e}"}
|
||||
shot_paths.append(shot)
|
||||
|
||||
try:
|
||||
judge = llm_judge_screenshot(shot, cfg)
|
||||
steps.append({"step": step_no, "complete": judge["complete"], "reason": judge["reason"]})
|
||||
except ValueError as ve:
|
||||
steps.append({"step": step_no, "complete": True, "reason": f"⚠️ 大模型判断失败，停止滚动: {ve}"})
|
||||
stop_reason = str(ve)
|
||||
break
|
||||
|
||||
if judge["complete"]:
|
||||
break
|
||||
|
||||
before = await _safe_scroll_y()
|
||||
try:
|
||||
await page.evaluate(f"window.scrollBy(0, {_scroll_pixels(vh, cfg.get('scroll_ratio', 0.85))})")
|
||||
await page.wait_for_timeout(wait_time if wait_time else 500)
|
||||
except Exception:
|
||||
# 滚动失败视为到页面底部
|
||||
steps.append({"step": step_no, "complete": True, "reason": "页面导航/滚动异常，视为已到末尾"})
|
||||
break
|
||||
after = await _safe_scroll_y()
|
||||
if before is not None and after is not None and after <= before:
|
||||
steps.append({"step": step_no, "complete": True, "reason": "已滚动到页面底部（滚动位置不再变化）"})
|
||||
break
|
||||
else:
|
||||
steps.append({"step": max_scrolls, "complete": True, "reason": "达到最大滚动次数上限，停止"})
|
||||
|
||||
out = CAPTURE_DATA_DIR / f"smart_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}.png"
|
||||
stitched = stitch_images(shot_paths, out)
|
||||
try:
|
||||
title = await page.title()
|
||||
except Exception:
|
||||
title = ""
|
||||
await browser.close()
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"title": title,
|
||||
"file_path": stitched,
|
||||
"steps": steps,
|
||||
"total_steps": len(shot_paths),
|
||||
"stop_reason": stop_reason,
|
||||
"backend": "playwright"
|
||||
}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
|
||||
def smart_capture(url, cfg, wait_time, viewport, backend="auto"):
|
||||
"""按需截图总入口：滚动截图 + 视觉大模型实时判断，返回拼接长图"""
|
||||
if backend == "auto":
|
||||
backend = "agent-browser" if AGENT_BROWSER_AVAILABLE else "playwright"
|
||||
|
||||
if backend == "agent-browser":
|
||||
return smart_capture_agent_browser(url, cfg, wait_time, viewport)
|
||||
elif backend == "playwright":
|
||||
try:
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
result = loop.run_until_complete(smart_capture_playwright(url, cfg, wait_time, viewport))
|
||||
loop.close()
|
||||
return result
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
return {"success": False, "error": f"Unknown backend: {backend}"}
|
||||
|
||||
|
||||
def capture_webpage(
|
||||
url: str,
|
||||
action: str = "screenshot",
|
||||
@@ -725,6 +1077,51 @@ def health():
|
||||
})
|
||||
|
||||
|
||||
@app.route('/api/smart/config', methods=['GET', 'POST'])
|
||||
def smart_config_endpoint():
|
||||
"""获取 / 保存按需截图 LLM 配置"""
|
||||
if request.method == 'GET':
|
||||
cfg = load_smart_config()
|
||||
return jsonify({"success": True, "config": cfg})
|
||||
data = request.get_json() or {}
|
||||
cfg = save_smart_config(data.get("config") or data)
|
||||
return jsonify({"success": True, "config": cfg})
|
||||
|
||||
|
||||
@app.route('/api/smart/test', methods=['POST'])
|
||||
def smart_test_endpoint():
|
||||
"""测试按需截图 LLM 接口连通性（发一条小文本消息验证 base_url/api_key/model）"""
|
||||
import requests
|
||||
data = request.get_json() or {}
|
||||
cfg = dict(load_smart_config())
|
||||
if data.get("config"):
|
||||
for k in DEFAULT_SMART_CONFIG:
|
||||
if k in data["config"] and data["config"][k] not in (None, ""):
|
||||
cfg[k] = data["config"][k]
|
||||
elif data.get("base_url") or data.get("api_key") or data.get("model"):
|
||||
for k in ("base_url", "api_key", "model"):
|
||||
if data.get(k):
|
||||
cfg[k] = data[k]
|
||||
|
||||
url = cfg["base_url"].rstrip("/") + "/chat/completions"
|
||||
payload = {"model": cfg["model"], "messages": [{"role": "user", "content": "ping"}], "max_tokens": 5}
|
||||
headers = {"Authorization": f"Bearer {cfg['api_key']}", "Content-Type": "application/json"}
|
||||
t0 = time.time()
|
||||
try:
|
||||
resp = requests.post(url, headers=headers, json=payload, timeout=int(cfg.get("timeout", 120)))
|
||||
cost = round(time.time() - t0, 2)
|
||||
if resp.status_code != 200:
|
||||
return jsonify({"success": False, "error": f"HTTP {resp.status_code}: {resp.text[:200]}", "latency": cost}), 400
|
||||
d = resp.json()
|
||||
try:
|
||||
reply = d["choices"][0]["message"]["content"][:80]
|
||||
except Exception:
|
||||
reply = "(无内容)"
|
||||
return jsonify({"success": True, "latency": cost, "model": cfg["model"], "reply": reply})
|
||||
except Exception as e:
|
||||
return jsonify({"success": False, "error": str(e)}), 400
|
||||
|
||||
|
||||
@app.route('/api/capture', methods=['POST'])
|
||||
def capture():
|
||||
"""
|
||||
@@ -733,14 +1130,15 @@ def capture():
|
||||
请求体:
|
||||
{
|
||||
"url": "https://example.com",
|
||||
"action": "screenshot" | "html",
|
||||
"action": "screenshot" | "html" | "text" | "smart",
|
||||
"scroll_times": 0,
|
||||
"scroll_delay": 1000,
|
||||
"full_page": false,
|
||||
"viewport": {"width": 1920, "height": 1080},
|
||||
"wait_time": 2000,
|
||||
"backend": "auto" | "agent-browser" | "playwright" | "chrome-cdp",
|
||||
"cdp_port": 9222 // 用于 chrome-cdp 后端，连接到已打开的 Chrome
|
||||
"cdp_port": 9222, // 用于 chrome-cdp 后端，连接到已打开的 Chrome
|
||||
"smart_config": {...} // 可选，按需截图时临时覆盖 LLM 配置
|
||||
}
|
||||
"""
|
||||
data = request.get_json()
|
||||
@@ -757,6 +1155,8 @@ def capture():
|
||||
url = "https://" + url
|
||||
|
||||
action = data.get("action", "screenshot")
|
||||
if action not in ("screenshot", "html", "text", "smart"):
|
||||
return jsonify({"success": False, "error": f"Unknown action: {action}（可选 screenshot/html/text/smart）"}), 400
|
||||
scroll_times = int(data.get("scroll_times", 0))
|
||||
scroll_delay = int(data.get("scroll_delay", 1000))
|
||||
full_page = bool(data.get("full_page", False))
|
||||
@@ -765,7 +1165,28 @@ def capture():
|
||||
backend = data.get("backend", "auto")
|
||||
cdp_port = int(data.get("cdp_port", 9222))
|
||||
url_hint = data.get("url_hint", "")
|
||||
|
||||
|
||||
# 按需截图：滚动截图 + 视觉大模型实时判断
|
||||
if action == "smart":
|
||||
smart_cfg = load_smart_config()
|
||||
override = data.get("smart_config") or {}
|
||||
if isinstance(override, dict):
|
||||
for k in DEFAULT_SMART_CONFIG:
|
||||
if k in override and override[k] not in (None, ""):
|
||||
smart_cfg[k] = override[k]
|
||||
result = smart_capture(url, smart_cfg, wait_time, viewport, backend)
|
||||
backend_used = result.get("backend", backend)
|
||||
title = result.get("title", "")
|
||||
if not result["success"]:
|
||||
save_capture(url, title, "smart", backend_used, "failed", error=result.get("error", ""))
|
||||
return jsonify(result), 400
|
||||
# 拼接长图已存持久化目录，直接入库
|
||||
steps_log = json.dumps(result.get("steps", []), ensure_ascii=False)
|
||||
rid = save_capture(url, title, "smart", backend_used, "success",
|
||||
file_path=result["file_path"], content=steps_log)
|
||||
result["history_id"] = rid
|
||||
return jsonify(result)
|
||||
|
||||
result = capture_webpage(
|
||||
url=url,
|
||||
action=action,
|
||||
|
||||
Reference in New Issue
Block a user