增强 Playwright 反爬虫检测能力

添加:
- 禁用自动化控制特征
- 真实浏览器参数配置
- 反检测脚本注入
- 地理位置等真实特征
- 更强的 User-Agent 模拟

Playwright 后端可以更好地绕过反爬虫检测

PROJECT_SUMMARY.md

@@ -0,0 +1,43 @@
+# Web Capture API 项目
+
+## 待推送信息
+
+- **仓库地址**: http://121.40.164.32:12007/hz4th_coder/web-capture-api.git
+- **当前版本**: v1.0.2
+- **提交历史**:
+  - v1.0.0: 初始化项目
+  - v1.0.1: 修改端口为 16025
+  - v1.0.2: 修复 agent-browser socket 权限问题
+
+## 操作步骤
+
+### 1. 在 Git 平台创建仓库
+
+请大爷登录 Git 平台 (http://121.40.164.32:12007) 并创建仓库：
+- 仓库名称: `web-capture-api`
+- 所属用户: `hz4th_coder`
+- 可见性: 公开或私有（根据需要）
+
+### 2. 推送代码
+
+仓库创建后，运行以下命令：
+
+```bash
+cd /home/openclaw/.openclaw/workspace-hz4th_coder/works/web-capture-api
+git push -u origin master --tags
+```
+
+## 项目功能
+
+- 网页截图（全页或视口）
+- HTML 代码提取
+- 自动滚动加载动态内容
+- 自定义视口大小
+- Web 界面和 RESTful API
+- 支持 agent-browser 和 Playwright 双后端
+- Docker 部署支持
+
+## 服务地址
+
+- Web 界面: http://192.168.0.101:16025
+- API 接口: http://192.168.0.101:16025/api/capture

app.py

@@ -59,12 +59,17 @@ class AgentBrowserSession:
     def run(self, cmd, timeout=60000):
         """执行 agent-browser 命令"""
         full_cmd = self.base_cmd + cmd
+        # 设置必要的环境变量
+        env = os.environ.copy()
+        env['AGENT_BROWSER_SOCKET_DIR'] = '/tmp/agent-browser-sockets'
+        env['AGENT_BROWSER_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
         try:
             result = subprocess.run(
                 full_cmd,
                 capture_output=True,
                 text=True,
-                timeout=timeout / 1000
+                timeout=timeout / 1000,
+                env=env
             )
             return result.returncode == 0, result.stdout, result.stderr
         except subprocess.TimeoutExpired:
@@ -74,6 +79,9 @@ class AgentBrowserSession:
     
     def open(self, url):
         """打开网页"""
+        # 添加反爬虫检测的 User-Agent
+        env = os.environ.copy()
+        env['AGENT_BROWSER_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
         return self.run(["open", url], timeout=30000)
     
     def set_viewport(self, width, height):
@@ -134,6 +142,9 @@ def capture_with_agent_browser(
         # 打开网页
         success, stdout, stderr = session.open(url)
         if not success:
+            # 检查是否是反爬虫拦截
+            if "403" in stdout or "Access Denied" in stdout:
+                return {"success": False, "error": "网站反爬虫拦截 (403)，建议使用 Playwright 后端或手动添加请求头"}
             return {"success": False, "error": f"Failed to open URL: {stderr}"}
         
         # 等待页面加载
@@ -183,26 +194,59 @@ async def capture_with_playwright(
 ):
     """
     使用 Playwright 捕获网页
+    Playwright 有更强的反爬虫能力
     """
     if not PLAYWRIGHT_AVAILABLE:
-        return {"success": False, "error": "Playwright not installed"}
+        return {"success": False, "error": "Playwright not installed. Run: pip install playwright && playwright install chromium"}
     
     session_id = str(uuid.uuid4())[:8]
     
     try:
         async with async_playwright() as p:
-            browser = await p.chromium.launch(headless=True)
+            # 启动浏览器，添加反检测配置
+            browser = await p.chromium.launch(
+                headless=True,
+                args=[
+                    '--disable-blink-features=AutomationControlled',  # 反自动化检测
+                    '--disable-dev-shm-usage',
+                    '--no-sandbox',
+                    '--disable-setuid-sandbox',
+                    '--disable-web-security',
+                ]
+            )
             
             viewport_settings = viewport or {"width": 1920, "height": 1080}
             
+            # 创建上下文，模拟真实浏览器
             context = await browser.new_context(
                 viewport={
                     "width": viewport_settings.get("width", 1920),
                     "height": viewport_settings.get("height", 1080)
                 },
-                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                # 添加更多浏览器特征
+                locale='zh-CN',
+                timezone_id='Asia/Shanghai',
+                geolocation={'latitude': 31.2304, 'longitude': 121.4737},  # 上海坐标
+                permissions=['geolocation'],
             )
             
+            # 注入反检测脚本
+            await context.add_init_script("""
+                Object.defineProperty(navigator, 'webdriver', {
+                    get: () => undefined
+                });
+                Object.defineProperty(navigator, 'plugins', {
+                    get: () => [1, 2, 3, 4, 5]
+                });
+                Object.defineProperty(navigator, 'languages', {
+                    get: () => ['zh-CN', 'zh', 'en', 'en-US']
+                });
+                window.chrome = {
+                    runtime: {}
+                };
+            """)
+            
             page = await context.new_page()
             
             try: