增强 Playwright 反爬虫检测能力

添加: - 禁用自动化控制特征 - 真实浏览器参数配置 - 反检测脚本注入 - 地理位置等真实特征 - 更强的 User-Agent 模拟 Playwright 后端可以更好地绕过反爬虫检测
增强 agent-browser 反爬虫支持
2026-07-05 00:55:09 +08:00 · 2026-07-05 00:19:18 +08:00 · 2026-07-05 00:12:48 +08:00 · 2026-07-04 23:21:52 +08:00
9 changed files with 118 additions and 27 deletions
--- a/6
+++ b/6
@@ -49,10 +49,10 @@ COPY . .
 # 创建临时目录
 RUN mkdir -p /tmp/web_captures

-EXPOSE 5000
+EXPOSE 16025

 # 健康检查
 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
-    CMD curl -f http://localhost:5000/health || exit 1
+    CMD curl -f http://localhost:16025/health || exit 1

-CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "--timeout", "120", "app:app"]
+CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:16025", "--timeout", "120", "app:app"]
--- a/PROJECT_SUMMARY.md
+++ b/PROJECT_SUMMARY.md
@@ -0,0 +1,43 @@
+# Web Capture API 项目
+
+## 待推送信息
+
+- **仓库地址**: http://121.40.164.32:12007/hz4th_coder/web-capture-api.git
+- **当前版本**: v1.0.2
+- **提交历史**:
+  - v1.0.0: 初始化项目
+  - v1.0.1: 修改端口为 16025
+  - v1.0.2: 修复 agent-browser socket 权限问题
+
+## 操作步骤
+
+### 1. 在 Git 平台创建仓库
+
+请大爷登录 Git 平台 (http://121.40.164.32:12007) 并创建仓库：
+- 仓库名称: `web-capture-api`
+- 所属用户: `hz4th_coder`
+- 可见性: 公开或私有（根据需要）
+
+### 2. 推送代码
+
+仓库创建后，运行以下命令：
+
+```bash
+cd /home/openclaw/.openclaw/workspace-hz4th_coder/works/web-capture-api
+git push -u origin master --tags
+```
+
+## 项目功能
+
+- 网页截图（全页或视口）
+- HTML 代码提取
+- 自动滚动加载动态内容
+- 自定义视口大小
+- Web 界面和 RESTful API
+- 支持 agent-browser 和 Playwright 双后端
+- Docker 部署支持
+
+## 服务地址
+
+- Web 界面: http://192.168.0.101:16025
+- API 接口: http://192.168.0.101:16025/api/capture
--- a/README.md
+++ b/README.md
@@ -104,18 +104,18 @@ API 信息接口

 ```bash
 # 基础截图
-curl -X POST http://localhost:5000/api/capture \
+curl -X POST http://localhost:16025/api/capture \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com", "action": "screenshot"}' \
  --output screenshot.png

 # 提取HTML
-curl -X POST http://localhost:5000/api/capture \
+curl -X POST http://localhost:16025/api/capture \
  -H "Content-Type: application/json" \
  -d '{"url": "https://example.com", "action": "html"}'

 # 滚动加载后全页截图
-curl -X POST http://localhost:5000/api/capture \
+curl -X POST http://localhost:16025/api/capture \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://news.ycombinator.com",
@@ -127,7 +127,7 @@ curl -X POST http://localhost:5000/api/capture \
  --output full_page.png

 # 自定义视口
-curl -X POST http://localhost:5000/api/capture \
+curl -X POST http://localhost:16025/api/capture \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com",
@@ -144,7 +144,7 @@ import requests

 # 截图
 response = requests.post(
-    'http://localhost:5000/api/capture',
+    'http://localhost:16025/api/capture',
    json={
        'url': 'https://example.com',
        'action': 'screenshot'
@@ -155,7 +155,7 @@ with open('screenshot.png', 'wb') as f:

 # 提取HTML
 response = requests.post(
-    'http://localhost:5000/api/capture',
+    'http://localhost:16025/api/capture',
    json={
        'url': 'https://example.com',
        'action': 'html'
@@ -169,7 +169,7 @@ print(html)

 ```javascript
 // 截图
-const response = await fetch('http://localhost:5000/api/capture', {
+const response = await fetch('http://localhost:16025/api/capture', {
  method: 'POST',
  headers: {'Content-Type': 'application/json'},
  body: JSON.stringify({
@@ -182,7 +182,7 @@ const response = await fetch('http://localhost:5000/api/capture', {
 const blob = await response.blob();

 // 提取HTML
-const response = await fetch('http://localhost:5000/api/capture', {
+const response = await fetch('http://localhost:16025/api/capture', {
  method: 'POST',
  headers: {'Content-Type': 'application/json'},
  body: JSON.stringify({
@@ -199,7 +199,7 @@ console.log(data.html);
 ### 使用 Gunicorn

 ```bash
-gunicorn -w 4 -b 0.0.0.0:5000 app:app
+gunicorn -w 4 -b 0.0.0.0:16025 app:app
 ```

 ### 使用 Systemd 服务
@@ -230,12 +230,12 @@ RUN playwright install chromium --with-deps

 COPY . .
 EXPOSE 5000
-CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "app:app"]
+CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:16025", "app:app"]
 ```

 ```bash
 docker build -t web-capture-api .
-docker run -p 5000:5000 web-capture-api
+docker run -p 16025:16025 web-capture-api
 ```

 ## 性能优化
--- a/app.py
+++ b/app.py
@@ -44,6 +44,10 @@ CORS(app)
 CAPTURE_DIR = Path(tempfile.gettempdir()) / "web_captures"
 CAPTURE_DIR.mkdir(exist_ok=True)

+# agent-browser socket 目录
+os.environ.setdefault("AGENT_BROWSER_SOCKET_DIR", "/tmp/agent-browser-sockets")
+Path(os.environ["AGENT_BROWSER_SOCKET_DIR"]).mkdir(exist_ok=True, parents=True)
+

 class AgentBrowserSession:
    """agent-browser 会话管理器"""
@@ -55,12 +59,17 @@ class AgentBrowserSession:
    def run(self, cmd, timeout=60000):
        """执行 agent-browser 命令"""
        full_cmd = self.base_cmd + cmd
+        # 设置必要的环境变量
+        env = os.environ.copy()
+        env['AGENT_BROWSER_SOCKET_DIR'] = '/tmp/agent-browser-sockets'
+        env['AGENT_BROWSER_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
        try:
            result = subprocess.run(
                full_cmd,
                capture_output=True,
                text=True,
-                timeout=timeout / 1000
+                timeout=timeout / 1000,
+                env=env
            )
            return result.returncode == 0, result.stdout, result.stderr
        except subprocess.TimeoutExpired:
@@ -70,6 +79,9 @@ class AgentBrowserSession:
    
    def open(self, url):
        """打开网页"""
+        # 添加反爬虫检测的 User-Agent
+        env = os.environ.copy()
+        env['AGENT_BROWSER_USER_AGENT'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
        return self.run(["open", url], timeout=30000)
    
    def set_viewport(self, width, height):
@@ -130,6 +142,9 @@ def capture_with_agent_browser(
        # 打开网页
        success, stdout, stderr = session.open(url)
        if not success:
+            # 检查是否是反爬虫拦截
+            if "403" in stdout or "Access Denied" in stdout:
+                return {"success": False, "error": "网站反爬虫拦截 (403)，建议使用 Playwright 后端或手动添加请求头"}
            return {"success": False, "error": f"Failed to open URL: {stderr}"}
        
        # 等待页面加载
@@ -179,26 +194,59 @@ async def capture_with_playwright(
 ):
    """
    使用 Playwright 捕获网页
+    Playwright 有更强的反爬虫能力
    """
    if not PLAYWRIGHT_AVAILABLE:
-        return {"success": False, "error": "Playwright not installed"}
+        return {"success": False, "error": "Playwright not installed. Run: pip install playwright && playwright install chromium"}
    
    session_id = str(uuid.uuid4())[:8]
    
    try:
        async with async_playwright() as p:
-            browser = await p.chromium.launch(headless=True)
+            # 启动浏览器，添加反检测配置
+            browser = await p.chromium.launch(
+                headless=True,
+                args=[
+                    '--disable-blink-features=AutomationControlled',  # 反自动化检测
+                    '--disable-dev-shm-usage',
+                    '--no-sandbox',
+                    '--disable-setuid-sandbox',
+                    '--disable-web-security',
+                ]
+            )
            
            viewport_settings = viewport or {"width": 1920, "height": 1080}
            
+            # 创建上下文，模拟真实浏览器
            context = await browser.new_context(
                viewport={
                    "width": viewport_settings.get("width", 1920),
                    "height": viewport_settings.get("height", 1080)
                },
-                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                # 添加更多浏览器特征
+                locale='zh-CN',
+                timezone_id='Asia/Shanghai',
+                geolocation={'latitude': 31.2304, 'longitude': 121.4737},  # 上海坐标
+                permissions=['geolocation'],
            )
            
+            # 注入反检测脚本
+            await context.add_init_script("""
+                Object.defineProperty(navigator, 'webdriver', {
+                    get: () => undefined
+                });
+                Object.defineProperty(navigator, 'plugins', {
+                    get: () => [1, 2, 3, 4, 5]
+                });
+                Object.defineProperty(navigator, 'languages', {
+                    get: () => ['zh-CN', 'zh', 'en', 'en-US']
+                });
+                window.chrome = {
+                    runtime: {}
+                };
+            """)
+            
            page = await context.new_page()
            
            try:
@@ -397,5 +445,5 @@ if __name__ == '__main__':
        print("   agent-browser: npm install -g agent-browser && agent-browser install")
        print("   playwright: pip install playwright && playwright install chromium")
    
-    print("\n🚀 Server running on http://0.0.0.0:5000")
-    app.run(host='0.0.0.0', port=5000, debug=True)
+    print("\n🚀 Server running on http://0.0.0.0:16025")
+    app.run(host='0.0.0.0', port=16025, debug=True)
--- a/deploy.sh
+++ b/deploy.sh
@@ -23,4 +23,4 @@ echo "运行方式："
 echo "  开发模式: python3.12 app.py"
 echo "  生产模式: gunicorn -w 4 -b 0.0.0.0:5000 app:app"
 echo ""
-echo "访问地址: http://localhost:5000"
+echo "访问地址: http://localhost:16025"
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -6,14 +6,14 @@ services:
    image: web-capture-api:latest
    container_name: web-capture-api
    ports:
-      - "5000:5000"
+      - "16025:16025"
    environment:
      - TZ=Asia/Shanghai
    volumes:
      - ./captures:/tmp/web_captures
    restart: unless-stopped
    healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:5000/health"]
+      test: ["CMD", "curl", "-f", "http://localhost:16025/health"]
      interval: 30s
      timeout: 10s
      retries: 3
--- a/install.sh
+++ b/install.sh
@@ -110,7 +110,7 @@ echo "   cd $(pwd)"
 echo "   $PYTHON app.py"
 echo ""
 echo "📖 访问地址:"
-echo "   http://localhost:5000"
+echo "   http://localhost:16025"
 echo ""
 echo "📚 API 文档:"
-echo "   http://localhost:5000/api"
+echo "   http://localhost:16025/api"
--- a/test_api.py
+++ b/test_api.py
@@ -7,7 +7,7 @@ import requests
 import json
 import sys

-API_URL = "http://localhost:5000"
+API_URL = "http://localhost:16025"


 def test_health():
--- a/web-capture-api.service
+++ b/web-capture-api.service
@@ -6,7 +6,7 @@ After=network.target
 Type=simple
 User=openclaw
 WorkingDirectory=/home/openclaw/.openclaw/workspace-hz4th_coder/works/web-capture-api
-ExecStart=/home/openclaw/.openclaw/workspace-hz4th_coder/works/web-capture-api/venv/bin/gunicorn -w 4 -b 0.0.0.0:5000 app:app
+ExecStart=/home/openclaw/.openclaw/workspace-hz4th_coder/works/web-capture-api/venv/bin/gunicorn -w 4 -b 0.0.0.0:16025 app:app
 Restart=always
 RestartSec=10
Author	SHA1	Message	Date
hz4th_coder	98ef180878	增强 Playwright 反爬虫检测能力添加: - 禁用自动化控制特征 - 真实浏览器参数配置 - 反检测脚本注入 - 地理位置等真实特征 - 更强的 User-Agent 模拟 Playwright 后端可以更好地绕过反爬虫检测	2026-07-05 00:55:09 +08:00
hz4th_coder	6b7e856e38	增强 agent-browser 反爬虫支持 - 自动设置 AGENT_BROWSER_SOCKET_DIR 环境变量 - 添加真实 User-Agent 绕过反爬虫检测 - 增加 403 错误检测和提示 - 解决 techpowerup.com 等网站拦截问题	2026-07-05 00:19:18 +08:00
hz4th_coder	ccb8479242	修复 agent-browser socket 目录权限问题 - 自动创建 /tmp/agent-browser-sockets 目录 - 设置 AGENT_BROWSER_SOCKET_DIR 环境变量 - 解决 Permission denied 错误	2026-07-05 00:12:48 +08:00
hz4th_coder	8f9bdff00c	修改部署端口为 16025 - Flask 服务端口: 16025 - Docker 端口映射: 16025:16025 - systemd 服务配置更新 - 测试脚本端口更新 - 文档示例更新	2026-07-04 23:21:52 +08:00