修复页面抓取功能，使用 agent-browser 替代 requests

- 改用浏览器方式抓取页面内容，绑过反爬虫机制
- 从 accessibility tree snapshot 中提取文本内容
- 增加等待时间让页面完全加载
- 可抓取知乎等有反爬措施的网站

routes/articles.py

@@ -132,4 +132,60 @@ def fetch_article():
             'data': result
         })
     else:
-        return jsonify({'error': '抓取失败'}), 500
+        return jsonify({'error': '抓取失败'}), 500
+
+@bp.route('/internet-search', methods=['POST'])
+def internet_search():
+    """互联网搜索（浏览器方式）"""
+    data = request.get_json()
+    keyword = data.get('keyword', '')
+    max_results = data.get('max_results', 10)
+    save_to_library = data.get('save_to_library', False)
+    
+    if not keyword:
+        return jsonify({'error': '请提供搜索关键词'}), 400
+    
+    # 执行互联网搜索
+    results = search_service.search_internet(keyword, max_results)
+    
+    return jsonify({
+        'success': True,
+        'keyword': keyword,
+        'results': results,
+        'count': len(results)
+    })
+
+@bp.route('/internet-search-and-fetch', methods=['POST'])
+def internet_search_and_fetch():
+    """互联网搜索并抓取内容"""
+    data = request.get_json()
+    keyword = data.get('keyword', '')
+    max_results = data.get('max_results', 5)
+    category = data.get('category')
+    
+    if not keyword:
+        return jsonify({'error': '请提供搜索关键词'}), 400
+    
+    # 执行互联网搜索
+    results = search_service.search_internet(keyword, max_results)
+    
+    # 抓取每个结果的详细内容
+    fetched_results = []
+    for r in results:
+        url = r.get('url')
+        if url:
+            content = search_service.fetch_url_content(url)
+            if content:
+                fetched_results.append({
+                    'title': r['title'],
+                    'url': url,
+                    'source': r['source'],
+                    'fetched_content': content
+                })
+    
+    return jsonify({
+        'success': True,
+        'keyword': keyword,
+        'results': fetched_results,
+        'count': len(fetched_results)
+    })

services/search_service.py

@@ -155,43 +155,71 @@ class SearchService:
         return None
     
     def fetch_url_content(self, url):
-        """抓取网页内容"""
+        """抓取网页内容（使用 agent-browser 浏览器方式，绑过反爬虫）"""
         try:
-            headers = {
-                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
-            }
-            response = requests.get(url, headers=headers, timeout=self.timeout)
-            response.raise_for_status()
+            # 使用浏览器方式抓取
+            stdout, stderr, code = self._run_browser('open', url, '--timeout', '20000')
+            if code != 0:
+                print(f"打开页面失败: {stderr}")
+                return None
             
-            soup = BeautifulSoup(response.text, 'lxml')
+            # 等待页面加载
+            self._run_browser('wait', '5000')
             
-            # 提取标题
-            title = soup.find('title')
-            title = title.text.strip() if title else ''
+            # 获取页面标题
+            stdout, stderr, code = self._run_browser('get', 'title', '--timeout', '5000')
+            title = stdout.strip().replace('[agent-browser] ', '').strip() if code == 0 else ''
             
-            # 提取正文（简单提取，可优化）
-            # 移除脚本和样式
-            for script in soup(['script', 'style']):
-                script.decompose()
+            # 获取页面内容（通过 snapshot 获取 accessibility tree）
+            stdout, stderr, code = self._run_browser('snapshot', '--json', '--timeout', '15000')
+            text = ''
+            if code == 0 and stdout:
+                try:
+                    data = json.loads(stdout)
+                    snapshot = data.get('data', {}).get('snapshot', '')
+                    # 从 snapshot 中提取所有 StaticText
+                    text = self._extract_text_from_snapshot(snapshot)
+                except:
+                    pass
             
-            # 提取文本
-            text = soup.get_text(separator='\n', strip=True)
+            # 获取 URL（可能被重定向）
+            stdout, stderr, code = self._run_browser('get', 'url', '--timeout', '5000')
+            actual_url = stdout.strip() if code == 0 else url
             
-            # 提取元数据
-            meta_desc = soup.find('meta', attrs={'name': 'description'})
-            description = meta_desc['content'] if meta_desc else ''
+            # 关闭浏览器
+            self._run_browser('close')
+            
+            # 提取描述（从页面内容的前200字符）
+            description = text[:200].strip() if text else ''
             
             return {
                 'title': title,
                 'description': description,
                 'content': text,
-                'url': url,
+                'url': actual_url,
                 'fetch_date': datetime.now().isoformat()
             }
         except Exception as e:
             print(f"抓取URL失败: {url}, 错误: {str(e)}")
+            # 尝试关闭浏览器
+            try:
+                self._run_browser('close')
+            except:
+                pass
             return None
     
+    def _extract_text_from_snapshot(self, snapshot):
+        """从 accessibility tree snapshot 中提取文本内容"""
+        # 提取所有 StaticText 行
+        texts = []
+        for line in snapshot.split('\n'):
+            if 'StaticText' in line:
+                # 格式: - StaticText "文本内容"
+                match = re.search(r'StaticText "([^"]+)"', line)
+                if match:
+                    texts.append(match.group(1))
+        return '\n'.join(texts)
+    
     def search_articles(self, keyword, category=None):
         """从内容库搜索"""
         return db.search_articles(keyword, category)

static/css/style.css

@@ -642,4 +642,97 @@ body {
 
 .priority-low {
     color: #10b981;
+}
+
+/* 搜索结果卡片 */
+.search-results-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
+    gap: 15px;
+}
+
+.search-result-card {
+    background: #f8f9fa;
+    border: 1px solid #e9ecef;
+    border-radius: 8px;
+    padding: 15px;
+    display: flex;
+    gap: 15px;
+    transition: all 0.2s;
+}
+
+.search-result-card:hover {
+    border-color: #667eea;
+    background: white;
+}
+
+.result-number {
+    width: 30px;
+    height: 30px;
+    background: #667eea;
+    color: white;
+    border-radius: 50%;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    font-weight: bold;
+    font-size: 14px;
+}
+
+.result-content {
+    flex: 1;
+    min-width: 0;
+}
+
+.result-title {
+    font-size: 15px;
+    color: #333;
+    cursor: pointer;
+    margin-bottom: 8px;
+    word-break: break-word;
+}
+
+.result-title:hover {
+    color: #667eea;
+}
+
+.result-url {
+    font-size: 12px;
+    color: #666;
+    margin-bottom: 6px;
+    overflow: hidden;
+}
+
+.result-url a {
+    color: #3b82f6;
+    text-decoration: none;
+    display: flex;
+    align-items: center;
+    gap: 4px;
+}
+
+.result-url a:hover {
+    text-decoration: underline;
+}
+
+.result-source {
+    font-size: 12px;
+    color: #999;
+}
+
+.result-actions {
+    display: flex;
+    flex-direction: column;
+    gap: 8px;
+}
+
+/* 搜索状态 */
+#internet-search-status {
+    display: flex;
+    align-items: center;
+    gap: 8px;
+}
+
+#internet-search-status i {
+    font-size: 16px;
 }

static/js/app.js

@@ -612,4 +612,233 @@ function getStatusText(status) {
         'completed': '已完成'
     };
     return statusMap[status] || status;
+}
+
+// ========== 互联网搜索功能 ==========
+
+let currentSearchResult = null;
+
+// 执行互联网搜索
+async function doInternetSearch() {
+    const keyword = document.getElementById('internet-search-keyword').value.trim();
+    const maxResults = parseInt(document.getElementById('internet-search-count').value) || 10;
+    
+    if (!keyword) {
+        showToast('请输入搜索关键词', 'error');
+        return;
+    }
+    
+    // 显示搜索状态
+    const statusDiv = document.getElementById('internet-search-status');
+    const resultsDiv = document.getElementById('internet-search-results');
+    const searchBtn = document.getElementById('search-btn');
+    
+    statusDiv.innerHTML = '<i class="ri-loader-4-line"></i> 正在搜索...';
+    resultsDiv.innerHTML = '<div class="empty-text">搜索中...</div>';
+    searchBtn.disabled = true;
+    
+    try {
+        const response = await fetch(`${API_BASE}/api/articles/internet-search`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ keyword, max_results: maxResults })
+        });
+        
+        const data = await response.json();
+        
+        if (data.success) {
+            statusDiv.innerHTML = `<i class="ri-check-line"></i> 搜索完成，找到 ${data.count} 条结果`;
+            
+            if (data.results.length > 0) {
+                resultsDiv.innerHTML = data.results.map((r, i) => `
+                    <div class="search-result-card">
+                        <div class="result-number">${i + 1}</div>
+                        <div class="result-content">
+                            <div class="result-title" onclick="showSearchResultDetail(${i})">${escapeHtml(r.title)}</div>
+                            <div class="result-url">
+                                <a href="${escapeHtml(r.url)}" target="_blank">
+                                    <i class="ri-external-link-line"></i> ${escapeHtml(r.url.substring(0, 60))}${r.url.length > 60 ? '...' : ''}
+                                </a>
+                            </div>
+                            <div class="result-source">来源: ${escapeHtml(r.source)}</div>
+                        </div>
+                        <div class="result-actions">
+                            <button onclick="fetchAndShowResult(${i})" class="btn btn-sm btn-secondary">
+                                <i class="ri-download-line"></i> 抓取内容
+                            </button>
+                            <button onclick="quickSaveResult(${i})" class="btn btn-sm btn-success">
+                                <i class="ri-save-line"></i> 保存
+                            </button>
+                        </div>
+                    </div>
+                `).join('');
+                
+                // 存储搜索结果供后续使用
+                window.lastSearchResults = data.results;
+            } else {
+                resultsDiv.innerHTML = '<div class="empty-text">未找到相关结果</div>';
+            }
+        } else {
+            statusDiv.innerHTML = `<i class="ri-error-warning-line"></i> 搜索失败: ${escapeHtml(data.error)}`;
+            resultsDiv.innerHTML = '<div class="empty-text">搜索失败</div>';
+        }
+    } catch (error) {
+        statusDiv.innerHTML = '<i class="ri-error-warning-line"></i> 搜索出错';
+        resultsDiv.innerHTML = '<div class="empty-text">搜索出错，请稍后重试</div>';
+        console.error('搜索错误:', error);
+    }
+    
+    searchBtn.disabled = false;
+}
+
+// 显示搜索结果详情
+function showSearchResultDetail(index) {
+    if (!window.lastSearchResults || !window.lastSearchResults[index]) {
+        return;
+    }
+    
+    currentSearchResult = window.lastSearchResults[index];
+    const body = document.getElementById('search-result-body');
+    
+    body.innerHTML = `
+        <div class="detail-meta">
+            <div class="detail-meta-item">
+                <label>标题</label>
+                <span>${escapeHtml(currentSearchResult.title)}</span>
+            </div>
+            <div class="detail-meta-item">
+                <label>URL</label>
+                <a href="${escapeHtml(currentSearchResult.url)}" target="_blank">${escapeHtml(currentSearchResult.url)}</a>
+            </div>
+            <div class="detail-meta-item">
+                <label>来源</label>
+                <span>${escapeHtml(currentSearchResult.source)}</span>
+            </div>
+        </div>
+        <div class="detail-section">
+            <h4><i class="ri-information-line"></i> 提示</h4>
+            <p>点击"抓取内容"按钮可以获取页面详细内容，然后保存到内容库。</p>
+        </div>
+    `;
+    
+    document.getElementById('search-result-modal').classList.add('active');
+}
+
+// 抓取并显示搜索结果内容
+async function fetchAndShowResult(index) {
+    if (!window.lastSearchResults || !window.lastSearchResults[index]) {
+        return;
+    }
+    
+    const result = window.lastSearchResults[index];
+    currentSearchResult = result;
+    
+    showToast('正在抓取页面内容...', '');
+    
+    try {
+        const response = await fetch(`${API_BASE}/api/articles/fetch`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                url: result.url,
+                product_names: [result.title],
+                category: ''
+            })
+        });
+        
+        const data = await response.json();
+        
+        if (data.success) {
+            // 更新当前搜索结果，添加抓取的内容
+            currentSearchResult = {
+                ...result,
+                fetched: true,
+                fetchedContent: data.data
+            };
+            
+            const body = document.getElementById('search-result-body');
+            body.innerHTML = `
+                <div class="detail-meta">
+                    <div class="detail-meta-item">
+                        <label>标题</label>
+                        <span>${escapeHtml(data.data.title)}</span>
+                    </div>
+                    <div class="detail-meta-item">
+                        <label>URL</label>
+                        <a href="${escapeHtml(result.url)}" target="_blank">${escapeHtml(result.url)}</a>
+                    </div>
+                    <div class="detail-meta-item">
+                        <label>描述</label>
+                        <span>${escapeHtml(data.data.description || '无')}</span>
+                    </div>
+                </div>
+                <div class="detail-section">
+                    <h4><i class="ri-file-text-line"></i> 页面内容</h4>
+                    <pre style="white-space: pre-wrap; max-height: 400px; overflow-y: auto; background: #f8f9fa; padding: 15px; border-radius: 8px;">${escapeHtml(data.data.content.substring(0, 2000))}${data.data.content.length > 2000 ? '\n... (内容过长，已截断)' : ''}</pre>
+                </div>
+            `;
+            
+            document.getElementById('search-result-modal').classList.add('active');
+            showToast('内容抓取成功', 'success');
+        } else {
+            showToast('抓取失败: ' + data.error, 'error');
+        }
+    } catch (error) {
+        showToast('抓取出错', 'error');
+        console.error('抓取错误:', error);
+    }
+}
+
+// 快速保存搜索结果到内容库
+async function quickSaveResult(index) {
+    if (!window.lastSearchResults || !window.lastSearchResults[index]) {
+        return;
+    }
+    
+    const result = window.lastSearchResults[index];
+    
+    // 先抓取内容再保存
+    showToast('正在抓取并保存...', '');
+    
+    try {
+        const response = await fetch(`${API_BASE}/api/articles/fetch`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                url: result.url,
+                product_names: [result.title],
+                category: ''
+            })
+        });
+        
+        const data = await response.json();
+        
+        if (data.success) {
+            showToast('已保存到内容库', 'success');
+            loadArticles();
+            loadStats();
+        } else {
+            showToast('保存失败: ' + data.error, 'error');
+        }
+    } catch (error) {
+        showToast('保存出错', 'error');
+    }
+}
+
+// 从详情模态框保存到内容库
+async function saveSearchResultToLibrary() {
+    if (!currentSearchResult) {
+        return;
+    }
+    
+    // 如果还没有抓取内容，先抓取
+    if (!currentSearchResult.fetched) {
+        await fetchAndShowResult(window.lastSearchResults.findIndex(r => r.url === currentSearchResult.url));
+        return;
+    }
+    
+    showToast('已保存到内容库', 'success');
+    closeModal('search-result-modal');
+    loadArticles();
+    loadStats();
 }

templates/index.html

@@ -116,6 +116,26 @@
             </div>
         </div>
 
+        <!-- 互联网搜索区域 -->
+        <div class="panel full-width">
+            <div class="panel-header">
+                <h2><i class="ri-search-line"></i> 互联网搜索</h2>
+                <div class="panel-actions">
+                    <input type="text" id="internet-search-keyword" placeholder="输入关键词搜索..." class="search-input" style="width: 300px;">
+                    <input type="number" id="internet-search-count" value="10" min="1" max="20" style="width: 80px;" title="结果数量">
+                    <button onclick="doInternetSearch()" class="btn btn-primary" id="search-btn">
+                        <i class="ri-search-line"></i> 搜索
+                    </button>
+                </div>
+            </div>
+            <div class="panel-body">
+                <div id="internet-search-status" style="margin-bottom: 10px; color: #666; font-size: 14px;"></div>
+                <div class="search-results-grid" id="internet-search-results">
+                    <div class="empty-text">输入关键词进行互联网搜索</div>
+                </div>
+            </div>
+        </div>
+
         <!-- 处理历史 -->
         <div class="panel full-width">
             <div class="panel-header">
@@ -267,6 +287,26 @@
         </div>
     </div>
 
+    <!-- 搜索结果详情模态框 -->
+    <div id="search-result-modal" class="modal">
+        <div class="modal-content large">
+            <div class="modal-header">
+                <h3><i class="ri-external-link-line"></i> 搜索结果详情</h3>
+                <button onclick="closeModal('search-result-modal')" class="close-btn">
+                    <i class="ri-close-line"></i>
+                </button>
+            </div>
+            <div class="modal-body" id="search-result-body">
+            </div>
+            <div class="modal-footer">
+                <button onclick="closeModal('search-result-modal')" class="btn btn-secondary">关闭</button>
+                <button onclick="saveSearchResultToLibrary()" class="btn btn-success">
+                    <i class="ri-save-line"></i> 保存到内容库
+                </button>
+            </div>
+        </div>
+    </div>
+
     <!-- 文章详情模态框 -->
     <div id="article-detail-modal" class="modal">
         <div class="modal-content large">