修复搜索抓取相关问题

1. 放宽搜索结果过滤条件，返回更多结果
2. 抓取返回详细错误信息（如超时、网络错误等）
3. 错误信息记录到失败URL库中
4. 统一返回格式包含success字段

routes/articles.py

@@ -114,7 +114,7 @@ def fetch_article():
     
     result = search_service.fetch_url_content(url)
     
-    if result:
+    if result and result.get('success'):
         # 自动保存到内容库
         article_id = search_service.save_to_articles(
             product_names=data.get('product_names', [result['title']]),
@@ -132,7 +132,8 @@ def fetch_article():
             'data': result
         })
     else:
-        return jsonify({'error': '抓取失败'}), 500
+        error_msg = result.get('error', '抓取失败') if result else '抓取失败'
+        return jsonify({'success': False, 'error': error_msg}), 500
 
 @bp.route('/internet-search', methods=['POST'])
 def internet_search():

services/search_service.py

@@ -151,14 +151,14 @@ class SearchService:
                 continue
             
             # 匹配标题链接：link "标题文字" [ref=eXX]
-            # 需要过滤域名链接（如 "zhihu.com"）和短链接
+            # 过滤域名链接（如 "zhihu.com"）和短链接
             if 'link "' in line and '[ref=' in line:
                 match = re.search(r'link "([^"]+)" \[ref=(e\d+)\]', line)
                 if match:
                     title = match.group(1)
                     ref = match.group(2)
-                    # 过滤短标题（域名链接如 "zhihu.com"）
+                    # 放宽过滤条件：只要不是纯域名格式就保留
-                    if len(title) > 20 and '.' not in title[:10]:  # 不是域名格式
+                    if not (title.endswith('.com') or title.endswith('.cn') or title.endswith('.net')):
                         refs.append((title, ref))
         
         # 获取每个结果的 URL
@@ -225,13 +225,18 @@ class SearchService:
     
     def fetch_url_content(self, url):
         """抓取网页内容（使用 agent-browser 浏览器方式，绕过反爬虫）"""
+        error_message = None
         try:
             # 使用浏览器方式抓取，增加超时时间到60秒
             stdout, stderr, code = self._run_browser('open', url, '--timeout', '60000')
             if code != 0:
+                error_message = stderr.strip() if stderr else '浏览器打开页面失败'
                 print(f"打开页面失败: {stderr}")
                 # 浏览器失败，尝试使用 requests 备用方案
-                return self._fetch_with_requests(url)
+                result = self._fetch_with_requests(url)
+                if result:
+                    return result
+                return {'success': False, 'error': error_message}
             
             # 等待页面加载（增加到10秒）
             self._run_browser('wait', '10000')
@@ -263,6 +268,7 @@ class SearchService:
             description = text[:200].strip() if text else ''
             
             return {
+                'success': True,
                 'title': title,
                 'description': description,
                 'content': text,
@@ -270,13 +276,14 @@ class SearchService:
                 'fetch_date': datetime.now().isoformat()
             }
         except Exception as e:
-            print(f"抓取URL失败: {url}, 错误: {str(e)}")
+            error_message = str(e)
+            print(f"抓取URL失败: {url}, 错误: {error_message}")
             # 尝试关闭浏览器
             try:
                 self._run_browser('close')
             except:
                 pass
-            return None
+            return {'success': False, 'error': error_message}
     
     def _extract_text_from_snapshot(self, snapshot):
         """从 accessibility tree snapshot 中提取文本内容"""
@@ -340,6 +347,7 @@ class SearchService:
             description = text[:200].strip() if text else ''
             
             return {
+                'success': True,
                 'title': title,
                 'description': description,
                 'content': text,

static/css/search.css

@@ -151,6 +151,13 @@
     font-size: 14px;
 }
 
+.progress-info {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-top: 10px;
+}
+
 /* 结果区域 */
 .results-section {
     background: white;

static/js/search.js

@@ -5,6 +5,9 @@ const API_BASE = '';
 let searchResults = [];
 let currentResultIndex = -1;
 
+// 自动流程控制
+let shouldStop = false;
+
 // 页面加载初始化
 document.addEventListener('DOMContentLoaded', () => {
     // 回车搜索
@@ -42,9 +45,11 @@ async function doSearch() {
     updateStatus('searching', `正在通过 ${engineNames[engine]} 搜索...`);
     document.getElementById('search-btn').disabled = true;
     
-    // 显示进度
+    // 显示进度和停止按钮
+    shouldStop = false;
     const progress = document.getElementById('search-progress');
     progress.style.display = 'block';
+    document.getElementById('stop-btn').style.display = 'inline-flex';
     document.getElementById('progress-fill').style.width = '0%';
     document.getElementById('progress-text').textContent = '正在搜索...';
 
@@ -67,6 +72,14 @@ async function doSearch() {
 
             displayResults();
             
+            // 检查是否被停止
+            if (shouldStop) {
+                document.getElementById('stop-btn').style.display = 'none';
+                document.getElementById('progress-text').textContent = '已停止';
+                updateStatus('warning', '搜索已停止');
+                return;
+            }
+            
             // 显示缓存状态
             if (data.cached) {
                 updateStatus('success', `找到 ${searchResults.length} 条结果（使用缓存）`);
@@ -80,25 +93,28 @@ async function doSearch() {
 
             document.getElementById('progress-fill').style.width = '100%';
             document.getElementById('progress-text').textContent = '搜索完成！';
+            document.getElementById('stop-btn').style.display = 'none';  // 隐藏停止按钮
 
             // 自动抓取和保存
             if (autoFetch && searchResults.length > 0) {
                 setTimeout(() => fetchAllResults(autoSave), 500);
+            } else {
+                setTimeout(() => {
+                    progress.style.display = 'none';
+                }, 1000);
             }
         } else {
+            document.getElementById('stop-btn').style.display = 'none';
             updateStatus('error', '搜索失败');
             showToast('搜索失败: ' + data.error, 'error');
         }
     } catch (error) {
+        document.getElementById('stop-btn').style.display = 'none';
         updateStatus('error', '搜索出错');
         showToast('搜索出错', 'error');
         console.error(error);
     }
 
-    setTimeout(() => {
-        progress.style.display = 'none';
-    }, 1000);
-
     document.getElementById('search-btn').disabled = false;
 }
 
@@ -338,13 +354,27 @@ async function saveAllResults() {
 
 // 抓取所有结果
 async function fetchAllResults(autoSave) {
+    shouldStop = false;  // 重置停止标志
+    
     const progress = document.getElementById('search-progress');
     progress.style.display = 'block';
+    document.getElementById('stop-btn').style.display = 'inline-flex';  // 显示停止按钮
 
     const total = searchResults.length;
     let fetched = 0;
 
     for (let i = 0; i < searchResults.length; i++) {
+        // 检查是否停止
+        if (shouldStop) {
+            document.getElementById('progress-text').textContent = `已停止（已抓取 ${fetched}/${total}）`;
+            document.getElementById('stop-btn').style.display = 'none';
+            setTimeout(() => {
+                progress.style.display = 'none';
+            }, 2000);
+            showToast('自动抓取已停止', 'warning');
+            return;
+        }
+        
         if (searchResults[i].fetched) {
             fetched++;
             continue;
@@ -361,6 +391,7 @@ async function fetchAllResults(autoSave) {
 
     document.getElementById('progress-fill').style.width = '100%';
     document.getElementById('progress-text').textContent = '抓取完成！';
+    document.getElementById('stop-btn').style.display = 'none';
 
     setTimeout(() => {
         progress.style.display = 'none';
@@ -372,6 +403,11 @@ async function fetchAllResults(autoSave) {
     }
 }
 
+// 停止自动处理
+function stopAutoProcess() {
+    shouldStop = true;
+}
+
 // 显示结果详情
 function showResultDetail(index) {
     currentResultIndex = index;

templates/search.html

@@ -62,7 +62,12 @@
             <div class="progress-bar">
                 <div class="progress-fill" id="progress-fill"></div>
             </div>
-            <div class="progress-text" id="progress-text">正在搜索...</div>
+            <div class="progress-info">
+                <span class="progress-text" id="progress-text">正在搜索...</span>
+                <button onclick="stopAutoProcess()" class="btn btn-danger btn-sm" id="stop-btn" style="display: none;">
+                    <i class="ri-stop-line"></i> 停止
+                </button>
+            </div>
         </div>
 
         <!-- 搜索结果 -->