修复搜索和重复检查问题

1. 搜索文章时增加URL字段搜索，解决重复抓取问题
2. 优化搜索结果解析，多获取几个结果确保返回足够数量

models/database.py

@@ -121,6 +121,20 @@ class Database:
                 )
             ''')
             
+            # 搜索缓存表
+            cursor.execute('''
+                CREATE TABLE IF NOT EXISTS search_cache (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    keyword TEXT NOT NULL,
+                    engine TEXT DEFAULT 'bing_cn',
+                    results TEXT NOT NULL,
+                    result_count INTEGER,
+                    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+                    expires_at DATETIME,
+                    UNIQUE(keyword, engine)
+                )
+            ''')
+            
             conn.commit()
 
 # ========== 内容库操作 ==========
@@ -143,16 +157,16 @@ class Database:
             if category:
                 cursor.execute('''
                     SELECT * FROM articles 
-                    WHERE (product_names LIKE ? OR keywords LIKE ? OR summary LIKE ? OR content LIKE ?)
+                    WHERE (product_names LIKE ? OR keywords LIKE ? OR summary LIKE ? OR content LIKE ? OR url LIKE ?)
                     AND category = ?
                     ORDER BY fetch_date DESC
-                ''', (f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', category))
+                ''', (f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', category))
             else:
                 cursor.execute('''
                     SELECT * FROM articles 
-                    WHERE product_names LIKE ? OR keywords LIKE ? OR summary LIKE ? OR content LIKE ?
+                    WHERE product_names LIKE ? OR keywords LIKE ? OR summary LIKE ? OR content LIKE ? OR url LIKE ?
                     ORDER BY fetch_date DESC
-                ''', (f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', f'%{keyword}%'))
+                ''', (f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', f'%{keyword}%'))
             return [dict(row) for row in cursor.fetchall()]
     
     def get_article_by_id(self, article_id):
@@ -396,5 +410,52 @@ class Database:
             conn.commit()
             return cursor.rowcount
 
+# ========== 搜索缓存操作 ==========
+    def save_search_cache(self, keyword, engine, results, expire_days=7):
+        """保存搜索结果缓存"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                INSERT OR REPLACE INTO search_cache (keyword, engine, results, result_count, created_at, expires_at)
+                VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP, datetime('now', '+' || ? || ' days'))
+            ''', (keyword, engine, json.dumps(results, ensure_ascii=False), len(results), expire_days))
+            conn.commit()
+            return cursor.lastrowid
+    
+    def get_search_cache(self, keyword, engine='bing_cn'):
+        """获取搜索结果缓存"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                SELECT results, result_count, created_at, expires_at 
+                FROM search_cache 
+                WHERE keyword = ? AND engine = ? AND expires_at > datetime('now')
+            ''', (keyword, engine))
+            row = cursor.fetchone()
+            if row:
+                return {
+                    'results': json.loads(row['results']),
+                    'count': row['result_count'],
+                    'cached_at': row['created_at'],
+                    'expires_at': row['expires_at']
+                }
+            return None
+    
+    def clear_expired_cache(self):
+        """清理过期缓存"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('DELETE FROM search_cache WHERE expires_at <= datetime("now")')
+            conn.commit()
+            return cursor.rowcount
+    
+    def clear_all_cache(self):
+        """清空所有缓存"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('DELETE FROM search_cache')
+            conn.commit()
+            return cursor.rowcount
+
 # 全局数据库实例
 db = Database()

routes/articles.py

@@ -114,7 +114,7 @@ def fetch_article():
     
     result = search_service.fetch_url_content(url)
     
-    if result:
+    if result and result.get('success'):
         # 自动保存到内容库
         article_id = search_service.save_to_articles(
             product_names=data.get('product_names', [result['title']]),
@@ -132,7 +132,8 @@ def fetch_article():
             'data': result
         })
     else:
-        return jsonify({'error': '抓取失败'}), 500
+        error_msg = result.get('error', '抓取失败') if result else '抓取失败'
+        return jsonify({'success': False, 'error': error_msg}), 500
 
 @bp.route('/internet-search', methods=['POST'])
 def internet_search():
@@ -141,19 +142,39 @@ def internet_search():
     keyword = data.get('keyword', '')
     max_results = data.get('max_results', 10)
     engine = data.get('engine', 'bing_cn')  # 默认 Bing 中国
+    use_cache = data.get('use_cache', True)  # 默认使用缓存
+    cache_days = data.get('cache_days', 7)  # 默认缓存7天
     
     if not keyword:
         return jsonify({'error': '请提供搜索关键词'}), 400
     
+    # 检查是否有缓存
+    cached = None
+    if use_cache:
+        cached = db.get_search_cache(keyword, engine)
+    
+    if cached:
+        return jsonify({
+            'success': True,
+            'keyword': keyword,
+            'engine': engine,
+            'results': cached['results'],
+            'count': cached['count'],
+            'cached': True,
+            'cached_at': cached['cached_at'],
+            'expires_at': cached['expires_at']
+        })
+    
     # 执行互联网搜索
-    results = search_service.search_internet(keyword, max_results, engine)
+    results = search_service.search_internet(keyword, max_results, engine, use_cache, cache_days)
     
     return jsonify({
         'success': True,
         'keyword': keyword,
         'engine': engine,
         'results': results,
-        'count': len(results)
+        'count': len(results),
+        'cached': False
     })
 
 @bp.route('/internet-search-and-fetch', methods=['POST'])

services/search_service.py

@@ -33,7 +33,7 @@ class SearchService:
         )
         return result.stdout, result.stderr, result.returncode
     
-    def search_internet(self, keyword, max_results=None, engine='bing_cn'):
+    def search_internet(self, keyword, max_results=None, engine='bing_cn', use_cache=True, cache_days=7):
         """
         从互联网搜索（使用 agent-browser 浏览器自动化）
         
@@ -42,10 +42,21 @@ class SearchService:
         - bing_global: Bing 国际版
         - google: Google
         - baidu: 百度
+        
+        参数:
+        - use_cache: 是否使用缓存（默认True）
+        - cache_days: 缓存有效天数（默认7天）
         """
         max_results = max_results or self.max_results
         results = []
         
+        # 优先查询缓存
+        if use_cache:
+            cached = db.get_search_cache(keyword, engine)
+            if cached:
+                print(f"使用缓存结果: {keyword} ({engine})")
+                return cached['results']
+        
         # 根据搜索引擎选择 URL
         encoded_keyword = urllib.parse.quote(keyword)
         search_urls = {
@@ -89,6 +100,10 @@ class SearchService:
             # 5. 关闭浏览器
             self._run_browser('close')
             
+            # 6. 保存到缓存
+            if results and use_cache:
+                db.save_search_cache(keyword, engine, results, cache_days)
+            
         except subprocess.TimeoutExpired:
             print(f"搜索超时: {keyword}")
         except Exception as e:
@@ -136,18 +151,23 @@ class SearchService:
                 continue
             
             # 匹配标题链接：link "标题文字" [ref=eXX]
-            # 需要过滤域名链接（如 "zhihu.com"）和短链接
+            # 过滤域名链接（如 "zhihu.com"）和短链接
             if 'link "' in line and '[ref=' in line:
                 match = re.search(r'link "([^"]+)" \[ref=(e\d+)\]', line)
                 if match:
                     title = match.group(1)
                     ref = match.group(2)
-                    # 过滤短标题（域名链接如 "zhihu.com"）
-                    if len(title) > 20 and '.' not in title[:10]:  # 不是域名格式
+                    # 过滤纯域名格式
+                    is_domain = (title.endswith('.com') or title.endswith('.cn') or 
+                                 title.endswith('.net') or title.endswith('.org') or
+                                 title.endswith('.edu') or title.endswith('.gov'))
+                    if not is_domain:
                         refs.append((title, ref))
         
-        # 获取每个结果的 URL
-        for title, ref in refs[:max_results]:
+        # 获取每个结果的 URL，多获取几个以防解析失败
+        for title, ref in refs[:max_results + 5]:
+            if len(results) >= max_results:
+                break
             url = self._get_link_url(ref)
             if url and 'bing.com/search' not in url:  # 过滤搜索结果页本身的链接
                 results.append({
@@ -209,16 +229,22 @@ class SearchService:
         return None
     
     def fetch_url_content(self, url):
-        """抓取网页内容（使用 agent-browser 浏览器方式，绑过反爬虫）"""
+        """抓取网页内容（使用 agent-browser 浏览器方式，绕过反爬虫）"""
+        error_message = None
         try:
-            # 使用浏览器方式抓取
-            stdout, stderr, code = self._run_browser('open', url, '--timeout', '20000')
+            # 使用浏览器方式抓取，增加超时时间到60秒
+            stdout, stderr, code = self._run_browser('open', url, '--timeout', '60000')
             if code != 0:
+                error_message = stderr.strip() if stderr else '浏览器打开页面失败'
                 print(f"打开页面失败: {stderr}")
-                return None
+                # 浏览器失败，尝试使用 requests 备用方案
+                result = self._fetch_with_requests(url)
+                if result:
+                    return result
+                return {'success': False, 'error': error_message}
             
-            # 等待页面加载
-            self._run_browser('wait', '5000')
+            # 等待页面加载（增加到10秒）
+            self._run_browser('wait', '10000')
             
             # 获取页面标题
             stdout, stderr, code = self._run_browser('get', 'title', '--timeout', '5000')
@@ -247,6 +273,7 @@ class SearchService:
             description = text[:200].strip() if text else ''
             
             return {
+                'success': True,
                 'title': title,
                 'description': description,
                 'content': text,
@@ -254,13 +281,14 @@ class SearchService:
                 'fetch_date': datetime.now().isoformat()
             }
         except Exception as e:
-            print(f"抓取URL失败: {url}, 错误: {str(e)}")
+            error_message = str(e)
+            print(f"抓取URL失败: {url}, 错误: {error_message}")
             # 尝试关闭浏览器
             try:
                 self._run_browser('close')
             except:
                 pass
-            return None
+            return {'success': False, 'error': error_message}
     
     def _extract_text_from_snapshot(self, snapshot):
         """从 accessibility tree snapshot 中提取文本内容"""
@@ -294,6 +322,47 @@ class SearchService:
         
         return result
     
+    def _fetch_with_requests(self, url):
+        """备用方案：使用 requests 抓取静态内容"""
+        try:
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+                'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'
+            }
+            response = requests.get(url, headers=headers, timeout=30)
+            response.raise_for_status()
+            
+            soup = BeautifulSoup(response.text, 'html.parser')
+            
+            # 获取标题
+            title = soup.title.string.strip() if soup.title else ''
+            
+            # 移除不需要的标签
+            for tag in soup(['script', 'style', 'nav', 'footer', 'header', 'aside']):
+                tag.decompose()
+            
+            # 获取主要内容
+            text = soup.get_text(separator='\n', strip=True)
+            # 清理多余空白行
+            lines = [line.strip() for line in text.split('\n') if line.strip()]
+            text = '\n'.join(lines)
+            
+            # 提取描述（前200字符）
+            description = text[:200].strip() if text else ''
+            
+            return {
+                'success': True,
+                'title': title,
+                'description': description,
+                'content': text,
+                'url': url,
+                'fetch_date': datetime.now().isoformat()
+            }
+        except Exception as e:
+            print(f"备用抓取失败: {url}, 错误: {str(e)}")
+            return None
+    
     def search_articles(self, keyword, category=None):
         """从内容库搜索"""
         return db.search_articles(keyword, category)

static/css/search.css

@@ -151,6 +151,13 @@
     font-size: 14px;
 }
 
+.progress-info {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-top: 10px;
+}
+
 /* 结果区域 */
 .results-section {
     background: white;

static/css/style.css

@@ -801,4 +801,37 @@ body {
 .library-entrance-link small {
     font-size: 13px;
     opacity: 0.8;
+}
+
+/* 快速统计卡片 */
+.quick-stats {
+    display: flex;
+    gap: 20px;
+    justify-content: center;
+}
+
+.quick-stat-item {
+    text-align: center;
+}
+
+.quick-stat-value {
+    font-size: 24px;
+    font-weight: bold;
+    color: #667eea;
+    display: block;
+}
+
+.quick-stat-label {
+    font-size: 13px;
+    color: #666;
+}
+
+.search-quick-info {
+    text-align: center;
+    margin-top: 10px;
+    color: #999;
+}
+
+.search-quick-info small {
+    font-size: 12px;
 }

static/js/app.js

@@ -24,8 +24,8 @@ async function refreshData() {
         await Promise.all([
             loadStats(),
             loadPendingProducts(),
-            loadArticles(),
-            loadHistory()
+            loadHistory(),
+            loadQuickStats()
         ]);
         updateSystemStatus(true);
     } catch (error) {
@@ -66,6 +66,38 @@ async function loadStats() {
     }
 }
 
+// 加载快速统计信息
+async function loadQuickStats() {
+    try {
+        // 加载文章统计
+        const articlesResponse = await fetch(`${API_BASE}/api/articles?limit=1000`);
+        const articlesData = await articlesResponse.json();
+        
+        if (articlesData.success) {
+            document.getElementById('quick-articles-count').textContent = articlesData.count;
+            
+            // 计算分类数
+            const categories = new Set(articlesData.articles.map(a => a.category).filter(c => c));
+            document.getElementById('quick-categories-count').textContent = categories.size;
+            
+            // 计算今日新增
+            const today = new Date().toISOString().split('T')[0];
+            const todayCount = articlesData.articles.filter(a => a.fetch_date && a.fetch_date.startsWith(today)).length;
+            document.getElementById('quick-today-count').textContent = todayCount;
+        }
+        
+        // 加载失败URL数
+        const failedResponse = await fetch(`${API_BASE}/api/articles/failed-urls`);
+        const failedData = await failedResponse.json();
+        
+        if (failedData.success) {
+            document.getElementById('quick-failed-count').textContent = failedData.count;
+        }
+    } catch (error) {
+        console.error('加载快速统计失败:', error);
+    }
+}
+
 // 加载待处理产品
 async function loadPendingProducts() {
     try {

static/js/search.js

@@ -5,6 +5,9 @@ const API_BASE = '';
 let searchResults = [];
 let currentResultIndex = -1;
 
+// 自动流程控制
+let shouldStop = false;
+
 // 页面加载初始化
 document.addEventListener('DOMContentLoaded', () => {
     // 回车搜索
@@ -23,6 +26,7 @@ async function doSearch() {
     const keyword = document.getElementById('search-keyword').value.trim();
     const maxResults = parseInt(document.getElementById('search-count').value) || 10;
     const engine = document.getElementById('search-engine').value;
+    const useCache = document.getElementById('use-cache').checked;
     const autoFetch = document.getElementById('auto-fetch').checked;
     const autoSave = document.getElementById('auto-save').checked;
 
@@ -41,9 +45,11 @@ async function doSearch() {
     updateStatus('searching', `正在通过 ${engineNames[engine]} 搜索...`);
     document.getElementById('search-btn').disabled = true;
     
-    // 显示进度
+    // 显示进度和停止按钮
+    shouldStop = false;
     const progress = document.getElementById('search-progress');
     progress.style.display = 'block';
+    document.getElementById('stop-btn').style.display = 'inline-flex';
     document.getElementById('progress-fill').style.width = '0%';
     document.getElementById('progress-text').textContent = '正在搜索...';
 
@@ -51,7 +57,7 @@ async function doSearch() {
         const response = await fetch(`${API_BASE}/api/articles/internet-search`, {
             method: 'POST',
             headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify({ keyword, max_results: maxResults, engine })
+            body: JSON.stringify({ keyword, max_results: maxResults, engine, use_cache: useCache })
         });
 
         const data = await response.json();
@@ -65,31 +71,50 @@ async function doSearch() {
             }));
 
             displayResults();
-            updateStatus('success', `找到 ${searchResults.length} 条结果`);
+            
+            // 检查是否被停止
+            if (shouldStop) {
+                document.getElementById('stop-btn').style.display = 'none';
+                document.getElementById('progress-text').textContent = '已停止';
+                updateStatus('warning', '搜索已停止');
+                return;
+            }
+            
+            // 显示缓存状态
+            if (data.cached) {
+                updateStatus('success', `找到 ${searchResults.length} 条结果（使用缓存）`);
+                showToast('使用缓存结果', 'success');
+            } else {
+                updateStatus('success', `找到 ${searchResults.length} 条结果`);
+            }
+            
             document.getElementById('results-count').textContent = `${searchResults.length} 条结果`;
             document.getElementById('save-all-btn').disabled = searchResults.length === 0;
 
             document.getElementById('progress-fill').style.width = '100%';
             document.getElementById('progress-text').textContent = '搜索完成！';
+            document.getElementById('stop-btn').style.display = 'none';  // 隐藏停止按钮
 
             // 自动抓取和保存
             if (autoFetch && searchResults.length > 0) {
                 setTimeout(() => fetchAllResults(autoSave), 500);
+            } else {
+                setTimeout(() => {
+                    progress.style.display = 'none';
+                }, 1000);
             }
         } else {
+            document.getElementById('stop-btn').style.display = 'none';
             updateStatus('error', '搜索失败');
             showToast('搜索失败: ' + data.error, 'error');
         }
     } catch (error) {
+        document.getElementById('stop-btn').style.display = 'none';
         updateStatus('error', '搜索出错');
         showToast('搜索出错', 'error');
         console.error(error);
     }
 
-    setTimeout(() => {
-        progress.style.display = 'none';
-    }, 1000);
-
     document.getElementById('search-btn').disabled = false;
 }
 
@@ -329,13 +354,27 @@ async function saveAllResults() {
 
 // 抓取所有结果
 async function fetchAllResults(autoSave) {
+    shouldStop = false;  // 重置停止标志
+    
     const progress = document.getElementById('search-progress');
     progress.style.display = 'block';
+    document.getElementById('stop-btn').style.display = 'inline-flex';  // 显示停止按钮
 
     const total = searchResults.length;
     let fetched = 0;
 
     for (let i = 0; i < searchResults.length; i++) {
+        // 检查是否停止
+        if (shouldStop) {
+            document.getElementById('progress-text').textContent = `已停止（已抓取 ${fetched}/${total}）`;
+            document.getElementById('stop-btn').style.display = 'none';
+            setTimeout(() => {
+                progress.style.display = 'none';
+            }, 2000);
+            showToast('自动抓取已停止', 'warning');
+            return;
+        }
+        
         if (searchResults[i].fetched) {
             fetched++;
             continue;
@@ -352,6 +391,7 @@ async function fetchAllResults(autoSave) {
 
     document.getElementById('progress-fill').style.width = '100%';
     document.getElementById('progress-text').textContent = '抓取完成！';
+    document.getElementById('stop-btn').style.display = 'none';
 
     setTimeout(() => {
         progress.style.display = 'none';
@@ -363,6 +403,11 @@ async function fetchAllResults(autoSave) {
     }
 }
 
+// 停止自动处理
+function stopAutoProcess() {
+    shouldStop = true;
+}
+
 // 显示结果详情
 function showResultDetail(index) {
     currentResultIndex = index;

templates/index.html

@@ -97,32 +97,54 @@
                 </div>
             </div>
 
-            <!-- 右侧：内容库入口 -->
+            <!-- 右侧：内容库和搜索入口 -->
             <div class="panel">
                 <div class="panel-header">
-                    <h2><i class="ri-folder-line"></i> 内容库文章</h2>
-                </div>
-                <div class="panel-body" style="text-align: center; padding: 40px;">
-                    <a href="/library" class="library-entrance-link">
-                        <i class="ri-folder-line"></i>
-                        <span>点击进入内容库页面</span>
-                        <small>查看、编辑、导出所有文章</small>
+                    <h2><i class="ri-folder-line"></i> 内容库</h2>
+                    <a href="/library" class="btn btn-primary btn-sm">
+                        <i class="ri-external-link-line"></i> 进入
                     </a>
                 </div>
+                <div class="panel-body">
+                    <div class="quick-stats">
+                        <div class="quick-stat-item">
+                            <span class="quick-stat-value" id="quick-articles-count">0</span>
+                            <span class="quick-stat-label">篇文章</span>
+                        </div>
+                        <div class="quick-stat-item">
+                            <span class="quick-stat-value" id="quick-categories-count">0</span>
+                            <span class="quick-stat-label">个分类</span>
+                        </div>
+                        <div class="quick-stat-item">
+                            <span class="quick-stat-value" id="quick-today-count">0</span>
+                            <span class="quick-stat-label">今日新增</span>
+                        </div>
+                    </div>
+                </div>
             </div>
-        </div>
 
-        <!-- 互联网搜索入口 -->
-        <div class="panel full-width">
-            <div class="panel-header">
-                <h2><i class="ri-search-line"></i> 互联网搜索</h2>
-            </div>
-            <div class="panel-body" style="text-align: center; padding: 40px;">
-                <a href="/search" class="search-entrance-link">
-                    <i class="ri-search-line"></i>
-                    <span>点击进入互联网搜索页面</span>
-                    <small>支持一键保存所有搜索结果到内容库</small>
-                </a>
+            <div class="panel">
+                <div class="panel-header">
+                    <h2><i class="ri-search-line"></i> 互联网搜索</h2>
+                    <a href="/search" class="btn btn-primary btn-sm">
+                        <i class="ri-external-link-line"></i> 进入
+                    </a>
+                </div>
+                <div class="panel-body">
+                    <div class="quick-stats">
+                        <div class="quick-stat-item">
+                            <span class="quick-stat-value" id="quick-failed-count">0</span>
+                            <span class="quick-stat-label">抓取失败</span>
+                        </div>
+                        <div class="quick-stat-item">
+                            <span class="quick-stat-label">支持引擎</span>
+                            <span class="quick-stat-value">Bing/百度/Google</span>
+                        </div>
+                    </div>
+                    <div class="search-quick-info">
+                        <small>支持一键保存所有结果到内容库</small>
+                    </div>
+                </div>
             </div>
         </div>
 

templates/search.html

@@ -42,6 +42,10 @@
                 </button>
             </div>
             <div class="search-options">
+                <label class="checkbox-label">
+                    <input type="checkbox" id="use-cache" checked>
+                    <span>使用缓存结果（7天有效）</span>
+                </label>
                 <label class="checkbox-label">
                     <input type="checkbox" id="auto-fetch" checked>
                     <span>搜索后自动抓取内容</span>
@@ -58,7 +62,12 @@
             <div class="progress-bar">
                 <div class="progress-fill" id="progress-fill"></div>
             </div>
-            <div class="progress-text" id="progress-text">正在搜索...</div>
+            <div class="progress-info">
+                <span class="progress-text" id="progress-text">正在搜索...</span>
+                <button onclick="stopAutoProcess()" class="btn btn-danger btn-sm" id="stop-btn" style="display: none;">
+                    <i class="ri-stop-line"></i> 停止
+                </button>
+            </div>
         </div>
 
         <!-- 搜索结果 -->