添加抓取失败网址记录功能

- 新增 failed_urls 数据库表存储失败URL - 抓取失败时自动记录URL、标题、错误信息 - 搜索页面显示失败URL列表 - 支持重试单个/全部失败URL - 支持删除和清空失败记录 - 显示重试次数和时间
抓取前检查内容库是否已存在相同链接
2026-07-13 17:28:44 +08:00 · 2026-07-13 17:20:33 +08:00 · 2026-07-13 17:03:26 +08:00
6 changed files with 562 additions and 14 deletions
@@ -106,6 +106,21 @@ class Database:
                )
            ''')
            
+            # 失败的URL记录表
+            cursor.execute('''
+                CREATE TABLE IF NOT EXISTS failed_urls (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    url TEXT NOT NULL,
+                    title TEXT,
+                    error_message TEXT,
+                    retry_count INTEGER DEFAULT 0,
+                    status TEXT DEFAULT 'failed',
+                    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+                    last_retry_at DATETIME,
+                    source TEXT DEFAULT 'search'
+                )
+            ''')
+            
            conn.commit()

 # ========== 内容库操作 ==========
@@ -309,5 +324,77 @@ class Database:
            ''', (key, value))
            conn.commit()

+# ========== 失败URL操作 ==========
+    def add_failed_url(self, url, title=None, error_message=None, source='search'):
+        """添加失败的URL"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            # 先检查是否已存在
+            cursor.execute('SELECT id, retry_count FROM failed_urls WHERE url = ?', (url,))
+            existing = cursor.fetchone()
+            
+            if existing:
+                # 更新重试次数和错误信息
+                cursor.execute('''
+                    UPDATE failed_urls 
+                    SET error_message = ?, last_retry_at = CURRENT_TIMESTAMP, retry_count = retry_count + 1
+                    WHERE url = ?
+                ''', (error_message, url))
+            else:
+                # 新增失败记录
+                cursor.execute('''
+                    INSERT INTO failed_urls (url, title, error_message, source)
+                    VALUES (?, ?, ?, ?)
+                ''', (url, title, error_message, source))
+            conn.commit()
+            return cursor.lastrowid
+    
+    def get_failed_urls(self, limit=100, status='failed'):
+        """获取失败的URL列表"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                SELECT * FROM failed_urls 
+                WHERE status = ?
+                ORDER BY created_at DESC 
+                LIMIT ?
+            ''', (status, limit))
+            return [dict(row) for row in cursor.fetchall()]
+    
+    def get_failed_url_count(self):
+        """获取失败URL数量"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('SELECT COUNT(*) FROM failed_urls WHERE status = "failed"')
+            return cursor.fetchone()[0]
+    
+    def mark_url_success(self, url):
+        """标记URL为成功（已处理）"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                UPDATE failed_urls 
+                SET status = 'success', last_retry_at = CURRENT_TIMESTAMP
+                WHERE url = ?
+            ''', (url,))
+            conn.commit()
+            return cursor.rowcount > 0
+    
+    def delete_failed_url(self, url_id):
+        """删除失败URL记录"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('DELETE FROM failed_urls WHERE id = ?', (url_id,))
+            conn.commit()
+            return cursor.rowcount > 0
+    
+    def clear_failed_urls(self):
+        """清空所有失败URL记录"""
+        with self.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('DELETE FROM failed_urls WHERE status = "failed"')
+            conn.commit()
+            return cursor.rowcount
+
 # 全局数据库实例
 db = Database()
@@ -140,17 +140,18 @@ def internet_search():
    data = request.get_json()
    keyword = data.get('keyword', '')
    max_results = data.get('max_results', 10)
-    save_to_library = data.get('save_to_library', False)
+    engine = data.get('engine', 'bing_cn')  # 默认 Bing 中国
    
    if not keyword:
        return jsonify({'error': '请提供搜索关键词'}), 400
    
    # 执行互联网搜索
-    results = search_service.search_internet(keyword, max_results)
+    results = search_service.search_internet(keyword, max_results, engine)
    
    return jsonify({
        'success': True,
        'keyword': keyword,
+        'engine': engine,
        'results': results,
        'count': len(results)
    })
@@ -188,4 +189,88 @@ def internet_search_and_fetch():
        'keyword': keyword,
        'results': fetched_results,
        'count': len(fetched_results)
-    })
+    })
+
+# ========== 失败URL管理 ==========
+
+@bp.route('/failed-urls', methods=['GET'])
+def get_failed_urls():
+    """获取失败的URL列表"""
+    limit = request.args.get('limit', 100, type=int)
+    urls = db.get_failed_urls(limit=limit)
+    return jsonify({
+        'success': True,
+        'urls': urls,
+        'count': len(urls)
+    })
+
+@bp.route('/failed-urls', methods=['POST'])
+def add_failed_url():
+    """记录失败的URL"""
+    data = request.get_json()
+    url = data.get('url')
+    title = data.get('title')
+    error_message = data.get('error_message')
+    source = data.get('source', 'search')
+    
+    if not url:
+        return jsonify({'error': '请提供URL'}), 400
+    
+    url_id = db.add_failed_url(url, title, error_message, source)
+    return jsonify({
+        'success': True,
+        'url_id': url_id,
+        'message': '失败URL已记录'
+    })
+
+@bp.route('/failed-urls/count', methods=['GET'])
+def get_failed_url_count():
+    """获取失败URL数量"""
+    count = db.get_failed_url_count()
+    return jsonify({
+        'success': True,
+        'count': count
+    })
+
+@bp.route('/failed-urls/<int:url_id>', methods=['DELETE'])
+def delete_failed_url(url_id):
+    """删除失败URL记录"""
+    success = db.delete_failed_url(url_id)
+    if success:
+        return jsonify({'success': True, 'message': '记录已删除'})
+    else:
+        return jsonify({'error': '记录不存在'}), 404
+
+@bp.route('/failed-urls/clear', methods=['POST'])
+def clear_failed_urls():
+    """清空所有失败URL记录"""
+    count = db.clear_failed_urls()
+    return jsonify({
+        'success': True,
+        'message': f'已清空 {count} 条记录'
+    })
+
+@bp.route('/failed-urls/retry', methods=['POST'])
+def retry_failed_url():
+    """重试失败的URL"""
+    data = request.get_json()
+    url = data.get('url')
+    
+    if not url:
+        return jsonify({'error': '请提供URL'}), 400
+    
+    # 尝试抓取
+    result = search_service.fetch_url_content(url)
+    
+    if result and result.get('content'):
+        # 成功，标记为已处理
+        db.mark_url_success(url)
+        return jsonify({
+            'success': True,
+            'message': '抓取成功',
+            'data': result
+        })
+    else:
+        # 仍然失败
+        db.add_failed_url(url, error_message='重试失败')
+        return jsonify({'error': '抓取仍然失败'}), 500
@@ -33,18 +33,32 @@ class SearchService:
        )
        return result.stdout, result.stderr, result.returncode
    
-    def search_internet(self, keyword, max_results=None):
+    def search_internet(self, keyword, max_results=None, engine='bing_cn'):
        """
        从互联网搜索（使用 agent-browser 浏览器自动化）
+        
+        支持的搜索引擎:
+        - bing_cn: Bing 中国（默认）
+        - bing_global: Bing 国际版
+        - google: Google
+        - baidu: 百度
        """
        max_results = max_results or self.max_results
        results = []
        
+        # 根据搜索引擎选择 URL
+        encoded_keyword = urllib.parse.quote(keyword)
+        search_urls = {
+            'bing_cn': f"https://cn.bing.com/search?q={encoded_keyword}",
+            'bing_global': f"https://www.bing.com/search?q={encoded_keyword}",
+            'google': f"https://www.google.com/search?q={encoded_keyword}",
+            'baidu': f"https://www.baidu.com/s?wd={encoded_keyword}"
+        }
+        
+        search_url = search_urls.get(engine, search_urls['bing_cn'])
+        
        try:
-            # 1. 打开 Bing 搜索
-            encoded_keyword = urllib.parse.quote(keyword)
-            search_url = f"https://www.bing.com/search?q={encoded_keyword}"
-            
+            # 1. 打开搜索引擎
            stdout, stderr, code = self._run_browser('open', search_url, '--timeout', '20000')
            if code != 0:
                print(f"打开搜索页面失败: {stderr}")
@@ -66,9 +80,11 @@ class SearchService:
                print(f"解析 JSON 失败: {stdout[:500]}")
                return results
            
-            # 4. 从 accessibility tree 中提取搜索结果
-            # Bing 搜索结果在 main[aria-label="搜索结果"] 区域内
-            results = self._parse_bing_results(data, max_results)
+            # 4. 根据搜索引擎选择解析方法
+            if engine == 'baidu':
+                results = self._parse_baidu_results(data, max_results)
+            else:
+                results = self._parse_bing_results(data, max_results)
            
            # 5. 关闭浏览器
            self._run_browser('close')
@@ -143,6 +159,44 @@ class SearchService:
        
        return results
    
+    def _parse_baidu_results(self, snapshot_data, max_results=10):
+        """从百度搜索结果中解析标题和链接"""
+        results = []
+        
+        snapshot = snapshot_data.get('data', {}).get('snapshot', '')
+        if not snapshot:
+            return results
+        
+        # 百度搜索结果解析
+        refs = []
+        lines = snapshot.split('\n')
+        
+        for line in lines:
+            line = line.strip()
+            
+            # 百度结果通常在 link 标签中
+            if 'link "' in line and '[ref=' in line:
+                match = re.search(r'link "([^"]+)" \[ref=(e\d+)\]', line)
+                if match:
+                    title = match.group(1)
+                    ref = match.group(2)
+                    # 过滤百度内部链接和广告
+                    if len(title) > 10 and '百度' not in title[:6]:
+                        refs.append((title, ref))
+        
+        # 获取每个结果的 URL
+        for title, ref in refs[:max_results]:
+            url = self._get_link_url(ref)
+            if url and 'baidu.com' not in url:
+                results.append({
+                    'title': title,
+                    'url': url,
+                    'snippet': '',
+                    'source': 'baidu'
+                })
+        
+        return results
+
    def _get_link_url(self, ref):
        """通过 agent-browser 获取链接的 URL"""
        try:
@@ -87,6 +87,21 @@
    font-size: 16px;
 }

+.engine-select {
+    padding: 15px;
+    border: 2px solid #e9ecef;
+    border-radius: 8px;
+    font-size: 16px;
+    background: white;
+    cursor: pointer;
+    min-width: 120px;
+}
+
+.engine-select:focus {
+    outline: none;
+    border-color: #667eea;
+}
+
 .search-options {
    display: flex;
    gap: 20px;
@@ -202,6 +217,11 @@
    background: #f0fdf4;
 }

+.result-item.exists {
+    border-color: #f59e0b;
+    background: #fffbeb;
+}
+
 .result-item.fetched {
    border-color: #3b82f6;
 }
@@ -331,4 +351,87 @@
        width: 100%;
        justify-content: space-between;
    }
+}
+
+/* 失败URL区域 */
+.failed-urls-section {
+    background: white;
+    border-radius: 12px;
+    box-shadow: 0 2px 4px rgba(0,0,0,0.05);
+    margin-top: 20px;
+}
+
+.failed-urls-section .panel-header {
+    background: #fef3c7;
+}
+
+.failed-urls-section .panel-header h2 {
+    color: #92400e;
+}
+
+.failed-count {
+    color: #92400e;
+    font-weight: bold;
+}
+
+.failed-urls-list {
+    display: flex;
+    flex-direction: column;
+    gap: 10px;
+}
+
+.failed-url-item {
+    display: flex;
+    justify-content: space-between;
+    align-items: flex-start;
+    padding: 15px;
+    border: 1px solid #fcd34d;
+    border-radius: 8px;
+    background: #fffbeb;
+}
+
+.failed-url-item.loading {
+    opacity: 0.5;
+}
+
+.failed-url-info {
+    flex: 1;
+}
+
+.failed-url-title {
+    font-weight: 500;
+    color: #333;
+    margin-bottom: 5px;
+}
+
+.failed-url-detail {
+    font-size: 13px;
+    margin-bottom: 5px;
+}
+
+.failed-url-detail a {
+    color: #3b82f6;
+    text-decoration: none;
+}
+
+.failed-url-detail a:hover {
+    text-decoration: underline;
+}
+
+.failed-error {
+    color: #dc2626;
+    margin-left: 10px;
+    font-size: 12px;
+}
+
+.failed-url-meta {
+    font-size: 12px;
+    color: #666;
+    display: flex;
+    gap: 15px;
+}
+
+.failed-url-actions {
+    display: flex;
+    gap: 8px;
 }
@@ -13,12 +13,16 @@ document.addEventListener('DOMContentLoaded', () => {
            doSearch();
        }
    });
+    
+    // 加载失败URL
+    loadFailedUrls();
 });

 // 执行搜索
 async function doSearch() {
    const keyword = document.getElementById('search-keyword').value.trim();
    const maxResults = parseInt(document.getElementById('search-count').value) || 10;
+    const engine = document.getElementById('search-engine').value;
    const autoFetch = document.getElementById('auto-fetch').checked;
    const autoSave = document.getElementById('auto-save').checked;

@@ -28,7 +32,13 @@ async function doSearch() {
    }

    // 更新状态
-    updateStatus('searching', '搜索中...');
+    const engineNames = {
+        'bing_cn': 'Bing 中国',
+        'bing_global': 'Bing 国际',
+        'google': 'Google',
+        'baidu': '百度'
+    };
+    updateStatus('searching', `正在通过 ${engineNames[engine]} 搜索...`);
    document.getElementById('search-btn').disabled = true;
    
    // 显示进度
@@ -41,7 +51,7 @@ async function doSearch() {
        const response = await fetch(`${API_BASE}/api/articles/internet-search`, {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
-            body: JSON.stringify({ keyword, max_results: maxResults })
+            body: JSON.stringify({ keyword, max_results: maxResults, engine })
        });

        const data = await response.json();
@@ -120,6 +130,21 @@ function displayResults() {
    `).join('');
 }

+// 检查URL是否已在内容库
+async function checkUrlExists(url) {
+    try {
+        const response = await fetch(`${API_BASE}/api/articles/search?q=${encodeURIComponent(url)}`);
+        const data = await response.json();
+        if (data.success && data.articles && data.articles.length > 0) {
+            // 检查是否有完全匹配的 URL
+            return data.articles.some(a => a.url === url);
+        }
+    } catch (error) {
+        console.error('检查URL失败:', error);
+    }
+    return false;
+}
+
 // 抓取单个结果
 async function fetchResult(index) {
    const result = searchResults[index];
@@ -127,6 +152,28 @@ async function fetchResult(index) {

    const btn = document.getElementById(`fetch-btn-${index}`);
    btn.disabled = true;
+    btn.innerHTML = '<i class="ri-loader-4-line"></i> 检查中...';
+
+    // 先检查内容库是否已存在
+    const exists = await checkUrlExists(result.url);
+    if (exists) {
+        searchResults[index].fetched = true;
+        searchResults[index].saved = true;
+        searchResults[index].content = '[内容库中已存在此链接]';
+        btn.innerHTML = '<i class="ri-check-line"></i> 已存在';
+        btn.className = 'btn btn-sm btn-warning';
+        
+        const saveBtn = document.getElementById(`save-btn-${index}`);
+        saveBtn.disabled = true;
+        saveBtn.innerHTML = '<i class="ri-check-line"></i> 已存在';
+        
+        const item = document.getElementById(`result-${index}`);
+        item.classList.add('saved');
+        
+        showToast('该链接已在内容库中', 'warning');
+        return;
+    }
+
    btn.innerHTML = '<i class="ri-loader-4-line"></i> 抓取中...';

    try {
@@ -155,26 +202,51 @@ async function fetchResult(index) {
            showToast('抓取成功', 'success');
            displayResults();
        } else {
+            // 记录失败URL
+            await recordFailedUrl(result.url, result.title, data.error);
            btn.disabled = false;
            btn.innerHTML = '<i class="ri-download-line"></i> 抓取';
+            btn.className = 'btn btn-sm btn-danger';
            showToast('抓取失败: ' + data.error, 'error');
        }
    } catch (error) {
+        // 记录失败URL
+        await recordFailedUrl(result.url, result.title, '抓取出错');
        btn.disabled = false;
        btn.innerHTML = '<i class="ri-download-line"></i> 抓取';
+        btn.className = 'btn btn-sm btn-danger';
        showToast('抓取出错', 'error');
    }
 }

+// 记录失败的URL
+async function recordFailedUrl(url, title, errorMessage) {
+    try {
+        await fetch(`${API_BASE}/api/articles/failed-urls`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ url, title, error_message: errorMessage })
+        });
+    } catch (error) {
+        console.error('记录失败URL出错:', error);
+    }
+}
+
 // 保存单个结果
 async function saveResult(index) {
    const result = searchResults[index];
    if (!result) return;

+    // 如果已经保存（可能是内容库中已存在），直接返回
+    if (result.saved) {
+        return;
+    }
+
    // 如果未抓取，先抓取
    if (!result.fetched) {
        await fetchResult(index);
-        if (!searchResults[index].fetched) return;
+        // 抓取后如果已标记为 saved（内容库已存在），不继续保存
+        if (searchResults[index].saved) return;
    }

    const btn = document.getElementById(`save-btn-${index}`);
@@ -381,4 +453,121 @@ function escapeHtml(text) {
    const div = document.createElement('div');
    div.textContent = text;
    return div.innerHTML;
+}
+
+// ========== 失败URL管理 ==========
+
+// 加载失败URL列表
+async function loadFailedUrls() {
+    try {
+        const response = await fetch(`${API_BASE}/api/articles/failed-urls`);
+        const data = await response.json();
+        
+        if (data.success) {
+            document.getElementById('failed-count').textContent = `${data.count} 条`;
+            
+            const container = document.getElementById('failed-urls-list');
+            
+            if (data.urls.length === 0) {
+                container.innerHTML = '<div class="empty-text">暂无失败记录</div>';
+            } else {
+                container.innerHTML = data.urls.map(url => `
+                    <div class="failed-url-item" id="failed-${url.id}">
+                        <div class="failed-url-info">
+                            <div class="failed-url-title">${escapeHtml(url.title || url.url.substring(0, 50))}</div>
+                            <div class="failed-url-detail">
+                                <a href="${escapeHtml(url.url)}" target="_blank">
+                                    <i class="ri-external-link-line"></i> ${escapeHtml(url.url.substring(0, 60))}${url.url.length > 60 ? '...' : ''}
+                                </a>
+                                <span class="failed-error">${escapeHtml(url.error_message || '未知错误')}</span>
+                            </div>
+                            <div class="failed-url-meta">
+                                <span>重试: ${url.retry_count || 0} 次</span>
+                                <span>${url.created_at || ''}</span>
+                            </div>
+                        </div>
+                        <div class="failed-url-actions">
+                            <button onclick="retryFailedUrl(${url.id}, '${escapeHtml(url.url)}')" class="btn btn-sm btn-warning">
+                                <i class="ri-restart-line"></i> 重试
+                            </button>
+                            <button onclick="deleteFailedUrl(${url.id})" class="btn btn-sm btn-danger">
+                                <i class="ri-delete-bin-line"></i>
+                            </button>
+                        </div>
+                    </div>
+                `).join('');
+            }
+        }
+    } catch (error) {
+        console.error('加载失败URL出错:', error);
+    }
+}
+
+// 重试单个失败URL
+async function retryFailedUrl(urlId, url) {
+    const item = document.getElementById(`failed-${urlId}`);
+    item.classList.add('loading');
+    
+    try {
+        const response = await fetch(`${API_BASE}/api/articles/failed-urls/retry`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ url })
+        });
+        
+        const data = await response.json();
+        
+        if (data.success) {
+            showToast('重试成功', 'success');
+            loadFailedUrls();
+        } else {
+            showToast('重试失败: ' + data.error, 'error');
+            item.classList.remove('loading');
+        }
+    } catch (error) {
+        showToast('重试出错', 'error');
+        item.classList.remove('loading');
+    }
+}
+
+// 全部重试
+async function retryAllFailed() {
+    showToast('正在重试所有失败URL...', '');
+    
+    const response = await fetch(`${API_BASE}/api/articles/failed-urls`);
+    const data = await response.json();
+    
+    if (data.success && data.urls.length > 0) {
+        for (const url of data.urls) {
+            await retryFailedUrl(url.id, url.url);
+            await new Promise(r => setTimeout(r, 500)); // 避免太快
+        }
+    }
+}
+
+// 删除失败URL记录
+async function deleteFailedUrl(urlId) {
+    try {
+        await fetch(`${API_BASE}/api/articles/failed-urls/${urlId}`, {
+            method: 'DELETE'
+        });
+        loadFailedUrls();
+    } catch (error) {
+        showToast('删除失败', 'error');
+    }
+}
+
+// 清空所有失败URL
+async function clearFailedUrls() {
+    if (!confirm('确定要清空所有失败记录吗？')) return;
+    
+    try {
+        await fetch(`${API_BASE}/api/articles/failed-urls/clear`, {
+            method: 'POST'
+        });
+        showToast('已清空', 'success');
+        loadFailedUrls();
+    } catch (error) {
+        showToast('清空失败', 'error');
+    }
 }
@@ -29,6 +29,12 @@
        <!-- 搜索区域 -->
        <div class="search-box">
            <div class="search-input-group">
+                <select id="search-engine" class="engine-select" title="选择搜索引擎">
+                    <option value="bing_cn" selected>Bing 中国</option>
+                    <option value="bing_global">Bing 国际</option>
+                    <option value="google">Google</option>
+                    <option value="baidu">百度</option>
+                </select>
                <input type="text" id="search-keyword" placeholder="输入关键词搜索..." class="search-input-large">
                <input type="number" id="search-count" value="10" min="1" max="20" class="count-input" title="结果数量">
                <button onclick="doSearch()" class="btn btn-primary btn-large" id="search-btn">
@@ -76,6 +82,30 @@
            </div>
        </div>

+        <!-- 失败URL区域 -->
+        <div class="panel failed-urls-section">
+            <div class="panel-header">
+                <h2><i class="ri-error-warning-line"></i> 抓取失败的网址</h2>
+                <div class="panel-actions">
+                    <span id="failed-count" class="failed-count">0 条</span>
+                    <button onclick="loadFailedUrls()" class="btn btn-secondary btn-sm">
+                        <i class="ri-refresh-line"></i> 刷新
+                    </button>
+                    <button onclick="retryAllFailed()" class="btn btn-warning btn-sm">
+                        <i class="ri-restart-line"></i> 全部重试
+                    </button>
+                    <button onclick="clearFailedUrls()" class="btn btn-danger btn-sm">
+                        <i class="ri-delete-bin-line"></i> 清空
+                    </button>
+                </div>
+            </div>
+            <div class="panel-body">
+                <div id="failed-urls-list" class="failed-urls-list">
+                    <div class="empty-text">暂无失败记录</div>
+                </div>
+            </div>
+        </div>
+
        <!-- 保存进度 -->
        <div id="save-progress" class="save-progress" style="display: none;">
            <div class="progress-info">