fix: 统一处理流程和后台任务显示

- 处理流程抓取失败时记录到失败URL库
- /search页面新增产品处理会话展示区
- 同时显示后台抓取任务和产品处理会话
- 自动刷新同时更新三种数据：任务、会话、失败URL

services/process_monitor.py

@@ -102,6 +102,7 @@ class ProcessMonitor:
                 self._start_step(session_id, product_name, 3, '抓取网页内容')
                 try:
                     fetched = []
+                    failed_count = 0
                     urls_to_fetch = [r['url'] for r in all_data['internet_results'][:5]]
                     
                     for i, url in enumerate(urls_to_fetch):
@@ -110,16 +111,46 @@ class ProcessMonitor:
                         
                         fetch_result = search_service.fetch_url_content(url)
                         if fetch_result.get('success'):
+                            title = fetch_result.get('title', '')
+                            content = fetch_result.get('content', '')
                             fetched.append({
                                 'url': url,
-                                'title': fetch_result.get('title', ''),
-                                'content': fetch_result.get('content', '')[:500]
+                                'title': title,
+                                'content': content[:500]
                             })
+                            
+                            # 保存到内容库
+                            try:
+                                existing = db.search_articles(url)
+                                if not any(a.get('url') == url for a in existing):
+                                    db.add_article(
+                                        product_names=[],
+                                        category=category or '',
+                                        keywords=[],
+                                        summary=content[:200] if content else '',
+                                        content=content,
+                                        source=url,
+                                        url=url,
+                                        search_title=title
+                                    )
+                                    logger.info(f"[{session_id}] 已保存到内容库: {title[:30]}")
+                            except Exception as save_error:
+                                logger.warning(f"[{session_id}] 保存内容库失败: {save_error}")
+                        else:
+                            # 记录失败URL
+                            failed_count += 1
+                            error_msg = fetch_result.get('error', '抓取失败')
+                            try:
+                                db.add_failed_url(url, product_name, error_msg, source='process_monitor')
+                                logger.warning(f"[{session_id}] 抓取失败，已记录: {url}")
+                            except Exception as e:
+                                logger.error(f"[{session_id}] 记录失败URL出错: {e}")
+                        
                         time.sleep(0.3)
                     
                     all_data['fetched_contents'] = fetched
-                    self._complete_step(session_id, 3, {'count': len(fetched)})
-                    logger.info(f"[{session_id}] 步骤3完成: 抓取 {len(fetched)} 个网页")
+                    self._complete_step(session_id, 3, {'count': len(fetched), 'failed': failed_count})
+                    logger.info(f"[{session_id}] 步骤3完成: 抓取 {len(fetched)} 个网页, 失败 {failed_count} 个")
                 except Exception as e:
                     self._fail_step(session_id, 3, str(e))
             

static/css/process.css

@@ -175,6 +175,31 @@
     max-width: 80px;
 }
 
+.step-value {
+    font-size: 12px;
+    font-weight: 600;
+    color: #10b981;
+    margin-top: 3px;
+    padding: 2px 6px;
+    background: #d1fae5;
+    border-radius: 3px;
+}
+
+.step-node.running .step-value {
+    color: #667eea;
+    background: #e0e7ff;
+}
+
+.step-node.failed .step-value {
+    color: #ef4444;
+    background: #fee2e2;
+}
+
+.step-node.skipped .step-value {
+    color: #6b7280;
+    background: #f3f4f6;
+}
+
 @keyframes pulse {
     0%, 100% { transform: scale(1); }
     50% { transform: scale(1.1); }

static/css/search.css

@@ -472,6 +472,22 @@
     color: #3730a3;
 }
 
+/* 产品处理会话区域 */
+.process-sessions-section {
+    background: white;
+    border-radius: 12px;
+    box-shadow: 0 2px 4px rgba(0,0,0,0.05);
+    margin-top: 20px;
+}
+
+.process-sessions-section .panel-header {
+    background: #fef3c7;
+}
+
+.process-sessions-section .panel-header h2 {
+    color: #92400e;
+}
+
 .background-tasks-list {
     display: flex;
     flex-direction: column;

static/js/app.js

@@ -396,30 +396,35 @@ async function processProduct(productName, category, subcategory) {
         return;
     }
     
+    showToast('正在启动处理...', '');
+    
     try {
-        const response = await fetch(`${API_BASE}/api/products/process`, {
+        const response = await fetch(`${API_BASE}/api/process/start`, {
             method: 'POST',
             headers: { 'Content-Type': 'application/json' },
             body: JSON.stringify({
                 product_name: productName,
-                category: category,
-                subcategory: subcategory
+                category: category || '',
+                subcategory: subcategory || ''
             })
         });
         
         const data = await response.json();
         
         if (data.success) {
-            showToast(data.message, 'success');
-            if (data.new_products && data.new_products.length > 0) {
-                showToast(`发现 ${data.new_products.length} 个新产品`, 'success');
-            }
+            showToast('处理流程已启动', 'success');
+            // 跳转到处理监控页面
+            setTimeout(() => {
+                window.location.href = '/process';
+            }, 1000);
         } else {
-            showToast('处理失败: ' + data.error || data.message, 'error');
+            const errorMsg = data.error || data.message || '未知错误';
+            showToast('处理失败: ' + errorMsg, 'error');
         }
         
         refreshData();
     } catch (error) {
+        console.error('处理产品失败:', error);
         showToast('处理产品失败', 'error');
     }
 }

static/js/process.js

@@ -133,17 +133,42 @@ function displayActiveProcesses(sessions) {
 function renderStepsProgress(steps, currentStep) {
     const totalSteps = 6;
     const stepStatuses = {};
+    const stepDataMap = {};
     
-    // 构建步骤状态映射
+    // 构建步骤状态和数据映射
     steps.forEach(s => {
         stepStatuses[s.step_number] = s.step_status;
+        if (s.step_data) {
+            try {
+                stepDataMap[s.step_number] = typeof s.step_data === 'string' ? JSON.parse(s.step_data) : s.step_data;
+            } catch (e) {
+                stepDataMap[s.step_number] = s.step_data;
+            }
+        }
     });
     
     const stepNames = ['搜索内容库', '搜索互联网', '抓取网页', '提取数据', '填充字段', '提交审核'];
     
-    let html = '';
+    // 获取每个步骤的简要数值
+    function getStepValue(stepNum) {
+        const data = stepDataMap[stepNum];
+        if (!data) return '';
+        
+        switch (stepNum) {
+            case 1: return data.count !== undefined ? `${data.count}篇` : '';
+            case 2: return data.count !== undefined ? `${data.count}条` : '';
+            case 3: return data.count !== undefined ? `${data.count}个` : '';
+            case 4: return data.has_data !== undefined ? (data.has_data ? '✓' : '✗') : (data.extracted ? '✓' : '');
+            case 5: return data.filled !== undefined ? (data.filled ? '✓' : '✗') : '';
+            case 6: return data.review_id ? '✓' : '';
+            default: return '';
+        }
+    }
+    
+    let html = '<div class="steps-progress">';
     for (let i = 1; i <= totalSteps; i++) {
         const status = stepStatuses[i] || (i > currentStep ? 'pending' : '');
+        const stepValue = getStepValue(i);
         let className = '';
         
         if (status === 'completed') className = 'completed';
@@ -155,9 +180,11 @@ function renderStepsProgress(steps, currentStep) {
             <div class="step-node ${className}">
                 <div class="step-circle">${i}</div>
                 <div class="step-label">${stepNames[i-1]}</div>
+                ${stepValue ? `<div class="step-value">${stepValue}</div>` : ''}
             </div>
         `;
     }
+    html += '</div>';
     
     return html;
 }

static/js/search.js

@@ -22,6 +22,7 @@ document.addEventListener('DOMContentLoaded', () => {
     // 加载失败URL和后台任务
     loadFailedUrls();
     loadBackgroundTasks();
+    loadProcessSessions();
     
     // 启动自动刷新后台任务列表（每3秒）
     startAutoRefreshTasks();
@@ -773,7 +774,12 @@ async function loadBackgroundTasks() {
                     const progressPercent = task.total > 0 ? 
                         Math.round((task.progress / task.total) * 100) : 0;
                     
-                    const result = task.result ? JSON.parse(task.result) : {};
+                    let result = {};
+                    try {
+                        result = task.result ? JSON.parse(task.result) : {};
+                    } catch (e) {
+                        result = {};
+                    }
                     
                     return `
                         <div class="background-task-item ${statusClass}">
@@ -875,6 +881,8 @@ function startAutoRefreshTasks() {
     
     autoRefreshTasksInterval = setInterval(() => {
         loadBackgroundTasks();
+        loadProcessSessions();  // 同时刷新处理会话
+        loadFailedUrls();  // 同时刷新失败URL
     }, 3000);  // 每3秒刷新一次
 }
 
@@ -884,4 +892,65 @@ function stopAutoRefreshTasks() {
         clearInterval(autoRefreshTasksInterval);
         autoRefreshTasksInterval = null;
     }
+}
+
+// 加载产品处理会话
+async function loadProcessSessions() {
+    try {
+        const response = await fetch(`${API_BASE}/api/process/recent?limit=10`);
+        const data = await response.json();
+        
+        if (data.success) {
+            const container = document.getElementById('process-sessions-list');
+            
+            if (data.sessions.length === 0) {
+                container.innerHTML = '<div class="empty-text">暂无处理会话</div>';
+            } else {
+                container.innerHTML = data.sessions.map(session => {
+                    const statusClass = {
+                        'pending': 'status-pending',
+                        'running': 'status-running',
+                        'paused': 'status-warning',
+                        'completed': 'status-success',
+                        'failed': 'status-error',
+                        'stopped': 'status-warning'
+                    }[session.status] || '';
+                    
+                    const statusText = {
+                        'pending': '等待中',
+                        'running': '处理中',
+                        'paused': '已暂停',
+                        'completed': '已完成',
+                        'failed': '失败',
+                        'stopped': '已停止'
+                    }[session.status] || session.status;
+                    
+                    return `
+                        <div class="background-task-item ${statusClass}">
+                            <div class="task-info">
+                                <div class="task-id">${escapeHtml(session.product_name)}</div>
+                                <div class="task-status">
+                                    <span class="status-badge ${statusClass}">${statusText}</span>
+                                    ${session.status === 'running' || session.status === 'paused' ? 
+                                        `<span class="task-progress">步骤 ${session.current_step || 0}/6</span>` : ''}
+                                    ${session.category ? `<span>分类: ${escapeHtml(session.category)}</span>` : ''}
+                                    ${session.review_id ? `<span>审核ID: ${escapeHtml(session.review_id)}</span>` : ''}
+                                </div>
+                                <div class="task-time">${session.created_at || ''}
+                                    ${session.finished_at ? ' → ' + session.finished_at : ''}
+                                </div>
+                            </div>
+                            <div class="task-actions">
+                                <a href="/process" class="btn btn-sm btn-secondary">
+                                    <i class="ri-eye-line"></i> 监控
+                                </a>
+                            </div>
+                        </div>
+                    `;
+                }).join('');
+            }
+        }
+    } catch (error) {
+        console.error('加载处理会话出错:', error);
+    }
 }

templates/index.html

@@ -13,6 +13,9 @@
         <header class="header">
             <h1><i class="ri-robot-line"></i> 参数数据自动化管理系统</h1>
             <div class="header-actions">
+                <a href="/process" class="btn btn-primary">
+                    <i class="ri-cpu-line"></i> 处理监控
+                </a>
                 <button onclick="refreshData()" class="btn btn-secondary">
                     <i class="ri-refresh-line"></i> 刷新数据
                 </button>

templates/search.html

@@ -111,6 +111,26 @@
             </div>
         </div>
 
+        <!-- 产品处理会话区域 -->
+        <div class="panel process-sessions-section">
+            <div class="panel-header">
+                <h2><i class="ri-play-circle-line"></i> 产品处理会话</h2>
+                <div class="panel-actions">
+                    <a href="/process" class="btn btn-primary btn-sm">
+                        <i class="ri-external-link-line"></i> 监控页面
+                    </a>
+                    <button onclick="loadProcessSessions()" class="btn btn-secondary btn-sm">
+                        <i class="ri-refresh-line"></i> 刷新
+                    </button>
+                </div>
+            </div>
+            <div class="panel-body">
+                <div id="process-sessions-list" class="background-tasks-list">
+                    <div class="empty-text">暂无处理会话</div>
+                </div>
+            </div>
+        </div>
+
         <!-- 失败URL区域 -->
         <div class="panel failed-urls-section">
             <div class="panel-header">