fix: 修复步骤3抓取网页问题

- 去掉[:5]限制,现在抓取所有互联网搜索结果
- 创建background_tasks记录,在/search页面显示抓取进度
- 每抓取一个URL更新进度和当前项
- 完成/失败/停止时更新任务状态
This commit is contained in:
2026-07-16 01:01:08 +08:00
parent 42c2a53623
commit a367887ace
+40 -1
View File
@@ -105,12 +105,36 @@ class ProcessMonitor:
try:
fetched = []
failed_count = 0
urls_to_fetch = [r['url'] for r in all_data['internet_results'][:5]]
urls_to_fetch = [r['url'] for r in all_data['internet_results']]
total_urls = len(urls_to_fetch)
# 创建后台任务记录,这样 /search 页面能看到进度
bg_task_id = f"fetch_{session_id}"
db.create_task(bg_task_id, 'fetch_urls', {
'total': total_urls,
'auto_save': True,
'category': category,
'source': 'process_monitor',
'product_name': product_name
})
db.update_task_status(bg_task_id, 'running', total=total_urls)
for i, url in enumerate(urls_to_fetch):
if self._check_pause(session_id):
db.update_task_status(bg_task_id, 'stopped', progress=i)
break
# 获取当前URL对应的标题
result_item = next((r for r in all_data['internet_results'] if r.get('url') == url), {})
current_title = result_item.get('title', url[:50])
# 更新后台任务进度
db.update_task_status(
bg_task_id, 'running',
progress=i,
current_item=current_title
)
fetch_result = search_service.fetch_url_content(url)
if fetch_result.get('success'):
title = fetch_result.get('title', '')
@@ -158,10 +182,25 @@ class ProcessMonitor:
time.sleep(0.3)
# 更新后台任务状态为完成
db.update_task_status(
bg_task_id, 'completed',
progress=total_urls,
result={
'total': total_urls,
'success': len(fetched),
'failed': failed_count,
'saved': len(fetched)
}
)
all_data['fetched_contents'] = fetched
self._complete_step(session_id, 3, {'count': len(fetched), 'failed': failed_count})
logger.info(f"[{session_id}] 步骤3完成: 抓取 {len(fetched)} 个网页, 失败 {failed_count}")
except Exception as e:
# 更新后台任务状态为失败
if 'bg_task_id' in locals():
db.update_task_status(bg_task_id, 'failed', error_message=str(e))
self._fail_step(session_id, 3, str(e))
# 步骤4: 提取产品数据(调用智能体执行)