fix: 修复步骤3抓取网页问题
- 去掉[:5]限制,现在抓取所有互联网搜索结果 - 创建background_tasks记录,在/search页面显示抓取进度 - 每抓取一个URL更新进度和当前项 - 完成/失败/停止时更新任务状态
This commit is contained in:
@@ -105,12 +105,36 @@ class ProcessMonitor:
|
||||
try:
|
||||
fetched = []
|
||||
failed_count = 0
|
||||
urls_to_fetch = [r['url'] for r in all_data['internet_results'][:5]]
|
||||
urls_to_fetch = [r['url'] for r in all_data['internet_results']]
|
||||
total_urls = len(urls_to_fetch)
|
||||
|
||||
# 创建后台任务记录,这样 /search 页面能看到进度
|
||||
bg_task_id = f"fetch_{session_id}"
|
||||
db.create_task(bg_task_id, 'fetch_urls', {
|
||||
'total': total_urls,
|
||||
'auto_save': True,
|
||||
'category': category,
|
||||
'source': 'process_monitor',
|
||||
'product_name': product_name
|
||||
})
|
||||
db.update_task_status(bg_task_id, 'running', total=total_urls)
|
||||
|
||||
for i, url in enumerate(urls_to_fetch):
|
||||
if self._check_pause(session_id):
|
||||
db.update_task_status(bg_task_id, 'stopped', progress=i)
|
||||
break
|
||||
|
||||
# 获取当前URL对应的标题
|
||||
result_item = next((r for r in all_data['internet_results'] if r.get('url') == url), {})
|
||||
current_title = result_item.get('title', url[:50])
|
||||
|
||||
# 更新后台任务进度
|
||||
db.update_task_status(
|
||||
bg_task_id, 'running',
|
||||
progress=i,
|
||||
current_item=current_title
|
||||
)
|
||||
|
||||
fetch_result = search_service.fetch_url_content(url)
|
||||
if fetch_result.get('success'):
|
||||
title = fetch_result.get('title', '')
|
||||
@@ -158,10 +182,25 @@ class ProcessMonitor:
|
||||
|
||||
time.sleep(0.3)
|
||||
|
||||
# 更新后台任务状态为完成
|
||||
db.update_task_status(
|
||||
bg_task_id, 'completed',
|
||||
progress=total_urls,
|
||||
result={
|
||||
'total': total_urls,
|
||||
'success': len(fetched),
|
||||
'failed': failed_count,
|
||||
'saved': len(fetched)
|
||||
}
|
||||
)
|
||||
|
||||
all_data['fetched_contents'] = fetched
|
||||
self._complete_step(session_id, 3, {'count': len(fetched), 'failed': failed_count})
|
||||
logger.info(f"[{session_id}] 步骤3完成: 抓取 {len(fetched)} 个网页, 失败 {failed_count} 个")
|
||||
except Exception as e:
|
||||
# 更新后台任务状态为失败
|
||||
if 'bg_task_id' in locals():
|
||||
db.update_task_status(bg_task_id, 'failed', error_message=str(e))
|
||||
self._fail_step(session_id, 3, str(e))
|
||||
|
||||
# 步骤4: 提取产品数据(调用智能体执行)
|
||||
|
||||
Reference in New Issue
Block a user