fix: 修复步骤3抓取网页问题
- 去掉[:5]限制,现在抓取所有互联网搜索结果 - 创建background_tasks记录,在/search页面显示抓取进度 - 每抓取一个URL更新进度和当前项 - 完成/失败/停止时更新任务状态
This commit is contained in:
@@ -105,12 +105,36 @@ class ProcessMonitor:
|
|||||||
try:
|
try:
|
||||||
fetched = []
|
fetched = []
|
||||||
failed_count = 0
|
failed_count = 0
|
||||||
urls_to_fetch = [r['url'] for r in all_data['internet_results'][:5]]
|
urls_to_fetch = [r['url'] for r in all_data['internet_results']]
|
||||||
|
total_urls = len(urls_to_fetch)
|
||||||
|
|
||||||
|
# 创建后台任务记录,这样 /search 页面能看到进度
|
||||||
|
bg_task_id = f"fetch_{session_id}"
|
||||||
|
db.create_task(bg_task_id, 'fetch_urls', {
|
||||||
|
'total': total_urls,
|
||||||
|
'auto_save': True,
|
||||||
|
'category': category,
|
||||||
|
'source': 'process_monitor',
|
||||||
|
'product_name': product_name
|
||||||
|
})
|
||||||
|
db.update_task_status(bg_task_id, 'running', total=total_urls)
|
||||||
|
|
||||||
for i, url in enumerate(urls_to_fetch):
|
for i, url in enumerate(urls_to_fetch):
|
||||||
if self._check_pause(session_id):
|
if self._check_pause(session_id):
|
||||||
|
db.update_task_status(bg_task_id, 'stopped', progress=i)
|
||||||
break
|
break
|
||||||
|
|
||||||
|
# 获取当前URL对应的标题
|
||||||
|
result_item = next((r for r in all_data['internet_results'] if r.get('url') == url), {})
|
||||||
|
current_title = result_item.get('title', url[:50])
|
||||||
|
|
||||||
|
# 更新后台任务进度
|
||||||
|
db.update_task_status(
|
||||||
|
bg_task_id, 'running',
|
||||||
|
progress=i,
|
||||||
|
current_item=current_title
|
||||||
|
)
|
||||||
|
|
||||||
fetch_result = search_service.fetch_url_content(url)
|
fetch_result = search_service.fetch_url_content(url)
|
||||||
if fetch_result.get('success'):
|
if fetch_result.get('success'):
|
||||||
title = fetch_result.get('title', '')
|
title = fetch_result.get('title', '')
|
||||||
@@ -158,10 +182,25 @@ class ProcessMonitor:
|
|||||||
|
|
||||||
time.sleep(0.3)
|
time.sleep(0.3)
|
||||||
|
|
||||||
|
# 更新后台任务状态为完成
|
||||||
|
db.update_task_status(
|
||||||
|
bg_task_id, 'completed',
|
||||||
|
progress=total_urls,
|
||||||
|
result={
|
||||||
|
'total': total_urls,
|
||||||
|
'success': len(fetched),
|
||||||
|
'failed': failed_count,
|
||||||
|
'saved': len(fetched)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
all_data['fetched_contents'] = fetched
|
all_data['fetched_contents'] = fetched
|
||||||
self._complete_step(session_id, 3, {'count': len(fetched), 'failed': failed_count})
|
self._complete_step(session_id, 3, {'count': len(fetched), 'failed': failed_count})
|
||||||
logger.info(f"[{session_id}] 步骤3完成: 抓取 {len(fetched)} 个网页, 失败 {failed_count} 个")
|
logger.info(f"[{session_id}] 步骤3完成: 抓取 {len(fetched)} 个网页, 失败 {failed_count} 个")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
# 更新后台任务状态为失败
|
||||||
|
if 'bg_task_id' in locals():
|
||||||
|
db.update_task_status(bg_task_id, 'failed', error_message=str(e))
|
||||||
self._fail_step(session_id, 3, str(e))
|
self._fail_step(session_id, 3, str(e))
|
||||||
|
|
||||||
# 步骤4: 提取产品数据(调用智能体执行)
|
# 步骤4: 提取产品数据(调用智能体执行)
|
||||||
|
|||||||
Reference in New Issue
Block a user