Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0229f2dfcf | ||
|
|
6534c3315b | ||
|
|
4f47bf2b91 | ||
|
|
6dabc4d0b7 |
@@ -10,6 +10,11 @@
|
||||
- 📝 **数据提取**:根据产品类别提取和填充字段
|
||||
- ✅ **审核提交**:自动提交到ParamHub后台管理待审核区
|
||||
- 🕐 **定时任务**:支持定时自动处理产品
|
||||
- 🔧 **后台任务系统**:抓取任务在后台持续运行,不受页面刷新影响
|
||||
- 一键抓取触发后,任务在后台独立运行
|
||||
- 可随时查看任务进度和状态
|
||||
- 支持手动停止正在运行的任务
|
||||
- 自动记录成功/失败/保存数量
|
||||
|
||||
## 系统架构
|
||||
|
||||
@@ -180,6 +185,51 @@ Content-Type: application/json
|
||||
}
|
||||
```
|
||||
|
||||
### 后台任务 API (`/api/tasks`)
|
||||
|
||||
#### 启动后台抓取任务
|
||||
```
|
||||
POST /api/tasks/start
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"results": [{"title": "标题", "url": "https://..."}],
|
||||
"auto_save": true,
|
||||
"category": "分类",
|
||||
"keywords": ["关键词"]
|
||||
}
|
||||
```
|
||||
|
||||
#### 获取任务状态
|
||||
```
|
||||
GET /api/tasks/{task_id}/status
|
||||
```
|
||||
|
||||
#### 停止任务
|
||||
```
|
||||
POST /api/tasks/{task_id}/stop
|
||||
```
|
||||
|
||||
#### 获取活动任务列表
|
||||
```
|
||||
GET /api/tasks/active
|
||||
```
|
||||
|
||||
#### 获取最近任务
|
||||
```
|
||||
GET /api/tasks/recent?limit=20
|
||||
```
|
||||
|
||||
#### 删除任务记录
|
||||
```
|
||||
DELETE /api/tasks/{task_id}
|
||||
```
|
||||
|
||||
#### 清理已完成任务
|
||||
```
|
||||
POST /api/tasks/clear
|
||||
```
|
||||
|
||||
### 系统管理 API (`/api/system`)
|
||||
|
||||
#### 获取系统配置
|
||||
@@ -263,6 +313,13 @@ BATCH_SIZE = 5 # 批量处理数量
|
||||
|
||||
## 版本历史
|
||||
|
||||
- v1.1.0 (2026-07-14): 后台任务系统
|
||||
- 新增后台任务API(/api/tasks)
|
||||
- 抓取任务在后台独立运行,不受页面刷新影响
|
||||
- 支持任务状态查询和手动停止
|
||||
- 新增后台任务管理界面
|
||||
- 数据库新增 background_tasks 表
|
||||
|
||||
- v1.0.0 (2026-07-12): 初始版本
|
||||
- 内容库管理
|
||||
- 产品处理流程
|
||||
|
||||
@@ -28,10 +28,12 @@ CORS(app)
|
||||
from routes.articles import bp as articles_bp
|
||||
from routes.products import bp as products_bp
|
||||
from routes.system import bp as system_bp
|
||||
from routes.tasks import bp as tasks_bp
|
||||
|
||||
app.register_blueprint(articles_bp)
|
||||
app.register_blueprint(products_bp)
|
||||
app.register_blueprint(system_bp)
|
||||
app.register_blueprint(tasks_bp)
|
||||
|
||||
# 首页
|
||||
@app.route('/')
|
||||
|
||||
@@ -142,6 +142,25 @@ class Database:
|
||||
)
|
||||
''')
|
||||
|
||||
# 后台任务表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS background_tasks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
task_id TEXT NOT NULL UNIQUE,
|
||||
task_type TEXT NOT NULL,
|
||||
status TEXT DEFAULT 'pending',
|
||||
params TEXT,
|
||||
progress INTEGER DEFAULT 0,
|
||||
total INTEGER DEFAULT 0,
|
||||
current_item TEXT,
|
||||
result TEXT,
|
||||
error_message TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
started_at DATETIME,
|
||||
finished_at DATETIME
|
||||
)
|
||||
''')
|
||||
|
||||
conn.commit()
|
||||
|
||||
# ========== 内容库操作 ==========
|
||||
@@ -472,5 +491,112 @@ class Database:
|
||||
conn.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
# ========== 后台任务操作 ==========
|
||||
def create_task(self, task_id, task_type, params=None):
|
||||
"""创建后台任务"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('''
|
||||
INSERT INTO background_tasks (task_id, task_type, params, status)
|
||||
VALUES (?, ?, ?, 'pending')
|
||||
''', (task_id, task_type, json.dumps(params, ensure_ascii=False) if params else None))
|
||||
conn.commit()
|
||||
return task_id
|
||||
|
||||
def get_task(self, task_id):
|
||||
"""获取任务详情"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('SELECT * FROM background_tasks WHERE task_id = ?', (task_id,))
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
result = dict(row)
|
||||
if result.get('params'):
|
||||
result['params'] = json.loads(result['params'])
|
||||
if result.get('result'):
|
||||
result['result'] = json.loads(result['result'])
|
||||
return result
|
||||
return None
|
||||
|
||||
def get_active_tasks(self, task_type=None):
|
||||
"""获取活动任务(running 或 pending)"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
if task_type:
|
||||
cursor.execute('''
|
||||
SELECT * FROM background_tasks
|
||||
WHERE status IN ('pending', 'running') AND task_type = ?
|
||||
ORDER BY created_at DESC
|
||||
''', (task_type,))
|
||||
else:
|
||||
cursor.execute('''
|
||||
SELECT * FROM background_tasks
|
||||
WHERE status IN ('pending', 'running')
|
||||
ORDER BY created_at DESC
|
||||
''')
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def update_task_status(self, task_id, status, **kwargs):
|
||||
"""更新任务状态"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
|
||||
updates = ['status = ?']
|
||||
values = [status]
|
||||
|
||||
if status == 'running' and 'started_at' not in kwargs:
|
||||
updates.append('started_at = CURRENT_TIMESTAMP')
|
||||
elif status in ('completed', 'failed', 'stopped'):
|
||||
updates.append('finished_at = CURRENT_TIMESTAMP')
|
||||
|
||||
for key in ['progress', 'total', 'current_item', 'error_message']:
|
||||
if key in kwargs:
|
||||
updates.append(f'{key} = ?')
|
||||
values.append(kwargs[key])
|
||||
|
||||
if 'result' in kwargs:
|
||||
updates.append('result = ?')
|
||||
values.append(json.dumps(kwargs['result'], ensure_ascii=False))
|
||||
|
||||
values.append(task_id)
|
||||
|
||||
cursor.execute(
|
||||
f'UPDATE background_tasks SET {" , ".join(updates)} WHERE task_id = ?',
|
||||
values
|
||||
)
|
||||
conn.commit()
|
||||
return cursor.rowcount > 0
|
||||
|
||||
def stop_task(self, task_id):
|
||||
"""停止任务"""
|
||||
return self.update_task_status(task_id, 'stopped')
|
||||
|
||||
def get_recent_tasks(self, limit=20):
|
||||
"""获取最近的任务"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('''
|
||||
SELECT * FROM background_tasks
|
||||
ORDER BY created_at DESC
|
||||
LIMIT ?
|
||||
''', (limit,))
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def delete_task(self, task_id):
|
||||
"""删除任务记录"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('DELETE FROM background_tasks WHERE task_id = ?', (task_id,))
|
||||
conn.commit()
|
||||
return cursor.rowcount > 0
|
||||
|
||||
def clear_completed_tasks(self):
|
||||
"""清理已完成的任务"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('DELETE FROM background_tasks WHERE status IN ("completed", "failed", "stopped")')
|
||||
conn.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
# 全局数据库实例
|
||||
db = Database()
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
"""
|
||||
后台任务 API 路由
|
||||
"""
|
||||
from flask import Blueprint, jsonify, request
|
||||
from services.task_service import task_service
|
||||
from models.database import db
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger('tasks_api')
|
||||
|
||||
bp = Blueprint('tasks', __name__, url_prefix='/api/tasks')
|
||||
|
||||
|
||||
@bp.route('/start', methods=['POST'])
|
||||
def start_task():
|
||||
"""
|
||||
启动后台抓取任务
|
||||
|
||||
请求体:
|
||||
{
|
||||
"results": [{"title": "...", "url": "..."}, ...],
|
||||
"auto_save": true,
|
||||
"category": "AI模型",
|
||||
"keywords": ["关键词1", "关键词2"]
|
||||
}
|
||||
"""
|
||||
try:
|
||||
data = request.get_json()
|
||||
|
||||
if not data or 'results' not in data:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': '缺少 results 参数'
|
||||
}), 400
|
||||
|
||||
results = data.get('results', [])
|
||||
auto_save = data.get('auto_save', False)
|
||||
category = data.get('category', '')
|
||||
keywords = data.get('keywords', [])
|
||||
|
||||
if not results:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': '结果列表为空'
|
||||
}), 400
|
||||
|
||||
# 启动后台任务
|
||||
task_id = task_service.start_fetch_task(
|
||||
results=results,
|
||||
auto_save=auto_save,
|
||||
category=category,
|
||||
keywords=keywords
|
||||
)
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'task_id': task_id,
|
||||
'message': f'任务已启动,共 {len(results)} 个URL'
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"启动任务失败: {e}")
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}), 500
|
||||
|
||||
|
||||
@bp.route('/<task_id>/status', methods=['GET'])
|
||||
def get_task_status(task_id):
|
||||
"""获取任务状态"""
|
||||
task = task_service.get_task_status(task_id)
|
||||
|
||||
if not task:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': '任务不存在'
|
||||
}), 404
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'task': task
|
||||
})
|
||||
|
||||
|
||||
@bp.route('/<task_id>/stop', methods=['POST'])
|
||||
def stop_task(task_id):
|
||||
"""停止任务"""
|
||||
task = db.get_task(task_id)
|
||||
|
||||
if not task:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': '任务不存在'
|
||||
}), 404
|
||||
|
||||
if task['status'] not in ('pending', 'running'):
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': '任务已完成或已停止'
|
||||
}), 400
|
||||
|
||||
# 设置停止标志
|
||||
if task_service.stop_task(task_id):
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'message': '已请求停止任务'
|
||||
})
|
||||
else:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': '停止任务失败'
|
||||
}), 500
|
||||
|
||||
|
||||
@bp.route('/active', methods=['GET'])
|
||||
def get_active_tasks():
|
||||
"""获取活动任务列表"""
|
||||
tasks = task_service.get_active_tasks()
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'tasks': tasks,
|
||||
'count': len(tasks)
|
||||
})
|
||||
|
||||
|
||||
@bp.route('/recent', methods=['GET'])
|
||||
def get_recent_tasks():
|
||||
"""获取最近的任务"""
|
||||
limit = request.args.get('limit', 20, type=int)
|
||||
tasks = db.get_recent_tasks(limit)
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'tasks': tasks,
|
||||
'count': len(tasks)
|
||||
})
|
||||
|
||||
|
||||
@bp.route('/<task_id>', methods=['DELETE'])
|
||||
def delete_task(task_id):
|
||||
"""删除任务记录"""
|
||||
task = db.get_task(task_id)
|
||||
|
||||
if not task:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': '任务不存在'
|
||||
}), 404
|
||||
|
||||
if task['status'] in ('pending', 'running'):
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': '任务仍在运行,请先停止'
|
||||
}), 400
|
||||
|
||||
if db.delete_task(task_id):
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'message': '任务已删除'
|
||||
})
|
||||
else:
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': '删除失败'
|
||||
}), 500
|
||||
|
||||
|
||||
@bp.route('/clear', methods=['POST'])
|
||||
def clear_completed_tasks():
|
||||
"""清理已完成的任务"""
|
||||
count = db.clear_completed_tasks()
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'message': f'已清理 {count} 个任务'
|
||||
})
|
||||
@@ -0,0 +1,191 @@
|
||||
"""
|
||||
后台任务服务 - 处理长时间的抓取任务
|
||||
"""
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from models.database import db
|
||||
from services.search_service import search_service
|
||||
|
||||
logger = logging.getLogger('task_service')
|
||||
|
||||
class BackgroundTaskService:
|
||||
"""后台任务服务"""
|
||||
|
||||
def __init__(self):
|
||||
self.active_threads = {} # task_id -> thread
|
||||
self.stop_flags = {} # task_id -> bool
|
||||
|
||||
def create_task_id(self):
|
||||
"""生成唯一任务ID"""
|
||||
return f"task_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}"
|
||||
|
||||
def start_fetch_task(self, results, auto_save=False, category=None, keywords=None):
|
||||
"""
|
||||
启动抓取任务
|
||||
|
||||
参数:
|
||||
- results: 搜索结果列表 [{title, url, ...}, ...]
|
||||
- auto_save: 是否自动保存到内容库
|
||||
- category: 分类(可选)
|
||||
- keywords: 关键词列表(可选)
|
||||
"""
|
||||
task_id = self.create_task_id()
|
||||
|
||||
# 创建任务记录
|
||||
db.create_task(task_id, 'fetch_urls', {
|
||||
'total': len(results),
|
||||
'auto_save': auto_save,
|
||||
'category': category,
|
||||
'keywords': keywords
|
||||
})
|
||||
|
||||
# 启动后台线程
|
||||
self.stop_flags[task_id] = False
|
||||
thread = threading.Thread(
|
||||
target=self._run_fetch_task,
|
||||
args=(task_id, results, auto_save, category, keywords),
|
||||
daemon=True
|
||||
)
|
||||
thread.start()
|
||||
self.active_threads[task_id] = thread
|
||||
|
||||
logger.info(f"任务已启动: {task_id}, 共 {len(results)} 个URL")
|
||||
return task_id
|
||||
|
||||
def _run_fetch_task(self, task_id, results, auto_save, category, keywords):
|
||||
"""执行抓取任务(后台线程)"""
|
||||
try:
|
||||
# 更新状态为 running
|
||||
db.update_task_status(task_id, 'running', total=len(results))
|
||||
|
||||
processed = 0
|
||||
success_count = 0
|
||||
failed_count = 0
|
||||
saved_count = 0
|
||||
fetch_results = []
|
||||
|
||||
for i, result in enumerate(results):
|
||||
# 检查是否被停止
|
||||
if self.stop_flags.get(task_id, False):
|
||||
db.update_task_status(
|
||||
task_id, 'stopped',
|
||||
progress=processed,
|
||||
result={'success': success_count, 'failed': failed_count, 'saved': saved_count}
|
||||
)
|
||||
logger.info(f"任务已停止: {task_id}")
|
||||
return
|
||||
|
||||
url = result.get('url', '')
|
||||
title = result.get('title', url[:50])
|
||||
|
||||
# 更新进度
|
||||
db.update_task_status(
|
||||
task_id, 'running',
|
||||
progress=processed,
|
||||
current_item=title
|
||||
)
|
||||
|
||||
# 抓取内容
|
||||
logger.info(f"[{task_id}] 抓取 {processed + 1}/{len(results)}: {title}")
|
||||
fetch_result = search_service.fetch_url_content(url)
|
||||
|
||||
if fetch_result.get('success'):
|
||||
success_count += 1
|
||||
content = fetch_result.get('content', '')
|
||||
|
||||
# 自动保存
|
||||
if auto_save and content:
|
||||
try:
|
||||
# 检查URL是否已存在
|
||||
existing = db.search_articles(url)
|
||||
if not any(a.get('url') == url for a in existing):
|
||||
article_id = db.add_article(
|
||||
product_names=[title],
|
||||
category=category or '',
|
||||
keywords=keywords or [],
|
||||
summary=content[:200],
|
||||
content=content,
|
||||
source=url,
|
||||
url=url,
|
||||
search_title=title
|
||||
)
|
||||
if article_id:
|
||||
saved_count += 1
|
||||
logger.info(f"[{task_id}] 已保存: {title}")
|
||||
else:
|
||||
logger.info(f"[{task_id}] URL已存在,跳过: {url}")
|
||||
except Exception as e:
|
||||
logger.error(f"[{task_id}] 保存失败: {e}")
|
||||
|
||||
fetch_results.append({
|
||||
'title': title,
|
||||
'url': url,
|
||||
'success': True,
|
||||
'content_length': len(content),
|
||||
'saved': auto_save
|
||||
})
|
||||
else:
|
||||
failed_count += 1
|
||||
error_msg = fetch_result.get('error', '未知错误')
|
||||
|
||||
# 记录失败URL
|
||||
db.add_failed_url(url, title, error_msg, source='background_task')
|
||||
|
||||
fetch_results.append({
|
||||
'title': title,
|
||||
'url': url,
|
||||
'success': False,
|
||||
'error': error_msg
|
||||
})
|
||||
logger.warning(f"[{task_id}] 抓取失败: {title} - {error_msg}")
|
||||
|
||||
processed += 1
|
||||
|
||||
# 短暂延迟,避免请求过快
|
||||
time.sleep(0.5)
|
||||
|
||||
# 任务完成
|
||||
db.update_task_status(
|
||||
task_id, 'completed',
|
||||
progress=processed,
|
||||
result={
|
||||
'total': len(results),
|
||||
'success': success_count,
|
||||
'failed': failed_count,
|
||||
'saved': saved_count
|
||||
}
|
||||
)
|
||||
logger.info(f"任务完成: {task_id}, 成功 {success_count}, 失败 {failed_count}, 已保存 {saved_count}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"任务异常: {task_id} - {e}")
|
||||
db.update_task_status(task_id, 'failed', error_message=str(e))
|
||||
|
||||
finally:
|
||||
# 清理
|
||||
if task_id in self.active_threads:
|
||||
del self.active_threads[task_id]
|
||||
if task_id in self.stop_flags:
|
||||
del self.stop_flags[task_id]
|
||||
|
||||
def stop_task(self, task_id):
|
||||
"""停止任务"""
|
||||
if task_id in self.stop_flags:
|
||||
self.stop_flags[task_id] = True
|
||||
logger.info(f"请求停止任务: {task_id}")
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_task_status(self, task_id):
|
||||
"""获取任务状态"""
|
||||
return db.get_task(task_id)
|
||||
|
||||
def get_active_tasks(self):
|
||||
"""获取活动任务列表"""
|
||||
return db.get_active_tasks()
|
||||
|
||||
# 全局任务服务实例
|
||||
task_service = BackgroundTaskService()
|
||||
@@ -263,6 +263,19 @@
|
||||
color: #4caf50;
|
||||
}
|
||||
|
||||
/* 上部分页 */
|
||||
.pagination-top {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
.pagination-top span {
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
}
|
||||
|
||||
.article-summary {
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
|
||||
@@ -454,4 +454,120 @@
|
||||
.failed-url-actions {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
/* 后台任务区域 */
|
||||
.background-tasks-section {
|
||||
background: white;
|
||||
border-radius: 12px;
|
||||
box-shadow: 0 2px 4px rgba(0,0,0,0.05);
|
||||
margin-top: 20px;
|
||||
}
|
||||
|
||||
.background-tasks-section .panel-header {
|
||||
background: #e0e7ff;
|
||||
}
|
||||
|
||||
.background-tasks-section .panel-header h2 {
|
||||
color: #3730a3;
|
||||
}
|
||||
|
||||
.background-tasks-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.background-task-item {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: flex-start;
|
||||
padding: 15px;
|
||||
border: 1px solid #c7d2fe;
|
||||
border-radius: 8px;
|
||||
background: #f5f3ff;
|
||||
}
|
||||
|
||||
.background-task-item.status-running {
|
||||
background: #fef3c7;
|
||||
border-color: #fcd34d;
|
||||
}
|
||||
|
||||
.background-task-item.status-success {
|
||||
background: #d1fae5;
|
||||
border-color: #6ee7b7;
|
||||
}
|
||||
|
||||
.background-task-item.status-error {
|
||||
background: #fee2e2;
|
||||
border-color: #fca5a5;
|
||||
}
|
||||
|
||||
.background-task-item.status-warning {
|
||||
background: #fef3c7;
|
||||
border-color: #fcd34d;
|
||||
}
|
||||
|
||||
.task-info {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
.task-id {
|
||||
font-size: 12px;
|
||||
color: #666;
|
||||
margin-bottom: 5px;
|
||||
}
|
||||
|
||||
.task-status {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
margin-bottom: 5px;
|
||||
}
|
||||
|
||||
.task-progress {
|
||||
color: #667eea;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.status-badge {
|
||||
padding: 2px 8px;
|
||||
border-radius: 4px;
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.status-badge.status-pending {
|
||||
background: #e0e7ff;
|
||||
color: #3730a3;
|
||||
}
|
||||
|
||||
.status-badge.status-running {
|
||||
background: #fef3c7;
|
||||
color: #92400e;
|
||||
}
|
||||
|
||||
.status-badge.status-success {
|
||||
background: #d1fae5;
|
||||
color: #065f46;
|
||||
}
|
||||
|
||||
.status-badge.status-error {
|
||||
background: #fee2e2;
|
||||
color: #991b1b;
|
||||
}
|
||||
|
||||
.status-badge.status-warning {
|
||||
background: #fef3c7;
|
||||
color: #92400e;
|
||||
}
|
||||
|
||||
.task-time {
|
||||
font-size: 11px;
|
||||
color: #999;
|
||||
}
|
||||
|
||||
.task-actions {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
}
|
||||
@@ -4,7 +4,7 @@ const API_BASE = '';
|
||||
// 状态
|
||||
let articles = [];
|
||||
let currentPage = 1;
|
||||
let pageSize = 100;
|
||||
let pageSize = 20; // 默认每页20条
|
||||
let totalCount = 0;
|
||||
let selectedIds = new Set();
|
||||
let currentArticleId = null;
|
||||
@@ -157,8 +157,14 @@ function displayArticles() {
|
||||
// 更新分页
|
||||
function updatePagination() {
|
||||
const totalPages = Math.ceil(totalCount / pageSize);
|
||||
document.getElementById('page-info').textContent = `第 ${currentPage} / ${totalPages || 1} 页`;
|
||||
|
||||
// 更新上部分页
|
||||
document.getElementById('page-info-top').textContent = `第 ${currentPage} / ${totalPages || 1} 页`;
|
||||
document.getElementById('prev-btn-top').disabled = currentPage <= 1;
|
||||
document.getElementById('next-btn-top').disabled = currentPage >= totalPages;
|
||||
|
||||
// 更新下部分页
|
||||
document.getElementById('page-info').textContent = `第 ${currentPage} / ${totalPages || 1} 页`;
|
||||
document.getElementById('prev-btn').disabled = currentPage <= 1;
|
||||
document.getElementById('next-btn').disabled = currentPage >= totalPages;
|
||||
}
|
||||
|
||||
+306
-43
@@ -5,8 +5,10 @@ const API_BASE = '';
|
||||
let searchResults = [];
|
||||
let currentResultIndex = -1;
|
||||
|
||||
// 自动流程控制
|
||||
let shouldStop = false;
|
||||
// 后台任务相关
|
||||
let currentTaskId = null;
|
||||
let taskPollInterval = null;
|
||||
let autoRefreshTasksInterval = null; // 自动刷新任务列表
|
||||
|
||||
// 页面加载初始化
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
@@ -17,8 +19,12 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
}
|
||||
});
|
||||
|
||||
// 加载失败URL
|
||||
// 加载失败URL和后台任务
|
||||
loadFailedUrls();
|
||||
loadBackgroundTasks();
|
||||
|
||||
// 启动自动刷新后台任务列表(每3秒)
|
||||
startAutoRefreshTasks();
|
||||
});
|
||||
|
||||
// 执行搜索
|
||||
@@ -95,7 +101,7 @@ async function doSearch() {
|
||||
document.getElementById('progress-text').textContent = '搜索完成!';
|
||||
document.getElementById('stop-btn').style.display = 'none'; // 隐藏停止按钮
|
||||
|
||||
// 自动抓取和保存
|
||||
// 自动抓取和保存(后台任务方式)
|
||||
if (autoFetch && searchResults.length > 0) {
|
||||
setTimeout(() => fetchAllResults(autoSave), 500);
|
||||
} else {
|
||||
@@ -358,60 +364,170 @@ async function saveAllResults() {
|
||||
showToast(`成功保存 ${saved} 条结果`, 'success');
|
||||
}
|
||||
|
||||
// 抓取所有结果
|
||||
// 抓取所有结果(后台任务方式)
|
||||
async function fetchAllResults(autoSave) {
|
||||
shouldStop = false; // 重置停止标志
|
||||
|
||||
const progress = document.getElementById('search-progress');
|
||||
progress.style.display = 'block';
|
||||
document.getElementById('stop-btn').style.display = 'inline-flex'; // 显示停止按钮
|
||||
document.getElementById('stop-btn').style.display = 'inline-flex';
|
||||
document.getElementById('progress-fill').style.width = '0%';
|
||||
document.getElementById('progress-text').textContent = '正在启动后台任务...';
|
||||
|
||||
const total = searchResults.length;
|
||||
let fetched = 0;
|
||||
// 准备结果数据
|
||||
const resultsToSend = searchResults.filter(r => !r.fetched).map(r => ({
|
||||
title: r.title,
|
||||
url: r.url
|
||||
}));
|
||||
|
||||
for (let i = 0; i < searchResults.length; i++) {
|
||||
// 检查是否停止
|
||||
if (shouldStop) {
|
||||
document.getElementById('progress-text').textContent = `已停止(已抓取 ${fetched}/${total})`;
|
||||
document.getElementById('stop-btn').style.display = 'none';
|
||||
setTimeout(() => {
|
||||
progress.style.display = 'none';
|
||||
}, 2000);
|
||||
showToast('自动抓取已停止', 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
if (searchResults[i].fetched) {
|
||||
fetched++;
|
||||
continue;
|
||||
}
|
||||
|
||||
document.getElementById('progress-text').textContent =
|
||||
`正在抓取 ${fetched + 1}/${total}...`;
|
||||
document.getElementById('progress-fill').style.width =
|
||||
`${(fetched / total) * 100}%`;
|
||||
|
||||
await fetchResult(i);
|
||||
fetched++;
|
||||
if (resultsToSend.length === 0) {
|
||||
document.getElementById('progress-text').textContent = '所有结果已抓取';
|
||||
document.getElementById('stop-btn').style.display = 'none';
|
||||
setTimeout(() => progress.style.display = 'none', 1000);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// 启动后台任务
|
||||
const response = await fetch(`${API_BASE}/api/tasks/start`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
results: resultsToSend,
|
||||
auto_save: autoSave,
|
||||
category: '',
|
||||
keywords: []
|
||||
})
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
currentTaskId = data.task_id;
|
||||
showToast('后台任务已启动', 'success');
|
||||
|
||||
// 开始轮询任务状态
|
||||
startTaskPolling(autoSave);
|
||||
} else {
|
||||
document.getElementById('stop-btn').style.display = 'none';
|
||||
showToast('启动后台任务失败: ' + data.error, 'error');
|
||||
progress.style.display = 'none';
|
||||
}
|
||||
} catch (error) {
|
||||
document.getElementById('stop-btn').style.display = 'none';
|
||||
showToast('启动任务出错', 'error');
|
||||
progress.style.display = 'none';
|
||||
console.error(error);
|
||||
}
|
||||
}
|
||||
|
||||
// 开始轮询任务状态
|
||||
function startTaskPolling(autoSave) {
|
||||
if (taskPollInterval) {
|
||||
clearInterval(taskPollInterval);
|
||||
}
|
||||
|
||||
taskPollInterval = setInterval(async () => {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/api/tasks/${currentTaskId}/status`);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
updateTaskProgress(data.task);
|
||||
|
||||
// 任务完成
|
||||
if (data.task.status === 'completed') {
|
||||
clearInterval(taskPollInterval);
|
||||
taskPollInterval = null;
|
||||
onTaskCompleted(data.task);
|
||||
} else if (data.task.status === 'failed') {
|
||||
clearInterval(taskPollInterval);
|
||||
taskPollInterval = null;
|
||||
showToast('任务失败: ' + data.task.error_message, 'error');
|
||||
document.getElementById('stop-btn').style.display = 'none';
|
||||
} else if (data.task.status === 'stopped') {
|
||||
clearInterval(taskPollInterval);
|
||||
taskPollInterval = null;
|
||||
showToast('任务已停止', 'warning');
|
||||
onTaskStopped(data.task);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('轮询任务状态失败:', error);
|
||||
}
|
||||
}, 1000); // 每秒轮询一次
|
||||
}
|
||||
|
||||
// 更新任务进度显示
|
||||
function updateTaskProgress(task) {
|
||||
const progress = document.getElementById('search-progress');
|
||||
const progressFill = document.getElementById('progress-fill');
|
||||
const progressText = document.getElementById('progress-text');
|
||||
|
||||
const total = task.total || 0;
|
||||
const current = task.progress || 0;
|
||||
const currentItem = task.current_item || '';
|
||||
|
||||
progressFill.style.width = total > 0 ? `${(current / total) * 100}%` : '0%';
|
||||
progressText.textContent = `正在抓取 ${current}/${total}... ${currentItem ? currentItem.substring(0, 30) : ''}`;
|
||||
}
|
||||
|
||||
// 任务完成处理
|
||||
function onTaskCompleted(task) {
|
||||
const progress = document.getElementById('search-progress');
|
||||
document.getElementById('progress-fill').style.width = '100%';
|
||||
document.getElementById('progress-text').textContent = '抓取完成!';
|
||||
|
||||
const result = task.result || {};
|
||||
document.getElementById('progress-text').textContent =
|
||||
`抓取完成!成功 ${result.success || 0},失败 ${result.failed || 0},已保存 ${result.saved || 0}`;
|
||||
document.getElementById('stop-btn').style.display = 'none';
|
||||
|
||||
setTimeout(() => {
|
||||
progress.style.display = 'none';
|
||||
}, 1000);
|
||||
}, 2000);
|
||||
|
||||
// 自动保存
|
||||
if (autoSave) {
|
||||
setTimeout(() => saveAllResults(), 500);
|
||||
}
|
||||
showToast(`抓取完成:成功 ${result.success || 0},失败 ${result.failed || 0}`, 'success');
|
||||
|
||||
// 刷新失败URL列表
|
||||
loadFailedUrls();
|
||||
|
||||
// 标记所有结果为已抓取
|
||||
searchResults.forEach(r => r.fetched = true);
|
||||
displayResults();
|
||||
}
|
||||
|
||||
// 停止自动处理
|
||||
function stopAutoProcess() {
|
||||
shouldStop = true;
|
||||
// 任务停止处理
|
||||
function onTaskStopped(task) {
|
||||
const progress = document.getElementById('search-progress');
|
||||
document.getElementById('progress-fill').style.width =
|
||||
task.total > 0 ? `${(task.progress / task.total) * 100}%` : '0%';
|
||||
document.getElementById('progress-text').textContent =
|
||||
`已停止(已处理 ${task.progress || 0}/${task.total || 0})`;
|
||||
document.getElementById('stop-btn').style.display = 'none';
|
||||
|
||||
setTimeout(() => {
|
||||
progress.style.display = 'none';
|
||||
}, 2000);
|
||||
|
||||
loadFailedUrls();
|
||||
}
|
||||
|
||||
// 停止自动处理(后台任务版本)
|
||||
async function stopAutoProcess() {
|
||||
if (!currentTaskId) return;
|
||||
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/api/tasks/${currentTaskId}/stop`, {
|
||||
method: 'POST'
|
||||
});
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
showToast('正在停止任务...', 'warning');
|
||||
} else {
|
||||
showToast('停止任务失败', 'error');
|
||||
}
|
||||
} catch (error) {
|
||||
showToast('停止任务出错', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 显示结果详情
|
||||
@@ -621,4 +737,151 @@ async function clearFailedUrls() {
|
||||
} catch (error) {
|
||||
showToast('清空失败', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// ========== 后台任务管理 ==========
|
||||
|
||||
// 加载后台任务列表
|
||||
async function loadBackgroundTasks() {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/api/tasks/recent?limit=10`);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
const container = document.getElementById('background-tasks-list');
|
||||
|
||||
if (data.tasks.length === 0) {
|
||||
container.innerHTML = '<div class="empty-text">暂无后台任务</div>';
|
||||
} else {
|
||||
container.innerHTML = data.tasks.map(task => {
|
||||
const statusClass = {
|
||||
'pending': 'status-pending',
|
||||
'running': 'status-running',
|
||||
'completed': 'status-success',
|
||||
'failed': 'status-error',
|
||||
'stopped': 'status-warning'
|
||||
}[task.status] || '';
|
||||
|
||||
const statusText = {
|
||||
'pending': '等待中',
|
||||
'running': '运行中',
|
||||
'completed': '已完成',
|
||||
'failed': '失败',
|
||||
'stopped': '已停止'
|
||||
}[task.status] || task.status;
|
||||
|
||||
const progressPercent = task.total > 0 ?
|
||||
Math.round((task.progress / task.total) * 100) : 0;
|
||||
|
||||
const result = task.result ? JSON.parse(task.result) : {};
|
||||
|
||||
return `
|
||||
<div class="background-task-item ${statusClass}">
|
||||
<div class="task-info">
|
||||
<div class="task-id">${escapeHtml(task.task_id)}</div>
|
||||
<div class="task-status">
|
||||
<span class="status-badge ${statusClass}">${statusText}</span>
|
||||
${task.status === 'running' ?
|
||||
`<span class="task-progress">${task.progress || 0}/${task.total || 0}</span>` : ''}
|
||||
${task.status === 'completed' && result.success ?
|
||||
`<span>成功 ${result.success},失败 ${result.failed},已保存 ${result.saved}</span>` : ''}
|
||||
</div>
|
||||
<div class="task-time">${task.created_at || ''}
|
||||
${task.finished_at ? ' → ' + task.finished_at : ''}
|
||||
</div>
|
||||
</div>
|
||||
<div class="task-actions">
|
||||
${task.status === 'running' ?
|
||||
`<button onclick="stopBackgroundTask('${task.task_id}')" class="btn btn-sm btn-danger">
|
||||
<i class="ri-stop-line"></i> 停止
|
||||
</button>` : ''}
|
||||
${task.status !== 'running' && task.status !== 'pending' ?
|
||||
`<button onclick="deleteBackgroundTask('${task.task_id}')" class="btn btn-sm btn-secondary">
|
||||
<i class="ri-delete-bin-line"></i>
|
||||
</button>` : ''}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('加载后台任务出错:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// 停止后台任务
|
||||
async function stopBackgroundTask(taskId) {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/api/tasks/${taskId}/stop`, {
|
||||
method: 'POST'
|
||||
});
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
showToast('任务已停止', 'success');
|
||||
loadBackgroundTasks();
|
||||
} else {
|
||||
showToast('停止失败: ' + data.error, 'error');
|
||||
}
|
||||
} catch (error) {
|
||||
showToast('停止任务出错', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 删除后台任务记录
|
||||
async function deleteBackgroundTask(taskId) {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/api/tasks/${taskId}`, {
|
||||
method: 'DELETE'
|
||||
});
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
showToast('任务已删除', 'success');
|
||||
loadBackgroundTasks();
|
||||
} else {
|
||||
showToast('删除失败: ' + data.error, 'error');
|
||||
}
|
||||
} catch (error) {
|
||||
showToast('删除任务出错', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 清理已完成的任务
|
||||
async function clearCompletedTasks() {
|
||||
try {
|
||||
const response = await fetch(`${API_BASE}/api/tasks/clear`, {
|
||||
method: 'POST'
|
||||
});
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
showToast(data.message, 'success');
|
||||
loadBackgroundTasks();
|
||||
} else {
|
||||
showToast('清理失败', 'error');
|
||||
}
|
||||
} catch (error) {
|
||||
showToast('清理任务出错', 'error');
|
||||
}
|
||||
}
|
||||
|
||||
// 启动自动刷新后台任务列表
|
||||
function startAutoRefreshTasks() {
|
||||
if (autoRefreshTasksInterval) {
|
||||
clearInterval(autoRefreshTasksInterval);
|
||||
}
|
||||
|
||||
autoRefreshTasksInterval = setInterval(() => {
|
||||
loadBackgroundTasks();
|
||||
}, 3000); // 每3秒刷新一次
|
||||
}
|
||||
|
||||
// 停止自动刷新
|
||||
function stopAutoRefreshTasks() {
|
||||
if (autoRefreshTasksInterval) {
|
||||
clearInterval(autoRefreshTasksInterval);
|
||||
autoRefreshTasksInterval = null;
|
||||
}
|
||||
}
|
||||
@@ -108,15 +108,15 @@
|
||||
<div class="panel-body">
|
||||
<div class="quick-stats">
|
||||
<div class="quick-stat-item">
|
||||
<span class="quick-stat-value" id="quick-articles-count">0</span>
|
||||
<span class="quick-stat-value" id="quick-articles-count">-</span>
|
||||
<span class="quick-stat-label">篇文章</span>
|
||||
</div>
|
||||
<div class="quick-stat-item">
|
||||
<span class="quick-stat-value" id="quick-categories-count">0</span>
|
||||
<span class="quick-stat-value" id="quick-categories-count">-</span>
|
||||
<span class="quick-stat-label">个分类</span>
|
||||
</div>
|
||||
<div class="quick-stat-item">
|
||||
<span class="quick-stat-value" id="quick-today-count">0</span>
|
||||
<span class="quick-stat-value" id="quick-today-count">-</span>
|
||||
<span class="quick-stat-label">今日新增</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -133,7 +133,7 @@
|
||||
<div class="panel-body">
|
||||
<div class="quick-stats">
|
||||
<div class="quick-stat-item">
|
||||
<span class="quick-stat-value" id="quick-failed-count">0</span>
|
||||
<span class="quick-stat-value" id="quick-failed-count">-</span>
|
||||
<span class="quick-stat-label">抓取失败</span>
|
||||
</div>
|
||||
<div class="quick-stat-item">
|
||||
|
||||
@@ -36,9 +36,9 @@
|
||||
<option value="">全部分类</option>
|
||||
</select>
|
||||
<select id="page-size" class="page-size-select" onchange="changePageSize()">
|
||||
<option value="20">每页 20 条</option>
|
||||
<option value="20" selected>每页 20 条</option>
|
||||
<option value="50">每页 50 条</option>
|
||||
<option value="100" selected>每页 100 条</option>
|
||||
<option value="100">每页 100 条</option>
|
||||
<option value="200">每页 200 条</option>
|
||||
<option value="500">每页 500 条</option>
|
||||
</select>
|
||||
@@ -69,6 +69,12 @@
|
||||
<i class="ri-delete-bin-line"></i> 批量删除
|
||||
</button>
|
||||
</div>
|
||||
<!-- 上部分页 -->
|
||||
<div class="pagination-top" id="pagination-top">
|
||||
<button onclick="prevPage()" class="btn btn-secondary btn-sm" id="prev-btn-top">上一页</button>
|
||||
<span id="page-info-top">第 1 页</span>
|
||||
<button onclick="nextPage()" class="btn btn-secondary btn-sm" id="next-btn-top">下一页</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="articles-body">
|
||||
<div id="articles-list" class="articles-list">
|
||||
|
||||
@@ -91,6 +91,26 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 后台任务区域 -->
|
||||
<div class="panel background-tasks-section">
|
||||
<div class="panel-header">
|
||||
<h2><i class="ri-cpu-line"></i> 后台抓取任务</h2>
|
||||
<div class="panel-actions">
|
||||
<button onclick="loadBackgroundTasks()" class="btn btn-secondary btn-sm">
|
||||
<i class="ri-refresh-line"></i> 刷新
|
||||
</button>
|
||||
<button onclick="clearCompletedTasks()" class="btn btn-warning btn-sm">
|
||||
<i class="ri-delete-bin-line"></i> 清理已完成
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div id="background-tasks-list" class="background-tasks-list">
|
||||
<div class="empty-text">暂无后台任务</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 失败URL区域 -->
|
||||
<div class="panel failed-urls-section">
|
||||
<div class="panel-header">
|
||||
|
||||
Reference in New Issue
Block a user