feat: 产品参数爬取系统 v1.0.0
功能: - 多步骤爬取流程（入口页→列表页→详情页） - 浏览器爬虫支持（Playwright，处理JS渲染） - 比亚迪汽车爬虫示例 - 后台管理界面 - 数据存储和导出 技术栈: - Python 3 + Flask - Playwright (浏览器自动化) - BeautifulSoup (HTML解析) 端口: - API服务: 19011 - 后台管理: 19012
This commit is contained in:
+185
@@ -0,0 +1,185 @@
|
||||
"""
|
||||
产品参数爬取系统 - 后台管理
|
||||
"""
|
||||
|
||||
from flask import Flask, render_template, jsonify, request
|
||||
from flask_cors import CORS
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
import asyncio
|
||||
|
||||
app = Flask(__name__)
|
||||
CORS(app)
|
||||
|
||||
BASE_DIR = Path(__file__).parent.parent
|
||||
DATA_DIR = BASE_DIR / 'data'
|
||||
PRODUCTS_FILE = DATA_DIR / 'products.json'
|
||||
TASKS_FILE = DATA_DIR / 'tasks.json'
|
||||
|
||||
|
||||
def load_products():
|
||||
if PRODUCTS_FILE.exists():
|
||||
return json.loads(PRODUCTS_FILE.read_text(encoding='utf-8'))
|
||||
return {"products": [], "last_update": None}
|
||||
|
||||
|
||||
def load_tasks():
|
||||
if TASKS_FILE.exists():
|
||||
return json.loads(TASKS_FILE.read_text(encoding='utf-8'))
|
||||
return {"tasks": []}
|
||||
|
||||
|
||||
# ============ 页面路由 ============
|
||||
|
||||
@app.route('/')
|
||||
def index():
|
||||
return render_template('index.html')
|
||||
|
||||
|
||||
@app.route('/products')
|
||||
def products_page():
|
||||
return render_template('products.html')
|
||||
|
||||
|
||||
@app.route('/spiders')
|
||||
def spiders_page():
|
||||
return render_template('spiders.html')
|
||||
|
||||
|
||||
@app.route('/tasks')
|
||||
def tasks_page():
|
||||
return render_template('tasks.html')
|
||||
|
||||
|
||||
@app.route('/config')
|
||||
def config_page():
|
||||
return render_template('config.html')
|
||||
|
||||
|
||||
# ============ API代理 ============
|
||||
|
||||
@app.route('/api/stats')
|
||||
def api_stats():
|
||||
"""获取统计信息"""
|
||||
data = load_products()
|
||||
tasks = load_tasks().get("tasks", [])
|
||||
|
||||
return jsonify({
|
||||
"total_products": len(data.get("products", [])),
|
||||
"last_update": data.get("last_update"),
|
||||
"total_tasks": len(tasks),
|
||||
"running_tasks": len([t for t in tasks if t.get("status") == "running"])
|
||||
})
|
||||
|
||||
|
||||
@app.route('/api/products')
|
||||
def api_products():
|
||||
"""获取产品列表"""
|
||||
data = load_products()
|
||||
return jsonify(data)
|
||||
|
||||
|
||||
@app.route('/api/spiders')
|
||||
def api_spiders():
|
||||
"""获取爬虫列表"""
|
||||
tasks = load_tasks().get("tasks", [])
|
||||
|
||||
spiders = [
|
||||
{
|
||||
"name": "byd",
|
||||
"display_name": "比亚迪汽车",
|
||||
"description": "爬取比亚迪官网车型参数",
|
||||
"url": "https://www.byd.com/cn/",
|
||||
"status": "available"
|
||||
}
|
||||
]
|
||||
|
||||
# 更新最后运行时间
|
||||
for spider in spiders:
|
||||
for task in reversed(tasks):
|
||||
if task.get("spider") == spider["name"]:
|
||||
spider["last_run"] = task.get("end_time")
|
||||
spider["last_status"] = task.get("status")
|
||||
break
|
||||
|
||||
return jsonify(spiders)
|
||||
|
||||
|
||||
@app.route('/api/tasks')
|
||||
def api_tasks():
|
||||
"""获取任务列表"""
|
||||
tasks = load_tasks().get("tasks", [])
|
||||
tasks.sort(key=lambda x: x.get("start_time", ""), reverse=True)
|
||||
return jsonify(tasks[:50])
|
||||
|
||||
|
||||
@app.route('/api/run/<spider_name>', methods=['POST'])
|
||||
def api_run_spider(spider_name):
|
||||
"""运行爬虫"""
|
||||
import sys
|
||||
sys.path.insert(0, str(BASE_DIR))
|
||||
|
||||
data = load_products()
|
||||
|
||||
async def run_spider():
|
||||
try:
|
||||
if spider_name == "byd":
|
||||
from spiders.byd import BYDSpider
|
||||
|
||||
spider = BYDSpider({"headless": True})
|
||||
results = await spider.run()
|
||||
|
||||
for item in results:
|
||||
existing = False
|
||||
for i, p in enumerate(data["products"]):
|
||||
if p.get("name") == item.get("name"):
|
||||
data["products"][i].update(item)
|
||||
data["products"][i]["updated_at"] = datetime.now().isoformat()
|
||||
existing = True
|
||||
break
|
||||
|
||||
if not existing:
|
||||
item["id"] = f"byd-{datetime.now().strftime('%Y%m%d%H%M%S')}-{len(data['products'])}"
|
||||
item["brand"] = "比亚迪"
|
||||
item["source"] = "byd.com"
|
||||
item["created_at"] = datetime.now().isoformat()
|
||||
data["products"].append(item)
|
||||
|
||||
PRODUCTS_FILE.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
return {"success": True, "count": len(results)}
|
||||
|
||||
return {"success": False, "error": f"Unknown spider: {spider_name}"}
|
||||
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
# 记录任务
|
||||
tasks = load_tasks()
|
||||
task = {
|
||||
"id": f"task-{datetime.now().strftime('%Y%m%d%H%M%S')}",
|
||||
"spider": spider_name,
|
||||
"status": "running",
|
||||
"start_time": datetime.now().isoformat()
|
||||
}
|
||||
tasks["tasks"].append(task)
|
||||
TASKS_FILE.write_text(json.dumps(tasks, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
|
||||
result = asyncio.run(run_spider())
|
||||
|
||||
task["status"] = "completed" if result.get("success") else "failed"
|
||||
task["end_time"] = datetime.now().isoformat()
|
||||
task["result"] = result
|
||||
TASKS_FILE.write_text(json.dumps(tasks, ensure_ascii=False, indent=2), encoding='utf-8')
|
||||
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("=" * 50)
|
||||
print("产品参数爬取系统 - 后台管理")
|
||||
print("=" * 50)
|
||||
print(f"访问地址: http://localhost:19012")
|
||||
print("=" * 50)
|
||||
|
||||
app.run(host='0.0.0.0', port=19012, debug=True)
|
||||
@@ -0,0 +1,102 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>系统配置 - 产品爬取系统</title>
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<link href="https://cdn.jsdelivr.net/npm/remixicon@3.5.0/fonts/remixicon.css" rel="stylesheet">
|
||||
</head>
|
||||
<body class="bg-gray-50 min-h-screen">
|
||||
<div class="flex">
|
||||
<!-- 侧边栏 -->
|
||||
<aside class="w-64 bg-slate-800 min-h-screen fixed left-0 top-0">
|
||||
<div class="p-6">
|
||||
<h1 class="text-white text-xl font-bold flex items-center gap-2">
|
||||
<i class="ri-spider-line text-2xl text-emerald-400"></i>
|
||||
Product Crawler
|
||||
</h1>
|
||||
<p class="text-slate-400 text-sm mt-1">产品爬取系统</p>
|
||||
</div>
|
||||
<nav class="mt-6">
|
||||
<a href="/" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-dashboard-line"></i><span>仪表盘</span>
|
||||
</a>
|
||||
<a href="/spiders" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-spider-line"></i><span>爬虫管理</span>
|
||||
</a>
|
||||
<a href="/products" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-product-hunt-line"></i><span>产品数据</span>
|
||||
</a>
|
||||
<a href="/tasks" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-task-line"></i><span>任务记录</span>
|
||||
</a>
|
||||
<a href="/config" class="flex items-center gap-3 px-6 py-3 bg-slate-700 text-white">
|
||||
<i class="ri-settings-3-line"></i><span>系统配置</span>
|
||||
</a>
|
||||
</nav>
|
||||
</aside>
|
||||
|
||||
<!-- 主内容 -->
|
||||
<main class="ml-64 flex-1 p-8">
|
||||
<div class="mb-8">
|
||||
<h1 class="text-2xl font-bold text-gray-800">系统配置</h1>
|
||||
<p class="text-gray-500 text-sm mt-1">爬虫系统配置信息</p>
|
||||
</div>
|
||||
|
||||
<div class="space-y-6">
|
||||
<!-- 服务配置 -->
|
||||
<div class="bg-white rounded-xl shadow-sm border border-gray-100 p-6">
|
||||
<h3 class="font-semibold text-gray-800 mb-4">服务配置</h3>
|
||||
<div class="grid grid-cols-2 gap-4 text-sm">
|
||||
<div class="bg-gray-50 p-3 rounded-lg">
|
||||
<div class="text-gray-500">API端口</div>
|
||||
<div class="font-medium text-gray-800">19011</div>
|
||||
</div>
|
||||
<div class="bg-gray-50 p-3 rounded-lg">
|
||||
<div class="text-gray-500">后台端口</div>
|
||||
<div class="font-medium text-gray-800">19012</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 爬虫配置 -->
|
||||
<div class="bg-white rounded-xl shadow-sm border border-gray-100 p-6">
|
||||
<h3 class="font-semibold text-gray-800 mb-4">爬虫配置</h3>
|
||||
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 text-sm">
|
||||
<div class="bg-gray-50 p-3 rounded-lg">
|
||||
<div class="text-gray-500">请求超时</div>
|
||||
<div class="font-medium text-gray-800">30秒</div>
|
||||
</div>
|
||||
<div class="bg-gray-50 p-3 rounded-lg">
|
||||
<div class="text-gray-500">重试次数</div>
|
||||
<div class="font-medium text-gray-800">3次</div>
|
||||
</div>
|
||||
<div class="bg-gray-50 p-3 rounded-lg">
|
||||
<div class="text-gray-500">请求间隔</div>
|
||||
<div class="font-medium text-gray-800">1秒</div>
|
||||
</div>
|
||||
<div class="bg-gray-50 p-3 rounded-lg">
|
||||
<div class="text-gray-500">并发限制</div>
|
||||
<div class="font-medium text-gray-800">3</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 使用说明 -->
|
||||
<div class="bg-blue-50 rounded-xl p-6">
|
||||
<h3 class="font-semibold text-blue-800 mb-4">
|
||||
<i class="ri-information-line mr-1"></i> 使用说明
|
||||
</h3>
|
||||
<div class="text-sm text-blue-700 space-y-2">
|
||||
<p>1. 进入「爬虫管理」页面，选择要运行的爬虫</p>
|
||||
<p>2. 点击「运行」按钮开始爬取数据</p>
|
||||
<p>3. 爬取完成后，在「产品数据」页面查看结果</p>
|
||||
<p>4. 支持导出JSON/CSV格式的数据</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,199 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>产品爬取系统 - 后台管理</title>
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<link href="https://cdn.jsdelivr.net/npm/remixicon@3.5.0/fonts/remixicon.css" rel="stylesheet">
|
||||
<style>
|
||||
.gradient-bg { background: linear-gradient(135deg, #10b981 0%, #059669 100%); }
|
||||
</style>
|
||||
</head>
|
||||
<body class="bg-gray-50 min-h-screen">
|
||||
<div class="flex">
|
||||
<!-- 侧边栏 -->
|
||||
<aside class="w-64 bg-slate-800 min-h-screen fixed left-0 top-0">
|
||||
<div class="p-6">
|
||||
<h1 class="text-white text-xl font-bold flex items-center gap-2">
|
||||
<i class="ri-spider-line text-2xl text-emerald-400"></i>
|
||||
Product Crawler
|
||||
</h1>
|
||||
<p class="text-slate-400 text-sm mt-1">产品爬取系统</p>
|
||||
</div>
|
||||
<nav class="mt-6">
|
||||
<a href="/" class="flex items-center gap-3 px-6 py-3 bg-slate-700 text-white">
|
||||
<i class="ri-dashboard-line"></i><span>仪表盘</span>
|
||||
</a>
|
||||
<a href="/spiders" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-spider-line"></i><span>爬虫管理</span>
|
||||
</a>
|
||||
<a href="/products" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-product-hunt-line"></i><span>产品数据</span>
|
||||
</a>
|
||||
<a href="/tasks" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-task-line"></i><span>任务记录</span>
|
||||
</a>
|
||||
<a href="/config" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-settings-3-line"></i><span>系统配置</span>
|
||||
</a>
|
||||
</nav>
|
||||
</aside>
|
||||
|
||||
<!-- 主内容 -->
|
||||
<main class="ml-64 flex-1 p-8">
|
||||
<!-- 顶部 -->
|
||||
<div class="mb-8">
|
||||
<h1 class="text-2xl font-bold text-gray-800">仪表盘</h1>
|
||||
<p class="text-gray-500 text-sm mt-1">产品参数爬取系统概览</p>
|
||||
</div>
|
||||
|
||||
<!-- 统计卡片 -->
|
||||
<div class="grid grid-cols-1 md:grid-cols-4 gap-6 mb-8">
|
||||
<div class="bg-white rounded-xl p-6 shadow-sm border border-gray-100">
|
||||
<div class="flex items-center justify-between">
|
||||
<div>
|
||||
<p class="text-gray-500 text-sm">产品总数</p>
|
||||
<p id="statProducts" class="text-3xl font-bold text-gray-800 mt-1">-</p>
|
||||
</div>
|
||||
<div class="w-12 h-12 bg-emerald-100 rounded-xl flex items-center justify-center">
|
||||
<i class="ri-product-hunt-line text-2xl text-emerald-600"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-white rounded-xl p-6 shadow-sm border border-gray-100">
|
||||
<div class="flex items-center justify-between">
|
||||
<div>
|
||||
<p class="text-gray-500 text-sm">爬虫数量</p>
|
||||
<p id="statSpiders" class="text-3xl font-bold text-gray-800 mt-1">1</p>
|
||||
</div>
|
||||
<div class="w-12 h-12 bg-blue-100 rounded-xl flex items-center justify-center">
|
||||
<i class="ri-spider-line text-2xl text-blue-600"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-white rounded-xl p-6 shadow-sm border border-gray-100">
|
||||
<div class="flex items-center justify-between">
|
||||
<div>
|
||||
<p class="text-gray-500 text-sm">任务总数</p>
|
||||
<p id="statTasks" class="text-3xl font-bold text-gray-800 mt-1">-</p>
|
||||
</div>
|
||||
<div class="w-12 h-12 bg-purple-100 rounded-xl flex items-center justify-center">
|
||||
<i class="ri-task-line text-2xl text-purple-600"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-white rounded-xl p-6 shadow-sm border border-gray-100">
|
||||
<div class="flex items-center justify-between">
|
||||
<div>
|
||||
<p class="text-gray-500 text-sm">最后更新</p>
|
||||
<p id="statUpdate" class="text-lg font-bold text-gray-800 mt-1">-</p>
|
||||
</div>
|
||||
<div class="w-12 h-12 bg-orange-100 rounded-xl flex items-center justify-center">
|
||||
<i class="ri-time-line text-2xl text-orange-600"></i>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 快速操作 -->
|
||||
<div class="bg-white rounded-xl shadow-sm border border-gray-100 mb-8">
|
||||
<div class="p-6 border-b">
|
||||
<h2 class="font-semibold text-gray-800">快速操作</h2>
|
||||
</div>
|
||||
<div class="p-6">
|
||||
<div class="grid grid-cols-1 md:grid-cols-3 gap-4">
|
||||
<a href="/spiders" class="flex items-center gap-4 p-4 border border-gray-200 rounded-xl hover:border-emerald-300 hover:bg-emerald-50 transition">
|
||||
<div class="w-10 h-10 bg-emerald-100 rounded-lg flex items-center justify-center">
|
||||
<i class="ri-play-line text-xl text-emerald-600"></i>
|
||||
</div>
|
||||
<div>
|
||||
<div class="font-medium text-gray-800">运行爬虫</div>
|
||||
<div class="text-sm text-gray-500">执行数据爬取</div>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
<a href="/products" class="flex items-center gap-4 p-4 border border-gray-200 rounded-xl hover:border-blue-300 hover:bg-blue-50 transition">
|
||||
<div class="w-10 h-10 bg-blue-100 rounded-lg flex items-center justify-center">
|
||||
<i class="ri-eye-line text-xl text-blue-600"></i>
|
||||
</div>
|
||||
<div>
|
||||
<div class="font-medium text-gray-800">查看数据</div>
|
||||
<div class="text-sm text-gray-500">浏览爬取结果</div>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
<a href="/tasks" class="flex items-center gap-4 p-4 border border-gray-200 rounded-xl hover:border-purple-300 hover:bg-purple-50 transition">
|
||||
<div class="w-10 h-10 bg-purple-100 rounded-lg flex items-center justify-center">
|
||||
<i class="ri-history-line text-xl text-purple-600"></i>
|
||||
</div>
|
||||
<div>
|
||||
<div class="font-medium text-gray-800">任务记录</div>
|
||||
<div class="text-sm text-gray-500">查看执行历史</div>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 最近任务 -->
|
||||
<div class="bg-white rounded-xl shadow-sm border border-gray-100">
|
||||
<div class="p-6 border-b flex justify-between items-center">
|
||||
<h2 class="font-semibold text-gray-800">最近任务</h2>
|
||||
<a href="/tasks" class="text-sm text-emerald-600 hover:text-emerald-700">查看全部</a>
|
||||
</div>
|
||||
<div id="recentTasks" class="p-6">
|
||||
<p class="text-gray-400 text-center py-4">加载中...</p>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
async function loadStats() {
|
||||
const res = await fetch('/api/stats');
|
||||
const stats = await res.json();
|
||||
|
||||
document.getElementById('statProducts').textContent = stats.total_products;
|
||||
document.getElementById('statTasks').textContent = stats.total_tasks;
|
||||
document.getElementById('statUpdate').textContent = stats.last_update ?
|
||||
new Date(stats.last_update).toLocaleString() : '-';
|
||||
}
|
||||
|
||||
async function loadRecentTasks() {
|
||||
const res = await fetch('/api/tasks');
|
||||
const tasks = await res.json();
|
||||
|
||||
const container = document.getElementById('recentTasks');
|
||||
|
||||
if (tasks.length === 0) {
|
||||
container.innerHTML = '<p class="text-gray-400 text-center py-4">暂无任务记录</p>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = tasks.slice(0, 5).map(task => `
|
||||
<div class="flex items-center justify-between py-3 border-b last:border-0">
|
||||
<div class="flex items-center gap-3">
|
||||
<div class="w-8 h-8 rounded-full ${task.status === 'completed' ? 'bg-green-100' : task.status === 'failed' ? 'bg-red-100' : 'bg-yellow-100'} flex items-center justify-center">
|
||||
<i class="ri-${task.status === 'completed' ? 'check text-green-600' : task.status === 'failed' ? 'close text-red-600' : 'loader text-yellow-600'}"></i>
|
||||
</div>
|
||||
<div>
|
||||
<div class="font-medium text-gray-800">${task.spider}</div>
|
||||
<div class="text-sm text-gray-500">${new Date(task.start_time).toLocaleString()}</div>
|
||||
</div>
|
||||
</div>
|
||||
<span class="px-2 py-1 text-xs rounded ${task.status === 'completed' ? 'bg-green-100 text-green-700' : task.status === 'failed' ? 'bg-red-100 text-red-700' : 'bg-yellow-100 text-yellow-700'}">
|
||||
${task.status === 'completed' ? '完成' : task.status === 'failed' ? '失败' : '运行中'}
|
||||
</span>
|
||||
</div>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
loadStats();
|
||||
loadRecentTasks();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,142 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>产品数据 - 产品爬取系统</title>
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<link href="https://cdn.jsdelivr.net/npm/remixicon@3.5.0/fonts/remixicon.css" rel="stylesheet">
|
||||
<style>
|
||||
.gradient-bg { background: linear-gradient(135deg, #10b981 0%, #059669 100%); }
|
||||
</style>
|
||||
</head>
|
||||
<body class="bg-gray-50 min-h-screen">
|
||||
<div class="flex">
|
||||
<!-- 侧边栏 -->
|
||||
<aside class="w-64 bg-slate-800 min-h-screen fixed left-0 top-0">
|
||||
<div class="p-6">
|
||||
<h1 class="text-white text-xl font-bold flex items-center gap-2">
|
||||
<i class="ri-spider-line text-2xl text-emerald-400"></i>
|
||||
Product Crawler
|
||||
</h1>
|
||||
<p class="text-slate-400 text-sm mt-1">产品爬取系统</p>
|
||||
</div>
|
||||
<nav class="mt-6">
|
||||
<a href="/" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-dashboard-line"></i><span>仪表盘</span>
|
||||
</a>
|
||||
<a href="/spiders" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-spider-line"></i><span>爬虫管理</span>
|
||||
</a>
|
||||
<a href="/products" class="flex items-center gap-3 px-6 py-3 bg-slate-700 text-white">
|
||||
<i class="ri-product-hunt-line"></i><span>产品数据</span>
|
||||
</a>
|
||||
<a href="/tasks" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-task-line"></i><span>任务记录</span>
|
||||
</a>
|
||||
<a href="/config" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-settings-3-line"></i><span>系统配置</span>
|
||||
</a>
|
||||
</nav>
|
||||
</aside>
|
||||
|
||||
<!-- 主内容 -->
|
||||
<main class="ml-64 flex-1 p-8">
|
||||
<div class="flex justify-between items-center mb-8">
|
||||
<div>
|
||||
<h1 class="text-2xl font-bold text-gray-800">产品数据</h1>
|
||||
<p class="text-gray-500 text-sm mt-1">查看爬取的产品参数数据</p>
|
||||
</div>
|
||||
<div class="flex gap-2">
|
||||
<input type="text" id="searchInput" placeholder="搜索产品..."
|
||||
class="px-4 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-emerald-500 focus:border-emerald-500">
|
||||
<button onclick="exportData()" class="px-4 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition">
|
||||
<i class="ri-download-line"></i> 导出
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 产品列表 -->
|
||||
<div class="bg-white rounded-xl shadow-sm border border-gray-100">
|
||||
<div id="productList" class="divide-y">
|
||||
<p class="text-gray-400 text-center py-8">加载中...</p>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
let products = [];
|
||||
|
||||
async function loadProducts() {
|
||||
const res = await fetch('/api/products');
|
||||
const data = await res.json();
|
||||
products = data.products || [];
|
||||
renderProducts(products);
|
||||
}
|
||||
|
||||
function renderProducts(list) {
|
||||
const container = document.getElementById('productList');
|
||||
|
||||
if (list.length === 0) {
|
||||
container.innerHTML = '<p class="text-gray-400 text-center py-8">暂无产品数据</p>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = list.map(p => `
|
||||
<div class="p-4 hover:bg-gray-50 cursor-pointer" onclick="toggleDetail('${p.id}')">
|
||||
<div class="flex items-center justify-between">
|
||||
<div class="flex items-center gap-4">
|
||||
<div class="w-12 h-12 bg-gray-100 rounded-lg flex items-center justify-center">
|
||||
<i class="ri-car-line text-2xl text-gray-400"></i>
|
||||
</div>
|
||||
<div>
|
||||
<h3 class="font-medium text-gray-800">${p.name}</h3>
|
||||
<div class="flex items-center gap-2 mt-1 text-sm text-gray-500">
|
||||
<span>${p.brand || '-'}</span>
|
||||
<span>·</span>
|
||||
<span>${Object.keys(p.params || {}).length} 个参数</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex items-center gap-4">
|
||||
<span class="text-sm text-gray-400">${p.updated_at ? new Date(p.updated_at).toLocaleDateString() : '-'}</span>
|
||||
<i class="ri-arrow-down-s-line text-gray-400"></i>
|
||||
</div>
|
||||
</div>
|
||||
<div id="detail-${p.id}" class="hidden mt-4 pt-4 border-t">
|
||||
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 text-sm">
|
||||
${Object.entries(p.params || {}).map(([k, v]) => `
|
||||
<div class="bg-gray-50 rounded p-2">
|
||||
<div class="text-gray-500">${k}</div>
|
||||
<div class="font-medium text-gray-800">${v || '-'}</div>
|
||||
</div>
|
||||
`).join('')}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
function toggleDetail(id) {
|
||||
const detail = document.getElementById(`detail-${id}`);
|
||||
detail.classList.toggle('hidden');
|
||||
}
|
||||
|
||||
function exportData() {
|
||||
window.open('/api/export?format=json', '_blank');
|
||||
}
|
||||
|
||||
document.getElementById('searchInput').addEventListener('input', (e) => {
|
||||
const keyword = e.target.value.toLowerCase();
|
||||
const filtered = products.filter(p =>
|
||||
p.name?.toLowerCase().includes(keyword) ||
|
||||
p.brand?.toLowerCase().includes(keyword)
|
||||
);
|
||||
renderProducts(filtered);
|
||||
});
|
||||
|
||||
loadProducts();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,136 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>爬虫管理 - 产品爬取系统</title>
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<link href="https://cdn.jsdelivr.net/npm/remixicon@3.5.0/fonts/remixicon.css" rel="stylesheet">
|
||||
<style>
|
||||
.gradient-bg { background: linear-gradient(135deg, #10b981 0%, #059669 100%); }
|
||||
</style>
|
||||
</head>
|
||||
<body class="bg-gray-50 min-h-screen">
|
||||
<div class="flex">
|
||||
<!-- 侧边栏 -->
|
||||
<aside class="w-64 bg-slate-800 min-h-screen fixed left-0 top-0">
|
||||
<div class="p-6">
|
||||
<h1 class="text-white text-xl font-bold flex items-center gap-2">
|
||||
<i class="ri-spider-line text-2xl text-emerald-400"></i>
|
||||
Product Crawler
|
||||
</h1>
|
||||
<p class="text-slate-400 text-sm mt-1">产品爬取系统</p>
|
||||
</div>
|
||||
<nav class="mt-6">
|
||||
<a href="/" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-dashboard-line"></i><span>仪表盘</span>
|
||||
</a>
|
||||
<a href="/spiders" class="flex items-center gap-3 px-6 py-3 bg-slate-700 text-white">
|
||||
<i class="ri-spider-line"></i><span>爬虫管理</span>
|
||||
</a>
|
||||
<a href="/products" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-product-hunt-line"></i><span>产品数据</span>
|
||||
</a>
|
||||
<a href="/tasks" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-task-line"></i><span>任务记录</span>
|
||||
</a>
|
||||
<a href="/config" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-settings-3-line"></i><span>系统配置</span>
|
||||
</a>
|
||||
</nav>
|
||||
</aside>
|
||||
|
||||
<!-- 主内容 -->
|
||||
<main class="ml-64 flex-1 p-8">
|
||||
<div class="mb-8">
|
||||
<h1 class="text-2xl font-bold text-gray-800">爬虫管理</h1>
|
||||
<p class="text-gray-500 text-sm mt-1">配置和运行爬虫任务</p>
|
||||
</div>
|
||||
|
||||
<!-- 爬虫列表 -->
|
||||
<div id="spiderList" class="space-y-4">
|
||||
<p class="text-gray-400 text-center py-8">加载中...</p>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
async function loadSpiders() {
|
||||
const res = await fetch('/api/spiders');
|
||||
const spiders = await res.json();
|
||||
|
||||
const container = document.getElementById('spiderList');
|
||||
|
||||
if (spiders.length === 0) {
|
||||
container.innerHTML = '<p class="text-gray-400 text-center py-8">暂无爬虫配置</p>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = spiders.map(spider => `
|
||||
<div class="bg-white rounded-xl shadow-sm border border-gray-100 p-6">
|
||||
<div class="flex items-start justify-between">
|
||||
<div class="flex items-start gap-4">
|
||||
<div class="w-12 h-12 bg-emerald-100 rounded-xl flex items-center justify-center">
|
||||
<i class="ri-spider-line text-2xl text-emerald-600"></i>
|
||||
</div>
|
||||
<div>
|
||||
<h3 class="font-semibold text-gray-800">${spider.display_name}</h3>
|
||||
<p class="text-sm text-gray-500 mt-1">${spider.description}</p>
|
||||
<div class="flex items-center gap-4 mt-3 text-sm">
|
||||
<span class="text-gray-400">
|
||||
<i class="ri-link"></i> ${spider.url || '-'}
|
||||
</span>
|
||||
${spider.last_run ? `
|
||||
<span class="text-gray-400">
|
||||
<i class="ri-time-line"></i> 最后运行: ${new Date(spider.last_run).toLocaleString()}
|
||||
</span>
|
||||
` : ''}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="px-3 py-1 text-xs rounded-full ${spider.status === 'running' ? 'bg-yellow-100 text-yellow-700' : 'bg-green-100 text-green-700'}">
|
||||
${spider.status === 'running' ? '运行中' : '可用'}
|
||||
</span>
|
||||
<button onclick="runSpider('${spider.name}')"
|
||||
class="px-4 py-2 gradient-bg text-white rounded-lg hover:opacity-90 transition flex items-center gap-2"
|
||||
${spider.status === 'running' ? 'disabled' : ''}>
|
||||
<i class="ri-play-line"></i>
|
||||
<span>${spider.status === 'running' ? '运行中...' : '运行'}</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
async function runSpider(name) {
|
||||
if (!confirm(`确定运行爬虫 "${name}" 吗？`)) return;
|
||||
|
||||
const btn = event.target.closest('button');
|
||||
btn.disabled = true;
|
||||
btn.innerHTML = '<i class="ri-loader-4-line animate-spin"></i> 运行中...';
|
||||
|
||||
try {
|
||||
const res = await fetch(`/api/run/${name}`, { method: 'POST' });
|
||||
const result = await res.json();
|
||||
|
||||
if (result.success) {
|
||||
alert(`爬取完成！获取 ${result.count} 条数据`);
|
||||
loadSpiders();
|
||||
} else {
|
||||
alert('爬取失败: ' + (result.error || '未知错误'));
|
||||
btn.disabled = false;
|
||||
btn.innerHTML = '<i class="ri-play-line"></i> 运行';
|
||||
}
|
||||
} catch (e) {
|
||||
alert('请求失败: ' + e.message);
|
||||
btn.disabled = false;
|
||||
btn.innerHTML = '<i class="ri-play-line"></i> 运行';
|
||||
}
|
||||
}
|
||||
|
||||
loadSpiders();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,102 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>任务记录 - 产品爬取系统</title>
|
||||
<script src="https://cdn.tailwindcss.com"></script>
|
||||
<link href="https://cdn.jsdelivr.net/npm/remixicon@3.5.0/fonts/remixicon.css" rel="stylesheet">
|
||||
</head>
|
||||
<body class="bg-gray-50 min-h-screen">
|
||||
<div class="flex">
|
||||
<!-- 侧边栏 -->
|
||||
<aside class="w-64 bg-slate-800 min-h-screen fixed left-0 top-0">
|
||||
<div class="p-6">
|
||||
<h1 class="text-white text-xl font-bold flex items-center gap-2">
|
||||
<i class="ri-spider-line text-2xl text-emerald-400"></i>
|
||||
Product Crawler
|
||||
</h1>
|
||||
<p class="text-slate-400 text-sm mt-1">产品爬取系统</p>
|
||||
</div>
|
||||
<nav class="mt-6">
|
||||
<a href="/" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-dashboard-line"></i><span>仪表盘</span>
|
||||
</a>
|
||||
<a href="/spiders" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-spider-line"></i><span>爬虫管理</span>
|
||||
</a>
|
||||
<a href="/products" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-product-hunt-line"></i><span>产品数据</span>
|
||||
</a>
|
||||
<a href="/tasks" class="flex items-center gap-3 px-6 py-3 bg-slate-700 text-white">
|
||||
<i class="ri-task-line"></i><span>任务记录</span>
|
||||
</a>
|
||||
<a href="/config" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
|
||||
<i class="ri-settings-3-line"></i><span>系统配置</span>
|
||||
</a>
|
||||
</nav>
|
||||
</aside>
|
||||
|
||||
<!-- 主内容 -->
|
||||
<main class="ml-64 flex-1 p-8">
|
||||
<div class="mb-8">
|
||||
<h1 class="text-2xl font-bold text-gray-800">任务记录</h1>
|
||||
<p class="text-gray-500 text-sm mt-1">查看爬虫任务执行历史</p>
|
||||
</div>
|
||||
|
||||
<!-- 任务列表 -->
|
||||
<div class="bg-white rounded-xl shadow-sm border border-gray-100">
|
||||
<div id="taskList" class="divide-y">
|
||||
<p class="text-gray-400 text-center py-8">加载中...</p>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
async function loadTasks() {
|
||||
const res = await fetch('/api/tasks');
|
||||
const tasks = await res.json();
|
||||
|
||||
const container = document.getElementById('taskList');
|
||||
|
||||
if (tasks.length === 0) {
|
||||
container.innerHTML = '<p class="text-gray-400 text-center py-8">暂无任务记录</p>';
|
||||
return;
|
||||
}
|
||||
|
||||
container.innerHTML = tasks.map(task => `
|
||||
<div class="p-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<div class="flex items-center gap-4">
|
||||
<div class="w-10 h-10 rounded-full ${task.status === 'completed' ? 'bg-green-100' : task.status === 'failed' ? 'bg-red-100' : 'bg-yellow-100'} flex items-center justify-center">
|
||||
<i class="ri-${task.status === 'completed' ? 'check text-green-600' : task.status === 'failed' ? 'close text-red-600' : 'loader text-yellow-600'}"></i>
|
||||
</div>
|
||||
<div>
|
||||
<div class="font-medium text-gray-800">爬虫: ${task.spider}</div>
|
||||
<div class="text-sm text-gray-500">
|
||||
${new Date(task.start_time).toLocaleString()}
|
||||
${task.end_time ? ' → ' + new Date(task.end_time).toLocaleTimeString() : ''}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex items-center gap-3">
|
||||
<span class="px-3 py-1 text-xs rounded-full ${task.status === 'completed' ? 'bg-green-100 text-green-700' : task.status === 'failed' ? 'bg-red-100 text-red-700' : 'bg-yellow-100 text-yellow-700'}">
|
||||
${task.status === 'completed' ? '完成' : task.status === 'failed' ? '失败' : '运行中'}
|
||||
</span>
|
||||
${task.result ? `<span class="text-sm text-gray-500">${task.result.count || 0} 条数据</span>` : ''}
|
||||
</div>
|
||||
</div>
|
||||
${task.result?.error ? `
|
||||
<div class="mt-2 p-2 bg-red-50 text-red-600 text-sm rounded">
|
||||
${task.result.error}
|
||||
</div>
|
||||
` : ''}
|
||||
</div>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
loadTasks();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Reference in New Issue
Block a user