coder/product-crawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 产品参数爬取系统 v1.0.0

Browse Source 
功能:
- 多步骤爬取流程（入口页→列表页→详情页）
- 浏览器爬虫支持（Playwright，处理JS渲染）
- 比亚迪汽车爬虫示例
- 后台管理界面
- 数据存储和导出

技术栈:
- Python 3 + Flask
- Playwright (浏览器自动化)
- BeautifulSoup (HTML解析)

端口:
- API服务: 19011
- 后台管理: 19012
This commit is contained in:
hubian
2026-04-10 00:45:51 +08:00
commit 0ee0abbbd1
16 changed files with 1847 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
admin/templates/products.html Normal file
View File

@@ -0,0 +1,142 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>产品数据 - 产品爬取系统</title>
<script src="https://cdn.tailwindcss.com"></script>
<link href="https://cdn.jsdelivr.net/npm/remixicon@3.5.0/fonts/remixicon.css" rel="stylesheet">
<style>
.gradient-bg { background: linear-gradient(135deg, #10b981 0%, #059669 100%); }
</style>
</head>
<body class="bg-gray-50 min-h-screen">
<div class="flex">
<!-- 侧边栏 -->
<aside class="w-64 bg-slate-800 min-h-screen fixed left-0 top-0">
<div class="p-6">
<h1 class="text-white text-xl font-bold flex items-center gap-2">
<i class="ri-spider-line text-2xl text-emerald-400"></i>
Product Crawler
</h1>
<p class="text-slate-400 text-sm mt-1">产品爬取系统</p>
</div>
<nav class="mt-6">
<a href="/" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
<i class="ri-dashboard-line"></i><span>仪表盘</span>
</a>
<a href="/spiders" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
<i class="ri-spider-line"></i><span>爬虫管理</span>
</a>
<a href="/products" class="flex items-center gap-3 px-6 py-3 bg-slate-700 text-white">
<i class="ri-product-hunt-line"></i><span>产品数据</span>
</a>
<a href="/tasks" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
<i class="ri-task-line"></i><span>任务记录</span>
</a>
<a href="/config" class="flex items-center gap-3 px-6 py-3 text-slate-300 hover:bg-slate-700 hover:text-white">
<i class="ri-settings-3-line"></i><span>系统配置</span>
</a>
</nav>
</aside>
<!-- 主内容 -->
<main class="ml-64 flex-1 p-8">
<div class="flex justify-between items-center mb-8">
<div>
<h1 class="text-2xl font-bold text-gray-800">产品数据</h1>
<p class="text-gray-500 text-sm mt-1">查看爬取的产品参数数据</p>
</div>
<div class="flex gap-2">
<input type="text" id="searchInput" placeholder="搜索产品..."
class="px-4 py-2 border border-gray-300 rounded-lg focus:ring-2 focus:ring-emerald-500 focus:border-emerald-500">
<button onclick="exportData()" class="px-4 py-2 bg-blue-600 text-white rounded-lg hover:bg-blue-700 transition">
<i class="ri-download-line"></i> 导出
</button>
</div>
</div>
<!-- 产品列表 -->
<div class="bg-white rounded-xl shadow-sm border border-gray-100">
<div id="productList" class="divide-y">
<p class="text-gray-400 text-center py-8">加载中...</p>
</div>
</div>
</main>
</div>
<script>
let products = [];
async function loadProducts() {
const res = await fetch('/api/products');
const data = await res.json();
products = data.products || [];
renderProducts(products);
}
function renderProducts(list) {
const container = document.getElementById('productList');
if (list.length === 0) {
container.innerHTML = '<p class="text-gray-400 text-center py-8">暂无产品数据</p>';
return;
}
container.innerHTML = list.map(p => `
<div class="p-4 hover:bg-gray-50 cursor-pointer" onclick="toggleDetail('${p.id}')">
<div class="flex items-center justify-between">
<div class="flex items-center gap-4">
<div class="w-12 h-12 bg-gray-100 rounded-lg flex items-center justify-center">
<i class="ri-car-line text-2xl text-gray-400"></i>
</div>
<div>
<h3 class="font-medium text-gray-800">${p.name}</h3>
<div class="flex items-center gap-2 mt-1 text-sm text-gray-500">
<span>${p.brand || '-'}</span>
<span>·</span>
<span>${Object.keys(p.params || {}).length} 个参数</span>
</div>
</div>
</div>
<div class="flex items-center gap-4">
<span class="text-sm text-gray-400">${p.updated_at ? new Date(p.updated_at).toLocaleDateString() : '-'}</span>
<i class="ri-arrow-down-s-line text-gray-400"></i>
</div>
</div>
<div id="detail-${p.id}" class="hidden mt-4 pt-4 border-t">
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 text-sm">
${Object.entries(p.params || {}).map(([k, v]) => `
<div class="bg-gray-50 rounded p-2">
<div class="text-gray-500">${k}</div>
<div class="font-medium text-gray-800">${v || '-'}</div>
</div>
`).join('')}
</div>
</div>
</div>
`).join('');
}
function toggleDetail(id) {
const detail = document.getElementById(`detail-${id}`);
detail.classList.toggle('hidden');
}
function exportData() {
window.open('/api/export?format=json', '_blank');
}
document.getElementById('searchInput').addEventListener('input', (e) => {
const keyword = e.target.value.toLowerCase();
const filtered = products.filter(p =>
p.name?.toLowerCase().includes(keyword) ||
p.brand?.toLowerCase().includes(keyword)
);
renderProducts(filtered);
});
loadProducts();
</script>
</body>
</html>