hz4th_coder
898c2407e9
- 新增 failed_urls 数据库表存储失败URL - 抓取失败时自动记录URL、标题、错误信息 - 搜索页面显示失败URL列表 - 支持重试单个/全部失败URL - 支持删除和清空失败记录 - 显示重试次数和时间
573 lines
20 KiB
JavaScript
573 lines
20 KiB
JavaScript
// API基础地址
|
|
const API_BASE = '';
|
|
|
|
// 搜索结果存储
|
|
let searchResults = [];
|
|
let currentResultIndex = -1;
|
|
|
|
// 页面加载初始化
|
|
document.addEventListener('DOMContentLoaded', () => {
|
|
// 回车搜索
|
|
document.getElementById('search-keyword').addEventListener('keypress', (e) => {
|
|
if (e.key === 'Enter') {
|
|
doSearch();
|
|
}
|
|
});
|
|
|
|
// 加载失败URL
|
|
loadFailedUrls();
|
|
});
|
|
|
|
// 执行搜索
|
|
async function doSearch() {
|
|
const keyword = document.getElementById('search-keyword').value.trim();
|
|
const maxResults = parseInt(document.getElementById('search-count').value) || 10;
|
|
const engine = document.getElementById('search-engine').value;
|
|
const autoFetch = document.getElementById('auto-fetch').checked;
|
|
const autoSave = document.getElementById('auto-save').checked;
|
|
|
|
if (!keyword) {
|
|
showToast('请输入搜索关键词', 'error');
|
|
return;
|
|
}
|
|
|
|
// 更新状态
|
|
const engineNames = {
|
|
'bing_cn': 'Bing 中国',
|
|
'bing_global': 'Bing 国际',
|
|
'google': 'Google',
|
|
'baidu': '百度'
|
|
};
|
|
updateStatus('searching', `正在通过 ${engineNames[engine]} 搜索...`);
|
|
document.getElementById('search-btn').disabled = true;
|
|
|
|
// 显示进度
|
|
const progress = document.getElementById('search-progress');
|
|
progress.style.display = 'block';
|
|
document.getElementById('progress-fill').style.width = '0%';
|
|
document.getElementById('progress-text').textContent = '正在搜索...';
|
|
|
|
try {
|
|
const response = await fetch(`${API_BASE}/api/articles/internet-search`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ keyword, max_results: maxResults, engine })
|
|
});
|
|
|
|
const data = await response.json();
|
|
|
|
if (data.success) {
|
|
searchResults = data.results.map(r => ({
|
|
...r,
|
|
fetched: false,
|
|
saved: false,
|
|
content: ''
|
|
}));
|
|
|
|
displayResults();
|
|
updateStatus('success', `找到 ${searchResults.length} 条结果`);
|
|
document.getElementById('results-count').textContent = `${searchResults.length} 条结果`;
|
|
document.getElementById('save-all-btn').disabled = searchResults.length === 0;
|
|
|
|
document.getElementById('progress-fill').style.width = '100%';
|
|
document.getElementById('progress-text').textContent = '搜索完成！';
|
|
|
|
// 自动抓取和保存
|
|
if (autoFetch && searchResults.length > 0) {
|
|
setTimeout(() => fetchAllResults(autoSave), 500);
|
|
}
|
|
} else {
|
|
updateStatus('error', '搜索失败');
|
|
showToast('搜索失败: ' + data.error, 'error');
|
|
}
|
|
} catch (error) {
|
|
updateStatus('error', '搜索出错');
|
|
showToast('搜索出错', 'error');
|
|
console.error(error);
|
|
}
|
|
|
|
setTimeout(() => {
|
|
progress.style.display = 'none';
|
|
}, 1000);
|
|
|
|
document.getElementById('search-btn').disabled = false;
|
|
}
|
|
|
|
// 显示搜索结果
|
|
function displayResults() {
|
|
const container = document.getElementById('search-results');
|
|
|
|
if (searchResults.length === 0) {
|
|
container.innerHTML = '<div class="empty-text">未找到相关结果</div>';
|
|
return;
|
|
}
|
|
|
|
container.innerHTML = searchResults.map((r, i) => `
|
|
<div class="result-item ${r.saved ? 'saved' : ''} ${r.fetched ? 'fetched' : ''}" id="result-${i}">
|
|
<div class="result-number">${i + 1}</div>
|
|
<div class="result-main">
|
|
<div class="result-title" onclick="showResultDetail(${i})">
|
|
${r.saved ? '<i class="ri-check-line saved-icon"></i>' : ''}
|
|
${escapeHtml(r.title)}
|
|
</div>
|
|
<div class="result-url">
|
|
<a href="${escapeHtml(r.url)}" target="_blank">
|
|
<i class="ri-external-link-line"></i>
|
|
${escapeHtml(r.url.substring(0, 70))}${r.url.length > 70 ? '...' : ''}
|
|
</a>
|
|
</div>
|
|
${r.content ? `<div class="result-content-preview">${escapeHtml(r.content.substring(0, 100))}...</div>` : ''}
|
|
</div>
|
|
<div class="result-actions">
|
|
<button onclick="fetchResult(${i})" class="btn btn-sm btn-secondary" id="fetch-btn-${i}" ${r.fetched ? 'disabled' : ''}>
|
|
<i class="ri-download-line"></i> ${r.fetched ? '已抓取' : '抓取'}
|
|
</button>
|
|
<button onclick="saveResult(${i})" class="btn btn-sm btn-success" id="save-btn-${i}" ${r.saved ? 'disabled' : ''}>
|
|
<i class="ri-save-line"></i> ${r.saved ? '已保存' : '保存'}
|
|
</button>
|
|
</div>
|
|
</div>
|
|
`).join('');
|
|
}
|
|
|
|
// 检查URL是否已在内容库
|
|
async function checkUrlExists(url) {
|
|
try {
|
|
const response = await fetch(`${API_BASE}/api/articles/search?q=${encodeURIComponent(url)}`);
|
|
const data = await response.json();
|
|
if (data.success && data.articles && data.articles.length > 0) {
|
|
// 检查是否有完全匹配的 URL
|
|
return data.articles.some(a => a.url === url);
|
|
}
|
|
} catch (error) {
|
|
console.error('检查URL失败:', error);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// 抓取单个结果
|
|
async function fetchResult(index) {
|
|
const result = searchResults[index];
|
|
if (!result || result.fetched) return;
|
|
|
|
const btn = document.getElementById(`fetch-btn-${index}`);
|
|
btn.disabled = true;
|
|
btn.innerHTML = '<i class="ri-loader-4-line"></i> 检查中...';
|
|
|
|
// 先检查内容库是否已存在
|
|
const exists = await checkUrlExists(result.url);
|
|
if (exists) {
|
|
searchResults[index].fetched = true;
|
|
searchResults[index].saved = true;
|
|
searchResults[index].content = '[内容库中已存在此链接]';
|
|
btn.innerHTML = '<i class="ri-check-line"></i> 已存在';
|
|
btn.className = 'btn btn-sm btn-warning';
|
|
|
|
const saveBtn = document.getElementById(`save-btn-${index}`);
|
|
saveBtn.disabled = true;
|
|
saveBtn.innerHTML = '<i class="ri-check-line"></i> 已存在';
|
|
|
|
const item = document.getElementById(`result-${index}`);
|
|
item.classList.add('saved');
|
|
|
|
showToast('该链接已在内容库中', 'warning');
|
|
return;
|
|
}
|
|
|
|
btn.innerHTML = '<i class="ri-loader-4-line"></i> 抓取中...';
|
|
|
|
try {
|
|
const response = await fetch(`${API_BASE}/api/articles/fetch`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
url: result.url,
|
|
product_names: [result.title]
|
|
})
|
|
});
|
|
|
|
const data = await response.json();
|
|
|
|
if (data.success) {
|
|
searchResults[index].fetched = true;
|
|
searchResults[index].content = data.data.content;
|
|
searchResults[index].title = data.data.title || result.title;
|
|
|
|
btn.innerHTML = '<i class="ri-check-line"></i> 已抓取';
|
|
|
|
// 更新结果显示
|
|
const item = document.getElementById(`result-${index}`);
|
|
item.classList.add('fetched');
|
|
|
|
showToast('抓取成功', 'success');
|
|
displayResults();
|
|
} else {
|
|
// 记录失败URL
|
|
await recordFailedUrl(result.url, result.title, data.error);
|
|
btn.disabled = false;
|
|
btn.innerHTML = '<i class="ri-download-line"></i> 抓取';
|
|
btn.className = 'btn btn-sm btn-danger';
|
|
showToast('抓取失败: ' + data.error, 'error');
|
|
}
|
|
} catch (error) {
|
|
// 记录失败URL
|
|
await recordFailedUrl(result.url, result.title, '抓取出错');
|
|
btn.disabled = false;
|
|
btn.innerHTML = '<i class="ri-download-line"></i> 抓取';
|
|
btn.className = 'btn btn-sm btn-danger';
|
|
showToast('抓取出错', 'error');
|
|
}
|
|
}
|
|
|
|
// 记录失败的URL
|
|
async function recordFailedUrl(url, title, errorMessage) {
|
|
try {
|
|
await fetch(`${API_BASE}/api/articles/failed-urls`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ url, title, error_message: errorMessage })
|
|
});
|
|
} catch (error) {
|
|
console.error('记录失败URL出错:', error);
|
|
}
|
|
}
|
|
|
|
// 保存单个结果
|
|
async function saveResult(index) {
|
|
const result = searchResults[index];
|
|
if (!result) return;
|
|
|
|
// 如果已经保存（可能是内容库中已存在），直接返回
|
|
if (result.saved) {
|
|
return;
|
|
}
|
|
|
|
// 如果未抓取，先抓取
|
|
if (!result.fetched) {
|
|
await fetchResult(index);
|
|
// 抓取后如果已标记为 saved（内容库已存在），不继续保存
|
|
if (searchResults[index].saved) return;
|
|
}
|
|
|
|
const btn = document.getElementById(`save-btn-${index}`);
|
|
btn.disabled = true;
|
|
btn.innerHTML = '<i class="ri-loader-4-line"></i> 保存中...';
|
|
|
|
try {
|
|
const response = await fetch(`${API_BASE}/api/articles`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({
|
|
product_names: [searchResults[index].title],
|
|
category: '',
|
|
keywords: [],
|
|
summary: searchResults[index].content.substring(0, 200),
|
|
content: searchResults[index].content,
|
|
source: searchResults[index].url,
|
|
url: searchResults[index].url
|
|
})
|
|
});
|
|
|
|
const data = await response.json();
|
|
|
|
if (data.success) {
|
|
searchResults[index].saved = true;
|
|
btn.innerHTML = '<i class="ri-check-line"></i> 已保存';
|
|
|
|
const item = document.getElementById(`result-${index}`);
|
|
item.classList.add('saved');
|
|
|
|
showToast('保存成功', 'success');
|
|
displayResults();
|
|
} else {
|
|
btn.disabled = false;
|
|
btn.innerHTML = '<i class="ri-save-line"></i> 保存';
|
|
showToast('保存失败: ' + data.error, 'error');
|
|
}
|
|
} catch (error) {
|
|
btn.disabled = false;
|
|
btn.innerHTML = '<i class="ri-save-line"></i> 保存';
|
|
showToast('保存出错', 'error');
|
|
}
|
|
}
|
|
|
|
// 一键保存全部
|
|
async function saveAllResults() {
|
|
const unsaved = searchResults.filter(r => !r.saved);
|
|
if (unsaved.length === 0) {
|
|
showToast('没有需要保存的结果', 'error');
|
|
return;
|
|
}
|
|
|
|
const progress = document.getElementById('save-progress');
|
|
progress.style.display = 'block';
|
|
|
|
let saved = 0;
|
|
const total = unsaved.length;
|
|
|
|
for (let i = 0; i < searchResults.length; i++) {
|
|
if (searchResults[i].saved) continue;
|
|
|
|
document.getElementById('save-progress-text').textContent =
|
|
`正在保存 ${saved + 1}/${total}...`;
|
|
document.getElementById('save-fill').style.width =
|
|
`${(saved / total) * 100}%`;
|
|
|
|
await saveResult(i);
|
|
saved++;
|
|
}
|
|
|
|
document.getElementById('save-fill').style.width = '100%';
|
|
document.getElementById('save-progress-text').textContent = '保存完成！';
|
|
|
|
setTimeout(() => {
|
|
progress.style.display = 'none';
|
|
}, 1500);
|
|
|
|
showToast(`成功保存 ${saved} 条结果`, 'success');
|
|
}
|
|
|
|
// 抓取所有结果
|
|
async function fetchAllResults(autoSave) {
|
|
const progress = document.getElementById('search-progress');
|
|
progress.style.display = 'block';
|
|
|
|
const total = searchResults.length;
|
|
let fetched = 0;
|
|
|
|
for (let i = 0; i < searchResults.length; i++) {
|
|
if (searchResults[i].fetched) {
|
|
fetched++;
|
|
continue;
|
|
}
|
|
|
|
document.getElementById('progress-text').textContent =
|
|
`正在抓取 ${fetched + 1}/${total}...`;
|
|
document.getElementById('progress-fill').style.width =
|
|
`${(fetched / total) * 100}%`;
|
|
|
|
await fetchResult(i);
|
|
fetched++;
|
|
}
|
|
|
|
document.getElementById('progress-fill').style.width = '100%';
|
|
document.getElementById('progress-text').textContent = '抓取完成！';
|
|
|
|
setTimeout(() => {
|
|
progress.style.display = 'none';
|
|
}, 1000);
|
|
|
|
// 自动保存
|
|
if (autoSave) {
|
|
setTimeout(() => saveAllResults(), 500);
|
|
}
|
|
}
|
|
|
|
// 显示结果详情
|
|
function showResultDetail(index) {
|
|
currentResultIndex = index;
|
|
const result = searchResults[index];
|
|
|
|
const body = document.getElementById('result-detail-body');
|
|
body.innerHTML = `
|
|
<div class="detail-meta">
|
|
<div class="detail-meta-item">
|
|
<label>标题</label>
|
|
<span>${escapeHtml(result.title)}</span>
|
|
</div>
|
|
<div class="detail-meta-item">
|
|
<label>URL</label>
|
|
<a href="${escapeHtml(result.url)}" target="_blank">${escapeHtml(result.url)}</a>
|
|
</div>
|
|
<div class="detail-meta-item">
|
|
<label>状态</label>
|
|
<span>${result.fetched ? '已抓取' : '未抓取'} / ${result.saved ? '已保存' : '未保存'}</span>
|
|
</div>
|
|
</div>
|
|
${result.content ? `
|
|
<div class="detail-section">
|
|
<h4><i class="ri-file-text-line"></i> 页面内容</h4>
|
|
<pre style="white-space: pre-wrap; max-height: 400px; overflow-y: auto; background: #f8f9fa; padding: 15px; border-radius: 8px;">${escapeHtml(result.content.substring(0, 3000))}${result.content.length > 3000 ? '\n...(内容过长，已截断)' : ''}</pre>
|
|
</div>
|
|
` : '<div class="detail-section"><p>尚未抓取内容，点击"保存到内容库"将自动抓取并保存</p></div>'}
|
|
`;
|
|
|
|
document.getElementById('result-detail-modal').classList.add('active');
|
|
}
|
|
|
|
// 保存当前结果
|
|
async function saveCurrentResult() {
|
|
if (currentResultIndex >= 0) {
|
|
await saveResult(currentResultIndex);
|
|
closeModal('result-detail-modal');
|
|
}
|
|
}
|
|
|
|
// 清空结果
|
|
function clearResults() {
|
|
searchResults = [];
|
|
displayResults();
|
|
document.getElementById('results-count').textContent = '0 条结果';
|
|
document.getElementById('save-all-btn').disabled = true;
|
|
updateStatus('ready', '就绪');
|
|
}
|
|
|
|
// 更新状态
|
|
function updateStatus(status, text) {
|
|
const dot = document.getElementById('search-status');
|
|
const statusText = document.getElementById('status-text');
|
|
|
|
dot.className = 'status-dot';
|
|
if (status === 'searching') {
|
|
dot.style.background = '#f59e0b';
|
|
} else if (status === 'success') {
|
|
dot.style.background = '#10b981';
|
|
} else if (status === 'error') {
|
|
dot.style.background = '#ef4444';
|
|
} else {
|
|
dot.style.background = '#10b981';
|
|
}
|
|
|
|
statusText.textContent = text;
|
|
}
|
|
|
|
// 关闭模态框
|
|
function closeModal(modalId) {
|
|
document.getElementById(modalId).classList.remove('active');
|
|
}
|
|
|
|
// 显示提示
|
|
function showToast(message, type = '') {
|
|
const toast = document.getElementById('toast');
|
|
toast.textContent = message;
|
|
toast.className = `toast active ${type}`;
|
|
|
|
setTimeout(() => {
|
|
toast.classList.remove('active');
|
|
}, 3000);
|
|
}
|
|
|
|
// HTML转义
|
|
function escapeHtml(text) {
|
|
if (!text) return '';
|
|
const div = document.createElement('div');
|
|
div.textContent = text;
|
|
return div.innerHTML;
|
|
}
|
|
|
|
// ========== 失败URL管理 ==========
|
|
|
|
// 加载失败URL列表
|
|
async function loadFailedUrls() {
|
|
try {
|
|
const response = await fetch(`${API_BASE}/api/articles/failed-urls`);
|
|
const data = await response.json();
|
|
|
|
if (data.success) {
|
|
document.getElementById('failed-count').textContent = `${data.count} 条`;
|
|
|
|
const container = document.getElementById('failed-urls-list');
|
|
|
|
if (data.urls.length === 0) {
|
|
container.innerHTML = '<div class="empty-text">暂无失败记录</div>';
|
|
} else {
|
|
container.innerHTML = data.urls.map(url => `
|
|
<div class="failed-url-item" id="failed-${url.id}">
|
|
<div class="failed-url-info">
|
|
<div class="failed-url-title">${escapeHtml(url.title || url.url.substring(0, 50))}</div>
|
|
<div class="failed-url-detail">
|
|
<a href="${escapeHtml(url.url)}" target="_blank">
|
|
<i class="ri-external-link-line"></i> ${escapeHtml(url.url.substring(0, 60))}${url.url.length > 60 ? '...' : ''}
|
|
</a>
|
|
<span class="failed-error">${escapeHtml(url.error_message || '未知错误')}</span>
|
|
</div>
|
|
<div class="failed-url-meta">
|
|
<span>重试: ${url.retry_count || 0} 次</span>
|
|
<span>${url.created_at || ''}</span>
|
|
</div>
|
|
</div>
|
|
<div class="failed-url-actions">
|
|
<button onclick="retryFailedUrl(${url.id}, '${escapeHtml(url.url)}')" class="btn btn-sm btn-warning">
|
|
<i class="ri-restart-line"></i> 重试
|
|
</button>
|
|
<button onclick="deleteFailedUrl(${url.id})" class="btn btn-sm btn-danger">
|
|
<i class="ri-delete-bin-line"></i>
|
|
</button>
|
|
</div>
|
|
</div>
|
|
`).join('');
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error('加载失败URL出错:', error);
|
|
}
|
|
}
|
|
|
|
// 重试单个失败URL
|
|
async function retryFailedUrl(urlId, url) {
|
|
const item = document.getElementById(`failed-${urlId}`);
|
|
item.classList.add('loading');
|
|
|
|
try {
|
|
const response = await fetch(`${API_BASE}/api/articles/failed-urls/retry`, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify({ url })
|
|
});
|
|
|
|
const data = await response.json();
|
|
|
|
if (data.success) {
|
|
showToast('重试成功', 'success');
|
|
loadFailedUrls();
|
|
} else {
|
|
showToast('重试失败: ' + data.error, 'error');
|
|
item.classList.remove('loading');
|
|
}
|
|
} catch (error) {
|
|
showToast('重试出错', 'error');
|
|
item.classList.remove('loading');
|
|
}
|
|
}
|
|
|
|
// 全部重试
|
|
async function retryAllFailed() {
|
|
showToast('正在重试所有失败URL...', '');
|
|
|
|
const response = await fetch(`${API_BASE}/api/articles/failed-urls`);
|
|
const data = await response.json();
|
|
|
|
if (data.success && data.urls.length > 0) {
|
|
for (const url of data.urls) {
|
|
await retryFailedUrl(url.id, url.url);
|
|
await new Promise(r => setTimeout(r, 500)); // 避免太快
|
|
}
|
|
}
|
|
}
|
|
|
|
// 删除失败URL记录
|
|
async function deleteFailedUrl(urlId) {
|
|
try {
|
|
await fetch(`${API_BASE}/api/articles/failed-urls/${urlId}`, {
|
|
method: 'DELETE'
|
|
});
|
|
loadFailedUrls();
|
|
} catch (error) {
|
|
showToast('删除失败', 'error');
|
|
}
|
|
}
|
|
|
|
// 清空所有失败URL
|
|
async function clearFailedUrls() {
|
|
if (!confirm('确定要清空所有失败记录吗？')) return;
|
|
|
|
try {
|
|
await fetch(`${API_BASE}/api/articles/failed-urls/clear`, {
|
|
method: 'POST'
|
|
});
|
|
showToast('已清空', 'success');
|
|
loadFailedUrls();
|
|
} catch (error) {
|
|
showToast('清空失败', 'error');
|
|
}
|
|
} |