304 lines
16 KiB
HTML
304 lines
16 KiB
HTML
{% extends "base.html" %}
|
||
{% set active = 'sources' %}
|
||
{% block title %}数据源 - 新闻智能跟踪{% endblock %}
|
||
{% block content %}
|
||
<div class="page-head">
|
||
<h1>📡 数据源管理</h1>
|
||
<button class="btn accent" onclick="openModal()">＋ 新增数据源</button>
|
||
</div>
|
||
|
||
<div class="card">
|
||
<table>
|
||
<thead><tr><th>ID</th><th>名称</th><th>监控方式</th><th>获取方式</th><th>类型</th><th>权重</th><th>采集周期</th><th>状态</th><th>最近采样</th><th>条数</th><th>操作</th></tr></thead>
|
||
<tbody>
|
||
{% for s in sources %}
|
||
<tr data-sid="{{ s.id }}" data-name="{{ s.name }}" data-type="{{ s.type }}" data-url="{{ s.url }}" data-desc="{{ s.description }}" data-weight="{{ s.weight }}" data-kind="{{ s.kind }}" data-standard="{{ s.monitor_standard or '' }}" data-interval="{{ s.scan_interval_min or 0 }}" data-fetch="{{ s.fetch_method or 'auto' }}" data-capture="{{ s.capture_params or '{}' }}">
|
||
<td>{{ s.id }}</td>
|
||
<td><b>{{ s.name }}</b><br><span class="muted small">{{ s.description }}</span><br><span class="muted small" style="word-break:break-all;">{{ s.url }}</span></td>
|
||
<td>{% if s.kind == 'custom' %}<span class="tag hot">🎯 定制监控</span>{% else %}<span class="tag ok">普通</span>{% endif %}</td>
|
||
<td>{% if s.fetch_method == 'direct' %}<span class="tag">⚡ 直接抓取</span>{% elif s.fetch_method == 'webcapture' %}<span class="tag">🌐 web-capture</span>{% else %}<span class="tag ok">🌐 自动</span>{% endif %}</td>
|
||
<td>{{ s.type }}</td>
|
||
<td>{% if s.kind == 'custom' %}<span class="muted small">—</span>{% else %}{{ s.weight }}{% endif %}</td>
|
||
<td>{% if s.scan_interval_min %}{{ s.scan_interval_min }}分{% else %}<span class="muted small">默认</span>{% endif %}</td>
|
||
<td>{% if s.enabled %}<span class="tag ok">启用</span>{% else %}<span class="tag">停用</span>{% endif %}</td>
|
||
<td>{{ s.last_fetch or '—' }}</td>
|
||
<td>{{ s.last_count }}</td>
|
||
<td>
|
||
<button class="btn mini" onclick="openModal({{ s.id }})">✏️ 编辑</button>
|
||
<button class="btn mini" onclick="openHistory({{ s.id }})">📜 历史采样</button>
|
||
<button class="btn mini" onclick="toggleSource({{ s.id }})">{{ '停用' if s.enabled else '启用' }}</button>
|
||
<button class="btn mini danger" onclick="delSource({{ s.id }})">删除</button>
|
||
</td>
|
||
</tr>
|
||
{% endfor %}
|
||
</tbody>
|
||
</table>
|
||
<div class="muted small" style="margin-top:8px;">
|
||
💡 真实 URL 源默认通过 web-capture-api 网页提取服务抓取（可每源改获取方式与参数）；example.com 等占位地址走模拟数据。<br>
|
||
⏱ 采集周期：0=跟随所属机制全局值（普通源=设置页采集间隔，定制源=定制监控间隔），填分钟数则本源自定义。<br>
|
||
🌐 获取方式：<b>自动</b>=优先 web-capture-api、失败回退直接抓取；<b>web-capture</b>=仅走网页提取服务；<b>直接抓取</b>=requests+正文清洗。设置页可改 web-capture-api 地址。<br>
|
||
🎯 <b>定制监控</b>：与新闻监控独立，无权重，按「推送标准」由大模型判断是否推送，命中实时推送 + 单独汇总。
|
||
</div>
|
||
</div>
|
||
|
||
<!-- 新增 / 编辑 数据源 模态框 -->
|
||
<div class="modal-mask" id="sourceModal" style="display:none;">
|
||
<div class="modal">
|
||
<div class="modal-head">
|
||
<h3 id="modalTitle">新增数据源</h3>
|
||
<span class="modal-x" onclick="closeModal()">✕</span>
|
||
</div>
|
||
<div class="modal-body">
|
||
<label>名称 *</label>
|
||
<input id="m_name" placeholder="如：36氪 AI频道">
|
||
|
||
<label>监控方式</label>
|
||
<select id="m_kind" onchange="onKindChange()">
|
||
<option value="normal">普通数据源（新闻监控机制，按权重打分推送）</option>
|
||
<option value="custom">定制监控（定制监控机制，无权重，按推送标准由大模型判断推送）</option>
|
||
</select>
|
||
|
||
<label>采集周期（分钟）<span class="muted small">— 0 表示跟随所属机制的全局间隔</span></label>
|
||
<input id="m_interval" type="number" min="0" step="1" value="0" placeholder="0=跟随全局（普通源30 / 定制源15）">
|
||
|
||
<label>获取网页方式<span class="muted small">— 真实 URL 网页通过 web-capture-api 抓取（设置页可改接口地址）</span></label>
|
||
<select id="m_fetch" onchange="onFetchChange()">
|
||
<option value="auto">🌐 自动（优先 web-capture-api，失败回退直接抓取）</option>
|
||
<option value="webcapture">🌐 web-capture-api（网页提取服务，反爬强）</option>
|
||
<option value="direct">⚡ 直接抓取（requests + 正文清洗）</option>
|
||
</select>
|
||
|
||
<div id="m_capture_row">
|
||
<label>抓取动作<span class="muted small">— html 提取资讯链接；text 整页作为一条资讯</span></label>
|
||
<select id="m_c_action">
|
||
<option value="html">html（提取链接）</option>
|
||
<option value="text">text（直接取正文）</option>
|
||
</select>
|
||
<div class="form-row" style="justify-content:flex-start;margin-top:6px;">
|
||
<label style="display:flex;align-items:center;gap:5px;font-size:12px;color:#6b7280;white-space:nowrap;">等待加载(ms)
|
||
<input id="m_c_wait" type="number" value="2000" style="width:90px;"></label>
|
||
<label style="display:flex;align-items:center;gap:5px;font-size:12px;color:#6b7280;white-space:nowrap;">滚动次数
|
||
<input id="m_c_scroll" type="number" value="0" style="width:70px;"></label>
|
||
<label style="display:flex;align-items:center;gap:5px;font-size:12px;color:#6b7280;white-space:nowrap;">后端
|
||
<select id="m_c_backend" style="width:130px;">
|
||
<option value="auto">auto</option>
|
||
<option value="agent-browser">agent-browser</option>
|
||
<option value="playwright">playwright</option>
|
||
</select></label>
|
||
</div>
|
||
<label>高级参数(JSON，可留空)</label>
|
||
<textarea id="m_c_extra" rows="2" placeholder='{"scroll_delay":1000,"full_page":false,"viewport":{"width":1280,"height":800}}'></textarea>
|
||
</div>
|
||
|
||
<div id="m_weight_row">
|
||
<label>权重（综合评分中该源占比，0.1~2.0）</label>
|
||
<input id="m_weight" type="number" step="0.1" min="0.1" max="2" value="0.8">
|
||
</div>
|
||
|
||
<div id="m_standard_row" style="display:none;">
|
||
<label>推送标准 *（大模型据此判断每条资讯是否达到推送条件）</label>
|
||
<textarea id="m_standard" rows="3" placeholder="例如：仅推送英伟达重大事件——股价单日涨跌超10%，或重大产品发布/回购/分拆/收购等实质性公告，或分析师大幅上调目标价；普通行情波动与日常新闻一律不推送。"></textarea>
|
||
<div class="form-row" style="justify-content:flex-start;">
|
||
<button class="btn mini" type="button" onclick="testStandard()">🧪 测试推送标准</button>
|
||
<span id="m_test_result" class="muted small"></span>
|
||
</div>
|
||
</div>
|
||
|
||
<label>类型（分类标签）</label>
|
||
<input id="m_type" placeholder="科技媒体/公司动态/学术/政策...">
|
||
|
||
<label>URL / RSS 地址</label>
|
||
<input id="m_url" placeholder="example.com 开头走模拟数据">
|
||
|
||
<label>描述</label>
|
||
<textarea id="m_desc" rows="2" placeholder="该源提供哪些内容"></textarea>
|
||
</div>
|
||
<div class="modal-foot">
|
||
<button class="btn" onclick="closeModal()">取消</button>
|
||
<button class="btn accent" onclick="saveSource()">保存</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- 历史采样 模态框 -->
|
||
<div class="modal-mask" id="histModal" style="display:none;">
|
||
<div class="modal" style="width:640px;">
|
||
<div class="modal-head">
|
||
<h3 id="histTitle">历史采样</h3>
|
||
<span class="modal-x" onclick="document.getElementById('histModal').style.display='none'">✕</span>
|
||
</div>
|
||
<div class="modal-body">
|
||
<div class="form-row" style="margin-bottom:8px;">
|
||
<button class="btn mini" onclick="openHistArticles()">📰 查看本源采集资讯</button>
|
||
<span id="hist_meta" class="muted small"></span>
|
||
</div>
|
||
<div id="hist_snaps"></div>
|
||
<div id="hist_articles" style="display:none;margin-top:8px;border-top:1px solid #f3f4f6;padding-top:8px;"></div>
|
||
</div>
|
||
<div class="modal-foot">
|
||
<button class="btn" onclick="document.getElementById('histModal').style.display='none'">关闭</button>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
{% endblock %}
|
||
{% block script %}
|
||
<script>
|
||
let editId = null;
|
||
let histSourceId = null;
|
||
|
||
function openModal(id){
|
||
editId = id || null;
|
||
document.getElementById('modalTitle').textContent = editId ? '编辑数据源' : '新增数据源';
|
||
document.getElementById('m_test_result').textContent = '';
|
||
if (editId){
|
||
const row = document.querySelector(`tr[data-sid="${editId}"]`);
|
||
if (!row) return;
|
||
setV('m_name', row.dataset.name);
|
||
setV('m_type', row.dataset.type);
|
||
setV('m_url', row.dataset.url);
|
||
setV('m_desc', row.dataset.desc);
|
||
setV('m_weight', row.dataset.weight || '0.8');
|
||
setV('m_kind', row.dataset.kind || 'normal');
|
||
setV('m_standard', row.dataset.standard || '');
|
||
setV('m_interval', row.dataset.interval || '0');
|
||
const cap = parseCapture(row.dataset.capture);
|
||
setV('m_fetch', row.dataset.fetch || 'auto');
|
||
setV('m_c_action', cap.action || 'html');
|
||
setV('m_c_wait', cap.wait_time != null ? cap.wait_time : '2000');
|
||
setV('m_c_scroll', cap.scroll_times != null ? cap.scroll_times : '0');
|
||
setV('m_c_backend', cap.backend || 'auto');
|
||
const extra = {...cap};
|
||
['action','wait_time','scroll_times','backend'].forEach(k => delete extra[k]);
|
||
setV('m_c_extra', Object.keys(extra).length ? JSON.stringify(extra) : '');
|
||
} else {
|
||
['m_name','m_type','m_url','m_desc','m_standard','m_c_extra'].forEach(k=>setV(k,''));
|
||
setV('m_weight', '0.8');
|
||
setV('m_kind', 'normal');
|
||
setV('m_interval', '0');
|
||
setV('m_fetch', 'auto');
|
||
setV('m_c_action', 'html');
|
||
setV('m_c_wait', '2000');
|
||
setV('m_c_scroll', '0');
|
||
setV('m_c_backend', 'auto');
|
||
}
|
||
onKindChange();
|
||
onFetchChange();
|
||
document.getElementById('sourceModal').style.display = 'flex';
|
||
}
|
||
function closeModal(){ document.getElementById('sourceModal').style.display = 'none'; editId = null; }
|
||
|
||
function onKindChange(){
|
||
const custom = getV('m_kind') === 'custom';
|
||
document.getElementById('m_weight_row').style.display = custom ? 'none' : 'block';
|
||
document.getElementById('m_standard_row').style.display = custom ? 'block' : 'none';
|
||
}
|
||
|
||
function onFetchChange(){
|
||
document.getElementById('m_capture_row').style.display =
|
||
getV('m_fetch') === 'direct' ? 'none' : 'block';
|
||
}
|
||
|
||
function parseCapture(s){
|
||
try { const o = JSON.parse(s || '{}'); return o && typeof o === 'object' ? o : {}; } catch(e){ return {}; }
|
||
}
|
||
|
||
async function saveSource(){
|
||
const kind = getV('m_kind');
|
||
let captureParams = '{}';
|
||
if (getV('m_fetch') !== 'direct'){
|
||
const base = {
|
||
action: getV('m_c_action') || 'html',
|
||
wait_time: parseInt(getV('m_c_wait')) || 2000,
|
||
scroll_times: parseInt(getV('m_c_scroll')) || 0,
|
||
backend: getV('m_c_backend') || 'auto'
|
||
};
|
||
let adv = {};
|
||
const extraTxt = getV('m_c_extra').trim();
|
||
if (extraTxt){
|
||
try { adv = JSON.parse(extraTxt); }
|
||
catch(e){ toast('高级参数 JSON 格式错误', false); return; }
|
||
}
|
||
captureParams = JSON.stringify({ ...base, ...adv });
|
||
}
|
||
const body = {
|
||
name: getV('m_name').trim(), type: getV('m_type').trim(), url: getV('m_url').trim(),
|
||
description: getV('m_desc').trim(), kind,
|
||
monitor_standard: kind === 'custom' ? getV('m_standard').trim() : '',
|
||
weight: parseFloat(getV('m_weight') || '1') || 1,
|
||
scan_interval_min: parseInt(getV('m_interval')) || 0,
|
||
fetch_method: getV('m_fetch'),
|
||
capture_params: captureParams
|
||
};
|
||
if (!body.name){ toast('请填写名称', false); return; }
|
||
if (kind === 'custom' && !body.monitor_standard){ toast('请填写推送标准', false); return; }
|
||
const r = await API.json('/api/sources',
|
||
editId ? {...body, action:'update', id: editId} : {...body, action:'add'});
|
||
toast(r.ok ? '✅ 已保存' : '❌ 保存失败', r.ok);
|
||
if (r.ok){ closeModal(); setTimeout(()=>location.reload(), 400); }
|
||
}
|
||
|
||
async function testStandard(){
|
||
const standard = getV('m_standard').trim();
|
||
if (!standard){ toast('请先填写推送标准', false); return; }
|
||
const el = document.getElementById('m_test_result');
|
||
el.textContent = '⏳ 大模型分析中...';
|
||
const r = await API.json('/api/sources', {action:'test_standard', monitor_standard: standard});
|
||
if (r.ok){
|
||
el.textContent = r.meets ? '✅ 命中标准，会推送 · ' + (r.reason||'') : '❌ 未达标准，不推送 · ' + (r.reason||'');
|
||
} else {
|
||
el.textContent = '❌ ' + (r.error || '测试失败');
|
||
}
|
||
}
|
||
|
||
// ---------- 历史采样 ----------
|
||
async function openHistory(id){
|
||
histSourceId = id;
|
||
const row = document.querySelector(`tr[data-sid="${id}"]`);
|
||
document.getElementById('histTitle').textContent = `📜 历史采样 · ${row ? row.dataset.name : ''}`;
|
||
document.getElementById('hist_articles').style.display = 'none';
|
||
document.getElementById('hist_articles').innerHTML = '';
|
||
document.getElementById('histModal').style.display = 'flex';
|
||
const r = await API.get('/api/sources/history', {source_id: id, limit: 100});
|
||
document.getElementById('hist_meta').textContent = r.ok ? `共 ${r.total} 次采样` : '加载失败';
|
||
if (!r.ok){ document.getElementById('hist_snaps').innerHTML = '<div class="empty">加载失败</div>'; return; }
|
||
const snaps = r.snapshots;
|
||
if (!snaps.length){
|
||
document.getElementById('hist_snaps').innerHTML = '<div class="empty">暂无采样记录（保存后由后台采集生成）</div>';
|
||
return;
|
||
}
|
||
document.getElementById('hist_snaps').innerHTML =
|
||
'<table><thead><tr><th>#</th><th>采样时间</th><th>采集条数</th><th>状态</th><th>备注</th></tr></thead><tbody>' +
|
||
snaps.map(s => `<tr>
|
||
<td>${s.id}</td><td>${s.fetched_at}</td><td>${s.count}</td>
|
||
<td>${s.status==='ok' ? '<span class="tag ok">成功</span>' : '<span class="tag err">失败</span>'}</td>
|
||
<td class="small muted">${escapeHtml(s.detail||'')}</td>
|
||
</tr>`).join('') + '</tbody></table>';
|
||
}
|
||
|
||
async function openHistArticles(){
|
||
if (!histSourceId) return;
|
||
const el = document.getElementById('hist_articles');
|
||
el.style.display = 'block';
|
||
el.innerHTML = '<div class="empty">⏳ 加载资讯列表...</div>';
|
||
const r = await API.get('/api/sources/articles', {source_id: histSourceId, page_size: 30});
|
||
if (!r.ok){ el.innerHTML = '<div class="empty">加载失败</div>'; return; }
|
||
const arts = r.articles;
|
||
if (!arts.length){ el.innerHTML = `<div class="empty">本源暂无采集到的资讯（共 ${r.total} 条）</div>`; return; }
|
||
el.innerHTML = `<div class="muted small" style="margin-bottom:6px;">本源共采集 ${r.total} 条资讯（最新 ${arts.length} 条）：</div>` +
|
||
'<div class="hist-list">' + arts.map(a =>
|
||
`<div class="hist-item"><a href="/news/${a.id}" target="_blank"><b>${escapeHtml(a.title)}</b></a>
|
||
<div class="muted small">${a.collected_at} · 综合分${a.total_score}${a.is_important ? ' · <span style="color:#dc2626;">重要</span>' : ''}</div></div>`
|
||
).join('') + '</div>';
|
||
}
|
||
|
||
async function toggleSource(id){ await API.json('/api/sources', {action:'toggle', id}); location.reload(); }
|
||
async function delSource(id){
|
||
if (!confirm('确认删除该数据源？')) return;
|
||
await API.json('/api/sources', {action:'delete', id}); toast('✅ 已删除'); setTimeout(()=>location.reload(), 400);
|
||
}
|
||
function getV(id){ return document.getElementById(id).value; }
|
||
function setV(id, v){ document.getElementById(id).value = v; }
|
||
function escapeHtml(s){ return (s||'').replace(/[&<>"']/g, c => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c])); }
|
||
</script>
|
||
{% endblock %}
|