v1.4.0: 通知日志分页+筛选 / 仪表盘TOP10+更多链接 / 数据源走web-capture-api抓取(获取方式与参数可编辑) / 系统错误邮件通知(频率+静默时段)

templates/sources.html

@@ -9,13 +9,14 @@
 
 <div class="card">
   <table>
-    <thead><tr><th>ID</th><th>名称</th><th>监控方式</th><th>类型</th><th>权重</th><th>采集周期</th><th>状态</th><th>最近采样</th><th>条数</th><th>操作</th></tr></thead>
+    <thead><tr><th>ID</th><th>名称</th><th>监控方式</th><th>获取方式</th><th>类型</th><th>权重</th><th>采集周期</th><th>状态</th><th>最近采样</th><th>条数</th><th>操作</th></tr></thead>
     <tbody>
     {% for s in sources %}
-      <tr data-sid="{{ s.id }}" data-name="{{ s.name }}" data-type="{{ s.type }}" data-url="{{ s.url }}" data-desc="{{ s.description }}" data-weight="{{ s.weight }}" data-kind="{{ s.kind }}" data-standard="{{ s.monitor_standard or '' }}" data-interval="{{ s.scan_interval_min or 0 }}">
+      <tr data-sid="{{ s.id }}" data-name="{{ s.name }}" data-type="{{ s.type }}" data-url="{{ s.url }}" data-desc="{{ s.description }}" data-weight="{{ s.weight }}" data-kind="{{ s.kind }}" data-standard="{{ s.monitor_standard or '' }}" data-interval="{{ s.scan_interval_min or 0 }}" data-fetch="{{ s.fetch_method or 'auto' }}" data-capture="{{ s.capture_params or '{}' }}">
         <td>{{ s.id }}</td>
         <td><b>{{ s.name }}</b><br><span class="muted small">{{ s.description }}</span><br><span class="muted small" style="word-break:break-all;">{{ s.url }}</span></td>
         <td>{% if s.kind == 'custom' %}<span class="tag hot">🎯 定制监控</span>{% else %}<span class="tag ok">普通</span>{% endif %}</td>
+        <td>{% if s.fetch_method == 'direct' %}<span class="tag">⚡ 直接抓取</span>{% elif s.fetch_method == 'webcapture' %}<span class="tag">🌐 web-capture</span>{% else %}<span class="tag ok">🌐 自动</span>{% endif %}</td>
         <td>{{ s.type }}</td>
         <td>{% if s.kind == 'custom' %}<span class="muted small">—</span>{% else %}{{ s.weight }}{% endif %}</td>
         <td>{% if s.scan_interval_min %}{{ s.scan_interval_min }}分{% else %}<span class="muted small">默认</span>{% endif %}</td>
@@ -33,8 +34,9 @@
     </tbody>
   </table>
   <div class="muted small" style="margin-top:8px;">
-    💡 真实 URL 源自动抓取页面并清洗可读正文入库；example.com 等占位地址走模拟数据。<br>
+    💡 真实 URL 源默认通过 web-capture-api 网页提取服务抓取（可每源改获取方式与参数）；example.com 等占位地址走模拟数据。<br>
     ⏱ 采集周期：0=跟随所属机制全局值（普通源=设置页采集间隔，定制源=定制监控间隔），填分钟数则本源自定义。<br>
+    🌐 获取方式：<b>自动</b>=优先 web-capture-api、失败回退直接抓取；<b>web-capture</b>=仅走网页提取服务；<b>直接抓取</b>=requests+正文清洗。设置页可改 web-capture-api 地址。<br>
     🎯 <b>定制监控</b>：与新闻监控独立，无权重，按「推送标准」由大模型判断是否推送，命中实时推送 + 单独汇总。
   </div>
 </div>
@@ -59,6 +61,35 @@
       <label>采集周期（分钟）<span class="muted small">— 0 表示跟随所属机制的全局间隔</span></label>
       <input id="m_interval" type="number" min="0" step="1" value="0" placeholder="0=跟随全局（普通源30 / 定制源15）">
 
+      <label>获取网页方式<span class="muted small">— 真实 URL 网页通过 web-capture-api 抓取（设置页可改接口地址）</span></label>
+      <select id="m_fetch" onchange="onFetchChange()">
+        <option value="auto">🌐 自动（优先 web-capture-api，失败回退直接抓取）</option>
+        <option value="webcapture">🌐 web-capture-api（网页提取服务，反爬强）</option>
+        <option value="direct">⚡ 直接抓取（requests + 正文清洗）</option>
+      </select>
+
+      <div id="m_capture_row">
+        <label>抓取动作<span class="muted small">— html 提取资讯链接；text 整页作为一条资讯</span></label>
+        <select id="m_c_action">
+          <option value="html">html（提取链接）</option>
+          <option value="text">text（直接取正文）</option>
+        </select>
+        <div class="form-row" style="justify-content:flex-start;margin-top:6px;">
+          <label style="display:flex;align-items:center;gap:5px;font-size:12px;color:#6b7280;white-space:nowrap;">等待加载(ms)
+            <input id="m_c_wait" type="number" value="2000" style="width:90px;"></label>
+          <label style="display:flex;align-items:center;gap:5px;font-size:12px;color:#6b7280;white-space:nowrap;">滚动次数
+            <input id="m_c_scroll" type="number" value="0" style="width:70px;"></label>
+          <label style="display:flex;align-items:center;gap:5px;font-size:12px;color:#6b7280;white-space:nowrap;">后端
+            <select id="m_c_backend" style="width:130px;">
+              <option value="auto">auto</option>
+              <option value="agent-browser">agent-browser</option>
+              <option value="playwright">playwright</option>
+            </select></label>
+        </div>
+        <label>高级参数(JSON，可留空)</label>
+        <textarea id="m_c_extra" rows="2" placeholder='{"scroll_delay":1000,"full_page":false,"viewport":{"width":1280,"height":800}}'></textarea>
+      </div>
+
       <div id="m_weight_row">
         <label>权重（综合评分中该源占比，0.1~2.0）</label>
         <input id="m_weight" type="number" step="0.1" min="0.1" max="2" value="0.8">
@@ -130,13 +161,28 @@ function openModal(id){
     setV('m_kind', row.dataset.kind || 'normal');
     setV('m_standard', row.dataset.standard || '');
     setV('m_interval', row.dataset.interval || '0');
+    const cap = parseCapture(row.dataset.capture);
+    setV('m_fetch', row.dataset.fetch || 'auto');
+    setV('m_c_action', cap.action || 'html');
+    setV('m_c_wait', cap.wait_time != null ? cap.wait_time : '2000');
+    setV('m_c_scroll', cap.scroll_times != null ? cap.scroll_times : '0');
+    setV('m_c_backend', cap.backend || 'auto');
+    const extra = {...cap};
+    ['action','wait_time','scroll_times','backend'].forEach(k => delete extra[k]);
+    setV('m_c_extra', Object.keys(extra).length ? JSON.stringify(extra) : '');
   } else {
-    ['m_name','m_type','m_url','m_desc','m_standard'].forEach(k=>setV(k,''));
+    ['m_name','m_type','m_url','m_desc','m_standard','m_c_extra'].forEach(k=>setV(k,''));
     setV('m_weight', '0.8');
     setV('m_kind', 'normal');
     setV('m_interval', '0');
+    setV('m_fetch', 'auto');
+    setV('m_c_action', 'html');
+    setV('m_c_wait', '2000');
+    setV('m_c_scroll', '0');
+    setV('m_c_backend', 'auto');
   }
   onKindChange();
+  onFetchChange();
   document.getElementById('sourceModal').style.display = 'flex';
 }
 function closeModal(){ document.getElementById('sourceModal').style.display = 'none'; editId = null; }
@@ -147,14 +193,41 @@ function onKindChange(){
   document.getElementById('m_standard_row').style.display = custom ? 'block' : 'none';
 }
 
+function onFetchChange(){
+  document.getElementById('m_capture_row').style.display =
+    getV('m_fetch') === 'direct' ? 'none' : 'block';
+}
+
+function parseCapture(s){
+  try { const o = JSON.parse(s || '{}'); return o && typeof o === 'object' ? o : {}; } catch(e){ return {}; }
+}
+
 async function saveSource(){
   const kind = getV('m_kind');
+  let captureParams = '{}';
+  if (getV('m_fetch') !== 'direct'){
+    const base = {
+      action: getV('m_c_action') || 'html',
+      wait_time: parseInt(getV('m_c_wait')) || 2000,
+      scroll_times: parseInt(getV('m_c_scroll')) || 0,
+      backend: getV('m_c_backend') || 'auto'
+    };
+    let adv = {};
+    const extraTxt = getV('m_c_extra').trim();
+    if (extraTxt){
+      try { adv = JSON.parse(extraTxt); }
+      catch(e){ toast('高级参数 JSON 格式错误', false); return; }
+    }
+    captureParams = JSON.stringify({ ...base, ...adv });
+  }
   const body = {
     name: getV('m_name').trim(), type: getV('m_type').trim(), url: getV('m_url').trim(),
     description: getV('m_desc').trim(), kind,
     monitor_standard: kind === 'custom' ? getV('m_standard').trim() : '',
     weight: parseFloat(getV('m_weight') || '1') || 1,
-    scan_interval_min: parseInt(getV('m_interval')) || 0
+    scan_interval_min: parseInt(getV('m_interval')) || 0,
+    fetch_method: getV('m_fetch'),
+    capture_params: captureParams
   };
   if (!body.name){ toast('请填写名称', false); return; }
   if (kind === 'custom' && !body.monitor_standard){ toast('请填写推送标准', false); return; }