Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
322ca28cb4 | ||
|
|
1a2d6ada88 | ||
|
|
00609980a8 | ||
|
|
76e24c4fa3 | ||
|
|
d23d526629 | ||
|
|
24f0d3c35b | ||
|
|
b0d98b78d9 | ||
|
|
d63fe2f671 | ||
|
|
c40acab1c9 | ||
|
|
4352b81c20 | ||
|
|
c960d959ad | ||
|
|
50379f5ecf |
+77
-9
@@ -32,6 +32,7 @@ class Database:
|
||||
CREATE TABLE IF NOT EXISTS articles (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
product_names TEXT NOT NULL,
|
||||
search_title TEXT,
|
||||
category TEXT,
|
||||
keywords TEXT,
|
||||
summary TEXT,
|
||||
@@ -44,6 +45,12 @@ class Database:
|
||||
)
|
||||
''')
|
||||
|
||||
# 为旧表添加字段(如果不存在)
|
||||
try:
|
||||
cursor.execute('ALTER TABLE articles ADD COLUMN search_title TEXT')
|
||||
except:
|
||||
pass
|
||||
|
||||
# 待处理产品列表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS pending_products (
|
||||
@@ -121,18 +128,32 @@ class Database:
|
||||
)
|
||||
''')
|
||||
|
||||
# 搜索缓存表
|
||||
cursor.execute('''
|
||||
CREATE TABLE IF NOT EXISTS search_cache (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
keyword TEXT NOT NULL,
|
||||
engine TEXT DEFAULT 'bing_cn',
|
||||
results TEXT NOT NULL,
|
||||
result_count INTEGER,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
expires_at DATETIME,
|
||||
UNIQUE(keyword, engine)
|
||||
)
|
||||
''')
|
||||
|
||||
conn.commit()
|
||||
|
||||
# ========== 内容库操作 ==========
|
||||
def add_article(self, product_names, category, keywords, summary, content, source, url=None):
|
||||
def add_article(self, product_names, category, keywords, summary, content, source, url=None, search_title=None):
|
||||
"""添加文章到内容库"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('''
|
||||
INSERT INTO articles (product_names, category, keywords, summary, content, source, url)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
''', (json.dumps(product_names, ensure_ascii=False), category,
|
||||
json.dumps(keywords, ensure_ascii=False), summary, content, source, url))
|
||||
INSERT INTO articles (product_names, search_title, category, keywords, summary, content, source, url)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
''', (json.dumps(product_names, ensure_ascii=False), search_title,
|
||||
category, json.dumps(keywords, ensure_ascii=False), summary, content, source, url))
|
||||
conn.commit()
|
||||
return cursor.lastrowid
|
||||
|
||||
@@ -143,16 +164,16 @@ class Database:
|
||||
if category:
|
||||
cursor.execute('''
|
||||
SELECT * FROM articles
|
||||
WHERE (product_names LIKE ? OR keywords LIKE ? OR summary LIKE ? OR content LIKE ?)
|
||||
WHERE (product_names LIKE ? OR keywords LIKE ? OR summary LIKE ? OR content LIKE ? OR url LIKE ?)
|
||||
AND category = ?
|
||||
ORDER BY fetch_date DESC
|
||||
''', (f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', category))
|
||||
''', (f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', category))
|
||||
else:
|
||||
cursor.execute('''
|
||||
SELECT * FROM articles
|
||||
WHERE product_names LIKE ? OR keywords LIKE ? OR summary LIKE ? OR content LIKE ?
|
||||
WHERE product_names LIKE ? OR keywords LIKE ? OR summary LIKE ? OR content LIKE ? OR url LIKE ?
|
||||
ORDER BY fetch_date DESC
|
||||
''', (f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', f'%{keyword}%'))
|
||||
''', (f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', f'%{keyword}%', f'%{keyword}%'))
|
||||
return [dict(row) for row in cursor.fetchall()]
|
||||
|
||||
def get_article_by_id(self, article_id):
|
||||
@@ -396,5 +417,52 @@ class Database:
|
||||
conn.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
# ========== 搜索缓存操作 ==========
|
||||
def save_search_cache(self, keyword, engine, results, expire_days=7):
|
||||
"""保存搜索结果缓存"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('''
|
||||
INSERT OR REPLACE INTO search_cache (keyword, engine, results, result_count, created_at, expires_at)
|
||||
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP, datetime('now', '+' || ? || ' days'))
|
||||
''', (keyword, engine, json.dumps(results, ensure_ascii=False), len(results), expire_days))
|
||||
conn.commit()
|
||||
return cursor.lastrowid
|
||||
|
||||
def get_search_cache(self, keyword, engine='bing_cn'):
|
||||
"""获取搜索结果缓存"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('''
|
||||
SELECT results, result_count, created_at, expires_at
|
||||
FROM search_cache
|
||||
WHERE keyword = ? AND engine = ? AND expires_at > datetime('now')
|
||||
''', (keyword, engine))
|
||||
row = cursor.fetchone()
|
||||
if row:
|
||||
return {
|
||||
'results': json.loads(row['results']),
|
||||
'count': row['result_count'],
|
||||
'cached_at': row['created_at'],
|
||||
'expires_at': row['expires_at']
|
||||
}
|
||||
return None
|
||||
|
||||
def clear_expired_cache(self):
|
||||
"""清理过期缓存"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('DELETE FROM search_cache WHERE expires_at <= datetime("now")')
|
||||
conn.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
def clear_all_cache(self):
|
||||
"""清空所有缓存"""
|
||||
with self.get_connection() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute('DELETE FROM search_cache')
|
||||
conn.commit()
|
||||
return cursor.rowcount
|
||||
|
||||
# 全局数据库实例
|
||||
db = Database()
|
||||
+34
-5
@@ -31,12 +31,17 @@ def search_articles():
|
||||
"""搜索文章"""
|
||||
keyword = request.args.get('q', '')
|
||||
category = request.args.get('category')
|
||||
limit = request.args.get('limit', type=int) # 支持limit参数
|
||||
|
||||
if not keyword:
|
||||
return jsonify({'error': '请提供搜索关键词'}), 400
|
||||
|
||||
articles = db.search_articles(keyword, category)
|
||||
|
||||
# 如果有limit参数,截取
|
||||
if limit:
|
||||
articles = articles[:limit]
|
||||
|
||||
# 解析JSON字段
|
||||
for article in articles:
|
||||
article['product_names'] = __import__('json').loads(article.get('product_names', '[]'))
|
||||
@@ -108,14 +113,16 @@ def fetch_article():
|
||||
"""从URL抓取文章"""
|
||||
data = request.get_json()
|
||||
url = data.get('url')
|
||||
search_title = data.get('search_title') # 搜索结果的标题
|
||||
|
||||
if not url:
|
||||
return jsonify({'error': '请提供URL'}), 400
|
||||
|
||||
result = search_service.fetch_url_content(url)
|
||||
|
||||
if result:
|
||||
if result and result.get('success'):
|
||||
# 自动保存到内容库
|
||||
# product_names 使用抓取到的标题,search_title 使用搜索结果的标题
|
||||
article_id = search_service.save_to_articles(
|
||||
product_names=data.get('product_names', [result['title']]),
|
||||
category=data.get('category'),
|
||||
@@ -123,7 +130,8 @@ def fetch_article():
|
||||
summary=result.get('description', ''),
|
||||
content=result['content'],
|
||||
source=url,
|
||||
url=url
|
||||
url=url,
|
||||
search_title=search_title
|
||||
)
|
||||
|
||||
return jsonify({
|
||||
@@ -132,7 +140,8 @@ def fetch_article():
|
||||
'data': result
|
||||
})
|
||||
else:
|
||||
return jsonify({'error': '抓取失败'}), 500
|
||||
error_msg = result.get('error', '抓取失败') if result else '抓取失败'
|
||||
return jsonify({'success': False, 'error': error_msg}), 500
|
||||
|
||||
@bp.route('/internet-search', methods=['POST'])
|
||||
def internet_search():
|
||||
@@ -141,19 +150,39 @@ def internet_search():
|
||||
keyword = data.get('keyword', '')
|
||||
max_results = data.get('max_results', 10)
|
||||
engine = data.get('engine', 'bing_cn') # 默认 Bing 中国
|
||||
use_cache = data.get('use_cache', True) # 默认使用缓存
|
||||
cache_days = data.get('cache_days', 7) # 默认缓存7天
|
||||
|
||||
if not keyword:
|
||||
return jsonify({'error': '请提供搜索关键词'}), 400
|
||||
|
||||
# 检查是否有缓存
|
||||
cached = None
|
||||
if use_cache:
|
||||
cached = db.get_search_cache(keyword, engine)
|
||||
|
||||
if cached:
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'keyword': keyword,
|
||||
'engine': engine,
|
||||
'results': cached['results'],
|
||||
'count': cached['count'],
|
||||
'cached': True,
|
||||
'cached_at': cached['cached_at'],
|
||||
'expires_at': cached['expires_at']
|
||||
})
|
||||
|
||||
# 执行互联网搜索
|
||||
results = search_service.search_internet(keyword, max_results, engine)
|
||||
results = search_service.search_internet(keyword, max_results, engine, use_cache, cache_days)
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'keyword': keyword,
|
||||
'engine': engine,
|
||||
'results': results,
|
||||
'count': len(results)
|
||||
'count': len(results),
|
||||
'cached': False
|
||||
})
|
||||
|
||||
@bp.route('/internet-search-and-fetch', methods=['POST'])
|
||||
|
||||
+85
-16
@@ -33,7 +33,7 @@ class SearchService:
|
||||
)
|
||||
return result.stdout, result.stderr, result.returncode
|
||||
|
||||
def search_internet(self, keyword, max_results=None, engine='bing_cn'):
|
||||
def search_internet(self, keyword, max_results=None, engine='bing_cn', use_cache=True, cache_days=7):
|
||||
"""
|
||||
从互联网搜索(使用 agent-browser 浏览器自动化)
|
||||
|
||||
@@ -42,10 +42,21 @@ class SearchService:
|
||||
- bing_global: Bing 国际版
|
||||
- google: Google
|
||||
- baidu: 百度
|
||||
|
||||
参数:
|
||||
- use_cache: 是否使用缓存(默认True)
|
||||
- cache_days: 缓存有效天数(默认7天)
|
||||
"""
|
||||
max_results = max_results or self.max_results
|
||||
results = []
|
||||
|
||||
# 优先查询缓存
|
||||
if use_cache:
|
||||
cached = db.get_search_cache(keyword, engine)
|
||||
if cached:
|
||||
print(f"使用缓存结果: {keyword} ({engine})")
|
||||
return cached['results']
|
||||
|
||||
# 根据搜索引擎选择 URL
|
||||
encoded_keyword = urllib.parse.quote(keyword)
|
||||
search_urls = {
|
||||
@@ -89,6 +100,10 @@ class SearchService:
|
||||
# 5. 关闭浏览器
|
||||
self._run_browser('close')
|
||||
|
||||
# 6. 保存到缓存
|
||||
if results and use_cache:
|
||||
db.save_search_cache(keyword, engine, results, cache_days)
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
print(f"搜索超时: {keyword}")
|
||||
except Exception as e:
|
||||
@@ -136,18 +151,23 @@ class SearchService:
|
||||
continue
|
||||
|
||||
# 匹配标题链接:link "标题文字" [ref=eXX]
|
||||
# 需要过滤域名链接(如 "zhihu.com")和短链接
|
||||
# 过滤域名链接(如 "zhihu.com")和短链接
|
||||
if 'link "' in line and '[ref=' in line:
|
||||
match = re.search(r'link "([^"]+)" \[ref=(e\d+)\]', line)
|
||||
if match:
|
||||
title = match.group(1)
|
||||
ref = match.group(2)
|
||||
# 过滤短标题(域名链接如 "zhihu.com")
|
||||
if len(title) > 20 and '.' not in title[:10]: # 不是域名格式
|
||||
# 过滤纯域名格式
|
||||
is_domain = (title.endswith('.com') or title.endswith('.cn') or
|
||||
title.endswith('.net') or title.endswith('.org') or
|
||||
title.endswith('.edu') or title.endswith('.gov'))
|
||||
if not is_domain:
|
||||
refs.append((title, ref))
|
||||
|
||||
# 获取每个结果的 URL
|
||||
for title, ref in refs[:max_results]:
|
||||
# 获取每个结果的 URL,多获取几个以防解析失败
|
||||
for title, ref in refs[:max_results + 5]:
|
||||
if len(results) >= max_results:
|
||||
break
|
||||
url = self._get_link_url(ref)
|
||||
if url and 'bing.com/search' not in url: # 过滤搜索结果页本身的链接
|
||||
results.append({
|
||||
@@ -209,16 +229,22 @@ class SearchService:
|
||||
return None
|
||||
|
||||
def fetch_url_content(self, url):
|
||||
"""抓取网页内容(使用 agent-browser 浏览器方式,绑过反爬虫)"""
|
||||
"""抓取网页内容(使用 agent-browser 浏览器方式,绕过反爬虫)"""
|
||||
error_message = None
|
||||
try:
|
||||
# 使用浏览器方式抓取
|
||||
stdout, stderr, code = self._run_browser('open', url, '--timeout', '20000')
|
||||
# 使用浏览器方式抓取,增加超时时间到60秒
|
||||
stdout, stderr, code = self._run_browser('open', url, '--timeout', '60000')
|
||||
if code != 0:
|
||||
error_message = stderr.strip() if stderr else '浏览器打开页面失败'
|
||||
print(f"打开页面失败: {stderr}")
|
||||
return None
|
||||
# 浏览器失败,尝试使用 requests 备用方案
|
||||
result = self._fetch_with_requests(url)
|
||||
if result:
|
||||
return result
|
||||
return {'success': False, 'error': error_message}
|
||||
|
||||
# 等待页面加载
|
||||
self._run_browser('wait', '5000')
|
||||
# 等待页面加载(增加到10秒)
|
||||
self._run_browser('wait', '10000')
|
||||
|
||||
# 获取页面标题
|
||||
stdout, stderr, code = self._run_browser('get', 'title', '--timeout', '5000')
|
||||
@@ -247,6 +273,7 @@ class SearchService:
|
||||
description = text[:200].strip() if text else ''
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'content': text,
|
||||
@@ -254,13 +281,14 @@ class SearchService:
|
||||
'fetch_date': datetime.now().isoformat()
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"抓取URL失败: {url}, 错误: {str(e)}")
|
||||
error_message = str(e)
|
||||
print(f"抓取URL失败: {url}, 错误: {error_message}")
|
||||
# 尝试关闭浏览器
|
||||
try:
|
||||
self._run_browser('close')
|
||||
except:
|
||||
pass
|
||||
return None
|
||||
return {'success': False, 'error': error_message}
|
||||
|
||||
def _extract_text_from_snapshot(self, snapshot):
|
||||
"""从 accessibility tree snapshot 中提取文本内容"""
|
||||
@@ -294,6 +322,47 @@ class SearchService:
|
||||
|
||||
return result
|
||||
|
||||
def _fetch_with_requests(self, url):
|
||||
"""备用方案:使用 requests 抓取静态内容"""
|
||||
try:
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'
|
||||
}
|
||||
response = requests.get(url, headers=headers, timeout=30)
|
||||
response.raise_for_status()
|
||||
|
||||
soup = BeautifulSoup(response.text, 'html.parser')
|
||||
|
||||
# 获取标题
|
||||
title = soup.title.string.strip() if soup.title else ''
|
||||
|
||||
# 移除不需要的标签
|
||||
for tag in soup(['script', 'style', 'nav', 'footer', 'header', 'aside']):
|
||||
tag.decompose()
|
||||
|
||||
# 获取主要内容
|
||||
text = soup.get_text(separator='\n', strip=True)
|
||||
# 清理多余空白行
|
||||
lines = [line.strip() for line in text.split('\n') if line.strip()]
|
||||
text = '\n'.join(lines)
|
||||
|
||||
# 提取描述(前200字符)
|
||||
description = text[:200].strip() if text else ''
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'title': title,
|
||||
'description': description,
|
||||
'content': text,
|
||||
'url': url,
|
||||
'fetch_date': datetime.now().isoformat()
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"备用抓取失败: {url}, 错误: {str(e)}")
|
||||
return None
|
||||
|
||||
def search_articles(self, keyword, category=None):
|
||||
"""从内容库搜索"""
|
||||
return db.search_articles(keyword, category)
|
||||
@@ -321,9 +390,9 @@ class SearchService:
|
||||
|
||||
return results
|
||||
|
||||
def save_to_articles(self, product_names, category, keywords, summary, content, source, url=None):
|
||||
def save_to_articles(self, product_names, category, keywords, summary, content, source, url=None, search_title=None):
|
||||
"""保存搜索结果到内容库"""
|
||||
return db.add_article(product_names, category, keywords, summary, content, source, url)
|
||||
return db.add_article(product_names, category, keywords, summary, content, source, url, search_title)
|
||||
|
||||
# 全局搜索服务实例
|
||||
search_service = SearchService()
|
||||
@@ -102,6 +102,15 @@
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.page-size-select {
|
||||
padding: 10px 15px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 6px;
|
||||
font-size: 14px;
|
||||
background: white;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.toolbar-right {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
@@ -228,6 +237,32 @@
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.article-search-title {
|
||||
font-size: 13px;
|
||||
color: #666;
|
||||
margin-bottom: 10px;
|
||||
padding: 4px 8px;
|
||||
background: #f0f4ff;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.article-search-title i {
|
||||
color: #667eea;
|
||||
}
|
||||
|
||||
.article-page-title {
|
||||
font-size: 13px;
|
||||
color: #666;
|
||||
margin-bottom: 10px;
|
||||
padding: 4px 8px;
|
||||
background: #e8f5e9;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.article-page-title i {
|
||||
color: #4caf50;
|
||||
}
|
||||
|
||||
.article-summary {
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
|
||||
@@ -151,6 +151,13 @@
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.progress-info {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
/* 结果区域 */
|
||||
.results-section {
|
||||
background: white;
|
||||
@@ -261,6 +268,19 @@
|
||||
color: #667eea;
|
||||
}
|
||||
|
||||
.result-page-title {
|
||||
font-size: 13px;
|
||||
color: #666;
|
||||
margin-bottom: 5px;
|
||||
padding: 4px 8px;
|
||||
background: #f0f4ff;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.result-page-title i {
|
||||
color: #667eea;
|
||||
}
|
||||
|
||||
.result-title .saved-icon {
|
||||
color: #10b981;
|
||||
}
|
||||
|
||||
@@ -801,4 +801,37 @@ body {
|
||||
.library-entrance-link small {
|
||||
font-size: 13px;
|
||||
opacity: 0.8;
|
||||
}
|
||||
|
||||
/* 快速统计卡片 */
|
||||
.quick-stats {
|
||||
display: flex;
|
||||
gap: 20px;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.quick-stat-item {
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.quick-stat-value {
|
||||
font-size: 24px;
|
||||
font-weight: bold;
|
||||
color: #667eea;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.quick-stat-label {
|
||||
font-size: 13px;
|
||||
color: #666;
|
||||
}
|
||||
|
||||
.search-quick-info {
|
||||
text-align: center;
|
||||
margin-top: 10px;
|
||||
color: #999;
|
||||
}
|
||||
|
||||
.search-quick-info small {
|
||||
font-size: 12px;
|
||||
}
|
||||
+34
-2
@@ -24,8 +24,8 @@ async function refreshData() {
|
||||
await Promise.all([
|
||||
loadStats(),
|
||||
loadPendingProducts(),
|
||||
loadArticles(),
|
||||
loadHistory()
|
||||
loadHistory(),
|
||||
loadQuickStats()
|
||||
]);
|
||||
updateSystemStatus(true);
|
||||
} catch (error) {
|
||||
@@ -66,6 +66,38 @@ async function loadStats() {
|
||||
}
|
||||
}
|
||||
|
||||
// 加载快速统计信息
|
||||
async function loadQuickStats() {
|
||||
try {
|
||||
// 加载文章统计
|
||||
const articlesResponse = await fetch(`${API_BASE}/api/articles?limit=1000`);
|
||||
const articlesData = await articlesResponse.json();
|
||||
|
||||
if (articlesData.success) {
|
||||
document.getElementById('quick-articles-count').textContent = articlesData.count;
|
||||
|
||||
// 计算分类数
|
||||
const categories = new Set(articlesData.articles.map(a => a.category).filter(c => c));
|
||||
document.getElementById('quick-categories-count').textContent = categories.size;
|
||||
|
||||
// 计算今日新增
|
||||
const today = new Date().toISOString().split('T')[0];
|
||||
const todayCount = articlesData.articles.filter(a => a.fetch_date && a.fetch_date.startsWith(today)).length;
|
||||
document.getElementById('quick-today-count').textContent = todayCount;
|
||||
}
|
||||
|
||||
// 加载失败URL数
|
||||
const failedResponse = await fetch(`${API_BASE}/api/articles/failed-urls`);
|
||||
const failedData = await failedResponse.json();
|
||||
|
||||
if (failedData.success) {
|
||||
document.getElementById('quick-failed-count').textContent = failedData.count;
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('加载快速统计失败:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// 加载待处理产品
|
||||
async function loadPendingProducts() {
|
||||
try {
|
||||
|
||||
+33
-7
@@ -4,7 +4,7 @@ const API_BASE = '';
|
||||
// 状态
|
||||
let articles = [];
|
||||
let currentPage = 1;
|
||||
let pageSize = 20;
|
||||
let pageSize = 100;
|
||||
let totalCount = 0;
|
||||
let selectedIds = new Set();
|
||||
let currentArticleId = null;
|
||||
@@ -31,6 +31,13 @@ document.addEventListener('DOMContentLoaded', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// 改变每页数量
|
||||
function changePageSize() {
|
||||
pageSize = parseInt(document.getElementById('page-size').value);
|
||||
currentPage = 1;
|
||||
loadArticles();
|
||||
}
|
||||
|
||||
// 加载文章列表
|
||||
async function loadArticles() {
|
||||
const keyword = document.getElementById('search-input').value.trim();
|
||||
@@ -42,7 +49,12 @@ async function loadArticles() {
|
||||
});
|
||||
|
||||
if (keyword) {
|
||||
const response = await fetch(`${API_BASE}/api/articles/search?q=${encodeURIComponent(keyword)}${category ? '&category=' + encodeURIComponent(category) : ''}`);
|
||||
// 搜索时也传递 limit 参数
|
||||
let searchUrl = `${API_BASE}/api/articles/search?q=${encodeURIComponent(keyword)}`;
|
||||
if (category) searchUrl += `&category=${encodeURIComponent(category)}`;
|
||||
searchUrl += `&limit=${pageSize}`;
|
||||
|
||||
const response = await fetch(searchUrl);
|
||||
const data = await response.json();
|
||||
|
||||
if (data.success) {
|
||||
@@ -98,6 +110,9 @@ function displayArticles() {
|
||||
const productNames = safeParseJSON(article.product_names, []);
|
||||
const keywords = safeParseJSON(article.keywords, []);
|
||||
|
||||
// 确定显示标题:优先 search_title,其次 product_names,最后来源
|
||||
const displayTitle = article.search_title || productNames.join(', ') || article.source || '未命名';
|
||||
|
||||
return `
|
||||
<div class="article-card ${selectedIds.has(article.id) ? 'selected' : ''}" id="article-${article.id}">
|
||||
<div class="article-card-header">
|
||||
@@ -107,7 +122,7 @@ function displayArticles() {
|
||||
onchange="toggleSelect(${article.id})">
|
||||
</label>
|
||||
<div class="article-title" onclick="showDetail(${article.id})">
|
||||
${escapeHtml(productNames.join(', ') || '未命名')}
|
||||
${escapeHtml(displayTitle)}
|
||||
</div>
|
||||
<div class="article-actions">
|
||||
<button onclick="editArticle(${article.id})" class="btn btn-sm btn-secondary">
|
||||
@@ -118,6 +133,11 @@ function displayArticles() {
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
${article.search_title && productNames.length > 0 && article.search_title !== productNames.join(', ') ? `
|
||||
<div class="article-page-title">
|
||||
<i class="ri-article-line"></i> 网页标题: ${escapeHtml(productNames.join(', '))}
|
||||
</div>
|
||||
` : ''}
|
||||
<div class="article-meta">
|
||||
${article.category ? `<span class="article-category">${escapeHtml(article.category)}</span>` : ''}
|
||||
<span><i class="ri-link"></i> ${escapeHtml(article.source || '未知来源')}</span>
|
||||
@@ -225,16 +245,22 @@ async function showDetail(id) {
|
||||
|
||||
const body = document.getElementById('detail-body');
|
||||
body.innerHTML = `
|
||||
${article.search_title ? `
|
||||
<div class="detail-section">
|
||||
<h4><i class="ri-search-line"></i> 搜索标题</h4>
|
||||
<p style="color: #667eea; font-weight: 500;">${escapeHtml(article.search_title)}</p>
|
||||
</div>
|
||||
` : ''}
|
||||
<div class="detail-section">
|
||||
<h4><i class="ri-price-tag-3-line"></i> 产品名称</h4>
|
||||
<p>${escapeHtml(productNames.join(', '))}</p>
|
||||
<h4><i class="ri-article-line"></i> 网页标题(产品名称)</h4>
|
||||
<p>${escapeHtml(productNames.join(', ') || '无')}</p>
|
||||
</div>
|
||||
<div class="detail-meta" style="display: flex; gap: 20px; background: #f8f9fa; padding: 15px; border-radius: 8px; margin-bottom: 20px;">
|
||||
<div class="detail-meta" style="display: flex; gap: 20px; background: #f8f9fa; padding: 15px; border-radius: 8px; margin-bottom: 20px; flex-wrap: wrap;">
|
||||
<div><strong>分类:</strong> ${escapeHtml(article.category || '未分类')}</div>
|
||||
<div><strong>来源:</strong> ${escapeHtml(article.source)}</div>
|
||||
<div><strong>时间:</strong> ${formatDate(article.fetch_date)}</div>
|
||||
</div>
|
||||
${article.url ? `<div class="detail-section"><h4><i class="ri-link"></i> 原文链接</h4><a href="${escapeHtml(article.url)}" target="_blank">${escapeHtml(article.url)}</a></div>` : ''}
|
||||
${article.url ? `<div class="detail-section"><h4><i class="ri-link"></i> 原文链接</h4><a href="${escapeHtml(article.url)}" target="_blank" style="color: #667eea;">${escapeHtml(article.url)}</a></div>` : ''}
|
||||
${keywords.length > 0 ? `<div class="detail-section"><h4><i class="ri-keyword-line"></i> 关键词</h4><p>${escapeHtml(keywords.join(', '))}</p></div>` : ''}
|
||||
<div class="detail-section">
|
||||
<h4><i class="ri-file-text-line"></i> 摘要</h4>
|
||||
|
||||
+60
-9
@@ -5,6 +5,9 @@ const API_BASE = '';
|
||||
let searchResults = [];
|
||||
let currentResultIndex = -1;
|
||||
|
||||
// 自动流程控制
|
||||
let shouldStop = false;
|
||||
|
||||
// 页面加载初始化
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
// 回车搜索
|
||||
@@ -23,6 +26,7 @@ async function doSearch() {
|
||||
const keyword = document.getElementById('search-keyword').value.trim();
|
||||
const maxResults = parseInt(document.getElementById('search-count').value) || 10;
|
||||
const engine = document.getElementById('search-engine').value;
|
||||
const useCache = document.getElementById('use-cache').checked;
|
||||
const autoFetch = document.getElementById('auto-fetch').checked;
|
||||
const autoSave = document.getElementById('auto-save').checked;
|
||||
|
||||
@@ -41,9 +45,11 @@ async function doSearch() {
|
||||
updateStatus('searching', `正在通过 ${engineNames[engine]} 搜索...`);
|
||||
document.getElementById('search-btn').disabled = true;
|
||||
|
||||
// 显示进度
|
||||
// 显示进度和停止按钮
|
||||
shouldStop = false;
|
||||
const progress = document.getElementById('search-progress');
|
||||
progress.style.display = 'block';
|
||||
document.getElementById('stop-btn').style.display = 'inline-flex';
|
||||
document.getElementById('progress-fill').style.width = '0%';
|
||||
document.getElementById('progress-text').textContent = '正在搜索...';
|
||||
|
||||
@@ -51,7 +57,7 @@ async function doSearch() {
|
||||
const response = await fetch(`${API_BASE}/api/articles/internet-search`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ keyword, max_results: maxResults, engine })
|
||||
body: JSON.stringify({ keyword, max_results: maxResults, engine, use_cache: useCache })
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
@@ -65,31 +71,50 @@ async function doSearch() {
|
||||
}));
|
||||
|
||||
displayResults();
|
||||
updateStatus('success', `找到 ${searchResults.length} 条结果`);
|
||||
|
||||
// 检查是否被停止
|
||||
if (shouldStop) {
|
||||
document.getElementById('stop-btn').style.display = 'none';
|
||||
document.getElementById('progress-text').textContent = '已停止';
|
||||
updateStatus('warning', '搜索已停止');
|
||||
return;
|
||||
}
|
||||
|
||||
// 显示缓存状态
|
||||
if (data.cached) {
|
||||
updateStatus('success', `找到 ${searchResults.length} 条结果(使用缓存)`);
|
||||
showToast('使用缓存结果', 'success');
|
||||
} else {
|
||||
updateStatus('success', `找到 ${searchResults.length} 条结果`);
|
||||
}
|
||||
|
||||
document.getElementById('results-count').textContent = `${searchResults.length} 条结果`;
|
||||
document.getElementById('save-all-btn').disabled = searchResults.length === 0;
|
||||
|
||||
document.getElementById('progress-fill').style.width = '100%';
|
||||
document.getElementById('progress-text').textContent = '搜索完成!';
|
||||
document.getElementById('stop-btn').style.display = 'none'; // 隐藏停止按钮
|
||||
|
||||
// 自动抓取和保存
|
||||
if (autoFetch && searchResults.length > 0) {
|
||||
setTimeout(() => fetchAllResults(autoSave), 500);
|
||||
} else {
|
||||
setTimeout(() => {
|
||||
progress.style.display = 'none';
|
||||
}, 1000);
|
||||
}
|
||||
} else {
|
||||
document.getElementById('stop-btn').style.display = 'none';
|
||||
updateStatus('error', '搜索失败');
|
||||
showToast('搜索失败: ' + data.error, 'error');
|
||||
}
|
||||
} catch (error) {
|
||||
document.getElementById('stop-btn').style.display = 'none';
|
||||
updateStatus('error', '搜索出错');
|
||||
showToast('搜索出错', 'error');
|
||||
console.error(error);
|
||||
}
|
||||
|
||||
setTimeout(() => {
|
||||
progress.style.display = 'none';
|
||||
}, 1000);
|
||||
|
||||
document.getElementById('search-btn').disabled = false;
|
||||
}
|
||||
|
||||
@@ -110,6 +135,11 @@ function displayResults() {
|
||||
${r.saved ? '<i class="ri-check-line saved-icon"></i>' : ''}
|
||||
${escapeHtml(r.title)}
|
||||
</div>
|
||||
${r.pageTitle && r.pageTitle !== r.title ? `
|
||||
<div class="result-page-title">
|
||||
<i class="ri-article-line"></i> 网页标题: ${escapeHtml(r.pageTitle)}
|
||||
</div>
|
||||
` : ''}
|
||||
<div class="result-url">
|
||||
<a href="${escapeHtml(r.url)}" target="_blank">
|
||||
<i class="ri-external-link-line"></i>
|
||||
@@ -182,7 +212,8 @@ async function fetchResult(index) {
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
url: result.url,
|
||||
product_names: [result.title]
|
||||
product_names: [result.title],
|
||||
search_title: result.title // 搜索结果的标题
|
||||
})
|
||||
});
|
||||
|
||||
@@ -191,7 +222,7 @@ async function fetchResult(index) {
|
||||
if (data.success) {
|
||||
searchResults[index].fetched = true;
|
||||
searchResults[index].content = data.data.content;
|
||||
searchResults[index].title = data.data.title || result.title;
|
||||
searchResults[index].pageTitle = data.data.title || result.title; // 抓取到的标题
|
||||
|
||||
btn.innerHTML = '<i class="ri-check-line"></i> 已抓取';
|
||||
|
||||
@@ -329,13 +360,27 @@ async function saveAllResults() {
|
||||
|
||||
// 抓取所有结果
|
||||
async function fetchAllResults(autoSave) {
|
||||
shouldStop = false; // 重置停止标志
|
||||
|
||||
const progress = document.getElementById('search-progress');
|
||||
progress.style.display = 'block';
|
||||
document.getElementById('stop-btn').style.display = 'inline-flex'; // 显示停止按钮
|
||||
|
||||
const total = searchResults.length;
|
||||
let fetched = 0;
|
||||
|
||||
for (let i = 0; i < searchResults.length; i++) {
|
||||
// 检查是否停止
|
||||
if (shouldStop) {
|
||||
document.getElementById('progress-text').textContent = `已停止(已抓取 ${fetched}/${total})`;
|
||||
document.getElementById('stop-btn').style.display = 'none';
|
||||
setTimeout(() => {
|
||||
progress.style.display = 'none';
|
||||
}, 2000);
|
||||
showToast('自动抓取已停止', 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
if (searchResults[i].fetched) {
|
||||
fetched++;
|
||||
continue;
|
||||
@@ -352,6 +397,7 @@ async function fetchAllResults(autoSave) {
|
||||
|
||||
document.getElementById('progress-fill').style.width = '100%';
|
||||
document.getElementById('progress-text').textContent = '抓取完成!';
|
||||
document.getElementById('stop-btn').style.display = 'none';
|
||||
|
||||
setTimeout(() => {
|
||||
progress.style.display = 'none';
|
||||
@@ -363,6 +409,11 @@ async function fetchAllResults(autoSave) {
|
||||
}
|
||||
}
|
||||
|
||||
// 停止自动处理
|
||||
function stopAutoProcess() {
|
||||
shouldStop = true;
|
||||
}
|
||||
|
||||
// 显示结果详情
|
||||
function showResultDetail(index) {
|
||||
currentResultIndex = index;
|
||||
|
||||
+42
-20
@@ -97,32 +97,54 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 右侧:内容库入口 -->
|
||||
<!-- 右侧:内容库和搜索入口 -->
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2><i class="ri-folder-line"></i> 内容库文章</h2>
|
||||
</div>
|
||||
<div class="panel-body" style="text-align: center; padding: 40px;">
|
||||
<a href="/library" class="library-entrance-link">
|
||||
<i class="ri-folder-line"></i>
|
||||
<span>点击进入内容库页面</span>
|
||||
<small>查看、编辑、导出所有文章</small>
|
||||
<h2><i class="ri-folder-line"></i> 内容库</h2>
|
||||
<a href="/library" class="btn btn-primary btn-sm">
|
||||
<i class="ri-external-link-line"></i> 进入
|
||||
</a>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div class="quick-stats">
|
||||
<div class="quick-stat-item">
|
||||
<span class="quick-stat-value" id="quick-articles-count">0</span>
|
||||
<span class="quick-stat-label">篇文章</span>
|
||||
</div>
|
||||
<div class="quick-stat-item">
|
||||
<span class="quick-stat-value" id="quick-categories-count">0</span>
|
||||
<span class="quick-stat-label">个分类</span>
|
||||
</div>
|
||||
<div class="quick-stat-item">
|
||||
<span class="quick-stat-value" id="quick-today-count">0</span>
|
||||
<span class="quick-stat-label">今日新增</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 互联网搜索入口 -->
|
||||
<div class="panel full-width">
|
||||
<div class="panel-header">
|
||||
<h2><i class="ri-search-line"></i> 互联网搜索</h2>
|
||||
</div>
|
||||
<div class="panel-body" style="text-align: center; padding: 40px;">
|
||||
<a href="/search" class="search-entrance-link">
|
||||
<i class="ri-search-line"></i>
|
||||
<span>点击进入互联网搜索页面</span>
|
||||
<small>支持一键保存所有搜索结果到内容库</small>
|
||||
</a>
|
||||
<div class="panel">
|
||||
<div class="panel-header">
|
||||
<h2><i class="ri-search-line"></i> 互联网搜索</h2>
|
||||
<a href="/search" class="btn btn-primary btn-sm">
|
||||
<i class="ri-external-link-line"></i> 进入
|
||||
</a>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div class="quick-stats">
|
||||
<div class="quick-stat-item">
|
||||
<span class="quick-stat-value" id="quick-failed-count">0</span>
|
||||
<span class="quick-stat-label">抓取失败</span>
|
||||
</div>
|
||||
<div class="quick-stat-item">
|
||||
<span class="quick-stat-label">支持引擎</span>
|
||||
<span class="quick-stat-value">Bing/百度/Google</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="search-quick-info">
|
||||
<small>支持一键保存所有结果到内容库</small>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -35,6 +35,13 @@
|
||||
<select id="category-filter" class="category-select">
|
||||
<option value="">全部分类</option>
|
||||
</select>
|
||||
<select id="page-size" class="page-size-select" onchange="changePageSize()">
|
||||
<option value="20">每页 20 条</option>
|
||||
<option value="50">每页 50 条</option>
|
||||
<option value="100" selected>每页 100 条</option>
|
||||
<option value="200">每页 200 条</option>
|
||||
<option value="500">每页 500 条</option>
|
||||
</select>
|
||||
</div>
|
||||
<div class="toolbar-right">
|
||||
<button onclick="showAddModal()" class="btn btn-primary">
|
||||
|
||||
+10
-1
@@ -42,6 +42,10 @@
|
||||
</button>
|
||||
</div>
|
||||
<div class="search-options">
|
||||
<label class="checkbox-label">
|
||||
<input type="checkbox" id="use-cache" checked>
|
||||
<span>使用缓存结果(7天有效)</span>
|
||||
</label>
|
||||
<label class="checkbox-label">
|
||||
<input type="checkbox" id="auto-fetch" checked>
|
||||
<span>搜索后自动抓取内容</span>
|
||||
@@ -58,7 +62,12 @@
|
||||
<div class="progress-bar">
|
||||
<div class="progress-fill" id="progress-fill"></div>
|
||||
</div>
|
||||
<div class="progress-text" id="progress-text">正在搜索...</div>
|
||||
<div class="progress-info">
|
||||
<span class="progress-text" id="progress-text">正在搜索...</span>
|
||||
<button onclick="stopAutoProcess()" class="btn btn-danger btn-sm" id="stop-btn" style="display: none;">
|
||||
<i class="ri-stop-line"></i> 停止
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 搜索结果 -->
|
||||
|
||||
Reference in New Issue
Block a user