V1.0.0: 基于索引的知识检索系统

核心功能:
- 文档索引:使用LLM分析提取关键词/摘要/主题/实体
- 查询处理:LLM分析查询意图并扩展关键词
- BM25检索:基于倒排索引的相关性排序
- RAG问答:检索增强生成

技术栈:
- Flask + SQLAlchemy
- OpenAI API兼容LLM
- BM25算法

特点: 不依赖向量模型和向量库
This commit is contained in:
2026-04-07 23:48:06 +08:00
commit cdaadef10c
10 changed files with 2079 additions and 0 deletions

217
templates/documents.html Normal file
View File

@@ -0,0 +1,217 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>文档管理 - LLM Index RAG</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
<link href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.0/font/bootstrap-icons.css" rel="stylesheet">
<style>
body { background-color: #f8f9fa; }
.doc-card { border-radius: 10px; transition: all 0.3s; }
.doc-card:hover { transform: translateY(-3px); box-shadow: 0 5px 20px rgba(0,0,0,0.1); }
</style>
</head>
<body>
<nav class="navbar navbar-expand-lg navbar-dark bg-dark">
<div class="container">
<a class="navbar-brand" href="/"><i class="bi bi-search"></i> LLM Index RAG</a>
<div class="navbar-nav ms-auto">
<a class="nav-link" href="/">首页</a>
<a class="nav-link active" href="/documents">文档管理</a>
<a class="nav-link" href="/search">知识检索</a>
</div>
</div>
</nav>
<div class="container mt-4">
<div class="d-flex justify-content-between align-items-center mb-4">
<h4><i class="bi bi-file-earmark-text"></i> 文档管理</h4>
<div>
<button class="btn btn-success" onclick="batchIndex()">
<i class="bi bi-arrow-repeat"></i> 批量索引
</button>
<button class="btn btn-warning" onclick="rebuildIndex()">
<i class="bi bi-trash"></i> 重建索引
</button>
</div>
</div>
<!-- 上传区域 -->
<div class="card mb-4">
<div class="card-header">
<h6 class="mb-0"><i class="bi bi-upload"></i> 上传文档</h6>
</div>
<div class="card-body">
<form id="uploadForm" enctype="multipart/form-data">
<div class="row">
<div class="col-md-6">
<input type="file" class="form-control" name="file" id="docFile" accept=".txt,.md,.pdf,.docx,.html,.json">
</div>
<div class="col-md-3">
<input type="text" class="form-control" name="title" placeholder="文档标题(可选)">
</div>
<div class="col-md-3">
<button type="submit" class="btn btn-primary w-100">
<i class="bi bi-upload"></i> 上传
</button>
</div>
</div>
</form>
</div>
</div>
<!-- 文档列表 -->
<div class="card">
<div class="card-header">
<h6 class="mb-0"><i class="bi bi-list"></i> 文档列表</h6>
</div>
<div class="card-body p-0">
<table class="table table-hover mb-0">
<thead class="table-light">
<tr>
<th>文件名</th>
<th>类型</th>
<th>大小</th>
<th>分块数</th>
<th>状态</th>
<th>索引时间</th>
<th>操作</th>
</tr>
</thead>
<tbody id="docList">
<tr><td colspan="7" class="text-center py-4">加载中...</td></tr>
</tbody>
</table>
</div>
</div>
<!-- 分页 -->
<nav class="mt-3">
<ul class="pagination justify-content-center" id="pagination"></ul>
</nav>
</div>
<script>
let currentPage = 1;
// 加载文档列表
function loadDocs(page = 1) {
currentPage = page;
fetch(`/api/documents?page=${page}`)
.then(r => r.json())
.then(data => {
const tbody = document.getElementById('docList');
tbody.innerHTML = '';
data.documents.forEach(doc => {
const statusBadge = doc.status === 'indexed'
? '<span class="badge bg-success">已索引</span>'
: doc.status === 'processing'
? '<span class="badge bg-warning">处理中</span>'
: doc.status === 'failed'
? '<span class="badge bg-danger">失败</span>'
: '<span class="badge bg-secondary">待索引</span>';
tbody.innerHTML += `
<tr>
<td>${doc.title || doc.filename}</td>
<td>${doc.file_type}</td>
<td>${(doc.file_size / 1024).toFixed(1)}KB</td>
<td>${doc.chunk_count}</td>
<td>${statusBadge}</td>
<td>${doc.indexed_at ? new Date(doc.indexed_at).toLocaleString() : '-'}</td>
<td>
${doc.status === 'pending' ? `<button class="btn btn-sm btn-outline-primary" onclick="indexDoc(${doc.id})"><i class="bi bi-arrow-repeat"></i></button>` : ''}
<button class="btn btn-sm btn-outline-info" onclick="viewDoc(${doc.id})"><i class="bi bi-eye"></i></button>
<button class="btn btn-sm btn-outline-danger" onclick="deleteDoc(${doc.id})"><i class="bi bi-trash"></i></button>
</td>
</tr>
`;
});
// 分页
const pagination = document.getElementById('pagination');
pagination.innerHTML = '';
for (let i = 1; i <= data.pages; i++) {
pagination.innerHTML += `
<li class="page-item ${i === page ? 'active' : ''}">
<a class="page-link" href="#" onclick="loadDocs(${i})">${i}</a>
</li>
`;
}
});
}
// 上传文档
document.getElementById('uploadForm').addEventListener('submit', function(e) {
e.preventDefault();
const formData = new FormData(this);
fetch('/api/documents', { method: 'POST', body: formData })
.then(r => r.json())
.then(data => {
if (data.success) {
alert('上传成功');
loadDocs();
} else {
alert('上传失败: ' + data.error);
}
});
});
// 索引文档
function indexDoc(id) {
fetch(`/api/index/${id}`, { method: 'POST' })
.then(r => r.json())
.then(data => {
alert(data.success ? '索引完成' : '索引失败');
loadDocs(currentPage);
});
}
// 批量索引
function batchIndex() {
if (!confirm('确定批量索引所有待索引文档?')) return;
fetch('/api/index/batch', { method: 'POST' })
.then(r => r.json())
.then(data => {
alert(`索引完成: 成功${data.success}, 失败${data.failed}`);
loadDocs();
});
}
// 重建索引
function rebuildIndex() {
if (!confirm('确定重建所有索引?这将清除现有索引!')) return;
fetch('/api/index/rebuild', { method: 'POST' })
.then(r => r.json())
.then(data => {
alert(`重建完成: 成功${data.success}, 失败${data.failed}`);
loadDocs();
});
}
// 删除文档
function deleteDoc(id) {
if (!confirm('确定删除此文档?')) return;
fetch(`/api/documents/${id}`, { method: 'DELETE' })
.then(r => r.json())
.then(data => {
if (data.success) loadDocs(currentPage);
});
}
// 查看文档
function viewDoc(id) {
window.location.href = `/documents/${id}`;
}
// 初始化
loadDocs();
</script>
</body>
</html>

162
templates/index.html Normal file
View File

@@ -0,0 +1,162 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>LLM Index RAG - 基于索引的知识检索系统</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
<link href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.0/font/bootstrap-icons.css" rel="stylesheet">
<style>
body { background-color: #f8f9fa; }
.hero { background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; padding: 60px 0; }
.hero h1 { font-size: 2.5rem; margin-bottom: 1rem; }
.search-box { background: white; border-radius: 10px; padding: 20px; box-shadow: 0 5px 20px rgba(0,0,0,0.1); }
.stat-card { border-radius: 10px; border: none; transition: transform 0.3s; }
.stat-card:hover { transform: translateY(-5px); }
.result-item { border-left: 4px solid #667eea; padding-left: 15px; margin-bottom: 20px; }
.source-tag { font-size: 0.8rem; background: #e9ecef; padding: 2px 8px; border-radius: 4px; }
</style>
</head>
<body>
<!-- 导航栏 -->
<nav class="navbar navbar-expand-lg navbar-dark bg-dark">
<div class="container">
<a class="navbar-brand" href="/">
<i class="bi bi-search"></i> LLM Index RAG
</a>
<div class="navbar-nav ms-auto">
<a class="nav-link" href="/">首页</a>
<a class="nav-link" href="/documents">文档管理</a>
<a class="nav-link" href="/search">知识检索</a>
</div>
</div>
</nav>
<!-- Hero区域 -->
<div class="hero">
<div class="container text-center">
<h1><i class="bi bi-diagram-3"></i> LLM Index RAG</h1>
<p class="lead">基于索引和搜索的知识检索系统(不使用向量模型)</p>
<p class="small">使用LLM构建索引 • 关键词检索 • BM25排序</p>
</div>
</div>
<!-- 搜索区域 -->
<div class="container" style="margin-top: -30px;">
<div class="search-box">
<form id="searchForm">
<div class="input-group input-group-lg">
<input type="text" class="form-control" id="queryInput"
placeholder="输入您的问题..." autocomplete="off">
<button class="btn btn-primary" type="submit">
<i class="bi bi-search"></i> 检索
</button>
</div>
<div class="mt-2">
<div class="form-check form-check-inline">
<input class="form-check-input" type="radio" name="mode" id="modeSearch" value="search" checked>
<label class="form-check-label" for="modeSearch">文档检索</label>
</div>
<div class="form-check form-check-inline">
<input class="form-check-input" type="radio" name="mode" id="modeRAG" value="rag">
<label class="form-check-label" for="modeRAG">智能问答</label>
</div>
</div>
</form>
</div>
</div>
<!-- 统计信息 -->
<div class="container mt-5">
<div class="row">
<div class="col-md-3">
<div class="card stat-card text-center p-4">
<i class="bi bi-file-earmark-text display-4 text-primary"></i>
<h3 class="mt-2" id="statDocs">{{ stats.total_documents or 0 }}</h3>
<small class="text-muted">文档数量</small>
</div>
</div>
<div class="col-md-3">
<div class="card stat-card text-center p-4">
<i class="bi bi-puzzle display-4 text-success"></i>
<h3 class="mt-2" id="statChunks">{{ stats.total_chunks or 0 }}</h3>
<small class="text-muted">文档分块</small>
</div>
</div>
<div class="col-md-3">
<div class="card stat-card text-center p-4">
<i class="bi bi-key display-4 text-warning"></i>
<h3 class="mt-2" id="statTerms">{{ stats.total_terms or 0 }}</h3>
<small class="text-muted">索引词条</small>
</div>
</div>
<div class="col-md-3">
<div class="card stat-card text-center p-4">
<i class="bi bi-file-word display-4 text-info"></i>
<h3 class="mt-2" id="statWords">{{ "{:,}".format(stats.total_words or 0) }}</h3>
<small class="text-muted">总字数</small>
</div>
</div>
</div>
</div>
<!-- 搜索结果 -->
<div class="container mt-4">
<div id="resultsSection" style="display:none;">
<h5><i class="bi bi-list-ul"></i> 检索结果 <span id="resultCount" class="badge bg-secondary"></span></h5>
<div id="resultsContainer"></div>
</div>
<!-- RAG回答 -->
<div id="ragSection" style="display:none;">
<h5><i class="bi bi-chat-dots"></i> 智能回答</h5>
<div id="ragAnswer" class="card p-4 mb-4"></div>
<h6><i class="bi bi-book"></i> 参考来源</h6>
<div id="ragSources"></div>
</div>
</div>
<!-- 工作原理 -->
<div class="container mt-5 mb-5">
<h4 class="text-center mb-4"><i class="bi bi-gear"></i> 工作原理</h4>
<div class="row">
<div class="col-md-4">
<div class="card h-100">
<div class="card-body text-center">
<i class="bi bi-file-earmark-plus display-4 text-primary"></i>
<h5 class="mt-3">1. 文档索引</h5>
<p class="text-muted">使用LLM分析文档提取关键词、摘要、主题、实体等信息构建索引</p>
</div>
</div>
</div>
<div class="col-md-4">
<div class="card h-100">
<div class="card-body text-center">
<i class="bi bi-search-heart display-4 text-success"></i>
<h5 class="mt-3">2. 查询处理</h5>
<p class="text-muted">LLM分析查询意图提取关键词并进行查询扩展</p>
</div>
</div>
</div>
<div class="col-md-4">
<div class="card h-100">
<div class="card-body text-center">
<i class="bi bi-sort-numeric-up display-4 text-warning"></i>
<h5 class="mt-3">3. BM25检索</h5>
<p class="text-muted">基于倒排索引和BM25算法计算相关性得分返回最相关文档</p>
</div>
</div>
</div>
</div>
</div>
<footer class="bg-dark text-white py-4 mt-5">
<div class="container text-center">
<p>LLM Index RAG v1.0.0 | 基于索引的知识检索系统</p>
</div>
</footer>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
<script src="/static/js/main.js"></script>
</body>
</html>

195
templates/search.html Normal file
View File

@@ -0,0 +1,195 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="UTF-8">
<title>知识检索 - LLM Index RAG</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
<link href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.10.0/font/bootstrap-icons.css" rel="stylesheet">
<style>
body { background-color: #f8f9fa; }
.search-box { background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); padding: 40px 0; }
.search-input { font-size: 1.2rem; border-radius: 10px; }
.result-card { border-radius: 10px; border-left: 4px solid #667eea; transition: all 0.3s; }
.result-card:hover { transform: translateX(5px); }
.rag-answer { background: linear-gradient(to right, #f8f9fa, #e9ecef); border-radius: 10px; }
.source-badge { font-size: 0.75rem; }
</style>
</head>
<body>
<nav class="navbar navbar-expand-lg navbar-dark bg-dark">
<div class="container">
<a class="navbar-brand" href="/"><i class="bi bi-search"></i> LLM Index RAG</a>
<div class="navbar-nav ms-auto">
<a class="nav-link" href="/">首页</a>
<a class="nav-link" href="/documents">文档管理</a>
<a class="nav-link active" href="/search">知识检索</a>
</div>
</div>
</nav>
<!-- 搜索框 -->
<div class="search-box">
<div class="container">
<div class="row justify-content-center">
<div class="col-md-8">
<form id="searchForm">
<div class="input-group input-group-lg">
<input type="text" class="form-control search-input" id="queryInput"
placeholder="输入您的问题或关键词..." autocomplete="off">
<button class="btn btn-light" type="submit">
<i class="bi bi-search"></i>
</button>
</div>
<div class="mt-3 text-white">
<div class="form-check form-check-inline">
<input class="form-check-input" type="radio" name="mode" id="modeSearch" value="search" checked>
<label class="form-check-label" for="modeSearch">文档检索</label>
</div>
<div class="form-check form-check-inline">
<input class="form-check-input" type="radio" name="mode" id="modeRAG" value="rag">
<label class="form-check-label" for="modeRAG">智能问答</label>
</div>
</div>
</form>
</div>
</div>
</div>
</div>
<!-- 结果区域 -->
<div class="container mt-4">
<!-- 加载中 -->
<div id="loadingSection" style="display:none;" class="text-center py-4">
<div class="spinner-border text-primary" role="status"></div>
<p class="mt-2">正在检索...</p>
</div>
<!-- RAG回答 -->
<div id="ragSection" style="display:none;">
<h5><i class="bi bi-chat-dots-fill"></i> 智能回答</h5>
<div id="ragAnswer" class="card rag-answer p-4 mb-4"></div>
<h6><i class="bi bi-book"></i> 参考来源</h6>
<div id="ragSources"></div>
</div>
<!-- 搜索结果 -->
<div id="resultsSection" style="display:none;">
<div class="d-flex justify-content-between align-items-center mb-3">
<h5><i class="bi bi-list-ul"></i> 检索结果</h5>
<span id="resultCount" class="badge bg-primary"></span>
</div>
<div id="resultsContainer"></div>
</div>
</div>
<script>
document.getElementById('searchForm').addEventListener('submit', function(e) {
e.preventDefault();
const query = document.getElementById('queryInput').value.trim();
const mode = document.querySelector('input[name="mode"]:checked').value;
if (!query) return;
// 显示加载
document.getElementById('loadingSection').style.display = 'block';
document.getElementById('ragSection').style.display = 'none';
document.getElementById('resultsSection').style.display = 'none';
if (mode === 'rag') {
// RAG问答
fetch('/api/rag/answer', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({query: query})
})
.then(r => r.json())
.then(data => {
document.getElementById('loadingSection').style.display = 'none';
document.getElementById('ragSection').style.display = 'block';
// 显示回答
document.getElementById('ragAnswer').innerHTML = `
<div class="d-flex align-items-start">
<i class="bi bi-robot fs-4 me-3 text-primary"></i>
<div>${data.answer.replace(/\n/g, '<br>')}</div>
</div>
<div class="mt-3 text-muted small">
<i class="bi bi-speedometer2"></i> 置信度: ${(data.confidence * 100).toFixed(0)}%
</div>
`;
// 显示来源
let sourcesHtml = '<div class="row">';
data.sources.forEach((s, i) => {
sourcesHtml += `
<div class="col-md-4 mb-3">
<div class="card result-card">
<div class="card-body">
<h6 class="card-title">${s.title}</h6>
<small class="text-muted">相关度: ${s.score.toFixed(2)}</small>
</div>
</div>
</div>
`;
});
sourcesHtml += '</div>';
document.getElementById('ragSources').innerHTML = sourcesHtml;
});
} else {
// 文档检索
fetch('/api/search', {
method: 'POST',
headers: {'Content-Type': 'application/json'},
body: JSON.stringify({query: query, top_k: 10})
})
.then(r => r.json())
.then(data => {
document.getElementById('loadingSection').style.display = 'none';
document.getElementById('resultsSection').style.display = 'block';
document.getElementById('resultCount').textContent = `${data.total} 条结果`;
let html = '';
data.results.forEach((r, i) => {
const chunks = r.matched_chunks.map(c =>
`<div class="bg-light p-2 rounded small mb-1">${c.content || c.summary || ''}</div>`
).join('');
html += `
<div class="card result-card mb-3">
<div class="card-body">
<div class="d-flex justify-content-between">
<h6 class="card-title">${r.doc.title || r.doc.filename}</h6>
<span class="badge bg-primary">${r.score.toFixed(2)}</span>
</div>
<div class="mb-2">
${r.matched_terms.map(t => `<span class="source-badge bg-info text-white px-2 py-1 rounded me-1">${t}</span>`).join('')}
</div>
<div class="mt-2">
${chunks}
</div>
</div>
</div>
`;
});
document.getElementById('resultsContainer').innerHTML = html || '<p class="text-muted">没有找到相关文档</p>';
});
}
});
// 自动补全
document.getElementById('queryInput').addEventListener('input', function() {
const prefix = this.value;
if (prefix.length < 2) return;
fetch(`/api/search/suggestions?prefix=${encodeURIComponent(prefix)}`)
.then(r => r.json())
.then(data => {
// 简单显示建议(可改用自动补全组件)
console.log('建议:', data.suggestions);
});
});
</script>
</body>
</html>