coder/llm-index-rag
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 添加配置管理功能和修复搜索问题

Browse Source 
新增:
- 系统设置页面 (/settings) - 支持动态配置LLM、索引、文档处理参数
- 配置API - 保存配置、测试LLM连接
- 前端JS交互文件 - 搜索、文档管理功能

修复:
- 首页搜索框无法正常工作的问题（缺少main.js）
- 服务支持动态读取配置（无需重启生效）

改进:
- LLM/索引/文档配置支持热更新
- 添加测试LLM连接功能
This commit is contained in:
coder
2026-04-09 12:54:31 +08:00
parent 4fb4d61877
commit 8c7a99d83f
6 changed files with 712 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
services.py
View File

@@ -16,17 +16,53 @@ from config import LLM_CONFIG, DOC_CONFIG, INDEX_CONFIG
from models import db, Document, DocumentChunk, InvertedIndex, IndexStats
def get_llm_config():
"""获取有效的LLM配置（支持动态更新）"""
user_config_file = os.path.join(os.path.dirname(__file__), 'user_config.json')
if os.path.exists(user_config_file):
with open(user_config_file, 'r', encoding='utf-8') as f:
user_config = json.load(f)
return {**LLM_CONFIG, **user_config.get('llm', {})}
return LLM_CONFIG
def get_doc_config():
"""获取有效的文档配置"""
user_config_file = os.path.join(os.path.dirname(__file__), 'user_config.json')
if os.path.exists(user_config_file):
with open(user_config_file, 'r', encoding='utf-8') as f:
user_config = json.load(f)
return {**DOC_CONFIG, **user_config.get('doc', {})}
return DOC_CONFIG
def get_index_config():
"""获取有效的索引配置"""
user_config_file = os.path.join(os.path.dirname(__file__), 'user_config.json')
if os.path.exists(user_config_file):
with open(user_config_file, 'r', encoding='utf-8') as f:
user_config = json.load(f)
return {**INDEX_CONFIG, **user_config.get('index', {})}
return INDEX_CONFIG
class LLMService:
"""LLM服务封装"""
def __init__(self):
self.client = OpenAI(
api_key=LLM_CONFIG['api_key'],
base_url=LLM_CONFIG['api_base'],
pass # 不再在初始化时设置配置
def _get_client(self):
"""获取LLM客户端"""
config = get_llm_config()
return OpenAI(
api_key=config['api_key'],
base_url=config['api_base'],
)
self.model = LLM_CONFIG['model']
self.max_tokens = LLM_CONFIG['max_tokens']
self.temperature = LLM_CONFIG['temperature']
def _get_config(self):
"""获取当前配置"""
return get_llm_config()
def analyze_document(self, content, title=None):
"""
@@ -60,8 +96,10 @@ class LLMService:
只返回JSON，不要其他内容。"""
try:
response = self.client.chat.completions.create(
model=self.model,
config = self._get_config()
client = self._get_client()
response = client.chat.completions.create(
model=config['model'],
messages=[{"role": "user", "content": prompt}],
max_tokens=1000,
temperature=0.3,
@@ -106,8 +144,10 @@ class LLMService:
只返回JSON。"""
try:
response = self.client.chat.completions.create(
model=self.model,
config = self._get_config()
client = self._get_client()
response = client.chat.completions.create(
model=config['model'],
messages=[{"role": "user", "content": prompt}],
max_tokens=500,
temperature=0.3,
@@ -150,8 +190,10 @@ class LLMService:
只返回JSON。"""
try:
response = self.client.chat.completions.create(
model=self.model,
config = self._get_config()
client = self._get_client()
response = client.chat.completions.create(
model=config['model'],
messages=[{"role": "user", "content": prompt}],
max_tokens=500,
temperature=0.3,
@@ -177,8 +219,11 @@ class DocumentIndexer:
def __init__(self):
self.llm = LLMService()
self.chunk_size = DOC_CONFIG['chunk_size']
self.chunk_overlap = DOC_CONFIG['chunk_overlap']
def _get_chunk_config(self):
"""获取分块配置"""
config = get_doc_config()
return config['chunk_size'], config['chunk_overlap']
def index_document(self, doc_id):
"""
@@ -330,6 +375,7 @@ class DocumentIndexer:
Returns:
list: 内容块列表
"""
chunk_size, _ = self._get_chunk_config()
chunks = []
# 按段落分割
@@ -337,7 +383,7 @@ class DocumentIndexer:
current_chunk = ""
for para in paragraphs:
if len(current_chunk) + len(para) < self.chunk_size:
if len(current_chunk) + len(para) < chunk_size:
current_chunk += para + '\n\n'
else:
if current_chunk.strip():
@@ -347,7 +393,7 @@ class DocumentIndexer:
if current_chunk.strip():
chunks.append(current_chunk.strip())
return chunks if chunks else [content[:self.chunk_size]]
return chunks if chunks else [content[:chunk_size]]
def _compute_term_freq(self, content):
"""计算词频"""
@@ -424,8 +470,11 @@ class SearchEngine:
def __init__(self):
self.llm = LLMService()
self.k1 = INDEX_CONFIG['bm25_k1']
self.b = INDEX_CONFIG['bm25_b']
def _get_bm25_params(self):
"""获取BM25参数"""
config = get_index_config()
return config['bm25_k1'], config['bm25_b']
def search(self, query, top_k=10):
"""
@@ -542,6 +591,7 @@ class SearchEngine:
continue
# BM25计算
k1, b = self._get_bm25_params()
score = 0
doc_len = doc.word_count or 1000
@@ -557,8 +607,8 @@ class SearchEngine:
tf = data['terms'].get(term, 0)
# BM25公式
tf_component = (tf * (self.k1 + 1)) / (
tf + self.k1 * (1 - self.b + self.b * doc_len / avg_doc_len)
tf_component = (tf * (k1 + 1)) / (
tf + k1 * (1 - b + b * doc_len / avg_doc_len)
)
score += idf * tf_component
@@ -653,13 +703,14 @@ class RAGGenerator:
请给出准确、简洁的回答，并标注信息来源。"""
try:
llm_config = get_llm_config()
client = OpenAI(
api_key=LLM_CONFIG['api_key'],
base_url=LLM_CONFIG['api_base'],
api_key=llm_config['api_key'],
base_url=llm_config['api_base'],
)
response = client.chat.completions.create(
model=LLM_CONFIG['model'],
model=llm_config['model'],
messages=[{"role": "user", "content": prompt}],
max_tokens=1000,
temperature=0.5,