coder
cdaadef10c
核心功能: - 文档索引：使用LLM分析提取关键词/摘要/主题/实体 - 查询处理：LLM分析查询意图并扩展关键词 - BM25检索：基于倒排索引的相关性排序 - RAG问答：检索增强生成 技术栈: - Flask + SQLAlchemy - OpenAI API兼容LLM - BM25算法 特点: 不依赖向量模型和向量库
275 lines
9.1 KiB
Python
275 lines
9.1 KiB
Python
"""
|
||
数据库模型定义
|
||
"""
|
||
|
||
from datetime import datetime
|
||
from flask_sqlalchemy import SQLAlchemy
|
||
import json
|
||
|
||
db = SQLAlchemy()
|
||
|
||
|
||
class Document(db.Model):
|
||
"""文档表"""
|
||
__tablename__ = 'documents'
|
||
|
||
id = db.Column(db.Integer, primary_key=True)
|
||
|
||
# 文件信息
|
||
filename = db.Column(db.String(255), nullable=False)
|
||
filepath = db.Column(db.String(512), nullable=False)
|
||
file_type = db.Column(db.String(20), nullable=False)
|
||
file_size = db.Column(db.Integer, default=0)
|
||
|
||
# 文档元数据
|
||
title = db.Column(db.String(500), nullable=True)
|
||
author = db.Column(db.String(100), nullable=True)
|
||
source = db.Column(db.String(255), nullable=True)
|
||
|
||
# 处理状态
|
||
status = db.Column(db.String(20), default='pending') # pending, processing, indexed, failed
|
||
error_message = db.Column(db.Text, nullable=True)
|
||
|
||
# 文档内容（可选存储原文）
|
||
content = db.Column(db.Text, nullable=True)
|
||
|
||
# 文档摘要（LLM生成）
|
||
summary = db.Column(db.Text, nullable=True)
|
||
|
||
# 主要关键词（JSON数组）
|
||
keywords = db.Column(db.Text, nullable=True)
|
||
|
||
# 文档分类/主题
|
||
category = db.Column(db.String(100), nullable=True)
|
||
topics = db.Column(db.Text, nullable=True) # JSON数组
|
||
|
||
# 文档实体（人物、地点、组织等）
|
||
entities = db.Column(db.Text, nullable=True) # JSON对象
|
||
|
||
# 统计信息
|
||
chunk_count = db.Column(db.Integer, default=0) # 分块数量
|
||
word_count = db.Column(db.Integer, default=0) # 字数
|
||
|
||
# 时间戳
|
||
created_at = db.Column(db.DateTime, default=datetime.utcnow)
|
||
indexed_at = db.Column(db.DateTime, nullable=True)
|
||
updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||
|
||
# 关系
|
||
chunks = db.relationship('DocumentChunk', backref='document', lazy=True, cascade='all, delete-orphan')
|
||
|
||
def get_keywords(self):
|
||
"""获取关键词列表"""
|
||
if self.keywords:
|
||
return json.loads(self.keywords)
|
||
return []
|
||
|
||
def set_keywords(self, keywords):
|
||
"""设置关键词"""
|
||
self.keywords = json.dumps(keywords, ensure_ascii=False)
|
||
|
||
def get_topics(self):
|
||
"""获取主题列表"""
|
||
if self.topics:
|
||
return json.loads(self.topics)
|
||
return []
|
||
|
||
def set_topics(self, topics):
|
||
"""设置主题"""
|
||
self.topics = json.dumps(topics, ensure_ascii=False)
|
||
|
||
def get_entities(self):
|
||
"""获取实体"""
|
||
if self.entities:
|
||
return json.loads(self.entities)
|
||
return {}
|
||
|
||
def set_entities(self, entities):
|
||
"""设置实体"""
|
||
self.entities = json.dumps(entities, ensure_ascii=False)
|
||
|
||
def to_dict(self):
|
||
return {
|
||
'id': self.id,
|
||
'filename': self.filename,
|
||
'title': self.title,
|
||
'status': self.status,
|
||
'summary': self.summary,
|
||
'keywords': self.get_keywords(),
|
||
'category': self.category,
|
||
'chunk_count': self.chunk_count,
|
||
'word_count': self.word_count,
|
||
'created_at': self.created_at.isoformat() if self.created_at else None,
|
||
'indexed_at': self.indexed_at.isoformat() if self.indexed_at else None,
|
||
}
|
||
|
||
|
||
class DocumentChunk(db.Model):
|
||
"""文档分块表"""
|
||
__tablename__ = 'document_chunks'
|
||
|
||
id = db.Column(db.Integer, primary_key=True)
|
||
document_id = db.Column(db.Integer, db.ForeignKey('documents.id'), nullable=False)
|
||
|
||
# 分块信息
|
||
chunk_index = db.Column(db.Integer, default=0) # 块序号
|
||
content = db.Column(db.Text, nullable=False) # 块内容
|
||
|
||
# LLM生成的索引信息
|
||
summary = db.Column(db.Text, nullable=True) # 块摘要
|
||
keywords = db.Column(db.Text, nullable=True) # 关键词JSON数组
|
||
topics = db.Column(db.Text, nullable=True) # 主题JSON数组
|
||
|
||
# 位置信息
|
||
start_char = db.Column(db.Integer, default=0)
|
||
end_char = db.Column(db.Integer, default=0)
|
||
|
||
# 词频统计（用于BM25）
|
||
term_freq = db.Column(db.Text, nullable=True) # JSON对象 {term: count}
|
||
|
||
created_at = db.Column(db.DateTime, default=datetime.utcnow)
|
||
|
||
def get_keywords(self):
|
||
if self.keywords:
|
||
return json.loads(self.keywords)
|
||
return []
|
||
|
||
def set_keywords(self, keywords):
|
||
self.keywords = json.dumps(keywords, ensure_ascii=False)
|
||
|
||
def get_term_freq(self):
|
||
if self.term_freq:
|
||
return json.loads(self.term_freq)
|
||
return {}
|
||
|
||
def set_term_freq(self, tf):
|
||
self.term_freq = json.dumps(tf, ensure_ascii=False)
|
||
|
||
def to_dict(self):
|
||
return {
|
||
'id': self.id,
|
||
'document_id': self.document_id,
|
||
'chunk_index': self.chunk_index,
|
||
'content': self.content[:200] + '...' if len(self.content) > 200 else self.content,
|
||
'summary': self.summary,
|
||
'keywords': self.get_keywords(),
|
||
}
|
||
|
||
|
||
class InvertedIndex(db.Model):
|
||
"""倒排索引表"""
|
||
__tablename__ = 'inverted_index'
|
||
|
||
id = db.Column(db.Integer, primary_key=True)
|
||
|
||
# 索引项
|
||
term = db.Column(db.String(100), nullable=False, index=True) # 关键词
|
||
term_type = db.Column(db.String(20), default='keyword') # keyword, entity, topic
|
||
|
||
# 文档频率
|
||
doc_freq = db.Column(db.Integer, default=0) # 包含该词的文档数
|
||
|
||
# 倒排列表（JSON：[{doc_id, chunk_id, tf, positions}]）
|
||
postings = db.Column(db.Text, nullable=False)
|
||
|
||
# 统计
|
||
total_freq = db.Column(db.Integer, default=0) # 总词频
|
||
|
||
updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||
|
||
def get_postings(self):
|
||
if self.postings:
|
||
return json.loads(self.postings)
|
||
return []
|
||
|
||
def set_postings(self, postings):
|
||
self.postings = json.dumps(postings, ensure_ascii=False)
|
||
self.doc_freq = len(set(p['doc_id'] for p in postings))
|
||
self.total_freq = sum(p.get('tf', 1) for p in postings)
|
||
|
||
@staticmethod
|
||
def get_or_create(term, term_type='keyword'):
|
||
"""获取或创建索引项"""
|
||
index = InvertedIndex.query.filter_by(term=term, term_type=term_type).first()
|
||
if not index:
|
||
index = InvertedIndex(term=term, term_type=term_type, postings='[]')
|
||
db.session.add(index)
|
||
return index
|
||
|
||
|
||
class QueryLog(db.Model):
|
||
"""查询日志表"""
|
||
__tablename__ = 'query_logs'
|
||
|
||
id = db.Column(db.Integer, primary_key=True)
|
||
|
||
# 查询信息
|
||
original_query = db.Column(db.Text, nullable=False) # 原始查询
|
||
processed_query = db.Column(db.Text, nullable=True) # 处理后的查询
|
||
expanded_terms = db.Column(db.Text, nullable=True) # 扩展词JSON数组
|
||
|
||
# 查询意图
|
||
intent = db.Column(db.String(50), nullable=True) # 查询意图
|
||
entities = db.Column(db.Text, nullable=True) # 识别的实体
|
||
|
||
# 检索结果
|
||
result_count = db.Column(db.Integer, default=0)
|
||
top_doc_ids = db.Column(db.Text, nullable=True) # 返回的文档ID JSON数组
|
||
|
||
# 性能
|
||
retrieval_time = db.Column(db.Float, default=0) # 检索耗时(秒)
|
||
total_time = db.Column(db.Float, default=0) # 总耗时(秒)
|
||
|
||
# 用户反馈
|
||
rating = db.Column(db.Integer, nullable=True) # 评分1-5
|
||
feedback = db.Column(db.Text, nullable=True) # 反馈文本
|
||
|
||
created_at = db.Column(db.DateTime, default=datetime.utcnow)
|
||
|
||
def to_dict(self):
|
||
return {
|
||
'id': self.id,
|
||
'query': self.original_query,
|
||
'result_count': self.result_count,
|
||
'retrieval_time': self.retrieval_time,
|
||
'created_at': self.created_at.isoformat() if self.created_at else None,
|
||
}
|
||
|
||
|
||
class IndexStats(db.Model):
|
||
"""索引统计表"""
|
||
__tablename__ = 'index_stats'
|
||
|
||
id = db.Column(db.Integer, primary_key=True)
|
||
|
||
# 统计信息
|
||
total_documents = db.Column(db.Integer, default=0)
|
||
total_chunks = db.Column(db.Integer, default=0)
|
||
total_terms = db.Column(db.Integer, default=0)
|
||
total_words = db.Column(db.Integer, default=0)
|
||
|
||
# 索引大小
|
||
index_size_mb = db.Column(db.Float, default=0)
|
||
|
||
# 最后更新
|
||
last_indexed_at = db.Column(db.DateTime, nullable=True)
|
||
|
||
updated_at = db.Column(db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||
|
||
@staticmethod
|
||
def get_stats():
|
||
stats = IndexStats.query.first()
|
||
if not stats:
|
||
stats = IndexStats()
|
||
db.session.add(stats)
|
||
db.session.commit()
|
||
return stats
|
||
|
||
def update_stats(self):
|
||
"""更新统计信息"""
|
||
self.total_documents = Document.query.filter_by(status='indexed').count()
|
||
self.total_chunks = DocumentChunk.query.count()
|
||
self.total_terms = InvertedIndex.query.count()
|
||
self.total_words = db.session.query(db.func.sum(Document.word_count)).scalar() or 0
|
||
self.last_indexed_at = datetime.utcnow()
|
||
db.session.commit() |