Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
06564babae |
@@ -1632,6 +1632,8 @@ pub fn kb_import_documents(
|
|||||||
file_size: bytes.len() as i64,
|
file_size: bytes.len() as i64,
|
||||||
char_count: text.chars().count() as i64,
|
char_count: text.chars().count() as i64,
|
||||||
chunk_count: chunks.len() as i64,
|
chunk_count: chunks.len() as i64,
|
||||||
|
file_path: String::new(),
|
||||||
|
source_id: None,
|
||||||
created_at: String::new(),
|
created_at: String::new(),
|
||||||
updated_at: String::new(),
|
updated_at: String::new(),
|
||||||
};
|
};
|
||||||
@@ -1684,6 +1686,193 @@ pub fn kb_search(
|
|||||||
.map_err(|e| e.to_string())
|
.map_err(|e| e.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct KbSourceInput {
|
||||||
|
pub kb_id: String,
|
||||||
|
pub path: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub extensions: String,
|
||||||
|
#[serde(default = "default_true")]
|
||||||
|
pub recursive: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct KbSourceScanResult {
|
||||||
|
pub imported: usize,
|
||||||
|
pub ignored: usize,
|
||||||
|
pub errors: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tauri::command]
|
||||||
|
pub fn kb_list_sources(
|
||||||
|
state: State<'_, App>,
|
||||||
|
kb_id: String,
|
||||||
|
) -> Result<Vec<xianren_core::KbSource>, String> {
|
||||||
|
let db = state.core.db.lock().unwrap();
|
||||||
|
kb_db::list_sources(&db, &kb_id).map_err(|e| e.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 添加目录来源并立即扫描导入:只收录文本 / 图片视频 / 音频三类文件,其余忽略。
|
||||||
|
#[tauri::command]
|
||||||
|
pub fn kb_add_source(
|
||||||
|
state: State<'_, App>,
|
||||||
|
input: KbSourceInput,
|
||||||
|
) -> Result<KbSourceScanResult, String> {
|
||||||
|
let dir = std::path::PathBuf::from(input.path.trim());
|
||||||
|
if !dir.is_dir() {
|
||||||
|
return Err(format!("目录不存在或不是文件夹:{}", dir.display()));
|
||||||
|
}
|
||||||
|
let exts = crate::knowledge::normalize_extensions(&input.extensions);
|
||||||
|
let db = state.core.db.lock().unwrap();
|
||||||
|
let kb = kb_db::get_knowledge_base(&db, &input.kb_id)
|
||||||
|
.map_err(|e| e.to_string())?
|
||||||
|
.ok_or_else(|| "知识库不存在".to_string())?;
|
||||||
|
let source_id = uuid::Uuid::new_v4().to_string();
|
||||||
|
let source = xianren_core::KbSource {
|
||||||
|
id: source_id.clone(),
|
||||||
|
kb_id: input.kb_id.clone(),
|
||||||
|
path: dir.to_string_lossy().to_string(),
|
||||||
|
extensions: input.extensions.trim().to_string(),
|
||||||
|
recursive: input.recursive,
|
||||||
|
created_at: String::new(),
|
||||||
|
};
|
||||||
|
kb_db::upsert_source(&db, &source).map_err(|e| e.to_string())?;
|
||||||
|
// 同目录重复添加时沿用已有来源 id,避免文档归属错乱
|
||||||
|
let source_id = kb_db::list_sources(&db, &input.kb_id)
|
||||||
|
.map_err(|e| e.to_string())?
|
||||||
|
.into_iter()
|
||||||
|
.find(|s| s.path == source.path && s.kb_id == source.kb_id)
|
||||||
|
.map(|s| s.id)
|
||||||
|
.unwrap_or(source_id);
|
||||||
|
Ok(scan_and_import(&db, &kb, &source_id, &dir, &exts, input.recursive))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 重新扫描该知识库的全部目录来源,导入新增文件(按文件路径去重)。
|
||||||
|
#[tauri::command]
|
||||||
|
pub fn kb_scan_sources(
|
||||||
|
state: State<'_, App>,
|
||||||
|
kb_id: String,
|
||||||
|
) -> Result<KbSourceScanResult, String> {
|
||||||
|
let db = state.core.db.lock().unwrap();
|
||||||
|
let kb = kb_db::get_knowledge_base(&db, &kb_id)
|
||||||
|
.map_err(|e| e.to_string())?
|
||||||
|
.ok_or_else(|| "知识库不存在".to_string())?;
|
||||||
|
let sources = kb_db::list_sources(&db, &kb_id).map_err(|e| e.to_string())?;
|
||||||
|
let mut total = KbSourceScanResult {
|
||||||
|
imported: 0,
|
||||||
|
ignored: 0,
|
||||||
|
errors: Vec::new(),
|
||||||
|
};
|
||||||
|
for src in sources {
|
||||||
|
let dir = std::path::PathBuf::from(&src.path);
|
||||||
|
if !dir.is_dir() {
|
||||||
|
total.errors.push(format!("目录不存在:{}", src.path));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let exts = crate::knowledge::normalize_extensions(&src.extensions);
|
||||||
|
let r = scan_and_import(&db, &kb, &src.id, &dir, &exts, src.recursive);
|
||||||
|
total.imported += r.imported;
|
||||||
|
total.ignored += r.ignored;
|
||||||
|
total.errors.extend(r.errors);
|
||||||
|
}
|
||||||
|
Ok(total)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 删除目录来源及其导入的全部文档。
|
||||||
|
#[tauri::command]
|
||||||
|
pub fn kb_remove_source(state: State<'_, App>, source_id: String) -> Result<usize, String> {
|
||||||
|
let db = state.core.db.lock().unwrap();
|
||||||
|
kb_db::get_source(&db, &source_id)
|
||||||
|
.map_err(|e| e.to_string())?
|
||||||
|
.ok_or_else(|| "目录来源不存在".to_string())?;
|
||||||
|
let removed = kb_db::delete_docs_by_source(&db, &source_id).map_err(|e| e.to_string())?;
|
||||||
|
kb_db::delete_source(&db, &source_id).map_err(|e| e.to_string())?;
|
||||||
|
Ok(removed)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 扫描目录并导入文档(文本提取切块;图片/视频/音频以文件名为内容登记;其他忽略)。
|
||||||
|
fn scan_and_import(
|
||||||
|
db: &rusqlite::Connection,
|
||||||
|
kb: &xianren_core::KnowledgeBase,
|
||||||
|
source_id: &str,
|
||||||
|
dir: &std::path::Path,
|
||||||
|
extensions: &[String],
|
||||||
|
recursive: bool,
|
||||||
|
) -> KbSourceScanResult {
|
||||||
|
let mut result = KbSourceScanResult {
|
||||||
|
imported: 0,
|
||||||
|
ignored: 0,
|
||||||
|
errors: Vec::new(),
|
||||||
|
};
|
||||||
|
let files = crate::knowledge::collect_files(dir, extensions, recursive);
|
||||||
|
for file in files {
|
||||||
|
let name = file
|
||||||
|
.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.to_string();
|
||||||
|
let file_path = file.to_string_lossy().to_string();
|
||||||
|
let category = crate::knowledge::classify(&name);
|
||||||
|
if category == crate::knowledge::FileCategory::Other {
|
||||||
|
result.ignored += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if kb_db::document_exists(db, &kb.id, &file_path).unwrap_or(false) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let bytes = match std::fs::read(&file) {
|
||||||
|
Ok(b) => b,
|
||||||
|
Err(e) => {
|
||||||
|
result.errors.push(format!("{name}: 读取失败 {e}"));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let content = if category == crate::knowledge::FileCategory::Text {
|
||||||
|
match crate::knowledge::extract_text(&name, &bytes) {
|
||||||
|
Ok(t) => t,
|
||||||
|
Err(e) => {
|
||||||
|
result.errors.push(format!("{name}: {e}"));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// 图片 / 视频 / 音频:以文件名为内容登记,便于按名检索
|
||||||
|
name.clone()
|
||||||
|
};
|
||||||
|
let chunks = if category == crate::knowledge::FileCategory::Text {
|
||||||
|
kb_db::chunk_text(&content, kb.chunk_size as usize, kb.chunk_overlap as usize)
|
||||||
|
} else {
|
||||||
|
vec![content.clone()]
|
||||||
|
};
|
||||||
|
if chunks.is_empty() {
|
||||||
|
result.errors.push(format!("{name}: 切分后没有有效分块"));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let doc = xianren_core::KbDocument {
|
||||||
|
id: uuid::Uuid::new_v4().to_string(),
|
||||||
|
kb_id: kb.id.clone(),
|
||||||
|
name,
|
||||||
|
file_type: file
|
||||||
|
.extension()
|
||||||
|
.map(|e| e.to_string_lossy().to_lowercase())
|
||||||
|
.unwrap_or_default(),
|
||||||
|
file_size: bytes.len() as i64,
|
||||||
|
char_count: content.chars().count() as i64,
|
||||||
|
chunk_count: chunks.len() as i64,
|
||||||
|
file_path,
|
||||||
|
source_id: Some(source_id.to_string()),
|
||||||
|
created_at: String::new(),
|
||||||
|
updated_at: String::new(),
|
||||||
|
};
|
||||||
|
if let Err(e) = kb_db::insert_document_with_chunks(db, &doc, &content, &chunks) {
|
||||||
|
result.errors.push(format!("{}: 写入失败 {e}", doc.name));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
result.imported += 1;
|
||||||
|
}
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
#[tauri::command]
|
#[tauri::command]
|
||||||
pub fn list_conversations(
|
pub fn list_conversations(
|
||||||
state: State<'_, App>,
|
state: State<'_, App>,
|
||||||
|
|||||||
@@ -1,10 +1,95 @@
|
|||||||
/// 按扩展名从文件字节提取纯文本:文本类文件直接按 UTF-8 读取,PDF 走 pdf-extract。
|
use std::path::Path;
|
||||||
pub fn extract_text(name: &str, bytes: &[u8]) -> Result<String, String> {
|
|
||||||
let ext = name
|
/// 文件分类:可读文本 / 图片视频 / 音频 / 其他(其他一律忽略)。
|
||||||
.rsplit('.')
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum FileCategory {
|
||||||
|
Text,
|
||||||
|
Media,
|
||||||
|
Other,
|
||||||
|
}
|
||||||
|
|
||||||
|
const TEXT_EXTS: &[&str] = &[
|
||||||
|
"txt", "md", "markdown", "json", "csv", "tsv", "log", "yml", "yaml", "toml", "ini",
|
||||||
|
"conf", "cfg", "xml", "html", "htm", "py", "rs", "ts", "tsx", "js", "jsx", "java", "c",
|
||||||
|
"cpp", "h", "hpp", "go", "php", "rb", "sh", "bat", "ps1", "sql", "css", "scss", "sass",
|
||||||
|
"pdf",
|
||||||
|
];
|
||||||
|
|
||||||
|
const IMAGE_EXTS: &[&str] = &[
|
||||||
|
"png", "jpg", "jpeg", "gif", "webp", "bmp", "svg", "tiff", "tif", "ico", "avif", "heic",
|
||||||
|
];
|
||||||
|
|
||||||
|
const VIDEO_EXTS: &[&str] = &[
|
||||||
|
"mp4", "mkv", "avi", "mov", "webm", "flv", "wmv", "m4v", "mpg", "mpeg", "3gp",
|
||||||
|
];
|
||||||
|
|
||||||
|
const AUDIO_EXTS: &[&str] = &[
|
||||||
|
"mp3", "wav", "flac", "ogg", "oga", "m4a", "aac", "wma", "opus", "aiff", "aif",
|
||||||
|
];
|
||||||
|
|
||||||
|
fn ext_of(name: &str) -> String {
|
||||||
|
name.rsplit('.')
|
||||||
.next()
|
.next()
|
||||||
.map(|e| e.to_lowercase())
|
.map(|e| e.to_lowercase())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 分类文件:文本(可提取内容)/ 媒体(图片、视频、音频)/ 其他。
|
||||||
|
pub fn classify(name: &str) -> FileCategory {
|
||||||
|
let ext = ext_of(name);
|
||||||
|
if TEXT_EXTS.contains(&ext.as_str()) {
|
||||||
|
FileCategory::Text
|
||||||
|
} else if IMAGE_EXTS.contains(&ext.as_str())
|
||||||
|
|| VIDEO_EXTS.contains(&ext.as_str())
|
||||||
|
|| AUDIO_EXTS.contains(&ext.as_str())
|
||||||
|
{
|
||||||
|
FileCategory::Media
|
||||||
|
} else {
|
||||||
|
FileCategory::Other
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 规范化用户输入的后缀列表:`.txt, .md;PDF` → ["txt", "md", "pdf"];空串表示全部。
|
||||||
|
pub fn normalize_extensions(raw: &str) -> Vec<String> {
|
||||||
|
raw.split([',', ';', ',', ';', ' '])
|
||||||
|
.map(|s| s.trim().trim_start_matches('.').to_lowercase())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn has_matching_ext(name: &str, extensions: &[String]) -> bool {
|
||||||
|
let ext = ext_of(name);
|
||||||
|
extensions.iter().any(|e| e == &ext)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 递归/非递归收集目录下符合后缀条件的文件,按路径排序。
|
||||||
|
pub fn collect_files(path: &Path, extensions: &[String], recursive: bool) -> Vec<std::path::PathBuf> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
fn walk(dir: &Path, exts: &[String], recursive: bool, out: &mut Vec<std::path::PathBuf>) {
|
||||||
|
let Ok(entries) = std::fs::read_dir(dir) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let p = entry.path();
|
||||||
|
if p.is_dir() {
|
||||||
|
if recursive {
|
||||||
|
walk(&p, exts, recursive, out);
|
||||||
|
}
|
||||||
|
} else if let Some(name) = p.file_name().and_then(|n| n.to_str()) {
|
||||||
|
if exts.is_empty() || has_matching_ext(name, exts) {
|
||||||
|
out.push(p);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
walk(path, extensions, recursive, &mut out);
|
||||||
|
out.sort();
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 按扩展名从文件字节提取纯文本:文本类文件直接按 UTF-8 读取,PDF 走 pdf-extract。
|
||||||
|
pub fn extract_text(name: &str, bytes: &[u8]) -> Result<String, String> {
|
||||||
|
let ext = ext_of(name);
|
||||||
let text = match ext.as_str() {
|
let text = match ext.as_str() {
|
||||||
"pdf" => pdf_extract::extract_text_from_mem(bytes)
|
"pdf" => pdf_extract::extract_text_from_mem(bytes)
|
||||||
.map_err(|e| format!("PDF 解析失败:{e}"))?,
|
.map_err(|e| format!("PDF 解析失败:{e}"))?,
|
||||||
@@ -31,3 +116,48 @@ pub fn extract_text(name: &str, bytes: &[u8]) -> Result<String, String> {
|
|||||||
}
|
}
|
||||||
Ok(cleaned)
|
Ok(cleaned)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn classify_three_categories() {
|
||||||
|
assert_eq!(classify("readme.md"), FileCategory::Text);
|
||||||
|
assert_eq!(classify("手册.pdf"), FileCategory::Text);
|
||||||
|
assert_eq!(classify("photo.PNG"), FileCategory::Media);
|
||||||
|
assert_eq!(classify("clip.mp4"), FileCategory::Media);
|
||||||
|
assert_eq!(classify("song.mp3"), FileCategory::Media);
|
||||||
|
assert_eq!(classify("app.exe"), FileCategory::Other);
|
||||||
|
assert_eq!(classify("archive.zip"), FileCategory::Other);
|
||||||
|
assert_eq!(classify("noext"), FileCategory::Other);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalize_extensions_handles_dots_and_separators() {
|
||||||
|
assert_eq!(
|
||||||
|
normalize_extensions(".txt, .md;PDF, png"),
|
||||||
|
vec!["txt", "md", "pdf", "png"]
|
||||||
|
);
|
||||||
|
assert!(normalize_extensions(" ").is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn collect_files_filters_and_recurses() {
|
||||||
|
let base = std::env::temp_dir().join(format!("kb_scan_test_{}", uuid::Uuid::new_v4()));
|
||||||
|
std::fs::create_dir_all(base.join("sub")).unwrap();
|
||||||
|
std::fs::write(base.join("a.txt"), "hello").unwrap();
|
||||||
|
std::fs::write(base.join("b.exe"), "x").unwrap();
|
||||||
|
std::fs::write(base.join("sub/c.md"), "hi").unwrap();
|
||||||
|
std::fs::write(base.join("sub/d.png"), "img").unwrap();
|
||||||
|
|
||||||
|
let all = collect_files(&base, &[], true);
|
||||||
|
assert_eq!(all.len(), 4);
|
||||||
|
let txt_only = collect_files(&base, &["txt".to_string()], true);
|
||||||
|
assert_eq!(txt_only.len(), 1);
|
||||||
|
let non_recursive = collect_files(&base, &[], false);
|
||||||
|
assert_eq!(non_recursive.len(), 2);
|
||||||
|
|
||||||
|
std::fs::remove_dir_all(&base).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -195,6 +195,10 @@ pub fn run() {
|
|||||||
commands::remove_kb_document,
|
commands::remove_kb_document,
|
||||||
commands::kb_rechunk_document,
|
commands::kb_rechunk_document,
|
||||||
commands::kb_search,
|
commands::kb_search,
|
||||||
|
commands::kb_list_sources,
|
||||||
|
commands::kb_add_source,
|
||||||
|
commands::kb_scan_sources,
|
||||||
|
commands::kb_remove_source,
|
||||||
commands::list_models,
|
commands::list_models,
|
||||||
commands::import_model,
|
commands::import_model,
|
||||||
commands::remove_model,
|
commands::remove_model,
|
||||||
|
|||||||
@@ -123,6 +123,8 @@ fn open_db(path: &Path) -> Result<Connection> {
|
|||||||
ensure_column(&conn, "scheduled_tasks", "email_mode", "TEXT NOT NULL DEFAULT 'fixed'")?;
|
ensure_column(&conn, "scheduled_tasks", "email_mode", "TEXT NOT NULL DEFAULT 'fixed'")?;
|
||||||
ensure_column(&conn, "scheduled_tasks", "email_subject", "TEXT NOT NULL DEFAULT ''")?;
|
ensure_column(&conn, "scheduled_tasks", "email_subject", "TEXT NOT NULL DEFAULT ''")?;
|
||||||
ensure_column(&conn, "scheduled_tasks", "email_body", "TEXT NOT NULL DEFAULT ''")?;
|
ensure_column(&conn, "scheduled_tasks", "email_body", "TEXT NOT NULL DEFAULT ''")?;
|
||||||
|
ensure_column(&conn, "kb_documents", "file_path", "TEXT NOT NULL DEFAULT ''")?;
|
||||||
|
ensure_column(&conn, "kb_documents", "source_id", "TEXT")?;
|
||||||
Ok(conn)
|
Ok(conn)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -24,10 +24,22 @@ pub struct KbDocument {
|
|||||||
pub file_size: i64,
|
pub file_size: i64,
|
||||||
pub char_count: i64,
|
pub char_count: i64,
|
||||||
pub chunk_count: i64,
|
pub chunk_count: i64,
|
||||||
|
pub file_path: String,
|
||||||
|
pub source_id: Option<String>,
|
||||||
pub created_at: String,
|
pub created_at: String,
|
||||||
pub updated_at: String,
|
pub updated_at: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct KbSource {
|
||||||
|
pub id: String,
|
||||||
|
pub kb_id: String,
|
||||||
|
pub path: String,
|
||||||
|
pub extensions: String,
|
||||||
|
pub recursive: bool,
|
||||||
|
pub created_at: String,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct KbDocumentDetail {
|
pub struct KbDocumentDetail {
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
@@ -45,7 +57,8 @@ pub struct KbSearchHit {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const KB_COLUMNS: &str = "id, name, description, chunk_size, chunk_overlap, created_at, updated_at";
|
const KB_COLUMNS: &str = "id, name, description, chunk_size, chunk_overlap, created_at, updated_at";
|
||||||
const DOC_COLUMNS: &str = "id, kb_id, name, file_type, file_size, char_count, chunk_count, created_at, updated_at";
|
const DOC_COLUMNS: &str = "id, kb_id, name, file_type, file_size, char_count, chunk_count, file_path, source_id, created_at, updated_at";
|
||||||
|
const SOURCE_COLUMNS: &str = "id, kb_id, path, extensions, recursive, created_at";
|
||||||
|
|
||||||
// ---------- 知识库 ----------
|
// ---------- 知识库 ----------
|
||||||
|
|
||||||
@@ -149,7 +162,7 @@ pub fn get_document_detail(db: &Connection, id: &str) -> Result<Option<KbDocumen
|
|||||||
))?;
|
))?;
|
||||||
let mut rows = stmt.query_map(params![id], |row| {
|
let mut rows = stmt.query_map(params![id], |row| {
|
||||||
let doc = row_to_doc(row)?;
|
let doc = row_to_doc(row)?;
|
||||||
let content: String = row.get(9)?;
|
let content: String = row.get(11)?;
|
||||||
Ok(KbDocumentDetail { doc, content })
|
Ok(KbDocumentDetail { doc, content })
|
||||||
})?;
|
})?;
|
||||||
match rows.next() {
|
match rows.next() {
|
||||||
@@ -168,8 +181,8 @@ pub fn insert_document_with_chunks(
|
|||||||
let tx = db.unchecked_transaction()?;
|
let tx = db.unchecked_transaction()?;
|
||||||
tx.execute(
|
tx.execute(
|
||||||
"INSERT INTO kb_documents
|
"INSERT INTO kb_documents
|
||||||
(id, kb_id, name, file_type, file_size, char_count, chunk_count, content)
|
(id, kb_id, name, file_type, file_size, char_count, chunk_count, content, file_path, source_id)
|
||||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
|
||||||
params![
|
params![
|
||||||
doc.id,
|
doc.id,
|
||||||
doc.kb_id,
|
doc.kb_id,
|
||||||
@@ -179,6 +192,8 @@ pub fn insert_document_with_chunks(
|
|||||||
doc.char_count,
|
doc.char_count,
|
||||||
chunks.len() as i64,
|
chunks.len() as i64,
|
||||||
content,
|
content,
|
||||||
|
doc.file_path,
|
||||||
|
doc.source_id,
|
||||||
],
|
],
|
||||||
)?;
|
)?;
|
||||||
for (i, chunk) in chunks.iter().enumerate() {
|
for (i, chunk) in chunks.iter().enumerate() {
|
||||||
@@ -203,6 +218,75 @@ pub fn delete_document(db: &Connection, id: &str) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 同一知识库下是否已存在来自该文件路径的文档(目录导入去重)。
|
||||||
|
pub fn document_exists(db: &Connection, kb_id: &str, file_path: &str) -> Result<bool> {
|
||||||
|
if file_path.is_empty() {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
let exists: bool = db.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM kb_documents WHERE kb_id = ?1 AND file_path = ?2)",
|
||||||
|
params![kb_id, file_path],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
Ok(exists)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------- 目录来源 ----------
|
||||||
|
|
||||||
|
pub fn list_sources(db: &Connection, kb_id: &str) -> Result<Vec<KbSource>> {
|
||||||
|
let mut stmt = db.prepare(&format!(
|
||||||
|
"SELECT {SOURCE_COLUMNS} FROM kb_sources WHERE kb_id = ?1 ORDER BY created_at ASC"
|
||||||
|
))?;
|
||||||
|
let rows = stmt.query_map(params![kb_id], row_to_source)?;
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for row in rows {
|
||||||
|
out.push(row?);
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_source(db: &Connection, id: &str) -> Result<Option<KbSource>> {
|
||||||
|
let mut stmt = db.prepare(&format!(
|
||||||
|
"SELECT {SOURCE_COLUMNS} FROM kb_sources WHERE id = ?1"
|
||||||
|
))?;
|
||||||
|
let mut rows = stmt.query_map(params![id], row_to_source)?;
|
||||||
|
match rows.next() {
|
||||||
|
Some(row) => Ok(Some(row?)),
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn upsert_source(db: &Connection, source: &KbSource) -> Result<()> {
|
||||||
|
db.execute(
|
||||||
|
"INSERT INTO kb_sources (id, kb_id, path, extensions, recursive)
|
||||||
|
VALUES (?1, ?2, ?3, ?4, ?5)
|
||||||
|
ON CONFLICT(kb_id, path) DO UPDATE SET
|
||||||
|
extensions = excluded.extensions, recursive = excluded.recursive",
|
||||||
|
params![
|
||||||
|
source.id,
|
||||||
|
source.kb_id,
|
||||||
|
source.path,
|
||||||
|
source.extensions,
|
||||||
|
source.recursive as i32,
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn delete_source(db: &Connection, id: &str) -> Result<()> {
|
||||||
|
db.execute("DELETE FROM kb_sources WHERE id = ?1", params![id])?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 删除某个目录来源导入的全部文档(分块随外键级联)。
|
||||||
|
pub fn delete_docs_by_source(db: &Connection, source_id: &str) -> Result<usize> {
|
||||||
|
let n = db.execute(
|
||||||
|
"DELETE FROM kb_documents WHERE source_id = ?1",
|
||||||
|
params![source_id],
|
||||||
|
)?;
|
||||||
|
Ok(n)
|
||||||
|
}
|
||||||
|
|
||||||
/// 重新切分文档:删除旧分块后按新参数重分并重建索引。
|
/// 重新切分文档:删除旧分块后按新参数重分并重建索引。
|
||||||
pub fn rechunk_document(
|
pub fn rechunk_document(
|
||||||
db: &Connection,
|
db: &Connection,
|
||||||
@@ -328,8 +412,22 @@ fn row_to_doc(row: &rusqlite::Row<'_>) -> rusqlite::Result<KbDocument> {
|
|||||||
file_size: row.get(4)?,
|
file_size: row.get(4)?,
|
||||||
char_count: row.get(5)?,
|
char_count: row.get(5)?,
|
||||||
chunk_count: row.get(6)?,
|
chunk_count: row.get(6)?,
|
||||||
created_at: row.get(7)?,
|
file_path: row.get(7)?,
|
||||||
updated_at: row.get(8)?,
|
source_id: row.get(8)?,
|
||||||
|
created_at: row.get(9)?,
|
||||||
|
updated_at: row.get(10)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn row_to_source(row: &rusqlite::Row<'_>) -> rusqlite::Result<KbSource> {
|
||||||
|
let recursive: i32 = row.get(4)?;
|
||||||
|
Ok(KbSource {
|
||||||
|
id: row.get(0)?,
|
||||||
|
kb_id: row.get(1)?,
|
||||||
|
path: row.get(2)?,
|
||||||
|
extensions: row.get(3)?,
|
||||||
|
recursive: recursive != 0,
|
||||||
|
created_at: row.get(5)?,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -363,6 +461,8 @@ mod tests {
|
|||||||
file_size: content.len() as i64,
|
file_size: content.len() as i64,
|
||||||
char_count: content.chars().count() as i64,
|
char_count: content.chars().count() as i64,
|
||||||
chunk_count: 0,
|
chunk_count: 0,
|
||||||
|
file_path: String::new(),
|
||||||
|
source_id: None,
|
||||||
created_at: String::new(),
|
created_at: String::new(),
|
||||||
updated_at: String::new(),
|
updated_at: String::new(),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ pub mod workflows;
|
|||||||
|
|
||||||
pub use app::CoreApp;
|
pub use app::CoreApp;
|
||||||
pub use error::{CoreError, Result};
|
pub use error::{CoreError, Result};
|
||||||
pub use knowledge_base::{KbDocument, KbDocumentDetail, KbSearchHit, KnowledgeBase};
|
pub use knowledge_base::{KbDocument, KbDocumentDetail, KbSearchHit, KbSource, KnowledgeBase};
|
||||||
pub use models::ModelInfo;
|
pub use models::ModelInfo;
|
||||||
pub use agents::Agent;
|
pub use agents::Agent;
|
||||||
pub use scheduled_tasks::ScheduledTask;
|
pub use scheduled_tasks::ScheduledTask;
|
||||||
|
|||||||
@@ -147,6 +147,8 @@ CREATE TABLE IF NOT EXISTS kb_documents (
|
|||||||
char_count INTEGER NOT NULL DEFAULT 0,
|
char_count INTEGER NOT NULL DEFAULT 0,
|
||||||
chunk_count INTEGER NOT NULL DEFAULT 0,
|
chunk_count INTEGER NOT NULL DEFAULT 0,
|
||||||
content TEXT NOT NULL DEFAULT '',
|
content TEXT NOT NULL DEFAULT '',
|
||||||
|
file_path TEXT NOT NULL DEFAULT '',
|
||||||
|
source_id TEXT,
|
||||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
);
|
);
|
||||||
@@ -160,6 +162,17 @@ CREATE TABLE IF NOT EXISTS kb_chunks (
|
|||||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
);
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS kb_sources (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
kb_id TEXT NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
|
||||||
|
path TEXT NOT NULL,
|
||||||
|
extensions TEXT NOT NULL DEFAULT '',
|
||||||
|
recursive INTEGER NOT NULL DEFAULT 1,
|
||||||
|
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_kb_sources_kb_path ON kb_sources(kb_id, path);
|
||||||
|
|
||||||
-- 全文检索索引(trigram 分词,对中文/短文本更友好;content='' 为 contentless-delete 模式)
|
-- 全文检索索引(trigram 分词,对中文/短文本更友好;content='' 为 contentless-delete 模式)
|
||||||
CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(content, content = '', tokenize = 'trigram');
|
CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(content, content = '', tokenize = 'trigram');
|
||||||
|
|
||||||
|
|||||||
@@ -12,7 +12,7 @@
|
|||||||
|
|
||||||
```
|
```
|
||||||
xianren_studio/
|
xianren_studio/
|
||||||
├── apps/desktop/ Tauri 桌面壳(Rust 命令层、tauri.conf.json、推荐模型/预制智能体/预制工作流默认 JSON、工作流执行引擎 workflow.rs、邮件发送 mail.rs、文档提取 knowledge.rs)
|
├── apps/desktop/ Tauri 桌面壳(Rust 命令层、tauri.conf.json、推荐模型/预制智能体/预制工作流默认 JSON、工作流执行引擎 workflow.rs、邮件发送 mail.rs、文档提取与目录扫描 knowledge.rs)
|
||||||
├── crates/core/ 领域核心:SQLite(模型注册表、智能体、工作流、知识库、定时计划、会话、消息、设置)
|
├── crates/core/ 领域核心:SQLite(模型注册表、智能体、工作流、知识库、定时计划、会话、消息、设置)
|
||||||
├── crates/engine/ llama-server 生命周期 + 流式/非流式聊天(本地与远程 OpenAI 兼容)
|
├── crates/engine/ llama-server 生命周期 + 流式/非流式聊天(本地与远程 OpenAI 兼容)
|
||||||
├── crates/download/ 分片断点续传下载器
|
├── crates/download/ 分片断点续传下载器
|
||||||
@@ -111,6 +111,7 @@ ui/src/
|
|||||||
| 定时计划 | `list_scheduled_tasks`、`add_scheduled_task`、`update_scheduled_task`、`remove_scheduled_task`、`set_scheduled_task_enabled`、`run_scheduled_task_now` |
|
| 定时计划 | `list_scheduled_tasks`、`add_scheduled_task`、`update_scheduled_task`、`remove_scheduled_task`、`set_scheduled_task_enabled`、`run_scheduled_task_now` |
|
||||||
| 邮件 | `mail_test`(用当前 SMTP 配置发送测试邮件) |
|
| 邮件 | `mail_test`(用当前 SMTP 配置发送测试邮件) |
|
||||||
| 知识库 | `list_knowledge_bases`、`add_knowledge_base`、`update_knowledge_base`、`remove_knowledge_base`、`list_kb_documents`、`get_kb_document`、`kb_import_documents`、`remove_kb_document`、`kb_rechunk_document`、`kb_search` |
|
| 知识库 | `list_knowledge_bases`、`add_knowledge_base`、`update_knowledge_base`、`remove_knowledge_base`、`list_kb_documents`、`get_kb_document`、`kb_import_documents`、`remove_kb_document`、`kb_rechunk_document`、`kb_search` |
|
||||||
|
| 知识库·目录来源 | `kb_list_sources`、`kb_add_source`、`kb_scan_sources`、`kb_remove_source` |
|
||||||
| 应用/设置 | `app_info`、`autostart_status`、`autostart_set`、`settings_get`、`settings_set` |
|
| 应用/设置 | `app_info`、`autostart_status`、`autostart_set`、`settings_get`、`settings_set` |
|
||||||
| 模型 | `list_models`、`import_model`、`remove_model`、`set_model_enabled`、`scan_models`、`add_remote_model` |
|
| 模型 | `list_models`、`import_model`、`remove_model`、`set_model_enabled`、`scan_models`、`add_remote_model` |
|
||||||
| 模型广场 | `search_models`、`list_model_files`、`list_recommended_models`、`import_recommendations`、`fetch_model_page` |
|
| 模型广场 | `search_models`、`list_model_files`、`list_recommended_models`、`import_recommendations`、`fetch_model_page` |
|
||||||
@@ -143,6 +144,7 @@ ui/src/
|
|||||||
- `send_task_email` / `load_smtp_config` / `split_recipients`:读取设置页「邮件」SMTP 配置、拆分多收件人并调用 `mail.rs` 发送。
|
- `send_task_email` / `load_smtp_config` / `split_recipients`:读取设置页「邮件」SMTP 配置、拆分多收件人并调用 `mail.rs` 发送。
|
||||||
- `kb_import_documents`:base64 解码 → `knowledge.rs::extract_text` 提取文本(文本类直接 UTF-8,PDF 走 pdf-extract)→ `kb_db::chunk_text` 切块(按知识库 chunk_size/overlap)→ `insert_document_with_chunks` 事务写入文档与分块,逐文件返回成功/失败结果。
|
- `kb_import_documents`:base64 解码 → `knowledge.rs::extract_text` 提取文本(文本类直接 UTF-8,PDF 走 pdf-extract)→ `kb_db::chunk_text` 切块(按知识库 chunk_size/overlap)→ `insert_document_with_chunks` 事务写入文档与分块,逐文件返回成功/失败结果。
|
||||||
- `kb_search` / `kb_rechunk_document`:检索走 `kb_db::search`(FTS5 trigram + BM25,短查询回退 LIKE);重切按知识库最新分块设置重建该文档分块与索引。
|
- `kb_search` / `kb_rechunk_document`:检索走 `kb_db::search`(FTS5 trigram + BM25,短查询回退 LIKE);重切按知识库最新分块设置重建该文档分块与索引。
|
||||||
|
- `kb_add_source` / `kb_scan_sources` / `kb_remove_source`:目录来源注册与扫描——`knowledge.rs::collect_files` 按后缀过滤(空=全部)与递归开关收集文件,`classify` 分成文本 / 图片视频 / 音频三类(其他忽略),文本提取切块入库、媒体以文件名为内容登记,`kb_db::document_exists` 按(kb_id + file_path)去重;删除来源级联删除其导入文档。
|
||||||
|
|
||||||
**邮件发送模块(`apps/desktop/src/mail.rs`):**
|
**邮件发送模块(`apps/desktop/src/mail.rs`):**
|
||||||
|
|
||||||
@@ -185,13 +187,14 @@ ui/src/
|
|||||||
| `skills` | `name`、`description`、`content`、`enabled` | 技能工具库 |
|
| `skills` | `name`、`description`、`content`、`enabled` | 技能工具库 |
|
||||||
| `mcp_servers` | `name`、`description`、`url`、`auth_token`、`enabled` | MCP 服务配置 |
|
| `mcp_servers` | `name`、`description`、`url`、`auth_token`、`enabled` | MCP 服务配置 |
|
||||||
| `knowledge_bases` | `name`、`description`、`chunk_size`、`chunk_overlap` | 知识库及分块设置 |
|
| `knowledge_bases` | `name`、`description`、`chunk_size`、`chunk_overlap` | 知识库及分块设置 |
|
||||||
| `kb_documents` | `kb_id`(级联删除)、`name`、`file_type`、`file_size`、`char_count`、`chunk_count`、`content`(提取后的纯文本) | 知识库文档;删除/重切分块由 FTS 触发器同步 |
|
| `kb_documents` | `kb_id`(级联删除)、`name`、`file_type`、`file_size`、`char_count`、`chunk_count`、`content`(提取后的纯文本/媒体文件名)、`file_path`、`source_id` | 知识库文档;目录导入记录文件路径与来源,`file_path` 用于去重;删除/重切分块由 FTS 触发器同步 |
|
||||||
| `kb_chunks` | `kb_id`、`document_id`(级联删除)、`seq`、`content` | 文档分块 |
|
| `kb_chunks` | `kb_id`、`document_id`(级联删除)、`seq`、`content` | 文档分块 |
|
||||||
| `kb_chunks_fts` | FTS5 虚拟表(contentless-delete + trigram 分词) | 全文检索索引,由 `kb_chunks` 触发器维护 |
|
| `kb_chunks_fts` | FTS5 虚拟表(contentless-delete + trigram 分词) | 全文检索索引,由 `kb_chunks` 触发器维护 |
|
||||||
|
| `kb_sources` | `kb_id`(级联删除)、`path`、`extensions`、`recursive`;`(kb_id, path)` 唯一 | 目录来源(多个);添加时立即扫描,可重新扫描增量导入 |
|
||||||
|
|
||||||
迁移清单(`ensure_column`):`models.kind/base_url/api_key/api_model/enabled`、`messages.elapsed_ms/first_token_ms/images_json/model_id`、`conversations.pinned/favorite/tools_json/agent_id`。
|
迁移清单(`ensure_column`):`models.kind/base_url/api_key/api_model/enabled`、`messages.elapsed_ms/first_token_ms/images_json/model_id`、`conversations.pinned/favorite/tools_json/agent_id`。
|
||||||
|
|
||||||
新增列迁移:`scheduled_tasks.email_enabled / email_to / email_mode / email_subject / email_body`(`ensure_column` 幂等补齐)。
|
新增列迁移:`scheduled_tasks.email_enabled / email_to / email_mode / email_subject / email_body`、`kb_documents.file_path / source_id`(`ensure_column` 幂等补齐)。
|
||||||
|
|
||||||
### 5.3 `models.rs`
|
### 5.3 `models.rs`
|
||||||
|
|
||||||
@@ -219,7 +222,7 @@ ui/src/
|
|||||||
|
|
||||||
### 5.9 `knowledge_base.rs`
|
### 5.9 `knowledge_base.rs`
|
||||||
|
|
||||||
知识库 CRUD(`list/get/insert/update/delete`)、文档 CRUD(`list/get/get_detail`)、`insert_document_with_chunks`(文档 + 分块同事务写入,FTS 触发器自动建索引)、`delete_document`(级联删除分块并同步索引)、`rechunk_document`、`chunk_text`(按字符切块 + 重叠)、`search`(FTS5 trigram BM25 排序,查询 <3 字或 MATCH 失败回退 LIKE)。
|
知识库 CRUD(`list/get/insert/update/delete`)、文档 CRUD(`list/get/get_detail`)、`insert_document_with_chunks`(文档 + 分块同事务写入,FTS 触发器自动建索引)、`delete_document`(级联删除分块并同步索引)、`document_exists`(目录导入按文件路径去重)、`rechunk_document`、`chunk_text`(按字符切块 + 重叠)、`search`(FTS5 trigram BM25 排序,查询 <3 字或 MATCH 失败回退 LIKE);目录来源 `list/get/upsert/delete_sources` 与 `delete_docs_by_source`。
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
@@ -133,6 +133,11 @@
|
|||||||
- 左侧导航新增「知识库」选项卡,页面分左右两栏:左侧知识库列表,右侧选中库的管理区。
|
- 左侧导航新增「知识库」选项卡,页面分左右两栏:左侧知识库列表,右侧选中库的管理区。
|
||||||
- **知识库管理**:新建 / 编辑 / 删除知识库,字段含名称、描述、**分块大小**(100–10000 字,默认 500)与**块间重叠**(默认 50);删除知识库会级联删除其文档与分块。
|
- **知识库管理**:新建 / 编辑 / 删除知识库,字段含名称、描述、**分块大小**(100–10000 字,默认 500)与**块间重叠**(默认 50);删除知识库会级联删除其文档与分块。
|
||||||
- **文档导入**:支持多文件上传,格式覆盖文本类文件(txt / md / json / csv / 代码 / HTML 等)与 **PDF**(后端提取文本);导入后按知识库分块设置自动切块并写入全文索引。
|
- **文档导入**:支持多文件上传,格式覆盖文本类文件(txt / md / json / csv / 代码 / HTML 等)与 **PDF**(后端提取文本);导入后按知识库分块设置自动切块并写入全文索引。
|
||||||
|
- **目录来源(批量导入)**:可为知识库添加多个本地目录,扫描目录批量导入文档:
|
||||||
|
- **后缀过滤**:可指定只导入的后缀列表(如 `.txt,.md,.pdf`),留空默认全部;
|
||||||
|
- **递归子目录**:可开关是否递归扫描所有子目录;
|
||||||
|
- **类型过滤**:只收录**可读文本**(提取内容并分块检索)、**图片 / 视频**、**音频**三类文件(图片视频音频以文件名为内容登记,可按文件名检索),检测到其他类型自动忽略;
|
||||||
|
- 同一目录重复添加自动更新配置不产生重复来源;按文件路径去重,可一键「重新扫描」增量导入新增文件;删除目录来源会同时删除它导入的文档。
|
||||||
- **文档操作**:文档列表展示类型 / 大小 / 字数 / 分块数;支持全文预览、「重切」(按最新分块设置重新切分,适用于修改设置后)与删除。
|
- **文档操作**:文档列表展示类型 / 大小 / 字数 / 分块数;支持全文预览、「重切」(按最新分块设置重新切分,适用于修改设置后)与删除。
|
||||||
- **检索**:全文检索基于 SQLite FTS5(trigram 分词,对中文友好),按 BM25 相关性排序;支持单库检索(也可全库检索),结果展示来源文档与分块序号,可一键复制分块内容用于对话。
|
- **检索**:全文检索基于 SQLite FTS5(trigram 分词,对中文友好),按 BM25 相关性排序;支持单库检索(也可全库检索),结果展示来源文档与分块序号,可一键复制分块内容用于对话。
|
||||||
- 数据表:`knowledge_bases`、`kb_documents`(含提取后的纯文本)、`kb_chunks`(分块)、`kb_chunks_fts`(FTS5 全文索引,contentless-delete 模式 + 触发器同步)。
|
- 数据表:`knowledge_bases`、`kb_documents`(含提取后的纯文本)、`kb_chunks`(分块)、`kb_chunks_fts`(FTS5 全文索引,contentless-delete 模式 + 触发器同步)。
|
||||||
@@ -224,6 +229,7 @@
|
|||||||
|
|
||||||
### 2026-08-17
|
### 2026-08-17
|
||||||
|
|
||||||
|
- 知识库新增「目录来源」:支持添加多个本地目录批量导入(后缀过滤默认全部、可开关递归子目录),只收录文本 / 图片视频 / 音频三类文件其余忽略,支持重新扫描增量导入与按来源删除。
|
||||||
- 新增「知识库」选项卡与完整操作界面:知识库管理(新建/编辑/删除)、多格式文档上传(文本类 + PDF)、自动分块与全文检索(FTS5 trigram + BM25)、文档预览 / 重新切分 / 删除、检索结果一键复制。
|
- 新增「知识库」选项卡与完整操作界面:知识库管理(新建/编辑/删除)、多格式文档上传(文本类 + PDF)、自动分块与全文检索(FTS5 trigram + BM25)、文档预览 / 重新切分 / 删除、检索结果一键复制。
|
||||||
- 计划任务支持发送邮件:任务设置中可开启邮件并填写收件人,内容支持固定内容与大模型生成两种模式;设置页新增「邮件」SMTP 配置与测试发送。
|
- 计划任务支持发送邮件:任务设置中可开启邮件并填写收件人,内容支持固定内容与大模型生成两种模式;设置页新增「邮件」SMTP 配置与测试发送。
|
||||||
- 设置页新增「启动」:开机启动开关(写入 Windows 注册表启动项);「自动加载模型」列表(多个本地模型按顺序在启动时逐个加载,最后一个保持运行)。
|
- 设置页新增「启动」:开机启动开关(写入 Windows 注册表启动项);「自动加载模型」列表(多个本地模型按顺序在启动时逐个加载,最后一个保持运行)。
|
||||||
|
|||||||
@@ -245,6 +245,28 @@ export interface KbImportResult {
|
|||||||
error: string | null;
|
error: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface KbSource {
|
||||||
|
id: string;
|
||||||
|
kb_id: string;
|
||||||
|
path: string;
|
||||||
|
extensions: string;
|
||||||
|
recursive: boolean;
|
||||||
|
created_at: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface KbSourceInput {
|
||||||
|
kb_id: string;
|
||||||
|
path: string;
|
||||||
|
extensions: string;
|
||||||
|
recursive: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface KbSourceScanResult {
|
||||||
|
imported: number;
|
||||||
|
ignored: number;
|
||||||
|
errors: string[];
|
||||||
|
}
|
||||||
|
|
||||||
export interface McpServer {
|
export interface McpServer {
|
||||||
id: string;
|
id: string;
|
||||||
name: string;
|
name: string;
|
||||||
@@ -483,6 +505,13 @@ export const api = {
|
|||||||
invoke<number>("kb_rechunk_document", { documentId }),
|
invoke<number>("kb_rechunk_document", { documentId }),
|
||||||
kbSearch: (kbId: string | null, query: string, limit?: number) =>
|
kbSearch: (kbId: string | null, query: string, limit?: number) =>
|
||||||
invoke<KbSearchHit[]>("kb_search", { kbId, query, limit: limit ?? 10 }),
|
invoke<KbSearchHit[]>("kb_search", { kbId, query, limit: limit ?? 10 }),
|
||||||
|
kbListSources: (kbId: string) => invoke<KbSource[]>("kb_list_sources", { kbId }),
|
||||||
|
kbAddSource: (input: KbSourceInput) =>
|
||||||
|
invoke<KbSourceScanResult>("kb_add_source", { input }),
|
||||||
|
kbScanSources: (kbId: string) =>
|
||||||
|
invoke<KbSourceScanResult>("kb_scan_sources", { kbId }),
|
||||||
|
kbRemoveSource: (sourceId: string) =>
|
||||||
|
invoke<number>("kb_remove_source", { sourceId }),
|
||||||
listModels: () => invoke<ModelInfo[]>("list_models"),
|
listModels: () => invoke<ModelInfo[]>("list_models"),
|
||||||
importModel: (path: string) => invoke<ModelInfo>("import_model", { path }),
|
importModel: (path: string) => invoke<ModelInfo>("import_model", { path }),
|
||||||
removeModel: (id: string) => invoke<void>("remove_model", { id }),
|
removeModel: (id: string) => invoke<void>("remove_model", { id }),
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ import {
|
|||||||
KbDocument,
|
KbDocument,
|
||||||
KbDocumentDetail,
|
KbDocumentDetail,
|
||||||
KbSearchHit,
|
KbSearchHit,
|
||||||
|
KbSource,
|
||||||
|
KbSourceScanResult,
|
||||||
KnowledgeBase,
|
KnowledgeBase,
|
||||||
KnowledgeBaseInput,
|
KnowledgeBaseInput,
|
||||||
} from "../api";
|
} from "../api";
|
||||||
@@ -56,6 +58,9 @@ export default function KnowledgeBasePage() {
|
|||||||
const [query, setQuery] = useState("");
|
const [query, setQuery] = useState("");
|
||||||
const [searching, setSearching] = useState(false);
|
const [searching, setSearching] = useState(false);
|
||||||
const [searchResults, setSearchResults] = useState<KbSearchHit[] | null>(null);
|
const [searchResults, setSearchResults] = useState<KbSearchHit[] | null>(null);
|
||||||
|
const [sources, setSources] = useState<KbSource[]>([]);
|
||||||
|
const [sourceModal, setSourceModal] = useState(false);
|
||||||
|
const [sourceMsg, setSourceMsg] = useState<string | null>(null);
|
||||||
const uploadInputRef = useRef<HTMLInputElement>(null);
|
const uploadInputRef = useRef<HTMLInputElement>(null);
|
||||||
|
|
||||||
const selected = useMemo(
|
const selected = useMemo(
|
||||||
@@ -88,6 +93,10 @@ export default function KnowledgeBasePage() {
|
|||||||
.listKbDocuments(selectedId)
|
.listKbDocuments(selectedId)
|
||||||
.then(setDocs)
|
.then(setDocs)
|
||||||
.catch((e) => setMsg(`加载文档失败:${String(e)}`));
|
.catch((e) => setMsg(`加载文档失败:${String(e)}`));
|
||||||
|
api
|
||||||
|
.kbListSources(selectedId)
|
||||||
|
.then(setSources)
|
||||||
|
.catch((e) => setMsg(`加载目录来源失败:${String(e)}`));
|
||||||
}, [selectedId]);
|
}, [selectedId]);
|
||||||
|
|
||||||
async function handleSaveKb(input: KnowledgeBaseInput) {
|
async function handleSaveKb(input: KnowledgeBaseInput) {
|
||||||
@@ -189,6 +198,76 @@ export default function KnowledgeBasePage() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function refreshSourcesAndDocs() {
|
||||||
|
if (!selected) return;
|
||||||
|
const [d, s] = await Promise.all([
|
||||||
|
api.listKbDocuments(selected.id),
|
||||||
|
api.kbListSources(selected.id),
|
||||||
|
]);
|
||||||
|
setDocs(d);
|
||||||
|
setSources(s);
|
||||||
|
await refreshKbs();
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatScanMsg(label: string, r: KbSourceScanResult) {
|
||||||
|
const errPart =
|
||||||
|
r.errors.length > 0
|
||||||
|
? `,${r.errors.length} 个错误(${r.errors
|
||||||
|
.slice(0, 3)
|
||||||
|
.join(";")}${r.errors.length > 3 ? "…" : ""})`
|
||||||
|
: "";
|
||||||
|
return `${label}完成:导入 ${r.imported} 个,忽略 ${r.ignored} 个${errPart}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleAddSource(path: string, extensions: string, recursive: boolean) {
|
||||||
|
if (!selected) return;
|
||||||
|
setBusy(true);
|
||||||
|
setSourceMsg(null);
|
||||||
|
try {
|
||||||
|
const r = await api.kbAddSource({
|
||||||
|
kb_id: selected.id,
|
||||||
|
path,
|
||||||
|
extensions,
|
||||||
|
recursive,
|
||||||
|
});
|
||||||
|
await refreshSourcesAndDocs();
|
||||||
|
setSourceMsg(formatScanMsg("添加目录", r));
|
||||||
|
} catch (e) {
|
||||||
|
setSourceMsg(`添加目录失败:${String(e)}`);
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
setSourceModal(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleScanAll() {
|
||||||
|
if (!selected) return;
|
||||||
|
setBusy(true);
|
||||||
|
setSourceMsg(null);
|
||||||
|
try {
|
||||||
|
const r = await api.kbScanSources(selected.id);
|
||||||
|
await refreshSourcesAndDocs();
|
||||||
|
setSourceMsg(formatScanMsg("重新扫描", r));
|
||||||
|
} catch (e) {
|
||||||
|
setSourceMsg(`重新扫描失败:${String(e)}`);
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handleRemoveSource(src: KbSource) {
|
||||||
|
if (!confirm(`确定删除目录来源「${src.path}」吗?将同时删除由它导入的文档。`)) return;
|
||||||
|
try {
|
||||||
|
const n = await api.kbRemoveSource(src.id);
|
||||||
|
setSourceMsg(`已删除目录来源及其 ${n} 个文档`);
|
||||||
|
if (selected) {
|
||||||
|
await refreshSourcesAndDocs();
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
setSourceMsg(`删除失败:${String(e)}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="flex h-full overflow-hidden">
|
<div className="flex h-full overflow-hidden">
|
||||||
{/* 左侧:知识库列表 */}
|
{/* 左侧:知识库列表 */}
|
||||||
@@ -343,6 +422,81 @@ export default function KnowledgeBasePage() {
|
|||||||
) : null}
|
) : null}
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
{/* 目录来源 */}
|
||||||
|
<section className="mb-6 rounded-xl border border-border bg-panel p-4">
|
||||||
|
<div className="mb-2 flex items-center justify-between">
|
||||||
|
<div className="text-xs uppercase text-slate-500">
|
||||||
|
目录来源({sources.length})
|
||||||
|
</div>
|
||||||
|
<div className="flex gap-2">
|
||||||
|
<button
|
||||||
|
className="btn-secondary !px-2.5 !py-1 text-xs"
|
||||||
|
onClick={handleScanAll}
|
||||||
|
disabled={busy || sources.length === 0}
|
||||||
|
>
|
||||||
|
<Icon name="refresh" className="mr-1 inline h-3 w-3" />
|
||||||
|
重新扫描
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
className="btn-primary !px-2.5 !py-1 text-xs"
|
||||||
|
onClick={() => setSourceModal(true)}
|
||||||
|
>
|
||||||
|
+ 添加目录
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<p className="mb-3 text-[11px] text-slate-500">
|
||||||
|
按目录批量导入文档:只收录可读文本、图片视频、音频三类文件,其余自动忽略;可指定后缀过滤(默认全部),可选是否递归子目录,同一知识库可添加多个目录
|
||||||
|
</p>
|
||||||
|
{sourceMsg ? (
|
||||||
|
<div className="mb-2 rounded-lg border border-border bg-panel-2/60 px-3 py-2 text-xs text-slate-400">
|
||||||
|
{sourceMsg}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
{sources.length === 0 ? (
|
||||||
|
<div className="rounded-lg border border-dashed border-border bg-panel-2/40 px-3 py-6 text-center text-xs text-slate-500">
|
||||||
|
还没有目录来源,点击「添加目录」批量导入文件夹中的文档
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="space-y-2">
|
||||||
|
{sources.map((src) => (
|
||||||
|
<div
|
||||||
|
key={src.id}
|
||||||
|
className="flex items-center gap-3 rounded-lg border border-border bg-panel-2/60 p-3"
|
||||||
|
>
|
||||||
|
<Icon name="folder" className="h-4 w-4 shrink-0 text-slate-400" />
|
||||||
|
<div className="min-w-0 flex-1">
|
||||||
|
<div className="truncate text-xs font-medium text-slate-300">
|
||||||
|
{src.path}
|
||||||
|
</div>
|
||||||
|
<div className="mt-0.5 flex items-center gap-1.5 text-[11px] text-slate-500">
|
||||||
|
<span className="rounded bg-panel-2 px-1.5 py-0.5">
|
||||||
|
{src.extensions.trim()
|
||||||
|
? src.extensions
|
||||||
|
.split(/[,;,;\s]+/)
|
||||||
|
.filter(Boolean)
|
||||||
|
.map((e) => (e.startsWith(".") ? e : `.${e}`))
|
||||||
|
.join(" ")
|
||||||
|
: "全部后缀"}
|
||||||
|
</span>
|
||||||
|
<span className="rounded bg-panel-2 px-1.5 py-0.5">
|
||||||
|
{src.recursive ? "递归子目录" : "仅当前目录"}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
className="rounded-lg border border-border bg-panel-2 px-2.5 py-1.5 text-xs text-slate-400 hover:text-red-400"
|
||||||
|
onClick={() => handleRemoveSource(src)}
|
||||||
|
title="删除该目录来源及其导入的文档"
|
||||||
|
>
|
||||||
|
<Icon name="trash" className="h-3.5 w-3.5" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</section>
|
||||||
|
|
||||||
{/* 文档列表 */}
|
{/* 文档列表 */}
|
||||||
<section>
|
<section>
|
||||||
<div className="mb-2 text-xs uppercase text-slate-500">
|
<div className="mb-2 text-xs uppercase text-slate-500">
|
||||||
@@ -381,6 +535,10 @@ export default function KnowledgeBasePage() {
|
|||||||
/>
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
|
{sourceModal ? (
|
||||||
|
<SourceModal busy={busy} onSave={handleAddSource} onClose={() => setSourceModal(false)} />
|
||||||
|
) : null}
|
||||||
|
|
||||||
{preview ? (
|
{preview ? (
|
||||||
<div
|
<div
|
||||||
className="fixed inset-0 z-50 flex items-center justify-center bg-black/70 p-6"
|
className="fixed inset-0 z-50 flex items-center justify-center bg-black/70 p-6"
|
||||||
@@ -605,3 +763,107 @@ function KbModal({
|
|||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function SourceModal({
|
||||||
|
busy,
|
||||||
|
onSave,
|
||||||
|
onClose,
|
||||||
|
}: {
|
||||||
|
busy: boolean;
|
||||||
|
onSave: (path: string, extensions: string, recursive: boolean) => void;
|
||||||
|
onClose: () => void;
|
||||||
|
}) {
|
||||||
|
const [path, setPath] = useState("");
|
||||||
|
const [extensions, setExtensions] = useState("");
|
||||||
|
const [recursive, setRecursive] = useState(true);
|
||||||
|
const valid = path.trim().length > 0;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className="fixed inset-0 z-50 flex items-center justify-center bg-black/70 p-6"
|
||||||
|
onClick={onClose}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
className="w-full max-w-md rounded-xl border border-border bg-panel"
|
||||||
|
onClick={(e) => e.stopPropagation()}
|
||||||
|
>
|
||||||
|
<div className="flex items-center justify-between border-b border-border px-5 py-3">
|
||||||
|
<span className="text-sm font-medium">添加目录来源</span>
|
||||||
|
<button className="text-slate-500 hover:text-slate-200" onClick={onClose}>
|
||||||
|
×
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div className="space-y-4 px-5 py-4">
|
||||||
|
<div>
|
||||||
|
<label className="mb-1 block text-xs text-slate-400">目录路径</label>
|
||||||
|
<input
|
||||||
|
className="input"
|
||||||
|
value={path}
|
||||||
|
placeholder="例如:D:\资料\项目文档"
|
||||||
|
onChange={(e) => setPath(e.target.value)}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label className="mb-1 block text-xs text-slate-400">后缀过滤(可选)</label>
|
||||||
|
<input
|
||||||
|
className="input"
|
||||||
|
value={extensions}
|
||||||
|
placeholder="留空表示全部;多个用逗号分隔,如 .txt,.md,.pdf,.png"
|
||||||
|
onChange={(e) => setExtensions(e.target.value)}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Toggle
|
||||||
|
checked={recursive}
|
||||||
|
onChange={(v) => setRecursive(v)}
|
||||||
|
/>
|
||||||
|
<span className="text-xs text-slate-400">
|
||||||
|
{recursive ? "递归扫描所有子目录" : "仅扫描当前目录"}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<p className="text-[11px] text-slate-500">
|
||||||
|
导入时只收录可读文本、图片视频、音频三类文件,检测到其他类型会自动忽略;图片 / 视频 / 音频以文件名为内容登记,可按文件名检索。
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div className="flex justify-end gap-2 border-t border-border px-5 py-3">
|
||||||
|
<button className="btn-secondary" onClick={onClose} disabled={busy}>
|
||||||
|
取消
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
className="btn-primary"
|
||||||
|
disabled={busy || !valid}
|
||||||
|
onClick={() => onSave(path.trim(), extensions.trim(), recursive)}
|
||||||
|
>
|
||||||
|
{busy ? "扫描中…" : "添加并扫描"}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function Toggle({
|
||||||
|
checked,
|
||||||
|
onChange,
|
||||||
|
}: {
|
||||||
|
checked: boolean;
|
||||||
|
onChange: (v: boolean) => void;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
role="switch"
|
||||||
|
aria-checked={checked}
|
||||||
|
onClick={() => onChange(!checked)}
|
||||||
|
className={`relative h-6 w-11 shrink-0 rounded-full transition-colors ${
|
||||||
|
checked ? "bg-accent" : "bg-panel-2"
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<span
|
||||||
|
className={`absolute top-0.5 h-5 w-5 rounded-full bg-white shadow transition-all ${
|
||||||
|
checked ? "left-[22px]" : "left-0.5"
|
||||||
|
}`}
|
||||||
|
/>
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user