feat: 知识库支持目录来源——多个目录批量导入、后缀过滤(默认全部)、可开关递归子目录,只收录文本/图片视频/音频三类其余忽略

This commit is contained in:
xianrenge
2026-08-17 21:39:06 +08:00
parent ab734478e8
commit bfec61d669
11 changed files with 754 additions and 16 deletions
+189
View File
@@ -1632,6 +1632,8 @@ pub fn kb_import_documents(
file_size: bytes.len() as i64,
char_count: text.chars().count() as i64,
chunk_count: chunks.len() as i64,
file_path: String::new(),
source_id: None,
created_at: String::new(),
updated_at: String::new(),
};
@@ -1684,6 +1686,193 @@ pub fn kb_search(
.map_err(|e| e.to_string())
}
#[derive(Deserialize)]
pub struct KbSourceInput {
pub kb_id: String,
pub path: String,
#[serde(default)]
pub extensions: String,
#[serde(default = "default_true")]
pub recursive: bool,
}
#[derive(Serialize)]
pub struct KbSourceScanResult {
pub imported: usize,
pub ignored: usize,
pub errors: Vec<String>,
}
#[tauri::command]
pub fn kb_list_sources(
state: State<'_, App>,
kb_id: String,
) -> Result<Vec<xianren_core::KbSource>, String> {
let db = state.core.db.lock().unwrap();
kb_db::list_sources(&db, &kb_id).map_err(|e| e.to_string())
}
/// 添加目录来源并立即扫描导入:只收录文本 / 图片视频 / 音频三类文件,其余忽略。
#[tauri::command]
pub fn kb_add_source(
state: State<'_, App>,
input: KbSourceInput,
) -> Result<KbSourceScanResult, String> {
let dir = std::path::PathBuf::from(input.path.trim());
if !dir.is_dir() {
return Err(format!("目录不存在或不是文件夹:{}", dir.display()));
}
let exts = crate::knowledge::normalize_extensions(&input.extensions);
let db = state.core.db.lock().unwrap();
let kb = kb_db::get_knowledge_base(&db, &input.kb_id)
.map_err(|e| e.to_string())?
.ok_or_else(|| "知识库不存在".to_string())?;
let source_id = uuid::Uuid::new_v4().to_string();
let source = xianren_core::KbSource {
id: source_id.clone(),
kb_id: input.kb_id.clone(),
path: dir.to_string_lossy().to_string(),
extensions: input.extensions.trim().to_string(),
recursive: input.recursive,
created_at: String::new(),
};
kb_db::upsert_source(&db, &source).map_err(|e| e.to_string())?;
// 同目录重复添加时沿用已有来源 id,避免文档归属错乱
let source_id = kb_db::list_sources(&db, &input.kb_id)
.map_err(|e| e.to_string())?
.into_iter()
.find(|s| s.path == source.path && s.kb_id == source.kb_id)
.map(|s| s.id)
.unwrap_or(source_id);
Ok(scan_and_import(&db, &kb, &source_id, &dir, &exts, input.recursive))
}
/// 重新扫描该知识库的全部目录来源,导入新增文件(按文件路径去重)。
#[tauri::command]
pub fn kb_scan_sources(
state: State<'_, App>,
kb_id: String,
) -> Result<KbSourceScanResult, String> {
let db = state.core.db.lock().unwrap();
let kb = kb_db::get_knowledge_base(&db, &kb_id)
.map_err(|e| e.to_string())?
.ok_or_else(|| "知识库不存在".to_string())?;
let sources = kb_db::list_sources(&db, &kb_id).map_err(|e| e.to_string())?;
let mut total = KbSourceScanResult {
imported: 0,
ignored: 0,
errors: Vec::new(),
};
for src in sources {
let dir = std::path::PathBuf::from(&src.path);
if !dir.is_dir() {
total.errors.push(format!("目录不存在:{}", src.path));
continue;
}
let exts = crate::knowledge::normalize_extensions(&src.extensions);
let r = scan_and_import(&db, &kb, &src.id, &dir, &exts, src.recursive);
total.imported += r.imported;
total.ignored += r.ignored;
total.errors.extend(r.errors);
}
Ok(total)
}
/// 删除目录来源及其导入的全部文档。
#[tauri::command]
pub fn kb_remove_source(state: State<'_, App>, source_id: String) -> Result<usize, String> {
let db = state.core.db.lock().unwrap();
kb_db::get_source(&db, &source_id)
.map_err(|e| e.to_string())?
.ok_or_else(|| "目录来源不存在".to_string())?;
let removed = kb_db::delete_docs_by_source(&db, &source_id).map_err(|e| e.to_string())?;
kb_db::delete_source(&db, &source_id).map_err(|e| e.to_string())?;
Ok(removed)
}
/// 扫描目录并导入文档(文本提取切块;图片/视频/音频以文件名为内容登记;其他忽略)。
fn scan_and_import(
db: &rusqlite::Connection,
kb: &xianren_core::KnowledgeBase,
source_id: &str,
dir: &std::path::Path,
extensions: &[String],
recursive: bool,
) -> KbSourceScanResult {
let mut result = KbSourceScanResult {
imported: 0,
ignored: 0,
errors: Vec::new(),
};
let files = crate::knowledge::collect_files(dir, extensions, recursive);
for file in files {
let name = file
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("")
.to_string();
let file_path = file.to_string_lossy().to_string();
let category = crate::knowledge::classify(&name);
if category == crate::knowledge::FileCategory::Other {
result.ignored += 1;
continue;
}
if kb_db::document_exists(db, &kb.id, &file_path).unwrap_or(false) {
continue;
}
let bytes = match std::fs::read(&file) {
Ok(b) => b,
Err(e) => {
result.errors.push(format!("{name}: 读取失败 {e}"));
continue;
}
};
let content = if category == crate::knowledge::FileCategory::Text {
match crate::knowledge::extract_text(&name, &bytes) {
Ok(t) => t,
Err(e) => {
result.errors.push(format!("{name}: {e}"));
continue;
}
}
} else {
// 图片 / 视频 / 音频:以文件名为内容登记,便于按名检索
name.clone()
};
let chunks = if category == crate::knowledge::FileCategory::Text {
kb_db::chunk_text(&content, kb.chunk_size as usize, kb.chunk_overlap as usize)
} else {
vec![content.clone()]
};
if chunks.is_empty() {
result.errors.push(format!("{name}: 切分后没有有效分块"));
continue;
}
let doc = xianren_core::KbDocument {
id: uuid::Uuid::new_v4().to_string(),
kb_id: kb.id.clone(),
name,
file_type: file
.extension()
.map(|e| e.to_string_lossy().to_lowercase())
.unwrap_or_default(),
file_size: bytes.len() as i64,
char_count: content.chars().count() as i64,
chunk_count: chunks.len() as i64,
file_path,
source_id: Some(source_id.to_string()),
created_at: String::new(),
updated_at: String::new(),
};
if let Err(e) = kb_db::insert_document_with_chunks(db, &doc, &content, &chunks) {
result.errors.push(format!("{}: 写入失败 {e}", doc.name));
continue;
}
result.imported += 1;
}
result
}
#[tauri::command]
pub fn list_conversations(
state: State<'_, App>,
+135 -5
View File
@@ -1,10 +1,95 @@
/// 按扩展名从文件字节提取纯文本:文本类文件直接按 UTF-8 读取,PDF 走 pdf-extract。
pub fn extract_text(name: &str, bytes: &[u8]) -> Result<String, String> {
let ext = name
.rsplit('.')
use std::path::Path;
/// 文件分类:可读文本 / 图片视频 / 音频 / 其他(其他一律忽略)。
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FileCategory {
Text,
Media,
Other,
}
const TEXT_EXTS: &[&str] = &[
"txt", "md", "markdown", "json", "csv", "tsv", "log", "yml", "yaml", "toml", "ini",
"conf", "cfg", "xml", "html", "htm", "py", "rs", "ts", "tsx", "js", "jsx", "java", "c",
"cpp", "h", "hpp", "go", "php", "rb", "sh", "bat", "ps1", "sql", "css", "scss", "sass",
"pdf",
];
const IMAGE_EXTS: &[&str] = &[
"png", "jpg", "jpeg", "gif", "webp", "bmp", "svg", "tiff", "tif", "ico", "avif", "heic",
];
const VIDEO_EXTS: &[&str] = &[
"mp4", "mkv", "avi", "mov", "webm", "flv", "wmv", "m4v", "mpg", "mpeg", "3gp",
];
const AUDIO_EXTS: &[&str] = &[
"mp3", "wav", "flac", "ogg", "oga", "m4a", "aac", "wma", "opus", "aiff", "aif",
];
fn ext_of(name: &str) -> String {
name.rsplit('.')
.next()
.map(|e| e.to_lowercase())
.unwrap_or_default();
.unwrap_or_default()
}
/// 分类文件:文本(可提取内容)/ 媒体(图片、视频、音频)/ 其他。
pub fn classify(name: &str) -> FileCategory {
let ext = ext_of(name);
if TEXT_EXTS.contains(&ext.as_str()) {
FileCategory::Text
} else if IMAGE_EXTS.contains(&ext.as_str())
|| VIDEO_EXTS.contains(&ext.as_str())
|| AUDIO_EXTS.contains(&ext.as_str())
{
FileCategory::Media
} else {
FileCategory::Other
}
}
/// 规范化用户输入的后缀列表:`.txt, .mdPDF` → ["txt", "md", "pdf"];空串表示全部。
pub fn normalize_extensions(raw: &str) -> Vec<String> {
raw.split([',', ';', '', '', ' '])
.map(|s| s.trim().trim_start_matches('.').to_lowercase())
.filter(|s| !s.is_empty())
.collect::<Vec<_>>()
}
fn has_matching_ext(name: &str, extensions: &[String]) -> bool {
let ext = ext_of(name);
extensions.iter().any(|e| e == &ext)
}
/// 递归/非递归收集目录下符合后缀条件的文件,按路径排序。
pub fn collect_files(path: &Path, extensions: &[String], recursive: bool) -> Vec<std::path::PathBuf> {
let mut out = Vec::new();
fn walk(dir: &Path, exts: &[String], recursive: bool, out: &mut Vec<std::path::PathBuf>) {
let Ok(entries) = std::fs::read_dir(dir) else {
return;
};
for entry in entries.flatten() {
let p = entry.path();
if p.is_dir() {
if recursive {
walk(&p, exts, recursive, out);
}
} else if let Some(name) = p.file_name().and_then(|n| n.to_str()) {
if exts.is_empty() || has_matching_ext(name, exts) {
out.push(p);
}
}
}
}
walk(path, extensions, recursive, &mut out);
out.sort();
out
}
/// 按扩展名从文件字节提取纯文本:文本类文件直接按 UTF-8 读取,PDF 走 pdf-extract。
pub fn extract_text(name: &str, bytes: &[u8]) -> Result<String, String> {
let ext = ext_of(name);
let text = match ext.as_str() {
"pdf" => pdf_extract::extract_text_from_mem(bytes)
.map_err(|e| format!("PDF 解析失败:{e}"))?,
@@ -31,3 +116,48 @@ pub fn extract_text(name: &str, bytes: &[u8]) -> Result<String, String> {
}
Ok(cleaned)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn classify_three_categories() {
assert_eq!(classify("readme.md"), FileCategory::Text);
assert_eq!(classify("手册.pdf"), FileCategory::Text);
assert_eq!(classify("photo.PNG"), FileCategory::Media);
assert_eq!(classify("clip.mp4"), FileCategory::Media);
assert_eq!(classify("song.mp3"), FileCategory::Media);
assert_eq!(classify("app.exe"), FileCategory::Other);
assert_eq!(classify("archive.zip"), FileCategory::Other);
assert_eq!(classify("noext"), FileCategory::Other);
}
#[test]
fn normalize_extensions_handles_dots_and_separators() {
assert_eq!(
normalize_extensions(".txt, .mdPDF, png"),
vec!["txt", "md", "pdf", "png"]
);
assert!(normalize_extensions(" ").is_empty());
}
#[test]
fn collect_files_filters_and_recurses() {
let base = std::env::temp_dir().join(format!("kb_scan_test_{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(base.join("sub")).unwrap();
std::fs::write(base.join("a.txt"), "hello").unwrap();
std::fs::write(base.join("b.exe"), "x").unwrap();
std::fs::write(base.join("sub/c.md"), "hi").unwrap();
std::fs::write(base.join("sub/d.png"), "img").unwrap();
let all = collect_files(&base, &[], true);
assert_eq!(all.len(), 4);
let txt_only = collect_files(&base, &["txt".to_string()], true);
assert_eq!(txt_only.len(), 1);
let non_recursive = collect_files(&base, &[], false);
assert_eq!(non_recursive.len(), 2);
std::fs::remove_dir_all(&base).unwrap();
}
}
+4
View File
@@ -195,6 +195,10 @@ pub fn run() {
commands::remove_kb_document,
commands::kb_rechunk_document,
commands::kb_search,
commands::kb_list_sources,
commands::kb_add_source,
commands::kb_scan_sources,
commands::kb_remove_source,
commands::list_models,
commands::import_model,
commands::remove_model,
+2
View File
@@ -123,6 +123,8 @@ fn open_db(path: &Path) -> Result<Connection> {
ensure_column(&conn, "scheduled_tasks", "email_mode", "TEXT NOT NULL DEFAULT 'fixed'")?;
ensure_column(&conn, "scheduled_tasks", "email_subject", "TEXT NOT NULL DEFAULT ''")?;
ensure_column(&conn, "scheduled_tasks", "email_body", "TEXT NOT NULL DEFAULT ''")?;
ensure_column(&conn, "kb_documents", "file_path", "TEXT NOT NULL DEFAULT ''")?;
ensure_column(&conn, "kb_documents", "source_id", "TEXT")?;
Ok(conn)
}
+106 -6
View File
@@ -24,10 +24,22 @@ pub struct KbDocument {
pub file_size: i64,
pub char_count: i64,
pub chunk_count: i64,
pub file_path: String,
pub source_id: Option<String>,
pub created_at: String,
pub updated_at: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KbSource {
pub id: String,
pub kb_id: String,
pub path: String,
pub extensions: String,
pub recursive: bool,
pub created_at: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct KbDocumentDetail {
#[serde(flatten)]
@@ -45,7 +57,8 @@ pub struct KbSearchHit {
}
const KB_COLUMNS: &str = "id, name, description, chunk_size, chunk_overlap, created_at, updated_at";
const DOC_COLUMNS: &str = "id, kb_id, name, file_type, file_size, char_count, chunk_count, created_at, updated_at";
const DOC_COLUMNS: &str = "id, kb_id, name, file_type, file_size, char_count, chunk_count, file_path, source_id, created_at, updated_at";
const SOURCE_COLUMNS: &str = "id, kb_id, path, extensions, recursive, created_at";
// ---------- 知识库 ----------
@@ -149,7 +162,7 @@ pub fn get_document_detail(db: &Connection, id: &str) -> Result<Option<KbDocumen
))?;
let mut rows = stmt.query_map(params![id], |row| {
let doc = row_to_doc(row)?;
let content: String = row.get(9)?;
let content: String = row.get(11)?;
Ok(KbDocumentDetail { doc, content })
})?;
match rows.next() {
@@ -168,8 +181,8 @@ pub fn insert_document_with_chunks(
let tx = db.unchecked_transaction()?;
tx.execute(
"INSERT INTO kb_documents
(id, kb_id, name, file_type, file_size, char_count, chunk_count, content)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)",
(id, kb_id, name, file_type, file_size, char_count, chunk_count, content, file_path, source_id)
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
params![
doc.id,
doc.kb_id,
@@ -179,6 +192,8 @@ pub fn insert_document_with_chunks(
doc.char_count,
chunks.len() as i64,
content,
doc.file_path,
doc.source_id,
],
)?;
for (i, chunk) in chunks.iter().enumerate() {
@@ -203,6 +218,75 @@ pub fn delete_document(db: &Connection, id: &str) -> Result<()> {
Ok(())
}
/// 同一知识库下是否已存在来自该文件路径的文档(目录导入去重)。
pub fn document_exists(db: &Connection, kb_id: &str, file_path: &str) -> Result<bool> {
if file_path.is_empty() {
return Ok(false);
}
let exists: bool = db.query_row(
"SELECT EXISTS(SELECT 1 FROM kb_documents WHERE kb_id = ?1 AND file_path = ?2)",
params![kb_id, file_path],
|row| row.get(0),
)?;
Ok(exists)
}
// ---------- 目录来源 ----------
pub fn list_sources(db: &Connection, kb_id: &str) -> Result<Vec<KbSource>> {
let mut stmt = db.prepare(&format!(
"SELECT {SOURCE_COLUMNS} FROM kb_sources WHERE kb_id = ?1 ORDER BY created_at ASC"
))?;
let rows = stmt.query_map(params![kb_id], row_to_source)?;
let mut out = Vec::new();
for row in rows {
out.push(row?);
}
Ok(out)
}
pub fn get_source(db: &Connection, id: &str) -> Result<Option<KbSource>> {
let mut stmt = db.prepare(&format!(
"SELECT {SOURCE_COLUMNS} FROM kb_sources WHERE id = ?1"
))?;
let mut rows = stmt.query_map(params![id], row_to_source)?;
match rows.next() {
Some(row) => Ok(Some(row?)),
None => Ok(None),
}
}
pub fn upsert_source(db: &Connection, source: &KbSource) -> Result<()> {
db.execute(
"INSERT INTO kb_sources (id, kb_id, path, extensions, recursive)
VALUES (?1, ?2, ?3, ?4, ?5)
ON CONFLICT(kb_id, path) DO UPDATE SET
extensions = excluded.extensions, recursive = excluded.recursive",
params![
source.id,
source.kb_id,
source.path,
source.extensions,
source.recursive as i32,
],
)?;
Ok(())
}
pub fn delete_source(db: &Connection, id: &str) -> Result<()> {
db.execute("DELETE FROM kb_sources WHERE id = ?1", params![id])?;
Ok(())
}
/// 删除某个目录来源导入的全部文档(分块随外键级联)。
pub fn delete_docs_by_source(db: &Connection, source_id: &str) -> Result<usize> {
let n = db.execute(
"DELETE FROM kb_documents WHERE source_id = ?1",
params![source_id],
)?;
Ok(n)
}
/// 重新切分文档:删除旧分块后按新参数重分并重建索引。
pub fn rechunk_document(
db: &Connection,
@@ -328,8 +412,22 @@ fn row_to_doc(row: &rusqlite::Row<'_>) -> rusqlite::Result<KbDocument> {
file_size: row.get(4)?,
char_count: row.get(5)?,
chunk_count: row.get(6)?,
created_at: row.get(7)?,
updated_at: row.get(8)?,
file_path: row.get(7)?,
source_id: row.get(8)?,
created_at: row.get(9)?,
updated_at: row.get(10)?,
})
}
fn row_to_source(row: &rusqlite::Row<'_>) -> rusqlite::Result<KbSource> {
let recursive: i32 = row.get(4)?;
Ok(KbSource {
id: row.get(0)?,
kb_id: row.get(1)?,
path: row.get(2)?,
extensions: row.get(3)?,
recursive: recursive != 0,
created_at: row.get(5)?,
})
}
@@ -363,6 +461,8 @@ mod tests {
file_size: content.len() as i64,
char_count: content.chars().count() as i64,
chunk_count: 0,
file_path: String::new(),
source_id: None,
created_at: String::new(),
updated_at: String::new(),
};
+1 -1
View File
@@ -12,7 +12,7 @@ pub mod workflows;
pub use app::CoreApp;
pub use error::{CoreError, Result};
pub use knowledge_base::{KbDocument, KbDocumentDetail, KbSearchHit, KnowledgeBase};
pub use knowledge_base::{KbDocument, KbDocumentDetail, KbSearchHit, KbSource, KnowledgeBase};
pub use models::ModelInfo;
pub use agents::Agent;
pub use scheduled_tasks::ScheduledTask;
+13
View File
@@ -147,6 +147,8 @@ CREATE TABLE IF NOT EXISTS kb_documents (
char_count INTEGER NOT NULL DEFAULT 0,
chunk_count INTEGER NOT NULL DEFAULT 0,
content TEXT NOT NULL DEFAULT '',
file_path TEXT NOT NULL DEFAULT '',
source_id TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
);
@@ -160,6 +162,17 @@ CREATE TABLE IF NOT EXISTS kb_chunks (
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS kb_sources (
id TEXT PRIMARY KEY,
kb_id TEXT NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
path TEXT NOT NULL,
extensions TEXT NOT NULL DEFAULT '',
recursive INTEGER NOT NULL DEFAULT 1,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_kb_sources_kb_path ON kb_sources(kb_id, path);
-- 全文检索索引(trigram 分词,对中文/短文本更友好;content='' 为 contentless-delete 模式)
CREATE VIRTUAL TABLE IF NOT EXISTS kb_chunks_fts USING fts5(content, content = '', tokenize = 'trigram');
+7 -4
View File
@@ -12,7 +12,7 @@
```
xianren_studio/
├── apps/desktop/ Tauri 桌面壳(Rust 命令层、tauri.conf.json、推荐模型/预制智能体/预制工作流默认 JSON、工作流执行引擎 workflow.rs、邮件发送 mail.rs、文档提取 knowledge.rs
├── apps/desktop/ Tauri 桌面壳(Rust 命令层、tauri.conf.json、推荐模型/预制智能体/预制工作流默认 JSON、工作流执行引擎 workflow.rs、邮件发送 mail.rs、文档提取与目录扫描 knowledge.rs
├── crates/core/ 领域核心:SQLite(模型注册表、智能体、工作流、知识库、定时计划、会话、消息、设置)
├── crates/engine/ llama-server 生命周期 + 流式/非流式聊天(本地与远程 OpenAI 兼容)
├── crates/download/ 分片断点续传下载器
@@ -111,6 +111,7 @@ ui/src/
| 定时计划 | `list_scheduled_tasks``add_scheduled_task``update_scheduled_task``remove_scheduled_task``set_scheduled_task_enabled``run_scheduled_task_now` |
| 邮件 | `mail_test`(用当前 SMTP 配置发送测试邮件) |
| 知识库 | `list_knowledge_bases``add_knowledge_base``update_knowledge_base``remove_knowledge_base``list_kb_documents``get_kb_document``kb_import_documents``remove_kb_document``kb_rechunk_document``kb_search` |
| 知识库·目录来源 | `kb_list_sources``kb_add_source``kb_scan_sources``kb_remove_source` |
| 应用/设置 | `app_info``autostart_status``autostart_set``settings_get``settings_set` |
| 模型 | `list_models``import_model``remove_model``set_model_enabled``scan_models``add_remote_model` |
| 模型广场 | `search_models``list_model_files``list_recommended_models``import_recommendations``fetch_model_page` |
@@ -143,6 +144,7 @@ ui/src/
- `send_task_email` / `load_smtp_config` / `split_recipients`:读取设置页「邮件」SMTP 配置、拆分多收件人并调用 `mail.rs` 发送。
- `kb_import_documents`base64 解码 → `knowledge.rs::extract_text` 提取文本(文本类直接 UTF-8PDF 走 pdf-extract)→ `kb_db::chunk_text` 切块(按知识库 chunk_size/overlap)→ `insert_document_with_chunks` 事务写入文档与分块,逐文件返回成功/失败结果。
- `kb_search` / `kb_rechunk_document`:检索走 `kb_db::search`FTS5 trigram + BM25,短查询回退 LIKE);重切按知识库最新分块设置重建该文档分块与索引。
- `kb_add_source` / `kb_scan_sources` / `kb_remove_source`:目录来源注册与扫描——`knowledge.rs::collect_files` 按后缀过滤(空=全部)与递归开关收集文件,`classify` 分成文本 / 图片视频 / 音频三类(其他忽略),文本提取切块入库、媒体以文件名为内容登记,`kb_db::document_exists` 按(kb_id + file_path)去重;删除来源级联删除其导入文档。
**邮件发送模块(`apps/desktop/src/mail.rs`):**
@@ -185,13 +187,14 @@ ui/src/
| `skills` | `name``description``content``enabled` | 技能工具库 |
| `mcp_servers` | `name``description``url``auth_token``enabled` | MCP 服务配置 |
| `knowledge_bases` | `name``description``chunk_size``chunk_overlap` | 知识库及分块设置 |
| `kb_documents` | `kb_id`(级联删除)、`name``file_type``file_size``char_count``chunk_count``content`(提取后的纯文本 | 知识库文档;删除/重切分块由 FTS 触发器同步 |
| `kb_documents` | `kb_id`(级联删除)、`name``file_type``file_size``char_count``chunk_count``content`(提取后的纯文本/媒体文件名)、`file_path``source_id` | 知识库文档;目录导入记录文件路径与来源,`file_path` 用于去重;删除/重切分块由 FTS 触发器同步 |
| `kb_chunks` | `kb_id``document_id`(级联删除)、`seq``content` | 文档分块 |
| `kb_chunks_fts` | FTS5 虚拟表(contentless-delete + trigram 分词) | 全文检索索引,由 `kb_chunks` 触发器维护 |
| `kb_sources` | `kb_id`(级联删除)、`path``extensions``recursive``(kb_id, path)` 唯一 | 目录来源(多个);添加时立即扫描,可重新扫描增量导入 |
迁移清单(`ensure_column`):`models.kind/base_url/api_key/api_model/enabled``messages.elapsed_ms/first_token_ms/images_json/model_id``conversations.pinned/favorite/tools_json/agent_id`
新增列迁移:`scheduled_tasks.email_enabled / email_to / email_mode / email_subject / email_body``ensure_column` 幂等补齐)。
新增列迁移:`scheduled_tasks.email_enabled / email_to / email_mode / email_subject / email_body``kb_documents.file_path / source_id``ensure_column` 幂等补齐)。
### 5.3 `models.rs`
@@ -219,7 +222,7 @@ ui/src/
### 5.9 `knowledge_base.rs`
知识库 CRUD`list/get/insert/update/delete`)、文档 CRUD`list/get/get_detail`)、`insert_document_with_chunks`(文档 + 分块同事务写入,FTS 触发器自动建索引)、`delete_document`(级联删除分块并同步索引)、`rechunk_document``chunk_text`(按字符切块 + 重叠)、`search`FTS5 trigram BM25 排序,查询 <3 字或 MATCH 失败回退 LIKE)。
知识库 CRUD`list/get/insert/update/delete`)、文档 CRUD`list/get/get_detail`)、`insert_document_with_chunks`(文档 + 分块同事务写入,FTS 触发器自动建索引)、`delete_document`(级联删除分块并同步索引)、`document_exists`(目录导入按文件路径去重)、`rechunk_document``chunk_text`(按字符切块 + 重叠)、`search`FTS5 trigram BM25 排序,查询 <3 字或 MATCH 失败回退 LIKE;目录来源 `list/get/upsert/delete_sources``delete_docs_by_source`
---
+6
View File
@@ -133,6 +133,11 @@
- 左侧导航新增「知识库」选项卡,页面分左右两栏:左侧知识库列表,右侧选中库的管理区。
- **知识库管理**:新建 / 编辑 / 删除知识库,字段含名称、描述、**分块大小**(100–10000 字,默认 500)与**块间重叠**(默认 50);删除知识库会级联删除其文档与分块。
- **文档导入**:支持多文件上传,格式覆盖文本类文件(txt / md / json / csv / 代码 / HTML 等)与 **PDF**(后端提取文本);导入后按知识库分块设置自动切块并写入全文索引。
- **目录来源(批量导入)**:可为知识库添加多个本地目录,扫描目录批量导入文档:
- **后缀过滤**:可指定只导入的后缀列表(如 `.txt,.md,.pdf`),留空默认全部;
- **递归子目录**:可开关是否递归扫描所有子目录;
- **类型过滤**:只收录**可读文本**(提取内容并分块检索)、**图片 / 视频**、**音频**三类文件(图片视频音频以文件名为内容登记,可按文件名检索),检测到其他类型自动忽略;
- 同一目录重复添加自动更新配置不产生重复来源;按文件路径去重,可一键「重新扫描」增量导入新增文件;删除目录来源会同时删除它导入的文档。
- **文档操作**:文档列表展示类型 / 大小 / 字数 / 分块数;支持全文预览、「重切」(按最新分块设置重新切分,适用于修改设置后)与删除。
- **检索**:全文检索基于 SQLite FTS5(trigram 分词,对中文友好),按 BM25 相关性排序;支持单库检索(也可全库检索),结果展示来源文档与分块序号,可一键复制分块内容用于对话。
- 数据表:`knowledge_bases``kb_documents`(含提取后的纯文本)、`kb_chunks`(分块)、`kb_chunks_fts`FTS5 全文索引,contentless-delete 模式 + 触发器同步)。
@@ -224,6 +229,7 @@
### 2026-08-17
- 知识库新增「目录来源」:支持添加多个本地目录批量导入(后缀过滤默认全部、可开关递归子目录),只收录文本 / 图片视频 / 音频三类文件其余忽略,支持重新扫描增量导入与按来源删除。
- 新增「知识库」选项卡与完整操作界面:知识库管理(新建/编辑/删除)、多格式文档上传(文本类 + PDF)、自动分块与全文检索(FTS5 trigram + BM25)、文档预览 / 重新切分 / 删除、检索结果一键复制。
- 计划任务支持发送邮件:任务设置中可开启邮件并填写收件人,内容支持固定内容与大模型生成两种模式;设置页新增「邮件」SMTP 配置与测试发送。
- 设置页新增「启动」:开机启动开关(写入 Windows 注册表启动项);「自动加载模型」列表(多个本地模型按顺序在启动时逐个加载,最后一个保持运行)。
+29
View File
@@ -245,6 +245,28 @@ export interface KbImportResult {
error: string | null;
}
export interface KbSource {
id: string;
kb_id: string;
path: string;
extensions: string;
recursive: boolean;
created_at: string;
}
export interface KbSourceInput {
kb_id: string;
path: string;
extensions: string;
recursive: boolean;
}
export interface KbSourceScanResult {
imported: number;
ignored: number;
errors: string[];
}
export interface McpServer {
id: string;
name: string;
@@ -483,6 +505,13 @@ export const api = {
invoke<number>("kb_rechunk_document", { documentId }),
kbSearch: (kbId: string | null, query: string, limit?: number) =>
invoke<KbSearchHit[]>("kb_search", { kbId, query, limit: limit ?? 10 }),
kbListSources: (kbId: string) => invoke<KbSource[]>("kb_list_sources", { kbId }),
kbAddSource: (input: KbSourceInput) =>
invoke<KbSourceScanResult>("kb_add_source", { input }),
kbScanSources: (kbId: string) =>
invoke<KbSourceScanResult>("kb_scan_sources", { kbId }),
kbRemoveSource: (sourceId: string) =>
invoke<number>("kb_remove_source", { sourceId }),
listModels: () => invoke<ModelInfo[]>("list_models"),
importModel: (path: string) => invoke<ModelInfo>("import_model", { path }),
removeModel: (id: string) => invoke<void>("remove_model", { id }),
+262
View File
@@ -4,6 +4,8 @@ import {
KbDocument,
KbDocumentDetail,
KbSearchHit,
KbSource,
KbSourceScanResult,
KnowledgeBase,
KnowledgeBaseInput,
} from "../api";
@@ -56,6 +58,9 @@ export default function KnowledgeBasePage() {
const [query, setQuery] = useState("");
const [searching, setSearching] = useState(false);
const [searchResults, setSearchResults] = useState<KbSearchHit[] | null>(null);
const [sources, setSources] = useState<KbSource[]>([]);
const [sourceModal, setSourceModal] = useState(false);
const [sourceMsg, setSourceMsg] = useState<string | null>(null);
const uploadInputRef = useRef<HTMLInputElement>(null);
const selected = useMemo(
@@ -88,6 +93,10 @@ export default function KnowledgeBasePage() {
.listKbDocuments(selectedId)
.then(setDocs)
.catch((e) => setMsg(`加载文档失败:${String(e)}`));
api
.kbListSources(selectedId)
.then(setSources)
.catch((e) => setMsg(`加载目录来源失败:${String(e)}`));
}, [selectedId]);
async function handleSaveKb(input: KnowledgeBaseInput) {
@@ -189,6 +198,76 @@ export default function KnowledgeBasePage() {
}
}
async function refreshSourcesAndDocs() {
if (!selected) return;
const [d, s] = await Promise.all([
api.listKbDocuments(selected.id),
api.kbListSources(selected.id),
]);
setDocs(d);
setSources(s);
await refreshKbs();
}
function formatScanMsg(label: string, r: KbSourceScanResult) {
const errPart =
r.errors.length > 0
? `${r.errors.length} 个错误(${r.errors
.slice(0, 3)
.join("")}${r.errors.length > 3 ? "…" : ""}`
: "";
return `${label}完成:导入 ${r.imported} 个,忽略 ${r.ignored}${errPart}`;
}
async function handleAddSource(path: string, extensions: string, recursive: boolean) {
if (!selected) return;
setBusy(true);
setSourceMsg(null);
try {
const r = await api.kbAddSource({
kb_id: selected.id,
path,
extensions,
recursive,
});
await refreshSourcesAndDocs();
setSourceMsg(formatScanMsg("添加目录", r));
} catch (e) {
setSourceMsg(`添加目录失败:${String(e)}`);
} finally {
setBusy(false);
setSourceModal(false);
}
}
async function handleScanAll() {
if (!selected) return;
setBusy(true);
setSourceMsg(null);
try {
const r = await api.kbScanSources(selected.id);
await refreshSourcesAndDocs();
setSourceMsg(formatScanMsg("重新扫描", r));
} catch (e) {
setSourceMsg(`重新扫描失败:${String(e)}`);
} finally {
setBusy(false);
}
}
async function handleRemoveSource(src: KbSource) {
if (!confirm(`确定删除目录来源「${src.path}」吗?将同时删除由它导入的文档。`)) return;
try {
const n = await api.kbRemoveSource(src.id);
setSourceMsg(`已删除目录来源及其 ${n} 个文档`);
if (selected) {
await refreshSourcesAndDocs();
}
} catch (e) {
setSourceMsg(`删除失败:${String(e)}`);
}
}
return (
<div className="flex h-full overflow-hidden">
{/* 左侧:知识库列表 */}
@@ -343,6 +422,81 @@ export default function KnowledgeBasePage() {
) : null}
</section>
{/* 目录来源 */}
<section className="mb-6 rounded-xl border border-border bg-panel p-4">
<div className="mb-2 flex items-center justify-between">
<div className="text-xs uppercase text-slate-500">
{sources.length}
</div>
<div className="flex gap-2">
<button
className="btn-secondary !px-2.5 !py-1 text-xs"
onClick={handleScanAll}
disabled={busy || sources.length === 0}
>
<Icon name="refresh" className="mr-1 inline h-3 w-3" />
</button>
<button
className="btn-primary !px-2.5 !py-1 text-xs"
onClick={() => setSourceModal(true)}
>
+
</button>
</div>
</div>
<p className="mb-3 text-[11px] text-slate-500">
</p>
{sourceMsg ? (
<div className="mb-2 rounded-lg border border-border bg-panel-2/60 px-3 py-2 text-xs text-slate-400">
{sourceMsg}
</div>
) : null}
{sources.length === 0 ? (
<div className="rounded-lg border border-dashed border-border bg-panel-2/40 px-3 py-6 text-center text-xs text-slate-500">
</div>
) : (
<div className="space-y-2">
{sources.map((src) => (
<div
key={src.id}
className="flex items-center gap-3 rounded-lg border border-border bg-panel-2/60 p-3"
>
<Icon name="folder" className="h-4 w-4 shrink-0 text-slate-400" />
<div className="min-w-0 flex-1">
<div className="truncate text-xs font-medium text-slate-300">
{src.path}
</div>
<div className="mt-0.5 flex items-center gap-1.5 text-[11px] text-slate-500">
<span className="rounded bg-panel-2 px-1.5 py-0.5">
{src.extensions.trim()
? src.extensions
.split(/[,;,;\s]+/)
.filter(Boolean)
.map((e) => (e.startsWith(".") ? e : `.${e}`))
.join(" ")
: "全部后缀"}
</span>
<span className="rounded bg-panel-2 px-1.5 py-0.5">
{src.recursive ? "递归子目录" : "仅当前目录"}
</span>
</div>
</div>
<button
className="rounded-lg border border-border bg-panel-2 px-2.5 py-1.5 text-xs text-slate-400 hover:text-red-400"
onClick={() => handleRemoveSource(src)}
title="删除该目录来源及其导入的文档"
>
<Icon name="trash" className="h-3.5 w-3.5" />
</button>
</div>
))}
</div>
)}
</section>
{/* 文档列表 */}
<section>
<div className="mb-2 text-xs uppercase text-slate-500">
@@ -381,6 +535,10 @@ export default function KnowledgeBasePage() {
/>
) : null}
{sourceModal ? (
<SourceModal busy={busy} onSave={handleAddSource} onClose={() => setSourceModal(false)} />
) : null}
{preview ? (
<div
className="fixed inset-0 z-50 flex items-center justify-center bg-black/70 p-6"
@@ -605,3 +763,107 @@ function KbModal({
</div>
);
}
function SourceModal({
busy,
onSave,
onClose,
}: {
busy: boolean;
onSave: (path: string, extensions: string, recursive: boolean) => void;
onClose: () => void;
}) {
const [path, setPath] = useState("");
const [extensions, setExtensions] = useState("");
const [recursive, setRecursive] = useState(true);
const valid = path.trim().length > 0;
return (
<div
className="fixed inset-0 z-50 flex items-center justify-center bg-black/70 p-6"
onClick={onClose}
>
<div
className="w-full max-w-md rounded-xl border border-border bg-panel"
onClick={(e) => e.stopPropagation()}
>
<div className="flex items-center justify-between border-b border-border px-5 py-3">
<span className="text-sm font-medium"></span>
<button className="text-slate-500 hover:text-slate-200" onClick={onClose}>
×
</button>
</div>
<div className="space-y-4 px-5 py-4">
<div>
<label className="mb-1 block text-xs text-slate-400"></label>
<input
className="input"
value={path}
placeholder="例如:D:\资料\项目文档"
onChange={(e) => setPath(e.target.value)}
/>
</div>
<div>
<label className="mb-1 block text-xs text-slate-400"></label>
<input
className="input"
value={extensions}
placeholder="留空表示全部;多个用逗号分隔,如 .txt,.md,.pdf,.png"
onChange={(e) => setExtensions(e.target.value)}
/>
</div>
<div className="flex items-center gap-2">
<Toggle
checked={recursive}
onChange={(v) => setRecursive(v)}
/>
<span className="text-xs text-slate-400">
{recursive ? "递归扫描所有子目录" : "仅扫描当前目录"}
</span>
</div>
<p className="text-[11px] text-slate-500">
/ /
</p>
</div>
<div className="flex justify-end gap-2 border-t border-border px-5 py-3">
<button className="btn-secondary" onClick={onClose} disabled={busy}>
</button>
<button
className="btn-primary"
disabled={busy || !valid}
onClick={() => onSave(path.trim(), extensions.trim(), recursive)}
>
{busy ? "扫描中…" : "添加并扫描"}
</button>
</div>
</div>
</div>
);
}
function Toggle({
checked,
onChange,
}: {
checked: boolean;
onChange: (v: boolean) => void;
}) {
return (
<button
type="button"
role="switch"
aria-checked={checked}
onClick={() => onChange(!checked)}
className={`relative h-6 w-11 shrink-0 rounded-full transition-colors ${
checked ? "bg-accent" : "bg-panel-2"
}`}
>
<span
className={`absolute top-0.5 h-5 w-5 rounded-full bg-white shadow transition-all ${
checked ? "left-[22px]" : "left-0.5"
}`}
/>
</button>
);
}