feat: 知识库页优化——文档列表与检索结果分页;目录来源/文档列表移到检索上方;目录扫描后台化并推送进度条与处理统计(处理/成功/失败/忽略/分块)

This commit is contained in:
xianrenge
2026-08-18 01:30:08 +08:00
parent 221a3f3e69
commit 405bdec2c6
6 changed files with 613 additions and 240 deletions
+258 -122
View File
@@ -1560,9 +1560,21 @@ pub fn remove_knowledge_base(state: State<'_, App>, id: String) -> Result<(), St
pub fn list_kb_documents(
state: State<'_, App>,
kb_id: String,
) -> Result<Vec<xianren_core::KbDocument>, String> {
page: Option<i64>,
page_size: Option<i64>,
) -> Result<KbDocumentPage, String> {
let page = page.unwrap_or(1).max(1);
let page_size = page_size.unwrap_or(10).clamp(1, 100);
let db = state.core.db.lock().unwrap();
kb_db::list_documents(&db, &kb_id).map_err(|e| e.to_string())
let total = kb_db::count_documents(&db, &kb_id).map_err(|e| e.to_string())?;
let items = kb_db::list_documents_page(&db, &kb_id, page, page_size)
.map_err(|e| e.to_string())?;
Ok(KbDocumentPage {
items,
total,
page,
page_size,
})
}
#[tauri::command]
@@ -1679,11 +1691,38 @@ pub fn kb_search(
state: State<'_, App>,
kb_id: Option<String>,
query: String,
limit: Option<usize>,
) -> Result<Vec<xianren_core::KbSearchHit>, String> {
page: Option<i64>,
page_size: Option<i64>,
) -> Result<KbSearchPage, String> {
let page = page.unwrap_or(1).max(1);
let page_size = page_size.unwrap_or(10).clamp(1, 100);
let db = state.core.db.lock().unwrap();
kb_db::search(&db, kb_id.as_deref(), &query, limit.unwrap_or(10))
.map_err(|e| e.to_string())
let total = kb_db::count_search(&db, kb_id.as_deref(), &query).map_err(|e| e.to_string())?;
let offset = ((page - 1) * page_size) as usize;
let items = kb_db::search(&db, kb_id.as_deref(), &query, page_size as usize, offset)
.map_err(|e| e.to_string())?;
Ok(KbSearchPage {
items,
total,
page,
page_size,
})
}
#[derive(Serialize)]
pub struct KbDocumentPage {
pub items: Vec<xianren_core::KbDocument>,
pub total: i64,
pub page: i64,
pub page_size: i64,
}
#[derive(Serialize)]
pub struct KbSearchPage {
pub items: Vec<xianren_core::KbSearchHit>,
pub total: i64,
pub page: i64,
pub page_size: i64,
}
#[derive(Deserialize)]
@@ -1696,10 +1735,18 @@ pub struct KbSourceInput {
pub recursive: bool,
}
#[derive(Serialize)]
pub struct KbSourceScanResult {
#[derive(Serialize, Clone)]
pub struct KbScanProgressEvent {
pub kb_id: String,
pub source_path: String,
/// scanning(进行中)/ done(完成)
pub status: String,
pub current: usize,
pub total: usize,
pub imported: usize,
pub failed: usize,
pub ignored: usize,
pub chunks: usize,
pub errors: Vec<String>,
}
@@ -1712,70 +1759,115 @@ pub fn kb_list_sources(
kb_db::list_sources(&db, &kb_id).map_err(|e| e.to_string())
}
/// 添加目录来源并立即扫描导入:只收录文本 / 图片视频 / 音频三类文件,其余忽略。
/// 添加目录来源并后台扫描导入:只收录文本 / 图片视频 / 音频三类文件,其余忽略;进度通过 `kb://scan-progress` 事件推送
#[tauri::command]
pub fn kb_add_source(
pub async fn kb_add_source(
app: AppHandle,
state: State<'_, App>,
input: KbSourceInput,
) -> Result<KbSourceScanResult, String> {
) -> Result<(), String> {
let dir = std::path::PathBuf::from(input.path.trim());
if !dir.is_dir() {
return Err(format!("目录不存在或不是文件夹:{}", dir.display()));
}
let exts = crate::knowledge::normalize_extensions(&input.extensions);
let db = state.core.db.lock().unwrap();
let kb = kb_db::get_knowledge_base(&db, &input.kb_id)
.map_err(|e| e.to_string())?
.ok_or_else(|| "知识库不存在".to_string())?;
let source_id = uuid::Uuid::new_v4().to_string();
let source = xianren_core::KbSource {
id: source_id.clone(),
kb_id: input.kb_id.clone(),
path: dir.to_string_lossy().to_string(),
extensions: input.extensions.trim().to_string(),
recursive: input.recursive,
created_at: String::new(),
let core = state.core.clone();
let (kb, source_id) = {
let db = core.db.lock().unwrap();
let kb = kb_db::get_knowledge_base(&db, &input.kb_id)
.map_err(|e| e.to_string())?
.ok_or_else(|| "知识库不存在".to_string())?;
let source_id = uuid::Uuid::new_v4().to_string();
let source = xianren_core::KbSource {
id: source_id.clone(),
kb_id: input.kb_id.clone(),
path: dir.to_string_lossy().to_string(),
extensions: input.extensions.trim().to_string(),
recursive: input.recursive,
created_at: String::new(),
};
kb_db::upsert_source(&db, &source).map_err(|e| e.to_string())?;
// 同目录重复添加时沿用已有来源 id,避免文档归属错乱
let source_id = kb_db::list_sources(&db, &input.kb_id)
.map_err(|e| e.to_string())?
.into_iter()
.find(|s| s.path == source.path && s.kb_id == source.kb_id)
.map(|s| s.id)
.unwrap_or(source_id);
(kb, source_id)
};
kb_db::upsert_source(&db, &source).map_err(|e| e.to_string())?;
// 同目录重复添加时沿用已有来源 id,避免文档归属错乱
let source_id = kb_db::list_sources(&db, &input.kb_id)
.map_err(|e| e.to_string())?
.into_iter()
.find(|s| s.path == source.path && s.kb_id == source.kb_id)
.map(|s| s.id)
.unwrap_or(source_id);
Ok(scan_and_import(&db, &kb, &source_id, &dir, &exts, input.recursive))
let app2 = app.clone();
tokio::spawn(async move {
run_scan_with_progress(&app2, &core, kb, Some(source_id), dir, exts, input.recursive, true)
.await;
});
Ok(())
}
/// 重新扫描该知识库的全部目录来源,导入新增文件(按文件路径去重)。
#[tauri::command]
pub fn kb_scan_sources(
pub async fn kb_scan_sources(
app: AppHandle,
state: State<'_, App>,
kb_id: String,
) -> Result<KbSourceScanResult, String> {
let db = state.core.db.lock().unwrap();
let kb = kb_db::get_knowledge_base(&db, &kb_id)
.map_err(|e| e.to_string())?
.ok_or_else(|| "知识库不存在".to_string())?;
let sources = kb_db::list_sources(&db, &kb_id).map_err(|e| e.to_string())?;
let mut total = KbSourceScanResult {
imported: 0,
ignored: 0,
errors: Vec::new(),
) -> Result<(), String> {
let core = state.core.clone();
let (kb, sources) = {
let db = core.db.lock().unwrap();
let kb = kb_db::get_knowledge_base(&db, &kb_id)
.map_err(|e| e.to_string())?
.ok_or_else(|| "知识库不存在".to_string())?;
let sources = kb_db::list_sources(&db, &kb_id).map_err(|e| e.to_string())?;
(kb, sources)
};
for src in sources {
let dir = std::path::PathBuf::from(&src.path);
if !dir.is_dir() {
total.errors.push(format!("目录不存在:{}", src.path));
continue;
let app2 = app.clone();
tokio::spawn(async move {
let mut total_imported = 0usize;
let mut total_failed = 0usize;
let mut total_ignored = 0usize;
let mut total_chunks = 0usize;
let mut total_current = 0usize;
let mut total_files = 0usize;
let mut all_errors: Vec<String> = Vec::new();
for src in sources {
let dir = std::path::PathBuf::from(&src.path);
if !dir.is_dir() {
all_errors.push(format!("目录不存在:{}", src.path));
continue;
}
let exts = crate::knowledge::normalize_extensions(&src.extensions);
let result =
run_scan_with_progress(&app2, &core, kb.clone(), Some(src.id), dir, exts, src.recursive, false)
.await;
total_imported += result.imported;
total_failed += result.failed;
total_ignored += result.ignored;
total_chunks += result.chunks;
total_current += result.current;
total_files += result.total;
for e in result.errors {
if all_errors.len() < 30 {
all_errors.push(e);
}
}
}
let exts = crate::knowledge::normalize_extensions(&src.extensions);
let r = scan_and_import(&db, &kb, &src.id, &dir, &exts, src.recursive);
total.imported += r.imported;
total.ignored += r.ignored;
total.errors.extend(r.errors);
}
Ok(total)
let _ = app2.emit(
"kb://scan-progress",
KbScanProgressEvent {
kb_id: kb.id,
source_path: "全部目录".into(),
status: "done".into(),
current: total_current,
total: total_files,
imported: total_imported,
failed: total_failed,
ignored: total_ignored,
chunks: total_chunks,
errors: all_errors,
},
);
});
Ok(())
}
/// 删除目录来源及其导入的全部文档。
@@ -1790,22 +1882,29 @@ pub fn kb_remove_source(state: State<'_, App>, source_id: String) -> Result<usiz
Ok(removed)
}
/// 扫描目录并导入文档(文本提取切块;图片/视频/音频以文件名为内容登记;其他忽略)
fn scan_and_import(
db: &rusqlite::Connection,
kb: &xianren_core::KnowledgeBase,
source_id: &str,
dir: &std::path::Path,
extensions: &[String],
/// 后台扫描一个目录来源并逐文件推送进度,结束推送 done 事件
async fn run_scan_with_progress(
app: &AppHandle,
core: &CoreApp,
kb: xianren_core::KnowledgeBase,
source_id: Option<String>,
dir: std::path::PathBuf,
extensions: Vec<String>,
recursive: bool,
) -> KbSourceScanResult {
let mut result = KbSourceScanResult {
imported: 0,
ignored: 0,
errors: Vec::new(),
};
let files = crate::knowledge::collect_files(dir, extensions, recursive);
emit_done: bool,
) -> KbScanProgressEvent {
let source_path = dir.to_string_lossy().to_string();
let files = crate::knowledge::collect_files(&dir, &extensions, recursive);
let total = files.len();
let mut current = 0usize;
let mut imported = 0usize;
let mut failed = 0usize;
let mut ignored = 0usize;
let mut chunks = 0usize;
let mut errors: Vec<String> = Vec::new();
for file in files {
current += 1;
let name = file
.file_name()
.and_then(|n| n.to_str())
@@ -1813,64 +1912,101 @@ fn scan_and_import(
.to_string();
let file_path = file.to_string_lossy().to_string();
let category = crate::knowledge::classify(&name);
if category == crate::knowledge::FileCategory::Other {
result.ignored += 1;
continue;
}
if kb_db::document_exists(db, &kb.id, &file_path).unwrap_or(false) {
continue;
}
let bytes = match std::fs::read(&file) {
Ok(b) => b,
Err(e) => {
result.errors.push(format!("{name}: 读取失败 {e}"));
continue;
}
};
let content = if category == crate::knowledge::FileCategory::Text {
match crate::knowledge::extract_text(&name, &bytes) {
Ok(t) => t,
let duplicate = kb_db::document_exists(&core.db.lock().unwrap(), &kb.id, &file_path)
.unwrap_or(false);
if category == crate::knowledge::FileCategory::Other || duplicate {
ignored += 1;
} else {
match process_scan_file(&core.db.lock().unwrap(), &kb, source_id.as_deref(), &file, &name, category) {
Ok(n) => {
imported += 1;
chunks += n;
}
Err(e) => {
result.errors.push(format!("{name}: {e}"));
continue;
failed += 1;
if errors.len() < 30 {
errors.push(format!("{name}: {e}"));
}
}
}
} else {
// 图片 / 视频 / 音频:以文件名为内容登记,便于按名检索
name.clone()
};
let chunks = if category == crate::knowledge::FileCategory::Text {
kb_db::chunk_text(&content, kb.chunk_size as usize, kb.chunk_overlap as usize)
} else {
vec![content.clone()]
};
if chunks.is_empty() {
result.errors.push(format!("{name}: 切分后没有有效分块"));
continue;
}
let doc = xianren_core::KbDocument {
id: uuid::Uuid::new_v4().to_string(),
kb_id: kb.id.clone(),
name,
file_type: file
.extension()
.map(|e| e.to_string_lossy().to_lowercase())
.unwrap_or_default(),
file_size: bytes.len() as i64,
char_count: content.chars().count() as i64,
chunk_count: chunks.len() as i64,
file_path,
source_id: Some(source_id.to_string()),
created_at: String::new(),
updated_at: String::new(),
};
if let Err(e) = kb_db::insert_document_with_chunks(db, &doc, &content, &chunks) {
result.errors.push(format!("{}: 写入失败 {e}", doc.name));
continue;
}
result.imported += 1;
let _ = app.emit(
"kb://scan-progress",
KbScanProgressEvent {
kb_id: kb.id.clone(),
source_path: source_path.clone(),
status: "scanning".into(),
current,
total,
imported,
failed,
ignored,
chunks,
errors: errors.clone(),
},
);
}
result
let done = KbScanProgressEvent {
kb_id: kb.id.clone(),
source_path,
status: "done".into(),
current: total,
total,
imported,
failed,
ignored,
chunks,
errors,
};
if emit_done {
let _ = app.emit("kb://scan-progress", done.clone());
}
done
}
/// 处理单个文件:文本提取切块,媒体以文件名为内容登记;返回导入的分块数。
fn process_scan_file(
db: &rusqlite::Connection,
kb: &xianren_core::KnowledgeBase,
source_id: Option<&str>,
file: &std::path::Path,
name: &str,
category: crate::knowledge::FileCategory,
) -> Result<usize, String> {
let bytes = std::fs::read(file).map_err(|e| format!("读取失败 {e}"))?;
let content = if category == crate::knowledge::FileCategory::Text {
crate::knowledge::extract_text(name, &bytes)?
} else {
// 图片 / 视频 / 音频:以文件名为内容登记,便于按名检索
name.to_string()
};
let chunks = if category == crate::knowledge::FileCategory::Text {
kb_db::chunk_text(&content, kb.chunk_size as usize, kb.chunk_overlap as usize)
} else {
vec![content.clone()]
};
if chunks.is_empty() {
return Err("切分后没有有效分块".into());
}
let doc = xianren_core::KbDocument {
id: uuid::Uuid::new_v4().to_string(),
kb_id: kb.id.clone(),
name: name.to_string(),
file_type: file
.extension()
.map(|e| e.to_string_lossy().to_lowercase())
.unwrap_or_default(),
file_size: bytes.len() as i64,
char_count: content.chars().count() as i64,
chunk_count: chunks.len() as i64,
file_path: file.to_string_lossy().to_string(),
source_id: source_id.map(str::to_string),
created_at: String::new(),
updated_at: String::new(),
};
kb_db::insert_document_with_chunks(db, &doc, &content, &chunks)
.map_err(|e| format!("写入失败 {e}"))?;
Ok(chunks.len())
}
#[tauri::command]
+72 -8
View File
@@ -145,6 +145,38 @@ pub fn list_documents(db: &Connection, kb_id: &str) -> Result<Vec<KbDocument>> {
Ok(out)
}
pub fn count_documents(db: &Connection, kb_id: &str) -> Result<i64> {
let n: i64 = db.query_row(
"SELECT COUNT(*) FROM kb_documents WHERE kb_id = ?1",
params![kb_id],
|row| row.get(0),
)?;
Ok(n)
}
/// 分页查询文档:`page` 从 1 开始,`page_size` 会被限制在 1100。
pub fn list_documents_page(
db: &Connection,
kb_id: &str,
page: i64,
page_size: i64,
) -> Result<Vec<KbDocument>> {
let page = page.max(1);
let page_size = page_size.clamp(1, 100);
let offset = (page - 1) * page_size;
let mut stmt = db.prepare(&format!(
"SELECT {DOC_COLUMNS} FROM kb_documents
WHERE kb_id = ?1 ORDER BY updated_at DESC, created_at DESC
LIMIT ?2 OFFSET ?3"
))?;
let rows = stmt.query_map(params![kb_id, page_size, offset], row_to_doc)?;
let mut out = Vec::new();
for row in rows {
out.push(row?);
}
Ok(out)
}
pub fn get_document(db: &Connection, id: &str) -> Result<Option<KbDocument>> {
let mut stmt = db.prepare(&format!(
"SELECT {DOC_COLUMNS} FROM kb_documents WHERE id = ?1"
@@ -349,12 +381,14 @@ pub fn search(
kb_id: Option<&str>,
query: &str,
limit: usize,
offset: usize,
) -> Result<Vec<KbSearchHit>> {
let q = query.trim();
if q.is_empty() {
return Ok(Vec::new());
}
let limit = limit.clamp(1, 100) as i64;
let offset = offset as i64;
if q.chars().count() >= 3 {
let fts_q = format!("\"{}\"", q.replace('"', "\"\""));
let result = db.prepare(
@@ -363,10 +397,10 @@ pub fn search(
JOIN kb_documents d ON d.id = c.document_id
JOIN kb_chunks_fts ON kb_chunks_fts.rowid = c.rowid
WHERE (?1 IS NULL OR c.kb_id = ?1) AND kb_chunks_fts MATCH ?2
ORDER BY bm25(kb_chunks_fts) LIMIT ?3",
ORDER BY bm25(kb_chunks_fts) LIMIT ?3 OFFSET ?4",
)
.and_then(|mut stmt| {
let rows = stmt.query_map(params![kb_id, fts_q, limit], row_to_hit)?;
let rows = stmt.query_map(params![kb_id, fts_q, limit, offset], row_to_hit)?;
rows.collect::<std::result::Result<Vec<_>, _>>()
});
if let Ok(hits) = result {
@@ -379,9 +413,9 @@ pub fn search(
FROM kb_chunks c
JOIN kb_documents d ON d.id = c.document_id
WHERE (?1 IS NULL OR c.kb_id = ?1) AND c.content LIKE '%' || ?2 || '%'
ORDER BY c.rowid LIMIT ?3",
ORDER BY c.rowid LIMIT ?3 OFFSET ?4",
)?;
let rows = stmt.query_map(params![kb_id, q, limit], row_to_hit)?;
let rows = stmt.query_map(params![kb_id, q, limit, offset], row_to_hit)?;
let mut out = Vec::new();
for row in rows {
out.push(row?);
@@ -389,6 +423,36 @@ pub fn search(
Ok(out)
}
/// 检索命中总数(与 `search` 使用同一套 FTS / LIKE 回退逻辑)。
pub fn count_search(db: &Connection, kb_id: Option<&str>, query: &str) -> Result<i64> {
let q = query.trim();
if q.is_empty() {
return Ok(0);
}
if q.chars().count() >= 3 {
let fts_q = format!("\"{}\"", q.replace('"', "\"\""));
let result = db.query_row(
"SELECT COUNT(*) FROM kb_chunks c
JOIN kb_documents d ON d.id = c.document_id
JOIN kb_chunks_fts ON kb_chunks_fts.rowid = c.rowid
WHERE (?1 IS NULL OR c.kb_id = ?1) AND kb_chunks_fts MATCH ?2",
params![kb_id, fts_q],
|row| row.get::<_, i64>(0),
);
if let Ok(n) = result {
return Ok(n);
}
}
let n: i64 = db.query_row(
"SELECT COUNT(*) FROM kb_chunks c
JOIN kb_documents d ON d.id = c.document_id
WHERE (?1 IS NULL OR c.kb_id = ?1) AND c.content LIKE '%' || ?2 || '%'",
params![kb_id, q],
|row| row.get(0),
)?;
Ok(n)
}
fn row_to_kb(row: &rusqlite::Row<'_>) -> rusqlite::Result<KnowledgeBase> {
Ok(KnowledgeBase {
id: row.get(0)?,
@@ -491,7 +555,7 @@ mod tests {
assert_eq!(kbs.len(), 1);
assert!(kbs[0].chunk_count >= 1);
let hits = search(&conn, Some("kb1"), "智能体", 10).unwrap();
let hits = search(&conn, Some("kb1"), "智能体", 10, 0).unwrap();
assert_eq!(hits.len(), 1);
assert!(hits[0].content.contains("智能体"));
}
@@ -515,7 +579,7 @@ mod tests {
)
.unwrap();
insert_doc(&conn, "d1", "今天是星期一,天气晴朗,适合出去散步。");
let hits = search(&conn, Some("kb1"), "散步", 10).unwrap();
let hits = search(&conn, Some("kb1"), "散步", 10, 0).unwrap();
assert_eq!(hits.len(), 1);
}
@@ -540,7 +604,7 @@ mod tests {
insert_doc(&conn, "d1", "知识库测试内容,用于验证删除级联。");
delete_document(&conn, "d1").unwrap();
assert!(get_document(&conn, "d1").unwrap().is_none());
let hits = search(&conn, Some("kb1"), "知识库", 10).unwrap();
let hits = search(&conn, Some("kb1"), "知识库", 10, 0).unwrap();
assert!(hits.is_empty());
}
@@ -569,6 +633,6 @@ mod tests {
let after = get_document(&conn, "d1").unwrap().unwrap().chunk_count;
assert_eq!(n as i64, after);
assert!(after < before);
assert!(!search(&conn, Some("kb1"), "测试文本", 5).unwrap().is_empty());
assert!(!search(&conn, Some("kb1"), "测试文本", 5, 0).unwrap().is_empty());
}
}
+5 -4
View File
@@ -110,8 +110,8 @@ ui/src/
| 工作流 | `list_workflows``add_workflow``update_workflow``remove_workflow``set_workflow_enabled``restore_preset_workflows``run_workflow` |
| 定时计划 | `list_scheduled_tasks``add_scheduled_task``update_scheduled_task``remove_scheduled_task``set_scheduled_task_enabled``run_scheduled_task_now` |
| 邮件 | `mail_test`(用当前 SMTP 配置发送测试邮件) |
| 知识库 | `list_knowledge_bases``add_knowledge_base``update_knowledge_base``remove_knowledge_base``list_kb_documents``get_kb_document``kb_import_documents``remove_kb_document``kb_rechunk_document``kb_search` |
| 知识库·目录来源 | `kb_list_sources``kb_add_source``kb_scan_sources``kb_remove_source` |
| 知识库 | `list_knowledge_bases``add_knowledge_base``update_knowledge_base``remove_knowledge_base``list_kb_documents`(分页)`get_kb_document``kb_import_documents``remove_kb_document``kb_rechunk_document``kb_search`(分页) |
| 知识库·目录来源 | `kb_list_sources``kb_add_source``kb_scan_sources`(后台扫描,`kb://scan-progress` 事件推送进度)`kb_remove_source` |
| 应用/设置 | `app_info``autostart_status``autostart_set``settings_get``settings_set` |
| 模型 | `list_models``import_model``remove_model``set_model_enabled``scan_models``add_remote_model` |
| 模型广场 | `search_models``list_model_files``list_recommended_models``import_recommendations``fetch_model_page` |
@@ -144,7 +144,7 @@ ui/src/
- `send_task_email` / `load_smtp_config` / `split_recipients`:读取设置页「邮件」SMTP 配置、拆分多收件人并调用 `mail.rs` 发送。
- `kb_import_documents`base64 解码 → `knowledge.rs::extract_text` 提取文本(文本类直接 UTF-8PDF 走 pdf-extract)→ `kb_db::chunk_text` 切块(按知识库 chunk_size/overlap)→ `insert_document_with_chunks` 事务写入文档与分块,逐文件返回成功/失败结果。
- `kb_search` / `kb_rechunk_document`:检索走 `kb_db::search`FTS5 trigram + BM25,短查询回退 LIKE);重切按知识库最新分块设置重建该文档分块与索引。
- `kb_add_source` / `kb_scan_sources` / `kb_remove_source`:目录来源注册与扫描——`knowledge.rs::collect_files` 按后缀过滤(空=全部)与递归开关收集文件,`classify` 分成文本 / 图片视频 / 音频三类(其他忽略),文本提取切块入库、媒体以文件名为内容登记,`kb_db::document_exists` 按(kb_id + file_path)去重;删除来源级联删除其导入文档。
- `kb_add_source` / `kb_scan_sources` / `kb_remove_source`:目录来源注册与后台扫描——`knowledge.rs::collect_files` 按后缀过滤(空=全部)与递归开关收集文件,`classify` 分成文本 / 图片视频 / 音频三类(其他忽略),文本提取切块入库、媒体以文件名为内容登记,`kb_db::document_exists` 按(kb_id + file_path)去重;扫描过程逐文件推送 `kb://scan-progress`(处理数/成功/失败/忽略/分块),多目录重扫最后推送一次汇总 done;删除来源级联删除其导入文档。
**邮件发送模块(`apps/desktop/src/mail.rs`):**
@@ -222,7 +222,7 @@ ui/src/
### 5.9 `knowledge_base.rs`
知识库 CRUD`list/get/insert/update/delete`)、文档 CRUD`list/get/get_detail`)、`insert_document_with_chunks`(文档 + 分块同事务写入,FTS 触发器自动建索引)、`delete_document`(级联删除分块并同步索引)、`document_exists`(目录导入按文件路径去重)、`rechunk_document``chunk_text`(按字符切块 + 重叠)、`search`FTS5 trigram BM25 排序,查询 <3 字或 MATCH 失败回退 LIKE);目录来源 `list/get/upsert/delete_sources``delete_docs_by_source`
知识库 CRUD`list/get/insert/update/delete`)、文档 CRUD`list/get/get_detail``count_documents` / `list_documents_page` 分页)、`insert_document_with_chunks`(文档 + 分块同事务写入,FTS 触发器自动建索引)、`delete_document`(级联删除分块并同步索引)、`document_exists`(目录导入按文件路径去重)、`rechunk_document``chunk_text`(按字符切块 + 重叠)、`search` / `count_search`FTS5 trigram BM25 排序 + 总数统计,查询 <3 字或 MATCH 失败回退 LIKE);目录来源 `list/get/upsert/delete_sources``delete_docs_by_source`
---
@@ -269,6 +269,7 @@ ui/src/
| `workflow://node-status` | 后端→前端 | 工作流节点运行状态(running/done/error)与输出文本 |
| `scheduled://updated` | 后端→前端 | 定时计划状态变化(创建/编辑/启停/执行完成),前端刷新列表 |
| `conversations://updated` | 后端→前端 | 定时计划执行后会话列表变化(供智能体会话页刷新) |
| `kb://scan-progress` | 后端→前端 | 知识库目录扫描进度(处理/成功/失败/忽略/分块数;status=scanning/done |
---
+5 -2
View File
@@ -139,8 +139,10 @@
- **递归子目录**:可开关是否递归扫描所有子目录;
- **类型过滤**:只收录**可读文本**(提取内容并分块检索)、**图片 / 视频**、**音频**三类文件(图片视频音频以文件名为内容登记,可按文件名检索),检测到其他类型自动忽略;
- 同一目录重复添加自动更新配置不产生重复来源;按文件路径去重,可一键「重新扫描」增量导入新增文件;删除目录来源会同时删除它导入的文档。
- **文档操作**:文档列表展示类型 / 大小 / 字数 / 分块数;支持全文预览、「重切」(按最新分块设置重新切分,适用于修改设置后)与删除
- **检索**:全文检索基于 SQLite FTS5(trigram 分词,对中文友好),按 BM25 相关性排序;支持单库检索(也可全库检索),结果展示来源文档与分块序号,可一键复制分块内容用于对话
- **扫描进度**:添加目录 / 重新扫描时显示进度条与处理统计(处理 X / Y 个文件、成功、失败、忽略、分块数),失败文件在结尾汇总提示
- **文档操作**:文档列表每页 10 条,带分页控件;展示类型 / 大小 / 字数 / 分块数;支持全文预览、「重切」(按最新分块设置重新切分,适用于修改设置后)与删除
- **检索**:全文检索基于 SQLite FTS5(trigram 分词,对中文友好),按 BM25 相关性排序;支持单库检索(也可全库检索),结果每页 10 条带分页控件,展示来源文档与分块序号,可一键复制分块内容用于对话。
- **页面布局**:知识库主区内自上而下依次为「目录来源」→「文档列表」→「检索」。
- 数据表:`knowledge_bases``kb_documents`(含提取后的纯文本)、`kb_chunks`(分块)、`kb_chunks_fts`FTS5 全文索引,contentless-delete 模式 + 触发器同步)。
## 6. 模型管理页(`/`
@@ -230,6 +232,7 @@
### 2026-08-17
- 知识库页调整:文档列表与检索结果增加分页控件;目录来源、文档列表区块移到检索上方;目录扫描增加进度条与处理统计(处理/成功/失败/忽略/分块数,事件驱动实时刷新)。
- 修复:知识库「添加目录 → 浏览…」无响应——为 dialog 插件在 capabilities 中补充权限(`dialog:default`),并让选择器异常在前端可见。
- 知识库「添加目录」弹窗支持两种方式选择目录:手动输入路径,或点击「浏览…」调起系统原生文件夹选择器(接入 tauri-plugin-dialog)。
- 知识库新增「目录来源」:支持添加多个本地目录批量导入(后缀过滤默认全部、可开关递归子目录),只收录文本 / 图片视频 / 音频三类文件其余忽略,支持重新扫描增量导入与按来源删除。
+37 -7
View File
@@ -261,9 +261,30 @@ export interface KbSourceInput {
recursive: boolean;
}
export interface KbSourceScanResult {
export interface KbDocumentPage {
items: KbDocument[];
total: number;
page: number;
page_size: number;
}
export interface KbSearchPage {
items: KbSearchHit[];
total: number;
page: number;
page_size: number;
}
export interface KbScanProgressEvent {
kb_id: string;
source_path: string;
status: string;
current: number;
total: number;
imported: number;
failed: number;
ignored: number;
chunks: number;
errors: string[];
}
@@ -491,8 +512,12 @@ export const api = {
invoke<void>("update_knowledge_base", { id, input }),
removeKnowledgeBase: (id: string) =>
invoke<void>("remove_knowledge_base", { id }),
listKbDocuments: (kbId: string) =>
invoke<KbDocument[]>("list_kb_documents", { kbId }),
listKbDocuments: (kbId: string, page?: number, pageSize?: number) =>
invoke<KbDocumentPage>("list_kb_documents", {
kbId,
page: page ?? 1,
pageSize: pageSize ?? 10,
}),
getKbDocument: (documentId: string) =>
invoke<KbDocumentDetail>("get_kb_document", { documentId }),
kbImportDocuments: (
@@ -503,13 +528,18 @@ export const api = {
invoke<void>("remove_kb_document", { documentId }),
kbRechunkDocument: (documentId: string) =>
invoke<number>("kb_rechunk_document", { documentId }),
kbSearch: (kbId: string | null, query: string, limit?: number) =>
invoke<KbSearchHit[]>("kb_search", { kbId, query, limit: limit ?? 10 }),
kbSearch: (kbId: string | null, query: string, page?: number, pageSize?: number) =>
invoke<KbSearchPage>("kb_search", {
kbId,
query,
page: page ?? 1,
pageSize: pageSize ?? 10,
}),
kbListSources: (kbId: string) => invoke<KbSource[]>("kb_list_sources", { kbId }),
kbAddSource: (input: KbSourceInput) =>
invoke<KbSourceScanResult>("kb_add_source", { input }),
invoke<void>("kb_add_source", { input }),
kbScanSources: (kbId: string) =>
invoke<KbSourceScanResult>("kb_scan_sources", { kbId }),
invoke<void>("kb_scan_sources", { kbId }),
kbRemoveSource: (sourceId: string) =>
invoke<number>("kb_remove_source", { sourceId }),
listModels: () => invoke<ModelInfo[]>("list_models"),
+236 -97
View File
@@ -4,14 +4,18 @@ import {
api,
KbDocument,
KbDocumentDetail,
KbScanProgressEvent,
KbSearchHit,
KbSource,
KbSourceScanResult,
KnowledgeBase,
KnowledgeBaseInput,
onEvent,
} from "../api";
import Icon from "../components/Icon";
const DOC_PAGE_SIZE = 10;
const SEARCH_PAGE_SIZE = 10;
function fmtSize(bytes: number) {
if (bytes >= 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
if (bytes >= 1024) return `${(bytes / 1024).toFixed(0)} KB`;
@@ -48,6 +52,8 @@ export default function KnowledgeBasePage() {
const [kbs, setKbs] = useState<KnowledgeBase[]>([]);
const [selectedId, setSelectedId] = useState<string | null>(null);
const [docs, setDocs] = useState<KbDocument[]>([]);
const [docTotal, setDocTotal] = useState(0);
const [docPage, setDocPage] = useState(1);
const [msg, setMsg] = useState<string | null>(null);
const [busy, setBusy] = useState(false);
const [kbModal, setKbModal] = useState<{
@@ -59,10 +65,16 @@ export default function KnowledgeBasePage() {
const [query, setQuery] = useState("");
const [searching, setSearching] = useState(false);
const [searchResults, setSearchResults] = useState<KbSearchHit[] | null>(null);
const [searchTotal, setSearchTotal] = useState(0);
const [searchPage, setSearchPage] = useState(1);
const [sources, setSources] = useState<KbSource[]>([]);
const [sourceModal, setSourceModal] = useState(false);
const [sourceMsg, setSourceMsg] = useState<string | null>(null);
const [scanning, setScanning] = useState(false);
const [scanProgress, setScanProgress] = useState<KbScanProgressEvent | null>(null);
const uploadInputRef = useRef<HTMLInputElement>(null);
const selectedIdRef = useRef<string | null>(null);
const docPageRef = useRef(1);
const selected = useMemo(
() => kbs.find((k) => k.id === selectedId) ?? null,
@@ -79,27 +91,73 @@ export default function KnowledgeBasePage() {
}
}
async function loadDocPage(kbId: string, page: number) {
const p = await api.listKbDocuments(kbId, page, DOC_PAGE_SIZE);
if (p.items.length === 0 && p.page > 1) {
const prev = await api.listKbDocuments(kbId, p.page - 1, DOC_PAGE_SIZE);
setDocs(prev.items);
setDocTotal(prev.total);
setDocPage(prev.page);
docPageRef.current = prev.page;
return;
}
setDocs(p.items);
setDocTotal(p.total);
setDocPage(p.page);
docPageRef.current = p.page;
}
useEffect(() => {
refreshKbs().catch((e) => setMsg(String(e)));
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
useEffect(() => {
selectedIdRef.current = selectedId;
}, [selectedId]);
useEffect(() => {
docPageRef.current = docPage;
}, [docPage]);
useEffect(() => {
if (!selectedId) {
setDocs([]);
setDocTotal(0);
setDocPage(1);
docPageRef.current = 1;
setSources([]);
setSearchResults(null);
setSearchTotal(0);
return;
}
api
.listKbDocuments(selectedId)
.then(setDocs)
.catch((e) => setMsg(`加载文档失败:${String(e)}`));
api
.kbListSources(selectedId)
.then(setSources)
.catch((e) => setMsg(`加载目录来源失败:${String(e)}`));
loadDocPage(selectedId, 1).catch((e) => setMsg(`加载文档失败:${String(e)}`));
}, [selectedId]);
useEffect(() => {
const un = onEvent<KbScanProgressEvent>("kb://scan-progress", (e) => {
setScanProgress(e);
setScanning(e.status !== "done");
if (e.status === "done") {
refreshKbs().catch(() => {});
if (selectedIdRef.current === e.kb_id) {
api
.kbListSources(e.kb_id)
.then(setSources)
.catch(() => {});
loadDocPage(e.kb_id, docPageRef.current).catch(() => {});
}
}
});
return () => {
un.then((f) => f());
};
}, []);
async function handleSaveKb(input: KnowledgeBaseInput) {
setBusy(true);
try {
@@ -121,7 +179,11 @@ export default function KnowledgeBasePage() {
}
async function handleDeleteKb(kb: KnowledgeBase) {
if (!confirm(`确定删除知识库「${kb.name}」吗?将同时删除其中 ${kb.doc_count} 个文档与 ${kb.chunk_count} 个分块。`)) {
if (
!confirm(
`确定删除知识库「${kb.name}」吗?将同时删除其中 ${kb.doc_count} 个文档与 ${kb.chunk_count} 个分块。`,
)
) {
return;
}
try {
@@ -147,10 +209,14 @@ export default function KnowledgeBasePage() {
const okCount = results.filter((r) => r.ok).length;
const failed = results.filter((r) => !r.ok);
setUploadMsg(
`导入完成:成功 ${okCount}${failed.length > 0 ? `,失败 ${failed.length} 个(${failed.map((f) => `${f.name}: ${f.error}`).join("")}` : ""}`,
`导入完成:成功 ${okCount}${
failed.length > 0
? `,失败 ${failed.length} 个(${failed.map((f) => `${f.name}: ${f.error}`).join("")}`
: ""
}`,
);
await refreshKbs();
setDocs(await api.listKbDocuments(selected.id));
await loadDocPage(selected.id, docPageRef.current);
} catch (e) {
setUploadMsg(`导入失败:${String(e)}`);
} finally {
@@ -165,7 +231,7 @@ export default function KnowledgeBasePage() {
const n = await api.kbRechunkDocument(doc.id);
setMsg(`已重新切分为 ${n} 个分块`);
if (selected) {
setDocs(await api.listKbDocuments(selected.id));
await loadDocPage(selected.id, docPageRef.current);
await refreshKbs();
}
} catch (e) {
@@ -179,7 +245,7 @@ export default function KnowledgeBasePage() {
await api.removeKbDocument(doc.id);
setMsg("文档已删除");
if (selected) {
setDocs(await api.listKbDocuments(selected.id));
await loadDocPage(selected.id, docPageRef.current);
await refreshKbs();
}
} catch (e) {
@@ -187,11 +253,14 @@ export default function KnowledgeBasePage() {
}
}
async function handleSearch() {
async function handleSearch(page = 1) {
if (!query.trim()) return;
setSearching(true);
try {
setSearchResults(await api.kbSearch(selectedId, query.trim(), 20));
const p = await api.kbSearch(selectedId, query.trim(), page, SEARCH_PAGE_SIZE);
setSearchResults(p.items);
setSearchTotal(p.total);
setSearchPage(p.page);
} catch (e) {
setMsg(`检索失败:${String(e)}`);
} finally {
@@ -199,42 +268,23 @@ export default function KnowledgeBasePage() {
}
}
async function refreshSourcesAndDocs() {
if (!selected) return;
const [d, s] = await Promise.all([
api.listKbDocuments(selected.id),
api.kbListSources(selected.id),
]);
setDocs(d);
setSources(s);
await refreshKbs();
}
function formatScanMsg(label: string, r: KbSourceScanResult) {
const errPart =
r.errors.length > 0
? `${r.errors.length} 个错误(${r.errors
.slice(0, 3)
.join("")}${r.errors.length > 3 ? "…" : ""}`
: "";
return `${label}完成:导入 ${r.imported} 个,忽略 ${r.ignored}${errPart}`;
}
async function handleAddSource(path: string, extensions: string, recursive: boolean) {
if (!selected) return;
setBusy(true);
setSourceMsg(null);
setScanProgress(null);
setScanning(true);
try {
const r = await api.kbAddSource({
await api.kbAddSource({
kb_id: selected.id,
path,
extensions,
recursive,
});
await refreshSourcesAndDocs();
setSourceMsg(formatScanMsg("添加目录", r));
setSourceMsg("开始扫描目录,请稍候…");
} catch (e) {
setSourceMsg(`添加目录失败:${String(e)}`);
setScanning(false);
} finally {
setBusy(false);
setSourceModal(false);
@@ -245,12 +295,13 @@ export default function KnowledgeBasePage() {
if (!selected) return;
setBusy(true);
setSourceMsg(null);
setScanProgress(null);
setScanning(true);
try {
const r = await api.kbScanSources(selected.id);
await refreshSourcesAndDocs();
setSourceMsg(formatScanMsg("重新扫描", r));
await api.kbScanSources(selected.id);
} catch (e) {
setSourceMsg(`重新扫描失败:${String(e)}`);
setScanning(false);
} finally {
setBusy(false);
}
@@ -262,7 +313,9 @@ export default function KnowledgeBasePage() {
const n = await api.kbRemoveSource(src.id);
setSourceMsg(`已删除目录来源及其 ${n} 个文档`);
if (selected) {
await refreshSourcesAndDocs();
setSources(await api.kbListSources(selected.id));
await loadDocPage(selected.id, docPageRef.current);
await refreshKbs();
}
} catch (e) {
setSourceMsg(`删除失败:${String(e)}`);
@@ -388,41 +441,6 @@ export default function KnowledgeBasePage() {
</div>
) : null}
{/* 检索 */}
<section className="mb-6 rounded-xl border border-border bg-panel p-4">
<div className="mb-2 text-xs uppercase text-slate-500"></div>
<div className="flex gap-2">
<input
className="input flex-1"
value={query}
placeholder="输入关键词,检索全部文档分块(支持中文)"
onChange={(e) => setQuery(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter") handleSearch();
}}
/>
<button
className="btn-primary shrink-0"
onClick={handleSearch}
disabled={searching || !query.trim()}
>
<Icon name="search" className="mr-1 inline h-3.5 w-3.5" />
{searching ? "检索中…" : "检索"}
</button>
</div>
{searchResults ? (
searchResults.length === 0 ? (
<div className="mt-3 text-xs text-slate-500"></div>
) : (
<div className="mt-3 space-y-2">
{searchResults.map((hit) => (
<SearchHitRow key={hit.chunk_id} hit={hit} />
))}
</div>
)
) : null}
</section>
{/* 目录来源 */}
<section className="mb-6 rounded-xl border border-border bg-panel p-4">
<div className="mb-2 flex items-center justify-between">
@@ -433,7 +451,7 @@ export default function KnowledgeBasePage() {
<button
className="btn-secondary !px-2.5 !py-1 text-xs"
onClick={handleScanAll}
disabled={busy || sources.length === 0}
disabled={busy || scanning || sources.length === 0}
>
<Icon name="refresh" className="mr-1 inline h-3 w-3" />
@@ -441,6 +459,7 @@ export default function KnowledgeBasePage() {
<button
className="btn-primary !px-2.5 !py-1 text-xs"
onClick={() => setSourceModal(true)}
disabled={scanning}
>
+
</button>
@@ -454,6 +473,40 @@ export default function KnowledgeBasePage() {
{sourceMsg}
</div>
) : null}
{scanProgress ? (
<div className="mb-3 rounded-lg border border-border bg-panel-2/60 p-3">
<div className="flex items-center justify-between gap-2 text-xs text-slate-300">
<span className="min-w-0 flex-1 truncate">{scanProgress.source_path}</span>
<span className="shrink-0">
{scanning
? `处理中 ${scanProgress.current}/${scanProgress.total}`
: "扫描完成"}
</span>
</div>
<div className="mt-2 h-1.5 overflow-hidden rounded-full bg-panel">
<div
className="h-full rounded-full bg-gradient-to-r from-accent to-accent-2 transition-all duration-200"
style={{
width: `${
scanProgress.total > 0
? (scanProgress.current / scanProgress.total) * 100
: 0
}%`,
}}
/>
</div>
<div className="mt-1.5 text-[11px] text-slate-400">
{scanProgress.current} / {scanProgress.total} · {" "}
{scanProgress.imported} · {scanProgress.failed} · {" "}
{scanProgress.ignored} · {scanProgress.chunks}
{scanProgress.errors.length > 0
? `${scanProgress.errors[0]}${
scanProgress.errors.length > 1 ? " 等" : ""
}`
: ""}
</div>
</div>
) : null}
{sources.length === 0 ? (
<div className="rounded-lg border border-dashed border-border bg-panel-2/40 px-3 py-6 text-center text-xs text-slate-500">
@@ -499,30 +552,84 @@ export default function KnowledgeBasePage() {
</section>
{/* 文档列表 */}
<section>
<section className="mb-6">
<div className="mb-2 text-xs uppercase text-slate-500">
{docs.length}
{docTotal}
</div>
{docs.length === 0 ? (
<div className="rounded-xl border border-dashed border-border bg-panel/50 px-4 py-10 text-center text-sm text-slate-500">
txt / md / PDF
txt / md / PDF
</div>
) : (
<div className="space-y-2">
{docs.map((doc) => (
<DocRow
key={doc.id}
doc={doc}
onPreview={() => {
api.getKbDocument(doc.id).then(setPreview).catch((e) => setMsg(String(e)));
}}
onRechunk={() => handleRechunk(doc)}
onDelete={() => handleDeleteDoc(doc)}
/>
))}
</div>
<>
<div className="space-y-2">
{docs.map((doc) => (
<DocRow
key={doc.id}
doc={doc}
onPreview={() => {
api
.getKbDocument(doc.id)
.then(setPreview)
.catch((e) => setMsg(String(e)));
}}
onRechunk={() => handleRechunk(doc)}
onDelete={() => handleDeleteDoc(doc)}
/>
))}
</div>
<Pager
page={docPage}
total={docTotal}
pageSize={DOC_PAGE_SIZE}
onChange={(p) => selected && loadDocPage(selected.id, p)}
/>
</>
)}
</section>
{/* 检索 */}
<section className="rounded-xl border border-border bg-panel p-4">
<div className="mb-2 text-xs uppercase text-slate-500"></div>
<div className="flex gap-2">
<input
className="input flex-1"
value={query}
placeholder="输入关键词,检索全部文档分块(支持中文)"
onChange={(e) => setQuery(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter") handleSearch(1);
}}
/>
<button
className="btn-primary shrink-0"
onClick={() => handleSearch(1)}
disabled={searching || !query.trim()}
>
<Icon name="search" className="mr-1 inline h-3.5 w-3.5" />
{searching ? "检索中…" : "检索"}
</button>
</div>
{searchResults ? (
searchResults.length === 0 ? (
<div className="mt-3 text-xs text-slate-500"></div>
) : (
<div className="mt-3">
<div className="space-y-2">
{searchResults.map((hit) => (
<SearchHitRow key={hit.chunk_id} hit={hit} />
))}
</div>
<Pager
page={searchPage}
total={searchTotal}
pageSize={SEARCH_PAGE_SIZE}
onChange={(p) => handleSearch(p)}
/>
</div>
)
) : null}
</section>
</div>
)}
</main>
@@ -571,6 +678,41 @@ export default function KnowledgeBasePage() {
);
}
function Pager({
page,
total,
pageSize,
onChange,
}: {
page: number;
total: number;
pageSize: number;
onChange: (p: number) => void;
}) {
const pages = Math.max(1, Math.ceil(total / pageSize));
return (
<div className="mt-3 flex items-center justify-center gap-3 text-xs text-slate-400">
<button
className="rounded border border-border bg-panel-2 px-2.5 py-1 hover:text-slate-200 disabled:opacity-40"
disabled={page <= 1}
onClick={() => onChange(page - 1)}
>
</button>
<span>
{page} / {pages} · {total}
</span>
<button
className="rounded border border-border bg-panel-2 px-2.5 py-1 hover:text-slate-200 disabled:opacity-40"
disabled={page >= pages}
onClick={() => onChange(page + 1)}
>
</button>
</div>
);
}
function SearchHitRow({ hit }: { hit: KbSearchHit }) {
const [copied, setCopied] = useState(false);
return (
@@ -845,10 +987,7 @@ function SourceModal({
/>
</div>
<div className="flex items-center gap-2">
<Toggle
checked={recursive}
onChange={(v) => setRecursive(v)}
/>
<Toggle checked={recursive} onChange={(v) => setRecursive(v)} />
<span className="text-xs text-slate-400">
{recursive ? "递归扫描所有子目录" : "仅扫描当前目录"}
</span>
@@ -866,7 +1005,7 @@ function SourceModal({
disabled={busy || !valid}
onClick={() => onSave(path.trim(), extensions.trim(), recursive)}
>
{busy ? "扫描中…" : "添加并扫描"}
{busy ? "添加中…" : "添加并扫描"}
</button>
</div>
</div>