diff --git a/apps/desktop/src/commands.rs b/apps/desktop/src/commands.rs index 8ca7f97..248a80a 100644 --- a/apps/desktop/src/commands.rs +++ b/apps/desktop/src/commands.rs @@ -1560,9 +1560,21 @@ pub fn remove_knowledge_base(state: State<'_, App>, id: String) -> Result<(), St pub fn list_kb_documents( state: State<'_, App>, kb_id: String, -) -> Result, String> { + page: Option, + page_size: Option, +) -> Result { + let page = page.unwrap_or(1).max(1); + let page_size = page_size.unwrap_or(10).clamp(1, 100); let db = state.core.db.lock().unwrap(); - kb_db::list_documents(&db, &kb_id).map_err(|e| e.to_string()) + let total = kb_db::count_documents(&db, &kb_id).map_err(|e| e.to_string())?; + let items = kb_db::list_documents_page(&db, &kb_id, page, page_size) + .map_err(|e| e.to_string())?; + Ok(KbDocumentPage { + items, + total, + page, + page_size, + }) } #[tauri::command] @@ -1679,11 +1691,38 @@ pub fn kb_search( state: State<'_, App>, kb_id: Option, query: String, - limit: Option, -) -> Result, String> { + page: Option, + page_size: Option, +) -> Result { + let page = page.unwrap_or(1).max(1); + let page_size = page_size.unwrap_or(10).clamp(1, 100); let db = state.core.db.lock().unwrap(); - kb_db::search(&db, kb_id.as_deref(), &query, limit.unwrap_or(10)) - .map_err(|e| e.to_string()) + let total = kb_db::count_search(&db, kb_id.as_deref(), &query).map_err(|e| e.to_string())?; + let offset = ((page - 1) * page_size) as usize; + let items = kb_db::search(&db, kb_id.as_deref(), &query, page_size as usize, offset) + .map_err(|e| e.to_string())?; + Ok(KbSearchPage { + items, + total, + page, + page_size, + }) +} + +#[derive(Serialize)] +pub struct KbDocumentPage { + pub items: Vec, + pub total: i64, + pub page: i64, + pub page_size: i64, +} + +#[derive(Serialize)] +pub struct KbSearchPage { + pub items: Vec, + pub total: i64, + pub page: i64, + pub page_size: i64, } #[derive(Deserialize)] @@ -1696,10 +1735,18 @@ pub struct KbSourceInput { pub recursive: bool, } -#[derive(Serialize)] -pub struct KbSourceScanResult { +#[derive(Serialize, Clone)] +pub struct KbScanProgressEvent { + pub kb_id: String, + pub source_path: String, + /// scanning（进行中）/ done（完成） + pub status: String, + pub current: usize, + pub total: usize, pub imported: usize, + pub failed: usize, pub ignored: usize, + pub chunks: usize, pub errors: Vec, } @@ -1712,70 +1759,115 @@ pub fn kb_list_sources( kb_db::list_sources(&db, &kb_id).map_err(|e| e.to_string()) } -/// 添加目录来源并立即扫描导入：只收录文本 / 图片视频 / 音频三类文件，其余忽略。 +/// 添加目录来源并后台扫描导入：只收录文本 / 图片视频 / 音频三类文件，其余忽略；进度通过 `kb://scan-progress` 事件推送。 #[tauri::command] -pub fn kb_add_source( +pub async fn kb_add_source( + app: AppHandle, state: State<'_, App>, input: KbSourceInput, -) -> Result { +) -> Result<(), String> { let dir = std::path::PathBuf::from(input.path.trim()); if !dir.is_dir() { return Err(format!("目录不存在或不是文件夹：{}", dir.display())); } let exts = crate::knowledge::normalize_extensions(&input.extensions); - let db = state.core.db.lock().unwrap(); - let kb = kb_db::get_knowledge_base(&db, &input.kb_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| "知识库不存在".to_string())?; - let source_id = uuid::Uuid::new_v4().to_string(); - let source = xianren_core::KbSource { - id: source_id.clone(), - kb_id: input.kb_id.clone(), - path: dir.to_string_lossy().to_string(), - extensions: input.extensions.trim().to_string(), - recursive: input.recursive, - created_at: String::new(), + let core = state.core.clone(); + let (kb, source_id) = { + let db = core.db.lock().unwrap(); + let kb = kb_db::get_knowledge_base(&db, &input.kb_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| "知识库不存在".to_string())?; + let source_id = uuid::Uuid::new_v4().to_string(); + let source = xianren_core::KbSource { + id: source_id.clone(), + kb_id: input.kb_id.clone(), + path: dir.to_string_lossy().to_string(), + extensions: input.extensions.trim().to_string(), + recursive: input.recursive, + created_at: String::new(), + }; + kb_db::upsert_source(&db, &source).map_err(|e| e.to_string())?; + // 同目录重复添加时沿用已有来源 id，避免文档归属错乱 + let source_id = kb_db::list_sources(&db, &input.kb_id) + .map_err(|e| e.to_string())? + .into_iter() + .find(|s| s.path == source.path && s.kb_id == source.kb_id) + .map(|s| s.id) + .unwrap_or(source_id); + (kb, source_id) }; - kb_db::upsert_source(&db, &source).map_err(|e| e.to_string())?; - // 同目录重复添加时沿用已有来源 id，避免文档归属错乱 - let source_id = kb_db::list_sources(&db, &input.kb_id) - .map_err(|e| e.to_string())? - .into_iter() - .find(|s| s.path == source.path && s.kb_id == source.kb_id) - .map(|s| s.id) - .unwrap_or(source_id); - Ok(scan_and_import(&db, &kb, &source_id, &dir, &exts, input.recursive)) + let app2 = app.clone(); + tokio::spawn(async move { + run_scan_with_progress(&app2, &core, kb, Some(source_id), dir, exts, input.recursive, true) + .await; + }); + Ok(()) } /// 重新扫描该知识库的全部目录来源，导入新增文件（按文件路径去重）。 #[tauri::command] -pub fn kb_scan_sources( +pub async fn kb_scan_sources( + app: AppHandle, state: State<'_, App>, kb_id: String, -) -> Result { - let db = state.core.db.lock().unwrap(); - let kb = kb_db::get_knowledge_base(&db, &kb_id) - .map_err(|e| e.to_string())? - .ok_or_else(|| "知识库不存在".to_string())?; - let sources = kb_db::list_sources(&db, &kb_id).map_err(|e| e.to_string())?; - let mut total = KbSourceScanResult { - imported: 0, - ignored: 0, - errors: Vec::new(), +) -> Result<(), String> { + let core = state.core.clone(); + let (kb, sources) = { + let db = core.db.lock().unwrap(); + let kb = kb_db::get_knowledge_base(&db, &kb_id) + .map_err(|e| e.to_string())? + .ok_or_else(|| "知识库不存在".to_string())?; + let sources = kb_db::list_sources(&db, &kb_id).map_err(|e| e.to_string())?; + (kb, sources) }; - for src in sources { - let dir = std::path::PathBuf::from(&src.path); - if !dir.is_dir() { - total.errors.push(format!("目录不存在：{}", src.path)); - continue; + let app2 = app.clone(); + tokio::spawn(async move { + let mut total_imported = 0usize; + let mut total_failed = 0usize; + let mut total_ignored = 0usize; + let mut total_chunks = 0usize; + let mut total_current = 0usize; + let mut total_files = 0usize; + let mut all_errors: Vec = Vec::new(); + for src in sources { + let dir = std::path::PathBuf::from(&src.path); + if !dir.is_dir() { + all_errors.push(format!("目录不存在：{}", src.path)); + continue; + } + let exts = crate::knowledge::normalize_extensions(&src.extensions); + let result = + run_scan_with_progress(&app2, &core, kb.clone(), Some(src.id), dir, exts, src.recursive, false) + .await; + total_imported += result.imported; + total_failed += result.failed; + total_ignored += result.ignored; + total_chunks += result.chunks; + total_current += result.current; + total_files += result.total; + for e in result.errors { + if all_errors.len() < 30 { + all_errors.push(e); + } + } } - let exts = crate::knowledge::normalize_extensions(&src.extensions); - let r = scan_and_import(&db, &kb, &src.id, &dir, &exts, src.recursive); - total.imported += r.imported; - total.ignored += r.ignored; - total.errors.extend(r.errors); - } - Ok(total) + let _ = app2.emit( + "kb://scan-progress", + KbScanProgressEvent { + kb_id: kb.id, + source_path: "全部目录".into(), + status: "done".into(), + current: total_current, + total: total_files, + imported: total_imported, + failed: total_failed, + ignored: total_ignored, + chunks: total_chunks, + errors: all_errors, + }, + ); + }); + Ok(()) } /// 删除目录来源及其导入的全部文档。 @@ -1790,22 +1882,29 @@ pub fn kb_remove_source(state: State<'_, App>, source_id: String) -> Result, + dir: std::path::PathBuf, + extensions: Vec, recursive: bool, -) -> KbSourceScanResult { - let mut result = KbSourceScanResult { - imported: 0, - ignored: 0, - errors: Vec::new(), - }; - let files = crate::knowledge::collect_files(dir, extensions, recursive); + emit_done: bool, +) -> KbScanProgressEvent { + let source_path = dir.to_string_lossy().to_string(); + let files = crate::knowledge::collect_files(&dir, &extensions, recursive); + let total = files.len(); + let mut current = 0usize; + let mut imported = 0usize; + let mut failed = 0usize; + let mut ignored = 0usize; + let mut chunks = 0usize; + let mut errors: Vec = Vec::new(); + for file in files { + current += 1; let name = file .file_name() .and_then(|n| n.to_str()) @@ -1813,64 +1912,101 @@ fn scan_and_import( .to_string(); let file_path = file.to_string_lossy().to_string(); let category = crate::knowledge::classify(&name); - if category == crate::knowledge::FileCategory::Other { - result.ignored += 1; - continue; - } - if kb_db::document_exists(db, &kb.id, &file_path).unwrap_or(false) { - continue; - } - let bytes = match std::fs::read(&file) { - Ok(b) => b, - Err(e) => { - result.errors.push(format!("{name}: 读取失败 {e}")); - continue; - } - }; - let content = if category == crate::knowledge::FileCategory::Text { - match crate::knowledge::extract_text(&name, &bytes) { - Ok(t) => t, + let duplicate = kb_db::document_exists(&core.db.lock().unwrap(), &kb.id, &file_path) + .unwrap_or(false); + if category == crate::knowledge::FileCategory::Other || duplicate { + ignored += 1; + } else { + match process_scan_file(&core.db.lock().unwrap(), &kb, source_id.as_deref(), &file, &name, category) { + Ok(n) => { + imported += 1; + chunks += n; + } Err(e) => { - result.errors.push(format!("{name}: {e}")); - continue; + failed += 1; + if errors.len() < 30 { + errors.push(format!("{name}: {e}")); + } } } - } else { - // 图片 / 视频 / 音频：以文件名为内容登记，便于按名检索 - name.clone() - }; - let chunks = if category == crate::knowledge::FileCategory::Text { - kb_db::chunk_text(&content, kb.chunk_size as usize, kb.chunk_overlap as usize) - } else { - vec![content.clone()] - }; - if chunks.is_empty() { - result.errors.push(format!("{name}: 切分后没有有效分块")); - continue; } - let doc = xianren_core::KbDocument { - id: uuid::Uuid::new_v4().to_string(), - kb_id: kb.id.clone(), - name, - file_type: file - .extension() - .map(|e| e.to_string_lossy().to_lowercase()) - .unwrap_or_default(), - file_size: bytes.len() as i64, - char_count: content.chars().count() as i64, - chunk_count: chunks.len() as i64, - file_path, - source_id: Some(source_id.to_string()), - created_at: String::new(), - updated_at: String::new(), - }; - if let Err(e) = kb_db::insert_document_with_chunks(db, &doc, &content, &chunks) { - result.errors.push(format!("{}: 写入失败 {e}", doc.name)); - continue; - } - result.imported += 1; + let _ = app.emit( + "kb://scan-progress", + KbScanProgressEvent { + kb_id: kb.id.clone(), + source_path: source_path.clone(), + status: "scanning".into(), + current, + total, + imported, + failed, + ignored, + chunks, + errors: errors.clone(), + }, + ); } - result + let done = KbScanProgressEvent { + kb_id: kb.id.clone(), + source_path, + status: "done".into(), + current: total, + total, + imported, + failed, + ignored, + chunks, + errors, + }; + if emit_done { + let _ = app.emit("kb://scan-progress", done.clone()); + } + done +} + +/// 处理单个文件：文本提取切块，媒体以文件名为内容登记；返回导入的分块数。 +fn process_scan_file( + db: &rusqlite::Connection, + kb: &xianren_core::KnowledgeBase, + source_id: Option<&str>, + file: &std::path::Path, + name: &str, + category: crate::knowledge::FileCategory, +) -> Result { + let bytes = std::fs::read(file).map_err(|e| format!("读取失败 {e}"))?; + let content = if category == crate::knowledge::FileCategory::Text { + crate::knowledge::extract_text(name, &bytes)? + } else { + // 图片 / 视频 / 音频：以文件名为内容登记，便于按名检索 + name.to_string() + }; + let chunks = if category == crate::knowledge::FileCategory::Text { + kb_db::chunk_text(&content, kb.chunk_size as usize, kb.chunk_overlap as usize) + } else { + vec![content.clone()] + }; + if chunks.is_empty() { + return Err("切分后没有有效分块".into()); + } + let doc = xianren_core::KbDocument { + id: uuid::Uuid::new_v4().to_string(), + kb_id: kb.id.clone(), + name: name.to_string(), + file_type: file + .extension() + .map(|e| e.to_string_lossy().to_lowercase()) + .unwrap_or_default(), + file_size: bytes.len() as i64, + char_count: content.chars().count() as i64, + chunk_count: chunks.len() as i64, + file_path: file.to_string_lossy().to_string(), + source_id: source_id.map(str::to_string), + created_at: String::new(), + updated_at: String::new(), + }; + kb_db::insert_document_with_chunks(db, &doc, &content, &chunks) + .map_err(|e| format!("写入失败 {e}"))?; + Ok(chunks.len()) } #[tauri::command] diff --git a/crates/core/src/knowledge_base.rs b/crates/core/src/knowledge_base.rs index 94858e9..b76f583 100644 --- a/crates/core/src/knowledge_base.rs +++ b/crates/core/src/knowledge_base.rs @@ -145,6 +145,38 @@ pub fn list_documents(db: &Connection, kb_id: &str) -> Result> { Ok(out) } +pub fn count_documents(db: &Connection, kb_id: &str) -> Result { + let n: i64 = db.query_row( + "SELECT COUNT(*) FROM kb_documents WHERE kb_id = ?1", + params![kb_id], + |row| row.get(0), + )?; + Ok(n) +} + +/// 分页查询文档：`page` 从 1 开始，`page_size` 会被限制在 1–100。 +pub fn list_documents_page( + db: &Connection, + kb_id: &str, + page: i64, + page_size: i64, +) -> Result> { + let page = page.max(1); + let page_size = page_size.clamp(1, 100); + let offset = (page - 1) * page_size; + let mut stmt = db.prepare(&format!( + "SELECT {DOC_COLUMNS} FROM kb_documents + WHERE kb_id = ?1 ORDER BY updated_at DESC, created_at DESC + LIMIT ?2 OFFSET ?3" + ))?; + let rows = stmt.query_map(params![kb_id, page_size, offset], row_to_doc)?; + let mut out = Vec::new(); + for row in rows { + out.push(row?); + } + Ok(out) +} + pub fn get_document(db: &Connection, id: &str) -> Result> { let mut stmt = db.prepare(&format!( "SELECT {DOC_COLUMNS} FROM kb_documents WHERE id = ?1" @@ -349,12 +381,14 @@ pub fn search( kb_id: Option<&str>, query: &str, limit: usize, + offset: usize, ) -> Result> { let q = query.trim(); if q.is_empty() { return Ok(Vec::new()); } let limit = limit.clamp(1, 100) as i64; + let offset = offset as i64; if q.chars().count() >= 3 { let fts_q = format!("\"{}\"", q.replace('"', "\"\"")); let result = db.prepare( @@ -363,10 +397,10 @@ pub fn search( JOIN kb_documents d ON d.id = c.document_id JOIN kb_chunks_fts ON kb_chunks_fts.rowid = c.rowid WHERE (?1 IS NULL OR c.kb_id = ?1) AND kb_chunks_fts MATCH ?2 - ORDER BY bm25(kb_chunks_fts) LIMIT ?3", + ORDER BY bm25(kb_chunks_fts) LIMIT ?3 OFFSET ?4", ) .and_then(|mut stmt| { - let rows = stmt.query_map(params![kb_id, fts_q, limit], row_to_hit)?; + let rows = stmt.query_map(params![kb_id, fts_q, limit, offset], row_to_hit)?; rows.collect::, _>>() }); if let Ok(hits) = result { @@ -379,9 +413,9 @@ pub fn search( FROM kb_chunks c JOIN kb_documents d ON d.id = c.document_id WHERE (?1 IS NULL OR c.kb_id = ?1) AND c.content LIKE '%' || ?2 || '%' - ORDER BY c.rowid LIMIT ?3", + ORDER BY c.rowid LIMIT ?3 OFFSET ?4", )?; - let rows = stmt.query_map(params![kb_id, q, limit], row_to_hit)?; + let rows = stmt.query_map(params![kb_id, q, limit, offset], row_to_hit)?; let mut out = Vec::new(); for row in rows { out.push(row?); @@ -389,6 +423,36 @@ pub fn search( Ok(out) } +/// 检索命中总数（与 `search` 使用同一套 FTS / LIKE 回退逻辑）。 +pub fn count_search(db: &Connection, kb_id: Option<&str>, query: &str) -> Result { + let q = query.trim(); + if q.is_empty() { + return Ok(0); + } + if q.chars().count() >= 3 { + let fts_q = format!("\"{}\"", q.replace('"', "\"\"")); + let result = db.query_row( + "SELECT COUNT(*) FROM kb_chunks c + JOIN kb_documents d ON d.id = c.document_id + JOIN kb_chunks_fts ON kb_chunks_fts.rowid = c.rowid + WHERE (?1 IS NULL OR c.kb_id = ?1) AND kb_chunks_fts MATCH ?2", + params![kb_id, fts_q], + |row| row.get::<_, i64>(0), + ); + if let Ok(n) = result { + return Ok(n); + } + } + let n: i64 = db.query_row( + "SELECT COUNT(*) FROM kb_chunks c + JOIN kb_documents d ON d.id = c.document_id + WHERE (?1 IS NULL OR c.kb_id = ?1) AND c.content LIKE '%' || ?2 || '%'", + params![kb_id, q], + |row| row.get(0), + )?; + Ok(n) +} + fn row_to_kb(row: &rusqlite::Row<'_>) -> rusqlite::Result { Ok(KnowledgeBase { id: row.get(0)?, @@ -491,7 +555,7 @@ mod tests { assert_eq!(kbs.len(), 1); assert!(kbs[0].chunk_count >= 1); - let hits = search(&conn, Some("kb1"), "智能体", 10).unwrap(); + let hits = search(&conn, Some("kb1"), "智能体", 10, 0).unwrap(); assert_eq!(hits.len(), 1); assert!(hits[0].content.contains("智能体")); } @@ -515,7 +579,7 @@ mod tests { ) .unwrap(); insert_doc(&conn, "d1", "今天是星期一，天气晴朗，适合出去散步。"); - let hits = search(&conn, Some("kb1"), "散步", 10).unwrap(); + let hits = search(&conn, Some("kb1"), "散步", 10, 0).unwrap(); assert_eq!(hits.len(), 1); } @@ -540,7 +604,7 @@ mod tests { insert_doc(&conn, "d1", "知识库测试内容，用于验证删除级联。"); delete_document(&conn, "d1").unwrap(); assert!(get_document(&conn, "d1").unwrap().is_none()); - let hits = search(&conn, Some("kb1"), "知识库", 10).unwrap(); + let hits = search(&conn, Some("kb1"), "知识库", 10, 0).unwrap(); assert!(hits.is_empty()); } @@ -569,6 +633,6 @@ mod tests { let after = get_document(&conn, "d1").unwrap().unwrap().chunk_count; assert_eq!(n as i64, after); assert!(after < before); - assert!(!search(&conn, Some("kb1"), "测试文本", 5).unwrap().is_empty()); + assert!(!search(&conn, Some("kb1"), "测试文本", 5, 0).unwrap().is_empty()); } } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a150bf3..ba0b320 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -110,8 +110,8 @@ ui/src/ | 工作流 | `list_workflows`、`add_workflow`、`update_workflow`、`remove_workflow`、`set_workflow_enabled`、`restore_preset_workflows`、`run_workflow` | | 定时计划 | `list_scheduled_tasks`、`add_scheduled_task`、`update_scheduled_task`、`remove_scheduled_task`、`set_scheduled_task_enabled`、`run_scheduled_task_now` | | 邮件 | `mail_test`（用当前 SMTP 配置发送测试邮件） | -| 知识库 | `list_knowledge_bases`、`add_knowledge_base`、`update_knowledge_base`、`remove_knowledge_base`、`list_kb_documents`、`get_kb_document`、`kb_import_documents`、`remove_kb_document`、`kb_rechunk_document`、`kb_search` | -| 知识库·目录来源 | `kb_list_sources`、`kb_add_source`、`kb_scan_sources`、`kb_remove_source` | +| 知识库 | `list_knowledge_bases`、`add_knowledge_base`、`update_knowledge_base`、`remove_knowledge_base`、`list_kb_documents`（分页）、`get_kb_document`、`kb_import_documents`、`remove_kb_document`、`kb_rechunk_document`、`kb_search`（分页） | +| 知识库·目录来源 | `kb_list_sources`、`kb_add_source`、`kb_scan_sources`（后台扫描，`kb://scan-progress` 事件推送进度）、`kb_remove_source` | | 应用/设置 | `app_info`、`autostart_status`、`autostart_set`、`settings_get`、`settings_set` | | 模型 | `list_models`、`import_model`、`remove_model`、`set_model_enabled`、`scan_models`、`add_remote_model` | | 模型广场 | `search_models`、`list_model_files`、`list_recommended_models`、`import_recommendations`、`fetch_model_page` | @@ -144,7 +144,7 @@ ui/src/ - `send_task_email` / `load_smtp_config` / `split_recipients`：读取设置页「邮件」SMTP 配置、拆分多收件人并调用 `mail.rs` 发送。 - `kb_import_documents`：base64 解码 → `knowledge.rs::extract_text` 提取文本（文本类直接 UTF-8，PDF 走 pdf-extract）→ `kb_db::chunk_text` 切块（按知识库 chunk_size/overlap）→ `insert_document_with_chunks` 事务写入文档与分块，逐文件返回成功/失败结果。 - `kb_search` / `kb_rechunk_document`：检索走 `kb_db::search`（FTS5 trigram + BM25，短查询回退 LIKE）；重切按知识库最新分块设置重建该文档分块与索引。 -- `kb_add_source` / `kb_scan_sources` / `kb_remove_source`：目录来源注册与扫描——`knowledge.rs::collect_files` 按后缀过滤（空=全部）与递归开关收集文件，`classify` 分成文本 / 图片视频 / 音频三类（其他忽略），文本提取切块入库、媒体以文件名为内容登记，`kb_db::document_exists` 按（kb_id + file_path）去重；删除来源级联删除其导入文档。 +- `kb_add_source` / `kb_scan_sources` / `kb_remove_source`：目录来源注册与后台扫描——`knowledge.rs::collect_files` 按后缀过滤（空=全部）与递归开关收集文件，`classify` 分成文本 / 图片视频 / 音频三类（其他忽略），文本提取切块入库、媒体以文件名为内容登记，`kb_db::document_exists` 按（kb_id + file_path）去重；扫描过程逐文件推送 `kb://scan-progress`（处理数/成功/失败/忽略/分块），多目录重扫最后推送一次汇总 done；删除来源级联删除其导入文档。 **邮件发送模块（`apps/desktop/src/mail.rs`）：** @@ -222,7 +222,7 @@ ui/src/ ### 5.9 `knowledge_base.rs` -知识库 CRUD（`list/get/insert/update/delete`）、文档 CRUD（`list/get/get_detail`）、`insert_document_with_chunks`（文档 + 分块同事务写入，FTS 触发器自动建索引）、`delete_document`（级联删除分块并同步索引）、`document_exists`（目录导入按文件路径去重）、`rechunk_document`、`chunk_text`（按字符切块 + 重叠）、`search`（FTS5 trigram BM25 排序，查询 <3 字或 MATCH 失败回退 LIKE）；目录来源 `list/get/upsert/delete_sources` 与 `delete_docs_by_source`。 +知识库 CRUD（`list/get/insert/update/delete`）、文档 CRUD（`list/get/get_detail`、`count_documents` / `list_documents_page` 分页）、`insert_document_with_chunks`（文档 + 分块同事务写入，FTS 触发器自动建索引）、`delete_document`（级联删除分块并同步索引）、`document_exists`（目录导入按文件路径去重）、`rechunk_document`、`chunk_text`（按字符切块 + 重叠）、`search` / `count_search`（FTS5 trigram BM25 排序 + 总数统计，查询 <3 字或 MATCH 失败回退 LIKE）；目录来源 `list/get/upsert/delete_sources` 与 `delete_docs_by_source`。 --- @@ -269,6 +269,7 @@ ui/src/ | `workflow://node-status` | 后端→前端 | 工作流节点运行状态（running/done/error）与输出文本 | | `scheduled://updated` | 后端→前端 | 定时计划状态变化（创建/编辑/启停/执行完成），前端刷新列表 | | `conversations://updated` | 后端→前端 | 定时计划执行后会话列表变化（供智能体会话页刷新） | +| `kb://scan-progress` | 后端→前端 | 知识库目录扫描进度（处理/成功/失败/忽略/分块数；status=scanning/done） | --- diff --git a/docs/FEATURES.md b/docs/FEATURES.md index faf518d..7d83fe2 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -139,8 +139,10 @@ - **递归子目录**：可开关是否递归扫描所有子目录； - **类型过滤**：只收录**可读文本**（提取内容并分块检索）、**图片 / 视频**、**音频**三类文件（图片视频音频以文件名为内容登记，可按文件名检索），检测到其他类型自动忽略； - 同一目录重复添加自动更新配置不产生重复来源；按文件路径去重，可一键「重新扫描」增量导入新增文件；删除目录来源会同时删除它导入的文档。 -- **文档操作**：文档列表展示类型 / 大小 / 字数 / 分块数；支持全文预览、「重切」（按最新分块设置重新切分，适用于修改设置后）与删除。 -- **检索**：全文检索基于 SQLite FTS5（trigram 分词，对中文友好），按 BM25 相关性排序；支持单库检索（也可全库检索），结果展示来源文档与分块序号，可一键复制分块内容用于对话。 + - **扫描进度**：添加目录 / 重新扫描时显示进度条与处理统计（处理 X / Y 个文件、成功、失败、忽略、分块数），失败文件在结尾汇总提示。 +- **文档操作**：文档列表每页 10 条，带分页控件；展示类型 / 大小 / 字数 / 分块数；支持全文预览、「重切」（按最新分块设置重新切分，适用于修改设置后）与删除。 +- **检索**：全文检索基于 SQLite FTS5（trigram 分词，对中文友好），按 BM25 相关性排序；支持单库检索（也可全库检索），结果每页 10 条带分页控件，展示来源文档与分块序号，可一键复制分块内容用于对话。 +- **页面布局**：知识库主区内自上而下依次为「目录来源」→「文档列表」→「检索」。 - 数据表：`knowledge_bases`、`kb_documents`（含提取后的纯文本）、`kb_chunks`（分块）、`kb_chunks_fts`（FTS5 全文索引，contentless-delete 模式 + 触发器同步）。 ## 6. 模型管理页（`/`） @@ -230,6 +232,7 @@ ### 2026-08-17 +- 知识库页调整：文档列表与检索结果增加分页控件；目录来源、文档列表区块移到检索上方；目录扫描增加进度条与处理统计（处理/成功/失败/忽略/分块数，事件驱动实时刷新）。 - 修复：知识库「添加目录 → 浏览…」无响应——为 dialog 插件在 capabilities 中补充权限（`dialog:default`），并让选择器异常在前端可见。 - 知识库「添加目录」弹窗支持两种方式选择目录：手动输入路径，或点击「浏览…」调起系统原生文件夹选择器（接入 tauri-plugin-dialog）。 - 知识库新增「目录来源」：支持添加多个本地目录批量导入（后缀过滤默认全部、可开关递归子目录），只收录文本 / 图片视频 / 音频三类文件其余忽略，支持重新扫描增量导入与按来源删除。 diff --git a/ui/src/api.ts b/ui/src/api.ts index 9a67fbd..a1e7d7a 100644 --- a/ui/src/api.ts +++ b/ui/src/api.ts @@ -261,9 +261,30 @@ export interface KbSourceInput { recursive: boolean; } -export interface KbSourceScanResult { +export interface KbDocumentPage { + items: KbDocument[]; + total: number; + page: number; + page_size: number; +} + +export interface KbSearchPage { + items: KbSearchHit[]; + total: number; + page: number; + page_size: number; +} + +export interface KbScanProgressEvent { + kb_id: string; + source_path: string; + status: string; + current: number; + total: number; imported: number; + failed: number; ignored: number; + chunks: number; errors: string[]; } @@ -491,8 +512,12 @@ export const api = { invoke("update_knowledge_base", { id, input }), removeKnowledgeBase: (id: string) => invoke("remove_knowledge_base", { id }), - listKbDocuments: (kbId: string) => - invoke("list_kb_documents", { kbId }), + listKbDocuments: (kbId: string, page?: number, pageSize?: number) => + invoke("list_kb_documents", { + kbId, + page: page ?? 1, + pageSize: pageSize ?? 10, + }), getKbDocument: (documentId: string) => invoke("get_kb_document", { documentId }), kbImportDocuments: ( @@ -503,13 +528,18 @@ export const api = { invoke("remove_kb_document", { documentId }), kbRechunkDocument: (documentId: string) => invoke("kb_rechunk_document", { documentId }), - kbSearch: (kbId: string | null, query: string, limit?: number) => - invoke("kb_search", { kbId, query, limit: limit ?? 10 }), + kbSearch: (kbId: string | null, query: string, page?: number, pageSize?: number) => + invoke("kb_search", { + kbId, + query, + page: page ?? 1, + pageSize: pageSize ?? 10, + }), kbListSources: (kbId: string) => invoke("kb_list_sources", { kbId }), kbAddSource: (input: KbSourceInput) => - invoke("kb_add_source", { input }), + invoke("kb_add_source", { input }), kbScanSources: (kbId: string) => - invoke("kb_scan_sources", { kbId }), + invoke("kb_scan_sources", { kbId }), kbRemoveSource: (sourceId: string) => invoke("kb_remove_source", { sourceId }), listModels: () => invoke("list_models"), diff --git a/ui/src/pages/KnowledgeBasePage.tsx b/ui/src/pages/KnowledgeBasePage.tsx index 247e301..5a87b81 100644 --- a/ui/src/pages/KnowledgeBasePage.tsx +++ b/ui/src/pages/KnowledgeBasePage.tsx @@ -4,14 +4,18 @@ import { api, KbDocument, KbDocumentDetail, + KbScanProgressEvent, KbSearchHit, KbSource, - KbSourceScanResult, KnowledgeBase, KnowledgeBaseInput, + onEvent, } from "../api"; import Icon from "../components/Icon"; +const DOC_PAGE_SIZE = 10; +const SEARCH_PAGE_SIZE = 10; + function fmtSize(bytes: number) { if (bytes >= 1024 * 1024) return `${(bytes / 1024 / 1024).toFixed(1)} MB`; if (bytes >= 1024) return `${(bytes / 1024).toFixed(0)} KB`; @@ -48,6 +52,8 @@ export default function KnowledgeBasePage() { const [kbs, setKbs] = useState([]); const [selectedId, setSelectedId] = useState(null); const [docs, setDocs] = useState([]); + const [docTotal, setDocTotal] = useState(0); + const [docPage, setDocPage] = useState(1); const [msg, setMsg] = useState(null); const [busy, setBusy] = useState(false); const [kbModal, setKbModal] = useState<{ @@ -59,10 +65,16 @@ export default function KnowledgeBasePage() { const [query, setQuery] = useState(""); const [searching, setSearching] = useState(false); const [searchResults, setSearchResults] = useState(null); + const [searchTotal, setSearchTotal] = useState(0); + const [searchPage, setSearchPage] = useState(1); const [sources, setSources] = useState([]); const [sourceModal, setSourceModal] = useState(false); const [sourceMsg, setSourceMsg] = useState(null); + const [scanning, setScanning] = useState(false); + const [scanProgress, setScanProgress] = useState(null); const uploadInputRef = useRef(null); + const selectedIdRef = useRef(null); + const docPageRef = useRef(1); const selected = useMemo( () => kbs.find((k) => k.id === selectedId) ?? null, @@ -79,27 +91,73 @@ export default function KnowledgeBasePage() { } } + async function loadDocPage(kbId: string, page: number) { + const p = await api.listKbDocuments(kbId, page, DOC_PAGE_SIZE); + if (p.items.length === 0 && p.page > 1) { + const prev = await api.listKbDocuments(kbId, p.page - 1, DOC_PAGE_SIZE); + setDocs(prev.items); + setDocTotal(prev.total); + setDocPage(prev.page); + docPageRef.current = prev.page; + return; + } + setDocs(p.items); + setDocTotal(p.total); + setDocPage(p.page); + docPageRef.current = p.page; + } + useEffect(() => { refreshKbs().catch((e) => setMsg(String(e))); // eslint-disable-next-line react-hooks/exhaustive-deps }, []); + useEffect(() => { + selectedIdRef.current = selectedId; + }, [selectedId]); + + useEffect(() => { + docPageRef.current = docPage; + }, [docPage]); + useEffect(() => { if (!selectedId) { setDocs([]); + setDocTotal(0); + setDocPage(1); + docPageRef.current = 1; + setSources([]); setSearchResults(null); + setSearchTotal(0); return; } - api - .listKbDocuments(selectedId) - .then(setDocs) - .catch((e) => setMsg(`加载文档失败：${String(e)}`)); api .kbListSources(selectedId) .then(setSources) .catch((e) => setMsg(`加载目录来源失败：${String(e)}`)); + loadDocPage(selectedId, 1).catch((e) => setMsg(`加载文档失败：${String(e)}`)); }, [selectedId]); + useEffect(() => { + const un = onEvent("kb://scan-progress", (e) => { + setScanProgress(e); + setScanning(e.status !== "done"); + if (e.status === "done") { + refreshKbs().catch(() => {}); + if (selectedIdRef.current === e.kb_id) { + api + .kbListSources(e.kb_id) + .then(setSources) + .catch(() => {}); + loadDocPage(e.kb_id, docPageRef.current).catch(() => {}); + } + } + }); + return () => { + un.then((f) => f()); + }; + }, []); + async function handleSaveKb(input: KnowledgeBaseInput) { setBusy(true); try { @@ -121,7 +179,11 @@ export default function KnowledgeBasePage() { } async function handleDeleteKb(kb: KnowledgeBase) { - if (!confirm(`确定删除知识库「${kb.name}」吗？将同时删除其中 ${kb.doc_count} 个文档与 ${kb.chunk_count} 个分块。`)) { + if ( + !confirm( + `确定删除知识库「${kb.name}」吗？将同时删除其中 ${kb.doc_count} 个文档与 ${kb.chunk_count} 个分块。`, + ) + ) { return; } try { @@ -147,10 +209,14 @@ export default function KnowledgeBasePage() { const okCount = results.filter((r) => r.ok).length; const failed = results.filter((r) => !r.ok); setUploadMsg( - `导入完成：成功 ${okCount} 个${failed.length > 0 ? `，失败 ${failed.length} 个（${failed.map((f) => `${f.name}: ${f.error}`).join("；")}）` : ""}`, + `导入完成：成功 ${okCount} 个${ + failed.length > 0 + ? `，失败 ${failed.length} 个（${failed.map((f) => `${f.name}: ${f.error}`).join("；")}）` + : "" + }`, ); await refreshKbs(); - setDocs(await api.listKbDocuments(selected.id)); + await loadDocPage(selected.id, docPageRef.current); } catch (e) { setUploadMsg(`导入失败：${String(e)}`); } finally { @@ -165,7 +231,7 @@ export default function KnowledgeBasePage() { const n = await api.kbRechunkDocument(doc.id); setMsg(`已重新切分为 ${n} 个分块`); if (selected) { - setDocs(await api.listKbDocuments(selected.id)); + await loadDocPage(selected.id, docPageRef.current); await refreshKbs(); } } catch (e) { @@ -179,7 +245,7 @@ export default function KnowledgeBasePage() { await api.removeKbDocument(doc.id); setMsg("文档已删除"); if (selected) { - setDocs(await api.listKbDocuments(selected.id)); + await loadDocPage(selected.id, docPageRef.current); await refreshKbs(); } } catch (e) { @@ -187,11 +253,14 @@ export default function KnowledgeBasePage() { } } - async function handleSearch() { + async function handleSearch(page = 1) { if (!query.trim()) return; setSearching(true); try { - setSearchResults(await api.kbSearch(selectedId, query.trim(), 20)); + const p = await api.kbSearch(selectedId, query.trim(), page, SEARCH_PAGE_SIZE); + setSearchResults(p.items); + setSearchTotal(p.total); + setSearchPage(p.page); } catch (e) { setMsg(`检索失败：${String(e)}`); } finally { @@ -199,42 +268,23 @@ export default function KnowledgeBasePage() { } } - async function refreshSourcesAndDocs() { - if (!selected) return; - const [d, s] = await Promise.all([ - api.listKbDocuments(selected.id), - api.kbListSources(selected.id), - ]); - setDocs(d); - setSources(s); - await refreshKbs(); - } - - function formatScanMsg(label: string, r: KbSourceScanResult) { - const errPart = - r.errors.length > 0 - ? `，${r.errors.length} 个错误（${r.errors - .slice(0, 3) - .join("；")}${r.errors.length > 3 ? "…" : ""}）` - : ""; - return `${label}完成：导入 ${r.imported} 个，忽略 ${r.ignored} 个${errPart}`; - } - async function handleAddSource(path: string, extensions: string, recursive: boolean) { if (!selected) return; setBusy(true); setSourceMsg(null); + setScanProgress(null); + setScanning(true); try { - const r = await api.kbAddSource({ + await api.kbAddSource({ kb_id: selected.id, path, extensions, recursive, }); - await refreshSourcesAndDocs(); - setSourceMsg(formatScanMsg("添加目录", r)); + setSourceMsg("开始扫描目录，请稍候…"); } catch (e) { setSourceMsg(`添加目录失败：${String(e)}`); + setScanning(false); } finally { setBusy(false); setSourceModal(false); @@ -245,12 +295,13 @@ export default function KnowledgeBasePage() { if (!selected) return; setBusy(true); setSourceMsg(null); + setScanProgress(null); + setScanning(true); try { - const r = await api.kbScanSources(selected.id); - await refreshSourcesAndDocs(); - setSourceMsg(formatScanMsg("重新扫描", r)); + await api.kbScanSources(selected.id); } catch (e) { setSourceMsg(`重新扫描失败：${String(e)}`); + setScanning(false); } finally { setBusy(false); } @@ -262,7 +313,9 @@ export default function KnowledgeBasePage() { const n = await api.kbRemoveSource(src.id); setSourceMsg(`已删除目录来源及其 ${n} 个文档`); if (selected) { - await refreshSourcesAndDocs(); + setSources(await api.kbListSources(selected.id)); + await loadDocPage(selected.id, docPageRef.current); + await refreshKbs(); } } catch (e) { setSourceMsg(`删除失败：${String(e)}`); @@ -388,41 +441,6 @@ export default function KnowledgeBasePage() { ) : null} - {/* 检索 */} -
-
检索知识库
-
- setQuery(e.target.value)} - onKeyDown={(e) => { - if (e.key === "Enter") handleSearch(); - }} - /> - -
- {searchResults ? ( - searchResults.length === 0 ? ( -
没有匹配的内容
- ) : ( -
- {searchResults.map((hit) => ( - - ))} -
- ) - ) : null} -
- {/* 目录来源 */}
@@ -433,7 +451,7 @@ export default function KnowledgeBasePage() { @@ -454,6 +473,40 @@ export default function KnowledgeBasePage() { {sourceMsg}
) : null} + {scanProgress ? ( +
+
+ {scanProgress.source_path} + + {scanning + ? `处理中 ${scanProgress.current}/${scanProgress.total}` + : "扫描完成"} + +
+
+
0 + ? (scanProgress.current / scanProgress.total) * 100 + : 0 + }%`, + }} + /> +
+
+ 处理 {scanProgress.current} / {scanProgress.total} 个文件 · 成功{" "} + {scanProgress.imported} · 失败 {scanProgress.failed} · 忽略{" "} + {scanProgress.ignored} · 分块 {scanProgress.chunks} + {scanProgress.errors.length > 0 + ? `（${scanProgress.errors[0]}${ + scanProgress.errors.length > 1 ? " 等" : "" + }）` + : ""} +
+
+ ) : null} {sources.length === 0 ? (
还没有目录来源，点击「添加目录」批量导入文件夹中的文档 @@ -499,30 +552,84 @@ export default function KnowledgeBasePage() {
{/* 文档列表 */} -
+
- 文档（{docs.length}） + 文档（共 {docTotal} 个）
{docs.length === 0 ? (
- 还没有文档，点击右上角「上传文档」导入 txt / md / PDF 等文件 + 还没有文档，可在上方「目录来源」批量导入，或点击右上角「上传文档」导入 txt / md / PDF 等文件
) : ( -
- {docs.map((doc) => ( - { - api.getKbDocument(doc.id).then(setPreview).catch((e) => setMsg(String(e))); - }} - onRechunk={() => handleRechunk(doc)} - onDelete={() => handleDeleteDoc(doc)} - /> - ))} -
+ <> +
+ {docs.map((doc) => ( + { + api + .getKbDocument(doc.id) + .then(setPreview) + .catch((e) => setMsg(String(e))); + }} + onRechunk={() => handleRechunk(doc)} + onDelete={() => handleDeleteDoc(doc)} + /> + ))} +
+ selected && loadDocPage(selected.id, p)} + /> + )}
+ + {/* 检索 */} +
+
检索知识库
+
+ setQuery(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") handleSearch(1); + }} + /> + +
+ {searchResults ? ( + searchResults.length === 0 ? ( +
没有匹配的内容
+ ) : ( +
+
+ {searchResults.map((hit) => ( + + ))} +
+ handleSearch(p)} + /> +
+ ) + ) : null} +
)} @@ -571,6 +678,41 @@ export default function KnowledgeBasePage() { ); } +function Pager({ + page, + total, + pageSize, + onChange, +}: { + page: number; + total: number; + pageSize: number; + onChange: (p: number) => void; +}) { + const pages = Math.max(1, Math.ceil(total / pageSize)); + return ( +
+ + + 第 {page} / {pages} 页 · 共 {total} 条 + + +
+ ); +} + function SearchHitRow({ hit }: { hit: KbSearchHit }) { const [copied, setCopied] = useState(false); return ( @@ -845,10 +987,7 @@ function SourceModal({ />
- setRecursive(v)} - /> + setRecursive(v)} /> {recursive ? "递归扫描所有子目录" : "仅扫描当前目录"} @@ -866,7 +1005,7 @@ function SourceModal({ disabled={busy || !valid} onClick={() => onSave(path.trim(), extensions.trim(), recursive)} > - {busy ? "扫描中…" : "添加并扫描"} + {busy ? "添加中…" : "添加并扫描"}