From dc767d83f91712fbc7b36d072b6467d2406a12a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 5 Aug 2026 15:35:55 +0800 Subject: [PATCH] 1 --- kb/controller/document_controller.go | 13 ++- kb/dao/document_dao.go | 11 +++ kb/model/dto/document_dto.go | 13 ++- kb/model/entity/document.go | 1 + kb/service/chunk_service.go | 54 ++++++++--- kb/service/document_service.go | 48 +++++++--- kb/service/parse_task_service.go | 6 +- ui-src/src/api/document.js | 4 + ui-src/src/views/DatasetDetail.vue | 134 ++++++++++++++++++++++----- ui-src/src/views/DatasetList.vue | 4 +- ui-src/src/views/Settings.vue | 4 +- 11 files changed, 239 insertions(+), 53 deletions(-) diff --git a/kb/controller/document_controller.go b/kb/controller/document_controller.go index 62eb1bf..4a20509 100644 --- a/kb/controller/document_controller.go +++ b/kb/controller/document_controller.go @@ -47,6 +47,14 @@ func (c *document) List(ctx context.Context, req *dto.ListDocumentReq) (*dto.Lis }, nil } +func (c *document) Detail(ctx context.Context, req *dto.GetDocumentDetailReq) (res *dto.GetDocumentDetailRes, err error) { + doc, err := service.DocumentService.Detail(ctx, req.Id) + if err != nil { + return nil, err + } + return &dto.GetDocumentDetailRes{Document: doc}, nil +} + func (c *document) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (*dto.DeleteDocumentRes, error) { if err := service.DocumentService.Delete(ctx, req.Id); err != nil { return nil, err @@ -55,8 +63,9 @@ func (c *document) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (*dto } func (c *document) Reembed(ctx context.Context, req *dto.ReembedDocumentReq) (*dto.ReembedDocumentRes, error) { - if err := service.DocumentService.Reembed(ctx, req.Id); err != nil { + n, err := service.DocumentService.Reembed(ctx, req.Id) + if err != nil { return nil, err } - return &dto.ReembedDocumentRes{}, nil + return &dto.ReembedDocumentRes{Count: n}, nil } diff --git a/kb/dao/document_dao.go b/kb/dao/document_dao.go index 2afbd83..cd60d0c 100644 --- a/kb/dao/document_dao.go +++ b/kb/dao/document_dao.go @@ -28,12 +28,22 @@ func init() { status INTEGER NOT NULL DEFAULT 0, chunk_count INTEGER NOT NULL DEFAULT 0, error_msg TEXT NOT NULL DEFAULT '', + content TEXT NOT NULL DEFAULT '', created_at DATETIME DEFAULT (datetime('now','localtime')), updated_at DATETIME DEFAULT (datetime('now','localtime')) )`) if err != nil { g.Log().Warningf(ctx, "create kb_document table failed: %v", err) } + // 迁移：旧库补 content 列（原始全文，抽屉展示用） + cnt, err := g.DB(consts.DbGroupDefault).Ctx(ctx).GetValue(ctx, + "SELECT COUNT(*) FROM pragma_table_info('"+consts.TableNameDocument+"') WHERE name=?", "content") + if err == nil && cnt.Int64() == 0 { + if _, err := g.DB(consts.DbGroupDefault).Exec(ctx, + "ALTER TABLE "+consts.TableNameDocument+" ADD COLUMN content TEXT NOT NULL DEFAULT ''"); err != nil { + g.Log().Warningf(ctx, "alter kb_document add content failed: %v", err) + } + } if _, err := g.DB(consts.DbGroupDefault).Exec(ctx, "CREATE INDEX IF NOT EXISTS idx_kb_document_dataset ON "+consts.TableNameDocument+"(dataset_id)"); err != nil { g.Log().Warningf(ctx, "create index idx_kb_document_dataset failed: %v", err) } @@ -64,6 +74,7 @@ func (d *documentDao) List(ctx context.Context, datasetId int64, page, pageSize } var list []*entity.Document err = g.DB(consts.DbGroupDefault).Model(consts.TableNameDocument).Ctx(ctx). + FieldsEx("content"). // 列表不返回全文，详情接口单独取 Where("dataset_id", datasetId).Page(page, pageSize).OrderDesc("id").Scan(&list) if list == nil { list = make([]*entity.Document, 0) diff --git a/kb/model/dto/document_dto.go b/kb/model/dto/document_dto.go index 9f1937e..732246c 100644 --- a/kb/model/dto/document_dto.go +++ b/kb/model/dto/document_dto.go @@ -31,6 +31,15 @@ type ListDocumentRes struct { PageSize int `json:"page_size"` } +type GetDocumentDetailReq struct { + g.Meta `path:"/detail" method:"get" tags:"文档" summary:"文档详情（含原始全文）"` + Id int64 `v:"required" json:"id"` +} + +type GetDocumentDetailRes struct { + Document *entity.Document `json:"document"` +} + type DeleteDocumentReq struct { g.Meta `path:"/delete" method:"post" tags:"文档" summary:"删除文档"` Id int64 `v:"required" json:"id"` @@ -43,4 +52,6 @@ type ReembedDocumentReq struct { Id int64 `v:"required" json:"id"` } -type ReembedDocumentRes struct{} +type ReembedDocumentRes struct { + Count int `json:"count"` +} diff --git a/kb/model/entity/document.go b/kb/model/entity/document.go index 126a647..d29745f 100644 --- a/kb/model/entity/document.go +++ b/kb/model/entity/document.go @@ -10,6 +10,7 @@ type Document struct { FileSize int64 `orm:"file_size" json:"file_size"` FileType string `orm:"file_type" json:"file_type"` Status int `orm:"status" json:"status"` + Content string `orm:"content" json:"content"` ChunkCount int `orm:"chunk_count" json:"chunk_count"` ErrorMsg string `orm:"error_msg" json:"error_msg"` CreatedAt *gtime.Time `orm:"created_at" json:"created_at"` diff --git a/kb/service/chunk_service.go b/kb/service/chunk_service.go index 8fc538d..6ed4cc7 100644 --- a/kb/service/chunk_service.go +++ b/kb/service/chunk_service.go @@ -28,17 +28,7 @@ var runeLen = utf8.RuneCountInString func (s *chunkService) SplitByStrategy(ctx context.Context, strategy, text string, chunkSize, overlap int, embedder eembedding.Embedder) ([]string, error) { switch strategy { case consts.ChunkStrategyRecursive: - sp, err := recursive.NewSplitter(ctx, &recursive.Config{ - ChunkSize: chunkSize, - OverlapSize: overlap, - Separators: chunkSeparators, - LenFunc: runeLen, - KeepType: recursive.KeepTypeEnd, - }) - if err != nil { - return nil, gerror.Wrap(err, "构建递归分块器失败") - } - return s.transformText(ctx, sp, text) + return s.splitRecursive(ctx, text, chunkSize, overlap) case consts.ChunkStrategySemantic: if embedder == nil { return nil, gerror.New("语义分块需要向量模型") @@ -53,12 +43,52 @@ func (s *chunkService) SplitByStrategy(ctx context.Context, strategy, text strin if err != nil { return nil, gerror.Wrap(err, "构建语义分块器失败") } - return s.transformText(ctx, sp, text) + chunks, err := s.transformText(ctx, sp, text) + if err != nil { + return nil, err + } + // chunkSize 作为安全上限：语义分块可能产出巨块（长连续语义段），超限块用递归二次切分兜底（不重叠） + return s.capChunks(ctx, chunks, chunkSize) default: return s.SplitText(text, chunkSize, overlap), nil } } +// splitRecursive 递归字符分块：按分隔符列表递归切分到目标大小，保留 overlap +func (s *chunkService) splitRecursive(ctx context.Context, text string, chunkSize, overlap int) ([]string, error) { + sp, err := recursive.NewSplitter(ctx, &recursive.Config{ + ChunkSize: chunkSize, + OverlapSize: overlap, + Separators: chunkSeparators, + LenFunc: runeLen, + KeepType: recursive.KeepTypeEnd, + }) + if err != nil { + return nil, gerror.Wrap(err, "构建递归分块器失败") + } + return s.transformText(ctx, sp, text) +} + +// capChunks 超过 maxSize 的块用递归分块器二次切分，防止语义分块产出巨块 +func (s *chunkService) capChunks(ctx context.Context, chunks []string, maxSize int) ([]string, error) { + if maxSize <= 0 { + return chunks, nil + } + out := make([]string, 0, len(chunks)) + for _, c := range chunks { + if runeLen(c) <= maxSize { + out = append(out, c) + continue + } + sub, err := s.splitRecursive(ctx, c, maxSize, 0) + if err != nil { + return nil, err + } + out = append(out, sub...) + } + return out, nil +} + func (s *chunkService) transformText(ctx context.Context, sp document.Transformer, text string) ([]string, error) { docs, err := sp.Transform(ctx, []*schema.Document{{ID: "0", Content: text}}) if err != nil { diff --git a/kb/service/document_service.go b/kb/service/document_service.go index f3e06fe..588f308 100644 --- a/kb/service/document_service.go +++ b/kb/service/document_service.go @@ -15,6 +15,7 @@ import ( "rag-local/kb/model/entity" "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" ) var DocumentService = &documentService{} @@ -81,29 +82,54 @@ func (s *documentService) List(ctx context.Context, datasetId int64, page, pageS return dao.Document.List(ctx, datasetId, page, pageSize) } -// Reembed 文档全部分块重新向量化（分块文本不变，仅重算向量） -func (s *documentService) Reembed(ctx context.Context, id int64) error { +// Detail 文档详情（含原始全文）。存量文档 content 为空时，现场从源文件解析并回填落库 +func (s *documentService) Detail(ctx context.Context, id int64) (*entity.Document, error) { doc, err := dao.Document.GetOne(ctx, id) if err != nil { - return err + return nil, err } if doc == nil { - return gerror.New("文档不存在") + return nil, gerror.New("文档不存在") + } + if doc.Content == "" && doc.FilePath != "" { + text, err := common.ParseFile(filepath.Join("workspace", doc.FilePath)) + if err != nil { + return nil, gerror.Wrap(err, "解析源文件失败") + } + if err := dao.Document.UpdateFields(ctx, id, g.Map{"content": text}); err != nil { + return nil, err + } + doc.Content = text + } + return doc, nil +} + +// Reembed 文档全部分块重新向量化（分块文本不变，仅重算向量），返回处理的分块数 +func (s *documentService) Reembed(ctx context.Context, id int64) (int, error) { + doc, err := dao.Document.GetOne(ctx, id) + if err != nil { + return 0, err + } + if doc == nil { + return 0, gerror.New("文档不存在") } cfgId, err := dao.Dataset.GetEmbeddingCfgId(ctx, doc.DatasetId) if err != nil { - return err + return 0, err } if cfgId <= 0 { - return gerror.New("数据集未绑定向量模型，无法向量化") + return 0, gerror.New("数据集未绑定向量模型，无法向量化") } em, err := BuildEmbedder(ctx, cfgId) if err != nil { - return err + return 0, err } chunks, _, err := dao.Chunk.ListByDocument(ctx, id, 1, 100000) if err != nil { - return err + return 0, err + } + if len(chunks) == 0 { + return 0, gerror.New("该文档无分块（解析失败或分块已删除），无法重新向量化，请先重新解析") } for start := 0; start < len(chunks); start += consts.EmbedBatchSize { end := min(start+consts.EmbedBatchSize, len(chunks)) @@ -113,15 +139,15 @@ func (s *documentService) Reembed(ctx context.Context, id int64) error { } vecs, err := em.EmbedStrings(ctx, texts) if err != nil { - return gerror.Wrap(err, "向量化失败") + return 0, gerror.Wrap(err, "向量化失败") } for j, c := range chunks[start:end] { if err := dao.Chunk.UpdateVec(ctx, c.Id, domain.VecJsonF64(vecs[j])); err != nil { - return err + return 0, err } } } - return nil + return len(chunks), nil } // Delete 删除文档：先删文件与索引数据，再删记录 diff --git a/kb/service/parse_task_service.go b/kb/service/parse_task_service.go index 4a0bfe8..ac33a6a 100644 --- a/kb/service/parse_task_service.go +++ b/kb/service/parse_task_service.go @@ -99,6 +99,10 @@ func (s *parseTaskService) processOne(ctx context.Context) { s.fail(ctx, task, "清理旧分块失败: "+err.Error()) return } + // 原始全文落库（分块抽屉展示用），失败不阻断流水线 + if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"content": text}); err != nil { + g.Log().Warningf(ctx, "save document content failed: %v", err) + } chunks, err := ChunkService.SplitByStrategy(ctx, strategy, text, chunkSize, chunkOverlap, embedder) if err != nil { s.fail(ctx, task, "分块失败: "+err.Error()) @@ -119,7 +123,7 @@ func (s *parseTaskService) processOne(ctx context.Context) { // processReembed 重新向量化任务：分块文本不变，用数据集当前绑定模型重算全部向量；失败不动文档状态 func (s *parseTaskService) processReembed(ctx context.Context, task *entity.ParseTask) { - if err := DocumentService.Reembed(ctx, task.DocumentId); err != nil { + if _, err := DocumentService.Reembed(ctx, task.DocumentId); err != nil { _ = dao.ParseTask.UpdateStatus(ctx, task.Id, consts.TaskStatusFailed, "重新向量化失败: "+err.Error()) g.Log().Errorf(ctx, "reembed task %d failed: %s", task.Id, err.Error()) return diff --git a/ui-src/src/api/document.js b/ui-src/src/api/document.js index dfee1c8..7e96b02 100644 --- a/ui-src/src/api/document.js +++ b/ui-src/src/api/document.js @@ -12,6 +12,10 @@ export function deleteDocument(id) { return request.post('/document/delete', { id }) } +export function getDocumentDetail(id) { + return request.get('/document/detail', { params: { id } }) +} + export function reembedDocument(id) { return request.post('/document/reembed', { id }) } diff --git a/ui-src/src/views/DatasetDetail.vue b/ui-src/src/views/DatasetDetail.vue index 7e51d4b..396de11 100644 --- a/ui-src/src/views/DatasetDetail.vue +++ b/ui-src/src/views/DatasetDetail.vue @@ -39,25 +39,38 @@ - -
- + +
+
+
原始内容（{{ rawContent.length }} 字）
+
+ 
{{ rawContent }}
+
{{ rawEmptyTip }}
+
+
+
+
+ +
+
+ + + + + + + + + +
+ +
- - - - - - - - - - @@ -79,6 +92,7 @@ import { ArrowLeft, UploadFilled } from '@element-plus/icons-vue' import { listDatasets } from '../api/dataset.js' import { listDocuments, uploadDocument, deleteDocument, reembedDocument, retryParseTask } from '../api/document.js' import { listChunks, updateChunk } from '../api/chunk.js' +import { getDocumentDetail } from '../api/document.js' const route = useRoute() const datasetId = computed(() => Number(route.params.id)) @@ -93,6 +107,9 @@ const uploading = ref(false) const chunkDrawer = ref(false) const chunkDrawerTitle = ref('') +const rawContent = ref('') +const rawLoading = ref(false) +const rawEmptyTip = ref('') const chunks = ref([]) const chunkTotal = ref(0) const chunkPage = ref(1) @@ -199,8 +216,8 @@ async function reembed(row) { } catch { return } - await reembedDocument(row.id) - ElMessage.success('重新向量化完成') + const d = await reembedDocument(row.id) + ElMessage.success(`已重新向量化 ${d.count} 个分块`) } async function openChunks(row) { @@ -209,9 +226,27 @@ async function openChunks(row) { chunkDrawer.value = true chunkPage.value = 1 chunkKeyword.value = '' + rawContent.value = '' + await loadRaw() await loadChunks() } +// 加载文档原始全文（存量文档后端会现场解析回填） +async function loadRaw() { + if (!currentDoc) return + rawLoading.value = true + rawEmptyTip.value = '' + try { + const d = await getDocumentDetail(currentDoc.id) + rawContent.value = (d.document && d.document.content) || '' + if (!rawContent.value) rawEmptyTip.value = '该文档无原始内容' + } catch (e) { + rawEmptyTip.value = '原始内容加载失败：' + (e.message || e) + } finally { + rawLoading.value = false + } +} + async function loadChunks() { if (!currentDoc) return chunkLoading.value = true @@ -265,15 +300,70 @@ async function saveChunk() { margin-top: 10px; justify-content: flex-end; } +.chunk-drawer :deep(.el-drawer__body) { + overflow: hidden; +} +.drawer-split { + display: flex; + gap: 16px; + height: 100%; +} +.drawer-left, +.drawer-right { + display: flex; + flex-direction: column; + min-height: 0; +} +.drawer-left { + flex: 0 0 42%; + border-right: 1px solid #ebeef5; + padding-right: 16px; +} +.drawer-right { + flex: 1; +} +.pane-title { + font-size: 13px; + font-weight: 600; + color: #303133; + margin-bottom: 8px; +} +.raw-scroll { + flex: 1; + overflow: auto; + min-height: 0; +} +.raw-content { + margin: 0; + padding: 10px 12px; + white-space: pre-wrap; + word-break: break-all; + background: #f8f8f8; + border-radius: 4px; + font-size: 13px; + line-height: 1.7; + color: #303133; +} +.raw-empty { + padding: 12px; + font-size: 13px; + color: #909399; +} .chunk-toolbar { margin-bottom: 10px; + flex-shrink: 0; +} +.chunk-table-scroll { + flex: 1; + overflow: auto; + min-height: 0; } .chunk-content { - max-height: 60px; - overflow: hidden; font-size: 13px; color: #606266; line-height: 1.6; + white-space: pre-wrap; + word-break: break-all; } .edit-tip { margin-top: 6px; diff --git a/ui-src/src/views/DatasetList.vue b/ui-src/src/views/DatasetList.vue index 4a10faa..d811d82 100644 --- a/ui-src/src/views/DatasetList.vue +++ b/ui-src/src/views/DatasetList.vue @@ -46,9 +46,9 @@
标题分块保留标题与段落结构；递归字符分块按分隔符切分，通用文本；语义分块按语义相似度切分，质量更高但解析更慢，且需绑定向量模型
- + -
{{ form.chunk_strategy === 'semantic' ? '语义分块按句子相似度切分，此值决定最小分块大小（低于该值的候选块会继续合并）' : '每块最大字数，超长段落自动按句号/换行切分' }}
+
{{ form.chunk_strategy === 'semantic' ? '语义分块按句子相似度切分，此值为安全上限（max_chunk_size），超过上限的块会再切分；语义分块不支持重叠' : '每块最大字数，超长段落自动按句号/换行切分' }}
 diff --git a/ui-src/src/views/Settings.vue b/ui-src/src/views/Settings.vue index 24c81cb..8886df6 100644 --- a/ui-src/src/views/Settings.vue +++ b/ui-src/src/views/Settings.vue @@ -11,8 +11,8 @@ - - + +