red-future/rag-local
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1

Browse Source
This commit is contained in:
admin
2026-08-05 15:35:55 +08:00
parent 20c1805efb
commit dc767d83f9
11 changed files with 239 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kb/controller/document_controller.go
+11 -2
View File
@@ -47,6 +47,14 @@ func (c *document) List(ctx context.Context, req *dto.ListDocumentReq) (*dto.Lis
}, nil
}
func (c *document) Detail(ctx context.Context, req *dto.GetDocumentDetailReq) (res *dto.GetDocumentDetailRes, err error) {
doc, err := service.DocumentService.Detail(ctx, req.Id)
if err != nil {
return nil, err
}
return &dto.GetDocumentDetailRes{Document: doc}, nil
}
func (c *document) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (*dto.DeleteDocumentRes, error) {
if err := service.DocumentService.Delete(ctx, req.Id); err != nil {
return nil, err
@@ -55,8 +63,9 @@ func (c *document) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (*dto
}
func (c *document) Reembed(ctx context.Context, req *dto.ReembedDocumentReq) (*dto.ReembedDocumentRes, error) {
if err := service.DocumentService.Reembed(ctx, req.Id); err != nil {
n, err := service.DocumentService.Reembed(ctx, req.Id)
if err != nil {
return nil, err
}
return &dto.ReembedDocumentRes{}, nil
return &dto.ReembedDocumentRes{Count: n}, nil
}
kb/dao/document_dao.go
+11
View File
@@ -28,12 +28,22 @@ func init() {
status INTEGER NOT NULL DEFAULT 0,
chunk_count INTEGER NOT NULL DEFAULT 0,
error_msg TEXT NOT NULL DEFAULT '',
content TEXT NOT NULL DEFAULT '',
created_at DATETIME DEFAULT (datetime('now','localtime')),
updated_at DATETIME DEFAULT (datetime('now','localtime'))
)`)
if err != nil {
g.Log().Warningf(ctx, "create kb_document table failed: %v", err)
}
// 迁移：旧库补 content 列（原始全文，抽屉展示用）
cnt, err := g.DB(consts.DbGroupDefault).Ctx(ctx).GetValue(ctx,
"SELECT COUNT(*) FROM pragma_table_info('"+consts.TableNameDocument+"') WHERE name=?", "content")
if err == nil && cnt.Int64() == 0 {
if _, err := g.DB(consts.DbGroupDefault).Exec(ctx,
"ALTER TABLE "+consts.TableNameDocument+" ADD COLUMN content TEXT NOT NULL DEFAULT ''"); err != nil {
g.Log().Warningf(ctx, "alter kb_document add content failed: %v", err)
}
}
if _, err := g.DB(consts.DbGroupDefault).Exec(ctx, "CREATE INDEX IF NOT EXISTS idx_kb_document_dataset ON "+consts.TableNameDocument+"(dataset_id)"); err != nil {
g.Log().Warningf(ctx, "create index idx_kb_document_dataset failed: %v", err)
}
@@ -64,6 +74,7 @@ func (d *documentDao) List(ctx context.Context, datasetId int64, page, pageSize
}
var list []*entity.Document
err = g.DB(consts.DbGroupDefault).Model(consts.TableNameDocument).Ctx(ctx).
FieldsEx("content"). // 列表不返回全文，详情接口单独取
Where("dataset_id", datasetId).Page(page, pageSize).OrderDesc("id").Scan(&list)
if list == nil {
list = make([]*entity.Document, 0)
kb/model/dto/document_dto.go
+12 -1
View File
@@ -31,6 +31,15 @@ type ListDocumentRes struct {
PageSize int `json:"page_size"`
}
type GetDocumentDetailReq struct {
g.Meta `path:"/detail" method:"get" tags:"文档" summary:"文档详情（含原始全文）"`
Id int64 `v:"required" json:"id"`
}
type GetDocumentDetailRes struct {
Document *entity.Document `json:"document"`
}
type DeleteDocumentReq struct {
g.Meta `path:"/delete" method:"post" tags:"文档" summary:"删除文档"`
Id int64 `v:"required" json:"id"`
@@ -43,4 +52,6 @@ type ReembedDocumentReq struct {
Id int64 `v:"required" json:"id"`
}
type ReembedDocumentRes struct{}
type ReembedDocumentRes struct {
Count int `json:"count"`
}
kb/model/entity/document.go
+1
View File
@@ -10,6 +10,7 @@ type Document struct {
FileSize int64 `orm:"file_size" json:"file_size"`
FileType string `orm:"file_type" json:"file_type"`
Status int `orm:"status" json:"status"`
Content string `orm:"content" json:"content"`
ChunkCount int `orm:"chunk_count" json:"chunk_count"`
ErrorMsg string `orm:"error_msg" json:"error_msg"`
CreatedAt *gtime.Time `orm:"created_at" json:"created_at"`
kb/service/chunk_service.go
+42 -12
View File
@@ -28,17 +28,7 @@ var runeLen = utf8.RuneCountInString
func (s *chunkService) SplitByStrategy(ctx context.Context, strategy, text string, chunkSize, overlap int, embedder eembedding.Embedder) ([]string, error) {
switch strategy {
case consts.ChunkStrategyRecursive:
sp, err := recursive.NewSplitter(ctx, &recursive.Config{
ChunkSize: chunkSize,
OverlapSize: overlap,
Separators: chunkSeparators,
LenFunc: runeLen,
KeepType: recursive.KeepTypeEnd,
})
if err != nil {
return nil, gerror.Wrap(err, "构建递归分块器失败")
}
return s.transformText(ctx, sp, text)
return s.splitRecursive(ctx, text, chunkSize, overlap)
case consts.ChunkStrategySemantic:
if embedder == nil {
return nil, gerror.New("语义分块需要向量模型")
@@ -53,12 +43,52 @@ func (s *chunkService) SplitByStrategy(ctx context.Context, strategy, text strin
if err != nil {
return nil, gerror.Wrap(err, "构建语义分块器失败")
}
return s.transformText(ctx, sp, text)
chunks, err := s.transformText(ctx, sp, text)
if err != nil {
return nil, err
}
// chunkSize 作为安全上限：语义分块可能产出巨块（长连续语义段），超限块用递归二次切分兜底（不重叠）
return s.capChunks(ctx, chunks, chunkSize)
default:
return s.SplitText(text, chunkSize, overlap), nil
}
}
// splitRecursive 递归字符分块：按分隔符列表递归切分到目标大小，保留 overlap
func (s *chunkService) splitRecursive(ctx context.Context, text string, chunkSize, overlap int) ([]string, error) {
sp, err := recursive.NewSplitter(ctx, &recursive.Config{
ChunkSize: chunkSize,
OverlapSize: overlap,
Separators: chunkSeparators,
LenFunc: runeLen,
KeepType: recursive.KeepTypeEnd,
})
if err != nil {
return nil, gerror.Wrap(err, "构建递归分块器失败")
}
return s.transformText(ctx, sp, text)
}
// capChunks 超过 maxSize 的块用递归分块器二次切分，防止语义分块产出巨块
func (s *chunkService) capChunks(ctx context.Context, chunks []string, maxSize int) ([]string, error) {
if maxSize <= 0 {
return chunks, nil
}
out := make([]string, 0, len(chunks))
for _, c := range chunks {
if runeLen(c) <= maxSize {
out = append(out, c)
continue
}
sub, err := s.splitRecursive(ctx, c, maxSize, 0)
if err != nil {
return nil, err
}
out = append(out, sub...)
}
return out, nil
}
func (s *chunkService) transformText(ctx context.Context, sp document.Transformer, text string) ([]string, error) {
docs, err := sp.Transform(ctx, []*schema.Document{{ID: "0", Content: text}})
if err != nil {
kb/service/document_service.go
+37 -11
View File
@@ -15,6 +15,7 @@ import (
"rag-local/kb/model/entity"
"github.com/gogf/gf/v2/errors/gerror"
"github.com/gogf/gf/v2/frame/g"
)
var DocumentService = &documentService{}
@@ -81,29 +82,54 @@ func (s *documentService) List(ctx context.Context, datasetId int64, page, pageS
return dao.Document.List(ctx, datasetId, page, pageSize)
}
// Reembed 文档全部分块重新向量化（分块文本不变，仅重算向量）
func (s *documentService) Reembed(ctx context.Context, id int64) error {
// Detail 文档详情（含原始全文）。存量文档 content 为空时，现场从源文件解析并回填落库
func (s *documentService) Detail(ctx context.Context, id int64) (*entity.Document, error) {
doc, err := dao.Document.GetOne(ctx, id)
if err != nil {
return err
return nil, err
}
if doc == nil {
return gerror.New("文档不存在")
return nil, gerror.New("文档不存在")
}
if doc.Content == "" && doc.FilePath != "" {
text, err := common.ParseFile(filepath.Join("workspace", doc.FilePath))
if err != nil {
return nil, gerror.Wrap(err, "解析源文件失败")
}
if err := dao.Document.UpdateFields(ctx, id, g.Map{"content": text}); err != nil {
return nil, err
}
doc.Content = text
}
return doc, nil
}
// Reembed 文档全部分块重新向量化（分块文本不变，仅重算向量），返回处理的分块数
func (s *documentService) Reembed(ctx context.Context, id int64) (int, error) {
doc, err := dao.Document.GetOne(ctx, id)
if err != nil {
return 0, err
}
if doc == nil {
return 0, gerror.New("文档不存在")
}
cfgId, err := dao.Dataset.GetEmbeddingCfgId(ctx, doc.DatasetId)
if err != nil {
return err
return 0, err
}
if cfgId <= 0 {
return gerror.New("数据集未绑定向量模型，无法向量化")
return 0, gerror.New("数据集未绑定向量模型，无法向量化")
}
em, err := BuildEmbedder(ctx, cfgId)
if err != nil {
return err
return 0, err
}
chunks, _, err := dao.Chunk.ListByDocument(ctx, id, 1, 100000)
if err != nil {
return err
return 0, err
}
if len(chunks) == 0 {
return 0, gerror.New("该文档无分块（解析失败或分块已删除），无法重新向量化，请先重新解析")
}
for start := 0; start < len(chunks); start += consts.EmbedBatchSize {
end := min(start+consts.EmbedBatchSize, len(chunks))
@@ -113,15 +139,15 @@ func (s *documentService) Reembed(ctx context.Context, id int64) error {
}
vecs, err := em.EmbedStrings(ctx, texts)
if err != nil {
return gerror.Wrap(err, "向量化失败")
return 0, gerror.Wrap(err, "向量化失败")
}
for j, c := range chunks[start:end] {
if err := dao.Chunk.UpdateVec(ctx, c.Id, domain.VecJsonF64(vecs[j])); err != nil {
return err
return 0, err
}
}
}
return nil
return len(chunks), nil
}
// Delete 删除文档：先删文件与索引数据，再删记录
kb/service/parse_task_service.go
+5 -1
View File
@@ -99,6 +99,10 @@ func (s *parseTaskService) processOne(ctx context.Context) {
s.fail(ctx, task, "清理旧分块失败: "+err.Error())
return
}
// 原始全文落库（分块抽屉展示用），失败不阻断流水线
if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"content": text}); err != nil {
g.Log().Warningf(ctx, "save document content failed: %v", err)
}
chunks, err := ChunkService.SplitByStrategy(ctx, strategy, text, chunkSize, chunkOverlap, embedder)
if err != nil {
s.fail(ctx, task, "分块失败: "+err.Error())
@@ -119,7 +123,7 @@ func (s *parseTaskService) processOne(ctx context.Context) {
// processReembed 重新向量化任务：分块文本不变，用数据集当前绑定模型重算全部向量；失败不动文档状态
func (s *parseTaskService) processReembed(ctx context.Context, task *entity.ParseTask) {
if err := DocumentService.Reembed(ctx, task.DocumentId); err != nil {
if _, err := DocumentService.Reembed(ctx, task.DocumentId); err != nil {
_ = dao.ParseTask.UpdateStatus(ctx, task.Id, consts.TaskStatusFailed, "重新向量化失败: "+err.Error())
g.Log().Errorf(ctx, "reembed task %d failed: %s", task.Id, err.Error())
return
ui-src/src/api/document.js
+4
View File
@@ -12,6 +12,10 @@ export function deleteDocument(id) {
return request.post('/document/delete', { id })
}
export function getDocumentDetail(id) {
return request.get('/document/detail', { params: { id } })
}
export function reembedDocument(id) {
return request.post('/document/reembed', { id })
}
ui-src/src/views/DatasetDetail.vue
+112 -22
View File
@@ -39,25 +39,38 @@
<el-pagination class="dd-page-bar" layout="total, prev, pager, next" :total="total"
:page-size="pageSize" v-model:current-page="page" @current-change="load" />
<el-drawer v-model="chunkDrawer" :title="chunkDrawerTitle" size="55%">
<div class="chunk-toolbar">
<el-input v-model="chunkKeyword" placeholder="搜索分块内容" clearable style="width: 240px" @change="loadChunks" />
<el-drawer v-model="chunkDrawer" :title="chunkDrawerTitle" size="70%" class="chunk-drawer">
<div class="drawer-split">
<div class="drawer-left">
<div class="pane-title">原始内容（{{ rawContent.length }} 字）</div>
<div v-loading="rawLoading" class="raw-scroll">
<pre v-if="rawContent" class="raw-content">{{ rawContent }}</pre>
<div v-else-if="!rawLoading" class="raw-empty">{{ rawEmptyTip }}</div>
</div>
</div>
<div class="drawer-right">
<div class="chunk-toolbar">
<el-input v-model="chunkKeyword" placeholder="搜索分块内容" clearable style="width: 240px" @change="loadChunks" />
</div>
<div class="chunk-table-scroll">
<el-table :data="chunks" v-loading="chunkLoading" size="small">
<el-table-column type="index" label="#" width="50" />
<el-table-column label="内容" min-width="300">
<template #default="{ row }">
<div class="chunk-content">{{ row.content }}</div>
</template>
</el-table-column>
<el-table-column label="操作" width="110" fixed="right">
<template #default="{ row }">
<el-button link type="primary" @click="editChunk(row)">编辑</el-button>
</template>
</el-table-column>
</el-table>
</div>
<el-pagination class="dd-page-bar" layout="total, prev, pager, next" :total="chunkTotal"
:page-size="chunkPageSize" v-model:current-page="chunkPage" @current-change="loadChunks" />
</div>
</div>
<el-table :data="chunks" v-loading="chunkLoading" size="small">
<el-table-column type="index" label="#" width="50" />
<el-table-column label="内容" min-width="300">
<template #default="{ row }">
<div class="chunk-content">{{ row.content }}</div>
</template>
</el-table-column>
<el-table-column label="操作" width="110" fixed="right">
<template #default="{ row }">
<el-button link type="primary" @click="editChunk(row)">编辑</el-button>
</template>
</el-table-column>
</el-table>
<el-pagination class="dd-page-bar" layout="total, prev, pager, next" :total="chunkTotal"
:page-size="chunkPageSize" v-model:current-page="chunkPage" @current-change="loadChunks" />
</el-drawer>
<el-dialog v-model="editDialog" title="编辑分块" width="640px">
@@ -79,6 +92,7 @@ import { ArrowLeft, UploadFilled } from '@element-plus/icons-vue'
import { listDatasets } from '../api/dataset.js'
import { listDocuments, uploadDocument, deleteDocument, reembedDocument, retryParseTask } from '../api/document.js'
import { listChunks, updateChunk } from '../api/chunk.js'
import { getDocumentDetail } from '../api/document.js'
const route = useRoute()
const datasetId = computed(() => Number(route.params.id))
@@ -93,6 +107,9 @@ const uploading = ref(false)
const chunkDrawer = ref(false)
const chunkDrawerTitle = ref('')
const rawContent = ref('')
const rawLoading = ref(false)
const rawEmptyTip = ref('')
const chunks = ref([])
const chunkTotal = ref(0)
const chunkPage = ref(1)
@@ -199,8 +216,8 @@ async function reembed(row) {
} catch {
return
}
await reembedDocument(row.id)
ElMessage.success('重新向量化完成')
const d = await reembedDocument(row.id)
ElMessage.success(`已重新向量化 ${d.count} 个分块`)
}
async function openChunks(row) {
@@ -209,9 +226,27 @@ async function openChunks(row) {
chunkDrawer.value = true
chunkPage.value = 1
chunkKeyword.value = ''
rawContent.value = ''
await loadRaw()
await loadChunks()
}
// 加载文档原始全文（存量文档后端会现场解析回填）
async function loadRaw() {
if (!currentDoc) return
rawLoading.value = true
rawEmptyTip.value = ''
try {
const d = await getDocumentDetail(currentDoc.id)
rawContent.value = (d.document && d.document.content) || ''
if (!rawContent.value) rawEmptyTip.value = '该文档无原始内容'
} catch (e) {
rawEmptyTip.value = '原始内容加载失败：' + (e.message || e)
} finally {
rawLoading.value = false
}
}
async function loadChunks() {
if (!currentDoc) return
chunkLoading.value = true
@@ -265,15 +300,70 @@ async function saveChunk() {
margin-top: 10px;
justify-content: flex-end;
}
.chunk-drawer :deep(.el-drawer__body) {
overflow: hidden;
}
.drawer-split {
display: flex;
gap: 16px;
height: 100%;
}
.drawer-left,
.drawer-right {
display: flex;
flex-direction: column;
min-height: 0;
}
.drawer-left {
flex: 0 0 42%;
border-right: 1px solid #ebeef5;
padding-right: 16px;
}
.drawer-right {
flex: 1;
}
.pane-title {
font-size: 13px;
font-weight: 600;
color: #303133;
margin-bottom: 8px;
}
.raw-scroll {
flex: 1;
overflow: auto;
min-height: 0;
}
.raw-content {
margin: 0;
padding: 10px 12px;
white-space: pre-wrap;
word-break: break-all;
background: #f8f8f8;
border-radius: 4px;
font-size: 13px;
line-height: 1.7;
color: #303133;
}
.raw-empty {
padding: 12px;
font-size: 13px;
color: #909399;
}
.chunk-toolbar {
margin-bottom: 10px;
flex-shrink: 0;
}
.chunk-table-scroll {
flex: 1;
overflow: auto;
min-height: 0;
}
.chunk-content {
max-height: 60px;
overflow: hidden;
font-size: 13px;
color: #606266;
line-height: 1.6;
white-space: pre-wrap;
word-break: break-all;
}
.edit-tip {
margin-top: 6px;
ui-src/src/views/DatasetList.vue
+2 -2
View File
@@ -46,9 +46,9 @@
</el-select>
<div class="ds-tip">标题分块保留标题与段落结构；递归字符分块按分隔符切分，通用文本；语义分块按语义相似度切分，质量更高但解析更慢，且需绑定向量模型</div>
</el-form-item>
<el-form-item label="分块大小">
<el-form-item :label="form.chunk_strategy === 'semantic' ? '上限大小' : '分块大小'">
<el-input-number v-model="form.chunk_size" :min="50" :max="5000" :step="50" style="width: 100%" />
<div class="ds-tip">{{ form.chunk_strategy === 'semantic' ? '语义分块按句子相似度切分，此值决定最小分块大小（低于该值的候选块会继续合并）' : '每块最大字数，超长段落自动按句号/换行切分' }}</div>
<div class="ds-tip">{{ form.chunk_strategy === 'semantic' ? '语义分块按句子相似度切分，此值为安全上限（max_chunk_size），超过上限的块会再切分；语义分块不支持重叠' : '每块最大字数，超长段落自动按句号/换行切分' }}</div>
</el-form-item>
<el-form-item v-if="form.chunk_strategy !== 'semantic'" label="重叠字数">
<el-input-number v-model="form.chunk_overlap" :min="0" :max="500" :step="10" style="width: 100%" />
ui-src/src/views/Settings.vue
+2 -2
View File
@@ -11,8 +11,8 @@
<el-table-column prop="name" label="名称" min-width="140" />
<el-table-column prop="model_name" label="模型" min-width="160" show-overflow-tooltip />
<el-table-column prop="endpoint_url" label="接口地址" min-width="220" show-overflow-tooltip />
<el-table-column prop="dimension" label="维度" width="80">
<template #default="{ row }">{{ row.model_type === 'embedding' ? row.dimension : '-' }}</template>
<el-table-column v-if="modelType === 'embedding'" prop="dimension" label="维度" width="80">
<template #default="{ row }">{{ row.dimension }}</template>
</el-table-column>
<el-table-column label="默认" width="110" align="center">
<template #default="{ row }">