This commit is contained in:
2026-08-06 10:36:12 +08:00
parent 74ecb6d277
commit 46f034d628
7 changed files with 44 additions and 20 deletions
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
+6 -4
View File
@@ -2,10 +2,12 @@ package consts
// 文档状态
const (
DocumentStatusPending = 0 // 待处理
DocumentStatusParsing = 1 // 处理中
DocumentStatusDone = 2 // 完成
DocumentStatusFailed = 3 // 失败
DocumentStatusPending = 0 // 待处理
DocumentStatusParsing = 1 // 解析中(读取文件 + 分块)
DocumentStatusEmbedding = 2 // 向量生成中
DocumentStatusKgBuilding = 3 // 知识图谱构建中
DocumentStatusDone = 4 // 已完成
DocumentStatusFailed = 5 // 失败
)
// 解析任务状态
-1
View File
@@ -409,7 +409,6 @@ func (s *chunkService) InsertAll(ctx context.Context, datasetId, documentId int6
}
return dao.Document.UpdateFields(ctx, documentId, g.Map{
"chunk_count": len(chunks),
"status": consts.DocumentStatusDone,
})
}
+7 -6
View File
@@ -35,23 +35,24 @@ type kgExtractResult struct {
}
// ExtractDocument 对文档全部新分块做 LLM 抽取(挂在解析流水线分块落库之后)。
// 每个分块一次调用,任何失败只记日志,不阻断解析流水线;未配置默认对话模型时直接跳过
func (s *kgEntityService) ExtractDocument(ctx context.Context, datasetId, documentId int64) error {
// 每个分块一次调用,任何失败只记日志,不阻断解析流水线;返回未成功抽取的分块数,供调用方标记图谱未构建
func (s *kgEntityService) ExtractDocument(ctx context.Context, datasetId, documentId int64) (int, error) {
chunks, _, err := dao.Chunk.ListByDocument(ctx, documentId, 1, 100000)
if err != nil {
return err
return 0, err
}
model, err := s.buildModel(ctx)
if err != nil {
g.Log().Infof(ctx, "kg extract skipped: %v", err)
return nil
return 0, err
}
failed := 0
for _, c := range chunks {
if err := s.extractChunk(ctx, model, datasetId, c); err != nil {
failed++
g.Log().Warningf(ctx, "kg extract chunk %d failed: %v", c.Id, err)
}
}
return nil
return failed, nil
}
func (s *kgEntityService) buildModel(ctx context.Context) (*OpenAIChatModel, error) {
+23 -4
View File
@@ -2,6 +2,7 @@ package service
import (
"context"
"fmt"
"path/filepath"
"time"
@@ -117,15 +118,33 @@ func (s *parseTaskService) processOne(ctx context.Context) {
g.Log().Warningf(ctx, "save detected structure patterns failed: %v", err)
}
}
if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"status": consts.DocumentStatusEmbedding}); err != nil {
s.fail(ctx, task, "更新文档状态失败: "+err.Error())
return
}
if err := ChunkService.InsertAll(ctx, task.DatasetId, doc.Id, chunks, embedder); err != nil {
s.fail(ctx, task, "写入分块失败: "+err.Error())
return
}
// 第 3.5 步:知识图谱抽取(失败不阻断流水线)
if err := KgEntityService.ExtractDocument(ctx, task.DatasetId, doc.Id); err != nil {
g.Log().Warningf(ctx, "kg extract skipped for doc %d: %v", doc.Id, err)
// 第 3.5 步:知识图谱抽取(失败不阻断流水线,但文档标记图谱未构建,避免误报已完成
if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"status": consts.DocumentStatusKgBuilding}); err != nil {
g.Log().Warningf(ctx, "mark kg building failed: %v", err)
}
if err := dao.ParseTask.UpdateStatus(ctx, task.Id, consts.TaskStatusDone, ""); err != nil {
kgFailed, kgErr := KgEntityService.ExtractDocument(ctx, task.DatasetId, doc.Id)
kgMsg := ""
switch {
case kgErr != nil:
kgMsg = "知识图谱未构建: " + kgErr.Error()
case kgFailed > 0:
kgMsg = fmt.Sprintf("知识图谱未构建:%d 个分块抽取失败", kgFailed)
}
if kgMsg != "" {
g.Log().Warningf(ctx, "kg extract incomplete for doc %d: %s", doc.Id, kgMsg)
}
if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"status": consts.DocumentStatusDone, "error_msg": kgMsg}); err != nil {
g.Log().Warningf(ctx, "mark document done failed: %v", err)
}
if err := dao.ParseTask.UpdateStatus(ctx, task.Id, consts.TaskStatusDone, kgMsg); err != nil {
g.Log().Errorf(ctx, "mark task done failed: %v", err)
}
}
+8 -5
View File
@@ -23,6 +23,7 @@
<el-tooltip :content="row.error_msg || ''" placement="top">
<el-tag :type="statusType(row.status)">{{ statusText(row.status) }}</el-tag>
</el-tooltip>
<el-tag v-if="row.status === 4 && row.error_msg" size="small" type="warning" style="margin-left: 4px">图谱未构建</el-tag>
</template>
</el-table-column>
<el-table-column prop="chunk_count" label="分块数" width="90" />
@@ -30,7 +31,7 @@
<el-table-column label="操作" width="240" fixed="right">
<template #default="{ row }">
<el-button link type="primary" @click="openChunks(row)">分块</el-button>
<el-button v-if="row.status === 3" link type="warning" @click="retry(row)">重试</el-button>
<el-button v-if="row.status === 5" link type="warning" @click="retry(row)">重试</el-button>
<el-button link type="primary" @click="reembed(row)">重新向量化</el-button>
<el-button link type="danger" @click="remove(row)">删除</el-button>
</template>
@@ -126,8 +127,10 @@ let editingChunk = null
const statusMap = {
0: { text: '待处理', type: 'info' },
1: { text: '解析中', type: 'warning' },
2: { text: '已完成', type: 'success' },
3: { text: '失败', type: 'danger' }
2: { text: '向量生成中', type: 'warning' },
3: { text: '图谱构建中', type: 'warning' },
4: { text: '已完成', type: 'success' },
5: { text: '失败', type: 'danger' }
}
function statusText(s) { return (statusMap[s] || {}).text || s }
@@ -172,7 +175,7 @@ async function startPolling() {
stopPolling()
pollTimer = setInterval(async () => {
await load()
if (!documents.value.some(d => d.status === 0 || d.status === 1)) stopPolling()
if (!documents.value.some(d => d.status < 4)) stopPolling()
}, 3000)
}
function stopPolling() {
@@ -189,7 +192,7 @@ onMounted(async () => {
datasetName.value = cur ? cur.name : '数据集'
} catch { /* 忽略 */ }
await load()
if (documents.value.some(d => d.status === 0 || d.status === 1)) startPolling()
if (documents.value.some(d => d.status < 4)) startPolling()
})
onUnmounted(stopPolling)