1
This commit is contained in:
@@ -62,6 +62,7 @@
|
||||
## 数据访问规范(硬性要求)
|
||||
|
||||
- **事务**:涉及多张表的增删改操作必须包数据库事务,禁止逐表裸调用。事务放 dao 层方法内,service 层负责编排;`tx.Begin` 后必须用 `defer` 防护已提交后的二次 Rollback
|
||||
- **SQL 单表约束**:每个 SQL 只允许访问一张表,禁止 JOIN 与跨表子查询(`IN (SELECT ...)` / `EXISTS`);跨表数据一律拆为多条单表 SQL + 应用层内存组装——先取外键 id 列表,再对目标表 `IN` 查询;`IN` 参数须按 ≤100 分批(SQLite 变量数上限 999)
|
||||
- **禁止 N+1 查询**:禁止在循环中逐条查库。循环场景一律改为批处理——一次 `ListByXxx` 取回后按外键在内存分组
|
||||
- **缓存一致性**:DAO 查询走缓存(TTL 来自 `database.cache.ttl`),写操作后必须清对应缓存
|
||||
- **批处理 SQL**:批量写入用 `InsertAll` 类方法,批量删除用 `IN` 子句,禁止循环单条 INSERT/DELETE
|
||||
|
||||
Binary file not shown.
@@ -38,6 +38,11 @@ const (
|
||||
AnnoMaxCandidates = 60 // 多数据集融合后的候选上限(喂给 LLM 判定)
|
||||
AnnoMaxClauseChars = 2000 // 合同条款全文上限(超长截断,控制 prompt)
|
||||
|
||||
// 判定 prompt 上下文预算(本地 4B 模型 context 8192 token,须留生成余量)
|
||||
AnnoJudgeMaxTokens = 1024 // 判定输出 token 上限(JSON 结果含 ≤3 条风险;须显式限制,防推理模型长思考烧光上下文)
|
||||
AnnoJudgeCandidateChars = 300 // 判定 prompt 单候选条文截断字数(精确条文文本由 ContentFull 抽取,不依赖 prompt 全文)
|
||||
AnnoJudgePromptBudget = 1500 // 判定 prompt 候选块总字数预算(候选按 RRF 相关度降序贪心填充,超预算截断后续候选)
|
||||
|
||||
// 合同风险识别
|
||||
RiskLevelHigh = "high" // 高风险(违反强制性规定、可能导致合同无效/赔偿)
|
||||
RiskLevelMid = "mid" // 中风险(约定与法律不符但可补救)
|
||||
|
||||
@@ -3,6 +3,7 @@ package controller
|
||||
import (
|
||||
"context"
|
||||
|
||||
"rag-local/kb/consts"
|
||||
"rag-local/kb/model/dto"
|
||||
"rag-local/kb/service"
|
||||
)
|
||||
@@ -48,9 +49,9 @@ func (c *document) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (*dto
|
||||
}
|
||||
|
||||
func (c *document) Reembed(ctx context.Context, req *dto.ReembedDocumentReq) (*dto.ReembedDocumentRes, error) {
|
||||
n, err := service.DocumentService.Reembed(ctx, req.Id)
|
||||
if err != nil {
|
||||
// 入队由轮询器串行消费,避免同步重算阻塞 HTTP 且无法展示过程状态;已排队/处理中会跳过
|
||||
if err := service.ParseTaskService.Enqueue(ctx, req.Id, consts.TaskTypeReembed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dto.ReembedDocumentRes{Count: n}, nil
|
||||
return &dto.ReembedDocumentRes{}, nil
|
||||
}
|
||||
|
||||
@@ -23,3 +23,15 @@ func (c *kgEntity) List(ctx context.Context, req *dto.ListKgEntityReq) (*dto.Lis
|
||||
PageSize: req.PageSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *kgEntity) Sources(ctx context.Context, req *dto.SourcesKgEntityReq) (*dto.SourcesKgEntityRes, error) {
|
||||
list, err := service.KgEntityService.Sources(ctx, req.DatasetId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]*dto.KgEntitySourceItem, 0, len(list))
|
||||
for _, s := range list {
|
||||
out = append(out, &dto.KgEntitySourceItem{Name: s.Name, Files: s.Files})
|
||||
}
|
||||
return &dto.SourcesKgEntityRes{List: out}, nil
|
||||
}
|
||||
|
||||
+73
-12
@@ -222,25 +222,86 @@ func (d *chunkDao) CountByDocument(ctx context.Context, documentId int64) (int,
|
||||
return g.DB(consts.DbGroupDefault).Model(consts.TableNameChunk).Ctx(ctx).Where("document_id", documentId).Count()
|
||||
}
|
||||
|
||||
// VecSearch 向量 KNN 检索:vec0 取最近 topK*4 后按数据集过滤(vec0 无 dataset 列)
|
||||
// ListByIds 批量按主键查分块(单表约束:IN ≤100 分批;调用方保证 ids 非空)
|
||||
func (d *chunkDao) ListByIds(ctx context.Context, ids []int64) ([]*entity.Chunk, error) {
|
||||
var list []*entity.Chunk
|
||||
for start := 0; start < len(ids); start += 100 {
|
||||
end := min(start+100, len(ids))
|
||||
var part []*entity.Chunk
|
||||
if err := g.DB(consts.DbGroupDefault).Model(consts.TableNameChunk).Ctx(ctx).
|
||||
WhereIn("id", ids[start:end]).Scan(&part); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
list = append(list, part...)
|
||||
}
|
||||
if list == nil {
|
||||
list = make([]*entity.Chunk, 0)
|
||||
}
|
||||
return list, nil
|
||||
}
|
||||
|
||||
// ListDocumentIdsByChunkIds 批量查 chunk 归属文档(单表约束:IN ≤100 分批;调用方保证 chunkIds 非空)
|
||||
func (d *chunkDao) ListDocumentIdsByChunkIds(ctx context.Context, chunkIds []int64) (map[int64]int64, error) {
|
||||
out := make(map[int64]int64, len(chunkIds))
|
||||
for start := 0; start < len(chunkIds); start += 100 {
|
||||
end := min(start+100, len(chunkIds))
|
||||
var rows []*entity.Chunk
|
||||
if err := g.DB(consts.DbGroupDefault).Model(consts.TableNameChunk).Ctx(ctx).
|
||||
Fields("id, document_id").WhereIn("id", chunkIds[start:end]).Scan(&rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, r := range rows {
|
||||
out[r.Id] = r.DocumentId
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// VecSearch 向量 KNN 检索:vec0 取最近 topK*4 候选,再按数据集过滤(单表约束:vec0 无 dataset 列,
|
||||
// 拆两条单表查询 + 内存过滤;候选已按距离升序,过滤后取前 topK 即等价原 JOIN 语义)
|
||||
func (d *chunkDao) VecSearch(ctx context.Context, datasetId int64, vecJson string, topK int) ([]domain.VecHit, error) {
|
||||
r, err := g.DB(consts.DbGroupDefault).Ctx(ctx).Raw(
|
||||
`SELECT v.chunk_id, v.distance FROM (
|
||||
SELECT chunk_id, distance FROM `+consts.TableNameChunkVec+`
|
||||
WHERE embedding MATCH ? ORDER BY distance LIMIT ?
|
||||
) v INNER JOIN `+consts.TableNameChunk+` c ON c.id = v.chunk_id
|
||||
WHERE c.dataset_id = ? ORDER BY v.distance LIMIT ?`,
|
||||
vecJson, topK*4, datasetId, topK,
|
||||
`SELECT chunk_id, distance FROM `+consts.TableNameChunkVec+
|
||||
` WHERE embedding MATCH ? ORDER BY distance LIMIT ?`,
|
||||
vecJson, topK*4,
|
||||
).All()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hits := make([]domain.VecHit, 0, len(r))
|
||||
if len(r) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
cand := make([]domain.VecHit, 0, len(r))
|
||||
ids := make([]int64, 0, len(r))
|
||||
for _, row := range r {
|
||||
hits = append(hits, domain.VecHit{
|
||||
ChunkId: row["chunk_id"].Int64(),
|
||||
Distance: row["distance"].Float64(),
|
||||
})
|
||||
id := row["chunk_id"].Int64()
|
||||
cand = append(cand, domain.VecHit{ChunkId: id, Distance: row["distance"].Float64()})
|
||||
ids = append(ids, id)
|
||||
}
|
||||
// 候选 id ≤ topK*4(默认 80),远低于 SQLite 变量上限,无需分批
|
||||
placeholders := strings.TrimSuffix(strings.Repeat("?,", len(ids)), ",")
|
||||
args := make([]any, 0, len(ids)+1)
|
||||
for _, id := range ids {
|
||||
args = append(args, id)
|
||||
}
|
||||
args = append(args, datasetId)
|
||||
chunks, err := g.DB(consts.DbGroupDefault).Ctx(ctx).Raw(
|
||||
"SELECT id FROM "+consts.TableNameChunk+" WHERE id IN ("+placeholders+") AND dataset_id = ?", args...).All()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
allowed := make(map[int64]bool, len(chunks))
|
||||
for _, row := range chunks {
|
||||
allowed[row["id"].Int64()] = true
|
||||
}
|
||||
hits := make([]domain.VecHit, 0, len(cand))
|
||||
for _, h := range cand {
|
||||
if allowed[h.ChunkId] {
|
||||
hits = append(hits, h)
|
||||
if len(hits) >= topK {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
@@ -36,31 +36,51 @@ func init() {
|
||||
}
|
||||
|
||||
func (d *contractClauseDao) InsertAll(ctx context.Context, taskId int64, clauses []entity.ContractClause) error {
|
||||
if len(clauses) == 0 {
|
||||
return nil
|
||||
}
|
||||
now := gtime.Now().Format("Y-m-d H:i:s")
|
||||
list := make(g.List, 0, len(clauses))
|
||||
for i := range clauses {
|
||||
list = append(list, g.Map{
|
||||
"task_id": taskId,
|
||||
"seq": clauses[i].Seq,
|
||||
"title": clauses[i].Title,
|
||||
"content": clauses[i].Content,
|
||||
"status": consts.TaskStatusPending,
|
||||
"error_msg": "",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
})
|
||||
}
|
||||
tx, err := g.DB(consts.DbGroupDefault).Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := gtime.Now().Format("Y-m-d H:i:s")
|
||||
for i := range clauses {
|
||||
clauses[i].TaskId = taskId
|
||||
clauses[i].Status = consts.TaskStatusPending
|
||||
clauses[i].CreatedAt = nil
|
||||
clauses[i].UpdatedAt = nil
|
||||
if _, err := tx.Model(consts.TableNameContractClause).Ctx(ctx).Data(g.Map{
|
||||
"task_id": clauses[i].TaskId,
|
||||
"seq": clauses[i].Seq,
|
||||
"title": clauses[i].Title,
|
||||
"content": clauses[i].Content,
|
||||
"status": clauses[i].Status,
|
||||
"error_msg": "",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}).Insert(); err != nil {
|
||||
// Commit 成功后 IsClosed 为 true,跳过 Rollback,避免对已提交事务回滚产生报错日志
|
||||
defer func() {
|
||||
if !tx.IsClosed() {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
if _, err := tx.Model(consts.TableNameContractClause).Ctx(ctx).Data(list).Batch(100).Insert(); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// UpdateStatuses 批量更新条款状态(单表约束:IN ≤100 分批)
|
||||
func (d *contractClauseDao) UpdateStatuses(ctx context.Context, ids []int64, status int, errorMsg string) error {
|
||||
now := gtime.Now().Format("Y-m-d H:i:s")
|
||||
for start := 0; start < len(ids); start += 100 {
|
||||
end := min(start+100, len(ids))
|
||||
if _, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameContractClause).Ctx(ctx).
|
||||
Data(g.Map{"status": status, "error_msg": errorMsg, "updated_at": now}).
|
||||
WhereIn("id", ids[start:end]).Update(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *contractClauseDao) ListByTask(ctx context.Context, taskId int64) ([]*entity.ContractClause, error) {
|
||||
|
||||
+22
-33
@@ -2,12 +2,12 @@ package dao
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
|
||||
"rag-local/kb/consts"
|
||||
"rag-local/kb/model/entity"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
"github.com/gogf/gf/v2/os/gtime"
|
||||
)
|
||||
|
||||
var ContractMark = &contractMarkDao{}
|
||||
@@ -36,34 +36,6 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
func (d *contractMarkDao) InsertAll(ctx context.Context, marks []*entity.ContractMark) error {
|
||||
if len(marks) == 0 {
|
||||
return nil
|
||||
}
|
||||
tx, err := g.DB(consts.DbGroupDefault).Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := gtime.Now().Format("Y-m-d H:i:s")
|
||||
for _, m := range marks {
|
||||
if _, err := tx.Model(consts.TableNameContractMark).Ctx(ctx).Data(g.Map{
|
||||
"clause_id": m.ClauseId,
|
||||
"chunk_id": m.ChunkId,
|
||||
"dataset_id": m.DatasetId,
|
||||
"law_title": m.LawTitle,
|
||||
"law_item": m.LawItem,
|
||||
"content": m.Content,
|
||||
"reason": m.Reason,
|
||||
"score": m.Score,
|
||||
"created_at": now,
|
||||
}).Insert(); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (d *contractMarkDao) ListByClause(ctx context.Context, clauseId int64) ([]*entity.ContractMark, error) {
|
||||
var list []*entity.ContractMark
|
||||
err := g.DB(consts.DbGroupDefault).Model(consts.TableNameContractMark).Ctx(ctx).
|
||||
@@ -74,15 +46,32 @@ func (d *contractMarkDao) ListByClause(ctx context.Context, clauseId int64) ([]*
|
||||
return list, err
|
||||
}
|
||||
|
||||
// ListByTask 某任务的全部标注(单表约束:先取条款 id,再按 IN 分批查询,内存按分排序)
|
||||
func (d *contractMarkDao) ListByTask(ctx context.Context, taskId int64) ([]*entity.ContractMark, error) {
|
||||
clauseIds, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameContractClause).Ctx(ctx).
|
||||
Fields("id").Where("task_id", taskId).Array()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var list []*entity.ContractMark
|
||||
err := g.DB(consts.DbGroupDefault).Model(consts.TableNameContractMark).Ctx(ctx).
|
||||
Where("clause_id IN (SELECT id FROM "+consts.TableNameContractClause+" WHERE task_id = ?)", taskId).
|
||||
OrderDesc("score").Scan(&list)
|
||||
for start := 0; start < len(clauseIds); start += 100 {
|
||||
end := min(start+100, len(clauseIds))
|
||||
ids := make([]int64, 0, end-start)
|
||||
for _, v := range clauseIds[start:end] {
|
||||
ids = append(ids, v.Int64())
|
||||
}
|
||||
var part []*entity.ContractMark
|
||||
if err := g.DB(consts.DbGroupDefault).Model(consts.TableNameContractMark).Ctx(ctx).
|
||||
WhereIn("clause_id", ids).Scan(&part); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
list = append(list, part...)
|
||||
}
|
||||
if list == nil {
|
||||
list = make([]*entity.ContractMark, 0)
|
||||
}
|
||||
return list, err
|
||||
sort.Slice(list, func(i, j int) bool { return list[i].Score > list[j].Score })
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func (d *contractMarkDao) DeleteByClause(ctx context.Context, clauseId int64) error {
|
||||
|
||||
@@ -40,23 +40,30 @@ func (d *contractRiskDao) InsertAll(ctx context.Context, risks []*entity.Contrac
|
||||
if len(risks) == 0 {
|
||||
return nil
|
||||
}
|
||||
tx, err := g.DB(consts.DbGroupDefault).Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
now := gtime.Now().Format("Y-m-d H:i:s")
|
||||
list := make(g.List, 0, len(risks))
|
||||
for _, r := range risks {
|
||||
if _, err := tx.Model(consts.TableNameContractRisk).Ctx(ctx).Data(g.Map{
|
||||
list = append(list, g.Map{
|
||||
"task_id": r.TaskId,
|
||||
"clause_id": r.ClauseId,
|
||||
"level": r.Level,
|
||||
"desc": r.Desc,
|
||||
"laws": r.Laws,
|
||||
"created_at": now,
|
||||
}).Insert(); err != nil {
|
||||
})
|
||||
}
|
||||
tx, err := g.DB(consts.DbGroupDefault).Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Commit 成功后 IsClosed 为 true,跳过 Rollback,避免对已提交事务回滚产生报错日志
|
||||
defer func() {
|
||||
if !tx.IsClosed() {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
}()
|
||||
if _, err := tx.Model(consts.TableNameContractRisk).Ctx(ctx).Data(list).Batch(100).Insert(); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
@@ -136,10 +136,23 @@ func (d *contractTaskDao) DeleteWithRelated(ctx context.Context, id int64) error
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
// 单表约束:先取条款 id,再按 IN 分批删除(条款数可能超 SQLite 变量上限)
|
||||
clauseIds, err := tx.Model(consts.TableNameContractClause).Ctx(ctx).Fields("id").Where("task_id", id).Array()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, table := range []string{consts.TableNameContractMark, consts.TableNameContractRisk} {
|
||||
if _, err := tx.Exec(
|
||||
"DELETE FROM "+table+" WHERE clause_id IN (SELECT id FROM "+consts.TableNameContractClause+" WHERE task_id = ?)", id); err != nil {
|
||||
return err
|
||||
for start := 0; start < len(clauseIds); start += 100 {
|
||||
end := min(start+100, len(clauseIds))
|
||||
ids := make([]int64, 0, end-start)
|
||||
for _, v := range clauseIds[start:end] {
|
||||
ids = append(ids, v.Int64())
|
||||
}
|
||||
if len(ids) > 0 {
|
||||
if _, err := tx.Model(table).Ctx(ctx).WhereIn("clause_id", ids).Delete(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := tx.Model(consts.TableNameContractClause).Ctx(ctx).Where("task_id", id).Delete(); err != nil {
|
||||
|
||||
@@ -82,6 +82,24 @@ func (d *documentDao) List(ctx context.Context, datasetId int64, page, pageSize
|
||||
return list, total, err
|
||||
}
|
||||
|
||||
// ListByIds 批量按主键查文档(单表约束:IN ≤100 分批;调用方保证 ids 非空)
|
||||
func (d *documentDao) ListByIds(ctx context.Context, ids []int64) ([]*entity.Document, error) {
|
||||
var list []*entity.Document
|
||||
for start := 0; start < len(ids); start += 100 {
|
||||
end := min(start+100, len(ids))
|
||||
var part []*entity.Document
|
||||
if err := g.DB(consts.DbGroupDefault).Model(consts.TableNameDocument).Ctx(ctx).
|
||||
FieldsEx("content").WhereIn("id", ids[start:end]).Scan(&part); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
list = append(list, part...)
|
||||
}
|
||||
if list == nil {
|
||||
list = make([]*entity.Document, 0)
|
||||
}
|
||||
return list, nil
|
||||
}
|
||||
|
||||
func (d *documentDao) Insert(ctx context.Context, data *entity.Document) (int64, error) {
|
||||
now := gtime.Now().Format("Y-m-d H:i:s")
|
||||
r, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameDocument).Ctx(ctx).Data(g.Map{
|
||||
|
||||
@@ -124,6 +124,34 @@ func (d *kgEntityDao) UpsertBatch(ctx context.Context, datasetId, chunkId int64,
|
||||
return nil
|
||||
}
|
||||
|
||||
// HasGraphByDocument 判定某文档是否产出过图谱(单表约束:先取分块 id,再数关系)。
|
||||
// 不用实体行计数:实体按 (dataset_id,name) 去重、chunk_id 会被跨文件 upsert 覆盖,按实体计数会漏计。
|
||||
// 关系每 chunk 至少一条、不受覆盖影响。
|
||||
func (d *kgEntityDao) HasGraphByDocument(ctx context.Context, documentId int64) (bool, error) {
|
||||
chunkIds, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameChunk).Ctx(ctx).
|
||||
Fields("id").Where("document_id", documentId).Array()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
// 分块数可能超 SQLite 变量上限,按 100 分批
|
||||
for start := 0; start < len(chunkIds); start += 100 {
|
||||
end := min(start+100, len(chunkIds))
|
||||
ids := make([]int64, 0, end-start)
|
||||
for _, v := range chunkIds[start:end] {
|
||||
ids = append(ids, v.Int64())
|
||||
}
|
||||
n, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameKgRelation).Ctx(ctx).
|
||||
WhereIn("chunk_id", ids).Count()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if n > 0 {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func (d *kgEntityDao) DeleteByChunkIds(ctx context.Context, chunkIds []int64) error {
|
||||
if len(chunkIds) == 0 {
|
||||
return nil
|
||||
@@ -143,3 +171,79 @@ func (d *kgEntityDao) DeleteByDataset(ctx context.Context, datasetId int64) erro
|
||||
Where("dataset_id", datasetId).Delete()
|
||||
return err
|
||||
}
|
||||
|
||||
// KgEntitySource 实体 → 出现文件清单(dao 内已按名聚合)
|
||||
type KgEntitySource struct {
|
||||
Name string
|
||||
Files []string
|
||||
}
|
||||
|
||||
// SourcesByDataset 实体名 → 出现文件清单(单表约束:拆 4 条单表查询 + 内存组装)。
|
||||
// 有关系的实体经关系表溯源(关系不去重、每条带来源 chunk,覆盖实体全部出现文件);
|
||||
// 孤立实体(无任何关系)由实体行 chunk_id 弱引用兜底(该弱引用是 upsert 的最近来源,不保证全部出现文件)。
|
||||
// 不再建实体-分块关联表:该组合已覆盖"图谱页标注来源文件"的展示需求。
|
||||
func (d *kgEntityDao) SourcesByDataset(ctx context.Context, datasetId int64) ([]KgEntitySource, error) {
|
||||
// 1. 关系表(单表):数据集全部三元组的 head/tail 及其来源分块
|
||||
relRows, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameKgRelation).Ctx(ctx).
|
||||
Fields("head", "tail", "chunk_id").Where("dataset_id", datasetId).All()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 2. 实体表(单表):孤立实体弱引用兜底
|
||||
entRows, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameKgEntity).Ctx(ctx).
|
||||
Fields("name", "chunk_id").Where("dataset_id", datasetId).All()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 3. 分块表(单表):数据集全部分块 chunk_id → document_id
|
||||
chunkRows, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameChunk).Ctx(ctx).
|
||||
Fields("id", "document_id").Where("dataset_id", datasetId).All()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// 4. 文档表(单表):数据集全部文档 id → 文件名
|
||||
docRows, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameDocument).Ctx(ctx).
|
||||
Fields("id", "filename").Where("dataset_id", datasetId).All()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
docByID := make(map[int64]string, len(docRows))
|
||||
for _, row := range docRows {
|
||||
docByID[row["id"].Int64()] = row["filename"].String()
|
||||
}
|
||||
docByChunk := make(map[int64]int64, len(chunkRows))
|
||||
for _, row := range chunkRows {
|
||||
docByChunk[row["id"].Int64()] = row["document_id"].Int64()
|
||||
}
|
||||
filesByName := make(map[string]map[string]bool)
|
||||
add := func(name string, chunkId int64) {
|
||||
docId, ok := docByChunk[chunkId]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
fname := docByID[docId]
|
||||
if name == "" || fname == "" {
|
||||
return
|
||||
}
|
||||
if filesByName[name] == nil {
|
||||
filesByName[name] = make(map[string]bool)
|
||||
}
|
||||
filesByName[name][fname] = true
|
||||
}
|
||||
for _, row := range relRows {
|
||||
add(row["head"].String(), row["chunk_id"].Int64())
|
||||
add(row["tail"].String(), row["chunk_id"].Int64())
|
||||
}
|
||||
for _, row := range entRows {
|
||||
add(row["name"].String(), row["chunk_id"].Int64())
|
||||
}
|
||||
out := make([]KgEntitySource, 0, len(filesByName))
|
||||
for name, set := range filesByName {
|
||||
files := make([]string, 0, len(set))
|
||||
for f := range set {
|
||||
files = append(files, f)
|
||||
}
|
||||
out = append(out, KgEntitySource{Name: name, Files: files})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -99,6 +99,30 @@ func (d *parseTaskDao) DeleteByDocument(ctx context.Context, documentId int64) e
|
||||
return err
|
||||
}
|
||||
|
||||
// ResetRunning 启动恢复:进程异常退出会孤儿化 running 任务(轮询器只消费 pending),重置回 pending 让轮询器重新捡起
|
||||
func (d *parseTaskDao) ResetRunning(ctx context.Context) (int64, error) {
|
||||
r, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameParseTask).Ctx(ctx).
|
||||
Data(g.Map{
|
||||
"status": consts.TaskStatusPending,
|
||||
"updated_at": gtime.Now().Format("Y-m-d H:i:s"),
|
||||
}).Where("status", consts.TaskStatusRunning).Update()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.RowsAffected()
|
||||
}
|
||||
|
||||
// HasPending 该文档是否存在待处理/运行中任务(重复入队防护)
|
||||
func (d *parseTaskDao) HasPending(ctx context.Context, documentId int64) (bool, error) {
|
||||
n, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameParseTask).Ctx(ctx).
|
||||
Where("document_id", documentId).
|
||||
WhereIn("status", []int{consts.TaskStatusPending, consts.TaskStatusRunning}).Count()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
func (d *parseTaskDao) NextPending(ctx context.Context) (*entity.ParseTask, error) {
|
||||
var m entity.ParseTask
|
||||
err := g.DB(consts.DbGroupDefault).Model(consts.TableNameParseTask).Ctx(ctx).
|
||||
|
||||
@@ -52,6 +52,4 @@ type ReembedDocumentReq struct {
|
||||
Id int64 `v:"required" json:"id"`
|
||||
}
|
||||
|
||||
type ReembedDocumentRes struct {
|
||||
Count int `json:"count"`
|
||||
}
|
||||
type ReembedDocumentRes struct{}
|
||||
|
||||
@@ -19,3 +19,17 @@ type ListKgEntityRes struct {
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
}
|
||||
|
||||
type SourcesKgEntityReq struct {
|
||||
g.Meta `path:"/sources" method:"get" tags:"知识图谱" summary:"实体来源文件清单"`
|
||||
DatasetId int64 `json:"dataset_id"`
|
||||
}
|
||||
|
||||
type KgEntitySourceItem struct {
|
||||
Name string `json:"name"`
|
||||
Files []string `json:"files"`
|
||||
}
|
||||
|
||||
type SourcesKgEntityRes struct {
|
||||
List []*KgEntitySourceItem `json:"list"`
|
||||
}
|
||||
|
||||
@@ -19,9 +19,10 @@ type ContractRisk struct {
|
||||
|
||||
// LawRef 支撑法条引用(laws JSON 元素)
|
||||
type LawRef struct {
|
||||
LawTitle string `json:"law_title"`
|
||||
LawItem string `json:"law_item"`
|
||||
Content string `json:"content"`
|
||||
LawTitle string `json:"law_title"`
|
||||
LawItem string `json:"law_item"`
|
||||
Content string `json:"content"`
|
||||
SourceFile string `json:"source_file"` // 法条所在语料文件名(判定时经 chunk 溯源填充)
|
||||
}
|
||||
|
||||
// LawsRefs 解析 laws JSON 文本为法条引用数组(非法 JSON 返回空数组)
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"rag-local/kb/model/domain"
|
||||
"rag-local/kb/model/entity"
|
||||
|
||||
emodel "github.com/cloudwego/eino/components/model"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
@@ -176,6 +177,8 @@ func (s *annotationService) processOne(ctx context.Context) {
|
||||
s.fail(ctx, task, "构建对话模型失败: "+err.Error())
|
||||
return
|
||||
}
|
||||
// 推理模型默认输出长思考链,烧光上下文(与 kg_extract 同策略);须在提交池前预置(共享实例可变字段)
|
||||
chatModel.DisableThinking()
|
||||
|
||||
embedders := make(map[int64]*OpenAIEmbedder)
|
||||
dsNames := make(map[int64]string)
|
||||
@@ -203,6 +206,7 @@ func (s *annotationService) processOne(ctx context.Context) {
|
||||
err error
|
||||
}
|
||||
ch := make(chan clauseJobOut, len(clauses))
|
||||
runIds := make([]int64, 0, len(clauses))
|
||||
var wg sync.WaitGroup
|
||||
for _, cl := range clauses {
|
||||
// 断点续跑:已完成且已有风险记录 → 跳过;已完成但无风险记录 → 仅当存在旧格式法条标注时重跑迁移
|
||||
@@ -216,10 +220,7 @@ func (s *annotationService) processOne(ctx context.Context) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err := dao.ContractClause.UpdateStatus(ctx, cl.Id, consts.TaskStatusRunning, ""); err != nil {
|
||||
g.Log().Errorf(ctx, "mark clause running failed: %v", err)
|
||||
continue
|
||||
}
|
||||
runIds = append(runIds, cl.Id)
|
||||
wg.Add(1)
|
||||
if err := common.AnnotationClausePool.AddWithRecover(ctx, func(ctx context.Context) {
|
||||
defer wg.Done()
|
||||
@@ -245,9 +246,31 @@ func (s *annotationService) processOne(ctx context.Context) {
|
||||
g.Log().Errorf(ctx, "submit clause %d failed: %v", cl.Id, err)
|
||||
}
|
||||
}
|
||||
// 批量标记进行中(避免逐条款 UPDATE)
|
||||
if len(runIds) > 0 {
|
||||
if err := dao.ContractClause.UpdateStatuses(ctx, runIds, consts.TaskStatusRunning, ""); err != nil {
|
||||
g.Log().Errorf(ctx, "mark clauses running failed: %v", err)
|
||||
}
|
||||
}
|
||||
go func() { wg.Wait(); close(ch) }()
|
||||
|
||||
// 进度按本地计数推进(断点续跑时已完成条款计入基数),每完成一条刷一次,避免逐条款读库统计
|
||||
progressDone := 0
|
||||
for _, cl := range clauses {
|
||||
if cl.Status == consts.TaskStatusDone {
|
||||
progressDone++
|
||||
}
|
||||
}
|
||||
totalClauses := len(clauses)
|
||||
tickProgress := func() {
|
||||
progressDone++
|
||||
if err := dao.ContractTask.UpdateProgress(ctx, task.Id, totalClauses, progressDone); err != nil {
|
||||
g.Log().Warningf(ctx, "update annotation progress failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
failed := 0
|
||||
doneIds := make([]int64, 0, len(runIds))
|
||||
for out := range ch {
|
||||
if out.err != nil {
|
||||
failed++
|
||||
@@ -256,8 +279,8 @@ func (s *annotationService) processOne(ctx context.Context) {
|
||||
}
|
||||
if out.noCands {
|
||||
// 无候选视为完成(无标注),避免卡住进度
|
||||
_ = dao.ContractClause.UpdateStatus(ctx, out.clauseId, consts.TaskStatusDone, "")
|
||||
s.updateProgress(ctx, task.Id)
|
||||
doneIds = append(doneIds, out.clauseId)
|
||||
tickProgress()
|
||||
continue
|
||||
}
|
||||
// 幂等:重跑前清旧风险与旧格式法条标注,避免重复记录
|
||||
@@ -278,8 +301,14 @@ func (s *annotationService) processOne(ctx context.Context) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
_ = dao.ContractClause.UpdateStatus(ctx, out.clauseId, consts.TaskStatusDone, "")
|
||||
s.updateProgress(ctx, task.Id)
|
||||
doneIds = append(doneIds, out.clauseId)
|
||||
tickProgress()
|
||||
}
|
||||
// 完成状态批量落库(成功与无候选统一刷一次;失败走上面逐条,error_msg 各异)
|
||||
if len(doneIds) > 0 {
|
||||
if err := dao.ContractClause.UpdateStatuses(ctx, doneIds, consts.TaskStatusDone, ""); err != nil {
|
||||
g.Log().Errorf(ctx, "mark clauses done failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
msg := ""
|
||||
@@ -291,23 +320,6 @@ func (s *annotationService) processOne(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// updateProgress 以库内实际完成数更新任务进度(断点续跑时跳过已 done 条款也能算对)
|
||||
func (s *annotationService) updateProgress(ctx context.Context, taskId int64) {
|
||||
doneList, err := dao.ContractClause.ListByTask(ctx, taskId)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
done := 0
|
||||
for _, c := range doneList {
|
||||
if c.Status == consts.TaskStatusDone {
|
||||
done++
|
||||
}
|
||||
}
|
||||
if err := dao.ContractTask.UpdateProgress(ctx, taskId, len(doneList), done); err != nil {
|
||||
g.Log().Warningf(ctx, "update annotation progress failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// annoRecallHit 单数据集召回结果(排名用于 RRF 融合)
|
||||
type annoRecallHit struct {
|
||||
ChunkId int64
|
||||
@@ -359,13 +371,31 @@ func (s *annotationService) recallCandidates(ctx context.Context, clause *entity
|
||||
if len(cands) > consts.AnnoMaxCandidates {
|
||||
cands = cands[:consts.AnnoMaxCandidates]
|
||||
}
|
||||
for i := range cands {
|
||||
if chunk, err := dao.Chunk.GetOne(ctx, cands[i].ChunkId); err == nil && chunk != nil {
|
||||
cands[i].ContentFull = chunk.Content
|
||||
cands[i].Content = truncateRunes(chunk.Content, consts.AnnoCandidateMaxChars)
|
||||
}
|
||||
// 候选内容批量加载(单次 IN 查回内存映射,禁止逐条 GetOne 的 N+1);chunk 已删除的候选丢弃
|
||||
chunkIds := make([]int64, 0, len(cands))
|
||||
for _, c := range cands {
|
||||
chunkIds = append(chunkIds, c.ChunkId)
|
||||
}
|
||||
return cands, nil
|
||||
chunks, err := dao.Chunk.ListByIds(ctx, chunkIds)
|
||||
if err != nil {
|
||||
g.Log().Warningf(ctx, "load candidate chunks failed: %v", err)
|
||||
chunks = nil
|
||||
}
|
||||
contentByChunk := make(map[int64]string, len(chunks))
|
||||
for _, ch := range chunks {
|
||||
contentByChunk[ch.Id] = ch.Content
|
||||
}
|
||||
kept := cands[:0]
|
||||
for _, c := range cands {
|
||||
content, ok := contentByChunk[c.ChunkId]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
c.ContentFull = content
|
||||
c.Content = truncateRunes(content, consts.AnnoCandidateMaxChars)
|
||||
kept = append(kept, c)
|
||||
}
|
||||
return kept, nil
|
||||
}
|
||||
|
||||
// recallOneDataset 单数据集召回:向量检索 + FTS 检索(纯读,供池内并发调用)
|
||||
@@ -412,8 +442,19 @@ func (s *annotationService) judgeRisks(ctx context.Context, model *OpenAIChatMod
|
||||
sb.WriteString("你是资深法律顾问,负责审查合同条款的法律风险。请结合候选法律条文,识别该合同条款存在的法律风险点(条款与法律强制性规定冲突、遗漏法定必备内容、赔偿/补偿标准低于法定标准、期限或程序违法、表述模糊导致争议等)。\n\n【合同条款】\n")
|
||||
sb.WriteString(clause.Title + " " + clause.Content)
|
||||
sb.WriteString("\n\n【候选法律条文】\n")
|
||||
// 候选按相关度降序,按上下文预算贪心填充(首条强制入队保底);被截断的候选不展示,
|
||||
// 编号连续映射回 cands,LLM 引用编号受展示条数约束
|
||||
shown := 0
|
||||
budget := consts.AnnoJudgePromptBudget
|
||||
for i, c := range cands {
|
||||
sb.WriteString(fmt.Sprintf("[%d]《%s》%s\n", i+1, c.LawTitle, c.Content))
|
||||
item := fmt.Sprintf("[%d]《%s》%s\n", i+1, c.LawTitle, truncateRunes(c.Content, consts.AnnoJudgeCandidateChars))
|
||||
itemLen := len([]rune(item))
|
||||
if shown > 0 && itemLen > budget {
|
||||
break
|
||||
}
|
||||
sb.WriteString(item)
|
||||
shown++
|
||||
budget -= itemLen
|
||||
}
|
||||
sb.WriteString(fmt.Sprintf("\n请输出该条款的风险点(0~%d 个,没有风险输出空数组)。每条风险点:\n", consts.RiskMaxPerClause))
|
||||
sb.WriteString("- level:风险等级,high=违反强制性规定/可能导致合同无效或赔偿,mid=约定与法律不符但可补救,low=表述瑕疵或建议性提示\n")
|
||||
@@ -422,7 +463,8 @@ func (s *annotationService) judgeRisks(ctx context.Context, model *OpenAIChatMod
|
||||
sb.WriteString("只输出 JSON,不要其他内容:")
|
||||
sb.WriteString(`{"risks":[{"level":"high|mid|low","desc":"...","laws":[{"cand":1,"law_item":"第九十二条"}]}]}`)
|
||||
|
||||
msg, err := model.Generate(ctx, []*schema.Message{{Role: schema.User, Content: sb.String()}})
|
||||
msg, err := model.Generate(ctx, []*schema.Message{{Role: schema.User, Content: sb.String()}},
|
||||
emodel.WithMaxTokens(consts.AnnoJudgeMaxTokens))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -443,6 +485,43 @@ func (s *annotationService) judgeRisks(ctx context.Context, model *OpenAIChatMod
|
||||
if err := json.Unmarshal([]byte(content), &resp); err != nil {
|
||||
return nil, gerror.Wrap(err, "解析风险判定结果失败: "+msg.Content)
|
||||
}
|
||||
// 法条来源溯源(展示增强,失败仅告警不阻断判定):候选 chunk → 文档名,单表两条 SQL + 内存组装
|
||||
srcByChunk := make(map[int64]string, len(cands))
|
||||
chunkIds := make([]int64, 0, len(cands))
|
||||
seenChunk := make(map[int64]bool, len(cands))
|
||||
for _, c := range cands {
|
||||
if c.ChunkId > 0 && !seenChunk[c.ChunkId] {
|
||||
seenChunk[c.ChunkId] = true
|
||||
chunkIds = append(chunkIds, c.ChunkId)
|
||||
}
|
||||
}
|
||||
if len(chunkIds) > 0 {
|
||||
docByChunk, err := dao.Chunk.ListDocumentIdsByChunkIds(ctx, chunkIds)
|
||||
if err != nil {
|
||||
g.Log().Warningf(ctx, "resolve chunk document failed: %v", err)
|
||||
} else {
|
||||
docIds := make([]int64, 0, len(docByChunk))
|
||||
for _, docId := range docByChunk {
|
||||
docIds = append(docIds, docId)
|
||||
}
|
||||
if len(docIds) > 0 {
|
||||
docs, err := dao.Document.ListByIds(ctx, docIds)
|
||||
if err != nil {
|
||||
g.Log().Warningf(ctx, "resolve document name failed: %v", err)
|
||||
} else {
|
||||
nameByDoc := make(map[int64]string, len(docs))
|
||||
for _, doc := range docs {
|
||||
nameByDoc[doc.Id] = doc.Filename
|
||||
}
|
||||
for chunkId, docId := range docByChunk {
|
||||
if f := nameByDoc[docId]; f != "" {
|
||||
srcByChunk[chunkId] = f
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
risks := make([]*entity.ContractRisk, 0, len(resp.Risks))
|
||||
for _, r := range resp.Risks {
|
||||
desc := strings.TrimSpace(r.Desc)
|
||||
@@ -457,7 +536,7 @@ func (s *annotationService) judgeRisks(ctx context.Context, model *OpenAIChatMod
|
||||
}
|
||||
refs := make([]entity.LawRef, 0, len(r.Laws))
|
||||
for _, lr := range r.Laws {
|
||||
if lr.Cand < 1 || lr.Cand > len(cands) {
|
||||
if lr.Cand < 1 || lr.Cand > shown {
|
||||
continue
|
||||
}
|
||||
c := cands[lr.Cand-1]
|
||||
@@ -466,9 +545,10 @@ func (s *annotationService) judgeRisks(ctx context.Context, model *OpenAIChatMod
|
||||
lawContent = c.Content
|
||||
}
|
||||
refs = append(refs, entity.LawRef{
|
||||
LawTitle: c.LawTitle,
|
||||
LawItem: lawItem,
|
||||
Content: lawContent,
|
||||
LawTitle: c.LawTitle,
|
||||
LawItem: lawItem,
|
||||
Content: lawContent,
|
||||
SourceFile: srcByChunk[c.ChunkId],
|
||||
})
|
||||
}
|
||||
lawsJson, _ := json.Marshal(refs)
|
||||
@@ -680,8 +760,12 @@ h1{font-size:20px;text-align:center;margin-bottom:4px}
|
||||
`<span class="risk-level" style="background:` + levelColor[r.Level] + `">` + levelText[r.Level] + `</span>` +
|
||||
`<div class="risk-desc">` + html.EscapeString(r.Desc) + `</div>`)
|
||||
for _, law := range r.LawsRefs() {
|
||||
lawSrc := ""
|
||||
if law.SourceFile != "" {
|
||||
lawSrc = "(来源:" + html.EscapeString(law.SourceFile) + ")"
|
||||
}
|
||||
sb.WriteString(`<div class="risk-law"><span class="law-title">《` + html.EscapeString(law.LawTitle) + `》` +
|
||||
html.EscapeString(law.LawItem) + `</span> ` + html.EscapeString(truncateRunes(law.Content, 200)) + `</div>`)
|
||||
html.EscapeString(law.LawItem) + lawSrc + `</span> ` + html.EscapeString(truncateRunes(law.Content, 200)) + `</div>`)
|
||||
}
|
||||
sb.WriteString(`</div>`)
|
||||
}
|
||||
|
||||
@@ -78,6 +78,12 @@ func (s *datasetService) enqueueTask(ctx context.Context, datasetId int64, taskT
|
||||
if _, err := dao.ParseTask.Insert(ctx, doc.Id, datasetId, taskType); err != nil {
|
||||
return err
|
||||
}
|
||||
// 重新向量化入队即刻置为向量生成中,与单文档入队(ParseTaskService.Enqueue)行为一致
|
||||
if taskType == consts.TaskTypeReembed {
|
||||
if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"status": consts.DocumentStatusEmbedding}); err != nil {
|
||||
g.Log().Warningf(ctx, "mark doc %d embedding failed: %v", doc.Id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -143,21 +143,47 @@ func (s *kgEntityService) callExtract(ctx context.Context, model *OpenAIChatMode
|
||||
return entities, relations, nil
|
||||
}
|
||||
|
||||
// saveExtract 主 goroutine 串行落库:实体与关系各一条批量 SQL(多行 VALUES)
|
||||
// saveExtract 主 goroutine 串行落库:实体与关系各一条批量 SQL(多行 VALUES)。
|
||||
// 落库前归一化(实体名/谓词,防别名分裂)+ chunk 内去重 + 过滤空名/自环。
|
||||
func (s *kgEntityService) saveExtract(ctx context.Context, datasetId, chunkId int64, entities []kgEntityItem, relations []kgRelationItem) error {
|
||||
if len(entities) > 0 {
|
||||
items := make([]dao.KgEntityItem, 0, len(entities))
|
||||
// 归一化 + chunk 内按名去重(同名取首个类型)
|
||||
seen := make(map[string]string, len(entities))
|
||||
for _, e := range entities {
|
||||
items = append(items, dao.KgEntityItem{Name: e.Name, EntityType: e.Type})
|
||||
name := common.NormalizeKgTerm(e.Name, consts.KgAliasMap)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[name]; dup {
|
||||
continue
|
||||
}
|
||||
seen[name] = strings.TrimSpace(e.Type)
|
||||
}
|
||||
items := make([]dao.KgEntityItem, 0, len(seen))
|
||||
for name, typ := range seen {
|
||||
items = append(items, dao.KgEntityItem{Name: name, EntityType: typ})
|
||||
}
|
||||
if err := dao.KgEntity.UpsertBatch(ctx, datasetId, chunkId, items); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(relations) > 0 {
|
||||
// 归一化(head/tail 用实体别名表、谓词用谓词表)+ chunk 内去重 + 空/自环过滤
|
||||
seen := make(map[string]bool, len(relations))
|
||||
items := make([]dao.KgRelationItem, 0, len(relations))
|
||||
for _, r := range relations {
|
||||
items = append(items, dao.KgRelationItem{Head: r.Head, Relation: r.Relation, Tail: r.Tail})
|
||||
head := common.NormalizeKgTerm(r.Head, consts.KgAliasMap)
|
||||
rel := common.NormalizeKgTerm(r.Relation, consts.KgPredicateAliasMap)
|
||||
tail := common.NormalizeKgTerm(r.Tail, consts.KgAliasMap)
|
||||
if head == "" || rel == "" || tail == "" || head == tail {
|
||||
continue
|
||||
}
|
||||
key := head + "|" + rel + "|" + tail
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
items = append(items, dao.KgRelationItem{Head: head, Relation: rel, Tail: tail})
|
||||
}
|
||||
if err := dao.KgRelation.InsertBatch(ctx, datasetId, chunkId, items); err != nil {
|
||||
return err
|
||||
@@ -166,6 +192,40 @@ func (s *kgEntityService) saveExtract(ctx context.Context, datasetId, chunkId in
|
||||
return nil
|
||||
}
|
||||
|
||||
// graphNotBuiltMsg 与 parse_task_service 写入 error_msg 的图谱未构建标记保持一致
|
||||
const graphNotBuiltMsg = "知识图谱未构建"
|
||||
|
||||
// IsGraphMissing 判定文档知识图谱是否缺失:error_msg 标记过未构建,或该文档分块未产出过关系
|
||||
func (s *kgEntityService) IsGraphMissing(ctx context.Context, doc *entity.Document) (bool, error) {
|
||||
if strings.Contains(doc.ErrorMsg, graphNotBuiltMsg) {
|
||||
return true, nil
|
||||
}
|
||||
has, err := dao.KgEntity.HasGraphByDocument(ctx, doc.Id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return !has, nil
|
||||
}
|
||||
|
||||
// RebuildDocument 重建文档图谱:删旧(实体+关系按该文档分块)→ 全量重抽,返回未成功抽取的分块数
|
||||
func (s *kgEntityService) RebuildDocument(ctx context.Context, datasetId, documentId int64) (int, error) {
|
||||
chunks, _, err := dao.Chunk.ListByDocument(ctx, documentId, 1, 100000)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
chunkIds := make([]int64, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
chunkIds = append(chunkIds, c.Id)
|
||||
}
|
||||
if err := dao.KgEntity.DeleteByChunkIds(ctx, chunkIds); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := dao.KgRelation.DeleteByChunkIds(ctx, chunkIds); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return s.ExtractDocument(ctx, datasetId, documentId)
|
||||
}
|
||||
|
||||
// parseKgJSON 解析模型输出的 JSON,容忍 ```json 代码块包裹及首个 JSON 对象后的多余内容(模型偶尔续写 ",{...}")
|
||||
func parseKgJSON(content string) (*kgExtractResult, error) {
|
||||
content = strings.TrimSpace(content)
|
||||
@@ -224,3 +284,22 @@ func extractFirstJSON(s string) (string, bool) {
|
||||
func (s *kgEntityService) List(ctx context.Context, datasetId int64, page, pageSize int) ([]*entity.KgEntity, int, error) {
|
||||
return dao.KgEntity.List(ctx, datasetId, page, pageSize)
|
||||
}
|
||||
|
||||
// KgEntitySource 实体 → 出现文件清单(图谱页节点标注来源)
|
||||
type KgEntitySource struct {
|
||||
Name string `json:"name"`
|
||||
Files []string `json:"files"`
|
||||
}
|
||||
|
||||
// Sources 数据集全部实体的来源文件(单表拆查 + 内存组装,见 dao.SourcesByDataset)
|
||||
func (s *kgEntityService) Sources(ctx context.Context, datasetId int64) ([]*KgEntitySource, error) {
|
||||
rows, err := dao.KgEntity.SourcesByDataset(ctx, datasetId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]*KgEntitySource, 0, len(rows))
|
||||
for _, r := range rows {
|
||||
out = append(out, &KgEntitySource{Name: r.Name, Files: r.Files})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"rag-local/common"
|
||||
"rag-local/kb/consts"
|
||||
"rag-local/kb/dao"
|
||||
"rag-local/kb/model/entity"
|
||||
)
|
||||
@@ -41,9 +42,12 @@ func (s *kgRelationService) GraphEnhance(ctx context.Context, datasetId int64, q
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// linkEntities 问题分词后按 token 命中实体名的个数打分,取前 topN;同分按名称长度优先
|
||||
// linkEntities 问题分词后按 token 命中实体名的个数打分,取前 topN;同分按名称长度优先。
|
||||
// 别名展开:归一化后的问题包含某别名(按 key 长度降序,最长优先)且标准名在库中,
|
||||
// 标准名以高分直接加入命中(用户用简称提问时能链接到全称实体)。
|
||||
func linkEntities(question string, names []string, topN int) []string {
|
||||
tokens := strings.Fields(common.Tokenize(question))
|
||||
normQ := common.NormalizeKgTerm(question, consts.KgAliasMap)
|
||||
tokens := strings.Fields(common.Tokenize(normQ))
|
||||
if len(tokens) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -51,13 +55,15 @@ func linkEntities(question string, names []string, topN int) []string {
|
||||
name string
|
||||
score int
|
||||
}
|
||||
byName := make(map[string]bool, len(names))
|
||||
var hits []scored
|
||||
for _, n := range names {
|
||||
if n == "" {
|
||||
continue
|
||||
}
|
||||
byName[n] = true
|
||||
score := 0
|
||||
if strings.Contains(question, n) {
|
||||
if strings.Contains(normQ, n) {
|
||||
score += 5 // 问题中出现完整实体名,强相关
|
||||
}
|
||||
for _, t := range tokens {
|
||||
@@ -69,6 +75,27 @@ func linkEntities(question string, names []string, topN int) []string {
|
||||
hits = append(hits, scored{name: n, score: score})
|
||||
}
|
||||
}
|
||||
// 别名展开:最长 key 优先(防"民诉法"吃掉"民诉法解释"),标准名须存在于库中
|
||||
aliasKeys := make([]string, 0, len(consts.KgAliasMap))
|
||||
for k := range consts.KgAliasMap {
|
||||
aliasKeys = append(aliasKeys, k)
|
||||
}
|
||||
sort.Slice(aliasKeys, func(i, j int) bool { return len(aliasKeys[i]) > len(aliasKeys[j]) })
|
||||
hitByName := make(map[string]bool, len(hits))
|
||||
for _, h := range hits {
|
||||
hitByName[h.name] = true
|
||||
}
|
||||
for _, k := range aliasKeys {
|
||||
if !strings.Contains(normQ, k) {
|
||||
continue
|
||||
}
|
||||
std := consts.KgAliasMap[k]
|
||||
if std == "" || !byName[std] || hitByName[std] {
|
||||
continue
|
||||
}
|
||||
hitByName[std] = true
|
||||
hits = append(hits, scored{name: std, score: 10}) // 明确以别名提及,强于 +5 的完整名命中
|
||||
}
|
||||
sort.Slice(hits, func(i, j int) bool {
|
||||
if hits[i].score != hits[j].score {
|
||||
return hits[i].score > hits[j].score
|
||||
|
||||
@@ -20,8 +20,14 @@ var ParseTaskService = &parseTaskService{}
|
||||
|
||||
type parseTaskService struct{}
|
||||
|
||||
// StartParsePoller 启动任务轮询:gtimer 单例定时器串行消费待处理任务(job 未结束不重入,与视频工厂同模式)
|
||||
// StartParsePoller 启动任务轮询:gtimer 单例定时器串行消费待处理任务(job 未结束不重入,与视频工厂同模式)。
|
||||
// 启动时先恢复上次运行遗留的 running 任务(进程异常退出会孤儿化,不恢复则永远不被消费)。
|
||||
func (s *parseTaskService) StartParsePoller(ctx context.Context) {
|
||||
if n, err := dao.ParseTask.ResetRunning(ctx); err != nil {
|
||||
g.Log().Warningf(ctx, "reset running parse tasks failed: %v", err)
|
||||
} else if n > 0 {
|
||||
g.Log().Infof(ctx, "recovered %d running parse task(s) to pending", n)
|
||||
}
|
||||
g.Log().Info(ctx, "parse task poller started")
|
||||
gtimer.AddSingleton(ctx, consts.ParsePollIntervalSeconds*time.Second, func(ctx context.Context) {
|
||||
s.processOne(ctx)
|
||||
@@ -53,7 +59,7 @@ func (s *parseTaskService) processOne(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
if task.TaskType == consts.TaskTypeReembed {
|
||||
s.processReembed(ctx, task)
|
||||
s.processReembed(ctx, task, doc)
|
||||
return
|
||||
}
|
||||
if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"status": consts.DocumentStatusParsing}); err != nil {
|
||||
@@ -128,9 +134,9 @@ func (s *parseTaskService) processOne(ctx context.Context) {
|
||||
kgMsg := ""
|
||||
switch {
|
||||
case kgErr != nil:
|
||||
kgMsg = "知识图谱未构建: " + kgErr.Error()
|
||||
kgMsg = graphNotBuiltMsg + ": " + kgErr.Error()
|
||||
case kgFailed > 0:
|
||||
kgMsg = fmt.Sprintf("知识图谱未构建:%d 个分块抽取失败", kgFailed)
|
||||
kgMsg = fmt.Sprintf(graphNotBuiltMsg+":%d 个分块抽取失败", kgFailed)
|
||||
}
|
||||
if kgMsg != "" {
|
||||
g.Log().Warningf(ctx, "kg extract incomplete for doc %d: %s", doc.Id, kgMsg)
|
||||
@@ -143,18 +149,73 @@ func (s *parseTaskService) processOne(ctx context.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
// processReembed 重新向量化任务:分块文本不变,用数据集当前绑定模型重算全部向量;失败不动文档状态
|
||||
func (s *parseTaskService) processReembed(ctx context.Context, task *entity.ParseTask) {
|
||||
// processReembed 重新向量化任务:分块文本不变,用数据集当前绑定模型重算全部向量。
|
||||
// 过程状态置为向量生成中,失败恢复原状态(旧向量仍可用);重算后若文档知识图谱缺失
|
||||
// (error_msg 标记未构建或实体为空),删旧重抽补建,成功后清空 error_msg 并置为已完成。
|
||||
func (s *parseTaskService) processReembed(ctx context.Context, task *entity.ParseTask, doc *entity.Document) {
|
||||
prevStatus := doc.Status
|
||||
if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"status": consts.DocumentStatusEmbedding}); err != nil {
|
||||
g.Log().Warningf(ctx, "mark doc embedding failed: %v", err)
|
||||
}
|
||||
if _, err := DocumentService.Reembed(ctx, task.DocumentId); err != nil {
|
||||
_ = dao.ParseTask.UpdateStatus(ctx, task.Id, consts.TaskStatusFailed, "重新向量化失败: "+err.Error())
|
||||
_ = dao.Document.UpdateFields(ctx, doc.Id, g.Map{"status": prevStatus})
|
||||
g.Log().Errorf(ctx, "reembed task %d failed: %s", task.Id, err.Error())
|
||||
return
|
||||
}
|
||||
if missing, err := KgEntityService.IsGraphMissing(ctx, doc); err != nil {
|
||||
g.Log().Warningf(ctx, "check kg missing failed for doc %d: %v", doc.Id, err)
|
||||
} else if missing {
|
||||
kgMsg := ""
|
||||
if kgFailed, kgErr := KgEntityService.RebuildDocument(ctx, doc.DatasetId, doc.Id); kgErr != nil {
|
||||
kgMsg = graphNotBuiltMsg + ": " + kgErr.Error()
|
||||
} else if kgFailed > 0 {
|
||||
kgMsg = fmt.Sprintf(graphNotBuiltMsg+":%d 个分块抽取失败", kgFailed)
|
||||
}
|
||||
if kgMsg != "" {
|
||||
g.Log().Warningf(ctx, "kg rebuild incomplete for doc %d: %s", doc.Id, kgMsg)
|
||||
}
|
||||
if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"error_msg": kgMsg}); err != nil {
|
||||
g.Log().Warningf(ctx, "update doc error_msg failed: %v", err)
|
||||
}
|
||||
}
|
||||
if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"status": consts.DocumentStatusDone}); err != nil {
|
||||
g.Log().Warningf(ctx, "mark doc done failed: %v", err)
|
||||
}
|
||||
if err := dao.ParseTask.UpdateStatus(ctx, task.Id, consts.TaskStatusDone, ""); err != nil {
|
||||
g.Log().Errorf(ctx, "mark task done failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Enqueue 单文档入队(解析/重新向量化)。
|
||||
// 已有待处理/运行中任务时返回错误,防重复排队;重新向量化入队即刻把文档置为向量生成中,
|
||||
// 避免等待轮询器捡起(最长一个轮询间隔)期间列表状态无变化、按钮仍可重复点击。
|
||||
func (s *parseTaskService) Enqueue(ctx context.Context, documentId int64, taskType string) error {
|
||||
doc, err := dao.Document.GetOne(ctx, documentId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if doc == nil {
|
||||
return gerror.New("文档不存在")
|
||||
}
|
||||
ok, err := dao.ParseTask.HasPending(ctx, documentId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
return gerror.New("该文档已有待处理或处理中的任务")
|
||||
}
|
||||
if _, err := dao.ParseTask.Insert(ctx, documentId, doc.DatasetId, taskType); err != nil {
|
||||
return err
|
||||
}
|
||||
if taskType == consts.TaskTypeReembed {
|
||||
if err := dao.Document.UpdateFields(ctx, documentId, g.Map{"status": consts.DocumentStatusEmbedding}); err != nil {
|
||||
g.Log().Warningf(ctx, "mark doc %d embedding failed: %v", documentId, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *parseTaskService) fail(ctx context.Context, task *entity.ParseTask, msg string) {
|
||||
_ = dao.ParseTask.UpdateStatus(ctx, task.Id, consts.TaskStatusFailed, msg)
|
||||
_ = dao.Document.UpdateFields(ctx, task.DocumentId, g.Map{
|
||||
|
||||
@@ -7,3 +7,7 @@ export function listEntities(params) {
|
||||
export function listRelations(params) {
|
||||
return request.get('/kg-relation/list', { params })
|
||||
}
|
||||
|
||||
export function listEntitySources(params) {
|
||||
return request.get('/kg-entity/sources', { params })
|
||||
}
|
||||
|
||||
@@ -83,6 +83,7 @@
|
||||
<el-collapse-item :title="`法律依据(${r.laws.length})`">
|
||||
<div v-for="(law, i) in r.laws" :key="i" class="risk-law">
|
||||
<span class="law-title">《{{ law.law_title }}》{{ law.law_item }}</span>
|
||||
<span v-if="law.source_file" class="law-src">来源:{{ law.source_file }}</span>
|
||||
<div class="law-content">{{ law.content }}</div>
|
||||
</div>
|
||||
</el-collapse-item>
|
||||
@@ -118,6 +119,7 @@
|
||||
<div class="summary-item-desc">{{ item.risks[0].desc }}</div>
|
||||
<div v-for="(law, j) in (item.risks[0].laws || [])" :key="j" class="summary-law">
|
||||
<span class="law-title">《{{ law.law_title }}》{{ law.law_item }}</span>
|
||||
<span v-if="law.source_file" class="law-src">来源:{{ law.source_file }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -401,6 +403,11 @@ const detailTitle = computed(() => detail.value ? detail.value.task.filename : '
|
||||
font-weight: 600;
|
||||
color: #303133;
|
||||
}
|
||||
.law-src {
|
||||
font-size: 12px;
|
||||
color: #909399;
|
||||
margin-left: 6px;
|
||||
}
|
||||
.law-content {
|
||||
font-size: 12px;
|
||||
color: #909399;
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
<template #default="{ row }">
|
||||
<el-button link type="primary" @click="openChunks(row)">分块</el-button>
|
||||
<el-button v-if="row.status === 5" link type="warning" @click="retry(row)">重试</el-button>
|
||||
<el-button link type="primary" @click="reembed(row)">重新向量化</el-button>
|
||||
<el-button link type="primary" :disabled="isBusy(row.status)" @click="reembed(row)">重新向量化</el-button>
|
||||
<el-button link type="danger" @click="remove(row)">删除</el-button>
|
||||
</template>
|
||||
</el-table-column>
|
||||
@@ -135,6 +135,8 @@ const statusMap = {
|
||||
|
||||
function statusText(s) { return (statusMap[s] || {}).text || s }
|
||||
function statusType(s) { return (statusMap[s] || {}).type || 'info' }
|
||||
// 处理中(解析/向量化/图谱构建),期间禁止重新向量化
|
||||
function isBusy(s) { return s === 1 || s === 2 || s === 3 }
|
||||
|
||||
function formatSize(bytes) {
|
||||
if (!bytes) return '-'
|
||||
@@ -219,8 +221,10 @@ async function reembed(row) {
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
const d = await reembedDocument(row.id)
|
||||
ElMessage.success(`已重新向量化 ${d.count} 个分块`)
|
||||
await reembedDocument(row.id)
|
||||
ElMessage.success('已入队,正在重新向量化')
|
||||
await load()
|
||||
startPolling()
|
||||
}
|
||||
|
||||
async function openChunks(row) {
|
||||
|
||||
@@ -27,7 +27,9 @@
|
||||
<el-table :data="entities" size="small" height="440" v-loading="loading">
|
||||
<el-table-column prop="name" label="实体" min-width="120" show-overflow-tooltip />
|
||||
<el-table-column prop="entity_type" label="类型" width="100" />
|
||||
<el-table-column prop="chunk_id" label="来源分块" width="90" />
|
||||
<el-table-column label="来源文件" min-width="150" show-overflow-tooltip>
|
||||
<template #default="{ row }">{{ srcText(row.name) }}</template>
|
||||
</el-table-column>
|
||||
</el-table>
|
||||
<el-pagination class="kg-page-bar" small layout="prev, pager, next" :total="entityTotal"
|
||||
:page-size="entityPageSize" v-model:current-page="entityPage" @current-change="loadEntities" />
|
||||
@@ -55,7 +57,7 @@ import { GraphChart } from 'echarts/charts'
|
||||
import { TooltipComponent, LegendComponent } from 'echarts/components'
|
||||
import { CanvasRenderer } from 'echarts/renderers'
|
||||
import { listDatasets } from '../api/dataset.js'
|
||||
import { listEntities, listRelations } from '../api/kg.js'
|
||||
import { listEntities, listRelations, listEntitySources } from '../api/kg.js'
|
||||
|
||||
echarts.use([GraphChart, TooltipComponent, LegendComponent, CanvasRenderer])
|
||||
|
||||
@@ -78,9 +80,15 @@ const relationPage = ref(1)
|
||||
const relationPageSize = 20
|
||||
|
||||
const sideTab = ref('entity')
|
||||
const srcMap = ref({}) // 实体名 → 来源文件列表(/kg-entity/sources)
|
||||
let chart = null
|
||||
let resizeObserver = null
|
||||
|
||||
function srcText(name) {
|
||||
const files = srcMap.value[name] || []
|
||||
return files.length ? files.join('、') : '-'
|
||||
}
|
||||
|
||||
const graphCountText = computed(() => `${truncated.value ? '≈' : ''}${graphNodes.value.length} 实体 / ${graphLinks.value.length} 关系`)
|
||||
const graphNodes = ref([])
|
||||
const graphLinks = ref([])
|
||||
@@ -104,10 +112,14 @@ async function load() {
|
||||
relationPage.value = 1
|
||||
loading.value = true
|
||||
try {
|
||||
const [allEntities, allRelations] = await Promise.all([
|
||||
const [allEntities, allRelations, allSources] = await Promise.all([
|
||||
listEntities({ dataset_id: datasetId.value, page: 1, page_size: 100000 }),
|
||||
listRelations({ dataset_id: datasetId.value, page: 1, page_size: 100000 }),
|
||||
listEntitySources({ dataset_id: datasetId.value }),
|
||||
])
|
||||
const map = {}
|
||||
for (const s of allSources?.list || []) map[s.name] = s.files || []
|
||||
srcMap.value = map
|
||||
buildGraph(allEntities?.list || [], allRelations?.list || [])
|
||||
await Promise.all([loadEntities(), loadRelations()])
|
||||
} finally {
|
||||
@@ -156,6 +168,7 @@ function buildGraph(entitiesList, relationsList) {
|
||||
name: n.name,
|
||||
category: n.entity_type || '未分类',
|
||||
symbolSize: sizeFor(degree.get(n.name) || 0),
|
||||
files: srcMap.value[n.name] || [],
|
||||
}))
|
||||
graphLinks.value = edges
|
||||
renderGraph()
|
||||
@@ -174,7 +187,9 @@ function renderGraph() {
|
||||
const e = graphLinks.value[p.dataIndex]
|
||||
return `${e.source} → ${e.target}<br/>关系:${e.relation}`
|
||||
}
|
||||
return `${p.name}<br/>类型:${p.data.category}`
|
||||
const files = p.data.files || []
|
||||
const src = files.length ? files.join('、') : '未知'
|
||||
return `${p.name}<br/>类型:${p.data.category}<br/>来源:${src}`
|
||||
},
|
||||
},
|
||||
series: [{
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
8. **文档状态机 6 态**:解析 → 向量生成 → 图谱构建分三段推进(`0 待处理/1 解析中/2 向量生成中/3 图谱构建中/4 已完成/5 失败`)。图谱抽取失败**不阻断**完成:文档仍置 4,`error_msg` 记录「知识图谱未构建」原因,前端以黄色标签提示(不再出现"显示已完成但图谱没建完"的假象)。
|
||||
9. **合同标注宁滥毋缺**:标注业务的召回策略与问答相反——问答要精(topK=5 + 重排门槛 max(最高分×50%, 6)),标注宁滥毋缺(漏标比多标严重)。召回放宽(每数据集向量+FTS 各 15 条)、不做重排门槛、全部候选交 LLM 判定后保留(含 0 分),见 §7.8。
|
||||
10. **轮询任务并发模型**:`StartParsePoller` 与 `StartAnnotationPoller` 各自**gtimer 单例定时器串行**消费(5 秒间隔,job 未结束不重入),不并发处理多个任务,避免 SQLite 写冲突;任务粒度(kb_parse_task / kb_contract_task)+ 子粒度(clause)断点续跑。任务内部热点(kg 逐 chunk 抽取、标注逐条款、多数据集召回、问答双路检索+图增强)用 **grpool 协程池并行化**,池大小 config.yml `pool` 段配置——并行段只做读查询与 LLM/Embedding 调用,SQLite 写全部收敛回主 goroutine 串行(见 §7.9)。
|
||||
11. **SQL 单表约束**:每个 SQL 只访问一张表,禁止 JOIN 与跨表子查询(`IN (SELECT ...)` / `EXISTS`);跨表数据拆多条单表 SQL + 应用层内存组装(先取外键 id 列表再 `IN` 目标表,`IN` 参数按 ≤100 分批防 SQLite 999 变量上限)。影响实例:vec0 检索(无 dataset 列,先 vec 候选再按 chunk 表过滤)、实体来源溯源(4 条单表查询内存组装)、文档图谱判定(先 chunk id 再数关系)、合同标注按任务查询(先 clause id 再 IN 分批)。
|
||||
|
||||
---
|
||||
|
||||
@@ -55,7 +56,7 @@
|
||||
2. **文件名不可信**:上传文件一律 `RandomToken(16)` 随机重命名,数据库不存用户原始路径(`filename` 仅作展示名);按日期分子目录,天然按月/日归档。
|
||||
3. **同生共死顺序**:新增 = 先写文件成功 → 再插记录(插失败删文件);删除 = 先删数据成功 → 再删文件(删文件失败仅记日志,孤儿文件可接受;反之删了文件留着记录会导致解析/导出直接报错)。
|
||||
4. **删除一致性**:
|
||||
- 删文档:`DocumentService.Delete` 单事务内删 `kg 两表(按 chunk_id)→ chunk → vec → fts → kb_parse_task` → 删 `kb_document` → `os.Remove` 文件(chunk_dao.go 的 `DeleteByDocument` 单事务完成,**无 FTS5 optimize**)
|
||||
- 删文档:`DocumentService.Delete` 单事务内删 `kg 两表(实体/关系,按 chunk_id)→ chunk → vec → fts → kb_parse_task` → 删 `kb_document` → `os.Remove` 文件(chunk_dao.go 的 `DeleteByDocument` 单事务完成,**无 FTS5 optimize**)
|
||||
- 删合同任务:`AnnotationService.Delete` 删 `mark(join clause 定位)→ clause → task` → `os.Remove` 文件
|
||||
5. **备份即两目录打包**:`data/`(3 个 db,小而关键)+ `workspace/`(大而可重建),二者独立迁移互不影响。
|
||||
|
||||
@@ -73,7 +74,7 @@
|
||||
kb_dataset --1:N-- kb_document --1:N-- kb_chunk --1:1-- kb_chunk_vec (vec0, chunk_id 主键)
|
||||
│ │ │ │ --1:1-- kb_chunk_fts (FTS5, chunk_id 唯一)
|
||||
│ │ └--1:N-- kb_parse_task (document_id)
|
||||
│ │ └--1:N-- kg_entity (chunk_id 来源, (dataset_id,name) 去重 upsert)
|
||||
│ │ └--1:N-- kg_entity (chunk_id 来源弱引用, (dataset_id,name) 去重 upsert)
|
||||
│ │ └--1:N-- kg_relation (chunk_id 来源, head/relation/tail 文本快照)
|
||||
│ └.. 弱引用: kb_contract_mark.chunk_id(标注命中法条分块)
|
||||
│
|
||||
@@ -155,7 +156,7 @@ func (h *HybridRetriever) Retrieve(ctx context.Context, query string, opts ...re
|
||||
```
|
||||
|
||||
流程(实际常量:`VectorTopK=FtsTopK=20`、`RrfK=60`、`RerankTopK=10`、`HybridTopK=5`):
|
||||
1. **向量检索**:query → Embedding → `vec0` KNN(L2 距离)预取 `topK*4` 后 join `kb_chunk` 按 dataset_id 过滤,取 20
|
||||
1. **向量检索**:query → Embedding → `vec0` KNN(L2 距离)预取 `topK*4` 候选,再单表查 `kb_chunk` 按 dataset_id 过滤(单表约束:内存组装,候选已按距离排序故过滤后取前 topK 等价原 JOIN),取 20
|
||||
2. **关键词检索**:query → `TokenizeQuery` 分词(引号词组 OR 语义、过滤 <2 rune 与 FTS 特殊字符)→ FTS5 MATCH(bm25 负分升序)取 20
|
||||
3. **RRF 融合**:`score = Σ 1/(60 + rank)` 全局融合,截 `RerankTopK=10`
|
||||
4. **LLM 重排**:`rerankByLLM` 一次调用为 10 个候选打 0-10 分(候选截 `RerankMaxChars=500` 字),**门槛 = max(最高分×`RerankKeepRatio=0.5`, `RerankMinScore=6`)**,低于门槛剔除
|
||||
@@ -252,6 +253,8 @@ StartParsePoller(main.go 启动,gtimer 单例 5 秒轮询,job 未结束不
|
||||
{"entities": [{"name": "张明", "type": "person"}], "relations": [{"head": "张明", "relation": "任职于", "tail": "XX科技"}]}
|
||||
```
|
||||
- 实体按 `(dataset_id, name)` 去重(同一实体多 chunk 出现只建一次,UNIQUE 约束 + ON CONFLICT upsert),关系带来源 chunk_id;删除文档时按分块 id 级联清理
|
||||
- **实体名/关系谓词归一化**(防别名分裂,见 §7.10):实体名与关系 head/tail 落库前经 `common.NormalizeKgTerm`(全半角、去空白/书名号/版本注记括号/「中华人民共和国」前缀、别名表全串映射),关系谓词经谓词别名表(`KgPredicateAliasMap`,如「颁布」→「发布」)归并;归一化后 chunk 内实体按名、关系按 `head|relation|tail` 去重。别名表为保守内置集(`kb/consts/kg_alias.go`),只做高置信合并,不做模糊相似度合并
|
||||
- **实体多文件溯源**(不建关联表,见 §7.10):有关系的实体经关系表溯源(关系不去重、每条带来源 chunk_id,JOIN `kb_chunk` → `kb_document` 得完整出现文件清单);孤立实体(无关系)由实体行 `chunk_id` 弱引用兜底(`GET /kg-entity/sources`,图谱页节点标注来源)
|
||||
- 抽取**失败不阻断**:无默认对话模型、或部分分块抽取失败,文档仍置 4 已完成,但 `error_msg` 记「知识图谱未构建:…」(未配置默认对话模型 / N 个分块抽取失败),前端文档列表黄色标签提示(`status===4 && error_msg`)
|
||||
|
||||
**使用(图增强检索,挂在 HybridRetriever 之后)**:
|
||||
@@ -287,9 +290,12 @@ StartParsePoller(main.go 启动,gtimer 单例 5 秒轮询,job 未结束不
|
||||
```
|
||||
|
||||
- **条款切分**:按优先级探测三种行首正则(`第X条` / `\d+(.\d+)*[、..]` / `中文数字[、..]`),命中 ≥2 采用,否则整篇单条(title=「全文」);title=标记、content=标记行至下一标记全文
|
||||
- **召回**:每 dataset 各调 `VecSearch(dsId, vec, 15)` + `FtsSearch(dsId, 分词截 200 字, 15)`,全局按 RRF(`1/(RrfK+rank+1)`) 融合截 `AnnoMaxCandidates=60`;embedder 按 dataset 绑定的 embedding 配置构建并缓存;**无候选的条款视为完成**(跳过 LLM 调用,无 mark)
|
||||
- **判定**:单次 LLM 调用,prompt 含条款全文(截 `AnnoMaxClauseChars=2000`)+ 编号候选(每条截 `RerankMaxChars=500` 字),输出 `{"marks":[{"cand_id":1,"law_item":"第四十四条","score":9,"reason":"..."}]}`;**law_item 由 LLM 判定输出**(法条编号,`第X条` 正则兜底提取);JSON 解析与 rerankByLLM 同套路(```json 提取 + 截首尾花括号);**不做门槛过滤,全部候选(含 0 分)按分降序保留**
|
||||
- **召回**:每 dataset 各调 `VecSearch(dsId, vec, 15)` + `FtsSearch(dsId, 分词截 200 字, 15)`,全局按 RRF(`1/(RrfK+rank+1)`) 融合截 `AnnoMaxCandidates=60`;候选 chunk 内容**批量加载**(`ListByIds` 一次 IN 查回内存映射,禁止逐条 GetOne——N+1);embedder 按 dataset 绑定的 embedding 配置构建并缓存;**无候选的条款视为完成**(跳过 LLM 调用,无 mark)
|
||||
- **判定**:单次 LLM 调用,prompt 含条款全文(截 `AnnoMaxClauseChars=2000`)+ 编号候选,输出 `{"marks":[{"cand_id":1,"law_item":"第四十四条","score":9,"reason":"..."}]}`;**law_item 由 LLM 判定输出**(法条编号,`第X条` 正则兜底提取);JSON 解析与 rerankByLLM 同套路(```json 提取 + 截首尾花括号);**不做门槛过滤,全部候选(含 0 分)按分降序保留**
|
||||
- **判定 prompt 上下文预算**:本地 4B 模型 context 8192 token,60 候选 × 2000 字会把上下文撑爆(`Context size has been exceeded`)。三道闸(缺一不可):① **关闭思维链**——gemma-4-E4B 是推理模型,默认输出长思考链占输出 token(`extra["thinking"]=false`,与 kg_extract 同策略,池提交前预置);② **显式 `max_tokens=AnnoJudgeMaxTokens=1024`**——不传时服务端默认预算大,推理模型可无限思考烧穿上下文;③ 候选块按 `AnnoJudgePromptBudget=1500` 字总预算**贪心填充**(按 RRF 相关度降序,每条截 `AnnoJudgeCandidateChars=300` 字,首条保底入队,超预算截断后续候选)。条文精确文本不依赖 prompt 全文——`extractLawItem` 用未截断的 `ContentFull` 抽取;LLM 引用编号受展示条数约束(越界引用丢弃)
|
||||
- **快照落库**:mark 存命中 chunk 的法条快照(law_title=dataset 名、law_item=LLM 判定的法条编号、content=chunk 内容截断 800 字),标注结果不随语料变更失效
|
||||
- **批量落库(禁逐条 SQL)**:条款插入与风险快照用 `Batch(100)` 多行 INSERT(GoFrame 一次语句写 100 行,8 列 ≈800 变量 < SQLite 999 上限);进行中/完成状态用 `UpdateStatuses`(WHERE id IN,≤100 分批)各刷一次;进度按**本地计数**每完成一条刷一次 `UpdateProgress`(不再逐条款 `ListByTask` 读库统计)。仅失败路径保留逐条 `UpdateStatus`(error_msg 各异的罕见路径,不批量)
|
||||
- **来源文件标注**:法条引用快照同时记录源文件名(`LawRef.source_file`,如「劳动合同法.pdf」)——law_title 只是 dataset 名(如「法律」),看不出出自哪部法文件,溯源到 chunk 才能定位。判定时按单表约束拆两条 SQL(`kb_chunk` 按 id 批量查 document_id、`kb_document` 按 id 批量查 filename,IN ≤100 分批)内存组装;界面「法律依据」与导出 HTML 显示「来源:xx.pdf」;溯源失败仅告警不阻断标注(展示增强,非判定依据)
|
||||
- **容错**:单条款 LLM 失败仅该条 failed(记 error_msg)不重试;**任务仍置 Done**,任务 error_msg 记「N 条条款标注失败」(前端可见);未配置默认 chat 模型 → 任务失败
|
||||
- **断点续跑**:任务按 `status IN (0,1)` 领取,clause 粒度续跑(已 done 不重复产生 mark;重跑任务先 DeleteByClause 幂等重建)
|
||||
- **导出**:`AnnotatedHTML` 生成自包含 HTML(条款 + 内嵌标注,score ≥8 绿 / ≥5 蓝 / 其余灰,打印按钮 `window.print()`);controller 直接写响应体(中间件检测已写入则不包装 JSON),前端原生 fetch + 手动 Authorization 获取 blob
|
||||
@@ -303,6 +309,34 @@ StartParsePoller(main.go 启动,gtimer 单例 5 秒轮询,job 未结束不
|
||||
- 问答(§7.3/§7.4):`Ask` 中 `retrieve`(ChatPool,内部再并行 vec/fts)与 `GraphEnhance`(纯读,主 goroutine 直接跑)并行;`Retrieve` 内 vec 段与 fts 段并行(`vecRetrieve`/`ftsRetrieve`),RRF 合并、LLM 重排保持串行
|
||||
- 任务级仍由轮询器单 goroutine 串行消费(决策 10),并行只发生在任务内部
|
||||
|
||||
### 7.10 实体名归一化与多文件溯源(P0)
|
||||
|
||||
**背景**:实体按 `(dataset_id, name)` 精确字符串去重,法律文档全称/简称混用(「中华人民共和国刑法」vs「刑法」、「民诉法」vs「民事诉讼法」)把同一实体拆成多行,切断了跨文件隐式关联(一跳邻居以共享实体名为桥)。
|
||||
|
||||
**归一化规则**(`common/kg_name_normalize.go`,纯函数、幂等、保守防误合并):
|
||||
|
||||
| 规则 | 示例 | 边界 |
|
||||
|---|---|---|
|
||||
| 全角→半角、去首尾/压缩内部空白 | `“中华人民共和国刑法 ”` → `中华人民共和国刑法` | — |
|
||||
| 去书名号《》 | `《民事诉讼法》` → `民事诉讼法` | — |
|
||||
| 去尾部标点 | `民事诉讼法。` → `民事诉讼法` | — |
|
||||
| 去版本注记括号及内容 | `刑法(2020修正)` → `刑法` | 仅剥含 年/修正/修订/施行/数字 的括号;「张三(北京分公司)」**不剥** |
|
||||
| 去「中华人民共和国」前缀 | `中华人民共和国刑法` → `刑法` | 仅前缀;结果为空则**保留原名**(防实体「中华人民共和国」被删) |
|
||||
| 别名全串精确映射 | `民诉法` → `民事诉讼法` | 全串匹配非 contains 替换(防「民诉法」吃掉「民诉法解释」);映射为空保留原名 |
|
||||
|
||||
别名表 `KgAliasMap`(变体→标准名)与谓词表 `KgPredicateAliasMap`(如 颁布→发布、施行→实施)内置在 `kb/consts/kg_alias.go`,由调用方传入 `common.NormalizeKgTerm`(common 不依赖 kb,分层约束)。
|
||||
|
||||
**写入路径**(`saveExtract`):实体名、关系 head/relation/tail 先归一化 → 过滤空名/自环 → chunk 内实体按名、关系按 `head|relation|tail` 去重 → 落库。
|
||||
|
||||
**溯源设计**(回答"去重后怎么知道实体出现在哪些文件"):
|
||||
|
||||
- 实体行 `chunk_id` 仅「最近来源」弱引用(每次 upsert 覆盖),**不作为溯源依据**
|
||||
- 溯源组合查询(单表约束:拆 4 条单表 SQL——kg_relation 全量 head/tail+chunk_id、kg_entity 全量 name+chunk_id、kb_chunk 全量 id+document_id、kb_document 全量 id+filename,内存组装):① 关系表溯源——有关系的实体得**完整**出现文件清单(关系不去重天然保留来源);② 实体行弱引用兜底——孤立实体(无任何关系)至少给出最近来源文件
|
||||
- **不建实体-分块关联表**(`kg_entity_chunk` 决策否决):关联表唯一增量价值是"孤立实体的精确多文件溯源",在图谱页标注来源的展示场景价值低,而成本实打实(一张表 + 写入多一次查询 + 删文档/重建图谱/迁移三个清理点)——够用即止
|
||||
- 图谱页(KgGraph.vue)节点 tooltip 与实体表格展示来源文件(`GET /kg-entity/sources`)
|
||||
|
||||
**图谱缺失判定修复**:`CountByDocument` 原按实体行 `chunk_id` 计数,跨文件 upsert 覆盖后**漏计**导致 `IsGraphMissing` 误报;改为关系表 EXISTS(关系每 chunk 至少一条、不受覆盖影响)。
|
||||
|
||||
---
|
||||
|
||||
## 8. 鉴权实现细节
|
||||
|
||||
Reference in New Issue
Block a user