This commit is contained in:
2026-08-05 16:42:12 +08:00
parent dc767d83f9
commit 139f6ccfd6
25 changed files with 647 additions and 104 deletions
-1
View File
@@ -1,5 +1,4 @@
# 运行时数据与本地环境
data/
workspace/
.idea/
.DS_Store
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
BIN
View File
Binary file not shown.
+3 -3
View File
@@ -11,9 +11,9 @@ const (
DefaultChunkSize = 800 // 分块最大字数(数据集默认值)
DefaultChunkOverlap = 150 // 分块重叠字数(数据集默认值)
ChunkStrategyTitle = "title" // 标题感知分块(自研,保留标题与段落
ChunkStrategyRecursive = "recursive" // 递归字符分块(Eino recursive
ChunkStrategySemantic = "semantic" // 语义分块(Eino semantic,需绑定向量模型)
// 全局设置键(app_config 表
SettingsKeyChunkSize = "chunk_default_size"
SettingsKeyChunkOverlap = "chunk_default_overlap"
ParsePollIntervalSeconds = 3 // 解析任务轮询间隔
+1
View File
@@ -2,6 +2,7 @@ package consts
const (
TableNameModelConfig = "model_config"
TableNameAppConfig = "app_config"
TableNameDataset = "kb_dataset"
TableNameDocument = "kb_document"
TableNameChunk = "kb_chunk"
-1
View File
@@ -28,7 +28,6 @@ func (c *dataset) Save(ctx context.Context, req *dto.SaveDatasetReq) (*dto.SaveD
EmbeddingCfgId: req.EmbeddingCfgId,
ChunkSize: req.ChunkSize,
ChunkOverlap: req.ChunkOverlap,
ChunkStrategy: req.ChunkStrategy,
Status: 1,
})
if err != nil {
+15
View File
@@ -18,3 +18,18 @@ func (c *systemConfig) Login(ctx context.Context, req *dto.LoginReq) (res *dto.L
}
return &dto.LoginRes{Token: token}, nil
}
func (c *systemConfig) GetSettings(ctx context.Context, _ *dto.GetSettingsReq) (*dto.GetSettingsRes, error) {
size, overlap, err := service.SystemConfigService.GetSettings(ctx)
if err != nil {
return nil, err
}
return &dto.GetSettingsRes{ChunkSize: size, ChunkOverlap: overlap}, nil
}
func (c *systemConfig) SaveSettings(ctx context.Context, req *dto.SaveSettingsReq) (*dto.SaveSettingsRes, error) {
if err := service.SystemConfigService.SaveSettings(ctx, req.ChunkSize, req.ChunkOverlap); err != nil {
return nil, err
}
return &dto.SaveSettingsRes{}, nil
}
+44
View File
@@ -0,0 +1,44 @@
package dao
import (
"context"
"rag-local/kb/consts"
"github.com/gogf/gf/v2/frame/g"
)
var AppConfig = &appConfigDao{}
type appConfigDao struct{}
func init() {
ctx := context.Background()
_, err := g.DB(consts.DbGroupSystem).Exec(ctx, `CREATE TABLE IF NOT EXISTS `+consts.TableNameAppConfig+` (
cfg_key TEXT PRIMARY KEY,
cfg_value TEXT NOT NULL DEFAULT '',
updated_at DATETIME DEFAULT (datetime('now','localtime'))
)`)
if err != nil {
g.Log().Warningf(ctx, "create app_config table failed: %v", err)
}
}
// GetInt 读取全局设置值,未设置或非法时返回默认值 def
func (d *appConfigDao) GetInt(ctx context.Context, key string, def int) int {
r, err := g.DB(consts.DbGroupSystem).Ctx(ctx).GetValue(ctx,
"SELECT cfg_value FROM "+consts.TableNameAppConfig+" WHERE cfg_key=?", key)
if err != nil || r.IsEmpty() {
return def
}
return r.Int()
}
// SetInt 写入全局设置值(UPSERT
func (d *appConfigDao) SetInt(ctx context.Context, key string, v int) error {
_, err := g.DB(consts.DbGroupSystem).Ctx(ctx).Exec(ctx,
"INSERT INTO "+consts.TableNameAppConfig+"(cfg_key, cfg_value) VALUES(?, ?) "+
"ON CONFLICT(cfg_key) DO UPDATE SET cfg_value=excluded.cfg_value, updated_at=datetime('now','localtime')",
key, v)
return err
}
+25 -3
View File
@@ -95,7 +95,12 @@ func (d *chunkDao) InsertWithVec(ctx context.Context, datasetId, documentId int6
if err != nil {
return 0, err
}
defer func() { _ = tx.Rollback() }()
// Commit 成功后 IsClosed 为 true,跳过 Rollback,避免对已提交事务回滚产生报错日志
defer func() {
if !tx.IsClosed() {
_ = tx.Rollback()
}
}()
r, err := tx.Model(consts.TableNameChunk).Ctx(ctx).Data(g.Map{
"dataset_id": datasetId,
@@ -133,7 +138,12 @@ func (d *chunkDao) DeleteByDocument(ctx context.Context, documentId int64) error
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
// Commit 成功后 IsClosed 为 true,跳过 Rollback,避免对已提交事务回滚产生报错日志
defer func() {
if !tx.IsClosed() {
_ = tx.Rollback()
}
}()
ids := tx.Model(consts.TableNameChunk).Ctx(ctx).Fields("id").Where("document_id", documentId)
r, err := ids.Array()
@@ -158,6 +168,13 @@ func (d *chunkDao) DeleteByDocument(ctx context.Context, documentId int64) error
if _, err := tx.Exec("DELETE FROM "+consts.TableNameChunkFts+" WHERE chunk_id IN ("+in+")", args...); err != nil {
return err
}
// 知识图谱数据按 chunk 关联,重新解析时旧 chunk 消失,一并清理避免孤儿数据
if _, err := tx.Exec("DELETE FROM "+consts.TableNameKgRelation+" WHERE chunk_id IN ("+in+")", args...); err != nil {
return err
}
if _, err := tx.Exec("DELETE FROM "+consts.TableNameKgEntity+" WHERE chunk_id IN ("+in+")", args...); err != nil {
return err
}
}
if _, err := tx.Model(consts.TableNameChunk).Ctx(ctx).Where("document_id", documentId).Delete(); err != nil {
return err
@@ -170,7 +187,12 @@ func (d *chunkDao) UpdateContent(ctx context.Context, id int64, content, vecJson
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
// Commit 成功后 IsClosed 为 true,跳过 Rollback,避免对已提交事务回滚产生报错日志
defer func() {
if !tx.IsClosed() {
_ = tx.Rollback()
}
}()
if _, err := tx.Model(consts.TableNameChunk).Ctx(ctx).Data(g.Map{"content": content}).Where("id", id).Update(); err != nil {
return err
+6 -1
View File
@@ -86,7 +86,12 @@ func (d *conversationDao) Delete(ctx context.Context, id int64) error {
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
// Commit 成功后 IsClosed 为 true,跳过 Rollback,避免对已提交事务回滚产生报错日志
defer func() {
if !tx.IsClosed() {
_ = tx.Rollback()
}
}()
if _, err := tx.Model(consts.TableNameMessage).Ctx(ctx).Where("conversation_id", id).Delete(); err != nil {
return err
}
+10 -2
View File
@@ -26,6 +26,8 @@ func init() {
chunk_size INTEGER NOT NULL DEFAULT 800,
chunk_overlap INTEGER NOT NULL DEFAULT 150,
chunk_strategy TEXT NOT NULL DEFAULT 'title',
unit_pattern TEXT NOT NULL DEFAULT '',
context_pattern TEXT NOT NULL DEFAULT '',
status INTEGER NOT NULL DEFAULT 1,
created_at DATETIME DEFAULT (datetime('now','localtime')),
updated_at DATETIME DEFAULT (datetime('now','localtime'))
@@ -41,6 +43,8 @@ func init() {
{"chunk_size", "chunk_size INTEGER NOT NULL DEFAULT 800"},
{"chunk_overlap", "chunk_overlap INTEGER NOT NULL DEFAULT 150"},
{"chunk_strategy", "chunk_strategy TEXT NOT NULL DEFAULT 'title'"},
{"unit_pattern", "unit_pattern TEXT NOT NULL DEFAULT ''"},
{"context_pattern", "context_pattern TEXT NOT NULL DEFAULT ''"},
} {
cnt, err := g.DB(consts.DbGroupDefault).Ctx(ctx).GetValue(ctx,
"SELECT COUNT(*) FROM pragma_table_info('"+consts.TableNameDataset+"') WHERE name=?", col.name)
@@ -83,7 +87,6 @@ func (d *datasetDao) Insert(ctx context.Context, data *entity.Dataset) (int64, e
"embedding_cfg_id": data.EmbeddingCfgId,
"chunk_size": data.ChunkSize,
"chunk_overlap": data.ChunkOverlap,
"chunk_strategy": data.ChunkStrategy,
"status": data.Status,
"created_at": now,
"updated_at": now,
@@ -101,7 +104,6 @@ func (d *datasetDao) Update(ctx context.Context, data *entity.Dataset) error {
"embedding_cfg_id": data.EmbeddingCfgId,
"chunk_size": data.ChunkSize,
"chunk_overlap": data.ChunkOverlap,
"chunk_strategy": data.ChunkStrategy,
"status": data.Status,
"updated_at": gtime.Now().Format("Y-m-d H:i:s"),
}).Where("id", data.Id).Update()
@@ -113,6 +115,12 @@ func (d *datasetDao) Delete(ctx context.Context, id int64) error {
return err
}
func (d *datasetDao) UpdateFields(ctx context.Context, id int64, data g.Map) error {
_, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameDataset).Ctx(ctx).Data(data).
Where("id", id).Update()
return err
}
func (d *datasetDao) GetEmbeddingCfgId(ctx context.Context, id int64) (int64, error) {
r, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameDataset).Ctx(ctx).
Fields("embedding_cfg_id").Where("id", id).One()
-1
View File
@@ -22,7 +22,6 @@ type SaveDatasetReq struct {
EmbeddingCfgId int64 `json:"embedding_cfg_id"`
ChunkSize int `json:"chunk_size"`
ChunkOverlap int `json:"chunk_overlap"`
ChunkStrategy string `json:"chunk_strategy"`
}
type SaveDatasetRes struct {
+17
View File
@@ -10,3 +10,20 @@ type LoginReq struct {
type LoginRes struct {
Token string `json:"token"`
}
type GetSettingsReq struct {
g.Meta `path:"/settings" method:"get" tags:"系统配置" summary:"获取全局设置"`
}
type GetSettingsRes struct {
ChunkSize int `json:"chunk_size"` // 默认分块大小
ChunkOverlap int `json:"chunk_overlap"` // 默认重叠字数
}
type SaveSettingsReq struct {
g.Meta `path:"/save-settings" method:"post" tags:"系统配置" summary:"保存全局设置"`
ChunkSize int `json:"chunk_size"`
ChunkOverlap int `json:"chunk_overlap"`
}
type SaveSettingsRes struct{}
+2 -1
View File
@@ -9,7 +9,8 @@ type Dataset struct {
EmbeddingCfgId int64 `orm:"embedding_cfg_id" json:"embedding_cfg_id"`
ChunkSize int `orm:"chunk_size" json:"chunk_size"`
ChunkOverlap int `orm:"chunk_overlap" json:"chunk_overlap"`
ChunkStrategy string `orm:"chunk_strategy" json:"chunk_strategy"`
UnitPattern string `orm:"unit_pattern" json:"unit_pattern"`
ContextPattern string `orm:"context_pattern" json:"context_pattern"`
Status int `orm:"status" json:"status"`
CreatedAt *gtime.Time `orm:"created_at" json:"created_at"`
UpdatedAt *gtime.Time `orm:"updated_at" json:"updated_at"`
+188 -20
View File
@@ -2,6 +2,7 @@ package service
import (
"context"
"regexp"
"strings"
"unicode/utf8"
@@ -24,34 +25,201 @@ import (
var chunkSeparators = []string{"\n\n", "\n", "。", "", "", "", "", " ", ""}
var runeLen = utf8.RuneCountInString
// SplitByStrategy 按数据集分块策略分块;semantic 需要 embedder,其余策略可传 nil
func (s *chunkService) SplitByStrategy(ctx context.Context, strategy, text string, chunkSize, overlap int, embedder eembedding.Embedder) ([]string, error) {
switch strategy {
case consts.ChunkStrategyRecursive:
return s.splitRecursive(ctx, text, chunkSize, overlap)
case consts.ChunkStrategySemantic:
if embedder == nil {
return nil, gerror.New("语义分块需要向量模型")
}
sp, err := semantic.NewSplitter(ctx, &semantic.Config{
// SplitAuto 内置组合分块管线(无需用户选择策略),按文档内容自动分派:
// 1. 结构识别(条文/条款/章回/编号等,行首命中达标)→ 结构化切分,带上下文前缀
// 2. 标题感知(# / 标题 / 第X章 行首)→ 标题并入段落切分
// 3. 语义分块(无结构无标题,需 embedder)→ 按句子语义切分,超限块递归兜底
// 4. 递归兜底(embedder 不可用或语义失败)→ 中文分隔符递归切分
//
// 返回的 unitPattern/ctxPattern 仅结构识别路径有值(自动识别出的结构模式,供落库展示)。
func (s *chunkService) SplitAuto(ctx context.Context, text string, chunkSize, overlap int, embedder eembedding.Embedder) (chunks []string, unitPattern, ctxPattern string, err error) {
if unit, ctxPat, ok := s.DetectStructure(text); ok {
g.Log().Infof(ctx, "分块管线:结构识别命中(%s / %s),按结构单元切分", unit, ctxPat)
return s.splitStructured(text, unit, ctxPat, chunkSize, overlap), unit, ctxPat, nil
}
if hasHeadings(text) {
g.Log().Infof(ctx, "分块管线:未识别结构,含标题行,走标题感知切分")
return s.SplitText(text, chunkSize, overlap), "", "", nil
}
if embedder != nil {
g.Log().Infof(ctx, "分块管线:无结构无标题,走语义切分")
sp, spErr := semantic.NewSplitter(ctx, &semantic.Config{
Embedding: embedder,
BufferSize: 1,
MinChunkSize: chunkSize / 2,
Separators: chunkSeparators[:len(chunkSeparators)-2],
LenFunc: runeLen,
})
if err != nil {
return nil, gerror.Wrap(err, "构建语义分块器失败")
if spErr != nil {
g.Log().Warningf(ctx, "构建语义分块器失败,回退递归分块: %v", spErr)
} else {
semanticChunks, tErr := s.transformText(ctx, sp, text)
if tErr != nil {
g.Log().Warningf(ctx, "语义分块失败,回退递归分块: %v", tErr)
} else if len(semanticChunks) > 0 {
// chunkSize 作为安全上限:语义分块可能产出巨块(长连续语义段),超限块递归二次切分兜底(不重叠)
chunks, err = s.capChunks(ctx, semanticChunks, chunkSize)
return chunks, "", "", err
}
}
chunks, err := s.transformText(ctx, sp, text)
if err != nil {
return nil, err
}
// chunkSize 作为安全上限:语义分块可能产出巨块(长连续语义段),超限块用递归二次切分兜底(不重叠)
return s.capChunks(ctx, chunks, chunkSize)
default:
return s.SplitText(text, chunkSize, overlap), nil
}
g.Log().Infof(ctx, "分块管线:无结构无标题,语义分块不可用,走递归切分")
chunks, err = s.splitRecursive(ctx, text, chunkSize, overlap)
return chunks, "", "", err
}
// hasHeadings 文本是否含标题行(Markdown # / 标题 / 第X章),标题感知分块保留标题结构
func hasHeadings(text string) bool {
for _, line := range strings.Split(text, "\n") {
if isHeading(strings.TrimSpace(line)) {
return true
}
}
return false
}
// 结构标记候选:单元模式按优先级排列(条文 > 章回 > 小节 > 章节 > 篇部 > 数字编号 > 括号序号)
var (
unitPatternCandidates = []string{
`第[一二三四五六七八九十百千]+条`,
`第[一二三四五六七八九十百千]+回`,
`第[一二三四五六七八九十百千]+节`,
`第[一二三四五六七八九十百千]+章`,
`第[一二三四五六七八九十百千]+[篇部]`,
`[一二三四五六七八九十]{1,3}[、.]`,
`\d{1,3}[、.]`,
`[(][一二三四五六七八九十\d]{1,3}[)]`,
}
ctxPatternCandidates = []string{
`第[一二三四五六七八九十百千]+[篇部]`,
`第[一二三四五六七八九十百千]+章`,
`第[一二三四五六七八九十百千]+节`,
}
minUnitHits = 5 // 行首命中至少 5 次才认定是结构单元(排除正文偶然提及)
minCtxHits = 3 // 上下文标记命中下限
)
// DetectStructure 自动识别文本的结构单元与上下文标记:统计各候选模式在行首的出现次数,
// 命中数达到阈值且单元优先顺序靠前者胜出;上下文取单元之外层级最高的候选。未识别出单元时 ok=false。
func (s *chunkService) DetectStructure(text string) (unit, ctx string, ok bool) {
count := func(pat string) int {
return len(regexp.MustCompile(`(?m)^\s*`+pat).FindAllStringIndex(text, -1))
}
hits := make(map[string]int, len(unitPatternCandidates)+len(ctxPatternCandidates))
for _, p := range unitPatternCandidates {
hits[p] = count(p)
}
for _, p := range ctxPatternCandidates {
hits[p] = count(p)
}
for _, p := range unitPatternCandidates {
if hits[p] >= minUnitHits {
unit = p
break
}
}
if unit == "" {
return "", "", false
}
for _, p := range ctxPatternCandidates {
if p == unit {
continue
}
if hits[p] >= minCtxHits {
ctx = p
break
}
}
return unit, ctx, true
}
// splitStructured 结构单元切分:按单元模式切分(标记保留在块内),上下文模式命中的整行作为前缀;
// 相邻短单元合并到 chunkSize/2(上限 300 字),超长单元按句兜底切分(后续片段补「(标记 续)」前缀)。
func (s *chunkService) splitStructured(text, unitPattern, ctxPattern string, chunkSize, overlap int) []string {
matches := regexp.MustCompile(`(?m)^\s*(`+unitPattern+`)`).FindAllStringIndex(text, -1)
if len(matches) == 0 {
return s.SplitText(text, chunkSize, overlap)
}
mergeMin := chunkSize / 2
if mergeMin > 300 {
mergeMin = 300
}
if mergeMin < 1 {
mergeMin = 1
}
var ctxMatches [][]int
if ctxPattern != "" {
ctxMatches = regexp.MustCompile(`(?m)^\s*(`+ctxPattern+`)`).FindAllStringIndex(text, -1)
}
// ctxBefore 返回 pos 之前最近的上下文标记整行(如「第三章 劳动合同的解除」)
ctxBefore := func(pos int) string {
cur := ""
for _, m := range ctxMatches {
if m[0] >= pos {
break
}
lineEnd := strings.IndexByte(text[m[0]:], '\n')
if lineEnd < 0 {
lineEnd = len(text) - m[0]
}
cur = strings.TrimSpace(text[m[0] : m[0]+lineEnd])
}
return cur
}
out := make([]string, 0, 16)
var buf []string
bufLen := 0
bufCtx := ""
flush := func() {
if len(buf) == 0 {
return
}
body := strings.Join(buf, "\n")
if bufCtx != "" {
body = "【" + bufCtx + "】" + body
}
out = append(out, body)
buf, bufLen, bufCtx = nil, 0, ""
}
for i, m := range matches {
start := m[1]
end := len(text)
if i+1 < len(matches) {
end = matches[i+1][0]
}
marker := strings.TrimSpace(text[m[0]:m[1]])
content := strings.TrimSpace(text[start:end])
if content == "" {
continue
}
item := marker + " " + content
ctx := ctxBefore(m[0])
if bufLen > 0 && (ctx != bufCtx || bufLen+runeLen(item) > chunkSize) {
flush()
}
if runeLen(item) > chunkSize {
pieces := s.SplitText(item, chunkSize, overlap)
if len(pieces) > 0 {
if ctx != "" {
pieces[0] = "【" + ctx + "】" + pieces[0]
}
out = append(out, pieces[0])
for _, p := range pieces[1:] {
out = append(out, ""+marker+" 续)"+p)
}
}
continue
}
buf = append(buf, item)
bufLen += runeLen(item)
bufCtx = ctx
if bufLen >= mergeMin {
flush()
}
}
flush()
return out
}
// splitRecursive 递归字符分块:按分隔符列表递归切分到目标大小,保留 overlap
+4 -21
View File
@@ -23,8 +23,8 @@ func (s *datasetService) Save(ctx context.Context, m *entity.Dataset) (int64, er
if m.Status == 0 {
m.Status = 1
}
if err := s.validateStrategy(m); err != nil {
return 0, err
if m.EmbeddingCfgId == 0 {
return 0, gerror.New("数据集必须绑定向量模型,请先选择向量模型")
}
if m.Id > 0 {
old, err := dao.Dataset.GetOne(ctx, m.Id)
@@ -40,8 +40,8 @@ func (s *datasetService) Save(ctx context.Context, m *entity.Dataset) (int64, er
g.Log().Warningf(ctx, "enqueue reembed for dataset %d failed: %v", m.Id, err)
}
}
// 分块策略/大小/重叠变更 → 入队完整重新解析任务(重新分词+向量化)
if old != nil && (old.ChunkSize != m.ChunkSize || old.ChunkOverlap != m.ChunkOverlap || old.ChunkStrategy != m.ChunkStrategy) {
// 分块大小/重叠变更 → 入队完整重新解析任务(重新分词+向量化)
if old != nil && (old.ChunkSize != m.ChunkSize || old.ChunkOverlap != m.ChunkOverlap) {
if err := s.enqueueTask(ctx, m.Id, consts.TaskTypeParse); err != nil {
g.Log().Warningf(ctx, "enqueue reparse for dataset %d failed: %v", m.Id, err)
}
@@ -54,26 +54,9 @@ func (s *datasetService) Save(ctx context.Context, m *entity.Dataset) (int64, er
if m.ChunkOverlap < 0 {
m.ChunkOverlap = consts.DefaultChunkOverlap
}
if m.ChunkStrategy == "" {
m.ChunkStrategy = consts.ChunkStrategyTitle
}
return dao.Dataset.Insert(ctx, m)
}
func (s *datasetService) validateStrategy(m *entity.Dataset) error {
if m.ChunkStrategy == "" {
m.ChunkStrategy = consts.ChunkStrategyTitle
}
if m.EmbeddingCfgId == 0 {
return gerror.New("数据集必须绑定向量模型,请先选择向量模型")
}
// 语义分块无重叠参数,清零避免误导
if m.ChunkStrategy == consts.ChunkStrategySemantic {
m.ChunkOverlap = 0
}
return nil
}
func (s *datasetService) enqueueTask(ctx context.Context, datasetId int64, taskType string) error {
docs, _, err := dao.Document.List(ctx, datasetId, 1, 100000)
if err != nil {
+13 -4
View File
@@ -71,10 +71,10 @@ func (s *parseTaskService) processOne(ctx context.Context) {
s.fail(ctx, task, "解析文件失败: "+err.Error())
return
}
// 数据集分块配置(策略/大小/重叠),未设置用默认值
chunkSize, chunkOverlap, strategy := consts.DefaultChunkSize, consts.DefaultChunkOverlap, consts.ChunkStrategyTitle
// 数据集分块配置(大小/重叠),未设置用默认值
chunkSize, chunkOverlap := consts.DefaultChunkSize, consts.DefaultChunkOverlap
if ds, err := dao.Dataset.GetOne(ctx, task.DatasetId); err == nil && ds != nil {
chunkSize, chunkOverlap, strategy = ds.ChunkSize, ds.ChunkOverlap, ds.ChunkStrategy
chunkSize, chunkOverlap = ds.ChunkSize, ds.ChunkOverlap
}
// 数据集必须绑定向量模型,构建失败直接失败任务(不降级全文索引)
cfgId, err := dao.Dataset.GetEmbeddingCfgId(ctx, task.DatasetId)
@@ -103,11 +103,20 @@ func (s *parseTaskService) processOne(ctx context.Context) {
if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"content": text}); err != nil {
g.Log().Warningf(ctx, "save document content failed: %v", err)
}
chunks, err := ChunkService.SplitByStrategy(ctx, strategy, text, chunkSize, chunkOverlap, embedder)
chunks, unitPattern, ctxPattern, err := ChunkService.SplitAuto(ctx, text, chunkSize, chunkOverlap, embedder)
if err != nil {
s.fail(ctx, task, "分块失败: "+err.Error())
return
}
// 结构识别出的模式落库(列表展示用);未识别出结构时保留旧值,避免清空已展示的模式
if unitPattern != "" {
if err := dao.Dataset.UpdateFields(ctx, task.DatasetId, g.Map{
"unit_pattern": unitPattern,
"context_pattern": ctxPattern,
}); err != nil {
g.Log().Warningf(ctx, "save detected structure patterns failed: %v", err)
}
}
if err := ChunkService.InsertAll(ctx, task.DatasetId, doc.Id, chunks, embedder); err != nil {
s.fail(ctx, task, "写入分块失败: "+err.Error())
return
+22
View File
@@ -4,6 +4,8 @@ import (
"context"
"rag-local/common"
"rag-local/kb/consts"
"rag-local/kb/dao"
"github.com/gogf/gf/v2/errors/gerror"
)
@@ -25,3 +27,23 @@ func (s *systemConfigService) Login(ctx context.Context, token string) (string,
}
return common.SignToken("owner", common.AccessTokenFingerprint(), common.TokenExpireSeconds)
}
// GetSettings 读取全局分块默认值(未设置时用内置默认值)
func (s *systemConfigService) GetSettings(ctx context.Context) (chunkSize, chunkOverlap int, err error) {
return dao.AppConfig.GetInt(ctx, consts.SettingsKeyChunkSize, consts.DefaultChunkSize),
dao.AppConfig.GetInt(ctx, consts.SettingsKeyChunkOverlap, consts.DefaultChunkOverlap), nil
}
// SaveSettings 保存全局分块默认值
func (s *systemConfigService) SaveSettings(ctx context.Context, chunkSize, chunkOverlap int) error {
if chunkSize < 50 || chunkSize > 5000 {
return gerror.New("分块大小需在 50~5000 之间")
}
if chunkOverlap < 0 || chunkOverlap > 500 {
return gerror.New("重叠字数需在 0~500 之间")
}
if err := dao.AppConfig.SetInt(ctx, consts.SettingsKeyChunkSize, chunkSize); err != nil {
return err
}
return dao.AppConfig.SetInt(ctx, consts.SettingsKeyChunkOverlap, chunkOverlap)
}
+26
View File
@@ -10,6 +10,7 @@
"dependencies": {
"@element-plus/icons-vue": "^2.3.0",
"axios": "^1.6.0",
"echarts": "^6.1.0",
"element-plus": "^2.5.0",
"pinia": "^2.1.0",
"vue": "^3.4.0",
@@ -1223,6 +1224,16 @@
"node": ">= 0.4"
}
},
"node_modules/echarts": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/echarts/-/echarts-6.1.0.tgz",
"integrity": "sha512-q0yaFPggC9FUdsWH4blavRWFmxdrIodbkoKNAjJudAI6CA9gNPxHtV2RcZNEepZVlk4yvBYkOkbk6HIVpIyHZA==",
"license": "Apache-2.0",
"dependencies": {
"tslib": "2.3.0",
"zrender": "6.1.0"
}
},
"node_modules/element-plus": {
"version": "2.14.3",
"resolved": "https://registry.npmjs.org/element-plus/-/element-plus-2.14.3.tgz",
@@ -1730,6 +1741,12 @@
"node": ">=0.10.0"
}
},
"node_modules/tslib": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz",
"integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==",
"license": "0BSD"
},
"node_modules/vite": {
"version": "5.4.21",
"resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz",
@@ -1857,6 +1874,15 @@
"peerDependencies": {
"vue": "^3.5.0"
}
},
"node_modules/zrender": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/zrender/-/zrender-6.1.0.tgz",
"integrity": "sha512-oEGMDB6pOP2S6OwRR4PdVv610zrjnA3Bh+JnSG12fYJlBKjtNAoEb5fSUoCOOINlH96I2fU38/A2UpRKs67xYQ==",
"license": "BSD-3-Clause",
"dependencies": {
"tslib": "2.3.0"
}
}
}
}
+1
View File
@@ -11,6 +11,7 @@
"dependencies": {
"@element-plus/icons-vue": "^2.3.0",
"axios": "^1.6.0",
"echarts": "^6.1.0",
"element-plus": "^2.5.0",
"pinia": "^2.1.0",
"vue": "^3.4.0",
+9
View File
@@ -0,0 +1,9 @@
import request from './request.js'
export function getSettings() {
return request.get('/system-config/settings')
}
export function saveSettings(data) {
return request.post('/system-config/save-settings', data)
}
+38 -24
View File
@@ -12,8 +12,10 @@
<el-tag v-else size="small" type="info">未设置</el-tag>
</template>
</el-table-column>
<el-table-column label="分块" width="140">
<template #default="{ row }">{{ STRATEGY_NAMES[row.chunk_strategy || 'title'] }} {{ row.chunk_size || 800 }}{{ row.chunk_strategy === 'semantic' ? '' : ' / ' + (row.chunk_overlap ?? 150) }}</template>
<el-table-column label="分块" width="210">
<template #default="{ row }">
{{ row.chunk_size || 800 }} / {{ row.chunk_overlap ?? 150 }}<template v-if="row.unit_pattern">自动识别{{ humanizePattern(row.unit_pattern) }}</template>
</template>
</el-table-column>
<el-table-column prop="created_at" label="创建时间" width="170" />
<el-table-column label="操作" width="190" fixed="right">
@@ -38,21 +40,13 @@
<el-option v-for="m in embedders" :key="m.id" :label="m.name" :value="m.id" />
</el-select>
</el-form-item>
<el-form-item label="分块策略">
<el-select v-model="form.chunk_strategy" style="width: 100%">
<el-option label="标题分块(推荐)" value="title" />
<el-option label="递归字符分块" value="recursive" />
<el-option label="语义分块" value="semantic" />
</el-select>
<div class="ds-tip">标题分块保留标题与段落结构递归字符分块按分隔符切分通用文本语义分块按语义相似度切分质量更高但解析更慢且需绑定向量模型</div>
</el-form-item>
<el-form-item :label="form.chunk_strategy === 'semantic' ? '上限大小' : '分块大小'">
<el-form-item label="分块大小">
<el-input-number v-model="form.chunk_size" :min="50" :max="5000" :step="50" style="width: 100%" />
<div class="ds-tip">{{ form.chunk_strategy === 'semantic' ? '语义分块按句子相似度切分,此值为安全上限(max_chunk_size),超过上限的块会再切分;语义分块不支持重叠' : '每块最大字数,超长段落自动按句号/换行切分' }}</div>
<div class="ds-tip">每块最大字数系统自动组合分块策略优先识别文档结构条文/章节/编号等按结构切分无结构时依次按标题感知语义递归切分</div>
</el-form-item>
<el-form-item v-if="form.chunk_strategy !== 'semantic'" label="重叠字数">
<el-form-item label="重叠字数">
<el-input-number v-model="form.chunk_overlap" :min="0" :max="500" :step="10" style="width: 100%" />
<div class="ds-tip">相邻分块间的重叠字数用于保持上下文连贯</div>
<div class="ds-tip">相邻分块间的重叠字数用于保持上下文连贯语义切分路径自动忽略</div>
</el-form-item>
</el-form>
<template #footer>
@@ -68,21 +62,41 @@ import { onMounted, ref } from 'vue'
import { ElMessage, ElMessageBox } from 'element-plus'
import { listDatasets, saveDataset, deleteDataset } from '../api/dataset.js'
import { listModelConfigs } from '../api/model_config.js'
import { getSettings } from '../api/settings.js'
const STRATEGY_NAMES = { title: '标题', recursive: '递归', semantic: '语义' }
// 把识别出的结构正则转成可读名称(第[一二三四五六七八九十百千]+条 → 条文)
function humanizePattern(p) {
if (!p) return ''
if (p.includes('条')) return '条文'
if (p.includes('回')) return '章回'
if (p.includes('节')) return '小节'
if (p.includes('章')) return '章节'
if (p.includes('篇') || p.includes('部')) return '篇章'
if (p.includes('') || p.includes('(')) return '序号'
if (/\d/.test(p)) return '编号'
return '结构单元'
}
const datasets = ref([])
const embedders = ref([])
const defaults = ref({ chunk_size: 800, chunk_overlap: 150 })
const loading = ref(false)
const saving = ref(false)
const dialogVisible = ref(false)
const editing = ref(false)
let editingRowCfgId = 0
let editingRowChunk = { chunk_size: 0, chunk_overlap: 0, chunk_strategy: '' }
const form = ref({ id: 0, name: '', description: '', embedding_cfg_id: 0, chunk_size: 800, chunk_overlap: 150, chunk_strategy: 'title' })
let editingRowChunk = { chunk_size: 0, chunk_overlap: 0 }
const form = ref({ id: 0, name: '', description: '', embedding_cfg_id: 0, chunk_size: 800, chunk_overlap: 150 })
onMounted(async () => {
await load()
try {
const s = await getSettings()
if (s) {
defaults.value.chunk_size = s.chunk_size || 800
defaults.value.chunk_overlap = s.chunk_overlap ?? 150
}
} catch { /* 忽略 */ }
try {
const m = await listModelConfigs('embedding')
if (m && m.list) embedders.value = m.list
@@ -107,20 +121,20 @@ function embeddingName(id) {
function openCreate() {
editing.value = false
editingRowCfgId = 0
editingRowChunk = { chunk_size: 0, chunk_overlap: 0, chunk_strategy: '' }
// 默认选中默认向量模型,其次第一个
editingRowChunk = { chunk_size: 0, chunk_overlap: 0 }
// 默认选中默认向量模型,其次第一个;分块大小/重叠带出全局设置值
const def = embedders.value.find(x => x.is_default === 1) || embedders.value[0]
form.value = { id: 0, name: '', description: '', embedding_cfg_id: def?.id || 0, chunk_size: 800, chunk_overlap: 150, chunk_strategy: 'title' }
form.value = { id: 0, name: '', description: '', embedding_cfg_id: def?.id || 0, chunk_size: defaults.value.chunk_size, chunk_overlap: defaults.value.chunk_overlap }
dialogVisible.value = true
}
function openEdit(row) {
editing.value = true
editingRowCfgId = row.embedding_cfg_id
editingRowChunk = { chunk_size: row.chunk_size || 800, chunk_overlap: row.chunk_overlap ?? 150, chunk_strategy: row.chunk_strategy || 'title' }
editingRowChunk = { chunk_size: row.chunk_size || 800, chunk_overlap: row.chunk_overlap ?? 150 }
form.value = {
id: row.id, name: row.name, description: row.description, embedding_cfg_id: row.embedding_cfg_id,
chunk_size: editingRowChunk.chunk_size, chunk_overlap: editingRowChunk.chunk_overlap, chunk_strategy: editingRowChunk.chunk_strategy,
chunk_size: editingRowChunk.chunk_size, chunk_overlap: editingRowChunk.chunk_overlap,
}
dialogVisible.value = true
}
@@ -135,11 +149,11 @@ async function save() {
return
}
const modelChanged = editing.value && form.value.embedding_cfg_id !== editingRowCfgId
const chunkChanged = editing.value && (form.value.chunk_size !== editingRowChunk.chunk_size || form.value.chunk_overlap !== editingRowChunk.chunk_overlap || form.value.chunk_strategy !== editingRowChunk.chunk_strategy)
const chunkChanged = editing.value && (form.value.chunk_size !== editingRowChunk.chunk_size || form.value.chunk_overlap !== editingRowChunk.chunk_overlap)
if (modelChanged || chunkChanged) {
const changed = []
if (modelChanged) changed.push('向量模型')
if (chunkChanged) changed.push('分块策略')
if (chunkChanged) changed.push('分块配置')
try {
await ElMessageBox.confirm(`${changed.join('与')}已变更,保存后将重新处理该数据集下所有文档(异步进行,可在文档列表查看进度)。是否继续?`, '重新处理文档', { type: 'warning' })
} catch {
+162 -22
View File
@@ -4,33 +4,44 @@
<el-select v-model="datasetId" placeholder="选择知识库" style="width: 260px" @change="load">
<el-option v-for="d in datasets" :key="d.id" :label="d.name" :value="d.id" />
</el-select>
<span class="kg-tip">实体与关系由解析流水线中的 LLM 抽取生成问答时命中实体自动注入一跳邻居</span>
<span class="kg-tip">实体与关系由解析流水线中的 LLM 抽取生成问答时命中实体自动注入一跳邻居点击节点高亮其关联实体</span>
</div>
<el-row :gutter="12">
<el-col :span="10">
<el-col :span="15">
<el-card shadow="never" class="kg-card">
<template #header>实体{{ entityTotal }}</template>
<el-table :data="entities" size="small" v-loading="loading">
<el-table-column prop="name" label="实体" min-width="140" show-overflow-tooltip />
<el-table-column prop="entity_type" label="类型" width="110" />
<el-table-column prop="chunk_id" label="来源分块" width="100" />
</el-table>
<el-pagination class="kg-page-bar" small layout="prev, pager, next" :total="entityTotal"
:page-size="entityPageSize" v-model:current-page="entityPage" @current-change="loadEntities" />
<template #header>
<div class="kg-graph-head">
<span>关系图谱{{ graphCountText }}</span>
<el-tag v-if="truncated" size="small" type="warning">实体过多仅展示度数最高的前 {{ MAX_NODES }} </el-tag>
</div>
</template>
<div ref="graphEl" class="kg-graph" v-loading="loading"></div>
<el-empty v-if="!loading && graphNodes.length === 0" description="暂无图谱数据,上传文档并完成解析后自动生成" />
</el-card>
</el-col>
<el-col :span="14">
<el-col :span="9">
<el-card shadow="never" class="kg-card">
<template #header>关系{{ relationTotal }}</template>
<el-table :data="relations" size="small" v-loading="loading">
<el-table-column prop="head" label="主体" min-width="140" show-overflow-tooltip />
<el-table-column prop="relation" label="关系" min-width="100" show-overflow-tooltip />
<el-table-column prop="tail" label="客体" min-width="140" show-overflow-tooltip />
<el-table-column prop="chunk_id" label="来源分块" width="100" />
</el-table>
<el-pagination class="kg-page-bar" small layout="prev, pager, next" :total="relationTotal"
:page-size="relationPageSize" v-model:current-page="relationPage" @current-change="loadRelations" />
<el-tabs v-model="sideTab">
<el-tab-pane :label="`实体(${entityTotal}`" name="entity">
<el-table :data="entities" size="small" height="440" v-loading="loading">
<el-table-column prop="name" label="实体" min-width="120" show-overflow-tooltip />
<el-table-column prop="entity_type" label="类型" width="100" />
<el-table-column prop="chunk_id" label="来源分块" width="90" />
</el-table>
<el-pagination class="kg-page-bar" small layout="prev, pager, next" :total="entityTotal"
:page-size="entityPageSize" v-model:current-page="entityPage" @current-change="loadEntities" />
</el-tab-pane>
<el-tab-pane :label="`关系(${relationTotal}`" name="relation">
<el-table :data="relations" size="small" height="440" v-loading="loading">
<el-table-column prop="head" label="主体" min-width="110" show-overflow-tooltip />
<el-table-column prop="relation" label="关系" min-width="90" show-overflow-tooltip />
<el-table-column prop="tail" label="客体" min-width="110" show-overflow-tooltip />
</el-table>
<el-pagination class="kg-page-bar" small layout="prev, pager, next" :total="relationTotal"
:page-size="relationPageSize" v-model:current-page="relationPage" @current-change="loadRelations" />
</el-tab-pane>
</el-tabs>
</el-card>
</el-col>
</el-row>
@@ -38,13 +49,23 @@
</template>
<script setup>
import { onMounted, ref } from 'vue'
import { computed, nextTick, onBeforeUnmount, onMounted, ref } from 'vue'
import * as echarts from 'echarts/core'
import { GraphChart } from 'echarts/charts'
import { TooltipComponent, LegendComponent } from 'echarts/components'
import { CanvasRenderer } from 'echarts/renderers'
import { listDatasets } from '../api/dataset.js'
import { listEntities, listRelations } from '../api/kg.js'
echarts.use([GraphChart, TooltipComponent, LegendComponent, CanvasRenderer])
const MAX_NODES = 400 // 力导向图节点上限,超过取度数最高者,避免布局卡顿
const datasets = ref([])
const datasetId = ref(null)
const loading = ref(false)
const graphEl = ref(null)
const truncated = ref(false)
const entities = ref([])
const entityTotal = ref(0)
@@ -56,6 +77,14 @@ const relationTotal = ref(0)
const relationPage = ref(1)
const relationPageSize = 20
const sideTab = ref('entity')
let chart = null
let resizeObserver = null
const graphCountText = computed(() => `${truncated.value ? '≈' : ''}${graphNodes.value.length} 实体 / ${graphLinks.value.length} 关系`)
const graphNodes = ref([])
const graphLinks = ref([])
onMounted(async () => {
try {
const d = await listDatasets()
@@ -65,10 +94,107 @@ onMounted(async () => {
await load()
})
onBeforeUnmount(() => {
resizeObserver?.disconnect()
chart?.dispose()
})
async function load() {
entityPage.value = 1
relationPage.value = 1
await Promise.all([loadEntities(), loadRelations()])
loading.value = true
try {
const [allEntities, allRelations] = await Promise.all([
listEntities({ dataset_id: datasetId.value, page: 1, page_size: 100000 }),
listRelations({ dataset_id: datasetId.value, page: 1, page_size: 100000 }),
])
buildGraph(allEntities?.list || [], allRelations?.list || [])
await Promise.all([loadEntities(), loadRelations()])
} finally {
loading.value = false
}
}
// 实体按名字去重合并(同一实体可能出现在多个分块),按出入度决定节点大小
function buildGraph(entitiesList, relationsList) {
const byName = new Map()
for (const e of entitiesList) {
const prev = byName.get(e.name)
if (prev) {
prev.chunks = [...new Set([...prev.chunks, e.chunk_id])]
} else {
byName.set(e.name, { name: e.name, entity_type: e.entity_type, chunks: [e.chunk_id] })
}
}
const degree = new Map()
const edges = []
const seenEdges = new Set()
for (const r of relationsList) {
if (!byName.has(r.head) || !byName.has(r.tail)) continue
const key = `${r.head}|${r.relation}|${r.tail}`
if (seenEdges.has(key)) continue
seenEdges.add(key)
edges.push({ source: r.head, target: r.tail, relation: r.relation })
degree.set(r.head, (degree.get(r.head) || 0) + 1)
degree.set(r.tail, (degree.get(r.tail) || 0) + 1)
}
let nodes = [...byName.values()]
truncated.value = nodes.length > MAX_NODES
if (truncated.value) {
nodes.sort((a, b) => (degree.get(b.name) || 0) - (degree.get(a.name) || 0))
const keep = new Set(nodes.slice(0, MAX_NODES).map(n => n.name))
nodes = nodes.filter(n => keep.has(n.name))
for (const e of [...edges]) {
if (!keep.has(e.source) || !keep.has(e.target)) edges.splice(edges.indexOf(e), 1)
}
}
const minD = Math.min(...nodes.map(n => degree.get(n.name) || 0))
const maxD = Math.max(...nodes.map(n => degree.get(n.name) || 0))
const sizeFor = d => (maxD === minD ? 30 : 20 + ((d - minD) / (maxD - minD)) * 36)
graphNodes.value = nodes.map(n => ({
id: n.name,
name: n.name,
category: n.entity_type || '未分类',
symbolSize: sizeFor(degree.get(n.name) || 0),
}))
graphLinks.value = edges
renderGraph()
}
function renderGraph() {
if (!graphEl.value) return
chart ??= echarts.init(graphEl.value)
const types = [...new Set(graphNodes.value.map(n => n.category))]
chart.setOption({
legend: { bottom: 0, type: 'scroll', data: types, textStyle: { fontSize: 11 } },
tooltip: {
trigger: 'item',
formatter: p => {
if (p.dataType === 'edge') {
const e = graphLinks.value[p.dataIndex]
return `${e.source}${e.target}<br/>关系:${e.relation}`
}
return `${p.name}<br/>类型:${p.data.category}`
},
},
series: [{
type: 'graph',
layout: 'force',
roam: true,
draggable: true,
data: graphNodes.value,
links: graphLinks.value,
categories: types.map(t => ({ name: t })),
force: { repulsion: 260, edgeLength: [40, 110], gravity: 0.08 },
label: { show: true, position: 'right', fontSize: 10, color: '#606266' },
edgeLabel: { show: false },
emphasis: {
focus: 'adjacency', // 点击节点自动高亮一跳邻居并淡化其余
label: { fontSize: 12, fontWeight: 600 },
lineStyle: { width: 2 },
},
}],
}, true)
}
async function loadEntities() {
@@ -94,6 +220,12 @@ async function loadRelations() {
loading.value = false
}
}
onMounted(() => {
// 布局稳定后渲染一次(图表容器尺寸依赖 el-row 布局),并监听容器尺寸变化
resizeObserver = new ResizeObserver(() => nextTick(renderGraph))
resizeObserver.observe(graphEl.value)
})
</script>
<style scoped>
@@ -107,6 +239,14 @@ async function loadRelations() {
font-size: 13px;
color: #909399;
}
.kg-graph {
height: 620px;
}
.kg-graph-head {
display: flex;
align-items: center;
gap: 10px;
}
.kg-page-bar {
margin-top: 10px;
justify-content: flex-end;
+61
View File
@@ -1,5 +1,17 @@
<template>
<div class="settings-page">
<div class="block-card">
<div class="block-title">分块默认值</div>
<div class="block-tip">新建数据集时自动带出单个数据集仍可在表单中修改</div>
<div class="block-row">
<span class="field-label">分块大小</span>
<el-input-number v-model="chunkForm.chunk_size" :min="50" :max="5000" :step="50" />
<span class="field-label">重叠字数</span>
<el-input-number v-model="chunkForm.chunk_overlap" :min="0" :max="500" :step="10" />
<el-button type="primary" :loading="chunkSaving" @click="saveChunkDefaults">保存</el-button>
</div>
</div>
<div class="section-head">
<el-radio-group v-model="modelType" @change="loadModels">
<el-radio-button value="chat">对话模型</el-radio-button>
@@ -65,8 +77,32 @@
import { onMounted, ref } from 'vue'
import { ElMessage, ElMessageBox } from 'element-plus'
import { listModelConfigs, saveModelConfig, deleteModelConfig, testModelConfig, setDefaultModelConfig } from '../api/model_config.js'
import { getSettings, saveSettings } from '../api/settings.js'
const modelType = ref('chat')
const chunkForm = ref({ chunk_size: 800, chunk_overlap: 150 })
const chunkSaving = ref(false)
async function loadChunkDefaults() {
try {
const s = await getSettings()
if (s) {
chunkForm.value.chunk_size = s.chunk_size || 800
chunkForm.value.chunk_overlap = s.chunk_overlap ?? 150
}
} catch { /* 忽略 */ }
}
async function saveChunkDefaults() {
chunkSaving.value = true
try {
await saveSettings(chunkForm.value)
ElMessage.success('分块默认值已保存')
} finally {
chunkSaving.value = false
}
}
const models = ref([])
const allModels = ref([])
const modelLoading = ref(false)
@@ -78,6 +114,7 @@ const modelForm = ref({ id: 0, name: '', model_type: 'chat', model_name: '', end
onMounted(async () => {
await loadModels()
await loadChunkDefaults()
try { const all = await listModelConfigs(''); if (all && all.list) allModels.value = all.list } catch { /* 忽略 */ }
})
@@ -180,4 +217,28 @@ async function test(row) {
align-items: center;
margin-bottom: 12px;
}
.block-card {
border: 1px solid #e4e7ed;
border-radius: 6px;
padding: 14px 16px;
margin-bottom: 16px;
}
.block-title {
font-weight: 600;
margin-bottom: 4px;
}
.block-tip {
font-size: 12px;
color: #909399;
margin-bottom: 10px;
}
.block-row {
display: flex;
align-items: center;
gap: 8px;
}
.field-label {
color: #606266;
font-size: 14px;
}
</style>