diff --git a/.gitignore b/.gitignore index f98bc2d..ea36b59 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ # 运行时数据与本地环境 -data/ workspace/ .idea/ .DS_Store diff --git a/data/business.db b/data/business.db new file mode 100644 index 0000000..000cdab Binary files /dev/null and b/data/business.db differ diff --git a/data/chat.db b/data/chat.db new file mode 100644 index 0000000..4c77dc7 Binary files /dev/null and b/data/chat.db differ diff --git a/data/system.db b/data/system.db new file mode 100644 index 0000000..7f6a2f8 Binary files /dev/null and b/data/system.db differ diff --git a/kb/consts/consts.go b/kb/consts/consts.go index 9e74358..920ba3a 100644 --- a/kb/consts/consts.go +++ b/kb/consts/consts.go @@ -11,9 +11,9 @@ const ( DefaultChunkSize = 800 // 分块最大字数（数据集默认值） DefaultChunkOverlap = 150 // 分块重叠字数（数据集默认值） - ChunkStrategyTitle = "title" // 标题感知分块（自研，保留标题与段落） - ChunkStrategyRecursive = "recursive" // 递归字符分块（Eino recursive） - ChunkStrategySemantic = "semantic" // 语义分块（Eino semantic，需绑定向量模型） + // 全局设置键（app_config 表） + SettingsKeyChunkSize = "chunk_default_size" + SettingsKeyChunkOverlap = "chunk_default_overlap" ParsePollIntervalSeconds = 3 // 解析任务轮询间隔 diff --git a/kb/consts/table_name.go b/kb/consts/table_name.go index 4465726..1332182 100644 --- a/kb/consts/table_name.go +++ b/kb/consts/table_name.go @@ -2,6 +2,7 @@ package consts const ( TableNameModelConfig = "model_config" + TableNameAppConfig = "app_config" TableNameDataset = "kb_dataset" TableNameDocument = "kb_document" TableNameChunk = "kb_chunk" diff --git a/kb/controller/dataset_controller.go b/kb/controller/dataset_controller.go index 9820ebb..058dcd4 100644 --- a/kb/controller/dataset_controller.go +++ b/kb/controller/dataset_controller.go @@ -28,7 +28,6 @@ func (c *dataset) Save(ctx context.Context, req *dto.SaveDatasetReq) (*dto.SaveD EmbeddingCfgId: req.EmbeddingCfgId, ChunkSize: req.ChunkSize, ChunkOverlap: req.ChunkOverlap, - ChunkStrategy: req.ChunkStrategy, Status: 1, }) if err != nil { diff --git a/kb/controller/system_config_controller.go b/kb/controller/system_config_controller.go index b6550e6..1b8495b 100644 --- a/kb/controller/system_config_controller.go +++ b/kb/controller/system_config_controller.go @@ -18,3 +18,18 @@ func (c *systemConfig) Login(ctx context.Context, req *dto.LoginReq) (res *dto.L } return &dto.LoginRes{Token: token}, nil } + +func (c *systemConfig) GetSettings(ctx context.Context, _ *dto.GetSettingsReq) (*dto.GetSettingsRes, error) { + size, overlap, err := service.SystemConfigService.GetSettings(ctx) + if err != nil { + return nil, err + } + return &dto.GetSettingsRes{ChunkSize: size, ChunkOverlap: overlap}, nil +} + +func (c *systemConfig) SaveSettings(ctx context.Context, req *dto.SaveSettingsReq) (*dto.SaveSettingsRes, error) { + if err := service.SystemConfigService.SaveSettings(ctx, req.ChunkSize, req.ChunkOverlap); err != nil { + return nil, err + } + return &dto.SaveSettingsRes{}, nil +} diff --git a/kb/dao/app_config_dao.go b/kb/dao/app_config_dao.go new file mode 100644 index 0000000..cb658ef --- /dev/null +++ b/kb/dao/app_config_dao.go @@ -0,0 +1,44 @@ +package dao + +import ( + "context" + + "rag-local/kb/consts" + + "github.com/gogf/gf/v2/frame/g" +) + +var AppConfig = &appConfigDao{} + +type appConfigDao struct{} + +func init() { + ctx := context.Background() + _, err := g.DB(consts.DbGroupSystem).Exec(ctx, `CREATE TABLE IF NOT EXISTS `+consts.TableNameAppConfig+` ( + cfg_key TEXT PRIMARY KEY, + cfg_value TEXT NOT NULL DEFAULT '', + updated_at DATETIME DEFAULT (datetime('now','localtime')) + )`) + if err != nil { + g.Log().Warningf(ctx, "create app_config table failed: %v", err) + } +} + +// GetInt 读取全局设置值，未设置或非法时返回默认值 def +func (d *appConfigDao) GetInt(ctx context.Context, key string, def int) int { + r, err := g.DB(consts.DbGroupSystem).Ctx(ctx).GetValue(ctx, + "SELECT cfg_value FROM "+consts.TableNameAppConfig+" WHERE cfg_key=?", key) + if err != nil || r.IsEmpty() { + return def + } + return r.Int() +} + +// SetInt 写入全局设置值（UPSERT） +func (d *appConfigDao) SetInt(ctx context.Context, key string, v int) error { + _, err := g.DB(consts.DbGroupSystem).Ctx(ctx).Exec(ctx, + "INSERT INTO "+consts.TableNameAppConfig+"(cfg_key, cfg_value) VALUES(?, ?) "+ + "ON CONFLICT(cfg_key) DO UPDATE SET cfg_value=excluded.cfg_value, updated_at=datetime('now','localtime')", + key, v) + return err +} diff --git a/kb/dao/chunk_dao.go b/kb/dao/chunk_dao.go index a20ecd5..425e1e9 100644 --- a/kb/dao/chunk_dao.go +++ b/kb/dao/chunk_dao.go @@ -95,7 +95,12 @@ func (d *chunkDao) InsertWithVec(ctx context.Context, datasetId, documentId int6 if err != nil { return 0, err } - defer func() { _ = tx.Rollback() }() + // Commit 成功后 IsClosed 为 true，跳过 Rollback，避免对已提交事务回滚产生报错日志 + defer func() { + if !tx.IsClosed() { + _ = tx.Rollback() + } + }() r, err := tx.Model(consts.TableNameChunk).Ctx(ctx).Data(g.Map{ "dataset_id": datasetId, @@ -133,7 +138,12 @@ func (d *chunkDao) DeleteByDocument(ctx context.Context, documentId int64) error if err != nil { return err } - defer func() { _ = tx.Rollback() }() + // Commit 成功后 IsClosed 为 true，跳过 Rollback，避免对已提交事务回滚产生报错日志 + defer func() { + if !tx.IsClosed() { + _ = tx.Rollback() + } + }() ids := tx.Model(consts.TableNameChunk).Ctx(ctx).Fields("id").Where("document_id", documentId) r, err := ids.Array() @@ -158,6 +168,13 @@ func (d *chunkDao) DeleteByDocument(ctx context.Context, documentId int64) error if _, err := tx.Exec("DELETE FROM "+consts.TableNameChunkFts+" WHERE chunk_id IN ("+in+")", args...); err != nil { return err } + // 知识图谱数据按 chunk 关联，重新解析时旧 chunk 消失，一并清理避免孤儿数据 + if _, err := tx.Exec("DELETE FROM "+consts.TableNameKgRelation+" WHERE chunk_id IN ("+in+")", args...); err != nil { + return err + } + if _, err := tx.Exec("DELETE FROM "+consts.TableNameKgEntity+" WHERE chunk_id IN ("+in+")", args...); err != nil { + return err + } } if _, err := tx.Model(consts.TableNameChunk).Ctx(ctx).Where("document_id", documentId).Delete(); err != nil { return err @@ -170,7 +187,12 @@ func (d *chunkDao) UpdateContent(ctx context.Context, id int64, content, vecJson if err != nil { return err } - defer func() { _ = tx.Rollback() }() + // Commit 成功后 IsClosed 为 true，跳过 Rollback，避免对已提交事务回滚产生报错日志 + defer func() { + if !tx.IsClosed() { + _ = tx.Rollback() + } + }() if _, err := tx.Model(consts.TableNameChunk).Ctx(ctx).Data(g.Map{"content": content}).Where("id", id).Update(); err != nil { return err diff --git a/kb/dao/conversation_dao.go b/kb/dao/conversation_dao.go index 3ef22e7..d7a5af4 100644 --- a/kb/dao/conversation_dao.go +++ b/kb/dao/conversation_dao.go @@ -86,7 +86,12 @@ func (d *conversationDao) Delete(ctx context.Context, id int64) error { if err != nil { return err } - defer func() { _ = tx.Rollback() }() + // Commit 成功后 IsClosed 为 true，跳过 Rollback，避免对已提交事务回滚产生报错日志 + defer func() { + if !tx.IsClosed() { + _ = tx.Rollback() + } + }() if _, err := tx.Model(consts.TableNameMessage).Ctx(ctx).Where("conversation_id", id).Delete(); err != nil { return err } diff --git a/kb/dao/dataset_dao.go b/kb/dao/dataset_dao.go index e5552d7..c8e6e16 100644 --- a/kb/dao/dataset_dao.go +++ b/kb/dao/dataset_dao.go @@ -26,6 +26,8 @@ func init() { chunk_size INTEGER NOT NULL DEFAULT 800, chunk_overlap INTEGER NOT NULL DEFAULT 150, chunk_strategy TEXT NOT NULL DEFAULT 'title', + unit_pattern TEXT NOT NULL DEFAULT '', + context_pattern TEXT NOT NULL DEFAULT '', status INTEGER NOT NULL DEFAULT 1, created_at DATETIME DEFAULT (datetime('now','localtime')), updated_at DATETIME DEFAULT (datetime('now','localtime')) @@ -41,6 +43,8 @@ func init() { {"chunk_size", "chunk_size INTEGER NOT NULL DEFAULT 800"}, {"chunk_overlap", "chunk_overlap INTEGER NOT NULL DEFAULT 150"}, {"chunk_strategy", "chunk_strategy TEXT NOT NULL DEFAULT 'title'"}, + {"unit_pattern", "unit_pattern TEXT NOT NULL DEFAULT ''"}, + {"context_pattern", "context_pattern TEXT NOT NULL DEFAULT ''"}, } { cnt, err := g.DB(consts.DbGroupDefault).Ctx(ctx).GetValue(ctx, "SELECT COUNT(*) FROM pragma_table_info('"+consts.TableNameDataset+"') WHERE name=?", col.name) @@ -83,7 +87,6 @@ func (d *datasetDao) Insert(ctx context.Context, data *entity.Dataset) (int64, e "embedding_cfg_id": data.EmbeddingCfgId, "chunk_size": data.ChunkSize, "chunk_overlap": data.ChunkOverlap, - "chunk_strategy": data.ChunkStrategy, "status": data.Status, "created_at": now, "updated_at": now, @@ -101,7 +104,6 @@ func (d *datasetDao) Update(ctx context.Context, data *entity.Dataset) error { "embedding_cfg_id": data.EmbeddingCfgId, "chunk_size": data.ChunkSize, "chunk_overlap": data.ChunkOverlap, - "chunk_strategy": data.ChunkStrategy, "status": data.Status, "updated_at": gtime.Now().Format("Y-m-d H:i:s"), }).Where("id", data.Id).Update() @@ -113,6 +115,12 @@ func (d *datasetDao) Delete(ctx context.Context, id int64) error { return err } +func (d *datasetDao) UpdateFields(ctx context.Context, id int64, data g.Map) error { + _, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameDataset).Ctx(ctx).Data(data). + Where("id", id).Update() + return err +} + func (d *datasetDao) GetEmbeddingCfgId(ctx context.Context, id int64) (int64, error) { r, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameDataset).Ctx(ctx). Fields("embedding_cfg_id").Where("id", id).One() diff --git a/kb/model/dto/dataset_dto.go b/kb/model/dto/dataset_dto.go index 077b7bc..9bc7dd8 100644 --- a/kb/model/dto/dataset_dto.go +++ b/kb/model/dto/dataset_dto.go @@ -22,7 +22,6 @@ type SaveDatasetReq struct { EmbeddingCfgId int64 `json:"embedding_cfg_id"` ChunkSize int `json:"chunk_size"` ChunkOverlap int `json:"chunk_overlap"` - ChunkStrategy string `json:"chunk_strategy"` } type SaveDatasetRes struct { diff --git a/kb/model/dto/system_config_dto.go b/kb/model/dto/system_config_dto.go index 44b74d7..00b084b 100644 --- a/kb/model/dto/system_config_dto.go +++ b/kb/model/dto/system_config_dto.go @@ -10,3 +10,20 @@ type LoginReq struct { type LoginRes struct { Token string `json:"token"` } + +type GetSettingsReq struct { + g.Meta `path:"/settings" method:"get" tags:"系统配置" summary:"获取全局设置"` +} + +type GetSettingsRes struct { + ChunkSize int `json:"chunk_size"` // 默认分块大小 + ChunkOverlap int `json:"chunk_overlap"` // 默认重叠字数 +} + +type SaveSettingsReq struct { + g.Meta `path:"/save-settings" method:"post" tags:"系统配置" summary:"保存全局设置"` + ChunkSize int `json:"chunk_size"` + ChunkOverlap int `json:"chunk_overlap"` +} + +type SaveSettingsRes struct{} diff --git a/kb/model/entity/dataset.go b/kb/model/entity/dataset.go index 3568993..3f60ba8 100644 --- a/kb/model/entity/dataset.go +++ b/kb/model/entity/dataset.go @@ -9,7 +9,8 @@ type Dataset struct { EmbeddingCfgId int64 `orm:"embedding_cfg_id" json:"embedding_cfg_id"` ChunkSize int `orm:"chunk_size" json:"chunk_size"` ChunkOverlap int `orm:"chunk_overlap" json:"chunk_overlap"` - ChunkStrategy string `orm:"chunk_strategy" json:"chunk_strategy"` + UnitPattern string `orm:"unit_pattern" json:"unit_pattern"` + ContextPattern string `orm:"context_pattern" json:"context_pattern"` Status int `orm:"status" json:"status"` CreatedAt *gtime.Time `orm:"created_at" json:"created_at"` UpdatedAt *gtime.Time `orm:"updated_at" json:"updated_at"` diff --git a/kb/service/chunk_service.go b/kb/service/chunk_service.go index 6ed4cc7..dc53a0b 100644 --- a/kb/service/chunk_service.go +++ b/kb/service/chunk_service.go @@ -2,6 +2,7 @@ package service import ( "context" + "regexp" "strings" "unicode/utf8" @@ -24,34 +25,201 @@ import ( var chunkSeparators = []string{"\n\n", "\n", "。", "！", "？", "；", "，", " ", ""} var runeLen = utf8.RuneCountInString -// SplitByStrategy 按数据集分块策略分块；semantic 需要 embedder，其余策略可传 nil -func (s *chunkService) SplitByStrategy(ctx context.Context, strategy, text string, chunkSize, overlap int, embedder eembedding.Embedder) ([]string, error) { - switch strategy { - case consts.ChunkStrategyRecursive: - return s.splitRecursive(ctx, text, chunkSize, overlap) - case consts.ChunkStrategySemantic: - if embedder == nil { - return nil, gerror.New("语义分块需要向量模型") - } - sp, err := semantic.NewSplitter(ctx, &semantic.Config{ +// SplitAuto 内置组合分块管线（无需用户选择策略），按文档内容自动分派： +// 1. 结构识别（条文/条款/章回/编号等，行首命中达标）→ 结构化切分，带上下文前缀 +// 2. 标题感知（# / 标题 / 第X章 行首）→ 标题并入段落切分 +// 3. 语义分块（无结构无标题，需 embedder）→ 按句子语义切分，超限块递归兜底 +// 4. 递归兜底（embedder 不可用或语义失败）→ 中文分隔符递归切分 +// +// 返回的 unitPattern/ctxPattern 仅结构识别路径有值（自动识别出的结构模式，供落库展示）。 +func (s *chunkService) SplitAuto(ctx context.Context, text string, chunkSize, overlap int, embedder eembedding.Embedder) (chunks []string, unitPattern, ctxPattern string, err error) { + if unit, ctxPat, ok := s.DetectStructure(text); ok { + g.Log().Infof(ctx, "分块管线：结构识别命中（%s / %s），按结构单元切分", unit, ctxPat) + return s.splitStructured(text, unit, ctxPat, chunkSize, overlap), unit, ctxPat, nil + } + if hasHeadings(text) { + g.Log().Infof(ctx, "分块管线：未识别结构，含标题行，走标题感知切分") + return s.SplitText(text, chunkSize, overlap), "", "", nil + } + if embedder != nil { + g.Log().Infof(ctx, "分块管线：无结构无标题，走语义切分") + sp, spErr := semantic.NewSplitter(ctx, &semantic.Config{ Embedding: embedder, BufferSize: 1, MinChunkSize: chunkSize / 2, Separators: chunkSeparators[:len(chunkSeparators)-2], LenFunc: runeLen, }) - if err != nil { - return nil, gerror.Wrap(err, "构建语义分块器失败") + if spErr != nil { + g.Log().Warningf(ctx, "构建语义分块器失败，回退递归分块: %v", spErr) + } else { + semanticChunks, tErr := s.transformText(ctx, sp, text) + if tErr != nil { + g.Log().Warningf(ctx, "语义分块失败，回退递归分块: %v", tErr) + } else if len(semanticChunks) > 0 { + // chunkSize 作为安全上限：语义分块可能产出巨块（长连续语义段），超限块递归二次切分兜底（不重叠） + chunks, err = s.capChunks(ctx, semanticChunks, chunkSize) + return chunks, "", "", err + } } - chunks, err := s.transformText(ctx, sp, text) - if err != nil { - return nil, err - } - // chunkSize 作为安全上限：语义分块可能产出巨块（长连续语义段），超限块用递归二次切分兜底（不重叠） - return s.capChunks(ctx, chunks, chunkSize) - default: - return s.SplitText(text, chunkSize, overlap), nil } + g.Log().Infof(ctx, "分块管线：无结构无标题，语义分块不可用，走递归切分") + chunks, err = s.splitRecursive(ctx, text, chunkSize, overlap) + return chunks, "", "", err +} + +// hasHeadings 文本是否含标题行（Markdown # / 标题 / 第X章），标题感知分块保留标题结构 +func hasHeadings(text string) bool { + for _, line := range strings.Split(text, "\n") { + if isHeading(strings.TrimSpace(line)) { + return true + } + } + return false +} + +// 结构标记候选：单元模式按优先级排列（条文 > 章回 > 小节 > 章节 > 篇部 > 数字编号 > 括号序号） +var ( + unitPatternCandidates = []string{ + `第[一二三四五六七八九十百千]+条`, + `第[一二三四五六七八九十百千]+回`, + `第[一二三四五六七八九十百千]+节`, + `第[一二三四五六七八九十百千]+章`, + `第[一二三四五六七八九十百千]+[篇部]`, + `[一二三四五六七八九十]{1,3}[、.]`, + `\d{1,3}[、.．]`, + `[（(][一二三四五六七八九十\d]{1,3}[)）]`, + } + ctxPatternCandidates = []string{ + `第[一二三四五六七八九十百千]+[篇部]`, + `第[一二三四五六七八九十百千]+章`, + `第[一二三四五六七八九十百千]+节`, + } + minUnitHits = 5 // 行首命中至少 5 次才认定是结构单元（排除正文偶然提及） + minCtxHits = 3 // 上下文标记命中下限 +) + +// DetectStructure 自动识别文本的结构单元与上下文标记：统计各候选模式在行首的出现次数， +// 命中数达到阈值且单元优先顺序靠前者胜出；上下文取单元之外层级最高的候选。未识别出单元时 ok=false。 +func (s *chunkService) DetectStructure(text string) (unit, ctx string, ok bool) { + count := func(pat string) int { + return len(regexp.MustCompile(`(?m)^\s*`+pat).FindAllStringIndex(text, -1)) + } + hits := make(map[string]int, len(unitPatternCandidates)+len(ctxPatternCandidates)) + for _, p := range unitPatternCandidates { + hits[p] = count(p) + } + for _, p := range ctxPatternCandidates { + hits[p] = count(p) + } + for _, p := range unitPatternCandidates { + if hits[p] >= minUnitHits { + unit = p + break + } + } + if unit == "" { + return "", "", false + } + for _, p := range ctxPatternCandidates { + if p == unit { + continue + } + if hits[p] >= minCtxHits { + ctx = p + break + } + } + return unit, ctx, true +} + +// splitStructured 结构单元切分：按单元模式切分（标记保留在块内），上下文模式命中的整行作为前缀； +// 相邻短单元合并到 chunkSize/2（上限 300 字），超长单元按句兜底切分（后续片段补「（标记 续）」前缀）。 +func (s *chunkService) splitStructured(text, unitPattern, ctxPattern string, chunkSize, overlap int) []string { + matches := regexp.MustCompile(`(?m)^\s*(`+unitPattern+`)`).FindAllStringIndex(text, -1) + if len(matches) == 0 { + return s.SplitText(text, chunkSize, overlap) + } + mergeMin := chunkSize / 2 + if mergeMin > 300 { + mergeMin = 300 + } + if mergeMin < 1 { + mergeMin = 1 + } + var ctxMatches [][]int + if ctxPattern != "" { + ctxMatches = regexp.MustCompile(`(?m)^\s*(`+ctxPattern+`)`).FindAllStringIndex(text, -1) + } + // ctxBefore 返回 pos 之前最近的上下文标记整行（如「第三章 劳动合同的解除」） + ctxBefore := func(pos int) string { + cur := "" + for _, m := range ctxMatches { + if m[0] >= pos { + break + } + lineEnd := strings.IndexByte(text[m[0]:], '\n') + if lineEnd < 0 { + lineEnd = len(text) - m[0] + } + cur = strings.TrimSpace(text[m[0] : m[0]+lineEnd]) + } + return cur + } + + out := make([]string, 0, 16) + var buf []string + bufLen := 0 + bufCtx := "" + flush := func() { + if len(buf) == 0 { + return + } + body := strings.Join(buf, "\n") + if bufCtx != "" { + body = "【" + bufCtx + "】" + body + } + out = append(out, body) + buf, bufLen, bufCtx = nil, 0, "" + } + for i, m := range matches { + start := m[1] + end := len(text) + if i+1 < len(matches) { + end = matches[i+1][0] + } + marker := strings.TrimSpace(text[m[0]:m[1]]) + content := strings.TrimSpace(text[start:end]) + if content == "" { + continue + } + item := marker + " " + content + ctx := ctxBefore(m[0]) + if bufLen > 0 && (ctx != bufCtx || bufLen+runeLen(item) > chunkSize) { + flush() + } + if runeLen(item) > chunkSize { + pieces := s.SplitText(item, chunkSize, overlap) + if len(pieces) > 0 { + if ctx != "" { + pieces[0] = "【" + ctx + "】" + pieces[0] + } + out = append(out, pieces[0]) + for _, p := range pieces[1:] { + out = append(out, "（"+marker+" 续）"+p) + } + } + continue + } + buf = append(buf, item) + bufLen += runeLen(item) + bufCtx = ctx + if bufLen >= mergeMin { + flush() + } + } + flush() + return out } // splitRecursive 递归字符分块：按分隔符列表递归切分到目标大小，保留 overlap diff --git a/kb/service/dataset_service.go b/kb/service/dataset_service.go index 8252213..d4a56e2 100644 --- a/kb/service/dataset_service.go +++ b/kb/service/dataset_service.go @@ -23,8 +23,8 @@ func (s *datasetService) Save(ctx context.Context, m *entity.Dataset) (int64, er if m.Status == 0 { m.Status = 1 } - if err := s.validateStrategy(m); err != nil { - return 0, err + if m.EmbeddingCfgId == 0 { + return 0, gerror.New("数据集必须绑定向量模型，请先选择向量模型") } if m.Id > 0 { old, err := dao.Dataset.GetOne(ctx, m.Id) @@ -40,8 +40,8 @@ func (s *datasetService) Save(ctx context.Context, m *entity.Dataset) (int64, er g.Log().Warningf(ctx, "enqueue reembed for dataset %d failed: %v", m.Id, err) } } - // 分块策略/大小/重叠变更 → 入队完整重新解析任务（重新分词+向量化） - if old != nil && (old.ChunkSize != m.ChunkSize || old.ChunkOverlap != m.ChunkOverlap || old.ChunkStrategy != m.ChunkStrategy) { + // 分块大小/重叠变更 → 入队完整重新解析任务（重新分词+向量化） + if old != nil && (old.ChunkSize != m.ChunkSize || old.ChunkOverlap != m.ChunkOverlap) { if err := s.enqueueTask(ctx, m.Id, consts.TaskTypeParse); err != nil { g.Log().Warningf(ctx, "enqueue reparse for dataset %d failed: %v", m.Id, err) } @@ -54,26 +54,9 @@ func (s *datasetService) Save(ctx context.Context, m *entity.Dataset) (int64, er if m.ChunkOverlap < 0 { m.ChunkOverlap = consts.DefaultChunkOverlap } - if m.ChunkStrategy == "" { - m.ChunkStrategy = consts.ChunkStrategyTitle - } return dao.Dataset.Insert(ctx, m) } -func (s *datasetService) validateStrategy(m *entity.Dataset) error { - if m.ChunkStrategy == "" { - m.ChunkStrategy = consts.ChunkStrategyTitle - } - if m.EmbeddingCfgId == 0 { - return gerror.New("数据集必须绑定向量模型，请先选择向量模型") - } - // 语义分块无重叠参数，清零避免误导 - if m.ChunkStrategy == consts.ChunkStrategySemantic { - m.ChunkOverlap = 0 - } - return nil -} - func (s *datasetService) enqueueTask(ctx context.Context, datasetId int64, taskType string) error { docs, _, err := dao.Document.List(ctx, datasetId, 1, 100000) if err != nil { diff --git a/kb/service/parse_task_service.go b/kb/service/parse_task_service.go index ac33a6a..f223cdf 100644 --- a/kb/service/parse_task_service.go +++ b/kb/service/parse_task_service.go @@ -71,10 +71,10 @@ func (s *parseTaskService) processOne(ctx context.Context) { s.fail(ctx, task, "解析文件失败: "+err.Error()) return } - // 数据集分块配置（策略/大小/重叠），未设置用默认值 - chunkSize, chunkOverlap, strategy := consts.DefaultChunkSize, consts.DefaultChunkOverlap, consts.ChunkStrategyTitle + // 数据集分块配置（大小/重叠），未设置用默认值 + chunkSize, chunkOverlap := consts.DefaultChunkSize, consts.DefaultChunkOverlap if ds, err := dao.Dataset.GetOne(ctx, task.DatasetId); err == nil && ds != nil { - chunkSize, chunkOverlap, strategy = ds.ChunkSize, ds.ChunkOverlap, ds.ChunkStrategy + chunkSize, chunkOverlap = ds.ChunkSize, ds.ChunkOverlap } // 数据集必须绑定向量模型，构建失败直接失败任务（不降级全文索引） cfgId, err := dao.Dataset.GetEmbeddingCfgId(ctx, task.DatasetId) @@ -103,11 +103,20 @@ func (s *parseTaskService) processOne(ctx context.Context) { if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"content": text}); err != nil { g.Log().Warningf(ctx, "save document content failed: %v", err) } - chunks, err := ChunkService.SplitByStrategy(ctx, strategy, text, chunkSize, chunkOverlap, embedder) + chunks, unitPattern, ctxPattern, err := ChunkService.SplitAuto(ctx, text, chunkSize, chunkOverlap, embedder) if err != nil { s.fail(ctx, task, "分块失败: "+err.Error()) return } + // 结构识别出的模式落库（列表展示用）；未识别出结构时保留旧值，避免清空已展示的模式 + if unitPattern != "" { + if err := dao.Dataset.UpdateFields(ctx, task.DatasetId, g.Map{ + "unit_pattern": unitPattern, + "context_pattern": ctxPattern, + }); err != nil { + g.Log().Warningf(ctx, "save detected structure patterns failed: %v", err) + } + } if err := ChunkService.InsertAll(ctx, task.DatasetId, doc.Id, chunks, embedder); err != nil { s.fail(ctx, task, "写入分块失败: "+err.Error()) return diff --git a/kb/service/system_config_service.go b/kb/service/system_config_service.go index b9e3dcc..053a302 100644 --- a/kb/service/system_config_service.go +++ b/kb/service/system_config_service.go @@ -4,6 +4,8 @@ import ( "context" "rag-local/common" + "rag-local/kb/consts" + "rag-local/kb/dao" "github.com/gogf/gf/v2/errors/gerror" ) @@ -25,3 +27,23 @@ func (s *systemConfigService) Login(ctx context.Context, token string) (string, } return common.SignToken("owner", common.AccessTokenFingerprint(), common.TokenExpireSeconds) } + +// GetSettings 读取全局分块默认值（未设置时用内置默认值） +func (s *systemConfigService) GetSettings(ctx context.Context) (chunkSize, chunkOverlap int, err error) { + return dao.AppConfig.GetInt(ctx, consts.SettingsKeyChunkSize, consts.DefaultChunkSize), + dao.AppConfig.GetInt(ctx, consts.SettingsKeyChunkOverlap, consts.DefaultChunkOverlap), nil +} + +// SaveSettings 保存全局分块默认值 +func (s *systemConfigService) SaveSettings(ctx context.Context, chunkSize, chunkOverlap int) error { + if chunkSize < 50 || chunkSize > 5000 { + return gerror.New("分块大小需在 50~5000 之间") + } + if chunkOverlap < 0 || chunkOverlap > 500 { + return gerror.New("重叠字数需在 0~500 之间") + } + if err := dao.AppConfig.SetInt(ctx, consts.SettingsKeyChunkSize, chunkSize); err != nil { + return err + } + return dao.AppConfig.SetInt(ctx, consts.SettingsKeyChunkOverlap, chunkOverlap) +} diff --git a/ui-src/package-lock.json b/ui-src/package-lock.json index 836bc2a..7b108b9 100644 --- a/ui-src/package-lock.json +++ b/ui-src/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@element-plus/icons-vue": "^2.3.0", "axios": "^1.6.0", + "echarts": "^6.1.0", "element-plus": "^2.5.0", "pinia": "^2.1.0", "vue": "^3.4.0", @@ -1223,6 +1224,16 @@ "node": ">= 0.4" } }, + "node_modules/echarts": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/echarts/-/echarts-6.1.0.tgz", + "integrity": "sha512-q0yaFPggC9FUdsWH4blavRWFmxdrIodbkoKNAjJudAI6CA9gNPxHtV2RcZNEepZVlk4yvBYkOkbk6HIVpIyHZA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "2.3.0", + "zrender": "6.1.0" + } + }, "node_modules/element-plus": { "version": "2.14.3", "resolved": "https://registry.npmjs.org/element-plus/-/element-plus-2.14.3.tgz", @@ -1730,6 +1741,12 @@ "node": ">=0.10.0" } }, + "node_modules/tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==", + "license": "0BSD" + }, "node_modules/vite": { "version": "5.4.21", "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", @@ -1857,6 +1874,15 @@ "peerDependencies": { "vue": "^3.5.0" } + }, + "node_modules/zrender": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/zrender/-/zrender-6.1.0.tgz", + "integrity": "sha512-oEGMDB6pOP2S6OwRR4PdVv610zrjnA3Bh+JnSG12fYJlBKjtNAoEb5fSUoCOOINlH96I2fU38/A2UpRKs67xYQ==", + "license": "BSD-3-Clause", + "dependencies": { + "tslib": "2.3.0" + } } } } diff --git a/ui-src/package.json b/ui-src/package.json index ae2b69d..6e5d53a 100644 --- a/ui-src/package.json +++ b/ui-src/package.json @@ -11,6 +11,7 @@ "dependencies": { "@element-plus/icons-vue": "^2.3.0", "axios": "^1.6.0", + "echarts": "^6.1.0", "element-plus": "^2.5.0", "pinia": "^2.1.0", "vue": "^3.4.0", diff --git a/ui-src/src/api/settings.js b/ui-src/src/api/settings.js new file mode 100644 index 0000000..695523d --- /dev/null +++ b/ui-src/src/api/settings.js @@ -0,0 +1,9 @@ +import request from './request.js' + +export function getSettings() { + return request.get('/system-config/settings') +} + +export function saveSettings(data) { + return request.post('/system-config/save-settings', data) +} diff --git a/ui-src/src/views/DatasetList.vue b/ui-src/src/views/DatasetList.vue index d811d82..50067f4 100644 --- a/ui-src/src/views/DatasetList.vue +++ b/ui-src/src/views/DatasetList.vue @@ -12,8 +12,10 @@ 未设置 - - + + @@ -38,21 +40,13 @@ - - - - - - -
标题分块保留标题与段落结构；递归字符分块按分隔符切分，通用文本；语义分块按语义相似度切分，质量更高但解析更慢，且需绑定向量模型
- - + -
{{ form.chunk_strategy === 'semantic' ? '语义分块按句子相似度切分，此值为安全上限（max_chunk_size），超过上限的块会再切分；语义分块不支持重叠' : '每块最大字数，超长段落自动按句号/换行切分' }}
+
每块最大字数。系统自动组合分块策略：优先识别文档结构（条文/章节/编号等）按结构切分，无结构时依次按标题感知、语义、递归切分
 - + -
相邻分块间的重叠字数，用于保持上下文连贯
+
相邻分块间的重叠字数，用于保持上下文连贯（语义切分路径自动忽略）