1
This commit is contained in:
@@ -1,5 +1,4 @@
|
||||
# 运行时数据与本地环境
|
||||
data/
|
||||
workspace/
|
||||
.idea/
|
||||
.DS_Store
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
+3
-3
@@ -11,9 +11,9 @@ const (
|
||||
DefaultChunkSize = 800 // 分块最大字数(数据集默认值)
|
||||
DefaultChunkOverlap = 150 // 分块重叠字数(数据集默认值)
|
||||
|
||||
ChunkStrategyTitle = "title" // 标题感知分块(自研,保留标题与段落)
|
||||
ChunkStrategyRecursive = "recursive" // 递归字符分块(Eino recursive)
|
||||
ChunkStrategySemantic = "semantic" // 语义分块(Eino semantic,需绑定向量模型)
|
||||
// 全局设置键(app_config 表)
|
||||
SettingsKeyChunkSize = "chunk_default_size"
|
||||
SettingsKeyChunkOverlap = "chunk_default_overlap"
|
||||
|
||||
ParsePollIntervalSeconds = 3 // 解析任务轮询间隔
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package consts
|
||||
|
||||
const (
|
||||
TableNameModelConfig = "model_config"
|
||||
TableNameAppConfig = "app_config"
|
||||
TableNameDataset = "kb_dataset"
|
||||
TableNameDocument = "kb_document"
|
||||
TableNameChunk = "kb_chunk"
|
||||
|
||||
@@ -28,7 +28,6 @@ func (c *dataset) Save(ctx context.Context, req *dto.SaveDatasetReq) (*dto.SaveD
|
||||
EmbeddingCfgId: req.EmbeddingCfgId,
|
||||
ChunkSize: req.ChunkSize,
|
||||
ChunkOverlap: req.ChunkOverlap,
|
||||
ChunkStrategy: req.ChunkStrategy,
|
||||
Status: 1,
|
||||
})
|
||||
if err != nil {
|
||||
|
||||
@@ -18,3 +18,18 @@ func (c *systemConfig) Login(ctx context.Context, req *dto.LoginReq) (res *dto.L
|
||||
}
|
||||
return &dto.LoginRes{Token: token}, nil
|
||||
}
|
||||
|
||||
func (c *systemConfig) GetSettings(ctx context.Context, _ *dto.GetSettingsReq) (*dto.GetSettingsRes, error) {
|
||||
size, overlap, err := service.SystemConfigService.GetSettings(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dto.GetSettingsRes{ChunkSize: size, ChunkOverlap: overlap}, nil
|
||||
}
|
||||
|
||||
func (c *systemConfig) SaveSettings(ctx context.Context, req *dto.SaveSettingsReq) (*dto.SaveSettingsRes, error) {
|
||||
if err := service.SystemConfigService.SaveSettings(ctx, req.ChunkSize, req.ChunkOverlap); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &dto.SaveSettingsRes{}, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package dao
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"rag-local/kb/consts"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
)
|
||||
|
||||
var AppConfig = &appConfigDao{}
|
||||
|
||||
type appConfigDao struct{}
|
||||
|
||||
func init() {
|
||||
ctx := context.Background()
|
||||
_, err := g.DB(consts.DbGroupSystem).Exec(ctx, `CREATE TABLE IF NOT EXISTS `+consts.TableNameAppConfig+` (
|
||||
cfg_key TEXT PRIMARY KEY,
|
||||
cfg_value TEXT NOT NULL DEFAULT '',
|
||||
updated_at DATETIME DEFAULT (datetime('now','localtime'))
|
||||
)`)
|
||||
if err != nil {
|
||||
g.Log().Warningf(ctx, "create app_config table failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// GetInt 读取全局设置值,未设置或非法时返回默认值 def
|
||||
func (d *appConfigDao) GetInt(ctx context.Context, key string, def int) int {
|
||||
r, err := g.DB(consts.DbGroupSystem).Ctx(ctx).GetValue(ctx,
|
||||
"SELECT cfg_value FROM "+consts.TableNameAppConfig+" WHERE cfg_key=?", key)
|
||||
if err != nil || r.IsEmpty() {
|
||||
return def
|
||||
}
|
||||
return r.Int()
|
||||
}
|
||||
|
||||
// SetInt 写入全局设置值(UPSERT)
|
||||
func (d *appConfigDao) SetInt(ctx context.Context, key string, v int) error {
|
||||
_, err := g.DB(consts.DbGroupSystem).Ctx(ctx).Exec(ctx,
|
||||
"INSERT INTO "+consts.TableNameAppConfig+"(cfg_key, cfg_value) VALUES(?, ?) "+
|
||||
"ON CONFLICT(cfg_key) DO UPDATE SET cfg_value=excluded.cfg_value, updated_at=datetime('now','localtime')",
|
||||
key, v)
|
||||
return err
|
||||
}
|
||||
+25
-3
@@ -95,7 +95,12 @@ func (d *chunkDao) InsertWithVec(ctx context.Context, datasetId, documentId int6
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
// Commit 成功后 IsClosed 为 true,跳过 Rollback,避免对已提交事务回滚产生报错日志
|
||||
defer func() {
|
||||
if !tx.IsClosed() {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
r, err := tx.Model(consts.TableNameChunk).Ctx(ctx).Data(g.Map{
|
||||
"dataset_id": datasetId,
|
||||
@@ -133,7 +138,12 @@ func (d *chunkDao) DeleteByDocument(ctx context.Context, documentId int64) error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
// Commit 成功后 IsClosed 为 true,跳过 Rollback,避免对已提交事务回滚产生报错日志
|
||||
defer func() {
|
||||
if !tx.IsClosed() {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
ids := tx.Model(consts.TableNameChunk).Ctx(ctx).Fields("id").Where("document_id", documentId)
|
||||
r, err := ids.Array()
|
||||
@@ -158,6 +168,13 @@ func (d *chunkDao) DeleteByDocument(ctx context.Context, documentId int64) error
|
||||
if _, err := tx.Exec("DELETE FROM "+consts.TableNameChunkFts+" WHERE chunk_id IN ("+in+")", args...); err != nil {
|
||||
return err
|
||||
}
|
||||
// 知识图谱数据按 chunk 关联,重新解析时旧 chunk 消失,一并清理避免孤儿数据
|
||||
if _, err := tx.Exec("DELETE FROM "+consts.TableNameKgRelation+" WHERE chunk_id IN ("+in+")", args...); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec("DELETE FROM "+consts.TableNameKgEntity+" WHERE chunk_id IN ("+in+")", args...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := tx.Model(consts.TableNameChunk).Ctx(ctx).Where("document_id", documentId).Delete(); err != nil {
|
||||
return err
|
||||
@@ -170,7 +187,12 @@ func (d *chunkDao) UpdateContent(ctx context.Context, id int64, content, vecJson
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
// Commit 成功后 IsClosed 为 true,跳过 Rollback,避免对已提交事务回滚产生报错日志
|
||||
defer func() {
|
||||
if !tx.IsClosed() {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := tx.Model(consts.TableNameChunk).Ctx(ctx).Data(g.Map{"content": content}).Where("id", id).Update(); err != nil {
|
||||
return err
|
||||
|
||||
@@ -86,7 +86,12 @@ func (d *conversationDao) Delete(ctx context.Context, id int64) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
// Commit 成功后 IsClosed 为 true,跳过 Rollback,避免对已提交事务回滚产生报错日志
|
||||
defer func() {
|
||||
if !tx.IsClosed() {
|
||||
_ = tx.Rollback()
|
||||
}
|
||||
}()
|
||||
if _, err := tx.Model(consts.TableNameMessage).Ctx(ctx).Where("conversation_id", id).Delete(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+10
-2
@@ -26,6 +26,8 @@ func init() {
|
||||
chunk_size INTEGER NOT NULL DEFAULT 800,
|
||||
chunk_overlap INTEGER NOT NULL DEFAULT 150,
|
||||
chunk_strategy TEXT NOT NULL DEFAULT 'title',
|
||||
unit_pattern TEXT NOT NULL DEFAULT '',
|
||||
context_pattern TEXT NOT NULL DEFAULT '',
|
||||
status INTEGER NOT NULL DEFAULT 1,
|
||||
created_at DATETIME DEFAULT (datetime('now','localtime')),
|
||||
updated_at DATETIME DEFAULT (datetime('now','localtime'))
|
||||
@@ -41,6 +43,8 @@ func init() {
|
||||
{"chunk_size", "chunk_size INTEGER NOT NULL DEFAULT 800"},
|
||||
{"chunk_overlap", "chunk_overlap INTEGER NOT NULL DEFAULT 150"},
|
||||
{"chunk_strategy", "chunk_strategy TEXT NOT NULL DEFAULT 'title'"},
|
||||
{"unit_pattern", "unit_pattern TEXT NOT NULL DEFAULT ''"},
|
||||
{"context_pattern", "context_pattern TEXT NOT NULL DEFAULT ''"},
|
||||
} {
|
||||
cnt, err := g.DB(consts.DbGroupDefault).Ctx(ctx).GetValue(ctx,
|
||||
"SELECT COUNT(*) FROM pragma_table_info('"+consts.TableNameDataset+"') WHERE name=?", col.name)
|
||||
@@ -83,7 +87,6 @@ func (d *datasetDao) Insert(ctx context.Context, data *entity.Dataset) (int64, e
|
||||
"embedding_cfg_id": data.EmbeddingCfgId,
|
||||
"chunk_size": data.ChunkSize,
|
||||
"chunk_overlap": data.ChunkOverlap,
|
||||
"chunk_strategy": data.ChunkStrategy,
|
||||
"status": data.Status,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
@@ -101,7 +104,6 @@ func (d *datasetDao) Update(ctx context.Context, data *entity.Dataset) error {
|
||||
"embedding_cfg_id": data.EmbeddingCfgId,
|
||||
"chunk_size": data.ChunkSize,
|
||||
"chunk_overlap": data.ChunkOverlap,
|
||||
"chunk_strategy": data.ChunkStrategy,
|
||||
"status": data.Status,
|
||||
"updated_at": gtime.Now().Format("Y-m-d H:i:s"),
|
||||
}).Where("id", data.Id).Update()
|
||||
@@ -113,6 +115,12 @@ func (d *datasetDao) Delete(ctx context.Context, id int64) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *datasetDao) UpdateFields(ctx context.Context, id int64, data g.Map) error {
|
||||
_, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameDataset).Ctx(ctx).Data(data).
|
||||
Where("id", id).Update()
|
||||
return err
|
||||
}
|
||||
|
||||
func (d *datasetDao) GetEmbeddingCfgId(ctx context.Context, id int64) (int64, error) {
|
||||
r, err := g.DB(consts.DbGroupDefault).Model(consts.TableNameDataset).Ctx(ctx).
|
||||
Fields("embedding_cfg_id").Where("id", id).One()
|
||||
|
||||
@@ -22,7 +22,6 @@ type SaveDatasetReq struct {
|
||||
EmbeddingCfgId int64 `json:"embedding_cfg_id"`
|
||||
ChunkSize int `json:"chunk_size"`
|
||||
ChunkOverlap int `json:"chunk_overlap"`
|
||||
ChunkStrategy string `json:"chunk_strategy"`
|
||||
}
|
||||
|
||||
type SaveDatasetRes struct {
|
||||
|
||||
@@ -10,3 +10,20 @@ type LoginReq struct {
|
||||
type LoginRes struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
type GetSettingsReq struct {
|
||||
g.Meta `path:"/settings" method:"get" tags:"系统配置" summary:"获取全局设置"`
|
||||
}
|
||||
|
||||
type GetSettingsRes struct {
|
||||
ChunkSize int `json:"chunk_size"` // 默认分块大小
|
||||
ChunkOverlap int `json:"chunk_overlap"` // 默认重叠字数
|
||||
}
|
||||
|
||||
type SaveSettingsReq struct {
|
||||
g.Meta `path:"/save-settings" method:"post" tags:"系统配置" summary:"保存全局设置"`
|
||||
ChunkSize int `json:"chunk_size"`
|
||||
ChunkOverlap int `json:"chunk_overlap"`
|
||||
}
|
||||
|
||||
type SaveSettingsRes struct{}
|
||||
|
||||
@@ -9,7 +9,8 @@ type Dataset struct {
|
||||
EmbeddingCfgId int64 `orm:"embedding_cfg_id" json:"embedding_cfg_id"`
|
||||
ChunkSize int `orm:"chunk_size" json:"chunk_size"`
|
||||
ChunkOverlap int `orm:"chunk_overlap" json:"chunk_overlap"`
|
||||
ChunkStrategy string `orm:"chunk_strategy" json:"chunk_strategy"`
|
||||
UnitPattern string `orm:"unit_pattern" json:"unit_pattern"`
|
||||
ContextPattern string `orm:"context_pattern" json:"context_pattern"`
|
||||
Status int `orm:"status" json:"status"`
|
||||
CreatedAt *gtime.Time `orm:"created_at" json:"created_at"`
|
||||
UpdatedAt *gtime.Time `orm:"updated_at" json:"updated_at"`
|
||||
|
||||
+188
-20
@@ -2,6 +2,7 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
@@ -24,34 +25,201 @@ import (
|
||||
var chunkSeparators = []string{"\n\n", "\n", "。", "!", "?", ";", ",", " ", ""}
|
||||
var runeLen = utf8.RuneCountInString
|
||||
|
||||
// SplitByStrategy 按数据集分块策略分块;semantic 需要 embedder,其余策略可传 nil
|
||||
func (s *chunkService) SplitByStrategy(ctx context.Context, strategy, text string, chunkSize, overlap int, embedder eembedding.Embedder) ([]string, error) {
|
||||
switch strategy {
|
||||
case consts.ChunkStrategyRecursive:
|
||||
return s.splitRecursive(ctx, text, chunkSize, overlap)
|
||||
case consts.ChunkStrategySemantic:
|
||||
if embedder == nil {
|
||||
return nil, gerror.New("语义分块需要向量模型")
|
||||
}
|
||||
sp, err := semantic.NewSplitter(ctx, &semantic.Config{
|
||||
// SplitAuto 内置组合分块管线(无需用户选择策略),按文档内容自动分派:
|
||||
// 1. 结构识别(条文/条款/章回/编号等,行首命中达标)→ 结构化切分,带上下文前缀
|
||||
// 2. 标题感知(# / 标题 / 第X章 行首)→ 标题并入段落切分
|
||||
// 3. 语义分块(无结构无标题,需 embedder)→ 按句子语义切分,超限块递归兜底
|
||||
// 4. 递归兜底(embedder 不可用或语义失败)→ 中文分隔符递归切分
|
||||
//
|
||||
// 返回的 unitPattern/ctxPattern 仅结构识别路径有值(自动识别出的结构模式,供落库展示)。
|
||||
func (s *chunkService) SplitAuto(ctx context.Context, text string, chunkSize, overlap int, embedder eembedding.Embedder) (chunks []string, unitPattern, ctxPattern string, err error) {
|
||||
if unit, ctxPat, ok := s.DetectStructure(text); ok {
|
||||
g.Log().Infof(ctx, "分块管线:结构识别命中(%s / %s),按结构单元切分", unit, ctxPat)
|
||||
return s.splitStructured(text, unit, ctxPat, chunkSize, overlap), unit, ctxPat, nil
|
||||
}
|
||||
if hasHeadings(text) {
|
||||
g.Log().Infof(ctx, "分块管线:未识别结构,含标题行,走标题感知切分")
|
||||
return s.SplitText(text, chunkSize, overlap), "", "", nil
|
||||
}
|
||||
if embedder != nil {
|
||||
g.Log().Infof(ctx, "分块管线:无结构无标题,走语义切分")
|
||||
sp, spErr := semantic.NewSplitter(ctx, &semantic.Config{
|
||||
Embedding: embedder,
|
||||
BufferSize: 1,
|
||||
MinChunkSize: chunkSize / 2,
|
||||
Separators: chunkSeparators[:len(chunkSeparators)-2],
|
||||
LenFunc: runeLen,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, gerror.Wrap(err, "构建语义分块器失败")
|
||||
if spErr != nil {
|
||||
g.Log().Warningf(ctx, "构建语义分块器失败,回退递归分块: %v", spErr)
|
||||
} else {
|
||||
semanticChunks, tErr := s.transformText(ctx, sp, text)
|
||||
if tErr != nil {
|
||||
g.Log().Warningf(ctx, "语义分块失败,回退递归分块: %v", tErr)
|
||||
} else if len(semanticChunks) > 0 {
|
||||
// chunkSize 作为安全上限:语义分块可能产出巨块(长连续语义段),超限块递归二次切分兜底(不重叠)
|
||||
chunks, err = s.capChunks(ctx, semanticChunks, chunkSize)
|
||||
return chunks, "", "", err
|
||||
}
|
||||
}
|
||||
chunks, err := s.transformText(ctx, sp, text)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// chunkSize 作为安全上限:语义分块可能产出巨块(长连续语义段),超限块用递归二次切分兜底(不重叠)
|
||||
return s.capChunks(ctx, chunks, chunkSize)
|
||||
default:
|
||||
return s.SplitText(text, chunkSize, overlap), nil
|
||||
}
|
||||
g.Log().Infof(ctx, "分块管线:无结构无标题,语义分块不可用,走递归切分")
|
||||
chunks, err = s.splitRecursive(ctx, text, chunkSize, overlap)
|
||||
return chunks, "", "", err
|
||||
}
|
||||
|
||||
// hasHeadings 文本是否含标题行(Markdown # / 标题 / 第X章),标题感知分块保留标题结构
|
||||
func hasHeadings(text string) bool {
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
if isHeading(strings.TrimSpace(line)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// 结构标记候选:单元模式按优先级排列(条文 > 章回 > 小节 > 章节 > 篇部 > 数字编号 > 括号序号)
|
||||
var (
|
||||
unitPatternCandidates = []string{
|
||||
`第[一二三四五六七八九十百千]+条`,
|
||||
`第[一二三四五六七八九十百千]+回`,
|
||||
`第[一二三四五六七八九十百千]+节`,
|
||||
`第[一二三四五六七八九十百千]+章`,
|
||||
`第[一二三四五六七八九十百千]+[篇部]`,
|
||||
`[一二三四五六七八九十]{1,3}[、.]`,
|
||||
`\d{1,3}[、..]`,
|
||||
`[((][一二三四五六七八九十\d]{1,3}[))]`,
|
||||
}
|
||||
ctxPatternCandidates = []string{
|
||||
`第[一二三四五六七八九十百千]+[篇部]`,
|
||||
`第[一二三四五六七八九十百千]+章`,
|
||||
`第[一二三四五六七八九十百千]+节`,
|
||||
}
|
||||
minUnitHits = 5 // 行首命中至少 5 次才认定是结构单元(排除正文偶然提及)
|
||||
minCtxHits = 3 // 上下文标记命中下限
|
||||
)
|
||||
|
||||
// DetectStructure 自动识别文本的结构单元与上下文标记:统计各候选模式在行首的出现次数,
|
||||
// 命中数达到阈值且单元优先顺序靠前者胜出;上下文取单元之外层级最高的候选。未识别出单元时 ok=false。
|
||||
func (s *chunkService) DetectStructure(text string) (unit, ctx string, ok bool) {
|
||||
count := func(pat string) int {
|
||||
return len(regexp.MustCompile(`(?m)^\s*`+pat).FindAllStringIndex(text, -1))
|
||||
}
|
||||
hits := make(map[string]int, len(unitPatternCandidates)+len(ctxPatternCandidates))
|
||||
for _, p := range unitPatternCandidates {
|
||||
hits[p] = count(p)
|
||||
}
|
||||
for _, p := range ctxPatternCandidates {
|
||||
hits[p] = count(p)
|
||||
}
|
||||
for _, p := range unitPatternCandidates {
|
||||
if hits[p] >= minUnitHits {
|
||||
unit = p
|
||||
break
|
||||
}
|
||||
}
|
||||
if unit == "" {
|
||||
return "", "", false
|
||||
}
|
||||
for _, p := range ctxPatternCandidates {
|
||||
if p == unit {
|
||||
continue
|
||||
}
|
||||
if hits[p] >= minCtxHits {
|
||||
ctx = p
|
||||
break
|
||||
}
|
||||
}
|
||||
return unit, ctx, true
|
||||
}
|
||||
|
||||
// splitStructured 结构单元切分:按单元模式切分(标记保留在块内),上下文模式命中的整行作为前缀;
|
||||
// 相邻短单元合并到 chunkSize/2(上限 300 字),超长单元按句兜底切分(后续片段补「(标记 续)」前缀)。
|
||||
func (s *chunkService) splitStructured(text, unitPattern, ctxPattern string, chunkSize, overlap int) []string {
|
||||
matches := regexp.MustCompile(`(?m)^\s*(`+unitPattern+`)`).FindAllStringIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return s.SplitText(text, chunkSize, overlap)
|
||||
}
|
||||
mergeMin := chunkSize / 2
|
||||
if mergeMin > 300 {
|
||||
mergeMin = 300
|
||||
}
|
||||
if mergeMin < 1 {
|
||||
mergeMin = 1
|
||||
}
|
||||
var ctxMatches [][]int
|
||||
if ctxPattern != "" {
|
||||
ctxMatches = regexp.MustCompile(`(?m)^\s*(`+ctxPattern+`)`).FindAllStringIndex(text, -1)
|
||||
}
|
||||
// ctxBefore 返回 pos 之前最近的上下文标记整行(如「第三章 劳动合同的解除」)
|
||||
ctxBefore := func(pos int) string {
|
||||
cur := ""
|
||||
for _, m := range ctxMatches {
|
||||
if m[0] >= pos {
|
||||
break
|
||||
}
|
||||
lineEnd := strings.IndexByte(text[m[0]:], '\n')
|
||||
if lineEnd < 0 {
|
||||
lineEnd = len(text) - m[0]
|
||||
}
|
||||
cur = strings.TrimSpace(text[m[0] : m[0]+lineEnd])
|
||||
}
|
||||
return cur
|
||||
}
|
||||
|
||||
out := make([]string, 0, 16)
|
||||
var buf []string
|
||||
bufLen := 0
|
||||
bufCtx := ""
|
||||
flush := func() {
|
||||
if len(buf) == 0 {
|
||||
return
|
||||
}
|
||||
body := strings.Join(buf, "\n")
|
||||
if bufCtx != "" {
|
||||
body = "【" + bufCtx + "】" + body
|
||||
}
|
||||
out = append(out, body)
|
||||
buf, bufLen, bufCtx = nil, 0, ""
|
||||
}
|
||||
for i, m := range matches {
|
||||
start := m[1]
|
||||
end := len(text)
|
||||
if i+1 < len(matches) {
|
||||
end = matches[i+1][0]
|
||||
}
|
||||
marker := strings.TrimSpace(text[m[0]:m[1]])
|
||||
content := strings.TrimSpace(text[start:end])
|
||||
if content == "" {
|
||||
continue
|
||||
}
|
||||
item := marker + " " + content
|
||||
ctx := ctxBefore(m[0])
|
||||
if bufLen > 0 && (ctx != bufCtx || bufLen+runeLen(item) > chunkSize) {
|
||||
flush()
|
||||
}
|
||||
if runeLen(item) > chunkSize {
|
||||
pieces := s.SplitText(item, chunkSize, overlap)
|
||||
if len(pieces) > 0 {
|
||||
if ctx != "" {
|
||||
pieces[0] = "【" + ctx + "】" + pieces[0]
|
||||
}
|
||||
out = append(out, pieces[0])
|
||||
for _, p := range pieces[1:] {
|
||||
out = append(out, "("+marker+" 续)"+p)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
buf = append(buf, item)
|
||||
bufLen += runeLen(item)
|
||||
bufCtx = ctx
|
||||
if bufLen >= mergeMin {
|
||||
flush()
|
||||
}
|
||||
}
|
||||
flush()
|
||||
return out
|
||||
}
|
||||
|
||||
// splitRecursive 递归字符分块:按分隔符列表递归切分到目标大小,保留 overlap
|
||||
|
||||
@@ -23,8 +23,8 @@ func (s *datasetService) Save(ctx context.Context, m *entity.Dataset) (int64, er
|
||||
if m.Status == 0 {
|
||||
m.Status = 1
|
||||
}
|
||||
if err := s.validateStrategy(m); err != nil {
|
||||
return 0, err
|
||||
if m.EmbeddingCfgId == 0 {
|
||||
return 0, gerror.New("数据集必须绑定向量模型,请先选择向量模型")
|
||||
}
|
||||
if m.Id > 0 {
|
||||
old, err := dao.Dataset.GetOne(ctx, m.Id)
|
||||
@@ -40,8 +40,8 @@ func (s *datasetService) Save(ctx context.Context, m *entity.Dataset) (int64, er
|
||||
g.Log().Warningf(ctx, "enqueue reembed for dataset %d failed: %v", m.Id, err)
|
||||
}
|
||||
}
|
||||
// 分块策略/大小/重叠变更 → 入队完整重新解析任务(重新分词+向量化)
|
||||
if old != nil && (old.ChunkSize != m.ChunkSize || old.ChunkOverlap != m.ChunkOverlap || old.ChunkStrategy != m.ChunkStrategy) {
|
||||
// 分块大小/重叠变更 → 入队完整重新解析任务(重新分词+向量化)
|
||||
if old != nil && (old.ChunkSize != m.ChunkSize || old.ChunkOverlap != m.ChunkOverlap) {
|
||||
if err := s.enqueueTask(ctx, m.Id, consts.TaskTypeParse); err != nil {
|
||||
g.Log().Warningf(ctx, "enqueue reparse for dataset %d failed: %v", m.Id, err)
|
||||
}
|
||||
@@ -54,26 +54,9 @@ func (s *datasetService) Save(ctx context.Context, m *entity.Dataset) (int64, er
|
||||
if m.ChunkOverlap < 0 {
|
||||
m.ChunkOverlap = consts.DefaultChunkOverlap
|
||||
}
|
||||
if m.ChunkStrategy == "" {
|
||||
m.ChunkStrategy = consts.ChunkStrategyTitle
|
||||
}
|
||||
return dao.Dataset.Insert(ctx, m)
|
||||
}
|
||||
|
||||
func (s *datasetService) validateStrategy(m *entity.Dataset) error {
|
||||
if m.ChunkStrategy == "" {
|
||||
m.ChunkStrategy = consts.ChunkStrategyTitle
|
||||
}
|
||||
if m.EmbeddingCfgId == 0 {
|
||||
return gerror.New("数据集必须绑定向量模型,请先选择向量模型")
|
||||
}
|
||||
// 语义分块无重叠参数,清零避免误导
|
||||
if m.ChunkStrategy == consts.ChunkStrategySemantic {
|
||||
m.ChunkOverlap = 0
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *datasetService) enqueueTask(ctx context.Context, datasetId int64, taskType string) error {
|
||||
docs, _, err := dao.Document.List(ctx, datasetId, 1, 100000)
|
||||
if err != nil {
|
||||
|
||||
@@ -71,10 +71,10 @@ func (s *parseTaskService) processOne(ctx context.Context) {
|
||||
s.fail(ctx, task, "解析文件失败: "+err.Error())
|
||||
return
|
||||
}
|
||||
// 数据集分块配置(策略/大小/重叠),未设置用默认值
|
||||
chunkSize, chunkOverlap, strategy := consts.DefaultChunkSize, consts.DefaultChunkOverlap, consts.ChunkStrategyTitle
|
||||
// 数据集分块配置(大小/重叠),未设置用默认值
|
||||
chunkSize, chunkOverlap := consts.DefaultChunkSize, consts.DefaultChunkOverlap
|
||||
if ds, err := dao.Dataset.GetOne(ctx, task.DatasetId); err == nil && ds != nil {
|
||||
chunkSize, chunkOverlap, strategy = ds.ChunkSize, ds.ChunkOverlap, ds.ChunkStrategy
|
||||
chunkSize, chunkOverlap = ds.ChunkSize, ds.ChunkOverlap
|
||||
}
|
||||
// 数据集必须绑定向量模型,构建失败直接失败任务(不降级全文索引)
|
||||
cfgId, err := dao.Dataset.GetEmbeddingCfgId(ctx, task.DatasetId)
|
||||
@@ -103,11 +103,20 @@ func (s *parseTaskService) processOne(ctx context.Context) {
|
||||
if err := dao.Document.UpdateFields(ctx, doc.Id, g.Map{"content": text}); err != nil {
|
||||
g.Log().Warningf(ctx, "save document content failed: %v", err)
|
||||
}
|
||||
chunks, err := ChunkService.SplitByStrategy(ctx, strategy, text, chunkSize, chunkOverlap, embedder)
|
||||
chunks, unitPattern, ctxPattern, err := ChunkService.SplitAuto(ctx, text, chunkSize, chunkOverlap, embedder)
|
||||
if err != nil {
|
||||
s.fail(ctx, task, "分块失败: "+err.Error())
|
||||
return
|
||||
}
|
||||
// 结构识别出的模式落库(列表展示用);未识别出结构时保留旧值,避免清空已展示的模式
|
||||
if unitPattern != "" {
|
||||
if err := dao.Dataset.UpdateFields(ctx, task.DatasetId, g.Map{
|
||||
"unit_pattern": unitPattern,
|
||||
"context_pattern": ctxPattern,
|
||||
}); err != nil {
|
||||
g.Log().Warningf(ctx, "save detected structure patterns failed: %v", err)
|
||||
}
|
||||
}
|
||||
if err := ChunkService.InsertAll(ctx, task.DatasetId, doc.Id, chunks, embedder); err != nil {
|
||||
s.fail(ctx, task, "写入分块失败: "+err.Error())
|
||||
return
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
|
||||
"rag-local/common"
|
||||
"rag-local/kb/consts"
|
||||
"rag-local/kb/dao"
|
||||
|
||||
"github.com/gogf/gf/v2/errors/gerror"
|
||||
)
|
||||
@@ -25,3 +27,23 @@ func (s *systemConfigService) Login(ctx context.Context, token string) (string,
|
||||
}
|
||||
return common.SignToken("owner", common.AccessTokenFingerprint(), common.TokenExpireSeconds)
|
||||
}
|
||||
|
||||
// GetSettings 读取全局分块默认值(未设置时用内置默认值)
|
||||
func (s *systemConfigService) GetSettings(ctx context.Context) (chunkSize, chunkOverlap int, err error) {
|
||||
return dao.AppConfig.GetInt(ctx, consts.SettingsKeyChunkSize, consts.DefaultChunkSize),
|
||||
dao.AppConfig.GetInt(ctx, consts.SettingsKeyChunkOverlap, consts.DefaultChunkOverlap), nil
|
||||
}
|
||||
|
||||
// SaveSettings 保存全局分块默认值
|
||||
func (s *systemConfigService) SaveSettings(ctx context.Context, chunkSize, chunkOverlap int) error {
|
||||
if chunkSize < 50 || chunkSize > 5000 {
|
||||
return gerror.New("分块大小需在 50~5000 之间")
|
||||
}
|
||||
if chunkOverlap < 0 || chunkOverlap > 500 {
|
||||
return gerror.New("重叠字数需在 0~500 之间")
|
||||
}
|
||||
if err := dao.AppConfig.SetInt(ctx, consts.SettingsKeyChunkSize, chunkSize); err != nil {
|
||||
return err
|
||||
}
|
||||
return dao.AppConfig.SetInt(ctx, consts.SettingsKeyChunkOverlap, chunkOverlap)
|
||||
}
|
||||
|
||||
Generated
+26
@@ -10,6 +10,7 @@
|
||||
"dependencies": {
|
||||
"@element-plus/icons-vue": "^2.3.0",
|
||||
"axios": "^1.6.0",
|
||||
"echarts": "^6.1.0",
|
||||
"element-plus": "^2.5.0",
|
||||
"pinia": "^2.1.0",
|
||||
"vue": "^3.4.0",
|
||||
@@ -1223,6 +1224,16 @@
|
||||
"node": ">= 0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/echarts": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/echarts/-/echarts-6.1.0.tgz",
|
||||
"integrity": "sha512-q0yaFPggC9FUdsWH4blavRWFmxdrIodbkoKNAjJudAI6CA9gNPxHtV2RcZNEepZVlk4yvBYkOkbk6HIVpIyHZA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"tslib": "2.3.0",
|
||||
"zrender": "6.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/element-plus": {
|
||||
"version": "2.14.3",
|
||||
"resolved": "https://registry.npmjs.org/element-plus/-/element-plus-2.14.3.tgz",
|
||||
@@ -1730,6 +1741,12 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz",
|
||||
"integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==",
|
||||
"license": "0BSD"
|
||||
},
|
||||
"node_modules/vite": {
|
||||
"version": "5.4.21",
|
||||
"resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz",
|
||||
@@ -1857,6 +1874,15 @@
|
||||
"peerDependencies": {
|
||||
"vue": "^3.5.0"
|
||||
}
|
||||
},
|
||||
"node_modules/zrender": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/zrender/-/zrender-6.1.0.tgz",
|
||||
"integrity": "sha512-oEGMDB6pOP2S6OwRR4PdVv610zrjnA3Bh+JnSG12fYJlBKjtNAoEb5fSUoCOOINlH96I2fU38/A2UpRKs67xYQ==",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"tslib": "2.3.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
"dependencies": {
|
||||
"@element-plus/icons-vue": "^2.3.0",
|
||||
"axios": "^1.6.0",
|
||||
"echarts": "^6.1.0",
|
||||
"element-plus": "^2.5.0",
|
||||
"pinia": "^2.1.0",
|
||||
"vue": "^3.4.0",
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
import request from './request.js'
|
||||
|
||||
export function getSettings() {
|
||||
return request.get('/system-config/settings')
|
||||
}
|
||||
|
||||
export function saveSettings(data) {
|
||||
return request.post('/system-config/save-settings', data)
|
||||
}
|
||||
@@ -12,8 +12,10 @@
|
||||
<el-tag v-else size="small" type="info">未设置</el-tag>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column label="分块" width="140">
|
||||
<template #default="{ row }">{{ STRATEGY_NAMES[row.chunk_strategy || 'title'] }} {{ row.chunk_size || 800 }}{{ row.chunk_strategy === 'semantic' ? '' : ' / ' + (row.chunk_overlap ?? 150) }}</template>
|
||||
<el-table-column label="分块" width="210">
|
||||
<template #default="{ row }">
|
||||
{{ row.chunk_size || 800 }} / {{ row.chunk_overlap ?? 150 }}<template v-if="row.unit_pattern">(自动识别:{{ humanizePattern(row.unit_pattern) }})</template>
|
||||
</template>
|
||||
</el-table-column>
|
||||
<el-table-column prop="created_at" label="创建时间" width="170" />
|
||||
<el-table-column label="操作" width="190" fixed="right">
|
||||
@@ -38,21 +40,13 @@
|
||||
<el-option v-for="m in embedders" :key="m.id" :label="m.name" :value="m.id" />
|
||||
</el-select>
|
||||
</el-form-item>
|
||||
<el-form-item label="分块策略">
|
||||
<el-select v-model="form.chunk_strategy" style="width: 100%">
|
||||
<el-option label="标题分块(推荐)" value="title" />
|
||||
<el-option label="递归字符分块" value="recursive" />
|
||||
<el-option label="语义分块" value="semantic" />
|
||||
</el-select>
|
||||
<div class="ds-tip">标题分块保留标题与段落结构;递归字符分块按分隔符切分,通用文本;语义分块按语义相似度切分,质量更高但解析更慢,且需绑定向量模型</div>
|
||||
</el-form-item>
|
||||
<el-form-item :label="form.chunk_strategy === 'semantic' ? '上限大小' : '分块大小'">
|
||||
<el-form-item label="分块大小">
|
||||
<el-input-number v-model="form.chunk_size" :min="50" :max="5000" :step="50" style="width: 100%" />
|
||||
<div class="ds-tip">{{ form.chunk_strategy === 'semantic' ? '语义分块按句子相似度切分,此值为安全上限(max_chunk_size),超过上限的块会再切分;语义分块不支持重叠' : '每块最大字数,超长段落自动按句号/换行切分' }}</div>
|
||||
<div class="ds-tip">每块最大字数。系统自动组合分块策略:优先识别文档结构(条文/章节/编号等)按结构切分,无结构时依次按标题感知、语义、递归切分</div>
|
||||
</el-form-item>
|
||||
<el-form-item v-if="form.chunk_strategy !== 'semantic'" label="重叠字数">
|
||||
<el-form-item label="重叠字数">
|
||||
<el-input-number v-model="form.chunk_overlap" :min="0" :max="500" :step="10" style="width: 100%" />
|
||||
<div class="ds-tip">相邻分块间的重叠字数,用于保持上下文连贯</div>
|
||||
<div class="ds-tip">相邻分块间的重叠字数,用于保持上下文连贯(语义切分路径自动忽略)</div>
|
||||
</el-form-item>
|
||||
</el-form>
|
||||
<template #footer>
|
||||
@@ -68,21 +62,41 @@ import { onMounted, ref } from 'vue'
|
||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||
import { listDatasets, saveDataset, deleteDataset } from '../api/dataset.js'
|
||||
import { listModelConfigs } from '../api/model_config.js'
|
||||
import { getSettings } from '../api/settings.js'
|
||||
|
||||
const STRATEGY_NAMES = { title: '标题', recursive: '递归', semantic: '语义' }
|
||||
// 把识别出的结构正则转成可读名称(第[一二三四五六七八九十百千]+条 → 条文)
|
||||
function humanizePattern(p) {
|
||||
if (!p) return ''
|
||||
if (p.includes('条')) return '条文'
|
||||
if (p.includes('回')) return '章回'
|
||||
if (p.includes('节')) return '小节'
|
||||
if (p.includes('章')) return '章节'
|
||||
if (p.includes('篇') || p.includes('部')) return '篇章'
|
||||
if (p.includes('(') || p.includes('(')) return '序号'
|
||||
if (/\d/.test(p)) return '编号'
|
||||
return '结构单元'
|
||||
}
|
||||
|
||||
const datasets = ref([])
|
||||
const embedders = ref([])
|
||||
const defaults = ref({ chunk_size: 800, chunk_overlap: 150 })
|
||||
const loading = ref(false)
|
||||
const saving = ref(false)
|
||||
const dialogVisible = ref(false)
|
||||
const editing = ref(false)
|
||||
let editingRowCfgId = 0
|
||||
let editingRowChunk = { chunk_size: 0, chunk_overlap: 0, chunk_strategy: '' }
|
||||
const form = ref({ id: 0, name: '', description: '', embedding_cfg_id: 0, chunk_size: 800, chunk_overlap: 150, chunk_strategy: 'title' })
|
||||
let editingRowChunk = { chunk_size: 0, chunk_overlap: 0 }
|
||||
const form = ref({ id: 0, name: '', description: '', embedding_cfg_id: 0, chunk_size: 800, chunk_overlap: 150 })
|
||||
|
||||
onMounted(async () => {
|
||||
await load()
|
||||
try {
|
||||
const s = await getSettings()
|
||||
if (s) {
|
||||
defaults.value.chunk_size = s.chunk_size || 800
|
||||
defaults.value.chunk_overlap = s.chunk_overlap ?? 150
|
||||
}
|
||||
} catch { /* 忽略 */ }
|
||||
try {
|
||||
const m = await listModelConfigs('embedding')
|
||||
if (m && m.list) embedders.value = m.list
|
||||
@@ -107,20 +121,20 @@ function embeddingName(id) {
|
||||
function openCreate() {
|
||||
editing.value = false
|
||||
editingRowCfgId = 0
|
||||
editingRowChunk = { chunk_size: 0, chunk_overlap: 0, chunk_strategy: '' }
|
||||
// 默认选中默认向量模型,其次第一个
|
||||
editingRowChunk = { chunk_size: 0, chunk_overlap: 0 }
|
||||
// 默认选中默认向量模型,其次第一个;分块大小/重叠带出全局设置值
|
||||
const def = embedders.value.find(x => x.is_default === 1) || embedders.value[0]
|
||||
form.value = { id: 0, name: '', description: '', embedding_cfg_id: def?.id || 0, chunk_size: 800, chunk_overlap: 150, chunk_strategy: 'title' }
|
||||
form.value = { id: 0, name: '', description: '', embedding_cfg_id: def?.id || 0, chunk_size: defaults.value.chunk_size, chunk_overlap: defaults.value.chunk_overlap }
|
||||
dialogVisible.value = true
|
||||
}
|
||||
|
||||
function openEdit(row) {
|
||||
editing.value = true
|
||||
editingRowCfgId = row.embedding_cfg_id
|
||||
editingRowChunk = { chunk_size: row.chunk_size || 800, chunk_overlap: row.chunk_overlap ?? 150, chunk_strategy: row.chunk_strategy || 'title' }
|
||||
editingRowChunk = { chunk_size: row.chunk_size || 800, chunk_overlap: row.chunk_overlap ?? 150 }
|
||||
form.value = {
|
||||
id: row.id, name: row.name, description: row.description, embedding_cfg_id: row.embedding_cfg_id,
|
||||
chunk_size: editingRowChunk.chunk_size, chunk_overlap: editingRowChunk.chunk_overlap, chunk_strategy: editingRowChunk.chunk_strategy,
|
||||
chunk_size: editingRowChunk.chunk_size, chunk_overlap: editingRowChunk.chunk_overlap,
|
||||
}
|
||||
dialogVisible.value = true
|
||||
}
|
||||
@@ -135,11 +149,11 @@ async function save() {
|
||||
return
|
||||
}
|
||||
const modelChanged = editing.value && form.value.embedding_cfg_id !== editingRowCfgId
|
||||
const chunkChanged = editing.value && (form.value.chunk_size !== editingRowChunk.chunk_size || form.value.chunk_overlap !== editingRowChunk.chunk_overlap || form.value.chunk_strategy !== editingRowChunk.chunk_strategy)
|
||||
const chunkChanged = editing.value && (form.value.chunk_size !== editingRowChunk.chunk_size || form.value.chunk_overlap !== editingRowChunk.chunk_overlap)
|
||||
if (modelChanged || chunkChanged) {
|
||||
const changed = []
|
||||
if (modelChanged) changed.push('向量模型')
|
||||
if (chunkChanged) changed.push('分块策略')
|
||||
if (chunkChanged) changed.push('分块配置')
|
||||
try {
|
||||
await ElMessageBox.confirm(`${changed.join('与')}已变更,保存后将重新处理该数据集下所有文档(异步进行,可在文档列表查看进度)。是否继续?`, '重新处理文档', { type: 'warning' })
|
||||
} catch {
|
||||
|
||||
+162
-22
@@ -4,33 +4,44 @@
|
||||
<el-select v-model="datasetId" placeholder="选择知识库" style="width: 260px" @change="load">
|
||||
<el-option v-for="d in datasets" :key="d.id" :label="d.name" :value="d.id" />
|
||||
</el-select>
|
||||
<span class="kg-tip">实体与关系由解析流水线中的 LLM 抽取生成,问答时命中实体自动注入一跳邻居</span>
|
||||
<span class="kg-tip">实体与关系由解析流水线中的 LLM 抽取生成,问答时命中实体自动注入一跳邻居;点击节点高亮其关联实体</span>
|
||||
</div>
|
||||
|
||||
<el-row :gutter="12">
|
||||
<el-col :span="10">
|
||||
<el-col :span="15">
|
||||
<el-card shadow="never" class="kg-card">
|
||||
<template #header>实体({{ entityTotal }})</template>
|
||||
<el-table :data="entities" size="small" v-loading="loading">
|
||||
<el-table-column prop="name" label="实体" min-width="140" show-overflow-tooltip />
|
||||
<el-table-column prop="entity_type" label="类型" width="110" />
|
||||
<el-table-column prop="chunk_id" label="来源分块" width="100" />
|
||||
</el-table>
|
||||
<el-pagination class="kg-page-bar" small layout="prev, pager, next" :total="entityTotal"
|
||||
:page-size="entityPageSize" v-model:current-page="entityPage" @current-change="loadEntities" />
|
||||
<template #header>
|
||||
<div class="kg-graph-head">
|
||||
<span>关系图谱({{ graphCountText }})</span>
|
||||
<el-tag v-if="truncated" size="small" type="warning">实体过多,仅展示度数最高的前 {{ MAX_NODES }} 个</el-tag>
|
||||
</div>
|
||||
</template>
|
||||
<div ref="graphEl" class="kg-graph" v-loading="loading"></div>
|
||||
<el-empty v-if="!loading && graphNodes.length === 0" description="暂无图谱数据,上传文档并完成解析后自动生成" />
|
||||
</el-card>
|
||||
</el-col>
|
||||
<el-col :span="14">
|
||||
<el-col :span="9">
|
||||
<el-card shadow="never" class="kg-card">
|
||||
<template #header>关系({{ relationTotal }})</template>
|
||||
<el-table :data="relations" size="small" v-loading="loading">
|
||||
<el-table-column prop="head" label="主体" min-width="140" show-overflow-tooltip />
|
||||
<el-table-column prop="relation" label="关系" min-width="100" show-overflow-tooltip />
|
||||
<el-table-column prop="tail" label="客体" min-width="140" show-overflow-tooltip />
|
||||
<el-table-column prop="chunk_id" label="来源分块" width="100" />
|
||||
</el-table>
|
||||
<el-pagination class="kg-page-bar" small layout="prev, pager, next" :total="relationTotal"
|
||||
:page-size="relationPageSize" v-model:current-page="relationPage" @current-change="loadRelations" />
|
||||
<el-tabs v-model="sideTab">
|
||||
<el-tab-pane :label="`实体(${entityTotal})`" name="entity">
|
||||
<el-table :data="entities" size="small" height="440" v-loading="loading">
|
||||
<el-table-column prop="name" label="实体" min-width="120" show-overflow-tooltip />
|
||||
<el-table-column prop="entity_type" label="类型" width="100" />
|
||||
<el-table-column prop="chunk_id" label="来源分块" width="90" />
|
||||
</el-table>
|
||||
<el-pagination class="kg-page-bar" small layout="prev, pager, next" :total="entityTotal"
|
||||
:page-size="entityPageSize" v-model:current-page="entityPage" @current-change="loadEntities" />
|
||||
</el-tab-pane>
|
||||
<el-tab-pane :label="`关系(${relationTotal})`" name="relation">
|
||||
<el-table :data="relations" size="small" height="440" v-loading="loading">
|
||||
<el-table-column prop="head" label="主体" min-width="110" show-overflow-tooltip />
|
||||
<el-table-column prop="relation" label="关系" min-width="90" show-overflow-tooltip />
|
||||
<el-table-column prop="tail" label="客体" min-width="110" show-overflow-tooltip />
|
||||
</el-table>
|
||||
<el-pagination class="kg-page-bar" small layout="prev, pager, next" :total="relationTotal"
|
||||
:page-size="relationPageSize" v-model:current-page="relationPage" @current-change="loadRelations" />
|
||||
</el-tab-pane>
|
||||
</el-tabs>
|
||||
</el-card>
|
||||
</el-col>
|
||||
</el-row>
|
||||
@@ -38,13 +49,23 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, ref } from 'vue'
|
||||
import { computed, nextTick, onBeforeUnmount, onMounted, ref } from 'vue'
|
||||
import * as echarts from 'echarts/core'
|
||||
import { GraphChart } from 'echarts/charts'
|
||||
import { TooltipComponent, LegendComponent } from 'echarts/components'
|
||||
import { CanvasRenderer } from 'echarts/renderers'
|
||||
import { listDatasets } from '../api/dataset.js'
|
||||
import { listEntities, listRelations } from '../api/kg.js'
|
||||
|
||||
echarts.use([GraphChart, TooltipComponent, LegendComponent, CanvasRenderer])
|
||||
|
||||
const MAX_NODES = 400 // 力导向图节点上限,超过取度数最高者,避免布局卡顿
|
||||
|
||||
const datasets = ref([])
|
||||
const datasetId = ref(null)
|
||||
const loading = ref(false)
|
||||
const graphEl = ref(null)
|
||||
const truncated = ref(false)
|
||||
|
||||
const entities = ref([])
|
||||
const entityTotal = ref(0)
|
||||
@@ -56,6 +77,14 @@ const relationTotal = ref(0)
|
||||
const relationPage = ref(1)
|
||||
const relationPageSize = 20
|
||||
|
||||
const sideTab = ref('entity')
|
||||
let chart = null
|
||||
let resizeObserver = null
|
||||
|
||||
const graphCountText = computed(() => `${truncated.value ? '≈' : ''}${graphNodes.value.length} 实体 / ${graphLinks.value.length} 关系`)
|
||||
const graphNodes = ref([])
|
||||
const graphLinks = ref([])
|
||||
|
||||
onMounted(async () => {
|
||||
try {
|
||||
const d = await listDatasets()
|
||||
@@ -65,10 +94,107 @@ onMounted(async () => {
|
||||
await load()
|
||||
})
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
resizeObserver?.disconnect()
|
||||
chart?.dispose()
|
||||
})
|
||||
|
||||
async function load() {
|
||||
entityPage.value = 1
|
||||
relationPage.value = 1
|
||||
await Promise.all([loadEntities(), loadRelations()])
|
||||
loading.value = true
|
||||
try {
|
||||
const [allEntities, allRelations] = await Promise.all([
|
||||
listEntities({ dataset_id: datasetId.value, page: 1, page_size: 100000 }),
|
||||
listRelations({ dataset_id: datasetId.value, page: 1, page_size: 100000 }),
|
||||
])
|
||||
buildGraph(allEntities?.list || [], allRelations?.list || [])
|
||||
await Promise.all([loadEntities(), loadRelations()])
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
// 实体按名字去重合并(同一实体可能出现在多个分块),按出入度决定节点大小
|
||||
function buildGraph(entitiesList, relationsList) {
|
||||
const byName = new Map()
|
||||
for (const e of entitiesList) {
|
||||
const prev = byName.get(e.name)
|
||||
if (prev) {
|
||||
prev.chunks = [...new Set([...prev.chunks, e.chunk_id])]
|
||||
} else {
|
||||
byName.set(e.name, { name: e.name, entity_type: e.entity_type, chunks: [e.chunk_id] })
|
||||
}
|
||||
}
|
||||
const degree = new Map()
|
||||
const edges = []
|
||||
const seenEdges = new Set()
|
||||
for (const r of relationsList) {
|
||||
if (!byName.has(r.head) || !byName.has(r.tail)) continue
|
||||
const key = `${r.head}|${r.relation}|${r.tail}`
|
||||
if (seenEdges.has(key)) continue
|
||||
seenEdges.add(key)
|
||||
edges.push({ source: r.head, target: r.tail, relation: r.relation })
|
||||
degree.set(r.head, (degree.get(r.head) || 0) + 1)
|
||||
degree.set(r.tail, (degree.get(r.tail) || 0) + 1)
|
||||
}
|
||||
let nodes = [...byName.values()]
|
||||
truncated.value = nodes.length > MAX_NODES
|
||||
if (truncated.value) {
|
||||
nodes.sort((a, b) => (degree.get(b.name) || 0) - (degree.get(a.name) || 0))
|
||||
const keep = new Set(nodes.slice(0, MAX_NODES).map(n => n.name))
|
||||
nodes = nodes.filter(n => keep.has(n.name))
|
||||
for (const e of [...edges]) {
|
||||
if (!keep.has(e.source) || !keep.has(e.target)) edges.splice(edges.indexOf(e), 1)
|
||||
}
|
||||
}
|
||||
const minD = Math.min(...nodes.map(n => degree.get(n.name) || 0))
|
||||
const maxD = Math.max(...nodes.map(n => degree.get(n.name) || 0))
|
||||
const sizeFor = d => (maxD === minD ? 30 : 20 + ((d - minD) / (maxD - minD)) * 36)
|
||||
graphNodes.value = nodes.map(n => ({
|
||||
id: n.name,
|
||||
name: n.name,
|
||||
category: n.entity_type || '未分类',
|
||||
symbolSize: sizeFor(degree.get(n.name) || 0),
|
||||
}))
|
||||
graphLinks.value = edges
|
||||
renderGraph()
|
||||
}
|
||||
|
||||
function renderGraph() {
|
||||
if (!graphEl.value) return
|
||||
chart ??= echarts.init(graphEl.value)
|
||||
const types = [...new Set(graphNodes.value.map(n => n.category))]
|
||||
chart.setOption({
|
||||
legend: { bottom: 0, type: 'scroll', data: types, textStyle: { fontSize: 11 } },
|
||||
tooltip: {
|
||||
trigger: 'item',
|
||||
formatter: p => {
|
||||
if (p.dataType === 'edge') {
|
||||
const e = graphLinks.value[p.dataIndex]
|
||||
return `${e.source} → ${e.target}<br/>关系:${e.relation}`
|
||||
}
|
||||
return `${p.name}<br/>类型:${p.data.category}`
|
||||
},
|
||||
},
|
||||
series: [{
|
||||
type: 'graph',
|
||||
layout: 'force',
|
||||
roam: true,
|
||||
draggable: true,
|
||||
data: graphNodes.value,
|
||||
links: graphLinks.value,
|
||||
categories: types.map(t => ({ name: t })),
|
||||
force: { repulsion: 260, edgeLength: [40, 110], gravity: 0.08 },
|
||||
label: { show: true, position: 'right', fontSize: 10, color: '#606266' },
|
||||
edgeLabel: { show: false },
|
||||
emphasis: {
|
||||
focus: 'adjacency', // 点击节点自动高亮一跳邻居并淡化其余
|
||||
label: { fontSize: 12, fontWeight: 600 },
|
||||
lineStyle: { width: 2 },
|
||||
},
|
||||
}],
|
||||
}, true)
|
||||
}
|
||||
|
||||
async function loadEntities() {
|
||||
@@ -94,6 +220,12 @@ async function loadRelations() {
|
||||
loading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
// 布局稳定后渲染一次(图表容器尺寸依赖 el-row 布局),并监听容器尺寸变化
|
||||
resizeObserver = new ResizeObserver(() => nextTick(renderGraph))
|
||||
resizeObserver.observe(graphEl.value)
|
||||
})
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
@@ -107,6 +239,14 @@ async function loadRelations() {
|
||||
font-size: 13px;
|
||||
color: #909399;
|
||||
}
|
||||
.kg-graph {
|
||||
height: 620px;
|
||||
}
|
||||
.kg-graph-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
.kg-page-bar {
|
||||
margin-top: 10px;
|
||||
justify-content: flex-end;
|
||||
|
||||
@@ -1,5 +1,17 @@
|
||||
<template>
|
||||
<div class="settings-page">
|
||||
<div class="block-card">
|
||||
<div class="block-title">分块默认值</div>
|
||||
<div class="block-tip">新建数据集时自动带出,单个数据集仍可在表单中修改</div>
|
||||
<div class="block-row">
|
||||
<span class="field-label">分块大小</span>
|
||||
<el-input-number v-model="chunkForm.chunk_size" :min="50" :max="5000" :step="50" />
|
||||
<span class="field-label">重叠字数</span>
|
||||
<el-input-number v-model="chunkForm.chunk_overlap" :min="0" :max="500" :step="10" />
|
||||
<el-button type="primary" :loading="chunkSaving" @click="saveChunkDefaults">保存</el-button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="section-head">
|
||||
<el-radio-group v-model="modelType" @change="loadModels">
|
||||
<el-radio-button value="chat">对话模型</el-radio-button>
|
||||
@@ -65,8 +77,32 @@
|
||||
import { onMounted, ref } from 'vue'
|
||||
import { ElMessage, ElMessageBox } from 'element-plus'
|
||||
import { listModelConfigs, saveModelConfig, deleteModelConfig, testModelConfig, setDefaultModelConfig } from '../api/model_config.js'
|
||||
import { getSettings, saveSettings } from '../api/settings.js'
|
||||
|
||||
const modelType = ref('chat')
|
||||
|
||||
const chunkForm = ref({ chunk_size: 800, chunk_overlap: 150 })
|
||||
const chunkSaving = ref(false)
|
||||
|
||||
async function loadChunkDefaults() {
|
||||
try {
|
||||
const s = await getSettings()
|
||||
if (s) {
|
||||
chunkForm.value.chunk_size = s.chunk_size || 800
|
||||
chunkForm.value.chunk_overlap = s.chunk_overlap ?? 150
|
||||
}
|
||||
} catch { /* 忽略 */ }
|
||||
}
|
||||
|
||||
async function saveChunkDefaults() {
|
||||
chunkSaving.value = true
|
||||
try {
|
||||
await saveSettings(chunkForm.value)
|
||||
ElMessage.success('分块默认值已保存')
|
||||
} finally {
|
||||
chunkSaving.value = false
|
||||
}
|
||||
}
|
||||
const models = ref([])
|
||||
const allModels = ref([])
|
||||
const modelLoading = ref(false)
|
||||
@@ -78,6 +114,7 @@ const modelForm = ref({ id: 0, name: '', model_type: 'chat', model_name: '', end
|
||||
|
||||
onMounted(async () => {
|
||||
await loadModels()
|
||||
await loadChunkDefaults()
|
||||
try { const all = await listModelConfigs(''); if (all && all.list) allModels.value = all.list } catch { /* 忽略 */ }
|
||||
})
|
||||
|
||||
@@ -180,4 +217,28 @@ async function test(row) {
|
||||
align-items: center;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
.block-card {
|
||||
border: 1px solid #e4e7ed;
|
||||
border-radius: 6px;
|
||||
padding: 14px 16px;
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
.block-title {
|
||||
font-weight: 600;
|
||||
margin-bottom: 4px;
|
||||
}
|
||||
.block-tip {
|
||||
font-size: 12px;
|
||||
color: #909399;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.block-row {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
.field-label {
|
||||
color: #606266;
|
||||
font-size: 14px;
|
||||
}
|
||||
</style>
|
||||
|
||||
Reference in New Issue
Block a user