golang版本升级1.26.0

.dockerignore

@@ -1 +0,0 @@
-.git

.gitignore

@@ -1 +1 @@
-/.idea/*
+/.idea/*

Dockerfile

@@ -1,25 +0,0 @@
-# 阶段1: 构建
-FROM golang:alpine AS builder
-
-RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories && \
-      apk add --no-cache git ca-certificates tzdata
-
-ENV TZ=Asia/Shanghai
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-
-ENV GO111MODULE=on
-ENV GOPROXY=https://goproxy.cn,direct
-ENV CGO_ENABLED=0
-ENV GOTOOLCHAIN=auto
-WORKDIR /build
-
-COPY . .
-
-RUN go mod download && go mod tidy
-
-RUN go build -ldflags="-s -w" -o main ./main.go
-
-
-EXPOSE 3006
-
-CMD ["./main"]

common/eino/chat.go

@@ -1,243 +0,0 @@
-package eino
-
-import (
-	"context"
-	"fmt"
-	"rag/consts/model"
-	"rag/dao"
-	"rag/model/dto"
-	"rag/model/entity"
-
-	"gitea.redpowerfuture.com/red-future/common/jaeger"
-	"gitea.redpowerfuture.com/red-future/common/utils"
-	"github.com/cloudwego/eino-ext/components/model/ark"
-	"github.com/cloudwego/eino-ext/components/model/arkbot"
-	"github.com/cloudwego/eino-ext/components/model/claude"
-	"github.com/cloudwego/eino-ext/components/model/deepseek"
-	"github.com/cloudwego/eino-ext/components/model/ollama"
-	"github.com/cloudwego/eino-ext/components/model/openai"
-	"github.com/cloudwego/eino-ext/components/model/qianfan"
-	"github.com/cloudwego/eino-ext/components/model/qwen"
-	modelChat "github.com/cloudwego/eino/components/model"
-	"github.com/gogf/gf/v2/frame/g"
-	"github.com/gogf/gf/v2/util/gconv"
-)
-
-type ChatModelSet struct {
-	Ark      *ark.ChatModel
-	ArkBot   *arkbot.ChatModel
-	Claude   *claude.ChatModel
-	DeepSeek *deepseek.ChatModel
-	Ollama   *ollama.ChatModel
-	OpenAI   *openai.ChatModel
-	Qianfan  *qianfan.ChatModel
-	Qwen     *qwen.ChatModel
-}
-
-// 全局租户容器：key=tenantId，value=该租户的对话模型
-var tenantChatModels = make(map[uint64]*ChatModelSet)
-
-func init() {
-	ctx := context.Background()
-	ctx, span := jaeger.NewSpan(ctx, "InitAllChat")
-	defer span.End()
-	InitAllChat(ctx)
-	return
-}
-
-// ===================== 1. 服务启动时：初始化所有租户对话模型 =====================
-func InitAllChat(ctx context.Context) {
-	list, err := dao.Model.GetNoTenantId(ctx, &dto.GetModelReq{
-		ModelType: model.ModelTypeChat.Code(),
-	})
-	if err != nil {
-		g.Log().Errorf(ctx, "获取所有租户对话模型失败: %v", err)
-		return
-	}
-
-	for _, l := range list {
-		err = InitChat(ctx, l)
-		if err != nil {
-			g.Log().Errorf(ctx, "初始化租户[%v]的对话模型失败: %v", l.TenantId, err)
-			continue
-		}
-	}
-}
-
-func InitChat(ctx context.Context, modelDO *entity.Model) (err error) {
-	set := &ChatModelSet{}
-	switch *modelDO.ConfigType {
-	case *model.ModelConfigTypeChatArk.Code():
-		var cfg entity.ChatModelConfigArk
-		if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
-			return fmt.Errorf("解析Ark配置失败: %v", err)
-		}
-		set.Ark, err = ark.NewChatModel(ctx, &ark.ChatModelConfig{
-			APIKey:      cfg.APIKey,
-			Model:       cfg.Model,
-			Temperature: gconv.PtrFloat32(0.7),
-			MaxTokens:   gconv.PtrInt(1024),
-			TopP:        gconv.PtrFloat32(1.0),
-		})
-
-	case *model.ModelConfigTypeChatArkBot.Code():
-		var cfg entity.ChatModelConfigArkBot
-		if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
-			return fmt.Errorf("解析ArkBot配置失败: %v", err)
-		}
-		set.ArkBot, err = arkbot.NewChatModel(ctx, &arkbot.Config{
-			APIKey:      cfg.APIKey,
-			Model:       cfg.Model,
-			Temperature: gconv.PtrFloat32(0.7),
-			MaxTokens:   gconv.PtrInt(1024),
-			TopP:        gconv.PtrFloat32(1.0),
-		})
-
-	case *model.ModelConfigTypeChatClaude.Code():
-		var cfg entity.ChatModelConfigClaude
-		if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
-			return fmt.Errorf("解析Claude配置失败: %v", err)
-		}
-		claudeCfg := claude.Config{
-			APIKey:          cfg.APIKey,
-			BaseURL:         gconv.PtrString(cfg.BaseURL),
-			Model:           cfg.Model,
-			Temperature:     gconv.PtrFloat32(0.7),
-			MaxTokens:       gconv.Int(1024),
-			TopP:            gconv.PtrFloat32(1.0),
-			ByBedrock:       cfg.ByBedrock,
-			AccessKey:       cfg.AccessKey,
-			SecretAccessKey: cfg.SecretAccessKey,
-			Region:          cfg.Region,
-		}
-		set.Claude, err = claude.NewChatModel(ctx, &claudeCfg)
-
-	case *model.ModelConfigTypeChatDeepSeek.Code():
-		var cfg entity.ChatModelConfigDeepSeek
-		if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
-			return fmt.Errorf("解析DeepSeek配置失败: %v", err)
-		}
-		set.DeepSeek, err = deepseek.NewChatModel(ctx, &deepseek.ChatModelConfig{
-			APIKey:      cfg.APIKey,
-			Model:       cfg.Model,
-			BaseURL:     cfg.BaseURL,
-			Temperature: gconv.Float32(0.7),
-			MaxTokens:   gconv.Int(1024),
-			TopP:        gconv.Float32(1.0),
-		})
-
-	case *model.ModelConfigTypeChatOllama.Code():
-		var cfg entity.ChatModelConfigOllama
-		if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
-			return fmt.Errorf("解析Ollama配置失败: %v", err)
-		}
-		set.Ollama, err = ollama.NewChatModel(ctx, &ollama.ChatModelConfig{
-			BaseURL: cfg.BaseURL,
-			Model:   cfg.Model,
-		})
-
-	case *model.ModelConfigTypeChatOpenAI.Code():
-		var cfg entity.ChatModelConfigOpenAI
-		if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
-			return fmt.Errorf("解析OpenAI配置失败: %v", err)
-		}
-		openAiCfg := openai.ChatModelConfig{
-			APIKey:              cfg.APIKey,
-			Model:               cfg.Model,
-			ByAzure:             cfg.ByAzure,
-			BaseURL:             cfg.BaseURL,
-			APIVersion:          cfg.APIVersion,
-			Temperature:         gconv.PtrFloat32(0.7),
-			MaxCompletionTokens: gconv.PtrInt(1024),
-			TopP:                gconv.PtrFloat32(1.0),
-		}
-		set.OpenAI, err = openai.NewChatModel(ctx, &openAiCfg)
-
-	case *model.ModelConfigTypeChatQianfan.Code():
-		var cfg entity.ChatModelConfigQianfan
-		if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
-			return fmt.Errorf("解析千帆配置失败: %v", err)
-		}
-		qcfg := qianfan.GetQianfanSingletonConfig()
-		qcfg.AccessKey = cfg.AccessKey
-		qcfg.SecretKey = cfg.SecretKey
-		set.Qianfan, err = qianfan.NewChatModel(ctx, &qianfan.ChatModelConfig{
-			Model:               cfg.Model,
-			Temperature:         gconv.PtrFloat32(0.7),
-			MaxCompletionTokens: gconv.PtrInt(1024),
-			TopP:                gconv.PtrFloat32(1.0),
-		})
-
-	case *model.ModelConfigTypeChatQwen.Code():
-		var cfg entity.ChatModelConfigQwen
-		if err = gconv.Struct(modelDO.ConfigContent, &cfg); err != nil {
-			return fmt.Errorf("解析Qwen配置失败: %v", err)
-		}
-		set.Qwen, err = qwen.NewChatModel(ctx, &qwen.ChatModelConfig{
-			APIKey:      cfg.APIKey,
-			Model:       cfg.Model,
-			BaseURL:     cfg.BaseURL,
-			Temperature: gconv.PtrFloat32(0.7),
-			MaxTokens:   gconv.PtrInt(1024),
-			TopP:        gconv.PtrFloat32(1.0),
-		})
-
-	default:
-		return fmt.Errorf("不支持的对话模型类型: %v", *modelDO.ConfigType)
-	}
-
-	if err != nil {
-		return fmt.Errorf("初始化对话模型失败: %v", err)
-	}
-
-	// 无锁存入租户 map
-	tenantChatModels[modelDO.TenantId] = set
-	g.Log().Infof(ctx, "租户[%v]对话模型[%v]初始化成功", modelDO.TenantId, *modelDO.ConfigType)
-	return
-}
-
-func GetTenantChatModel(tenantId uint64) (*ChatModelSet, error) {
-	set := tenantChatModels[tenantId]
-	if set == nil {
-		return nil, fmt.Errorf("租户[%v]对话模型未初始化", tenantId)
-	}
-	return set, nil
-}
-
-func GetTenantChatModelByType(ctx context.Context, configType model.ModelConfigType) (modelChat.BaseChatModel, error) {
-	userInfo, err := utils.GetUserInfo(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	set, err := GetTenantChatModel(userInfo.TenantId)
-	if set == nil {
-		return nil, err
-	}
-
-	switch *configType {
-	case *model.ModelConfigTypeChatArk.Code():
-		return set.Ark, nil
-	case *model.ModelConfigTypeChatArkBot.Code():
-		return set.ArkBot, nil
-	case *model.ModelConfigTypeChatClaude.Code():
-		return set.Claude, nil
-	case *model.ModelConfigTypeChatDeepSeek.Code():
-		return set.DeepSeek, nil
-	case *model.ModelConfigTypeChatOllama.Code():
-		return set.Ollama, nil
-	case *model.ModelConfigTypeChatOpenAI.Code():
-		return set.OpenAI, nil
-	case *model.ModelConfigTypeChatQianfan.Code():
-		return set.Qianfan, nil
-	case *model.ModelConfigTypeChatQwen.Code():
-		return set.Qwen, nil
-	default:
-		return nil, fmt.Errorf("不支持的对话模型类型: %v", configType)
-	}
-}
-
-func RefreshTenantChatModel(ctx context.Context, modelDO *entity.Model) error {
-	delete(tenantChatModels, modelDO.TenantId)
-	return InitChat(ctx, modelDO)
-}

common/eino/chat_model.go

@@ -1,125 +0,0 @@
-package eino
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"rag/consts/model"
-
-	"github.com/cloudwego/eino/components/prompt"
-	"github.com/cloudwego/eino/schema"
-)
-
-const (
-	MaxHistoryTurns = 5 // 最大历史轮数
-)
-
-var (
-	ragPromptTemplate prompt.ChatTemplate // EINO 官方模板
-)
-
-func init() {
-	// 初始化 EINO 提示词模板
-	initRAGPromptTemplate()
-	return
-}
-
-// 初始化 EINO 官方提示词模板（最关键！）
-func initRAGPromptTemplate() {
-	ragPromptTemplate = prompt.FromMessages(
-		schema.FString,
-		// 系统提示（带参考知识）
-		&schema.Message{
-			Role: schema.System,
-			Content: `你是专业客服，语气友好简洁。
-请依据参考知识回答，不知道就说：抱歉，我暂时无法回答这个问题。
-
-参考知识：
-{knowledge}`,
-		},
-		// 用户问题
-		&schema.Message{
-			Role:    schema.User,
-			Content: "{question}",
-		},
-	)
-}
-
-// NewChatModel 只处理逻辑，不复用创建模型
-func NewChatModel(ctx context.Context, question string, docs []*schema.Document, history []*schema.Message, chatModel model.ModelConfigType) (replyMsg *schema.Message, err error) {
-	// 1. 构建参考知识
-	knowledge := buildKnowledgeAndSources(docs)
-	// 2. 历史精简
-	history = limitHistory(history)
-	// 3. ✅ EINO 官方模板格式化（超级干净）
-	msgs, err := ragPromptTemplate.Format(ctx, map[string]any{
-		"knowledge": knowledge,
-		"question":  question,
-	})
-	if err != nil {
-		return nil, err
-	}
-	// 4. 历史插入到模板消息中间（标准EINO用法）
-	if len(history) > 0 {
-		msgs = append(msgs[:1], append(history, msgs[1:]...)...)
-	}
-	// 5. 🔥 直接使用全局单例，不重复创建
-	replyMsg, err = streamGenerateAnswer(ctx, msgs, chatModel)
-
-	return
-}
-
-func limitHistory(history []*schema.Message) []*schema.Message {
-	valid := make([]*schema.Message, 0, len(history))
-	for _, m := range history {
-		if m.Role == schema.User || m.Role == schema.Assistant {
-			valid = append(valid, m)
-		}
-	}
-
-	keep := 2 * MaxHistoryTurns
-	if len(valid) > keep {
-		valid = valid[len(valid)-keep:]
-	}
-	return valid
-}
-
-// buildKnowledgeAndSources 拼接参考知识
-func buildKnowledgeAndSources(docs []*schema.Document) string {
-	var knowledge string
-
-	for i, doc := range docs {
-		knowledge += fmt.Sprintf("[参考%d] %s\n", i+1, doc.Content)
-
-	}
-	return knowledge
-}
-
-// streamGenerateAnswer 流式生成
-func streamGenerateAnswer(ctx context.Context, msgs []*schema.Message, chatModel model.ModelConfigType) (reply *schema.Message, err error) {
-
-	cm, err := GetTenantChatModelByType(ctx, chatModel)
-	if err != nil {
-		return nil, err
-	}
-
-	sr, err := cm.Stream(ctx, msgs)
-	if err != nil {
-		return nil, fmt.Errorf("stream failed: %w", err)
-	}
-
-	var chunks []*schema.Message
-	for {
-		chunk, err := sr.Recv()
-		if errors.Is(err, io.EOF) {
-			break
-		}
-		if err != nil {
-			return nil, fmt.Errorf("stream recv failed: %w", err)
-		}
-		chunks = append(chunks, chunk)
-	}
-
-	return schema.ConcatMessages(chunks)
-}

common/eino/document_loader.go

@@ -1,74 +0,0 @@
-package eino
-
-import (
-	"context"
-	"fmt"
-
-	"gitea.redpowerfuture.com/red-future/common/utils"
-	"github.com/cloudwego/eino-ext/components/document/loader/file"
-	"github.com/cloudwego/eino-ext/components/document/loader/url"
-	"github.com/cloudwego/eino-ext/components/document/parser/docx"
-	"github.com/cloudwego/eino-ext/components/document/parser/pdf"
-	"github.com/cloudwego/eino-ext/components/document/parser/xlsx"
-	"github.com/cloudwego/eino/components/document"
-	"github.com/cloudwego/eino/components/document/parser"
-	"github.com/cloudwego/eino/schema"
-)
-
-// LoadDocument 业务函数：加载文件
-func a(ctx context.Context, filePath, fileFormat string) (docs []*schema.Document, err error) {
-	p, err := docsParser(ctx, fileFormat)
-	if err != nil {
-		return
-	}
-	loader, err := url.NewLoader(ctx, &url.LoaderConfig{
-		Parser: p,
-	})
-	imageUrl, err := utils.GetFileAddressPrefix(ctx)
-	if err != nil {
-		return
-	}
-	docs, err = loader.Load(ctx, document.Source{
-		URI: fmt.Sprintf("%s%s", imageUrl, filePath),
-	})
-	return
-}
-
-func LoadDocument(ctx context.Context, filePath, fileFormat string) (docs []*schema.Document, err error) {
-	p, err := docsParser(ctx, fileFormat)
-	if err != nil {
-		return
-	}
-	// 1. 创建文件加载器
-	loader, err := file.NewFileLoader(ctx, &file.FileLoaderConfig{
-		UseNameAsID: false, // 使用文件名作为文档ID
-		Parser:      p,
-	})
-	if err != nil {
-		return
-	}
-
-	// 2. 加载本地文件
-	docs, err = loader.Load(ctx, document.Source{
-		URI: "C:\\Users\\AI\\Desktop\\手机发展史.txt",
-	})
-
-	return
-}
-
-func docsParser(ctx context.Context, fileFormat string) (p parser.Parser, err error) {
-	switch fileFormat {
-	case "docx":
-		p, err = docx.NewDocxParser(ctx, &docx.Config{
-			ToSections:     true,
-			IncludeHeaders: true,
-			IncludeFooters: true,
-			IncludeTables:  true,
-		})
-	case "pdf":
-		p, err = pdf.NewPDFParser(ctx, &pdf.Config{})
-	case "xlsx":
-		p, err = xlsx.NewXlsxParser(ctx, &xlsx.Config{})
-	}
-	return
-}

common/eino/document_semantic.go

@@ -1,55 +0,0 @@
-package eino
-
-import (
-	"context"
-	"rag/consts/model"
-
-	"github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive"
-	"github.com/cloudwego/eino-ext/components/document/transformer/splitter/semantic"
-	"github.com/cloudwego/eino/schema"
-	"github.com/gogf/gf/v2/frame/g"
-)
-
-// SemanticSplitDocument 语义分割文档
-func SemanticSplitDocument(ctx context.Context, docs []*schema.Document, vectorModel model.ModelConfigType) (res []*schema.Document, err error) {
-	// 默认分隔符（支持中英文）
-	separators := []string{"\n\n", "\n", "。", "！", "？", "；", ".", "!", "?", ";"}
-	// 读取配置，使用合理的默认值
-	bufferSize := g.Cfg().MustGet(ctx, "eino.splitter.bufferSize").Int()
-	minChunkSize := g.Cfg().MustGet(ctx, "eino.splitter.minChunkSize").Int()
-	percentile := g.Cfg().MustGet(ctx, "eino.splitter.percentile").Float64()
-	batchSize := g.Cfg().MustGet(ctx, "eino.splitter.batchSize").Int()
-
-	embedder, err := GetTenantEmbedderByType(ctx, vectorModel)
-	if err != nil {
-		return nil, err
-	}
-
-	splitter, err := semantic.NewSplitter(ctx, &semantic.Config{
-		Embedding:    NewBatchEmbedder(embedder, batchSize),
-		BufferSize:   bufferSize,
-		MinChunkSize: minChunkSize,
-		Percentile:   percentile,
-		Separators:   separators,
-	})
-	if err != nil {
-		return
-	}
-	return splitter.Transform(ctx, docs)
-}
-
-// RecursiveSplitDocument 递归分割文档
-func RecursiveSplitDocument(ctx context.Context, docs []*schema.Document) (res []*schema.Document, err error) {
-	// 默认分隔符（支持中英文）
-	separators := []string{"\n\n", "\n", "。", "！", "？", "；", ".", "!", "?", ";"}
-	splitter, err := recursive.NewSplitter(ctx, &recursive.Config{
-		ChunkSize:   512,
-		OverlapSize: 100,
-		KeepType:    recursive.KeepTypeNone,
-		Separators:  separators,
-	})
-	if err != nil {
-		return
-	}
-	return splitter.Transform(ctx, docs)
-}

common/eino/embedding.go

@@ -1,213 +0,0 @@
-package eino
-
-import (
-	"context"
-	"fmt"
-	"rag/consts/model"
-	"rag/model/entity"
-
-	"gitea.redpowerfuture.com/red-future/common/jaeger"
-	"gitea.redpowerfuture.com/red-future/common/utils"
-	"github.com/cloudwego/eino-ext/components/embedding/ark"
-	"github.com/cloudwego/eino-ext/components/embedding/dashscope"
-	"github.com/cloudwego/eino-ext/components/embedding/ollama"
-	"github.com/cloudwego/eino-ext/components/embedding/openai"
-	"github.com/cloudwego/eino-ext/components/embedding/qianfan"
-	"github.com/cloudwego/eino-ext/components/embedding/tencentcloud"
-	"github.com/cloudwego/eino/components/embedding"
-	"github.com/gogf/gf/v2/frame/g"
-	"github.com/gogf/gf/v2/util/gconv"
-)
-
-type EmbedderSet struct {
-	Ark          *ark.Embedder
-	Ollama       *ollama.Embedder
-	OpenAI       *openai.Embedder
-	Qianfan      *qianfan.Embedder
-	TencentCloud *tencentcloud.Embedder
-	DashScope    *dashscope.Embedder
-}
-
-// 全局租户容器：key=tenantId，value=该租户的向量模型
-var tenantEmbedders = make(map[uint64]*EmbedderSet)
-
-func init() {
-	ctx := context.Background()
-	ctx, span := jaeger.NewSpan(ctx, "InitAllVector")
-	defer span.End()
-	InitAllVector(ctx)
-	return
-}
-
-// ===================== 1. 服务启动时调用：初始化所有租户 =====================
-func InitAllVector(ctx context.Context) {
-	//list, err := dao.Model.GetNoTenantId(ctx, &dto.GetModelReq{
-	//	ModelType: model.ModelTypeVector.Code(),
-	//})
-	//if err != nil {
-	//	g.Log().Errorf(ctx, "获取所有租户ID失败: %v", err)
-	//	return
-	//}
-	//
-	//for _, l := range list {
-	//	err = InitVector(ctx, l)
-	//	if err != nil {
-	//		g.Log().Errorf(ctx, "初始化租户[%v]的向量模型失败: %v", l.TenantId, err)
-	//		continue
-	//	}
-	//}
-	modelDO := new(entity.Model)
-	modelDO.TenantId = 1
-	modelDO.ConfigType = model.ModelConfigTypeVectorDashScope.Code()
-	var cfg entity.VectorModelConfigDashScope
-	cfg.APIKey = "sk-4a8b82770bf74bc490eb3e4c5a8e2be9"
-	cfg.Model = "text-embedding-v3"
-	modelDO.ConfigContent = gconv.Map(&cfg)
-	err := InitVector(ctx, modelDO)
-	if err != nil {
-		g.Log().Errorf(ctx, "初始化向量模型失败: %v", err)
-		return
-	}
-}
-
-func InitVector(ctx context.Context, modelDO *entity.Model) (err error) {
-	set := &EmbedderSet{}
-	switch *modelDO.ConfigType {
-	case *model.ModelConfigTypeVectorArk.Code():
-		// 解析 Ark 向量配置
-		var cfg entity.VectorModelConfigArk
-		err = gconv.Struct(modelDO.ConfigContent, &cfg)
-		if err != nil {
-			return fmt.Errorf("解析Ark向量配置失败: %v", err)
-		}
-		arkCfg := &ark.EmbeddingConfig{
-			APIKey: cfg.APIKey,
-			Model:  cfg.Model,
-		}
-		if !g.IsEmpty(cfg.APIType) {
-			arkCfg.APIType = new(ark.APIType(cfg.APIType))
-		}
-		set.Ark, err = ark.NewEmbedder(ctx, arkCfg)
-
-	case *model.ModelConfigTypeVectorOllama.Code():
-		// 解析 Ollama 向量配置
-		var cfg entity.VectorModelConfigOllama
-		err = gconv.Struct(modelDO.ConfigContent, &cfg)
-		if err != nil {
-			return fmt.Errorf("解析Ollama向量配置失败: %v", err)
-		}
-		set.Ollama, err = ollama.NewEmbedder(ctx, &ollama.EmbeddingConfig{
-			BaseURL: cfg.BaseURL,
-			Model:   cfg.Model,
-		})
-
-	case *model.ModelConfigTypeVectorOpenAI.Code():
-		// 解析 OpenAI 向量配置
-		var cfg entity.VectorModelConfigOpenAI
-		err = gconv.Struct(modelDO.ConfigContent, &cfg)
-		if err != nil {
-			return fmt.Errorf("解析OpenAI向量配置失败: %v", err)
-		}
-		openaiCfg := &openai.EmbeddingConfig{
-			APIKey:     cfg.APIKey,
-			Model:      cfg.Model,
-			ByAzure:    cfg.ByAzure,
-			BaseURL:    cfg.BaseURL,
-			APIVersion: cfg.APIVersion,
-		}
-		set.OpenAI, err = openai.NewEmbedder(ctx, openaiCfg)
-
-	case *model.ModelConfigTypeVectorQianfan.Code():
-		// 解析 千帆 向量配置
-		var cfg entity.VectorModelConfigQianfan
-		err = gconv.Struct(modelDO.ConfigContent, &cfg)
-		if err != nil {
-			return fmt.Errorf("解析千帆向量配置失败: %v", err)
-		}
-		qcfg := qianfan.GetQianfanSingletonConfig()
-		qcfg.AccessKey = cfg.AccessKey
-		qcfg.SecretKey = cfg.SecretKey
-		set.Qianfan, err = qianfan.NewEmbedder(ctx, &qianfan.EmbeddingConfig{
-			Model: cfg.Model,
-		})
-
-	case *model.ModelConfigTypeVectorTencentCloud.Code():
-		// 解析 腾讯云 向量配置
-		var cfg entity.VectorModelConfigTencentCloud
-		err = gconv.Struct(modelDO.ConfigContent, &cfg)
-		if err != nil {
-			return fmt.Errorf("解析腾讯云向量配置失败: %v", err)
-		}
-		set.TencentCloud, err = tencentcloud.NewEmbedder(ctx, &tencentcloud.EmbeddingConfig{
-			SecretID:  cfg.SecretID,
-			SecretKey: cfg.SecretKey,
-			Region:    cfg.Region,
-		})
-
-	case *model.ModelConfigTypeVectorDashScope.Code():
-		// 解析 阿里 dashscope 向量配置
-		var cfg entity.VectorModelConfigDashScope
-		err = gconv.Struct(modelDO.ConfigContent, &cfg)
-		if err != nil {
-			return fmt.Errorf("解析阿里dashscope向量配置失败: %v", err)
-		}
-		set.DashScope, err = dashscope.NewEmbedder(ctx, &dashscope.EmbeddingConfig{
-			APIKey: cfg.APIKey,
-			Model:  cfg.Model,
-		})
-
-	default:
-		return fmt.Errorf("不支持的向量模型配置类型: %v", *modelDO.ConfigType)
-	}
-
-	// 统一错误处理
-	if err != nil {
-		return fmt.Errorf("初始化向量模型失败: %v", err)
-	}
-	// 直接存入 map（无锁，重复初始化会直接覆盖）
-	tenantEmbedders[modelDO.TenantId] = set
-	g.Log().Infof(ctx, "向量模型[%v]初始化成功", modelDO.ConfigType)
-	return
-}
-
-func GetTenantEmbedder(tenantId uint64) (*EmbedderSet, error) {
-	set := tenantEmbedders[tenantId]
-	if set == nil {
-		return nil, fmt.Errorf("租户[%v]的向量模型未初始化", tenantId)
-	}
-	return set, nil
-}
-
-func GetTenantEmbedderByType(ctx context.Context, configType model.ModelConfigType) (embedding.Embedder, error) {
-	userInfo, err := utils.GetUserInfo(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	set, err := GetTenantEmbedder(userInfo.TenantId)
-	if set == nil {
-		return nil, err
-	}
-
-	switch *configType {
-	case *model.ModelConfigTypeVectorArk.Code():
-		return set.Ark, nil
-	case *model.ModelConfigTypeVectorOllama.Code():
-		return set.Ollama, nil
-	case *model.ModelConfigTypeVectorOpenAI.Code():
-		return set.OpenAI, nil
-	case *model.ModelConfigTypeVectorQianfan.Code():
-		return set.Qianfan, nil
-	case *model.ModelConfigTypeVectorTencentCloud.Code():
-		return set.TencentCloud, nil
-	case *model.ModelConfigTypeVectorDashScope.Code():
-		return set.DashScope, nil
-	default:
-		return nil, fmt.Errorf("不支持的向量模型配置类型: %v", *configType)
-	}
-}
-
-func RefreshTenantEmbedder(ctx context.Context, modelDO *entity.Model) error {
-	delete(tenantEmbedders, modelDO.TenantId)
-	return InitVector(ctx, modelDO)
-}

common/eino/embedding_batch.go

@@ -1,47 +0,0 @@
-package eino
-
-import (
-	"context"
-
-	"github.com/cloudwego/eino/components/embedding"
-)
-
-// BatchEmbedder 包装器，支持批量限制
-type BatchEmbedder struct {
-	embedder  embedding.Embedder
-	batchSize int
-}
-
-// NewBatchEmbedder 创建支持批量限制的 embedding 包装器
-func NewBatchEmbedder(embedder embedding.Embedder, batchSize int) *BatchEmbedder {
-	if batchSize <= 0 {
-		batchSize = 10 // 默认每批 10 个
-	}
-	return &BatchEmbedder{
-		embedder:  embedder,
-		batchSize: batchSize,
-	}
-}
-
-// EmbedStrings 分批调用 embedding
-func (b *BatchEmbedder) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) {
-	if len(texts) <= b.batchSize {
-		return b.embedder.EmbedStrings(ctx, texts, opts...)
-	}
-
-	var allEmbeddings [][]float64
-	for i := 0; i < len(texts); i += b.batchSize {
-		end := i + b.batchSize
-		if end > len(texts) {
-			end = len(texts)
-		}
-
-		batch := texts[i:end]
-		embeddings, err := b.embedder.EmbedStrings(ctx, batch, opts...)
-		if err != nil {
-			return nil, err
-		}
-		allEmbeddings = append(allEmbeddings, embeddings...)
-	}
-	return allEmbeddings, nil
-}

common/eino/indexer.go

@@ -1,185 +0,0 @@
-package eino
-
-import (
-	"context"
-	"database/sql"
-	"errors"
-	"fmt"
-	"rag/consts/model"
-	"rag/dao"
-	"rag/model/dto"
-	"rag/model/entity"
-
-	"gitea.redpowerfuture.com/red-future/common/beans"
-	"github.com/cloudwego/eino/callbacks"
-	"github.com/cloudwego/eino/components/indexer"
-	"github.com/cloudwego/eino/schema"
-	"github.com/gogf/gf/v2/os/glog"
-	"github.com/gogf/gf/v2/util/gconv"
-	"github.com/pgvector/pgvector-go"
-)
-
-type PGVectorIndexerOptions struct {
-	BatchSize int // 每批处理多少条
-}
-
-type PGVectorIndexer struct {
-	opts *PGVectorIndexerOptions
-}
-
-func NewPGVectorIndexer(opts *PGVectorIndexerOptions) *PGVectorIndexer {
-	// 默认值
-	if opts.BatchSize <= 0 {
-		opts.BatchSize = 5
-	}
-	return &PGVectorIndexer{opts: opts}
-}
-
-func (i *PGVectorIndexer) Store(ctx context.Context, docs []*schema.Document, configType model.ModelConfigType, opts ...indexer.Option) (rows int64, err error) {
-
-	embedderByType, err := GetTenantEmbedderByType(ctx, configType)
-	if err != nil {
-		return
-	}
-	indexer.WithEmbedding(embedderByType)
-
-	commonOpts := indexer.GetCommonOptions(&indexer.Options{}, opts...)
-
-	if commonOpts.Embedding == nil {
-		return 0, errors.New("embedding model not set")
-	}
-
-	// 回调
-	ctx = callbacks.OnStart(ctx, &indexer.CallbackInput{Docs: docs})
-
-	rows, err = i.bulkStore(ctx, docs, commonOpts)
-	if err != nil {
-		callbacks.OnError(ctx, err)
-		return
-	}
-
-	callbacks.OnEnd(ctx, &indexer.CallbackOutput{IDs: gconv.Strings(rows)})
-	return
-}
-
-func (i *PGVectorIndexer) bulkStore(ctx context.Context, docs []*schema.Document, opts *indexer.Options) (rows int64, err error) {
-	var batchDocs []*schema.Document
-
-	// 官方ES同款逻辑：满 BatchSize 就处理一批
-	for _, doc := range docs {
-		batchDocs = append(batchDocs, doc)
-
-		// 满了 → 处理
-		if len(batchDocs) >= i.opts.BatchSize {
-			var r int64
-			r, err = i.doStore(ctx, batchDocs, opts)
-			if err != nil {
-				return
-			}
-			rows = rows + r
-			batchDocs = nil
-		}
-	}
-
-	// 最后一批
-	if len(batchDocs) > 0 {
-		var r int64
-		r, err = i.doStore(ctx, batchDocs, opts)
-		if err != nil {
-			return
-		}
-		rows = rows + r
-	}
-
-	return
-}
-
-func (i *PGVectorIndexer) doStore(ctx context.Context, docs []*schema.Document, opts *indexer.Options) (rows int64, err error) {
-
-	texts := make([]string, len(docs))
-	for i, d := range docs {
-		texts[i] = d.Content
-	}
-
-	// 向量化（官方ES也没有重试！）
-	vectors, err := opts.Embedding.EmbedStrings(ctx, texts)
-	if err != nil {
-		return
-	}
-
-	// 转成业务实体
-	var chunks []*dto.VectorDocumentVectorMsg
-	for idx, doc := range docs {
-		ck := new(dto.VectorDocumentVectorMsg)
-		err = gconv.Struct(doc.MetaData, ck)
-		if err != nil {
-			glog.Errorf(ctx, "doStore err: %v", err)
-			continue
-		}
-		ck.Content = doc.Content
-		ck.Vector = pgvector.NewVector(gconv.Float32s(vectors[idx]))
-		ck.VectorStatus = gconv.PtrInt8(1)
-		ck.Status = gconv.PtrInt8(1)
-		chunks = append(chunks, ck)
-	}
-	if len(chunks) == 0 {
-		return
-	}
-	ctx = context.WithValue(ctx, "user", &beans.User{
-		TenantId: chunks[0].TenantId,
-		UserName: chunks[0].Creator,
-	})
-	// 创建索引
-	if err = i.createOrUpdateDatasetIndex(ctx, chunks[0].DatasetId, len(vectors[0]), int64(len(chunks))); err != nil {
-		return
-	}
-	// 入库
-	rows, err = dao.DocumentVector.BatchInsert(ctx, chunks)
-
-	return
-}
-
-func (i *PGVectorIndexer) createOrUpdateDatasetIndex(ctx context.Context, datasetId int64, dimension int, vectorCount int64) error {
-	exist, err := dao.DatasetIndex.GetByDatasetId(ctx, datasetId)
-	if err != nil && !errors.Is(err, sql.ErrNoRows) {
-		return err
-	}
-	if exist != nil {
-		_ = dao.DatasetIndex.IncVectorCount(ctx, exist.Id, vectorCount)
-		return nil
-	}
-
-	indexName := fmt.Sprintf("idx_dataset_%d_vector", datasetId)
-	idx := &entity.DatasetIndex{
-		DatasetId:   datasetId,
-		Name:        indexName,
-		Dimension:   dimension,
-		FieldType:   "float",
-		MetricType:  "COSINE",
-		Status:      gconv.PtrInt8(1),
-		VectorCount: vectorCount,
-		Description: fmt.Sprintf("数据集%d向量索引", datasetId),
-	}
-	_, err = dao.DatasetIndex.Insert(ctx, idx)
-	if err != nil {
-		return err
-	}
-	return i.createRealPGVectorIndex(ctx, indexName)
-}
-
-func (i *PGVectorIndexer) createRealPGVectorIndex(ctx context.Context, indexName string) error {
-	if err := dao.DatasetIndex.InsertIndex(ctx, indexName); err != nil {
-		glog.Errorf(ctx, "create vector index failed: %v", err)
-		return err
-	}
-	glog.Infof(ctx, "created pgvector index: %s", indexName)
-	return nil
-}
-
-func (i *PGVectorIndexer) GetType() string {
-	return "pgvector_indexer"
-}
-
-func (i *PGVectorIndexer) IsCallbacksEnabled() bool {
-	return true
-}

common/eino/rerank.go

@@ -1,116 +0,0 @@
-package eino
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"time"
-
-	"github.com/cloudwego/eino/schema"
-	"github.com/gogf/gf/v2/frame/g"
-)
-
-// DashScopeReranker 通义百炼 Rerank 精排（Cross-Encoder）
-type DashScopeReranker struct {
-	httpClient *http.Client
-}
-
-func NewDashScopeReranker() *DashScopeReranker {
-	return &DashScopeReranker{
-		httpClient: &http.Client{
-			Timeout: 10 * time.Second,
-		},
-	}
-}
-
-// Rerank 对文档进行精排（Cross-Encoder 核心）
-func (d *DashScopeReranker) Rerank(ctx context.Context, query string, docs []*schema.Document) ([]*schema.Document, error) {
-	if len(docs) == 0 {
-		return docs, nil
-	}
-
-	// 官方必过 URL
-	url := "https://dashscope.aliyuncs.com/api/v1/services/rerank/text-rerank/text-rerank"
-	apiKey := g.Cfg().MustGet(ctx, "eino.rerank.apiKey").String()
-	model := g.Cfg().MustGet(ctx, "eino.rerank.model").String()
-
-	documents := make([]string, len(docs))
-	for i, doc := range docs {
-		documents[i] = doc.Content
-	}
-
-	reqBody := map[string]any{
-		"model": model,
-		"input": map[string]any{
-			"query":     query,
-			"documents": documents,
-		},
-		"parameters": map[string]any{
-			"top_n": len(docs),
-		},
-	}
-	bs, _ := json.Marshal(reqBody)
-
-	req, _ := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(bs))
-	req.Header.Set("Authorization", "Bearer "+apiKey)
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := d.httpClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		body, _ := io.ReadAll(resp.Body)
-		return nil, fmt.Errorf("rerank api error: status=%d, body=%s", resp.StatusCode, string(body))
-	}
-
-	// 解析结果
-	var result struct {
-		Output struct {
-			Results []struct {
-				Index          int     `json:"index"`
-				RelevanceScore float64 `json:"relevance_score"`
-			} `json:"results"`
-		} `json:"output"`
-	}
-
-	if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
-		return nil, err
-	}
-
-	// 按分数排序
-	type scoredDoc struct {
-		doc   *schema.Document
-		score float64
-	}
-	scored := make([]scoredDoc, len(docs))
-	for i, doc := range docs {
-		scored[i] = scoredDoc{doc: doc, score: 0}
-	}
-	for _, res := range result.Output.Results {
-		scored[res.Index].score = res.RelevanceScore
-	}
-
-	// 分数从高到低排序
-	for i := 0; i < len(scored); i++ {
-		for j := i + 1; j < len(scored); j++ {
-			if scored[j].score > scored[i].score {
-				scored[i], scored[j] = scored[j], scored[i]
-			}
-		}
-	}
-
-	// 输出最终排好的文档
-	ranked := make([]*schema.Document, 0, len(scored))
-	for _, s := range scored {
-		s.doc.MetaData["rerank_score"] = s.score
-		ranked = append(ranked, s.doc)
-	}
-
-	return ranked, nil
-}

common/eino/retriever.go

@@ -1,300 +0,0 @@
-package eino
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"rag/consts/model"
-	"rag/dao"
-	"sort"
-	"time"
-
-	"github.com/cloudwego/eino/callbacks"
-	"github.com/cloudwego/eino/components/embedding"
-	"github.com/cloudwego/eino/components/retriever"
-	"github.com/cloudwego/eino/schema"
-	"github.com/gogf/gf/v2/frame/g"
-	"github.com/gogf/gf/v2/os/grpool"
-	"github.com/gogf/gf/v2/util/gconv"
-	"github.com/pgvector/pgvector-go"
-)
-
-type PGVectorRetrieverConfig struct {
-	Embedder     embedding.Embedder
-	DefaultTopK  int
-	DefaultIndex string
-	DSLInfo      map[string]any
-}
-
-type PGVectorRetriever struct {
-	embedder embedding.Embedder
-	topK     int
-	index    string
-	dslInfo  map[string]any
-	reranker *DashScopeReranker // 通义精排
-}
-
-func NewPGVectorRetriever(ctx context.Context, config *PGVectorRetrieverConfig, configType model.ModelConfigType) (*PGVectorRetriever, error) {
-	if config.DefaultTopK <= 0 {
-		config.DefaultTopK = 5
-	}
-
-	e, err := GetTenantEmbedderByType(ctx, configType)
-	if err != nil {
-		return nil, err
-	}
-
-	return &PGVectorRetriever{
-		embedder: e,
-		topK:     config.DefaultTopK,
-		index:    config.DefaultIndex,
-		dslInfo:  config.DSLInfo,
-		//reranker: NewDashScopeReranker(), // 👈 直接初始化你的精排
-	}, nil
-}
-
-func (r *PGVectorRetriever) Retrieve(ctx context.Context, query string, opts ...retriever.Option) ([]*schema.Document, error) {
-	options := &retriever.Options{
-		Index:     &r.index,
-		TopK:      &r.topK,
-		DSLInfo:   r.dslInfo,
-		Embedding: r.embedder,
-	}
-	options = retriever.GetCommonOptions(options, opts...)
-
-	// 安全保护：防止 nil 指针 panic
-	topK := 10
-	if options.TopK != nil {
-		topK = *options.TopK
-	}
-
-	ctx = callbacks.OnStart(ctx, &retriever.CallbackInput{
-		Query: query,
-		TopK:  *options.TopK,
-	})
-
-	// ==========================================
-	// 🔥 优化版：grpool 并行双路检索（安全、健壮、无泄漏）
-	// ==========================================
-	var (
-		docsVector   []*schema.Document
-		docsFulltext []*schema.Document
-		errVector    error
-		errFulltext  error
-
-		// 缓冲通道=2，确保无死锁等待
-		done = make(chan struct{}, 2)
-	)
-
-	// 上下文：超时 + 可取消双保障（建议5s超时，根据业务调整）
-	taskCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
-	defer cancel()
-
-	// 封装并行任务函数，消除重复代码
-	runTask := func(task func() error, errTarget *error) {
-		defer func() {
-			// 任务结束必发信号，确保通道不阻塞
-			done <- struct{}{}
-		}()
-
-		// 捕获 panic + 执行业务逻辑
-		g.TryCatch(taskCtx, func(ctx context.Context) {
-			*errTarget = task()
-		}, func(ctx context.Context, panicErr error) {
-			*errTarget = panicErr
-		})
-
-		// 任务失败：立即取消另一个任务（快速失败）
-		if *errTarget != nil {
-			cancel()
-		}
-	}
-
-	// ----------------------
-	// 并行提交两个检索任务
-	// ----------------------
-	// 任务1：向量检索
-	grpool.Add(taskCtx, func(ctx context.Context) {
-		runTask(func() error {
-			docsVector, errVector = r.doRetrieveVector(ctx, query, options)
-			return errVector
-		}, &errVector)
-	})
-
-	// 任务2：全文检索
-	grpool.Add(taskCtx, func(ctx context.Context) {
-		runTask(func() error {
-			docsFulltext, errFulltext = r.doRetrieveMeilisearch(ctx, query, options)
-			return errFulltext
-		}, &errFulltext)
-	})
-
-	// ----------------------
-	// 安全等待所有任务完成
-	// ----------------------
-	<-done
-	<-done
-
-	// ----------------------
-	// 统一错误处理
-	// ----------------------
-	// 用 errors.Join 合并所有错误，不丢失信息
-	if err := errors.Join(errVector, errFulltext); err != nil {
-		return nil, err
-	}
-
-	// 合并 + 智能去重（保留最优分数）
-	mergedDocs := mergeAndDeduplicate(docsVector, docsFulltext)
-
-	// =========================
-	// 🔥 Cross-Encoder 精排
-	// =========================
-	var finalDocs []*schema.Document
-	if r.reranker != nil {
-		ranked, err := r.reranker.Rerank(ctx, query, mergedDocs)
-		if err != nil {
-			return nil, fmt.Errorf("rerank failed: %w", err)
-		}
-		finalDocs = ranked
-	} else {
-		sort.Slice(mergedDocs, func(i, j int) bool {
-			d1 := gconv.Float64(mergedDocs[i].MetaData["distance"])
-			d2 := gconv.Float64(mergedDocs[j].MetaData["distance"])
-			return d1 < d2
-		})
-		finalDocs = mergedDocs
-	}
-
-	// =========================
-	// 过滤无效文档
-	// =========================
-	const maxDistance = 0.8
-	validDocs := make([]*schema.Document, 0, len(finalDocs))
-	for _, doc := range finalDocs {
-		dist := gconv.Float64(doc.MetaData["distance"])
-		if dist <= maxDistance {
-			validDocs = append(validDocs, doc)
-		}
-	}
-
-	// 最多保留 topK
-	if len(validDocs) > topK {
-		validDocs = validDocs[:topK]
-	}
-
-	callbacks.OnEnd(ctx, &retriever.CallbackOutput{Docs: validDocs})
-	return validDocs, nil
-}
-
-// ==========================================
-// 1. 向量检索（PG）
-// ==========================================
-func (r *PGVectorRetriever) doRetrieveVector(ctx context.Context, query string, opts *retriever.Options) ([]*schema.Document, error) {
-	vectors, err := opts.Embedding.EmbedStrings(ctx, []string{query})
-	if err != nil {
-		return nil, err
-	}
-	if len(vectors) == 0 {
-		return nil, errors.New("empty query vector")
-	}
-
-	queryVec := pgvector.NewVector(gconv.Float32s(vectors[0]))
-	topK := 10
-	if opts.TopK != nil {
-		topK = *opts.TopK
-	}
-	var datasetIds, documentIds []int64
-	if g.IsEmpty(opts.DSLInfo["dataset_ids"]) {
-		datasetIds = gconv.Int64s(opts.DSLInfo["dataset_ids"])
-	}
-	if g.IsEmpty(opts.DSLInfo["document_ids"]) {
-		documentIds = gconv.Int64s(opts.DSLInfo["document_ids"])
-	}
-
-	rows, err := dao.DocumentVector.GetAllByVector(ctx, datasetIds, documentIds, queryVec, topK)
-	if err != nil {
-		return nil, err
-	}
-
-	docs := make([]*schema.Document, 0, len(rows))
-	for _, row := range rows {
-		docs = append(docs, &schema.Document{
-			ID:      gconv.String(row["id"]),
-			Content: gconv.String(row["content"]),
-			MetaData: map[string]any{
-				"dataset_id":  gconv.Int64(row["dataset_id"]),
-				"document_id": gconv.Int64(row["document_id"]),
-				"distance":    gconv.Float64(row["distance"]),
-				"retrieve_by": "vector",
-			},
-		})
-	}
-	return docs, nil
-}
-
-// ==========================================
-// 2. 全文检索（Meilisearch）🔥 新增
-// ==========================================
-func (r *PGVectorRetriever) doRetrieveMeilisearch(ctx context.Context, query string, opts *retriever.Options) ([]*schema.Document, error) {
-	topK := *opts.TopK
-
-	var datasetIds, documentIds []int64
-	if g.IsEmpty(opts.DSLInfo["dataset_ids"]) {
-		datasetIds = gconv.Int64s(opts.DSLInfo["dataset_ids"])
-	}
-	if g.IsEmpty(opts.DSLInfo["document_ids"]) {
-		documentIds = gconv.Int64s(opts.DSLInfo["document_ids"])
-	}
-
-	// 调用你已有的 Meilisearch DAO
-	rows, err := dao.DocumentVector.SearchByKeywords(ctx, query, datasetIds, documentIds, topK)
-	if err != nil {
-		return nil, err
-	}
-
-	docs := make([]*schema.Document, 0, len(rows))
-	for _, row := range rows {
-		score := gconv.Float64(row["_rankingScore"])
-		distance := score
-
-		docs = append(docs, &schema.Document{
-			ID:      gconv.String(row["id"]),
-			Content: gconv.String(row["content"]),
-			MetaData: map[string]any{
-				"dataset_id":  gconv.Int64(row["dataset_id"]),
-				"document_id": gconv.Int64(row["document_id"]),
-				"distance":    distance,
-				"retrieve_by": "fulltext",
-			},
-		})
-	}
-	return docs, nil
-}
-
-// ==========================================
-// 合并去重（智能版：两路都命中时，保留向量结果 + 全文标记）
-// ==========================================
-func mergeAndDeduplicate(vecDocs, fullDocs []*schema.Document) []*schema.Document {
-	idMap := make(map[string]*schema.Document)
-
-	// 先存入向量结果
-	for _, d := range vecDocs {
-		idMap[d.ID] = d
-	}
-
-	// 再处理全文：不存在则添加；存在则标记“双路命中”，不覆盖向量分数
-	for _, d := range fullDocs {
-		if existDoc, ok := idMap[d.ID]; ok {
-			// 标记同时被向量和全文检索到
-			existDoc.MetaData["retrieve_by"] = "both"
-		} else {
-			idMap[d.ID] = d
-		}
-	}
-
-	merged := make([]*schema.Document, 0, len(idMap))
-	for _, d := range idMap {
-		merged = append(merged, d)
-	}
-	return merged
-}

config.yml

@@ -1,19 +1,19 @@
 server:
   address: :3006
   name: rag
+  workerId: 1
 
 # Database.
 database:
-  rag_knowledge:
+  default:
     - type: "pgsql"
-      host: "192.168.0.83"
+      host: "116.204.74.41"
       port: "15432"
       user: "postgres"
       pass: "Bjang09@686^*^"
-      name: "tenant-1"
+      name: "rag"
-      prefix: "rag_knowledge_"        # (可选)表名前缀
+      role: "master"              # (可选)数据库主从角色(master/slave)，默认为master。如果不使用应用主从机制请不配置或留空即可。
-      role: "master"
+      debug: false                # (可选)开启调试模式
-      debug: true                # (可选)开启调试模式
       dryRun: false               # (可选)ORM空跑(只读不写)
       charset: "utf8"             # (可选)数据库编码(如: utf8mb4/utf8/gbk/gb2312)，一般设置为utf8mb4。默认为utf8。
       timezone: "Asia/Shanghai"           # (可选)时区配置，例如:Local
@@ -25,15 +25,34 @@ database:
       updatedAt: "updated_at"     # (可选)自动更新时间字段名称
       deletedAt: "deleted_at"     # (可选)软删除时间字段名称
       timeMaintainDisabled: false # (可选)是否完全关闭时间更新特性，为true时CreatedAt/UpdatedAt/DeletedAt都将失效
-  rag_vector:
     - type: "pgsql"
-      host: "192.168.0.83"
+      host: "116.204.74.41"
       port: "15432"
       user: "postgres"
       pass: "Bjang09@686^*^"
-      name: "tenant-1"
+      name: "rag"
-      prefix: "rag_vector_"        # (可选)表名前缀
+      role: "slave"               # (可选)数据库主从角色(master/slave)，默认为master。如果不使用应用主从机制请不配置或留空即可。
+      debug: false                # (可选)开启调试模式
+      dryRun: false               # (可选)ORM空跑(只读不写)
+      charset: "utf8"             # (可选)数据库编码(如: utf8mb4/utf8/gbk/gb2312)，一般设置为utf8mb4。默认为utf8。
+      timezone: "Asia/Shanghai"           # (可选)时区配置，例如:Local
+      maxIdle: 5                  # (可选)连接池最大闲置的连接数(默认10)
+      maxOpen: 20                 # (可选)连接池最大打开的连接数(默认无限制)
+      maxLifetime: "30s"          # (可选)连接对象可重复使用的时间长度(默认30秒)
+      maxIdleConnTime: "30s"      # (可选，v2.10新增)连接池中空闲连接的最大生存时间(默认30秒)。可以通过配置文件或SetConnMaxIdleTime方法设置，避免长时间空闲连接占用资源。
+      createdAt: "created_at"     # (可选)自动创建时间字段名称
+      updatedAt: "updated_at"     # (可选)自动更新时间字段名称
+      deletedAt: "deleted_at"     # (可选)软删除时间字段名称
+      timeMaintainDisabled: false # (可选)是否完全关闭时间更新特性，为true时CreatedAt/UpdatedAt/DeletedAt都将失效
+  tenant-1:
+    - type: "pgsql"
+      host: "localhost"
+      port: "5432"
+      user: "postgres"
+      pass: "123456"
+      name: "tenant"
       role: "master"
+      prefix: "rag_"        # (可选)表名前缀
       debug: true                # (可选)开启调试模式
       dryRun: false               # (可选)ORM空跑(只读不写)
       charset: "utf8"             # (可选)数据库编码(如: utf8mb4/utf8/gbk/gb2312)，一般设置为utf8mb4。默认为utf8。
@@ -49,22 +68,22 @@ database:
 
 redis:
   default:
-    address: "192.168.0.83:6379"
+    address: "localhost:6379"
     db: 0
 
 consul:
-  address: 192.168.0.83:8500
+  address: localhost:8500
 
 jaeger:
-  addr: 192.168.0.83:4318
+  addr: localhost:4318
 
 # eino框架配置
 eino:
   # 文件切分配置
   splitter:
-    bufferSize: 3      # 必须 >=3 才能识别上下文语义
+    bufferSize: 1
-    minChunkSize: 1  # 避免切碎
+    minChunkSize: 64
-    percentile: 0.75   # 保持不变
+    percentile: 0.75
   # 向量化配置
   embedding:
     provider: "dashscope"
@@ -73,22 +92,27 @@ eino:
 #    apiType: "multi_modal_api"
     apiKey: "sk-4a8b82770bf74bc490eb3e4c5a8e2be9"
     model: "text-embedding-v3"
-  chatmodel:
-    provider: "dashscope"
-    apiKey: "sk-4a8b82770bf74bc490eb3e4c5a8e2be9"
-    model: "qwen-turbo"
-  rerank:
-    provider: "dashscope"
-    apiKey: "sk-4a8b82770bf74bc490eb3e4c5a8e2be9"
-    model: "qwen3-rerank"
 
 # 文件上传服务地址，与oss模块minio中的endpoint一致
-filePrefix: "http://192.168.0.83:9000"
+filePrefix: "http://116.204.74.41:9000"
+
+gmq:
+  redis:
+    primary:
+      addr: "localhost"
+      port: "6379"
+      db: 0
+      username: ""
+      password: ""
+      poolSize: 10
+      minIdleConn: 5
+      maxActiveConn: 10
+      maxRetries: 30
 
 # Meilisearch 全文检索配置
 meilisearch:
   default:
-    host: "http://192.168.0.83"
+    host: "http://localhost"
     port: 7700
     apiKey: "admin"
 #    apiKey: "6b8b6062bcb5e31f150427961d9da1a9e81758aa"

consts/keyword/type.go

@@ -1,26 +0,0 @@
-package keyword
-
-import "github.com/gogf/gf/v2/util/gconv"
-
-var (
-	KeywordTypeDefined = newKeywordType(gconv.PtrInt8(1), "自定义")
-	KeywordTypeInitial = newKeywordType(gconv.PtrInt8(2), "初始化")
-)
-
-type KeywordType *int8
-
-type keywordType struct {
-	code KeywordType
-	desc string
-}
-
-func (s keywordType) Code() KeywordType {
-	return s.code
-}
-func (s keywordType) Desc() string {
-	return s.desc
-}
-
-func newKeywordType(code KeywordType, desc string) keywordType {
-	return keywordType{code: code, desc: desc}
-}

consts/model/config_type.go

@@ -1,132 +0,0 @@
-package model
-
-import (
-	"github.com/gogf/gf/v2/util/gconv"
-)
-
-var (
-	ModelConfigTypeVectorArk          = newModelConfigType(gconv.PtrString("ark"), "字节跳动火山引擎方舟大模型服务")
-	ModelConfigTypeVectorOllama       = newModelConfigType(gconv.PtrString("ollama"), "Ollama 本地大模型运行工具")
-	ModelConfigTypeVectorOpenAI       = newModelConfigType(gconv.PtrString("openAI"), "OpenAI 官方大模型服务")
-	ModelConfigTypeVectorQianfan      = newModelConfigType(gconv.PtrString("qianfan"), "百度文心一言千帆大模型平台")
-	ModelConfigTypeVectorTencentCloud = newModelConfigType(gconv.PtrString("tencentCloud"), "腾讯云大模型服务")
-	ModelConfigTypeVectorDashScope    = newModelConfigType(gconv.PtrString("dashScope"), "阿里云通义千问 DashScope 平台")
-
-	ModelConfigTypeChatArk      = newModelConfigType(gconv.PtrString("ark"), "字节跳动火山引擎方舟大模型服务")
-	ModelConfigTypeChatArkBot   = newModelConfigType(gconv.PtrString("arkBot"), "火山引擎 ARK Bot 智能体服务")
-	ModelConfigTypeChatClaude   = newModelConfigType(gconv.PtrString("claude"), "Anthropic Claude 系列大模型")
-	ModelConfigTypeChatDeepSeek = newModelConfigType(gconv.PtrString("deepSeek"), "DeepSeek 深度求索大模型")
-	ModelConfigTypeChatOllama   = newModelConfigType(gconv.PtrString("ollama"), "Ollama 本地大模型运行工具")
-	ModelConfigTypeChatOpenAI   = newModelConfigType(gconv.PtrString("openAI"), "OpenAI 官方大模型服务")
-	ModelConfigTypeChatQianfan  = newModelConfigType(gconv.PtrString("qianfan"), "百度文心一言千帆大模型平台")
-	ModelConfigTypeChatQwen     = newModelConfigType(gconv.PtrString("qwen"), "腾讯文心千问大模型平台")
-)
-
-type ModelConfigType *string
-
-type modelConfigType struct {
-	code ModelConfigType
-	desc string
-}
-
-func (s modelConfigType) Code() ModelConfigType {
-	return s.code
-}
-func (s modelConfigType) Desc() string {
-	return s.desc
-}
-
-func newModelConfigType(code ModelConfigType, desc string) modelConfigType {
-	return modelConfigType{code: code, desc: desc}
-}
-
-func GetVectorDescByCode(code ModelConfigType) string {
-	switch *code {
-	case *ModelConfigTypeVectorArk.Code():
-		return ModelConfigTypeVectorArk.Desc()
-	case *ModelConfigTypeVectorOllama.Code():
-		return ModelConfigTypeVectorOllama.Desc()
-	case *ModelConfigTypeVectorOpenAI.Code():
-		return ModelConfigTypeVectorOpenAI.Desc()
-	case *ModelConfigTypeVectorQianfan.Code():
-		return ModelConfigTypeVectorQianfan.Desc()
-	case *ModelConfigTypeVectorTencentCloud.Code():
-		return ModelConfigTypeVectorTencentCloud.Desc()
-	case *ModelConfigTypeVectorDashScope.Code():
-		return ModelConfigTypeVectorDashScope.Desc()
-	}
-	return "未知类型"
-}
-
-func GetChatDescByCode(code ModelConfigType) string {
-	switch *code {
-	case *ModelConfigTypeChatArk.Code():
-		return ModelConfigTypeChatArk.Desc()
-	case *ModelConfigTypeChatArkBot.Code():
-		return ModelConfigTypeChatArkBot.Desc()
-	case *ModelConfigTypeChatClaude.Code():
-		return ModelConfigTypeChatClaude.Desc()
-	case *ModelConfigTypeChatDeepSeek.Code():
-		return ModelConfigTypeChatDeepSeek.Desc()
-	case *ModelConfigTypeChatOllama.Code():
-		return ModelConfigTypeChatOllama.Desc()
-	case *ModelConfigTypeChatOpenAI.Code():
-		return ModelConfigTypeChatOpenAI.Desc()
-	case *ModelConfigTypeChatQianfan.Code():
-		return ModelConfigTypeChatQianfan.Desc()
-	case *ModelConfigTypeChatQwen.Code():
-		return ModelConfigTypeChatQwen.Desc()
-	}
-	return "未知类型"
-}
-
-type GetModelConfigTypeEnumRes struct {
-	Options []ConfigTypeKeyValue `json:"options"`
-}
-
-type ConfigTypeKeyValue struct {
-	Key   interface{} `json:"key"`   // 对应原有常量值
-	Value interface{} `json:"value"` // 对应描述信息
-}
-
-func GetAllModelConfigTypeEnums(modelType ModelType) *GetModelConfigTypeEnumRes {
-	// 枚举列表
-	var list []modelConfigType
-
-	if *modelType == *ModelTypeVector.Code() {
-		list = []modelConfigType{
-			ModelConfigTypeVectorArk,
-			ModelConfigTypeVectorOllama,
-			ModelConfigTypeVectorOpenAI,
-			ModelConfigTypeVectorQianfan,
-			ModelConfigTypeVectorTencentCloud,
-			ModelConfigTypeVectorDashScope,
-		}
-	}
-
-	if *modelType == *ModelTypeChat.Code() {
-		list = []modelConfigType{
-			ModelConfigTypeChatArk,
-			ModelConfigTypeChatArkBot,
-			ModelConfigTypeChatClaude,
-			ModelConfigTypeChatDeepSeek,
-			ModelConfigTypeChatOllama,
-			ModelConfigTypeChatOpenAI,
-			ModelConfigTypeChatQianfan,
-			ModelConfigTypeChatQwen,
-		}
-	}
-
-	// 组装返回格式
-	options := make([]ConfigTypeKeyValue, 0, len(list))
-	for _, item := range list {
-		options = append(options, ConfigTypeKeyValue{
-			Key:   *item.Code(),
-			Value: item.Desc(),
-		})
-	}
-
-	return &GetModelConfigTypeEnumRes{
-		Options: options,
-	}
-}

consts/model/type.go

@@ -1,66 +0,0 @@
-package model
-
-import "github.com/gogf/gf/v2/util/gconv"
-
-var (
-	ModelTypeVector = newModelType(gconv.PtrString("vector"), "向量")
-	ModelTypeChat   = newModelType(gconv.PtrString("chat"), "对话")
-)
-
-type ModelType *string
-
-type modelType struct {
-	code ModelType
-	desc string
-}
-
-func (s modelType) Code() ModelType {
-	return s.code
-}
-func (s modelType) Desc() string {
-	return s.desc
-}
-
-func newModelType(code ModelType, desc string) modelType {
-	return modelType{code: code, desc: desc}
-}
-
-func GetModelTypeDescByCode(code ModelType) string {
-	switch *code {
-	case *ModelTypeVector.Code():
-		return ModelTypeVector.Desc()
-	case *ModelTypeChat.Code():
-		return ModelTypeChat.Desc()
-	}
-	return "未知类型"
-}
-
-type GetModelTypeEnumRes struct {
-	Options []TypeKeyValue `json:"options"`
-}
-
-type TypeKeyValue struct {
-	Key   interface{} `json:"key"`   // 对应原有常量值
-	Value interface{} `json:"value"` // 对应描述信息
-}
-
-func GetAllModelTypeEnums() *GetModelTypeEnumRes {
-	// 枚举列表
-	list := []modelType{
-		//ModelTypeVector,
-		ModelTypeChat,
-	}
-
-	// 组装返回格式
-	options := make([]TypeKeyValue, 0, len(list))
-	for _, item := range list {
-		options = append(options, TypeKeyValue{
-			Key:   *item.Code(),
-			Value: item.Desc(),
-		})
-	}
-
-	return &GetModelTypeEnumRes{
-		Options: options,
-	}
-}

consts/public/msg_key.go

@@ -1,15 +0,0 @@
-package public
-
-const GmqMsgPluginsName = "gmq_msg"
-
-const KnowledgeLockEsKey = "knowledge:lock:knowledgeIdEs-%v"
-const KnowledgeLockSqlKey = "knowledge:lock:knowledgeIdSql-%v"
-const KnowledgeContentHashEsKey = "knowledge:knowledgeId:contentHashEs-%v"
-const KnowledgeContentHashSqlKey = "knowledge:knowledgeId:contentHashSql-%v"
-
-const (
-	KnowledgeDocumentVectorTopic    = "knowledge:document:vector:stream"   // 请求 Stream 键名（与发消息的key一致）
-	KnowledgeDocumentVectorConsumer = "knowledge-document-vector-consumer" // 消费者名称（唯一标识）
-	KnowledgeDocumentVectorCount    = 1                                    // 批处理大小（每次读取1条）
-	KnowledgeDocumentVectorAutoAck  = false                                // ACK是否自动确认（true自动确认，false不确认）
-)

consts/public/redis_key.go

@@ -0,0 +1,20 @@
+package public
+
+const KnowledgeLockEsKey = "rag:knowledge:lock:knowledgeIdEs-%v"
+const KnowledgeLockSqlKey = "rag:knowledge:lock:knowledgeIdSql-%v"
+const KnowledgeContentHashEsKey = "rag:knowledge:knowledgeId:contentHashEs-%v"
+const KnowledgeContentHashSqlKey = "rag:knowledge:knowledgeId:contentHashSql-%v"
+
+const (
+	KnowledgeDocumentVectorStatusTopic     = "knowledge:document:vector:status:stream"
+	KnowledgeDocumentVectorStatusConsumer  = "knowledge-document-vector-status-consumer"
+	KnowledgeDocumentVectorStatusBatchSize = 1
+	KnowledgeDocumentVectorStatusAutoAck   = false
+)
+
+const (
+	KnowledgeDocumentChunkTopic     = "knowledge:document:chunk:stream"   // 请求 Stream 键名（与发消息的key一致）
+	KnowledgeDocumentChunkConsumer  = "knowledge-document-chunk-consumer" // 消费者名称（唯一标识）
+	KnowledgeDocumentChunkBatchSize = 1                                   // 批处理大小（每次读取1条）
+	KnowledgeDocumentChunkAutoAck   = false                               // ACK是否自动确认（true自动确认，false不确认）
+)

consts/public/table_name.go

@@ -1,20 +1,12 @@
 package public
 
-// 数据库名称
-const (
-	DbNameKnowledge = "rag_knowledge"
-	DbNameVector    = "rag_vector"
-)
-
 // sql 数据库表名
 const (
-	TableNameDocument       = "document"
+	TableNameDocument      = "document"
-	TableNameDataset        = "dataset"
+	TableNameDataset       = "dataset"
-	TableNameKeyword        = "keyword"
+	TableNameKeyword       = "keyword"
-	TableNameTask           = "task"
+	TableNameDatasetIndex  = "dataset_index"
-	TableNameModel          = "model"
+	TableNameDocumentChunk = "document_chunk"
-	TableNameDatasetIndex   = "dataset_index"
-	TableNameDocumentVector = "document_vector"
 )
 
 // es 索引名称

consts/task/consts.go

@@ -1,30 +0,0 @@
-package task
-
-// TaskType 任务类型枚举：文档解析的三个子任务
-type TaskType string
-
-const (
-	TaskTypeExtractKeywords TaskType = "EXTRACT_KEYWORDS" // 提取关键词
-	TaskTypeGenerateVector  TaskType = "GENERATE_VECTOR"  // 生成向量
-	TaskTypeFullTextSearch  TaskType = "FULL_TEXT_SEARCH" // 全文检索
-	TaskTypeDocParse        TaskType = "DOC_PARSE"        // 顶层文档解析总任务
-)
-
-// TaskStatus 任务状态枚举
-type TaskStatus string
-
-const (
-	TaskStatusPending   TaskStatus = "PENDING"   // 待执行
-	TaskStatusRunning   TaskStatus = "RUNNING"   // 执行中
-	TaskStatusCompleted TaskStatus = "COMPLETED" // 已完成
-	TaskStatusFailed    TaskStatus = "FAILED"    // 执行失败
-)
-
-// TaskPriority 任务优先级
-type TaskPriority int
-
-const (
-	TaskPriorityLow    TaskPriority = 1 // 低
-	TaskPriorityMedium TaskPriority = 2 // 中
-	TaskPriorityHigh   TaskPriority = 3 // 高
-)

controller/dataset.go

@@ -6,7 +6,7 @@ import (
 	"rag/model/dto"
 	"rag/service"
 
-	"gitea.redpowerfuture.com/red-future/common/beans"
+	"gitea.com/red-future/common/beans"
 	"github.com/gogf/gf/v2/frame/g"
 )
 
@@ -40,3 +40,9 @@ func (c *dataset) List(ctx context.Context, req *dto.ListDatasetReq) (res *dto.L
 	res, err = service.Dataset.List(ctx, req)
 	return
 }
+
+// Search 搜索
+//func (c *dataset) Search(ctx context.Context, req *dto.SearchReq) (res *dto.SearchRes, err error) {
+//	res, err = service.Dataset.Search(ctx, req)
+//	return
+//}

controller/dataset_index.go

@@ -0,0 +1,5 @@
+package controller
+
+type datasetIndex struct{}
+
+var DatasetIndex = new(datasetIndex)

controller/document.go

@@ -2,10 +2,11 @@ package controller
 
 import (
 	"context"
+
 	"rag/model/dto"
 	"rag/service"
 
-	"gitea.redpowerfuture.com/red-future/common/beans"
+	"gitea.com/red-future/common/beans"
 	"github.com/gogf/gf/v2/frame/g"
 )
 
@@ -32,7 +33,7 @@ func (c *document) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (res
 }
 
 // Get 获取文件详情
-func (c *document) Get(ctx context.Context, req *dto.GetDocumentReq) (res *dto.GetDocumentRes, err error) {
+func (c *document) Get(ctx context.Context, req *dto.GetDocumentReq) (res *dto.DocumentVO, err error) {
 	res, err = service.Document.Get(ctx, req)
 	return
 }
@@ -46,23 +47,8 @@ func (c *document) List(ctx context.Context, req *dto.ListDocumentReq) (res *dto
 	return
 }
 
-// DocumentVector 处理文件(向量化)
+// Process 处理文件(向量化)
-func (c *document) DocumentVector(ctx context.Context, req *dto.DocumentVectorReq) (res *beans.ResponseEmpty, err error) {
+func (c *document) Process(ctx context.Context, req *dto.ProcessDocumentReq) (res *dto.ProcessDocumentRes, err error) {
-	err = service.Document.Vector(ctx, req)
+	res, err = service.Document.Process(ctx, req)
-	return
-}
-
-func (c *document) VectorSemanticSplit(ctx context.Context, req *dto.VectorSemanticSplitReq) (res *beans.ResponseEmpty, err error) {
-	err = service.Document.VectorSemanticSplit(ctx, req)
-	return
-}
-
-func (c *document) SearchRecursiveSplit(ctx context.Context, req *dto.SearchRecursiveSplitReq) (res *beans.ResponseEmpty, err error) {
-	err = service.Document.SearchRecursiveSplit(ctx, req)
-	return
-}
-
-func (c *document) KeywordExtract(ctx context.Context, req *dto.KeywordExtractReq) (res *beans.ResponseEmpty, err error) {
-	err = service.Document.KeywordExtract(ctx, req)
 	return
 }

controller/document_chunk.go

@@ -0,0 +1,29 @@
+package controller
+
+import (
+	"context"
+	"rag/model/dto"
+	"rag/service"
+
+	"gitea.com/red-future/common/beans"
+	"github.com/gogf/gf/v2/frame/g"
+)
+
+type documentChunk struct{}
+
+var DocumentChunk = new(documentChunk)
+
+// Update 更新文件片段
+func (c *documentChunk) Update(ctx context.Context, req *dto.UpdateDocumentChunkReq) (res *beans.ResponseEmpty, err error) {
+	err = service.DocumentChunk.Update(ctx, req)
+	return
+}
+
+// List 文件片段列表
+func (c *documentChunk) List(ctx context.Context, req *dto.ListDocumentChunkReq) (res *dto.ListDocumentChunkRes, err error) {
+	if !g.IsEmpty(req.Page) {
+		req.Page = &beans.Page{PageNum: 1, PageSize: 20}
+	}
+	res, err = service.DocumentChunk.List(ctx, req)
+	return
+}

controller/document_vector.go

@@ -1,35 +0,0 @@
-package controller
-
-import (
-	"context"
-	"rag/model/dto"
-	"rag/service"
-
-	"gitea.redpowerfuture.com/red-future/common/beans"
-	"github.com/gogf/gf/v2/frame/g"
-)
-
-type documentVector struct{}
-
-var DocumentVector = new(documentVector)
-
-// Query 执行RAG查询
-func (c *documentVector) Query(ctx context.Context, req *dto.RAGQueryReq) (res *dto.RAGQueryRes, err error) {
-	res, err = service.DocumentVector.Query(ctx, req)
-	return
-}
-
-// Update 更新文件片段
-func (c *documentVector) Update(ctx context.Context, req *dto.UpdateDocumentVectorReq) (res *beans.ResponseEmpty, err error) {
-	err = service.DocumentVector.Update(ctx, req)
-	return
-}
-
-// List 文件片段列表
-func (c *documentVector) List(ctx context.Context, req *dto.ListDocumentVectorReq) (res *dto.ListDocumentVectorRes, err error) {
-	if !g.IsEmpty(req.Page) {
-		req.Page = &beans.Page{PageNum: 1, PageSize: 20}
-	}
-	res, err = service.DocumentVector.List(ctx, req)
-	return
-}

controller/keyword.go

@@ -2,10 +2,11 @@ package controller
 
 import (
 	"context"
+
 	"rag/model/dto"
 	"rag/service"
 
-	"gitea.redpowerfuture.com/red-future/common/beans"
+	"gitea.com/red-future/common/beans"
 	"github.com/gogf/gf/v2/frame/g"
 )
 

controller/model.go

@@ -1,52 +0,0 @@
-package controller
-
-import (
-	"context"
-	"rag/model/dto"
-	"rag/service"
-
-	"gitea.redpowerfuture.com/red-future/common/beans"
-	"github.com/gogf/gf/v2/frame/g"
-)
-
-type model struct{}
-
-var Model = new(model)
-
-func (c *model) GetModelTypeEnums(ctx context.Context, req *dto.GetModelAllEnumsReq) (res *dto.GetModelEnumRes, err error) {
-	res, err = service.ModelService.GetModelAllEnums(ctx, req)
-	return
-}
-
-func (c *model) GetModelConfigFormFields(ctx context.Context, req *dto.GetModelConfigFormFieldsReq) (res *dto.GetModelConfigFormFieldsRes, err error) {
-	res, err = service.ModelService.GetModelConfigFormFields(ctx, req)
-	return
-}
-
-func (c *model) Create(ctx context.Context, req *dto.CreateModelReq) (res *dto.CreateModelRes, err error) {
-	res, err = service.ModelService.Create(ctx, req)
-	return
-}
-
-func (c *model) Update(ctx context.Context, req *dto.UpdateModelReq) (res *beans.ResponseEmpty, err error) {
-	err = service.ModelService.Update(ctx, req)
-	return
-}
-
-func (c *model) Delete(ctx context.Context, req *dto.DeleteModelReq) (res *beans.ResponseEmpty, err error) {
-	err = service.ModelService.Delete(ctx, req)
-	return
-}
-
-func (c *model) Get(ctx context.Context, req *dto.GetModelReq) (res *dto.ModelVO, err error) {
-	res, err = service.ModelService.Get(ctx, req)
-	return
-}
-
-func (c *model) List(ctx context.Context, req *dto.ListModelReq) (res *dto.ListModelRes, err error) {
-	if !g.IsEmpty(req.Page) {
-		req.Page = &beans.Page{PageNum: 1, PageSize: 20}
-	}
-	res, err = service.ModelService.List(ctx, req)
-	return
-}

controller/task.go

@@ -1,16 +0,0 @@
-package controller
-
-import (
-	"context"
-	"rag/model/dto"
-	"rag/service"
-)
-
-type task struct{}
-
-var Task = new(task)
-
-func (c *task) Get(ctx context.Context, req *dto.GetTaskReq) (res *dto.ListTaskRes, err error) {
-	res, err = service.Task.Get(ctx, req)
-	return
-}

dao/dataset.go

@@ -6,7 +6,7 @@ import (
 	"rag/model/dto"
 	"rag/model/entity"
 
-	"gitea.redpowerfuture.com/red-future/common/db/gfdb"
+	"gitea.com/red-future/common/db/gfdb"
 	"github.com/gogf/gf/v2/database/gdb"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/util/gconv"
@@ -22,7 +22,7 @@ func (d *datasetDao) Insert(ctx context.Context, req *dto.CreateDatasetReq) (id
 	if err = gconv.Struct(req, &res); err != nil {
 		return
 	}
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDataset).Data(&res).Insert()
+	r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDataset).Data(&res).Insert()
 	if err != nil {
 		return
 	}
@@ -31,7 +31,7 @@ func (d *datasetDao) Insert(ctx context.Context, req *dto.CreateDatasetReq) (id
 
 // Update 更新数据集
 func (d *datasetDao) Update(ctx context.Context, req *dto.UpdateDatasetReq) (rows int64, err error) {
-	model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDataset).OmitEmpty()
+	model := gfdb.DB(ctx).Model(ctx, public.TableNameDataset).OmitEmpty()
 	if !g.IsEmpty(req.DocumentCount) {
 		model.Data(entity.DatasetCol.DocumentCount, &gdb.Counter{
 			Field: entity.DatasetCol.DocumentCount,
@@ -53,7 +53,7 @@ func (d *datasetDao) Update(ctx context.Context, req *dto.UpdateDatasetReq) (row
 
 // Delete 删除数据集
 func (d *datasetDao) Delete(ctx context.Context, req *dto.DeleteDatasetReq) (rows int64, err error) {
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDataset).Where(entity.DatasetCol.Id, req.Id).Delete()
+	r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDataset).Where(entity.DatasetCol.Id, req.Id).Delete()
 	if err != nil {
 		return
 	}
@@ -61,7 +61,7 @@ func (d *datasetDao) Delete(ctx context.Context, req *dto.DeleteDatasetReq) (row
 }
 
 func (d *datasetDao) GetByID(ctx context.Context, req *dto.GetDatasetReq, fields ...string) (res *entity.Dataset, err error) {
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDataset).Where(entity.DatasetCol.Id, req.Id).Fields(fields).One()
+	r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDataset).Where(entity.DatasetCol.Id, req.Id).Fields(fields).One()
 	if err != nil {
 		return
 	}
@@ -71,7 +71,7 @@ func (d *datasetDao) GetByID(ctx context.Context, req *dto.GetDatasetReq, fields
 
 // List 获取数据集列表
 func (d *datasetDao) List(ctx context.Context, req *dto.ListDatasetReq, fields ...string) (res []*entity.Dataset, total int, err error) {
-	model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDataset).Fields(fields).OmitEmpty()
+	model := gfdb.DB(ctx).Model(ctx, public.TableNameDataset).Fields(fields).OmitEmpty()
 	if !g.IsEmpty(req.Keyword) {
 		model.WhereLike(entity.DatasetCol.Name, "%"+req.Keyword+"%")
 	}

dao/dataset_index.go

@@ -7,7 +7,7 @@ import (
 	"rag/consts/public"
 	"rag/model/entity"
 
-	"gitea.redpowerfuture.com/red-future/common/db/gfdb"
+	"gitea.com/red-future/common/db/gfdb"
 )
 
 var DatasetIndex = new(datasetIndexDao)
@@ -16,7 +16,7 @@ type datasetIndexDao struct{}
 
 // Insert 插入数据集索引
 func (d *datasetIndexDao) Insert(ctx context.Context, index *entity.DatasetIndex) (id int64, err error) {
-	_, err = gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDatasetIndex).Data(index).Insert()
+	_, err = gfdb.DB(ctx).Model(ctx, public.TableNameDatasetIndex).Data(index).Insert()
 	if err != nil {
 		return
 	}
@@ -25,7 +25,7 @@ func (d *datasetIndexDao) Insert(ctx context.Context, index *entity.DatasetIndex
 
 // GetByDatasetId 根据数据集ID获取索引
 func (d *datasetIndexDao) GetByDatasetId(ctx context.Context, datasetId int64) (result *entity.DatasetIndex, err error) {
-	err = gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDatasetIndex).Where(entity.DatasetIndexCol.DatasetId, datasetId).Scan(&result)
+	err = gfdb.DB(ctx).Model(ctx, public.TableNameDatasetIndex).Where(entity.DatasetIndexCol.DatasetId, datasetId).Scan(&result)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil, nil
@@ -37,21 +37,23 @@ func (d *datasetIndexDao) GetByDatasetId(ctx context.Context, datasetId int64) (
 
 // IncVectorCount 增加或减少向量数量
 func (d *datasetIndexDao) IncVectorCount(ctx context.Context, id int64, delta int64) (err error) {
-	_, err = gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDatasetIndex).
+	_, err = gfdb.DB(ctx).Model(ctx, public.TableNameDatasetIndex).
 		Where(entity.DatasetIndexCol.Id, id).
 		Increment(entity.DatasetIndexCol.VectorCount, delta)
 	return
 }
 
 func (d *datasetIndexDao) InsertIndex(ctx context.Context, indexName string) (err error) {
-	db := gfdb.DB(ctx, public.DbNameVector)
+	prefix, err := gfdb.GetTablePrefix(ctx)
+	if err != nil {
+		return
+	}
 	sqlStr := fmt.Sprintf(`
 		CREATE INDEX IF NOT EXISTS %s
 		ON %s
 		USING ivfflat (vector vector_cosine_ops)
-		WITH (lists = 100)
 		WHERE vector IS NOT NULL;
-	`, indexName, gfdb.TablePrefix+public.TableNameDocumentVector)
+	`, indexName, prefix+public.TableNameDocumentChunk)
-	_, err = db.Exec(ctx, sqlStr)
+	_, err = gfdb.DB(ctx).Exec(ctx, sqlStr)
 	return
 }

dao/document.go

@@ -6,7 +6,7 @@ import (
 	"rag/model/dto"
 	"rag/model/entity"
 
-	"gitea.redpowerfuture.com/red-future/common/db/gfdb"
+	"gitea.com/red-future/common/db/gfdb"
 	"github.com/gogf/gf/v2/database/gdb"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/util/gconv"
@@ -22,7 +22,7 @@ func (d *documentDao) Insert(ctx context.Context, req *dto.CreateDocumentReq) (i
 	if err = gconv.Struct(req, &res); err != nil {
 		return
 	}
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).Data(&res).Insert()
+	r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDocument).Data(&res).Insert()
 	if err != nil {
 		return
 	}
@@ -31,7 +31,7 @@ func (d *documentDao) Insert(ctx context.Context, req *dto.CreateDocumentReq) (i
 
 // Update 更新文件
 func (d *documentDao) Update(ctx context.Context, req *dto.UpdateDocumentReq) (rows int64, err error) {
-	model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).OmitEmpty()
+	model := gfdb.DB(ctx).Model(ctx, public.TableNameDocument).OmitEmpty()
 	if !g.IsEmpty(req.ChunkCount) {
 		model.Data(entity.DocumentCol.ChunkCount, &gdb.Counter{
 			Field: entity.DocumentCol.ChunkCount,
@@ -48,28 +48,16 @@ func (d *documentDao) Update(ctx context.Context, req *dto.UpdateDocumentReq) (r
 
 // Delete 删除文件
 func (d *documentDao) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (rows int64, err error) {
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).OmitEmpty().
+	r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDocument).Where(entity.DocumentCol.Id, req.Id).Delete()
-		Where(entity.DocumentCol.Id, req.Id).
-		Delete()
 	if err != nil {
 		return
 	}
 	return r.RowsAffected()
 }
 
-func (d *documentDao) Count(ctx context.Context, req *dto.ListDocumentReq) (count int, err error) {
+// GetByID 根据ID获取文件
-	count, err = gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).OmitEmpty().
+func (d *documentDao) GetByID(ctx context.Context, req *dto.GetDocumentReq, fields ...string) (res *entity.Document, err error) {
-		Where(entity.DocumentCol.DatasetId, req.DatasetId).
+	r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDocument).Where(entity.DocumentCol.Id, req.Id).Fields(fields).One()
-		Where(entity.DocumentCol.Title, req.Title).Count()
-	return
-}
-
-// Get 根据ID获取文件
-func (d *documentDao) Get(ctx context.Context, req *dto.GetDocumentReq, fields ...string) (res *entity.Document, err error) {
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).Fields(fields).OmitEmpty().
-		Where(entity.DocumentCol.Id, req.Id).
-		Where(entity.DocumentCol.Title, req.Title).
-		Where(entity.DocumentCol.DatasetId, req.DatasetId).One()
 	if err != nil {
 		return
 	}
@@ -79,7 +67,7 @@ func (d *documentDao) Get(ctx context.Context, req *dto.GetDocumentReq, fields .
 
 // List 获取文件列表
 func (d *documentDao) List(ctx context.Context, req *dto.ListDocumentReq, fields ...string) (res []*entity.Document, total int, err error) {
-	model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameDocument).OmitEmpty()
+	model := gfdb.DB(ctx).Model(ctx, public.TableNameDocument).OmitEmpty()
 	if !g.IsEmpty(req.Keyword) {
 		model.WhereLike(entity.DocumentCol.Title, "%"+req.Keyword+"%")
 	}

dao/document_chunk.go

@@ -0,0 +1,104 @@
+package dao
+
+import (
+	"context"
+	"rag/consts/public"
+	"rag/model/dto"
+	"rag/model/entity"
+
+	"gitea.com/red-future/common/db/gfdb"
+	"github.com/gogf/gf/v2/util/gconv"
+)
+
+var DocumentChunk = new(documentChunkDao)
+
+type documentChunkDao struct{}
+
+// BatchInsert 批量插入文件块
+func (d *documentChunkDao) BatchInsert(ctx context.Context, req []*dto.VectorDocumentChunkMsg) (rows int64, err error) {
+	var res []*entity.DocumentChunk
+	if err = gconv.Structs(req, &res); err != nil {
+		return
+	}
+	r, err := gfdb.DB(ctx).Model(ctx, public.TableNameDocumentChunk).Data(&res).Insert()
+	if err != nil {
+		return
+	}
+	return r.RowsAffected()
+}
+
+// Update 更新文件块
+func (d *documentChunkDao) Update(ctx context.Context, req *dto.UpdateDocumentChunkReq) (rows int64, err error) {
+	model := gfdb.DB(ctx).Model(ctx, public.TableNameDocumentChunk)
+	r, err := model.Data(&req).Where(entity.DocumentChunkCol.Id, req.Id).Update()
+	if err != nil {
+		return
+	}
+	return r.RowsAffected()
+}
+
+// List 文件块列表
+func (d *documentChunkDao) List(ctx context.Context, req *dto.ListDocumentChunkReq, fields ...string) (res []*entity.DocumentChunk, total int, err error) {
+	model := gfdb.DB(ctx).Model(ctx, public.TableNameDocumentChunk).Fields(fields).OmitEmpty().
+		Where(entity.DocumentChunkCol.DatasetId, req.DatasetId).
+		Where(entity.DocumentChunkCol.DocumentId, req.DocumentId).
+		Where(entity.DocumentChunkCol.Status, req.Status).
+		Where(entity.DocumentChunkCol.VectorStatus, req.VectorStatus).
+		OrderDesc(entity.DocumentChunkCol.CreatedAt)
+	if req.Page != nil {
+		model.Page(int(req.Page.PageNum), int(req.Page.PageSize))
+	}
+	r, total, err := model.AllAndCount(false)
+	if err != nil {
+		return
+	}
+	err = r.Structs(&res)
+	return
+}
+
+//// Insert 插入向量文档
+//func (d *vectorDocumentDao) Insert(ctx context.Context, docs []*entity.DocumentChunk) (ids []interface{}, err error) {
+//	if len(docs) == 0 {
+//		return
+//	}
+//	interfaces := make([]interface{}, len(docs))
+//	for i := range docs {
+//		interfaces[i] = docs[i]
+//	}
+//	return mongoDB.Insert(ctx, interfaces, CollectionVectorDoc)
+//}
+//
+//// DeleteByIDs 根据ID删除向量文档
+//func (d *vectorDocumentDao) DeleteByIDs(ctx context.Context, ids []string) (err error) {
+//	if len(ids) == 0 {
+//		return
+//	}
+//	objectIDs := make([]bson.ObjectID, len(ids))
+//	for i, id := range ids {
+//		objectIDs[i], err = bson.ObjectIDFromHex(id)
+//		if err != nil {
+//			return err
+//		}
+//	}
+//	filter := bson.M{"_id": bson.M{"$in": objectIDs}}
+//	_, err = mongoDB.Delete(ctx, filter, CollectionVectorDoc)
+//	return
+//}
+//
+//// GetByIndexID 根据索引ID获取向量文档
+//func (d *vectorDocumentDao) GetByIndexID(ctx context.Context, indexID string, limit int) (result []*entity.DocumentChunk, err error) {
+//	filter := bson.M{"indexId": indexID}
+//	page := &beans.Page{PageNum: 1, PageSize: int64(limit)}
+//	_, err = mongoDB.Find(ctx, filter, &result, CollectionVectorDoc, page, nil)
+//	return
+//}
+//
+//// GetByVectorIDs 根据向量ID获取向量文档
+//func (d *vectorDocumentDao) GetByVectorIDs(ctx context.Context, vectorIDs []string) (result []*entity.DocumentChunk, err error) {
+//	if len(vectorIDs) == 0 {
+//		return
+//	}
+//	filter := bson.M{"vectorId": bson.M{"$in": vectorIDs}}
+//	_, err = mongoDB.Find(ctx, filter, &result, CollectionVectorDoc, &beans.Page{PageSize: -1}, nil)
+//	return
+//}

dao/document_vector.go

@@ -1,152 +0,0 @@
-package dao
-
-import (
-	"context"
-	"fmt"
-	"rag/consts/public"
-	"rag/model/dto"
-	"rag/model/entity"
-
-	"gitea.redpowerfuture.com/red-future/common/db/gfdb"
-	"gitea.redpowerfuture.com/red-future/common/full-text-search/meilisearch"
-	"github.com/gogf/gf/v2/database/gdb"
-	"github.com/gogf/gf/v2/frame/g"
-	"github.com/gogf/gf/v2/text/gstr"
-	"github.com/gogf/gf/v2/util/gconv"
-	"github.com/pgvector/pgvector-go"
-)
-
-var DocumentVector = new(documentVectorDao)
-
-type documentVectorDao struct{}
-
-// BatchInsert 批量插入文件块
-func (d *documentVectorDao) BatchInsert(ctx context.Context, req []*dto.VectorDocumentVectorMsg) (rows int64, err error) {
-	var res []*entity.DocumentVector
-	if err = gconv.Structs(req, &res); err != nil {
-		return
-	}
-	r, err := gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDocumentVector).Data(&res).Insert()
-	if err != nil {
-		return
-	}
-	return r.RowsAffected()
-}
-
-// Update 更新文件块
-func (d *documentVectorDao) Update(ctx context.Context, req *dto.UpdateDocumentVectorReq) (rows int64, err error) {
-	model := gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDocumentVector).OmitEmpty()
-	r, err := model.Data(&req).Where(entity.DocumentVectorCol.Id, req.Id).Update()
-	if err != nil {
-		return
-	}
-	return r.RowsAffected()
-}
-
-func (d *documentVectorDao) Delete(ctx context.Context, req *dto.DeleteDocumentVectorReq) (rows int64, err error) {
-	result, err := gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDocumentVector).OmitEmpty().
-		Where(entity.DocumentVectorCol.Id, req.Id).
-		Where(entity.DocumentVectorCol.DocumentId, req.DocumentId).
-		Delete()
-	if err != nil {
-		return
-	}
-	return result.RowsAffected()
-}
-
-// List 文件块列表
-func (d *documentVectorDao) List(ctx context.Context, req *dto.ListDocumentVectorReq, fields ...string) (res []*entity.DocumentVector, total int, err error) {
-	model := gfdb.DB(ctx, public.DbNameVector).Model(ctx, public.TableNameDocumentVector).Fields(fields).OmitEmpty().
-		Where(entity.DocumentVectorCol.DatasetId, req.DatasetId).
-		Where(entity.DocumentVectorCol.DocumentId, req.DocumentId).
-		Where(entity.DocumentVectorCol.Status, req.Status).
-		Where(entity.DocumentVectorCol.VectorStatus, req.VectorStatus).
-		WhereIn(entity.DocumentVectorCol.DocumentId, req.DocumentIds)
-	if !g.IsEmpty(req.Keyword) {
-		model.WhereLike(entity.DocumentVectorCol.Content, "%"+req.Keyword+"%")
-	}
-	model.OrderAsc(entity.DocumentVectorCol.ChunkIndex)
-	if req.Page != nil {
-		model.Page(int(req.Page.PageNum), int(req.Page.PageSize))
-	}
-	r, total, err := model.AllAndCount(false)
-	if err != nil {
-		return
-	}
-	err = r.Structs(&res)
-	return
-}
-
-func (d *documentVectorDao) GetAllByVector(ctx context.Context, datasetIds, documentIds []int64, vector pgvector.Vector, topK int) (list gdb.List, err error) {
-	// 动态拼接 WHERE 条件
-	var whereCondition string
-	var queryParams []interface{}
-
-	// 优先使用 documentIds 查询
-	if len(documentIds) > 0 {
-		whereCondition = fmt.Sprintf(" AND %s IN (?) ", entity.DocumentVectorCol.DocumentId)
-		queryParams = append(queryParams, documentIds)
-	}
-	if len(datasetIds) > 0 {
-		whereCondition = fmt.Sprintf(" AND %s IN (?) ", entity.DocumentVectorCol.DatasetId)
-		queryParams = append(queryParams, datasetIds)
-	}
-
-	// 完整 SQL
-	sql := `
-	SELECT id, content, dataset_id, document_id,vector <=> ? AS distance
-	FROM rag_vector_document_vector
-	WHERE 1=1 ` + whereCondition + ` AND vector IS NOT NULL ORDER BY distance ASC LIMIT ?`
-	// 拼接参数：vector + 条件参数 + topK
-	queryParams = append([]interface{}{vector}, queryParams...)
-	queryParams = append(queryParams, topK)
-
-	// 执行查询
-	result, err := gfdb.DB(ctx, public.DbNameVector).GetAll(ctx, sql, queryParams...)
-	if err != nil {
-		return nil, err
-	}
-	return result.List(), nil
-}
-
-// SearchByKeywords 通过关键词全文检索文档块
-func (d *documentVectorDao) SearchByKeywords(ctx context.Context, query string, datasetIds, documentIds []int64, topK int) (list gdb.List, err error) {
-	// 构建 meilisearch 查询参数
-	searchParams := &meilisearch.SearchParams{
-		Query:            query,
-		Limit:            int64(topK),
-		ShowRankingScore: true,
-	}
-
-	if len(datasetIds) > 0 {
-		datasetIdStrs := gconv.Strings(datasetIds)
-		quotedIds := make([]string, len(datasetIdStrs))
-		for i, id := range datasetIdStrs {
-			quotedIds[i] = fmt.Sprintf("%s", id)
-		}
-		searchParams.Filter = fmt.Sprintf("%s IN [%s]", entity.DocumentVectorCol.DatasetId, gstr.Implode(", ", quotedIds))
-	}
-
-	if len(documentIds) > 0 {
-		documentIdStrs := gconv.Strings(documentIds)
-		quotedIds := make([]string, len(documentIdStrs))
-		for i, id := range documentIdStrs {
-			quotedIds[i] = fmt.Sprintf("%s", id)
-		}
-		searchParams.Filter = fmt.Sprintf("%s IN [%s]", entity.DocumentVectorCol.DocumentId, gstr.Implode(", ", quotedIds))
-	}
-
-	// 执行搜索
-	var hits []map[string]interface{}
-	_, err = meilisearch.DB().Search(ctx, searchParams, public.IndexNameDocumentChunk, &hits)
-	if err != nil {
-		return nil, err
-	}
-
-	// 转换查询结果为 gdb.List
-	resultList := make(gdb.List, 0, len(hits))
-	for _, hit := range hits {
-		resultList = append(resultList, hit)
-	}
-	return resultList, nil
-}

dao/keyword.go

@@ -6,7 +6,7 @@ import (
 	"rag/model/dto"
 	"rag/model/entity"
 
-	"gitea.redpowerfuture.com/red-future/common/db/gfdb"
+	"gitea.com/red-future/common/db/gfdb"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/util/gconv"
 )
@@ -20,7 +20,7 @@ func (d *keywordDao) Insert(ctx context.Context, req *dto.CreateKeywordReq) (id
 	if err = gconv.Struct(req, &res); err != nil {
 		return
 	}
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameKeyword).Data(&res).Insert()
+	r, err := gfdb.DB(ctx).Model(ctx, public.TableNameKeyword).Data(&res).Insert()
 	if err != nil {
 		return
 	}
@@ -32,7 +32,7 @@ func (d *keywordDao) BatchSaveOrUpdate(ctx context.Context, req []*dto.CreateKey
 	if err = gconv.Structs(req, &res); err != nil {
 		return
 	}
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameKeyword).Data(&res).OnConflict(
+	r, err := gfdb.DB(ctx).Model(ctx, public.TableNameKeyword).Data(&res).OnConflict(
 		entity.KeywordCol.TenantId,
 		entity.KeywordCol.DatasetId,
 		entity.KeywordCol.DocumentId,
@@ -44,7 +44,7 @@ func (d *keywordDao) BatchSaveOrUpdate(ctx context.Context, req []*dto.CreateKey
 }
 
 func (d *keywordDao) Update(ctx context.Context, req *dto.UpdateKeywordReq) (rows int64, err error) {
-	model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameKeyword).OmitEmpty()
+	model := gfdb.DB(ctx).Model(ctx, public.TableNameKeyword)
 	r, err := model.Data(&req).Where(entity.KeywordCol.Id, req.Id).Update()
 	if err != nil {
 		return
@@ -53,10 +53,7 @@ func (d *keywordDao) Update(ctx context.Context, req *dto.UpdateKeywordReq) (row
 }
 
 func (d *keywordDao) Delete(ctx context.Context, req *dto.DeleteKeywordReq) (rows int64, err error) {
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameKeyword).OmitEmpty().
+	r, err := gfdb.DB(ctx).Model(ctx, public.TableNameKeyword).Where(entity.KeywordCol.Id, req.Id).Delete()
-		Where(entity.KeywordCol.Id, req.Id).
-		Where(entity.KeywordCol.DocumentId, req.DocumentId).
-		Delete()
 	if err != nil {
 		return
 	}
@@ -64,7 +61,7 @@ func (d *keywordDao) Delete(ctx context.Context, req *dto.DeleteKeywordReq) (row
 }
 
 func (d *keywordDao) Count(ctx context.Context, req *dto.ListKeywordReq) (count int, err error) {
-	count, err = gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameKeyword).OmitEmpty().
+	count, err = gfdb.DB(ctx).Model(ctx, public.TableNameKeyword).OmitEmpty().
 		Where(entity.KeywordCol.DatasetId, req.DatasetId).
 		Where(entity.KeywordCol.DocumentId, req.DocumentId).
 		Where(entity.KeywordCol.Word, req.Word).Count()
@@ -72,7 +69,7 @@ func (d *keywordDao) Count(ctx context.Context, req *dto.ListKeywordReq) (count
 }
 
 func (d *keywordDao) GetByID(ctx context.Context, req *dto.GetKeywordReq, fields ...string) (res *entity.Document, err error) {
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameKeyword).Where(entity.KeywordCol.Id, req.Id).Fields(fields).One()
+	r, err := gfdb.DB(ctx).Model(ctx, public.TableNameKeyword).Where(entity.KeywordCol.Id, req.Id).Fields(fields).One()
 	if err != nil {
 		return
 	}
@@ -81,13 +78,10 @@ func (d *keywordDao) GetByID(ctx context.Context, req *dto.GetKeywordReq, fields
 }
 
 func (d *keywordDao) List(ctx context.Context, req *dto.ListKeywordReq, fields ...string) (res []*entity.Keyword, total int, err error) {
-	model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameKeyword).Fields(fields).OmitEmpty()
+	model := gfdb.DB(ctx).Model(ctx, public.TableNameKeyword).Fields(fields).OmitEmpty()
 	if !g.IsEmpty(req.Keyword) {
 		model.WhereLike(entity.KeywordCol.Word, "%"+req.Keyword+"%")
 	}
-	model.WhereIn(entity.KeywordCol.Word, req.Words)
-	model.Where(entity.KeywordCol.DatasetId, req.DatasetId)
-	model.Where(entity.KeywordCol.DocumentId, req.DocumentId)
 	model.OrderDesc(entity.KeywordCol.Weight)
 	model.OrderDesc(entity.KeywordCol.CreatedAt)
 	if req.Page != nil {

dao/model.go

@@ -1,88 +0,0 @@
-package dao
-
-import (
-	"context"
-	"rag/consts/public"
-	"rag/model/dto"
-	"rag/model/entity"
-
-	"gitea.redpowerfuture.com/red-future/common/db/gfdb"
-	"github.com/gogf/gf/v2/frame/g"
-	"github.com/gogf/gf/v2/util/gconv"
-)
-
-var Model = new(modelDao)
-
-type modelDao struct{}
-
-func (d *modelDao) Insert(ctx context.Context, req *dto.CreateModelReq) (id int64, err error) {
-	var res *entity.Model
-	if err = gconv.Struct(req, &res); err != nil {
-		return
-	}
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).Data(&res).Insert()
-	if err != nil {
-		return
-	}
-	return r.LastInsertId()
-}
-
-func (d *modelDao) Update(ctx context.Context, req *dto.UpdateModelReq) (rows int64, err error) {
-	model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).OmitEmpty()
-	r, err := model.Data(&req).Where(entity.ModelCol.Id, req.Id).Update()
-	if err != nil {
-		return
-	}
-	return r.RowsAffected()
-}
-
-func (d *modelDao) Delete(ctx context.Context, req *dto.DeleteModelReq) (rows int64, err error) {
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).Where(entity.ModelCol.Id, req.Id).Delete()
-	if err != nil {
-		return
-	}
-	return r.RowsAffected()
-}
-
-func (d *modelDao) Count(ctx context.Context, req *dto.GetModelReq) (count int, err error) {
-	count, err = gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).OmitEmpty().Where(entity.ModelCol.ModelType, req.ModelType).Count()
-	return
-}
-
-func (d *modelDao) Get(ctx context.Context, req *dto.GetModelReq, fields ...string) (res *entity.Model, err error) {
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).Fields(fields).OmitEmpty().
-		Where(entity.ModelCol.Id, req.Id).
-		Where(entity.ModelCol.ModelType, req.ModelType).One()
-	if err != nil {
-		return
-	}
-	err = r.Struct(&res)
-	return
-}
-
-func (d *modelDao) GetNoTenantId(ctx context.Context, req *dto.GetModelReq, fields ...string) (res []*entity.Model, err error) {
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).NoTenantId(ctx).Where(entity.ModelCol.ModelType, req.ModelType).Fields(fields).All()
-	if err != nil {
-		return
-	}
-	err = r.Structs(&res)
-	return
-}
-
-func (d *modelDao) List(ctx context.Context, req *dto.ListModelReq, fields ...string) (res []*entity.Model, total int, err error) {
-	model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameModel).Fields(fields).OmitEmpty()
-	if !g.IsEmpty(req.ModelName) {
-		model.WhereLike(entity.ModelCol.ModelName, "%"+req.ModelName+"%")
-	}
-	model.Where(entity.ModelCol.ModelType, req.ModelType)
-	model.OrderDesc(entity.KeywordCol.CreatedAt)
-	if req.Page != nil {
-		model.Page(int(req.Page.PageNum), int(req.Page.PageSize))
-	}
-	r, total, err := model.AllAndCount(false)
-	if err != nil {
-		return
-	}
-	err = r.Structs(&res)
-	return
-}

dao/task.go

@@ -1,67 +0,0 @@
-package dao
-
-import (
-	"context"
-	"rag/consts/public"
-	"rag/model/dto"
-	"rag/model/entity"
-
-	"gitea.redpowerfuture.com/red-future/common/db/gfdb"
-	"github.com/gogf/gf/v2/util/gconv"
-)
-
-var Task = new(taskDao)
-
-type taskDao struct{}
-
-// Insert 创建任务
-func (d *taskDao) Insert(ctx context.Context, req *dto.CreateTaskReq) (id int64, err error) {
-	var res *entity.Task
-	if err = gconv.Struct(req, &res); err != nil {
-		return
-	}
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameTask).Data(&res).Insert()
-	if err != nil {
-		return
-	}
-	return r.LastInsertId()
-}
-
-// Update 更新任务
-func (d *taskDao) Update(ctx context.Context, req *dto.UpdateTaskReq) (rows int64, err error) {
-	model := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameTask).OmitEmpty()
-	r, err := model.Data(&req).Where(entity.TaskCol.Id, req.Id).Where(entity.TaskCol.TaskId, req.TaskId).Update()
-	if err != nil {
-		return
-	}
-	return r.RowsAffected()
-}
-
-func (d *taskDao) Count(ctx context.Context, req *dto.GetTaskReq) (count int, err error) {
-	count, err = gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameTask).OmitEmpty().
-		Where(entity.TaskCol.TaskId, req.TaskId).
-		Where(entity.TaskCol.TaskType, req.TaskType).
-		Where(entity.TaskCol.Status, req.TaskStatus).
-		Count()
-	return
-}
-
-func (d *taskDao) Get(ctx context.Context, req *dto.GetTaskReq) (res []*entity.Task, total int, err error) {
-	r, total, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameTask).OmitEmpty().
-		Where(entity.TaskCol.Id, req.Id).
-		Where(entity.TaskCol.TaskId, req.TaskId).
-		Where(entity.TaskCol.TaskType, req.TaskType).AllAndCount(false)
-	if err != nil {
-		return
-	}
-	err = r.Structs(&res)
-	return
-}
-
-func (d *taskDao) DeleteByTaskId(ctx context.Context, req *dto.DeleteTaskByTaskIdReq) (rows int64, err error) {
-	r, err := gfdb.DB(ctx, public.DbNameKnowledge).Model(ctx, public.TableNameTask).Where(entity.TaskCol.TaskId, req.TaskId).Delete()
-	if err != nil {
-		return
-	}
-	return r.RowsAffected()
-}

go.mod

@@ -1,90 +1,66 @@
 module rag
 
-go 1.26.1
+go 1.26.0
 
 require (
-	gitea.redpowerfuture.com/red-future/common v0.0.29
+	gitea.com/red-future/common v0.0.6
-	github.com/bjang03/gmq v0.0.1
+	github.com/bjang03/gmq v0.0.0-00010101000000-000000000000
-	github.com/cloudwego/eino v0.9.9
+	github.com/cloudwego/eino v0.8.6
-	github.com/cloudwego/eino-ext/components/document/loader/file v0.0.0-20260610041010-8fe947165ad0
+	github.com/gogf/gf/contrib/drivers/pgsql/v2 v2.10.0
-	github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20260610041010-8fe947165ad0
+	github.com/gogf/gf/v2 v2.10.0
-	github.com/cloudwego/eino-ext/components/document/parser/docx v0.0.0-20260610041010-8fe947165ad0
+	github.com/pgvector/pgvector-go v0.3.0
-	github.com/cloudwego/eino-ext/components/document/parser/pdf v0.0.0-20260610041010-8fe947165ad0
-	github.com/cloudwego/eino-ext/components/document/parser/xlsx v0.0.0-20260610041010-8fe947165ad0
-	github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive v0.0.0-20260610041010-8fe947165ad0
-	github.com/cloudwego/eino-ext/components/document/transformer/splitter/semantic v0.0.0-20260610041010-8fe947165ad0
-	github.com/cloudwego/eino-ext/components/embedding/ark v0.1.2
-	github.com/cloudwego/eino-ext/components/embedding/dashscope v0.0.0-20260610041010-8fe947165ad0
-	github.com/cloudwego/eino-ext/components/embedding/ollama v0.0.0-20260610041010-8fe947165ad0
-	github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260610041010-8fe947165ad0
-	github.com/cloudwego/eino-ext/components/embedding/qianfan v0.0.0-20260610041010-8fe947165ad0
-	github.com/cloudwego/eino-ext/components/embedding/tencentcloud v0.0.0-20260610041010-8fe947165ad0
-	github.com/cloudwego/eino-ext/components/model/ark v0.1.68
-	github.com/cloudwego/eino-ext/components/model/arkbot v0.1.2
-	github.com/cloudwego/eino-ext/components/model/claude v0.1.19
-	github.com/cloudwego/eino-ext/components/model/deepseek v0.1.6
-	github.com/cloudwego/eino-ext/components/model/ollama v0.1.9
-	github.com/cloudwego/eino-ext/components/model/openai v0.1.13
-	github.com/cloudwego/eino-ext/components/model/qianfan v0.1.4
-	github.com/cloudwego/eino-ext/components/model/qwen v0.1.9
-	github.com/gogf/gf/contrib/drivers/pgsql/v2 v2.10.2
-	github.com/gogf/gf/v2 v2.10.2
-	github.com/pgvector/pgvector-go v0.4.0
 )
 
+replace gitea.com/red-future/common v0.0.6 => ../common
+
+replace github.com/bjang03/gmq => ../gmq
+
 require (
-	cloud.google.com/go/auth v0.7.2 // indirect
-	cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
-	cloud.google.com/go/compute/metadata v0.9.0 // indirect
 	github.com/BurntSushi/toml v1.6.0 // indirect
 	github.com/PuerkitoBio/goquery v1.8.1 // indirect
 	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
-	github.com/anthropics/anthropic-sdk-go v1.26.0 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.33.0 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.29.1 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.54 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect
-	github.com/aws/smithy-go v1.22.1 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/bahlo/generic-list-go v0.2.0 // indirect
-	github.com/baidubce/bce-qianfan-sdk/go/qianfan v0.0.14 // indirect
+	github.com/buger/jsonparser v1.1.1 // indirect
-	github.com/baidubce/bce-sdk-go v0.9.164 // indirect
-	github.com/buger/jsonparser v1.2.0 // indirect
 	github.com/bwmarrin/snowflake v0.3.0 // indirect
-	github.com/bytedance/gopkg v0.1.4 // indirect
+	github.com/bytedance/gopkg v0.1.3 // indirect
-	github.com/bytedance/sonic v1.15.2 // indirect
+	github.com/bytedance/sonic v1.15.0 // indirect
-	github.com/bytedance/sonic/loader v0.5.1 // indirect
+	github.com/bytedance/sonic/loader v0.5.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/clbanning/mxj/v2 v2.7.0 // indirect
 	github.com/clipperhouse/displaywidth v0.11.0 // indirect
 	github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
-	github.com/cloudwego/base64x v0.1.7 // indirect
+	github.com/cloudwego/base64x v0.1.6 // indirect
+	github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20260323112355-f061db7e8419 // indirect
+	github.com/cloudwego/eino-ext/components/document/parser/docx v0.0.0-20260323112355-f061db7e8419 // indirect
 	github.com/cloudwego/eino-ext/components/document/parser/html v0.0.0-20241224063832-9fbcc0e56c28 // indirect
-	github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17 // indirect
+	github.com/cloudwego/eino-ext/components/document/parser/pdf v0.0.0-20260323112355-f061db7e8419 // indirect
-	github.com/cohesion-org/deepseek-go v1.3.4 // indirect
+	github.com/cloudwego/eino-ext/components/document/parser/xlsx v0.0.0-20260323112355-f061db7e8419 // indirect
+	github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive v0.0.0-20260323112355-f061db7e8419 // indirect
+	github.com/cloudwego/eino-ext/components/document/transformer/splitter/semantic v0.0.0-20260323112355-f061db7e8419 // indirect
+	github.com/cloudwego/eino-ext/components/embedding/ark v0.1.1 // indirect
+	github.com/cloudwego/eino-ext/components/embedding/dashscope v0.0.0-20260323112355-f061db7e8419 // indirect
+	github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20260323112355-f061db7e8419 // indirect
+	github.com/cloudwego/eino-ext/components/indexer/es8 v0.0.0-20260331071634-4f359694d2d9 // indirect
+	github.com/cloudwego/eino-ext/components/retriever/es8 v0.0.0-20260331071634-4f359694d2d9 // indirect
+	github.com/cloudwego/eino-ext/libs/acl/openai v0.1.14 // indirect
 	github.com/dgraph-io/badger/v4 v4.2.0 // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
+	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/dslipak/pdf v0.0.2 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/eino-contrib/docx2md v0.0.1 // indirect
 	github.com/eino-contrib/jsonschema v1.0.3 // indirect
-	github.com/eino-contrib/ollama v0.1.0 // indirect
+	github.com/elastic/elastic-transport-go/v8 v8.10.0 // indirect
+	github.com/elastic/go-elasticsearch/v8 v8.16.0 // indirect
 	github.com/emirpasic/gods/v2 v2.0.0-alpha // indirect
 	github.com/evanphx/json-patch v0.5.2 // indirect
 	github.com/fatih/color v1.19.0 // indirect
-	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
-	github.com/fsnotify/fsnotify v1.10.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.13 // indirect
 	github.com/go-ego/gse v1.0.2 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
@@ -92,9 +68,8 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.30.1 // indirect
-	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
 	github.com/goccy/go-json v0.10.6 // indirect
-	github.com/gogf/gf/contrib/nosql/redis/v2 v2.10.2 // indirect
+	github.com/gogf/gf/contrib/nosql/redis/v2 v2.9.1 // indirect
 	github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5 // indirect
 	github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -103,122 +78,90 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v1.0.0 // indirect
-	github.com/google/btree v1.1.3 // indirect
+	github.com/google/flatbuffers v1.12.1 // indirect
-	github.com/google/flatbuffers v25.12.19+incompatible // indirect
-	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/uuid v1.6.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/goph/emperror v0.17.2 // indirect
 	github.com/gorilla/css v1.0.1 // indirect
-	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
+	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/grokify/html-strip-tags-go v0.1.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
-	github.com/hashicorp/consul/api v1.33.5 // indirect
+	github.com/hashicorp/consul/api v1.26.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
-	github.com/hashicorp/go-version v1.9.0 // indirect
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hashicorp/serf v0.10.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.18.6 // indirect
+	github.com/klauspost/compress v1.18.4 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
-	github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
-	github.com/lib/pq v1.12.3 // indirect
+	github.com/lib/pq v1.12.1 // indirect
 	github.com/magiconair/properties v1.8.10 // indirect
-	github.com/mailru/easyjson v0.9.2 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
-	github.com/mattn/go-colorable v0.1.15 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
-	github.com/mattn/go-isatty v0.0.22 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/mattn/go-runewidth v0.0.24 // indirect
+	github.com/mattn/go-runewidth v0.0.21 // indirect
-	github.com/meguminnnnnnnnn/go-openai v0.1.5 // indirect
+	github.com/meguminnnnnnnnn/go-openai v0.1.1 // indirect
 	github.com/meilisearch/meilisearch-go v0.36.1 // indirect
 	github.com/microcosm-cc/bluemonday v1.0.27 // indirect
-	github.com/miekg/dns v1.1.72 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/nats-io/nats.go v1.49.0 // indirect
 	github.com/nats-io/nkeys v0.4.15 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/nikolalohinski/gonja v1.5.3 // indirect
 	github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
-	github.com/olekukonko/errors v1.3.0 // indirect
+	github.com/olekukonko/errors v1.2.0 // indirect
 	github.com/olekukonko/ll v0.1.8 // indirect
 	github.com/olekukonko/tablewriter v1.1.4 // indirect
-	github.com/ollama/ollama v0.9.6 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
-	github.com/pelletier/go-toml/v2 v2.4.1 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/r3labs/diff/v2 v2.15.1 // indirect
 	github.com/rabbitmq/amqp091-go v1.10.0 // indirect
-	github.com/redis/go-redis/v9 v9.21.0 // indirect
+	github.com/redis/go-redis/v9 v9.18.0 // indirect
-	github.com/richardlehane/mscfb v1.0.6 // indirect
+	github.com/richardlehane/mscfb v1.0.4 // indirect
-	github.com/richardlehane/msoleps v1.0.6 // indirect
+	github.com/richardlehane/msoleps v1.0.4 // indirect
-	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
-	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/sirupsen/logrus v1.9.4 // indirect
 	github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect
-	github.com/sourcegraph/conc v0.3.0 // indirect
-	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.10.0 // indirect
-	github.com/spf13/pflag v1.0.10 // indirect
-	github.com/spf13/viper v1.18.2 // indirect
-	github.com/subosito/gotenv v1.6.0 // indirect
-	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.1093 // indirect
-	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/hunyuan v1.0.1093 // indirect
-	github.com/tidwall/gjson v1.19.0 // indirect
-	github.com/tidwall/match v1.2.0 // indirect
-	github.com/tidwall/pretty v1.2.1 // indirect
-	github.com/tidwall/sjson v1.2.5 // indirect
-	github.com/tiendc/go-deepcopy v1.7.2 // indirect
 	github.com/tiger1103/gfast-token v1.0.10 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/vcaesar/cedar v0.30.0 // indirect
-	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
+	github.com/volcengine/volc-sdk-golang v1.0.199 // indirect
-	github.com/volcengine/volc-sdk-golang v1.0.23 // indirect
+	github.com/volcengine/volcengine-go-sdk v1.0.181 // indirect
-	github.com/volcengine/volcengine-go-sdk v1.2.30 // indirect
 	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
-	github.com/xuri/efp v0.0.1 // indirect
+	github.com/xuri/efp v0.0.0-20240408161823-9ad904a10d6d // indirect
-	github.com/xuri/excelize/v2 v2.10.1 // indirect
+	github.com/xuri/excelize/v2 v2.9.0 // indirect
-	github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect
+	github.com/xuri/nfp v0.0.0-20240318013403-ab9948c2c4a7 // indirect
 	github.com/yargevad/filepathx v1.0.0 // indirect
-	go.mongodb.org/mongo-driver/v2 v2.6.0 // indirect
+	go.mongodb.org/mongo-driver/v2 v2.4.0 // indirect
-	go.opencensus.io v0.24.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.42.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
-	go.opentelemetry.io/otel v1.44.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 // indirect
-	go.opentelemetry.io/otel/metric v1.44.0 // indirect
+	go.opentelemetry.io/otel/metric v1.42.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.44.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.42.0 // indirect
-	go.opentelemetry.io/otel/trace v1.44.0 // indirect
+	go.opentelemetry.io/otel/trace v1.42.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.7.1 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	go.uber.org/multierr v1.10.0 // indirect
+	golang.org/x/arch v0.15.0 // indirect
-	golang.org/x/arch v0.28.0 // indirect
+	golang.org/x/crypto v0.49.0 // indirect
-	golang.org/x/crypto v0.53.0 // indirect
+	golang.org/x/exp v0.0.0-20260312153236-7ab1446f8b90 // indirect
-	golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect
+	golang.org/x/net v0.52.0 // indirect
-	golang.org/x/net v0.56.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
-	golang.org/x/oauth2 v0.35.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
-	golang.org/x/sync v0.21.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
-	golang.org/x/sys v0.46.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
-	golang.org/x/text v0.38.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect
-	golang.org/x/time v0.9.0 // indirect
+	google.golang.org/grpc v1.75.0 // indirect
-	google.golang.org/api v0.189.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect
-	google.golang.org/grpc v1.79.3 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
-	gopkg.in/ini.v1 v1.67.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

main.go

@@ -7,12 +7,10 @@ import (
 	"rag/consts/public"
 	"rag/controller"
 	"rag/service"
-	"strings"
 	"syscall"
 
-	"gitea.redpowerfuture.com/red-future/common/http"
+	"gitea.com/red-future/common/http"
-	"gitea.redpowerfuture.com/red-future/common/jaeger"
+	"gitea.com/red-future/common/jaeger"
-	"gitea.redpowerfuture.com/red-future/common/utils"
 	gmq "github.com/bjang03/gmq/core/gmq"
 	"github.com/bjang03/gmq/mq"
 	"github.com/bjang03/gmq/types"
@@ -28,31 +26,30 @@ func main() {
 	http.RouteRegister([]interface{}{
 		controller.Dataset,
 		controller.Document,
-		controller.DocumentVector,
+		controller.DocumentChunk,
-		controller.Model,
-		controller.Keyword,
-		controller.Task,
 	})
 
-	if err := utils.InitGseTool(ctx); err != nil {
+	gmq.Init("config.yml")
-		g.Log().Error(ctx, "gse 分词工具初始化失败:", err)
+
+	if err := gmq.GetGmq("primary").GmqSubscribe(ctx, &mq.RedisSubMessage{
+		SubMessage: types.SubMessage{
+			Topic:        public.KnowledgeDocumentVectorStatusTopic,
+			ConsumerName: public.KnowledgeDocumentVectorStatusConsumer,
+			AutoAck:      public.KnowledgeDocumentVectorStatusAutoAck,
+			FetchCount:   public.KnowledgeDocumentVectorStatusBatchSize,
+			HandleFunc:   service.Document.DocsVectorStatusMsg,
+		},
+	}); err != nil {
+		return
 	}
 
-	redisAddress := g.Cfg().MustGet(ctx, "redis.default.address").String()
+	if err := gmq.GetGmq("primary").GmqSubscribe(ctx, &mq.RedisSubMessage{
-	redisAddressList := strings.Split(redisAddress, ":")
-	gmq.GmqRegister(public.GmqMsgPluginsName, &mq.RedisConn{
-		RedisConfig: mq.RedisConfig{
-			Addr: redisAddressList[0],
-			Port: redisAddressList[1],
-		},
-	})
-	if err := gmq.GetGmq(public.GmqMsgPluginsName).GmqSubscribe(ctx, &mq.RedisSubMessage{
 		SubMessage: types.SubMessage{
-			Topic:        public.KnowledgeDocumentVectorTopic,
+			Topic:        public.KnowledgeDocumentChunkTopic,
-			ConsumerName: public.KnowledgeDocumentVectorConsumer,
+			ConsumerName: public.KnowledgeDocumentChunkConsumer,
-			AutoAck:      public.KnowledgeDocumentVectorAutoAck,
+			AutoAck:      public.KnowledgeDocumentChunkAutoAck,
-			FetchCount:   public.KnowledgeDocumentVectorCount,
+			FetchCount:   public.KnowledgeDocumentChunkBatchSize,
-			HandleFunc:   service.DocumentVector.DocsChunkMsg,
+			HandleFunc:   service.DocumentChunk.DocsChunkMsg,
 		},
 	}); err != nil {
 		return

model/dto/dataset.go

@@ -1,14 +1,14 @@
 package dto
 
 import (
-	"gitea.redpowerfuture.com/red-future/common/beans"
+	"gitea.com/red-future/common/beans"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/os/gtime"
 )
 
 // CreateDatasetReq 创建数据集请求
 type CreateDatasetReq struct {
-	g.Meta `path:"/create" method:"post" tags:"知识库（数据集）管理" summary:"创建知识库（数据集）" dc:"创建知识库（数据集）"`
+	g.Meta `path:"/createDataset" method:"post" tags:"知识库（数据集）管理" summary:"创建知识库（数据集）" dc:"创建知识库（数据集）"`
 
 	Name        string `json:"name" v:"required#名称不能为空"`
 	Description string `json:"description"`
@@ -21,7 +21,7 @@ type CreateDatasetRes struct {
 
 // UpdateDatasetReq 更新数据集请求
 type UpdateDatasetReq struct {
-	g.Meta `path:"/update" method:"put" tags:"知识库（数据集）管理" summary:"更新知识库（数据集）" dc:"更新知识库（数据集）"`
+	g.Meta `path:"/updateDataset" method:"put" tags:"知识库（数据集）管理" summary:"更新知识库（数据集）" dc:"更新知识库（数据集）"`
 
 	Id            int64  `json:"id" v:"required#ID不能为空"`
 	Name          string `json:"name"`
@@ -32,24 +32,23 @@ type UpdateDatasetReq struct {
 
 // DeleteDatasetReq 删除数据集请求
 type DeleteDatasetReq struct {
-	g.Meta `path:"/delete" method:"delete" tags:"知识库（数据集）管理" summary:"删除知识库（数据集）" dc:"删除知识库（数据集）"`
+	g.Meta `path:"/deleteDataset" method:"delete" tags:"知识库（数据集）管理" summary:"删除知识库（数据集）" dc:"删除知识库（数据集）"`
 
 	Id int64 `json:"id" v:"required#ID不能为空"`
 }
 
 // GetDatasetReq 获取数据集请求
 type GetDatasetReq struct {
-	g.Meta `path:"/get" method:"get" tags:"知识库（数据集）管理" summary:"获取知识库（数据集）详情" dc:"获取知识库（数据集）详情"`
+	g.Meta `path:"/getDataset" method:"get" tags:"知识库（数据集）管理" summary:"获取知识库（数据集）详情" dc:"获取知识库（数据集）详情"`
 
 	Id int64 `json:"id" v:"required#ID不能为空"`
 }
 
 // ListDatasetReq 数据集列表请求
 type ListDatasetReq struct {
-	g.Meta `path:"/list" method:"get" tags:"知识库（数据集）管理" summary:"获取知识库（数据集）列表" dc:"分页查询知识库（数据集）列表，支持多条件筛选"`
+	g.Meta `path:"/listDataset" method:"get" tags:"知识库（数据集）管理" summary:"获取知识库（数据集）列表" dc:"分页查询知识库（数据集）列表，支持多条件筛选"`
 
 	Page    *beans.Page `json:"page"`
-	Ids     []int64     `json:"ids" dc:"数据集ID列表"`
 	Keyword string      `json:"keyword" dc:"关键词搜索"`
 }
 

model/dto/dataset_index.go

@@ -0,0 +1 @@
+package dto

model/dto/document.go

@@ -3,15 +3,14 @@ package dto
 import (
 	"rag/consts/document"
 
-	"gitea.redpowerfuture.com/red-future/common/beans"
+	"gitea.com/red-future/common/beans"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/os/gtime"
-	"github.com/pgvector/pgvector-go"
 )
 
 // CreateDocumentReq 创建文件请求
 type CreateDocumentReq struct {
-	g.Meta `path:"/create" method:"post" tags:"文件管理" summary:"创建文件" dc:"创建文件"`
+	g.Meta `path:"/createDocument" method:"post" tags:"文件管理" summary:"创建文件" dc:"创建文件"`
 
 	DatasetId int64  `json:"datasetId" v:"required#数据集ID不能为空"`
 	Title     string `json:"title" v:"required#标题不能为空"`
@@ -27,7 +26,7 @@ type CreateDocumentRes struct {
 
 // UpdateDocumentReq 更新文件请求
 type UpdateDocumentReq struct {
-	g.Meta `path:"/update" method:"put" tags:"文件管理" summary:"更新文件" dc:"更新文件"`
+	g.Meta `path:"/updateDocument" method:"put" tags:"文件管理" summary:"更新文件" dc:"更新文件"`
 
 	Id           int64                 `json:"id" v:"required#ID不能为空"`
 	Status       document.Status       `json:"status"`
@@ -37,33 +36,25 @@ type UpdateDocumentReq struct {
 
 // DeleteDocumentReq 删除文件请求
 type DeleteDocumentReq struct {
-	g.Meta `path:"/delete" method:"delete" tags:"文件管理" summary:"删除文件" dc:"删除文件"`
+	g.Meta `path:"/deleteDocument" method:"delete" tags:"文件管理" summary:"删除文件" dc:"删除文件"`
 
 	Id int64 `json:"id" v:"required#ID不能为空"`
 }
 
 // GetDocumentReq 获取文件请求
 type GetDocumentReq struct {
-	g.Meta `path:"/get" method:"get" tags:"文件管理" summary:"获取文件详情" dc:"获取文件详情"`
+	g.Meta `path:"/getDocument" method:"get" tags:"文件管理" summary:"获取文件详情" dc:"获取文件详情"`
 
-	Id        int64  `json:"id" v:"required#ID不能为空"`
+	Id int64 `json:"id" v:"required#ID不能为空"`
-	DatasetId int64  `json:"datasetId"`
-	Title     string `json:"title"`
-}
-
-type GetDocumentRes struct {
-	*DocumentVO
-	ImgAddressPrefix string `json:"imgAddressPrefix"`
 }
 
 // ListDocumentReq 文件列表请求
 type ListDocumentReq struct {
-	g.Meta `path:"/list" method:"get" tags:"文件管理" summary:"获取文件列表" dc:"分页查询文件列表，支持多条件筛选"`
+	g.Meta `path:"/listDocument" method:"get" tags:"文件管理" summary:"获取文件列表" dc:"分页查询文件列表，支持多条件筛选"`
 
 	Page      *beans.Page     `json:"page"`
 	DatasetId int64           `json:"datasetId"`
 	Keyword   string          `json:"keyword" dc:"关键词搜索"`
-	Title     string          `json:"title" dc:"文件标题"`
 	Status    document.Status `json:"status"`
 }
 
@@ -77,8 +68,6 @@ type DocumentVO struct {
 	Id           int64                 `json:"id,string" dc:"id"`
 	DatasetId    int64                 `json:"datasetId,string"`
 	Title        string                `json:"title" dc:"文件标题"`
-	Format       string                `orm:"format" json:"format" dc:"文件格式"`
-	FilePath     string                `orm:"file_path" json:"filePath" dc:"文件存储路径"`
 	Status       document.Status       `json:"status" dc:"状态1启用/0停用"`
 	VectorStatus document.VectorStatus `json:"vectorStatus" dc:"向量化状态 状态: 1 待定, 2 处理, 3 完成, 4 失败"`
 	ChunkCount   int64                 `json:"chunkCount" dc:"分块数"`
@@ -87,36 +76,33 @@ type DocumentVO struct {
 	UpdatedAt    *gtime.Time           `json:"updatedAt" dc:"更新时间"`
 }
 
-// DocumentVectorReq 处理文件请求(向量化)
+// ProcessDocumentReq 处理文件请求(向量化)
-type DocumentVectorReq struct {
+type ProcessDocumentReq struct {
-	g.Meta `path:"/vectorization" method:"post" tags:"文件管理" summary:"文件向量化处理" dc:"文件向量化处理"`
+	g.Meta `path:"/getProcess" method:"get" tags:"文件管理" summary:"文件向量化处理" dc:"文件向量化处理"`
 
 	Id        int64 `json:"id" v:"required#ID不能为空"`
 	DatasetId int64 `json:"datasetId" v:"required#数据集ID不能为空"`
 }
 
-type DocumentVectorRPC struct {
+// ProcessDocumentRes 处理文件响应
-	Id          int64           `json:"id" dc:"id"`
+type ProcessDocumentRes struct {
-	DatasetId   int64           `json:"datasetId" dc:"所属数据集ID"`
+	ChunkCount int64 `json:"chunkCount"`
-	DocumentId  int64           `json:"documentId" dc:"文件ID"`
+	CostTime   int64 `json:"costTime"`
-	ContentHash string          `json:"contentHash" dc:"内容hash"`
-	Vector      pgvector.Vector `json:"vector" dc:"向量"`
 }
 
-type VectorSemanticSplitReq struct {
+type ListDocumentChunkRPC struct {
-	g.Meta `path:"/vectorSemanticSplit" method:"post" tags:"文件管理" summary:"向量化生成" dc:"向量化生成"`
+	List []*DocumentChunkRPC `json:"list"`
-
-	Id int64 `json:"id" v:"required#ID不能为空"`
 }
 
-type SearchRecursiveSplitReq struct {
+type DocumentChunkRPC struct {
-	g.Meta `path:"/searchRecursiveSplit" method:"post" tags:"文件管理" summary:"全文检索生成" dc:"全文检索生成"`
+	Id          int64  `json:"id" dc:"id"`
-
+	DatasetId   int64  `json:"datasetId" dc:"所属数据集ID"`
-	Id int64 `json:"id" v:"required#ID不能为空"`
+	ContentHash string `json:"contentHash" dc:"内容hash"`
 }
 
-type KeywordExtractReq struct {
+type KnowledgeDocumentMsg struct {
-	g.Meta `path:"/keywordExtract" method:"post" tags:"文件管理" summary:"关键词提取" dc:"关键词提取"`
+	TenantId     uint64                `json:"tenantId"`
-
+	Creator      string                `json:"creator"`
-	Id int64 `json:"id" v:"required#ID不能为空"`
+	Id           int64                 `json:"id"`
+	VectorStatus document.VectorStatus `json:"vectorStatus"`
 }

model/dto/document_chunk.go

@@ -3,69 +3,38 @@ package dto
 import (
 	"rag/consts/document"
 
-	"gitea.redpowerfuture.com/red-future/common/beans"
+	"gitea.com/red-future/common/beans"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/os/gtime"
 	"github.com/pgvector/pgvector-go"
 )
 
-// RAGQueryReq RAG查询请求
+// UpdateDocumentChunkReq 更新文件块向量请求
-type RAGQueryReq struct {
+type UpdateDocumentChunkReq struct {
-	g.Meta `path:"/ragQuery" method:"post" tags:"RAG查询" summary:"执行RAG查询" dc:"执行RAG查询"`
+	g.Meta `path:"/updateDocumentChunk" method:"put" tags:"文件块向量管理" summary:"更新文件块" dc:"更新文件块"`
-
-	Content     string     `json:"content" v:"required#查询内容不能为空" dc:"用户问题"`
-	DatasetIds  []int64    `json:"datasetIds" dc:"数据集ID"`
-	DocumentIds []int64    `json:"documentIds" dc:"文档ID"`
-	History     []*Message `json:"history" dc:"历史对话"`
-	TopK        int        `json:"topK" d:"5" dc:"检索topK，默认5"`
-}
-
-type Message struct {
-	Role    string `json:"role"`
-	Content string `json:"content"`
-}
-
-// RAGQueryRes RAG查询响应
-type RAGQueryRes struct {
-	Answer string `json:"answer" dc:"生成的答案"`
-}
-
-// UpdateDocumentVectorReq 更新文件块向量请求
-type UpdateDocumentVectorReq struct {
-	g.Meta `path:"/update" method:"put" tags:"文件块向量管理" summary:"更新文件块" dc:"更新文件块"`
 
 	Id     int64           `json:"id" v:"required#ID不能为空"`
 	Status document.Status `json:"status"`
 }
 
-type DeleteDocumentVectorReq struct {
+// ListDocumentChunkReq 文件块向量列表请求
-	g.Meta `path:"/delete" method:"put" tags:"文件块向量管理" summary:"删除文件块" dc:"删除文件块"`
+type ListDocumentChunkReq struct {
-
+	g.Meta `path:"/listDocumentChunk" method:"get" tags:"文件块向量管理" summary:"获取文件块向量列表" dc:"分页查询文件块向量列表，支持多条件筛选"`
-	Id         int64 `json:"id"`
-	DocumentId int64 `json:"documentId"`
-}
-
-// ListDocumentVectorReq 文件块向量列表请求
-type ListDocumentVectorReq struct {
-	g.Meta `path:"/list" method:"get" tags:"文件块向量管理" summary:"获取文件块向量列表" dc:"分页查询文件块向量列表，支持多条件筛选"`
 
 	Page         *beans.Page           `json:"page"`
-	Keyword      string                `json:"keyword" dc:"关键词搜索"`
 	DatasetId    int64                 `json:"datasetId"`
 	DocumentId   int64                 `json:"documentId"`
-	DocumentIds  []int64               `json:"documentIds"`
-	ContentHashs []string              `json:"contentHash"`
 	Status       document.Status       `json:"status"`
 	VectorStatus document.VectorStatus `json:"vectorStatus"`
 }
 
-// ListDocumentVectorRes 文件块向量列表响应
+// ListDocumentChunkRes 文件块向量列表响应
-type ListDocumentVectorRes struct {
+type ListDocumentChunkRes struct {
-	List  []*DocumentVectorVO `json:"list"`
+	List  []*DocumentChunkItem `json:"list"`
-	Total int                 `json:"total"`
+	Total int                  `json:"total"`
 }
 
-type DocumentVectorVO struct {
+type DocumentChunkItem struct {
 	Id           int64                  `json:"id,string" dc:"id"`
 	Status       document.Status        `json:"status" dc:"状态"`
 	VectorStatus document.VectorStatus  `json:"vectorStatus" dc:"向量状态"`
@@ -80,7 +49,7 @@ type DocumentVectorVO struct {
 	UpdatedAt    *gtime.Time            `json:"updatedAt" dc:"更新时间"`
 }
 
-type VectorDocumentVectorMsg struct {
+type VectorDocumentChunkMsg struct {
 	TenantId     uint64                 `json:"tenantId"`
 	Creator      string                 `json:"creator"`
 	DatasetId    int64                  `json:"datasetId"`   // 数据集ID

model/dto/keyword.go

@@ -1,22 +1,19 @@
 package dto
 
 import (
-	"rag/consts/keyword"
+	"gitea.com/red-future/common/beans"
-
-	"gitea.redpowerfuture.com/red-future/common/beans"
 	"github.com/gogf/gf/v2/frame/g"
 	"github.com/gogf/gf/v2/os/gtime"
 )
 
 // CreateKeywordReq 创建关键词请求
 type CreateKeywordReq struct {
-	g.Meta `path:"/create" method:"post" tags:"关键词管理" summary:"创建关键词" dc:"创建关键词"`
+	g.Meta `path:"/createKeyword" method:"post" tags:"关键词管理" summary:"创建关键词" dc:"创建关键词"`
 
-	DatasetId   int64               `json:"datasetId" v:"required#数据集ID不能为空"`
+	DatasetId  int64  `json:"datasetId" v:"required#数据集ID不能为空"`
-	DocumentId  int64               `json:"documentId" v:"required#文档ID不能为空"`
+	DocumentId int64  `json:"documentId" v:"required#文档ID不能为空"`
-	Word        string              `json:"word" v:"required#名称不能为空"`
+	Word       string `json:"word" v:"required#名称不能为空"`
-	Weight      int16               `json:"weight" v:"required#权重不能为空"`
+	Weight     int16  `json:"weight" v:"required#权重不能为空"`
-	KeywordType keyword.KeywordType `json:"keywordType" v:"required#类型不能为空"`
 }
 
 // CreateKeywordRes 创建关键词响应
@@ -26,7 +23,7 @@ type CreateKeywordRes struct {
 
 // UpdateKeywordReq 更新关键词请求
 type UpdateKeywordReq struct {
-	g.Meta `path:"/update" method:"put" tags:"关键词管理" summary:"更新关键词" dc:"更新关键词"`
+	g.Meta `path:"/updateKeyword" method:"put" tags:"关键词管理" summary:"更新关键词" dc:"更新关键词"`
 
 	Id     int64  `json:"id" v:"required#ID不能为空"`
 	Word   string `json:"word"`
@@ -35,30 +32,27 @@ type UpdateKeywordReq struct {
 
 // DeleteKeywordReq 删除关键词请求
 type DeleteKeywordReq struct {
-	g.Meta `path:"/delete" method:"delete" tags:"关键词管理" summary:"删除关键词" dc:"删除关键词"`
+	g.Meta `path:"/deleteKeyword" method:"delete" tags:"关键词管理" summary:"删除关键词" dc:"删除关键词"`
 
-	Id         int64 `json:"id"`
+	Id int64 `json:"id" v:"required#ID不能为空"`
-	DocumentId int64 `json:"documentId"`
 }
 
 // GetKeywordReq 获取关键词请求
 type GetKeywordReq struct {
-	g.Meta `path:"/get" method:"get" tags:"关键词管理" summary:"获取关键词详情" dc:"获取关键词详情"`
+	g.Meta `path:"/getKeyword" method:"get" tags:"关键词管理" summary:"获取关键词详情" dc:"获取关键词详情"`
 
 	Id int64 `json:"id" v:"required#ID不能为空"`
 }
 
 // ListKeywordReq 关键词列表请求
 type ListKeywordReq struct {
-	g.Meta `path:"/list" method:"get" tags:"关键词管理" summary:"获取关键词列表" dc:"分页查询关键词列表，支持多条件筛选"`
+	g.Meta `path:"/listKeyword" method:"get" tags:"关键词管理" summary:"获取关键词列表" dc:"分页查询关键词列表，支持多条件筛选"`
 
-	Page        *beans.Page         `json:"page"`
+	Page       *beans.Page `json:"page"`
-	DatasetId   int64               `json:"datasetId"`
+	DatasetId  int64       `json:"datasetId"`
-	DocumentId  int64               `json:"documentId"`
+	DocumentId int64       `json:"documentId"`
-	Word        string              `json:"word"`
+	Word       string      `json:"word"`
-	Words       []string            `json:"words"`
+	Keyword    string      `json:"keyword" dc:"关键词搜索"`
-	Keyword     string              `json:"keyword" dc:"关键词搜索"`
-	KeywordType keyword.KeywordType `json:"keywordType"`
 }
 
 // ListKeywordRes 关键词列表响应
@@ -68,12 +62,9 @@ type ListKeywordRes struct {
 }
 
 type KeywordVO struct {
-	Id          int64               `json:"id,string" dc:"id"`
+	Id        int64       `json:"id,string" dc:"id"`
-	Word        string              `json:"word" dc:"关键词名称"`
+	Word      string      `json:"word" dc:"关键词名称"`
-	Weight      int16               `json:"weight" dc:"权重"`
+	Weight    int16       `json:"weight" dc:"权重"`
-	KeywordType keyword.KeywordType `json:"keywordType" dc:"类型"`
+	CreatedAt *gtime.Time `json:"createdAt" dc:"创建时间"`
-	DatasetId   int64               `json:"datasetId,string" dc:"数据集ID"`
+	UpdatedAt *gtime.Time `json:"updatedAt" dc:"更新时间"`
-	DocumentId  int64               `json:"documentId,string" dc:"文档ID"`
-	CreatedAt   *gtime.Time         `json:"createdAt" dc:"创建时间"`
-	UpdatedAt   *gtime.Time         `json:"updatedAt" dc:"更新时间"`
 }

model/dto/model.go

@@ -1,114 +0,0 @@
-package dto
-
-import (
-	"rag/consts/model"
-	"time"
-
-	"gitea.redpowerfuture.com/red-future/common/beans"
-	"github.com/gogf/gf/v2/frame/g"
-)
-
-type GetModelAllEnumsReq struct {
-	g.Meta `path:"/getAllEnums" method:"get" tags:"模型配置管理" summary:"获取全量模型枚举（类型+配置）"`
-}
-
-type GetModelEnumRes struct {
-	Options []ModelEnumOption `json:"options"`
-}
-
-// ModelEnumOption 主类型：模型类型（vector/chat）
-type ModelEnumOption struct {
-	Key         interface{}     `json:"key"`
-	Value       interface{}     `json:"value"`
-	ConfigTypes []ModelKeyValue `json:"configTypes"` // 这里统一！
-}
-
-// ModelKeyValue 统一的 KV 结构 → 给模型类型 + 配置类型共用
-type ModelKeyValue struct {
-	Key   interface{} `json:"key"`
-	Value interface{} `json:"value"`
-}
-
-// GetModelConfigFormFieldsReq 获取模型配置表单请求
-type GetModelConfigFormFieldsReq struct {
-	g.Meta `path:"/getModelFormField" method:"get" tags:"模型配置管理" summary:"获取模型表单" dc:"获取模型表单列表"`
-
-	ModelType  model.ModelType       `json:"modelType"`  // 模型类型 vector/chat
-	ConfigType model.ModelConfigType `json:"configType"` // 配置类型 ark/ollama/openai...
-}
-
-// GetModelConfigFormFieldsRes 获取模型配置表单响应
-type GetModelConfigFormFieldsRes struct {
-	ModelType  model.ModelType          `json:"modelType"`
-	ConfigType model.ModelConfigType    `json:"configType"`
-	Fields     []map[string]interface{} `json:"fields"`
-}
-
-// CreateModelReq 创建模型请求
-type CreateModelReq struct {
-	g.Meta `path:"/create" method:"post" tags:"模型配置管理" summary:"创建模型配置" dc:"创建模型配置"`
-
-	ModelType     model.ModelType        `json:"modelType" v:"required#模型类型不能为空"`
-	ModelName     string                 `json:"modelName" v:"required#模型名称不能为空"`
-	ModelDesc     string                 `json:"modelDesc"`
-	ConfigType    model.ModelConfigType  `json:"configType"`
-	ConfigContent map[string]interface{} `json:"configContent"`
-}
-
-// CreateModelRes 创建模型响应
-type CreateModelRes struct {
-	Id int64 `json:"id,string"`
-}
-
-// UpdateModelReq 更新模型请求
-type UpdateModelReq struct {
-	g.Meta `path:"/update" method:"put" tags:"模型配置管理" summary:"更新模型配置" dc:"更新模型配置"`
-
-	Id            int64                  `json:"id" v:"required#ID不能为空"`
-	ModelType     model.ModelType        `json:"modelType"`
-	ModelName     string                 `json:"modelName"`
-	ModelDesc     string                 `json:"modelDesc"`
-	ConfigType    model.ModelConfigType  `json:"configType"`
-	ConfigContent map[string]interface{} `json:"configContent"`
-}
-
-// DeleteModelReq 删除模型请求
-type DeleteModelReq struct {
-	g.Meta `path:"/delete" method:"delete" tags:"模型配置管理" summary:"删除模型配置" dc:"删除模型配置"`
-
-	Id int64 `json:"id" v:"required#ID不能为空"`
-}
-
-// GetModelReq 获取模型请求
-type GetModelReq struct {
-	g.Meta `path:"/get" method:"get" tags:"模型配置管理" summary:"获取模型配置详情" dc:"获取模型配置详情"`
-
-	Id        int64           `json:"id"`
-	ModelType model.ModelType `json:"modelType"`
-}
-
-// ListModelReq 获取模型列表请求
-type ListModelReq struct {
-	g.Meta `path:"/list" method:"get" tags:"模型配置管理" summary:"获取模型配置列表" dc:"分页查询模型配置列表，支持多条件筛选"`
-
-	Page      *beans.Page     `json:"page"`
-	ModelType model.ModelType `json:"modelType"`
-	ModelName string          `json:"modelName"`
-}
-
-// ListModelRes 获取模型列表响应
-type ListModelRes struct {
-	List  []*ModelVO `json:"list"`
-	Total int        `json:"total"`
-}
-
-type ModelVO struct {
-	Id            int64                  `json:"id,string"`
-	ModelType     model.ModelType        `json:"modelType"`
-	ModelName     string                 `json:"modelName"`
-	ModelDesc     string                 `json:"modelDesc"`
-	ConfigType    model.ModelConfigType  `json:"configType"`
-	ConfigContent map[string]interface{} `json:"configContent"`
-	CreateTime    time.Time              `json:"createTime"`
-	UpdateTime    time.Time              `json:"updateTime"`
-}

model/dto/task.go

@@ -1,75 +0,0 @@
-package dto
-
-import (
-	"rag/consts/task"
-
-	"github.com/gogf/gf/v2/frame/g"
-)
-
-// WriteTaskProgressReq 写入任务进度请求
-type WriteTaskProgressReq struct {
-	TaskType task.TaskType   `json:"taskType" dc:"任务类型"`
-	Status   task.TaskStatus `json:"status" dc:"任务状态"`
-	TaskId   int64           `json:"taskId" dc:"任务ID"`
-	Remark   string          `json:"remark" dc:"备注"`
-}
-
-// CreateTaskReq 创建任务请求
-type CreateTaskReq struct {
-	TaskType task.TaskType   `json:"taskType" dc:"任务类型"`
-	Status   task.TaskStatus `json:"status" dc:"任务状态"`
-	TaskId   int64           `json:"taskId" dc:"任务ID"`
-	Remark   string          `json:"remark" dc:"备注"`
-}
-
-// UpdateTaskReq 更新任务请求
-type UpdateTaskReq struct {
-	Id     int64           `json:"id" dc:"任务ID"`
-	TaskId int64           `json:"taskId" dc:"任务ID"`
-	Status task.TaskStatus `json:"status" dc:"任务状态"`
-	Remark string          `json:"remark" dc:"备注"`
-}
-
-// DeleteTaskByTaskIdReq 删除任务请求
-type DeleteTaskByTaskIdReq struct {
-	TaskId int64 `json:"taskId" v:"required#任务id不能为空"`
-}
-
-// GetTaskReq 获取任务请求
-type GetTaskReq struct {
-	g.Meta `path:"/get" method:"get" tags:"任务管理" summary:"获取任务详情" dc:"获取任务详情"`
-
-	Id         int64           `json:"id" dc:"任务ID"`
-	TaskId     int64           `json:"taskId" dc:"任务ID"`
-	TaskType   task.TaskType   `json:"taskType" dc:"任务类型"`
-	TaskStatus task.TaskStatus `json:"taskStatus" dc:"任务状态"`
-}
-
-type ListTaskRes struct {
-	List  []*TaskVO `json:"list"`
-	Total int       `json:"total"`
-}
-
-// TaskVO 任务视图对象
-type TaskVO struct {
-	Id             int64             `json:"id" dc:"任务ID"`
-	TaskType       task.TaskType     `json:"taskType" dc:"任务类型"`
-	Status         task.TaskStatus   `json:"status" dc:"任务状态"`
-	Priority       task.TaskPriority `json:"priority" dc:"任务优先级"`
-	ParentTaskID   int64             `json:"parentTaskId" dc:"父任务ID"`
-	TotalItems     int64             `json:"totalItems" dc:"总项数"`
-	ProcessedItems int64             `json:"processedItems" dc:"已处理项数"`
-	Progress       float64           `json:"progress" dc:"进度百分比"`
-	StartTime      *int64            `json:"startTime" dc:"开始时间戳"`
-	EndTime        *int64            `json:"endTime" dc:"结束时间戳"`
-	Duration       int64             `json:"duration" dc:"耗时(毫秒)"`
-	SuccessCount   int64             `json:"successCount" dc:"成功数"`
-	FailCount      int64             `json:"failCount" dc:"失败数"`
-	Executor       string            `json:"executor" dc:"执行器"`
-	DocumentID     int64             `json:"documentId" dc:"文档ID"`
-	Remark         string            `json:"remark" dc:"备注"`
-	Creator        string            `json:"creator" dc:"创建人"`
-	CreatedAt      int64             `json:"createdAt" dc:"创建时间"`
-	Updater        string            `json:"updater" dc:"更新人"`
-	UpdatedAt      int64             `json:"updatedAt" dc:"更新时间"`
-}

model/entity/dataset.go

@@ -1,7 +1,7 @@
 package entity
 
 import (
-	"gitea.redpowerfuture.com/red-future/common/beans"
+	"gitea.com/red-future/common/beans"
 )
 
 type datasetCol struct {

model/entity/dataset_index.go

@@ -1,6 +1,6 @@
 package entity
 
-import "gitea.redpowerfuture.com/red-future/common/beans"
+import "gitea.com/red-future/common/beans"
 
 type datasetIndexCol struct {
 	beans.SQLBaseCol

model/entity/document.go

@@ -1,7 +1,7 @@
 package entity
 
 import (
-	"gitea.redpowerfuture.com/red-future/common/beans"
+	"gitea.com/red-future/common/beans"
 
 	"rag/consts/document"
 )

model/entity/document_chunk.go

@@ -3,11 +3,11 @@ package entity
 import (
 	"rag/consts/document"
 
-	"gitea.redpowerfuture.com/red-future/common/beans"
+	"gitea.com/red-future/common/beans"
 	"github.com/pgvector/pgvector-go"
 )
 
-type documentVectorCol struct {
+type documentChunkCol struct {
 	beans.SQLBaseCol
 	Status       string
 	VectorStatus string
@@ -20,7 +20,7 @@ type documentVectorCol struct {
 	Metadata     string
 }
 
-var DocumentVectorCol = documentVectorCol{
+var DocumentChunkCol = documentChunkCol{
 	SQLBaseCol:   beans.DefSQLBaseCol,
 	Status:       "status",
 	VectorStatus: "vector_status",
@@ -33,8 +33,8 @@ var DocumentVectorCol = documentVectorCol{
 	Metadata:     "metadata",
 }
 
-// DocumentVector 文档切分块实体
+// DocumentChunk 文档切分块实体
-type DocumentVector struct {
+type DocumentChunk struct {
 	beans.SQLBaseDO `orm:",inline"`
 
 	Status       document.Status        `orm:"status" json:"status" dc:"状态"`

model/entity/keyword.go

@@ -1,34 +1,27 @@
 package entity
 
-import (
+import "gitea.com/red-future/common/beans"
-	"rag/consts/keyword"
-
-	"gitea.redpowerfuture.com/red-future/common/beans"
-)
 
 type keywordCol struct {
 	beans.SQLBaseCol
-	DatasetId   string
+	DatasetId  string
-	DocumentId  string
+	DocumentId string
-	Word        string
+	Word       string
-	Weight      string
+	Weight     string
-	KeywordType string
 }
 
 var KeywordCol = keywordCol{
-	SQLBaseCol:  beans.DefSQLBaseCol,
+	SQLBaseCol: beans.DefSQLBaseCol,
-	DatasetId:   "dataset_id",
+	DatasetId:  "dataset_id",
-	DocumentId:  "document_id",
+	DocumentId: "document_id",
-	Word:        "word",
+	Word:       "word",
-	Weight:      "weight",
+	Weight:     "weight",
-	KeywordType: "keyword_type",
 }
 
 type Keyword struct {
 	beans.SQLBaseDO `orm:",inline"`
-	DatasetId       int64               `orm:"dataset_id" json:"datasetId" dc:"数据集ID"`
+	DatasetId       int64  `orm:"dataset_id" json:"datasetId" dc:"数据集ID"`
-	DocumentId      int64               `orm:"document_id" json:"documentId" dc:"文件ID"`
+	DocumentId      int64  `orm:"document_id" json:"documentId" dc:"文件ID"`
-	Word            string              `orm:"word" json:"word" dc:"关键词"`
+	Word            string `orm:"word" json:"word" dc:"关键词"`
-	Weight          int16               `orm:"weight" json:"weight" dc:"权重"`
+	Weight          int16  `orm:"weight" json:"weight" dc:"权重"`
-	KeywordType     keyword.KeywordType `orm:"keyword_type" json:"keywordType" dc:"类型"`
 }

model/entity/model.go

@@ -1,119 +0,0 @@
-package entity
-
-import (
-	"rag/consts/model"
-
-	"gitea.redpowerfuture.com/red-future/common/beans"
-)
-
-type modelCol struct {
-	beans.SQLBaseCol
-	DatasetId     string
-	ModelType     string
-	ModelName     string
-	ModelDesc     string
-	ConfigType    string
-	ConfigContent string
-}
-
-var ModelCol = modelCol{
-	SQLBaseCol:    beans.DefSQLBaseCol,
-	DatasetId:     "dataset_id",
-	ModelType:     "model_type",
-	ModelName:     "model_name",
-	ModelDesc:     "model_desc",
-	ConfigType:    "config_type",
-	ConfigContent: "config_content",
-}
-
-type Model struct {
-	beans.SQLBaseDO `orm:",inline"`
-	DatasetId       int64                  `orm:"dataset_id" json:"datasetId" dc:"数据集ID"`
-	ModelType       model.ModelType        `orm:"model_type" json:"modelType" dc:"模型类型"` // 向量/对话
-	ModelName       string                 `orm:"model_name" json:"modelName" dc:"模型名称"`
-	ModelDesc       string                 `orm:"model_desc" json:"modelDesc" dc:"模型描述"`
-	ConfigType      model.ModelConfigType  `orm:"config_type" json:"configType" dc:"配置类型"`       // ark/ollama等
-	ConfigContent   map[string]interface{} `orm:"config_content" json:"configContent" dc:"配置详情"` // 存JSON
-}
-
-// -------------------------- 通用配置结构体（抽离重复字段）--------------------------
-
-// OllamaConfig 通用配置（向量/对话完全一致）
-type OllamaConfig struct {
-	BaseURL string `json:"base_url"`
-	Model   string `json:"model"`
-}
-
-// OpenAIConfig 通用配置
-type OpenAIConfig struct {
-	APIKey     string `json:"api_key"`
-	Model      string `json:"model"`
-	ByAzure    bool   `json:"by_azure"`
-	BaseURL    string `json:"base_url"`
-	APIVersion string `json:"api_version"`
-}
-
-// QianfanConfig 千帆通用配置
-type QianfanConfig struct {
-	AccessKey string `json:"access_key"`
-	SecretKey string `json:"secret_key"`
-	Model     string `json:"model"`
-}
-
-// ArkConfig 通用配置
-type ArkConfig struct {
-	APIKey string `json:"api_key"`
-	Model  string `json:"model"`
-}
-
-// -------------------------- 向量模型配置 --------------------------
-
-type VectorModelConfigOllama = OllamaConfig   // 直接复用
-type VectorModelConfigOpenAI = OpenAIConfig   // 直接复用
-type VectorModelConfigQianfan = QianfanConfig // 直接复用
-
-type VectorModelConfigArk struct {
-	ArkConfig
-	APIType string `json:"api_type"`
-}
-
-type VectorModelConfigTencentCloud struct {
-	SecretID  string `json:"secret_id"`
-	SecretKey string `json:"secret_key"`
-	Region    string `json:"region"`
-}
-
-type VectorModelConfigDashScope struct {
-	APIKey string `json:"api_key"`
-	Model  string `json:"model"`
-}
-
-// -------------------------- 对话模型配置 --------------------------
-
-type ChatModelConfigArk = ArkConfig         // 直接复用
-type ChatModelConfigArkBot = ArkConfig      // 直接复用
-type ChatModelConfigOllama = OllamaConfig   // 直接复用
-type ChatModelConfigOpenAI = OpenAIConfig   // 直接复用
-type ChatModelConfigQianfan = QianfanConfig // 直接复用
-
-type ChatModelConfigClaude struct {
-	ByBedrock       bool   `json:"by_bedrock"`
-	AccessKey       string `json:"access_key"`
-	SecretAccessKey string `json:"secret_access_key"`
-	Region          string `json:"region"`
-	APIKey          string `json:"api_key"`
-	Model           string `json:"model"`
-	BaseURL         string `json:"base_url"`
-}
-
-type ChatModelConfigDeepSeek struct {
-	APIKey  string `json:"api_key"`
-	Model   string `json:"model"`
-	BaseURL string `json:"base_url"`
-}
-
-type ChatModelConfigQwen struct {
-	APIKey  string `json:"api_key"`
-	Model   string `json:"model"`
-	BaseURL string `json:"base_url"`
-}

model/entity/task.go

@@ -1,66 +0,0 @@
-package entity
-
-import (
-	"rag/consts/task"
-
-	"gitea.redpowerfuture.com/red-future/common/beans"
-)
-
-type taskCol struct {
-	beans.SQLBaseCol
-	TaskId   string
-	TaskType string
-	Status   string
-	Executor string
-	Remark   string
-	//Priority       string
-	//ParentTaskId   string
-	//TotalItems     string
-	//ProcessedItems string
-	//Progress       string
-	//StartTime      string
-	//EndTime        string
-	//Duration       string
-	//SuccessCount   string
-	//FailCount      string
-}
-
-var TaskCol = taskCol{
-	SQLBaseCol: beans.DefSQLBaseCol,
-	TaskId:     "task_id",
-	TaskType:   "task_type",
-	Status:     "status",
-	Executor:   "executor",
-	Remark:     "remark",
-	//Priority:       "priority",
-	//ParentTaskId:   "parent_task_id",
-	//TotalItems:     "total_items",
-	//ProcessedItems: "processed_items",
-	//Progress:       "progress",
-	//StartTime:      "start_time",
-	//EndTime:        "end_time",
-	//Duration:       "duration",
-	//SuccessCount:   "success_count",
-	//FailCount:      "fail_count",
-}
-
-// Task 任务记录表
-type Task struct {
-	beans.SQLBaseDO `orm:",inline"`
-
-	TaskId   int64           `orm:"task_id" json:"taskId" dc:"任务ID"`
-	TaskType task.TaskType   `orm:"task_type" json:"taskType" dc:"任务类型"`
-	Status   task.TaskStatus `orm:"status" json:"status" dc:"任务状态"`
-	Executor string          `orm:"executor" json:"executor" dc:"执行器"`
-	Remark   string          `orm:"remark" json:"remark" dc:"备注"`
-	//Priority       task.TaskPriority `orm:"priority" json:"priority" dc:"任务优先级"`
-	//ParentTaskId   int64             `orm:"parent_task_id" json:"parentTaskId" dc:"父任务ID"`
-	//TotalItems     int64             `orm:"total_items" json:"totalItems" dc:"总项数"`
-	//ProcessedItems int64             `orm:"processed_items" json:"processedItems" dc:"已处理项数"`
-	//SuccessCount   int64             `orm:"success_count" json:"successCount" dc:"成功数"`
-	//FailCount      int64             `orm:"fail_count" json:"failCount" dc:"失败数"`
-	//Progress       float64           `orm:"progress" json:"progress" dc:"进度百分比"`
-	//StartTime      *gtime.Time       `orm:"start_time" json:"startTime" dc:"开始时间戳"`
-	//EndTime        *gtime.Time       `orm:"end_time" json:"endTime" dc:"结束时间戳"`
-	//Duration       int64             `orm:"duration" json:"duration" dc:"耗时(毫秒)"`
-}

service/dataset.go

@@ -45,3 +45,43 @@ func (s *datasetService) List(ctx context.Context, req *dto.ListDatasetReq) (res
 	err = gconv.Struct(list, &res.List)
 	return
 }
+
+//// Search 搜索(示例,实际需要调用向量库)
+//func (s *datasetService) Search(ctx context.Context, req *dto.SearchReq) (res *dto.SearchRes, err error) {
+//	// 1. 获取数据集信息
+//	kb, err := dao.Dataset.GetByID(ctx, req)
+//	if err != nil {
+//		return nil, err
+//	}
+//
+//	// 2. 获取文件块
+//	chunks, err := dao.Chunk.FindChunksByKBIDWithLimit(ctx, req.KBID, 0, req.TopK)
+//	if err != nil {
+//		return nil, err
+//	}
+//
+//	// 3. TODO: 使用向量检索(需要集成向量库)
+//	// 暂时使用简单的关键词匹配
+//	results := make([]dto.SearchResult, 0)
+//	for _, chunk := range chunks {
+//		results = append(results, dto.SearchResult{
+//			Content:    chunk.Content,
+//			Score:      0.8, // TODO: 计算实际向量相似度
+//			DocumentID: chunk.DocumentID,
+//			ChunkIndex: chunk.Index,
+//		})
+//	}
+//
+//	g.Log().Infof(ctx, "数据集[%s]搜索完成,查询:%s,结果数:%d", kb.Name, req.Query, len(results))
+//
+//	return &dto.SearchRes{Results: results}, nil
+//}
+//
+//// formatChunks 格式化文件块为上下文
+//func (s *datasetService) formatChunks(chunks []*entity.DocumentChunk) string {
+//	var sb strings.Builder
+//	for i, chunk := range chunks {
+//		sb.WriteString(fmt.Sprintf("[%d] %s\n\n", i+1, chunk.Content))
+//	}
+//	return sb.String()
+//}

service/dataset_index.go

@@ -0,0 +1,5 @@
+package service
+
+var DatasetIndex = new(datasetIndexService)
+
+type datasetIndexService struct{}

service/document.go

@@ -2,31 +2,32 @@ package service
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"rag/common/eino"
 	"rag/consts/document"
-	"rag/consts/keyword"
-	"rag/consts/model"
 	"rag/consts/public"
-	"rag/consts/task"
 	"rag/dao"
 	"rag/model/dto"
 	"rag/model/entity"
 	"strings"
+	"sync"
 	"time"
 
-	"gitea.redpowerfuture.com/red-future/common/db/gfdb"
+	"gitea.com/red-future/common/beans"
-	"gitea.redpowerfuture.com/red-future/common/full-text-search/meilisearch"
+	"gitea.com/red-future/common/db/gfdb"
-	"gitea.redpowerfuture.com/red-future/common/utils"
+	"gitea.com/red-future/common/full-text-search/meilisearch"
+	"gitea.com/red-future/common/http"
+	"gitea.com/red-future/common/rag/eino"
+	"gitea.com/red-future/common/rag/gse"
+	"gitea.com/red-future/common/utils"
 	gmq "github.com/bjang03/gmq/core/gmq"
 	"github.com/bjang03/gmq/mq"
 	"github.com/bjang03/gmq/types"
 	"github.com/cloudwego/eino/schema"
+	"github.com/gogf/gf/v2/container/gvar"
 	"github.com/gogf/gf/v2/crypto/gmd5"
 	"github.com/gogf/gf/v2/database/gdb"
+	"github.com/gogf/gf/v2/database/gredis"
 	"github.com/gogf/gf/v2/frame/g"
-	"github.com/gogf/gf/v2/os/grpool"
 	"github.com/gogf/gf/v2/util/gconv"
 )
 
@@ -36,35 +37,7 @@ type documentService struct{}
 
 // Create 创建文件
 func (s *documentService) Create(ctx context.Context, req *dto.CreateDocumentReq) (res *dto.CreateDocumentRes, err error) {
-	err = gfdb.DB(ctx, public.DbNameKnowledge).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) {
+	err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) {
-		doc, err := dao.Document.Get(ctx, &dto.GetDocumentReq{
-			DatasetId: req.DatasetId,
-			Title:     req.Title,
-		})
-		if err != nil {
-			return
-		}
-		if !g.IsEmpty(doc) && doc.Id > 0 {
-			_, err = dao.Keyword.Delete(ctx, &dto.DeleteKeywordReq{
-				DocumentId: doc.Id,
-			})
-			if err != nil {
-				return err
-			}
-			_, err = dao.DocumentVector.Delete(ctx, &dto.DeleteDocumentVectorReq{
-				DocumentId: doc.Id,
-			})
-			if err != nil {
-				return err
-			}
-			_, err = dao.Document.Delete(ctx, &dto.DeleteDocumentReq{
-				Id: doc.Id,
-			})
-			if err != nil {
-				return err
-			}
-		}
-
 		var id int64
 		id, err = dao.Document.Insert(ctx, req)
 		if err != nil {
@@ -80,16 +53,7 @@ func (s *documentService) Create(ctx context.Context, req *dto.CreateDocumentReq
 			return
 		}
 		res = &dto.CreateDocumentRes{Id: id}
-		// 写入任务进度进行中 任务类型为文档解析
+
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   id,
-			TaskType: task.TaskTypeDocParse,
-			Status:   task.TaskStatusCompleted,
-			Remark:   "文档上传完成",
-		})
-		if err != nil {
-			return
-		}
 		return
 	})
 
@@ -104,42 +68,21 @@ func (s *documentService) Update(ctx context.Context, req *dto.UpdateDocumentReq
 
 // Delete 删除文件
 func (s *documentService) Delete(ctx context.Context, req *dto.DeleteDocumentReq) (err error) {
-	docs, err := dao.Document.Get(ctx, &dto.GetDocumentReq{Id: req.Id})
+	docs, err := dao.Document.GetByID(ctx, &dto.GetDocumentReq{Id: req.Id})
 	if err != nil {
 		return
 	}
-	err = gfdb.DB(ctx, public.DbNameKnowledge).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) {
+	err = gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) (err error) {
 		datasetReq := &dto.UpdateDatasetReq{
 			Id:            docs.DatasetId,
 			DocumentCount: -1,
 			DocumentSize:  -docs.FileSize,
 		}
-		if _, err = dao.Dataset.Update(ctx, datasetReq); err != nil {
+		_, err = dao.Dataset.Update(ctx, datasetReq)
+		if err != nil {
 			return
 		}
-
+		_, err = dao.Document.Delete(ctx, req)
-		if _, err = dao.Document.Delete(ctx, req); err != nil {
-			return
-		}
-
-		if _, err = dao.Keyword.Delete(ctx, &dto.DeleteKeywordReq{
-			DocumentId: docs.Id,
-		}); err != nil {
-			return err
-		}
-
-		if _, err = dao.DocumentVector.Delete(ctx, &dto.DeleteDocumentVectorReq{
-			DocumentId: docs.Id,
-		}); err != nil {
-			return err
-		}
-
-		if _, err = dao.Task.DeleteByTaskId(ctx, &dto.DeleteTaskByTaskIdReq{
-			TaskId: docs.Id,
-		}); err != nil {
-			return
-		}
-
 		return
 	})
 
@@ -147,17 +90,9 @@ func (s *documentService) Delete(ctx context.Context, req *dto.DeleteDocumentReq
 }
 
 // Get 获取文件详情
-func (s *documentService) Get(ctx context.Context, req *dto.GetDocumentReq) (res *dto.GetDocumentRes, err error) {
+func (s *documentService) Get(ctx context.Context, req *dto.GetDocumentReq) (res *dto.DocumentVO, err error) {
-	r, err := dao.Document.Get(ctx, req)
+	r, err := dao.Document.GetByID(ctx, req)
-	if err != nil {
+	err = gconv.Struct(r, &res)
-		return
-	}
-	res = &dto.GetDocumentRes{}
-	err = gconv.Struct(r, &res.DocumentVO)
-	if err != nil {
-		return
-	}
-	res.ImgAddressPrefix, err = utils.GetFileAddressPrefix(ctx)
 	return
 }
 
@@ -171,516 +106,232 @@ func (s *documentService) List(ctx context.Context, req *dto.ListDocumentReq) (r
 		Total: total,
 	}
 	err = gconv.Struct(list, &res.List)
+
+	//eino.TestIndexer()
+	//eino.TestRetriever()
+
 	return
 }
 
-func (s *documentService) VectorSemanticSplit(ctx context.Context, req *dto.VectorSemanticSplitReq) (err error) {
+// Process 处理文件(使用eino框架切分和向量化)
+func (s *documentService) Process(ctx context.Context, req *dto.ProcessDocumentReq) (res *dto.ProcessDocumentRes, err error) {
+	startTime := time.Now()
+
 	// 1. 查询文件信息
 	documentReq := dto.GetDocumentReq{Id: req.Id}
-	doc, err := dao.Document.Get(ctx, &documentReq)
+	doc, err := dao.Document.GetByID(ctx, &documentReq)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	if g.IsEmpty(doc) {
-		return errors.New("document not found")
-	}
-	err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-		TaskId:   req.Id,
-		TaskType: task.TaskTypeGenerateVector,
-		Status:   task.TaskStatusRunning,
-		Remark:   "向量化执行中",
-	})
-	return s.semanticSplitDocument(ctx, doc)
-}
 
-func (s *documentService) SearchRecursiveSplit(ctx context.Context, req *dto.SearchRecursiveSplitReq) (err error) {
+	// 2. 使用eino框架进行文件切分（并发执行）
-	// 1. 查询文件信息
+	var vectorDocsCount, chunks int64
-	documentReq := dto.GetDocumentReq{Id: req.Id}
+	// 用 gopool 或者简单的错误等待，绝对不用裸 goroutine
-	doc, err := dao.Document.Get(ctx, &documentReq)
+	var err1, err2, err3 error
-	if err != nil {
+	var wg sync.WaitGroup
-		return err
+	wg.Add(3)
-	}
-	if g.IsEmpty(doc) {
-		return errors.New("document not found")
-	}
-	err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-		TaskId:   req.Id,
-		TaskType: task.TaskTypeFullTextSearch,
-		Status:   task.TaskStatusRunning,
-		Remark:   "全文检索执行中",
-	})
-	return s.recursiveSplitDocument(ctx, doc)
-}
 
-func (s *documentService) KeywordExtract(ctx context.Context, req *dto.KeywordExtractReq) (err error) {
+	// 任务1
-	// 1. 查询文件信息
+	go func() {
-	documentReq := dto.GetDocumentReq{Id: req.Id}
+		defer wg.Done()
-	doc, err := dao.Document.Get(ctx, &documentReq)
+		vectorDocsCount, chunks, err1 = s.sqlSplitDocument(ctx, doc)
-	if err != nil {
+	}()
-		return err
+
-	}
+	// 任务2
-	if g.IsEmpty(doc) {
+	go func() {
-		return errors.New("document not found")
+		defer wg.Done()
-	}
+		err2 = s.esSplitDocument(ctx, doc)
-	err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
+	}()
-		TaskId:   req.Id,
+
-		TaskType: task.TaskTypeExtractKeywords,
+	// 任务3
-		Status:   task.TaskStatusRunning,
+	go func() {
-		Remark:   "提取关键词执行中",
+		defer wg.Done()
-	})
+		err3 = s.extractDocument(ctx, doc)
-	return s.extractDocument(ctx, doc)
+	}()
-}
+
+	// 直接等待，不使用通道，避免泄漏
+	wg.Wait()
 
-// Vector 处理文件(使用eino框架切分和向量化)
-func (s *documentService) Vector(ctx context.Context, req *dto.DocumentVectorReq) (err error) {
-	// 更新文档状态为处理中
 	updateDocumentReq := new(dto.UpdateDocumentReq)
 	updateDocumentReq.Id = req.Id
-	updateDocumentReq.VectorStatus = document.VectorStatusProcessing.Code()
+
+	// 统一判断错误
+	if err1 != nil || err2 != nil || err3 != nil {
+		// 更新文档状态
+		updateDocumentReq.VectorStatus = document.VectorStatusFailed.Code()
+		if _, err = dao.Document.Update(ctx, updateDocumentReq); err != nil {
+			return nil, err
+		}
+		if err1 != nil {
+			return nil, err1
+		}
+		if err2 != nil {
+			return nil, err2
+		}
+		return nil, err3
+	}
+
+	// 4. 更新文件状态为处理中和切分数量
+	if vectorDocsCount > 0 {
+		updateDocumentReq.VectorStatus = document.VectorStatusProcessing.Code()
+	} else {
+		updateDocumentReq.VectorStatus = document.VectorStatusCompleted.Code()
+	}
+	updateDocumentReq.ChunkCount = chunks
 	if _, err = dao.Document.Update(ctx, updateDocumentReq); err != nil {
-		// 写入任务进度失败 任务类型为文档解析
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   req.Id,
-			TaskType: task.TaskTypeDocParse,
-			Status:   task.TaskStatusFailed,
-			Remark:   "更新文档状态失败: " + err.Error(),
-		})
 		return
 	}
 
-	user, err := utils.GetUserInfo(ctx)
+	costTime := time.Since(startTime).Milliseconds()
-	if err != nil {
-		return err
-	}
-	// 使用带超时的background context，避免HTTP请求完成后context被取消
-	taskCtx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
-	taskCtx = context.WithValue(taskCtx, "user", user)
-	// 任务1: 语义 切分文档
-	grpool.Add(taskCtx, func(ctx context.Context) {
-		g.TryCatch(ctx, func(ctx context.Context) {
-			if innerErr := s.VectorSemanticSplit(ctx, &dto.VectorSemanticSplitReq{Id: req.Id}); innerErr != nil {
-				cancel()
-			}
-		}, func(ctx context.Context, err error) {
-			cancel()
-		})
-	})
 
-	// 任务2: 递归 切分文档
+	return &dto.ProcessDocumentRes{
-	grpool.Add(taskCtx, func(ctx context.Context) {
+		ChunkCount: chunks,
-		g.TryCatch(ctx, func(ctx context.Context) {
+		CostTime:   costTime,
-			if innerErr := s.SearchRecursiveSplit(ctx, &dto.SearchRecursiveSplitReq{Id: req.Id}); innerErr != nil {
+	}, nil
-				cancel()
-			}
-		}, func(ctx context.Context, err error) {
-			cancel()
-		})
-	})
-
-	// 任务3: 提取文档
-	grpool.Add(taskCtx, func(ctx context.Context) {
-		g.TryCatch(ctx, func(ctx context.Context) {
-			if innerErr := s.KeywordExtract(ctx, &dto.KeywordExtractReq{Id: req.Id}); innerErr != nil {
-				cancel()
-			}
-		}, func(ctx context.Context, err error) {
-			cancel()
-		})
-	})
-
-	return nil
 }
 
-// extractDocument 关键词提取（支持取消）
 func (s *documentService) extractDocument(ctx context.Context, doc *entity.Document) (err error) {
-	// ========== 取消检查 1：方法入口 ==========
-	if ctx.Err() != nil {
-		// 写入任务进度失败 任务类型为关键字存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeExtractKeywords,
-			Status:   task.TaskStatusFailed,
-			Remark:   "ctx取消: " + ctx.Err().Error(),
-		})
-		return ctx.Err()
-	}
-
 	// 1. 加载文件
 	docs, err := s.loadDocument(ctx, doc)
 	if err != nil {
-		// 写入任务进度失败 任务类型为关键字存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeExtractKeywords,
-			Status:   task.TaskStatusFailed,
-			Remark:   "加载文件失败: " + err.Error(),
-		})
 		return
 	}
 
-	var words []utils.Keyword
+	var words []gse.Keyword
 	if len(docs[0].Content) < 500 {
-		words = utils.GseTool.Extract(docs[0].Content, 4)
+		words = gse.GseTool.Extract(docs[0].Content, 4)
 	} else if len(docs[0].Content) < 2000 {
-		words = utils.GseTool.Extract(docs[0].Content, 8)
+		words = gse.GseTool.Extract(docs[0].Content, 8)
 	} else if len(docs[0].Content) < 5000 {
-		words = utils.GseTool.Extract(docs[0].Content, 13)
+		words = gse.GseTool.Extract(docs[0].Content, 13)
 	} else {
 		var docsSplit []*schema.Document
 		docsSplit, err = eino.RecursiveSplitDocument(ctx, docs)
 		if err != nil {
-			err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-				TaskId:   doc.Id,
-				TaskType: task.TaskTypeExtractKeywords,
-				Status:   task.TaskStatusFailed,
-				Remark:   "递归分割文档失败: " + err.Error(),
-			})
 			return
 		}
-		// ========== 取消检查 2：循环内部 ==========
 		for _, t := range docsSplit {
-			if ctx.Err() != nil {
+			words = append(words, gse.GseTool.Extract(t.Content, 6)...)
-				err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-					TaskId:   doc.Id,
-					TaskType: task.TaskTypeExtractKeywords,
-					Status:   task.TaskStatusFailed,
-					Remark:   "ctx取消: " + ctx.Err().Error(),
-				})
-				return ctx.Err()
-			}
-			words = append(words, utils.GseTool.Extract(t.Content, 6)...)
 		}
 	}
 
-	// ========== 取消检查 3：批量操作前 ==========
-	if ctx.Err() != nil {
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeExtractKeywords,
-			Status:   task.TaskStatusFailed,
-			Remark:   "ctx取消: " + ctx.Err().Error(),
-		})
-		return ctx.Err()
-	}
-
 	var keywordReqs = make([]*dto.CreateKeywordReq, 0)
 	for _, word := range words {
 		keywordReqs = append(keywordReqs, &dto.CreateKeywordReq{
-			DatasetId:   doc.DatasetId,
+			DatasetId:  doc.DatasetId,
-			DocumentId:  doc.Id,
+			DocumentId: doc.Id,
-			Word:        word.Word,
+			Word:       word.Word,
-			Weight:      gconv.Int16(word.Score),
+			Weight:     gconv.Int16(word.Score),
-			KeywordType: keyword.KeywordTypeInitial.Code(),
 		})
 	}
 	if len(keywordReqs) > 0 {
 		_, err = dao.Keyword.BatchSaveOrUpdate(ctx, keywordReqs)
 		if err != nil {
-			// 写入任务进度失败 任务类型为关键字存储
-			err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-				TaskId:   doc.Id,
-				TaskType: task.TaskTypeExtractKeywords,
-				Status:   task.TaskStatusFailed,
-				Remark:   "关键字存储失败: " + err.Error(),
-			})
 			return
 		}
-		// 写入任务进度已完成 任务类型为关键字存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeExtractKeywords,
-			Status:   task.TaskStatusCompleted,
-			Remark:   "关键字提取完成",
-		})
-	} else {
-		// 写入任务进度已完成 任务类型为关键字存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeExtractKeywords,
-			Status:   task.TaskStatusCompleted,
-			Remark:   "没有提取到关键词，关键字提取完成",
-		})
 	}
 	return
 }
 
-// semanticSplitDocument 语义切分
+func (s *documentService) sqlSplitDocument(ctx context.Context, doc *entity.Document) (vectorDocsCount, docsSplitCount int64, err error) {
-func (s *documentService) semanticSplitDocument(ctx context.Context, doc *entity.Document) (err error) {
-	// ========== 取消检查 1：方法入口 ==========
-	if ctx.Err() != nil {
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeGenerateVector,
-			Status:   task.TaskStatusFailed,
-			Remark:   "ctx取消: " + ctx.Err().Error(),
-		})
-		return ctx.Err()
-	}
-
 	// 1. 加载文件
 	docs, err := s.loadDocument(ctx, doc)
 	if err != nil {
-		// 写入任务进度失败 任务类型为sql存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeGenerateVector,
-			Status:   task.TaskStatusFailed,
-			Remark:   "加载文件失败: " + err.Error(),
-		})
 		return
 	}
-
 	// 2. 语义切分文件
-	docsSplit, err := eino.SemanticSplitDocument(ctx, docs, model.ModelConfigTypeVectorDashScope.Code()) //TODO 后续替换成本地模型
+	docsSplit, err := eino.SemanticSplitDocument(ctx, docs)
 	if err != nil {
-		// 写入任务进度失败 任务类型为sql存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeGenerateVector,
-			Status:   task.TaskStatusFailed,
-			Remark:   "文档切分失败: " + err.Error(),
-		})
 		return
 	}
-
+	docsSplitCount = gconv.Int64(len(docsSplit))
 	// 2. 获取历史数据
 	err = s.getHistoryData(ctx, doc, public.KnowledgeLockSqlKey, public.KnowledgeContentHashSqlKey)
 	if err != nil {
-		// 写入任务进度失败 任务类型为sql存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeGenerateVector,
-			Status:   task.TaskStatusFailed,
-			Remark:   "获取历史数据失败: " + err.Error(),
-		})
 		return
 	}
-
 	// 3. 组装向量文档
-	var docsChunk = make([]*schema.Document, 0)
+	var vectorDocs = make([]dto.VectorDocumentChunkMsg, 0)
 	for i, t := range docsSplit {
-		// ========== 取消检查 2：循环内部 ==========
-		if ctx.Err() != nil {
-			err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-				TaskId:   doc.Id,
-				TaskType: task.TaskTypeGenerateVector,
-				Status:   task.TaskStatusFailed,
-				Remark:   "ctx取消: " + ctx.Err().Error(),
-			})
-			return ctx.Err()
-		}
-
 		contentHash := gmd5.MustEncryptString(t.Content)
-		var isNew, needCopy bool
+		// 检查是否重复
-		isNew, needCopy, err = s.checkRepeatWithDocId(ctx, public.KnowledgeContentHashSqlKey, contentHash, doc.Id)
+		var success bool
+		success, err = s.checkRepeat(ctx, public.KnowledgeContentHashSqlKey, contentHash)
 		if err != nil {
-			// 写入任务进度失败 任务类型为sql存储
-			err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-				TaskId:   doc.Id,
-				TaskType: task.TaskTypeGenerateVector,
-				Status:   task.TaskStatusFailed,
-				Remark:   "检查重复数据失败: " + err.Error(),
-			})
 			return
 		}
-		if !isNew && !needCopy {
+		if !success {
 			continue
 		}
-		var metaData = make(map[string]any)
+		vectorDocs = append(vectorDocs, dto.VectorDocumentChunkMsg{
-		metaData[entity.DocumentCol.TenantId] = doc.TenantId
+			TenantId:    doc.TenantId,
-		metaData[entity.DocumentCol.Creator] = doc.Creator
+			Creator:     doc.Creator,
-		metaData[entity.DocumentCol.DatasetId] = doc.DatasetId
+			DatasetId:   doc.DatasetId,
-		metaData[entity.DocumentVectorCol.DocumentId] = doc.Id
+			DocumentId:  doc.Id,
-		metaData[entity.DocumentVectorCol.ContentHash] = contentHash
+			Content:     t.Content,
-		metaData[entity.DocumentVectorCol.ChunkIndex] = gconv.Int64(i + 1)
+			ContentHash: contentHash,
-		if isNew {
+			ChunkIndex:  gconv.Int64(i),
-			metaData["isNew"] = true
-		}
-		if needCopy {
-			metaData["isNew"] = false
-		}
-		t.MetaData = metaData
-		docsChunk = append(docsChunk, t)
-	}
-
-	// ========== 取消检查 3：批量发送前 ==========
-	if ctx.Err() != nil {
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeGenerateVector,
-			Status:   task.TaskStatusFailed,
-			Remark:   "ctx取消: " + ctx.Err().Error(),
 		})
-		return ctx.Err()
-	}
 
+	}
 	// 4. 发送消息到队列
-	if len(docsChunk) > 0 {
+	if len(vectorDocs) > 0 {
-		err = gmq.GetGmq(public.GmqMsgPluginsName).GmqPublish(ctx, &mq.RedisPubMessage{
+		err = gmq.GetGmq("primary").GmqPublish(ctx, &mq.RedisPubMessage{
 			PubMessage: types.PubMessage{
-				Topic: public.KnowledgeDocumentVectorTopic,
+				Topic: public.KnowledgeDocumentChunkTopic,
-				Data:  docsChunk,
+				Data:  vectorDocs,
 			},
 		})
-		if err != nil {
-			// 写入任务进度失败 任务类型为sql存储
-			err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-				TaskId:   doc.Id,
-				TaskType: task.TaskTypeGenerateVector,
-				Status:   task.TaskStatusFailed,
-				Remark:   "发送消息到队列失败: " + err.Error(),
-			})
-			return
-		}
-		// 写入任务进度进行中 任务类型为sql存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeGenerateVector,
-			Status:   task.TaskStatusRunning,
-			Remark:   "向量生成任务已提交到队列",
-		})
-	} else {
-		// 写入任务进度已完成 任务类型为sql存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeGenerateVector,
-			Status:   task.TaskStatusCompleted,
-			Remark:   "无需生成向量，任务完成",
-		})
 	}
+	vectorDocsCount = gconv.Int64(len(vectorDocs))
 	return
 }
 
-// recursiveSplitDocument 递归切分
+func (s *documentService) esSplitDocument(ctx context.Context, doc *entity.Document) (err error) {
-func (s *documentService) recursiveSplitDocument(ctx context.Context, doc *entity.Document) (err error) {
-	// ========== 取消检查 1：方法入口 ==========
-	if ctx.Err() != nil {
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeFullTextSearch,
-			Status:   task.TaskStatusFailed,
-			Remark:   "ctx取消: " + ctx.Err().Error(),
-		})
-		return ctx.Err()
-	}
-
 	// 1. 加载文件
 	docs, err := s.loadDocument(ctx, doc)
 	if err != nil {
-		// 写入任务进度失败 任务类型为es存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeFullTextSearch,
-			Status:   task.TaskStatusFailed,
-			Remark:   "加载文件失败: " + err.Error(),
-		})
 		return
 	}
-
 	// 2. 递归切分文件
 	docsSplit, err := eino.RecursiveSplitDocument(ctx, docs)
 	if err != nil {
-		// 写入任务进度失败 任务类型为es存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeFullTextSearch,
-			Status:   task.TaskStatusFailed,
-			Remark:   "文档切分失败: " + err.Error(),
-		})
 		return
 	}
-
 	// 2. 获取历史数据
 	err = s.getHistoryData(ctx, doc, public.KnowledgeLockEsKey, public.KnowledgeContentHashEsKey)
 	if err != nil {
-		// 写入任务进度失败 任务类型为es存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeFullTextSearch,
-			Status:   task.TaskStatusFailed,
-			Remark:   "获取历史数据失败: " + err.Error(),
-		})
 		return
 	}
-
 	// 3. 组装向量文档并同时构建meilisearch文档
 	var meiliDocs = make([]interface{}, 0)
 	for i, t := range docsSplit {
-		// ========== 取消检查 2：循环内部 ==========
-		if ctx.Err() != nil {
-			err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-				TaskId:   doc.Id,
-				TaskType: task.TaskTypeFullTextSearch,
-				Status:   task.TaskStatusFailed,
-				Remark:   "ctx取消: " + ctx.Err().Error(),
-			})
-			return ctx.Err()
-		}
-
 		contentHash := gmd5.MustEncryptString(t.Content)
-		var isNew, needCopy bool
+		// 检查是否重复
-		isNew, needCopy, err = s.checkRepeatWithDocId(ctx, public.KnowledgeContentHashEsKey, contentHash, doc.Id)
+		var success bool
+		success, err = s.checkRepeat(ctx, public.KnowledgeContentHashEsKey, contentHash)
 		if err != nil {
-			// 写入任务进度失败 任务类型为es存储
-			err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-				TaskId:   doc.Id,
-				TaskType: task.TaskTypeFullTextSearch,
-				Status:   task.TaskStatusFailed,
-				Remark:   "检查重复数据失败: " + err.Error(),
-			})
 			return
 		}
-		if !isNew && !needCopy {
+		if !success {
 			continue
 		}
+		// 构建Meilisearch文档
 		meiliDocs = append(meiliDocs, map[string]interface{}{
-			entity.DocumentVectorCol.Id:          contentHash,
+			"id":          contentHash,
-			entity.DocumentVectorCol.DatasetId:   doc.DatasetId,
+			"datasetId":   doc.DatasetId,
-			entity.DocumentVectorCol.DocumentId:  doc.Id,
+			"documentId":  doc.Id,
-			entity.DocumentVectorCol.Content:     t.Content,
+			"content":     t.Content,
-			entity.DocumentVectorCol.ContentHash: contentHash,
+			"contentHash": contentHash,
-			entity.DocumentVectorCol.ChunkIndex:  i + 1,
+			"chunkIndex":  i,
 		})
 	}
-
-	// ========== 取消检查 3：批量写入前 ==========
-	if ctx.Err() != nil {
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeFullTextSearch,
-			Status:   task.TaskStatusFailed,
-			Remark:   "ctx取消: " + ctx.Err().Error(),
-		})
-		return ctx.Err()
-	}
-
 	// 4. 写入到meilisearch数据库中
 	if len(meiliDocs) > 0 {
 		if _, err = meilisearch.DB().InsertMany(ctx, meiliDocs, public.IndexNameDocumentChunk); err != nil {
-			// 写入任务进度失败 任务类型为meilisearch存储
+			g.Log().Errorf(ctx, "写入meilisearch失败: %v", err)
-			err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-				TaskId:   doc.Id,
-				TaskType: task.TaskTypeFullTextSearch,
-				Status:   task.TaskStatusFailed,
-				Remark:   "写入meilisearch失败: " + err.Error(),
-			})
 			return
 		}
-		// 写入任务进度已完成 任务类型为meilisearch存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeFullTextSearch,
-			Status:   task.TaskStatusCompleted,
-			Remark:   "全文检索数据写入完成",
-		})
-	} else {
-		// 写入任务进度已完成 任务类型为meilisearch存储
-		err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-			TaskId:   doc.Id,
-			TaskType: task.TaskTypeFullTextSearch,
-			Status:   task.TaskStatusCompleted,
-			Remark:   "无需生成全文检索数据，任务完成",
-		})
 	}
 	return
 }
@@ -714,8 +365,7 @@ func (s *documentService) getHistoryData(ctx context.Context, doc *entity.Docume
 		}
 
 		// 3. Redis 无数据：根据 contentKey 类型选择查询方式
-		var dictData = make([]*dto.DocumentVectorRPC, 0)
+		var dictData = make([]*dto.DocumentChunkRPC, 0)
-
 		if public.KnowledgeContentHashSqlKey == contentKey {
 			// SQL 方式：调用 HTTP 接口查询
 			dictData, err = s.getHistoryDataFromHttp(ctx, doc)
@@ -727,16 +377,20 @@ func (s *documentService) getHistoryData(ctx context.Context, doc *entity.Docume
 			return err
 		}
 
+		// 4. 把查询到的数据写入 Redis（600s过期）
 		for _, item := range dictData {
+			// 去除可能的 JSON 引号
 			contentHash := strings.Trim(item.ContentHash, `"`)
 			key := fmt.Sprintf(contentKey, contentHash)
-			// SAdd：把文档ID加入集合（自动去重，可存多个）
+			_, err = g.Redis().Set(ctx, key, true, gredis.SetOption{
-			_, err = g.Redis().SAdd(ctx, key, item.DocumentId)
+				TTLOption: gredis.TTLOption{
+					EX: gconv.PtrInt64(600),
+				},
+				NX: true,
+			})
 			if err != nil {
 				return err
 			}
-			// 设置过期时间
-			_, _ = g.Redis().Expire(ctx, key, 600)
 		}
 
 		return nil
@@ -748,20 +402,29 @@ func (s *documentService) getHistoryData(ctx context.Context, doc *entity.Docume
 }
 
 // getHistoryDataFromHttp 通过 HTTP 接口查询历史数据
-func (s *documentService) getHistoryDataFromHttp(ctx context.Context, doc *entity.Document) (dictData []*dto.DocumentVectorRPC, err error) {
+func (s *documentService) getHistoryDataFromHttp(ctx context.Context, doc *entity.Document) (dictData []*dto.DocumentChunkRPC, err error) {
+	headers := make(map[string]string)
+	if r := g.RequestFromCtx(ctx); r != nil {
+		for k, v := range r.Request.Header {
+			if len(v) > 0 {
+				headers[k] = v[0]
+			}
+		}
+	}
+
 	// 调用接口获取数据
-	res, _, err := dao.DocumentVector.List(ctx, &dto.ListDocumentVectorReq{
+	d := &dto.ListDocumentChunkRPC{}
-		DatasetId: doc.DatasetId,
+	if err = http.Get(ctx, "rag-vector/document/chunk/listDocumentChunk", headers, &d,
-	})
+		"datasetId", gconv.String(doc.DatasetId),
-	if err != nil {
+		"status", 1); err != nil {
 		return
 	}
-	err = gconv.Struct(res, &dictData)
+	dictData = d.List
 	return
 }
 
 // getHistoryDataFromMeilisearch 通过 meilisearch 查询历史数据
-func (s *documentService) getHistoryDataFromMeilisearch(ctx context.Context, doc *entity.Document) (dictData []*dto.DocumentVectorRPC, err error) {
+func (s *documentService) getHistoryDataFromMeilisearch(ctx context.Context, doc *entity.Document) (dictData []*dto.DocumentChunkRPC, err error) {
 	// 构建 meilisearch 查询参数
 	searchParams := &meilisearch.SearchParams{
 		Filter: fmt.Sprintf("datasetId = %d", doc.DatasetId),
@@ -776,9 +439,9 @@ func (s *documentService) getHistoryDataFromMeilisearch(ctx context.Context, doc
 	}
 
 	// 转换查询结果
-	dictData = make([]*dto.DocumentVectorRPC, 0)
+	dictData = make([]*dto.DocumentChunkRPC, 0)
 	for _, hit := range hits {
-		item := &dto.DocumentVectorRPC{}
+		item := &dto.DocumentChunkRPC{}
 		if err = gconv.Struct(hit, item); err != nil {
 			return
 		}
@@ -787,39 +450,34 @@ func (s *documentService) getHistoryDataFromMeilisearch(ctx context.Context, doc
 	return
 }
 
-// checkRepeatWithDocId 正确版：检查当前文档是否已存在该分片
+// checkRepeat 检查是否重复
-// 返回：isNew(是否需要生成向量)、isCrossDoc(是否跨文档需拷贝)、err
+func (s *documentService) checkRepeat(ctx context.Context, contentKey, contentHash string) (success bool, err error) {
-func (s *documentService) checkRepeatWithDocId(ctx context.Context, contentKey string, contentHash string, currentDocId int64) (isNew bool, needCopy bool, err error) {
+	var val *gvar.Var
-	key := fmt.Sprintf(contentKey, contentHash)
+	if val, err = g.Redis().Set(ctx, fmt.Sprintf(contentKey, contentHash), true, gredis.SetOption{
-
+		TTLOption: gredis.TTLOption{
-	// 1. 检查当前文档ID是否在集合中
+			EX: gconv.PtrInt64(600),
-	exists, err := g.Redis().SIsMember(ctx, key, currentDocId)
+		},
-	if err != nil {
+		NX: true,
-		return false, false, err
+	}); err != nil {
+		return
 	}
+	success = val.Bool()
+	return
+}
 
-	// 情况1：当前文档已存在 → 完全跳过，不生成、不拷贝
+func (s *documentService) DocsVectorStatusMsg(ctx context.Context, msg any) (err error) {
-	if !g.IsEmpty(exists) {
+	var req = new(dto.KnowledgeDocumentMsg)
-		return false, false, nil
+	if err = gconv.Struct(msg, &req); err != nil {
+		g.Log().Error(ctx, "DocsVectorStatusMsg err:", err)
+		return
 	}
-
+	ctx = context.WithValue(ctx, "user", &beans.User{
-	// 2. 检查 key 是否存在（是否有任何文档拥有该分片）
+		TenantId: req.TenantId,
-	keyExists, err := g.Redis().Exists(ctx, key)
+		UserName: req.Creator,
-	if err != nil {
+	})
-		return false, false, err
+	_, err = dao.Document.Update(ctx, &dto.UpdateDocumentReq{
-	}
+		Id:           req.Id,
-
+		VectorStatus: req.VectorStatus,
-	// 情况2：key 不存在 = 全新数据 → 需要生成向量
+	})
-	if g.IsEmpty(keyExists) {
+	return
-		// 把当前文档ID加入集合
-		_, err = g.Redis().SAdd(ctx, key, currentDocId)
-		_, _ = g.Redis().Expire(ctx, key, 600)
-		return true, false, err
-	}
-
-	// 情况3：key 存在，但当前文档不在集合中 = 跨文档重复 → 不生成，需拷贝
-	// 把当前文档ID加入集合（记录归属关系）
-	_, err = g.Redis().SAdd(ctx, key, currentDocId)
-	_, _ = g.Redis().Expire(ctx, key, 600)
-	return false, true, err
 }

service/document_chunk.go

@@ -0,0 +1,176 @@
+package service
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"rag/consts/document"
+	"rag/consts/public"
+	"rag/dao"
+	"rag/model/dto"
+	"rag/model/entity"
+
+	"gitea.com/red-future/common/beans"
+	"gitea.com/red-future/common/rag/eino"
+	gmq "github.com/bjang03/gmq/core/gmq"
+	"github.com/bjang03/gmq/mq"
+	"github.com/bjang03/gmq/types"
+	"github.com/gogf/gf/v2/frame/g"
+	"github.com/gogf/gf/v2/util/gconv"
+	"github.com/pgvector/pgvector-go"
+)
+
+var DocumentChunk = new(documentChunkService)
+
+type documentChunkService struct{}
+
+const (
+	DatasetIndexStatusReady = "ready"
+)
+
+// Update 更新文件块
+func (s *documentChunkService) Update(ctx context.Context, req *dto.UpdateDocumentChunkReq) (err error) {
+	_, err = dao.DocumentChunk.Update(ctx, req)
+	return
+}
+
+// List 获取文件块列表
+func (s *documentChunkService) List(ctx context.Context, req *dto.ListDocumentChunkReq) (res *dto.ListDocumentChunkRes, err error) {
+	list, total, err := dao.DocumentChunk.List(ctx, req)
+	if err != nil {
+		return
+	}
+	res = &dto.ListDocumentChunkRes{
+		Total: total,
+	}
+	err = gconv.Struct(list, &res.List)
+	return
+}
+
+func (s *documentChunkService) DocsChunkMsg(ctx context.Context, msg any) (err error) {
+	var req = make([]*dto.VectorDocumentChunkMsg, 0)
+	msgMap := gconv.Map(msg)
+	if err = gconv.Structs(msgMap["data"], &req); err != nil {
+		g.Log().Error(ctx, "DocsChunkMsg err:", err)
+		return
+	}
+	if len(req) == 0 {
+		g.Log().Error(ctx, "DocsChunkMsg err:", "msg is empty")
+		return
+	}
+
+	ctx = context.WithValue(ctx, "user", &beans.User{
+		TenantId: req[0].TenantId,
+		UserName: req[0].Creator,
+	})
+
+	// 调用eino接口获取向量
+	var vectorDocsStr = make([]string, 0, len(req))
+	for _, t := range req {
+		vectorDocsStr = append(vectorDocsStr, t.Content)
+	}
+	embeddings, err := eino.EmbedStrings(ctx, vectorDocsStr)
+	if err != nil {
+		g.Log().Error(ctx, "DocsChunkMsg err:", err)
+		err = s.publishKnowledgeDocumentMsg(ctx, req[0].TenantId, req[0].Creator, req[0].DocumentId, document.VectorStatusFailed.Code())
+		return
+	}
+
+	// 获取向量维度
+	dimension := 0
+	if len(embeddings) > 0 {
+		dimension = len(embeddings[0])
+	}
+
+	// 创建或更新DatasetIndex
+	err = s.createOrUpdateDatasetIndex(ctx, req[0].DatasetId, dimension, int64(len(req)))
+	if err != nil {
+		g.Log().Error(ctx, "CreateOrUpdateDatasetIndex err:", err)
+		err = s.publishKnowledgeDocumentMsg(ctx, req[0].TenantId, req[0].Creator, req[0].DocumentId, document.VectorStatusFailed.Code())
+		return
+	}
+
+	// 更新向量文档
+	for i, embedding := range embeddings {
+		req[i].Vector = pgvector.NewVector(gconv.Float32s(embedding))
+		req[i].VectorStatus = document.VectorStatusCompleted.Code()
+		req[i].Status = document.StatusEnable.Code()
+	}
+	_, err = dao.DocumentChunk.BatchInsert(ctx, req)
+	if err != nil {
+		g.Log().Error(ctx, "DocsChunkMsg err:", err)
+		err = s.publishKnowledgeDocumentMsg(ctx, req[0].TenantId, req[0].Creator, req[0].DocumentId, document.VectorStatusFailed.Code())
+		return
+	}
+
+	err = s.publishKnowledgeDocumentMsg(ctx, req[0].TenantId, req[0].Creator, req[0].DocumentId, document.VectorStatusCompleted.Code())
+
+	return
+}
+
+// createOrUpdateDatasetIndex 创建或更新数据集索引
+func (s *documentChunkService) createOrUpdateDatasetIndex(ctx context.Context, datasetId int64, dimension int, vectorCount int64) (err error) {
+	// 查询数据集是否已有索引
+	existIndex, err := dao.DatasetIndex.GetByDatasetId(ctx, datasetId)
+	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+		return err
+	}
+
+	// 已有索引 → 只更新数量
+	if existIndex != nil {
+		_ = dao.DatasetIndex.IncVectorCount(ctx, existIndex.Id, vectorCount)
+		return nil
+	}
+
+	// ====================== 创建新索引 ======================
+	indexName := fmt.Sprintf("idx_dataset_%d_vector", datasetId) // 真实PG索引名
+	// 1. 插入索引配置
+	index := &entity.DatasetIndex{
+		DatasetId:   datasetId,
+		Name:        indexName,
+		Dimension:   dimension,
+		FieldType:   "float",
+		MetricType:  "COSINE",
+		Status:      gconv.PtrInt8(1),
+		VectorCount: vectorCount,
+		Description: fmt.Sprintf("数据集%d向量索引", datasetId),
+	}
+	_, err = dao.DatasetIndex.Insert(ctx, index)
+	if err != nil {
+		return err
+	}
+
+	// 2. 真正创建 PGVector 索引（唯一真实索引！）
+	err = s.createRealPGVectorIndex(ctx, indexName)
+	return err
+}
+
+// createRealPGVectorIndex 真正在PostgreSQL创建向量索引（真实可用）
+func (s *documentChunkService) createRealPGVectorIndex(ctx context.Context, indexName string) error {
+	// 执行真实建索引语句
+	err := dao.DatasetIndex.InsertIndex(ctx, indexName)
+	if err != nil {
+		g.Log().Error(ctx, "创建向量索引失败：", err)
+		return err
+	}
+	g.Log().Info(ctx, "PGVector真实索引创建成功："+indexName)
+	return nil
+}
+
+// publishKnowledgeDocumentMsg 发布消息
+func (s *documentChunkService) publishKnowledgeDocumentMsg(ctx context.Context, tenantId uint64, creator string, documentId int64, vectorStatus document.VectorStatus) (err error) {
+	knowledgeDocumentMsg := dto.KnowledgeDocumentMsg{
+		TenantId:     tenantId,
+		Creator:      creator,
+		Id:           documentId,
+		VectorStatus: vectorStatus,
+	}
+	err = gmq.GetGmq("primary").GmqPublish(ctx, &mq.RedisPubMessage{
+		PubMessage: types.PubMessage{
+			Topic: public.KnowledgeDocumentVectorStatusTopic,
+			Data:  knowledgeDocumentMsg,
+		},
+	})
+	return
+}

service/document_vector.go

@@ -1,221 +0,0 @@
-package service
-
-import (
-	"context"
-	"fmt"
-	"rag/common/eino"
-	"rag/consts/model"
-	"rag/consts/task"
-	"rag/dao"
-	"rag/model/dto"
-	"rag/model/entity"
-
-	"gitea.redpowerfuture.com/red-future/common/beans"
-	"github.com/cloudwego/eino/components/retriever"
-	"github.com/cloudwego/eino/schema"
-	"github.com/gogf/gf/v2/frame/g"
-	"github.com/gogf/gf/v2/util/gconv"
-	"github.com/pgvector/pgvector-go"
-)
-
-var DocumentVector = new(documentVectorService)
-
-type documentVectorService struct{}
-
-// Query 执行RAG查询
-func (s *documentVectorService) Query(ctx context.Context, req *dto.RAGQueryReq) (*dto.RAGQueryRes, error) {
-
-	modelInfo, err := dao.Model.Get(ctx, &dto.GetModelReq{
-		ModelType: model.ModelTypeChat.Code(),
-	})
-	if err != nil {
-		g.Log().Errorf(ctx, "获取模型失败: %v", err)
-		return nil, fmt.Errorf("获取模型失败: %w", err)
-	}
-	if modelInfo == nil {
-		g.Log().Errorf(ctx, "模型不存在: %v", model.ModelTypeChat.Code())
-		return nil, fmt.Errorf("模型不存在: %w", err)
-	}
-
-	// 4. 使用向量检索器进行查询
-	r, err := eino.NewPGVectorRetriever(ctx, &eino.PGVectorRetrieverConfig{
-		DefaultTopK: req.TopK,
-	}, model.ModelConfigTypeVectorDashScope.Code()) //TODO 后续替换成本地模型
-	if err != nil {
-		g.Log().Errorf(ctx, "初始化向量检索器失败: %v", err)
-		return nil, fmt.Errorf("初始化向量检索器失败: %w", err)
-	}
-
-	// 5. 执行向量检索
-	docs, err := r.Retrieve(ctx, req.Content, retriever.WithDSLInfo(map[string]any{
-		"dataset_ids":  req.DatasetIds,
-		"document_ids": req.DocumentIds,
-	}))
-	if err != nil {
-		g.Log().Errorf(ctx, "向量检索失败: %v", err)
-		return nil, fmt.Errorf("向量检索失败: %w", err)
-	}
-
-	messages := make([]*schema.Message, 0)
-	err = gconv.Struct(req.History, &messages)
-	if err != nil {
-		g.Log().Errorf(ctx, "转换历史消息失败: %v", err)
-		return nil, fmt.Errorf("转换历史消息失败: %w", err)
-	}
-
-	replyMsg, err := eino.NewChatModel(ctx, req.Content, docs, messages, modelInfo.ConfigType)
-	if err != nil {
-		g.Log().Errorf(ctx, "向量检索失败: %v", err)
-		return nil, fmt.Errorf("向量检索失败: %w", err)
-	}
-
-	return &dto.RAGQueryRes{
-		Answer: replyMsg.Content,
-	}, nil
-}
-
-// Update 更新文件块
-func (s *documentVectorService) Update(ctx context.Context, req *dto.UpdateDocumentVectorReq) (err error) {
-	_, err = dao.DocumentVector.Update(ctx, req)
-	return
-}
-
-// List 获取文件块列表
-func (s *documentVectorService) List(ctx context.Context, req *dto.ListDocumentVectorReq) (res *dto.ListDocumentVectorRes, err error) {
-	list, total, err := dao.DocumentVector.List(ctx, req)
-	if err != nil {
-		return
-	}
-	res = &dto.ListDocumentVectorRes{
-		Total: total,
-	}
-	err = gconv.Struct(list, &res.List)
-	return
-}
-
-func (s *documentVectorService) DocsChunkMsg(ctx context.Context, msg any) (err error) {
-	var docs = make([]*schema.Document, 0)
-	msgMap := gconv.Map(msg)
-	if err = gconv.Structs(msgMap["data"], &docs); err != nil {
-		g.Log().Error(ctx, "DocsChunkMsg err:", err)
-		return
-	}
-	if len(docs) == 0 {
-		g.Log().Error(ctx, "DocsChunkMsg err:", "msg is empty")
-		return
-	}
-	ctx = context.WithValue(ctx, "user", &beans.User{
-		TenantId: gconv.Uint64(docs[0].MetaData[entity.DocumentVectorCol.TenantId]),
-		UserName: gconv.String(docs[0].MetaData[entity.DocumentVectorCol.Creator]),
-	})
-
-	documentId := gconv.Int64(docs[0].MetaData[entity.DocumentVectorCol.DocumentId])
-
-	var docsStore = make([]*schema.Document, 0)
-	var docsInsert = make([]*dto.VectorDocumentVectorMsg, 0)
-	for _, doc := range docs {
-		if gconv.Bool(doc.MetaData["isNew"]) {
-			docsStore = append(docsStore, doc)
-		} else {
-			ck := new(dto.VectorDocumentVectorMsg)
-			err = gconv.Struct(doc.MetaData, ck)
-			ck.Content = doc.Content
-			ck.VectorStatus = gconv.PtrInt8(1)
-			ck.Status = gconv.PtrInt8(1)
-			docsInsert = append(docsInsert, ck)
-		}
-	}
-
-	if !g.IsEmpty(docsStore) {
-		idx := eino.NewPGVectorIndexer(&eino.PGVectorIndexerOptions{
-			BatchSize: 10,
-		})
-		var rows int64
-		rows, err = idx.Store(ctx, docsStore, model.ModelConfigTypeVectorDashScope.Code()) //TODO 后续替换成本地模型
-
-		if err != nil || rows == 0 {
-			g.Log().Error(ctx, "DocsChunkMsg rows: , err:", rows, err)
-			// 写入任务进度失败 任务类型为sql存储
-			remark := " 向量存储数量: " + gconv.String(rows)
-			if err != nil {
-				remark = "向量存储失败: " + err.Error()
-			}
-			err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-				TaskId:   documentId,
-				TaskType: task.TaskTypeGenerateVector,
-				Status:   task.TaskStatusFailed,
-				Remark:   remark,
-			})
-			return
-		}
-	}
-
-	if !g.IsEmpty(docsInsert) {
-		// 1. 提取所有 contentHash
-		contentHashs := make([]string, 0, len(docsInsert))
-		for _, d := range docsInsert {
-			contentHashs = append(contentHashs, d.ContentHash)
-		}
-
-		// 2. 分页查询已存在的向量（一页1000，避免大查询）
-		var existVectors []*entity.DocumentVector
-		for page := 1; ; page++ {
-			res, total, err := dao.DocumentVector.List(ctx, &dto.ListDocumentVectorReq{
-				Page:         &beans.Page{PageSize: 1000, PageNum: int64(page)},
-				ContentHashs: contentHashs,
-			})
-			if err != nil {
-				return err
-			}
-			if len(res) == 0 {
-				break
-			}
-			existVectors = append(existVectors, res...)
-			if len(existVectors) >= total {
-				break
-			}
-		}
-
-		// 3. 构建哈希 -> 向量 的映射表（O(1) 查找，性能提升巨大）
-		vectorMap := make(map[string]pgvector.Vector, len(existVectors))
-		for _, v := range existVectors {
-			vectorMap[v.ContentHash] = v.Vector
-		}
-
-		// 4. 回填向量 + 过滤掉数据库已存在的数据（避免重复插入）
-		for _, d := range docsInsert {
-			// 回填已有向量
-			if vec, ok := vectorMap[d.ContentHash]; ok {
-				d.Vector = vec
-			}
-		}
-
-		var rows int64
-		rows, err = dao.DocumentVector.BatchInsert(ctx, docsInsert)
-
-		if err != nil || rows == 0 {
-			g.Log().Error(ctx, "DocsChunkMsg rows: , err:", rows, err)
-			// 写入任务进度失败 任务类型为sql存储
-			remark := " 向量存储数量: " + gconv.String(rows)
-			if err != nil {
-				remark = "向量存储失败: " + err.Error()
-			}
-			err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-				TaskId:   documentId,
-				TaskType: task.TaskTypeGenerateVector,
-				Status:   task.TaskStatusFailed,
-				Remark:   remark,
-			})
-			return
-		}
-	}
-
-	// 写入任务进度成功 任务类型为sql存储
-	err = Task.WriteTaskProgress(ctx, &dto.WriteTaskProgressReq{
-		TaskId:   documentId,
-		TaskType: task.TaskTypeGenerateVector,
-		Status:   task.TaskStatusCompleted,
-		Remark:   "向量生成完成",
-	})
-	return
-}

service/model.go

@@ -1,299 +0,0 @@
-package service
-
-import (
-	"context"
-	"rag/common/eino"
-	"rag/consts/model"
-	"rag/consts/task"
-	"rag/dao"
-	"rag/model/dto"
-	"rag/model/entity"
-
-	"github.com/gogf/gf/v2/errors/gerror"
-	"github.com/gogf/gf/v2/frame/g"
-	"github.com/gogf/gf/v2/util/gconv"
-)
-
-var ModelService = new(modelService)
-
-type modelService struct{}
-
-// GetModelAllEnums 获取模型全量枚举（模型类型 + 配置类型 合并）
-func (s *modelService) GetModelAllEnums(ctx context.Context, req *dto.GetModelAllEnumsReq) (res *dto.GetModelEnumRes, err error) {
-	_, _ = ctx, req
-	res = new(dto.GetModelEnumRes)
-
-	// 获取所有模型类型
-	modelTypeRes := model.GetAllModelTypeEnums()
-
-	var options []dto.ModelEnumOption
-	for _, mt := range modelTypeRes.Options {
-		// 构造 modelType
-		modelTypeStr := gconv.String(mt.Key)
-		modelType := model.ModelType(gconv.PtrString(modelTypeStr))
-
-		// 获取对应配置类型
-		configRes := model.GetAllModelConfigTypeEnums(modelType)
-
-		// 把 configRes.Options 转成目标类型
-		var configList []dto.ModelKeyValue
-		err = gconv.Structs(configRes.Options, &configList)
-		if err != nil {
-			return
-		}
-		options = append(options, dto.ModelEnumOption{
-			Key:         mt.Key,
-			Value:       mt.Value,
-			ConfigTypes: configList,
-		})
-	}
-
-	res.Options = options
-	return
-}
-
-func (s *modelService) GetModelConfigFormFields(ctx context.Context, req *dto.GetModelConfigFormFieldsReq) (*dto.GetModelConfigFormFieldsRes, error) {
-	_ = ctx
-
-	fields := make([]map[string]interface{}, 0)
-
-	// ===================== 固定基础字段（CreateModelReq 前4个）=====================
-	// 1. 模型类型：固定只读字段
-	fields = append(fields, map[string]interface{}{
-		"name":     "modelType",
-		"label":    "模型类型",
-		"type":     "text",
-		"disabled": true,
-		"required": true,
-		"value":    model.GetModelTypeDescByCode(req.ModelType),
-	})
-
-	var configTypeValue = "未知类型"
-	if *req.ModelType == *model.ModelTypeVector.Code() {
-		configTypeValue = model.GetVectorDescByCode(req.ConfigType)
-	} else if *req.ModelType == *model.ModelTypeChat.Code() {
-		configTypeValue = model.GetChatDescByCode(req.ConfigType)
-	}
-
-	// 2. 配置类型：固定只读字段
-	fields = append(fields, map[string]interface{}{
-		"name":     "configType",
-		"label":    "配置类型",
-		"type":     "text",
-		"disabled": true,
-		"required": true,
-		"value":    configTypeValue,
-	})
-
-	// 3. 基础信息
-	fields = append(fields, []map[string]interface{}{
-		{
-			"name":        "modelName",
-			"label":       "模型名称",
-			"type":        "input",
-			"required":    true,
-			"placeholder": "例如：DeepSeek 对话模型",
-		},
-		{
-			"name":     "modelDesc",
-			"label":    "模型描述",
-			"type":     "textarea",
-			"required": false,
-		},
-	}...)
-
-	// 4. 通用模型名称字段
-	fields = append(fields, map[string]interface{}{
-		"name":        "model",
-		"label":       "模型类型",
-		"type":        "input",
-		"required":    true,
-		"placeholder": "例如：deepseek-chat / text-embedding-3-small",
-	})
-
-	// ===================== 动态配置内容 ConfigContent =====================
-
-	// 根据模型类型 + 配置类型生成动态字段
-	switch *req.ModelType {
-	case *model.ModelTypeChat.Code():
-		switch *req.ConfigType {
-		case *model.ModelConfigTypeChatArk.Code():
-			fields = append(fields, map[string]interface{}{"name": "api_key", "label": "API Key", "type": "input", "required": true})
-
-		case *model.ModelConfigTypeChatArkBot.Code():
-			fields = append(fields, map[string]interface{}{"name": "api_key", "label": "API Key", "type": "input", "required": true})
-
-		case *model.ModelConfigTypeChatClaude.Code():
-			fields = append(fields, []map[string]interface{}{
-				{"name": "by_bedrock", "label": "使用 AWS Bedrock", "type": "switch", "default": true},
-				{"name": "access_key", "label": "Access Key", "type": "input"},
-				{"name": "secret_access_key", "label": "Secret Access Key", "type": "input"},
-				{"name": "region", "label": "Region", "type": "input"},
-				{"name": "api_key", "label": "API Key", "type": "input"},
-				{"name": "base_url", "label": "Base URL", "type": "input"},
-			}...)
-
-		case *model.ModelConfigTypeChatDeepSeek.Code():
-			fields = append(fields, []map[string]interface{}{
-				{"name": "api_key", "label": "API Key", "type": "input", "required": true},
-				{"name": "base_url", "label": "Base URL", "type": "input", "default": "https://api.deepseek.com"},
-			}...)
-
-		case *model.ModelConfigTypeChatOllama.Code():
-			fields = append(fields, map[string]interface{}{"name": "base_url", "label": "Base URL", "type": "input", "required": true, "default": "http://127.0.0.1:11434"})
-
-		case *model.ModelConfigTypeChatOpenAI.Code():
-			fields = append(fields, []map[string]interface{}{
-				{"name": "api_key", "label": "API Key", "type": "input", "required": true},
-				{"name": "by_azure", "label": "使用 Azure", "type": "switch", "default": true},
-				{"name": "base_url", "label": "Base URL", "type": "input"},
-				{"name": "api_version", "label": "API Version", "type": "input"},
-			}...)
-
-		case *model.ModelConfigTypeChatQianfan.Code():
-			fields = append(fields, []map[string]interface{}{
-				{"name": "access_key", "label": "Access Key", "type": "input", "required": true},
-				{"name": "secret_key", "label": "Secret Key", "type": "input", "required": true},
-			}...)
-
-		case *model.ModelConfigTypeChatQwen.Code():
-			fields = append(fields, []map[string]interface{}{
-				{"name": "api_key", "label": "API Key", "type": "input", "required": true},
-				{"name": "base_url", "label": "Base URL", "type": "input"},
-			}...)
-		}
-
-	case *model.ModelTypeVector.Code():
-		switch *req.ConfigType {
-		case *model.ModelConfigTypeVectorArk.Code():
-			fields = append(fields, []map[string]interface{}{
-				{"name": "api_key", "label": "API Key", "type": "input", "required": true},
-				{"name": "api_type", "label": "API Type", "type": "input"},
-			}...)
-
-		case *model.ModelConfigTypeVectorOllama.Code():
-			fields = append(fields, map[string]interface{}{"name": "base_url", "label": "Base URL", "type": "input", "required": true, "default": "http://127.0.0.1:11434"})
-
-		case *model.ModelConfigTypeVectorOpenAI.Code():
-			fields = append(fields, []map[string]interface{}{
-				{"name": "api_key", "label": "API Key", "type": "input", "required": true},
-				{"name": "by_azure", "label": "使用 Azure", "type": "switch", "default": true},
-				{"name": "base_url", "label": "Base URL", "type": "input"},
-				{"name": "api_version", "label": "API Version", "type": "input"},
-			}...)
-
-		case *model.ModelConfigTypeVectorQianfan.Code():
-			fields = append(fields, []map[string]interface{}{
-				{"name": "access_key", "label": "Access Key", "type": "input", "required": true},
-				{"name": "secret_key", "label": "Secret Key", "type": "input", "required": true},
-			}...)
-
-		case *model.ModelConfigTypeVectorTencentCloud.Code():
-			fields = append(fields, []map[string]interface{}{
-				{"name": "secret_id", "label": "Secret ID", "type": "input", "required": true},
-				{"name": "secret_key", "label": "Secret Key", "type": "input", "required": true},
-				{"name": "region", "label": "Region", "type": "input", "required": true, "default": "ap-beijing"},
-			}...)
-		case *model.ModelConfigTypeVectorDashScope.Code():
-			fields = append(fields, map[string]interface{}{"name": "api_key", "label": "API Key", "type": "input", "required": true})
-		}
-	}
-
-	return &dto.GetModelConfigFormFieldsRes{
-		ModelType:  req.ModelType,
-		ConfigType: req.ConfigType,
-		Fields:     fields,
-	}, nil
-}
-
-func (s *modelService) Create(ctx context.Context, req *dto.CreateModelReq) (res *dto.CreateModelRes, err error) {
-	count, err := dao.Model.Count(ctx, &dto.GetModelReq{
-		ModelType: req.ModelType,
-	})
-	if err != nil {
-		return
-	}
-	if count > 0 {
-		err = gerror.New("模型配置已存在")
-		return
-	}
-	var id int64
-	id, err = dao.Model.Insert(ctx, req)
-	if err != nil {
-		return
-	}
-	res = &dto.CreateModelRes{Id: id}
-	err = s.refresh(ctx, id)
-	return
-}
-
-func (s *modelService) Update(ctx context.Context, req *dto.UpdateModelReq) (err error) {
-	count, err := dao.Task.Count(ctx, &dto.GetTaskReq{
-		TaskStatus: task.TaskStatusRunning,
-	})
-	if err != nil {
-		return err
-	}
-	if !g.IsEmpty(count) {
-		err = gerror.New("任务正在执行中，模型配置暂时不可修改，请稍后再试")
-		return
-	}
-	var updateCount int64
-	updateCount, err = dao.Model.Update(ctx, req)
-	if err != nil {
-		return
-	}
-	if !g.IsEmpty(updateCount) {
-		err = s.refresh(ctx, req.Id)
-		if err != nil {
-			return err
-		}
-	}
-	return
-}
-
-func (s *modelService) refresh(ctx context.Context, id int64) (err error) {
-	var modelDO *entity.Model
-	modelDO, err = dao.Model.Get(ctx, &dto.GetModelReq{
-		Id: id,
-	})
-	if err != nil {
-		return err
-	}
-	if *modelDO.ModelType == *model.ModelTypeChat.Code() {
-		if err = eino.RefreshTenantChatModel(ctx, modelDO); err != nil {
-			return err
-		}
-	}
-
-	if *modelDO.ModelType == *model.ModelTypeVector.Code() {
-		if err = eino.RefreshTenantEmbedder(ctx, modelDO); err != nil {
-			return err
-		}
-	}
-
-	return
-}
-
-func (s *modelService) Delete(ctx context.Context, req *dto.DeleteModelReq) (err error) {
-	_, err = dao.Model.Delete(ctx, req)
-	return
-}
-
-func (s *modelService) Get(ctx context.Context, req *dto.GetModelReq) (res *dto.ModelVO, err error) {
-	r, err := dao.Model.Get(ctx, req)
-	err = gconv.Struct(r, &res)
-	return
-}
-
-func (s *modelService) List(ctx context.Context, req *dto.ListModelReq) (res *dto.ListModelRes, err error) {
-	list, total, err := dao.Model.List(ctx, req)
-	if err != nil {
-		return nil, err
-	}
-	res = &dto.ListModelRes{
-		Total: total,
-	}
-	err = gconv.Struct(list, &res.List)
-	return
-}

service/task.go

@@ -1,148 +0,0 @@
-package service
-
-import (
-	"context"
-	"rag/consts/document"
-	"rag/consts/public"
-	"rag/consts/task"
-	"rag/dao"
-	"rag/model/dto"
-
-	"gitea.redpowerfuture.com/red-future/common/db/gfdb"
-	"github.com/gogf/gf/v2/database/gdb"
-	"github.com/gogf/gf/v2/frame/g"
-	"github.com/gogf/gf/v2/util/gconv"
-)
-
-var Task = new(taskService)
-
-type taskService struct{}
-
-// WriteTaskProgress 写入任务进度（核心方法）
-func (s *taskService) WriteTaskProgress(ctx context.Context, req *dto.WriteTaskProgressReq) (err error) {
-	t, total, err := dao.Task.Get(ctx, &dto.GetTaskReq{
-		TaskId: req.TaskId,
-	})
-	if err != nil {
-		g.Log().Errorf(ctx, "查询任务失败: %v", err)
-		return err
-	}
-	completed := false
-	if total != 0 {
-		taskVO := make([]*dto.TaskVO, 0, total)
-		err = gconv.Struct(t, &taskVO)
-		if err != nil {
-			g.Log().Errorf(ctx, "转换任务失败: %v", err)
-			return err
-		}
-		taskVO = append(taskVO, &dto.TaskVO{
-			TaskType: req.TaskType,
-			Status:   req.Status,
-		})
-		completed = IsAllSubTasks(taskVO, task.TaskStatusCompleted)
-	}
-
-	// 1. 查询是否已存在该文档的该类型任务
-	existTask, _, err := dao.Task.Get(ctx, &dto.GetTaskReq{
-		TaskId:   req.TaskId,
-		TaskType: req.TaskType,
-	})
-	if err != nil {
-		g.Log().Errorf(ctx, "查询任务失败: %v", err)
-		return err
-	}
-	err = gfdb.DB(ctx, public.DbNameKnowledge).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
-		// 2. 如果不存在，则创建新任务
-		if g.IsEmpty(existTask) {
-			createReq := &dto.CreateTaskReq{
-				TaskId:   req.TaskId,
-				TaskType: req.TaskType,
-				Status:   req.Status,
-				Remark:   req.Remark,
-			}
-			_, err = dao.Task.Insert(ctx, createReq)
-		} else {
-			// 3. 如果已存在，则更新任务
-			updateReq := &dto.UpdateTaskReq{
-				Id:     existTask[0].Id,
-				Status: req.Status,
-				Remark: req.Remark,
-			}
-			_, err = dao.Task.Update(ctx, updateReq)
-			if err != nil {
-				g.Log().Errorf(ctx, "更新任务失败: %v", err)
-				return err
-			}
-		}
-
-		if completed {
-			// 3. 如果已存在，则更新任务
-			_, err = dao.Task.Update(ctx, &dto.UpdateTaskReq{
-				TaskId: req.TaskId,
-				Status: task.TaskStatusCompleted,
-				Remark: "文档解析完成",
-			})
-			if err != nil {
-				g.Log().Errorf(ctx, "更新任务失败: %v", err)
-				return err
-			}
-			_, err = dao.Document.Update(ctx, &dto.UpdateDocumentReq{
-				Id:           req.TaskId,
-				VectorStatus: document.VectorStatusCompleted.Code(),
-			})
-			if err != nil {
-				return err
-			}
-		} else {
-			if task.TaskStatusFailed == req.Status {
-				_, err = dao.Document.Update(ctx, &dto.UpdateDocumentReq{
-					Id:           req.TaskId,
-					VectorStatus: document.VectorStatusFailed.Code(),
-				})
-				if err != nil {
-					return err
-				}
-			}
-		}
-		return nil
-	})
-
-	return
-}
-
-// IsAllSubTasks 判断三个子任务
-func IsAllSubTasks(subTasks []*dto.TaskVO, taskStatus task.TaskStatus) bool {
-	// 必须包含 3 种任务类型
-	hasKeywords := false
-	hasVector := false
-	hasFullText := false
-
-	for _, t := range subTasks {
-		// 子任务必须是【已完成】状态才计数
-		if t.Status == taskStatus {
-			switch t.TaskType {
-			case task.TaskTypeExtractKeywords:
-				hasKeywords = true
-			case task.TaskTypeGenerateVector:
-				hasVector = true
-			case task.TaskTypeFullTextSearch:
-				hasFullText = true
-			}
-		}
-	}
-
-	// 三个任务全部完成 → 返回true
-	return hasKeywords && hasVector && hasFullText
-}
-
-func (s *taskService) Get(ctx context.Context, req *dto.GetTaskReq) (res *dto.ListTaskRes, err error) {
-	list, total, err := dao.Task.Get(ctx, req)
-	if err != nil {
-		return
-	}
-	res = &dto.ListTaskRes{
-		Total: total,
-	}
-	err = gconv.Struct(list, &res.List)
-	return
-}

update.sql

@@ -114,7 +114,6 @@ COMMENT ON COLUMN rag_knowledge_document.file_path IS '文件存储路径（如M
 COMMENT ON COLUMN rag_knowledge_document.metadata IS '文件元数据，结构：{"author":"作者","tags":["标签1","标签2"],"custom":{"key":"值"}}';
 
 --------------------pgsql创建rag_knowledge_document表语句---------------------------
-
 --------------------pgsql创建rag_knowledge_keyword表语句---------------------------
 -- 关键词表（文档关键词+权重）
 CREATE TABLE IF NOT EXISTS rag_knowledge_keyword (
@@ -135,9 +134,9 @@ CREATE TABLE IF NOT EXISTS rag_knowledge_keyword (
 );
 
 -- 唯一索引：保证 租户 + 数据集 + 文档 + 关键词 全局唯一
--- CREATE UNIQUE INDEX uk_rag_knowledge_keyword_tenant_dataset_doc_word
+CREATE UNIQUE INDEX uk_rag_knowledge_keyword_tenant_dataset_doc_word
---     ON rag_knowledge_keyword(tenant_id, dataset_id, document_id, word)
+    ON rag_knowledge_keyword(tenant_id, dataset_id, document_id, word)
---     WHERE deleted_at IS NULL;
+    WHERE deleted_at IS NULL;
 
 -- 索引（按业务高频查询）
 CREATE INDEX idx_keyword_tenant_id ON rag_knowledge_keyword(tenant_id);
@@ -159,206 +158,5 @@ COMMENT ON COLUMN rag_knowledge_keyword.dataset_id IS '数据集ID';
 COMMENT ON COLUMN rag_knowledge_keyword.document_id IS '文档ID';
 COMMENT ON COLUMN rag_knowledge_keyword.word IS '关键词';
 COMMENT ON COLUMN rag_knowledge_keyword.weight IS '权重';
-CREATE UNIQUE INDEX uk_rag_knowledge_keyword_tenant_dataset_doc_word ON rag_knowledge_keyword (tenant_id, dataset_id, document_id, word);
---------------------pgsql创建rag_knowledge_keyword表语句---------------------------
 
---------------------pgsql创建rag_knowledge_task表语句---------------------------
+--------------------pgsql创建rag_knowledge_keyword表语句---------------------------
--- 知识库任务表
-CREATE TABLE IF NOT EXISTS rag_knowledge_task (
-    -- 基础字段（完全对齐项目规范）
-    id BIGINT PRIMARY KEY, -- 主键ID（非自增）
-    tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID int8
-    creator VARCHAR(64) NOT NULL,
-    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    updater VARCHAR(64) NOT NULL,
-    updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    deleted_at timestamp(6),
-
-    -- 业务字段
-    task_id BIGINT NOT NULL, -- 任务ID
-    task_type VARCHAR(32) NOT NULL, -- 任务类型
-    status VARCHAR(32) NOT NULL, -- 任务状态
-    executor VARCHAR(128) DEFAULT '', -- 执行器
-    remark TEXT DEFAULT '' -- 备注
-    );
-
--- 索引（高频查询）
-CREATE INDEX idx_rkt_tenant_id ON rag_knowledge_task(tenant_id);
-CREATE INDEX idx_rkt_task_id ON rag_knowledge_task(task_id);
-CREATE INDEX idx_rkt_task_type ON rag_knowledge_task(task_type);
-CREATE INDEX idx_rkt_status ON rag_knowledge_task(status);
-CREATE INDEX idx_rkt_deleted_at ON rag_knowledge_task(deleted_at);
-
--- 表和字段注释
-COMMENT ON TABLE rag_knowledge_task IS '知识库任务表';
-COMMENT ON COLUMN rag_knowledge_task.id IS '主键ID（非自增）';
-COMMENT ON COLUMN rag_knowledge_task.tenant_id IS '租户ID';
-COMMENT ON COLUMN rag_knowledge_task.creator IS '创建人';
-COMMENT ON COLUMN rag_knowledge_task.created_at IS '创建时间';
-COMMENT ON COLUMN rag_knowledge_task.updater IS '更新人';
-COMMENT ON COLUMN rag_knowledge_task.updated_at IS '更新时间';
-COMMENT ON COLUMN rag_knowledge_task.deleted_at IS '删除时间（软删）';
-COMMENT ON COLUMN rag_knowledge_task.task_id IS '任务ID';
-COMMENT ON COLUMN rag_knowledge_task.task_type IS '任务类型';
-COMMENT ON COLUMN rag_knowledge_task.status IS '任务状态';
-COMMENT ON COLUMN rag_knowledge_task.executor IS '执行器';
-COMMENT ON COLUMN rag_knowledge_task.remark IS '备注';
-
---------------------pgsql创建rag_knowledge_task表语句---------------------------
-
---------------------pgsql创建rag_knowledge_model表语句---------------------------
--- 知识库模型配置表
-CREATE TABLE IF NOT EXISTS rag_knowledge_model (
-    -- 基础字段（完全对齐项目规范）
-    id BIGINT PRIMARY KEY, -- 主键ID（非自增）
-    tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID int8
-    creator VARCHAR(64) NOT NULL,
-    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    updater VARCHAR(64) NOT NULL,
-    updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    deleted_at timestamp(6),
-
-    -- 业务字段
-    dataset_id BIGINT NOT NULL, -- 数据集ID
-    model_type VARCHAR(32) NOT NULL, -- 模型类型
-    model_name VARCHAR(128) NOT NULL, -- 模型名称
-    model_desc TEXT DEFAULT '', -- 模型描述
-    model_config JSONB DEFAULT '{}'::JSONB -- 模型配置(JSONB)
-    );
-
--- 索引（高频查询）
-CREATE INDEX idx_rkm_tenant_id ON rag_knowledge_model(tenant_id);
-CREATE INDEX idx_rkm_dataset_id ON rag_knowledge_model(dataset_id);
-CREATE INDEX idx_rkm_model_type ON rag_knowledge_model(model_type);
-CREATE INDEX idx_rkm_deleted_at ON rag_knowledge_model(deleted_at);
-
--- 表和字段注释
-COMMENT ON TABLE rag_knowledge_model IS '知识库模型配置表';
-COMMENT ON COLUMN rag_knowledge_model.id IS '主键ID（非自增）';
-COMMENT ON COLUMN rag_knowledge_model.tenant_id IS '租户ID';
-COMMENT ON COLUMN rag_knowledge_model.creator IS '创建人';
-COMMENT ON COLUMN rag_knowledge_model.created_at IS '创建时间';
-COMMENT ON COLUMN rag_knowledge_model.updater IS '更新人';
-COMMENT ON COLUMN rag_knowledge_model.updated_at IS '更新时间';
-COMMENT ON COLUMN rag_knowledge_model.deleted_at IS '删除时间（软删）';
-COMMENT ON COLUMN rag_knowledge_model.dataset_id IS '数据集ID';
-COMMENT ON COLUMN rag_knowledge_model.model_type IS '模型类型';
-COMMENT ON COLUMN rag_knowledge_model.model_name IS '模型名称';
-COMMENT ON COLUMN rag_knowledge_model.model_desc IS '模型描述';
-COMMENT ON COLUMN rag_knowledge_model.model_config IS '模型配置(JSONB)';
-
---------------------pgsql创建rag_knowledge_model表语句---------------------------
-
-
---------------------pgsql创建rag_vector_dataset_index表语句---------------------------
--- 向量数据集索引表
-CREATE TABLE IF NOT EXISTS rag_vector_dataset_index (
-    -- 基础字段
-    id BIGINT PRIMARY KEY, -- 主键ID（非自增）
-    tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID int8
-    creator VARCHAR(64) NOT NULL,
-    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    updater VARCHAR(64) NOT NULL,
-    updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    deleted_at timestamp(6),
-
-    -- 核心字段
-    dataset_id INT8 NOT NULL,
-    name VARCHAR(255) NOT NULL,
-    collection VARCHAR(255) NOT NULL,
-    dimension INT NOT NULL,
-    field_type VARCHAR(50) NOT NULL,
-    metric_type VARCHAR(50) NOT NULL,
-    status SMALLINT NOT NULL DEFAULT 1, -- 状态：1启用/0停用
-    vector_count INT8 NOT NULL DEFAULT 0,
-    description TEXT
-    );
-
--- 唯一约束
-ALTER TABLE rag_vector_dataset_index ADD CONSTRAINT uk_dataset_id_name UNIQUE (dataset_id, name);
-
--- 索引
-CREATE INDEX idx_dataset_index_tenant_id ON rag_vector_dataset_index(tenant_id);
-CREATE INDEX idx_dataset_index_dataset_id ON rag_vector_dataset_index(dataset_id);
-CREATE INDEX idx_dataset_index_status ON rag_vector_dataset_index(status);
-
--- 注释
-COMMENT ON TABLE rag_vector_dataset_index IS '向量数据集索引表';
-COMMENT ON COLUMN rag_vector_dataset_index.id IS '主键ID（非自增）';
-COMMENT ON COLUMN rag_vector_dataset_index.tenant_id IS '租户ID';
-COMMENT ON COLUMN rag_vector_dataset_index.creator IS '创建人';
-COMMENT ON COLUMN rag_vector_dataset_index.created_at IS '创建时间';
-COMMENT ON COLUMN rag_vector_dataset_index.updater IS '更新人';
-COMMENT ON COLUMN rag_vector_dataset_index.updated_at IS '更新时间';
-COMMENT ON COLUMN rag_vector_dataset_index.deleted_at IS '删除时间（软删）';
-COMMENT ON COLUMN rag_vector_dataset_index.dataset_id IS '数据集ID';
-COMMENT ON COLUMN rag_vector_dataset_index.name IS '索引名称';
-COMMENT ON COLUMN rag_vector_dataset_index.collection IS '向量集合名称';
-COMMENT ON COLUMN rag_vector_dataset_index.dimension IS '向量维度';
-COMMENT ON COLUMN rag_vector_dataset_index.field_type IS '字段类型';
-COMMENT ON COLUMN rag_vector_dataset_index.metric_type IS '度量类型';
-COMMENT ON COLUMN rag_vector_dataset_index.status IS '状态';
-COMMENT ON COLUMN rag_vector_dataset_index.vector_count IS '向量数量';
-COMMENT ON COLUMN rag_vector_dataset_index.description IS '描述';
-
---------------------pgsql创建rag_vector_dataset_index表语句---------------------------
-
---------------------pgsql创建rag_vector_document_vector表语句---------------------------
-
-CREATE EXTENSION IF NOT EXISTS vector;
-
--- 文档分块向量表
-CREATE TABLE IF NOT EXISTS rag_vector_document_vector (
-    -- 基础字段
-    id BIGINT PRIMARY KEY, -- 主键ID（非自增）
-    tenant_id BIGINT NOT NULL DEFAULT 0, -- 租户ID int8
-    creator VARCHAR(64) NOT NULL,
-    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    updater VARCHAR(64) NOT NULL,
-    updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    deleted_at timestamp(6),
-
-    -- 核心字段
-    status SMALLINT NOT NULL DEFAULT 1, -- 状态：1启用/0停用
-    vector_status SMALLINT NOT NULL DEFAULT 1, -- 向量化状态: 1pending, 2processing, 3completed, 4failed,5partCompleted
-    dataset_id INT8 NOT NULL,
-    document_id INT8 NOT NULL,
-    content TEXT NOT NULL,
-    content_hash VARCHAR(128) NOT NULL,
-    chunk_index INT8 NOT NULL,
-
-    -- 向量字段（pgvector）
-    vector vector(1024) NOT NULL,
-
-    -- 扩展信息
-    metadata JSONB
-    );
-
--- 索引
-CREATE INDEX idx_vector_tenant_id ON rag_vector_document_vector(tenant_id);
-CREATE INDEX idx_vector_dataset_id ON rag_vector_document_vector(dataset_id);
-CREATE INDEX idx_vector_document_id ON rag_vector_document_vector(document_id);
-CREATE INDEX idx_vector_content_hash ON rag_vector_document_vector(content_hash);
-CREATE INDEX idx_vector_status ON rag_vector_document_vector(status);
-CREATE INDEX idx_vector_vector_status ON rag_vector_document_vector(vector_status);
-
--- 注释
-COMMENT ON TABLE rag_vector_document_vector IS '文档分块向量表';
-COMMENT ON COLUMN rag_vector_document_vector.id IS '主键ID（非自增）';
-COMMENT ON COLUMN rag_vector_document_vector.tenant_id IS '租户ID';
-COMMENT ON COLUMN rag_vector_document_vector.creator IS '创建人';
-COMMENT ON COLUMN rag_vector_document_vector.created_at IS '创建时间';
-COMMENT ON COLUMN rag_vector_document_vector.updater IS '更新人';
-COMMENT ON COLUMN rag_vector_document_vector.updated_at IS '更新时间';
-COMMENT ON COLUMN rag_vector_document_vector.deleted_at IS '删除时间（软删）';
-COMMENT ON COLUMN rag_vector_document_vector.status IS '状态';
-COMMENT ON COLUMN rag_vector_document_vector.vector_status IS '向量生成状态';
-COMMENT ON COLUMN rag_vector_document_vector.dataset_id IS '数据集ID';
-COMMENT ON COLUMN rag_vector_document_vector.document_id IS '文档ID';
-COMMENT ON COLUMN rag_vector_document_vector.content IS '分块内容';
-COMMENT ON COLUMN rag_vector_document_vector.content_hash IS '内容哈希';
-COMMENT ON COLUMN rag_vector_document_vector.chunk_index IS '分块序号';
-COMMENT ON COLUMN rag_vector_document_vector.vector IS '向量数据';
-COMMENT ON COLUMN rag_vector_document_vector.metadata IS '扩展元数据';
-
---------------------pgsql创建rag_vector_document_vector表语句---------------------------