feat: 支持模板子流程递归拷贝与段级续跑优化

* 模板拷贝用户流程时递归复制子流程,并重写 sub_flow 节点引用
* 段级续跑改用列表位置作为段身份,移除对 segment_index 的依赖
* 段结果保存移到每段生成完成时立即落库,降低崩溃丢失风险
* 移除视频分段续跑设计与对应测试
This commit is contained in:
2026-08-25 13:56:55 +08:00
parent 855d0cca72
commit b3f4b94b21
8 changed files with 123 additions and 426 deletions
@@ -1,113 +0,0 @@
package flow
import (
"context"
"os"
"testing"
"time"
"gitea.redpowerfuture.com/red-future/common/beans"
// 空导入:pgsql 驱动。main.go 里已有,但 go test ./workflow/dao/flow/ 单测包不经 main
// 不加则报 "cannot find database driver for specified database type pgsql"
_ "github.com/gogf/gf/contrib/drivers/pgsql/v2"
)
// TestFlowSegmentResultDao 段结果 DAO 集成测试:Save / ListByNode / DeleteByExecution。
// 依赖本地 black_deacon + 用户信息注入;默认跳过,设置 AI_AGENT_TEST_DB=1 时运行。
func TestFlowSegmentResultDao(t *testing.T) {
if os.Getenv("AI_AGENT_TEST_DB") == "" {
t.Skip("设置 AI_AGENT_TEST_DB=1 且本地 black_deacon 可用时运行")
}
// gfdb Insert/Select 钩子从 ctx 读用户信息补 tenant_id/creator 并追加租户过滤
ctx := context.WithValue(context.Background(), "user", &beans.User{UserName: "unit_test", TenantId: 999999})
execId := time.Now().UnixMilli()
nodeId := "seg_test_node"
dao := FlowSegmentResultDao
if err := dao.Save(ctx, execId, nodeId, 1, "video_url", "http://host/u1.mp4"); err != nil {
t.Fatalf("Save 段1失败: %v", err)
}
if err := dao.Save(ctx, execId, nodeId, 2, "video_url", "http://host/u2.mp4"); err != nil {
t.Fatalf("Save 段2失败: %v", err)
}
// 唯一键冲突 → 更新不报错
if err := dao.Save(ctx, execId, nodeId, 2, "video_url", "http://host/u2-new.mp4"); err != nil {
t.Fatalf("Save 段2(冲突更新)失败: %v", err)
}
m, err := dao.ListByNode(ctx, execId, nodeId)
if err != nil {
t.Fatalf("ListByNode失败: %v", err)
}
if len(m) != 2 {
t.Fatalf("期望 2 段,实际 %d", len(m))
}
if m[1].URL != "http://host/u1.mp4" || m[2].URL != "http://host/u2-new.mp4" {
t.Fatalf("段内容不符: %+v", m)
}
if m[2].Key != "video_url" {
t.Fatalf("段2 key 不符: %s", m[2].Key)
}
if err := dao.DeleteByExecution(ctx, execId); err != nil {
t.Fatalf("DeleteByExecution失败: %v", err)
}
m, err = dao.ListByNode(ctx, execId, nodeId)
if err != nil {
t.Fatalf("删除后 ListByNode失败: %v", err)
}
if len(m) != 0 {
t.Fatalf("删除后应无段结果,实际 %d", len(m))
}
}
// TestFlowSegmentResultDaoReuseAfterCleanup 硬删→重存的"续跑复用"保证。
// DeleteByExecution 必须物理删除:实体嵌入 beans.SQLBaseDO(含 deleted_at),若走
// gfdb Model.Delete() 会退化为软删(UPDATE ... SET deleted_at=now()),而后续 Save
// 的 ON CONFLICT ... DO UPDATE SET 不含 deleted_atOmitNil 丢弃 nil),软删行无法复活,
// ListByNode 永远看不到 → 每次重执行都全量重生成。本测试验证:清理(物理删两行)后
// 对同一 (execution_id, node_id, segment_index) 重存段2,仅段2可见(len==1)且为更新后 URL。
func TestFlowSegmentResultDaoReuseAfterCleanup(t *testing.T) {
if os.Getenv("AI_AGENT_TEST_DB") == "" {
t.Skip("设置 AI_AGENT_TEST_DB=1 且本地 black_deacon 可用时运行")
}
ctx := context.WithValue(context.Background(), "user", &beans.User{UserName: "unit_test", TenantId: 999999})
execId := time.Now().UnixMilli()
nodeId := "seg_test_cleanup"
dao := FlowSegmentResultDao
// 物理残留由硬清理(docker exec psql DELETE FROM)兜底
t.Cleanup(func() { _ = dao.DeleteByExecution(ctx, execId) })
if err := dao.Save(ctx, execId, nodeId, 1, "video_url", "http://host/u1-cleanup.mp4"); err != nil {
t.Fatalf("Save 段1失败: %v", err)
}
if err := dao.Save(ctx, execId, nodeId, 2, "video_url", "http://host/u2-cleanup.mp4"); err != nil {
t.Fatalf("Save 段2失败: %v", err)
}
// 物理删除整条执行(DELETE FROM ... WHERE execution_id=?
if err := dao.DeleteByExecution(ctx, execId); err != nil {
t.Fatalf("DeleteByExecution失败: %v", err)
}
// 清理后重新保存段2(同一 exec/node/idx,不同 URL):
// 物理删除已移除该行,Save 走纯 INSERT,段2 在 ListByNode 中可见且为更新后的 URL
if err := dao.Save(ctx, execId, nodeId, 2, "video_url", "http://host/u2-cleanup-new.mp4"); err != nil {
t.Fatalf("Save 段2(清理后重存)失败: %v", err)
}
m, err := dao.ListByNode(ctx, execId, nodeId)
if err != nil {
t.Fatalf("ListByNode失败: %v", err)
}
// 期望:段1 已物理删除,仅段2 可见(len==1)且为更新后的 URL
if len(m) != 1 {
t.Fatalf("硬删后重存应只剩段2(len=1),实际 len=%d: %+v", len(m), m)
}
if m[2].URL != "http://host/u2-cleanup-new.mp4" {
t.Fatalf("段2 应为更新后的 URL,实际: %+v", m[2])
}
}
+1
View File
@@ -34,6 +34,7 @@ type UpdateFlowUserReq struct {
NodeInputParams []*entity.FlowNode `json:"nodeInputParams" description:"节点输入参数"`
AccessLevel flow.FlowUserAccessLevel `json:"accessLevel" description:"访问权限:1私有,2团队,3公开"`
SourceFlowTemplateId int64 `json:"sourceFlowTemplateId" description:"来源流程模板ID"`
SubFlows []UpdateFlowUserReq `json:"subFlows" description:"子流程"`
}
type DeleteFlowUserReq struct {
+1 -1
View File
@@ -65,7 +65,7 @@ type ValueSource struct {
// SubFlowConfig 子流程节点配置
type SubFlowConfig struct {
WorkflowId int64 `json:"workflowId"`
WorkflowId int64 `json:"workflowId,string"`
WorkflowName string `json:"workflowName"`
Fields []map[string]any `json:"fields"`
MaxConcurrency int `json:"maxConcurrency"` // 子流程并发数
+86 -8
View File
@@ -2,6 +2,7 @@ package flow
import (
"ai-agent/workflow/consts/flow"
"ai-agent/workflow/consts/public"
flowDao "ai-agent/workflow/dao/flow"
flowDto "ai-agent/workflow/model/dto/flow"
"ai-agent/workflow/model/entity"
@@ -9,7 +10,9 @@ import (
"context"
"gitea.redpowerfuture.com/red-future/common/beans"
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
"gitea.redpowerfuture.com/red-future/common/utils"
"github.com/gogf/gf/v2/database/gdb"
"github.com/gogf/gf/v2/frame/g"
"github.com/gogf/gf/v2/util/gconv"
)
@@ -65,15 +68,35 @@ func (s *flowUserService) Update(ctx context.Context, req *flowDto.UpdateFlowUse
}
if !g.IsEmpty(get) {
id, err = flowDao.FlowUserDao.Insert(ctx, &flowDto.CreateFlowUserReq{
FlowName: req.FlowName,
Description: req.Description,
FlowContent: req.FlowContent,
NodeInputParams: req.NodeInputParams,
SourceFlowTemplateId: get.Id,
// 模版 → 用户流程拷贝(含递归子流程拷贝)整体放一个事务:任一拷贝失败则整体回滚,
// 避免主流程已落库而子流程缺失/引用错乱的脏数据。
// Transaction 会把 tx 注入回调 ctx,回调内用该 ctx 调 DAO 即自动进入事务。
txErr := gfdb.DB(ctx, public.DbNameBlackDeacon).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
// 子流程引用重写:模版里的 sub_flow 节点指向子模版/子流程 id,
// 拷贝成用户自己的流程时要把子工作流也复制一份(子流程可能再嵌套子流程,递归),
// 并把主流程 SubConfig.WorkflowId 改写为新拷贝的 id
subIdMap, copyErr := copySubFlows(ctx, req.SubFlows)
if copyErr != nil {
return copyErr
}
rewriteSubFlowWorkflowIds(req.FlowContent, subIdMap)
// SubConfig 改写后重新提取节点参数,保证落库的 NodeInputParams 与 FlowContent 一致
req.NodeInputParams = ExtractFlowNodeFrom(req.FlowContent)
newId, insertErr := flowDao.FlowUserDao.Insert(ctx, &flowDto.CreateFlowUserReq{
FlowName: req.FlowName,
Description: req.Description,
FlowContent: req.FlowContent,
NodeInputParams: req.NodeInputParams,
SourceFlowTemplateId: get.Id,
})
if insertErr != nil {
return insertErr
}
id = newId
return nil
})
if err != nil {
return
if txErr != nil {
return nil, txErr
}
} else {
_, err = flowDao.FlowUserDao.Update(ctx, req)
@@ -196,3 +219,58 @@ func (s *flowUserService) List(ctx context.Context, req *flowDto.ListFlowUserReq
return
}
// copySubFlows 递归拷贝子工作流(子流程可能再包含子流程),返回 旧id→新id 映射。
// 映射键是子工作流在来源(模版)中的 id,即父流程 sub_flow 节点 SubConfig.WorkflowId 指向的值;
// 拷贝时把子工作流落为当前用户自己的流程(SourceFlowTemplateId 记录来源)。
// 同一子流程被多个节点引用时只拷贝一份,避免产生孤儿副本。
func copySubFlows(ctx context.Context, subs []flowDto.UpdateFlowUserReq) (map[int64]int64, error) {
idMap := make(map[int64]int64)
for i := range subs {
sub := subs[i]
if _, done := idMap[sub.Id]; done {
continue
}
nestedMap, err := copySubFlows(ctx, sub.SubFlows)
if err != nil {
return nil, err
}
for k, v := range nestedMap {
idMap[k] = v
}
// 改写本子流程对更深层子流程的引用后,再落库为新的用户流程
rewriteSubFlowWorkflowIds(sub.FlowContent, idMap)
var nodeInputParams []*entity.FlowNode
if sub.FlowContent != nil {
nodeInputParams = ExtractFlowNodeFrom(sub.FlowContent)
}
newId, err := flowDao.FlowUserDao.Insert(ctx, &flowDto.CreateFlowUserReq{
FlowName: sub.FlowName,
Description: sub.Description,
FlowContent: sub.FlowContent,
NodeInputParams: nodeInputParams,
SourceFlowTemplateId: sub.Id,
})
if err != nil {
return nil, err
}
idMap[sub.Id] = newId
}
return idMap, nil
}
// rewriteSubFlowWorkflowIds 把 flowContent 中 sub_flow 节点的 WorkflowId 按 idMap 改写为新拷贝的流程 id
func rewriteSubFlowWorkflowIds(flowContent *entity.FlowInfo, idMap map[int64]int64) {
if flowContent == nil {
return
}
for i := range flowContent.Nodes {
n := &flowContent.Nodes[i]
if n.SubConfig == nil || n.SubConfig.WorkflowId == 0 {
continue
}
if newId, ok := idMap[n.SubConfig.WorkflowId]; ok {
n.SubConfig.WorkflowId = newId
}
}
}
+25 -25
View File
@@ -77,12 +77,13 @@ func ModelLambda(ctx context.Context, input any) (any, error) {
var totalCost float64
if len(paramsList) > 1 {
// 段级续跑仅在"多段 + 视频模型"启用;非视频分段(批量文本等)走原逻辑零影响。
// 段级复用/落库还要求各段带互不重复的 segment_index(目前仅 split_shots_pipeline 产出):
// 视频节点若被批量拆分(如 split_batch_model_params),paramsList>1 但无 segment_index
// 此时不得启用段级续跑,否则续跑会拿同一段结果拼出 N 段重复视频(静默损坏)。
// segVideo 为 false 时走既有非段级合并路径(全量生成、不落库、不复用),全新执行行为不变,
// 段身份 = 列表位置(0-based):paramsList 顺序即段序,concat 按列表顺序拼接;
// 位置互不重复且跨 reExecute 稳定(参数一致 → 段数/顺序不变)。不依赖 params 里的
// segment_index——真实链路(上游 split_shots_pipeline 转写 → 下游 split_segment 按
// __segment_fields 拆分,invokePreTool 剥离 __ 内部键)下 paramsList 只有模型参数。
// segVideo=false 时走既有非段级合并路径(全量生成、不落库、不复用),全新执行行为不变,
// 后续自动 concat 判断(独立的 isVideoModel 调用)仍正常执行。
segVideo := isVideoModel(ctx, nodeInput.Config.ModelConfig.ModelId) && hasDistinctSegmentIndex(paramsList)
segVideo := isVideoModel(ctx, nodeInput.Config.ModelConfig.ModelId)
// 续跑(!ForceNewRun)时读取该节点已成功段;全新执行不查(BuildExecution 已清旧段),saved 为 nil → 全量重生成
var saved map[int]entity.SegmentRef
@@ -98,6 +99,7 @@ func ModelLambda(ctx context.Context, input any) (any, error) {
results := make([][]map[string]any, len(paramsList))
tokenRes := make([]*gateway.ModelCallRes, len(paramsList))
errs := make([]error, len(paramsList))
saveErrs := make([]error, len(paramsList))
isInference := make([]bool, len(paramsList))
var wg sync.WaitGroup
for i, params := range paramsList {
@@ -114,31 +116,29 @@ func ModelLambda(ctx context.Context, input any) (any, error) {
break
}
}
// 每段成功立即落库:该段刚成功即持久化,其他段仍在跑/重试时已成功段也不丢;
// 后续段失败或进程崩溃(panic/OOM/kill)时,已完成段已在库中,reExecute 可直接复用
if segVideo && errs[i] == nil {
for _, rec := range results[i] {
key := media.FindVideoKey(rec)
url := media.FindVideoURL(ctx, rec)
if key == "" || url == "" {
continue
}
if err := flowDao.FlowSegmentResultDao.Save(ctx, nodeInput.Global.ExecutionId, nodeInput.Config.Id, idxList[i], key, url); err != nil {
saveErrs[i] = err
}
}
}
}(i, params)
}
wg.Wait()
// 每段成功立即落库:即使后续某段失败导致节点失败成功段也保留供下次续跑复用
if segVideo {
for i := range results {
if !needGen[i] || errs[i] != nil {
continue
}
for _, rec := range results[i] {
key := media.FindVideoKey(rec)
url := media.FindVideoURL(ctx, rec)
if key == "" || url == "" {
continue
}
if err := flowDao.FlowSegmentResultDao.Save(ctx, nodeInput.Global.ExecutionId, nodeInput.Config.Id, idxList[i], key, url); err != nil {
return nil, err
}
}
}
}
// 仍有失败段 → 节点失败(成功段已落库,供下次 reExecute 复用)
// 仍有失败段或落库失败 → 节点失败成功段已立即落库,供下次 reExecute 复用
for i := range results {
if saveErrs[i] != nil {
return nil, saveErrs[i]
}
if needGen[i] && errs[i] != nil {
return nil, errs[i]
}
+10 -28
View File
@@ -4,42 +4,24 @@ import (
"sort"
"ai-agent/workflow/model/entity"
"github.com/gogf/gf/v2/util/gconv"
)
// segmentGenerateMaxAttempts 视频段生成最大尝试次数(失败自动重试 1 次,共 2 次尝试),参数化可调
const segmentGenerateMaxAttempts = 2
// hasDistinctSegmentIndex 判断各段参数是否都带 segment_index 且互不重复。
// 段级续跑依赖 segment_index 作为段的稳定身份;缺失或重复(如视频节点误用批量拆分)时
// 不得启用复用/落库,否则续跑会拿同一段结果拼出 N 段重复视频(静默损坏)
func hasDistinctSegmentIndex(paramsList []map[string]any) bool {
seen := make(map[int]bool, len(paramsList))
for _, params := range paramsList {
v, ok := params["segment_index"]
if !ok {
return false
}
idx := gconv.Int(v)
if seen[idx] {
return false
}
seen[idx] = true
}
return len(paramsList) > 0
}
// planSegmentResume 段级续跑决策:把 paramsList 各段映射到"是否需重新生成"。
// savedMap 为该节点已成功段(段序号 → {key,url});段在表中缺失或地址为空则需重新生成。
// 返回值与 paramsList 对齐。全新执行(savedMap 为 nil/空)时全部需生成。
// planSegmentResume 段级续跑决策:段身份取列表位置(0-based,paramsList 顺序即段序),
// 把各段映射到"是否需重新生成"。savedMap 为该节点已成功段(段序号 → {key,url});
// 段在表中缺失或地址为空则需重新生成。返回值与 paramsList 对齐
// 全新执行(savedMap 为 nil/空)时全部需生成。
// 不用 params["segment_index"] 作为段身份:真实链路(上游 split_shots_pipeline 转写 →
// 下游 split_segment 按 __segment_fields 拆分,invokePreTool 剥离 __ 内部键)下
// paramsList 只有模型参数;列表位置互不重复、顺序即段序、参数一致时跨 reExecute 稳定。
func planSegmentResume(paramsList []map[string]any, savedMap map[int]entity.SegmentRef) (idxList []int, needGen []bool) {
idxList = make([]int, len(paramsList))
needGen = make([]bool, len(paramsList))
for i, params := range paramsList {
idx := gconv.Int(params["segment_index"])
idxList[i] = idx
ref, ok := savedMap[idx]
for i := range paramsList {
idxList[i] = i
ref, ok := savedMap[i]
needGen[i] = !ok || ref.URL == ""
}
return
@@ -1,124 +0,0 @@
package flow
import (
"testing"
"ai-agent/workflow/model/entity"
)
// hasDistinctSegmentIndex:各段都带 segment_index 且互不重复 → 允许段级续跑
func TestHasDistinctSegmentIndex(t *testing.T) {
distinct := []map[string]any{{"segment_index": 1}, {"segment_index": 2}, {"segment_index": 3}}
if !hasDistinctSegmentIndex(distinct) {
t.Fatal("互不重复的 segment_index 应返回 true")
}
missing := []map[string]any{{"segment_index": 1}, {"other": 2}}
if hasDistinctSegmentIndex(missing) {
t.Fatal("缺失 segment_index 应返回 false")
}
duplicate := []map[string]any{{"segment_index": 1}, {"segment_index": 1}}
if hasDistinctSegmentIndex(duplicate) {
t.Fatal("重复 segment_index 应返回 false")
}
if hasDistinctSegmentIndex(nil) {
t.Fatal("空列表应返回 false")
}
if hasDistinctSegmentIndex([]map[string]any{}) {
t.Fatal("空列表(非 nil)应返回 false")
}
}
// planSegmentResume:段序号从 params 的 segment_index 读出;已成功段复用,缺失/空地址需生成
func TestPlanSegmentResume(t *testing.T) {
paramsList := []map[string]any{
{"segment_index": 1}, {"segment_index": 2}, {"segment_index": 3}, {"segment_index": 4}, {"segment_index": 5},
}
saved := map[int]entity.SegmentRef{
1: {Key: "video_url", URL: "u1"}, 2: {Key: "video_url", URL: "u2"},
3: {Key: "video_url", URL: "u3"}, 5: {Key: "video_url", URL: "u5"},
}
idxList, needGen := planSegmentResume(paramsList, saved)
if len(idxList) != 5 || len(needGen) != 5 {
t.Fatalf("长度不符: idxList=%v needGen=%v", idxList, needGen)
}
want := []bool{false, false, false, true, false} // 仅段4需重新生成
for i := range want {
if needGen[i] != want[i] {
t.Fatalf("needGen[%d]=%v 期望 %v", i, needGen[i], want[i])
}
if idxList[i] != i+1 {
t.Fatalf("idxList[%d]=%d 期望 %d", i, idxList[i], i+1)
}
}
}
// planSegmentResume:无已成功段(全新执行)→ 全部需生成
func TestPlanSegmentResumeEmptySaved(t *testing.T) {
paramsList := []map[string]any{{"segment_index": 0}, {"segment_index": 1}}
_, needGen := planSegmentResume(paramsList, nil)
for i := range needGen {
if !needGen[i] {
t.Fatalf("needGen[%d] 期望 true(无复用)", i)
}
}
}
// mergeSegmentOutputs:复用段 + 新生段按段序号升序,顺序即拼接顺序
func TestMergeSegmentOutputs(t *testing.T) {
idxList := []int{1, 2, 3, 4, 5}
needGen := []bool{false, false, false, true, false}
newRes := make([][]map[string]any, 5)
newRes[3] = []map[string]any{{"video_url": "new4"}}
saved := map[int]entity.SegmentRef{
1: {Key: "video_url", URL: "u1"}, 2: {Key: "video_url", URL: "u2"},
3: {Key: "video_url", URL: "u3"}, 5: {Key: "video_url", URL: "u5"},
}
got := mergeSegmentOutputs(idxList, needGen, newRes, saved)
if len(got) != 5 {
t.Fatalf("合并后应有 5 段,实际 %d: %v", len(got), got)
}
wantURLs := []string{"u1", "u2", "u3", "new4", "u5"}
for i, rec := range got {
if rec["video_url"] != wantURLs[i] {
t.Fatalf("第 %d 段=%v 期望 %s", i, rec, wantURLs[i])
}
}
}
// mergeSegmentOutputs:复用段 key 保持模型原字段,重建记录与新生记录 key 不一致也不影响顺序
func TestMergeSegmentOutputsMixedKeys(t *testing.T) {
idxList := []int{1, 2, 3}
needGen := []bool{false, true, false}
newRes := make([][]map[string]any, 3)
newRes[1] = []map[string]any{{"video_oss_url": "new2"}}
saved := map[int]entity.SegmentRef{
1: {Key: "video_url", URL: "u1"}, 3: {Key: "file_url", URL: "u3"},
}
got := mergeSegmentOutputs(idxList, needGen, newRes, saved)
if len(got) != 3 {
t.Fatalf("合并后应有 3 段,实际 %d: %v", len(got), got)
}
if got[0]["video_url"] != "u1" || got[1]["video_oss_url"] != "new2" || got[2]["file_url"] != "u3" {
t.Fatalf("合并结果不符: %v", got)
}
}
// mergeSegmentOutputsparamsList 乱序时仍按段序号升序输出(顺序保证)
func TestMergeSegmentOutputsUnorderedParams(t *testing.T) {
idxList := []int{3, 1, 2}
needGen := []bool{true, false, true}
newRes := make([][]map[string]any, 3)
newRes[0] = []map[string]any{{"video_url": "new3"}}
newRes[2] = []map[string]any{{"video_url": "new2"}}
saved := map[int]entity.SegmentRef{1: {Key: "video_url", URL: "u1"}}
got := mergeSegmentOutputs(idxList, needGen, newRes, saved)
if len(got) != 3 {
t.Fatalf("合并后应有 3 段,实际 %d: %v", len(got), got)
}
wantURLs := []string{"u1", "new2", "new3"}
for i, rec := range got {
if rec["video_url"] != wantURLs[i] {
t.Fatalf("第 %d 段=%v 期望 %s(应段序升序)", i, rec, wantURLs[i])
}
}
}