From 67d049e58679314a76a9ea0334ede8b7ab7202eb Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Sat, 29 Aug 2026 11:51:30 +0800 Subject: [PATCH] =?UTF-8?q?refactor(workflow):=20=E7=BB=9F=E4=B8=80=20OSS?= =?UTF-8?q?=20=E6=8E=A5=E5=8F=A3=E5=B9=B6=E9=87=8D=E6=9E=84=E6=81=A2?= =?UTF-8?q?=E5=A4=8D=E9=94=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 将文件地址前缀与上传逻辑迁移至 common/oss - 恢复执行改用 utils.WithLock 自动续期锁 - 转写提示词增加单镜头最小时长约束 --- gateway/file.go | 46 +--- workflow/service/creation_info_service.go | 3 +- workflow/service/einograph/lambda_func.go | 24 +- .../service/flow/flow_execution_service.go | 5 +- workflow/service/flow/flow_ws_exec.go | 4 +- workflow/service/flow/lambda_node.go | 4 +- workflow/service/flow/lambda_node_util.go | 9 +- .../service/flow/lambda_script_transcribe.go | 20 +- .../flow/processor/builtin/media/media.go | 4 +- workflow/service/flow/recover_execution.go | 223 +++++++++--------- workflow/service/flow/redis_lock.go | 43 ---- workflow/service/session/session_service.go | 5 +- workflow/service/skill/skill_user_service.go | 9 +- 13 files changed, 173 insertions(+), 226 deletions(-) delete mode 100644 workflow/service/flow/redis_lock.go diff --git a/gateway/file.go b/gateway/file.go index 85c68c8..22703c7 100644 --- a/gateway/file.go +++ b/gateway/file.go @@ -1,65 +1,27 @@ package gateway import ( - "bytes" "context" "io" - "mime/multipart" "net/http" "strings" - commonHttp "gitea.redpowerfuture.com/red-future/common/http" + "gitea.redpowerfuture.com/red-future/common/oss" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/util/gconv" ) -// Upload 以 multipart 方式上传文件字节到 OSS,返回可访问 URL。 +// Upload 上传文件字节到 OSS,返回可访问 URL。 // 原签名基于 workflow/model/dto 的 UploadFileBytesReq/Res,抽离时简化为直接传文件名与字节,便于业务侧解耦 workflow。 +// 统一走 common/oss:multipart field=file、X-User-Info 三态注入(透传请求头 / ctx 注入 user / 解析 token)。 func Upload(ctx context.Context, fileName string, fileBytes []byte) (string, error) { - body := &bytes.Buffer{} - writer := multipart.NewWriter(body) - - part, err := writer.CreateFormFile("file", fileName) + res, err := oss.UploadFileBytes(ctx, fileName, fileBytes) if err != nil { return "", err } - if _, err = part.Write(fileBytes); err != nil { - return "", err - } - if err = writer.Close(); err != nil { - return "", err - } - - headers := make(map[string]string) - if r := g.RequestFromCtx(ctx); r != nil { - for k, v := range r.Request.Header { - if len(v) > 0 { - headers[k] = v[0] - } - } - } - // 后台恢复续跑无 HTTP 请求时(ctx 携带合成 user),补充 X-User-Info 供 OSS GetUserInfo 识别用户与桶名; - // 与 gateway/model.go requestHeaders 的恢复兜底保持一致,否则 OSS GetBucketName 落到空 token 解析报错 - if headers["X-User-Info"] == "" { - if u := ctx.Value("user"); u != nil { - headers["X-User-Info"] = gconv.String(u) - } - } - headers["Content-Type"] = writer.FormDataContentType() - - res := &uploadFileRes{} - if err = commonHttp.Post(ctx, "oss/file/uploadFile", headers, res, body.Bytes()); err != nil { - return "", err - } return res.FileURL, nil } -// uploadFileRes OSS 上传响应的最小字段(原 workflow/model/dto.UploadFileBytesRes 的 URL 部分)。 -type uploadFileRes struct { - FileURL string `json:"fileURL"` -} - // GetFileBytesFromURL 下载远程文件内容(把 filePrefix 前缀替换为 minioPrefix 后经 GoFrame 客户端下载) func GetFileBytesFromURL(ctx context.Context, fileUrl string) ([]byte, error) { newS := strings.ReplaceAll(fileUrl, g.Cfg().MustGet(ctx, "filePrefix").String(), g.Cfg().MustGet(ctx, "minioPrefix").String()) diff --git a/workflow/service/creation_info_service.go b/workflow/service/creation_info_service.go index 4138a23..d0c11a8 100644 --- a/workflow/service/creation_info_service.go +++ b/workflow/service/creation_info_service.go @@ -9,6 +9,7 @@ import ( "fmt" "sort" + "gitea.redpowerfuture.com/red-future/common/oss" "gitea.redpowerfuture.com/red-future/common/utils" "github.com/gogf/gf/v2/util/gconv" ) @@ -66,7 +67,7 @@ func (s *creationInfoService) List(ctx context.Context, req *dto.ListCreationInf res = &dto.ListCreationInfoRes{ Total: total, } - res.ImgAddressPrefix, err = utils.GetFileAddressPrefix(ctx) + res.ImgAddressPrefix, err = oss.GetFileAddressPrefix(ctx) if err != nil { return } diff --git a/workflow/service/einograph/lambda_func.go b/workflow/service/einograph/lambda_func.go index 10df44f..736fb58 100644 --- a/workflow/service/einograph/lambda_func.go +++ b/workflow/service/einograph/lambda_func.go @@ -9,8 +9,7 @@ import ( "net/http" "strings" - commonHttp "gitea.redpowerfuture.com/red-future/common/http" - "gitea.redpowerfuture.com/red-future/common/utils" + "gitea.redpowerfuture.com/red-future/common/oss" "github.com/cloudwego/eino/schema" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/glog" @@ -260,7 +259,7 @@ func GenerateImageLambda(ctx context.Context, input any) (any, error) { items = append(items, s) } } - imgAddressPrefix, err := utils.GetFileAddressPrefix(ctx) + imgAddressPrefix, err := oss.GetFileAddressPrefix(ctx) if err != nil { return nil, err @@ -393,17 +392,18 @@ func getImageBytesFromURL(url string) ([]byte, error) { } func Upload(ctx context.Context, req *dto.UploadFileBytesReq) (*dto.UploadFileBytesRes, error) { - headers := make(map[string]string) - if r := g.RequestFromCtx(ctx); r != nil { - for k, v := range r.Header { - headers[k] = v[0] - } - } - res := &dto.UploadFileBytesRes{} - err := commonHttp.Post(ctx, "oss/file/uploadFileBytes", headers, res, req) + // 统一走 common/oss:旧实现 POST oss/file/uploadFileBytes 是坏链(oss 服务从未注册该路由,必然 404), + // common/oss 打真实 oss/file/uploadFile,且 multipart field=file、X-User-Info 注入与旧透传等价 + res, err := oss.UploadFileBytes(ctx, req.FileName, req.FileBytes) if err != nil { glog.Error(ctx, err) return nil, err } - return res, nil + return &dto.UploadFileBytesRes{ + FileURL: res.FileURL, + FileSize: res.FileSize, + FileName: res.FileName, + FileFormat: res.FileFormat, + FileAddressPrefix: res.FileAddressPrefix, + }, nil } diff --git a/workflow/service/flow/flow_execution_service.go b/workflow/service/flow/flow_execution_service.go index da2edcc..6b70582 100644 --- a/workflow/service/flow/flow_execution_service.go +++ b/workflow/service/flow/flow_execution_service.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" + "gitea.redpowerfuture.com/red-future/common/oss" "gitea.redpowerfuture.com/red-future/common/utils" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gtime" @@ -25,7 +26,7 @@ func (s *flowExecutionService) Get(ctx context.Context, req *flowDto.GetFlowExec return nil, err } res = new(flowDto.VOFlowExecution) - res.ImgAddressPrefix, err = utils.GetFileAddressPrefix(ctx) + res.ImgAddressPrefix, err = oss.GetFileAddressPrefix(ctx) if err != nil { return nil, err } @@ -155,7 +156,7 @@ func (s *flowExecutionService) List(ctx context.Context, req *flowDto.ListFlowEx return tree[i].CreateDate > tree[j].CreateDate }) - imgPrefix, err := utils.GetFileAddressPrefix(ctx) + imgPrefix, err := oss.GetFileAddressPrefix(ctx) return &flowDto.ListFlowExecutionTreeRes{ Tree: tree, ImgAddressPrefix: imgPrefix, diff --git a/workflow/service/flow/flow_ws_exec.go b/workflow/service/flow/flow_ws_exec.go index b0ff776..6f9d404 100644 --- a/workflow/service/flow/flow_ws_exec.go +++ b/workflow/service/flow/flow_ws_exec.go @@ -14,7 +14,7 @@ import ( "fmt" "time" - "gitea.redpowerfuture.com/red-future/common/utils" + "gitea.redpowerfuture.com/red-future/common/oss" wsCommon "gitea.redpowerfuture.com/red-future/common/websocket" "github.com/cloudwego/eino/compose" "github.com/gogf/gf/v2/frame/g" @@ -298,7 +298,7 @@ func workflowResultFileUrls(ctx context.Context, execId int64) []string { glog.Errorf(ctx, "查询工作流结果路径失败: %v", err) return nil } - prefix, _ := utils.GetFileAddressPrefix(ctx) + prefix, _ := oss.GetFileAddressPrefix(ctx) urls := make([]string, 0, len(results)) for _, r := range results { if r.ResultFileUrl != "" { diff --git a/workflow/service/flow/lambda_node.go b/workflow/service/flow/lambda_node.go index 95a0192..583e0c9 100644 --- a/workflow/service/flow/lambda_node.go +++ b/workflow/service/flow/lambda_node.go @@ -23,7 +23,7 @@ import ( "sync" "gitea.redpowerfuture.com/red-future/common/db/gfdb" - "gitea.redpowerfuture.com/red-future/common/utils" + "gitea.redpowerfuture.com/red-future/common/oss" "github.com/cloudwego/eino-examples/compose/batch/batch" "github.com/cloudwego/eino/compose" "github.com/gogf/gf/v2/database/gdb" @@ -670,7 +670,7 @@ func resolveFileContent(val any) (isPath bool, path string, fileBytes []byte, ex return true, s, nil, "" } // MinIO 对象裸路径(无 http 前缀,模型网关转存 OSS 后返回) - if utils.IsOSSPath(s) { + if oss.IsOSSPath(s) { return true, s, nil, "" } // data URI:data:;base64, diff --git a/workflow/service/flow/lambda_node_util.go b/workflow/service/flow/lambda_node_util.go index ec5195e..8d0fc3b 100644 --- a/workflow/service/flow/lambda_node_util.go +++ b/workflow/service/flow/lambda_node_util.go @@ -13,6 +13,7 @@ import ( "unicode/utf8" commonHttp "gitea.redpowerfuture.com/red-future/common/http" + "gitea.redpowerfuture.com/red-future/common/oss" "gitea.redpowerfuture.com/red-future/common/utils" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" @@ -249,12 +250,16 @@ func HttpCallResultLambda(ctx context.Context, nodeInput *flowDto.NodeExecutionI // addFilePathPrefix 递归把 body 中的 MinIO 裸路径(模型网关转存 OSS 后返回,无 http 前缀)补上文件前缀, // 供目标 HTTP 服务直接下载文件;已是完整 URL 的值保持不变 func addFilePathPrefix(ctx context.Context, url string, body map[string]any) { - prefix, err := utils.GetFileAddressPrefix(ctx) + prefix, err := oss.GetFileAddressPrefix(ctx) if err != nil { g.Log().Warningf(ctx, "获取文件前缀失败,保持原路径: %v", err) return } for k, v := range body { + if k == "templates" && g.IsEmpty(v) { + delete(body, k) + continue + } body[k] = prependFilePathPrefix(prefix, v) } // template/template 模板接口要求 video_urls 为数组:标量值包装为单元素数组 @@ -290,7 +295,7 @@ func toVideoURLsArray(v any) any { func prependFilePathPrefix(prefix string, v any) any { switch val := v.(type) { case string: - if utils.IsOSSPath(val) { + if oss.IsOSSPath(val) { return prefix + val } return val diff --git a/workflow/service/flow/lambda_script_transcribe.go b/workflow/service/flow/lambda_script_transcribe.go index aee5ca4..52dcbc5 100644 --- a/workflow/service/flow/lambda_script_transcribe.go +++ b/workflow/service/flow/lambda_script_transcribe.go @@ -101,12 +101,13 @@ func ScriptTranscribeLambda(ctx context.Context, input any) (any, error) { if totalDuration > 0 { systemPrompt += fmt.Sprintf("\n\n视频总时长 %d 秒(MM:SS 为 %s):所有镜头的时间码需前后衔接并完整覆盖该总时长,最后一镜的 endTime 对齐到总时长。", totalDuration, formatSecondsToMMSS(totalDuration)) } - // 单镜头时长上限约束:按视频模型推导单段最大时长注入转写提示词,从源头避免超长镜头 - //(SplitOversized 仍是机械兜底);推导失败仅降级跳过约束注入,不影响转写主流程。 - if maxSeg, _, err := split_shots_pipeline.SegmentBounds(ctx, modelId); err != nil { + // 单镜头时长约束:按视频模型推导单段最大/最小时长注入转写提示词,从源头避免超长/超短镜头 + //(SplitOversized / GroupSegments 咬取补齐仍是机械兜底);推导失败仅降级跳过约束注入,不影响转写主流程。 + if maxSeg, minSeg, err := split_shots_pipeline.SegmentBounds(ctx, modelId); err != nil { g.Log().Warningf(ctx, "获取视频模型单段时长约束失败,跳过单镜时长约束注入: %v", err) } else { systemPrompt += shotDurationConstraintPrompt(maxSeg) + systemPrompt += shotMinDurationConstraintPrompt(minSeg, totalDuration) } // 静音模式硬约束:从转写源头杜绝对白/旁白/开口说话,后续清洗只做兜底 if noSpeech { @@ -373,6 +374,19 @@ func shotDurationConstraintPrompt(maxSeg int) string { return fmt.Sprintf("\n\n单个镜头时长不超过 %d 秒:每镜的 startTime 与 endTime 之差必须 ≤ %d 秒。", maxSeg, maxSeg) } +// shotMinDurationConstraintPrompt 生成"单个镜头时长不少于 minSeg 秒"的转写约束提示词片段。 +// minSeg<=0 返回空串;minSeg 超过视频总时长时也返回空串——此时与"末镜 endTime 对齐总时长"的约束 +// 自相矛盾、模型无法满足,强行注入反而会让模型困惑(GroupSegments 对末段残余本就放行短于 min)。 +func shotMinDurationConstraintPrompt(minSeg, totalDuration int) string { + if minSeg <= 0 { + return "" + } + if totalDuration > 0 && minSeg > totalDuration { + return "" + } + return fmt.Sprintf("\n\n单个镜头时长不少于 %d 秒:每镜的 startTime 与 endTime 之差必须 ≥ %d 秒。", minSeg, minSeg) +} + // noSpeechSystemPromptConstraint 静音模式的转写硬约束:要求模型从源头就不产出对白/旁白/说话动词, // 后续 noSpeech 清洗(清空台词旁白 + cleanSpeechVerbs)只做机械兜底。 func noSpeechSystemPromptConstraint() string { diff --git a/workflow/service/flow/processor/builtin/media/media.go b/workflow/service/flow/processor/builtin/media/media.go index 75e4f43..f54e1be 100644 --- a/workflow/service/flow/processor/builtin/media/media.go +++ b/workflow/service/flow/processor/builtin/media/media.go @@ -8,7 +8,7 @@ import ( "time" commonHttp "gitea.redpowerfuture.com/red-future/common/http" - "gitea.redpowerfuture.com/red-future/common/utils" + "gitea.redpowerfuture.com/red-future/common/oss" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/util/gconv" ) @@ -300,7 +300,7 @@ func normalizeVideoURL(ctx context.Context, url string) string { if url == "" || strings.HasPrefix(url, "http") { return url } - prefix, err := utils.GetFileAddressPrefix(ctx) + prefix, err := oss.GetFileAddressPrefix(ctx) if err != nil { g.Log().Warningf(ctx, "获取文件前缀失败,视频地址保持相对路径: %s err=%v", url, err) return url diff --git a/workflow/service/flow/recover_execution.go b/workflow/service/flow/recover_execution.go index 29e5e0e..a5a2f97 100644 --- a/workflow/service/flow/recover_execution.go +++ b/workflow/service/flow/recover_execution.go @@ -13,6 +13,7 @@ import ( "ai-agent/workflow/model/entity" "gitea.redpowerfuture.com/red-future/common/beans" + "gitea.redpowerfuture.com/red-future/common/utils" wsCommon "gitea.redpowerfuture.com/red-future/common/websocket" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/gtrace" @@ -135,12 +136,123 @@ func recoverExecution(parentCtx context.Context, execId int64, attach *execAttac subscribeConnToHub(attach.conn, hub) } - lock := NewRedisLock(fmt.Sprintf("workflow:exec:recover:%d", execId), int64(recoverLockTTL/time.Second)) + lockKey := fmt.Sprintf("workflow:exec:recover:%d", execId) // 抢锁/前置判定用独立短超时 ctx:该阶段只做 Redis SET + 一次 DB 读 + 一次条件重置,应毫秒级完成 lockCtx, lockCancel := context.WithTimeout(context.WithoutCancel(parentCtx), recoverLockPhase) defer lockCancel() + // 恢复体整体包进 utils.WithLock(自动续期 + 单次尝试,替代本地对象形态锁 redis_lock.go): + // - 自动续期:15min TTL 覆盖整段执行。长执行原本就靠心跳陈旧 + 条件重置防双跑, + // 锁持满只是让其它节点提前「锁忙跳过」;节点崩溃续期停 → TTL 过期 → 其它节点照常捞起; + // - 单次尝试(retryTimes=1):原「被其它节点持有就跳过」语义,抢不到不等待。 + ok, err := utils.WithLock(lockCtx, lockKey, int64(recoverLockTTL/time.Second), func(_ context.Context) error { - ok, err := lock.Acquire(lockCtx) + // 抢锁后重读:此刻租户未知(该 exec 可能属于任意租户),必须跨租户读 + exec, err := sessionDao.ExecWorkflowDao.GetByIdNoTenant(lockCtx, execId) + if err != nil || exec == nil { + return nil + } + if !isRecoverable(exec, time.Now().UnixMilli()) { + return nil // 锁等待期间状态已变化(其它节点已恢复/用户取消/已耗尽) + } + + // 事件中枢(scan 路径,无附着连接):此刻才知道 exec 的 session+flow,建 hub 供后续用户连接附着 + //(所有权在重置成功后统一接管,见下) + if attach == nil { + hub, _ = registerHubIfAbsent(exec.SessionId, exec.FlowId, newExecHub(exec.SessionId, exec.FlowId)) + } + + // 置运行中并续跑(BuildExecution false → 断点续跑:checkpoint 有则从中继续,无则从头 + 段/memo 复用)。 + // 心跳在条件重置成功后才启动:避免对未抢到重置权的 exec 空跑心跳 + saveCtx := context.WithoutCancel(parentCtx) + // 恢复无 HTTP 用户,但图节点 lambda 的 INSERT(node_execution/flow_async_task/segment_result)走 + // insertHook 硬性要求 user;用 exec 所属租户合成系统用户 ctx(保留 span),供后续落库使用 + userCtx := context.WithValue(saveCtx, "user", &beans.User{UserName: exec.Creator, TenantId: exec.TenantId}) + nodeGroupId := uuid.NewString() // 置运行中与图执行的节点组标识须一致 + // 条件重置(原子防与用户断点续跑双跑):仅当仍可恢复(status=3 或 status=1 心跳陈旧)时才抢到重置权 + staleBeforeMs := time.Now().UnixMilli() - int64(heartbeatStaleAfter/time.Millisecond) + reset, err := sessionDao.ExecWorkflowDao.ResetRunningIfRecoverable(userCtx, execId, nodeGroupId, staleBeforeMs) + if err != nil { + g.Log().Errorf(lockCtx, "恢复置运行中失败 execId=%d: %v", execId, err) + return nil + } + if !reset { + // 状态已被其它路径(用户 reExecute/execute 或其它节点恢复)抢先重置:放弃续跑,状态由持有方收敛,不落终态 + g.Log().Infof(lockCtx, "execId=%d 已被其它路径抢先重置为运行中,跳过恢复", execId) + return nil + } + + // 事件中枢所有权:此时已抢到锁+重置权,本 goroutine 是实际执行者,接管 hub(TryOwn 原子置 owned+cancel)。 + // 已有持有者(同 session+flow 另有执行在跑)则本恢复不持有 hub:锁与重置权已到手,放弃会留 status=1 + // 无主,继续执行但进度不广播(恢复仍正常完成落终态)。 + if hub != nil { + if !hub.TryOwn(topCancel) { + hub = nil + } else { + defer hub.Close() + // 恢复建立前用户已取消(占位期 workflow_cancel 只置标志、cancel 尚未设置):立即中止, + // 走下方 UserCancelled 分类写永久取消(retryable=0) + if hub.UserCancelled() { + topCancel() + } + } + } + + // 执行与心跳绑定同一 execCtx(带 recoverExecTimeout 上限): + // 心跳停止(超时/进程死/租约丢失)与 BuildExecution 中止必须同步,否则出现 + // "心跳已过期但执行还活着" 的窗口,被其它节点扫描恢复导致双跑。 + // 心跳连续失败达陈旧阈值时回调 execCancel 中止执行(心跳与执行同生共死)。 + // 从函数顶部登记的 topCtx 派生执行 ctx:注入用户信息(供 insertHook 落库)+ 12h 执行超时上限。 + // 关停时 cancelAllExecRuns 取消 topCancel → 本 ctx 随之取消,BuildExecution 中止后走下方错误分类落终态。 + execCtx, execCancel := context.WithTimeout(context.WithValue(topCtx, "user", &beans.User{UserName: exec.Creator, TenantId: exec.TenantId}), recoverExecTimeout) + defer execCancel() + stop := startHeartbeat(execCtx, execId, execCancel) + defer stop() + // 图节点进度经 hub 广播(无 hub 时 getProgressHub 返回 nil,reporter nil 安全) + progressCtx := execCtx + if hub != nil { + progressCtx = context.WithValue(execCtx, wsProgressCtxKey{}, hub) + } + err = BuildExecution(progressCtx, false, exec.FlowId, execId, nodeGroupId, exec.SessionId, exec.RequestParams) + if err != nil { + // 用户显式取消(附着连接 workflow_cancel):永久取消,retryable=0,恢复扫描不再捞起, + // 杜绝"取消→恢复→再取消"循环;清 flow_async_task 孤儿缓存,落"用户已终止执行" + if hub != nil && hub.UserCancelled() { + retryable, retryCnt := 0, 0 + _ = flowDao.FlowAsyncTaskDao.DeleteByExecution(userCtx, execId) + recordWorkflow(userCtx, execId, 0, context.Canceled, &retryable, &retryCnt) + hub.Publish(&wsCommon.WsPushMsg{Type: "error", Message: errWorkflowTerminated}) + return nil + } + // 续跑失败:恢复例程无用户,任意错误(含执行超时/租约丢失取消/模型/DB/网络/panic) + // 一律 retryable=1 交下一轮扫描决定是否再恢复;重试耗尽才终局失败。 + retryable, retryCnt := 1, exec.RetryCount+1 + // 优雅关停导致的取消:换错误标记让 recordWorkflow 写"程序关停中断"(与 WS 路径一致), + // 仍 retryable=1 下次启动扫描捞起续跑;非关停的取消(租约丢失/执行超时)保留原错误 + if errors.Is(err, context.Canceled) && IsShuttingDown() { + err = errInterruptedByShutdown + } + // 终局清理(Task 5 用户裁定):重试耗尽 → 执行永久失败, + // 清理该 exec 残留的 flow_async_task 孤儿缓存,避免未来复用同一 execId 的运行误取到过期 done 结果 + if exec.RetryCount+1 >= execMaxRetryCount { + _ = flowDao.FlowAsyncTaskDao.DeleteByExecution(userCtx, execId) + } + recordWorkflow(userCtx, execId, 0, err, &retryable, &retryCnt) + // 终态广播(附着连接;scan 路径无连接则无人接收) + if hub != nil { + hub.Publish(&wsCommon.WsPushMsg{Type: "error", Message: "工作流执行失败", Error: err.Error()}) + } + return nil + } + // 续跑成功:recordWorkflow 落 status=2;BuildExecution 已清理 checkpoint/segment_result/flow_async_task + recordWorkflow(userCtx, execId, 0, nil, nil, nil) + // 终态广播:把本次执行保存的结果文件路径一并推给前端 + if hub != nil { + hub.Publish(&wsCommon.WsPushMsg{Type: "flow_complete", Message: "工作流执行完成", Data: map[string]interface{}{ + "resultFileUrls": workflowResultFileUrls(userCtx, execId), + }}) + } + return nil + }) if err != nil { g.Log().Errorf(lockCtx, "恢复抢锁失败 execId=%d: %v", execId, err) return @@ -149,113 +261,6 @@ func recoverExecution(parentCtx context.Context, execId int64, attach *execAttac g.Log().Infof(lockCtx, "恢复锁被其它节点持有,跳过 execId=%d", execId) return } - defer lock.Release(context.Background()) - - // 抢锁后重读:此刻租户未知(该 exec 可能属于任意租户),必须跨租户读 - exec, err := sessionDao.ExecWorkflowDao.GetByIdNoTenant(lockCtx, execId) - if err != nil || exec == nil { - return - } - if !isRecoverable(exec, time.Now().UnixMilli()) { - return // 锁等待期间状态已变化(其它节点已恢复/用户取消/已耗尽) - } - - // 事件中枢(scan 路径,无附着连接):此刻才知道 exec 的 session+flow,建 hub 供后续用户连接附着 - //(所有权在重置成功后统一接管,见下) - if attach == nil { - hub, _ = registerHubIfAbsent(exec.SessionId, exec.FlowId, newExecHub(exec.SessionId, exec.FlowId)) - } - - // 置运行中并续跑(BuildExecution false → 断点续跑:checkpoint 有则从中继续,无则从头 + 段/memo 复用)。 - // 心跳在条件重置成功后才启动:避免对未抢到重置权的 exec 空跑心跳 - saveCtx := context.WithoutCancel(parentCtx) - // 恢复无 HTTP 用户,但图节点 lambda 的 INSERT(node_execution/flow_async_task/segment_result)走 - // insertHook 硬性要求 user;用 exec 所属租户合成系统用户 ctx(保留 span),供后续落库使用 - userCtx := context.WithValue(saveCtx, "user", &beans.User{UserName: exec.Creator, TenantId: exec.TenantId}) - nodeGroupId := uuid.NewString() // 置运行中与图执行的节点组标识须一致 - // 条件重置(原子防与用户断点续跑双跑):仅当仍可恢复(status=3 或 status=1 心跳陈旧)时才抢到重置权 - staleBeforeMs := time.Now().UnixMilli() - int64(heartbeatStaleAfter/time.Millisecond) - reset, err := sessionDao.ExecWorkflowDao.ResetRunningIfRecoverable(userCtx, execId, nodeGroupId, staleBeforeMs) - if err != nil { - g.Log().Errorf(lockCtx, "恢复置运行中失败 execId=%d: %v", execId, err) - return - } - if !reset { - // 状态已被其它路径(用户 reExecute/execute 或其它节点恢复)抢先重置:放弃续跑,状态由持有方收敛,不落终态 - g.Log().Infof(lockCtx, "execId=%d 已被其它路径抢先重置为运行中,跳过恢复", execId) - return - } - - // 事件中枢所有权:此时已抢到锁+重置权,本 goroutine 是实际执行者,接管 hub(TryOwn 原子置 owned+cancel)。 - // 已有持有者(同 session+flow 另有执行在跑)则本恢复不持有 hub:锁与重置权已到手,放弃会留 status=1 - // 无主,继续执行但进度不广播(恢复仍正常完成落终态)。 - if hub != nil { - if !hub.TryOwn(topCancel) { - hub = nil - } else { - defer hub.Close() - // 恢复建立前用户已取消(占位期 workflow_cancel 只置标志、cancel 尚未设置):立即中止, - // 走下方 UserCancelled 分类写永久取消(retryable=0) - if hub.UserCancelled() { - topCancel() - } - } - } - - // 执行与心跳绑定同一 execCtx(带 recoverExecTimeout 上限): - // 心跳停止(超时/进程死/租约丢失)与 BuildExecution 中止必须同步,否则出现 - // "心跳已过期但执行还活着" 的窗口,被其它节点扫描恢复导致双跑。 - // 心跳连续失败达陈旧阈值时回调 execCancel 中止执行(心跳与执行同生共死)。 - // 从函数顶部登记的 topCtx 派生执行 ctx:注入用户信息(供 insertHook 落库)+ 12h 执行超时上限。 - // 关停时 cancelAllExecRuns 取消 topCancel → 本 ctx 随之取消,BuildExecution 中止后走下方错误分类落终态。 - execCtx, execCancel := context.WithTimeout(context.WithValue(topCtx, "user", &beans.User{UserName: exec.Creator, TenantId: exec.TenantId}), recoverExecTimeout) - defer execCancel() - stop := startHeartbeat(execCtx, execId, execCancel) - defer stop() - // 图节点进度经 hub 广播(无 hub 时 getProgressHub 返回 nil,reporter nil 安全) - progressCtx := execCtx - if hub != nil { - progressCtx = context.WithValue(execCtx, wsProgressCtxKey{}, hub) - } - err = BuildExecution(progressCtx, false, exec.FlowId, execId, nodeGroupId, exec.SessionId, exec.RequestParams) - if err != nil { - // 用户显式取消(附着连接 workflow_cancel):永久取消,retryable=0,恢复扫描不再捞起, - // 杜绝"取消→恢复→再取消"循环;清 flow_async_task 孤儿缓存,落"用户已终止执行" - if hub != nil && hub.UserCancelled() { - retryable, retryCnt := 0, 0 - _ = flowDao.FlowAsyncTaskDao.DeleteByExecution(userCtx, execId) - recordWorkflow(userCtx, execId, 0, context.Canceled, &retryable, &retryCnt) - hub.Publish(&wsCommon.WsPushMsg{Type: "error", Message: errWorkflowTerminated}) - return - } - // 续跑失败:恢复例程无用户,任意错误(含执行超时/租约丢失取消/模型/DB/网络/panic) - // 一律 retryable=1 交下一轮扫描决定是否再恢复;重试耗尽才终局失败。 - retryable, retryCnt := 1, exec.RetryCount+1 - // 优雅关停导致的取消:换错误标记让 recordWorkflow 写"程序关停中断"(与 WS 路径一致), - // 仍 retryable=1 下次启动扫描捞起续跑;非关停的取消(租约丢失/执行超时)保留原错误 - if errors.Is(err, context.Canceled) && IsShuttingDown() { - err = errInterruptedByShutdown - } - // 终局清理(Task 5 用户裁定):重试耗尽 → 执行永久失败, - // 清理该 exec 残留的 flow_async_task 孤儿缓存,避免未来复用同一 execId 的运行误取到过期 done 结果 - if exec.RetryCount+1 >= execMaxRetryCount { - _ = flowDao.FlowAsyncTaskDao.DeleteByExecution(userCtx, execId) - } - recordWorkflow(userCtx, execId, 0, err, &retryable, &retryCnt) - // 终态广播(附着连接;scan 路径无连接则无人接收) - if hub != nil { - hub.Publish(&wsCommon.WsPushMsg{Type: "error", Message: "工作流执行失败", Error: err.Error()}) - } - return - } - // 续跑成功:recordWorkflow 落 status=2;BuildExecution 已清理 checkpoint/segment_result/flow_async_task - recordWorkflow(userCtx, execId, 0, nil, nil, nil) - // 终态广播:把本次执行保存的结果文件路径一并推给前端 - if hub != nil { - hub.Publish(&wsCommon.WsPushMsg{Type: "flow_complete", Message: "工作流执行完成", Data: map[string]interface{}{ - "resultFileUrls": workflowResultFileUrls(userCtx, execId), - }}) - } } // isRecoverable 判定可恢复(spec §3):僵尸运行中(status=1 且心跳陈旧)或可重试失败(status=3 且 retryable=1 且未耗尽)。 diff --git a/workflow/service/flow/redis_lock.go b/workflow/service/flow/redis_lock.go deleted file mode 100644 index 33686fd..0000000 --- a/workflow/service/flow/redis_lock.go +++ /dev/null @@ -1,43 +0,0 @@ -package flow - -import ( - "context" - - "github.com/gogf/gf/v2/database/gredis" - "github.com/gogf/gf/v2/frame/g" - "github.com/google/uuid" -) - -// RedisLock 基于 SETNX 的分布式锁:token 标识持有者身份,Lua 原子释放防误删他人锁。 -// 用途:恢复例程抢锁防多节点对同一 exec 重复恢复(spec §6) -type RedisLock struct { - key string - token string - ttl int64 // 秒 -} - -func NewRedisLock(key string, ttlSec int64) *RedisLock { - return &RedisLock{key: key, token: uuid.NewString(), ttl: ttlSec} -} - -// Acquire 抢锁;返回 true 表示抢到(SET NX 成功)。 -// 注意:gogf v2.10.2 的 SetNX(ctx,key,value) 不接收 TTL 参数,直接 SETNX 会永不过期(崩溃后死锁); -// 故改用 Set + SetOption{NX,TTLOption{EX}},原子地执行 `SET key value EX NX`。 -func (l *RedisLock) Acquire(ctx context.Context) (bool, error) { - r, err := g.Redis().Set(ctx, l.key, l.token, gredis.SetOption{ - TTLOption: gredis.TTLOption{EX: &l.ttl}, - NX: true, - }) - if err != nil { - return false, err - } - // SET NX 失败时 Redis 返回 nil(空回复),gogf 会转换为值 nil 的 gvar;成功时返回 "OK" - return !r.IsNil(), nil -} - -// Release 释放锁:仅当 key 值仍为本锁 token 时删除(Lua 原子),防止超时后误删他人已续的锁 -func (l *RedisLock) Release(ctx context.Context) error { - const luaRelease = `if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end` - _, err := g.Redis().Eval(ctx, luaRelease, 1, []string{l.key}, []any{l.token}) - return err -} diff --git a/workflow/service/session/session_service.go b/workflow/service/session/session_service.go index 87e05ae..8df79ca 100644 --- a/workflow/service/session/session_service.go +++ b/workflow/service/session/session_service.go @@ -15,6 +15,7 @@ import ( "strings" "gitea.redpowerfuture.com/red-future/common/beans" + "gitea.redpowerfuture.com/red-future/common/oss" "gitea.redpowerfuture.com/red-future/common/utils" "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/util/gconv" @@ -96,7 +97,7 @@ func (s *sessionService) Get(ctx context.Context, req *sessionDto.GetSessionInfo if err != nil { return } - prefix, _ := utils.GetFileAddressPrefix(ctx) + prefix, _ := oss.GetFileAddressPrefix(ctx) // 工作流结果按 exec_id 分组,合并到对应执行记录的结果文件URL resultByExec := make(map[int64][]string) for _, wr := range wfResultList { @@ -230,7 +231,7 @@ func (s *sessionService) ResultList(ctx context.Context, req *sessionDto.ListWor return } res = &flowDto.ListFlowExecutionTreeRes{} - res.ImgAddressPrefix, _ = utils.GetFileAddressPrefix(ctx) + res.ImgAddressPrefix, _ = oss.GetFileAddressPrefix(ctx) if len(dates) == 0 { return } diff --git a/workflow/service/skill/skill_user_service.go b/workflow/service/skill/skill_user_service.go index 3fd3602..5bd3b68 100644 --- a/workflow/service/skill/skill_user_service.go +++ b/workflow/service/skill/skill_user_service.go @@ -11,6 +11,7 @@ import ( "strings" "gitea.redpowerfuture.com/red-future/common/beans" + "gitea.redpowerfuture.com/red-future/common/oss" "gitea.redpowerfuture.com/red-future/common/utils" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/util/gconv" @@ -149,7 +150,7 @@ func (s *skillUserService) Get(ctx context.Context, req *skillDto.GetSkillUserRe return nil, err } res = &skillDto.SkillUserVO{} - res.ImgAddressPrefix, err = utils.GetFileAddressPrefix(ctx) + res.ImgAddressPrefix, err = oss.GetFileAddressPrefix(ctx) if err != nil { return nil, err } @@ -169,7 +170,7 @@ func (s *skillUserService) Get(ctx context.Context, req *skillDto.GetSkillUserRe return nil, err } res = &skillDto.SkillUserVO{} - res.ImgAddressPrefix, err = utils.GetFileAddressPrefix(ctx) + res.ImgAddressPrefix, err = oss.GetFileAddressPrefix(ctx) if err != nil { return nil, err } @@ -250,7 +251,7 @@ func (s *skillUserService) GetUserOrTemplate(ctx context.Context, req *skillDto. } if !g.IsEmpty(list) { res = &skillDto.SkillUserVO{} - res.ImgAddressPrefix, err = utils.GetFileAddressPrefix(ctx) + res.ImgAddressPrefix, err = oss.GetFileAddressPrefix(ctx) if err != nil { return nil, err } @@ -273,7 +274,7 @@ func (s *skillUserService) GetUserOrTemplate(ctx context.Context, req *skillDto. return nil, err } res = &skillDto.SkillUserVO{} - res.ImgAddressPrefix, err = utils.GetFileAddressPrefix(ctx) + res.ImgAddressPrefix, err = oss.GetFileAddressPrefix(ctx) if err != nil { return nil, err }