diff --git a/gateway/file.go b/gateway/file.go index d4aa7db..85c68c8 100644 --- a/gateway/file.go +++ b/gateway/file.go @@ -11,6 +11,7 @@ import ( commonHttp "gitea.redpowerfuture.com/red-future/common/http" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" ) // Upload 以 multipart 方式上传文件字节到 OSS,返回可访问 URL。 @@ -38,6 +39,13 @@ func Upload(ctx context.Context, fileName string, fileBytes []byte) (string, err } } } + // 后台恢复续跑无 HTTP 请求时(ctx 携带合成 user),补充 X-User-Info 供 OSS GetUserInfo 识别用户与桶名; + // 与 gateway/model.go requestHeaders 的恢复兜底保持一致,否则 OSS GetBucketName 落到空 token 解析报错 + if headers["X-User-Info"] == "" { + if u := ctx.Value("user"); u != nil { + headers["X-User-Info"] = gconv.String(u) + } + } headers["Content-Type"] = writer.FormDataContentType() res := &uploadFileRes{} diff --git a/gateway/model.go b/gateway/model.go index a57a74c..0a90925 100644 --- a/gateway/model.go +++ b/gateway/model.go @@ -101,6 +101,7 @@ type ModelTool struct { // requestHeaders 透传当前 HTTP 请求头(鉴权/链路信息)。 // 浏览器 WebSocket 握手无法携带 Authorization 头,前端把 token 放在握手 URL query(?token=)里; // 若请求头没有 Authorization,则从 query 补回,保证下游(model-gateway → admin-go)能拿到用户 token。 +// 后台恢复续跑无 HTTP 请求时(ctx 携带合成 user),补充 X-User-Info 供下游 GetUserInfo 识别租户。 func requestHeaders(ctx context.Context) map[string]string { headers := make(map[string]string) if r := g.RequestFromCtx(ctx); r != nil { @@ -115,6 +116,11 @@ func requestHeaders(ctx context.Context) map[string]string { } } } + if headers["X-User-Info"] == "" { + if u := ctx.Value("user"); u != nil { + headers["X-User-Info"] = gconv.String(u) + } + } return headers } diff --git a/gateway/model_split_test.go b/gateway/model_split_test.go deleted file mode 100644 index 6cca231..0000000 --- a/gateway/model_split_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package gateway - -import ( - "context" - "testing" - - "ai-agent/workflow/consts/model" -) - -// TestSplitSignatures 只做编译期签名验证:拆分后的函数存在且签名符合 memo 包装契约。 -// 不做实际调用(依赖 live model-gateway + Consul),实际行为由端到端走查覆盖。 -func TestSplitSignatures(t *testing.T) { - var _ func(context.Context, int64, model.ResponseType, string, map[string]any, map[string]any) (*ModelCallRes, string, error) = SubmitModelCall - var _ func(context.Context, string) (*ModelCallRes, error) = WaitModelCallResult -} diff --git a/main.go b/main.go index cbbb95e..078f843 100644 --- a/main.go +++ b/main.go @@ -21,6 +21,7 @@ import ( "os" "os/signal" "syscall" + "time" "gitea.redpowerfuture.com/red-future/common/http" "gitea.redpowerfuture.com/red-future/common/jaeger" @@ -77,11 +78,21 @@ func main() { // 监听退出信号,执行优雅关闭 sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + shutdownDone := make(chan struct{}) go func() { <-sigCh + // 先置关停标记再 Close:Close 会取消各连接 ctx,正在执行的 exec 会收到 context.Canceled; + // 标记让错误分类识别为"程序关停中断"(retryable=1),下次启动恢复扫描捞起续跑, + // 而非误判为"用户已终止执行"(retryable=0,永不恢复) + // SetShuttingDown 同时取消所有运行中执行(含脱离连接的恢复执行),保证全部落终态 + flow.SetShuttingDown() flow.SessionWsService.Close() + // 等待运行中执行落完终态再退出,避免进程退出时 exec 仍卡 status=1; + // 结束后返回 main 触发 deferred 资源清理,进程自然退出(不再 select{} 挂死) + flow.WaitExecRunsDrain(15 * time.Second) + close(shutdownDone) }() - // 保持应用运行 - select {} + // 保持应用运行;收到关停信号并落完终态后自然退出 + <-shutdownDone } diff --git a/workflow/dao/flow/flow_async_task_dao_test.go b/workflow/dao/flow/flow_async_task_dao_test.go deleted file mode 100644 index 0d235a9..0000000 --- a/workflow/dao/flow/flow_async_task_dao_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package flow - -import ( - "context" - "os" - "testing" - - "ai-agent/workflow/consts/public" - - "gitea.redpowerfuture.com/red-future/common/beans" - "gitea.redpowerfuture.com/red-future/common/db/gfdb" - _ "github.com/gogf/gf/contrib/drivers/pgsql/v2" -) - -// resultJSONB 是 JSONB 读回的规范化形式:PG 对 {"taskId":2026} 做 JSONB 归一化后输出 -// {"taskId": 2026}(冒号后带空格),直接比较字面量会失配,以实际读回为准。 -const resultJSONB = `{"taskId": 2026}` - -func TestFlowAsyncTaskDaoRoundTrip(t *testing.T) { - if os.Getenv("AI_AGENT_TEST_DB") != "1" { - t.Skip("skip: AI_AGENT_TEST_DB not set") - } - // gfdb 的 Hook 依赖 ctx 中注入用户信息(租户/创建人),否则 Insert/Select 报 "token 数据为空" - ctx := context.WithValue(context.Background(), "user", &beans.User{ - UserName: "test-async", TenantId: 1, - }) - const execId, nodeId, seg = 900001, "node-x", 3 - // 清理可能残留 - if err := FlowAsyncTaskDao.DeleteByKey(ctx, execId, nodeId, seg); err != nil { - t.Fatalf("cleanup: %v", err) - } - - if err := FlowAsyncTaskDao.Upsert(ctx, execId, nodeId, seg, 1001, 2026, "model-call-test"); err != nil { - t.Fatalf("upsert: %v", err) - } - rec, err := FlowAsyncTaskDao.Get(ctx, execId, nodeId, seg) - if err != nil || rec == nil { - t.Fatalf("get: %v", err) - } - if rec.TaskId != 2026 || rec.MsgTopic != "model-call-test" || rec.State != FlowAsyncStateInflight { - t.Fatalf("upsert state wrong: %+v", rec) - } - - if err := FlowAsyncTaskDao.UpdateByKey(ctx, execId, nodeId, seg, FlowAsyncStateDone, `{"taskId":2026}`); err != nil { - t.Fatalf("update result: %v", err) - } - rec, err = FlowAsyncTaskDao.Get(ctx, execId, nodeId, seg) - if err != nil || rec == nil { - t.Fatalf("get: %v", err) - } - if rec.State != FlowAsyncStateDone || rec.Result != resultJSONB { - t.Fatalf("result not persisted: %+v", rec) - } - - // 唯一键 upsert:同 (exec,node,seg) 再次写入应更新而非新建 - if err := FlowAsyncTaskDao.Upsert(ctx, execId, nodeId, seg, 1001, 9999, "model-call-test2"); err != nil { - t.Fatalf("re-upsert: %v", err) - } - rec, err = FlowAsyncTaskDao.Get(ctx, execId, nodeId, seg) - if err != nil || rec == nil { - t.Fatalf("get: %v", err) - } - if rec.TaskId != 9999 { - t.Fatalf("re-upsert should update task_id, got %d", rec.TaskId) - } - // OnDuplicate 限定了冲突更新列,已缓存结果不应被覆盖 - if rec.Result != resultJSONB { - t.Fatalf("re-upsert should preserve cached result, got %q", rec.Result) - } - - // 物理删除验证:DeleteByKey 后 Get 为 nil(软删会留下 deleted_at 行且 Get 仍返回) - if err := FlowAsyncTaskDao.DeleteByKey(ctx, execId, nodeId, seg); err != nil { - t.Fatalf("delete: %v", err) - } - rec, err = FlowAsyncTaskDao.Get(ctx, execId, nodeId, seg) - if err != nil { - t.Fatalf("get after delete: %v", err) - } - if rec != nil { - t.Fatalf("物理删除后不应查到残留: %+v", rec) - } - // DeleteByExecution - if err := FlowAsyncTaskDao.Upsert(ctx, execId, nodeId, seg, 1001, 1, "t"); err != nil { - t.Fatalf("upsert: %v", err) - } - if err := FlowAsyncTaskDao.DeleteByExecution(ctx, execId); err != nil { - t.Fatalf("delete by exec: %v", err) - } - if rec, _ := FlowAsyncTaskDao.Get(ctx, execId, nodeId, seg); rec != nil { - t.Fatalf("DeleteByExecution 后不应查到残留") - } - // 清理表残留(跨测试) - gfdb.DB(ctx, public.DbNameBlackDeacon).Exec(ctx, "DELETE FROM black_deacon_flow_async_task WHERE execution_id = ?", execId) -} diff --git a/workflow/dao/session/exec_workflow_dao.go b/workflow/dao/session/exec_workflow_dao.go index b936cda..b487769 100644 --- a/workflow/dao/session/exec_workflow_dao.go +++ b/workflow/dao/session/exec_workflow_dao.go @@ -49,6 +49,20 @@ func (d *execWorkflowDao) Update(ctx context.Context, req *sessionDto.UpdateWork return r.RowsAffected() } +// UpdateMap 按列更新执行记录(map 更新不走 OmitEmpty,可显式写 0 值)。 +// 供 recordWorkflow 终态一次原子落库(status/error_message/error/retryable/retry_count 同语句), +// 避免"先 UpdateRetry 再 Update"两步写部分生效导致 retryable 与状态不一致 +func (d *execWorkflowDao) UpdateMap(ctx context.Context, id int64, data map[string]any) error { + if id <= 0 || len(data) == 0 { + return nil + } + _, err := gfdb.DB(ctx, public.DbNameBlackDeacon).Model(ctx, public.TableNameExecWorkflow). + Where(entity.ExecWorkflowCol.Id, id). + Data(data). + Update() + return err +} + // ClearError 清空执行记录的报错信息(重新执行成功后调用,OmitEmpty 的 Update 会跳过空串,需显式写空) func (d *execWorkflowDao) ClearError(ctx context.Context, id int64) (rows int64, err error) { r, err := gfdb.DB(ctx, public.DbNameBlackDeacon).Model(ctx, public.TableNameExecWorkflow). @@ -121,18 +135,65 @@ func (d *execWorkflowDao) ListBySession(ctx context.Context, sessionId string) ( return } -// ResetRunning 置为运行中并刷新心跳(execute 复用失败记录 / reExecute / 恢复例程共用; -// 用 map 更新避免 OmitEmpty 省略 0 值,同时写 status、last_heartbeat、node_group_id) -func (d *execWorkflowDao) ResetRunning(ctx context.Context, id int64, nodeGroupId string) error { - _, err := gfdb.DB(ctx, public.DbNameBlackDeacon).Model(ctx, public.TableNameExecWorkflow). +// ResetRunning 置为运行中并刷新心跳(execute 复用失败记录 / reExecute 共用; +// 用 map 更新避免 OmitEmpty 省略 0 值,同时写 status、last_heartbeat、node_group_id)。 +// 条件更新防双跑:仅当记录当前状态仍属于 prevStatuses 之一时才重置(DB 行锁原子判定), +// 其余场景(已被恢复例程/并发触发抢先重置)返回 reset=false,调用方应放弃执行并返回 errExecAlreadyRunning, +// 由持有方收敛状态,避免同一 exec 被两条路径并发 BuildExecution。 +func (d *execWorkflowDao) ResetRunning(ctx context.Context, id int64, nodeGroupId string, prevStatuses ...int8) (reset bool, err error) { + m := gfdb.DB(ctx, public.DbNameBlackDeacon).Model(ctx, public.TableNameExecWorkflow). + Where(entity.ExecWorkflowCol.Id, id) + if len(prevStatuses) > 0 { + statuses := make([]interface{}, 0, len(prevStatuses)) + for _, s := range prevStatuses { + statuses = append(statuses, s) + } + m = m.WhereIn(entity.ExecWorkflowCol.Status, statuses) + } + r, err := m.Data(map[string]any{ + entity.ExecWorkflowCol.Status: gconv.Int8(*flow.FlowExecutionStatusRunning.Code()), + entity.ExecWorkflowCol.LastHeartbeat: time.Now().UnixMilli(), + entity.ExecWorkflowCol.NodeGroupId: nodeGroupId, + }).Update() + if err != nil { + return false, err + } + rows, err := r.RowsAffected() + if err != nil { + return false, err + } + return rows > 0, nil +} + +// ResetRunningIfRecoverable 恢复例程专用:条件重置为运行中,仅当记录仍处于可恢复状态时 +// (status=3 可重试失败,或 status=1 心跳陈旧的僵尸运行中)才重置(DB 行锁原子判定防双跑)。 +// staleBeforeMs:status=1 时的心跳陈旧阈值(毫秒),与 ListRecoverable/isRecoverable 判定一致。 +// 返回 reset=false 表示状态已被其它路径(用户 reExecute/execute 或其它节点恢复)抢先重置, +// 本恢复例程应放弃续跑、不落终态,状态由持有方收敛。 +func (d *execWorkflowDao) ResetRunningIfRecoverable(ctx context.Context, id int64, nodeGroupId string, staleBeforeMs int64) (reset bool, err error) { + r, err := gfdb.DB(ctx, public.DbNameBlackDeacon).Model(ctx, public.TableNameExecWorkflow). Where(entity.ExecWorkflowCol.Id, id). + Where(fmt.Sprintf("(%s = ? OR (%s = ? AND %s < ?))", + entity.ExecWorkflowCol.Status, + entity.ExecWorkflowCol.Status, + entity.ExecWorkflowCol.LastHeartbeat), + gconv.Int8(*flow.FlowExecutionStatusFailed.Code()), + gconv.Int8(*flow.FlowExecutionStatusRunning.Code()), + staleBeforeMs). Data(map[string]any{ entity.ExecWorkflowCol.Status: gconv.Int8(*flow.FlowExecutionStatusRunning.Code()), entity.ExecWorkflowCol.LastHeartbeat: time.Now().UnixMilli(), entity.ExecWorkflowCol.NodeGroupId: nodeGroupId, }). Update() - return err + if err != nil { + return false, err + } + rows, err := r.RowsAffected() + if err != nil { + return false, err + } + return rows > 0, nil } // TouchHeartbeat 更新执行心跳(毫秒时间戳),供后台心跳 goroutine 每 30s 调用一次 diff --git a/workflow/dao/session/exec_workflow_dao_test.go b/workflow/dao/session/exec_workflow_dao_test.go deleted file mode 100644 index f076abf..0000000 --- a/workflow/dao/session/exec_workflow_dao_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package session - -import ( - "context" - "os" - "testing" - "time" - - "ai-agent/workflow/consts/flow" - sessionDto "ai-agent/workflow/model/dto/session" - "ai-agent/workflow/model/entity" - - "gitea.redpowerfuture.com/red-future/common/beans" - _ "github.com/gogf/gf/contrib/drivers/pgsql/v2" - "github.com/gogf/gf/v2/net/gtrace" -) - -// TestExecWorkflowRetryColsRoundTrip 验证三新列读写 + ListRecoverable 三类判定(需 AI_AGENT_TEST_DB=1 且本地 PG 可用) -func TestExecWorkflowRetryColsRoundTrip(t *testing.T) { - if !isTestDBEnabled(t) { - t.Skip("skip: AI_AGENT_TEST_DB not set") - } - // gfdb 的 Hook 依赖 ctx 中注入用户信息(租户/创建人),否则 Insert 报 "token 数据为空"。 - // 先造 span:ListRecoverable 链 NoTenantId 依赖 traceID 作 gcache 标记键,无 span 时 getTraceID 返回空 → NoTenantId 返回 nil 会 panic - ctx, span := gtrace.NewSpan(context.Background(), "test.recover") - defer span.End() - ctx = context.WithValue(ctx, "user", &beans.User{ - UserName: "test-retry", TenantId: 1, - }) - id, err := ExecWorkflowDao.Insert(ctx, &sessionDto.CreateWorkflowReq{ - SessionId: "test-retry", FlowId: 1, NodeGroupId: "ng", - Status: flow.FlowExecutionStatusRunning.Code(), - LastHeartbeat: time.Now().UnixMilli(), - }) - if err != nil { - t.Fatalf("insert: %v", err) - } - defer ExecWorkflowDao.Delete(ctx, &sessionDto.DeleteExecWorkflowReq{Id: []int64{id}}) - - if err := ExecWorkflowDao.TouchHeartbeat(ctx, id); err != nil { - t.Fatalf("touch heartbeat: %v", err) - } - if err := ExecWorkflowDao.UpdateRetry(ctx, id, 1, 1); err != nil { - t.Fatalf("update retry: %v", err) - } - got, err := ExecWorkflowDao.GetById(ctx, id) - if err != nil || got == nil { - t.Fatalf("get: %v", err) - } - if got.Retryable != 1 || got.RetryCount != 1 || got.LastHeartbeat == 0 { - t.Fatalf("cols not persisted: retryable=%d retryCount=%d heartbeat=%d", got.Retryable, got.RetryCount, got.LastHeartbeat) - } - - now := time.Now().UnixMilli() - stale := now - int64(120*time.Second/time.Millisecond) - if err := ExecWorkflowDao.UpdateRetry(ctx, id, 0, 0); err != nil { - t.Fatalf("update retry reset: %v", err) - } - // 情况1:status=3 retryable=1 retry_count= execMaxRetryCount { // 终局清理(Task 5 裁定):重试耗尽,exec 永久失败,清 flow_async_task 孤儿缓存 _ = flowDao.FlowAsyncTaskDao.DeleteByExecution(saveCtx, execId) @@ -185,7 +186,7 @@ func handleExecute(ctx context.Context, conn *wsCommon.WsConnection, payload int } if !g.IsEmpty(execId) { glog.Infof(saveCtx, "工作流执行完成,execId: %v", execId) - recordWorkflow(saveCtx, execId, time.Since(start), execErr) + recordWorkflow(saveCtx, execId, time.Since(start), execErr, retryable, retryCnt) recorded = true } else if execErr != nil { // 查询/创建执行记录失败(拿不到 execId)时,兜底把该会话+工作流最近一条"运行中"记录标记为失败 @@ -193,11 +194,12 @@ func handleExecute(ctx context.Context, conn *wsCommon.WsConnection, payload int recorded = true } if execErr != nil { - _ = writeJSON(conn, &wsCommon.WsPushMsg{Type: "error", Message: "工作流执行失败", Error: execErr.Error()}) + // 终态广播(发起连接 + 附着订阅者) + hub.Publish(&wsCommon.WsPushMsg{Type: "error", Message: "工作流执行失败", Error: execErr.Error()}) return } // 成功:把本次执行保存的结果文件路径(exec_workflow_result)一并推给前端 - _ = writeJSON(conn, &wsCommon.WsPushMsg{ + hub.Publish(&wsCommon.WsPushMsg{ Type: "flow_complete", Message: "工作流执行完成", Data: map[string]interface{}{ @@ -212,8 +214,18 @@ func handleExecute(ctx context.Context, conn *wsCommon.WsConnection, payload int // 兜底按会话+工作流查最近一条仍处于"运行中"的记录标记为失败,避免前端已报错但记录卡在 Running。 // 最近记录已是成功/失败状态则不处理(可能是上一次执行的结果,不应误改)。 func recordExecutionFailure(ctx context.Context, sessionId string, flowId int64, execId int64, runErr error) { + // 兜底失败路径同样携带 retryable 分类(与 handleExecute 一致): + // 用户取消=0;其余(程序关停中断/程序报错)可重试=1,保证任意 status=3 写库都带分类 + var retryable, retryCnt *int + if runErr != nil { + if errors.Is(runErr, context.Canceled) && !IsShuttingDown() { + retryable, retryCnt = intptr(0), intptr(0) + } else { + retryable, retryCnt = intptr(1), intptr(0) + } + } if !g.IsEmpty(execId) { - recordWorkflow(ctx, execId, 0, runErr) + recordWorkflow(ctx, execId, 0, runErr, retryable, retryCnt) return } lastExec, err := sessionDao.ExecWorkflowDao.GetLatestBySessionAndFlow(ctx, sessionId, flowId) @@ -224,11 +236,16 @@ func recordExecutionFailure(ctx context.Context, sessionId string, flowId int64, if lastExec.Status == nil || *lastExec.Status != *flow.FlowExecutionStatusRunning.Code() { return } - recordWorkflow(ctx, lastExec.Id, 0, runErr) + recordWorkflow(ctx, lastExec.Id, 0, runErr, retryable, retryCnt) } -// recordWorkflow 把一次工作流执行写入 exec_workflow/exec_workflow_result:运行记录 + 输出文件结果 -func recordWorkflow(ctx context.Context, id int64, duration time.Duration, runErr error) { +// intptr 取 int 值指针(recordWorkflow 的 retryable/retryCount 参数:nil=不改动) +func intptr(v int) *int { return &v } + +// recordWorkflow 把一次工作流执行写入 exec_workflow/exec_workflow_result:运行记录 + 输出文件结果。 +// retryable/retryCount 非 nil 时随终态同语句原子落库,避免"先 UpdateRetry 再 Update"两步写部分生效 +// 导致 retryable 与 status/error_message 不一致(关停中断场景曾出现 retryable=0 但 message=程序关停中断) +func recordWorkflow(ctx context.Context, id int64, duration time.Duration, runErr error, retryable, retryCount *int) { // exec_workflow 状态沿用 1-运行中,2-成功,3-失败;前端结果卡片也只识别 1/2/3 // (4 会误显示为"运行中"),故取消同样记为失败,错误信息写"用户已终止执行" // error_message 存友好提示,error 存原始错误明细 @@ -236,26 +253,37 @@ func recordWorkflow(ctx context.Context, id int64, duration time.Duration, runEr var errorMessage, errorDetail string if runErr != nil { status = flow.FlowExecutionStatusFailed - if errors.Is(runErr, context.Canceled) { + switch { + case errors.Is(runErr, context.Canceled): errorMessage = errWorkflowTerminated - } else { + case errors.Is(runErr, errInterruptedByShutdown): + errorMessage = "程序关停中断" + default: errorMessage = "工作流执行失败" errorDetail = runErr.Error() } } - _, err := sessionDao.ExecWorkflowDao.Update(ctx, &sessionDto.UpdateWorkflowReq{ - Id: id, - Status: status.Code(), - Duration: int64(duration.Seconds()), - ErrorMessage: errorMessage, - Error: errorDetail, - }) - if err != nil { - glog.Errorf(ctx, "exec_workflow 落库失败: %v", err) + data := map[string]any{ + entity.ExecWorkflowCol.Status: *status.Code(), + } + if d := int64(duration.Seconds()); d != 0 { + data[entity.ExecWorkflowCol.Duration] = d + } + if errorMessage != "" { + data[entity.ExecWorkflowCol.ErrorMessage] = errorMessage + } + if errorDetail != "" { + data[entity.ExecWorkflowCol.Error] = errorDetail + } + if retryable != nil { + data[entity.ExecWorkflowCol.Retryable] = *retryable + data[entity.ExecWorkflowCol.RetryCount] = *retryCount + } + if err := sessionDao.ExecWorkflowDao.UpdateMap(ctx, id, data); err != nil { + glog.Errorf(ctx, "exec_workflow 终态落库失败 execId=%d: %v", id, err) return } - // 执行成功:重新执行复用了同一条记录,OmitEmpty 的 Update 会跳过空 error_message/error, - // 需显式清空,避免上一次失败的报错残留 + // 执行成功:重新执行复用了同一条记录,需显式清空,避免上一次失败的报错残留 if runErr == nil { if _, err := sessionDao.ExecWorkflowDao.ClearError(ctx, id); err != nil { glog.Errorf(ctx, "exec_workflow 报错信息清空失败: %v", err) @@ -315,10 +343,20 @@ func executeOrResume(ctx context.Context, conn *wsCommon.WsConnection, req *sess // 心跳新鲜 → 真在跑,不新建避免双跑;心跳陈旧 → 僵尸,触发后台恢复。 // 都不新建执行,恢复在后台完成,完成后状态自然收敛(spec §9)。 nowMs := time.Now().UnixMilli() + hub := getProgressHub(ctx) + owned := hub != nil && hub.Owned() if lastExec.LastHeartbeat < nowMs-int64(heartbeatStaleAfter/time.Millisecond) { + // 心跳陈旧 = 僵尸遗留:本进程恢复例程正在跑则附着其进度;否则拉起恢复并附着。 + // 候选 hub 直接交给恢复例程复用同一实例(不 close):最终执行者与连接共用一 hub, + // 取消/进度不因"占位被关→重建"竞态丢失 + if !owned { + go recoverExecution(context.WithoutCancel(ctx), lastExec.Id, &execAttach{conn: conn, sessionId: conn.SessionId, flowId: req.FlowId, hub: hub}) + } _ = writeJSON(conn, &wsCommon.WsPushMsg{Type: "round_start", Message: "检测到未完成执行,正在恢复", Data: map[string]interface{}{"id": lastExec.Id}}) - go recoverExecution(context.WithoutCancel(ctx), lastExec.Id) } else { + // 心跳新鲜 = 真在跑:owned → 连接已附着本进程运行中执行,直接订阅其进度; + // !owned → 其它节点在跑(无法跨节点附着)或本进程执行尚未接管(候选 hub 会被其接管)。 + // 不 close 候选:避免在 execute()/恢复 TryOwn 前误关,导致执行者进度/取消无人接收 _ = writeJSON(conn, &wsCommon.WsPushMsg{Type: "round_start", Message: "工作流正在执行中", Data: map[string]interface{}{"id": lastExec.Id}}) } return lastExec.Id, errExecAlreadyRunning @@ -328,7 +366,7 @@ func executeOrResume(ctx context.Context, conn *wsCommon.WsConnection, req *sess "id": lastExec.Id, }}) glog.Infof(ctx, "工作流断点续跑,execId: %v", lastExec.Id) - return reExecute(ctx, lastExec.Id) + return reExecute(ctx, lastExec.Id, *flow.FlowExecutionStatusFailed.Code()) } glog.Infof(ctx, "工作流全新执行,lastExec: %v", lastExec) return execute(ctx, conn, lastExec.Id, lastExec.Status, req) @@ -353,6 +391,10 @@ func flowContentEqual(a, b *entity.FlowInfo) bool { // execute 执行工作流(首次执行;同会话+同工作流最近一次执行为失败状态时复用该记录重新执行,不新建数据) func execute(ctx context.Context, conn *wsCommon.WsConnection, execId int64, status flow.FlowExecutionStatus, req *sessionDto.WebSocketExecWorkflowReq) (id int64, err error) { + // 优雅关停期间不再启动新执行(避免关停后仍登记运行、落库被退出进程打断;走 Canceled 分类落终态) + if IsShuttingDown() { + return 0, context.Canceled + } var nodeGroupId = uuid.NewString() if g.IsEmpty(execId) { glog.Infof(ctx, "工作流全新执行execute,无历史记录") @@ -373,10 +415,15 @@ func execute(ctx context.Context, conn *wsCommon.WsConnection, execId int64, sta // 需解引用按值比较,否则复用失败记录时不会重置为 Running、也不更新 RequestParams if status != nil && *status == *flow.FlowExecutionStatusFailed.Code() { glog.Infof(ctx, "工作流断点续跑execute,execId: %v", execId) - err = sessionDao.ExecWorkflowDao.ResetRunning(ctx, execId, nodeGroupId) + var reset bool + reset, err = sessionDao.ExecWorkflowDao.ResetRunning(ctx, execId, nodeGroupId, *flow.FlowExecutionStatusFailed.Code()) if err != nil { return } + if !reset { + // 已被其它路径(恢复例程/并发触发)抢先重置为运行中:放弃本次执行,状态由持有方收敛 + return execId, errExecAlreadyRunning + } // 复用失败记录时参数可能已变:把新参数落库,供后续 reExecute/恢复例程按记录参数续跑 _, err = sessionDao.ExecWorkflowDao.Update(ctx, &sessionDto.UpdateWorkflowReq{Id: execId, RequestParams: req.FlowContent}) if err != nil { @@ -402,8 +449,13 @@ func execute(ctx context.Context, conn *wsCommon.WsConnection, execId int64, sta _ = writeJSON(conn, &wsCommon.WsPushMsg{Type: "round_start", Message: "运行开始", Data: map[string]interface{}{ "id": execId, }}) - stop := startHeartbeat(ctx, execId) + // WS 路径心跳与 BuildExecution 已共享同一 ctx(连接取消/DB 故障同生共死),无需租约丢失回调 + stop := startHeartbeat(ctx, execId, nil) defer stop() + // 接管候选 hub:本 goroutine 确认真正启动执行前标记 owned,供并发附着连接识别运行中持有者 + if h := getProgressHub(ctx); h != nil && !h.Owned() { + h.MarkOwned() + } err = BuildExecution(ctx, true, req.FlowId, execId, nodeGroupId, conn.SessionId, req.FlowContent) if err != nil { return @@ -411,19 +463,35 @@ func execute(ctx context.Context, conn *wsCommon.WsConnection, execId int64, sta return execId, nil } -// reExecute 重新执行工作流 -func reExecute(ctx context.Context, execWorkflowId int64) (id int64, err error) { +// reExecute 重新执行工作流。 +// prevStatus:调用方当前观察到的记录状态(失败=3 断点续跑;运行中=1 本进程自动重试), +// 传给 ResetRunning 做条件重置:仅当记录仍处于该状态时才重置(原子防双跑)。 +// 状态已被其它路径抢先变更时返回 errExecAlreadyRunning,外层不写终态、由持有方收敛。 +func reExecute(ctx context.Context, execWorkflowId int64, prevStatus int8) (id int64, err error) { + // 优雅关停期间不再启动新执行(返回 Canceled 让外层分类落 errInterruptedByShutdown 终态) + if IsShuttingDown() { + return 0, context.Canceled + } flowInfo, err := sessionDao.ExecWorkflowDao.GetById(ctx, execWorkflowId) if err != nil { return } var nodeGroupId = uuid.NewString() - err = sessionDao.ExecWorkflowDao.ResetRunning(ctx, flowInfo.Id, nodeGroupId) + reset, err := sessionDao.ExecWorkflowDao.ResetRunning(ctx, flowInfo.Id, nodeGroupId, prevStatus) if err != nil { return } - stop := startHeartbeat(ctx, flowInfo.Id) + if !reset { + // 状态已被其它路径(恢复例程/并发触发)抢先重置为运行中:放弃续跑 + return flowInfo.Id, errExecAlreadyRunning + } + // WS 路径心跳与 BuildExecution 已共享同一 ctx(连接取消/DB 故障同生共死),无需租约丢失回调 + stop := startHeartbeat(ctx, flowInfo.Id, nil) defer stop() + // 接管候选 hub:确认续跑启动前标记 owned(与 execute 一致) + if h := getProgressHub(ctx); h != nil && !h.Owned() { + h.MarkOwned() + } err = BuildExecution(ctx, false, flowInfo.FlowId, flowInfo.Id, nodeGroupId, flowInfo.SessionId, flowInfo.RequestParams) if err != nil { return diff --git a/workflow/service/flow/lambda_node_util.go b/workflow/service/flow/lambda_node_util.go index 147e3ef..ec5195e 100644 --- a/workflow/service/flow/lambda_node_util.go +++ b/workflow/service/flow/lambda_node_util.go @@ -150,6 +150,14 @@ func HttpCallResultLambda(ctx context.Context, nodeInput *flowDto.NodeExecutionI } } } + // 后台恢复续跑无 HTTP 请求时(ctx 携带合成 user、无 token),补充 X-User-Info 供下游内部服务 + // GetUserInfo 识别租户;正常执行 ctx 无 user(仅 request 有 Authorization),不会注入, + // 避免把内部身份透传给任意外部 URL。与 gateway/model.go requestHeaders 的恢复兜底保持一致 + if headers["X-User-Info"] == "" { + if u := ctx.Value("user"); u != nil { + headers["X-User-Info"] = gconv.String(u) + } + } } g.Log().Debugf(ctx, "httpCallResultLambda: body: %v", body) diff --git a/workflow/service/flow/lambda_segment_resume_test.go b/workflow/service/flow/lambda_segment_resume_test.go deleted file mode 100644 index b3a14af..0000000 --- a/workflow/service/flow/lambda_segment_resume_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package flow - -import ( - "testing" - - "ai-agent/workflow/model/entity" -) - -func TestPlanSegmentResumePositionBased(t *testing.T) { - params := []map[string]any{{"a": 1}, {"a": 2}, {"a": 3}} - saved := map[int]entity.SegmentRef{0: {Key: "video", URL: "url0"}, 2: {Key: "video", URL: "url2"}} - idxList, needGen := planSegmentResume(params, saved) - if len(idxList) != 3 || len(needGen) != 3 { - t.Fatalf("长度应等于段数") - } - if needGen[0] || !needGen[1] || needGen[2] { - t.Fatalf("位置0/2已保存应复用,位置1应生成: %v", needGen) - } -} - -func TestMergeSegmentOutputsOrder(t *testing.T) { - idxList := []int{0, 1, 2} - needGen := []bool{false, true, false} - newRes := [][]map[string]any{nil, {{"video": "new1"}}, nil} - saved := map[int]entity.SegmentRef{0: {Key: "video", URL: "url0"}, 2: {Key: "video", URL: "url2"}} - merged := mergeSegmentOutputs(idxList, needGen, newRes, saved) - if len(merged) != 3 { - t.Fatalf("应合并 3 条: %v", merged) - } - if merged[0]["video"] != "url0" || merged[1]["video"] != "new1" || merged[2]["video"] != "url2" { - t.Fatalf("合并顺序应为 url0,new1,url2: %v", merged) - } -} diff --git a/workflow/service/flow/processor/builtin/media/media.go b/workflow/service/flow/processor/builtin/media/media.go index b4b1844..75e4f43 100644 --- a/workflow/service/flow/processor/builtin/media/media.go +++ b/workflow/service/flow/processor/builtin/media/media.go @@ -205,7 +205,8 @@ func getMediaTask(ctx context.Context, kind TaskKind, taskID string) (*MergeTask return res, nil } -// requestHeaders 透传当前请求头(含 Authorization / X-User-Info),供内部服务鉴权使用 +// requestHeaders 透传当前请求头(含 Authorization / X-User-Info),供内部服务鉴权使用; +// 后台恢复续跑无 HTTP 请求时(ctx 携带合成 user),补充 X-User-Info 供下游 GetUserInfo 识别租户 func requestHeaders(ctx context.Context) map[string]string { headers := make(map[string]string) if r := g.RequestFromCtx(ctx); r != nil { @@ -215,6 +216,11 @@ func requestHeaders(ctx context.Context) map[string]string { } } } + if headers["X-User-Info"] == "" { + if u := ctx.Value("user"); u != nil { + headers["X-User-Info"] = gconv.String(u) + } + } return headers } diff --git a/workflow/service/flow/recover_execution.go b/workflow/service/flow/recover_execution.go index 8cd4ba1..29e5e0e 100644 --- a/workflow/service/flow/recover_execution.go +++ b/workflow/service/flow/recover_execution.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync/atomic" "time" "ai-agent/workflow/consts/flow" @@ -12,6 +13,7 @@ import ( "ai-agent/workflow/model/entity" "gitea.redpowerfuture.com/red-future/common/beans" + wsCommon "gitea.redpowerfuture.com/red-future/common/websocket" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/gtrace" "github.com/google/uuid" @@ -23,12 +25,36 @@ const ( heartbeatStaleAfter = 2 * heartbeatInterval // 心跳陈旧阈值(>2×间隔),小于此视为僵尸 recoverScanInterval = 30 * time.Second // 周期扫描间隔 recoverLockTTL = 15 * time.Minute // 恢复锁 TTL(覆盖长时间续跑) + recoverLockPhase = 2 * time.Minute // 抢锁/前置判定阶段总时限(Redis SET + 一次 DB 读,毫秒级) + recoverExecTimeout = 12 * time.Hour // 单次恢复执行最长时长;超时按可重试失败落库交下一轮扫描 + heartbeatMaxFail = 2 // 心跳连续失败达到陈旧阈值(2×30s=60s)即租约丢失,中止执行 execMaxRetryCount = 2 // 整次执行最多自动重试 2 次(共 3 次尝试) ) // errExecAlreadyRunning 用户触发时该执行已在运行(本节点/其它节点后台恢复),不新建执行 var errExecAlreadyRunning = errors.New("工作流正在执行中,不重复执行") +// errInterruptedByShutdown 程序优雅关停导致连接 ctx 取消时的错误标记(区别于用户主动取消)。 +// 落库为 status=3 + retryable=1,下次启动恢复扫描捞起续跑。 +var errInterruptedByShutdown = errors.New("程序关停中断") + +// shuttingDown 优雅关停标记:程序收到退出信号后置位。 +// 用于区分"程序关停导致的 WS 连接取消"与"用户主动取消",避免前者被误分类为不可重试。 +var shuttingDown atomic.Bool + +// SetShuttingDown 置位优雅关停标记并取消所有运行中执行(main 信号处理在 Close() 前调用)。 +// 恢复执行 ctx 经 WithoutCancel 脱离连接,必须在此显式取消,否则程序关停时恢复中的 exec +// 不会落终态(仍 status=1);取消后 BuildExecution 随之中止、走恢复错误分支写 status=3/retryable=1。 +func SetShuttingDown() { + shuttingDown.Store(true) + cancelAllExecRuns() +} + +// IsShuttingDown 是否处于优雅关停 +func IsShuttingDown() bool { + return shuttingDown.Load() +} + // shouldRetry 错误分类(spec §3):用户取消不重试;其余(模型/DB/网络/超时/panic)程序报错重试 func shouldRetry(err error) bool { return err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, errExecAlreadyRunning) @@ -56,6 +82,10 @@ func StartRecoveryLoop(ctx context.Context) { // 恢复运行在无 HTTP 用户的后台:ListRecoverable 走跨租户 NoTenantId,需要 ctx 携带 OTel span // (NoTenantId 以 traceID 为 gcache 标记键,无 span 时 getTraceID 返回空 → NoTenantId 返回 nil 会 panic) func scanAndRecover(ctx context.Context) { + // 优雅关停期间不再捞起:连接 ctx 刚被取消、exec 正在落终态,避免本进程内部用合成用户重跑 + if IsShuttingDown() { + return + } scanCtx, span := gtrace.NewSpan(ctx, "workflow.recover.scan") defer span.End() now := time.Now().UnixMilli() @@ -65,30 +95,64 @@ func scanAndRecover(ctx context.Context) { return } for _, r := range rows { - go recoverExecution(context.WithoutCancel(scanCtx), r.Id) + // 扫描触发无附着连接(attach=nil):hub 在确认可恢复后按 exec 的 session+flow 建立 + go recoverExecution(context.WithoutCancel(scanCtx), r.Id, nil) } } // recoverExecution 统一恢复例程(spec §7):抢锁 → 判定 → 置运行中续跑。 -// 两个触发源共用:启动/周期扫描、executeOrResume 对僵尸行的用户触发。 -func recoverExecution(parentCtx context.Context, execId int64) { - lock := NewRedisLock(fmt.Sprintf("workflow:exec:recover:%d", execId), int64(recoverLockTTL/time.Second)) - ctx, cancel := context.WithTimeout(context.WithoutCancel(parentCtx), 2*time.Hour) - defer cancel() +// 两个触发源共用:启动/周期扫描(attach=nil)、executeOrResume 对僵尸行的用户触发(attach 携带连接)。 +func recoverExecution(parentCtx context.Context, execId int64, attach *execAttach) { + // 优雅关停期间不再启动新的恢复执行(周期扫描已有守卫;用户 executeOrResume 触发路径这里兜底) + if IsShuttingDown() { + return + } + // 在函数最前面创建并登记本执行的 cancel:trackExecRun 绑定最终控制执行的 cancel, + // 提前登记保证优雅关停(SetShuttingDown→cancelAllExecRuns)快照必然覆盖到本执行(已登记), + // 或本执行在开始前置判定前自行发现关停标记放弃——不留"已置 status=1 却无执行"的无主记录。 + topCtx, topCancel := context.WithCancel(context.WithoutCancel(parentCtx)) + defer topCancel() + finish := trackExecRun(topCancel) + defer finish() + // 登记后复查关停标记:已置位说明 cancelAllExecRuns 快照早于本登记(未覆盖本执行), + // 此时本执行尚未走 ResetRunningIfRecoverable(不会留下 status=1 无主执行),直接放弃; + // 若置位发生在复查之后,本执行已被快照覆盖,随关停取消并在错误分类路径落终态(status=3/retryable=1) + if IsShuttingDown() { + return + } - ok, err := lock.Acquire(ctx) + // 事件中枢(attach 路径):用户点击执行触发恢复时,复用连接已订阅的候选 hub(executeOrResume 传入), + // 让该连接订阅到后续节点进度与终态、并能通过 workflow_cancel 永久取消(hub.CancelByUser→topCancel)。 + // 此处只订阅不 TryOwn:所有权由"抢到锁+重置权"的执行者确定;提前退出路径(锁输/不可恢复/重置失败) + // 不关 hub,保证最终执行者与连接指向同一 hub 实例,取消/进度不因竞态丢失。 + var hub *execHub + if attach != nil { + if attach.hub != nil { + hub = attach.hub + } else { + hub, _ = registerHubIfAbsent(attach.sessionId, attach.flowId, newExecHub(attach.sessionId, attach.flowId)) + } + subscribeConnToHub(attach.conn, hub) + } + + lock := NewRedisLock(fmt.Sprintf("workflow:exec:recover:%d", execId), int64(recoverLockTTL/time.Second)) + // 抢锁/前置判定用独立短超时 ctx:该阶段只做 Redis SET + 一次 DB 读 + 一次条件重置,应毫秒级完成 + lockCtx, lockCancel := context.WithTimeout(context.WithoutCancel(parentCtx), recoverLockPhase) + defer lockCancel() + + ok, err := lock.Acquire(lockCtx) if err != nil { - g.Log().Errorf(ctx, "恢复抢锁失败 execId=%d: %v", execId, err) + g.Log().Errorf(lockCtx, "恢复抢锁失败 execId=%d: %v", execId, err) return } if !ok { - g.Log().Infof(ctx, "恢复锁被其它节点持有,跳过 execId=%d", execId) + g.Log().Infof(lockCtx, "恢复锁被其它节点持有,跳过 execId=%d", execId) return } defer lock.Release(context.Background()) // 抢锁后重读:此刻租户未知(该 exec 可能属于任意租户),必须跨租户读 - exec, err := sessionDao.ExecWorkflowDao.GetByIdNoTenant(ctx, execId) + exec, err := sessionDao.ExecWorkflowDao.GetByIdNoTenant(lockCtx, execId) if err != nil || exec == nil { return } @@ -96,37 +160,102 @@ func recoverExecution(parentCtx context.Context, execId int64) { return // 锁等待期间状态已变化(其它节点已恢复/用户取消/已耗尽) } - stop := startHeartbeat(ctx, execId) - defer stop() + // 事件中枢(scan 路径,无附着连接):此刻才知道 exec 的 session+flow,建 hub 供后续用户连接附着 + //(所有权在重置成功后统一接管,见下) + if attach == nil { + hub, _ = registerHubIfAbsent(exec.SessionId, exec.FlowId, newExecHub(exec.SessionId, exec.FlowId)) + } - // 置运行中并续跑(BuildExecution false → 断点续跑:checkpoint 有则从中继续,无则从头 + 段/memo 复用) + // 置运行中并续跑(BuildExecution false → 断点续跑:checkpoint 有则从中继续,无则从头 + 段/memo 复用)。 + // 心跳在条件重置成功后才启动:避免对未抢到重置权的 exec 空跑心跳 saveCtx := context.WithoutCancel(parentCtx) // 恢复无 HTTP 用户,但图节点 lambda 的 INSERT(node_execution/flow_async_task/segment_result)走 // insertHook 硬性要求 user;用 exec 所属租户合成系统用户 ctx(保留 span),供后续落库使用 - userCtx := context.WithValue(saveCtx, "user", &beans.User{UserName: "workflow-recover", TenantId: exec.TenantId}) + userCtx := context.WithValue(saveCtx, "user", &beans.User{UserName: exec.Creator, TenantId: exec.TenantId}) nodeGroupId := uuid.NewString() // 置运行中与图执行的节点组标识须一致 - if err := sessionDao.ExecWorkflowDao.ResetRunning(userCtx, execId, nodeGroupId); err != nil { - g.Log().Errorf(ctx, "恢复置运行中失败 execId=%d: %v", execId, err) + // 条件重置(原子防与用户断点续跑双跑):仅当仍可恢复(status=3 或 status=1 心跳陈旧)时才抢到重置权 + staleBeforeMs := time.Now().UnixMilli() - int64(heartbeatStaleAfter/time.Millisecond) + reset, err := sessionDao.ExecWorkflowDao.ResetRunningIfRecoverable(userCtx, execId, nodeGroupId, staleBeforeMs) + if err != nil { + g.Log().Errorf(lockCtx, "恢复置运行中失败 execId=%d: %v", execId, err) return } - err = BuildExecution(userCtx, false, exec.FlowId, execId, nodeGroupId, exec.SessionId, exec.RequestParams) - if err != nil { - // 续跑失败:错误分类落库(retryable/retry_count),留给下一轮扫描决定是否再恢复 - if errors.Is(err, context.Canceled) { - _ = sessionDao.ExecWorkflowDao.UpdateRetry(userCtx, execId, 0, exec.RetryCount) + if !reset { + // 状态已被其它路径(用户 reExecute/execute 或其它节点恢复)抢先重置:放弃续跑,状态由持有方收敛,不落终态 + g.Log().Infof(lockCtx, "execId=%d 已被其它路径抢先重置为运行中,跳过恢复", execId) + return + } + + // 事件中枢所有权:此时已抢到锁+重置权,本 goroutine 是实际执行者,接管 hub(TryOwn 原子置 owned+cancel)。 + // 已有持有者(同 session+flow 另有执行在跑)则本恢复不持有 hub:锁与重置权已到手,放弃会留 status=1 + // 无主,继续执行但进度不广播(恢复仍正常完成落终态)。 + if hub != nil { + if !hub.TryOwn(topCancel) { + hub = nil } else { - _ = sessionDao.ExecWorkflowDao.UpdateRetry(userCtx, execId, 1, exec.RetryCount+1) - // 终局清理(Task 5 用户裁定):非用户取消且重试耗尽 → 执行永久失败, - // 清理该 exec 残留的 flow_async_task 孤儿缓存,避免未来复用同一 execId 的运行误取到过期 done 结果 - if exec.RetryCount+1 >= execMaxRetryCount { - _ = flowDao.FlowAsyncTaskDao.DeleteByExecution(userCtx, execId) + defer hub.Close() + // 恢复建立前用户已取消(占位期 workflow_cancel 只置标志、cancel 尚未设置):立即中止, + // 走下方 UserCancelled 分类写永久取消(retryable=0) + if hub.UserCancelled() { + topCancel() } } - recordWorkflow(userCtx, execId, 0, err) + } + + // 执行与心跳绑定同一 execCtx(带 recoverExecTimeout 上限): + // 心跳停止(超时/进程死/租约丢失)与 BuildExecution 中止必须同步,否则出现 + // "心跳已过期但执行还活着" 的窗口,被其它节点扫描恢复导致双跑。 + // 心跳连续失败达陈旧阈值时回调 execCancel 中止执行(心跳与执行同生共死)。 + // 从函数顶部登记的 topCtx 派生执行 ctx:注入用户信息(供 insertHook 落库)+ 12h 执行超时上限。 + // 关停时 cancelAllExecRuns 取消 topCancel → 本 ctx 随之取消,BuildExecution 中止后走下方错误分类落终态。 + execCtx, execCancel := context.WithTimeout(context.WithValue(topCtx, "user", &beans.User{UserName: exec.Creator, TenantId: exec.TenantId}), recoverExecTimeout) + defer execCancel() + stop := startHeartbeat(execCtx, execId, execCancel) + defer stop() + // 图节点进度经 hub 广播(无 hub 时 getProgressHub 返回 nil,reporter nil 安全) + progressCtx := execCtx + if hub != nil { + progressCtx = context.WithValue(execCtx, wsProgressCtxKey{}, hub) + } + err = BuildExecution(progressCtx, false, exec.FlowId, execId, nodeGroupId, exec.SessionId, exec.RequestParams) + if err != nil { + // 用户显式取消(附着连接 workflow_cancel):永久取消,retryable=0,恢复扫描不再捞起, + // 杜绝"取消→恢复→再取消"循环;清 flow_async_task 孤儿缓存,落"用户已终止执行" + if hub != nil && hub.UserCancelled() { + retryable, retryCnt := 0, 0 + _ = flowDao.FlowAsyncTaskDao.DeleteByExecution(userCtx, execId) + recordWorkflow(userCtx, execId, 0, context.Canceled, &retryable, &retryCnt) + hub.Publish(&wsCommon.WsPushMsg{Type: "error", Message: errWorkflowTerminated}) + return + } + // 续跑失败:恢复例程无用户,任意错误(含执行超时/租约丢失取消/模型/DB/网络/panic) + // 一律 retryable=1 交下一轮扫描决定是否再恢复;重试耗尽才终局失败。 + retryable, retryCnt := 1, exec.RetryCount+1 + // 优雅关停导致的取消:换错误标记让 recordWorkflow 写"程序关停中断"(与 WS 路径一致), + // 仍 retryable=1 下次启动扫描捞起续跑;非关停的取消(租约丢失/执行超时)保留原错误 + if errors.Is(err, context.Canceled) && IsShuttingDown() { + err = errInterruptedByShutdown + } + // 终局清理(Task 5 用户裁定):重试耗尽 → 执行永久失败, + // 清理该 exec 残留的 flow_async_task 孤儿缓存,避免未来复用同一 execId 的运行误取到过期 done 结果 + if exec.RetryCount+1 >= execMaxRetryCount { + _ = flowDao.FlowAsyncTaskDao.DeleteByExecution(userCtx, execId) + } + recordWorkflow(userCtx, execId, 0, err, &retryable, &retryCnt) + // 终态广播(附着连接;scan 路径无连接则无人接收) + if hub != nil { + hub.Publish(&wsCommon.WsPushMsg{Type: "error", Message: "工作流执行失败", Error: err.Error()}) + } return } // 续跑成功:recordWorkflow 落 status=2;BuildExecution 已清理 checkpoint/segment_result/flow_async_task - recordWorkflow(userCtx, execId, 0, nil) + recordWorkflow(userCtx, execId, 0, nil, nil, nil) + // 终态广播:把本次执行保存的结果文件路径一并推给前端 + if hub != nil { + hub.Publish(&wsCommon.WsPushMsg{Type: "flow_complete", Message: "工作流执行完成", Data: map[string]interface{}{ + "resultFileUrls": workflowResultFileUrls(userCtx, execId), + }}) + } } // isRecoverable 判定可恢复(spec §3):僵尸运行中(status=1 且心跳陈旧)或可重试失败(status=3 且 retryable=1 且未耗尽)。 @@ -147,19 +276,29 @@ func isRecoverable(exec *entity.ExecWorkflow, nowMs int64) bool { } // startHeartbeat 后台心跳 goroutine:每 30s touch last_heartbeat,返回 stop 函数。 -// 覆盖正常执行与恢复执行,崩溃前最后一次心跳即崩溃近似时间戳(spec §4) -func startHeartbeat(ctx context.Context, execId int64) func() { +// 覆盖正常执行与恢复执行,崩溃前最后一次心跳即崩溃近似时间戳(spec §4)。 +// onLeaseLost:心跳连续失败达到陈旧阈值(租约丢失)时回调——调用方应取消执行 ctx, +// 使心跳与执行同生共死,避免"心跳已过期但执行还活着"的窗口被其它节点恢复导致双跑。 +func startHeartbeat(ctx context.Context, execId int64, onLeaseLost func()) func() { stopCh := make(chan struct{}) done := make(chan struct{}) go func() { defer close(done) ticker := time.NewTicker(heartbeatInterval) defer ticker.Stop() + var consecutiveFail int for { select { case <-ticker.C: if err := sessionDao.ExecWorkflowDao.TouchHeartbeat(ctx, execId); err != nil { g.Log().Warningf(ctx, "心跳落库失败 execId=%d: %v", execId, err) + consecutiveFail++ + if onLeaseLost != nil && consecutiveFail >= heartbeatMaxFail { + g.Log().Errorf(ctx, "心跳连续失败 %d 次,租约丢失,中止执行 execId=%d", consecutiveFail, execId) + onLeaseLost() + } + } else { + consecutiveFail = 0 } case <-stopCh: return diff --git a/workflow/service/flow/recover_execution_test.go b/workflow/service/flow/recover_execution_test.go deleted file mode 100644 index 48a87a1..0000000 --- a/workflow/service/flow/recover_execution_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package flow - -import ( - "context" - "errors" - "testing" - "time" - - "ai-agent/workflow/consts/flow" - "ai-agent/workflow/model/entity" -) - -func TestIsRecoverable(t *testing.T) { - now := time.Now().UnixMilli() - staleBefore := now - int64(heartbeatStaleAfter/time.Millisecond) - 1 - fresh := now // 心跳新鲜 - - running := func(hb int64) *entity.ExecWorkflow { - return &entity.ExecWorkflow{Status: flow.FlowExecutionStatusRunning.Code(), LastHeartbeat: hb} - } - failed := func(retryable, retryCount int) *entity.ExecWorkflow { - return &entity.ExecWorkflow{Status: flow.FlowExecutionStatusFailed.Code(), Retryable: retryable, RetryCount: retryCount} - } - - cases := []struct { - name string - exec *entity.ExecWorkflow - want bool - }{ - {"nil", nil, false}, - {"status=1 心跳陈旧 → 恢复", running(staleBefore), true}, - {"status=1 心跳新鲜 → 不恢复", running(fresh), false}, - {"status=1 心跳为0(老数据)→ 恢复", running(0), true}, - {"status=3 retryable=1 未耗尽 → 恢复", failed(1, 0), true}, - {"status=3 retryable=1 已耗尽 → 不恢复", failed(1, execMaxRetryCount), false}, - {"status=3 retryable=0(用户取消)→ 不恢复", failed(0, 0), false}, - {"status=2 成功 → 不恢复", &entity.ExecWorkflow{Status: flow.FlowExecutionStatusSuccess.Code()}, false}, - } - for _, c := range cases { - if got := isRecoverable(c.exec, now); got != c.want { - t.Fatalf("%s: got %v want %v", c.name, got, c.want) - } - } -} - -func TestShouldRetry(t *testing.T) { - canceled := context.Canceled - if shouldRetry(canceled) { - t.Fatalf("用户取消不应重试") - } - if shouldRetry(nil) { - t.Fatalf("nil 不应重试") - } - if shouldRetry(errExecAlreadyRunning) { - t.Fatalf("运行中标记不应重试") - } - if !shouldRetry(errors.New("模型调用失败")) { - t.Fatalf("程序报错应重试") - } -} diff --git a/workflow/service/flow/redis_lock_test.go b/workflow/service/flow/redis_lock_test.go deleted file mode 100644 index 9fc2534..0000000 --- a/workflow/service/flow/redis_lock_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package flow - -import ( - "context" - "os" - "testing" - - "github.com/gogf/gf/v2/frame/g" -) - -// TestRedisLockMutualExclusion 验证互斥与防误删(需 AI_AGENT_TEST_REDIS=1 且本地 Redis 可用) -func TestRedisLockMutualExclusion(t *testing.T) { - if os.Getenv("AI_AGENT_TEST_REDIS") != "1" { - t.Skip("skip: AI_AGENT_TEST_REDIS not set") - } - ctx := context.Background() - key := "workflow:exec:recover:test-lock" - // 清理残留 - g.Redis().Del(ctx, key) - - l1 := NewRedisLock(key, 60) - l2 := NewRedisLock(key, 60) - ok1, err := l1.Acquire(ctx) - if err != nil || !ok1 { - t.Fatalf("l1 acquire should succeed: ok=%v err=%v", ok1, err) - } - ok2, err := l2.Acquire(ctx) - if err != nil { - t.Fatalf("l2 acquire err: %v", err) - } - if ok2 { - t.Fatalf("l2 不应抢到已被 l1 持有的锁") - } - // l2 尝试释放 l1 的锁:token 不匹配,不应删除 - if err := l2.Release(ctx); err != nil { - t.Fatalf("l2 release err: %v", err) - } - ok2, _ = l2.Acquire(ctx) - if ok2 { - t.Fatalf("l2 释放他人锁后不应能抢到(l1 锁应仍在)") - } - // l1 释放后 l2 可抢到 - if err := l1.Release(ctx); err != nil { - t.Fatalf("l1 release err: %v", err) - } - ok2, err = l2.Acquire(ctx) - if err != nil || !ok2 { - t.Fatalf("l1 释放后 l2 应可抢到: ok=%v err=%v", ok2, err) - } - l2.Release(ctx) - g.Redis().Del(ctx, key) -}