package flow import ( "ai-agent/workflow/consts/node" "ai-agent/workflow/service/flow/processor" "ai-agent/workflow/service/flow/processor/builtin/split_shots_pipeline" "context" "encoding/json" "fmt" "strings" "ai-agent/gateway" flowDto "ai-agent/workflow/model/dto/flow" "ai-agent/workflow/service/flow/processor/builtin/split_shots_pipeline/pipeline" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/util/gconv" ) // 脚本转写节点默认系统提示词(字段与 domain.Shot 的 JSON tag 对齐,结构化输出与 content 兜底两条路径一致) const defaultScriptTranscribeSystemPrompt = `你是短剧分镜脚本师。请根据提供的文案/视频分析结果,把内容拆分为连续的分镜镜头脚本。 每个镜头输出一个 JSON 对象,字段固定为: - index:镜头序号(数字) - startTime:开始时间,格式 MM:SS - endTime:结束时间,格式 MM:SS - event:事件描述/动作描写 - narration:旁白/画外音,没有则省略 - dialogue:角色开口说的主台词,没有则省略 - ambientSound:环境音,没有则省略 - cameraMovement:运镜描述 - shotSize:景别 - characters:出演人物名列表(字符串数组) - scene:场景名 - props:道具名列表(字符串数组) 时间码需前后衔接、覆盖整个内容时长。直接输出 JSON 数组,不要输出其他文字。` // ScriptTranscribeLambda 脚本转写节点: // 把节点输入(文案/视频分析结果,经 valueSource 解析)通过大模型转写为 []pipeline.Shot, // 再经 split_shots_pipeline 前置处理器拆成各段扁平请求参数列表([{"prompt","duration","seed",...},...]), // 供下游视频生成节点逐段引用聚合。 func ScriptTranscribeLambda(ctx context.Context, input any) (any, error) { nodeInput, ok := input.(*flowDto.NodeExecutionInput) if !ok { return nil, fmt.Errorf("入参类型错误") } n := new([]node.NodePresetField) err := gconv.Structs(nodeInput.Config.OutputConfig, n) if err != nil { return nil, err } var totalDuration int var modelId int64 // 静音模式:清空镜头台词/旁白,生成视频不含口播、旁白、字幕与人物开口动作,默认开启 noSpeech := true for _, item := range *n { switch item.Field { case "noSpeech": if !g.IsEmpty(item.Value) { noSpeech = gconv.Bool(item.Value) } case "totalDuration": if !g.IsEmpty(item.Value) { totalDuration = gconv.Int(item.Value) } else { if !g.IsEmpty(item.ValueSource) { for _, k := range item.ValueSource { nodeConfig := nodeInput.Global.ConfigMap[k.NodeId] if nodeConfig != nil { for _, output := range nodeConfig.OutputResult { if !g.IsEmpty(output[k.Field]) { totalDuration = gconv.Int(output[k.Field]) } } } } } } case "modelId": modelId = gconv.Int64(item.Value) } } modelParams, err := BuildModelRequestBody(nodeInput.Config.ModelConfig.ModelRequestParamsPath, nodeInput.Global) if err != nil { return nil, err } // 2. 构建系统提示词 + 用户输入 systemPrompt := nodeInput.Config.Prompt if systemPrompt == "" { systemPrompt = defaultScriptTranscribeSystemPrompt } // 参考素材名单注入:转写模型只从名单选名,保证镜头里的角色/场景/道具名与参考素材精确一致 // (名字绑定/类别推断都依赖名字对上) refsName, refsItem := pipeline.ExtractRefs(nodeInput.Config.ModelConfig.ModelRequestParams) if len(refsName) > 0 { systemPrompt += "\n\n参考素材名单:" + strings.Join(refsName, "、") + "\n约束:镜头里的 characters/scene/props 必须原样使用名单中的名字,不得改写、不得加修饰(如“主角小明”)、不得造新名;名单外的名字按原文输出。" } if totalDuration > 0 { systemPrompt += fmt.Sprintf("\n\n视频总时长 %d 秒(MM:SS 为 %s):所有镜头的时间码需前后衔接并完整覆盖该总时长,最后一镜的 endTime 对齐到总时长。", totalDuration, formatSecondsToMMSS(totalDuration)) } // 单镜头时长上限约束:按视频模型推导单段最大时长注入转写提示词,从源头避免超长镜头 //(SplitOversized 仍是机械兜底);推导失败仅降级跳过约束注入,不影响转写主流程。 if maxSeg, _, err := split_shots_pipeline.SegmentBounds(ctx, modelId); err != nil { g.Log().Warningf(ctx, "获取视频模型单段时长约束失败,跳过单镜时长约束注入: %v", err) } else { systemPrompt += shotDurationConstraintPrompt(maxSeg) } // 静音模式硬约束:从转写源头杜绝对白/旁白/开口说话,后续清洗只做兜底 if noSpeech { systemPrompt += noSpeechSystemPromptConstraint() } info, err := gateway.GetModelInfoById(ctx, &gateway.GetModelInfoByIdReq{ModelId: nodeInput.Config.ModelConfig.ModelId}) if err != nil { return nil, err } params := map[string]any{ "system_prompt": systemPrompt, } // 结构化输出:chat 模型映射配置了 response_format(StructuredOutput)即走原生 json_schema 保证结构; // 否则模型 content 直接出 JSON,靠容错解析兜底 val, ok := info.ModelManage.RequestBusinessFieldMapping["response_format"] if ok { if !g.IsEmpty(val) { params["response_format"] = pipeline.ShotsStructuredFormat() } } modelInfo, err := gateway.GetModelInfoById(ctx, &gateway.GetModelInfoByIdReq{ModelId: nodeInput.Config.ModelConfig.ModelId}) if err != nil { return "", fmt.Errorf("获取模型配置失败: %w", err) } result, err := gateway.ModelCallResult(ctx, nodeInput.Config.ModelConfig.ModelId, modelInfo.ModelManage.ResponseType, nodeInput.Global.SessionId, modelParams, params) if err != nil { return nil, err } var content string for _, v := range gconv.Map(result.Content) { content += v.(string) } // 4. 解析镜头数组(兼容英文键结构化输出与中文键 content 兜底) shots, err := unmarshalShots(content) if err != nil { return nil, err } // 静音模式:清空台词与旁白,保证写进视频 prompt 的只有事件/环境音/运镜/景别, // 视频模型不会产生口播、旁白配音、字幕烧录,也不会把角色标记为开口优先做口型动画 if noSpeech { for i := range shots { shots[i].Dialogue = "" shots[i].Narration = "" // event 里的说话动词仍会经 事件:%s 块写进分段 prompt,导致视频模型生成口型/字幕,需确定性清洗 shots[i].Event = cleanSpeechVerbs(shots[i].Event) } } // 5. 产出固定结构 {"shots": [...]} var arr []any if b, err := json.Marshal(shots); err == nil { _ = json.Unmarshal(b, &arr) } args := split_shots_pipeline.SplitShotsInput{ ModelID: modelId, Shots: shots, TotalDuration: totalDuration, FlatRefs: refsItem, Seed: nodeInput.Global.ExecutionId % 1000000, NegativePrompt: nodeInput.Config.NegativePrompt, NoSpeech: noSpeech, } data, err := processor.Call(ctx, "split_shots_pipeline", gconv.Map(args)) if err != nil { return nil, err } nodeInput.Config.OutputResult = gconv.Maps(data) return nodeInput, nil } // unmarshalShots 解析镜头数组 JSON,容忍 markdown 代码围栏、json_schema 结构化输出的 {"shots":[...]} 包装, // 以及模型按中文键输出(时间码/事件/台词旁白/景别/运镜/出演角色/场景/道具)的容错映射。 func unmarshalShots(s string) ([]pipeline.Shot, error) { s = strings.TrimSpace(s) if strings.HasPrefix(s, "```") { s = strings.TrimPrefix(s, "```json") s = strings.TrimPrefix(s, "```") s = strings.TrimSuffix(s, "```") s = strings.TrimSpace(s) } var raw []map[string]any if err := json.Unmarshal([]byte(s), &raw); err == nil && len(raw) > 0 { return parseShots(raw) } var wrapped struct { Shots []map[string]any `json:"shots"` } if err := json.Unmarshal([]byte(s), &wrapped); err == nil && len(wrapped.Shots) > 0 { return parseShots(wrapped.Shots) } return nil, fmt.Errorf("解析分镜脚本失败: %v", s) } // parseShots 把原始镜头对象数组归一为 domain.Shot,跳过没有内容字段的镜头。 func parseShots(raw []map[string]any) ([]pipeline.Shot, error) { shots := make([]pipeline.Shot, 0, len(raw)) for i, m := range raw { shot := shotFromMap(m) if shot.Index == 0 { shot.Index = i + 1 } if isEmptyShot(shot) { continue } shots = append(shots, shot) } if len(shots) == 0 { return nil, fmt.Errorf("解析分镜脚本失败: 镜头内容为空") } return shots, nil } // isEmptyShot 镜头是否没有可用内容(仅有时间码/序号,或字段名对不上导致全空)。 func isEmptyShot(s pipeline.Shot) bool { return s.Event == "" && s.Dialogue == "" && s.Narration == "" && s.AmbientSound == "" && s.CameraMovement == "" && s.ShotSize == "" && s.Scene == "" && len(s.Characters) == 0 && len(s.Props) == 0 } // shotFromMap 把单个镜头对象映射为 domain.Shot,兼容英文键(结构化输出)与中文键(提示词兜底输出)。 func shotFromMap(m map[string]any) pipeline.Shot { get := func(keys ...string) string { for _, k := range keys { switch v := m[k].(type) { case string: if t := strings.TrimSpace(v); t != "" { return t } case float64: return gconv.String(v) } } return "" } getSlice := func(keys ...string) []string { for _, k := range keys { switch v := m[k].(type) { case []any: var out []string for _, e := range v { if t, ok := e.(string); ok && strings.TrimSpace(t) != "" { out = append(out, strings.TrimSpace(t)) } } if len(out) > 0 { return out } case string: if out := splitList(v); len(out) > 0 { return out } } } return nil } shot := pipeline.Shot{ Index: gconv.Int(m["index"]), StartTime: get("startTime", "开始时间"), EndTime: get("endTime", "结束时间"), Event: get("event", "事件"), Dialogue: get("dialogue", "台词"), Narration: get("narration", "旁白"), AmbientSound: get("ambientSound", "环境音"), CameraMovement: get("cameraMovement", "运镜"), ShotSize: get("shotSize", "景别"), Scene: get("scene", "场景"), Characters: getSlice("characters", "出演角色"), Props: getSlice("props", "道具"), } if shot.StartTime == "" && shot.EndTime == "" { shot.StartTime, shot.EndTime = splitTimeRange(get("时间码")) } if shot.Dialogue == "" && shot.Narration == "" { shot.Dialogue, shot.Narration = splitDialogueNarration(get("台词/旁白")) } return shot } // formatSecondsToMMSS 秒转 MM:SS 时间码。 func formatSecondsToMMSS(sec int) string { if sec < 0 { sec = 0 } return fmt.Sprintf("%02d:%02d", sec/60, sec%60) } // splitTimeRange 解析时间码 "MM:SS-MM:SS"(兼容 "—"/"~"/"到" 等分隔,或单个时间点)。 func splitTimeRange(s string) (start, end string) { if s == "" { return "", "" } normalized := strings.NewReplacer("—", "-", "–", "-", "~", "-", "~", "-", "到", "-", "至", "-").Replace(s) parts := strings.Split(normalized, "-") start = strings.TrimSpace(parts[0]) if len(parts) > 1 { end = strings.TrimSpace(parts[1]) } else { end = start } return start, end } // splitDialogueNarration 把"台词/旁白"合字段拆成 dialogue 与 narration: // 以"旁白"/"画外音"开头的内容归为旁白,其余视为角色开口的主台词。 func splitDialogueNarration(s string) (dialogue, narration string) { s = strings.TrimSpace(s) switch { case strings.HasPrefix(s, "旁白"): return "", strings.TrimSpace(strings.TrimLeft(strings.TrimPrefix(s, "旁白"), "::")) case strings.HasPrefix(s, "画外音"): return "", strings.TrimSpace(strings.TrimLeft(strings.TrimPrefix(s, "画外音"), "::")) default: return s, "" } } // cleanSpeechVerbs 静音模式下清洗 event 中的说话动词:把常见说话/喊叫/对白表达替换为空串, // 避免"事件:%s"块里残留的说话动词让视频模型生成口型/字幕。仅做机械兜底,硬约束在转写提示词。 // NewReplacer 按最长匹配替换,故先列含"说/喊"的非开口语义词做保护(no-op,如"说明""呐喊"), // 再列开口表达;"叫"语义多变(呼叫/叫停/叫住),不做裸清洗以免误伤。 func cleanSpeechVerbs(s string) string { if s == "" { return "" } repl := strings.NewReplacer( "说明", "说明", "解说", "解说", "据说", "据说", "传说", "传说", "小说", "小说", "学说", "学说", "说法", "说法", "说服", "说服", "呐喊", "呐喊", "开口说话", "", "开口说", "", "开口", "", "说道:", "", "说道:", "", "说道", "", "说着", "", "说话", "", "讲话", "", "台词", "", "对白", "", "喊道:", "", "喊道:", "", "喊道", "", "大喊", "", "喊叫", "", "叫道:", "", "叫道:", "", "叫道", "", "叫到", "", "叫喊", "", "回答", "", "答道", "", "回应", "", "回话", "", "问道:", "", "问道:", "", "问道", "", "念叨", "", "嘟囔", "", "嘀咕", "", "自言自语", "", "呼唤", "", "呼叫", "", "叫唤", "", "惊叫", "", "惨叫", "", "说:", "", "说:", "", "说", "", "喊:", "", "喊:", "", "喊", "", ) return strings.TrimSpace(repl.Replace(s)) } // splitList 按常见分隔符拆分人名/道具列表(兼容中英文顿号、逗号、分号、"和""及"等)。 func splitList(s string) []string { repl := strings.NewReplacer("、", "|", ",", "|", ",", "|", ";", "|", ";", "|", "和", "|", "及", "|", "&", "|", "/", "|", " ", "|") var out []string for _, p := range strings.Split(repl.Replace(strings.TrimSpace(s)), "|") { if t := strings.TrimSpace(p); t != "" { out = append(out, t) } } return out } // shotDurationConstraintPrompt 生成"单个镜头时长不超过 maxSeg 秒"的转写约束提示词片段;maxSeg<=0 返回空串。 func shotDurationConstraintPrompt(maxSeg int) string { if maxSeg <= 0 { return "" } return fmt.Sprintf("\n\n单个镜头时长不超过 %d 秒:每镜的 startTime 与 endTime 之差必须 ≤ %d 秒。", maxSeg, maxSeg) } // noSpeechSystemPromptConstraint 静音模式的转写硬约束:要求模型从源头就不产出对白/旁白/说话动词, // 后续 noSpeech 清洗(清空台词旁白 + cleanSpeechVerbs)只做机械兜底。 func noSpeechSystemPromptConstraint() string { return "\n\n本片为静音模式,镜头里禁止任何声音类内容:\n" + "- 所有镜头禁止出现台词、旁白、画外音,narration 与 dialogue 一律留空、不要输出;\n" + "- 禁止角色开口说话,事件描述只能写无声的动作、表情、神态、场景变化,不要出现“说”“喊”“叫”“对白”“讲话”“开口”“问”“回答”“念叨”等说话类动词;\n" + "- characters 只是出镜角色名,不代表开口说话;\n" + "- 视频不包含口型动作与字幕,据此调整分镜描写。" }