342 lines
12 KiB
Go
342 lines
12 KiB
Go
package flow
|
||
|
||
import (
|
||
"ai-agent/workflow/consts/node"
|
||
"ai-agent/workflow/service/flow/processor"
|
||
"ai-agent/workflow/service/flow/processor/builtin/split_shots_pipeline"
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"strings"
|
||
|
||
"ai-agent/gateway"
|
||
flowDto "ai-agent/workflow/model/dto/flow"
|
||
"ai-agent/workflow/service/flow/processor/builtin/split_shots_pipeline/pipeline"
|
||
|
||
"github.com/gogf/gf/v2/frame/g"
|
||
"github.com/gogf/gf/v2/util/gconv"
|
||
)
|
||
|
||
// 脚本转写节点默认系统提示词（字段与 domain.Shot 的 JSON tag 对齐，结构化输出与 content 兜底两条路径一致）
|
||
const defaultScriptTranscribeSystemPrompt = `你是短剧分镜脚本师。请根据提供的文案/视频分析结果，把内容拆分为连续的分镜镜头脚本。
|
||
每个镜头输出一个 JSON 对象，字段固定为：
|
||
- index：镜头序号（数字）
|
||
- startTime：开始时间，格式 MM:SS
|
||
- endTime：结束时间，格式 MM:SS
|
||
- event：事件描述/动作描写
|
||
- narration：旁白/画外音，没有则省略
|
||
- dialogue：角色开口说的主台词，没有则省略
|
||
- ambientSound：环境音，没有则省略
|
||
- cameraMovement：运镜描述
|
||
- shotSize：景别
|
||
- characters：出演人物名列表（字符串数组）
|
||
- scene：场景名
|
||
- props：道具名列表（字符串数组）
|
||
时间码需前后衔接、覆盖整个内容时长。直接输出 JSON 数组，不要输出其他文字。`
|
||
|
||
// ScriptTranscribeLambda 脚本转写节点：
|
||
// 把节点输入（文案/视频分析结果，经 valueSource 解析）通过大模型转写为 []pipeline.Shot，
|
||
// 再经 split_shots_pipeline 前置处理器拆成各段扁平请求参数列表（[{"prompt","duration","seed",...},...]），
|
||
// 供下游视频生成节点逐段引用聚合。
|
||
func ScriptTranscribeLambda(ctx context.Context, input any) (any, error) {
|
||
nodeInput, ok := input.(*flowDto.NodeExecutionInput)
|
||
if !ok {
|
||
return nil, fmt.Errorf("入参类型错误")
|
||
}
|
||
|
||
n := new([]node.NodePresetField)
|
||
err := gconv.Structs(nodeInput.Config.OutputConfig, n)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
var totalDuration int
|
||
var modelId int64
|
||
// 静音模式：清空镜头台词/旁白，生成视频不含口播、旁白、字幕与人物开口动作，默认开启
|
||
noSpeech := true
|
||
for _, item := range *n {
|
||
switch item.Field {
|
||
case "noSpeech":
|
||
if !g.IsEmpty(item.Value) {
|
||
noSpeech = gconv.Bool(item.Value)
|
||
}
|
||
case "totalDuration":
|
||
if !g.IsEmpty(item.Value) {
|
||
totalDuration = gconv.Int(item.Value)
|
||
} else {
|
||
if !g.IsEmpty(item.ValueSource) {
|
||
for _, k := range item.ValueSource {
|
||
nodeConfig := nodeInput.Global.ConfigMap[k.NodeId]
|
||
if nodeConfig != nil {
|
||
for _, output := range nodeConfig.OutputResult {
|
||
if !g.IsEmpty(output[k.Field]) {
|
||
totalDuration = gconv.Int(output[k.Field])
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
case "modelId":
|
||
modelId = gconv.Int64(item.Value)
|
||
}
|
||
}
|
||
|
||
modelParams, err := BuildModelRequestBody(nodeInput.Config.ModelConfig.ModelRequestParamsPath, nodeInput.Global)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
// 2. 构建系统提示词 + 用户输入
|
||
systemPrompt := nodeInput.Config.Prompt
|
||
if systemPrompt == "" {
|
||
systemPrompt = defaultScriptTranscribeSystemPrompt
|
||
}
|
||
// 参考素材名单注入：转写模型只从名单选名，保证镜头里的角色/场景/道具名与参考素材精确一致
|
||
// （名字绑定/类别推断都依赖名字对上）
|
||
refsName, refsItem := pipeline.ExtractRefs(nodeInput.Config.ModelConfig.ModelRequestParams)
|
||
if len(refsName) > 0 {
|
||
systemPrompt += "\n\n参考素材名单：" + strings.Join(refsName, "、") +
|
||
"\n约束：镜头里的 characters/scene/props 必须原样使用名单中的名字，不得改写、不得加修饰（如“主角小明”）、不得造新名；名单外的名字按原文输出。"
|
||
}
|
||
if totalDuration > 0 {
|
||
systemPrompt += fmt.Sprintf("\n\n视频总时长 %d 秒（MM:SS 为 %s）：所有镜头的时间码需前后衔接并完整覆盖该总时长，最后一镜的 endTime 对齐到总时长。", totalDuration, formatSecondsToMMSS(totalDuration))
|
||
}
|
||
// 单镜头时长上限约束：按视频模型推导单段最大时长注入转写提示词，从源头避免超长镜头
|
||
//（SplitOversized 仍是机械兜底）；推导失败仅降级跳过约束注入，不影响转写主流程。
|
||
if maxSeg, _, err := split_shots_pipeline.SegmentBounds(ctx, modelId); err != nil {
|
||
g.Log().Warningf(ctx, "获取视频模型单段时长约束失败，跳过单镜时长约束注入: %v", err)
|
||
} else {
|
||
systemPrompt += shotDurationConstraintPrompt(maxSeg)
|
||
}
|
||
|
||
info, err := gateway.GetModelInfoById(ctx, &gateway.GetModelInfoByIdReq{ModelId: nodeInput.Config.ModelConfig.ModelId})
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
params := map[string]any{
|
||
"system_prompt": systemPrompt,
|
||
}
|
||
// 结构化输出：chat 模型映射配置了 response_format（StructuredOutput）即走原生 json_schema 保证结构；
|
||
// 否则模型 content 直接出 JSON，靠容错解析兜底
|
||
val, ok := info.ModelManage.RequestBusinessFieldMapping["response_format"]
|
||
if ok {
|
||
if !g.IsEmpty(val) {
|
||
params["response_format"] = pipeline.ShotsStructuredFormat()
|
||
}
|
||
}
|
||
modelInfo, err := gateway.GetModelInfoById(ctx, &gateway.GetModelInfoByIdReq{ModelId: nodeInput.Config.ModelConfig.ModelId})
|
||
if err != nil {
|
||
return "", fmt.Errorf("获取模型配置失败: %w", err)
|
||
}
|
||
result, err := gateway.ModelCallResult(ctx, nodeInput.Config.ModelConfig.ModelId, modelInfo.ModelManage.ResponseType, nodeInput.Global.SessionId, modelParams, params)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
var content string
|
||
for _, v := range gconv.Map(result.Content) {
|
||
content += v.(string)
|
||
}
|
||
// 4. 解析镜头数组（兼容英文键结构化输出与中文键 content 兜底）
|
||
shots, err := unmarshalShots(content)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
// 静音模式：清空台词与旁白，保证写进视频 prompt 的只有事件/环境音/运镜/景别，
|
||
// 视频模型不会产生口播、旁白配音、字幕烧录，也不会把角色标记为开口优先做口型动画
|
||
if noSpeech {
|
||
for i := range shots {
|
||
shots[i].Dialogue = ""
|
||
shots[i].Narration = ""
|
||
}
|
||
}
|
||
|
||
// 5. 产出固定结构 {"shots": [...]}
|
||
var arr []any
|
||
if b, err := json.Marshal(shots); err == nil {
|
||
_ = json.Unmarshal(b, &arr)
|
||
}
|
||
|
||
args := split_shots_pipeline.SplitShotsInput{
|
||
ModelID: modelId,
|
||
Shots: shots,
|
||
TotalDuration: totalDuration,
|
||
FlatRefs: refsItem,
|
||
Seed: nodeInput.Global.ExecutionId % 1000000,
|
||
NegativePrompt: nodeInput.Config.NegativePrompt,
|
||
}
|
||
data, err := processor.Call(ctx, "split_shots_pipeline", gconv.Map(args))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
nodeInput.Config.OutputResult = gconv.Maps(data)
|
||
return nodeInput, nil
|
||
}
|
||
|
||
// unmarshalShots 解析镜头数组 JSON，容忍 markdown 代码围栏、json_schema 结构化输出的 {"shots":[...]} 包装，
|
||
// 以及模型按中文键输出（时间码/事件/台词旁白/景别/运镜/出演角色/场景/道具）的容错映射。
|
||
func unmarshalShots(s string) ([]pipeline.Shot, error) {
|
||
s = strings.TrimSpace(s)
|
||
if strings.HasPrefix(s, "```") {
|
||
s = strings.TrimPrefix(s, "```json")
|
||
s = strings.TrimPrefix(s, "```")
|
||
s = strings.TrimSuffix(s, "```")
|
||
s = strings.TrimSpace(s)
|
||
}
|
||
var raw []map[string]any
|
||
if err := json.Unmarshal([]byte(s), &raw); err == nil && len(raw) > 0 {
|
||
return parseShots(raw)
|
||
}
|
||
var wrapped struct {
|
||
Shots []map[string]any `json:"shots"`
|
||
}
|
||
if err := json.Unmarshal([]byte(s), &wrapped); err == nil && len(wrapped.Shots) > 0 {
|
||
return parseShots(wrapped.Shots)
|
||
}
|
||
return nil, fmt.Errorf("解析分镜脚本失败: %v", s)
|
||
}
|
||
|
||
// parseShots 把原始镜头对象数组归一为 domain.Shot，跳过没有内容字段的镜头。
|
||
func parseShots(raw []map[string]any) ([]pipeline.Shot, error) {
|
||
shots := make([]pipeline.Shot, 0, len(raw))
|
||
for i, m := range raw {
|
||
shot := shotFromMap(m)
|
||
if shot.Index == 0 {
|
||
shot.Index = i + 1
|
||
}
|
||
if isEmptyShot(shot) {
|
||
continue
|
||
}
|
||
shots = append(shots, shot)
|
||
}
|
||
if len(shots) == 0 {
|
||
return nil, fmt.Errorf("解析分镜脚本失败: 镜头内容为空")
|
||
}
|
||
return shots, nil
|
||
}
|
||
|
||
// isEmptyShot 镜头是否没有可用内容（仅有时间码/序号，或字段名对不上导致全空）。
|
||
func isEmptyShot(s pipeline.Shot) bool {
|
||
return s.Event == "" && s.Dialogue == "" && s.Narration == "" && s.AmbientSound == "" &&
|
||
s.CameraMovement == "" && s.ShotSize == "" && s.Scene == "" &&
|
||
len(s.Characters) == 0 && len(s.Props) == 0
|
||
}
|
||
|
||
// shotFromMap 把单个镜头对象映射为 domain.Shot，兼容英文键（结构化输出）与中文键（提示词兜底输出）。
|
||
func shotFromMap(m map[string]any) pipeline.Shot {
|
||
get := func(keys ...string) string {
|
||
for _, k := range keys {
|
||
switch v := m[k].(type) {
|
||
case string:
|
||
if t := strings.TrimSpace(v); t != "" {
|
||
return t
|
||
}
|
||
case float64:
|
||
return gconv.String(v)
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
getSlice := func(keys ...string) []string {
|
||
for _, k := range keys {
|
||
switch v := m[k].(type) {
|
||
case []any:
|
||
var out []string
|
||
for _, e := range v {
|
||
if t, ok := e.(string); ok && strings.TrimSpace(t) != "" {
|
||
out = append(out, strings.TrimSpace(t))
|
||
}
|
||
}
|
||
if len(out) > 0 {
|
||
return out
|
||
}
|
||
case string:
|
||
if out := splitList(v); len(out) > 0 {
|
||
return out
|
||
}
|
||
}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
shot := pipeline.Shot{
|
||
Index: gconv.Int(m["index"]),
|
||
StartTime: get("startTime", "开始时间"),
|
||
EndTime: get("endTime", "结束时间"),
|
||
Event: get("event", "事件"),
|
||
Dialogue: get("dialogue", "台词"),
|
||
Narration: get("narration", "旁白"),
|
||
AmbientSound: get("ambientSound", "环境音"),
|
||
CameraMovement: get("cameraMovement", "运镜"),
|
||
ShotSize: get("shotSize", "景别"),
|
||
Scene: get("scene", "场景"),
|
||
Characters: getSlice("characters", "出演角色"),
|
||
Props: getSlice("props", "道具"),
|
||
}
|
||
if shot.StartTime == "" && shot.EndTime == "" {
|
||
shot.StartTime, shot.EndTime = splitTimeRange(get("时间码"))
|
||
}
|
||
if shot.Dialogue == "" && shot.Narration == "" {
|
||
shot.Dialogue, shot.Narration = splitDialogueNarration(get("台词/旁白"))
|
||
}
|
||
return shot
|
||
}
|
||
|
||
// formatSecondsToMMSS 秒转 MM:SS 时间码。
|
||
func formatSecondsToMMSS(sec int) string {
|
||
if sec < 0 {
|
||
sec = 0
|
||
}
|
||
return fmt.Sprintf("%02d:%02d", sec/60, sec%60)
|
||
}
|
||
|
||
// splitTimeRange 解析时间码 "MM:SS-MM:SS"（兼容 "—"/"~"/"到" 等分隔，或单个时间点）。
|
||
func splitTimeRange(s string) (start, end string) {
|
||
if s == "" {
|
||
return "", ""
|
||
}
|
||
normalized := strings.NewReplacer("—", "-", "–", "-", "～", "-", "~", "-", "到", "-", "至", "-").Replace(s)
|
||
parts := strings.Split(normalized, "-")
|
||
start = strings.TrimSpace(parts[0])
|
||
if len(parts) > 1 {
|
||
end = strings.TrimSpace(parts[1])
|
||
} else {
|
||
end = start
|
||
}
|
||
return start, end
|
||
}
|
||
|
||
// splitDialogueNarration 把"台词/旁白"合字段拆成 dialogue 与 narration：
|
||
// 以"旁白"/"画外音"开头的内容归为旁白，其余视为角色开口的主台词。
|
||
func splitDialogueNarration(s string) (dialogue, narration string) {
|
||
s = strings.TrimSpace(s)
|
||
switch {
|
||
case strings.HasPrefix(s, "旁白"):
|
||
return "", strings.TrimSpace(strings.TrimLeft(strings.TrimPrefix(s, "旁白"), "：:"))
|
||
case strings.HasPrefix(s, "画外音"):
|
||
return "", strings.TrimSpace(strings.TrimLeft(strings.TrimPrefix(s, "画外音"), "：:"))
|
||
default:
|
||
return s, ""
|
||
}
|
||
}
|
||
|
||
// splitList 按常见分隔符拆分人名/道具列表（兼容中英文顿号、逗号、分号、"和""及"等）。
|
||
func splitList(s string) []string {
|
||
repl := strings.NewReplacer("、", "|", "，", "|", ",", "|", "；", "|", ";", "|", "和", "|", "及", "|", "&", "|", "/", "|", " ", "|")
|
||
var out []string
|
||
for _, p := range strings.Split(repl.Replace(strings.TrimSpace(s)), "|") {
|
||
if t := strings.TrimSpace(p); t != "" {
|
||
out = append(out, t)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// shotDurationConstraintPrompt 生成"单个镜头时长不超过 maxSeg 秒"的转写约束提示词片段；maxSeg<=0 返回空串。
|
||
func shotDurationConstraintPrompt(maxSeg int) string {
|
||
if maxSeg <= 0 {
|
||
return ""
|
||
}
|
||
return fmt.Sprintf("\n\n单个镜头时长不超过 %d 秒：每镜的 startTime 与 endTime 之差必须 ≤ %d 秒。", maxSeg, maxSeg)
|
||
}
|