From e889059d3e5a7bf289b9e72b13057c178c9e438a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 27 Aug 2026 09:15:32 +0800 Subject: [PATCH] 1 --- server/biz/dao/model_training.go | 12 ++++++++++++ server/biz/model/dto/training.go | 1 + server/biz/service/dataset.go | 1 + server/biz/service/training.go | 25 +++++++++++++++++++------ server_admin/src/views/Datasets.vue | 7 ++++++- 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/server/biz/dao/model_training.go b/server/biz/dao/model_training.go index 5d6de9b..6832fc7 100644 --- a/server/biz/dao/model_training.go +++ b/server/biz/dao/model_training.go @@ -184,6 +184,18 @@ func (d *modelTrainingDao) ListRunning(ctx context.Context) ([]*entity.ModelTrai return list, nil } +// FailUnstarted 重启恢复：running 且 pid 未落（发起准备阶段服务中断，进程已随服务消亡）的任务置 failed +func (d *modelTrainingDao) FailUnstarted(ctx context.Context) error { + _, err := g.DB().Model(consts.TableTraining).Ctx(ctx). + Where("status", consts.TrainingStatusRunning).Where("pid", 0). + Data(g.Map{ + "status": consts.TrainingStatusFailed, + "error": "服务器重启，训练发起未完成", + "finished_at": gtime.Now(), + }).Update() + return err +} + // LatestByDatasets 批量取各数据集最新一条训练记录（列表卡片训练状态用； // IN 一次取回按 id 倒序，应用层按 dataset_id 去重；数据集表小、记录少，单次查询足够） func (d *modelTrainingDao) LatestByDatasets(ctx context.Context, datasetIds []int64) (map[int64]*entity.ModelTraining, error) { diff --git a/server/biz/model/dto/training.go b/server/biz/model/dto/training.go index a84f4b0..efb1168 100644 --- a/server/biz/model/dto/training.go +++ b/server/biz/model/dto/training.go @@ -33,6 +33,7 @@ type AdminDatasetItem struct { Description string `json:"description"` // 描述 TrainingId int64 `json:"trainingId"` // 最新训练记录 id（发布/详情用） TrainingStatus string `json:"trainingStatus"` // 最新训练记录状态 running|success|failed|空 + TrainingError string `json:"trainingError"` // 最新训练记录失败原因（failed 时展示用） TrainingCurrentEpoch int `json:"trainingCurrentEpoch"` TrainingTotalEpochs int `json:"trainingTotalEpochs"` CreatedAt *gtime.Time `json:"createdAt"` diff --git a/server/biz/service/dataset.go b/server/biz/service/dataset.go index fa53106..e54732b 100644 --- a/server/biz/service/dataset.go +++ b/server/biz/service/dataset.go @@ -75,6 +75,7 @@ func (s *datasetService) AdminListDatasets(ctx context.Context, req *dto.AdminDa if t, ok := latest[v.Id]; ok { item.TrainingId = t.Id item.TrainingStatus = t.Status + item.TrainingError = t.Error item.TrainingCurrentEpoch = t.CurrentEpoch item.TrainingTotalEpochs = t.TotalEpochs } diff --git a/server/biz/service/training.go b/server/biz/service/training.go index 46ce43f..e07a448 100644 --- a/server/biz/service/training.go +++ b/server/biz/service/training.go @@ -31,6 +31,9 @@ var Training = &trainingService{} // 单协程生命周期任务（非并行工作负载），不做池封装。 func (s *trainingService) StartBackgroundJobs(ctx context.Context) { LabelTask.recoverLabelTasks(ctx) + if err := dao.Training.FailUnstarted(ctx); err != nil { + g.Log().Errorf(ctx, "恢复未启动训练任务失败: %+v", err) + } go s.pollTrainings(ctx) } @@ -92,12 +95,16 @@ func (s *trainingService) pollOne(ctx context.Context, runner common.TrainingRun s.finishSuccess(ctx, runner, job, t, result, tail) return } - alive, err := runner.IsAlive(ctx, job) - if err != nil { - g.Log().Errorf(ctx, "训练 %d 存活探测失败: %+v", t.Id, err) - return + alive := true + if t.Pid != 0 { + a, err := runner.IsAlive(ctx, job) + if err != nil { + g.Log().Errorf(ctx, "训练 %d 存活探测失败: %+v", t.Id, err) + return + } + alive = a } - // 超时判死（started_at 起算） + // 超时判死（started_at 起算；含发起准备阶段） cfg, _ := common.TrainingConfigOf(ctx) timeout := time.Duration(cfg.TimeoutMins) * time.Minute if t.StartedAt != nil && time.Since(t.StartedAt.Time) > timeout { @@ -105,6 +112,10 @@ func (s *trainingService) pollOne(ctx context.Context, runner common.TrainingRun _ = s.finishFailed(ctx, t, "训练超时（超过 %d 分钟无结果，已终止）", cfg.TimeoutMins) return } + // pid 未落 = 发起准备阶段（写任务参数/同步数据集/启动进程）尚未完成，不判死 + if t.Pid == 0 { + return + } if !alive { _ = s.finishFailed(ctx, t, "训练进程已退出（无结果文件）") return @@ -186,7 +197,9 @@ func mergeNamesIntoMetrics(metricsJSON, namesJSON string) string { } func (s *trainingService) finishFailed(ctx context.Context, t *entity.ModelTraining, format string, args ...any) error { - return dao.Training.Finish(ctx, t.Id, consts.TrainingStatusFailed, "", "", fmt.Sprintf(format, args...)) + msg := fmt.Sprintf(format, args...) + g.Log().Errorf(ctx, "训练 %d 置失败: %s", t.Id, msg) + return dao.Training.Finish(ctx, t.Id, consts.TrainingStatusFailed, "", "", msg) } // buildJob 组装训练机路径布局的 runner 任务 diff --git a/server_admin/src/views/Datasets.vue b/server_admin/src/views/Datasets.vue index 38a3537..4c1a9a6 100644 --- a/server_admin/src/views/Datasets.vue +++ b/server_admin/src/views/Datasets.vue @@ -278,7 +278,12 @@ onBeforeUnmount(() => {