red-future/cid
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

提交代码

Browse Source
This commit is contained in:
lmk
2026-07-02 10:37:59 +08:00
parent aced1aa3a6
commit d930266fbf
17 changed files with 411 additions and 445 deletions
Download Patch File Download Diff File Expand all files Collapse all files
service/dataengine/material_verify_service.go
+114 -132
View File
@@ -10,6 +10,7 @@ import (
"fmt"
"time"
"gitea.redpowerfuture.com/red-future/common/beans"
"github.com/gogf/gf/v2/frame/g"
)
@@ -41,7 +42,7 @@ func SuggestionToVerifyStatus(suggestion int) string {
case consts.SuggestionPass:
return entity.VerifyStatusVerified // 通过
case consts.SuggestionReview:
return entity.VerifyStatusPending // 嫌疑，需要人工审核，暂不更新状态
return entity.VerifyStatusReview // 嫌疑，需人工复核
case consts.SuggestionBlock:
return entity.VerifyStatusRejected // 不通过
default:
@@ -55,22 +56,28 @@ func SuggestionToVerifyStatus(suggestion int) string {
// VerifyImageByID 根据图片ID执行校验
func (s *MaterialVerifyService) VerifyImageByID(ctx context.Context, imageID string) (*entity.MaterialVerifyLog, error) {
// 1. 获取图片数据
image, err := dao.TencentImage.GetByImageID(ctx, imageID)
if err != nil {
return nil, fmt.Errorf("查询图片数据失败: %w", err)
return nil, fmt.Errorf("unknown error: %w", err)
}
if image == nil {
return nil, fmt.Errorf("未找到图片数据, imageID=%s", imageID)
}
// 2. 创建校验日志
// 幂等性检查：如果已在送检中，直接返回已有日志
if image.VerifyStatus == consts.CheckStatusSubmitting {
logs, err := dao.MaterialVerifyLog.GetByMaterialID(ctx, imageID)
if err == nil && len(logs) > 0 {
g.Log().Infof(ctx, "图片已在送检中, imageID=%s, logId=%d", imageID, logs[0].Id)
return &logs[0], nil
}
}
log := s.createVerifyLog(ctx, entity.MaterialTypeImage, imageID, consts.SourceTableTencentImage, image.Id, image.AccountID)
if log == nil {
return nil, fmt.Errorf("创建校验日志失败")
}
// 3. 执行校验
err = s.submitImageCheck(ctx, image, log)
if err != nil {
return nil, err
@@ -83,10 +90,8 @@ func (s *MaterialVerifyService) VerifyImageByID(ctx context.Context, imageID str
func (s *MaterialVerifyService) submitImageCheck(ctx context.Context, image *entity.TencentImage, log *entity.MaterialVerifyLog) error {
startTime := time.Now()
// 获取回调模式开关
callbackMode := g.Cfg().MustGet(ctx, "yidun.callback_mode").Bool()
// 构建请求参数
requestParams := map[string]interface{}{
"imageURL": image.PreviewURL,
"dataID": image.ImageID,
@@ -99,7 +104,6 @@ func (s *MaterialVerifyService) submitImageCheck(ctx context.Context, image *ent
)
if callbackMode {
// 回调模式：使用异步检测，易盾处理完成后会回调
callbackURL := g.Cfg().MustGet(ctx, "yidun.image.callback_url").String()
requestParams["callbackURL"] = callbackURL
@@ -109,35 +113,30 @@ func (s *MaterialVerifyService) submitImageCheck(ctx context.Context, image *ent
dao.MaterialVerifyLog.UpdateError(ctx, log.Id, entity.VerifyStatusPending, err.Error())
dao.MaterialVerifyLog.UpdateDuration(ctx, log.Id, duration)
g.Log().Warningf(ctx, "图片异步检测失败（保持待检验）, id=%d, imageId=%s, error=%v", image.Id, image.ImageID, err)
return fmt.Errorf("图片异步检测失败: %w", err)
return fmt.Errorf("unknown error: %w", err)
}
taskID = result.TaskID
// 保存任务ID和请求参数
dao.MaterialVerifyLog.UpdateTaskID(ctx, log.Id, taskID)
dao.MaterialVerifyLog.UpdateRequestParams(ctx, log.Id, string(requestParamsJSON))
// 更新原表状态为 submitting（等待回调）
s.updateImageStatus(ctx, image.Id, StatusSubmitting)
g.Log().Infof(ctx, "图片异步检测已提交, id=%d, imageId=%s, taskId=%s, duration=%dms",
image.Id, image.ImageID, taskID, duration)
} else {
// 轮询模式：使用同步检测，直接返回结果
syncResult, err := yidunService.ImageDetection.DetectImageSync(ctx, image.PreviewURL, image.ImageID)
duration = time.Since(startTime).Milliseconds()
if err != nil {
dao.MaterialVerifyLog.UpdateError(ctx, log.Id, entity.VerifyStatusPending, err.Error())
dao.MaterialVerifyLog.UpdateDuration(ctx, log.Id, duration)
g.Log().Warningf(ctx, "图片同步检测失败（保持待检验）, id=%d, imageId=%s, error=%v", image.Id, image.ImageID, err)
return fmt.Errorf("图片同步检测失败: %w", err)
return fmt.Errorf("unknown error: %w", err)
}
taskID = syncResult.TaskID
// 保存任务ID和请求参数
dao.MaterialVerifyLog.UpdateTaskID(ctx, log.Id, taskID)
dao.MaterialVerifyLog.UpdateRequestParams(ctx, log.Id, string(requestParamsJSON))
// 根据同步结果更新状态
verifyStatus := SuggestionToVerifyStatus(syncResult.Suggestion)
responseJSON, _ := json.Marshal(syncResult)
dao.MaterialVerifyLog.UpdateVerifyResult(ctx, log.Id, verifyStatus,
@@ -157,22 +156,28 @@ func (s *MaterialVerifyService) submitImageCheck(ctx context.Context, image *ent
// VerifyVideoByID 根据视频ID执行校验
func (s *MaterialVerifyService) VerifyVideoByID(ctx context.Context, videoID string) (*entity.MaterialVerifyLog, error) {
// 1. 获取视频数据
video, err := dao.TencentVideo.GetByVideoID(ctx, videoID)
if err != nil {
return nil, fmt.Errorf("查询视频数据失败: %w", err)
return nil, fmt.Errorf("unknown error: %w", err)
}
if video == nil {
return nil, fmt.Errorf("未找到视频数据, videoID=%s", videoID)
}
// 2. 创建校验日志
// 幂等性检查：如果已在送检中，直接返回已有日志
if video.VerifyStatus == consts.CheckStatusSubmitting {
logs, err := dao.MaterialVerifyLog.GetByMaterialID(ctx, videoID)
if err == nil && len(logs) > 0 {
g.Log().Infof(ctx, "视频已在送检中, videoID=%s, logId=%d", videoID, logs[0].Id)
return &logs[0], nil
}
}
log := s.createVerifyLog(ctx, entity.MaterialTypeVideo, videoID, consts.SourceTableTencentVideo, video.Id, video.AccountID)
if log == nil {
return nil, fmt.Errorf("创建校验日志失败")
}
// 3. 执行校验
err = s.submitVideoCheck(ctx, video, log)
if err != nil {
return nil, err
@@ -185,16 +190,13 @@ func (s *MaterialVerifyService) VerifyVideoByID(ctx context.Context, videoID str
func (s *MaterialVerifyService) submitVideoCheck(ctx context.Context, video *entity.TencentVideo, log *entity.MaterialVerifyLog) error {
startTime := time.Now()
// 获取回调模式开关
callbackMode := g.Cfg().MustGet(ctx, "yidun.callback_mode").Bool()
// 根据开关决定回调地址
var callbackURL string
if callbackMode {
callbackURL = g.Cfg().MustGet(ctx, "yidun.video.callback_url").String()
}
// 构建请求参数
requestParams := map[string]interface{}{
"videoURL": video.PreviewURL,
"dataID": video.VideoID,
@@ -202,30 +204,22 @@ func (s *MaterialVerifyService) submitVideoCheck(ctx context.Context, video *ent
}
requestParamsJSON, _ := json.Marshal(requestParams)
// 调用易盾视频检测
result, err := yidunService.VideoDetection.DetectVideo(ctx, video.PreviewURL, video.VideoID, callbackURL)
duration := time.Since(startTime).Milliseconds()
if err != nil {
// 调用易盾接口失败（如额度用光、网络错误、超时等），不更新状态，保持待检验
// 只有易盾明确返回检测结果且suggestion=BLOCK时才标记为失败
dao.MaterialVerifyLog.UpdateError(ctx, log.Id, entity.VerifyStatusPending, err.Error())
dao.MaterialVerifyLog.UpdateDuration(ctx, log.Id, duration)
g.Log().Warningf(ctx, "视频校验接口调用失败（保持待检验）, id=%d, videoId=%s, error=%v", video.Id, video.VideoID, err)
return fmt.Errorf("视频校验调用失败: %w", err)
return fmt.Errorf("unknown error: %w", err)
}
// 保存任务ID和请求参数
dao.MaterialVerifyLog.UpdateTaskID(ctx, log.Id, result.TaskID)
dao.MaterialVerifyLog.UpdateRequestParams(ctx, log.Id, string(requestParamsJSON))
// 更新原表状态为 submitting
s.updateVideoStatus(ctx, video.Id, StatusSubmitting)
// 轮询模式（无回调）：提交后立即尝试查询检测结果
if !callbackMode {
g.Log().Infof(ctx, "轮询模式：提交后立即查询结果, taskId=%s", result.TaskID)
// 等待500ms让易盾有时间处理
time.Sleep(500 * time.Millisecond)
if err := s.ProcessVideoResultByTask(ctx, result.TaskID); err != nil {
g.Log().Warningf(ctx, "提交后立即查询结果失败（不影响状态，后续轮询继续）, taskId=%s, error=%v", result.TaskID, err)
@@ -249,7 +243,7 @@ func (s *MaterialVerifyService) ProcessImageCallback(ctx context.Context, callba
var callback yidunService.ImageCallbackData
if err := json.Unmarshal([]byte(callbackData), &callback); err != nil {
g.Log().Errorf(ctx, "解析图片回调数据失败: %v", err)
return fmt.Errorf("解析回调数据失败: %w", err)
return fmt.Errorf("unknown error: %w", err)
}
if callback.Antispam == nil {
@@ -260,30 +254,23 @@ func (s *MaterialVerifyService) ProcessImageCallback(ctx context.Context, callba
g.Log().Infof(ctx, "处理图片校验结果 - taskId: %s, suggestion: %d, resultType: %d",
antispam.TaskId, antispam.Suggestion, antispam.ResultType)
// 根据 taskId 查找校验日志
log, err := dao.MaterialVerifyLog.GetByTaskID(ctx, antispam.TaskId)
if err != nil {
return fmt.Errorf("查找校验日志失败: %w", err)
return fmt.Errorf("unknown error: %w", err)
}
if log == nil {
g.Log().Warningf(ctx, "未找到校验日志, taskId=%s", antispam.TaskId)
return nil
}
// 构建响应结果
responseResult := callbackData
// 根据 suggestion 确定校验状态
verifyStatus := SuggestionToVerifyStatus(antispam.Suggestion)
// 更新日志
err = dao.MaterialVerifyLog.UpdateVerifyResult(ctx, log.Id, verifyStatus,
antispam.Suggestion, antispam.Label, antispam.ResultType, responseResult, antispam.CensorTime)
antispam.Suggestion, antispam.Label, antispam.ResultType, callbackData, antispam.CensorTime)
if err != nil {
return fmt.Errorf("更新校验日志失败: %w", err)
return fmt.Errorf("unknown error: %w", err)
}
// 更新原表状态（图片回调只处理图片来源）
if log.SourceTable == consts.SourceTableTencentImage {
s.updateImageStatus(ctx, log.SourceID, verifyStatus)
}
@@ -301,7 +288,7 @@ func (s *MaterialVerifyService) ProcessVideoCallback(ctx context.Context, callba
var callback yidunService.VideoCallbackData
if err := json.Unmarshal([]byte(callbackData), &callback); err != nil {
g.Log().Errorf(ctx, "解析视频回调数据失败: %v", err)
return fmt.Errorf("解析回调数据失败: %w", err)
return fmt.Errorf("unknown error: %w", err)
}
if callback.Antispam == nil {
@@ -312,36 +299,28 @@ func (s *MaterialVerifyService) ProcessVideoCallback(ctx context.Context, callba
g.Log().Infof(ctx, "处理视频校验结果 - taskId: %s, suggestion: %d, resultType: %d",
antispam.TaskID, antispam.Suggestion, antispam.ResultType)
// 根据 taskId 查找校验日志
log, err := dao.MaterialVerifyLog.GetByTaskID(ctx, antispam.TaskID)
if err != nil {
return fmt.Errorf("查找校验日志失败: %w", err)
return fmt.Errorf("unknown error: %w", err)
}
if log == nil {
g.Log().Warningf(ctx, "未找到校验日志, taskId=%s", antispam.TaskID)
return nil
}
// 构建响应结果
responseResult := callbackData
// 根据 suggestion 确定校验状态
verifyStatus := SuggestionToVerifyStatus(antispam.Suggestion)
// 审核时间
checkTime := antispam.CensorTime
if checkTime == 0 {
checkTime = antispam.CheckTime
}
// 更新日志
err = dao.MaterialVerifyLog.UpdateVerifyResult(ctx, log.Id, verifyStatus,
antispam.Suggestion, antispam.Label, antispam.ResultType, responseResult, checkTime)
antispam.Suggestion, antispam.Label, antispam.ResultType, callbackData, checkTime)
if err != nil {
return fmt.Errorf("更新校验日志失败: %w", err)
return fmt.Errorf("unknown error: %w", err)
}
// 更新原表状态（视频回调只处理视频来源）
if log.SourceTable == consts.SourceTableTencentVideo {
s.updateVideoStatus(ctx, log.SourceID, verifyStatus)
}
@@ -373,36 +352,27 @@ func (s *MaterialVerifyService) ProcessImageResultByTask(ctx context.Context, ta
result, err := yidunService.ImageDetection.GetImageResult(ctx, taskID)
if err != nil {
// 判断是否是未找到结果或仍在检测中的错误
if err == yidunService.ErrImageResultNotFound || err == yidunService.ErrImageStillProcessing {
// 未获取到结果（任务不存在或仍在处理），不更新状态，保持等待下次轮询
g.Log().Infof(ctx, "图片检测结果未就绪, taskId=%s, 保持pending状态, err=%v", taskID, err)
return nil
}
// 其他错误（如额度用光、网络错误、API错误等），不更新状态，保持待检验
// 只有易盾明确返回suggestion=BLOCK时才标记为失败
dao.MaterialVerifyLog.UpdateError(ctx, log.Id, entity.VerifyStatusPending, err.Error())
g.Log().Warningf(ctx, "图片检测查询失败（保持待检验）, taskId=%s, error=%v", taskID, err)
return nil // 返回nil避免日志被反复处理，但保持pending状态
return nil
}
// 判断检测状态
if result.Status == YidunStatusProcessing || result.Status == YidunStatusNotStart {
// 检测仍在进行中，保持pending状态
g.Log().Infof(ctx, "图片检测仍在进行中, taskId=%s, status=%d, 保持pending状态", taskID, result.Status)
return nil
}
if result.Status == YidunStatusFailed {
// 易盾检测失败（如额度用光、服务端错误等），不更新状态，保持待检验
// 只有易盾明确返回suggestion=BLOCK时才标记为失败
errMsg := fmt.Sprintf("易盾检测失败, status=%d", result.Status)
dao.MaterialVerifyLog.UpdateError(ctx, log.Id, entity.VerifyStatusPending, errMsg)
dao.MaterialVerifyLog.UpdateError(ctx, log.Id, entity.VerifyStatusPending,
fmt.Sprintf("易盾检测失败, status=%d", result.Status))
g.Log().Warningf(ctx, "图片检测失败（保持待检验）, taskId=%s, status=%d", taskID, result.Status)
return nil
}
// status == YidunStatusSuccess，检测成功，根据suggestion更新状态
verifyStatus := SuggestionToVerifyStatus(result.Suggestion)
responseJSON, _ := json.Marshal(result)
@@ -427,36 +397,27 @@ func (s *MaterialVerifyService) ProcessVideoResultByTask(ctx context.Context, ta
result, err := yidunService.VideoDetection.GetVideoResult(ctx, taskID)
if err != nil {
// 判断是否是未找到结果或仍在检测中的错误
if err == yidunService.ErrVideoResultNotFound || err == yidunService.ErrVideoStillProcessing {
// 未获取到结果（任务不存在或仍在处理），不更新状态，保持等待下次轮询
g.Log().Infof(ctx, "视频检测结果未就绪, taskId=%s, 保持pending状态, err=%v", taskID, err)
return nil
}
// 其他错误（如额度用光、网络错误、API错误等），不更新状态，保持待检验
// 只有易盾明确返回suggestion=BLOCK时才标记为失败
dao.MaterialVerifyLog.UpdateError(ctx, log.Id, entity.VerifyStatusPending, err.Error())
g.Log().Warningf(ctx, "视频检测查询失败（保持待检验）, taskId=%s, error=%v", taskID, err)
return nil // 返回nil避免日志被反复处理，但保持pending状态
return nil
}
// 判断检测状态
if result.Status == YidunStatusProcessing || result.Status == YidunStatusNotStart {
// 检测仍在进行中，保持pending状态
g.Log().Infof(ctx, "视频检测仍在进行中, taskId=%s, status=%d, 保持pending状态", taskID, result.Status)
return nil
}
if result.Status == YidunStatusFailed {
// 易盾检测失败（如额度用光、服务端错误等），不更新状态，保持待检验
// 只有易盾明确返回suggestion=BLOCK时才标记为失败
errMsg := fmt.Sprintf("易盾检测失败, status=%d", result.Status)
dao.MaterialVerifyLog.UpdateError(ctx, log.Id, entity.VerifyStatusPending, errMsg)
dao.MaterialVerifyLog.UpdateError(ctx, log.Id, entity.VerifyStatusPending,
fmt.Sprintf("易盾检测失败, status=%d", result.Status))
g.Log().Warningf(ctx, "视频检测失败（保持待检验）, taskId=%s, status=%d", taskID, result.Status)
return nil
}
// status == YidunStatusSuccess，检测成功，根据suggestion更新状态
verifyStatus := SuggestionToVerifyStatus(result.Suggestion)
responseJSON, _ := json.Marshal(result)
@@ -478,8 +439,16 @@ func (s *MaterialVerifyService) ProcessVideoResultByTask(ctx context.Context, ta
// createVerifyLog 创建校验日志
func (s *MaterialVerifyService) createVerifyLog(ctx context.Context, materialType, materialID, sourceTable string, sourceID, accountID int64) *entity.MaterialVerifyLog {
// 从上下文提取租户ID
var tenantID int64
if user := ctx.Value("user"); user != nil {
if u, ok := user.(*beans.User); ok {
tenantID = int64(u.TenantId)
}
}
log := &entity.MaterialVerifyLog{
TenantID: 0,
TenantID: tenantID,
MaterialType: materialType,
MaterialID: materialID,
SourceTable: sourceTable,
@@ -498,7 +467,7 @@ func (s *MaterialVerifyService) createVerifyLog(ctx context.Context, materialTyp
return log
}
// updateImageStatus 更新图片状态
// updateImageStatus 更新图片状态（已记录日志则同步更新，失败仅记录日志不影响主流程）
func (s *MaterialVerifyService) updateImageStatus(ctx context.Context, imageID int64, verifyStatus string) {
_, err := dao.TencentImage.UpdateStatus(ctx, imageID, verifyStatus)
if err != nil {
@@ -508,7 +477,7 @@ func (s *MaterialVerifyService) updateImageStatus(ctx context.Context, imageID i
}
}
// updateVideoStatus 更新视频状态
// updateVideoStatus 更新视频状态（已记录日志则同步更新，失败仅记录日志不影响主流程）
func (s *MaterialVerifyService) updateVideoStatus(ctx context.Context, videoID int64, verifyStatus string) {
_, err := dao.TencentVideo.UpdateStatus(ctx, videoID, verifyStatus)
if err != nil {
@@ -706,11 +675,12 @@ func getFailureReason(log *entity.MaterialVerifyLog) string {
return suggestionText
}
// ExportRejectedData 导出不通过数据
const exportBatchSize = 1000
// ExportRejectedData 导出不通过数据（分批加载，避免OOM）
func (s *MaterialVerifyService) ExportRejectedData(ctx context.Context, materialType string) ([]ExportRejectedItem, error) {
var items []ExportRejectedItem
// 加载账户名称映射
accountMap := make(map[int64]string)
if accounts, err := dao.TencentAccountRelation.GetAll(ctx); err == nil {
for _, acc := range accounts {
@@ -720,70 +690,82 @@ func (s *MaterialVerifyService) ExportRejectedData(ctx context.Context, material
}
}
// 处理图片
if materialType == "" || materialType == entity.MaterialTypeImage {
condition := map[string]interface{}{
entity.TencentImageCols.VerifyStatus: entity.VerifyStatusRejected,
}
images, total, err := dao.TencentImage.GetByCondition(ctx, condition, 1, 100000)
if err != nil {
g.Log().Errorf(ctx, "查询不通过图片失败: %v", err)
return nil, fmt.Errorf("查询不通过图片失败: %w", err)
}
g.Log().Infof(ctx, "导出不通过图片: total=%d, got=%d", total, len(images))
for _, img := range images {
// 查询最后一条失败的校验日志
log, _ := dao.MaterialVerifyLog.GetLastRejectedLogByMaterialID(ctx, img.ImageID, entity.VerifyStatusRejected)
var createdAtStr string
if log != nil && log.CreatedAt != nil {
createdAtStr = log.CreatedAt.Format("Y-m-d H:i:s")
page := 1
for {
images, total, err := dao.TencentImage.GetByCondition(ctx, condition, page, exportBatchSize)
if err != nil {
g.Log().Errorf(ctx, "查询不通过图片失败: %v", err)
return nil, fmt.Errorf("unknown error: %w", err)
}
items = append(items, ExportRejectedItem{
ID: img.Id,
MaterialID: img.ImageID,
AccountID: img.AccountID,
CorporationName: accountMap[img.AccountID],
PreviewURL: img.PreviewURL,
Description: img.Description,
ErrorMsg: getFailureReason(log),
MaterialType: entity.MaterialTypeImage,
ImageUsage: img.ImageUsage,
CreatedAt: createdAtStr,
})
for _, img := range images {
log, _ := dao.MaterialVerifyLog.GetLastRejectedLogByMaterialID(ctx, img.ImageID, entity.VerifyStatusRejected)
var createdAtStr string
if log != nil && log.CreatedAt != nil {
createdAtStr = log.CreatedAt.Format("2006-01-02 15:04:05")
}
items = append(items, ExportRejectedItem{
ID: img.Id,
MaterialID: img.ImageID,
AccountID: img.AccountID,
CorporationName: accountMap[img.AccountID],
PreviewURL: img.PreviewURL,
Description: img.Description,
ErrorMsg: getFailureReason(log),
MaterialType: entity.MaterialTypeImage,
ImageUsage: img.ImageUsage,
CreatedAt: createdAtStr,
})
}
if page*exportBatchSize >= total {
break
}
page++
}
}
// 处理视频
if materialType == "" || materialType == entity.MaterialTypeVideo {
condition := map[string]interface{}{
entity.TencentVideoCols.VerifyStatus: entity.VerifyStatusRejected,
}
videos, total, err := dao.TencentVideo.GetByCondition(ctx, condition, 1, 100000)
if err != nil {
g.Log().Errorf(ctx, "查询不通过视频失败: %v", err)
return nil, fmt.Errorf("查询不通过视频失败: %w", err)
}
g.Log().Infof(ctx, "导出不通过视频: total=%d, got=%d", total, len(videos))
for _, vid := range videos {
// 查询最后一条失败的校验日志
log, _ := dao.MaterialVerifyLog.GetLastRejectedLogByMaterialID(ctx, vid.VideoID, entity.VerifyStatusRejected)
var createdAtStr string
if log != nil && log.CreatedAt != nil {
createdAtStr = log.CreatedAt.Format("Y-m-d H:i:s")
page := 1
for {
videos, total, err := dao.TencentVideo.GetByCondition(ctx, condition, page, exportBatchSize)
if err != nil {
g.Log().Errorf(ctx, "查询不通过视频失败: %v", err)
return nil, fmt.Errorf("unknown error: %w", err)
}
items = append(items, ExportRejectedItem{
ID: vid.Id,
MaterialID: vid.VideoID,
AccountID: vid.AccountID,
CorporationName: accountMap[vid.AccountID],
PreviewURL: vid.PreviewURL,
Description: vid.Description,
ErrorMsg: getFailureReason(log),
MaterialType: entity.MaterialTypeVideo,
CreatedAt: createdAtStr,
})
for _, vid := range videos {
log, _ := dao.MaterialVerifyLog.GetLastRejectedLogByMaterialID(ctx, vid.VideoID, entity.VerifyStatusRejected)
var createdAtStr string
if log != nil && log.CreatedAt != nil {
createdAtStr = log.CreatedAt.Format("2006-01-02 15:04:05")
}
items = append(items, ExportRejectedItem{
ID: vid.Id,
MaterialID: vid.VideoID,
AccountID: vid.AccountID,
CorporationName: accountMap[vid.AccountID],
PreviewURL: vid.PreviewURL,
Description: vid.Description,
ErrorMsg: getFailureReason(log),
MaterialType: entity.MaterialTypeVideo,
CreatedAt: createdAtStr,
})
}
if page*exportBatchSize >= total {
break
}
page++
}
}
service/dataengine/tencent_content_callback_service.go
+7 -23
View File
@@ -36,26 +36,21 @@ func (s *TencentContentCallbackService) ProcessImageCallback(ctx context.Context
g.Log().Infof(ctx, "处理图片检测结果 - taskId: %s, suggestion: %d, resultType: %d",
antispam.TaskId, antispam.Suggestion, antispam.ResultType)
// 根据 taskId 查找送检日志
log, err := dao.TencentContentCheckLog.GetByTaskID(ctx, antispam.TaskId)
if err != nil {
g.Log().Errorf(ctx, "查找送检日志失败, taskId=%s: %v", antispam.TaskId, err)
return fmt.Errorf("查找送检日志失败: %w", err)
}
if log == nil {
g.Log().Warningf(ctx, "未找到送检日志, taskId=%s", antispam.TaskId)
return nil
}
// 更新送检日志
checkTime := antispam.CensorTime
err = dao.TencentContentCheckLog.UpdateCheckResult(ctx, log.Id,
antispam.Suggestion, antispam.Label, antispam.ResultType, checkTime)
antispam.Suggestion, antispam.Label, antispam.ResultType, antispam.CensorTime)
if err != nil {
g.Log().Errorf(ctx, "更新送检日志检测结果失败: %v", err)
return err
return fmt.Errorf("更新送检日志检测结果失败: %w", err)
}
g.Log().Infof(ctx, "图片检测回调处理完成, taskId=%s, suggestion=%d", antispam.TaskId, antispam.Suggestion)
@@ -80,19 +75,16 @@ func (s *TencentContentCallbackService) ProcessVideoCallback(ctx context.Context
g.Log().Infof(ctx, "处理视频检测结果 - taskId: %s, suggestion: %d, resultType: %d, censorSource: %d",
antispam.TaskID, antispam.Suggestion, antispam.ResultType, antispam.CensorSource)
// 根据 taskId 查找送检日志
log, err := dao.TencentContentCheckLog.GetByTaskID(ctx, antispam.TaskID)
if err != nil {
g.Log().Errorf(ctx, "查找送检日志失败, taskId=%s: %v", antispam.TaskID, err)
return fmt.Errorf("查找送检日志失败: %w", err)
}
if log == nil {
g.Log().Warningf(ctx, "未找到送检日志, taskId=%s", antispam.TaskID)
return nil
}
// 更新送检日志
checkTime := antispam.CensorTime
if checkTime == 0 {
checkTime = antispam.CheckTime
@@ -102,7 +94,7 @@ func (s *TencentContentCallbackService) ProcessVideoCallback(ctx context.Context
antispam.Suggestion, antispam.Label, antispam.ResultType, checkTime)
if err != nil {
g.Log().Errorf(ctx, "更新送检日志检测结果失败: %v", err)
return err
return fmt.Errorf("更新送检日志检测结果失败: %w", err)
}
g.Log().Infof(ctx, "视频检测回调处理完成, taskId=%s, suggestion=%d", antispam.TaskID, antispam.Suggestion)
@@ -113,25 +105,22 @@ func (s *TencentContentCallbackService) ProcessVideoCallback(ctx context.Context
func (s *TencentContentCallbackService) ProcessImageResult(ctx context.Context, taskID string) error {
g.Log().Infof(ctx, "查询图片检测结果, taskId: %s", taskID)
// 查找送检日志
log, err := dao.TencentContentCheckLog.GetByTaskID(ctx, taskID)
if err != nil || log == nil {
return fmt.Errorf("未找到送检日志, taskId=%s", taskID)
}
// 调用易盾查询结果
result, err := yidunService.ImageDetection.GetImageResult(ctx, taskID)
if err != nil {
g.Log().Errorf(ctx, "查询图片检测结果失败: %v", err)
return err
return fmt.Errorf("查询图片检测结果失败: %w", err)
}
// 更新日志
err = dao.TencentContentCheckLog.UpdateCheckResult(ctx, log.Id,
result.Suggestion, result.Label, result.ResultType, result.CensorTime)
if err != nil {
g.Log().Errorf(ctx, "更新送检日志检测结果失败: %v", err)
return err
return fmt.Errorf("更新送检日志检测结果失败: %w", err)
}
g.Log().Infof(ctx, "图片检测结果处理完成, taskId=%s, suggestion=%d", taskID, result.Suggestion)
@@ -142,25 +131,22 @@ func (s *TencentContentCallbackService) ProcessImageResult(ctx context.Context,
func (s *TencentContentCallbackService) ProcessVideoResult(ctx context.Context, taskID string) error {
g.Log().Infof(ctx, "查询视频检测结果, taskId: %s", taskID)
// 查找送检日志
log, err := dao.TencentContentCheckLog.GetByTaskID(ctx, taskID)
if err != nil || log == nil {
return fmt.Errorf("未找到送检日志, taskId=%s", taskID)
}
// 调用易盾查询结果
result, err := yidunService.VideoDetection.GetVideoResult(ctx, taskID)
if err != nil {
g.Log().Errorf(ctx, "查询视频检测结果失败: %v", err)
return err
return fmt.Errorf("查询视频检测结果失败: %w", err)
}
// 更新日志
err = dao.TencentContentCheckLog.UpdateCheckResult(ctx, log.Id,
result.Suggestion, result.Label, result.ResultType, result.CensorTime)
if err != nil {
g.Log().Errorf(ctx, "更新送检日志检测结果失败: %v", err)
return err
return fmt.Errorf("更新送检日志检测结果失败: %w", err)
}
g.Log().Infof(ctx, "视频检测结果处理完成, taskId=%s, suggestion=%d", taskID, result.Suggestion)
@@ -169,7 +155,6 @@ func (s *TencentContentCallbackService) ProcessVideoResult(ctx context.Context,
// GetCheckLogsByImageID 根据图片ID获取送检日志
func (s *TencentContentCallbackService) GetCheckLogsByImageID(ctx context.Context, imageID string) ([]entity.TencentContentCheckLog, error) {
// 先获取图片数据
image, err := dao.TencentImage.GetByImageID(ctx, imageID)
if err != nil || image == nil {
return nil, fmt.Errorf("未找到图片数据")
@@ -180,7 +165,6 @@ func (s *TencentContentCallbackService) GetCheckLogsByImageID(ctx context.Contex
// GetCheckLogsByVideoID 根据视频ID获取送检日志
func (s *TencentContentCallbackService) GetCheckLogsByVideoID(ctx context.Context, videoID string) ([]entity.TencentContentCheckLog, error) {
// 先获取视频数据
video, err := dao.TencentVideo.GetByVideoID(ctx, videoID)
if err != nil || video == nil {
return nil, fmt.Errorf("未找到视频数据")
service/dataengine/tencent_content_check_service.go
+49 -166
View File
@@ -7,7 +7,6 @@ import (
yidunService "cid/service/yidun"
"context"
"encoding/json"
"fmt"
"time"
"gitea.redpowerfuture.com/red-future/common/beans"
@@ -16,14 +15,10 @@ import (
// ContentCheckConfig 送检配置
type ContentCheckConfig struct {
// 每批处理数量
BatchSize int `json:"batch_size"`
// 图片检测启用
ImageEnabled bool `json:"image_enabled"`
// 视频检测启用
VideoEnabled bool `json:"video_enabled"`
// 定时任务间隔（秒）
IntervalSeconds int `json:"interval_seconds"`
BatchSize int `json:"batch_size"`
ImageEnabled bool `json:"image_enabled"`
VideoEnabled bool `json:"video_enabled"`
IntervalSeconds int `json:"interval_seconds"`
}
// DefaultConfig 默认配置
@@ -61,7 +56,11 @@ func (s *TencentContentCheckService) Start(ctx context.Context) error {
g.Log().Infof(ctx, "启动内容送检服务，配置: batch_size=%d, interval=%ds, image=%v, video=%v",
s.config.BatchSize, s.config.IntervalSeconds, s.config.ImageEnabled, s.config.VideoEnabled)
go s.runScheduler(ctx)
schedCtx := context.Background()
if user := ctx.Value("user"); user != nil {
schedCtx = context.WithValue(schedCtx, "user", user)
}
go s.runScheduler(schedCtx)
return nil
}
@@ -76,23 +75,18 @@ func (s *TencentContentCheckService) runScheduler(ctx context.Context) {
ticker := time.NewTicker(time.Duration(s.config.IntervalSeconds) * time.Second)
defer ticker.Stop()
// 启动时先执行一次
s.processAll(ctx)
for s.isRunning {
select {
case <-ticker.C:
s.processAll(ctx)
case <-ctx.Done():
s.isRunning = false
for range ticker.C {
if !s.isRunning {
return
}
s.processAll(ctx)
}
}
// processAll 处理所有待送检数据
func (s *TencentContentCheckService) processAll(ctx context.Context) {
// 添加系统用户上下文，绕过gfdb租户验证
ctx = context.WithValue(ctx, "user", &beans.User{UserName: "system", TenantId: 1})
startTime := time.Now()
@@ -100,7 +94,6 @@ func (s *TencentContentCheckService) processAll(ctx context.Context) {
var totalProcessed int
// 处理图片
if s.config.ImageEnabled {
imageCount, _ := dao.TencentImage.CountPending(ctx)
if imageCount > 0 {
@@ -109,7 +102,6 @@ func (s *TencentContentCheckService) processAll(ctx context.Context) {
}
}
// 处理视频
if s.config.VideoEnabled {
videoCount, _ := dao.TencentVideo.CountPending(ctx)
if videoCount > 0 {
@@ -122,9 +114,8 @@ func (s *TencentContentCheckService) processAll(ctx context.Context) {
g.Log().Infof(ctx, "处理完成，共处理 %d 条数据，耗时 %dms", totalProcessed, duration)
}
// processImages 处理图片送检
// processImages 处理图片送检（统一走 MaterialVerify 系统）
func (s *TencentContentCheckService) processImages(ctx context.Context) (int, error) {
// 获取待送检图片
images, err := dao.TencentImage.GetPendingList(ctx, s.config.BatchSize)
if err != nil {
g.Log().Errorf(ctx, "获取待送检图片失败: %v", err)
@@ -141,22 +132,17 @@ func (s *TencentContentCheckService) processImages(ctx context.Context) (int, er
failedCount := 0
for _, img := range images {
// 创建送检日志
log := s.createCheckLog(ctx, consts.SourceTableTencentImage, img.Id, img.ImageID, img.PreviewURL)
// 提交送检
err := s.submitImageCheck(ctx, &img, log)
// 统一走 MaterialVerify 系统提交（处理完整校验流程：日志→提交→状态反写）
mLog, err := MaterialVerify.VerifyImageByID(ctx, img.ImageID)
if err != nil {
failedCount++
// 更新日志为失败
if log != nil {
dao.TencentContentCheckLog.UpdateStatus(ctx, log.Id, consts.CheckStatusFailed, "", err.Error())
}
g.Log().Errorf(ctx, "图片送检失败, imageId=%s, error=%v", img.ImageID, err)
} else {
successCount++
// 审计日志：同步写入 tencent_content_check_log
s.writeAuditLog(ctx, consts.SourceTableTencentImage, img.Id, img.ImageID, img.PreviewURL, mLog.TaskID)
}
// 避免请求过快
time.Sleep(100 * time.Millisecond)
}
@@ -164,9 +150,8 @@ func (s *TencentContentCheckService) processImages(ctx context.Context) (int, er
return len(images), nil
}
// processVideos 处理视频送检
// processVideos 处理视频送检（统一走 MaterialVerify 系统）
func (s *TencentContentCheckService) processVideos(ctx context.Context) (int, error) {
// 获取待送检视频
videos, err := dao.TencentVideo.GetPendingList(ctx, s.config.BatchSize)
if err != nil {
g.Log().Errorf(ctx, "获取待送检视频失败: %v", err)
@@ -183,22 +168,15 @@ func (s *TencentContentCheckService) processVideos(ctx context.Context) (int, er
failedCount := 0
for _, video := range videos {
// 创建送检日志
log := s.createCheckLog(ctx, consts.SourceTableTencentVideo, video.Id, video.VideoID, video.PreviewURL)
// 提交送检
err := s.submitVideoCheck(ctx, &video, log)
mLog, err := MaterialVerify.VerifyVideoByID(ctx, video.VideoID)
if err != nil {
failedCount++
// 更新日志为失败
if log != nil {
dao.TencentContentCheckLog.UpdateStatus(ctx, log.Id, consts.CheckStatusFailed, "", err.Error())
}
g.Log().Errorf(ctx, "视频送检失败, videoId=%s, error=%v", video.VideoID, err)
} else {
successCount++
s.writeAuditLog(ctx, consts.SourceTableTencentVideo, video.Id, video.VideoID, video.PreviewURL, mLog.TaskID)
}
// 避免请求过快
time.Sleep(100 * time.Millisecond)
}
@@ -206,8 +184,8 @@ func (s *TencentContentCheckService) processVideos(ctx context.Context) (int, er
return len(videos), nil
}
// createCheckLog 创建送检日志
func (s *TencentContentCheckService) createCheckLog(ctx context.Context, sourceTable string, sourceID int64, mediaID string, mediaURL string) *entity.TencentContentCheckLog {
// writeAuditLog 写入审计日志（tencent_content_check_log）
func (s *TencentContentCheckService) writeAuditLog(ctx context.Context, sourceTable string, sourceID int64, mediaID string, mediaURL string, taskID string) {
requestParam := map[string]interface{}{
"media_id": mediaID,
"url": mediaURL,
@@ -219,147 +197,52 @@ func (s *TencentContentCheckService) createCheckLog(ctx context.Context, sourceT
SourceID: sourceID,
RequestURL: "易盾内容安全检测接口",
RequestParam: string(requestParamJSON),
Status: consts.CheckStatusPending,
Status: consts.CheckStatusSuccess,
CheckTime: time.Now().UnixMilli(),
TaskID: taskID,
}
id, err := dao.TencentContentCheckLog.Create(ctx, log)
if err != nil {
g.Log().Errorf(ctx, "创建送检日志失败: %v", err)
return nil
g.Log().Errorf(ctx, "创建送检审计日志失败: %v", err)
return
}
log.Id = id
g.Log().Debugf(ctx, "创建送检日志成功, id=%d, sourceTable=%s, sourceID=%d", id, sourceTable, sourceID)
return log
g.Log().Debugf(ctx, "创建送检审计日志成功, id=%d, sourceTable=%s, sourceID=%d, taskId=%s", id, sourceTable, sourceID, taskID)
}
// submitImageCheck 提交图片送检
func (s *TencentContentCheckService) submitImageCheck(ctx context.Context, image *entity.TencentImage, log *entity.TencentContentCheckLog) error {
startTime := time.Now()
// 更新日志状态为送检中
if log != nil {
dao.TencentContentCheckLog.UpdateStatus(ctx, log.Id, consts.CheckStatusSubmitting, "", "")
}
// 获取回调地址
callbackURL := g.Cfg().MustGet(ctx, "yidun.image.callback_url").String()
// 调用易盾图片检测
result, err := yidunService.ImageDetection.DetectImage(ctx, image.PreviewURL, image.ImageID, callbackURL)
duration := time.Since(startTime).Milliseconds()
// 更新日志
if log != nil {
if err != nil {
dao.TencentContentCheckLog.UpdateDuration(ctx, log.Id, duration)
dao.TencentContentCheckLog.UpdateStatus(ctx, log.Id, consts.CheckStatusFailed, "", err.Error())
g.Log().Errorf(ctx, "图片送检失败, id=%d, url=%s, error=%v", image.Id, image.PreviewURL, err)
return err
}
// 更新日志和图片状态
responseData, _ := json.Marshal(result)
dao.TencentContentCheckLog.UpdateStatus(ctx, log.Id, consts.CheckStatusSuccess, string(responseData), "")
dao.TencentContentCheckLog.UpdateTaskID(ctx, log.Id, result.TaskID)
dao.TencentContentCheckLog.UpdateDuration(ctx, log.Id, duration)
}
g.Log().Infof(ctx, "图片送检成功, id=%d, imageId=%s, taskId=%s", image.Id, image.ImageID, result.TaskID)
return nil
}
// submitVideoCheck 提交视频送检
func (s *TencentContentCheckService) submitVideoCheck(ctx context.Context, video *entity.TencentVideo, log *entity.TencentContentCheckLog) error {
startTime := time.Now()
// 更新日志状态为送检中
if log != nil {
dao.TencentContentCheckLog.UpdateStatus(ctx, log.Id, consts.CheckStatusSubmitting, "", "")
}
// 获取回调地址
callbackURL := g.Cfg().MustGet(ctx, "yidun.video.callback_url").String()
// 调用易盾视频检测
result, err := yidunService.VideoDetection.DetectVideo(ctx, video.PreviewURL, video.VideoID, callbackURL)
duration := time.Since(startTime).Milliseconds()
// 更新日志
if log != nil {
if err != nil {
dao.TencentContentCheckLog.UpdateDuration(ctx, log.Id, duration)
dao.TencentContentCheckLog.UpdateStatus(ctx, log.Id, consts.CheckStatusFailed, "", err.Error())
g.Log().Errorf(ctx, "视频送检失败, id=%d, url=%s, error=%v", video.Id, video.PreviewURL, err)
return err
}
// 更新日志和视频状态
responseData, _ := json.Marshal(result)
dao.TencentContentCheckLog.UpdateStatus(ctx, log.Id, consts.CheckStatusSuccess, string(responseData), "")
dao.TencentContentCheckLog.UpdateTaskID(ctx, log.Id, result.TaskID)
dao.TencentContentCheckLog.UpdateDuration(ctx, log.Id, duration)
}
g.Log().Infof(ctx, "视频送检成功, id=%d, videoId=%s, taskId=%s", video.Id, video.VideoID, result.TaskID)
return nil
}
// SubmitImageByID 根据图片ID手动提交送检
// SubmitImageByID 根据图片ID手动提交送检（统一走 MaterialVerify 系统）
func (s *TencentContentCheckService) SubmitImageByID(ctx context.Context, imageID string) (*yidunService.ImageSubmitResult, error) {
// 根据图片ID获取数据
image, err := dao.TencentImage.GetByImageID(ctx, imageID)
if err != nil {
return nil, fmt.Errorf("查询图片数据失败: %w", err)
}
if image == nil {
return nil, fmt.Errorf("未找到图片数据, imageID=%s", imageID)
}
// 创建送检日志
log := s.createCheckLog(ctx, consts.SourceTableTencentImage, image.Id, image.ImageID, image.PreviewURL)
if log == nil {
return nil, fmt.Errorf("创建送检日志失败")
}
// 提交送检
err = s.submitImageCheck(ctx, image, log)
mLog, err := MaterialVerify.VerifyImageByID(ctx, imageID)
if err != nil {
return nil, err
}
// 获取送检结果
return dao.TencentContentCheckLog.GetImageSubmitResult(ctx, log.Id)
image, err := dao.TencentImage.GetByImageID(ctx, imageID)
if err == nil && image != nil {
s.writeAuditLog(ctx, consts.SourceTableTencentImage, image.Id, imageID, image.PreviewURL, mLog.TaskID)
}
return &yidunService.ImageSubmitResult{
TaskID: mLog.TaskID,
}, nil
}
// SubmitVideoByID 根据视频ID手动提交送检
// SubmitVideoByID 根据视频ID手动提交送检（统一走 MaterialVerify 系统）
func (s *TencentContentCheckService) SubmitVideoByID(ctx context.Context, videoID string) (*yidunService.VideoSubmitResult, error) {
// 根据视频ID获取数据
video, err := dao.TencentVideo.GetByVideoID(ctx, videoID)
if err != nil {
return nil, fmt.Errorf("查询视频数据失败: %w", err)
}
if video == nil {
return nil, fmt.Errorf("未找到视频数据, videoID=%s", videoID)
}
// 创建送检日志
log := s.createCheckLog(ctx, consts.SourceTableTencentVideo, video.Id, video.VideoID, video.PreviewURL)
if log == nil {
return nil, fmt.Errorf("创建送检日志失败")
}
// 提交送检
err = s.submitVideoCheck(ctx, video, log)
mLog, err := MaterialVerify.VerifyVideoByID(ctx, videoID)
if err != nil {
return nil, err
}
// 获取送检结果
return dao.TencentContentCheckLog.GetVideoSubmitResult(ctx, log.Id)
video, err := dao.TencentVideo.GetByVideoID(ctx, videoID)
if err == nil && video != nil {
s.writeAuditLog(ctx, consts.SourceTableTencentVideo, video.Id, videoID, video.PreviewURL, mLog.TaskID)
}
return &yidunService.VideoSubmitResult{
TaskID: mLog.TaskID,
}, nil
}
// GetPendingStats 获取待送检统计