diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..fa2f322 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,30 @@ +{ + "permissions": { + "allow": [ + "Bash(go build *)", + "Bash(go test *)", + "Bash(go run *)", + "Bash(go mod *)", + "Bash(curl *)", + "Bash(npx *)", + "Bash(brew *)", + "Bash(npm install *)", + "Bash(npm run *)", + "Bash(python3 *)", + "Bash(psql *)", + "Bash(PGPASSWORD=*)", + "Bash(lsof *)", + "Bash(pkill *)", + "Bash(sleep *)", + "Bash(mkdir *)", + "Bash(which *)", + "Bash(pwd *)", + "Bash(echo *)", + "Bash(source *)", + "Bash(export *)", + "Bash(unset *)", + "Bash(package *)", + "Bash(kill *)" + ] + } +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 61829ef..6873603 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /.idea/* /data-engine /resource/* +.gstack/ +/docs/* diff --git a/CLAUDE.md b/CLAUDE.md index bac59a9..317ddc8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,7 @@ # data-engine +> ⚠️ **启动时必读:** 每次会话开始时，先读取 `docs/INDEX.md` 了解项目全景和最近变更。 + ## Skill routing When the user's request matches an available skill, invoke it via the Skill tool. When in doubt, invoke the skill. @@ -18,3 +20,17 @@ Key routing rules: - Save progress → invoke /context-save - Resume context → invoke /context-restore - Author a backlog-ready spec/issue → invoke /spec + +## Agent skills + +### Issue tracker + +Issues tracked as local markdown files under `.scratch/`. See `docs/agents/issue-tracker.md`. + +### Triage labels + +Default five-role vocabulary (`needs-triage`, `needs-info`, `ready-for-agent`, `ready-for-human`, `wontfix`). See `docs/agents/triage-labels.md`. + +### Domain docs + +Single-context repo. See `docs/agents/domain.md`. diff --git a/common/report/api.go b/common/report/api.go index 25ce1d4..8b31a01 100644 --- a/common/report/api.go +++ b/common/report/api.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "dataengine/common/report/config" "dataengine/common/report/ddlsync" @@ -13,8 +14,18 @@ import ( "gitea.redpowerfuture.com/red-future/common/beans" "gitea.redpowerfuture.com/red-future/common/db/gfdb" + + "github.com/gogf/gf/v2/database/gdb" ) +// getUserNameFromCtx 从上下文中获取当前用户名 +func getUserNameFromCtx(ctx context.Context) string { + if user, ok := ctx.Value("user").(*beans.User); ok && user != nil && user.UserName != "" { + return user.UserName + } + return "system" +} + // ReportService 报表公共服务 // 对外暴露的统一接口 type ReportService struct { @@ -108,7 +119,68 @@ func (s *ReportService) ExtractDailyData(ctx context.Context, businessCode, repo } // ============================================================ -// 核心接口 3: 用户选择查询（最核心） +// 核心接口 3: 批量回填数据 +// 首次接入时对日期范围逐天抽取 +// ============================================================ + +// Backfill 批量回填数据 +// 遍历 startDate → endDate 逐天调用 ExtractDailyData +// 返回汇总统计 +func (s *ReportService) Backfill(ctx context.Context, req *model.BackfillReq) (*model.BackfillResp, error) { + start := time.Now() + + startDate, err := time.Parse("2006-01-02", req.StartDate) + if err != nil { + return nil, fmt.Errorf("开始日期格式错误: %w", err) + } + endDate, err := time.Parse("2006-01-02", req.EndDate) + if err != nil { + return nil, fmt.Errorf("结束日期格式错误: %w", err) + } + if endDate.Before(startDate) { + return nil, fmt.Errorf("结束日期不能早于开始日期") + } + + totalDays := 0 + successDays := 0 + failDays := 0 + var lastErr error + + for d := startDate; !d.After(endDate); d = d.AddDate(0, 0, 1) { + totalDays++ + dateStr := d.Format("2006-01-02") + resp, err := s.ExtractDailyData(ctx, req.BusinessCode, req.ReportCode, dateStr, req.Executor) + if err != nil { + failDays++ + lastErr = fmt.Errorf("回填 %s 失败: %w", dateStr, err) + continue + } + if !resp.Success { + failDays++ + lastErr = fmt.Errorf("回填 %s 异常: %s", dateStr, resp.ErrorMsg) + continue + } + successDays++ + } + + execTime := time.Since(start).Milliseconds() + + resp := &model.BackfillResp{ + Success: failDays == 0, + TotalDays: totalDays, + SuccessDays: successDays, + FailDays: failDays, + ExecTimeMs: execTime, + } + if lastErr != nil { + resp.ErrorMsg = lastErr.Error() + } + + return resp, nil +} + +// ============================================================ +// 核心接口 4: 用户选择查询（最核心） // 前端用户选择条件 → 实时构建SQL → 返回报表数据 // ============================================================ @@ -145,11 +217,21 @@ func (s *ReportService) GetReportFields(ctx context.Context, businessCode, repor } // GetAllBusinesses 获取所有启用业务列表 + +// ListBusinesses 分页获取业务列表 +func (s *ReportService) ListBusinesses(ctx context.Context, pageNum, pageSize int) ([]model.BusinessConfig, int, error) { + return s.configLoader.ListBusinesses(ctx, pageNum, pageSize) +} func (s *ReportService) GetAllBusinesses(ctx context.Context) ([]model.BusinessConfig, error) { return s.configLoader.GetAllBusinesses(ctx) } // GetAllReports 获取业务下所有报表列表 + +// ListReports 分页获取报表列表 +func (s *ReportService) ListReports(ctx context.Context, businessCode, reportName string, pageNum, pageSize int) ([]model.ReportConfig, int, error) { + return s.configLoader.ListReports(ctx, businessCode, reportName, pageNum, pageSize) +} func (s *ReportService) GetAllReports(ctx context.Context, businessCode string) ([]model.ReportConfig, error) { return s.configLoader.GetAllReports(ctx, businessCode) } @@ -174,8 +256,20 @@ func (s *ReportService) SaveBusiness(ctx context.Context, req *model.SaveBusines return nil, fmt.Errorf("初始化系统表失败: %w", err) } + // 新增时校验 businessCode 唯一性 + if req.ID == nil || *req.ID == 0 { + r, err := gfdb.DB(ctx).GetAll(ctx, "SELECT id FROM report_business_config WHERE business_code = $1 LIMIT 1", req.BusinessCode) + if err != nil { + return nil, fmt.Errorf("校验业务编码失败: %w", err) + } + if !r.IsEmpty() { + return nil, fmt.Errorf("业务编码 %s 已存在", req.BusinessCode) + } + } + + userName := getUserNameFromCtx(ctx) biz := &model.BusinessConfig{ - SQLBaseDO: beans.SQLBaseDO{Creator: req.Operator, Updater: req.Operator}, + SQLBaseDO: beans.SQLBaseDO{Creator: userName, Updater: userName, TenantId: 1}, BusinessCode: req.BusinessCode, BusinessName: req.BusinessName, Description: req.Description, @@ -213,6 +307,14 @@ func (s *ReportService) DeleteBusiness(ctx context.Context, id int64) (*model.De if err != nil { return nil, err } + // 级联检查：该业务下是否存在报表 + r, err := gfdb.DB(ctx).GetAll(ctx, "SELECT COUNT(1) AS cnt FROM report_report_config WHERE business_code = $1 AND deleted_at IS NULL LIMIT 1", biz.BusinessCode) + if err == nil && !r.IsEmpty() { + cnt := r[0]["cnt"].Int() + if cnt > 0 { + return &model.DeleteResult{Success: false, Message: fmt.Sprintf("业务 %s 下存在 %d 个报表配置，请先删除关联报表后再删除业务", biz.BusinessCode, cnt)}, nil + } + } if err := s.configLoader.DeleteBusiness(ctx, id, biz.BusinessCode); err != nil { return nil, err } @@ -234,8 +336,21 @@ func (s *ReportService) SaveReport(ctx context.Context, req *model.SaveReportReq return nil, fmt.Errorf("初始化系统表失败: %w", err) } + // 新增时校验 reportCode 唯一性（同业务下） + if req.ID == nil || *req.ID == 0 { + r, err := gfdb.DB(ctx).GetAll(ctx, "SELECT id FROM report_report_config WHERE business_code = $1 AND report_code = $2 LIMIT 1", + req.BusinessCode, req.ReportCode) + if err != nil { + return nil, fmt.Errorf("校验报表编码失败: %w", err) + } + if !r.IsEmpty() { + return nil, fmt.Errorf("业务 %s 下报表编码 %s 已存在", req.BusinessCode, req.ReportCode) + } + } + + userName := getUserNameFromCtx(ctx) rpt := &model.ReportConfig{ - SQLBaseDO: beans.SQLBaseDO{Creator: req.Operator, Updater: req.Operator}, + SQLBaseDO: beans.SQLBaseDO{Creator: userName, Updater: userName, TenantId: 1}, BusinessCode: req.BusinessCode, ReportCode: req.ReportCode, ReportName: req.ReportName, @@ -286,6 +401,18 @@ func (s *ReportService) DeleteReport(ctx context.Context, id int64) (*model.Dele if err != nil { return nil, err } + // 级联检查：该报表下是否存在未删除的字段 + r, err := gfdb.DB(ctx).GetAll(ctx, "SELECT COUNT(1) AS cnt FROM report_field_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL LIMIT 1", + rpt.BusinessCode, rpt.ReportCode) + if err == nil && !r.IsEmpty() && r[0]["cnt"].Int() > 0 { + return &model.DeleteResult{Success: false, Message: fmt.Sprintf("报表 %s 下存在字段配置，请先删除关联字段后再删除报表", rpt.ReportCode)}, nil + } + // 级联检查：该报表下是否存在抽取配置 + r, err = gfdb.DB(ctx).GetAll(ctx, "SELECT COUNT(1) AS cnt FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL LIMIT 1", + rpt.BusinessCode, rpt.ReportCode) + if err == nil && !r.IsEmpty() && r[0]["cnt"].Int() > 0 { + return &model.DeleteResult{Success: false, Message: fmt.Sprintf("报表 %s 下存在抽取配置，请先删除关联抽取配置后再删除报表", rpt.ReportCode)}, nil + } if err := s.configLoader.DeleteReport(ctx, id, rpt.BusinessCode, rpt.ReportCode); err != nil { return nil, err } @@ -294,7 +421,223 @@ func (s *ReportService) DeleteReport(ctx context.Context, id int64) (*model.Dele // GetReport 获取单个报表配置 func (s *ReportService) GetReport(ctx context.Context, id int64) (*model.ReportConfig, error) { - return s.configLoader.GetReportByID(ctx, id) + rpt, err := s.configLoader.GetReportByID(ctx, id) + if err != nil { + return nil, err + } + // 同时加载全部字段（含已失效），编辑时前端可直接回显 + allFields, err := s.configLoader.GetAllFields(ctx, rpt.BusinessCode, rpt.ReportCode) + if err == nil { + rpt.Fields = allFields + } + return rpt, nil +} + +// ============================================================ +// 合并操作：报表+字段一起保存 +// ============================================================ + +// SaveReportWithFields 保存报表及字段配置（报表+字段全量替换） +func (s *ReportService) SaveReportWithFields(ctx context.Context, req *model.SaveReportWithFieldsReq) (*model.SaveResult, error) { + if err := initTables(ctx); err != nil { + return nil, fmt.Errorf("初始化系统表失败: %w", err) + } + + // 自动填充每个字段的 businessCode/reportCode（从报表级继承） + for i := range req.Fields { + if req.Fields[i].BusinessCode == "" { + req.Fields[i].BusinessCode = req.BusinessCode + } + if req.Fields[i].ReportCode == "" { + req.Fields[i].ReportCode = req.ReportCode + } + } + + var reportID int64 + userName := getUserNameFromCtx(ctx) + + // 事务：要么全成功，要么不动 + err := gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error { + // 1. 保存报表（新增或更新） + now := time.Now() + if req.ID != nil && *req.ID > 0 { + // 更新报表 + data := map[string]interface{}{ + "business_code": req.BusinessCode, + "report_code": req.ReportCode, + "report_name": req.ReportName, + "description": req.Description, + "status": req.Status, + "stat_table_name": req.StatTableName, + "stat_table_comment": req.StatTableComment, + "date_field": req.DateField, + "primary_keys": req.PrimaryKeys, + "conflict_keys": req.ConflictKeys, + "config": req.Config, + "updated_at": now, + "updater": userName, + } + if _, err := tx.Model("report_report_config").Where("id", *req.ID).Data(data).Update(); err != nil { + return fmt.Errorf("更新报表失败: %w", err) + } + reportID = *req.ID + } else { + // 新增时校验 reportCode 唯一性 + cnt, err := tx.Model("report_report_config").Where("business_code", req.BusinessCode).Where("report_code", req.ReportCode).Count() + if err != nil { + return fmt.Errorf("校验报表编码失败: %w", err) + } + if cnt > 0 { + return fmt.Errorf("业务 %s 下报表编码 %s 已存在", req.BusinessCode, req.ReportCode) + } + data := map[string]interface{}{ + "business_code": req.BusinessCode, + "report_code": req.ReportCode, + "report_name": req.ReportName, + "description": req.Description, + "status": ifVal(req.Status, model.StatusActive), + "stat_table_name": req.StatTableName, + "stat_table_comment": req.StatTableComment, + "date_field": ifVal(req.DateField, "stat_date"), + "primary_keys": req.PrimaryKeys, + "conflict_keys": req.ConflictKeys, + "config": req.Config, + "created_at": now, + "updated_at": now, + "creator": userName, + "updater": userName, + "tenant_id": 1, + } + if data["primary_keys"] == nil || len(data["primary_keys"].([]string)) == 0 { + data["primary_keys"] = []string{"id"} + } + if data["conflict_keys"] == nil || len(data["conflict_keys"].([]string)) == 0 { + data["conflict_keys"] = []string{ifVal(req.DateField, "stat_date")} + } + result, err := tx.Model("report_report_config").Data(data).Insert() + if err != nil { + if strings.Contains(err.Error(), "duplicate key") { + return fmt.Errorf("业务 %s 下报表编码 %s 已存在", req.BusinessCode, req.ReportCode) + } + return fmt.Errorf("创建报表失败: %w", err) + } + rid, err := result.LastInsertId() + if err != nil { + return fmt.Errorf("获取自增ID失败: %w", err) + } + reportID = rid + } + + // 2. 获取报表下现有字段（含已删除，编辑时重新添加的字段需走 UPDATE 而非 INSERT） + rows, err := tx.Model("report_field_config"). + Where("business_code", req.BusinessCode). + Where("report_code", req.ReportCode). + All() + if err != nil { + return fmt.Errorf("获取现有字段失败: %w", err) + } + existingMap := make(map[string]int64) // fieldCode -> id + for _, row := range rows { + existingMap[row["field_code"].String()] = row["id"].Int64() + } + + // 3. 处理请求中的字段（新增或更新） + for i := range req.Fields { + f := &req.Fields[i] + status := ifVal(f.Status, model.StatusActive) + dataType := ifVal(f.DataType, model.FieldTypeString) + + if existingID, ok := existingMap[f.FieldCode]; ok { + // 已存在 → 更新 + updateData := map[string]interface{}{ + "field_name": f.FieldName, + "field_type": f.FieldType, + "data_type": dataType, + "field_role": f.FieldRole, + "is_aggregatable": f.IsAggregatable, + "is_filterable": f.IsFilterable, + "is_queryable": f.IsQueryable, + "is_sortable": f.IsSortable, + "default_aggregate": f.DefaultAggregate, + "valid_aggregates": f.ValidAggregates, + "filter_operators": f.FilterOperators, + "expression": f.Expression, + "expression_type": f.ExpressionType, + "format_pattern": f.FormatPattern, + "unit": f.Unit, + "dict_code": f.DictCode, + "sort_order": f.SortOrder, + "group_name": f.GroupName, + "status": status, + "updated_at": now, + "updater": userName, + } + if _, err := tx.Model("report_field_config").Where("id", existingID).Data(updateData).Update(); err != nil { + return fmt.Errorf("更新字段 %s 失败: %w", f.FieldCode, err) + } + delete(existingMap, f.FieldCode) + } else { + // 不存在 → 新增 + insertData := map[string]interface{}{ + "business_code": req.BusinessCode, + "report_code": req.ReportCode, + "field_code": f.FieldCode, + "field_name": f.FieldName, + "field_type": f.FieldType, + "data_type": dataType, + "field_role": f.FieldRole, + "is_aggregatable": f.IsAggregatable, + "is_filterable": f.IsFilterable, + "is_queryable": f.IsQueryable, + "is_sortable": f.IsSortable, + "default_aggregate": f.DefaultAggregate, + "valid_aggregates": f.ValidAggregates, + "filter_operators": f.FilterOperators, + "expression": f.Expression, + "expression_type": f.ExpressionType, + "format_pattern": f.FormatPattern, + "unit": f.Unit, + "dict_code": f.DictCode, + "sort_order": f.SortOrder, + "group_name": f.GroupName, + "status": status, + "created_at": now, + "updated_at": now, + "creator": userName, + "updater": userName, + "tenant_id": 1, + } + if _, err := tx.Model("report_field_config").Data(insertData).Insert(); err != nil { + if strings.Contains(err.Error(), "duplicate key") { + return fmt.Errorf("字段编码 %s 在当前报表下已存在，请更换字段编码", f.FieldCode) + } + return fmt.Errorf("创建字段 %s 失败: %w", f.FieldCode, err) + } + } + } + + // 4. 删除剩余字段（不在请求中的旧字段） + for fieldCode, existingID := range existingMap { + if _, err := tx.Model("report_field_config").Where("id", existingID).Data(map[string]interface{}{ + "status": model.StatusInactive, + "deleted_at": now, + "updated_at": now, + "updater": userName, + }).Update(); err != nil { + return fmt.Errorf("删除字段 %s 失败: %w", fieldCode, err) + } + } + + return nil + }) + if err != nil { + return nil, err + } + + // 清理缓存 + s.configLoader.InvalidateCache(req.BusinessCode, req.ReportCode) + + return &model.SaveResult{Success: true, ID: reportID, Message: "报表及字段保存成功"}, nil } // ============================================================ @@ -307,8 +650,16 @@ func (s *ReportService) SaveField(ctx context.Context, req *model.SaveFieldReq) return nil, fmt.Errorf("初始化系统表失败: %w", err) } + if req.BusinessCode == "" { + return nil, fmt.Errorf("businessCode 不能为空") + } + if req.ReportCode == "" { + return nil, fmt.Errorf("reportCode 不能为空") + } + + userName := getUserNameFromCtx(ctx) field := &model.FieldConfig{ - SQLBaseDO: beans.SQLBaseDO{Creator: req.Operator, Updater: req.Operator}, + SQLBaseDO: beans.SQLBaseDO{Creator: userName, Updater: userName, TenantId: 1}, BusinessCode: req.BusinessCode, ReportCode: req.ReportCode, FieldCode: req.FieldCode, @@ -418,8 +769,21 @@ func (s *ReportService) SaveExtractConfig(ctx context.Context, req *model.SaveEx return nil, fmt.Errorf("初始化系统表失败: %w", err) } + // 新增时校验 extractCode 唯一性（同业务+报表下） + if req.ID == nil || *req.ID == 0 { + r, err := gfdb.DB(ctx).GetAll(ctx, "SELECT id FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND extract_code = $3 LIMIT 1", + req.BusinessCode, req.ReportCode, req.ExtractCode) + if err != nil { + return nil, fmt.Errorf("校验抽取编码失败: %w", err) + } + if !r.IsEmpty() { + return nil, fmt.Errorf("业务 %s/报表 %s 下抽取编码 %s 已存在", req.BusinessCode, req.ReportCode, req.ExtractCode) + } + } + + userName := getUserNameFromCtx(ctx) ec := &model.ExtractConfig{ - SQLBaseDO: beans.SQLBaseDO{Creator: req.Operator, Updater: req.Operator}, + SQLBaseDO: beans.SQLBaseDO{Creator: userName, Updater: userName, TenantId: 1}, BusinessCode: req.BusinessCode, ReportCode: req.ReportCode, ExtractCode: req.ExtractCode, @@ -499,6 +863,38 @@ func (s *ReportService) GetExtractConfig(ctx context.Context, id int64) (*model. } // GetExtractConfigs 获取业务报表下所有抽取配置 + +// ListExtractConfigs 分页获取抽取配置列表 +func (s *ReportService) ListExtractConfigs(ctx context.Context, businessCode, reportCode string, pageNum, pageSize int) ([]model.ExtractConfig, int, error) { + return s.configLoader.ListExtractConfigs(ctx, businessCode, reportCode, pageNum, pageSize) +} func (s *ReportService) GetExtractConfigs(ctx context.Context, businessCode, reportCode string) ([]model.ExtractConfig, error) { return s.configLoader.GetExtractConfigs(ctx, businessCode, reportCode) } + +// GetExtractConfigsAll 获取业务报表下所有抽取配置（不限 status，用于填充字段展示） +func (s *ReportService) GetExtractConfigsAll(ctx context.Context, businessCode, reportCode string) ([]model.ExtractConfig, error) { + r, err := gfdb.DB(ctx).GetAll(ctx, + "SELECT id, extract_code, extract_name, status FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND status != $3 ORDER BY id ASC", + businessCode, reportCode, model.StatusInactive) + if err != nil { + return nil, err + } + var configs []model.ExtractConfig + for _, record := range r { + var ec model.ExtractConfig + if err := record.Struct(&ec); err != nil { + return nil, err + } + configs = append(configs, ec) + } + return configs, nil +} + +// ifVal 三目运算：非空返回 s，否则返回 fallback +func ifVal(s, fallback string) string { + if s == "" { + return fallback + } + return s +} diff --git a/common/report/builder/sql_builder.go b/common/report/builder/sql_builder.go index 11a4357..6ead390 100644 --- a/common/report/builder/sql_builder.go +++ b/common/report/builder/sql_builder.go @@ -92,8 +92,13 @@ func (b *SQLBuilder) BuildQuerySQL(ctx context.Context, req *model.UserSelectQue countSql += " WHERE " + whereClause } if groupByClause != "" { - countSql = fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE %s GROUP BY %s) AS t", - fromClause, whereClause, groupByClause) + if whereClause != "" { + countSql = fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE %s GROUP BY %s) AS t", + fromClause, whereClause, groupByClause) + } else { + countSql = fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s GROUP BY %s) AS t", + fromClause, groupByClause) + } } metadata := map[string]interface{}{ @@ -352,6 +357,8 @@ func (b *SQLBuilder) buildTimeGroupExpr(dateField, timeGroup string) string { return fmt.Sprintf("DATE_TRUNC('week', %s::date)::text AS time_group", dateField) case "month": return fmt.Sprintf("TO_CHAR(%s::date, 'YYYY-MM') AS time_group", dateField) + case "year": + return fmt.Sprintf("TO_CHAR(%s::date, 'YYYY') AS time_group", dateField) case "quarter": return "TO_CHAR(" + dateField + "::date, 'YYYY-\"Q\"Q') AS time_group" default: diff --git a/common/report/builder/sql_builder_test.go b/common/report/builder/sql_builder_test.go index 20d0cc0..8828005 100644 --- a/common/report/builder/sql_builder_test.go +++ b/common/report/builder/sql_builder_test.go @@ -9,12 +9,12 @@ import ( // Test data - reusable field configs var testFields = map[string]*model.FieldConfig{ - "shop_id": {FieldCode: "shop_id", FieldName: "店铺ID", FieldType: "STRING", FieldRole: "DIMENSION", IsFilterable: true, IsSortable: true, FilterOperators: []string{"=", "IN"}}, - "shop_name": {FieldCode: "shop_name", FieldName: "店铺名称", FieldType: "STRING", FieldRole: "DIMENSION", IsFilterable: true, IsSortable: true, FilterOperators: []string{"=", "LIKE"}}, - "stat_date": {FieldCode: "stat_date", FieldName: "统计日期", FieldType: "DATE", FieldRole: "DIMENSION", IsFilterable: true, IsSortable: true, FilterOperators: []string{"=", ">=", "<=", "BETWEEN"}}, - "order_count": {FieldCode: "order_count", FieldName: "订单数", FieldType: "INT", FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "SUM", ValidAggregates: []string{"SUM", "COUNT"}}, + "shop_id": {FieldCode: "shop_id", FieldName: "店铺ID", FieldType: "STRING", FieldRole: "DIMENSION", IsFilterable: true, IsSortable: true, FilterOperators: []string{"=", "IN"}}, + "shop_name": {FieldCode: "shop_name", FieldName: "店铺名称", FieldType: "STRING", FieldRole: "DIMENSION", IsFilterable: true, IsSortable: true, FilterOperators: []string{"=", "LIKE"}}, + "stat_date": {FieldCode: "stat_date", FieldName: "统计日期", FieldType: "DATE", FieldRole: "DIMENSION", IsFilterable: true, IsSortable: true, FilterOperators: []string{"=", ">=", "<=", "BETWEEN"}}, + "order_count": {FieldCode: "order_count", FieldName: "订单数", FieldType: "INT", FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "SUM", ValidAggregates: []string{"SUM", "COUNT"}}, "order_amount": {FieldCode: "order_amount", FieldName: "订单金额", FieldType: "FLOAT", FieldRole: "INDICATOR", IsAggregatable: true, IsSortable: true, DefaultAggregate: "SUM", ValidAggregates: []string{"SUM", "AVG", "MAX", "MIN"}}, - "refund_rate": {FieldCode: "refund_rate", FieldName: "退款率", FieldType: "FLOAT", FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "AVG", ValidAggregates: []string{"AVG"}, Expression: "{refund_amount} / NULLIF({order_amount}, 0) * 100", ExpressionType: "CALCULATED"}, + "refund_rate": {FieldCode: "refund_rate", FieldName: "退款率", FieldType: "FLOAT", FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "AVG", ValidAggregates: []string{"AVG"}, Expression: "{refund_amount} / NULLIF({order_amount}, 0) * 100", ExpressionType: "CALCULATED"}, "order_status": {FieldCode: "order_status", FieldName: "订单状态", FieldType: "STRING", FieldRole: "FILTER", IsFilterable: true, FilterOperators: []string{"=", "IN"}}, } @@ -303,6 +303,7 @@ func TestBuildTimeGroupExpr(t *testing.T) { group string wantSub string }{ + {"year", "YYYY"}, {"week", "DATE_TRUNC"}, {"month", "YYYY-MM"}, {"quarter", "Q"}, diff --git a/common/report/config/loader.go b/common/report/config/loader.go index 3579737..0830637 100644 --- a/common/report/config/loader.go +++ b/common/report/config/loader.go @@ -51,8 +51,8 @@ func (l *ConfigLoader) GetBusiness(ctx context.Context, businessCode string) (*m var biz model.BusinessConfig r, err := gfdb.DB(ctx).GetAll(ctx, - "SELECT * FROM report_business_config WHERE business_code = $1 AND status = $2 LIMIT 1", - businessCode, model.StatusActive) + "SELECT * FROM report_business_config WHERE business_code = $1 AND deleted_at IS NULL LIMIT 1", + businessCode) if err != nil { return nil, fmt.Errorf("查询业务配置失败: %w", err) } @@ -86,8 +86,8 @@ func (l *ConfigLoader) GetReport(ctx context.Context, businessCode, reportCode s var rpt model.ReportConfig r, err := gfdb.DB(ctx).GetAll(ctx, - "SELECT * FROM report_report_config WHERE business_code = $1 AND report_code = $2 AND status = $3 LIMIT 1", - businessCode, reportCode, model.StatusActive) + "SELECT * FROM report_report_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL LIMIT 1", + businessCode, reportCode) if err != nil { return nil, fmt.Errorf("查询报表配置失败: %w", err) } @@ -126,8 +126,8 @@ func (l *ConfigLoader) GetFields(ctx context.Context, businessCode, reportCode s l.mu.RUnlock() r, err := gfdb.DB(ctx).GetAll(ctx, - "SELECT * FROM report_field_config WHERE business_code = $1 AND report_code = $2 AND status = $3 ORDER BY sort_order ASC", - businessCode, reportCode, model.StatusActive) + "SELECT * FROM report_field_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY sort_order ASC", + businessCode, reportCode) if err != nil { return nil, err } @@ -153,6 +153,31 @@ func (l *ConfigLoader) GetFields(ctx context.Context, businessCode, reportCode s return fields, nil } +// GetAllFields 获取报表全部字段（含 INACTIVE，不含已删除，用于编辑回显） +func (l *ConfigLoader) GetAllFields(ctx context.Context, businessCode, reportCode string) ([]model.FieldConfig, error) { + r, err := gfdb.DB(ctx).GetAll(ctx, + "SELECT * FROM report_field_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY sort_order ASC", + businessCode, reportCode) + if err != nil { + return nil, err + } + var fields []model.FieldConfig + for _, record := range r { + var f model.FieldConfig + if err := record.Struct(&f); err != nil { + return nil, err + } + if f.ValidAggregates == nil { + f.ValidAggregates = []string{} + } + if f.FilterOperators == nil { + f.FilterOperators = []string{"=", "!=", ">", "<", ">=", "<=", "IN", "LIKE", "BETWEEN"} + } + fields = append(fields, f) + } + return fields, nil +} + // GetFieldMap 获取字段配置Map func (l *ConfigLoader) GetFieldMap(ctx context.Context, businessCode, reportCode string) (map[string]*model.FieldConfig, error) { fields, err := l.GetFields(ctx, businessCode, reportCode) @@ -178,8 +203,8 @@ func (l *ConfigLoader) GetExtractConfigs(ctx context.Context, businessCode, repo l.mu.RUnlock() r, err := gfdb.DB(ctx).GetAll(ctx, - "SELECT * FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND status = $3 AND is_enabled = $4", - businessCode, reportCode, model.StatusActive, true) + "SELECT * FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY id ASC", + businessCode, reportCode) if err != nil { return nil, err } @@ -586,8 +611,9 @@ func (l *ConfigLoader) GetExtractConfigByID(ctx context.Context, id int64) (*mod // GetAllBusinesses 获取所有业务配置 func (l *ConfigLoader) GetAllBusinesses(ctx context.Context) ([]model.BusinessConfig, error) { r, err := gfdb.DB(ctx).GetAll(ctx, - "SELECT * FROM report_business_config WHERE status = $1 ORDER BY id ASC", - model.StatusActive) + "SELECT * FROM report_business_config WHERE deleted_at IS NULL ORDER BY id ASC", + ) + // (removed status filter) if err != nil { return nil, err } @@ -605,8 +631,8 @@ func (l *ConfigLoader) GetAllBusinesses(ctx context.Context) ([]model.BusinessCo // GetAllReports 获取所有报表配置 func (l *ConfigLoader) GetAllReports(ctx context.Context, businessCode string) ([]model.ReportConfig, error) { r, err := gfdb.DB(ctx).GetAll(ctx, - "SELECT * FROM report_report_config WHERE business_code = $1 AND status = $2 ORDER BY id ASC", - businessCode, model.StatusActive) + "SELECT * FROM report_report_config WHERE business_code = $1 AND deleted_at IS NULL ORDER BY id ASC", + businessCode) if err != nil { return nil, err } @@ -621,6 +647,99 @@ func (l *ConfigLoader) GetAllReports(ctx context.Context, businessCode string) ( return reports, nil } +// ListBusinesses 分页获取业务列表 +func (l *ConfigLoader) ListBusinesses(ctx context.Context, pageNum, pageSize int) ([]model.BusinessConfig, int, error) { + total, err := gfdb.DB(ctx).GetAll(ctx, + "SELECT COUNT(*) AS cnt FROM report_business_config WHERE deleted_at IS NULL") + if err != nil { + return nil, 0, err + } + count := 0 + if !total.IsEmpty() { + count = total[0]["cnt"].Int() + } + offset := (pageNum - 1) * pageSize + r, err := gfdb.DB(ctx).GetAll(ctx, + "SELECT * FROM report_business_config WHERE deleted_at IS NULL ORDER BY id ASC LIMIT $1 OFFSET $2", + pageSize, offset) + if err != nil { + return nil, 0, err + } + var businesses []model.BusinessConfig + for _, record := range r { + var biz model.BusinessConfig + if err := record.Struct(&biz); err != nil { + return nil, 0, err + } + businesses = append(businesses, biz) + } + return businesses, count, nil +} + +// ListReports 分页获取报表列表 +func (l *ConfigLoader) ListReports(ctx context.Context, businessCode, reportName string, pageNum, pageSize int) ([]model.ReportConfig, int, error) { + namePattern := "%" + reportName + "%" + total, err := gfdb.DB(ctx).GetAll(ctx, + "SELECT COUNT(*) AS cnt FROM report_report_config WHERE business_code = $1 AND report_name ILIKE $2 AND deleted_at IS NULL", + businessCode, namePattern) + if err != nil { + return nil, 0, err + } + count := 0 + if !total.IsEmpty() { + count = total[0]["cnt"].Int() + } + offset := (pageNum - 1) * pageSize + r, err := gfdb.DB(ctx).GetAll(ctx, + "SELECT * FROM report_report_config WHERE business_code = $1 AND report_name ILIKE $2 AND deleted_at IS NULL ORDER BY id ASC LIMIT $3 OFFSET $4", + businessCode, namePattern, pageSize, offset) + if err != nil { + return nil, 0, err + } + var reports []model.ReportConfig + for _, record := range r { + var rpt model.ReportConfig + if err := record.Struct(&rpt); err != nil { + return nil, 0, err + } + reports = append(reports, rpt) + } + return reports, count, nil +} + +// ListExtractConfigs 分页获取抽取配置列表 +func (l *ConfigLoader) ListExtractConfigs(ctx context.Context, businessCode, reportCode string, pageNum, pageSize int) ([]model.ExtractConfig, int, error) { + total, err := gfdb.DB(ctx).GetAll(ctx, + "SELECT COUNT(*) AS cnt FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL", + businessCode, reportCode) + if err != nil { + return nil, 0, err + } + count := 0 + if !total.IsEmpty() { + count = total[0]["cnt"].Int() + } + offset := (pageNum - 1) * pageSize + r, err := gfdb.DB(ctx).GetAll(ctx, + "SELECT * FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY id ASC LIMIT $3 OFFSET $4", + businessCode, reportCode, pageSize, offset) + if err != nil { + return nil, 0, err + } + var configs []model.ExtractConfig + for _, record := range r { + var ec model.ExtractConfig + if err := record.Struct(&ec); err != nil { + return nil, 0, err + } + configs = append(configs, ec) + } + l.mu.Lock() + l.extractCache[businessCode+":"+reportCode] = configs + l.mu.Unlock() + return configs, count, nil +} + // GetReportFields 获取报表可用字段(按角色分类) func (l *ConfigLoader) GetReportFields(ctx context.Context, businessCode, reportCode string) (*model.GetReportFieldsResp, error) { fields, err := l.GetFields(ctx, businessCode, reportCode) diff --git a/common/report/cron.go b/common/report/cron.go new file mode 100644 index 0000000..a364f3b --- /dev/null +++ b/common/report/cron.go @@ -0,0 +1,66 @@ +package report + +import ( + "context" + "time" + + "github.com/gogf/gf/v2/os/gcron" + "github.com/gogf/gf/v2/os/gctx" + "github.com/sirupsen/logrus" +) + +// StartDailyExtractJob 启动每日自动抽取定时任务 +// 每天凌晨 2:00 遍历所有已启用的业务+报表，抽取前一天数据 +// 在 main.go 中调用：report.StartDailyExtractJob() +func StartDailyExtractJob() { + svc := GetService() + + // 每天凌晨 2:00 执行 + _, err := gcron.Add(gctx.New(), "0 0 2 * * *", func(ctx context.Context) { + logrus.Info("[报表引擎] 开始每日自动抽取任务") + + // 获取所有启用业务 + businesses, err := svc.GetAllBusinesses(ctx) + if err != nil { + logrus.Errorf("[报表引擎] 获取业务列表失败: %v", err) + return + } + + yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02") + + for _, biz := range businesses { + // 获取业务下所有启用报表 + reports, err := svc.GetAllReports(ctx, biz.BusinessCode) + if err != nil { + logrus.Errorf("[报表引擎] 获取报表列表失败 [%s]: %v", biz.BusinessCode, err) + continue + } + + for _, rpt := range reports { + logrus.Infof("[报表引擎] 开始抽取 %s/%s 日期 %s", biz.BusinessCode, rpt.ReportCode, yesterday) + + resp, err := svc.ExtractDailyData(ctx, biz.BusinessCode, rpt.ReportCode, yesterday, "system") + if err != nil { + logrus.Errorf("[报表引擎] 抽取失败 %s/%s %s: %v", biz.BusinessCode, rpt.ReportCode, yesterday, err) + continue + } + + if resp.Success { + logrus.Infof("[报表引擎] 抽取成功 %s/%s %s 总数:%d 耗时:%dms", + biz.BusinessCode, rpt.ReportCode, yesterday, resp.TotalCount, resp.ExecTimeMs) + } else { + logrus.Errorf("[报表引擎] 抽取异常 %s/%s %s: %s", biz.BusinessCode, rpt.ReportCode, yesterday, resp.ErrorMsg) + } + } + } + + logrus.Info("[报表引擎] 每日自动抽取任务完成") + }, "daily-report-extract") + + if err != nil { + logrus.Errorf("[报表引擎] 启动定时任务失败: %v", err) + return + } + + logrus.Info("[报表引擎] 每日自动抽取定时任务已启动 (02:00)") +} diff --git a/common/report/ddlsync/creator.go b/common/report/ddlsync/creator.go index c6a0ab9..e2b00a3 100644 --- a/common/report/ddlsync/creator.go +++ b/common/report/ddlsync/creator.go @@ -115,8 +115,10 @@ func (c *StatTableCreator) buildCreateTableSQL(tableName string, report *model.R // 业务字段 for _, f := range fields { - colDef := c.fieldTypeToColumn(f.FieldCode, f.FieldType) - cols = append(cols, colDef) + if f.FieldCode == dateField { + continue + } + cols = append(cols, fmt.Sprintf("%s %s", f.FieldCode, model.FieldTypeToPG(f.FieldType))) } // 原始数据 @@ -127,22 +129,6 @@ func (c *StatTableCreator) buildCreateTableSQL(tableName string, report *model.R return sql } -// fieldTypeToColumn 字段类型转PG列类型 -func (c *StatTableCreator) fieldTypeToColumn(fieldCode, fieldType string) string { - switch fieldType { - case model.FieldTypeInt, model.FieldTypeFloat: - return fmt.Sprintf("%s NUMERIC(20,4) DEFAULT 0", fieldCode) - case model.FieldTypeDate: - return fmt.Sprintf("%s VARCHAR(16) DEFAULT ''", fieldCode) - case model.FieldTypeDatetime: - return fmt.Sprintf("%s TIMESTAMP WITH TIME ZONE", fieldCode) - case model.FieldTypeJsonb: - return fmt.Sprintf("%s JSONB DEFAULT '{}'", fieldCode) - default: // STRING - return fmt.Sprintf("%s VARCHAR(256) DEFAULT ''", fieldCode) - } -} - // DropStatTable 删除统计宽表 func (c *StatTableCreator) DropStatTable(ctx context.Context, businessCode, reportCode string) error { report, err := c.loader.GetReport(ctx, businessCode, reportCode) @@ -200,8 +186,7 @@ func (c *StatTableCreator) AlterTableAddColumns(ctx context.Context, tableName s if existingMap[f.FieldCode] { continue } - colDef := c.fieldTypeToColumn(f.FieldCode, f.FieldType) - sql := fmt.Sprintf("ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s", tableName, colDef) + sql := fmt.Sprintf("ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s %s", tableName, f.FieldCode, model.FieldTypeToPG(f.FieldType)) if _, err := gfdb.DB(ctx).Exec(ctx, sql); err != nil { return fmt.Errorf("添加字段 %s 失败: %w", f.FieldCode, err) } diff --git a/common/report/executor/executor.go b/common/report/executor/executor.go index 78320d5..4b56564 100644 --- a/common/report/executor/executor.go +++ b/common/report/executor/executor.go @@ -169,6 +169,7 @@ func (e *QueryExecutor) validateReq(req *model.UserSelectQueryReq) error { "week": true, "month": true, "quarter": true, + "year": true, } if !validGroups[req.TimeGroup] { return fmt.Errorf("不支持的时间分组: %s", req.TimeGroup) diff --git a/common/report/extract/extract.go b/common/report/extract/extract.go index aad19de..4d76618 100644 --- a/common/report/extract/extract.go +++ b/common/report/extract/extract.go @@ -3,6 +3,7 @@ package extract import ( "context" "fmt" + "sort" "strings" "time" @@ -91,7 +92,9 @@ func (e *DailyExtractor) ExtractDailyData(ctx context.Context, businessCode, rep Executor: executor, StartTime: &start, } - _ = e.loader.CreateExtractLog(ctx, extractLog) + if err := e.loader.CreateExtractLog(ctx, extractLog); err != nil { + logger.Errorf("创建抽取日志失败: %v", err) + } // 执行抽取 c, s, f, err := e.executeExtract(ctx, &ec, report, fieldMap, statDate) @@ -476,7 +479,7 @@ func (e *DailyExtractor) applyTransformRules(ec *model.ExtractConfig, row map[st } } -// ensureStatTableExists 确保统计宽表存在 +// ensureStatTableExists 确保统计宽表存在，如有新增字段自动加列 func (e *DailyExtractor) ensureStatTableExists(ctx context.Context, report *model.ReportConfig, fieldMap map[string]*model.FieldConfig) error { tableName := report.StatTableName @@ -495,10 +498,48 @@ func (e *DailyExtractor) ensureStatTableExists(ctx context.Context, report *mode return e.createStatTable(ctx, report, fieldMap) } + // 表已存在，检查并添加新增字段 + if err := e.alterTableAddColumns(ctx, tableName, fieldMap); err != nil { + logrus.Warnf("自动加列失败 %s: %v", tableName, err) + // 不返回 error，加列失败不应阻塞抽取 + } + logrus.Infof("统计宽表 %s 已存在", tableName) return nil } +// alterTableAddColumns 为已存在的统计宽表添加新增字段 +func (e *DailyExtractor) alterTableAddColumns(ctx context.Context, tableName string, fieldMap map[string]*model.FieldConfig) error { + // 获取已有列 + rows, err := gfdb.DB(ctx).GetAll(ctx, + "SELECT column_name FROM information_schema.columns WHERE table_name = $1 AND table_schema = 'public'", + strings.ToLower(tableName)) + if err != nil { + return fmt.Errorf("查询表结构失败: %w", err) + } + + existingCols := make(map[string]bool) + for _, row := range rows.List() { + if col, ok := row["column_name"].(string); ok { + existingCols[col] = true + } + } + + for _, fc := range fieldMap { + if existingCols[fc.FieldCode] { + continue + } + colType := model.FieldTypeToPG(fc.FieldType) + sql := fmt.Sprintf("ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s %s", tableName, fc.FieldCode, colType) + if _, err := gfdb.DB(ctx).Exec(ctx, sql); err != nil { + return fmt.Errorf("添加字段 %s 失败: %w", fc.FieldCode, err) + } + logrus.Infof("统计宽表 %s 自动添加字段 %s %s", tableName, fc.FieldCode, colType) + } + + return nil +} + // createStatTable 创建统计宽表 func (e *DailyExtractor) createStatTable(ctx context.Context, report *model.ReportConfig, fieldMap map[string]*model.FieldConfig) error { var cols []string @@ -523,7 +564,10 @@ func (e *DailyExtractor) createStatTable(ctx context.Context, report *model.Repo // 业务字段 for _, fc := range fieldMap { fc := fc - colType := fieldTypeToPG(fc.FieldType) + if fc.FieldCode == dateField { + continue + } + colType := model.FieldTypeToPG(fc.FieldType) cols = append(cols, fmt.Sprintf("%s %s", fc.FieldCode, colType)) } @@ -564,23 +608,43 @@ func (e *DailyExtractor) createStatTable(ctx context.Context, report *model.Repo return nil } -// batchUpsert 批量upsert写入 +// batchUpsert 批量写入统计宽表（使用原始SQL INSERT，避免ORM兼容问题） func (e *DailyExtractor) batchUpsert(ctx context.Context, tableName string, conflictKeys []string, rows []map[string]interface{}) (int, []string, error) { if len(rows) == 0 { return 0, nil, nil } - now := time.Now() - for i := range rows { - if rows[i] == nil { - rows[i] = make(map[string]interface{}) + // 去除 id 和 created_at 列（数据库自增/默认值） + var keys []string + for k := range rows[0] { + if k == "id" || k == "created_at" || k == "updated_at" || k == "deleted_at" { + continue } - rows[i]["updated_at"] = now + keys = append(keys, k) + } + sort.Strings(keys) + + cols := strings.Join(keys, ", ") + placeholders := make([]string, len(keys)) + for i := range keys { + placeholders[i] = fmt.Sprintf("$%d", i+1) + } + phStr := strings.Join(placeholders, ", ") + + sqlStr := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", tableName, cols, phStr) + if len(conflictKeys) > 0 { + var updateParts []string + for _, k := range keys { + if k == "id" || k == "created_at" { + continue + } + updateParts = append(updateParts, fmt.Sprintf("%s = EXCLUDED.%s", k, k)) + } + sqlStr += " ON CONFLICT (" + strings.Join(conflictKeys, ", ") + ") DO UPDATE SET " + strings.Join(updateParts, ", ") } batchSize := 100 total := 0 - var allColumns []string for i := 0; i < len(rows); i += batchSize { end := i + batchSize @@ -589,56 +653,18 @@ func (e *DailyExtractor) batchUpsert(ctx context.Context, tableName string, conf } batch := rows[i:end] - m := gfdb.DB(ctx).Model(ctx, tableName).Data(batch) - if len(conflictKeys) > 0 { - keys := make([]interface{}, len(conflictKeys)) - for j, k := range conflictKeys { - keys[j] = k + for _, row := range batch { + var args []interface{} + for _, k := range keys { + args = append(args, row[k]) } - m = m.OnConflict(keys...) - } - - _, err := m.Save() - if err != nil { - logrus.Errorf("批量写入 %s 失败: %v", tableName, err) - // 逐条重试 - for _, row := range batch { - mm := gfdb.DB(ctx).Model(ctx, tableName).Data(row) - if len(conflictKeys) > 0 { - keys := make([]interface{}, len(conflictKeys)) - for j, k := range conflictKeys { - keys[j] = k - } - mm = mm.OnConflict(keys...) - } - if _, e := mm.Save(); e != nil { - logrus.Errorf("逐条写入失败: %v", e) - } else { - total++ - } + if _, err := gfdb.DB(ctx).Exec(ctx, sqlStr, args...); err != nil { + logrus.Errorf("逐行写入 %s 失败: %v", tableName, err) + } else { + total++ } - } else { - total += len(batch) } } - return total, allColumns, nil -} - -// fieldTypeToPG 字段类型转PG类型 -func fieldTypeToPG(fieldType string) string { - switch fieldType { - case model.FieldTypeInt: - return "NUMERIC(20,0) DEFAULT 0" - case model.FieldTypeFloat: - return "NUMERIC(20,4) DEFAULT 0" - case model.FieldTypeDate: - return "VARCHAR(16) DEFAULT ''" - case model.FieldTypeDatetime: - return "TIMESTAMP WITH TIME ZONE" - case model.FieldTypeJsonb: - return "JSONB DEFAULT '{}'" - default: - return "VARCHAR(256) DEFAULT ''" - } + return total, nil, nil } diff --git a/common/report/model/model.go b/common/report/model/model.go index 75d47f9..ecc6501 100644 --- a/common/report/model/model.go +++ b/common/report/model/model.go @@ -13,99 +13,101 @@ import ( // BusinessConfig 业务配置 type BusinessConfig struct { beans.SQLBaseDO `orm:",inherit"` - BusinessCode string `orm:"business_code" json:"businessCode"` - BusinessName string `orm:"business_name" json:"businessName"` - Description string `orm:"description" json:"description"` - Status string `orm:"status" json:"status"` - Config map[string]interface{} `orm:"config" json:"config"` + BusinessCode string `orm:"business_code" json:"businessCode" dc:"业务编码" eg:"KUAISHOU"` + BusinessName string `orm:"business_name" json:"businessName" dc:"业务名称" eg:"快手电商"` + Description string `orm:"description" json:"description" dc:"描述" eg:"快手电商店铺每日统计报表"` + Status string `orm:"status" json:"status" dc:"status" eg:"ACTIVE"` + Config map[string]interface{} `orm:"config" json:"config" dc:"扩展配置"` } // ReportConfig 报表配置 type ReportConfig struct { beans.SQLBaseDO `orm:",inherit"` - BusinessCode string `orm:"business_code" json:"businessCode"` - ReportCode string `orm:"report_code" json:"reportCode"` - ReportName string `orm:"report_name" json:"reportName"` - Description string `orm:"description" json:"description"` - Status string `orm:"status" json:"status"` - StatTableName string `orm:"stat_table_name" json:"statTableName"` - StatTableComment string `orm:"stat_table_comment" json:"statTableComment"` - DateField string `orm:"date_field" json:"dateField"` - PrimaryKeys []string `orm:"primary_keys" json:"primaryKeys"` - ConflictKeys []string `orm:"conflict_keys" json:"conflictKeys"` - Config map[string]interface{} `orm:"config" json:"config"` + BusinessCode string `orm:"business_code" json:"businessCode" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `orm:"report_code" json:"reportCode" dc:"报表编码" eg:"shop_daily"` + ReportName string `orm:"report_name" json:"reportName" dc:"报表名称" eg:"店铺日报"` + Description string `orm:"description" json:"description" dc:"描述" eg:"快手电商店铺每日统计报表"` + Status string `orm:"status" json:"status" dc:"状态 ACTIVE/INACTIVE" eg:"ACTIVE"` + StatTableName string `orm:"stat_table_name" json:"statTableName" dc:"统计宽表名" eg:"stat_kuaishou_shop_daily"` + StatTableComment string `orm:"stat_table_comment" json:"statTableComment" dc:"统计宽表注释" eg:"快手电商-店铺日报"` + DateField string `orm:"date_field" json:"dateField" dc:"日期字段名" eg:"stat_date"` + PrimaryKeys []string `orm:"primary_keys" json:"primaryKeys" dc:"主键字段列表"` + ConflictKeys []string `orm:"conflict_keys" json:"conflictKeys" dc:"冲突键（唯一索引）"` + Config map[string]interface{} `orm:"config" json:"config" dc:"扩展配置"` + ExtractConfigIDs string `orm:"-" json:"extractConfigIds" dc:"抽取配置ID列表，逗号分隔" eg:"1,2,3"` + Fields []FieldConfig `orm:"-" json:"fields,omitempty" dc:"字段配置列表（查询时返回）"` } // FieldConfig 字段配置 type FieldConfig struct { beans.SQLBaseDO `orm:",inherit"` - BusinessCode string `orm:"business_code" json:"businessCode"` - ReportCode string `orm:"report_code" json:"reportCode"` - FieldCode string `orm:"field_code" json:"fieldCode"` - FieldName string `orm:"field_name" json:"fieldName"` - FieldType string `orm:"field_type" json:"fieldType"` - DataType string `orm:"data_type" json:"dataType"` - FieldRole string `orm:"field_role" json:"fieldRole"` - IsAggregatable bool `orm:"is_aggregatable" json:"isAggregatable"` - IsFilterable bool `orm:"is_filterable" json:"isFilterable"` - IsQueryable bool `orm:"is_queryable" json:"isQueryable"` - IsSortable bool `orm:"is_sortable" json:"isSortable"` - DefaultAggregate string `orm:"default_aggregate" json:"defaultAggregate"` - ValidAggregates []string `orm:"valid_aggregates" json:"validAggregates"` - FilterOperators []string `orm:"filter_operators" json:"filterOperators"` - Expression string `orm:"expression" json:"expression"` - ExpressionType string `orm:"expression_type" json:"expressionType"` - FormatPattern string `orm:"format_pattern" json:"formatPattern"` - Unit string `orm:"unit" json:"unit"` - DictCode string `orm:"dict_code" json:"dictCode"` - SortOrder int `orm:"sort_order" json:"sortOrder"` - GroupName string `orm:"group_name" json:"groupName"` - Status string `orm:"status" json:"status"` + BusinessCode string `orm:"business_code" json:"businessCode" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `orm:"report_code" json:"reportCode" dc:"reportCode" eg:"shop_daily"` + FieldCode string `orm:"field_code" json:"fieldCode" dc:"字段编码" eg:"order_amount"` + FieldName string `orm:"field_name" json:"fieldName" dc:"字段名称" eg:"订单金额"` + FieldType string `orm:"field_type" json:"fieldType" dc:"字段类型 STRING/INT/FLOAT/DATE/DATETIME/JSONB" eg:"FLOAT"` + DataType string `orm:"data_type" json:"dataType" dc:"数据存储类型" eg:"STRING"` + FieldRole string `orm:"field_role" json:"fieldRole" dc:"字段角色 DIMENSION(维度)/INDICATOR(指标)/FILTER(筛选)/FILTER_ONLY(仅筛选)" eg:"DIMENSION"` + IsAggregatable bool `orm:"is_aggregatable" json:"isAggregatable" dc:"是否可聚合" eg:"true"` + IsFilterable bool `orm:"is_filterable" json:"isFilterable" dc:"是否可筛选" eg:"true"` + IsQueryable bool `orm:"is_queryable" json:"isQueryable" dc:"是否可查询" eg:"true"` + IsSortable bool `orm:"is_sortable" json:"isSortable" dc:"是否可排序" eg:"true"` + DefaultAggregate string `orm:"default_aggregate" json:"defaultAggregate" dc:"默认聚合方式 SUM/COUNT/AVG/MAX/MIN" eg:"SUM"` + ValidAggregates []string `orm:"valid_aggregates" json:"validAggregates" dc:"可选聚合列表"` + FilterOperators []string `orm:"filter_operators" json:"filterOperators" dc:"可选操作符列表 =/!=/>/=/<=/IN/LIKE/BETWEEN"` + Expression string `orm:"expression" json:"expression" dc:"表达式（衍生字段）" eg:"order_amount - refund_amount"` + ExpressionType string `orm:"expression_type" json:"expressionType" dc:"表达式类型 DIRECT/CALCULATED" eg:"DIRECT"` + FormatPattern string `orm:"format_pattern" json:"formatPattern" dc:"格式化模板" eg:"#,##0.00"` + Unit string `orm:"unit" json:"unit" dc:"单位"` + DictCode string `orm:"dict_code" json:"dictCode" dc:"字典编码" eg:"dict_order_type"` + SortOrder int `orm:"sort_order" json:"sortOrder" dc:"排序号" eg:"1"` + GroupName string `orm:"group_name" json:"groupName" dc:"分组名称" eg:"交易指标"` + Status string `orm:"status" json:"status" dc:"status" eg:"ACTIVE"` } // ExtractConfig 抽取配置 type ExtractConfig struct { beans.SQLBaseDO `orm:",inherit"` - BusinessCode string `orm:"business_code" json:"businessCode"` - ReportCode string `orm:"report_code" json:"reportCode"` - ExtractCode string `orm:"extract_code" json:"extractCode"` - ExtractName string `orm:"extract_name" json:"extractName"` - SourceTableName string `orm:"source_table_name" json:"sourceTableName"` - SourceTableAlias string `orm:"source_table_alias" json:"sourceTableAlias"` - TargetTableName string `orm:"target_table_name" json:"targetTableName"` - IsEnabled bool `orm:"is_enabled" json:"isEnabled"` - ExtractType string `orm:"extract_type" json:"extractType"` - ExtractMode string `orm:"extract_mode" json:"extractMode"` - ExtractKeyField string `orm:"extract_key_field" json:"extractKeyField"` - ExtractKeyFormat string `orm:"extract_key_format" json:"extractKeyFormat"` - GroupByFields []string `orm:"group_by_fields" json:"groupByFields"` - FilterExpression string `orm:"filter_expression" json:"filterExpression"` - JoinConfigs []JoinConfig `orm:"join_configs" json:"joinConfigs"` - FieldMappings []FieldMapping `orm:"field_mappings" json:"fieldMappings"` - TransformRules []TransformRule `orm:"transform_rules" json:"transformRules"` - BatchSize int `orm:"batch_size" json:"batchSize"` - Status string `orm:"status" json:"status"` + BusinessCode string `orm:"business_code" json:"businessCode" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `orm:"report_code" json:"reportCode" dc:"reportCode" eg:"shop_daily"` + ExtractCode string `orm:"extract_code" json:"extractCode" dc:"抽取编码" eg:"kuaishou_order_extract"` + ExtractName string `orm:"extract_name" json:"extractName" dc:"抽取名称" eg:"订单抽取"` + SourceTableName string `orm:"source_table_name" json:"sourceTableName" dc:"源表名" eg:"ods_kuaishou_order"` + SourceTableAlias string `orm:"source_table_alias" json:"sourceTableAlias" dc:"源表别名" eg:"o"` + TargetTableName string `orm:"target_table_name" json:"targetTableName" dc:"目标表名" eg:"stat_kuaishou_shop_daily"` + IsEnabled bool `orm:"is_enabled" json:"isEnabled" dc:"是否启用" eg:"true"` + ExtractType string `orm:"extract_type" json:"extractType" dc:"抽取类型 FULL/INCREMENTAL" eg:"INCREMENTAL"` + ExtractMode string `orm:"extract_mode" json:"extractMode" dc:"抽取模式 DIRECT(逐行)/AGGREGATE(聚合)" eg:"DIRECT"` + ExtractKeyField string `orm:"extract_key_field" json:"extractKeyField" dc:"抽取关键字段（增量依据）" eg:"updated_at"` + ExtractKeyFormat string `orm:"extract_key_format" json:"extractKeyFormat" dc:"关键字段格式" eg:"yyyy-MM-dd HH:mm:ss"` + GroupByFields []string `orm:"group_by_fields" json:"groupByFields" dc:"GROUP BY 字段列表"` + FilterExpression string `orm:"filter_expression" json:"filterExpression" dc:"过滤表达式" eg:"status = 'PAID'"` + JoinConfigs []JoinConfig `orm:"join_configs" json:"joinConfigs" dc:"JOIN配置"` + FieldMappings []FieldMapping `orm:"field_mappings" json:"fieldMappings" dc:"字段映射列表"` + TransformRules []TransformRule `orm:"transform_rules" json:"transformRules" dc:"转换规则列表"` + BatchSize int `orm:"batch_size" json:"batchSize" dc:"批处理大小" eg:"1000"` + Status string `orm:"status" json:"status" dc:"status" eg:"ACTIVE"` } // ExtractLog 抽取记录（不含 Creator/Updater/DeletedAt，表结构不同） type ExtractLog struct { - ID int64 `orm:"id" json:"id"` - TenantId uint64 `orm:"tenant_id" json:"tenantId"` - BusinessCode string `orm:"business_code" json:"businessCode"` - ReportCode string `orm:"report_code" json:"reportCode"` - ExtractCode string `orm:"extract_code" json:"extractCode"` - StatDate string `orm:"stat_date" json:"statDate"` - ExtractType string `orm:"extract_type" json:"extractType"` - Status string `orm:"status" json:"status"` - TotalCount int `orm:"total_count" json:"totalCount"` - SuccessCount int `orm:"success_count" json:"successCount"` - FailCount int `orm:"fail_count" json:"failCount"` - StartTime *time.Time `orm:"start_time" json:"startTime"` - EndTime *time.Time `orm:"end_time" json:"endTime"` - ErrorMessage string `orm:"error_message" json:"errorMessage"` - Executor string `orm:"executor" json:"executor"` - CreatedAt *time.Time `orm:"created_at" json:"createdAt"` - UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt"` + ID int64 `orm:"id" json:"id" dc:"主键ID" eg:"1"` + TenantId uint64 `orm:"tenant_id" json:"tenantId" dc:"租户ID" eg:"1"` + BusinessCode string `orm:"business_code" json:"businessCode" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `orm:"report_code" json:"reportCode" dc:"reportCode" eg:"shop_daily"` + ExtractCode string `orm:"extract_code" json:"extractCode" dc:"抽取编码" eg:"kuaishou_order_extract"` + StatDate string `orm:"stat_date" json:"statDate" dc:"统计日期" eg:"2026-06-01"` + ExtractType string `orm:"extract_type" json:"extractType" dc:"抽取类型 FULL/INCREMENTAL" eg:"INCREMENTAL"` + Status string `orm:"status" json:"status" dc:"status" eg:"ACTIVE"` + TotalCount int `orm:"total_count" json:"totalCount" dc:"总记录数" eg:"100"` + SuccessCount int `orm:"success_count" json:"successCount" dc:"成功记录数" eg:"98"` + FailCount int `orm:"fail_count" json:"failCount" dc:"失败记录数" eg:"2"` + StartTime *time.Time `orm:"start_time" json:"startTime" dc:"开始时间"` + EndTime *time.Time `orm:"end_time" json:"endTime" dc:"结束时间"` + ErrorMessage string `orm:"error_message" json:"errorMessage" dc:"错误信息"` + Executor string `orm:"executor" json:"executor" dc:"执行人" eg:"system"` + CreatedAt *time.Time `orm:"created_at" json:"createdAt" dc:"创建时间"` + UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt" dc:"更新时间"` } // ============================================================ @@ -114,30 +116,30 @@ type ExtractLog struct { // JoinConfig 关联配置 type JoinConfig struct { - JoinTable string `json:"joinTable"` - JoinAlias string `json:"joinAlias"` - JoinType string `json:"joinType"` // LEFT/RIGHT/INNER - JoinCondition string `json:"joinCondition"` - FieldMappings []FieldMapping `json:"fieldMappings"` + JoinTable string `json:"joinTable" dc:"关联表名" eg:"ods_kuaishou_shop"` + JoinAlias string `json:"joinAlias" dc:"关联表别名" eg:"s"` + JoinType string `json:"joinType" dc:"LEFT/RIGHT/INNER" eg:"LEFT"` + JoinCondition string `json:"joinCondition" dc:"关联条件" eg:"o.shop_id = s.shop_id"` + FieldMappings []FieldMapping `json:"fieldMappings" dc:"字段映射列表"` } // FieldMapping 字段映射 type FieldMapping struct { - SourceField string `json:"sourceField"` - TargetField string `json:"targetField"` - FieldType string `json:"fieldType"` - AggregateFunction string `json:"aggregateFunction"` - DefaultValue interface{} `json:"defaultValue"` - TransformRule *TransformRule `json:"transformRule,omitempty"` + SourceField string `json:"sourceField" dc:"源字段"` + TargetField string `json:"targetField" dc:"目标字段"` + FieldType string `json:"fieldType" dc:"字段类型 STRING/INT/FLOAT/DATE/DATETIME/JSONB" eg:"FLOAT"` + AggregateFunction string `json:"aggregateFunction" dc:"聚合函数 SUM/COUNT/AVG/MAX/MIN"` + DefaultValue interface{} `json:"defaultValue" dc:"默认值"` + TransformRule *TransformRule `json:"transformRule,omitempty" dc:"transformRule"` } // TransformRule 转换规则 type TransformRule struct { - RuleCode string `json:"ruleCode"` - RuleType string `json:"ruleType"` // DIRECT/MAPPING/FORMAT/CALCULATE - Expression string `json:"expression"` - Format string `json:"format"` - Mapping map[string]interface{} `json:"mapping"` + RuleCode string `json:"ruleCode" dc:"规则编码" eg:"AMOUNT_CONVERT"` + RuleType string `json:"ruleType" dc:"DIRECT/MAPPING/FORMAT/CALCULATE" eg:"FORMAT"` + Expression string `json:"expression" dc:"表达式（衍生字段）" eg:"order_amount - refund_amount"` + Format string `json:"format" dc:"format" eg:"%Y-%m-%d"` + Mapping map[string]interface{} `json:"mapping" dc:"映射表（MAPPING类型时使用）"` } // ============================================================ @@ -146,205 +148,255 @@ type TransformRule struct { // UserSelectQueryReq 用户选择查询请求 type UserSelectQueryReq struct { - BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"` - ReportCode string `json:"reportCode" v:"required" dc:"报表编码"` + BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"` Dimensions []string `json:"dimensions" dc:"统计维度列表，如 shop_id/anchor_id/date"` Indicators []IndicatorSelect `json:"indicators" dc:"统计指标列表(含聚合方式)"` Filters []FilterCondition `json:"filters" dc:"筛选条件列表"` TimeRange *TimeRange `json:"timeRange" dc:"时间范围"` - TimeGroup string `json:"timeGroup" dc:"时间分组: day/week/month/quarter"` + TimeGroup string `json:"timeGroup" dc:"时间分组: day/week/month/quarter" eg:"day"` OrderBy []OrderCondition `json:"orderBy" dc:"排序条件"` - Page int `json:"page" dc:"页码" d:"1"` - PageSize int `json:"pageSize" dc:"每页条数" d:"20"` + Page int `json:"page" dc:"页码" d:"1" eg:"1"` + PageSize int `json:"pageSize" dc:"每页条数" d:"20" eg:"20"` } // IndicatorSelect 指标选择 type IndicatorSelect struct { - FieldCode string `json:"fieldCode" dc:"字段编码"` - Aggregate string `json:"aggregate" dc:"聚合方式: SUM/COUNT/AVG/MAX/MIN"` - Alias string `json:"alias" dc:"别名"` + FieldCode string `json:"fieldCode" dc:"字段编码" eg:"order_amount"` + Aggregate string `json:"aggregate" dc:"聚合方式: SUM/COUNT/AVG/MAX/MIN" eg:"SUM"` + Alias string `json:"alias" dc:"别名" eg:"总订单金额"` } // FilterCondition 筛选条件 type FilterCondition struct { - FieldCode string `json:"fieldCode" dc:"字段编码"` - Operator string `json:"operator" dc:"操作符: =/!=/>/=/<=/IN/LIKE/BETWEEN"` + FieldCode string `json:"fieldCode" dc:"字段编码" eg:"order_amount"` + Operator string `json:"operator" dc:"操作符: =/!=/>/=/<=/IN/LIKE/BETWEEN" eg:"="` Value interface{} `json:"value" dc:"值"` Value2 interface{} `json:"value2" dc:"第二个值(BETWEEN时使用)"` } // TimeRange 时间范围 type TimeRange struct { - StartDate string `json:"startDate" dc:"开始日期 yyyy-MM-dd"` - EndDate string `json:"endDate" dc:"结束日期 yyyy-MM-dd"` + StartDate string `json:"startDate" dc:"开始日期 yyyy-MM-dd" eg:"2026-06-01"` + EndDate string `json:"endDate" dc:"结束日期 yyyy-MM-dd" eg:"2026-06-30"` } // OrderCondition 排序条件 type OrderCondition struct { - FieldCode string `json:"fieldCode" dc:"字段编码"` - Direction string `json:"direction" dc:"排序方向: ASC/DESC"` + FieldCode string `json:"fieldCode" dc:"字段编码" eg:"order_amount"` + Direction string `json:"direction" dc:"排序方向: ASC/DESC" eg:"DESC"` } // UserSelectQueryResp 用户选择查询响应 type UserSelectQueryResp struct { List []map[string]interface{} `json:"list" dc:"数据列表"` - Total int64 `json:"total" dc:"总数"` - Page int `json:"page" dc:"当前页"` - PageSize int `json:"pageSize" dc:"每页条数"` - TotalPages int `json:"totalPages" dc:"总页数"` - Sql string `json:"sql,omitempty" dc:"执行的SQL(调试用)"` - ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)"` + Total int64 `json:"total" dc:"总数" eg:"100"` + Page int `json:"page" dc:"当前页" eg:"1"` + PageSize int `json:"pageSize" dc:"每页条数" eg:"20"` + TotalPages int `json:"totalPages" dc:"总页数" eg:"5"` + Sql string `json:"sql,omitempty" dc:"执行的SQL(调试用)" eg:"SELECT 1"` + ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)" eg:"1234"` } // ExtractDailyDataReq 按天抽取数据请求 type ExtractDailyDataReq struct { - BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"` - ReportCode string `json:"reportCode" v:"required" dc:"报表编码"` - StatDate string `json:"statDate" v:"required" dc:"统计日期 yyyy-MM-dd"` - Executor string `json:"executor" dc:"执行人"` + BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"` + StatDate string `json:"statDate" v:"required" dc:"统计日期 yyyy-MM-dd" eg:"2026-06-01"` + Executor string `json:"executor" dc:"执行人" eg:"system"` } // ExtractDailyDataResp 按天抽取数据响应 type ExtractDailyDataResp struct { - Success bool `json:"success" dc:"是否成功"` - TotalCount int `json:"totalCount" dc:"总记录数"` - SuccessCount int `json:"successCount" dc:"成功记录数"` - FailCount int `json:"failCount" dc:"失败记录数"` - ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)"` + Success bool `json:"success" dc:"是否成功" eg:"true"` + TotalCount int `json:"totalCount" dc:"总记录数" eg:"100"` + SuccessCount int `json:"successCount" dc:"成功记录数" eg:"98"` + FailCount int `json:"failCount" dc:"失败记录数" eg:"2"` + ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)" eg:"1234"` ErrorMsg string `json:"errorMsg" dc:"错误信息"` } // AutoCreateStatTableReq 自动创建统计宽表请求 type AutoCreateStatTableReq struct { - BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"` - ReportCode string `json:"reportCode" v:"required" dc:"报表编码"` - Creator string `json:"creator" dc:"创建人"` + BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"` + Creator string `json:"creator" dc:"创建人" eg:"admin"` } // AutoCreateStatTableResp 自动创建统计宽表响应 type AutoCreateStatTableResp struct { - Success bool `json:"success" dc:"是否成功"` - TableName string `json:"tableName" dc:"创建的表名"` - ColumnCount int `json:"columnCount" dc:"字段数量"` - ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)"` + Success bool `json:"success" dc:"是否成功" eg:"true"` + TableName string `json:"tableName" dc:"创建的表名" eg:"stat_kuaishou_shop_daily"` + ColumnCount int `json:"columnCount" dc:"字段数量" eg:"15"` + ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)" eg:"1234"` } // GetReportFieldsResp 获取报表可用字段响应 type GetReportFieldsResp struct { - BusinessCode string `json:"businessCode" dc:"业务编码"` - ReportCode string `json:"reportCode" dc:"报表编码"` + BusinessCode string `json:"businessCode" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `json:"reportCode" dc:"报表编码" eg:"shop_daily"` Dimensions []FieldConfig `json:"dimensions" dc:"维度字段列表"` Indicators []FieldConfig `json:"indicators" dc:"指标字段列表"` Filters []FieldConfig `json:"filters" dc:"筛选字段列表"` } +// BackfillReq 批量回填请求 +type BackfillReq struct { + BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"` + StartDate string `json:"startDate" v:"required" dc:"开始日期 yyyy-MM-dd" eg:"2026-06-01"` + EndDate string `json:"endDate" v:"required" dc:"结束日期 yyyy-MM-dd" eg:"2026-06-30"` + Executor string `json:"executor" dc:"执行人" eg:"admin"` +} + +// BackfillResp 批量回填响应 +type BackfillResp struct { + Success bool `json:"success" dc:"是否全部成功" eg:"true"` + TotalDays int `json:"totalDays" dc:"总天数" eg:"30"` + SuccessDays int `json:"successDays" dc:"成功天数" eg:"28"` + FailDays int `json:"failDays" dc:"失败天数" eg:"2"` + ExecTimeMs int64 `json:"execTimeMs" dc:"总耗时(毫秒)" eg:"5678"` + ErrorMsg string `json:"errorMsg" dc:"错误信息"` +} + // ============================================================ // 配置 CRUD 请求/响应 // ============================================================ // SaveBusinessReq 保存业务配置请求（新增/修改合一） type SaveBusinessReq struct { - ID *int64 `json:"id"` // 有值为更新，无值为新增 - BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"` - BusinessName string `json:"businessName" v:"required" dc:"业务名称"` - Description string `json:"description" dc:"描述"` - Status string `json:"status" dc:"状态 ACTIVE/INACTIVE" d:"ACTIVE"` + ID *int64 `json:"id" dc:"有值为更新，无值为新增" eg:"1"` + BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"` + BusinessName string `json:"businessName" v:"required" dc:"业务名称" eg:"快手电商"` + Description string `json:"description" dc:"描述" eg:"快手电商店铺每日统计报表"` + Status string `json:"status" dc:"状态 ACTIVE/INACTIVE" d:"ACTIVE" eg:"ACTIVE"` Config map[string]interface{} `json:"config" dc:"扩展配置"` - Operator string `json:"operator" dc:"操作人"` + Operator string `json:"operator" dc:"操作人" eg:"="` } // SaveReportReq 保存报表配置请求 type SaveReportReq struct { - ID *int64 `json:"id"` - BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"` - ReportCode string `json:"reportCode" v:"required" dc:"报表编码"` - ReportName string `json:"reportName" v:"required" dc:"报表名称"` - Description string `json:"description" dc:"描述"` - Status string `json:"status" dc:"状态" d:"ACTIVE"` + ID *int64 `json:"id" dc:"主键ID" eg:"1"` + BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"` + ReportName string `json:"reportName" v:"required" dc:"报表名称" eg:"店铺日报"` + Description string `json:"description" dc:"描述" eg:"快手电商店铺每日统计报表"` + Status string `json:"status" dc:"状态" d:"ACTIVE" eg:"ACTIVE"` StatTableName string `json:"statTableName" v:"required" dc:"统计宽表名"` StatTableComment string `json:"statTableComment" dc:"统计宽表注释"` - DateField string `json:"dateField" dc:"日期字段" d:"stat_date"` + DateField string `json:"dateField" dc:"日期字段" d:"stat_date" eg:"stat_date"` PrimaryKeys []string `json:"primaryKeys" dc:"主键字段"` ConflictKeys []string `json:"conflictKeys" dc:"冲突键（唯一索引）"` Config map[string]interface{} `json:"config" dc:"扩展配置"` - Operator string `json:"operator" dc:"操作人"` + Operator string `json:"operator" dc:"操作人" eg:"="` } // SaveFieldReq 保存字段配置请求 type SaveFieldReq struct { - ID *int64 `json:"id"` - BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"` - ReportCode string `json:"reportCode" v:"required" dc:"报表编码"` - FieldCode string `json:"fieldCode" v:"required" dc:"字段编码"` - FieldName string `json:"fieldName" v:"required" dc:"字段名称"` - FieldType string `json:"fieldType" v:"required" dc:"字段类型 STRING/INT/FLOAT/DATE/DATETIME/JSONB"` - DataType string `json:"dataType" dc:"数据存储类型" d:"STRING"` - FieldRole string `json:"fieldRole" v:"required" dc:"字段角色 DIMENSION/INDICATOR/FILTER/FILTER_ONLY"` - IsAggregatable bool `json:"isAggregatable" dc:"是否可聚合"` - IsFilterable bool `json:"isFilterable" dc:"是否可筛选" d:"true"` - IsQueryable bool `json:"isQueryable" dc:"是否可查询" d:"true"` - IsSortable bool `json:"isSortable" dc:"是否可排序" d:"true"` - DefaultAggregate string `json:"defaultAggregate" dc:"默认聚合方式"` + ID *int64 `json:"id" dc:"主键ID" eg:"1"` + BusinessCode string `json:"businessCode" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `json:"reportCode" dc:"报表编码" eg:"shop_daily"` + FieldCode string `json:"fieldCode" v:"required" dc:"字段编码" eg:"order_amount"` + FieldName string `json:"fieldName" v:"required" dc:"字段名称" eg:"订单金额"` + FieldType string `json:"fieldType" v:"required" dc:"字段类型 STRING/INT/FLOAT/DATE/DATETIME/JSONB" eg:"FLOAT"` + DataType string `json:"dataType" dc:"数据存储类型" d:"STRING" eg:"STRING"` + FieldRole string `json:"fieldRole" v:"required" dc:"字段角色 DIMENSION/INDICATOR/FILTER/FILTER_ONLY" eg:"DIMENSION"` + IsAggregatable bool `json:"isAggregatable" dc:"是否可聚合" eg:"true"` + IsFilterable bool `json:"isFilterable" dc:"是否可筛选" d:"true" eg:"true"` + IsQueryable bool `json:"isQueryable" dc:"是否可查询" d:"true" eg:"true"` + IsSortable bool `json:"isSortable" dc:"是否可排序" d:"true" eg:"true"` + DefaultAggregate string `json:"defaultAggregate" dc:"默认聚合方式" eg:"SUM"` ValidAggregates []string `json:"validAggregates" dc:"可选聚合列表"` FilterOperators []string `json:"filterOperators" dc:"可选操作符列表"` - Expression string `json:"expression" dc:"表达式（衍生字段）"` - ExpressionType string `json:"expressionType" dc:"表达式类型 DIRECT/CALCULATED"` - FormatPattern string `json:"formatPattern" dc:"格式化模板"` - Unit string `json:"unit" dc:"单位"` - DictCode string `json:"dictCode" dc:"字典编码"` - SortOrder int `json:"sortOrder" dc:"排序"` - GroupName string `json:"groupName" dc:"分组名称"` - Status string `json:"status" dc:"状态" d:"ACTIVE"` - Operator string `json:"operator" dc:"操作人"` + Expression string `json:"expression" dc:"表达式（衍生字段）" eg:"order_amount - refund_amount"` + ExpressionType string `json:"expressionType" dc:"表达式类型 DIRECT/CALCULATED" eg:"DIRECT"` + FormatPattern string `json:"formatPattern" dc:"格式化模板" eg:"#,##0.00"` + Unit string `json:"unit" dc:"单位" eg:"元"` + DictCode string `json:"dictCode" dc:"字典编码" eg:"dict_order_type"` + SortOrder int `json:"sortOrder" dc:"排序" eg:"1"` + GroupName string `json:"groupName" dc:"分组名称" eg:"交易指标"` + Status string `json:"status" dc:"状态" d:"ACTIVE" eg:"ACTIVE"` + Operator string `json:"operator" dc:"操作人" eg:"admin"` +} + +// SaveReportWithFieldsReq 保存报表及字段配置请求（字段全量替换） +type SaveReportWithFieldsReq struct { + ID *int64 `json:"id" dc:"主键ID（更新时必填，新建为空）"` + BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"` + ReportName string `json:"reportName" v:"required" dc:"报表名称" eg:"店铺日报"` + Description string `json:"description" dc:"描述" eg:"快手电商店铺每日统计报表"` + Status string `json:"status" dc:"状态" d:"ACTIVE" eg:"ACTIVE"` + StatTableName string `json:"statTableName" v:"required" dc:"统计宽表名" eg:"stat_kuaishou_shop_daily"` + StatTableComment string `json:"statTableComment" dc:"统计宽表注释" eg:"快手电商-店铺日报"` + DateField string `json:"dateField" dc:"日期字段" d:"stat_date" eg:"stat_date"` + PrimaryKeys []string `json:"primaryKeys" dc:"主键字段"` + ConflictKeys []string `json:"conflictKeys" dc:"冲突键（唯一索引）"` + Config map[string]interface{} `json:"config" dc:"扩展配置"` + Fields []SaveFieldReq `json:"fields" dc:"字段配置列表（全量替换，不在列表中的字段将被删除）"` } // SaveExtractConfigReq 保存抽取配置请求 type SaveExtractConfigReq struct { - ID *int64 `json:"id"` - BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"` - ReportCode string `json:"reportCode" v:"required" dc:"报表编码"` - ExtractCode string `json:"extractCode" v:"required" dc:"抽取编码"` - ExtractName string `json:"extractName" v:"required" dc:"抽取名称"` - SourceTableName string `json:"sourceTableName" v:"required" dc:"源表名"` - SourceTableAlias string `json:"sourceTableAlias" dc:"源表别名"` - TargetTableName string `json:"targetTableName" v:"required" dc:"目标表名"` - IsEnabled bool `json:"isEnabled" dc:"是否启用" d:"true"` - ExtractType string `json:"extractType" dc:"抽取类型 FULL/INCREMENTAL" d:"INCREMENTAL"` - ExtractMode string `json:"extractMode" dc:"抽取模式 DIRECT/AGGREGATE" d:"DIRECT"` - ExtractKeyField string `json:"extractKeyField" dc:"抽取关键字段（增量依据）"` - ExtractKeyFormat string `json:"extractKeyFormat" dc:"关键字段格式"` + ID *int64 `json:"id" dc:"主键ID" eg:"1"` + BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"` + ExtractCode string `json:"extractCode" v:"required" dc:"抽取编码" eg:"kuaishou_order_extract"` + ExtractName string `json:"extractName" v:"required" dc:"抽取名称" eg:"订单抽取"` + SourceTableName string `json:"sourceTableName" v:"required" dc:"源表名" eg:"ods_kuaishou_order"` + SourceTableAlias string `json:"sourceTableAlias" dc:"源表别名" eg:"o"` + TargetTableName string `json:"targetTableName" v:"required" dc:"目标表名" eg:"stat_kuaishou_shop_daily"` + IsEnabled bool `json:"isEnabled" dc:"是否启用" d:"true" eg:"true"` + ExtractType string `json:"extractType" dc:"抽取类型 FULL/INCREMENTAL" d:"INCREMENTAL" eg:"INCREMENTAL"` + ExtractMode string `json:"extractMode" dc:"抽取模式 DIRECT/AGGREGATE" d:"DIRECT" eg:"DIRECT"` + ExtractKeyField string `json:"extractKeyField" dc:"抽取关键字段（增量依据）" eg:"updated_at"` + ExtractKeyFormat string `json:"extractKeyFormat" dc:"关键字段格式" eg:"yyyy-MM-dd HH:mm:ss"` GroupByFields []string `json:"groupByFields" dc:"GROUP BY 字段列表"` - FilterExpression string `json:"filterExpression" dc:"过滤表达式"` + FilterExpression string `json:"filterExpression" dc:"过滤表达式" eg:"status = 'PAID'"` JoinConfigs []JoinConfig `json:"joinConfigs" dc:"JOIN配置"` FieldMappings []FieldMapping `json:"fieldMappings" dc:"字段映射列表"` TransformRules []TransformRule `json:"transformRules" dc:"转换规则列表"` - BatchSize int `json:"batchSize" dc:"批处理大小" d:"1000"` - Status string `json:"status" dc:"状态" d:"ACTIVE"` - Operator string `json:"operator" dc:"操作人"` + BatchSize int `json:"batchSize" dc:"批处理大小" d:"1000" eg:"1000"` + Status string `json:"status" dc:"状态" d:"ACTIVE" eg:"ACTIVE"` + Operator string `json:"operator" dc:"操作人" eg:"="` +} + +// PageReq 分页请求参数 +type PageReq struct { + PageNum int `json:"pageNum" d:"1" dc:"页码"` + PageSize int `json:"pageSize" d:"20" dc:"每页条数"` +} + +// PageRes 分页响应 +type PageRes struct { + Total int `json:"total" dc:"总数"` + Page int `json:"page" dc:"当前页"` + PageSize int `json:"pageSize" dc:"每页条数"` + TotalPages int `json:"totalPages" dc:"总页数"` } // IdReq 通用 ID 请求 type IdReq struct { - ID int64 `json:"id" v:"required" dc:"主键ID"` + ID int64 `json:"id" v:"required" dc:"主键ID" eg:"1"` } // SaveResult 写操作通用返回 type SaveResult struct { - Success bool `json:"success"` - ID int64 `json:"id"` - Message string `json:"message"` + Success bool `json:"success" dc:"是否成功" eg:"true"` + ID int64 `json:"id" dc:"主键ID" eg:"1"` + Message string `json:"message" dc:"结果消息" eg:"操作成功"` } // DeleteResult 删除结果 type DeleteResult struct { - Success bool `json:"success"` - Message string `json:"message"` + Success bool `json:"success" dc:"是否成功" eg:"true"` + Message string `json:"message" dc:"结果消息" eg:"操作成功"` } // GetExtractConfigsReq 获取抽取配置列表请求 type GetExtractConfigsReq struct { - BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"` - ReportCode string `json:"reportCode" v:"required" dc:"报表编码"` + BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"` + ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"` } // ============================================================ @@ -401,3 +453,21 @@ const ( ExtractStatusSuccess = "SUCCESS" ExtractStatusFailed = "FAILED" ) + +// FieldTypeToPG 字段类型转PG列类型定义 +func FieldTypeToPG(fieldType string) string { + switch fieldType { + case FieldTypeInt: + return "NUMERIC(20,0) DEFAULT 0" + case FieldTypeFloat: + return "NUMERIC(20,4) DEFAULT 0" + case FieldTypeDate: + return "VARCHAR(16) DEFAULT ''" + case FieldTypeDatetime: + return "TIMESTAMP WITH TIME ZONE" + case FieldTypeJsonb: + return "JSONB DEFAULT '{}'" + default: + return "VARCHAR(256) DEFAULT ''" + } +} diff --git a/config.yml b/config.yml index e112cab..a1fa4d9 100644 --- a/config.yml +++ b/config.yml @@ -16,7 +16,7 @@ sync: retry_count: 3 # 最大重试次数 sync_interval_minutes: 60 # 自动同步间隔（分钟） compensation_interval_seconds: 300 # 补偿调度器扫描间隔（秒） - auto_sync_enabled: true # 是否启用自动同步 + auto_sync_enabled: false # 是否启用自动同步　 sync_timeout_minutes: 120 # 单次同步超时（分钟），全量超大表可适当调大 default_lookback_days: 89 # 全量同步默认回溯天数（快手限制90天内，留1天余量）（单接口可通过 request_config.full_sync_start_time 覆盖） default_tenant_id: 1 # 自动同步使用的租户 ID（多租户部署时配置） @@ -55,7 +55,7 @@ redis: writeTimeout: "30s" #TCP的Write操作超时时间，使用时间字符串例如30s/1m/1d maxActive: 100 consul: - address: 116.204.74.41:8500 + address: 192.168.3.135:8500 #k8s: # apiServer: "https://192.168.3.37:6443" # token: "eyJhbGciOiJSUzI1NiIsImtpZCI6IlR6X0QtelVDYkZPYnpjTDlfamZzOEFjbERJb2dUMTRqMjNfYUNncGcwRW8ifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiLCJrM3MiXSwiZXhwIjoxODA3ODY3MTA1LCJpYXQiOjE3NzYzMzExMDUsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwianRpIjoiNjAwMGY0ODctZTQ4Ni00NTUxLWIyNjgtMzE1MWE1MDE5YjU4Iiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJkYXNoYm9hcmQtYWRtaW4iLCJ1aWQiOiJlZjAxY2IxNS0xNTc0LTRlYTYtODg4Ny03YjZhODY3OWFkYmIifX0sIm5iZiI6MTc3NjMzMTEwNSwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmVybmV0ZXMtZGFzaGJvYXJkOmRhc2hib2FyZC1hZG1pbiJ9.Z5MEy-dhWq6lruuR6QSI--cYRZzDoeEes78a4lLMtXa8lSLt1FZy16kNVn2pJli74RSk0kKS2F5GAatyuyGjF_yB7Tm1Tb7iCjzeUM7xRpdvXEmk4MUImlzwGeW31NXpwFrfR0oPS-9TLlbahEmsgwL0DUOC0CekSiToVbIduwEmmrB0FMPFayr5_wqGDmxdqtaAH3K-LJNCqPiAMqevsxTebrhyJSrThU-Pi7iJYm_sUd8VHKrRxrwDiGA77j4lfita_hZSdyrZe8qsrbGqomHxFWk9ZzcJJ9q7OLS13OXo-Xbcu-_TZcfDOreZhune9ctM2lmuOtmYnad47gYRcA" diff --git a/controller/report/report_admin_controller.go b/controller/report/report_admin_controller.go index 02a15e3..b41c8a2 100644 --- a/controller/report/report_admin_controller.go +++ b/controller/report/report_admin_controller.go @@ -174,6 +174,7 @@ tr:hover td { background: #fafafa; }
+
@@ -734,6 +735,22 @@ async function execExtract() { // ==================== 数据查询 ==================== let queryCtx = { dimensions: [], indicators: [], filters: [], availableFields: null }; + +async function execBackfill() { + const bc = v('extract-biz-select'); + const rc = v('extract-report-select'); + if (!bc || !rc) { toast('请先选择业务和报表', 'error'); return; } + const start = prompt('请输入开始日期 (yyyy-MM-dd)：'); + if (!start) return; + const end = prompt('请输入结束日期 (yyyy-MM-dd)：'); + if (!end) return; + if (!confirm('确定回填 ' + start + ' ~ ' + end + ' 的数据？')) return; + try { + const res = await api('POST', '/report/backfill', { businessCode: bc, reportCode: rc, startDate: start, endDate: end, executor: 'admin' }); + toast('回填完成！成功: ' + res.successDays + '/' + res.totalDays + ' 天, 耗时: ' + res.execTimeMs + 'ms' + (res.failDays > 0 ? ' 失败: ' + res.failDays + ' 天' : ''), res.success ? 'success' : 'error'); + } catch(e) { toast(e.message, 'error'); } +} + async function onQueryBizChange() { const bc = v('query-biz-select'); const rs = document.getElementById('query-report-select'); diff --git a/controller/report/report_controller.go b/controller/report/report_controller.go index 7b0d451..5b255a6 100644 --- a/controller/report/report_controller.go +++ b/controller/report/report_controller.go @@ -2,12 +2,15 @@ package report import ( "context" + "strconv" + "strings" reportSvc "dataengine/common/report" "dataengine/common/report/model" "gitea.redpowerfuture.com/red-future/common/beans" "github.com/gogf/gf/v2/frame/g" + "github.com/sirupsen/logrus" ) type report struct{} @@ -28,19 +31,23 @@ func ctxWithUser(ctx context.Context) context.Context { type listBusinessesReq struct { g.Meta `path:"/businesses" method:"get" tags:"报表引擎" summary:"业务列表"` + model.PageReq } type listBusinessesRes struct { List []model.BusinessConfig `json:"list"` + model.PageRes } func (c *report) ListBusinesses(ctx context.Context, req *listBusinessesReq) (*listBusinessesRes, error) { ctx = ctxWithUser(ctx) - list, err := svc().GetAllBusinesses(ctx) + list, total, err := svc().ListBusinesses(ctx, req.PageNum, req.PageSize) if err != nil { + logrus.Errorf("[报表引擎] 获取业务列表失败: %v", err) return nil, err } - return &listBusinessesRes{List: list}, nil + totalPages := (total + req.PageSize - 1) / req.PageSize + return &listBusinessesRes{List: list, PageRes: model.PageRes{Total: total, Page: req.PageNum, PageSize: req.PageSize, TotalPages: totalPages}}, nil } type getBusinessReq struct { @@ -56,6 +63,7 @@ func (c *report) GetBusiness(ctx context.Context, req *getBusinessReq) (*getBusi ctx = ctxWithUser(ctx) data, err := svc().GetBusiness(ctx, req.ID) if err != nil { + logrus.Errorf("[报表引擎] 获取业务失败 id=%d: %v", req.ID, err) return nil, err } return &getBusinessRes{Data: data}, nil @@ -74,6 +82,7 @@ func (c *report) SaveBusiness(ctx context.Context, req *saveBusinessReq) (*saveB ctx = ctxWithUser(ctx) result, err := svc().SaveBusiness(ctx, &req.SaveBusinessReq) if err != nil { + logrus.Errorf("[报表引擎] 保存业务失败: %v", err) return nil, err } return &saveBusinessRes{SaveResult: result}, nil @@ -92,6 +101,7 @@ func (c *report) DeleteBusiness(ctx context.Context, req *deleteBusinessReq) (*d ctx = ctxWithUser(ctx) result, err := svc().DeleteBusiness(ctx, req.ID) if err != nil { + logrus.Errorf("[报表引擎] 删除业务失败 id=%d: %v", req.ID, err) return nil, err } return &deleteBusinessRes{DeleteResult: result}, nil @@ -104,19 +114,37 @@ func (c *report) DeleteBusiness(ctx context.Context, req *deleteBusinessReq) (*d type listReportsReq struct { g.Meta `path:"/reports" method:"get" tags:"报表引擎" summary:"报表列表"` BusinessCode string `json:"businessCode" v:"required"` + ReportName string `json:"reportName" dc:"报表名称（模糊搜索）"` + model.PageReq } type listReportsRes struct { List []model.ReportConfig `json:"list"` + model.PageRes } func (c *report) ListReports(ctx context.Context, req *listReportsReq) (*listReportsRes, error) { ctx = ctxWithUser(ctx) - list, err := svc().GetAllReports(ctx, req.BusinessCode) + list, total, err := svc().ListReports(ctx, req.BusinessCode, req.ReportName, req.PageNum, req.PageSize) if err != nil { + logrus.Errorf("[报表引擎] 获取报表列表失败 %s: %v", req.BusinessCode, err) return nil, err } - return &listReportsRes{List: list}, nil + + // 填充抽取配置ID（逗号分隔，不限 status） + for i := range list { + configs, err := svc().GetExtractConfigsAll(ctx, list[i].BusinessCode, list[i].ReportCode) + if err == nil { + ids := make([]string, 0, len(configs)) + for _, ec := range configs { + ids = append(ids, strconv.FormatInt(ec.Id, 10)) + } + list[i].ExtractConfigIDs = strings.Join(ids, ",") + } + } + + totalPages := (total + req.PageSize - 1) / req.PageSize + return &listReportsRes{List: list, PageRes: model.PageRes{Total: total, Page: req.PageNum, PageSize: req.PageSize, TotalPages: totalPages}}, nil } type getReportReq struct { @@ -132,8 +160,20 @@ func (c *report) GetReport(ctx context.Context, req *getReportReq) (*getReportRe ctx = ctxWithUser(ctx) data, err := svc().GetReport(ctx, req.ID) if err != nil { + logrus.Errorf("[报表引擎] 获取报表失败 id=%d: %v", req.ID, err) return nil, err } + + // 填充抽取配置ID（逗号分隔，不限 status） + configs, err := svc().GetExtractConfigsAll(ctx, data.BusinessCode, data.ReportCode) + if err == nil { + ids := make([]string, 0, len(configs)) + for _, ec := range configs { + ids = append(ids, strconv.FormatInt(ec.Id, 10)) + } + data.ExtractConfigIDs = strings.Join(ids, ",") + } + return &getReportRes{Data: data}, nil } @@ -150,11 +190,32 @@ func (c *report) SaveReport(ctx context.Context, req *saveReportReq) (*saveRepor ctx = ctxWithUser(ctx) result, err := svc().SaveReport(ctx, &req.SaveReportReq) if err != nil { + logrus.Errorf("[报表引擎] 保存报表失败: %v", err) return nil, err } return &saveReportRes{SaveResult: result}, nil } +type saveReportWithFieldsReq struct { + g.Meta `path:"/report/saveWithFields" method:"post" tags:"报表引擎" summary:"保存报表及字段配置（全量替换）"` + model.SaveReportWithFieldsReq +} + +type saveReportWithFieldsRes struct { + *model.SaveResult +} + +func (c *report) SaveReportWithFields(ctx context.Context, req *saveReportWithFieldsReq) (*saveReportWithFieldsRes, error) { + ctx = ctxWithUser(ctx) + result, err := svc().SaveReportWithFields(ctx, &req.SaveReportWithFieldsReq) + if err != nil { + logrus.Errorf("[报表引擎] 保存报表及字段失败 %s/%s: %v", req.BusinessCode, req.ReportCode, err) + return nil, err + } + logrus.Infof("[报表引擎] 保存报表及字段成功 %s/%s id=%d", req.BusinessCode, req.ReportCode, result.ID) + return &saveReportWithFieldsRes{SaveResult: result}, nil +} + type deleteReportReq struct { g.Meta `path:"/report" method:"delete" tags:"报表引擎" summary:"删除报表"` ID int64 `json:"id" v:"required"` @@ -168,6 +229,7 @@ func (c *report) DeleteReport(ctx context.Context, req *deleteReportReq) (*delet ctx = ctxWithUser(ctx) result, err := svc().DeleteReport(ctx, req.ID) if err != nil { + logrus.Errorf("[报表引擎] 删除报表失败 id=%d: %v", req.ID, err) return nil, err } return &deleteReportRes{DeleteResult: result}, nil @@ -181,11 +243,17 @@ type getReportFieldsReq struct { g.Meta `path:"/fields" method:"get" tags:"报表引擎" summary:"报表字段列表（按角色分组）"` BusinessCode string `json:"businessCode" v:"required"` ReportCode string `json:"reportCode" v:"required"` + model.PageReq } func (c *report) GetReportFields(ctx context.Context, req *getReportFieldsReq) (*model.GetReportFieldsResp, error) { ctx = ctxWithUser(ctx) - return svc().GetReportFields(ctx, req.BusinessCode, req.ReportCode) + resp, err := svc().GetReportFields(ctx, req.BusinessCode, req.ReportCode) + if err != nil { + logrus.Errorf("[报表引擎] 获取报表字段失败 %s/%s: %v", req.BusinessCode, req.ReportCode, err) + return nil, err + } + return resp, nil } type getFieldReq struct { @@ -201,6 +269,7 @@ func (c *report) GetField(ctx context.Context, req *getFieldReq) (*getFieldRes, ctx = ctxWithUser(ctx) data, err := svc().GetField(ctx, req.ID) if err != nil { + logrus.Errorf("[报表引擎] 获取字段失败 id=%d: %v", req.ID, err) return nil, err } return &getFieldRes{Data: data}, nil @@ -219,6 +288,7 @@ func (c *report) SaveField(ctx context.Context, req *saveFieldReq) (*saveFieldRe ctx = ctxWithUser(ctx) result, err := svc().SaveField(ctx, &req.SaveFieldReq) if err != nil { + logrus.Errorf("[报表引擎] 保存字段失败: %v", err) return nil, err } return &saveFieldRes{SaveResult: result}, nil @@ -237,6 +307,7 @@ func (c *report) DeleteField(ctx context.Context, req *deleteFieldReq) (*deleteF ctx = ctxWithUser(ctx) result, err := svc().DeleteField(ctx, req.ID) if err != nil { + logrus.Errorf("[报表引擎] 删除字段失败 id=%d: %v", req.ID, err) return nil, err } return &deleteFieldRes{DeleteResult: result}, nil @@ -250,19 +321,23 @@ type getExtractConfigsReq struct { g.Meta `path:"/extractConfigs" method:"get" tags:"报表引擎" summary:"抽取配置列表"` BusinessCode string `json:"businessCode" v:"required"` ReportCode string `json:"reportCode" v:"required"` + model.PageReq } type getExtractConfigsRes struct { List []model.ExtractConfig `json:"list"` + model.PageRes } func (c *report) GetExtractConfigs(ctx context.Context, req *getExtractConfigsReq) (*getExtractConfigsRes, error) { ctx = ctxWithUser(ctx) - list, err := svc().GetExtractConfigs(ctx, req.BusinessCode, req.ReportCode) + list, total, err := svc().ListExtractConfigs(ctx, req.BusinessCode, req.ReportCode, req.PageNum, req.PageSize) if err != nil { + logrus.Errorf("[报表引擎] 获取抽取配置列表失败 %s/%s: %v", req.BusinessCode, req.ReportCode, err) return nil, err } - return &getExtractConfigsRes{List: list}, nil + totalPages := (total + req.PageSize - 1) / req.PageSize + return &getExtractConfigsRes{List: list, PageRes: model.PageRes{Total: total, Page: req.PageNum, PageSize: req.PageSize, TotalPages: totalPages}}, nil } type getExtractConfigReq struct { @@ -278,6 +353,7 @@ func (c *report) GetExtractConfig(ctx context.Context, req *getExtractConfigReq) ctx = ctxWithUser(ctx) data, err := svc().GetExtractConfig(ctx, req.ID) if err != nil { + logrus.Errorf("[报表引擎] 获取抽取配置失败 id=%d: %v", req.ID, err) return nil, err } return &getExtractConfigRes{Data: data}, nil @@ -296,6 +372,7 @@ func (c *report) SaveExtractConfig(ctx context.Context, req *saveExtractConfigRe ctx = ctxWithUser(ctx) result, err := svc().SaveExtractConfig(ctx, &req.SaveExtractConfigReq) if err != nil { + logrus.Errorf("[报表引擎] 保存抽取配置失败: %v", err) return nil, err } return &saveExtractConfigRes{SaveResult: result}, nil @@ -314,6 +391,7 @@ func (c *report) DeleteExtractConfig(ctx context.Context, req *deleteExtractConf ctx = ctxWithUser(ctx) result, err := svc().DeleteExtractConfig(ctx, req.ID) if err != nil { + logrus.Errorf("[报表引擎] 删除抽取配置失败 id=%d: %v", req.ID, err) return nil, err } return &deleteExtractConfigRes{DeleteResult: result}, nil @@ -330,7 +408,27 @@ type extractDataReq struct { func (c *report) ExtractData(ctx context.Context, req *extractDataReq) (*model.ExtractDailyDataResp, error) { ctx = ctxWithUser(ctx) - return svc().ExtractDailyData(ctx, req.BusinessCode, req.ReportCode, req.StatDate, req.Executor) + resp, err := svc().ExtractDailyData(ctx, req.BusinessCode, req.ReportCode, req.StatDate, req.Executor) + if err != nil { + logrus.Errorf("[报表引擎] 按天抽取失败 %s/%s %s: %v", req.BusinessCode, req.ReportCode, req.StatDate, err) + return nil, err + } + return resp, nil +} + +type backfillReq struct { + g.Meta `path:"/backfill" method:"post" tags:"报表引擎" summary:"批量回填数据"` + model.BackfillReq +} + +func (c *report) Backfill(ctx context.Context, req *backfillReq) (*model.BackfillResp, error) { + ctx = ctxWithUser(ctx) + resp, err := svc().Backfill(ctx, &req.BackfillReq) + if err != nil { + logrus.Errorf("[报表引擎] 批量回填失败 %s/%s %s~%s: %v", req.BusinessCode, req.ReportCode, req.StartDate, req.EndDate, err) + return nil, err + } + return resp, nil } type autoCreateTableReq struct { @@ -340,7 +438,12 @@ type autoCreateTableReq struct { func (c *report) AutoCreateTable(ctx context.Context, req *autoCreateTableReq) (*model.AutoCreateStatTableResp, error) { ctx = ctxWithUser(ctx) - return svc().AutoCreateStatTable(ctx, req.BusinessCode, req.ReportCode) + resp, err := svc().AutoCreateStatTable(ctx, req.BusinessCode, req.ReportCode) + if err != nil { + logrus.Errorf("[报表引擎] 自动建表失败 %s/%s: %v", req.BusinessCode, req.ReportCode, err) + return nil, err + } + return resp, nil } type queryReportReq struct { @@ -350,7 +453,12 @@ type queryReportReq struct { func (c *report) QueryReport(ctx context.Context, req *queryReportReq) (*model.UserSelectQueryResp, error) { ctx = ctxWithUser(ctx) - return svc().QueryReportByUserSelect(ctx, &req.UserSelectQueryReq) + resp, err := svc().QueryReportByUserSelect(ctx, &req.UserSelectQueryReq) + if err != nil { + logrus.Errorf("[报表引擎] 查询报表失败 %s/%s: %v", req.BusinessCode, req.ReportCode, err) + return nil, err + } + return resp, nil } type initTablesReq struct { diff --git a/demo_kuaishou.sql b/demo_kuaishou.sql new file mode 100644 index 0000000..f31b7b4 --- /dev/null +++ b/demo_kuaishou.sql @@ -0,0 +1,359 @@ +-- ============================================================ +-- 报表引擎 — 快手电商演示数据初始化脚本 +-- ============================================================ +-- 使用方式： +-- PGPASSWORD='Bjang09@686^*^' psql -h 116.204.74.41 \ +-- -p 15432 -U postgres -d engine -f demo_kuaishou.sql +-- ============================================================ + +BEGIN; + +-- ============================================================ +-- 第〇步：确保系统表存在（CREATE TABLE IF NOT EXISTS） +-- ============================================================ + +CREATE TABLE IF NOT EXISTS report_business_config ( + id BIGSERIAL PRIMARY KEY, + tenant_id BIGINT NOT NULL DEFAULT 0, + business_code VARCHAR(64) NOT NULL, + business_name VARCHAR(128) NOT NULL, + description TEXT DEFAULT '', + status VARCHAR(16) NOT NULL DEFAULT 'ACTIVE', + config JSONB DEFAULT '{}', + creator VARCHAR(64) DEFAULT '', + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updater VARCHAR(64) DEFAULT '', + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + deleted_at TIMESTAMP WITH TIME ZONE +); + +CREATE TABLE IF NOT EXISTS report_report_config ( + id BIGSERIAL PRIMARY KEY, + tenant_id BIGINT NOT NULL DEFAULT 0, + business_code VARCHAR(64) NOT NULL, + report_code VARCHAR(64) NOT NULL, + report_name VARCHAR(128) NOT NULL, + description TEXT DEFAULT '', + status VARCHAR(16) NOT NULL DEFAULT 'ACTIVE', + stat_table_name VARCHAR(128) NOT NULL, + stat_table_comment VARCHAR(256) DEFAULT '', + date_field VARCHAR(64) DEFAULT 'stat_date', + primary_keys JSONB DEFAULT '["id"]'::jsonb, + conflict_keys JSONB DEFAULT '["stat_date"]'::jsonb, + config JSONB DEFAULT '{}', + creator VARCHAR(64) DEFAULT '', + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updater VARCHAR(64) DEFAULT '', + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + deleted_at TIMESTAMP WITH TIME ZONE, + CONSTRAINT uk_business_report_code UNIQUE (tenant_id, business_code, report_code) +); + +CREATE TABLE IF NOT EXISTS report_field_config ( + id BIGSERIAL PRIMARY KEY, + tenant_id BIGINT NOT NULL DEFAULT 0, + business_code VARCHAR(64) NOT NULL, + report_code VARCHAR(64) NOT NULL, + field_code VARCHAR(64) NOT NULL, + field_name VARCHAR(128) NOT NULL, + field_type VARCHAR(32) NOT NULL, + data_type VARCHAR(32) NOT NULL DEFAULT 'STRING', + field_role VARCHAR(32) NOT NULL, + is_aggregatable BOOLEAN DEFAULT FALSE, + is_filterable BOOLEAN DEFAULT TRUE, + is_queryable BOOLEAN DEFAULT TRUE, + is_sortable BOOLEAN DEFAULT TRUE, + default_aggregate VARCHAR(32) DEFAULT '', + valid_aggregates JSONB DEFAULT '[]'::jsonb, + filter_operators JSONB DEFAULT '["=","!=",">","<",">=","<=","IN","LIKE","BETWEEN"]'::jsonb, + expression VARCHAR(512) DEFAULT '', + expression_type VARCHAR(32) DEFAULT '', + format_pattern VARCHAR(64) DEFAULT '', + unit VARCHAR(32) DEFAULT '', + dict_code VARCHAR(64) DEFAULT '', + sort_order INT DEFAULT 0, + group_name VARCHAR(64) DEFAULT '', + status VARCHAR(16) NOT NULL DEFAULT 'ACTIVE', + creator VARCHAR(64) DEFAULT '', + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updater VARCHAR(64) DEFAULT '', + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + deleted_at TIMESTAMP WITH TIME ZONE, + CONSTRAINT uk_business_report_field_code UNIQUE (tenant_id, business_code, report_code, field_code) +); + +CREATE TABLE IF NOT EXISTS report_extract_config ( + id BIGSERIAL PRIMARY KEY, + tenant_id BIGINT NOT NULL DEFAULT 0, + business_code VARCHAR(64) NOT NULL, + report_code VARCHAR(64) NOT NULL, + extract_code VARCHAR(64) NOT NULL, + extract_name VARCHAR(128) NOT NULL, + source_table_name VARCHAR(128) NOT NULL, + source_table_alias VARCHAR(64) DEFAULT '', + target_table_name VARCHAR(128) NOT NULL, + is_enabled BOOLEAN DEFAULT TRUE, + extract_type VARCHAR(32) NOT NULL DEFAULT 'FULL', + extract_mode VARCHAR(32) NOT NULL DEFAULT 'DIRECT', + extract_key_field VARCHAR(64) DEFAULT '', + extract_key_format VARCHAR(64) DEFAULT '', + group_by_fields JSONB DEFAULT '[]'::jsonb, + filter_expression TEXT DEFAULT '', + join_configs JSONB DEFAULT '[]'::jsonb, + field_mappings JSONB DEFAULT '[]'::jsonb, + transform_rules JSONB DEFAULT '[]'::jsonb, + batch_size INT DEFAULT 1000, + status VARCHAR(16) NOT NULL DEFAULT 'ACTIVE', + creator VARCHAR(64) DEFAULT '', + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updater VARCHAR(64) DEFAULT '', + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + deleted_at TIMESTAMP WITH TIME ZONE, + CONSTRAINT uk_business_report_extract_code UNIQUE (tenant_id, business_code, report_code, extract_code) +); + +CREATE TABLE IF NOT EXISTS report_extract_log ( + id BIGSERIAL PRIMARY KEY, + tenant_id BIGINT NOT NULL DEFAULT 0, + business_code VARCHAR(64) NOT NULL, + report_code VARCHAR(64) NOT NULL, + extract_code VARCHAR(64) NOT NULL, + stat_date VARCHAR(16) NOT NULL, + extract_type VARCHAR(32) NOT NULL, + status VARCHAR(16) NOT NULL DEFAULT 'RUNNING', + total_count INT DEFAULT 0, + success_count INT DEFAULT 0, + fail_count INT DEFAULT 0, + start_time TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + end_time TIMESTAMP WITH TIME ZONE, + error_message TEXT DEFAULT '', + executor VARCHAR(64) DEFAULT '', + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + CONSTRAINT uk_extract_keys UNIQUE (tenant_id, business_code, report_code, extract_code, stat_date) +); + +-- ============================================================ +-- 第一步：创建源数据表 + 模拟数据 +-- ============================================================ + +-- 店铺表 +DROP TABLE IF EXISTS demo_kuaishou_shop CASCADE; +CREATE TABLE demo_kuaishou_shop ( + shop_id VARCHAR(64) PRIMARY KEY, + shop_name VARCHAR(128) NOT NULL, + shop_level VARCHAR(16) DEFAULT '普通', + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); + +INSERT INTO demo_kuaishou_shop VALUES +('KS001', '快手官方旗舰店', '旗舰'), +('KS002', '数码3C专营店', '高级'), +('KS003', '美妆个护优选', '普通'), +('KS004', '食品生鲜超市', '高级'), +('KS005', '服装潮流馆', '普通'); + +-- 订单明细表 +DROP TABLE IF EXISTS demo_kuaishou_orders CASCADE; +CREATE TABLE demo_kuaishou_orders ( + id BIGSERIAL PRIMARY KEY, + order_no VARCHAR(64) NOT NULL, + shop_id VARCHAR(64) NOT NULL, + order_status VARCHAR(16) NOT NULL DEFAULT '已完成', + product_name VARCHAR(256), + product_category VARCHAR(64), + quantity INT DEFAULT 1, + unit_price NUMERIC(12,2) DEFAULT 0, + order_amount NUMERIC(12,2) DEFAULT 0, + refund_amount NUMERIC(12,2) DEFAULT 0, + shipping_cost NUMERIC(10,2) DEFAULT 0, + payment_method VARCHAR(32) DEFAULT '微信', + buyer_id VARCHAR(64) DEFAULT '', + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(), + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() +); + +-- 生成30天模拟订单 +DO $$ +DECLARE + v_date DATE; + v_order_date DATE := CURRENT_DATE - INTERVAL '30 days'; + v_order_count INT; + v_shop_id VARCHAR(64); + v_shop_ids VARCHAR[] := ARRAY['KS001','KS002','KS003','KS004','KS005']; + v_categories VARCHAR[] := ARRAY['手机','电脑','美妆','食品','服装']; + v_payments VARCHAR[] := ARRAY['微信','支付宝','快手支付']; + v_statuses VARCHAR[] := ARRAY['已完成','已完成','已完成','已完成','已完成','已完成','已完成','已取消','已完成']; + v_price NUMERIC(12,2); + v_qty INT; +BEGIN + FOR day_offset IN 0..29 LOOP + v_date := v_order_date + day_offset; + v_order_count := 20 + floor(random() * 31)::int; + FOR i IN 1..v_order_count LOOP + v_shop_id := v_shop_ids[1 + floor(random() * 5)::int]; + v_qty := 1 + floor(random() * 5)::int; + v_price := round((50 + random() * 2000)::numeric, 2); + INSERT INTO demo_kuaishou_orders (order_no, shop_id, order_status, product_name, + product_category, quantity, unit_price, order_amount, refund_amount, + shipping_cost, payment_method, buyer_id, created_at, updated_at) + VALUES ( + 'KS' || to_char(v_date, 'YYYYMMDD') || '-' || lpad(i::text, 4, '0'), + v_shop_id, v_statuses[1 + floor(random() * 9)::int], + '商品' || (1000 + floor(random() * 9000)::int), + v_categories[1 + floor(random() * 5)::int], + v_qty, v_price, round((v_price * v_qty)::numeric, 2), + CASE WHEN random() < 0.1 THEN round((v_price * v_qty * random() * 0.5)::numeric, 2) ELSE 0 END, + round((5 + random() * 15)::numeric, 2), + v_payments[1 + floor(random() * 3)::int], + 'buyer_' || (10000 + floor(random() * 90000)::int), + v_date + time '08:00:00' + (random() * interval '12 hours'), + v_date + time '08:00:00' + (random() * interval '12 hours') + ); + END LOOP; + END LOOP; +END $$; + +ANALYZE demo_kuaishou_orders; + +-- ============================================================ +-- 第二步：插入系统配置（先清空避免重复） +-- ============================================================ + +DELETE FROM report_business_config WHERE business_code = 'KUAISHOU'; +DELETE FROM report_report_config WHERE business_code = 'KUAISHOU'; +DELETE FROM report_field_config WHERE business_code = 'KUAISHOU'; +DELETE FROM report_extract_config WHERE business_code = 'KUAISHOU'; + +-- 1. 业务配置 +INSERT INTO report_business_config (tenant_id, business_code, business_name, description, status, config, creator) +VALUES (1, 'KUAISHOU', '快手电商', '快手电商平台订单数据', 'ACTIVE', '{}', 'system'); + +-- 2. 报表配置 +INSERT INTO report_report_config (tenant_id, business_code, report_code, report_name, description, + status, stat_table_name, stat_table_comment, date_field, primary_keys, conflict_keys, config, creator) +VALUES (1, 'KUAISHOU', 'shop_daily', '店铺日报', + '按店铺+日期统计订单数据', 'ACTIVE', + 'stat_kuaishou_shop_daily', '快手电商店铺日报统计宽表', + 'stat_date', '["id"]'::jsonb, '["stat_date", "shop_id"]'::jsonb, '{}', 'system'); + +-- 3. 字段配置 +INSERT INTO report_field_config (tenant_id, business_code, report_code, field_code, field_name, + field_type, data_type, field_role, is_aggregatable, is_filterable, is_queryable, is_sortable, + default_aggregate, valid_aggregates, filter_operators, sort_order, group_name, status, creator) +VALUES +-- 维度 +(1, 'KUAISHOU', 'shop_daily', 'stat_date', '统计日期', + 'DATE', 'STRING', 'DIMENSION', false, true, true, true, + '', '[]'::jsonb, '["=",">=","<=","BETWEEN"]'::jsonb, 1, '时间维度', 'ACTIVE', 'system'), +(1, 'KUAISHOU', 'shop_daily', 'order_no', '订单号', + 'STRING', 'STRING', 'DIMENSION', false, true, true, true, + '', '[]'::jsonb, '["=","IN"]'::jsonb, 0, '订单维度', 'ACTIVE', 'system'), +(1, 'KUAISHOU', 'shop_daily', 'shop_id', '店铺ID', + 'STRING', 'STRING', 'DIMENSION', false, true, true, true, + '', '[]'::jsonb, '["=","IN"]'::jsonb, 2, '店铺维度', 'ACTIVE', 'system'), +(1, 'KUAISHOU', 'shop_daily', 'shop_name', '店铺名称', + 'STRING', 'STRING', 'DIMENSION', false, true, true, true, + '', '[]'::jsonb, '["=","LIKE"]'::jsonb, 3, '店铺维度', 'ACTIVE', 'system'), +(1, 'KUAISHOU', 'shop_daily', 'product_category', '商品类目', + 'STRING', 'STRING', 'DIMENSION', false, true, true, true, + '', '[]'::jsonb, '["=","IN"]'::jsonb, 4, '商品维度', 'ACTIVE', 'system'), +(1, 'KUAISHOU', 'shop_daily', 'order_status', '订单状态', + 'STRING', 'STRING', 'DIMENSION', false, true, true, true, + '', '[]'::jsonb, '["=","IN"]'::jsonb, 5, '订单维度', 'ACTIVE', 'system'), +-- 指标 +(1, 'KUAISHOU', 'shop_daily', 'order_count', '订单数', + 'INT', 'INT', 'INDICATOR', true, false, true, true, + 'SUM', '["SUM","COUNT"]'::jsonb, '[]'::jsonb, 10, '交易指标', 'ACTIVE', 'system'), +(1, 'KUAISHOU', 'shop_daily', 'order_amount', '订单金额', + 'FLOAT', 'FLOAT', 'INDICATOR', true, false, true, true, + 'SUM', '["SUM","AVG","MAX","MIN"]'::jsonb, '[]'::jsonb, 11, '交易指标', 'ACTIVE', 'system'), +(1, 'KUAISHOU', 'shop_daily', 'refund_amount', '退款金额', + 'FLOAT', 'FLOAT', 'INDICATOR', true, false, true, true, + 'SUM', '["SUM","AVG"]'::jsonb, '[]'::jsonb, 12, '交易指标', 'ACTIVE', 'system'), +(1, 'KUAISHOU', 'shop_daily', 'shipping_cost', '运费', + 'FLOAT', 'FLOAT', 'INDICATOR', true, false, true, true, + 'SUM', '["SUM","AVG"]'::jsonb, '[]'::jsonb, 13, '交易指标', 'ACTIVE', 'system'), +(1, 'KUAISHOU', 'shop_daily', 'total_quantity', '商品数量', + 'INT', 'INT', 'INDICATOR', true, false, true, true, + 'SUM', '["SUM","AVG"]'::jsonb, '[]'::jsonb, 14, '交易指标', 'ACTIVE', 'system'), +(1, 'KUAISHOU', 'shop_daily', 'avg_price', '平均单价', + 'FLOAT', 'FLOAT', 'INDICATOR', true, false, true, true, + 'AVG', '["AVG","MAX","MIN"]'::jsonb, '[]'::jsonb, 15, '交易指标', 'ACTIVE', 'system'); + +-- 4. 抽取配置（DIRECT 模式：逐行抽取，查询时动态聚合） +INSERT INTO report_extract_config (tenant_id, business_code, report_code, extract_code, extract_name, + source_table_name, source_table_alias, target_table_name, is_enabled, + extract_type, extract_mode, extract_key_field, group_by_fields, filter_expression, + field_mappings, join_configs, batch_size, status, creator) +VALUES (1, 'KUAISHOU', 'shop_daily', 'shop_daily_direct', '店铺日订单(逐行,可自由组合查询)', + 'demo_kuaishou_orders', 'o', 'stat_kuaishou_shop_daily', + true, 'INCREMENTAL', 'DIRECT', 'created_at', + '[]'::jsonb, + 'o.order_status = ''已完成''', + '[ + {"sourceField":"order_no", "targetField":"order_no", "fieldType":"STRING", "aggregateFunction":"", "defaultValue":""}, + {"sourceField":"shop_id", "targetField":"shop_id", "fieldType":"STRING", "aggregateFunction":"", "defaultValue":""}, + {"sourceField":"product_category", "targetField":"product_category", "fieldType":"STRING", "aggregateFunction":"", "defaultValue":""}, + {"sourceField":"order_status", "targetField":"order_status", "fieldType":"STRING", "aggregateFunction":"", "defaultValue":""}, + {"sourceField":"order_amount", "targetField":"order_amount", "fieldType":"FLOAT", "aggregateFunction":"", "defaultValue":0}, + {"sourceField":"refund_amount", "targetField":"refund_amount", "fieldType":"FLOAT", "aggregateFunction":"", "defaultValue":0}, + {"sourceField":"shipping_cost", "targetField":"shipping_cost", "fieldType":"FLOAT", "aggregateFunction":"", "defaultValue":0}, + {"sourceField":"quantity", "targetField":"total_quantity", "fieldType":"INT", "aggregateFunction":"", "defaultValue":0}, + {"sourceField":"unit_price", "targetField":"unit_price", "fieldType":"FLOAT", "aggregateFunction":"", "defaultValue":0} + ]'::jsonb, + '[ + { + "joinTable":"demo_kuaishou_shop", + "joinAlias":"s", + "joinType":"LEFT", + "joinCondition":"o.shop_id = s.shop_id", + "fieldMappings":[ + {"sourceField":"shop_name", "targetField":"shop_name", "fieldType":"STRING", "aggregateFunction":""}, + {"sourceField":"shop_level", "targetField":"shop_level", "fieldType":"STRING", "aggregateFunction":""} + ] + } + ]'::jsonb, + 1000, 'ACTIVE', 'system' +); + +-- ============================================================ +-- 第三步：验证 +-- ============================================================ + +SELECT '✅ 源表数据' AS info, COUNT(*) AS 行数 FROM demo_kuaishou_orders +UNION ALL +SELECT '✅ 店铺数', COUNT(*) FROM demo_kuaishou_shop +UNION ALL +SELECT '✅ 业务配置', COUNT(*) FROM report_business_config WHERE business_code = 'KUAISHOU' +UNION ALL +SELECT '✅ 报表配置', COUNT(*) FROM report_report_config WHERE business_code = 'KUAISHOU' +UNION ALL +SELECT '✅ 字段配置', COUNT(*) FROM report_field_config WHERE business_code = 'KUAISHOU' +UNION ALL +SELECT '✅ 抽取配置', COUNT(*) FROM report_extract_config WHERE business_code = 'KUAISHOU'; + +SELECT '📅 数据日期范围:' AS info, MIN(created_at::date)::text || ' ~ ' || MAX(created_at::date)::text AS 范围 FROM demo_kuaishou_orders; + +COMMIT; + +-- ============================================================ +-- 使用说明 +-- ============================================================ +-- 1. 启动项目（会自动创建系统表） +-- 2. 执行本脚本 +-- 3. 打开 http://localhost:3013/admin/report +-- 4. 抽取配置 → 选"快手电商/shop_daily" → 点"执行抽取" +-- 5. 数据查询 → 选"快手电商/shop_daily" → 自由选维度/指标查询 +-- ============================================================ +-- +-- 上线前清理 Demo 数据： +-- DELETE FROM report_business_config WHERE business_code = 'KUAISHOU'; +-- DELETE FROM report_report_config WHERE business_code = 'KUAISHOU'; +-- DELETE FROM report_field_config WHERE business_code = 'KUAISHOU'; +-- DELETE FROM report_extract_config WHERE business_code = 'KUAISHOU'; +-- DELETE FROM report_extract_log WHERE business_code = 'KUAISHOU'; +-- DROP TABLE IF EXISTS stat_kuaishou_shop_daily CASCADE; +-- DROP TABLE IF EXISTS demo_kuaishou_orders CASCADE; +-- DROP TABLE IF EXISTS demo_kuaishou_shop CASCADE; +-- ============================================================ diff --git a/fix_all_lists.py b/fix_all_lists.py new file mode 100644 index 0000000..8a15701 --- /dev/null +++ b/fix_all_lists.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +"""Add pagination to all list endpoints + change status filter to deleted_at.""" + +# ========== 1. Controller: request/response structs + handlers ========== +ctrl_path = '/Users/xujiaqian/GolandProjects/HDWL/data-engine/controller/report/report_controller.go' +with open(ctrl_path) as f: + ctrl = f.read() + +# 1a. listBusinessesReq - add PageReq embed +ctrl = ctrl.replace( + 'type listBusinessesReq struct {\n\t\tg.Meta `path:"/businesses" method:"get" tags:"报表引擎" summary:"业务列表"`\n\t}', + 'type listBusinessesReq struct {\n\t\tg.Meta `path:"/businesses" method:"get" tags:"报表引擎" summary:"业务列表"`\n\t\tmodel.PageReq\n\t}' +) + +# 1b. listBusinessesRes - add PageRes embed + rename List +ctrl = ctrl.replace( + 'type listBusinessesRes struct {\n\t\tList []model.BusinessConfig `json:"list"`\n\t}', + 'type listBusinessesRes struct {\n\t\tList []model.BusinessConfig `json:"list"`\n\t\tmodel.PageRes\n\t}' +) + +# 1c. ListBusinesses handler - pass page params + calc TotalPages +ctrl = ctrl.replace( + '\t\tlist, err := svc().GetAllBusinesses(ctx)', + '\t\tlist, total, err := svc().GetAllBusinesses(ctx, req.PageNum, req.PageSize)' +) + +ctrl = ctrl.replace( + '\t\treturn &listBusinessesRes{List: list}, nil', + '\t\ttotalPages := (total + req.PageSize - 1) / req.PageSize\n\t\treturn &listBusinessesRes{List: list, PageRes: model.PageRes{Total: total, Page: req.PageNum, PageSize: req.PageSize, TotalPages: totalPages}}, nil' +) + +# 1d. listReportsReq - add PageReq embed +ctrl = ctrl.replace( + 'type listReportsReq struct {\n\t\tg.Meta `path:"/reports" method:"get" tags:"报表引擎" summary:"报表列表"`\n\t\tBusinessCode string `json:"businessCode" v:"required"`\n\t}', + 'type listReportsReq struct {\n\t\tg.Meta `path:"/reports" method:"get" tags:"报表引擎" summary:"报表列表"`\n\t\tBusinessCode string `json:"businessCode" v:"required"`\n\t\tmodel.PageReq\n\t}' +) + +# 1e. listReportsRes - add PageRes embed +ctrl = ctrl.replace( + 'type listReportsRes struct {\n\t\tList []model.ReportConfig `json:"list"`\n\t}', + 'type listReportsRes struct {\n\t\tList []model.ReportConfig `json:"list"`\n\t\tmodel.PageRes\n\t}' +) + +# 1f. ListReports handler +ctrl = ctrl.replace( + '\t\tlist, err := svc().GetAllReports(ctx, req.BusinessCode)', + '\t\tlist, total, err := svc().GetAllReports(ctx, req.BusinessCode, req.PageNum, req.PageSize)' +) + +ctrl = ctrl.replace( + '\t\treturn &listReportsRes{List: list}, nil', + '\t\ttotalPages := (total + req.PageSize - 1) / req.PageSize\n\t\treturn &listReportsRes{List: list, PageRes: model.PageRes{Total: total, Page: req.PageNum, PageSize: req.PageSize, TotalPages: totalPages}}, nil' +) + +# 1g. getExtractConfigsReq - add PageReq embed +ctrl = ctrl.replace( + 'type getExtractConfigsReq struct {\n\t\tg.Meta `path:"/extractConfigs" method:"get" tags:"报表引擎" summary:"抽取配置列表"`\n\t\tBusinessCode string `json:"businessCode" v:"required"`\n\t\tReportCode string `json:"reportCode" v:"required"`\n\t}', + 'type getExtractConfigsReq struct {\n\t\tg.Meta `path:"/extractConfigs" method:"get" tags:"报表引擎" summary:"抽取配置列表"`\n\t\tBusinessCode string `json:"businessCode" v:"required"`\n\t\tReportCode string `json:"reportCode" v:"required"`\n\t\tmodel.PageReq\n\t}' +) + +# 1h. getExtractConfigsRes - add PageRes embed +ctrl = ctrl.replace( + 'type getExtractConfigsRes struct {\n\t\tList []model.ExtractConfig `json:"list"`\n\t}', + 'type getExtractConfigsRes struct {\n\t\tList []model.ExtractConfig `json:"list"`\n\t\tmodel.PageRes\n\t}' +) + +# 1i. GetExtractConfigs handler +ctrl = ctrl.replace( + '\t\tlist, err := svc().GetExtractConfigs(ctx, req.BusinessCode, req.ReportCode)', + '\t\tlist, total, err := svc().GetExtractConfigs(ctx, req.BusinessCode, req.ReportCode, req.PageNum, req.PageSize)' +) + +ctrl = ctrl.replace( + '\t\treturn &getExtractConfigsRes{List: list}, nil', + '\t\ttotalPages := (total + req.PageSize - 1) / req.PageSize\n\t\treturn &getExtractConfigsRes{List: list, PageRes: model.PageRes{Total: total, Page: req.PageNum, PageSize: req.PageSize, TotalPages: totalPages}}, nil' +) + +with open(ctrl_path, 'w') as f: + f.write(ctrl) +print('✅ Controller done') + +# ========== 2. API service layer ========== +api_path = '/Users/xujiaqian/GolandProjects/HDWL/data-engine/common/report/api.go' +with open(api_path) as f: + api = f.read() + +# 2a. GetAllBusinesses +api = api.replace( + 'func (s *ReportService) GetAllBusinesses(ctx context.Context) ([]model.BusinessConfig, error) {\n\t\treturn s.configLoader.GetAllBusinesses(ctx)', + 'func (s *ReportService) GetAllBusinesses(ctx context.Context, pageNum, pageSize int) ([]model.BusinessConfig, int, error) {\n\t\treturn s.configLoader.GetAllBusinesses(ctx, pageNum, pageSize)' +) + +# 2b. GetAllReports +api = api.replace( + 'func (s *ReportService) GetAllReports(ctx context.Context, businessCode string) ([]model.ReportConfig, error) {\n\t\treturn s.configLoader.GetAllReports(ctx, businessCode)', + 'func (s *ReportService) GetAllReports(ctx context.Context, businessCode string, pageNum, pageSize int) ([]model.ReportConfig, int, error) {\n\t\treturn s.configLoader.GetAllReports(ctx, businessCode, pageNum, pageSize)' +) + +# 2c. GetExtractConfigs +api = api.replace( + 'func (s *ReportService) GetExtractConfigs(ctx context.Context, businessCode, reportCode string) ([]model.ExtractConfig, error) {\n\t\treturn s.configLoader.GetExtractConfigs(ctx, businessCode, reportCode)', + 'func (s *ReportService) GetExtractConfigs(ctx context.Context, businessCode, reportCode string, pageNum, pageSize int) ([]model.ExtractConfig, int, error) {\n\t\treturn s.configLoader.GetExtractConfigs(ctx, businessCode, reportCode, pageNum, pageSize)' +) + +with open(api_path, 'w') as f: + f.write(api) +print('✅ Service done') + +# ========== 3. Loader layer ========== +loader_path = '/Users/xujiaqian/GolandProjects/HDWL/data-engine/common/report/config/loader.go' +with open(loader_path) as f: + loader = f.read() + +# 3a. GetAllBusinesses - add deleted_at + pagination +old_biz = '''func (l *ConfigLoader) GetAllBusinesses(ctx context.Context) ([]model.BusinessConfig, error) { +\tr, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT * FROM report_business_config WHERE status = $1 ORDER BY id ASC", +\t\tmodel.StatusActive) +\tif err != nil { +\t\treturn nil, err +\t} +\tvar businesses []model.BusinessConfig +\tfor _, record := range r { +\t\tvar biz model.BusinessConfig +\t\tif err := record.Struct(&biz); err != nil { +\t\t\treturn nil, err +\t\t} +\t\tbusinesses = append(businesses, biz) +\t} +\treturn businesses, nil +}''' + +new_biz = '''func (l *ConfigLoader) GetAllBusinesses(ctx context.Context, pageNum, pageSize int) ([]model.BusinessConfig, int, error) { +\t// 查询总数 +\ttotal, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT COUNT(*) AS cnt FROM report_business_config WHERE deleted_at IS NULL") +\tif err != nil { +\t\treturn nil, 0, err +\t} +\tcount := 0 +\tif !total.IsEmpty() { +\t\tcount = total[0]["cnt"].Int() +\t} +\toffset := (pageNum - 1) * pageSize +\tr, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT * FROM report_business_config WHERE deleted_at IS NULL ORDER BY id ASC LIMIT $1 OFFSET $2", +\t\tpageSize, offset) +\tif err != nil { +\t\treturn nil, 0, err +\t} +\tvar businesses []model.BusinessConfig +\tfor _, record := range r { +\t\tvar biz model.BusinessConfig +\t\tif err := record.Struct(&biz); err != nil { +\t\t\treturn nil, 0, err +\t\t} +\t\tbusinesses = append(businesses, biz) +\t} +\treturn businesses, count, nil +}''' + +loader = loader.replace(old_biz, new_biz, 1) + +# 3b. GetAllReports - add deleted_at + pagination +old_rpt = '''func (l *ConfigLoader) GetAllReports(ctx context.Context, businessCode string) ([]model.ReportConfig, error) { +\tr, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT * FROM report_report_config WHERE business_code = $1 ORDER BY id ASC", +\t\tbusinessCode) +\tif err != nil { +\t\treturn nil, err +\t} +\tvar reports []model.ReportConfig +\tfor _, record := range r { +\t\tvar rpt model.ReportConfig +\t\tif err := record.Struct(&rpt); err != nil { +\t\t\treturn nil, err +\t\t} +\t\treports = append(reports, rpt) +\t} +\treturn reports, nil +}''' + +new_rpt = '''func (l *ConfigLoader) GetAllReports(ctx context.Context, businessCode string, pageNum, pageSize int) ([]model.ReportConfig, int, error) { +\t// 查询总数 +\ttotal, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT COUNT(*) AS cnt FROM report_report_config WHERE business_code = $1 AND deleted_at IS NULL", +\t\tbusinessCode) +\tif err != nil { +\t\treturn nil, 0, err +\t} +\tcount := 0 +\tif !total.IsEmpty() { +\t\tcount = total[0]["cnt"].Int() +\t} +\toffset := (pageNum - 1) * pageSize +\tr, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT * FROM report_report_config WHERE business_code = $1 AND deleted_at IS NULL ORDER BY id ASC LIMIT $2 OFFSET $3", +\t\tbusinessCode, pageSize, offset) +\tif err != nil { +\t\treturn nil, 0, err +\t} +\tvar reports []model.ReportConfig +\tfor _, record := range r { +\t\tvar rpt model.ReportConfig +\t\tif err := record.Struct(&rpt); err != nil { +\t\t\treturn nil, 0, err +\t\t} +\t\treports = append(reports, rpt) +\t} +\treturn reports, count, nil +}''' + +loader = loader.replace(old_rpt, new_rpt, 1) + +# 3c. GetExtractConfigs - add deleted_at + pagination +old_ec = '''func (l *ConfigLoader) GetExtractConfigs(ctx context.Context, businessCode, reportCode string) ([]model.ExtractConfig, error) { +\tr, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT * FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND status = $3 AND is_enabled = $4", +\t\tbusinessCode, reportCode, model.StatusActive, true) +\tif err != nil { +\t\treturn nil, err +\t} +\tvar configs []model.ExtractConfig +\tfor _, record := range r { +\t\tvar ec model.ExtractConfig +\t\tif err := record.Struct(&ec); err != nil { +\t\t\treturn nil, err +\t\t} +\t\tconfigs = append(configs, ec) +\t} +\tl.mu.Lock() +\tl.extractCache[businessCode+":"+reportCode] = configs +\tl.mu.Unlock() +\treturn configs, nil +}''' + +new_ec = '''func (l *ConfigLoader) GetExtractConfigs(ctx context.Context, businessCode, reportCode string, pageNum, pageSize int) ([]model.ExtractConfig, int, error) { +\t// 查询总数 +\ttotal, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT COUNT(*) AS cnt FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL", +\t\tbusinessCode, reportCode) +\tif err != nil { +\t\treturn nil, 0, err +\t} +\tcount := 0 +\tif !total.IsEmpty() { +\t\tcount = total[0]["cnt"].Int() +\t} +\toffset := (pageNum - 1) * pageSize +\tr, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT * FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY id ASC LIMIT $3 OFFSET $4", +\t\tbusinessCode, reportCode, pageSize, offset) +\tif err != nil { +\t\treturn nil, 0, err +\t} +\tvar configs []model.ExtractConfig +\tfor _, record := range r { +\t\tvar ec model.ExtractConfig +\t\tif err := record.Struct(&ec); err != nil { +\t\t\treturn nil, 0, err +\t\t} +\t\tconfigs = append(configs, ec) +\t} +\treturn configs, count, nil +}''' + +loader = loader.replace(old_ec, new_ec, 1) + +with open(loader_path, 'w') as f: + f.write(loader) +print('✅ Loader done') diff --git a/fix_dollar.py b/fix_dollar.py new file mode 100644 index 0000000..acb1e0a --- /dev/null +++ b/fix_dollar.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +import re + +path = '/Users/xujiaqian/GolandProjects/HDWL/data-engine/common/report/config/loader.go' +with open(path) as f: + content = f.read() + +# Fix `\$1` → `$1` (the \ came from Python escaping in previous script) +content = content.replace('\\$1', '$1') +content = content.replace('\\$2', '$2') + +with open(path, 'w') as f: + f.write(content) + +print('Fixed') diff --git a/fix_final.py b/fix_final.py new file mode 100644 index 0000000..48558f4 --- /dev/null +++ b/fix_final.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +"""Final pass: apply all list-pagination + deleted_at filter changes.""" +import re + +# ============= LOADER ============= +lp = '/Users/xujiaqian/GolandProjects/HDWL/data-engine/common/report/config/loader.go' +with open(lp) as f: + l = f.read() + +# 1. GetAllBusinesses: status=ACTIVE -> deleted_at IS NULL +l = l.replace( + '"SELECT * FROM report_business_config WHERE status = $1 ORDER BY id ASC"', + '"SELECT * FROM report_business_config WHERE deleted_at IS NULL ORDER BY id ASC"' +) +l = l.replace('\t\tmodel.StatusActive)', ')\n\t\t// (removed status filter)') + +# 2. GetAllReports: add deleted_at IS NULL (keep existing) +# Already has no status filter, just verify +# (The user already removed status filter from GetAllReports earlier) + +# 3. GetFields: status=ACTIVE -> deleted_at IS NULL +l = l.replace( + '"SELECT * FROM report_field_config WHERE business_code = $1 AND report_code = $2 AND status = $3 ORDER BY sort_order ASC"', + '"SELECT * FROM report_field_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY sort_order ASC"' +) +# Remove StatusActive arg from GetFields call +l = l.replace( + 'businessCode, reportCode, model.StatusActive)', + 'businessCode, reportCode)' +) + +# 4. GetExtractConfigs: status=ACTIVE AND is_enabled=true -> deleted_at IS NULL +l = l.replace( + '"SELECT * FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND status = $3 AND is_enabled = $4"', + '"SELECT * FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY id ASC"' +) +l = l.replace( + 'businessCode, reportCode, model.StatusActive, true)', + 'businessCode, reportCode)' +) + +# 5. Add GetAllFields (for edit query, returns all non-deleted regardless of status) +if 'GetAllFields' not in l: + # Add after GetFields function ends + marker = 'return fields, nil\n}\n\n// GetFieldMap 获取字段配置Map' + add = '''return fields, nil +} + +// GetAllFields 获取报表全部字段（含 INACTIVE，不含已删除，用于编辑回显） +func (l *ConfigLoader) GetAllFields(ctx context.Context, businessCode, reportCode string) ([]model.FieldConfig, error) { +\tr, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT * FROM report_field_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY sort_order ASC", +\t\tbusinessCode, reportCode) +\tif err != nil { +\t\treturn nil, err +\t} +\tvar fields []model.FieldConfig +\tfor _, record := range r { +\t\tvar f model.FieldConfig +\t\tif err := record.Struct(&f); err != nil { +\t\t\treturn nil, err +\t\t} +\t\tif f.ValidAggregates == nil { +\t\t\tf.ValidAggregates = []string{} +\t\t} +\t\tif f.FilterOperators == nil { +\t\t\tf.FilterOperators = []string{"=", "!=", ">", "<", ">=", "<=", "IN", "LIKE", "BETWEEN"} +\t\t} +\t\tfields = append(fields, f) +\t} +\treturn fields, nil +} + +// GetFieldMap 获取字段配置Map''' + l = l.replace(marker, add, 1) + +# 6. Add paginated list methods +# ListBusinesses +if 'ListBusinesses' not in l: + add = '''// ListBusinesses 分页获取业务列表 +func (l *ConfigLoader) ListBusinesses(ctx context.Context, pageNum, pageSize int) ([]model.BusinessConfig, int, error) { +\ttotal, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT COUNT(*) AS cnt FROM report_business_config WHERE deleted_at IS NULL") +\tif err != nil { +\t\treturn nil, 0, err +\t} +\tcount := 0 +\tif !total.IsEmpty() { +\t\tcount = total[0]["cnt"].Int() +\t} +\toffset := (pageNum - 1) * pageSize +\tr, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT * FROM report_business_config WHERE deleted_at IS NULL ORDER BY id ASC LIMIT $1 OFFSET $2", +\t\tpageSize, offset) +\tif err != nil { +\t\treturn nil, 0, err +\t} +\tvar businesses []model.BusinessConfig +\tfor _, record := range r { +\t\tvar biz model.BusinessConfig +\t\tif err := record.Struct(&biz); err != nil { +\t\t\treturn nil, 0, err +\t\t} +\t\tbusinesses = append(businesses, biz) +\t} +\treturn businesses, count, nil +} + +// ListReports 分页获取报表列表 +func (l *ConfigLoader) ListReports(ctx context.Context, businessCode string, pageNum, pageSize int) ([]model.ReportConfig, int, error) { +\ttotal, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT COUNT(*) AS cnt FROM report_report_config WHERE business_code = $1 AND deleted_at IS NULL", +\t\tbusinessCode) +\tif err != nil { +\t\treturn nil, 0, err +\t} +\tcount := 0 +\tif !total.IsEmpty() { +\t\tcount = total[0]["cnt"].Int() +\t} +\toffset := (pageNum - 1) * pageSize +\tr, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT * FROM report_report_config WHERE business_code = $1 AND deleted_at IS NULL ORDER BY id ASC LIMIT $2 OFFSET $3", +\t\tbusinessCode, pageSize, offset) +\tif err != nil { +\t\treturn nil, 0, err +\t} +\tvar reports []model.ReportConfig +\tfor _, record := range r { +\t\tvar rpt model.ReportConfig +\t\tif err := record.Struct(&rpt); err != nil { +\t\t\treturn nil, 0, err +\t\t} +\t\treports = append(reports, rpt) +\t} +\treturn reports, count, nil +} + +// ListExtractConfigs 分页获取抽取配置列表 +func (l *ConfigLoader) ListExtractConfigs(ctx context.Context, businessCode, reportCode string, pageNum, pageSize int) ([]model.ExtractConfig, int, error) { +\ttotal, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT COUNT(*) AS cnt FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL", +\t\tbusinessCode, reportCode) +\tif err != nil { +\t\treturn nil, 0, err +\t} +\tcount := 0 +\tif !total.IsEmpty() { +\t\tcount = total[0]["cnt"].Int() +\t} +\toffset := (pageNum - 1) * pageSize +\tr, err := gfdb.DB(ctx).GetAll(ctx, +\t\t"SELECT * FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY id ASC LIMIT $3 OFFSET $4", +\t\tbusinessCode, reportCode, pageSize, offset) +\tif err != nil { +\t\treturn nil, 0, err +\t} +\tvar configs []model.ExtractConfig +\tfor _, record := range r { +\t\tvar ec model.ExtractConfig +\t\tif err := record.Struct(&ec); err != nil { +\t\t\treturn nil, 0, err +\t\t} +\t\tconfigs = append(configs, ec) +\t} +\tl.mu.Lock() +\tl.extractCache[businessCode+":"+reportCode] = configs +\tl.mu.Unlock() +\treturn configs, count, nil +} +''' + # Insert before the "//" comment separator near the end + marker2 = '\n// GetReportFields 获取报表可用字段(按角色分类)' + if marker2 in l: + l = l.replace(marker2, add + marker2, 1) + else: + l += add + +with open(lp, 'w') as f: + f.write(l) +print('✅ Loader') + +# ============= SERVICE (api.go) ============= +ap = '/Users/xujiaqian/GolandProjects/HDWL/data-engine/common/report/api.go' +with open(ap) as f: + a = f.read() + +# Add ListBusinesses/ListReports/ListExtractConfigs service methods +if 'func (s *ReportService) ListBusinesses' not in a: + add_svc = ''' +// ListBusinesses 分页获取业务列表 +func (s *ReportService) ListBusinesses(ctx context.Context, pageNum, pageSize int) ([]model.BusinessConfig, int, error) { +\treturn s.configLoader.ListBusinesses(ctx, pageNum, pageSize) +} + +// ListReports 分页获取报表列表 +func (s *ReportService) ListReports(ctx context.Context, businessCode string, pageNum, pageSize int) ([]model.ReportConfig, int, error) { +\treturn s.configLoader.ListReports(ctx, businessCode, pageNum, pageSize) +} + +// ListExtractConfigs 分页获取抽取配置列表 +func (s *ReportService) ListExtractConfigs(ctx context.Context, businessCode, reportCode string, pageNum, pageSize int) ([]model.ExtractConfig, int, error) { +\treturn s.configLoader.ListExtractConfigs(ctx, businessCode, reportCode, pageNum, pageSize) +} +''' + # Insert after the service type definition + a = a.replace( + 's.configLoader.GetAllBusinesses(ctx)\n}', + 's.configLoader.GetAllBusinesses(ctx)\n}' + add_svc + ) + +with open(ap, 'w') as f: + f.write(a) +print('✅ Service') + +# ============= CONTROLLER ============= +cp = '/Users/xujiaqian/GolandProjects/HDWL/data-engine/controller/report/report_controller.go' +with open(cp) as f: + c = f.read() + +# Fix ListBusinesses handler +c = c.replace( + 'list, err := svc().GetAllBusinesses(ctx)', + 'list, total, err := svc().ListBusinesses(ctx, req.PageNum, req.PageSize)' +) +c = c.replace( + 'return &listBusinessesRes{List: list}, nil', + 'totalPages := (total + req.PageSize - 1) / req.PageSize\n\t\treturn &listBusinessesRes{List: list, PageRes: model.PageRes{Total: total, Page: req.PageNum, PageSize: req.PageSize, TotalPages: totalPages}}, nil' +) + +# Fix ListReports handler +c = c.replace( + 'list, err := svc().GetAllReports(ctx, req.BusinessCode)', + 'list, total, err := svc().ListReports(ctx, req.BusinessCode, req.PageNum, req.PageSize)' +) +c = c.replace( + 'return &listReportsRes{List: list}, nil', + 'totalPages := (total + req.PageSize - 1) / req.PageSize\n\t\treturn &listReportsRes{List: list, PageRes: model.PageRes{Total: total, Page: req.PageNum, PageSize: req.PageSize, TotalPages: totalPages}}, nil' +) + +# Fix GetExtractConfigs handler +c = c.replace( + 'list, err := svc().GetExtractConfigs(ctx, req.BusinessCode, req.ReportCode)', + 'list, total, err := svc().ListExtractConfigs(ctx, req.BusinessCode, req.ReportCode, req.PageNum, req.PageSize)' +) +c = c.replace( + 'return &getExtractConfigsRes{List: list}, nil', + 'totalPages := (total + req.PageSize - 1) / req.PageSize\n\t\treturn &getExtractConfigsRes{List: list, PageRes: model.PageRes{Total: total, Page: req.PageNum, PageSize: req.PageSize, TotalPages: totalPages}}, nil' +) + +with open(cp, 'w') as f: + f.write(c) +print('✅ Controller') + +print('\nAll done!') diff --git a/go.mod b/go.mod index 6fa02ec..7a3605c 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( require ( github.com/BurntSushi/toml v1.6.0 // indirect - github.com/Masterminds/semver/v3 v3.5.0 // indirect github.com/armon/go-metrics v0.4.1 // indirect github.com/bwmarrin/snowflake v0.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect @@ -41,7 +40,6 @@ require ( github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5 // indirect - github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/glog v1.2.5 // indirect @@ -61,20 +59,17 @@ require ( github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect - github.com/hashicorp/go-version v1.9.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/serf v0.10.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.6 // indirect - github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/lib/pq v1.12.3 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mailru/easyjson v0.9.2 // indirect github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-runewidth v0.0.24 // indirect - github.com/miekg/dns v1.1.72 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect diff --git a/main.go b/main.go index 47f3987..229ac11 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "dataengine/common/report" "dataengine/controller/debug" "dataengine/controller/dict" "dataengine/controller/public" @@ -28,6 +29,9 @@ func main() { // 启动自动同步（后台循环执行，首次全量后续增量） syncSvc.InitAndStartAutoSync(ctx) + // 启动每日自动抽取（每天凌晨2点抽取前一天数据） + report.StartDailyExtractJob() + http.RouteRegister([]interface{}{ // 接口管理 dict.ApiInterface, diff --git a/report-engine-openapi.json b/report-engine-openapi.json new file mode 100644 index 0000000..c86d38a --- /dev/null +++ b/report-engine-openapi.json @@ -0,0 +1,4849 @@ +{ + "openapi": "3.0.0", + "components": { + "schemas": { + "dataengine.model.dto.dict.CreateApiInterfaceReq": { + "description": "创建新的数据接口", + "properties": { + "platformId": { + "description": "所属平台ID", + "format": "int64", + "type": "integer" + }, + "name": { + "description": "接口名称", + "format": "string", + "type": "string" + }, + "code": { + "description": "接口编码", + "format": "string", + "type": "string" + }, + "url": { + "description": "接口地址", + "format": "string", + "type": "string" + }, + "method": { + "description": "请求方法", + "format": "api_feature.ApiMethod", + "type": "string" + }, + "status": { + "default": "active", + "description": "接口状态", + "format": "api_feature.PlatformStatus", + "type": "string" + }, + "authType": { + "description": "认证类型", + "format": "string", + "type": "string" + }, + "requestConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "请求配置", + "format": "map[string]interface {}", + "type": "object" + }, + "responseConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "响应配置", + "format": "map[string]interface {}", + "type": "object" + }, + "limitConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "接口独立限流配置", + "format": "map[string]interface {}", + "type": "object" + }, + "tableDefinition": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "表结构定义(JSON)，用于自动建表", + "format": "map[string]interface {}", + "type": "object" + } + }, + "required": [ + "platformId", + "name", + "code", + "url", + "method" + ], + "type": "object" + }, + "interface": { + "properties": {}, + "type": "object" + }, + "dataengine.model.dto.dict.CreateApiInterfaceRes": { + "properties": { + "id": { + "description": "接口ID", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, + "dataengine.model.dto.dict.DeleteApiInterfaceReq": { + "description": "删除接口", + "properties": { + "id": { + "description": "接口ID", + "format": "int64", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "gitea.redpowerfuture.com.red-future.common.beans.ResponseEmpty": { + "properties": {}, + "type": "object" + }, + "dataengine.model.dto.dict.GetApiInterfaceReq": { + "description": "获取接口详情", + "properties": { + "id": { + "description": "接口ID", + "format": "int64", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.model.dto.dict.GetApiInterfaceRes": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + }, + "tenantId": { + "format": "uint64", + "type": "integer" + }, + "creator": { + "format": "string", + "type": "string" + }, + "createdAt": { + "format": "*gtime.Time", + "type": "string" + }, + "updater": { + "format": "string", + "type": "string" + }, + "updatedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "deletedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "platformId": { + "description": "所属平台ID", + "format": "int64", + "type": "integer" + }, + "name": { + "description": "接口名称", + "format": "string", + "type": "string" + }, + "code": { + "description": "接口编码", + "format": "string", + "type": "string" + }, + "url": { + "description": "接口地址", + "format": "string", + "type": "string" + }, + "method": { + "description": "请求方法：GET/POST/PUT/DELETE等", + "format": "api_feature.ApiMethod", + "type": "string" + }, + "status": { + "description": "接口状态：active启用/inactive停用", + "format": "api_feature.PlatformStatus", + "type": "string" + }, + "authType": { + "description": "认证类型：oauth2/apikey/basic等", + "format": "string", + "type": "string" + }, + "requestConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "请求配置", + "format": "map[string]interface {}", + "type": "object" + }, + "responseConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "响应配置", + "format": "map[string]interface {}", + "type": "object" + }, + "limitConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "接口独立限流配置（可选，覆盖平台配置）", + "format": "map[string]interface {}", + "type": "object" + }, + "tableDefinition": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "表结构定义，用于自动建表", + "format": "map[string]interface {}", + "type": "object" + }, + "platformName": { + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.model.dto.dict.ListApiInterfaceReq": { + "description": "分页查询接口列表", + "properties": { + "pageNum": { + "format": "int64", + "type": "integer" + }, + "pageSize": { + "format": "int64", + "type": "integer" + }, + "Total": { + "format": "int64", + "type": "integer" + }, + "platformId": { + "description": "平台ID", + "format": "int64", + "type": "integer" + }, + "name": { + "description": "接口名称", + "format": "string", + "type": "string" + }, + "code": { + "description": "接口编码", + "format": "string", + "type": "string" + }, + "method": { + "description": "请求方法", + "format": "api_feature.ApiMethod", + "type": "string" + }, + "status": { + "description": "接口状态", + "format": "api_feature.PlatformStatus", + "type": "string" + }, + "keyword": { + "description": "关键字（搜索名称或编码）", + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.model.dto.dict.ListApiInterfaceRes": { + "properties": { + "list": { + "description": "接口列表", + "format": "[]dict.ApiInterfaceItem", + "items": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.ApiInterfaceItem", + "description": "" + }, + "type": "array" + }, + "total": { + "description": "总数", + "format": "int", + "type": "integer" + } + }, + "type": "object" + }, + "dataengine.model.dto.dict.ApiInterfaceItem": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + }, + "platformId": { + "format": "int64", + "type": "integer" + }, + "platformName": { + "format": "string", + "type": "string" + }, + "name": { + "format": "string", + "type": "string" + }, + "code": { + "format": "string", + "type": "string" + }, + "url": { + "format": "string", + "type": "string" + }, + "method": { + "format": "api_feature.ApiMethod", + "type": "string" + }, + "status": { + "format": "api_feature.PlatformStatus", + "type": "string" + }, + "statusName": { + "format": "string", + "type": "string" + }, + "createdBy": { + "format": "string", + "type": "string" + }, + "createdAt": { + "format": "int64", + "type": "integer" + }, + "updatedBy": { + "format": "string", + "type": "string" + }, + "updatedAt": { + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, + "dataengine.model.dto.dict.UpdateApiInterfaceReq": { + "description": "更新接口信息", + "properties": { + "id": { + "description": "接口ID", + "format": "int64", + "type": "integer" + }, + "platformId": { + "description": "所属平台ID", + "format": "int64", + "type": "integer" + }, + "name": { + "description": "接口名称", + "format": "string", + "type": "string" + }, + "code": { + "description": "接口编码", + "format": "string", + "type": "string" + }, + "url": { + "description": "接口地址", + "format": "string", + "type": "string" + }, + "method": { + "description": "请求方法", + "format": "api_feature.ApiMethod", + "type": "string" + }, + "status": { + "description": "接口状态", + "format": "api_feature.PlatformStatus", + "type": "string" + }, + "authType": { + "description": "认证类型", + "format": "string", + "type": "string" + }, + "requestConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "请求配置", + "format": "map[string]interface {}", + "type": "object" + }, + "responseConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "响应配置", + "format": "map[string]interface {}", + "type": "object" + }, + "limitConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "接口独立限流配置", + "format": "map[string]interface {}", + "type": "object" + }, + "tableDefinition": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "表结构定义(JSON)，用于自动建表", + "format": "map[string]interface {}", + "type": "object" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.model.dto.dict.UpdateApiInterfaceStatusReq": { + "description": "更新接口状态", + "properties": { + "id": { + "description": "接口ID", + "format": "int64", + "type": "integer" + }, + "status": { + "description": "状态：active启用/inactive停用", + "enum": [ + "active", + "inactive" + ], + "format": "api_feature.PlatformStatus", + "type": "string" + } + }, + "required": [ + "id", + "status" + ], + "type": "object" + }, + "dataengine.model.dto.dict.CreateDatasourcePlatformReq": { + "description": "创建新的数据源平台配置", + "properties": { + "platformCode": { + "description": "平台编码(唯一标识)", + "format": "string", + "type": "string" + }, + "platformName": { + "description": "平台名称", + "format": "string", + "type": "string" + }, + "description": { + "description": "平台描述", + "format": "string", + "type": "string" + }, + "status": { + "default": "ACTIVE", + "description": "平台状态", + "format": "api_feature.PlatformStatus", + "type": "string" + }, + "apiBaseUrl": { + "description": "API基础地址", + "format": "string", + "type": "string" + }, + "authType": { + "description": "认证类型: TOKEN/API_KEY/OAUTH2/BASIC", + "enum": [ + "TOKEN", + "API_KEY", + "OAUTH2", + "BASIC" + ], + "format": "string", + "type": "string" + }, + "token": { + "description": "认证token/密钥", + "format": "string", + "type": "string" + }, + "apiKey": { + "description": "API Key", + "format": "string", + "type": "string" + }, + "clientId": { + "description": "OAuth2 Client ID", + "format": "string", + "type": "string" + }, + "clientSecret": { + "description": "OAuth2 Client Secret", + "format": "string", + "type": "string" + }, + "rateLimitPerMinute": { + "default": 60, + "description": "每分钟请求限制", + "format": "int", + "type": "integer" + }, + "rateLimitPerHour": { + "default": 1000, + "description": "每小时请求限制", + "format": "int", + "type": "integer" + }, + "concurrencyLimit": { + "default": 10, + "description": "并发连接限制", + "format": "int", + "type": "integer" + }, + "requestTimeoutMs": { + "default": 30000, + "description": "请求超时时间(毫秒)", + "format": "int", + "type": "integer" + }, + "maxRetries": { + "default": 3, + "description": "最大重试次数", + "format": "int", + "type": "integer" + }, + "retryDelayMs": { + "default": 1000, + "description": "重试延迟(毫秒)", + "format": "int", + "type": "integer" + }, + "authConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "自定义认证配置(JSON)，支持各平台特有认证方式", + "format": "map[string]interface {}", + "type": "object" + } + }, + "required": [ + "platformCode", + "platformName", + "authType" + ], + "type": "object" + }, + "dataengine.model.dto.dict.CreateDatasourcePlatformRes": { + "properties": { + "id": { + "description": "平台ID", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, + "dataengine.model.dto.dict.DeleteDatasourcePlatformReq": { + "description": "删除数据源平台", + "properties": { + "id": { + "description": "平台ID", + "format": "int64", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.model.dto.dict.GetDatasourcePlatformReq": { + "description": "获取数据源平台详情", + "properties": { + "id": { + "description": "平台ID", + "format": "int64", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.model.dto.dict.GetDatasourcePlatformRes": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + }, + "tenantId": { + "format": "uint64", + "type": "integer" + }, + "creator": { + "format": "string", + "type": "string" + }, + "createdAt": { + "format": "*gtime.Time", + "type": "string" + }, + "updater": { + "format": "string", + "type": "string" + }, + "updatedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "deletedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "platformCode": { + "description": "平台编码，唯一标识", + "format": "string", + "type": "string" + }, + "platformName": { + "description": "平台名称", + "format": "string", + "type": "string" + }, + "description": { + "description": "平台描述", + "format": "string", + "type": "string" + }, + "status": { + "description": "状态: ACTIVE启用, INACTIVE停用", + "format": "string", + "type": "string" + }, + "apiBaseUrl": { + "description": "API基础地址", + "format": "string", + "type": "string" + }, + "authType": { + "description": "认证类型: TOKEN/API_KEY/OAUTH2/BASIC", + "format": "string", + "type": "string" + }, + "token": { + "description": "认证token/密钥", + "format": "string", + "type": "string" + }, + "apiKey": { + "description": "API Key", + "format": "string", + "type": "string" + }, + "clientId": { + "description": "OAuth2 Client ID", + "format": "string", + "type": "string" + }, + "clientSecret": { + "description": "OAuth2 Client Secret", + "format": "string", + "type": "string" + }, + "rateLimitPerMinute": { + "description": "每分钟请求限制", + "format": "int", + "type": "integer" + }, + "rateLimitPerHour": { + "description": "每小时请求限制", + "format": "int", + "type": "integer" + }, + "concurrencyLimit": { + "description": "并发连接限制", + "format": "int", + "type": "integer" + }, + "requestTimeoutMs": { + "description": "请求超时时间(毫秒)", + "format": "int", + "type": "integer" + }, + "maxRetries": { + "description": "最大重试次数", + "format": "int", + "type": "integer" + }, + "retryDelayMs": { + "description": "重试延迟(毫秒)", + "format": "int", + "type": "integer" + }, + "authConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "自定义认证配置，支持各平台特有的认证方式", + "format": "map[string]interface {}", + "type": "object" + }, + "version": { + "description": "版本号(乐观锁)", + "format": "int", + "type": "integer" + } + }, + "type": "object" + }, + "dataengine.model.dto.dict.GetPlatformByCodeReq": { + "description": "根据平台编码获取平台配置信息", + "properties": { + "platformCode": { + "description": "平台编码", + "format": "string", + "type": "string" + } + }, + "required": [ + "platformCode" + ], + "type": "object" + }, + "dataengine.model.dto.dict.GetPlatformByCodeRes": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + }, + "tenantId": { + "format": "uint64", + "type": "integer" + }, + "creator": { + "format": "string", + "type": "string" + }, + "createdAt": { + "format": "*gtime.Time", + "type": "string" + }, + "updater": { + "format": "string", + "type": "string" + }, + "updatedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "deletedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "platformCode": { + "description": "平台编码，唯一标识", + "format": "string", + "type": "string" + }, + "platformName": { + "description": "平台名称", + "format": "string", + "type": "string" + }, + "description": { + "description": "平台描述", + "format": "string", + "type": "string" + }, + "status": { + "description": "状态: ACTIVE启用, INACTIVE停用", + "format": "string", + "type": "string" + }, + "apiBaseUrl": { + "description": "API基础地址", + "format": "string", + "type": "string" + }, + "authType": { + "description": "认证类型: TOKEN/API_KEY/OAUTH2/BASIC", + "format": "string", + "type": "string" + }, + "token": { + "description": "认证token/密钥", + "format": "string", + "type": "string" + }, + "apiKey": { + "description": "API Key", + "format": "string", + "type": "string" + }, + "clientId": { + "description": "OAuth2 Client ID", + "format": "string", + "type": "string" + }, + "clientSecret": { + "description": "OAuth2 Client Secret", + "format": "string", + "type": "string" + }, + "rateLimitPerMinute": { + "description": "每分钟请求限制", + "format": "int", + "type": "integer" + }, + "rateLimitPerHour": { + "description": "每小时请求限制", + "format": "int", + "type": "integer" + }, + "concurrencyLimit": { + "description": "并发连接限制", + "format": "int", + "type": "integer" + }, + "requestTimeoutMs": { + "description": "请求超时时间(毫秒)", + "format": "int", + "type": "integer" + }, + "maxRetries": { + "description": "最大重试次数", + "format": "int", + "type": "integer" + }, + "retryDelayMs": { + "description": "重试延迟(毫秒)", + "format": "int", + "type": "integer" + }, + "authConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "自定义认证配置，支持各平台特有的认证方式", + "format": "map[string]interface {}", + "type": "object" + }, + "version": { + "description": "版本号(乐观锁)", + "format": "int", + "type": "integer" + } + }, + "type": "object" + }, + "dataengine.model.dto.dict.GetPlatformStatisticsReq": { + "description": "获取数据源平台统计信息", + "properties": {}, + "type": "object" + }, + "dataengine.model.dto.dict.GetPlatformStatisticsRes": { + "properties": { + "totalPlatforms": { + "description": "总平台数", + "format": "int", + "type": "integer" + }, + "activePlatforms": { + "description": "启用平台数", + "format": "int", + "type": "integer" + }, + "inactivePlatforms": { + "description": "停用平台数", + "format": "int", + "type": "integer" + }, + "tokenAuthPlatforms": { + "description": "TOKEN认证平台数", + "format": "int", + "type": "integer" + }, + "apiKeyAuthPlatforms": { + "description": "API_KEY认证平台数", + "format": "int", + "type": "integer" + }, + "oauth2AuthPlatforms": { + "description": "OAUTH2认证平台数", + "format": "int", + "type": "integer" + }, + "basicAuthPlatforms": { + "description": "BASIC认证平台数", + "format": "int", + "type": "integer" + } + }, + "type": "object" + }, + "dataengine.model.dto.dict.ListDatasourcePlatformReq": { + "description": "分页查询数据源平台列表", + "properties": { + "pageNum": { + "format": "int64", + "type": "integer" + }, + "pageSize": { + "format": "int64", + "type": "integer" + }, + "Total": { + "format": "int64", + "type": "integer" + }, + "platformCode": { + "description": "平台编码", + "format": "string", + "type": "string" + }, + "platformName": { + "description": "平台名称", + "format": "string", + "type": "string" + }, + "status": { + "description": "平台状态", + "format": "api_feature.PlatformStatus", + "type": "string" + }, + "authType": { + "description": "认证类型", + "format": "string", + "type": "string" + }, + "keyword": { + "description": "关键字（搜索平台名称或编码）", + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.model.dto.dict.ListDatasourcePlatformRes": { + "properties": { + "list": { + "description": "平台列表", + "format": "[]dict.DatasourcePlatformItem", + "items": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.DatasourcePlatformItem", + "description": "" + }, + "type": "array" + }, + "total": { + "description": "总数", + "format": "int", + "type": "integer" + } + }, + "type": "object" + }, + "dataengine.model.dto.dict.DatasourcePlatformItem": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + }, + "tenantId": { + "format": "uint64", + "type": "integer" + }, + "platformCode": { + "format": "string", + "type": "string" + }, + "platformName": { + "format": "string", + "type": "string" + }, + "description": { + "format": "string", + "type": "string" + }, + "status": { + "format": "api_feature.PlatformStatus", + "type": "string" + }, + "statusName": { + "format": "string", + "type": "string" + }, + "apiBaseUrl": { + "format": "string", + "type": "string" + }, + "authType": { + "format": "string", + "type": "string" + }, + "authTypeName": { + "format": "string", + "type": "string" + }, + "rateLimitPerMinute": { + "format": "int", + "type": "integer" + }, + "rateLimitPerHour": { + "format": "int", + "type": "integer" + }, + "concurrencyLimit": { + "format": "int", + "type": "integer" + }, + "requestTimeoutMs": { + "format": "int", + "type": "integer" + }, + "maxRetries": { + "format": "int", + "type": "integer" + }, + "retryDelayMs": { + "format": "int", + "type": "integer" + }, + "createdBy": { + "format": "string", + "type": "string" + }, + "createdAt": { + "format": "int64", + "type": "integer" + }, + "updatedBy": { + "format": "string", + "type": "string" + }, + "updatedAt": { + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, + "dataengine.model.dto.dict.UpdateDatasourcePlatformReq": { + "description": "更新数据源平台配置", + "properties": { + "id": { + "description": "平台ID", + "format": "int64", + "type": "integer" + }, + "platformCode": { + "description": "平台编码", + "format": "string", + "type": "string" + }, + "platformName": { + "description": "平台名称", + "format": "string", + "type": "string" + }, + "description": { + "description": "平台描述", + "format": "string", + "type": "string" + }, + "status": { + "description": "平台状态", + "format": "api_feature.PlatformStatus", + "type": "string" + }, + "apiBaseUrl": { + "description": "API基础地址", + "format": "string", + "type": "string" + }, + "authType": { + "description": "认证类型", + "format": "string", + "type": "string" + }, + "token": { + "description": "认证token/密钥", + "format": "string", + "type": "string" + }, + "apiKey": { + "description": "API Key", + "format": "string", + "type": "string" + }, + "clientId": { + "description": "OAuth2 Client ID", + "format": "string", + "type": "string" + }, + "clientSecret": { + "description": "OAuth2 Client Secret", + "format": "string", + "type": "string" + }, + "rateLimitPerMinute": { + "description": "每分钟请求限制", + "format": "int", + "type": "integer" + }, + "rateLimitPerHour": { + "description": "每小时请求限制", + "format": "int", + "type": "integer" + }, + "concurrencyLimit": { + "description": "并发连接限制", + "format": "int", + "type": "integer" + }, + "requestTimeoutMs": { + "description": "请求超时时间(毫秒)", + "format": "int", + "type": "integer" + }, + "maxRetries": { + "description": "最大重试次数", + "format": "int", + "type": "integer" + }, + "retryDelayMs": { + "description": "重试延迟(毫秒)", + "format": "int", + "type": "integer" + }, + "authConfig": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "自定义认证配置(JSON)", + "format": "map[string]interface {}", + "type": "object" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.model.dto.dict.UpdateDatasourcePlatformStatusReq": { + "description": "更新数据源平台状态", + "properties": { + "id": { + "description": "平台ID", + "format": "int64", + "type": "integer" + }, + "status": { + "description": "状态：ACTIVE启用/INACTIVE停用", + "enum": [ + "ACTIVE", + "INACTIVE" + ], + "format": "api_feature.PlatformStatus", + "type": "string" + } + }, + "required": [ + "id", + "status" + ], + "type": "object" + }, + "dataengine.controller.public.ClearCacheReq": { + "properties": {}, + "type": "object" + }, + "dataengine.model.dto.public.TableListRes": { + "properties": { + "list": { + "description": "表列表", + "format": "[]public.TableInfo", + "items": { + "$ref": "#/components/schemas/dataengine.model.dto.public.TableInfo", + "description": "" + }, + "type": "array" + } + }, + "type": "object" + }, + "dataengine.model.dto.public.TableInfo": { + "properties": { + "tableName": { + "description": "表名", + "format": "string", + "type": "string" + }, + "platformName": { + "description": "平台名称", + "format": "string", + "type": "string" + }, + "interfaceName": { + "description": "接口名称", + "format": "string", + "type": "string" + }, + "columns": { + "description": "可用字段列表", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "dataengine.model.dto.public.QueryReq": { + "description": "支持字段/表名/过滤/分组/分页的通用查询接口", + "properties": { + "table": { + "description": "表名（白名单限制）", + "format": "string", + "type": "string" + }, + "fields": { + "description": "查询字段，逗号分隔，默认 *", + "format": "string", + "type": "string" + }, + "where": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "过滤条件，支持操作符: =, !=, >, <, >=, <=, like, in, between", + "format": "map[string]interface {}", + "type": "object" + }, + "groupBy": { + "description": "分组字段，逗号分隔", + "format": "string", + "type": "string" + }, + "orderBy": { + "description": "排序字段，如: create_time desc", + "format": "string", + "type": "string" + }, + "page": { + "default": 1, + "description": "页码，默认1", + "format": "int", + "type": "integer" + }, + "pageSize": { + "default": 20, + "description": "每页条数，默认20，最大100", + "format": "int", + "type": "integer" + } + }, + "required": [ + "table" + ], + "type": "object" + }, + "dataengine.model.dto.public.QueryRes": { + "properties": { + "list": { + "description": "数据列表", + "format": "[]map[string]interface {}", + "items": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "format": "map[string]interface {}", + "type": "object" + }, + "type": "array" + }, + "total": { + "description": "总数", + "format": "int64", + "type": "integer" + }, + "page": { + "description": "当前页码", + "format": "int", + "type": "integer" + }, + "size": { + "description": "每页条数", + "format": "int", + "type": "integer" + } + }, + "type": "object" + }, + "dataengine.model.dto.public.TableListReq": { + "description": "获取配置了表结构的同步表列表", + "properties": {}, + "type": "object" + }, + "dataengine.model.dto.public.ColumnListReq": { + "description": "获取指定表的字段列表", + "properties": { + "table": { + "description": "表名", + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.model.dto.public.ColumnListRes": { + "properties": { + "tableName": { + "description": "表名", + "format": "string", + "type": "string" + }, + "columns": { + "description": "字段列表", + "format": "[]public.Column", + "items": { + "$ref": "#/components/schemas/dataengine.model.dto.public.Column", + "description": "" + }, + "type": "array" + } + }, + "type": "object" + }, + "dataengine.model.dto.public.Column": { + "properties": { + "name": { + "description": "字段名", + "format": "string", + "type": "string" + }, + "type": { + "description": "字段类型", + "format": "string", + "type": "string" + }, + "comment": { + "description": "字段说明", + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.controller.report.autoCreateTableReq": { + "properties": { + "businessCode": { + "description": "业务编码", + "format": "string", + "type": "string" + }, + "reportCode": { + "description": "报表编码", + "format": "string", + "type": "string" + }, + "creator": { + "description": "创建人", + "format": "string", + "type": "string" + } + }, + "required": [ + "businessCode", + "reportCode" + ], + "type": "object", + "example": { + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "creator": "admin" + } + }, + "dataengine.common.report.model.AutoCreateStatTableResp": { + "properties": { + "success": { + "description": "是否成功", + "format": "bool", + "type": "boolean" + }, + "tableName": { + "description": "创建的表名", + "format": "string", + "type": "string" + }, + "columnCount": { + "description": "字段数量", + "format": "int", + "type": "integer" + }, + "execTimeMs": { + "description": "执行耗时(毫秒)", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, + "dataengine.controller.report.backfillReq": { + "properties": { + "businessCode": { + "description": "业务编码", + "format": "string", + "type": "string" + }, + "reportCode": { + "description": "报表编码", + "format": "string", + "type": "string" + }, + "startDate": { + "description": "开始日期 yyyy-MM-dd", + "format": "string", + "type": "string" + }, + "endDate": { + "description": "结束日期 yyyy-MM-dd", + "format": "string", + "type": "string" + }, + "executor": { + "description": "执行人", + "format": "string", + "type": "string" + } + }, + "required": [ + "businessCode", + "reportCode", + "startDate", + "endDate" + ], + "type": "object", + "example": { + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "startDate": "2026-06-01", + "endDate": "2026-06-17", + "executor": "admin" + } + }, + "dataengine.common.report.model.BackfillResp": { + "properties": { + "success": { + "description": "是否全部成功", + "format": "bool", + "type": "boolean" + }, + "totalDays": { + "description": "总天数", + "format": "int", + "type": "integer" + }, + "successDays": { + "description": "成功天数", + "format": "int", + "type": "integer" + }, + "failDays": { + "description": "失败天数", + "format": "int", + "type": "integer" + }, + "execTimeMs": { + "description": "总耗时(毫秒)", + "format": "int64", + "type": "integer" + }, + "errorMsg": { + "description": "错误信息", + "format": "string", + "type": "string" + } + }, + "type": "object", + "example": { + "success": true, + "totalDays": 30, + "successDays": 30, + "execTimeMs": 95000 + } + }, + "dataengine.controller.report.deleteBusinessReq": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.controller.report.deleteBusinessRes": { + "properties": { + "success": { + "format": "bool", + "type": "boolean" + }, + "message": { + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.controller.report.getBusinessReq": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.controller.report.getBusinessRes": { + "properties": { + "data": { + "$ref": "#/components/schemas/dataengine.common.report.model.BusinessConfig", + "description": "" + } + }, + "type": "object" + }, + "dataengine.common.report.model.BusinessConfig": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + }, + "tenantId": { + "format": "uint64", + "type": "integer" + }, + "creator": { + "format": "string", + "type": "string" + }, + "createdAt": { + "format": "*gtime.Time", + "type": "string" + }, + "updater": { + "format": "string", + "type": "string" + }, + "updatedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "deletedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "businessCode": { + "format": "string", + "type": "string" + }, + "businessName": { + "format": "string", + "type": "string" + }, + "description": { + "format": "string", + "type": "string" + }, + "status": { + "format": "string", + "type": "string" + }, + "config": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "format": "map[string]interface {}", + "type": "object" + } + }, + "type": "object", + "example": { + "id": 1, + "businessCode": "KUAISHOU", + "businessName": "快手电商", + "status": "ACTIVE" + } + }, + "dataengine.controller.report.saveBusinessReq": { + "properties": { + "id": { + "format": "*int64", + "type": "integer" + }, + "businessCode": { + "description": "业务编码", + "format": "string", + "type": "string" + }, + "businessName": { + "description": "业务名称", + "format": "string", + "type": "string" + }, + "description": { + "description": "描述", + "format": "string", + "type": "string" + }, + "status": { + "default": "ACTIVE", + "description": "状态 ACTIVE/INACTIVE", + "format": "string", + "type": "string" + }, + "config": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "扩展配置", + "format": "map[string]interface {}", + "type": "object" + } + }, + "required": [ + "businessCode", + "businessName" + ], + "type": "object", + "example": { + "id": null, + "businessCode": "KUAISHOU", + "businessName": "快手电商", + "description": "快手电商业务", + "status": "ACTIVE", + "config": { + "sourceSystem": "kuaishou", + "owner": "data-team" + } + } + }, + "dataengine.controller.report.saveBusinessRes": { + "properties": { + "success": { + "format": "bool", + "type": "boolean" + }, + "id": { + "format": "int64", + "type": "integer" + }, + "message": { + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.controller.report.listBusinessesReq": { + "properties": {}, + "type": "object" + }, + "dataengine.controller.report.listBusinessesRes": { + "properties": { + "list": { + "format": "[]model.BusinessConfig", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.BusinessConfig", + "description": "" + }, + "type": "array" + } + }, + "type": "object" + }, + "dataengine.controller.report.extractDataReq": { + "properties": { + "businessCode": { + "description": "业务编码", + "format": "string", + "type": "string" + }, + "reportCode": { + "description": "报表编码", + "format": "string", + "type": "string" + }, + "statDate": { + "description": "统计日期 yyyy-MM-dd", + "format": "string", + "type": "string" + }, + "executor": { + "description": "执行人", + "format": "string", + "type": "string" + } + }, + "required": [ + "businessCode", + "reportCode", + "statDate" + ], + "type": "object", + "example": { + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "statDate": "2026-06-17", + "executor": "admin" + } + }, + "dataengine.common.report.model.ExtractDailyDataResp": { + "properties": { + "success": { + "description": "是否成功", + "format": "bool", + "type": "boolean" + }, + "totalCount": { + "description": "总记录数", + "format": "int", + "type": "integer" + }, + "successCount": { + "description": "成功记录数", + "format": "int", + "type": "integer" + }, + "failCount": { + "description": "失败记录数", + "format": "int", + "type": "integer" + }, + "execTimeMs": { + "description": "执行耗时(毫秒)", + "format": "int64", + "type": "integer" + }, + "errorMsg": { + "description": "错误信息", + "format": "string", + "type": "string" + } + }, + "type": "object", + "example": { + "success": true, + "totalCount": 31, + "successCount": 31, + "execTimeMs": 3200 + } + }, + "dataengine.controller.report.deleteExtractConfigReq": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.controller.report.deleteExtractConfigRes": { + "properties": { + "success": { + "format": "bool", + "type": "boolean" + }, + "message": { + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.controller.report.getExtractConfigReq": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.controller.report.getExtractConfigRes": { + "properties": { + "data": { + "$ref": "#/components/schemas/dataengine.common.report.model.ExtractConfig", + "description": "" + } + }, + "type": "object" + }, + "dataengine.common.report.model.ExtractConfig": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + }, + "tenantId": { + "format": "uint64", + "type": "integer" + }, + "creator": { + "format": "string", + "type": "string" + }, + "createdAt": { + "format": "*gtime.Time", + "type": "string" + }, + "updater": { + "format": "string", + "type": "string" + }, + "updatedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "deletedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "businessCode": { + "format": "string", + "type": "string" + }, + "reportCode": { + "format": "string", + "type": "string" + }, + "extractCode": { + "format": "string", + "type": "string" + }, + "extractName": { + "format": "string", + "type": "string" + }, + "sourceTableName": { + "format": "string", + "type": "string" + }, + "sourceTableAlias": { + "format": "string", + "type": "string" + }, + "targetTableName": { + "format": "string", + "type": "string" + }, + "isEnabled": { + "format": "bool", + "type": "boolean" + }, + "extractType": { + "format": "string", + "type": "string" + }, + "extractMode": { + "format": "string", + "type": "string" + }, + "extractKeyField": { + "format": "string", + "type": "string" + }, + "extractKeyFormat": { + "format": "string", + "type": "string" + }, + "groupByFields": { + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "filterExpression": { + "format": "string", + "type": "string" + }, + "joinConfigs": { + "format": "[]model.JoinConfig", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.JoinConfig", + "description": "" + }, + "type": "array" + }, + "fieldMappings": { + "format": "[]model.FieldMapping", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.FieldMapping", + "description": "" + }, + "type": "array" + }, + "transformRules": { + "format": "[]model.TransformRule", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.TransformRule", + "description": "" + }, + "type": "array" + }, + "batchSize": { + "format": "int", + "type": "integer" + }, + "status": { + "format": "string", + "type": "string" + } + }, + "type": "object", + "example": { + "id": 1, + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "extractCode": "shop_daily_direct", + "extractName": "店铺日订单", + "extractMode": "DIRECT" + } + }, + "dataengine.common.report.model.JoinConfig": { + "properties": { + "joinTable": { + "format": "string", + "type": "string" + }, + "joinAlias": { + "format": "string", + "type": "string" + }, + "joinType": { + "format": "string", + "type": "string" + }, + "joinCondition": { + "format": "string", + "type": "string" + }, + "fieldMappings": { + "format": "[]model.FieldMapping", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.FieldMapping", + "description": "" + }, + "type": "array" + } + }, + "type": "object" + }, + "dataengine.common.report.model.FieldMapping": { + "properties": { + "sourceField": { + "format": "string", + "type": "string" + }, + "targetField": { + "format": "string", + "type": "string" + }, + "fieldType": { + "format": "string", + "type": "string" + }, + "aggregateFunction": { + "format": "string", + "type": "string" + }, + "defaultValue": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "transformRule": { + "$ref": "#/components/schemas/dataengine.common.report.model.TransformRule", + "description": "" + } + }, + "type": "object" + }, + "dataengine.common.report.model.TransformRule": { + "properties": { + "ruleCode": { + "format": "string", + "type": "string" + }, + "ruleType": { + "format": "string", + "type": "string" + }, + "expression": { + "format": "string", + "type": "string" + }, + "format": { + "format": "string", + "type": "string" + }, + "mapping": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "format": "map[string]interface {}", + "type": "object" + } + }, + "type": "object" + }, + "dataengine.controller.report.saveExtractConfigReq": { + "properties": { + "id": { + "format": "*int64", + "type": "integer" + }, + "businessCode": { + "description": "业务编码", + "format": "string", + "type": "string" + }, + "reportCode": { + "description": "报表编码", + "format": "string", + "type": "string" + }, + "extractCode": { + "description": "抽取编码", + "format": "string", + "type": "string" + }, + "extractName": { + "description": "抽取名称", + "format": "string", + "type": "string" + }, + "sourceTableName": { + "description": "源表名", + "format": "string", + "type": "string" + }, + "sourceTableAlias": { + "description": "源表别名", + "format": "string", + "type": "string" + }, + "targetTableName": { + "description": "目标表名", + "format": "string", + "type": "string" + }, + "isEnabled": { + "default": true, + "description": "是否启用", + "format": "bool", + "type": "boolean" + }, + "extractType": { + "default": "INCREMENTAL", + "description": "抽取类型 FULL/INCREMENTAL", + "format": "string", + "type": "string" + }, + "extractMode": { + "default": "DIRECT", + "description": "抽取模式 DIRECT/AGGREGATE", + "format": "string", + "type": "string" + }, + "extractKeyField": { + "description": "抽取关键字段（增量依据）", + "format": "string", + "type": "string" + }, + "extractKeyFormat": { + "description": "关键字段格式", + "format": "string", + "type": "string" + }, + "groupByFields": { + "description": "GROUP BY 字段列表", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "filterExpression": { + "description": "过滤表达式", + "format": "string", + "type": "string" + }, + "joinConfigs": { + "description": "JOIN配置", + "format": "[]model.JoinConfig", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.JoinConfig", + "description": "" + }, + "type": "array" + }, + "fieldMappings": { + "description": "字段映射列表", + "format": "[]model.FieldMapping", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.FieldMapping", + "description": "" + }, + "type": "array" + }, + "transformRules": { + "description": "转换规则列表", + "format": "[]model.TransformRule", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.TransformRule", + "description": "" + }, + "type": "array" + }, + "batchSize": { + "default": 1000, + "description": "批处理大小", + "format": "int", + "type": "integer" + }, + "status": { + "default": "ACTIVE", + "description": "状态", + "format": "string", + "type": "string" + } + }, + "required": [ + "businessCode", + "reportCode", + "extractCode", + "extractName", + "sourceTableName", + "targetTableName" + ], + "type": "object", + "example": { + "id": 9, + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "extractCode": "shop_daily_direct", + "extractName": "店铺日订单(逐行,可自由组合查询)", + "sourceTableName": "demo_kuaishou_orders", + "sourceTableAlias": "o", + "targetTableName": "stat_kuaishou_shop_daily", + "isEnabled": true, + "extractType": "INCREMENTAL", + "extractMode": "DIRECT", + "extractKeyField": "created_at", + "extractKeyFormat": "yyyy-MM-dd", + "groupByFields": [], + "filterExpression": "o.order_status = '已完成'", + "fieldMappings": [], + "joinConfigs": [], + "transformRules": [], + "batchSize": 1000, + "status": "ACTIVE" + } + }, + "dataengine.controller.report.saveExtractConfigRes": { + "properties": { + "success": { + "format": "bool", + "type": "boolean" + }, + "id": { + "format": "int64", + "type": "integer" + }, + "message": { + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.controller.report.getExtractConfigsReq": { + "properties": { + "businessCode": { + "format": "string", + "type": "string" + }, + "reportCode": { + "format": "string", + "type": "string" + } + }, + "required": [ + "businessCode", + "reportCode" + ], + "type": "object" + }, + "dataengine.controller.report.getExtractConfigsRes": { + "properties": { + "list": { + "format": "[]model.ExtractConfig", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.ExtractConfig", + "description": "" + }, + "type": "array" + } + }, + "type": "object" + }, + "dataengine.controller.report.deleteFieldReq": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.controller.report.deleteFieldRes": { + "properties": { + "success": { + "format": "bool", + "type": "boolean" + }, + "message": { + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.controller.report.getFieldReq": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.controller.report.getFieldRes": { + "properties": { + "data": { + "$ref": "#/components/schemas/dataengine.common.report.model.FieldConfig", + "description": "" + } + }, + "type": "object" + }, + "dataengine.common.report.model.FieldConfig": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + }, + "tenantId": { + "format": "uint64", + "type": "integer" + }, + "creator": { + "format": "string", + "type": "string" + }, + "createdAt": { + "format": "*gtime.Time", + "type": "string" + }, + "updater": { + "format": "string", + "type": "string" + }, + "updatedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "deletedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "businessCode": { + "format": "string", + "type": "string" + }, + "reportCode": { + "format": "string", + "type": "string" + }, + "fieldCode": { + "format": "string", + "type": "string" + }, + "fieldName": { + "format": "string", + "type": "string" + }, + "fieldType": { + "format": "string", + "type": "string" + }, + "dataType": { + "format": "string", + "type": "string" + }, + "fieldRole": { + "format": "string", + "type": "string" + }, + "isAggregatable": { + "format": "bool", + "type": "boolean" + }, + "isFilterable": { + "format": "bool", + "type": "boolean" + }, + "isQueryable": { + "format": "bool", + "type": "boolean" + }, + "isSortable": { + "format": "bool", + "type": "boolean" + }, + "defaultAggregate": { + "format": "string", + "type": "string" + }, + "validAggregates": { + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "filterOperators": { + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "expression": { + "format": "string", + "type": "string" + }, + "expressionType": { + "format": "string", + "type": "string" + }, + "formatPattern": { + "format": "string", + "type": "string" + }, + "unit": { + "format": "string", + "type": "string" + }, + "dictCode": { + "format": "string", + "type": "string" + }, + "sortOrder": { + "format": "int", + "type": "integer" + }, + "groupName": { + "format": "string", + "type": "string" + }, + "status": { + "format": "string", + "type": "string" + } + }, + "type": "object", + "example": { + "id": 1, + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "fieldCode": "order_amount", + "fieldName": "订单金额", + "fieldType": "FLOAT", + "fieldRole": "INDICATOR", + "defaultAggregate": "SUM" + } + }, + "dataengine.controller.report.saveFieldReq": { + "properties": { + "id": { + "format": "*int64", + "type": "integer" + }, + "businessCode": { + "description": "业务编码", + "format": "string", + "type": "string" + }, + "reportCode": { + "description": "报表编码", + "format": "string", + "type": "string" + }, + "fieldCode": { + "description": "字段编码", + "format": "string", + "type": "string" + }, + "fieldName": { + "description": "字段名称", + "format": "string", + "type": "string" + }, + "fieldType": { + "description": "字段类型 STRING/INT/FLOAT/DATE/DATETIME/JSONB", + "format": "string", + "type": "string" + }, + "dataType": { + "default": "STRING", + "description": "数据存储类型", + "format": "string", + "type": "string" + }, + "fieldRole": { + "description": "字段角色 DIMENSION/INDICATOR/FILTER/FILTER_ONLY", + "format": "string", + "type": "string" + }, + "isAggregatable": { + "description": "是否可聚合", + "format": "bool", + "type": "boolean" + }, + "isFilterable": { + "default": true, + "description": "是否可筛选", + "format": "bool", + "type": "boolean" + }, + "isQueryable": { + "default": true, + "description": "是否可查询", + "format": "bool", + "type": "boolean" + }, + "isSortable": { + "default": true, + "description": "是否可排序", + "format": "bool", + "type": "boolean" + }, + "defaultAggregate": { + "description": "默认聚合方式", + "format": "string", + "type": "string" + }, + "validAggregates": { + "description": "可选聚合列表", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "filterOperators": { + "description": "可选操作符列表", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "expression": { + "description": "表达式（衍生字段）", + "format": "string", + "type": "string" + }, + "expressionType": { + "description": "表达式类型 DIRECT/CALCULATED", + "format": "string", + "type": "string" + }, + "formatPattern": { + "description": "格式化模板", + "format": "string", + "type": "string" + }, + "unit": { + "description": "单位", + "format": "string", + "type": "string" + }, + "dictCode": { + "description": "字典编码", + "format": "string", + "type": "string" + }, + "sortOrder": { + "description": "排序", + "format": "int", + "type": "integer" + }, + "groupName": { + "description": "分组名称", + "format": "string", + "type": "string" + }, + "status": { + "default": "ACTIVE", + "description": "状态", + "format": "string", + "type": "string" + } + }, + "required": [ + "businessCode", + "reportCode", + "fieldCode", + "fieldName", + "fieldType", + "fieldRole" + ], + "type": "object", + "example": { + "id": null, + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "fieldCode": "order_amount", + "fieldName": "订单金额", + "fieldType": "FLOAT", + "dataType": "FLOAT", + "fieldRole": "INDICATOR", + "isAggregatable": true, + "isFilterable": true, + "isQueryable": true, + "isSortable": true, + "defaultAggregate": "SUM", + "validAggregates": [ + "SUM", + "COUNT", + "AVG", + "MAX", + "MIN" + ], + "filterOperators": [ + "=", + "!=", + ">", + "<", + ">=", + "<=" + ], + "expression": "", + "expressionType": "DIRECT", + "formatPattern": "#,##0.00", + "unit": "分", + "dictCode": "", + "sortOrder": 1, + "groupName": "交易指标", + "status": "ACTIVE" + } + }, + "dataengine.controller.report.saveFieldRes": { + "properties": { + "success": { + "format": "bool", + "type": "boolean" + }, + "id": { + "format": "int64", + "type": "integer" + }, + "message": { + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.controller.report.getReportFieldsReq": { + "properties": { + "businessCode": { + "format": "string", + "type": "string" + }, + "reportCode": { + "format": "string", + "type": "string" + } + }, + "required": [ + "businessCode", + "reportCode" + ], + "type": "object" + }, + "dataengine.common.report.model.GetReportFieldsResp": { + "properties": { + "businessCode": { + "description": "业务编码", + "format": "string", + "type": "string" + }, + "reportCode": { + "description": "报表编码", + "format": "string", + "type": "string" + }, + "dimensions": { + "description": "维度字段列表", + "format": "[]model.FieldConfig", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.FieldConfig", + "description": "" + }, + "type": "array" + }, + "indicators": { + "description": "指标字段列表", + "format": "[]model.FieldConfig", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.FieldConfig", + "description": "" + }, + "type": "array" + }, + "filters": { + "description": "筛选字段列表", + "format": "[]model.FieldConfig", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.FieldConfig", + "description": "" + }, + "type": "array" + } + }, + "type": "object" + }, + "dataengine.controller.report.initTablesReq": { + "properties": {}, + "type": "object" + }, + "dataengine.controller.report.initTablesRes": { + "properties": { + "success": { + "format": "bool", + "type": "boolean" + }, + "message": { + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.controller.report.queryReportReq": { + "properties": { + "businessCode": { + "description": "业务编码", + "format": "string", + "type": "string" + }, + "reportCode": { + "description": "报表编码", + "format": "string", + "type": "string" + }, + "dimensions": { + "description": "统计维度列表，如 shop_id/anchor_id/date", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "indicators": { + "description": "统计指标列表(含聚合方式)", + "format": "[]model.IndicatorSelect", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.IndicatorSelect", + "description": "" + }, + "type": "array" + }, + "filters": { + "description": "筛选条件列表", + "format": "[]model.FilterCondition", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.FilterCondition", + "description": "" + }, + "type": "array" + }, + "timeRange": { + "$ref": "#/components/schemas/dataengine.common.report.model.TimeRange", + "description": "时间范围" + }, + "timeGroup": { + "description": "时间分组: day/week/month/quarter", + "format": "string", + "type": "string" + }, + "orderBy": { + "description": "排序条件", + "format": "[]model.OrderCondition", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.OrderCondition", + "description": "" + }, + "type": "array" + }, + "page": { + "default": 1, + "description": "页码", + "format": "int", + "type": "integer" + }, + "pageSize": { + "default": 20, + "description": "每页条数", + "format": "int", + "type": "integer" + } + }, + "required": [ + "businessCode", + "reportCode" + ], + "type": "object", + "example": { + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "dimensions": [ + "shop_id", + "stat_date" + ], + "indicators": [ + { + "fieldCode": "order_amount", + "aggregate": "SUM", + "alias": "总订单金额" + }, + { + "fieldCode": "order_count", + "aggregate": "COUNT", + "alias": "订单数" + } + ], + "filters": [ + { + "fieldCode": "shop_id", + "operator": "=", + "value": "KS001" + } + ], + "timeRange": { + "startDate": "2026-06-01", + "endDate": "2026-06-17" + }, + "timeGroup": "day", + "orderBy": [ + { + "fieldCode": "order_amount", + "direction": "DESC" + } + ], + "page": 1, + "pageSize": 20 + } + }, + "dataengine.common.report.model.IndicatorSelect": { + "properties": { + "fieldCode": { + "description": "字段编码", + "format": "string", + "type": "string" + }, + "aggregate": { + "description": "聚合方式: SUM/COUNT/AVG/MAX/MIN", + "format": "string", + "type": "string" + }, + "alias": { + "description": "别名", + "format": "string", + "type": "string" + } + }, + "type": "object", + "example": { + "fieldCode": "order_amount", + "aggregate": "SUM" + } + }, + "dataengine.common.report.model.FilterCondition": { + "properties": { + "fieldCode": { + "description": "字段编码", + "format": "string", + "type": "string" + }, + "operator": { + "description": "操作符: =/!=/>/=/<=/IN/LIKE/BETWEEN", + "format": "string", + "type": "string" + }, + "value": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "value2": { + "$ref": "#/components/schemas/interface", + "description": "" + } + }, + "type": "object", + "example": { + "fieldCode": "shop_id", + "operator": "=", + "value": "KS001" + } + }, + "dataengine.common.report.model.TimeRange": { + "properties": { + "startDate": { + "description": "开始日期 yyyy-MM-dd", + "format": "string", + "type": "string" + }, + "endDate": { + "description": "结束日期 yyyy-MM-dd", + "format": "string", + "type": "string" + } + }, + "type": "object", + "example": { + "startDate": "2026-06-01", + "endDate": "2026-06-17" + } + }, + "dataengine.common.report.model.OrderCondition": { + "properties": { + "fieldCode": { + "description": "字段编码", + "format": "string", + "type": "string" + }, + "direction": { + "description": "排序方向: ASC/DESC", + "format": "string", + "type": "string" + } + }, + "type": "object", + "example": { + "fieldCode": "order_amount", + "direction": "DESC" + } + }, + "dataengine.common.report.model.UserSelectQueryResp": { + "properties": { + "list": { + "description": "数据列表", + "format": "[]map[string]interface {}", + "items": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "format": "map[string]interface {}", + "type": "object" + }, + "type": "array" + }, + "total": { + "description": "总数", + "format": "int64", + "type": "integer" + }, + "page": { + "description": "当前页", + "format": "int", + "type": "integer" + }, + "pageSize": { + "description": "每页条数", + "format": "int", + "type": "integer" + }, + "totalPages": { + "description": "总页数", + "format": "int", + "type": "integer" + }, + "sql": { + "description": "执行的SQL(调试用)", + "format": "string", + "type": "string" + }, + "execTimeMs": { + "description": "执行耗时(毫秒)", + "format": "int64", + "type": "integer" + } + }, + "type": "object" + }, + "dataengine.controller.report.deleteReportReq": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.controller.report.deleteReportRes": { + "properties": { + "success": { + "format": "bool", + "type": "boolean" + }, + "message": { + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.controller.report.getReportReq": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + } + }, + "required": [ + "id" + ], + "type": "object" + }, + "dataengine.controller.report.getReportRes": { + "properties": { + "data": { + "$ref": "#/components/schemas/dataengine.common.report.model.ReportConfig", + "description": "" + } + }, + "type": "object" + }, + "dataengine.common.report.model.ReportConfig": { + "properties": { + "id": { + "format": "int64", + "type": "integer" + }, + "tenantId": { + "format": "uint64", + "type": "integer" + }, + "creator": { + "format": "string", + "type": "string" + }, + "createdAt": { + "format": "*gtime.Time", + "type": "string" + }, + "updater": { + "format": "string", + "type": "string" + }, + "updatedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "deletedAt": { + "format": "*gtime.Time", + "type": "string" + }, + "businessCode": { + "format": "string", + "type": "string" + }, + "reportCode": { + "format": "string", + "type": "string" + }, + "reportName": { + "format": "string", + "type": "string" + }, + "description": { + "format": "string", + "type": "string" + }, + "status": { + "format": "string", + "type": "string" + }, + "statTableName": { + "format": "string", + "type": "string" + }, + "statTableComment": { + "format": "string", + "type": "string" + }, + "dateField": { + "format": "string", + "type": "string" + }, + "primaryKeys": { + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "conflictKeys": { + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "config": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "format": "map[string]interface {}", + "type": "object" + }, + "extractConfigIds": { + "description": "抽取配置ID列表，逗号分隔", + "format": "string", + "type": "string" + } + }, + "type": "object", + "example": { + "id": 1, + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "reportName": "店铺日报", + "statTableName": "stat_kuaishou_shop_daily", + "dateField": "stat_date", + "extractConfigIds": "1,2" + } + }, + "dataengine.controller.report.saveReportReq": { + "properties": { + "id": { + "format": "*int64", + "type": "integer" + }, + "businessCode": { + "description": "业务编码", + "format": "string", + "type": "string" + }, + "reportCode": { + "description": "报表编码", + "format": "string", + "type": "string" + }, + "reportName": { + "description": "报表名称", + "format": "string", + "type": "string" + }, + "description": { + "description": "描述", + "format": "string", + "type": "string" + }, + "status": { + "default": "ACTIVE", + "description": "状态", + "format": "string", + "type": "string" + }, + "statTableName": { + "description": "统计宽表名", + "format": "string", + "type": "string" + }, + "statTableComment": { + "description": "统计宽表注释", + "format": "string", + "type": "string" + }, + "dateField": { + "default": "stat_date", + "description": "日期字段", + "format": "string", + "type": "string" + }, + "primaryKeys": { + "description": "主键字段", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "conflictKeys": { + "description": "冲突键（唯一索引）", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "config": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "扩展配置", + "format": "map[string]interface {}", + "type": "object" + } + }, + "required": [ + "businessCode", + "reportCode", + "reportName", + "statTableName" + ], + "type": "object", + "example": { + "id": null, + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "reportName": "店铺日报", + "description": "快手电商店铺每日统计报表", + "status": "ACTIVE", + "statTableName": "stat_kuaishou_shop_daily", + "statTableComment": "快手电商-店铺日报", + "dateField": "stat_date", + "primaryKeys": [ + "shop_id", + "stat_date" + ], + "conflictKeys": [ + "shop_id", + "stat_date" + ], + "config": { + "owner": "data-team", + "sourceSystem": "kuaishou" + } + } + }, + "dataengine.controller.report.saveReportRes": { + "properties": { + "success": { + "format": "bool", + "type": "boolean" + }, + "id": { + "format": "int64", + "type": "integer" + }, + "message": { + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.controller.report.saveReportWithFieldsReq": { + "properties": { + "id": { + "format": "*int64", + "type": "integer" + }, + "businessCode": { + "description": "业务编码", + "format": "string", + "type": "string" + }, + "reportCode": { + "description": "报表编码", + "format": "string", + "type": "string" + }, + "reportName": { + "description": "报表名称", + "format": "string", + "type": "string" + }, + "description": { + "description": "描述", + "format": "string", + "type": "string" + }, + "status": { + "default": "ACTIVE", + "description": "状态", + "format": "string", + "type": "string" + }, + "statTableName": { + "description": "统计宽表名", + "format": "string", + "type": "string" + }, + "statTableComment": { + "description": "统计宽表注释", + "format": "string", + "type": "string" + }, + "dateField": { + "default": "stat_date", + "description": "日期字段", + "format": "string", + "type": "string" + }, + "primaryKeys": { + "description": "主键字段", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "conflictKeys": { + "description": "冲突键（唯一索引）", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array" + }, + "config": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "扩展配置", + "format": "map[string]interface {}", + "type": "object" + }, + "fields": { + "description": "字段配置列表（全量替换，不在列表中的字段将被删除）", + "format": "[]model.SaveFieldReq", + "items": { + "$ref": "#/components/schemas/dataengine.controller.report.saveFieldReq", + "description": "" + }, + "type": "array" + } + }, + "required": [ + "businessCode", + "reportCode", + "reportName", + "statTableName" + ], + "type": "object", + "example": { + "id": null, + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "reportName": "店铺日报", + "description": "快手电商店铺每日统计报表", + "status": "ACTIVE", + "statTableName": "stat_kuaishou_shop_daily", + "statTableComment": "快手电商-店铺日报", + "dateField": "stat_date", + "primaryKeys": ["shop_id", "stat_date"], + "conflictKeys": ["shop_id", "stat_date"], + "fields": [ + { + "fieldCode": "shop_id", + "fieldName": "店铺ID", + "fieldType": "STRING", + "fieldRole": "DIMENSION", + "isFilterable": true, + "isQueryable": true, + "sortOrder": 1, + "status": "ACTIVE" + }, + { + "fieldCode": "order_amount", + "fieldName": "订单金额", + "fieldType": "FLOAT", + "fieldRole": "INDICATOR", + "isAggregatable": true, + "defaultAggregate": "SUM", + "validAggregates": ["SUM", "COUNT", "AVG"], + "sortOrder": 2, + "status": "ACTIVE" + } + ] + } + }, + "dataengine.controller.report.saveReportWithFieldsRes": { + "properties": { + "success": { + "format": "bool", + "type": "boolean" + }, + "id": { + "format": "int64", + "type": "integer" + }, + "message": { + "format": "string", + "type": "string" + } + }, + "type": "object" + }, + "dataengine.controller.report.listReportsReq": { + "properties": { + "businessCode": { + "format": "string", + "type": "string" + } + }, + "required": [ + "businessCode" + ], + "type": "object" + }, + "dataengine.controller.report.listReportsRes": { + "properties": { + "list": { + "format": "[]model.ReportConfig", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.ReportConfig", + "description": "" + }, + "type": "array" + } + }, + "type": "object" + }, + "dataengine.controller.sync.QueryPlatformConfigReq": { + "description": "查看平台和接口配置", + "properties": { + "platformCode": { + "description": "平台编码", + "format": "string", + "type": "string" + } + }, + "required": [ + "platformCode" + ], + "type": "object" + }, + "dataengine.controller.sync.QueryPlatformConfigRes": { + "properties": { + "platformName": { + "format": "string", + "type": "string" + }, + "platformCode": { + "format": "string", + "type": "string" + }, + "apiBaseUrl": { + "format": "string", + "type": "string" + }, + "authType": { + "format": "string", + "type": "string" + }, + "hasToken": { + "format": "bool", + "type": "boolean" + }, + "interfaceCount": { + "format": "int", + "type": "integer" + }, + "interfaces": { + "format": "EmbeddedStructDefinition", + "items": { + "properties": { + "code": { + "format": "string", + "type": "string" + }, + "name": { + "format": "string", + "type": "string" + }, + "url": { + "format": "string", + "type": "string" + }, + "hasTableDef": { + "format": "bool", + "type": "boolean" + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "dataengine.controller.sync.TriggerSyncReq": { + "description": "根据平台编码和接口编码触发数据同步", + "properties": { + "platformCode": { + "description": "平台编码", + "format": "string", + "type": "string" + }, + "interfaceCode": { + "description": "接口编码", + "format": "string", + "type": "string" + }, + "fullSync": { + "description": "是否全量同步，true=全量 false=增量", + "format": "bool", + "type": "boolean" + } + }, + "required": [ + "platformCode", + "interfaceCode" + ], + "type": "object" + }, + "dataengine.controller.sync.TriggerSyncRes": { + "properties": { + "success": { + "format": "bool", + "type": "boolean" + }, + "tableName": { + "format": "string", + "type": "string" + }, + "totalRows": { + "format": "int", + "type": "integer" + }, + "insertedRows": { + "format": "int", + "type": "integer" + }, + "duration": { + "format": "string", + "type": "string" + } + }, + "type": "object" + } + } + }, + "info": { + "title": "", + "version": "" + }, + "paths": { + "/api/interface/controller/createApiInterface": { + "post": { + "description": "创建新的数据接口", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.CreateApiInterfaceReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.CreateApiInterfaceRes" + } + } + }, + "description": "" + } + }, + "summary": "创建接口", + "tags": [ + "接口管理" + ] + } + }, + "/api/interface/controller/deleteApiInterface": { + "delete": { + "description": "删除接口", + "parameters": [ + { + "description": "接口ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/gitea.redpowerfuture.com.red-future.common.beans.ResponseEmpty" + } + } + }, + "description": "" + } + }, + "summary": "删除接口", + "tags": [ + "接口管理" + ] + } + }, + "/api/interface/controller/getApiInterface": { + "get": { + "description": "获取接口详情", + "parameters": [ + { + "description": "接口ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.GetApiInterfaceRes" + } + } + }, + "description": "" + } + }, + "summary": "获取接口详情", + "tags": [ + "接口管理" + ] + } + }, + "/api/interface/controller/listApiInterfaces": { + "get": { + "description": "获取接口列表", + "parameters": [ + { + "description": "平台ID", + "in": "query", + "name": "platformId", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.ListApiInterfaceRes" + } + } + }, + "description": "" + } + }, + "summary": "获取接口列表", + "tags": [ + "接口管理" + ] + } + }, + "/api/interface/controller/updateApiInterface": { + "post": { + "description": "更新接口配置", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.UpdateApiInterfaceReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/gitea.redpowerfuture.com.red-future.common.beans.ResponseEmpty" + } + } + }, + "description": "" + } + }, + "summary": "更新接口", + "tags": [ + "接口管理" + ] + } + }, + "/api/interface/controller/updateApiInterfaceStatus": { + "post": { + "description": "更新接口状态", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.UpdateApiInterfaceStatusReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/gitea.redpowerfuture.com.red-future.common.beans.ResponseEmpty" + } + } + }, + "description": "" + } + }, + "summary": "更新接口状态", + "tags": [ + "接口管理" + ] + } + }, + "/datasource/platform/controller/createDatasourcePlatform": { + "post": { + "description": "创建新的数据源平台配置", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.CreateDatasourcePlatformReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.CreateDatasourcePlatformRes" + } + } + }, + "description": "" + } + }, + "summary": "创建数据源平台", + "tags": [ + "数据源平台" + ] + } + }, + "/datasource/platform/controller/deleteDatasourcePlatform": { + "delete": { + "description": "删除数据源平台", + "parameters": [ + { + "description": "平台ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/gitea.redpowerfuture.com.red-future.common.beans.ResponseEmpty" + } + } + }, + "description": "" + } + }, + "summary": "删除数据源平台", + "tags": [ + "数据源平台" + ] + } + }, + "/datasource/platform/controller/getDatasourcePlatform": { + "get": { + "description": "获取数据源平台详情", + "parameters": [ + { + "description": "平台ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.GetDatasourcePlatformRes" + } + } + }, + "description": "" + } + }, + "summary": "获取数据源平台详情", + "tags": [ + "数据源平台" + ] + } + }, + "/datasource/platform/controller/getPlatformByCode": { + "get": { + "description": "根据编码获取平台信息", + "parameters": [ + { + "description": "平台编码", + "in": "query", + "name": "code", + "required": true, + "schema": { + "format": "string", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.GetPlatformByCodeRes" + } + } + }, + "description": "" + } + }, + "summary": "根据编码获取平台信息", + "tags": [ + "数据源平台" + ] + } + }, + "/datasource/platform/controller/getPlatformStatistics": { + "get": { + "description": "获取平台统计信息（接口数、同步状态等）", + "parameters": [ + { + "description": "平台ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.GetPlatformStatisticsRes" + } + } + }, + "description": "" + } + }, + "summary": "获取平台统计信息", + "tags": [ + "数据源平台" + ] + } + }, + "/datasource/platform/controller/listDatasourcePlatforms": { + "get": { + "description": "获取数据源平台列表", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.ListDatasourcePlatformRes" + } + } + }, + "description": "" + } + }, + "summary": "获取数据源平台列表", + "tags": [ + "数据源平台" + ] + } + }, + "/datasource/platform/controller/updateDatasourcePlatform": { + "post": { + "description": "更新数据源平台配置", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.UpdateDatasourcePlatformReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/gitea.redpowerfuture.com.red-future.common.beans.ResponseEmpty" + } + } + }, + "description": "" + } + }, + "summary": "更新数据源平台", + "tags": [ + "数据源平台" + ] + } + }, + "/datasource/platform/controller/updateDatasourcePlatformStatus": { + "post": { + "description": "更新数据源平台状态", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.dict.UpdateDatasourcePlatformStatusReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/gitea.redpowerfuture.com.red-future.common.beans.ResponseEmpty" + } + } + }, + "description": "" + } + }, + "summary": "更新数据源平台状态", + "tags": [ + "数据源平台" + ] + } + }, + "/public/query/controller/public/cache/clear": { + "delete": { + "description": "清除查询缓存", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.public.TableListRes" + } + } + }, + "description": "" + } + }, + "summary": "清除查询缓存", + "tags": [ + "公共查询" + ] + } + }, + "/public/query/controller/public/query": { + "post": { + "description": "执行通用 SQL 查询", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.public.QueryReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.public.QueryRes" + } + } + }, + "description": "" + } + }, + "summary": "执行公共查询", + "tags": [ + "公共查询" + ] + } + }, + "/public/query/controller/public/tables": { + "get": { + "description": "获取可查询表列表", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.public.TableListRes" + } + } + }, + "description": "" + } + }, + "summary": "获取表列表", + "tags": [ + "公共查询" + ] + } + }, + "/public/query/controller/public/tables/{table}/columns": { + "get": { + "description": "获取指定表的字段列表", + "parameters": [ + { + "description": "表名", + "in": "path", + "name": "table", + "required": true, + "schema": { + "format": "string", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.model.dto.public.ColumnListRes" + } + } + }, + "description": "" + } + }, + "summary": "获取表字段列表", + "tags": [ + "公共查询" + ] + } + }, + "/sync/ctrl/config": { + "get": { + "description": "查看平台和接口配置", + "parameters": [ + { + "description": "平台编码", + "in": "query", + "name": "platformCode", + "required": true, + "schema": { + "format": "string", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.sync.QueryPlatformConfigRes" + } + } + }, + "description": "" + } + }, + "summary": "查询平台配置", + "tags": [ + "平台同步" + ] + } + }, + "/sync/ctrl/trigger": { + "post": { + "description": "根据平台编码和接口编码触发数据同步", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.sync.TriggerSyncReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.sync.TriggerSyncRes" + } + } + }, + "description": "" + } + }, + "summary": "触发同步", + "tags": [ + "平台同步" + ] + } + }, + "/report/businesses": { + "get": { + "description": "获取所有业务配置列表", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.listBusinessesRes" + } + } + }, + "description": "" + } + }, + "summary": "业务列表", + "tags": [ + "报表引擎" + ] + } + }, + "/report/business": { + "get": { + "description": "获取业务配置详情", + "parameters": [ + { + "description": "业务ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.getBusinessRes" + } + } + }, + "description": "" + } + }, + "summary": "获取业务", + "tags": [ + "报表引擎" + ] + }, + "delete": { + "description": "删除业务配置", + "parameters": [ + { + "description": "业务ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.deleteBusinessRes" + } + } + }, + "description": "" + } + }, + "summary": "删除业务", + "tags": [ + "报表引擎" + ] + } + }, + "/report/business/save": { + "post": { + "description": "新增或修改业务配置，id为空则新增，有值则更新", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.saveBusinessReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.saveBusinessRes" + } + } + }, + "description": "" + } + }, + "summary": "保存业务", + "tags": [ + "报表引擎" + ] + } + }, + "/report/reports": { + "get": { + "description": "获取指定业务编码下的所有报表", + "parameters": [ + { + "description": "业务编码", + "in": "query", + "name": "businessCode", + "required": true, + "schema": { + "format": "string", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.listReportsRes" + } + } + }, + "description": "" + } + }, + "summary": "报表列表", + "tags": [ + "报表引擎" + ] + } + }, + "/report/report": { + "get": { + "description": "获取报表配置详情", + "parameters": [ + { + "description": "报表ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.getReportRes" + } + } + }, + "description": "" + } + }, + "summary": "获取报表", + "tags": [ + "报表引擎" + ] + }, + "delete": { + "description": "删除报表配置", + "parameters": [ + { + "description": "报表ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.deleteReportRes" + } + } + }, + "description": "" + } + }, + "summary": "删除报表", + "tags": [ + "报表引擎" + ] + } + }, + "/report/report/save": { + "post": { + "description": "新增或修改报表配置，id为空则新增，有值则更新", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.saveReportReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.saveReportRes" + } + } + }, + "description": "" + } + }, + "summary": "保存报表", + "tags": [ + "报表引擎" + ] + } + }, + "/report/report/saveWithFields": { + "post": { + "description": "保存报表及字段配置（全量替换），不在fields列表中的旧字段将被自动删除", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.saveReportWithFieldsReq", + "description": "" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.saveReportWithFieldsRes", + "description": "" + } + } + }, + "description": "" + } + }, + "summary": "保存报表及字段配置（全量替换）", + "tags": [ + "报表引擎" + ] + } + }, + "/report/fields": { + "get": { + "description": "获取指定报表的所有可用字段，按角色（维度/指标/筛选）分组", + "parameters": [ + { + "description": "业务编码", + "in": "query", + "name": "businessCode", + "required": true, + "schema": { + "format": "string", + "type": "string" + } + }, + { + "description": "报表编码", + "in": "query", + "name": "reportCode", + "required": true, + "schema": { + "format": "string", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.common.report.model.GetReportFieldsResp" + } + } + }, + "description": "" + } + }, + "summary": "报表字段列表（按角色分组）", + "tags": [ + "报表引擎" + ] + } + }, + "/report/field": { + "get": { + "description": "获取字段配置详情", + "parameters": [ + { + "description": "字段ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.getFieldRes" + } + } + }, + "description": "" + } + }, + "summary": "获取字段", + "tags": [ + "报表引擎" + ] + }, + "delete": { + "description": "删除字段配置", + "parameters": [ + { + "description": "字段ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.deleteFieldRes" + } + } + }, + "description": "" + } + }, + "summary": "删除字段", + "tags": [ + "报表引擎" + ] + } + }, + "/report/field/save": { + "post": { + "description": "新增或修改字段配置，id为空则新增，有值则更新", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.saveFieldReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.saveFieldRes" + } + } + }, + "description": "" + } + }, + "summary": "保存字段", + "tags": [ + "报表引擎" + ] + } + }, + "/report/extractConfigs": { + "get": { + "description": "获取指定报表的所有抽取配置", + "parameters": [ + { + "description": "业务编码", + "in": "query", + "name": "businessCode", + "required": true, + "schema": { + "format": "string", + "type": "string" + } + }, + { + "description": "报表编码", + "in": "query", + "name": "reportCode", + "required": true, + "schema": { + "format": "string", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.getExtractConfigsRes" + } + } + }, + "description": "" + } + }, + "summary": "抽取配置列表", + "tags": [ + "报表引擎" + ] + } + }, + "/report/extractConfig": { + "get": { + "description": "获取抽取配置详情", + "parameters": [ + { + "description": "抽取配置ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.getExtractConfigRes" + } + } + }, + "description": "" + } + }, + "summary": "获取抽取配置", + "tags": [ + "报表引擎" + ] + }, + "delete": { + "description": "删除抽取配置", + "parameters": [ + { + "description": "抽取配置ID", + "in": "query", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.deleteExtractConfigRes" + } + } + }, + "description": "" + } + }, + "summary": "删除抽取配置", + "tags": [ + "报表引擎" + ] + } + }, + "/report/extractConfig/save": { + "post": { + "description": "新增或修改抽取配置，id为空则新增，有值则更新", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.saveExtractConfigReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.saveExtractConfigRes" + } + } + }, + "description": "" + } + }, + "summary": "保存抽取配置", + "tags": [ + "报表引擎" + ] + } + }, + "/report/extract": { + "post": { + "description": "执行按天数据抽取，将源表数据抽取到统计宽表", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.extractDataReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.common.report.model.ExtractDailyDataResp" + } + } + }, + "description": "" + } + }, + "summary": "执行按天数据抽取", + "tags": [ + "报表引擎" + ] + } + }, + "/report/backfill": { + "post": { + "description": "批量回填指定日期范围内的数据", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.backfillReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.common.report.model.BackfillResp" + } + } + }, + "description": "" + } + }, + "summary": "批量回填数据", + "tags": [ + "报表引擎" + ] + } + }, + "/report/autoCreateTable": { + "post": { + "description": "根据报表字段配置自动创建/更新统计宽表", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.autoCreateTableReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.common.report.model.AutoCreateStatTableResp" + } + } + }, + "description": "" + } + }, + "summary": "自动创建统计宽表", + "tags": [ + "报表引擎" + ] + } + }, + "/report/query": { + "post": { + "description": "用户选择维度、指标、筛选条件、时间范围后查询数据", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.queryReportReq" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.common.report.model.UserSelectQueryResp" + } + } + }, + "description": "" + } + }, + "summary": "用户选择查询", + "tags": [ + "报表引擎" + ] + } + }, + "/report/initTables": { + "post": { + "description": "初始化报表引擎所需的系统表（如业务表、报表表、字段表、抽取配置表等）", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.initTablesRes" + } + } + }, + "description": "" + } + }, + "summary": "初始化系统表", + "tags": [ + "报表引擎" + ] + } + } + } +} \ No newline at end of file diff --git a/saveWithFields-openapi.json b/saveWithFields-openapi.json new file mode 100644 index 0000000..524a224 --- /dev/null +++ b/saveWithFields-openapi.json @@ -0,0 +1,468 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "报表引擎 - saveWithFields", + "version": "1.0.0" + }, + "servers": [ + { + "url": "http://localhost:3013", + "description": "本地开发" + } + ], + "paths": { + "/report/report/saveWithFields": { + "post": { + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.saveReportWithFieldsReq", + "description": "" + }, + "example": { + "id": null, + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "reportName": "店铺日报", + "description": "快手电商店铺每日统计报表", + "status": "ACTIVE", + "statTableName": "stat_kuaishou_shop_daily", + "statTableComment": "快手电商-店铺日报", + "dateField": "stat_date", + "primaryKeys": [ + "shop_id", + "stat_date" + ], + "conflictKeys": [ + "shop_id", + "stat_date" + ], + "config": { + "sync_frequency": "daily", + "retention_days": 90 + }, + "fields": [ + { + "id": null, + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "fieldCode": "shop_id", + "fieldName": "店铺ID", + "fieldType": "STRING", + "dataType": "STRING", + "fieldRole": "DIMENSION", + "isAggregatable": false, + "isFilterable": true, + "isQueryable": true, + "isSortable": false, + "defaultAggregate": "", + "validAggregates": [], + "filterOperators": [ + "=", + "!=", + "IN" + ], + "expression": "", + "expressionType": "DIRECT", + "formatPattern": "", + "unit": "", + "dictCode": "", + "sortOrder": 1, + "groupName": "基本信息", + "status": "ACTIVE" + }, + { + "id": null, + "businessCode": "KUAISHOU", + "reportCode": "shop_daily", + "fieldCode": "order_amount", + "fieldName": "订单金额", + "fieldType": "FLOAT", + "dataType": "STRING", + "fieldRole": "INDICATOR", + "isAggregatable": true, + "isFilterable": false, + "isQueryable": true, + "isSortable": true, + "defaultAggregate": "SUM", + "validAggregates": [ + "SUM", + "COUNT", + "AVG", + "MAX", + "MIN" + ], + "filterOperators": [], + "expression": "", + "expressionType": "DIRECT", + "formatPattern": "#,##0.00", + "unit": "元", + "dictCode": "", + "sortOrder": 2, + "groupName": "交易指标", + "status": "ACTIVE" + } + ] + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dataengine.controller.report.saveReportWithFieldsRes", + "description": "" + }, + "example": { + "success": true, + "id": 1, + "message": "报表及字段保存成功" + } + } + }, + "description": "" + } + }, + "summary": "保存报表及字段配置（全量替换）", + "tags": [ + "报表引擎" + ] + } + } + }, + "components": { + "schemas": { + "dataengine.controller.report.saveReportWithFieldsRes": { + "properties": { + "success": { + "format": "bool", + "type": "boolean", + "example": true + }, + "id": { + "format": "int64", + "type": "integer", + "example": 1 + }, + "message": { + "format": "string", + "type": "string", + "example": "报表及字段保存成功" + } + }, + "type": "object" + }, + "dataengine.controller.report.saveReportWithFieldsReq": { + "properties": { + "id": { + "format": "*int64", + "type": "integer" + }, + "businessCode": { + "description": "业务编码", + "format": "string", + "type": "string", + "example": "KUAISHOU" + }, + "reportCode": { + "description": "报表编码", + "format": "string", + "type": "string", + "example": "shop_daily" + }, + "reportName": { + "description": "报表名称", + "format": "string", + "type": "string", + "example": "店铺日报" + }, + "description": { + "description": "描述", + "format": "string", + "type": "string", + "example": "快手电商店铺每日统计报表" + }, + "status": { + "default": "ACTIVE", + "description": "状态", + "format": "string", + "type": "string", + "example": "ACTIVE" + }, + "statTableName": { + "description": "统计宽表名", + "format": "string", + "type": "string", + "example": "stat_kuaishou_shop_daily" + }, + "statTableComment": { + "description": "统计宽表注释", + "format": "string", + "type": "string", + "example": "快手电商-店铺日报" + }, + "dateField": { + "default": "stat_date", + "description": "日期字段", + "format": "string", + "type": "string", + "example": "stat_date" + }, + "primaryKeys": { + "description": "主键字段", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array", + "example": [ + "shop_id", + "stat_date" + ] + }, + "conflictKeys": { + "description": "冲突键（唯一索引）", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array", + "example": [ + "shop_id", + "stat_date" + ] + }, + "config": { + "additionalProperties": { + "$ref": "#/components/schemas/interface", + "description": "" + }, + "description": "扩展配置", + "format": "map[string]interface {}", + "type": "object" + }, + "fields": { + "description": "字段配置列表（全量替换，不在列表中的字段将被删除）", + "format": "[]model.SaveFieldReq", + "items": { + "$ref": "#/components/schemas/dataengine.common.report.model.SaveFieldReq", + "description": "" + }, + "type": "array", + "example": [ + { + "fieldCode": "shop_id", + "fieldName": "店铺ID", + "fieldType": "STRING", + "fieldRole": "DIMENSION", + "isFilterable": true, + "isQueryable": true, + "sortOrder": 1, + "status": "ACTIVE" + }, + { + "fieldCode": "order_amount", + "fieldName": "订单金额", + "fieldType": "FLOAT", + "fieldRole": "INDICATOR", + "isAggregatable": true, + "defaultAggregate": "SUM", + "validAggregates": [ + "SUM", + "COUNT", + "AVG" + ], + "sortOrder": 2, + "status": "ACTIVE" + } + ] + } + }, + "required": [ + "businessCode", + "reportCode", + "reportName", + "statTableName" + ], + "type": "object" + }, + "dataengine.common.report.model.SaveFieldReq": { + "properties": { + "id": { + "format": "*int64", + "type": "integer" + }, + "businessCode": { + "description": "业务编码", + "format": "string", + "type": "string" + }, + "reportCode": { + "description": "报表编码", + "format": "string", + "type": "string" + }, + "fieldCode": { + "description": "字段编码", + "format": "string", + "type": "string", + "example": "order_amount" + }, + "fieldName": { + "description": "字段名称", + "format": "string", + "type": "string", + "example": "订单金额" + }, + "fieldType": { + "description": "字段类型 STRING/INT/FLOAT/DATE/DATETIME/JSONB", + "format": "string", + "type": "string", + "example": "FLOAT" + }, + "dataType": { + "default": "STRING", + "description": "数据存储类型", + "format": "string", + "type": "string", + "example": "STRING" + }, + "fieldRole": { + "description": "字段角色 DIMENSION/INDICATOR/FILTER/FILTER_ONLY", + "format": "string", + "type": "string", + "example": "INDICATOR" + }, + "isAggregatable": { + "description": "是否可聚合", + "format": "bool", + "type": "boolean", + "example": true + }, + "isFilterable": { + "default": true, + "description": "是否可筛选", + "format": "bool", + "type": "boolean", + "example": true + }, + "isQueryable": { + "default": true, + "description": "是否可查询", + "format": "bool", + "type": "boolean", + "example": true + }, + "isSortable": { + "default": true, + "description": "是否可排序", + "format": "bool", + "type": "boolean", + "example": true + }, + "defaultAggregate": { + "description": "默认聚合方式", + "format": "string", + "type": "string", + "example": "SUM" + }, + "validAggregates": { + "description": "可选聚合列表", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array", + "example": [ + "SUM", + "COUNT", + "AVG" + ] + }, + "filterOperators": { + "description": "可选操作符列表", + "format": "[]string", + "items": { + "format": "string", + "type": "string" + }, + "type": "array", + "example": [ + "=", + "!=", + ">", + "<", + "IN" + ] + }, + "expression": { + "description": "表达式（衍生字段）", + "format": "string", + "type": "string", + "example": "" + }, + "expressionType": { + "description": "表达式类型 DIRECT/CALCULATED", + "format": "string", + "type": "string", + "example": "DIRECT" + }, + "formatPattern": { + "description": "格式化模板", + "format": "string", + "type": "string", + "example": "#,##0.00" + }, + "unit": { + "description": "单位", + "format": "string", + "type": "string", + "example": "元" + }, + "dictCode": { + "description": "字典编码", + "format": "string", + "type": "string", + "example": "" + }, + "sortOrder": { + "description": "排序", + "format": "int", + "type": "integer", + "example": 1 + }, + "groupName": { + "description": "分组名称", + "format": "string", + "type": "string", + "example": "交易指标" + }, + "status": { + "default": "ACTIVE", + "description": "状态", + "format": "string", + "type": "string", + "example": "ACTIVE" + } + }, + "required": [ + "fieldCode", + "fieldName", + "fieldType", + "fieldRole" + ], + "type": "object" + }, + "interface": { + "properties": {}, + "type": "object" + } + } + } +} \ No newline at end of file diff --git a/service/sync/dynamic_sync.go b/service/sync/dynamic_sync.go index 3a823fa..8b01048 100644 --- a/service/sync/dynamic_sync.go +++ b/service/sync/dynamic_sync.go @@ -15,6 +15,7 @@ import ( entity "dataengine/model/entity/dict" "gitea.redpowerfuture.com/red-future/common/db/gfdb" + "github.com/gogf/gf/v2/os/grpool" "github.com/sirupsen/logrus" ) @@ -595,16 +596,14 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon var mu sync.Mutex var wg sync.WaitGroup - sem := make(chan struct{}, concurrency) + pool := grpool.New(concurrency) globalMaxTime := lastSyncTime for idx, entityVal := range allEntities { + idx, val := idx, entityVal wg.Add(1) - sem <- struct{}{} - - go func(idx int, val interface{}) { + pool.Add(ctx, func(ctx context.Context) { defer wg.Done() - defer func() { <-sem }() logrus.Infof(" 处理实体 [%d/%d]: %v", idx+1, len(allEntities), val) entityMaxTime := int64(0) @@ -728,7 +727,7 @@ func syncWithPrefetch(ctx context.Context, api *ApiClient, platform *PlatformCon } mu.Unlock() } - }(idx, entityVal) + }) } wg.Wait()