red-future/data-engine
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

提交代码

Browse Source
This commit is contained in:
lmk
2026-07-02 10:23:07 +08:00
parent 7f6ec3a757
commit 802139c349
24 changed files with 7373 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/report/api.go
+402 -6
View File
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"strings"
"time"
"dataengine/common/report/config"
"dataengine/common/report/ddlsync"
@@ -13,8 +14,18 @@ import (
"gitea.redpowerfuture.com/red-future/common/beans"
"gitea.redpowerfuture.com/red-future/common/db/gfdb"
"github.com/gogf/gf/v2/database/gdb"
)
// getUserNameFromCtx 从上下文中获取当前用户名
func getUserNameFromCtx(ctx context.Context) string {
if user, ok := ctx.Value("user").(*beans.User); ok && user != nil && user.UserName != "" {
return user.UserName
}
return "system"
}
// ReportService 报表公共服务
// 对外暴露的统一接口
type ReportService struct {
@@ -108,7 +119,68 @@ func (s *ReportService) ExtractDailyData(ctx context.Context, businessCode, repo
}
// ============================================================
// 核心接口 3: 用户选择查询（最核心）
// 核心接口 3: 批量回填数据
// 首次接入时对日期范围逐天抽取
// ============================================================
// Backfill 批量回填数据
// 遍历 startDate → endDate 逐天调用 ExtractDailyData
// 返回汇总统计
func (s *ReportService) Backfill(ctx context.Context, req *model.BackfillReq) (*model.BackfillResp, error) {
start := time.Now()
startDate, err := time.Parse("2006-01-02", req.StartDate)
if err != nil {
return nil, fmt.Errorf("开始日期格式错误: %w", err)
}
endDate, err := time.Parse("2006-01-02", req.EndDate)
if err != nil {
return nil, fmt.Errorf("结束日期格式错误: %w", err)
}
if endDate.Before(startDate) {
return nil, fmt.Errorf("结束日期不能早于开始日期")
}
totalDays := 0
successDays := 0
failDays := 0
var lastErr error
for d := startDate; !d.After(endDate); d = d.AddDate(0, 0, 1) {
totalDays++
dateStr := d.Format("2006-01-02")
resp, err := s.ExtractDailyData(ctx, req.BusinessCode, req.ReportCode, dateStr, req.Executor)
if err != nil {
failDays++
lastErr = fmt.Errorf("回填 %s 失败: %w", dateStr, err)
continue
}
if !resp.Success {
failDays++
lastErr = fmt.Errorf("回填 %s 异常: %s", dateStr, resp.ErrorMsg)
continue
}
successDays++
}
execTime := time.Since(start).Milliseconds()
resp := &model.BackfillResp{
Success: failDays == 0,
TotalDays: totalDays,
SuccessDays: successDays,
FailDays: failDays,
ExecTimeMs: execTime,
}
if lastErr != nil {
resp.ErrorMsg = lastErr.Error()
}
return resp, nil
}
// ============================================================
// 核心接口 4: 用户选择查询（最核心）
// 前端用户选择条件 → 实时构建SQL → 返回报表数据
// ============================================================
@@ -145,11 +217,21 @@ func (s *ReportService) GetReportFields(ctx context.Context, businessCode, repor
}
// GetAllBusinesses 获取所有启用业务列表
// ListBusinesses 分页获取业务列表
func (s *ReportService) ListBusinesses(ctx context.Context, pageNum, pageSize int) ([]model.BusinessConfig, int, error) {
return s.configLoader.ListBusinesses(ctx, pageNum, pageSize)
}
func (s *ReportService) GetAllBusinesses(ctx context.Context) ([]model.BusinessConfig, error) {
return s.configLoader.GetAllBusinesses(ctx)
}
// GetAllReports 获取业务下所有报表列表
// ListReports 分页获取报表列表
func (s *ReportService) ListReports(ctx context.Context, businessCode, reportName string, pageNum, pageSize int) ([]model.ReportConfig, int, error) {
return s.configLoader.ListReports(ctx, businessCode, reportName, pageNum, pageSize)
}
func (s *ReportService) GetAllReports(ctx context.Context, businessCode string) ([]model.ReportConfig, error) {
return s.configLoader.GetAllReports(ctx, businessCode)
}
@@ -174,8 +256,20 @@ func (s *ReportService) SaveBusiness(ctx context.Context, req *model.SaveBusines
return nil, fmt.Errorf("初始化系统表失败: %w", err)
}
// 新增时校验 businessCode 唯一性
if req.ID == nil || *req.ID == 0 {
r, err := gfdb.DB(ctx).GetAll(ctx, "SELECT id FROM report_business_config WHERE business_code = $1 LIMIT 1", req.BusinessCode)
if err != nil {
return nil, fmt.Errorf("校验业务编码失败: %w", err)
}
if !r.IsEmpty() {
return nil, fmt.Errorf("业务编码 %s 已存在", req.BusinessCode)
}
}
userName := getUserNameFromCtx(ctx)
biz := &model.BusinessConfig{
SQLBaseDO: beans.SQLBaseDO{Creator: req.Operator, Updater: req.Operator},
SQLBaseDO: beans.SQLBaseDO{Creator: userName, Updater: userName, TenantId: 1},
BusinessCode: req.BusinessCode,
BusinessName: req.BusinessName,
Description: req.Description,
@@ -213,6 +307,14 @@ func (s *ReportService) DeleteBusiness(ctx context.Context, id int64) (*model.De
if err != nil {
return nil, err
}
// 级联检查：该业务下是否存在报表
r, err := gfdb.DB(ctx).GetAll(ctx, "SELECT COUNT(1) AS cnt FROM report_report_config WHERE business_code = $1 AND deleted_at IS NULL LIMIT 1", biz.BusinessCode)
if err == nil && !r.IsEmpty() {
cnt := r[0]["cnt"].Int()
if cnt > 0 {
return &model.DeleteResult{Success: false, Message: fmt.Sprintf("业务 %s 下存在 %d 个报表配置，请先删除关联报表后再删除业务", biz.BusinessCode, cnt)}, nil
}
}
if err := s.configLoader.DeleteBusiness(ctx, id, biz.BusinessCode); err != nil {
return nil, err
}
@@ -234,8 +336,21 @@ func (s *ReportService) SaveReport(ctx context.Context, req *model.SaveReportReq
return nil, fmt.Errorf("初始化系统表失败: %w", err)
}
// 新增时校验 reportCode 唯一性（同业务下）
if req.ID == nil || *req.ID == 0 {
r, err := gfdb.DB(ctx).GetAll(ctx, "SELECT id FROM report_report_config WHERE business_code = $1 AND report_code = $2 LIMIT 1",
req.BusinessCode, req.ReportCode)
if err != nil {
return nil, fmt.Errorf("校验报表编码失败: %w", err)
}
if !r.IsEmpty() {
return nil, fmt.Errorf("业务 %s 下报表编码 %s 已存在", req.BusinessCode, req.ReportCode)
}
}
userName := getUserNameFromCtx(ctx)
rpt := &model.ReportConfig{
SQLBaseDO: beans.SQLBaseDO{Creator: req.Operator, Updater: req.Operator},
SQLBaseDO: beans.SQLBaseDO{Creator: userName, Updater: userName, TenantId: 1},
BusinessCode: req.BusinessCode,
ReportCode: req.ReportCode,
ReportName: req.ReportName,
@@ -286,6 +401,18 @@ func (s *ReportService) DeleteReport(ctx context.Context, id int64) (*model.Dele
if err != nil {
return nil, err
}
// 级联检查：该报表下是否存在未删除的字段
r, err := gfdb.DB(ctx).GetAll(ctx, "SELECT COUNT(1) AS cnt FROM report_field_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL LIMIT 1",
rpt.BusinessCode, rpt.ReportCode)
if err == nil && !r.IsEmpty() && r[0]["cnt"].Int() > 0 {
return &model.DeleteResult{Success: false, Message: fmt.Sprintf("报表 %s 下存在字段配置，请先删除关联字段后再删除报表", rpt.ReportCode)}, nil
}
// 级联检查：该报表下是否存在抽取配置
r, err = gfdb.DB(ctx).GetAll(ctx, "SELECT COUNT(1) AS cnt FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL LIMIT 1",
rpt.BusinessCode, rpt.ReportCode)
if err == nil && !r.IsEmpty() && r[0]["cnt"].Int() > 0 {
return &model.DeleteResult{Success: false, Message: fmt.Sprintf("报表 %s 下存在抽取配置，请先删除关联抽取配置后再删除报表", rpt.ReportCode)}, nil
}
if err := s.configLoader.DeleteReport(ctx, id, rpt.BusinessCode, rpt.ReportCode); err != nil {
return nil, err
}
@@ -294,7 +421,223 @@ func (s *ReportService) DeleteReport(ctx context.Context, id int64) (*model.Dele
// GetReport 获取单个报表配置
func (s *ReportService) GetReport(ctx context.Context, id int64) (*model.ReportConfig, error) {
return s.configLoader.GetReportByID(ctx, id)
rpt, err := s.configLoader.GetReportByID(ctx, id)
if err != nil {
return nil, err
}
// 同时加载全部字段（含已失效），编辑时前端可直接回显
allFields, err := s.configLoader.GetAllFields(ctx, rpt.BusinessCode, rpt.ReportCode)
if err == nil {
rpt.Fields = allFields
}
return rpt, nil
}
// ============================================================
// 合并操作：报表+字段一起保存
// ============================================================
// SaveReportWithFields 保存报表及字段配置（报表+字段全量替换）
func (s *ReportService) SaveReportWithFields(ctx context.Context, req *model.SaveReportWithFieldsReq) (*model.SaveResult, error) {
if err := initTables(ctx); err != nil {
return nil, fmt.Errorf("初始化系统表失败: %w", err)
}
// 自动填充每个字段的 businessCode/reportCode（从报表级继承）
for i := range req.Fields {
if req.Fields[i].BusinessCode == "" {
req.Fields[i].BusinessCode = req.BusinessCode
}
if req.Fields[i].ReportCode == "" {
req.Fields[i].ReportCode = req.ReportCode
}
}
var reportID int64
userName := getUserNameFromCtx(ctx)
// 事务：要么全成功，要么不动
err := gfdb.DB(ctx).Transaction(ctx, func(ctx context.Context, tx gdb.TX) error {
// 1. 保存报表（新增或更新）
now := time.Now()
if req.ID != nil && *req.ID > 0 {
// 更新报表
data := map[string]interface{}{
"business_code": req.BusinessCode,
"report_code": req.ReportCode,
"report_name": req.ReportName,
"description": req.Description,
"status": req.Status,
"stat_table_name": req.StatTableName,
"stat_table_comment": req.StatTableComment,
"date_field": req.DateField,
"primary_keys": req.PrimaryKeys,
"conflict_keys": req.ConflictKeys,
"config": req.Config,
"updated_at": now,
"updater": userName,
}
if _, err := tx.Model("report_report_config").Where("id", *req.ID).Data(data).Update(); err != nil {
return fmt.Errorf("更新报表失败: %w", err)
}
reportID = *req.ID
} else {
// 新增时校验 reportCode 唯一性
cnt, err := tx.Model("report_report_config").Where("business_code", req.BusinessCode).Where("report_code", req.ReportCode).Count()
if err != nil {
return fmt.Errorf("校验报表编码失败: %w", err)
}
if cnt > 0 {
return fmt.Errorf("业务 %s 下报表编码 %s 已存在", req.BusinessCode, req.ReportCode)
}
data := map[string]interface{}{
"business_code": req.BusinessCode,
"report_code": req.ReportCode,
"report_name": req.ReportName,
"description": req.Description,
"status": ifVal(req.Status, model.StatusActive),
"stat_table_name": req.StatTableName,
"stat_table_comment": req.StatTableComment,
"date_field": ifVal(req.DateField, "stat_date"),
"primary_keys": req.PrimaryKeys,
"conflict_keys": req.ConflictKeys,
"config": req.Config,
"created_at": now,
"updated_at": now,
"creator": userName,
"updater": userName,
"tenant_id": 1,
}
if data["primary_keys"] == nil || len(data["primary_keys"].([]string)) == 0 {
data["primary_keys"] = []string{"id"}
}
if data["conflict_keys"] == nil || len(data["conflict_keys"].([]string)) == 0 {
data["conflict_keys"] = []string{ifVal(req.DateField, "stat_date")}
}
result, err := tx.Model("report_report_config").Data(data).Insert()
if err != nil {
if strings.Contains(err.Error(), "duplicate key") {
return fmt.Errorf("业务 %s 下报表编码 %s 已存在", req.BusinessCode, req.ReportCode)
}
return fmt.Errorf("创建报表失败: %w", err)
}
rid, err := result.LastInsertId()
if err != nil {
return fmt.Errorf("获取自增ID失败: %w", err)
}
reportID = rid
}
// 2. 获取报表下现有字段（含已删除，编辑时重新添加的字段需走 UPDATE 而非 INSERT）
rows, err := tx.Model("report_field_config").
Where("business_code", req.BusinessCode).
Where("report_code", req.ReportCode).
All()
if err != nil {
return fmt.Errorf("获取现有字段失败: %w", err)
}
existingMap := make(map[string]int64) // fieldCode -> id
for _, row := range rows {
existingMap[row["field_code"].String()] = row["id"].Int64()
}
// 3. 处理请求中的字段（新增或更新）
for i := range req.Fields {
f := &req.Fields[i]
status := ifVal(f.Status, model.StatusActive)
dataType := ifVal(f.DataType, model.FieldTypeString)
if existingID, ok := existingMap[f.FieldCode]; ok {
// 已存在 → 更新
updateData := map[string]interface{}{
"field_name": f.FieldName,
"field_type": f.FieldType,
"data_type": dataType,
"field_role": f.FieldRole,
"is_aggregatable": f.IsAggregatable,
"is_filterable": f.IsFilterable,
"is_queryable": f.IsQueryable,
"is_sortable": f.IsSortable,
"default_aggregate": f.DefaultAggregate,
"valid_aggregates": f.ValidAggregates,
"filter_operators": f.FilterOperators,
"expression": f.Expression,
"expression_type": f.ExpressionType,
"format_pattern": f.FormatPattern,
"unit": f.Unit,
"dict_code": f.DictCode,
"sort_order": f.SortOrder,
"group_name": f.GroupName,
"status": status,
"updated_at": now,
"updater": userName,
}
if _, err := tx.Model("report_field_config").Where("id", existingID).Data(updateData).Update(); err != nil {
return fmt.Errorf("更新字段 %s 失败: %w", f.FieldCode, err)
}
delete(existingMap, f.FieldCode)
} else {
// 不存在 → 新增
insertData := map[string]interface{}{
"business_code": req.BusinessCode,
"report_code": req.ReportCode,
"field_code": f.FieldCode,
"field_name": f.FieldName,
"field_type": f.FieldType,
"data_type": dataType,
"field_role": f.FieldRole,
"is_aggregatable": f.IsAggregatable,
"is_filterable": f.IsFilterable,
"is_queryable": f.IsQueryable,
"is_sortable": f.IsSortable,
"default_aggregate": f.DefaultAggregate,
"valid_aggregates": f.ValidAggregates,
"filter_operators": f.FilterOperators,
"expression": f.Expression,
"expression_type": f.ExpressionType,
"format_pattern": f.FormatPattern,
"unit": f.Unit,
"dict_code": f.DictCode,
"sort_order": f.SortOrder,
"group_name": f.GroupName,
"status": status,
"created_at": now,
"updated_at": now,
"creator": userName,
"updater": userName,
"tenant_id": 1,
}
if _, err := tx.Model("report_field_config").Data(insertData).Insert(); err != nil {
if strings.Contains(err.Error(), "duplicate key") {
return fmt.Errorf("字段编码 %s 在当前报表下已存在，请更换字段编码", f.FieldCode)
}
return fmt.Errorf("创建字段 %s 失败: %w", f.FieldCode, err)
}
}
}
// 4. 删除剩余字段（不在请求中的旧字段）
for fieldCode, existingID := range existingMap {
if _, err := tx.Model("report_field_config").Where("id", existingID).Data(map[string]interface{}{
"status": model.StatusInactive,
"deleted_at": now,
"updated_at": now,
"updater": userName,
}).Update(); err != nil {
return fmt.Errorf("删除字段 %s 失败: %w", fieldCode, err)
}
}
return nil
})
if err != nil {
return nil, err
}
// 清理缓存
s.configLoader.InvalidateCache(req.BusinessCode, req.ReportCode)
return &model.SaveResult{Success: true, ID: reportID, Message: "报表及字段保存成功"}, nil
}
// ============================================================
@@ -307,8 +650,16 @@ func (s *ReportService) SaveField(ctx context.Context, req *model.SaveFieldReq)
return nil, fmt.Errorf("初始化系统表失败: %w", err)
}
if req.BusinessCode == "" {
return nil, fmt.Errorf("businessCode 不能为空")
}
if req.ReportCode == "" {
return nil, fmt.Errorf("reportCode 不能为空")
}
userName := getUserNameFromCtx(ctx)
field := &model.FieldConfig{
SQLBaseDO: beans.SQLBaseDO{Creator: req.Operator, Updater: req.Operator},
SQLBaseDO: beans.SQLBaseDO{Creator: userName, Updater: userName, TenantId: 1},
BusinessCode: req.BusinessCode,
ReportCode: req.ReportCode,
FieldCode: req.FieldCode,
@@ -418,8 +769,21 @@ func (s *ReportService) SaveExtractConfig(ctx context.Context, req *model.SaveEx
return nil, fmt.Errorf("初始化系统表失败: %w", err)
}
// 新增时校验 extractCode 唯一性（同业务+报表下）
if req.ID == nil || *req.ID == 0 {
r, err := gfdb.DB(ctx).GetAll(ctx, "SELECT id FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND extract_code = $3 LIMIT 1",
req.BusinessCode, req.ReportCode, req.ExtractCode)
if err != nil {
return nil, fmt.Errorf("校验抽取编码失败: %w", err)
}
if !r.IsEmpty() {
return nil, fmt.Errorf("业务 %s/报表 %s 下抽取编码 %s 已存在", req.BusinessCode, req.ReportCode, req.ExtractCode)
}
}
userName := getUserNameFromCtx(ctx)
ec := &model.ExtractConfig{
SQLBaseDO: beans.SQLBaseDO{Creator: req.Operator, Updater: req.Operator},
SQLBaseDO: beans.SQLBaseDO{Creator: userName, Updater: userName, TenantId: 1},
BusinessCode: req.BusinessCode,
ReportCode: req.ReportCode,
ExtractCode: req.ExtractCode,
@@ -499,6 +863,38 @@ func (s *ReportService) GetExtractConfig(ctx context.Context, id int64) (*model.
}
// GetExtractConfigs 获取业务报表下所有抽取配置
// ListExtractConfigs 分页获取抽取配置列表
func (s *ReportService) ListExtractConfigs(ctx context.Context, businessCode, reportCode string, pageNum, pageSize int) ([]model.ExtractConfig, int, error) {
return s.configLoader.ListExtractConfigs(ctx, businessCode, reportCode, pageNum, pageSize)
}
func (s *ReportService) GetExtractConfigs(ctx context.Context, businessCode, reportCode string) ([]model.ExtractConfig, error) {
return s.configLoader.GetExtractConfigs(ctx, businessCode, reportCode)
}
// GetExtractConfigsAll 获取业务报表下所有抽取配置（不限 status，用于填充字段展示）
func (s *ReportService) GetExtractConfigsAll(ctx context.Context, businessCode, reportCode string) ([]model.ExtractConfig, error) {
r, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT id, extract_code, extract_name, status FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND status != $3 ORDER BY id ASC",
businessCode, reportCode, model.StatusInactive)
if err != nil {
return nil, err
}
var configs []model.ExtractConfig
for _, record := range r {
var ec model.ExtractConfig
if err := record.Struct(&ec); err != nil {
return nil, err
}
configs = append(configs, ec)
}
return configs, nil
}
// ifVal 三目运算：非空返回 s，否则返回 fallback
func ifVal(s, fallback string) string {
if s == "" {
return fallback
}
return s
}
common/report/builder/sql_builder.go
+9 -2
View File
@@ -92,8 +92,13 @@ func (b *SQLBuilder) BuildQuerySQL(ctx context.Context, req *model.UserSelectQue
countSql += " WHERE " + whereClause
}
if groupByClause != "" {
countSql = fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE %s GROUP BY %s) AS t",
fromClause, whereClause, groupByClause)
if whereClause != "" {
countSql = fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s WHERE %s GROUP BY %s) AS t",
fromClause, whereClause, groupByClause)
} else {
countSql = fmt.Sprintf("SELECT COUNT(*) FROM (SELECT 1 FROM %s GROUP BY %s) AS t",
fromClause, groupByClause)
}
}
metadata := map[string]interface{}{
@@ -352,6 +357,8 @@ func (b *SQLBuilder) buildTimeGroupExpr(dateField, timeGroup string) string {
return fmt.Sprintf("DATE_TRUNC('week', %s::date)::text AS time_group", dateField)
case "month":
return fmt.Sprintf("TO_CHAR(%s::date, 'YYYY-MM') AS time_group", dateField)
case "year":
return fmt.Sprintf("TO_CHAR(%s::date, 'YYYY') AS time_group", dateField)
case "quarter":
return "TO_CHAR(" + dateField + "::date, 'YYYY-\"Q\"Q') AS time_group"
default:
common/report/builder/sql_builder_test.go
+6 -5
View File
@@ -9,12 +9,12 @@ import (
// Test data - reusable field configs
var testFields = map[string]*model.FieldConfig{
"shop_id": {FieldCode: "shop_id", FieldName: "店铺ID", FieldType: "STRING", FieldRole: "DIMENSION", IsFilterable: true, IsSortable: true, FilterOperators: []string{"=", "IN"}},
"shop_name": {FieldCode: "shop_name", FieldName: "店铺名称", FieldType: "STRING", FieldRole: "DIMENSION", IsFilterable: true, IsSortable: true, FilterOperators: []string{"=", "LIKE"}},
"stat_date": {FieldCode: "stat_date", FieldName: "统计日期", FieldType: "DATE", FieldRole: "DIMENSION", IsFilterable: true, IsSortable: true, FilterOperators: []string{"=", ">=", "<=", "BETWEEN"}},
"order_count": {FieldCode: "order_count", FieldName: "订单数", FieldType: "INT", FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "SUM", ValidAggregates: []string{"SUM", "COUNT"}},
"shop_id": {FieldCode: "shop_id", FieldName: "店铺ID", FieldType: "STRING", FieldRole: "DIMENSION", IsFilterable: true, IsSortable: true, FilterOperators: []string{"=", "IN"}},
"shop_name": {FieldCode: "shop_name", FieldName: "店铺名称", FieldType: "STRING", FieldRole: "DIMENSION", IsFilterable: true, IsSortable: true, FilterOperators: []string{"=", "LIKE"}},
"stat_date": {FieldCode: "stat_date", FieldName: "统计日期", FieldType: "DATE", FieldRole: "DIMENSION", IsFilterable: true, IsSortable: true, FilterOperators: []string{"=", ">=", "<=", "BETWEEN"}},
"order_count": {FieldCode: "order_count", FieldName: "订单数", FieldType: "INT", FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "SUM", ValidAggregates: []string{"SUM", "COUNT"}},
"order_amount": {FieldCode: "order_amount", FieldName: "订单金额", FieldType: "FLOAT", FieldRole: "INDICATOR", IsAggregatable: true, IsSortable: true, DefaultAggregate: "SUM", ValidAggregates: []string{"SUM", "AVG", "MAX", "MIN"}},
"refund_rate": {FieldCode: "refund_rate", FieldName: "退款率", FieldType: "FLOAT", FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "AVG", ValidAggregates: []string{"AVG"}, Expression: "{refund_amount} / NULLIF({order_amount}, 0) * 100", ExpressionType: "CALCULATED"},
"refund_rate": {FieldCode: "refund_rate", FieldName: "退款率", FieldType: "FLOAT", FieldRole: "INDICATOR", IsAggregatable: true, DefaultAggregate: "AVG", ValidAggregates: []string{"AVG"}, Expression: "{refund_amount} / NULLIF({order_amount}, 0) * 100", ExpressionType: "CALCULATED"},
"order_status": {FieldCode: "order_status", FieldName: "订单状态", FieldType: "STRING", FieldRole: "FILTER", IsFilterable: true, FilterOperators: []string{"=", "IN"}},
}
@@ -303,6 +303,7 @@ func TestBuildTimeGroupExpr(t *testing.T) {
group string
wantSub string
}{
{"year", "YYYY"},
{"week", "DATE_TRUNC"},
{"month", "YYYY-MM"},
{"quarter", "Q"},
common/report/config/loader.go
+131 -12
View File
@@ -51,8 +51,8 @@ func (l *ConfigLoader) GetBusiness(ctx context.Context, businessCode string) (*m
var biz model.BusinessConfig
r, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT * FROM report_business_config WHERE business_code = $1 AND status = $2 LIMIT 1",
businessCode, model.StatusActive)
"SELECT * FROM report_business_config WHERE business_code = $1 AND deleted_at IS NULL LIMIT 1",
businessCode)
if err != nil {
return nil, fmt.Errorf("查询业务配置失败: %w", err)
}
@@ -86,8 +86,8 @@ func (l *ConfigLoader) GetReport(ctx context.Context, businessCode, reportCode s
var rpt model.ReportConfig
r, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT * FROM report_report_config WHERE business_code = $1 AND report_code = $2 AND status = $3 LIMIT 1",
businessCode, reportCode, model.StatusActive)
"SELECT * FROM report_report_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL LIMIT 1",
businessCode, reportCode)
if err != nil {
return nil, fmt.Errorf("查询报表配置失败: %w", err)
}
@@ -126,8 +126,8 @@ func (l *ConfigLoader) GetFields(ctx context.Context, businessCode, reportCode s
l.mu.RUnlock()
r, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT * FROM report_field_config WHERE business_code = $1 AND report_code = $2 AND status = $3 ORDER BY sort_order ASC",
businessCode, reportCode, model.StatusActive)
"SELECT * FROM report_field_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY sort_order ASC",
businessCode, reportCode)
if err != nil {
return nil, err
}
@@ -153,6 +153,31 @@ func (l *ConfigLoader) GetFields(ctx context.Context, businessCode, reportCode s
return fields, nil
}
// GetAllFields 获取报表全部字段（含 INACTIVE，不含已删除，用于编辑回显）
func (l *ConfigLoader) GetAllFields(ctx context.Context, businessCode, reportCode string) ([]model.FieldConfig, error) {
r, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT * FROM report_field_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY sort_order ASC",
businessCode, reportCode)
if err != nil {
return nil, err
}
var fields []model.FieldConfig
for _, record := range r {
var f model.FieldConfig
if err := record.Struct(&f); err != nil {
return nil, err
}
if f.ValidAggregates == nil {
f.ValidAggregates = []string{}
}
if f.FilterOperators == nil {
f.FilterOperators = []string{"=", "!=", ">", "<", ">=", "<=", "IN", "LIKE", "BETWEEN"}
}
fields = append(fields, f)
}
return fields, nil
}
// GetFieldMap 获取字段配置Map
func (l *ConfigLoader) GetFieldMap(ctx context.Context, businessCode, reportCode string) (map[string]*model.FieldConfig, error) {
fields, err := l.GetFields(ctx, businessCode, reportCode)
@@ -178,8 +203,8 @@ func (l *ConfigLoader) GetExtractConfigs(ctx context.Context, businessCode, repo
l.mu.RUnlock()
r, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT * FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND status = $3 AND is_enabled = $4",
businessCode, reportCode, model.StatusActive, true)
"SELECT * FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY id ASC",
businessCode, reportCode)
if err != nil {
return nil, err
}
@@ -586,8 +611,9 @@ func (l *ConfigLoader) GetExtractConfigByID(ctx context.Context, id int64) (*mod
// GetAllBusinesses 获取所有业务配置
func (l *ConfigLoader) GetAllBusinesses(ctx context.Context) ([]model.BusinessConfig, error) {
r, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT * FROM report_business_config WHERE status = $1 ORDER BY id ASC",
model.StatusActive)
"SELECT * FROM report_business_config WHERE deleted_at IS NULL ORDER BY id ASC",
)
// (removed status filter)
if err != nil {
return nil, err
}
@@ -605,8 +631,8 @@ func (l *ConfigLoader) GetAllBusinesses(ctx context.Context) ([]model.BusinessCo
// GetAllReports 获取所有报表配置
func (l *ConfigLoader) GetAllReports(ctx context.Context, businessCode string) ([]model.ReportConfig, error) {
r, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT * FROM report_report_config WHERE business_code = $1 AND status = $2 ORDER BY id ASC",
businessCode, model.StatusActive)
"SELECT * FROM report_report_config WHERE business_code = $1 AND deleted_at IS NULL ORDER BY id ASC",
businessCode)
if err != nil {
return nil, err
}
@@ -621,6 +647,99 @@ func (l *ConfigLoader) GetAllReports(ctx context.Context, businessCode string) (
return reports, nil
}
// ListBusinesses 分页获取业务列表
func (l *ConfigLoader) ListBusinesses(ctx context.Context, pageNum, pageSize int) ([]model.BusinessConfig, int, error) {
total, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT COUNT(*) AS cnt FROM report_business_config WHERE deleted_at IS NULL")
if err != nil {
return nil, 0, err
}
count := 0
if !total.IsEmpty() {
count = total[0]["cnt"].Int()
}
offset := (pageNum - 1) * pageSize
r, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT * FROM report_business_config WHERE deleted_at IS NULL ORDER BY id ASC LIMIT $1 OFFSET $2",
pageSize, offset)
if err != nil {
return nil, 0, err
}
var businesses []model.BusinessConfig
for _, record := range r {
var biz model.BusinessConfig
if err := record.Struct(&biz); err != nil {
return nil, 0, err
}
businesses = append(businesses, biz)
}
return businesses, count, nil
}
// ListReports 分页获取报表列表
func (l *ConfigLoader) ListReports(ctx context.Context, businessCode, reportName string, pageNum, pageSize int) ([]model.ReportConfig, int, error) {
namePattern := "%" + reportName + "%"
total, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT COUNT(*) AS cnt FROM report_report_config WHERE business_code = $1 AND report_name ILIKE $2 AND deleted_at IS NULL",
businessCode, namePattern)
if err != nil {
return nil, 0, err
}
count := 0
if !total.IsEmpty() {
count = total[0]["cnt"].Int()
}
offset := (pageNum - 1) * pageSize
r, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT * FROM report_report_config WHERE business_code = $1 AND report_name ILIKE $2 AND deleted_at IS NULL ORDER BY id ASC LIMIT $3 OFFSET $4",
businessCode, namePattern, pageSize, offset)
if err != nil {
return nil, 0, err
}
var reports []model.ReportConfig
for _, record := range r {
var rpt model.ReportConfig
if err := record.Struct(&rpt); err != nil {
return nil, 0, err
}
reports = append(reports, rpt)
}
return reports, count, nil
}
// ListExtractConfigs 分页获取抽取配置列表
func (l *ConfigLoader) ListExtractConfigs(ctx context.Context, businessCode, reportCode string, pageNum, pageSize int) ([]model.ExtractConfig, int, error) {
total, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT COUNT(*) AS cnt FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL",
businessCode, reportCode)
if err != nil {
return nil, 0, err
}
count := 0
if !total.IsEmpty() {
count = total[0]["cnt"].Int()
}
offset := (pageNum - 1) * pageSize
r, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT * FROM report_extract_config WHERE business_code = $1 AND report_code = $2 AND deleted_at IS NULL ORDER BY id ASC LIMIT $3 OFFSET $4",
businessCode, reportCode, pageSize, offset)
if err != nil {
return nil, 0, err
}
var configs []model.ExtractConfig
for _, record := range r {
var ec model.ExtractConfig
if err := record.Struct(&ec); err != nil {
return nil, 0, err
}
configs = append(configs, ec)
}
l.mu.Lock()
l.extractCache[businessCode+":"+reportCode] = configs
l.mu.Unlock()
return configs, count, nil
}
// GetReportFields 获取报表可用字段(按角色分类)
func (l *ConfigLoader) GetReportFields(ctx context.Context, businessCode, reportCode string) (*model.GetReportFieldsResp, error) {
fields, err := l.GetFields(ctx, businessCode, reportCode)
common/report/cron.go
+66
View File
@@ -0,0 +1,66 @@
package report
import (
"context"
"time"
"github.com/gogf/gf/v2/os/gcron"
"github.com/gogf/gf/v2/os/gctx"
"github.com/sirupsen/logrus"
)
// StartDailyExtractJob 启动每日自动抽取定时任务
// 每天凌晨 2:00 遍历所有已启用的业务+报表，抽取前一天数据
// 在 main.go 中调用：report.StartDailyExtractJob()
func StartDailyExtractJob() {
svc := GetService()
// 每天凌晨 2:00 执行
_, err := gcron.Add(gctx.New(), "0 0 2 * * *", func(ctx context.Context) {
logrus.Info("[报表引擎] 开始每日自动抽取任务")
// 获取所有启用业务
businesses, err := svc.GetAllBusinesses(ctx)
if err != nil {
logrus.Errorf("[报表引擎] 获取业务列表失败: %v", err)
return
}
yesterday := time.Now().AddDate(0, 0, -1).Format("2006-01-02")
for _, biz := range businesses {
// 获取业务下所有启用报表
reports, err := svc.GetAllReports(ctx, biz.BusinessCode)
if err != nil {
logrus.Errorf("[报表引擎] 获取报表列表失败 [%s]: %v", biz.BusinessCode, err)
continue
}
for _, rpt := range reports {
logrus.Infof("[报表引擎] 开始抽取 %s/%s 日期 %s", biz.BusinessCode, rpt.ReportCode, yesterday)
resp, err := svc.ExtractDailyData(ctx, biz.BusinessCode, rpt.ReportCode, yesterday, "system")
if err != nil {
logrus.Errorf("[报表引擎] 抽取失败 %s/%s %s: %v", biz.BusinessCode, rpt.ReportCode, yesterday, err)
continue
}
if resp.Success {
logrus.Infof("[报表引擎] 抽取成功 %s/%s %s 总数:%d 耗时:%dms",
biz.BusinessCode, rpt.ReportCode, yesterday, resp.TotalCount, resp.ExecTimeMs)
} else {
logrus.Errorf("[报表引擎] 抽取异常 %s/%s %s: %s", biz.BusinessCode, rpt.ReportCode, yesterday, resp.ErrorMsg)
}
}
}
logrus.Info("[报表引擎] 每日自动抽取任务完成")
}, "daily-report-extract")
if err != nil {
logrus.Errorf("[报表引擎] 启动定时任务失败: %v", err)
return
}
logrus.Info("[报表引擎] 每日自动抽取定时任务已启动 (02:00)")
}
common/report/ddlsync/creator.go
+5 -20
View File
@@ -115,8 +115,10 @@ func (c *StatTableCreator) buildCreateTableSQL(tableName string, report *model.R
// 业务字段
for _, f := range fields {
colDef := c.fieldTypeToColumn(f.FieldCode, f.FieldType)
cols = append(cols, colDef)
if f.FieldCode == dateField {
continue
}
cols = append(cols, fmt.Sprintf("%s %s", f.FieldCode, model.FieldTypeToPG(f.FieldType)))
}
// 原始数据
@@ -127,22 +129,6 @@ func (c *StatTableCreator) buildCreateTableSQL(tableName string, report *model.R
return sql
}
// fieldTypeToColumn 字段类型转PG列类型
func (c *StatTableCreator) fieldTypeToColumn(fieldCode, fieldType string) string {
switch fieldType {
case model.FieldTypeInt, model.FieldTypeFloat:
return fmt.Sprintf("%s NUMERIC(20,4) DEFAULT 0", fieldCode)
case model.FieldTypeDate:
return fmt.Sprintf("%s VARCHAR(16) DEFAULT ''", fieldCode)
case model.FieldTypeDatetime:
return fmt.Sprintf("%s TIMESTAMP WITH TIME ZONE", fieldCode)
case model.FieldTypeJsonb:
return fmt.Sprintf("%s JSONB DEFAULT '{}'", fieldCode)
default: // STRING
return fmt.Sprintf("%s VARCHAR(256) DEFAULT ''", fieldCode)
}
}
// DropStatTable 删除统计宽表
func (c *StatTableCreator) DropStatTable(ctx context.Context, businessCode, reportCode string) error {
report, err := c.loader.GetReport(ctx, businessCode, reportCode)
@@ -200,8 +186,7 @@ func (c *StatTableCreator) AlterTableAddColumns(ctx context.Context, tableName s
if existingMap[f.FieldCode] {
continue
}
colDef := c.fieldTypeToColumn(f.FieldCode, f.FieldType)
sql := fmt.Sprintf("ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s", tableName, colDef)
sql := fmt.Sprintf("ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s %s", tableName, f.FieldCode, model.FieldTypeToPG(f.FieldType))
if _, err := gfdb.DB(ctx).Exec(ctx, sql); err != nil {
return fmt.Errorf("添加字段 %s 失败: %w", f.FieldCode, err)
}
common/report/executor/executor.go
+1
View File
@@ -169,6 +169,7 @@ func (e *QueryExecutor) validateReq(req *model.UserSelectQueryReq) error {
"week": true,
"month": true,
"quarter": true,
"year": true,
}
if !validGroups[req.TimeGroup] {
return fmt.Errorf("不支持的时间分组: %s", req.TimeGroup)
common/report/extract/extract.go
+83 -57
View File
@@ -3,6 +3,7 @@ package extract
import (
"context"
"fmt"
"sort"
"strings"
"time"
@@ -91,7 +92,9 @@ func (e *DailyExtractor) ExtractDailyData(ctx context.Context, businessCode, rep
Executor: executor,
StartTime: &start,
}
_ = e.loader.CreateExtractLog(ctx, extractLog)
if err := e.loader.CreateExtractLog(ctx, extractLog); err != nil {
logger.Errorf("创建抽取日志失败: %v", err)
}
// 执行抽取
c, s, f, err := e.executeExtract(ctx, &ec, report, fieldMap, statDate)
@@ -476,7 +479,7 @@ func (e *DailyExtractor) applyTransformRules(ec *model.ExtractConfig, row map[st
}
}
// ensureStatTableExists 确保统计宽表存在
// ensureStatTableExists 确保统计宽表存在，如有新增字段自动加列
func (e *DailyExtractor) ensureStatTableExists(ctx context.Context, report *model.ReportConfig, fieldMap map[string]*model.FieldConfig) error {
tableName := report.StatTableName
@@ -495,10 +498,48 @@ func (e *DailyExtractor) ensureStatTableExists(ctx context.Context, report *mode
return e.createStatTable(ctx, report, fieldMap)
}
// 表已存在，检查并添加新增字段
if err := e.alterTableAddColumns(ctx, tableName, fieldMap); err != nil {
logrus.Warnf("自动加列失败 %s: %v", tableName, err)
// 不返回 error，加列失败不应阻塞抽取
}
logrus.Infof("统计宽表 %s 已存在", tableName)
return nil
}
// alterTableAddColumns 为已存在的统计宽表添加新增字段
func (e *DailyExtractor) alterTableAddColumns(ctx context.Context, tableName string, fieldMap map[string]*model.FieldConfig) error {
// 获取已有列
rows, err := gfdb.DB(ctx).GetAll(ctx,
"SELECT column_name FROM information_schema.columns WHERE table_name = $1 AND table_schema = 'public'",
strings.ToLower(tableName))
if err != nil {
return fmt.Errorf("查询表结构失败: %w", err)
}
existingCols := make(map[string]bool)
for _, row := range rows.List() {
if col, ok := row["column_name"].(string); ok {
existingCols[col] = true
}
}
for _, fc := range fieldMap {
if existingCols[fc.FieldCode] {
continue
}
colType := model.FieldTypeToPG(fc.FieldType)
sql := fmt.Sprintf("ALTER TABLE %s ADD COLUMN IF NOT EXISTS %s %s", tableName, fc.FieldCode, colType)
if _, err := gfdb.DB(ctx).Exec(ctx, sql); err != nil {
return fmt.Errorf("添加字段 %s 失败: %w", fc.FieldCode, err)
}
logrus.Infof("统计宽表 %s 自动添加字段 %s %s", tableName, fc.FieldCode, colType)
}
return nil
}
// createStatTable 创建统计宽表
func (e *DailyExtractor) createStatTable(ctx context.Context, report *model.ReportConfig, fieldMap map[string]*model.FieldConfig) error {
var cols []string
@@ -523,7 +564,10 @@ func (e *DailyExtractor) createStatTable(ctx context.Context, report *model.Repo
// 业务字段
for _, fc := range fieldMap {
fc := fc
colType := fieldTypeToPG(fc.FieldType)
if fc.FieldCode == dateField {
continue
}
colType := model.FieldTypeToPG(fc.FieldType)
cols = append(cols, fmt.Sprintf("%s %s", fc.FieldCode, colType))
}
@@ -564,23 +608,43 @@ func (e *DailyExtractor) createStatTable(ctx context.Context, report *model.Repo
return nil
}
// batchUpsert 批量upsert写入
// batchUpsert 批量写入统计宽表（使用原始SQL INSERT，避免ORM兼容问题）
func (e *DailyExtractor) batchUpsert(ctx context.Context, tableName string, conflictKeys []string, rows []map[string]interface{}) (int, []string, error) {
if len(rows) == 0 {
return 0, nil, nil
}
now := time.Now()
for i := range rows {
if rows[i] == nil {
rows[i] = make(map[string]interface{})
// 去除 id 和 created_at 列（数据库自增/默认值）
var keys []string
for k := range rows[0] {
if k == "id" || k == "created_at" || k == "updated_at" || k == "deleted_at" {
continue
}
rows[i]["updated_at"] = now
keys = append(keys, k)
}
sort.Strings(keys)
cols := strings.Join(keys, ", ")
placeholders := make([]string, len(keys))
for i := range keys {
placeholders[i] = fmt.Sprintf("$%d", i+1)
}
phStr := strings.Join(placeholders, ", ")
sqlStr := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", tableName, cols, phStr)
if len(conflictKeys) > 0 {
var updateParts []string
for _, k := range keys {
if k == "id" || k == "created_at" {
continue
}
updateParts = append(updateParts, fmt.Sprintf("%s = EXCLUDED.%s", k, k))
}
sqlStr += " ON CONFLICT (" + strings.Join(conflictKeys, ", ") + ") DO UPDATE SET " + strings.Join(updateParts, ", ")
}
batchSize := 100
total := 0
var allColumns []string
for i := 0; i < len(rows); i += batchSize {
end := i + batchSize
@@ -589,56 +653,18 @@ func (e *DailyExtractor) batchUpsert(ctx context.Context, tableName string, conf
}
batch := rows[i:end]
m := gfdb.DB(ctx).Model(ctx, tableName).Data(batch)
if len(conflictKeys) > 0 {
keys := make([]interface{}, len(conflictKeys))
for j, k := range conflictKeys {
keys[j] = k
for _, row := range batch {
var args []interface{}
for _, k := range keys {
args = append(args, row[k])
}
m = m.OnConflict(keys...)
}
_, err := m.Save()
if err != nil {
logrus.Errorf("批量写入 %s 失败: %v", tableName, err)
// 逐条重试
for _, row := range batch {
mm := gfdb.DB(ctx).Model(ctx, tableName).Data(row)
if len(conflictKeys) > 0 {
keys := make([]interface{}, len(conflictKeys))
for j, k := range conflictKeys {
keys[j] = k
}
mm = mm.OnConflict(keys...)
}
if _, e := mm.Save(); e != nil {
logrus.Errorf("逐条写入失败: %v", e)
} else {
total++
}
if _, err := gfdb.DB(ctx).Exec(ctx, sqlStr, args...); err != nil {
logrus.Errorf("逐行写入 %s 失败: %v", tableName, err)
} else {
total++
}
} else {
total += len(batch)
}
}
return total, allColumns, nil
}
// fieldTypeToPG 字段类型转PG类型
func fieldTypeToPG(fieldType string) string {
switch fieldType {
case model.FieldTypeInt:
return "NUMERIC(20,0) DEFAULT 0"
case model.FieldTypeFloat:
return "NUMERIC(20,4) DEFAULT 0"
case model.FieldTypeDate:
return "VARCHAR(16) DEFAULT ''"
case model.FieldTypeDatetime:
return "TIMESTAMP WITH TIME ZONE"
case model.FieldTypeJsonb:
return "JSONB DEFAULT '{}'"
default:
return "VARCHAR(256) DEFAULT ''"
}
return total, nil, nil
}
common/report/model/model.go
+259 -189
View File
@@ -13,99 +13,101 @@ import (
// BusinessConfig 业务配置
type BusinessConfig struct {
beans.SQLBaseDO `orm:",inherit"`
BusinessCode string `orm:"business_code" json:"businessCode"`
BusinessName string `orm:"business_name" json:"businessName"`
Description string `orm:"description" json:"description"`
Status string `orm:"status" json:"status"`
Config map[string]interface{} `orm:"config" json:"config"`
BusinessCode string `orm:"business_code" json:"businessCode" dc:"业务编码" eg:"KUAISHOU"`
BusinessName string `orm:"business_name" json:"businessName" dc:"业务名称" eg:"快手电商"`
Description string `orm:"description" json:"description" dc:"描述" eg:"快手电商店铺每日统计报表"`
Status string `orm:"status" json:"status" dc:"status" eg:"ACTIVE"`
Config map[string]interface{} `orm:"config" json:"config" dc:"扩展配置"`
}
// ReportConfig 报表配置
type ReportConfig struct {
beans.SQLBaseDO `orm:",inherit"`
BusinessCode string `orm:"business_code" json:"businessCode"`
ReportCode string `orm:"report_code" json:"reportCode"`
ReportName string `orm:"report_name" json:"reportName"`
Description string `orm:"description" json:"description"`
Status string `orm:"status" json:"status"`
StatTableName string `orm:"stat_table_name" json:"statTableName"`
StatTableComment string `orm:"stat_table_comment" json:"statTableComment"`
DateField string `orm:"date_field" json:"dateField"`
PrimaryKeys []string `orm:"primary_keys" json:"primaryKeys"`
ConflictKeys []string `orm:"conflict_keys" json:"conflictKeys"`
Config map[string]interface{} `orm:"config" json:"config"`
BusinessCode string `orm:"business_code" json:"businessCode" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `orm:"report_code" json:"reportCode" dc:"报表编码" eg:"shop_daily"`
ReportName string `orm:"report_name" json:"reportName" dc:"报表名称" eg:"店铺日报"`
Description string `orm:"description" json:"description" dc:"描述" eg:"快手电商店铺每日统计报表"`
Status string `orm:"status" json:"status" dc:"状态 ACTIVE/INACTIVE" eg:"ACTIVE"`
StatTableName string `orm:"stat_table_name" json:"statTableName" dc:"统计宽表名" eg:"stat_kuaishou_shop_daily"`
StatTableComment string `orm:"stat_table_comment" json:"statTableComment" dc:"统计宽表注释" eg:"快手电商-店铺日报"`
DateField string `orm:"date_field" json:"dateField" dc:"日期字段名" eg:"stat_date"`
PrimaryKeys []string `orm:"primary_keys" json:"primaryKeys" dc:"主键字段列表"`
ConflictKeys []string `orm:"conflict_keys" json:"conflictKeys" dc:"冲突键（唯一索引）"`
Config map[string]interface{} `orm:"config" json:"config" dc:"扩展配置"`
ExtractConfigIDs string `orm:"-" json:"extractConfigIds" dc:"抽取配置ID列表，逗号分隔" eg:"1,2,3"`
Fields []FieldConfig `orm:"-" json:"fields,omitempty" dc:"字段配置列表（查询时返回）"`
}
// FieldConfig 字段配置
type FieldConfig struct {
beans.SQLBaseDO `orm:",inherit"`
BusinessCode string `orm:"business_code" json:"businessCode"`
ReportCode string `orm:"report_code" json:"reportCode"`
FieldCode string `orm:"field_code" json:"fieldCode"`
FieldName string `orm:"field_name" json:"fieldName"`
FieldType string `orm:"field_type" json:"fieldType"`
DataType string `orm:"data_type" json:"dataType"`
FieldRole string `orm:"field_role" json:"fieldRole"`
IsAggregatable bool `orm:"is_aggregatable" json:"isAggregatable"`
IsFilterable bool `orm:"is_filterable" json:"isFilterable"`
IsQueryable bool `orm:"is_queryable" json:"isQueryable"`
IsSortable bool `orm:"is_sortable" json:"isSortable"`
DefaultAggregate string `orm:"default_aggregate" json:"defaultAggregate"`
ValidAggregates []string `orm:"valid_aggregates" json:"validAggregates"`
FilterOperators []string `orm:"filter_operators" json:"filterOperators"`
Expression string `orm:"expression" json:"expression"`
ExpressionType string `orm:"expression_type" json:"expressionType"`
FormatPattern string `orm:"format_pattern" json:"formatPattern"`
Unit string `orm:"unit" json:"unit"`
DictCode string `orm:"dict_code" json:"dictCode"`
SortOrder int `orm:"sort_order" json:"sortOrder"`
GroupName string `orm:"group_name" json:"groupName"`
Status string `orm:"status" json:"status"`
BusinessCode string `orm:"business_code" json:"businessCode" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `orm:"report_code" json:"reportCode" dc:"reportCode" eg:"shop_daily"`
FieldCode string `orm:"field_code" json:"fieldCode" dc:"字段编码" eg:"order_amount"`
FieldName string `orm:"field_name" json:"fieldName" dc:"字段名称" eg:"订单金额"`
FieldType string `orm:"field_type" json:"fieldType" dc:"字段类型 STRING/INT/FLOAT/DATE/DATETIME/JSONB" eg:"FLOAT"`
DataType string `orm:"data_type" json:"dataType" dc:"数据存储类型" eg:"STRING"`
FieldRole string `orm:"field_role" json:"fieldRole" dc:"字段角色 DIMENSION(维度)/INDICATOR(指标)/FILTER(筛选)/FILTER_ONLY(仅筛选)" eg:"DIMENSION"`
IsAggregatable bool `orm:"is_aggregatable" json:"isAggregatable" dc:"是否可聚合" eg:"true"`
IsFilterable bool `orm:"is_filterable" json:"isFilterable" dc:"是否可筛选" eg:"true"`
IsQueryable bool `orm:"is_queryable" json:"isQueryable" dc:"是否可查询" eg:"true"`
IsSortable bool `orm:"is_sortable" json:"isSortable" dc:"是否可排序" eg:"true"`
DefaultAggregate string `orm:"default_aggregate" json:"defaultAggregate" dc:"默认聚合方式 SUM/COUNT/AVG/MAX/MIN" eg:"SUM"`
ValidAggregates []string `orm:"valid_aggregates" json:"validAggregates" dc:"可选聚合列表"`
FilterOperators []string `orm:"filter_operators" json:"filterOperators" dc:"可选操作符列表 =/!=/>/</>=/<=/IN/LIKE/BETWEEN"`
Expression string `orm:"expression" json:"expression" dc:"表达式（衍生字段）" eg:"order_amount - refund_amount"`
ExpressionType string `orm:"expression_type" json:"expressionType" dc:"表达式类型 DIRECT/CALCULATED" eg:"DIRECT"`
FormatPattern string `orm:"format_pattern" json:"formatPattern" dc:"格式化模板" eg:"#,##0.00"`
Unit string `orm:"unit" json:"unit" dc:"单位"`
DictCode string `orm:"dict_code" json:"dictCode" dc:"字典编码" eg:"dict_order_type"`
SortOrder int `orm:"sort_order" json:"sortOrder" dc:"排序号" eg:"1"`
GroupName string `orm:"group_name" json:"groupName" dc:"分组名称" eg:"交易指标"`
Status string `orm:"status" json:"status" dc:"status" eg:"ACTIVE"`
}
// ExtractConfig 抽取配置
type ExtractConfig struct {
beans.SQLBaseDO `orm:",inherit"`
BusinessCode string `orm:"business_code" json:"businessCode"`
ReportCode string `orm:"report_code" json:"reportCode"`
ExtractCode string `orm:"extract_code" json:"extractCode"`
ExtractName string `orm:"extract_name" json:"extractName"`
SourceTableName string `orm:"source_table_name" json:"sourceTableName"`
SourceTableAlias string `orm:"source_table_alias" json:"sourceTableAlias"`
TargetTableName string `orm:"target_table_name" json:"targetTableName"`
IsEnabled bool `orm:"is_enabled" json:"isEnabled"`
ExtractType string `orm:"extract_type" json:"extractType"`
ExtractMode string `orm:"extract_mode" json:"extractMode"`
ExtractKeyField string `orm:"extract_key_field" json:"extractKeyField"`
ExtractKeyFormat string `orm:"extract_key_format" json:"extractKeyFormat"`
GroupByFields []string `orm:"group_by_fields" json:"groupByFields"`
FilterExpression string `orm:"filter_expression" json:"filterExpression"`
JoinConfigs []JoinConfig `orm:"join_configs" json:"joinConfigs"`
FieldMappings []FieldMapping `orm:"field_mappings" json:"fieldMappings"`
TransformRules []TransformRule `orm:"transform_rules" json:"transformRules"`
BatchSize int `orm:"batch_size" json:"batchSize"`
Status string `orm:"status" json:"status"`
BusinessCode string `orm:"business_code" json:"businessCode" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `orm:"report_code" json:"reportCode" dc:"reportCode" eg:"shop_daily"`
ExtractCode string `orm:"extract_code" json:"extractCode" dc:"抽取编码" eg:"kuaishou_order_extract"`
ExtractName string `orm:"extract_name" json:"extractName" dc:"抽取名称" eg:"订单抽取"`
SourceTableName string `orm:"source_table_name" json:"sourceTableName" dc:"源表名" eg:"ods_kuaishou_order"`
SourceTableAlias string `orm:"source_table_alias" json:"sourceTableAlias" dc:"源表别名" eg:"o"`
TargetTableName string `orm:"target_table_name" json:"targetTableName" dc:"目标表名" eg:"stat_kuaishou_shop_daily"`
IsEnabled bool `orm:"is_enabled" json:"isEnabled" dc:"是否启用" eg:"true"`
ExtractType string `orm:"extract_type" json:"extractType" dc:"抽取类型 FULL/INCREMENTAL" eg:"INCREMENTAL"`
ExtractMode string `orm:"extract_mode" json:"extractMode" dc:"抽取模式 DIRECT(逐行)/AGGREGATE(聚合)" eg:"DIRECT"`
ExtractKeyField string `orm:"extract_key_field" json:"extractKeyField" dc:"抽取关键字段（增量依据）" eg:"updated_at"`
ExtractKeyFormat string `orm:"extract_key_format" json:"extractKeyFormat" dc:"关键字段格式" eg:"yyyy-MM-dd HH:mm:ss"`
GroupByFields []string `orm:"group_by_fields" json:"groupByFields" dc:"GROUP BY 字段列表"`
FilterExpression string `orm:"filter_expression" json:"filterExpression" dc:"过滤表达式" eg:"status = 'PAID'"`
JoinConfigs []JoinConfig `orm:"join_configs" json:"joinConfigs" dc:"JOIN配置"`
FieldMappings []FieldMapping `orm:"field_mappings" json:"fieldMappings" dc:"字段映射列表"`
TransformRules []TransformRule `orm:"transform_rules" json:"transformRules" dc:"转换规则列表"`
BatchSize int `orm:"batch_size" json:"batchSize" dc:"批处理大小" eg:"1000"`
Status string `orm:"status" json:"status" dc:"status" eg:"ACTIVE"`
}
// ExtractLog 抽取记录（不含 Creator/Updater/DeletedAt，表结构不同）
type ExtractLog struct {
ID int64 `orm:"id" json:"id"`
TenantId uint64 `orm:"tenant_id" json:"tenantId"`
BusinessCode string `orm:"business_code" json:"businessCode"`
ReportCode string `orm:"report_code" json:"reportCode"`
ExtractCode string `orm:"extract_code" json:"extractCode"`
StatDate string `orm:"stat_date" json:"statDate"`
ExtractType string `orm:"extract_type" json:"extractType"`
Status string `orm:"status" json:"status"`
TotalCount int `orm:"total_count" json:"totalCount"`
SuccessCount int `orm:"success_count" json:"successCount"`
FailCount int `orm:"fail_count" json:"failCount"`
StartTime *time.Time `orm:"start_time" json:"startTime"`
EndTime *time.Time `orm:"end_time" json:"endTime"`
ErrorMessage string `orm:"error_message" json:"errorMessage"`
Executor string `orm:"executor" json:"executor"`
CreatedAt *time.Time `orm:"created_at" json:"createdAt"`
UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt"`
ID int64 `orm:"id" json:"id" dc:"主键ID" eg:"1"`
TenantId uint64 `orm:"tenant_id" json:"tenantId" dc:"租户ID" eg:"1"`
BusinessCode string `orm:"business_code" json:"businessCode" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `orm:"report_code" json:"reportCode" dc:"reportCode" eg:"shop_daily"`
ExtractCode string `orm:"extract_code" json:"extractCode" dc:"抽取编码" eg:"kuaishou_order_extract"`
StatDate string `orm:"stat_date" json:"statDate" dc:"统计日期" eg:"2026-06-01"`
ExtractType string `orm:"extract_type" json:"extractType" dc:"抽取类型 FULL/INCREMENTAL" eg:"INCREMENTAL"`
Status string `orm:"status" json:"status" dc:"status" eg:"ACTIVE"`
TotalCount int `orm:"total_count" json:"totalCount" dc:"总记录数" eg:"100"`
SuccessCount int `orm:"success_count" json:"successCount" dc:"成功记录数" eg:"98"`
FailCount int `orm:"fail_count" json:"failCount" dc:"失败记录数" eg:"2"`
StartTime *time.Time `orm:"start_time" json:"startTime" dc:"开始时间"`
EndTime *time.Time `orm:"end_time" json:"endTime" dc:"结束时间"`
ErrorMessage string `orm:"error_message" json:"errorMessage" dc:"错误信息"`
Executor string `orm:"executor" json:"executor" dc:"执行人" eg:"system"`
CreatedAt *time.Time `orm:"created_at" json:"createdAt" dc:"创建时间"`
UpdatedAt *time.Time `orm:"updated_at" json:"updatedAt" dc:"更新时间"`
}
// ============================================================
@@ -114,30 +116,30 @@ type ExtractLog struct {
// JoinConfig 关联配置
type JoinConfig struct {
JoinTable string `json:"joinTable"`
JoinAlias string `json:"joinAlias"`
JoinType string `json:"joinType"` // LEFT/RIGHT/INNER
JoinCondition string `json:"joinCondition"`
FieldMappings []FieldMapping `json:"fieldMappings"`
JoinTable string `json:"joinTable" dc:"关联表名" eg:"ods_kuaishou_shop"`
JoinAlias string `json:"joinAlias" dc:"关联表别名" eg:"s"`
JoinType string `json:"joinType" dc:"LEFT/RIGHT/INNER" eg:"LEFT"`
JoinCondition string `json:"joinCondition" dc:"关联条件" eg:"o.shop_id = s.shop_id"`
FieldMappings []FieldMapping `json:"fieldMappings" dc:"字段映射列表"`
}
// FieldMapping 字段映射
type FieldMapping struct {
SourceField string `json:"sourceField"`
TargetField string `json:"targetField"`
FieldType string `json:"fieldType"`
AggregateFunction string `json:"aggregateFunction"`
DefaultValue interface{} `json:"defaultValue"`
TransformRule *TransformRule `json:"transformRule,omitempty"`
SourceField string `json:"sourceField" dc:"源字段"`
TargetField string `json:"targetField" dc:"目标字段"`
FieldType string `json:"fieldType" dc:"字段类型 STRING/INT/FLOAT/DATE/DATETIME/JSONB" eg:"FLOAT"`
AggregateFunction string `json:"aggregateFunction" dc:"聚合函数 SUM/COUNT/AVG/MAX/MIN"`
DefaultValue interface{} `json:"defaultValue" dc:"默认值"`
TransformRule *TransformRule `json:"transformRule,omitempty" dc:"transformRule"`
}
// TransformRule 转换规则
type TransformRule struct {
RuleCode string `json:"ruleCode"`
RuleType string `json:"ruleType"` // DIRECT/MAPPING/FORMAT/CALCULATE
Expression string `json:"expression"`
Format string `json:"format"`
Mapping map[string]interface{} `json:"mapping"`
RuleCode string `json:"ruleCode" dc:"规则编码" eg:"AMOUNT_CONVERT"`
RuleType string `json:"ruleType" dc:"DIRECT/MAPPING/FORMAT/CALCULATE" eg:"FORMAT"`
Expression string `json:"expression" dc:"表达式（衍生字段）" eg:"order_amount - refund_amount"`
Format string `json:"format" dc:"format" eg:"%Y-%m-%d"`
Mapping map[string]interface{} `json:"mapping" dc:"映射表（MAPPING类型时使用）"`
}
// ============================================================
@@ -146,205 +148,255 @@ type TransformRule struct {
// UserSelectQueryReq 用户选择查询请求
type UserSelectQueryReq struct {
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"`
Dimensions []string `json:"dimensions" dc:"统计维度列表，如 shop_id/anchor_id/date"`
Indicators []IndicatorSelect `json:"indicators" dc:"统计指标列表(含聚合方式)"`
Filters []FilterCondition `json:"filters" dc:"筛选条件列表"`
TimeRange *TimeRange `json:"timeRange" dc:"时间范围"`
TimeGroup string `json:"timeGroup" dc:"时间分组: day/week/month/quarter"`
TimeGroup string `json:"timeGroup" dc:"时间分组: day/week/month/quarter" eg:"day"`
OrderBy []OrderCondition `json:"orderBy" dc:"排序条件"`
Page int `json:"page" dc:"页码" d:"1"`
PageSize int `json:"pageSize" dc:"每页条数" d:"20"`
Page int `json:"page" dc:"页码" d:"1" eg:"1"`
PageSize int `json:"pageSize" dc:"每页条数" d:"20" eg:"20"`
}
// IndicatorSelect 指标选择
type IndicatorSelect struct {
FieldCode string `json:"fieldCode" dc:"字段编码"`
Aggregate string `json:"aggregate" dc:"聚合方式: SUM/COUNT/AVG/MAX/MIN"`
Alias string `json:"alias" dc:"别名"`
FieldCode string `json:"fieldCode" dc:"字段编码" eg:"order_amount"`
Aggregate string `json:"aggregate" dc:"聚合方式: SUM/COUNT/AVG/MAX/MIN" eg:"SUM"`
Alias string `json:"alias" dc:"别名" eg:"总订单金额"`
}
// FilterCondition 筛选条件
type FilterCondition struct {
FieldCode string `json:"fieldCode" dc:"字段编码"`
Operator string `json:"operator" dc:"操作符: =/!=/>/</>=/<=/IN/LIKE/BETWEEN"`
FieldCode string `json:"fieldCode" dc:"字段编码" eg:"order_amount"`
Operator string `json:"operator" dc:"操作符: =/!=/>/</>=/<=/IN/LIKE/BETWEEN" eg:"="`
Value interface{} `json:"value" dc:"值"`
Value2 interface{} `json:"value2" dc:"第二个值(BETWEEN时使用)"`
}
// TimeRange 时间范围
type TimeRange struct {
StartDate string `json:"startDate" dc:"开始日期 yyyy-MM-dd"`
EndDate string `json:"endDate" dc:"结束日期 yyyy-MM-dd"`
StartDate string `json:"startDate" dc:"开始日期 yyyy-MM-dd" eg:"2026-06-01"`
EndDate string `json:"endDate" dc:"结束日期 yyyy-MM-dd" eg:"2026-06-30"`
}
// OrderCondition 排序条件
type OrderCondition struct {
FieldCode string `json:"fieldCode" dc:"字段编码"`
Direction string `json:"direction" dc:"排序方向: ASC/DESC"`
FieldCode string `json:"fieldCode" dc:"字段编码" eg:"order_amount"`
Direction string `json:"direction" dc:"排序方向: ASC/DESC" eg:"DESC"`
}
// UserSelectQueryResp 用户选择查询响应
type UserSelectQueryResp struct {
List []map[string]interface{} `json:"list" dc:"数据列表"`
Total int64 `json:"total" dc:"总数"`
Page int `json:"page" dc:"当前页"`
PageSize int `json:"pageSize" dc:"每页条数"`
TotalPages int `json:"totalPages" dc:"总页数"`
Sql string `json:"sql,omitempty" dc:"执行的SQL(调试用)"`
ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)"`
Total int64 `json:"total" dc:"总数" eg:"100"`
Page int `json:"page" dc:"当前页" eg:"1"`
PageSize int `json:"pageSize" dc:"每页条数" eg:"20"`
TotalPages int `json:"totalPages" dc:"总页数" eg:"5"`
Sql string `json:"sql,omitempty" dc:"执行的SQL(调试用)" eg:"SELECT 1"`
ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)" eg:"1234"`
}
// ExtractDailyDataReq 按天抽取数据请求
type ExtractDailyDataReq struct {
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
StatDate string `json:"statDate" v:"required" dc:"统计日期 yyyy-MM-dd"`
Executor string `json:"executor" dc:"执行人"`
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"`
StatDate string `json:"statDate" v:"required" dc:"统计日期 yyyy-MM-dd" eg:"2026-06-01"`
Executor string `json:"executor" dc:"执行人" eg:"system"`
}
// ExtractDailyDataResp 按天抽取数据响应
type ExtractDailyDataResp struct {
Success bool `json:"success" dc:"是否成功"`
TotalCount int `json:"totalCount" dc:"总记录数"`
SuccessCount int `json:"successCount" dc:"成功记录数"`
FailCount int `json:"failCount" dc:"失败记录数"`
ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)"`
Success bool `json:"success" dc:"是否成功" eg:"true"`
TotalCount int `json:"totalCount" dc:"总记录数" eg:"100"`
SuccessCount int `json:"successCount" dc:"成功记录数" eg:"98"`
FailCount int `json:"failCount" dc:"失败记录数" eg:"2"`
ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)" eg:"1234"`
ErrorMsg string `json:"errorMsg" dc:"错误信息"`
}
// AutoCreateStatTableReq 自动创建统计宽表请求
type AutoCreateStatTableReq struct {
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
Creator string `json:"creator" dc:"创建人"`
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"`
Creator string `json:"creator" dc:"创建人" eg:"admin"`
}
// AutoCreateStatTableResp 自动创建统计宽表响应
type AutoCreateStatTableResp struct {
Success bool `json:"success" dc:"是否成功"`
TableName string `json:"tableName" dc:"创建的表名"`
ColumnCount int `json:"columnCount" dc:"字段数量"`
ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)"`
Success bool `json:"success" dc:"是否成功" eg:"true"`
TableName string `json:"tableName" dc:"创建的表名" eg:"stat_kuaishou_shop_daily"`
ColumnCount int `json:"columnCount" dc:"字段数量" eg:"15"`
ExecTimeMs int64 `json:"execTimeMs" dc:"执行耗时(毫秒)" eg:"1234"`
}
// GetReportFieldsResp 获取报表可用字段响应
type GetReportFieldsResp struct {
BusinessCode string `json:"businessCode" dc:"业务编码"`
ReportCode string `json:"reportCode" dc:"报表编码"`
BusinessCode string `json:"businessCode" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `json:"reportCode" dc:"报表编码" eg:"shop_daily"`
Dimensions []FieldConfig `json:"dimensions" dc:"维度字段列表"`
Indicators []FieldConfig `json:"indicators" dc:"指标字段列表"`
Filters []FieldConfig `json:"filters" dc:"筛选字段列表"`
}
// BackfillReq 批量回填请求
type BackfillReq struct {
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"`
StartDate string `json:"startDate" v:"required" dc:"开始日期 yyyy-MM-dd" eg:"2026-06-01"`
EndDate string `json:"endDate" v:"required" dc:"结束日期 yyyy-MM-dd" eg:"2026-06-30"`
Executor string `json:"executor" dc:"执行人" eg:"admin"`
}
// BackfillResp 批量回填响应
type BackfillResp struct {
Success bool `json:"success" dc:"是否全部成功" eg:"true"`
TotalDays int `json:"totalDays" dc:"总天数" eg:"30"`
SuccessDays int `json:"successDays" dc:"成功天数" eg:"28"`
FailDays int `json:"failDays" dc:"失败天数" eg:"2"`
ExecTimeMs int64 `json:"execTimeMs" dc:"总耗时(毫秒)" eg:"5678"`
ErrorMsg string `json:"errorMsg" dc:"错误信息"`
}
// ============================================================
// 配置 CRUD 请求/响应
// ============================================================
// SaveBusinessReq 保存业务配置请求（新增/修改合一）
type SaveBusinessReq struct {
ID *int64 `json:"id"` // 有值为更新，无值为新增
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
BusinessName string `json:"businessName" v:"required" dc:"业务名称"`
Description string `json:"description" dc:"描述"`
Status string `json:"status" dc:"状态 ACTIVE/INACTIVE" d:"ACTIVE"`
ID *int64 `json:"id" dc:"有值为更新，无值为新增" eg:"1"`
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"`
BusinessName string `json:"businessName" v:"required" dc:"业务名称" eg:"快手电商"`
Description string `json:"description" dc:"描述" eg:"快手电商店铺每日统计报表"`
Status string `json:"status" dc:"状态 ACTIVE/INACTIVE" d:"ACTIVE" eg:"ACTIVE"`
Config map[string]interface{} `json:"config" dc:"扩展配置"`
Operator string `json:"operator" dc:"操作人"`
Operator string `json:"operator" dc:"操作人" eg:"="`
}
// SaveReportReq 保存报表配置请求
type SaveReportReq struct {
ID *int64 `json:"id"`
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
ReportName string `json:"reportName" v:"required" dc:"报表名称"`
Description string `json:"description" dc:"描述"`
Status string `json:"status" dc:"状态" d:"ACTIVE"`
ID *int64 `json:"id" dc:"主键ID" eg:"1"`
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"`
ReportName string `json:"reportName" v:"required" dc:"报表名称" eg:"店铺日报"`
Description string `json:"description" dc:"描述" eg:"快手电商店铺每日统计报表"`
Status string `json:"status" dc:"状态" d:"ACTIVE" eg:"ACTIVE"`
StatTableName string `json:"statTableName" v:"required" dc:"统计宽表名"`
StatTableComment string `json:"statTableComment" dc:"统计宽表注释"`
DateField string `json:"dateField" dc:"日期字段" d:"stat_date"`
DateField string `json:"dateField" dc:"日期字段" d:"stat_date" eg:"stat_date"`
PrimaryKeys []string `json:"primaryKeys" dc:"主键字段"`
ConflictKeys []string `json:"conflictKeys" dc:"冲突键（唯一索引）"`
Config map[string]interface{} `json:"config" dc:"扩展配置"`
Operator string `json:"operator" dc:"操作人"`
Operator string `json:"operator" dc:"操作人" eg:"="`
}
// SaveFieldReq 保存字段配置请求
type SaveFieldReq struct {
ID *int64 `json:"id"`
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
FieldCode string `json:"fieldCode" v:"required" dc:"字段编码"`
FieldName string `json:"fieldName" v:"required" dc:"字段名称"`
FieldType string `json:"fieldType" v:"required" dc:"字段类型 STRING/INT/FLOAT/DATE/DATETIME/JSONB"`
DataType string `json:"dataType" dc:"数据存储类型" d:"STRING"`
FieldRole string `json:"fieldRole" v:"required" dc:"字段角色 DIMENSION/INDICATOR/FILTER/FILTER_ONLY"`
IsAggregatable bool `json:"isAggregatable" dc:"是否可聚合"`
IsFilterable bool `json:"isFilterable" dc:"是否可筛选" d:"true"`
IsQueryable bool `json:"isQueryable" dc:"是否可查询" d:"true"`
IsSortable bool `json:"isSortable" dc:"是否可排序" d:"true"`
DefaultAggregate string `json:"defaultAggregate" dc:"默认聚合方式"`
ID *int64 `json:"id" dc:"主键ID" eg:"1"`
BusinessCode string `json:"businessCode" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `json:"reportCode" dc:"报表编码" eg:"shop_daily"`
FieldCode string `json:"fieldCode" v:"required" dc:"字段编码" eg:"order_amount"`
FieldName string `json:"fieldName" v:"required" dc:"字段名称" eg:"订单金额"`
FieldType string `json:"fieldType" v:"required" dc:"字段类型 STRING/INT/FLOAT/DATE/DATETIME/JSONB" eg:"FLOAT"`
DataType string `json:"dataType" dc:"数据存储类型" d:"STRING" eg:"STRING"`
FieldRole string `json:"fieldRole" v:"required" dc:"字段角色 DIMENSION/INDICATOR/FILTER/FILTER_ONLY" eg:"DIMENSION"`
IsAggregatable bool `json:"isAggregatable" dc:"是否可聚合" eg:"true"`
IsFilterable bool `json:"isFilterable" dc:"是否可筛选" d:"true" eg:"true"`
IsQueryable bool `json:"isQueryable" dc:"是否可查询" d:"true" eg:"true"`
IsSortable bool `json:"isSortable" dc:"是否可排序" d:"true" eg:"true"`
DefaultAggregate string `json:"defaultAggregate" dc:"默认聚合方式" eg:"SUM"`
ValidAggregates []string `json:"validAggregates" dc:"可选聚合列表"`
FilterOperators []string `json:"filterOperators" dc:"可选操作符列表"`
Expression string `json:"expression" dc:"表达式（衍生字段）"`
ExpressionType string `json:"expressionType" dc:"表达式类型 DIRECT/CALCULATED"`
FormatPattern string `json:"formatPattern" dc:"格式化模板"`
Unit string `json:"unit" dc:"单位"`
DictCode string `json:"dictCode" dc:"字典编码"`
SortOrder int `json:"sortOrder" dc:"排序"`
GroupName string `json:"groupName" dc:"分组名称"`
Status string `json:"status" dc:"状态" d:"ACTIVE"`
Operator string `json:"operator" dc:"操作人"`
Expression string `json:"expression" dc:"表达式（衍生字段）" eg:"order_amount - refund_amount"`
ExpressionType string `json:"expressionType" dc:"表达式类型 DIRECT/CALCULATED" eg:"DIRECT"`
FormatPattern string `json:"formatPattern" dc:"格式化模板" eg:"#,##0.00"`
Unit string `json:"unit" dc:"单位" eg:"元"`
DictCode string `json:"dictCode" dc:"字典编码" eg:"dict_order_type"`
SortOrder int `json:"sortOrder" dc:"排序" eg:"1"`
GroupName string `json:"groupName" dc:"分组名称" eg:"交易指标"`
Status string `json:"status" dc:"状态" d:"ACTIVE" eg:"ACTIVE"`
Operator string `json:"operator" dc:"操作人" eg:"admin"`
}
// SaveReportWithFieldsReq 保存报表及字段配置请求（字段全量替换）
type SaveReportWithFieldsReq struct {
ID *int64 `json:"id" dc:"主键ID（更新时必填，新建为空）"`
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"`
ReportName string `json:"reportName" v:"required" dc:"报表名称" eg:"店铺日报"`
Description string `json:"description" dc:"描述" eg:"快手电商店铺每日统计报表"`
Status string `json:"status" dc:"状态" d:"ACTIVE" eg:"ACTIVE"`
StatTableName string `json:"statTableName" v:"required" dc:"统计宽表名" eg:"stat_kuaishou_shop_daily"`
StatTableComment string `json:"statTableComment" dc:"统计宽表注释" eg:"快手电商-店铺日报"`
DateField string `json:"dateField" dc:"日期字段" d:"stat_date" eg:"stat_date"`
PrimaryKeys []string `json:"primaryKeys" dc:"主键字段"`
ConflictKeys []string `json:"conflictKeys" dc:"冲突键（唯一索引）"`
Config map[string]interface{} `json:"config" dc:"扩展配置"`
Fields []SaveFieldReq `json:"fields" dc:"字段配置列表（全量替换，不在列表中的字段将被删除）"`
}
// SaveExtractConfigReq 保存抽取配置请求
type SaveExtractConfigReq struct {
ID *int64 `json:"id"`
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
ExtractCode string `json:"extractCode" v:"required" dc:"抽取编码"`
ExtractName string `json:"extractName" v:"required" dc:"抽取名称"`
SourceTableName string `json:"sourceTableName" v:"required" dc:"源表名"`
SourceTableAlias string `json:"sourceTableAlias" dc:"源表别名"`
TargetTableName string `json:"targetTableName" v:"required" dc:"目标表名"`
IsEnabled bool `json:"isEnabled" dc:"是否启用" d:"true"`
ExtractType string `json:"extractType" dc:"抽取类型 FULL/INCREMENTAL" d:"INCREMENTAL"`
ExtractMode string `json:"extractMode" dc:"抽取模式 DIRECT/AGGREGATE" d:"DIRECT"`
ExtractKeyField string `json:"extractKeyField" dc:"抽取关键字段（增量依据）"`
ExtractKeyFormat string `json:"extractKeyFormat" dc:"关键字段格式"`
ID *int64 `json:"id" dc:"主键ID" eg:"1"`
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"`
ExtractCode string `json:"extractCode" v:"required" dc:"抽取编码" eg:"kuaishou_order_extract"`
ExtractName string `json:"extractName" v:"required" dc:"抽取名称" eg:"订单抽取"`
SourceTableName string `json:"sourceTableName" v:"required" dc:"源表名" eg:"ods_kuaishou_order"`
SourceTableAlias string `json:"sourceTableAlias" dc:"源表别名" eg:"o"`
TargetTableName string `json:"targetTableName" v:"required" dc:"目标表名" eg:"stat_kuaishou_shop_daily"`
IsEnabled bool `json:"isEnabled" dc:"是否启用" d:"true" eg:"true"`
ExtractType string `json:"extractType" dc:"抽取类型 FULL/INCREMENTAL" d:"INCREMENTAL" eg:"INCREMENTAL"`
ExtractMode string `json:"extractMode" dc:"抽取模式 DIRECT/AGGREGATE" d:"DIRECT" eg:"DIRECT"`
ExtractKeyField string `json:"extractKeyField" dc:"抽取关键字段（增量依据）" eg:"updated_at"`
ExtractKeyFormat string `json:"extractKeyFormat" dc:"关键字段格式" eg:"yyyy-MM-dd HH:mm:ss"`
GroupByFields []string `json:"groupByFields" dc:"GROUP BY 字段列表"`
FilterExpression string `json:"filterExpression" dc:"过滤表达式"`
FilterExpression string `json:"filterExpression" dc:"过滤表达式" eg:"status = 'PAID'"`
JoinConfigs []JoinConfig `json:"joinConfigs" dc:"JOIN配置"`
FieldMappings []FieldMapping `json:"fieldMappings" dc:"字段映射列表"`
TransformRules []TransformRule `json:"transformRules" dc:"转换规则列表"`
BatchSize int `json:"batchSize" dc:"批处理大小" d:"1000"`
Status string `json:"status" dc:"状态" d:"ACTIVE"`
Operator string `json:"operator" dc:"操作人"`
BatchSize int `json:"batchSize" dc:"批处理大小" d:"1000" eg:"1000"`
Status string `json:"status" dc:"状态" d:"ACTIVE" eg:"ACTIVE"`
Operator string `json:"operator" dc:"操作人" eg:"="`
}
// PageReq 分页请求参数
type PageReq struct {
PageNum int `json:"pageNum" d:"1" dc:"页码"`
PageSize int `json:"pageSize" d:"20" dc:"每页条数"`
}
// PageRes 分页响应
type PageRes struct {
Total int `json:"total" dc:"总数"`
Page int `json:"page" dc:"当前页"`
PageSize int `json:"pageSize" dc:"每页条数"`
TotalPages int `json:"totalPages" dc:"总页数"`
}
// IdReq 通用 ID 请求
type IdReq struct {
ID int64 `json:"id" v:"required" dc:"主键ID"`
ID int64 `json:"id" v:"required" dc:"主键ID" eg:"1"`
}
// SaveResult 写操作通用返回
type SaveResult struct {
Success bool `json:"success"`
ID int64 `json:"id"`
Message string `json:"message"`
Success bool `json:"success" dc:"是否成功" eg:"true"`
ID int64 `json:"id" dc:"主键ID" eg:"1"`
Message string `json:"message" dc:"结果消息" eg:"操作成功"`
}
// DeleteResult 删除结果
type DeleteResult struct {
Success bool `json:"success"`
Message string `json:"message"`
Success bool `json:"success" dc:"是否成功" eg:"true"`
Message string `json:"message" dc:"结果消息" eg:"操作成功"`
}
// GetExtractConfigsReq 获取抽取配置列表请求
type GetExtractConfigsReq struct {
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码"`
BusinessCode string `json:"businessCode" v:"required" dc:"业务编码" eg:"KUAISHOU"`
ReportCode string `json:"reportCode" v:"required" dc:"报表编码" eg:"shop_daily"`
}
// ============================================================
@@ -401,3 +453,21 @@ const (
ExtractStatusSuccess = "SUCCESS"
ExtractStatusFailed = "FAILED"
)
// FieldTypeToPG 字段类型转PG列类型定义
func FieldTypeToPG(fieldType string) string {
switch fieldType {
case FieldTypeInt:
return "NUMERIC(20,0) DEFAULT 0"
case FieldTypeFloat:
return "NUMERIC(20,4) DEFAULT 0"
case FieldTypeDate:
return "VARCHAR(16) DEFAULT ''"
case FieldTypeDatetime:
return "TIMESTAMP WITH TIME ZONE"
case FieldTypeJsonb:
return "JSONB DEFAULT '{}'"
default:
return "VARCHAR(256) DEFAULT ''"
}
}