From ea1d5c2fd854489ccb98d0a182fd55939fd5f46f Mon Sep 17 00:00:00 2001 From: admin <259278618@qq.com> Date: Wed, 11 Mar 2026 19:05:02 +0800 Subject: [PATCH 001/225] Initial commit --- .gitignore | 27 +++++++++++++++++++++++++++ README.md | 2 ++ 2 files changed, 29 insertions(+) create mode 100644 .gitignore create mode 100644 README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5b90e79 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# ---> Go +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +go.work.sum + +# env file +.env + diff --git a/README.md b/README.md new file mode 100644 index 0000000..5b26276 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# common + From ea5dfe629ff8d2a93466b328e602c30e38bc0a1b Mon Sep 17 00:00:00 2001 From: admin <259278618@qq.com> Date: Wed, 11 Mar 2026 19:10:45 +0800 Subject: [PATCH 002/225] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E8=87=B3=E3=80=8C/=E3=80=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 32 +- beans.go | 54 +++ go.mod | 166 +++++++++ go.sum | 918 +++++++++++++++++++++++++++++++++++++++++++++++ module_tenant.go | 81 +++++ 5 files changed, 1224 insertions(+), 27 deletions(-) create mode 100644 beans.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 module_tenant.go diff --git a/.gitignore b/.gitignore index 5b90e79..406a7f0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,27 +1,5 @@ -# ---> Go -# If you prefer the allow list template instead of the deny list, see community template: -# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore -# -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib - -# Test binary, built with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Dependency directories (remove the comment below to include it) -# vendor/ - -# Go workspace file -go.work -go.work.sum - -# env file -.env - +/.idea/* +rabbitmq/开发指南.md +ragflow/agent文档.md +ragflow/README_GLOBAL.md +redis/stream使用示例.md diff --git a/beans.go b/beans.go new file mode 100644 index 0000000..31806bd --- /dev/null +++ b/beans.go @@ -0,0 +1,54 @@ +package beans + +import ( + "time" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +type ResponseEmpty struct { +} + +type Page struct { + PageNum int64 `p:"pageNum"` //当前页码 + PageSize int64 `p:"pageSize"` //每页数 + Total int64 //总页数 +} + +type OrderEnum string + +const ( + Asc OrderEnum = "asc" // 正序 + Desc OrderEnum = "desc" // 倒序 +) + +type OrderBy struct { + Field string `p:"field"` //排序字段 + Order OrderEnum `p:"order"` //排序方式 +} +type MongoBaseDO struct { + Id *bson.ObjectID `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID + Creator interface{} `bson:"creator,omitempty" json:"creator"` + CreatedAt *time.Time `bson:"createdAt,omitempty" json:"createdAt"` + Updater interface{} `bson:"updater,omitempty" json:"updater"` + UpdatedAt *time.Time `bson:"updatedAt,omitempty" json:"updatedAt"` + TenantId interface{} `bson:"tenantId" json:"tenantId" default:"1"` // 租户ID + IsDeleted bool `bson:"isDeleted" json:"isDeleted" default:"false"` +} + +// SQLBaseDO SQL数据库基础实体 +type SQLBaseDO struct { + Id uint64 `json:"id"` // 主键ID + Creator string `json:"creator"` // 创建人 + CreatedAt *time.Time `json:"createdAt"` // 创建时间 + Updater string `json:"updater"` // 更新人 + UpdatedAt *time.Time `json:"updatedAt"` // 更新时间 + TenantId string `json:"tenantId"` // 租户ID + IsDeleted bool `json:"isDeleted"` // 是否删除 +} + +type User struct { + UserId interface{} `bson:"userId" json:"userId"` // 用户ID + UserName interface{} `bson:"userName" json:"userName"` // 用户名 + TenantId interface{} `bson:"tenantId" json:"tenantId"` // 租户ID +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..33e76c0 --- /dev/null +++ b/go.mod @@ -0,0 +1,166 @@ +module gitea.com/red-future/common + +go 1.25.3 + +require ( + github.com/alibaba/sentinel-golang v1.0.4 + github.com/cloudwego/eino v0.7.26 + github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5 + github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 + github.com/gogf/gf/v2 v2.9.5 + github.com/google/uuid v1.6.0 + github.com/meilisearch/meilisearch-go v0.36.1 + github.com/minio/minio-go/v7 v7.0.97 + github.com/nats-io/nats.go v1.48.0 + github.com/olivere/elastic/v7 v7.0.32 + github.com/rabbitmq/amqp091-go v1.10.0 + github.com/smallnest/rpcx v1.9.1 + github.com/tiger1103/gfast-token v1.0.10 + go.mongodb.org/mongo-driver/v2 v2.4.0 + go.opentelemetry.io/otel v1.38.0 + go.opentelemetry.io/otel/trace v1.38.0 +) + +require ( + github.com/BurntSushi/toml v1.5.0 // indirect + github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/akutz/memconn v0.1.0 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/apache/thrift v0.21.0 // indirect + github.com/armon/go-metrics v0.4.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/buger/jsonparser v1.1.1 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.14.1 // indirect + github.com/bytedance/sonic/loader v0.3.0 // indirect + github.com/cenk/backoff v2.2.1+incompatible // indirect + github.com/cenkalti/backoff v2.2.1+incompatible // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/clbanning/mxj/v2 v2.7.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/dgraph-io/badger/v4 v4.2.0 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgryski/go-jump v0.0.0-20211018200510-ba001c3ffce0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/edwingeng/doublejump v1.0.1 // indirect + github.com/eino-contrib/jsonschema v1.0.3 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.4 // indirect + github.com/go-ping/ping v1.2.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/gogf/gf/contrib/nosql/redis/v2 v2.9.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/golang/glog v1.2.5 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/google/flatbuffers v1.12.1 // indirect + github.com/google/pprof v0.0.0-20250128161936-077ca0a936bf // indirect + github.com/goph/emperror v0.17.2 // indirect + github.com/gorilla/websocket v1.5.3 // indirect + github.com/grandcat/zeroconf v1.0.0 // indirect + github.com/grokify/html-strip-tags-go v0.1.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect + github.com/hashicorp/consul/api v1.26.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v1.5.0 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/serf v0.10.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/juju/ratelimit v1.0.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/klauspost/reedsolomon v1.12.4 // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/miekg/dns v1.1.63 // indirect + github.com/minio/crc64nvme v1.1.0 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nats-io/nkeys v0.4.11 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/nikolalohinski/gonja v1.5.3 // indirect + github.com/olekukonko/errors v1.1.0 // indirect + github.com/olekukonko/ll v0.0.9 // indirect + github.com/olekukonko/tablewriter v1.1.0 // indirect + github.com/onsi/ginkgo/v2 v2.22.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.9 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/quic-go/quic-go v0.49.0 // indirect + github.com/redis/go-redis/v9 v9.12.1 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/rubyist/circuitbreaker v2.2.1+incompatible // indirect + github.com/shirou/gopsutil/v3 v3.21.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect + github.com/smallnest/quick v0.2.0 // indirect + github.com/smallnest/rsocket v0.0.0-20241130031020-4a72eb6ff62a // indirect + github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 // indirect + github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b // indirect + github.com/tinylib/msgp v1.3.0 // indirect + github.com/tjfoc/gmsm v1.4.1 // indirect + github.com/tklauser/go-sysconf v0.3.6 // indirect + github.com/tklauser/numcpus v0.2.2 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/valyala/fastrand v1.1.0 // indirect + github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/xtaci/kcp-go v5.4.20+incompatible // indirect + github.com/yargevad/filepathx v1.0.0 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + go.opencensus.io v0.23.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk v1.38.0 // indirect + go.opentelemetry.io/proto/otlp v1.7.1 // indirect + go.uber.org/mock v0.5.0 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/exp v0.0.0-20250128144449-3edf0e91c1ae // indirect + golang.org/x/mod v0.26.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sync v0.16.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/tools v0.35.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect + google.golang.org/grpc v1.75.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..68fb0ce --- /dev/null +++ b/go.sum @@ -0,0 +1,918 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= +github.com/akutz/memconn v0.1.0 h1:NawI0TORU4hcOMsMr11g7vwlCdkYeLKXBcxWu2W/P8A= +github.com/akutz/memconn v0.1.0/go.mod h1:Jo8rI7m0NieZyLI5e2CDlRdRqRRB4S7Xp77ukDjH+Fw= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alibaba/sentinel-golang v1.0.4 h1:i0wtMvNVdy7vM4DdzYrlC4r/Mpk1OKUUBurKKkWhEo8= +github.com/alibaba/sentinel-golang v1.0.4/go.mod h1:Lag5rIYyJiPOylK8Kku2P+a23gdKMMqzQS7wTnjWEpk= +github.com/alitto/pond v1.9.2 h1:9Qb75z/scEZVCoSU+osVmQ0I0JOeLfdTDafrbcJ8CLs= +github.com/alitto/pond v1.9.2/go.mod h1:xQn3P/sHTYcU/1BR3i86IGIrilcrGC2LiS+E2+CJWsI= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= +github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= +github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.14.1 h1:FBMC0zVz5XUmE4z9wF4Jey0An5FueFvOsTKKKtwIl7w= +github.com/bytedance/sonic v1.14.1/go.mod h1:gi6uhQLMbTdeP0muCnrjHLeCUPyb70ujhnNlhOylAFc= +github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= +github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenk/backoff v2.2.1+incompatible h1:djdFT7f4gF2ttuzRKPbMOWgZajgesItGLwG5FTQKmmE= +github.com/cenk/backoff v2.2.1+incompatible/go.mod h1:7FtoeaSnHoZnmZzz47cM35Y9nSW7tNyaidugnHTaFDE= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME= +github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/cloudwego/eino v0.7.26 h1:FD8xnhd8WyV495eJs0Rka8aHnknqj/ljq3ZbwGH/SiQ= +github.com/cloudwego/eino v0.7.26/go.mod h1:nA8Vacmuqv3pqKBQbTWENBLQ8MmGmPt/WqiyLeB8ohQ= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-jump v0.0.0-20170409065014-e1f439676b57/go.mod h1:4hKCXuwrJoYvHZxJ86+bRVTOMyJ0Ej+RqfSm8mHi6KA= +github.com/dgryski/go-jump v0.0.0-20211018200510-ba001c3ffce0 h1:0wH6nO9QEa02Qx8sIQGw6ieKdz+BXjpccSOo9vXNl4U= +github.com/dgryski/go-jump v0.0.0-20211018200510-ba001c3ffce0/go.mod h1:4hKCXuwrJoYvHZxJ86+bRVTOMyJ0Ej+RqfSm8mHi6KA= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/edwingeng/doublejump v1.0.1 h1:wJ6QgNyyF23Of9vw+ThbwJ/obe9KdxaWEg/Brpv5S1o= +github.com/edwingeng/doublejump v1.0.1/go.mod h1:ykMWX8JWePtMtk2OGjNE9kwtgpI+SF2FNIyXV4gS36k= +github.com/eino-contrib/jsonschema v1.0.3 h1:2Kfsm1xlMV0ssY2nuxshS4AwbLFuqmPmzIjLVJ1Fsp0= +github.com/eino-contrib/jsonschema v1.0.3/go.mod h1:cpnX4SyKjWjGC7iN2EbhxaTdLqGjCi0e9DxpLYxddD4= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= +github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a/go.mod h1:7Ga40egUymuWXxAe151lTNnCv97MddSOVsjpPPkityA= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-ping/ping v1.2.0 h1:vsJ8slZBZAXNCK4dPcI2PEE9eM9n9RbXbGouVQ/Y4yQ= +github.com/go-ping/ping v1.2.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/godzie44/go-uring v0.0.0-20220926161041-69611e8b13d5 h1:5zELAgnSz0gqmr4Q5DWCoOzNHoeBAxVUXB7LS1eG+sw= +github.com/godzie44/go-uring v0.0.0-20220926161041-69611e8b13d5/go.mod h1:ermjEDUoT/fS+3Ona5Vd6t6mZkw1eHp99ILO5jGRBkM= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogf/gf/contrib/nosql/redis/v2 v2.9.1 h1:egobo4YfQX3C4NtrEFunBqMX3jsddagklgut9u91+BM= +github.com/gogf/gf/contrib/nosql/redis/v2 v2.9.1/go.mod h1:YQ+u5Cs5N2ETCeQaLbv29z/UWYuxw27mJpUkOLj0kJ8= +github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5 h1:eUqwJ/qNH8lJ6yssiqskazgp1ACQuNU6zXlLOZVuXTQ= +github.com/gogf/gf/contrib/registry/consul/v2 v2.9.5/go.mod h1:sjQyMry9+0POYZCA6lHXBxO77WoNKkruJpRB4xKqk5k= +github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5 h1:tHUEZYB5GTqEYYVDYnlGobf1xISARKDE4KHVlgjwTec= +github.com/gogf/gf/contrib/trace/otlphttp/v2 v2.9.5/go.mod h1:cfzTn2HS9RDX8f5pUVkbGxUWcSosouqfNQ1G6cY0V88= +github.com/gogf/gf/v2 v2.9.5 h1:1scfOdHbMP854oQaiLejl+eL+c4xfuvtWmmZiDJxbKs= +github.com/gogf/gf/v2 v2.9.5/go.mod h1:VUb5eyJKpvW77O/dXsbbLNO/Kjrg0UycIiq0lRiBjjo= +github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.2.5 h1:DrW6hGnjIhtvhOIiAKT6Psh/Kd/ldepEa81DKeiRJ5I= +github.com/golang/glog v1.2.5/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20250128161936-077ca0a936bf h1:BvBLUD2hkvLI3dJTJMiopAq8/wp43AAZKTP7qdpptbU= +github.com/google/pprof v0.0.0-20250128161936-077ca0a936bf/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grandcat/zeroconf v1.0.0 h1:uHhahLBKqwWBV6WZUDAT71044vwOTL+McW0mBJvo6kE= +github.com/grandcat/zeroconf v1.0.0/go.mod h1:lTKmG1zh86XyCoUeIHSA4FJMBwCJiQmGfcP2PdzytEs= +github.com/grokify/html-strip-tags-go v0.1.0 h1:03UrQLjAny8xci+R+qjCce/MYnpNXCtgzltlQbOBae4= +github.com/grokify/html-strip-tags-go v0.1.0/go.mod h1:ZdzgfHEzAfz9X6Xe5eBLVblWIxXfYSQ40S/VKrAOGpc= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= +github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/api v1.26.1 h1:5oSXOO5fboPZeW5SN+TdGFP/BILDgBm19OrPZ/pICIM= +github.com/hashicorp/consul/api v1.26.1/go.mod h1:B4sQTeaSO16NtynqrAdwOlahJ7IUDZM9cj2420xYL8A= +github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.15.0 h1:2qK9nDrr4tiJKRoxPGhm6B7xJjLVIQqkjiab2M4aKjU= +github.com/hashicorp/consul/sdk v0.15.0/go.mod h1:r/OmRRPbHOe0yxNahLw7G9x5WG17E1BIECMtCjcPSNo= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= +github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= +github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= +github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kavu/go_reuseport v1.5.0 h1:UNuiY2OblcqAtVDE8Gsg1kZz8zbBWg907sP1ceBV+bk= +github.com/kavu/go_reuseport v1.5.0/go.mod h1:CG8Ee7ceMFSMnx/xr25Vm0qXaj2Z4i5PWoUx+JZ5/CU= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/klauspost/reedsolomon v1.12.4 h1:5aDr3ZGoJbgu/8+j45KtUJxzYm8k08JGtB9Wx1VQ4OA= +github.com/klauspost/reedsolomon v1.12.4/go.mod h1:d3CzOMOt0JXGIFZm1StgkyF14EYr3xneR2rNWo7NcMU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/libp2p/go-sockaddr v0.2.0 h1:Alhhj6lGxVAon9O32tOO89T601EugSx6YiGjy5BVjWk= +github.com/libp2p/go-sockaddr v0.2.0/go.mod h1:5NxulaB17yJ07IpzRIleys4un0PJ7WLWgMDLBBWrGw8= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/meilisearch/meilisearch-go v0.36.1 h1:mJTCJE5g7tRvaqKco6DfqOuJEjX+rRltDEnkEC02Y0M= +github.com/meilisearch/meilisearch-go v0.36.1/go.mod h1:hWcR0MuWLSzHfbz9GGzIr3s9rnXLm1jqkmHkJPbUSvM= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY= +github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs= +github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= +github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= +github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U= +github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= +github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= +github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/olekukonko/errors v1.1.0 h1:RNuGIh15QdDenh+hNvKrJkmxxjV4hcS50Db478Ou5sM= +github.com/olekukonko/errors v1.1.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= +github.com/olekukonko/ll v0.0.9 h1:Y+1YqDfVkqMWuEQMclsF9HUR5+a82+dxJuL1HHSRpxI= +github.com/olekukonko/ll v0.0.9/go.mod h1:En+sEW0JNETl26+K8eZ6/W4UQ7CYSrrgg/EdIYT2H8g= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v1.1.0 h1:N0LHrshF4T39KvI96fn6GT8HEjXRXYNDrDjKFDB7RIY= +github.com/olekukonko/tablewriter v1.1.0/go.mod h1:5c+EBPeSqvXnLLgkm9isDdzR3wjfBkHR9Nhfp3NWrzo= +github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E= +github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= +github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.36.2 h1:koNYke6TVk6ZmnyHrCXba/T/MoLBXFjeC1PtvYgw0A8= +github.com/onsi/gomega v1.36.2/go.mod h1:DdwyADRjrc825LhMEkD76cHR5+pUnjhUN8GlHlRPHzY= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/peterbourgon/g2s v0.0.0-20140925154142-ec76db4c1ac1 h1:5Dl+ADmsGerAqHwWzyLqkNaUBQ+48DQwfDCaW1gHAQM= +github.com/peterbourgon/g2s v0.0.0-20140925154142-ec76db4c1ac1/go.mod h1:1VcHEd3ro4QMoHfiNl/j7Jkln9+KQuorp0PItHMJYNg= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/quic-go/quic-go v0.49.0 h1:w5iJHXwHxs1QxyBv1EHKuC50GX5to8mJAxvtnttJp94= +github.com/quic-go/quic-go v0.49.0/go.mod h1:s2wDnmCdooUQBmQfpUSTCYBl1/D4FcqbULMMkASvR6s= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/redis/go-redis/v9 v9.12.1 h1:k5iquqv27aBtnTm2tIkROUDp8JBXhXZIVu1InSgvovg= +github.com/redis/go-redis/v9 v9.12.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rubyist/circuitbreaker v2.2.1+incompatible h1:KUKd/pV8Geg77+8LNDwdow6rVCAYOp8+kHUyFvL6Mhk= +github.com/rubyist/circuitbreaker v2.2.1+incompatible/go.mod h1:Ycs3JgJADPuzJDwffe12k6BZT8hxVi6lFK+gWYJLN4A= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shirou/gopsutil/v3 v3.21.6 h1:vU7jrp1Ic/2sHB7w6UNs7MIkn7ebVtTb5D9j45o9VYE= +github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+LwHTKj0ST88= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg= +github.com/smallnest/quick v0.2.0 h1:AEvm7ZovZ6Utv+asFDBh866G4ufMNhRNMKbZHVMFYPE= +github.com/smallnest/quick v0.2.0/go.mod h1:ODNivpfZTaMgYrNb/fhDtqoEe2TTPxSRo8JaIT/QThI= +github.com/smallnest/rpcx v1.9.1 h1:fGw+qMcDRPm7Ei9fdEfqteYY6qQqgVsoyxoUw6hEXy0= +github.com/smallnest/rpcx v1.9.1/go.mod h1:owr4mDCReTn+dy9m5ilof0mBivFBeK0XrkYfZYdDGb4= +github.com/smallnest/rsocket v0.0.0-20241130031020-4a72eb6ff62a h1:GI6kCNC5AVFbKA6ZKbVd4r+fk+Z7XZCRQm9LURZY4t4= +github.com/smallnest/rsocket v0.0.0-20241130031020-4a72eb6ff62a/go.mod h1:VJeIKKrDEzT4ZNVe87JN9uRLw1XLp/ZnnE9PfsyJ1jY= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161 h1:89CEmDvlq/F7SJEOqkIdNDGJXrQIhuIx9D2DBXjavSU= +github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU= +github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b h1:fj5tQ8acgNUr6O8LEplsxDhUIe2573iLkJc+PqnzZTI= +github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4= +github.com/tiger1103/gfast-token v1.0.10 h1:fNiBE/Dq5iTHvTGlCx3DmXa2o4hr0NtumFpffZ39k6s= +github.com/tiger1103/gfast-token v1.0.10/go.mod h1:a/21mxmj7zFeNvjhZSC0XpEAFHfb1aT2k6DXnufFU1s= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= +github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= +github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4= +github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= +github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= +github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xtaci/kcp-go v5.4.20+incompatible h1:TN1uey3Raw0sTz0Fg8GkfM0uH3YwzhnZWQ1bABv5xAg= +github.com/xtaci/kcp-go v5.4.20+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= +github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37 h1:EWU6Pktpas0n8lLQwDsRyZfmkPeRbdgPtW609es+/9E= +github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37/go.mod h1:HpMP7DB2CyokmAh4lp0EQnnWhmycP/TvwBGzvuie+H0= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= +go.mongodb.org/mongo-driver/v2 v2.4.0 h1:Oq6BmUAAFTzMeh6AonuDlgZMuAuEiUxoAD1koK5MuFo= +go.mongodb.org/mongo-driver/v2 v2.4.0/go.mod h1:jHeEDJHJq7tm6ZF45Issun9dbogjfnPySb1vXA7EeAI= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 h1:GqRJVj7UmLjCVyVJ3ZFLdPRmhDUp2zFmQe3RHIOsw24= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0/go.mod h1:ri3aaHSmCTVYu2AWv44YMauwAQc0aqI9gHKIcSbI1pU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/proto/otlp v1.7.1 h1:gTOMpGDb0WTBOP8JaO72iL3auEZhVmAQg4ipjOVAtj4= +go.opentelemetry.io/proto/otlp v1.7.1/go.mod h1:b2rVh6rfI/s2pHWNlB7ILJcRALpcNDzKhACevjI+ZnE= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= +go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= +golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20250128144449-3edf0e91c1ae h1:COZdc9Ut6wLq7MO9GIYxfZl4n4ScmgqQLoHocKXrxco= +golang.org/x/exp v0.0.0-20250128144449-3edf0e91c1ae/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg= +golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 h1:BIRfGDEjiHRrk0QKZe3Xv2ieMhtgRGeLcZQ0mIVn4EY= +google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5/go.mod h1:j3QtIyytwqGr1JUDtYXwtMXWPKsEa5LtzIFN1Wn5WvE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 h1:eaY8u2EuxbRv7c3NiGK0/NedzVsCcV6hDuU5qPX5EGE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5/go.mod h1:M4/wBTSeyLxupu3W3tJtOgB14jILAS/XWPSSa3TAlJc= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/module_tenant.go b/module_tenant.go new file mode 100644 index 0000000..4b53f8b --- /dev/null +++ b/module_tenant.go @@ -0,0 +1,81 @@ +package beans + +import ( + "github.com/gogf/gf/v2/os/gtime" +) + +// ModuleAssetId 模块资产ID映射(key-value结构) +// Key: 服务名,Value: 资产ID +var ModuleAssetId = map[string]string{ + "assets": "696b4acd1be1c8b76c4b4c15", // 资产模块 + "cid": "696f423705e496ba4ccbe665", // 广告模块 + "customerService": "696f421205e496ba4ccbe662", // AI客服模块 +} + +// 模块类型(值从ModuleAssetId map获取) +var ( + TenantModuleAssets = ModuleAssetId["assets"] // 资产模块 + TenantModuleAd = ModuleAssetId["cid"] // 广告模块 + TenantModuleAICs = ModuleAssetId["customerService"] // AI客服模块 +) + +type TenantModuleType string + +const ( + TenantModuleTypePlatform TenantModuleType = "platform" + TenantModuleTypePrivate TenantModuleType = "private" + TenantModuleTypeSupplier TenantModuleType = "supplier" + TenantModuleTypeSmallShop TenantModuleType = "small_shop" +) + +// TenantModuleTypeKV 租户类型 +type TenantModuleTypeKV struct { + Key string + Value string +} + +// TenantModuleTypesAssets 资产模块租户类型 +var TenantModuleTypesAssets = []TenantModuleTypeKV{ + {Key: string(TenantModuleTypePrivate), Value: "私域租户"}, + {Key: string(TenantModuleTypeSupplier), Value: "供应商"}, + {Key: string(TenantModuleTypeSmallShop), Value: "电商小店"}, +} + +// TenantModuleTypesAd 广告模块租户类型(待定) +var TenantModuleTypesAd []TenantModuleTypeKV + +// TenantModuleTypesAICs AI客服模块租户类型(待定) +var TenantModuleTypesAICs []TenantModuleTypeKV + +// GetTenantModuleTypes 获取模块的租户类型列表 +func GetTenantModuleTypes(module string) []TenantModuleTypeKV { + switch module { + case TenantModuleAssets: + return TenantModuleTypesAssets + case TenantModuleAd: + return TenantModuleTypesAd + case TenantModuleAICs: + return TenantModuleTypesAICs + default: + return []TenantModuleTypeKV{} + } +} + +type ModuleTenantCheckReq struct { + ModuleKey string `p:"moduleKey" v:"required#模块Key不能为空"` + TenantId uint64 `p:"tenantId" v:"required#租户ID不能为空"` +} + +// ModuleTenantCheckRes 调用admin-go设置模块租户关系的响应 +type ModuleTenantCheckRes struct { + Status bool `json:"status"` + CertificationStatus bool `json:"certificationStatus"` + Message string `json:"message"` // 状态描述 +} + +// ModuleTenant 模块租户关系实体(引用自admin-go) +type ModuleTenant struct { + ExpireAt *gtime.Time `json:"expireAt" description:"到期时间"` + TenantModuleType TenantModuleType `json:"tenantModuleType" description:"租户模块类型"` + CertificationStatus int `json:"certificationStatus" description:"认证状态"` +} From 3d8af8183b83aa9a68cd0142ed2875c7212a5cef Mon Sep 17 00:00:00 2001 From: admin <259278618@qq.com> Date: Wed, 11 Mar 2026 19:11:01 +0800 Subject: [PATCH 003/225] =?UTF-8?q?=E4=B8=8A=E4=BC=A0=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E8=87=B3=E3=80=8C/=E3=80=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From f6b00baae77daa3978685f630cfb8b2ed3f4e088 Mon Sep 17 00:00:00 2001 From: admin <259278618@qq.com> Date: Wed, 11 Mar 2026 19:11:29 +0800 Subject: [PATCH 004/225] =?UTF-8?q?=E5=88=A0=E9=99=A4=20beans.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- beans.go | 54 ------------------------------------------------------ 1 file changed, 54 deletions(-) delete mode 100644 beans.go diff --git a/beans.go b/beans.go deleted file mode 100644 index 31806bd..0000000 --- a/beans.go +++ /dev/null @@ -1,54 +0,0 @@ -package beans - -import ( - "time" - - "go.mongodb.org/mongo-driver/v2/bson" -) - -type ResponseEmpty struct { -} - -type Page struct { - PageNum int64 `p:"pageNum"` //当前页码 - PageSize int64 `p:"pageSize"` //每页数 - Total int64 //总页数 -} - -type OrderEnum string - -const ( - Asc OrderEnum = "asc" // 正序 - Desc OrderEnum = "desc" // 倒序 -) - -type OrderBy struct { - Field string `p:"field"` //排序字段 - Order OrderEnum `p:"order"` //排序方式 -} -type MongoBaseDO struct { - Id *bson.ObjectID `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID - Creator interface{} `bson:"creator,omitempty" json:"creator"` - CreatedAt *time.Time `bson:"createdAt,omitempty" json:"createdAt"` - Updater interface{} `bson:"updater,omitempty" json:"updater"` - UpdatedAt *time.Time `bson:"updatedAt,omitempty" json:"updatedAt"` - TenantId interface{} `bson:"tenantId" json:"tenantId" default:"1"` // 租户ID - IsDeleted bool `bson:"isDeleted" json:"isDeleted" default:"false"` -} - -// SQLBaseDO SQL数据库基础实体 -type SQLBaseDO struct { - Id uint64 `json:"id"` // 主键ID - Creator string `json:"creator"` // 创建人 - CreatedAt *time.Time `json:"createdAt"` // 创建时间 - Updater string `json:"updater"` // 更新人 - UpdatedAt *time.Time `json:"updatedAt"` // 更新时间 - TenantId string `json:"tenantId"` // 租户ID - IsDeleted bool `json:"isDeleted"` // 是否删除 -} - -type User struct { - UserId interface{} `bson:"userId" json:"userId"` // 用户ID - UserName interface{} `bson:"userName" json:"userName"` // 用户名 - TenantId interface{} `bson:"tenantId" json:"tenantId"` // 租户ID -} From 252c98b14966963cfea442e00609fce066b17225 Mon Sep 17 00:00:00 2001 From: admin <259278618@qq.com> Date: Wed, 11 Mar 2026 19:11:44 +0800 Subject: [PATCH 005/225] =?UTF-8?q?=E5=88=A0=E9=99=A4=20module=5Ftenant.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- module_tenant.go | 81 ------------------------------------------------ 1 file changed, 81 deletions(-) delete mode 100644 module_tenant.go diff --git a/module_tenant.go b/module_tenant.go deleted file mode 100644 index 4b53f8b..0000000 --- a/module_tenant.go +++ /dev/null @@ -1,81 +0,0 @@ -package beans - -import ( - "github.com/gogf/gf/v2/os/gtime" -) - -// ModuleAssetId 模块资产ID映射(key-value结构) -// Key: 服务名,Value: 资产ID -var ModuleAssetId = map[string]string{ - "assets": "696b4acd1be1c8b76c4b4c15", // 资产模块 - "cid": "696f423705e496ba4ccbe665", // 广告模块 - "customerService": "696f421205e496ba4ccbe662", // AI客服模块 -} - -// 模块类型(值从ModuleAssetId map获取) -var ( - TenantModuleAssets = ModuleAssetId["assets"] // 资产模块 - TenantModuleAd = ModuleAssetId["cid"] // 广告模块 - TenantModuleAICs = ModuleAssetId["customerService"] // AI客服模块 -) - -type TenantModuleType string - -const ( - TenantModuleTypePlatform TenantModuleType = "platform" - TenantModuleTypePrivate TenantModuleType = "private" - TenantModuleTypeSupplier TenantModuleType = "supplier" - TenantModuleTypeSmallShop TenantModuleType = "small_shop" -) - -// TenantModuleTypeKV 租户类型 -type TenantModuleTypeKV struct { - Key string - Value string -} - -// TenantModuleTypesAssets 资产模块租户类型 -var TenantModuleTypesAssets = []TenantModuleTypeKV{ - {Key: string(TenantModuleTypePrivate), Value: "私域租户"}, - {Key: string(TenantModuleTypeSupplier), Value: "供应商"}, - {Key: string(TenantModuleTypeSmallShop), Value: "电商小店"}, -} - -// TenantModuleTypesAd 广告模块租户类型(待定) -var TenantModuleTypesAd []TenantModuleTypeKV - -// TenantModuleTypesAICs AI客服模块租户类型(待定) -var TenantModuleTypesAICs []TenantModuleTypeKV - -// GetTenantModuleTypes 获取模块的租户类型列表 -func GetTenantModuleTypes(module string) []TenantModuleTypeKV { - switch module { - case TenantModuleAssets: - return TenantModuleTypesAssets - case TenantModuleAd: - return TenantModuleTypesAd - case TenantModuleAICs: - return TenantModuleTypesAICs - default: - return []TenantModuleTypeKV{} - } -} - -type ModuleTenantCheckReq struct { - ModuleKey string `p:"moduleKey" v:"required#模块Key不能为空"` - TenantId uint64 `p:"tenantId" v:"required#租户ID不能为空"` -} - -// ModuleTenantCheckRes 调用admin-go设置模块租户关系的响应 -type ModuleTenantCheckRes struct { - Status bool `json:"status"` - CertificationStatus bool `json:"certificationStatus"` - Message string `json:"message"` // 状态描述 -} - -// ModuleTenant 模块租户关系实体(引用自admin-go) -type ModuleTenant struct { - ExpireAt *gtime.Time `json:"expireAt" description:"到期时间"` - TenantModuleType TenantModuleType `json:"tenantModuleType" description:"租户模块类型"` - CertificationStatus int `json:"certificationStatus" description:"认证状态"` -} From f87bd9f52a51ec9a4760d2c1dcd5fc849836aace Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 25 Nov 2025 11:51:16 +0800 Subject: [PATCH 006/225] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/consul.go | 24 ++++++++++++ http/http.go | 77 ++++++++++++++++++++++++++++++++++++++ jaeger/jaeger.go | 75 +++++++++++++++++++++++++++++++++++++ mongo/mongo.go | 96 ++++++++++++++++++++++++++++++++++++++++++++++++ utils/utils.go | 53 ++++++++++++++++++++++++++ 5 files changed, 325 insertions(+) create mode 100644 consul/consul.go create mode 100644 http/http.go create mode 100644 jaeger/jaeger.go create mode 100644 mongo/mongo.go create mode 100644 utils/utils.go diff --git a/consul/consul.go b/consul/consul.go new file mode 100644 index 0000000..177a74b --- /dev/null +++ b/consul/consul.go @@ -0,0 +1,24 @@ +package consul + +import ( + "context" + _ "github.com/gogf/gf/contrib/drivers/mysql/v2" + _ "github.com/gogf/gf/contrib/nosql/redis/v2" + "github.com/gogf/gf/contrib/registry/consul/v2" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/gsvc" + _ "go.mongodb.org/mongo-driver/mongo" +) + +func init() { + consulCfg, err := g.Cfg().Get(context.Background(), "consul.address") + if err != nil { + panic(err) + } + consulAddr := consulCfg.String() + registry, err := consul.New(consul.WithAddress(consulAddr)) + if err != nil { + panic(err) + } + gsvc.SetRegistry(registry) +} diff --git a/http/http.go b/http/http.go new file mode 100644 index 0000000..f4e2efb --- /dev/null +++ b/http/http.go @@ -0,0 +1,77 @@ +package http + +import ( + "context" + "errors" + "gitee.com/red-future---jilin-g/common/utils" + "github.com/gogf/gf/contrib/registry/consul/v2" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/gclient" + "github.com/gogf/gf/v2/net/ghttp" + "github.com/gogf/gf/v2/net/gsel" + "github.com/gogf/gf/v2/net/gsvc" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/util/gconv" + "net/http" +) + +type ResponseEmpty struct { +} + +const PageSize = 20 + +type Page struct { + PageNum int `p:"pageNum"` //当前页码 + PageSize int `p:"pageSize"` //每页数 + Total int //总页数 +} + +func getHttpClient(ctx context.Context) (client *gclient.Client, err error) { + consulCfg, _ := g.Cfg().Get(context.Background(), "consul.address") + consulAddr := consulCfg.String() + registry, err := consul.New(consul.WithAddress(consulAddr)) + if err != nil { + return + } + gsvc.SetRegistry(registry) + gsel.SetBuilder(gsel.NewBuilderRoundRobin()) + client = g.Client() + client.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) + client.SetDiscovery(gsvc.GetRegistry()) + return +} +func doRequest(ctx context.Context, method string, url string, target any, data ...any) (err error) { + err = utils.ValidStructPtr(target) + if err != nil { + return + } + client, err := getHttpClient(ctx) + if err != nil { + return + } + response, err := client.Get(ctx, method, url, data) + if err != nil { + return + } + defer func() { + if err = response.Close(); err != nil { + glog.Errorf(ctx, `%+v`, err) + } + }() + result := response.ReadAll() + resultStrut := &ghttp.DefaultHandlerResponse{} + if gconv.Struct(result, resultStrut); resultStrut.Code != 200 { + err = errors.New(resultStrut.Message) + } else { + gconv.Struct(resultStrut.Data, target) + } + return +} +func Get(ctx context.Context, url string, target any, data ...any) (err error) { + err = doRequest(ctx, http.MethodGet, url, target, data) + return +} +func Post(ctx context.Context, url string, target any, data ...any) (err error) { + err = doRequest(ctx, http.MethodPost, url, target, data) + return +} diff --git a/jaeger/jaeger.go b/jaeger/jaeger.go new file mode 100644 index 0000000..ee0c491 --- /dev/null +++ b/jaeger/jaeger.go @@ -0,0 +1,75 @@ +package jaeger + +import ( + "context" + "encoding/json" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/ghttp" + "github.com/gogf/gf/v2/util/gconv" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/jaeger" + "go.opentelemetry.io/otel/sdk/resource" + "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.21.0" + "strconv" +) + +var Tp = new(trace.TracerProvider) + +func init() { + jaegerAgent, err := g.Cfg().Get(context.Background(), "jaeger.addr") + if err != nil { + panic(err) + } + + exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(gconv.String(jaegerAgent)))) + if err != nil { + panic(err) + } + serverName, err := g.Cfg().Get(context.Background(), "consul.Name") + if err != nil { + panic(err) + } + // 创建一个 TracerProvider,并将 Jaeger exporter 设置为其处理器 + Tp = trace.NewTracerProvider( + // 使用 BatchSpanProcessor 可以提高性能,它会批量发送 span + trace.WithBatcher(exp), + // 设置资源属性,这些属性会附加到所有导出的 span 上 + trace.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceName(gconv.String(serverName)), // 服务名称,在 Jaeger UI 中会显示 + )), + ) + // 将 TracerProvider 设置为全局,方便在应用的任何地方通过 otel.Tracer() 获取 + otel.SetTracerProvider(Tp) +} +func NewTracer(r *ghttp.Request) { + // 从传入的上下文中获取 Tracer,或者直接使用全局的 + tracer := otel.Tracer(r.GetServeHandler().GetMetaTag("summary")) + _, span := tracer.Start(r.Context(), r.GetServeHandler().GetMetaTag("summary")) + defer span.End() // 非常重要:确保 span 在函数结束时被关闭 + span.SetAttributes(attribute.String("request", getParams(r))) + r.Middleware.Next() + span.SetAttributes(attribute.String("response", r.Response.BufferString())) + //span.AddEvent("Saying hello is done") +} +func getParams(r *ghttp.Request) string { + params := map[string]interface{}{} + if r.Method == "POST" { + json.Unmarshal(r.GetBody(), ¶ms) //获取raw传参 + } + if r.Method == "GET" { + r.Request.ParseForm() + form := r.Form + for k, v := range form { + if vl, e := strconv.Atoi(v[0]); e == nil { + params[k] = vl + } else { + params[k] = v[0] + } + } + } + rp, _ := json.Marshal(¶ms) + return string(rp) +} diff --git a/mongo/mongo.go b/mongo/mongo.go new file mode 100644 index 0000000..41fc717 --- /dev/null +++ b/mongo/mongo.go @@ -0,0 +1,96 @@ +package mongo + +import ( + "context" + "gitee.com/red-future---jilin-g/common/utils" + "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/text/gstr" + "go.mongodb.org/mongo-driver/bson/primitive" + "strings" + "time" + + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +var db = new(mongo.Database) + +func init() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + link, _ := g.Cfg().Get(context.Background(), "mongo.address") + mongoAddr := link.String() + client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoAddr)) + if err != nil { + glog.Error(ctx, "mongodb连接失败") + } + dbName := gstr.SubStr(mongoAddr, strings.LastIndex(mongoAddr, "/")+1, len(mongoAddr)) + db = client.Database(dbName) +} + +// Find 查询多条记录 +func Find(ctx context.Context, filter *primitive.M, result interface{}, collection string, opts ...*options.FindOptions) (err error) { + if err = utils.ValidStructPtr(result); err != nil { + return + } + cur, err := db.Collection(collection).Find(ctx, filter, opts...) + if err != nil { + return + } + err = cur.All(ctx, result) + return +} + +// FindOne 查询1条记录 +func FindOne(ctx context.Context, filter *primitive.M, result interface{}, collection string, opts ...*options.FindOneOptions) (err error) { + if len(*filter) == 0 { + err = gerror.New("缺少查询条件") + return + } + if err = utils.ValidStructPtr(result); err != nil { + return + } + cur := db.Collection(collection).FindOne(ctx, filter, opts...) + if err != nil { + return + } + err = cur.Decode(result) + if err == mongo.ErrNoDocuments { + err = nil + } + return +} + +// Delete 删除记录 +func Delete(ctx context.Context, filter *primitive.M, collection string, opts ...*options.DeleteOptions) (count int64, err error) { + r, err := db.Collection(collection).DeleteMany(ctx, filter, opts...) + if err != nil { + return + } + count = r.DeletedCount + return +} + +// Update 修改记录 +func Update(ctx context.Context, filter *primitive.M, update interface{}, result *mongo.UpdateResult, collection string, opts ...*options.UpdateOptions) (err error) { + if err = utils.ValidStructPtr(result); err != nil { + return + } + result, err = db.Collection(collection).UpdateMany(ctx, filter, update, opts...) + if err != nil { + return + } + return +} + +// Insert 修改记录 +func Insert(ctx context.Context, documents []interface{}, collection string, opts ...*options.InsertManyOptions) (ids []interface{}, err error) { + r, err := db.Collection(collection).InsertMany(ctx, documents, opts...) + if err != nil { + return + } + ids = r.InsertedIDs + return +} diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 0000000..5c24731 --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,53 @@ +package utils + +import ( + "fmt" + "github.com/gogf/gf/v2/errors/gcode" + "github.com/gogf/gf/v2/errors/gerror" + "reflect" + "time" +) + +// ValidStructPtr 验证是否为结构体指针 +func ValidStructPtr(req any) (err error) { + //验证请求参数必须为指针 + var ( + reflectValue reflect.Value + reflectKind reflect.Kind + ) + if v, ok := req.(reflect.Value); ok { + reflectValue = v + } else { + reflectValue = reflect.ValueOf(req) + } + + reflectKind = reflectValue.Kind() + if reflectKind != reflect.Ptr { + err = gerror.NewCode(gcode.CodeInvalidParameter, `the parameter "req" for function Find should type of *struct/*[]struct`) + } + return +} + +// GetMonthToday 获取N个月前的某日 +func GetMonthToday(t time.Time, month int) time.Time { + // today + fmt.Printf("today: [%s]\n", t) + // 判断天数范围 小于等于28天的计算,覆盖大多数情况 + if t.Day() <= 28 { + return t.AddDate(0, -month, 0) + } + // 月份的天数数组 + monthDay := [13]int{0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} + // 计算目标所在日期 + target := t.AddDate(0, 0, 1-t.Day()).AddDate(0, -month, 0) + // 计算当月最大天数 + targetDay := monthDay[target.Month()] + // 计算闰年 + if target.Month() == time.February && (target.Year()%400 == 0 || (target.Year()%100 != 0 && target.Year()%4 == 0)) { + targetDay++ + } + if t.Day() > targetDay { + return target.AddDate(0, 0, targetDay-1) + } + return target.AddDate(0, 0, t.Day()-1) +} From 8432b09dd49eaa3b0ad8a38dec7bad38b9a125a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 25 Nov 2025 13:07:27 +0800 Subject: [PATCH 007/225] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 2 +- jaeger/jaeger.go | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/http/http.go b/http/http.go index f4e2efb..dfccac5 100644 --- a/http/http.go +++ b/http/http.go @@ -49,7 +49,7 @@ func doRequest(ctx context.Context, method string, url string, target any, data if err != nil { return } - response, err := client.Get(ctx, method, url, data) + response, err := client.DoRequest(ctx, method, url, data) if err != nil { return } diff --git a/jaeger/jaeger.go b/jaeger/jaeger.go index ee0c491..0e7b716 100644 --- a/jaeger/jaeger.go +++ b/jaeger/jaeger.go @@ -3,6 +3,7 @@ package jaeger import ( "context" "encoding/json" + "fmt" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/util/gconv" @@ -22,12 +23,12 @@ func init() { if err != nil { panic(err) } - + fmt.Println(jaegerAgent) exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(gconv.String(jaegerAgent)))) if err != nil { panic(err) } - serverName, err := g.Cfg().Get(context.Background(), "consul.Name") + serverName, err := g.Cfg().Get(context.Background(), "server.Name") if err != nil { panic(err) } From 7e8f10e5576cd6428739ea425d02bb7692317d2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 25 Nov 2025 13:25:56 +0800 Subject: [PATCH 008/225] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jaeger/jaeger.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/jaeger/jaeger.go b/jaeger/jaeger.go index 0e7b716..4126d01 100644 --- a/jaeger/jaeger.go +++ b/jaeger/jaeger.go @@ -3,7 +3,6 @@ package jaeger import ( "context" "encoding/json" - "fmt" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/util/gconv" @@ -23,7 +22,6 @@ func init() { if err != nil { panic(err) } - fmt.Println(jaegerAgent) exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(gconv.String(jaegerAgent)))) if err != nil { panic(err) From 53243082fd042a5efe9e8bb1a783170a5ed8ec8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 25 Nov 2025 13:52:20 +0800 Subject: [PATCH 009/225] =?UTF-8?q?http=E6=9C=8D=E5=8A=A1=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E3=80=81=E8=B7=AF=E7=94=B1=E6=B3=A8=E5=86=8C=E6=96=B9?= =?UTF-8?q?=E6=B3=95=E5=B0=81=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/http/http.go b/http/http.go index dfccac5..05e8dac 100644 --- a/http/http.go +++ b/http/http.go @@ -3,6 +3,8 @@ package http import ( "context" "errors" + "fmt" + "gitee.com/red-future---jilin-g/common/jaeger" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/contrib/registry/consul/v2" "github.com/gogf/gf/v2/frame/g" @@ -13,6 +15,9 @@ import ( "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/util/gconv" "net/http" + "reflect" + "regexp" + "strings" ) type ResponseEmpty struct { @@ -26,6 +31,29 @@ type Page struct { Total int //总页数 } +var HttpServer = g.Server() + +func init() { + //s.Use(common.Cors) //中间件验证 + //s.EnablePProf() //启用性能分析 + HttpServer.SetOpenApiPath("/api.json") + HttpServer.SetSwaggerPath("/swagger") //api文档访问路径 + HttpServer.SetDumpRouterMap(true) //关闭打印路由注册信息 + HttpServer.BindMiddlewareDefault(ghttp.MiddlewareHandlerResponse, jaeger.NewTracer) //使用默认http返回结构 + go HttpServer.Run() +} +func RouteRegister(controllers []interface{}) { + re := regexp.MustCompile("[A-Z]") + for _, t := range controllers { + sName := reflect.ValueOf(t).Elem().Type().Name() + convertedStr := re.ReplaceAllStringFunc(sName, func(s string) string { + return fmt.Sprintf("/%s", strings.ToLower(s)) + }) + HttpServer.Group(convertedStr, func(group *ghttp.RouterGroup) { + group.Bind(t) + }) + } +} func getHttpClient(ctx context.Context) (client *gclient.Client, err error) { consulCfg, _ := g.Cfg().Get(context.Background(), "consul.address") consulAddr := consulCfg.String() From 718d6a25288f008b7016e679bd3ca301d812db16 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 25 Nov 2025 13:47:20 +0800 Subject: [PATCH 010/225] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=B8=BAMongo=20=20D?= =?UTF-8?q?river=20V2=E5=86=99=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 41fc717..d41faf3 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -2,17 +2,17 @@ package mongo import ( "context" + "strings" + "time" + "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/text/gstr" - "go.mongodb.org/mongo-driver/bson/primitive" - "strings" - "time" - - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" ) var db = new(mongo.Database) @@ -22,16 +22,22 @@ func init() { defer cancel() link, _ := g.Cfg().Get(context.Background(), "mongo.address") mongoAddr := link.String() - client, err := mongo.Connect(ctx, options.Client().ApplyURI(mongoAddr)) + opt := options.Client().ApplyURI(mongoAddr) + client, err := mongo.Connect(opt) if err != nil { - glog.Error(ctx, "mongodb连接失败") + glog.Error(ctx, "mongodb连接失败", err) } + // 从连接串中解析数据库名 dbName := gstr.SubStr(mongoAddr, strings.LastIndex(mongoAddr, "/")+1, len(mongoAddr)) + // 如果连接串带有参数(如 ?retryWrites=true),需要去掉参数部分 + if strings.Contains(dbName, "?") { + dbName = gstr.SubStr(dbName, 0, strings.Index(dbName, "?")) + } db = client.Database(dbName) } // Find 查询多条记录 -func Find(ctx context.Context, filter *primitive.M, result interface{}, collection string, opts ...*options.FindOptions) (err error) { +func Find(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.FindOptions) (err error) { if err = utils.ValidStructPtr(result); err != nil { return } @@ -44,8 +50,8 @@ func Find(ctx context.Context, filter *primitive.M, result interface{}, collecti } // FindOne 查询1条记录 -func FindOne(ctx context.Context, filter *primitive.M, result interface{}, collection string, opts ...*options.FindOneOptions) (err error) { - if len(*filter) == 0 { +func FindOne(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.FindOneOptions) (err error) { + if len(filter) == 0 { err = gerror.New("缺少查询条件") return } @@ -53,9 +59,6 @@ func FindOne(ctx context.Context, filter *primitive.M, result interface{}, colle return } cur := db.Collection(collection).FindOne(ctx, filter, opts...) - if err != nil { - return - } err = cur.Decode(result) if err == mongo.ErrNoDocuments { err = nil @@ -64,7 +67,7 @@ func FindOne(ctx context.Context, filter *primitive.M, result interface{}, colle } // Delete 删除记录 -func Delete(ctx context.Context, filter *primitive.M, collection string, opts ...*options.DeleteOptions) (count int64, err error) { +func Delete(ctx context.Context, filter bson.M, collection string, opts ...options.DeleteOptions) (count int64, err error) { r, err := db.Collection(collection).DeleteMany(ctx, filter, opts...) if err != nil { return @@ -74,10 +77,7 @@ func Delete(ctx context.Context, filter *primitive.M, collection string, opts .. } // Update 修改记录 -func Update(ctx context.Context, filter *primitive.M, update interface{}, result *mongo.UpdateResult, collection string, opts ...*options.UpdateOptions) (err error) { - if err = utils.ValidStructPtr(result); err != nil { - return - } +func Update(ctx context.Context, filter bson.M, update interface{}, collection string, opts ...options.UpdateOptions) (result *mongo.UpdateResult, err error) { result, err = db.Collection(collection).UpdateMany(ctx, filter, update, opts...) if err != nil { return @@ -85,8 +85,8 @@ func Update(ctx context.Context, filter *primitive.M, update interface{}, result return } -// Insert 修改记录 -func Insert(ctx context.Context, documents []interface{}, collection string, opts ...*options.InsertManyOptions) (ids []interface{}, err error) { +// Insert 插入多条记录 +func Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.InsertManyOptions) (ids []interface{}, err error) { r, err := db.Collection(collection).InsertMany(ctx, documents, opts...) if err != nil { return From 8665705f269cb7ef83dbc070f0854156b35bd1f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 25 Nov 2025 14:19:52 +0800 Subject: [PATCH 011/225] =?UTF-8?q?mongo=E5=B7=A5=E5=85=B7=E7=B1=BB?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index d41faf3..aadc838 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -37,7 +37,7 @@ func init() { } // Find 查询多条记录 -func Find(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.FindOptions) (err error) { +func Find(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOptions]) (err error) { if err = utils.ValidStructPtr(result); err != nil { return } @@ -50,7 +50,7 @@ func Find(ctx context.Context, filter bson.M, result interface{}, collection str } // FindOne 查询1条记录 -func FindOne(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.FindOneOptions) (err error) { +func FindOne(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOneOptions]) (err error) { if len(filter) == 0 { err = gerror.New("缺少查询条件") return @@ -67,7 +67,7 @@ func FindOne(ctx context.Context, filter bson.M, result interface{}, collection } // Delete 删除记录 -func Delete(ctx context.Context, filter bson.M, collection string, opts ...options.DeleteOptions) (count int64, err error) { +func Delete(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.DeleteManyOptions]) (count int64, err error) { r, err := db.Collection(collection).DeleteMany(ctx, filter, opts...) if err != nil { return @@ -77,7 +77,7 @@ func Delete(ctx context.Context, filter bson.M, collection string, opts ...optio } // Update 修改记录 -func Update(ctx context.Context, filter bson.M, update interface{}, collection string, opts ...options.UpdateOptions) (result *mongo.UpdateResult, err error) { +func Update(ctx context.Context, filter bson.M, update interface{}, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { result, err = db.Collection(collection).UpdateMany(ctx, filter, update, opts...) if err != nil { return @@ -86,7 +86,7 @@ func Update(ctx context.Context, filter bson.M, update interface{}, collection s } // Insert 插入多条记录 -func Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.InsertManyOptions) (ids []interface{}, err error) { +func Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { r, err := db.Collection(collection).InsertMany(ctx, documents, opts...) if err != nil { return From 62f545affd814cfc219874685503b0fdc1172f27 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 25 Nov 2025 16:37:29 +0800 Subject: [PATCH 012/225] =?UTF-8?q?mongo=20driver=20=20=E7=9A=84=20countdo?= =?UTF-8?q?cument=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mongo/mongo.go b/mongo/mongo.go index aadc838..f0a8eb2 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -94,3 +94,10 @@ func Insert(ctx context.Context, documents []interface{}, collection string, opt ids = r.InsertedIDs return } + +// Count 查询总数 +func Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { + // 调用驱动的 CountDocuments,在数据库端执行的 + count, err = db.Collection(collection).CountDocuments(ctx, filter) + return +} \ No newline at end of file From cf65e0383cf66c192f43d4a848647d73a1a79c98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 25 Nov 2025 16:55:37 +0800 Subject: [PATCH 013/225] =?UTF-8?q?mongo=E5=B7=A5=E5=85=B7=E7=B1=BB?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/ragflow.go | 5 +++++ ragflow/ragflow_dto.go | 1 + 2 files changed, 6 insertions(+) create mode 100644 ragflow/ragflow.go create mode 100644 ragflow/ragflow_dto.go diff --git a/ragflow/ragflow.go b/ragflow/ragflow.go new file mode 100644 index 0000000..78bf86d --- /dev/null +++ b/ragflow/ragflow.go @@ -0,0 +1,5 @@ +package ragflow + +func init() { + +} diff --git a/ragflow/ragflow_dto.go b/ragflow/ragflow_dto.go new file mode 100644 index 0000000..3cdcbe1 --- /dev/null +++ b/ragflow/ragflow_dto.go @@ -0,0 +1 @@ +package ragflow From 15a5311288e2409a115989bf9c016d4f259d163a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 26 Nov 2025 10:38:15 +0800 Subject: [PATCH 014/225] =?UTF-8?q?=E4=BC=98=E5=8C=96jaeger=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=EF=BC=8C=E4=BD=BF=E7=94=A8otlp=E8=A7=84=E8=8C=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 15 ++++++++------- jaeger/jaeger.go | 37 +++++++++---------------------------- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/http/http.go b/http/http.go index 05e8dac..6b82dec 100644 --- a/http/http.go +++ b/http/http.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + _ "gitee.com/red-future---jilin-g/common/consul" "gitee.com/red-future---jilin-g/common/jaeger" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/contrib/registry/consul/v2" @@ -31,16 +32,16 @@ type Page struct { Total int //总页数 } -var HttpServer = g.Server() +var Httpserver = g.Server() func init() { //s.Use(common.Cors) //中间件验证 //s.EnablePProf() //启用性能分析 - HttpServer.SetOpenApiPath("/api.json") - HttpServer.SetSwaggerPath("/swagger") //api文档访问路径 - HttpServer.SetDumpRouterMap(true) //关闭打印路由注册信息 - HttpServer.BindMiddlewareDefault(ghttp.MiddlewareHandlerResponse, jaeger.NewTracer) //使用默认http返回结构 - go HttpServer.Run() + Httpserver.SetOpenApiPath("/api.json") + Httpserver.SetSwaggerPath("/swagger") //api文档访问路径 + Httpserver.SetDumpRouterMap(true) //关闭打印路由注册信息 + Httpserver.BindMiddlewareDefault(ghttp.MiddlewareHandlerResponse, jaeger.NewTracer) //使用默认http返回结构 + go Httpserver.Run() } func RouteRegister(controllers []interface{}) { re := regexp.MustCompile("[A-Z]") @@ -49,7 +50,7 @@ func RouteRegister(controllers []interface{}) { convertedStr := re.ReplaceAllStringFunc(sName, func(s string) string { return fmt.Sprintf("/%s", strings.ToLower(s)) }) - HttpServer.Group(convertedStr, func(group *ghttp.RouterGroup) { + Httpserver.Group(convertedStr, func(group *ghttp.RouterGroup) { group.Bind(t) }) } diff --git a/jaeger/jaeger.go b/jaeger/jaeger.go index 4126d01..ea8fa94 100644 --- a/jaeger/jaeger.go +++ b/jaeger/jaeger.go @@ -3,55 +3,36 @@ package jaeger import ( "context" "encoding/json" + "github.com/gogf/gf/contrib/trace/otlphttp/v2" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" - "github.com/gogf/gf/v2/util/gconv" - "go.opentelemetry.io/otel" + "github.com/gogf/gf/v2/net/gtrace" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/exporters/jaeger" - "go.opentelemetry.io/otel/sdk/resource" - "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.21.0" "strconv" ) -var Tp = new(trace.TracerProvider) +var ShutDown func(ctx context.Context) func init() { jaegerAgent, err := g.Cfg().Get(context.Background(), "jaeger.addr") if err != nil { panic(err) } - exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(gconv.String(jaegerAgent)))) - if err != nil { - panic(err) - } serverName, err := g.Cfg().Get(context.Background(), "server.Name") if err != nil { panic(err) } - // 创建一个 TracerProvider,并将 Jaeger exporter 设置为其处理器 - Tp = trace.NewTracerProvider( - // 使用 BatchSpanProcessor 可以提高性能,它会批量发送 span - trace.WithBatcher(exp), - // 设置资源属性,这些属性会附加到所有导出的 span 上 - trace.WithResource(resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceName(gconv.String(serverName)), // 服务名称,在 Jaeger UI 中会显示 - )), - ) - // 将 TracerProvider 设置为全局,方便在应用的任何地方通过 otel.Tracer() 获取 - otel.SetTracerProvider(Tp) + ShutDown, err = otlphttp.Init(serverName.String(), jaegerAgent.String(), "/v1/traces") + if err != nil { + panic(err) + } } func NewTracer(r *ghttp.Request) { - // 从传入的上下文中获取 Tracer,或者直接使用全局的 - tracer := otel.Tracer(r.GetServeHandler().GetMetaTag("summary")) - _, span := tracer.Start(r.Context(), r.GetServeHandler().GetMetaTag("summary")) - defer span.End() // 非常重要:确保 span 在函数结束时被关闭 + _, span := gtrace.NewSpan(r.Context(), r.GetServeHandler().GetMetaTag("summary")) + defer span.End() span.SetAttributes(attribute.String("request", getParams(r))) r.Middleware.Next() span.SetAttributes(attribute.String("response", r.Response.BufferString())) - //span.AddEvent("Saying hello is done") } func getParams(r *ghttp.Request) string { params := map[string]interface{}{} From e9f3aa5b223d3e0b1ee187a24251550e34abf020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 26 Nov 2025 11:10:01 +0800 Subject: [PATCH 015/225] =?UTF-8?q?consul.go=E5=8C=85=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/consul.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/consul/consul.go b/consul/consul.go index 177a74b..d674451 100644 --- a/consul/consul.go +++ b/consul/consul.go @@ -2,12 +2,9 @@ package consul import ( "context" - _ "github.com/gogf/gf/contrib/drivers/mysql/v2" - _ "github.com/gogf/gf/contrib/nosql/redis/v2" "github.com/gogf/gf/contrib/registry/consul/v2" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/gsvc" - _ "go.mongodb.org/mongo-driver/mongo" ) func init() { From b57c8de47ff433d8caf92dcf8bf433f9f3bd69d3 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Thu, 27 Nov 2025 09:50:12 +0800 Subject: [PATCH 016/225] =?UTF-8?q?ragflow=E7=9A=84dto?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/chat.go | 172 +++++++++++++++++++++++++++++++++++++ ragflow/chunk.go | 174 ++++++++++++++++++++++++++++++++++++++ ragflow/dataset.go | 161 +++++++++++++++++++++++++++++++++++ ragflow/document.go | 132 +++++++++++++++++++++++++++++ ragflow/ragflow.go | 5 -- ragflow/ragflow_dto.go | 1 - ragflow/service/client.go | 77 +++++++++++++++++ ragflow/session.go | 164 +++++++++++++++++++++++++++++++++++ 8 files changed, 880 insertions(+), 6 deletions(-) create mode 100644 ragflow/chat.go create mode 100644 ragflow/chunk.go create mode 100644 ragflow/dataset.go create mode 100644 ragflow/document.go delete mode 100644 ragflow/ragflow.go delete mode 100644 ragflow/ragflow_dto.go create mode 100644 ragflow/service/client.go create mode 100644 ragflow/session.go diff --git a/ragflow/chat.go b/ragflow/chat.go new file mode 100644 index 0000000..addc182 --- /dev/null +++ b/ragflow/chat.go @@ -0,0 +1,172 @@ +package ragflow + +import ( + "context" + "fmt" +) + +// Chat 结构体 +type Chat struct { + Id string `json:"id"` + Name string `json:"name"` + Avatar string `json:"avatar"` + DatasetIds []string `json:"dataset_ids"` + Llm Llm `json:"llm"` + Prompt Prompt `json:"prompt"` + Description string `json:"description"` + DoRefer string `json:"do_refer"` + Language string `json:"language"` + PromptType string `json:"prompt_type"` + Status string `json:"status"` + TenantId string `json:"tenant_id"` + TopK int `json:"top_k"` + CreateDate string `json:"create_date"` + CreateTime int64 `json:"create_time"` + UpdateDate string `json:"update_date"` + UpdateTime int64 `json:"update_time"` +} + +type Llm struct { + ModelName string `json:"model_name,omitempty"` + Temperature float64 `json:"temperature,omitempty"` + TopP float64 `json:"top_p,omitempty"` + PresencePenalty float64 `json:"presence_penalty,omitempty"` + FrequencyPenalty float64 `json:"frequency_penalty,omitempty"` +} + +type Prompt struct { + SimilarityThreshold float64 `json:"similarity_threshold,omitempty"` + KeywordsSimilarityWeight float64 `json:"keywords_similarity_weight,omitempty"` + Opener string `json:"opener,omitempty"` + Prompt string `json:"prompt,omitempty"` + RerankModel string `json:"rerank_model,omitempty"` + TopN int `json:"top_n,omitempty"` + Variables []Variable `json:"variables,omitempty"` + EmptyResponse string `json:"empty_response,omitempty"` +} + +type Variable struct { + Key string `json:"key"` + Optional bool `json:"optional"` +} + +// CreateChatReq 创建聊天助手请求 +type CreateChatReq struct { + Name string `json:"name"` + Avatar string `json:"avatar,omitempty"` + DatasetIds []string `json:"dataset_ids,omitempty"` + Llm *Llm `json:"llm,omitempty"` + Prompt *Prompt `json:"prompt,omitempty"` +} + +// UpdateChatReq 更新聊天助手请求 +type UpdateChatReq struct { + Name string `json:"name,omitempty"` + Avatar string `json:"avatar,omitempty"` + DatasetIds []string `json:"dataset_ids,omitempty"` + Llm *Llm `json:"llm,omitempty"` + Prompt *Prompt `json:"prompt,omitempty"` +} + +// ListChatsReq 列出聊天助手请求 +type ListChatsReq struct { + Page int `json:"page,omitempty"` + PageSize int `json:"page_size,omitempty"` + OrderBy string `json:"orderby,omitempty"` + Desc bool `json:"desc,omitempty"` + Name string `json:"name,omitempty"` + Id string `json:"id,omitempty"` +} + +// ListChatsRes 列出聊天助手响应 +type ListChatsRes struct { + Code int `json:"code"` + Data []*Chat `json:"data"` + Total int `json:"total"` // API 文档中未明确 total 字段,但通常列表接口会有 +} + +// DeleteChatsReq 删除聊天助手请求 +type DeleteChatsReq struct { + Ids []string `json:"ids"` +} + +// CreateChat 创建聊天助手 +func (c *Client) CreateChat(ctx context.Context, req *CreateChatReq) (*Chat, error) { + var res struct { + Code int `json:"code"` + Data *Chat `json:"data"` + Msg string `json:"message"` + } + if err := c.request(ctx, "POST", "/api/v1/chats", req, &res); err != nil { + return nil, err + } + if res.Code != 0 { + return nil, fmt.Errorf("create chat failed: %s", res.Msg) + } + return res.Data, nil +} + +// ListChats 列出聊天助手 +func (c *Client) ListChats(ctx context.Context, req *ListChatsReq) (*ListChatsRes, error) { + path := "/api/v1/chats?" + params := map[string]interface{}{} + if req.Page > 0 { + params["page"] = req.Page + } + if req.PageSize > 0 { + params["page_size"] = req.PageSize + } + if req.OrderBy != "" { + params["orderby"] = req.OrderBy + } + if req.Desc { + params["desc"] = "true" + } else { + params["desc"] = "false" + } + if req.Name != "" { + params["name"] = req.Name + } + if req.Id != "" { + params["id"] = req.Id + } + + for k, v := range params { + path += fmt.Sprintf("%s=%v&", k, v) + } + + var res ListChatsRes + if err := c.request(ctx, "GET", path, nil, &res); err != nil { + return nil, err + } + if res.Code != 0 { + return nil, fmt.Errorf("list chats failed: code=%d", res.Code) + } + return &res, nil +} + +// DeleteChats 删除聊天助手 +func (c *Client) DeleteChats(ctx context.Context, ids []string) error { + req := DeleteChatsReq{Ids: ids} + var res CommonResponse + if err := c.request(ctx, "DELETE", "/api/v1/chats", req, &res); err != nil { + return err + } + if !res.IsSuccess() { + return fmt.Errorf("delete chats failed: %s", res.Message) + } + return nil +} + +// UpdateChat 更新聊天助手 +func (c *Client) UpdateChat(ctx context.Context, id string, req *UpdateChatReq) error { + var res CommonResponse + path := fmt.Sprintf("/api/v1/chats/%s", id) + if err := c.request(ctx, "PUT", path, req, &res); err != nil { + return err + } + if !res.IsSuccess() { + return fmt.Errorf("update chat failed: %s", res.Message) + } + return nil +} diff --git a/ragflow/chunk.go b/ragflow/chunk.go new file mode 100644 index 0000000..e2c2182 --- /dev/null +++ b/ragflow/chunk.go @@ -0,0 +1,174 @@ +package ragflow + +import ( + "context" + "fmt" +) + +// Chunk 结构体 +type Chunk struct { + Id string `json:"id"` + Content string `json:"content"` + DocumentId string `json:"document_id"` + DatasetId string `json:"dataset_id"` + CreateTime string `json:"create_time"` + CreateTimestamp float64 `json:"create_timestamp"` + ImportantKeywords []string `json:"important_keywords"` + Questions []string `json:"questions"` + Available bool `json:"available"` + ImageId string `json:"image_id"` + Positions []string `json:"positions"` +} + +// AddChunkReq 添加知识块请求 +type AddChunkReq struct { + Content string `json:"content"` + ImportantKeywords []string `json:"important_keywords,omitempty"` + Questions []string `json:"questions,omitempty"` +} + +// ListChunksReq 列出知识块请求 +type ListChunksReq struct { + Keywords string `json:"keywords,omitempty"` + Page int `json:"page,omitempty"` + PageSize int `json:"page_size,omitempty"` + Id string `json:"id,omitempty"` +} + +// ListChunksRes 列出知识块响应 +type ListChunksRes struct { + Code int `json:"code"` + Data struct { + Chunks []*Chunk `json:"chunks"` + Doc interface{} `json:"doc"` // 文档信息,暂时用 interface{} + Total int `json:"total"` + } `json:"data"` +} + +// DeleteChunksReq 删除知识块请求 +type DeleteChunksReq struct { + ChunkIds []string `json:"chunk_ids,omitempty"` // 如果为空,删除所有 +} + +// UpdateChunkReq 更新知识块请求 +type UpdateChunkReq struct { + Content string `json:"content,omitempty"` + ImportantKeywords []string `json:"important_keywords,omitempty"` + Available *bool `json:"available,omitempty"` +} + +// RetrieveChunksReq 检索知识块请求 +type RetrieveChunksReq struct { + Question string `json:"question"` + DatasetIds []string `json:"dataset_ids,omitempty"` + DocumentIds []string `json:"document_ids,omitempty"` + Page int `json:"page,omitempty"` + PageSize int `json:"page_size,omitempty"` + SimilarityThreshold float64 `json:"similarity_threshold,omitempty"` + VectorSimilarityWeight float64 `json:"vector_similarity_weight,omitempty"` + TopK int `json:"top_k,omitempty"` + RerankId string `json:"rerank_id,omitempty"` + Keyword bool `json:"keyword,omitempty"` + Highlight bool `json:"highlight,omitempty"` + CrossLanguages []string `json:"cross_languages,omitempty"` + MetadataCondition map[string]interface{} `json:"metadata_condition,omitempty"` +} + +// RetrieveChunksRes 检索知识块响应 (结构比较复杂,暂时简化,根据实际返回调整) +// 官方文档未给出详细响应结构,假设返回 chunks 列表 +type RetrieveChunksRes struct { + Code int `json:"code"` + Data struct { + Chunks []interface{} `json:"chunks"` // 检索结果可能包含额外信息 + Total int `json:"total"` + } `json:"data"` +} + +// AddChunk 添加知识块 +func (c *Client) AddChunk(ctx context.Context, datasetId, documentId string, req *AddChunkReq) (*Chunk, error) { + path := fmt.Sprintf("/api/v1/datasets/%s/documents/%s/chunks", datasetId, documentId) + var res struct { + Code int `json:"code"` + Data struct { + Chunk *Chunk `json:"chunk"` + } `json:"data"` + Msg string `json:"message"` + } + if err := c.request(ctx, "POST", path, req, &res); err != nil { + return nil, err + } + if res.Code != 0 { + return nil, fmt.Errorf("add chunk failed: %s", res.Msg) + } + return res.Data.Chunk, nil +} + +// ListChunks 列出知识块 +func (c *Client) ListChunks(ctx context.Context, datasetId, documentId string, req *ListChunksReq) (*ListChunksRes, error) { + path := fmt.Sprintf("/api/v1/datasets/%s/documents/%s/chunks?", datasetId, documentId) + params := map[string]interface{}{} + if req.Keywords != "" { + params["keywords"] = req.Keywords + } + if req.Page > 0 { + params["page"] = req.Page + } + if req.PageSize > 0 { + params["page_size"] = req.PageSize + } + if req.Id != "" { + params["id"] = req.Id + } + + for k, v := range params { + path += fmt.Sprintf("%s=%v&", k, v) + } + + var res ListChunksRes + if err := c.request(ctx, "GET", path, nil, &res); err != nil { + return nil, err + } + if res.Code != 0 { + return nil, fmt.Errorf("list chunks failed: code=%d", res.Code) + } + return &res, nil +} + +// DeleteChunks 删除知识块 +func (c *Client) DeleteChunks(ctx context.Context, datasetId, documentId string, chunkIds []string) error { + req := DeleteChunksReq{ChunkIds: chunkIds} + var res CommonResponse + path := fmt.Sprintf("/api/v1/datasets/%s/documents/%s/chunks", datasetId, documentId) + if err := c.request(ctx, "DELETE", path, req, &res); err != nil { + return err + } + if !res.IsSuccess() { + return fmt.Errorf("delete chunks failed: %s", res.Message) + } + return nil +} + +// UpdateChunk 更新知识块 +func (c *Client) UpdateChunk(ctx context.Context, datasetId, documentId, chunkId string, req *UpdateChunkReq) error { + var res CommonResponse + path := fmt.Sprintf("/api/v1/datasets/%s/documents/%s/chunks/%s", datasetId, documentId, chunkId) + if err := c.request(ctx, "PUT", path, req, &res); err != nil { + return err + } + if !res.IsSuccess() { + return fmt.Errorf("update chunk failed: %s", res.Message) + } + return nil +} + +// RetrieveChunks 检索知识块 +func (c *Client) RetrieveChunks(ctx context.Context, req *RetrieveChunksReq) (*RetrieveChunksRes, error) { + var res RetrieveChunksRes + if err := c.request(ctx, "POST", "/api/v1/retrieval", req, &res); err != nil { + return nil, err + } + if res.Code != 0 { + return nil, fmt.Errorf("retrieve chunks failed: code=%d", res.Code) + } + return &res, nil +} diff --git a/ragflow/dataset.go b/ragflow/dataset.go new file mode 100644 index 0000000..ea0de16 --- /dev/null +++ b/ragflow/dataset.go @@ -0,0 +1,161 @@ +package ragflow + +import ( + "context" + "fmt" +) + +// Dataset 结构体 +type Dataset struct { + Id string `json:"id"` + Name string `json:"name"` + Avatar string `json:"avatar"` + TenantId string `json:"tenant_id"` + Description string `json:"description"` + Language string `json:"language"` + EmbeddingModel string `json:"embedding_model"` + Permission string `json:"permission"` + DocumentCount int `json:"document_count"` + ChunkCount int `json:"chunk_count"` + ParseStatus string `json:"parse_status"` + CreatedBy string `json:"created_by"` + CreateTime int64 `json:"create_time"` + UpdateDate string `json:"update_date"` + UpdateTime int64 `json:"update_time"` + Status string `json:"status"` + ChunkMethod string `json:"chunk_method"` + ParserConfig map[string]interface{} `json:"parser_config"` + VectorSimilarityWeight float64 `json:"vector_similarity_weight"` + SimilarityThreshold float64 `json:"similarity_threshold"` + TokenNum int `json:"token_num"` +} + +// CreateDatasetReq 创建数据集请求 +type CreateDatasetReq struct { + Name string `json:"name"` + Avatar string `json:"avatar,omitempty"` + Description string `json:"description,omitempty"` + EmbeddingModel string `json:"embedding_model,omitempty"` + Permission string `json:"permission,omitempty"` + ChunkMethod string `json:"chunk_method,omitempty"` + ParserConfig map[string]interface{} `json:"parser_config,omitempty"` +} + +// UpdateDatasetReq 更新数据集请求 +type UpdateDatasetReq struct { + Name string `json:"name,omitempty"` + Avatar string `json:"avatar,omitempty"` + Description string `json:"description,omitempty"` + EmbeddingModel string `json:"embedding_model,omitempty"` + Permission string `json:"permission,omitempty"` + ChunkMethod string `json:"chunk_method,omitempty"` + PageRank int `json:"pagerank,omitempty"` + ParserConfig map[string]interface{} `json:"parser_config,omitempty"` +} + +// ListDatasetsReq 列出数据集请求 +type ListDatasetsReq struct { + Page int `json:"page,omitempty"` + PageSize int `json:"page_size,omitempty"` + OrderBy string `json:"orderby,omitempty"` + Desc bool `json:"desc,omitempty"` + Name string `json:"name,omitempty"` + Id string `json:"id,omitempty"` +} + +// ListDatasetsRes 列出数据集响应 +type ListDatasetsRes struct { + Code int `json:"code"` + Data []*Dataset `json:"data"` + Total int `json:"total"` +} + +// DeleteDatasetsReq 删除数据集请求 +type DeleteDatasetsReq struct { + Ids []string `json:"ids"` +} + +// CreateDataset 创建数据集 +func (c *Client) CreateDataset(ctx context.Context, req *CreateDatasetReq) (*Dataset, error) { + var res struct { + Code int `json:"code"` + Data *Dataset `json:"data"` + Msg string `json:"message"` + } + if err := c.request(ctx, "POST", "/api/v1/datasets", req, &res); err != nil { + return nil, err + } + if res.Code != 0 { + return nil, fmt.Errorf("create dataset failed: %s", res.Msg) + } + return res.Data, nil +} + +// ListDatasets 列出数据集 +func (c *Client) ListDatasets(ctx context.Context, req *ListDatasetsReq) (*ListDatasetsRes, error) { + // 构建查询参数 + path := "/api/v1/datasets?" + params := map[string]interface{}{} + if req.Page > 0 { + params["page"] = req.Page + } + if req.PageSize > 0 { + params["page_size"] = req.PageSize + } + if req.OrderBy != "" { + params["orderby"] = req.OrderBy + } + // desc 默认为 true,如果显式设置为 false 才传递,或者根据 API 行为调整 + // 这里简单处理,如果设置了就传 + if req.Desc { + params["desc"] = "true" + } else { + params["desc"] = "false" + } + if req.Name != "" { + params["name"] = req.Name + } + if req.Id != "" { + params["id"] = req.Id + } + + // 拼接 query string + for k, v := range params { + path += fmt.Sprintf("%s=%v&", k, v) + } + + var res ListDatasetsRes + if err := c.request(ctx, "GET", path, nil, &res); err != nil { + return nil, err + } + if res.Code != 0 { + return nil, fmt.Errorf("list datasets failed: code=%d", res.Code) + } + return &res, nil +} + +// DeleteDataset 删除数据集 +func (c *Client) DeleteDataset(ctx context.Context, ids []string) error { + req := DeleteDatasetsReq{Ids: ids} + var res CommonResponse + if err := c.request(ctx, "DELETE", "/api/v1/datasets", req, &res); err != nil { + return err + } + if !res.IsSuccess() { + return fmt.Errorf("delete dataset failed: %s", res.Message) + } + return nil +} + +// UpdateDataset 更新数据集 +func (c *Client) UpdateDataset(ctx context.Context, id string, req *UpdateDatasetReq) error { + var res CommonResponse + path := fmt.Sprintf("/api/v1/datasets/%s", id) + if err := c.request(ctx, "PUT", path, req, &res); err != nil { + return err + } + if !res.IsSuccess() { + return fmt.Errorf("update dataset failed: %s", res.Message) + } + return nil +} diff --git a/ragflow/document.go b/ragflow/document.go new file mode 100644 index 0000000..0d2e5e8 --- /dev/null +++ b/ragflow/document.go @@ -0,0 +1,132 @@ +package ragflow + +import ( + "context" + "fmt" +) + +// Document 结构体 +type Document struct { + Id string `json:"id"` + DatasetId string `json:"dataset_id"` + Name string `json:"name"` + Size int64 `json:"size"` + Location string `json:"location"` + CreatedBy string `json:"created_by"` + CreateTime int64 `json:"create_time"` + Thumbnail string `json:"thumbnail"` + Type string `json:"type"` + RunStatus string `json:"run_status"` // 对应 API 返回的 "run" 字段,可能需要确认 + Status string `json:"status"` + ChunkMethod string `json:"chunk_method"` + ParserConfig map[string]interface{} `json:"parser_config"` + TokenNum int `json:"token_num"` + ChunkCount int `json:"chunk_count"` + ProcessBegin int64 `json:"process_begin"` + ProcessDu int64 `json:"process_du"` + Progress float64 `json:"progress"` + ProgressMsg string `json:"progress_msg"` +} + +// UploadDocumentReq 上传文档请求 +// 注意:上传文件通常需要 multipart/form-data,这里仅定义结构,实际逻辑在方法中处理 +type UploadDocumentReq struct { + FilePaths []string // 本地文件路径列表 +} + +// ListDocumentsReq 列出文档请求 +type ListDocumentsReq struct { + Page int `json:"page,omitempty"` + PageSize int `json:"page_size,omitempty"` + OrderBy string `json:"orderby,omitempty"` + Desc bool `json:"desc,omitempty"` + Keywords string `json:"keywords,omitempty"` + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + CreateTimeFrom int64 `json:"create_time_from,omitempty"` + CreateTimeTo int64 `json:"create_time_to,omitempty"` +} + +// ListDocumentsRes 列出文档响应 +type ListDocumentsRes struct { + Code int `json:"code"` + Data []*Document `json:"data"` + Total int `json:"total"` +} + +// DeleteDocumentsReq 删除文档请求 +type DeleteDocumentsReq struct { + Ids []string `json:"ids"` +} + +// ListDocuments 列出文档 +func (c *Client) ListDocuments(ctx context.Context, datasetId string, req *ListDocumentsReq) (*ListDocumentsRes, error) { + path := fmt.Sprintf("/api/v1/datasets/%s/documents?", datasetId) + params := map[string]interface{}{} + if req.Page > 0 { + params["page"] = req.Page + } + if req.PageSize > 0 { + params["page_size"] = req.PageSize + } + if req.OrderBy != "" { + params["orderby"] = req.OrderBy + } + if req.Desc { + params["desc"] = "true" + } else { + params["desc"] = "false" + } + if req.Keywords != "" { + params["keywords"] = req.Keywords + } + if req.Id != "" { + params["id"] = req.Id + } + if req.Name != "" { + params["name"] = req.Name + } + if req.CreateTimeFrom > 0 { + params["create_time_from"] = req.CreateTimeFrom + } + if req.CreateTimeTo > 0 { + params["create_time_to"] = req.CreateTimeTo + } + + for k, v := range params { + path += fmt.Sprintf("%s=%v&", k, v) + } + + var res ListDocumentsRes + if err := c.request(ctx, "GET", path, nil, &res); err != nil { + return nil, err + } + if res.Code != 0 { + return nil, fmt.Errorf("list documents failed: code=%d", res.Code) + } + return &res, nil +} + +// UploadDocument 上传文档 +// 注意:此方法需要特殊处理 multipart/form-data,目前的 request 方法可能不支持 +// 我们需要扩展 request 方法或在此处单独实现 +func (c *Client) UploadDocument(ctx context.Context, datasetId string, filePaths []string) error { + // TODO: 实现文件上传逻辑,需要使用 gclient 的 UploadFile 功能 + // 由于 request 方法封装了 JSON 处理,这里可能需要绕过 request 方法直接使用 c.Client + // 暂时留空或仅做简单提示,待完善 Client 封装以支持文件上传 + return fmt.Errorf("upload document not implemented yet") +} + +// DeleteDocument 删除文档 +func (c *Client) DeleteDocument(ctx context.Context, datasetId string, ids []string) error { + req := DeleteDocumentsReq{Ids: ids} + var res CommonResponse + path := fmt.Sprintf("/api/v1/datasets/%s/documents", datasetId) + if err := c.request(ctx, "DELETE", path, req, &res); err != nil { + return err + } + if !res.IsSuccess() { + return fmt.Errorf("delete document failed: %s", res.Message) + } + return nil +} diff --git a/ragflow/ragflow.go b/ragflow/ragflow.go deleted file mode 100644 index 78bf86d..0000000 --- a/ragflow/ragflow.go +++ /dev/null @@ -1,5 +0,0 @@ -package ragflow - -func init() { - -} diff --git a/ragflow/ragflow_dto.go b/ragflow/ragflow_dto.go deleted file mode 100644 index 3cdcbe1..0000000 --- a/ragflow/ragflow_dto.go +++ /dev/null @@ -1 +0,0 @@ -package ragflow diff --git a/ragflow/service/client.go b/ragflow/service/client.go new file mode 100644 index 0000000..4a45dfe --- /dev/null +++ b/ragflow/service/client.go @@ -0,0 +1,77 @@ +package service + +import ( + "context" + "fmt" + "time" + + "gitee.com/red-future---jilin-g/common/ragflow/dto" + "github.com/gogf/gf/v2/encoding/gjson" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/gclient" +) + +type Client struct { + BaseURL string + ApiKey string + Client *gclient.Client +} + +// NewClient 创建一个新的 RAGFlow 客户端 +func NewClient(baseUrl, apiKey string) *Client { + return &Client{ + BaseURL: baseUrl, + ApiKey: apiKey, + Client: g.Client().SetTimeout(30 * time.Second), + } +} + +// request 发送 HTTP 请求 +func (c *Client) request(ctx context.Context, method, path string, data interface{}, result interface{}) error { + url := fmt.Sprintf("%s%s", c.BaseURL, path) + + req := c.Client.Header(map[string]string{ + "Authorization": fmt.Sprintf("Bearer %s", c.ApiKey), + "Content-Type": "application/json", + }) + + var res *gclient.Response + var err error + + switch method { + case "GET": + res, err = req.Get(ctx, url, data) + case "POST": + res, err = req.Post(ctx, url, data) + case "PUT": + res, err = req.Put(ctx, url, data) + case "DELETE": + res, err = req.Delete(ctx, url, data) + default: + return fmt.Errorf("unsupported method: %s", method) + } + + if err != nil { + return err + } + defer res.Close() + + // 读取响应体 + body := res.ReadAllString() + + // 解析响应 + if result != nil { + if err := gjson.DecodeTo(body, result); err != nil { + return fmt.Errorf("failed to decode response: %v, body: %s", err, body) + } + + // 检查业务错误码 + if commonRes, ok := result.(*dto.CommonResponse); ok { + if !commonRes.IsSuccess() { + return fmt.Errorf("api error: code=%d, message=%s", commonRes.Code, commonRes.Message) + } + } + } + + return nil +} diff --git a/ragflow/session.go b/ragflow/session.go new file mode 100644 index 0000000..0c15136 --- /dev/null +++ b/ragflow/session.go @@ -0,0 +1,164 @@ +package ragflow + +import ( + "context" + "fmt" +) + +// Session 结构体 +type Session struct { + Id string `json:"id"` + Name string `json:"name"` + ChatId string `json:"chat_id"` // 响应中是 "chat" 或 "chat_id",根据文档示例调整 + Messages []Message `json:"messages"` + CreateDate string `json:"create_date"` + CreateTime int64 `json:"create_time"` + UpdateDate string `json:"update_date"` + UpdateTime int64 `json:"update_time"` +} + +type Message struct { + Content string `json:"content"` + Role string `json:"role"` +} + +// CreateSessionReq 创建会话请求 +type CreateSessionReq struct { + Name string `json:"name"` + UserId string `json:"user_id,omitempty"` +} + +// ListSessionsReq 列出会话请求 +type ListSessionsReq struct { + Page int `json:"page,omitempty"` + PageSize int `json:"page_size,omitempty"` + OrderBy string `json:"orderby,omitempty"` + Desc bool `json:"desc,omitempty"` + Name string `json:"name,omitempty"` + Id string `json:"id,omitempty"` + UserId string `json:"user_id,omitempty"` +} + +// ListSessionsRes 列出会话响应 +type ListSessionsRes struct { + Code int `json:"code"` + Data []*Session `json:"data"` + Total int `json:"total"` // API 文档未明确 +} + +// DeleteSessionsReq 删除会话请求 +type DeleteSessionsReq struct { + Ids []string `json:"ids"` +} + +// ChatCompletionReq 对话请求 +type ChatCompletionReq struct { + Question string `json:"question"` + Stream bool `json:"stream"` + SessionId string `json:"session_id,omitempty"` + UserId string `json:"user_id,omitempty"` +} + +// ChatCompletionRes 对话响应 (非流式) +type ChatCompletionRes struct { + Code int `json:"code"` + Data struct { + Answer string `json:"answer"` + Reference interface{} `json:"reference"` + AudioBinary interface{} `json:"audio_binary"` + Id interface{} `json:"id"` + SessionId string `json:"session_id"` + } `json:"data"` +} + +// CreateSession 创建会话 +func (c *Client) CreateSession(ctx context.Context, chatId string, req *CreateSessionReq) (*Session, error) { + path := fmt.Sprintf("/api/v1/chats/%s/sessions", chatId) + var res struct { + Code int `json:"code"` + Data *Session `json:"data"` + Msg string `json:"message"` + } + if err := c.request(ctx, "POST", path, req, &res); err != nil { + return nil, err + } + if res.Code != 0 { + return nil, fmt.Errorf("create session failed: %s", res.Msg) + } + return res.Data, nil +} + +// ListSessions 列出会话 +func (c *Client) ListSessions(ctx context.Context, chatId string, req *ListSessionsReq) (*ListSessionsRes, error) { + path := fmt.Sprintf("/api/v1/chats/%s/sessions?", chatId) + params := map[string]interface{}{} + if req.Page > 0 { + params["page"] = req.Page + } + if req.PageSize > 0 { + params["page_size"] = req.PageSize + } + if req.OrderBy != "" { + params["orderby"] = req.OrderBy + } + if req.Desc { + params["desc"] = "true" + } else { + params["desc"] = "false" + } + if req.Name != "" { + params["name"] = req.Name + } + if req.Id != "" { + params["id"] = req.Id + } + if req.UserId != "" { + params["user_id"] = req.UserId + } + + for k, v := range params { + path += fmt.Sprintf("%s=%v&", k, v) + } + + var res ListSessionsRes + if err := c.request(ctx, "GET", path, nil, &res); err != nil { + return nil, err + } + if res.Code != 0 { + return nil, fmt.Errorf("list sessions failed: code=%d", res.Code) + } + return &res, nil +} + +// DeleteSessions 删除会话 +func (c *Client) DeleteSessions(ctx context.Context, chatId string, ids []string) error { + req := DeleteSessionsReq{Ids: ids} + var res CommonResponse + path := fmt.Sprintf("/api/v1/chats/%s/sessions", chatId) + if err := c.request(ctx, "DELETE", path, req, &res); err != nil { + return err + } + if !res.IsSuccess() { + return fmt.Errorf("delete sessions failed: %s", res.Message) + } + return nil +} + +// ChatCompletion 对话 (目前仅支持非流式) +func (c *Client) ChatCompletion(ctx context.Context, chatId string, req *ChatCompletionReq) (*ChatCompletionRes, error) { + path := fmt.Sprintf("/api/v1/chats/%s/completions", chatId) + var res ChatCompletionRes + + // 如果需要流式支持,需要使用 gclient 的流式处理能力,这里暂只实现非流式 + if req.Stream { + return nil, fmt.Errorf("stream mode not supported yet") + } + + if err := c.request(ctx, "POST", path, req, &res); err != nil { + return nil, err + } + if res.Code != 0 { + return nil, fmt.Errorf("chat completion failed: code=%d", res.Code) + } + return &res, nil +} From f2ba4b66b9d8e58bf3f5003c180ed04bfbf323c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 26 Nov 2025 11:56:05 +0800 Subject: [PATCH 017/225] =?UTF-8?q?consul.go=E5=8C=85=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/http/http.go b/http/http.go index 6b82dec..8d84822 100644 --- a/http/http.go +++ b/http/http.go @@ -9,7 +9,6 @@ import ( "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/contrib/registry/consul/v2" "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/net/gclient" "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/net/gsel" "github.com/gogf/gf/v2/net/gsvc" @@ -33,6 +32,7 @@ type Page struct { } var Httpserver = g.Server() +var Httpclient = g.Client() func init() { //s.Use(common.Cors) //中间件验证 @@ -42,6 +42,17 @@ func init() { Httpserver.SetDumpRouterMap(true) //关闭打印路由注册信息 Httpserver.BindMiddlewareDefault(ghttp.MiddlewareHandlerResponse, jaeger.NewTracer) //使用默认http返回结构 go Httpserver.Run() + + consulCfg, _ := g.Cfg().Get(context.Background(), "consul.address") + consulAddr := consulCfg.String() + registry, err := consul.New(consul.WithAddress(consulAddr)) + if err != nil { + panic(err) + } + gsvc.SetRegistry(registry) + gsel.SetBuilder(gsel.NewBuilderRoundRobin()) + Httpclient.SetHeader("Authorization", g.RequestFromCtx(context.TODO()).GetHeader("Authorization")) + Httpclient.SetDiscovery(gsvc.GetRegistry()) } func RouteRegister(controllers []interface{}) { re := regexp.MustCompile("[A-Z]") @@ -55,30 +66,12 @@ func RouteRegister(controllers []interface{}) { }) } } -func getHttpClient(ctx context.Context) (client *gclient.Client, err error) { - consulCfg, _ := g.Cfg().Get(context.Background(), "consul.address") - consulAddr := consulCfg.String() - registry, err := consul.New(consul.WithAddress(consulAddr)) - if err != nil { - return - } - gsvc.SetRegistry(registry) - gsel.SetBuilder(gsel.NewBuilderRoundRobin()) - client = g.Client() - client.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) - client.SetDiscovery(gsvc.GetRegistry()) - return -} func doRequest(ctx context.Context, method string, url string, target any, data ...any) (err error) { err = utils.ValidStructPtr(target) if err != nil { return } - client, err := getHttpClient(ctx) - if err != nil { - return - } - response, err := client.DoRequest(ctx, method, url, data) + response, err := Httpclient.DoRequest(ctx, method, url, data) if err != nil { return } From c75166912da87f5dc45962a36ddd193da692214f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 27 Nov 2025 09:46:47 +0800 Subject: [PATCH 018/225] =?UTF-8?q?swagger=E4=BF=AE=E6=94=B9=EF=BC=8C?= =?UTF-8?q?=E9=99=90=E6=B5=81=E4=BB=A4=E7=89=8C=E6=A1=B6=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 33 +++++++++++++++++++++--- middleware/middleware.go | 55 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 middleware/middleware.go diff --git a/http/http.go b/http/http.go index 8d84822..c675ec4 100644 --- a/http/http.go +++ b/http/http.go @@ -6,6 +6,7 @@ import ( "fmt" _ "gitee.com/red-future---jilin-g/common/consul" "gitee.com/red-future---jilin-g/common/jaeger" + "gitee.com/red-future---jilin-g/common/middleware" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/contrib/registry/consul/v2" "github.com/gogf/gf/v2/frame/g" @@ -38,9 +39,33 @@ func init() { //s.Use(common.Cors) //中间件验证 //s.EnablePProf() //启用性能分析 Httpserver.SetOpenApiPath("/api.json") - Httpserver.SetSwaggerPath("/swagger") //api文档访问路径 - Httpserver.SetDumpRouterMap(true) //关闭打印路由注册信息 - Httpserver.BindMiddlewareDefault(ghttp.MiddlewareHandlerResponse, jaeger.NewTracer) //使用默认http返回结构 + Httpserver.SetSwaggerPath("/docs") //api文档访问路径 + Httpserver.SetDumpRouterMap(true) //关闭打印路由注册信息 + Httpserver.SetSwaggerUITemplate(` + + + + + + + SwaggerUI + + + +
+ + + + + `) + Httpserver.BindMiddlewareDefault(ghttp.MiddlewareCORS, ghttp.MiddlewareHandlerResponse, middleware.Limiter, jaeger.NewTracer) //使用默认http返回结构 go Httpserver.Run() consulCfg, _ := g.Cfg().Get(context.Background(), "consul.address") @@ -51,7 +76,6 @@ func init() { } gsvc.SetRegistry(registry) gsel.SetBuilder(gsel.NewBuilderRoundRobin()) - Httpclient.SetHeader("Authorization", g.RequestFromCtx(context.TODO()).GetHeader("Authorization")) Httpclient.SetDiscovery(gsvc.GetRegistry()) } func RouteRegister(controllers []interface{}) { @@ -71,6 +95,7 @@ func doRequest(ctx context.Context, method string, url string, target any, data if err != nil { return } + Httpclient.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) response, err := Httpclient.DoRequest(ctx, method, url, data) if err != nil { return diff --git a/middleware/middleware.go b/middleware/middleware.go new file mode 100644 index 0000000..593c25c --- /dev/null +++ b/middleware/middleware.go @@ -0,0 +1,55 @@ +package middleware + +import ( + "context" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/ghttp" + "github.com/gogf/gf/v2/os/gtime" + "github.com/gogf/gf/v2/text/gstr" + "golang.org/x/time/rate" +) + +// Logger 中间件 +func Logger(r *ghttp.Request) { + startTime := gtime.TimestampMilli() + r.Middleware.Next() + endTime := gtime.TimestampMilli() + g.Log().Infof(r.GetCtx(), + "request: %s %s | status: %d | time: %dms", + r.Method, + r.URL.Path, + r.Response.Status, + endTime-startTime, + ) +} + +var rateLimit, _ = g.Cfg().Get(context.TODO(), "rate.limit") +var rateBurst, _ = g.Cfg().Get(context.TODO(), "rate.burst") +var limiter = rate.NewLimiter(rate.Limit(rateLimit.Int()), rateBurst.Int()) + +func Limiter(r *ghttp.Request) { + if !limiter.Allow() { + r.Response.WriteStatusExit(429) // Return 429 Too Many Requests + r.ExitAll() + } + r.Middleware.Next() +} +func Auth(r *ghttp.Request) { + token := r.Header.Get("Authorization") + if token == "" || !gstr.HasPrefix(token, "Bearer ") { + r.Response.WriteStatusExit(401, "Unauthorized") + return + } + + // 验证 token + if !validateToken(gstr.SubStrFrom(token, "7")) { + r.Response.WriteStatusExit(401, "Unauthorized") + return + } + + r.Middleware.Next() +} +func validateToken(token string) bool { + // 实现 token 验证逻辑 + return token == "valid-token" +} From fb0cb27d1d907c518a075359d31ccc76f19498e4 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Thu, 27 Nov 2025 17:38:42 +0800 Subject: [PATCH 019/225] =?UTF-8?q?=E6=9A=82=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/agent.go | 132 ++++++++++++++++++++++++++++++++++++++ ragflow/chat.go | 10 ++- ragflow/chunk.go | 10 ++- ragflow/client.go | 109 +++++++++++++++++++++++++++++++ ragflow/dataset.go | 10 ++- ragflow/document.go | 10 ++- ragflow/openai.go | 122 +++++++++++++++++++++++++++++++++++ ragflow/service/client.go | 77 ---------------------- ragflow/session.go | 10 ++- ragflow/system.go | 39 +++++++++++ 10 files changed, 437 insertions(+), 92 deletions(-) create mode 100644 ragflow/agent.go create mode 100644 ragflow/client.go create mode 100644 ragflow/openai.go delete mode 100644 ragflow/service/client.go create mode 100644 ragflow/system.go diff --git a/ragflow/agent.go b/ragflow/agent.go new file mode 100644 index 0000000..7e0ed1e --- /dev/null +++ b/ragflow/agent.go @@ -0,0 +1,132 @@ +package ragflow + +import ( + "context" + "fmt" +) + +// Agent AGENT 管理 +// 参考: https://ragflow.com.cn/docs/dev/http_api_reference#agent-管理 + +// Agent Agent 结构体 +type Agent struct { + ID string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + DSL map[string]interface{} `json:"dsl"` // Canvas DSL 对象 +} + +// CreateAgentReq 创建 Agent 请求 +type CreateAgentReq struct { + Title string `json:"title"` // 必需 + Description string `json:"description,omitempty"` // 可选,默认为 None + DSL map[string]interface{} `json:"dsl"` // 必需,Canvas DSL 对象 +} + +// UpdateAgentReq 更新 Agent 请求 +type UpdateAgentReq struct { + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + DSL map[string]interface{} `json:"dsl,omitempty"` +} + +// ListAgentsReq 列出 Agent 请求 +type ListAgentsReq struct { + Page int `json:"page,omitempty"` + PageSize int `json:"page_size,omitempty"` + OrderBy string `json:"orderby,omitempty"` + Desc bool `json:"desc,omitempty"` + Title string `json:"title,omitempty"` + ID string `json:"id,omitempty"` +} + +// ListAgentsRes 列出 Agent 响应 +type ListAgentsRes struct { + Code int `json:"code"` + Data []*Agent `json:"data"` + Total int `json:"total"` +} + +// CreateAgent 创建 Agent +// POST /api/v1/agents +func (c *Client) CreateAgent(ctx context.Context, req *CreateAgentReq) error { + var res CommonResponse + if err := c.request(ctx, "POST", "/api/v1/agents", req, &res); err != nil { + return fmt.Errorf("create agent failed: %w", err) + } + if !res.IsSuccess() { + return fmt.Errorf("create agent failed: %s", res.Message) + } + return nil +} + +// UpdateAgent 更新 Agent +// PUT /api/v1/agents/{agent_id} +func (c *Client) UpdateAgent(ctx context.Context, agentID string, req *UpdateAgentReq) error { + path := fmt.Sprintf("/api/v1/agents/%s", agentID) + var res CommonResponse + if err := c.request(ctx, "PUT", path, req, &res); err != nil { + return fmt.Errorf("update agent failed: %w", err) + } + if !res.IsSuccess() { + return fmt.Errorf("update agent failed: %s", res.Message) + } + return nil +} + +// DeleteAgent 删除 Agent +// DELETE /api/v1/agents/{agent_id} +func (c *Client) DeleteAgent(ctx context.Context, agentID string) error { + path := fmt.Sprintf("/api/v1/agents/%s", agentID) + var res CommonResponse + if err := c.request(ctx, "DELETE", path, nil, &res); err != nil { + return fmt.Errorf("delete agent failed: %w", err) + } + if !res.IsSuccess() { + return fmt.Errorf("delete agent failed: %s", res.Message) + } + return nil +} + +// ListAgents 列出 Agent +// GET /api/v1/agents +func (c *Client) ListAgents(ctx context.Context, req *ListAgentsReq) (*ListAgentsRes, error) { + path := "/api/v1/agents" + if req != nil { + params := map[string]interface{}{} + if req.Page > 0 { + params["page"] = req.Page + } + if req.PageSize > 0 { + params["page_size"] = req.PageSize + } + if req.OrderBy != "" { + params["orderby"] = req.OrderBy + } + if req.Desc { + params["desc"] = "true" + } else { + params["desc"] = "false" + } + if req.Title != "" { + params["title"] = req.Title + } + if req.ID != "" { + params["id"] = req.ID + } + + query := buildQueryString(params) + if query != "" { + path += "?" + query + } + } + + var res ListAgentsRes + if err := c.request(ctx, "GET", path, nil, &res); err != nil { + return nil, fmt.Errorf("list agents failed: %w", err) + } + if res.Code != 0 { + return nil, fmt.Errorf("list agents failed: code=%d", res.Code) + } + return &res, nil +} diff --git a/ragflow/chat.go b/ragflow/chat.go index addc182..bb19251 100644 --- a/ragflow/chat.go +++ b/ragflow/chat.go @@ -5,7 +5,10 @@ import ( "fmt" ) -// Chat 结构体 +// 聊天助手管理 +// 参考: https://ragflow.com.cn/docs/dev/http_api_reference#聊天助手管理 + +// Chat 聊天助手结构体 type Chat struct { Id string `json:"id"` Name string `json:"name"` @@ -131,8 +134,9 @@ func (c *Client) ListChats(ctx context.Context, req *ListChatsReq) (*ListChatsRe params["id"] = req.Id } - for k, v := range params { - path += fmt.Sprintf("%s=%v&", k, v) + query := buildQueryString(params) + if query != "" { + path += "?" + query } var res ListChatsRes diff --git a/ragflow/chunk.go b/ragflow/chunk.go index e2c2182..dad9e59 100644 --- a/ragflow/chunk.go +++ b/ragflow/chunk.go @@ -5,7 +5,10 @@ import ( "fmt" ) -// Chunk 结构体 +// 数据集内知识块管理 +// 参考: https://ragflow.com.cn/docs/dev/http_api_reference#数据集内知识块管理 + +// Chunk 知识块结构体 type Chunk struct { Id string `json:"id"` Content string `json:"content"` @@ -120,8 +123,9 @@ func (c *Client) ListChunks(ctx context.Context, datasetId, documentId string, r params["id"] = req.Id } - for k, v := range params { - path += fmt.Sprintf("%s=%v&", k, v) + query := buildQueryString(params) + if query != "" { + path += "?" + query } var res ListChunksRes diff --git a/ragflow/client.go b/ragflow/client.go new file mode 100644 index 0000000..44d028c --- /dev/null +++ b/ragflow/client.go @@ -0,0 +1,109 @@ +package ragflow + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/gogf/gf/v2/net/gclient" +) + +// Client RAGFlow API 客户端 +type Client struct { + BaseURL string + APIKey string + HTTPClient *gclient.Client +} + +// NewClient 创建新的 RAGFlow 客户端 +func NewClient(baseURL, apiKey string) *Client { + client := gclient.New() + client.SetHeader("Authorization", fmt.Sprintf("Bearer %s", apiKey)) + client.SetHeader("Content-Type", "application/json") + + return &Client{ + BaseURL: strings.TrimSuffix(baseURL, "/"), + APIKey: apiKey, + HTTPClient: client, + } +} + +// CommonResponse 通用响应结构 +type CommonResponse struct { + Code int `json:"code"` + Message string `json:"message"` + Data interface{} `json:"data,omitempty"` +} + +// IsSuccess 检查响应是否成功 +func (r *CommonResponse) IsSuccess() bool { + return r.Code == 0 +} + +// request 发送 HTTP 请求 +func (c *Client) request(ctx context.Context, method, path string, body interface{}, result interface{}) error { + fullURL := c.BaseURL + path + + var reqBody io.Reader + if body != nil { + jsonData, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("marshal request body failed: %w", err) + } + reqBody = strings.NewReader(string(jsonData)) + } + + var resp *gclient.Response + var err error + + switch method { + case "GET": + resp, err = c.HTTPClient.Get(ctx, fullURL) + case "POST": + resp, err = c.HTTPClient.Post(ctx, fullURL, reqBody) + case "PUT": + resp, err = c.HTTPClient.Put(ctx, fullURL, reqBody) + case "DELETE": + resp, err = c.HTTPClient.Delete(ctx, fullURL, reqBody) + default: + return fmt.Errorf("unsupported method: %s", method) + } + + if err != nil { + return fmt.Errorf("http request failed: %w", err) + } + defer resp.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("http request failed with status: %d", resp.StatusCode) + } + + respBody, err := resp.ReadAll() + if err != nil { + return fmt.Errorf("read response body failed: %w", err) + } + + if err := json.Unmarshal(respBody, result); err != nil { + return fmt.Errorf("unmarshal response failed: %w", err) + } + + return nil +} + +// buildQueryString 构建查询字符串 +func buildQueryString(params map[string]interface{}) string { + if len(params) == 0 { + return "" + } + + var parts []string + for k, v := range params { + parts = append(parts, fmt.Sprintf("%s=%v", url.QueryEscape(k), url.QueryEscape(fmt.Sprintf("%v", v)))) + } + return strings.Join(parts, "&") +} + diff --git a/ragflow/dataset.go b/ragflow/dataset.go index ea0de16..0301d70 100644 --- a/ragflow/dataset.go +++ b/ragflow/dataset.go @@ -5,7 +5,10 @@ import ( "fmt" ) -// Dataset 结构体 +// 数据集管理 +// 参考: https://ragflow.com.cn/docs/dev/http_api_reference#数据集管理 + +// Dataset 数据集结构体 type Dataset struct { Id string `json:"id"` Name string `json:"name"` @@ -120,8 +123,9 @@ func (c *Client) ListDatasets(ctx context.Context, req *ListDatasetsReq) (*ListD } // 拼接 query string - for k, v := range params { - path += fmt.Sprintf("%s=%v&", k, v) + query := buildQueryString(params) + if query != "" { + path += "?" + query } var res ListDatasetsRes diff --git a/ragflow/document.go b/ragflow/document.go index 0d2e5e8..44827ca 100644 --- a/ragflow/document.go +++ b/ragflow/document.go @@ -5,7 +5,10 @@ import ( "fmt" ) -// Document 结构体 +// 数据集内文件管理 +// 参考: https://ragflow.com.cn/docs/dev/http_api_reference#数据集内文件管理 + +// Document 文档结构体 type Document struct { Id string `json:"id"` DatasetId string `json:"dataset_id"` @@ -93,8 +96,9 @@ func (c *Client) ListDocuments(ctx context.Context, datasetId string, req *ListD params["create_time_to"] = req.CreateTimeTo } - for k, v := range params { - path += fmt.Sprintf("%s=%v&", k, v) + query := buildQueryString(params) + if query != "" { + path += "?" + query } var res ListDocumentsRes diff --git a/ragflow/openai.go b/ragflow/openai.go new file mode 100644 index 0000000..b56008b --- /dev/null +++ b/ragflow/openai.go @@ -0,0 +1,122 @@ +package ragflow + +import ( + "context" + "encoding/json" + "fmt" +) + +// OpenAICompatibleAPI 与 OpenAI 兼容的 API +// 参考: https://ragflow.com.cn/docs/dev/http_api_reference#与-openai-兼容的-api + +// ChatCompletionMessage OpenAI 格式的消息 +type ChatCompletionMessage struct { + Role string `json:"role"` // "user", "assistant", "system" + Content string `json:"content"` +} + +// ChatCompletionRequest OpenAI 格式的聊天补全请求 +type ChatCompletionRequest struct { + Model string `json:"model"` // 模型名称(服务器会自动解析,可设置为任意值) + Messages []ChatCompletionMessage `json:"messages"` // 消息列表,必须至少包含一条 user 消息 + Stream bool `json:"stream,omitempty"` // 是否流式返回,默认 false +} + +// ChatCompletionResponse OpenAI 格式的聊天补全响应(非流式) +type ChatCompletionResponse struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Model string `json:"model"` + Choices []struct { + Index int `json:"index"` + Message ChatCompletionMessage `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage"` +} + +// ChatCompletionChunk 流式响应块 +type ChatCompletionChunk struct { + ID string `json:"id"` + Object string `json:"object"` + Created int64 `json:"created"` + Model string `json:"model"` + Choices []struct { + Index int `json:"index"` + Delta struct { + Content string `json:"content"` + Role string `json:"role"` + } `json:"delta"` + FinishReason *string `json:"finish_reason"` + } `json:"choices"` + Usage *struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + TotalTokens int `json:"total_tokens"` + } `json:"usage,omitempty"` +} + +// CreateChatCompletion 创建聊天补全(与聊天助手) +// POST /api/v1/chats_openai/{chat_id}/chat/completions +func (c *Client) CreateChatCompletion(ctx context.Context, chatID string, req *ChatCompletionRequest) (*ChatCompletionResponse, error) { + path := fmt.Sprintf("/api/v1/chats_openai/%s/chat/completions", chatID) + + var resp ChatCompletionResponse + if err := c.request(ctx, "POST", path, req, &resp); err != nil { + return nil, fmt.Errorf("create chat completion failed: %w", err) + } + + return &resp, nil +} + +// CreateAgentCompletion 创建 Agent 补全 +// POST /api/v1/agents_openai/{agent_id}/chat/completions +func (c *Client) CreateAgentCompletion(ctx context.Context, agentID string, req *ChatCompletionRequest) (*ChatCompletionResponse, error) { + path := fmt.Sprintf("/api/v1/agents_openai/%s/chat/completions", agentID) + + var resp ChatCompletionResponse + if err := c.request(ctx, "POST", path, req, &resp); err != nil { + return nil, fmt.Errorf("create agent completion failed: %w", err) + } + + return &resp, nil +} + +// CreateChatCompletionStream 创建流式聊天补全(与聊天助手) +// 注意:流式响应需要特殊处理,这里返回一个可用于读取流的接口 +func (c *Client) CreateChatCompletionStream(ctx context.Context, chatID string, req *ChatCompletionRequest) (*StreamReader, error) { + req.Stream = true + apiPath := fmt.Sprintf("/api/v1/chats_openai/%s/chat/completions", chatID) + + // TODO: 实现流式读取逻辑 + return nil, fmt.Errorf("stream mode not implemented yet") +} + +// StreamReader 流式响应读取器 +type StreamReader struct { + decoder *json.Decoder + close func() error +} + +// ReadChunk 读取下一个响应块 +func (sr *StreamReader) ReadChunk() (*ChatCompletionChunk, error) { + var chunk ChatCompletionChunk + if err := sr.decoder.Decode(&chunk); err != nil { + return nil, err + } + return &chunk, nil +} + +// Close 关闭流 +func (sr *StreamReader) Close() error { + if sr.close != nil { + return sr.close() + } + return nil +} + diff --git a/ragflow/service/client.go b/ragflow/service/client.go deleted file mode 100644 index 4a45dfe..0000000 --- a/ragflow/service/client.go +++ /dev/null @@ -1,77 +0,0 @@ -package service - -import ( - "context" - "fmt" - "time" - - "gitee.com/red-future---jilin-g/common/ragflow/dto" - "github.com/gogf/gf/v2/encoding/gjson" - "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/net/gclient" -) - -type Client struct { - BaseURL string - ApiKey string - Client *gclient.Client -} - -// NewClient 创建一个新的 RAGFlow 客户端 -func NewClient(baseUrl, apiKey string) *Client { - return &Client{ - BaseURL: baseUrl, - ApiKey: apiKey, - Client: g.Client().SetTimeout(30 * time.Second), - } -} - -// request 发送 HTTP 请求 -func (c *Client) request(ctx context.Context, method, path string, data interface{}, result interface{}) error { - url := fmt.Sprintf("%s%s", c.BaseURL, path) - - req := c.Client.Header(map[string]string{ - "Authorization": fmt.Sprintf("Bearer %s", c.ApiKey), - "Content-Type": "application/json", - }) - - var res *gclient.Response - var err error - - switch method { - case "GET": - res, err = req.Get(ctx, url, data) - case "POST": - res, err = req.Post(ctx, url, data) - case "PUT": - res, err = req.Put(ctx, url, data) - case "DELETE": - res, err = req.Delete(ctx, url, data) - default: - return fmt.Errorf("unsupported method: %s", method) - } - - if err != nil { - return err - } - defer res.Close() - - // 读取响应体 - body := res.ReadAllString() - - // 解析响应 - if result != nil { - if err := gjson.DecodeTo(body, result); err != nil { - return fmt.Errorf("failed to decode response: %v, body: %s", err, body) - } - - // 检查业务错误码 - if commonRes, ok := result.(*dto.CommonResponse); ok { - if !commonRes.IsSuccess() { - return fmt.Errorf("api error: code=%d, message=%s", commonRes.Code, commonRes.Message) - } - } - } - - return nil -} diff --git a/ragflow/session.go b/ragflow/session.go index 0c15136..3f8a240 100644 --- a/ragflow/session.go +++ b/ragflow/session.go @@ -5,7 +5,10 @@ import ( "fmt" ) -// Session 结构体 +// 会话管理 +// 参考: https://ragflow.com.cn/docs/dev/http_api_reference#会话管理 + +// Session 会话结构体 type Session struct { Id string `json:"id"` Name string `json:"name"` @@ -116,8 +119,9 @@ func (c *Client) ListSessions(ctx context.Context, chatId string, req *ListSessi params["user_id"] = req.UserId } - for k, v := range params { - path += fmt.Sprintf("%s=%v&", k, v) + query := buildQueryString(params) + if query != "" { + path += "?" + query } var res ListSessionsRes diff --git a/ragflow/system.go b/ragflow/system.go new file mode 100644 index 0000000..f294c91 --- /dev/null +++ b/ragflow/system.go @@ -0,0 +1,39 @@ +package ragflow + +import ( + "context" + "fmt" +) + +// System 系统管理 +// 参考: https://ragflow.com.cn/docs/dev/http_api_reference#系统 + +// HealthStatus 健康状态 +type HealthStatus struct { + DB string `json:"db"` // "ok" 或 "nok" + Redis string `json:"redis"` // "ok" 或 "nok" + DocEngine string `json:"doc_engine"` // "ok" 或 "nok" + Storage string `json:"storage"` // "ok" 或 "nok" + Status string `json:"status"` // 整体状态: "ok" 或 "nok" + Meta map[string]interface{} `json:"_meta,omitempty"` // 详细错误信息 +} + +// CheckHealth 检查系统健康状况 +// GET /v1/system/healthz +func (c *Client) CheckHealth(ctx context.Context) (*HealthStatus, error) { + var status HealthStatus + if err := c.request(ctx, "GET", "/v1/system/healthz", nil, &status); err != nil { + return nil, fmt.Errorf("check health failed: %w", err) + } + return &status, nil +} + +// IsHealthy 检查系统是否健康 +func (c *Client) IsHealthy(ctx context.Context) (bool, error) { + status, err := c.CheckHealth(ctx) + if err != nil { + return false, err + } + return status.Status == "ok", nil +} + From ad1ccc2bc16b8920b18e143f191593ad45aa9f60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 27 Nov 2025 14:42:51 +0800 Subject: [PATCH 020/225] =?UTF-8?q?gateway=E5=8F=8D=E5=90=91=E4=BB=A3?= =?UTF-8?q?=E7=90=86=E5=AF=BC=E8=87=B4=E7=9A=84header=E4=BF=A1=E6=81=AF?= =?UTF-8?q?=E9=87=8D=E5=A4=8DBUG=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/consul.go | 2 ++ http/http.go | 14 ++------------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/consul/consul.go b/consul/consul.go index d674451..bc72dd2 100644 --- a/consul/consul.go +++ b/consul/consul.go @@ -4,6 +4,7 @@ import ( "context" "github.com/gogf/gf/contrib/registry/consul/v2" "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/gsel" "github.com/gogf/gf/v2/net/gsvc" ) @@ -18,4 +19,5 @@ func init() { panic(err) } gsvc.SetRegistry(registry) + gsel.SetBuilder(gsel.NewBuilderRoundRobin()) } diff --git a/http/http.go b/http/http.go index c675ec4..afcad70 100644 --- a/http/http.go +++ b/http/http.go @@ -8,10 +8,8 @@ import ( "gitee.com/red-future---jilin-g/common/jaeger" "gitee.com/red-future---jilin-g/common/middleware" "gitee.com/red-future---jilin-g/common/utils" - "github.com/gogf/gf/contrib/registry/consul/v2" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" - "github.com/gogf/gf/v2/net/gsel" "github.com/gogf/gf/v2/net/gsvc" "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/util/gconv" @@ -65,17 +63,8 @@ func init() { `) - Httpserver.BindMiddlewareDefault(ghttp.MiddlewareCORS, ghttp.MiddlewareHandlerResponse, middleware.Limiter, jaeger.NewTracer) //使用默认http返回结构 + Httpserver.BindMiddlewareDefault(ghttp.MiddlewareHandlerResponse) go Httpserver.Run() - - consulCfg, _ := g.Cfg().Get(context.Background(), "consul.address") - consulAddr := consulCfg.String() - registry, err := consul.New(consul.WithAddress(consulAddr)) - if err != nil { - panic(err) - } - gsvc.SetRegistry(registry) - gsel.SetBuilder(gsel.NewBuilderRoundRobin()) Httpclient.SetDiscovery(gsvc.GetRegistry()) } func RouteRegister(controllers []interface{}) { @@ -86,6 +75,7 @@ func RouteRegister(controllers []interface{}) { return fmt.Sprintf("/%s", strings.ToLower(s)) }) Httpserver.Group(convertedStr, func(group *ghttp.RouterGroup) { + group.Middleware(middleware.Limiter, jaeger.NewTracer) group.Bind(t) }) } From d8410fab37e7cea1f88ad68e5d29912f7bd25db8 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Thu, 27 Nov 2025 18:03:01 +0800 Subject: [PATCH 021/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/client.go | 27 +++++++++++++-------------- ragflow/openai.go | 29 ++++++++++++++--------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/ragflow/client.go b/ragflow/client.go index 44d028c..c340f73 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -24,7 +24,7 @@ func NewClient(baseURL, apiKey string) *Client { client := gclient.New() client.SetHeader("Authorization", fmt.Sprintf("Bearer %s", apiKey)) client.SetHeader("Content-Type", "application/json") - + return &Client{ BaseURL: strings.TrimSuffix(baseURL, "/"), APIKey: apiKey, @@ -34,8 +34,8 @@ func NewClient(baseURL, apiKey string) *Client { // CommonResponse 通用响应结构 type CommonResponse struct { - Code int `json:"code"` - Message string `json:"message"` + Code int `json:"code"` + Message string `json:"message"` Data interface{} `json:"data,omitempty"` } @@ -47,7 +47,7 @@ func (r *CommonResponse) IsSuccess() bool { // request 发送 HTTP 请求 func (c *Client) request(ctx context.Context, method, path string, body interface{}, result interface{}) error { fullURL := c.BaseURL + path - + var reqBody io.Reader if body != nil { jsonData, err := json.Marshal(body) @@ -56,10 +56,10 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac } reqBody = strings.NewReader(string(jsonData)) } - + var resp *gclient.Response var err error - + switch method { case "GET": resp, err = c.HTTPClient.Get(ctx, fullURL) @@ -72,25 +72,25 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac default: return fmt.Errorf("unsupported method: %s", method) } - + if err != nil { return fmt.Errorf("http request failed: %w", err) } defer resp.Close() - + if resp.StatusCode != http.StatusOK { return fmt.Errorf("http request failed with status: %d", resp.StatusCode) } - - respBody, err := resp.ReadAll() + + respBody := resp.ReadAll() if err != nil { return fmt.Errorf("read response body failed: %w", err) } - + if err := json.Unmarshal(respBody, result); err != nil { return fmt.Errorf("unmarshal response failed: %w", err) } - + return nil } @@ -99,11 +99,10 @@ func buildQueryString(params map[string]interface{}) string { if len(params) == 0 { return "" } - + var parts []string for k, v := range params { parts = append(parts, fmt.Sprintf("%s=%v", url.QueryEscape(k), url.QueryEscape(fmt.Sprintf("%v", v)))) } return strings.Join(parts, "&") } - diff --git a/ragflow/openai.go b/ragflow/openai.go index b56008b..acaa2ff 100644 --- a/ragflow/openai.go +++ b/ragflow/openai.go @@ -11,15 +11,15 @@ import ( // ChatCompletionMessage OpenAI 格式的消息 type ChatCompletionMessage struct { - Role string `json:"role"` // "user", "assistant", "system" + Role string `json:"role"` // "user", "assistant", "system" Content string `json:"content"` } // ChatCompletionRequest OpenAI 格式的聊天补全请求 type ChatCompletionRequest struct { - Model string `json:"model"` // 模型名称(服务器会自动解析,可设置为任意值) - Messages []ChatCompletionMessage `json:"messages"` // 消息列表,必须至少包含一条 user 消息 - Stream bool `json:"stream,omitempty"` // 是否流式返回,默认 false + Model string `json:"model"` // 模型名称(服务器会自动解析,可设置为任意值) + Messages []ChatCompletionMessage `json:"messages"` // 消息列表,必须至少包含一条 user 消息 + Stream bool `json:"stream,omitempty"` // 是否流式返回,默认 false } // ChatCompletionResponse OpenAI 格式的聊天补全响应(非流式) @@ -29,9 +29,9 @@ type ChatCompletionResponse struct { Created int64 `json:"created"` Model string `json:"model"` Choices []struct { - Index int `json:"index"` + Index int `json:"index"` Message ChatCompletionMessage `json:"message"` - FinishReason string `json:"finish_reason"` + FinishReason string `json:"finish_reason"` } `json:"choices"` Usage struct { PromptTokens int `json:"prompt_tokens"` @@ -47,8 +47,8 @@ type ChatCompletionChunk struct { Created int64 `json:"created"` Model string `json:"model"` Choices []struct { - Index int `json:"index"` - Delta struct { + Index int `json:"index"` + Delta struct { Content string `json:"content"` Role string `json:"role"` } `json:"delta"` @@ -65,12 +65,12 @@ type ChatCompletionChunk struct { // POST /api/v1/chats_openai/{chat_id}/chat/completions func (c *Client) CreateChatCompletion(ctx context.Context, chatID string, req *ChatCompletionRequest) (*ChatCompletionResponse, error) { path := fmt.Sprintf("/api/v1/chats_openai/%s/chat/completions", chatID) - + var resp ChatCompletionResponse if err := c.request(ctx, "POST", path, req, &resp); err != nil { return nil, fmt.Errorf("create chat completion failed: %w", err) } - + return &resp, nil } @@ -78,12 +78,12 @@ func (c *Client) CreateChatCompletion(ctx context.Context, chatID string, req *C // POST /api/v1/agents_openai/{agent_id}/chat/completions func (c *Client) CreateAgentCompletion(ctx context.Context, agentID string, req *ChatCompletionRequest) (*ChatCompletionResponse, error) { path := fmt.Sprintf("/api/v1/agents_openai/%s/chat/completions", agentID) - + var resp ChatCompletionResponse if err := c.request(ctx, "POST", path, req, &resp); err != nil { return nil, fmt.Errorf("create agent completion failed: %w", err) } - + return &resp, nil } @@ -91,8 +91,8 @@ func (c *Client) CreateAgentCompletion(ctx context.Context, agentID string, req // 注意:流式响应需要特殊处理,这里返回一个可用于读取流的接口 func (c *Client) CreateChatCompletionStream(ctx context.Context, chatID string, req *ChatCompletionRequest) (*StreamReader, error) { req.Stream = true - apiPath := fmt.Sprintf("/api/v1/chats_openai/%s/chat/completions", chatID) - + _ = fmt.Sprintf("/api/v1/chats_openai/%s/chat/completions", chatID) + // TODO: 实现流式读取逻辑 return nil, fmt.Errorf("stream mode not implemented yet") } @@ -119,4 +119,3 @@ func (sr *StreamReader) Close() error { } return nil } - From 067c16b7ec8862bc8cd9fc7b42af7002a45f810a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 28 Nov 2025 08:52:22 +0800 Subject: [PATCH 022/225] =?UTF-8?q?swagger=E8=81=9A=E5=90=88=E5=B0=81?= =?UTF-8?q?=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/consul.go | 14 ++++++ http/http.go | 4 +- swagger/swagger.go | 105 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 swagger/swagger.go diff --git a/consul/consul.go b/consul/consul.go index bc72dd2..699f582 100644 --- a/consul/consul.go +++ b/consul/consul.go @@ -2,10 +2,12 @@ package consul import ( "context" + "errors" "github.com/gogf/gf/contrib/registry/consul/v2" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/gsel" "github.com/gogf/gf/v2/net/gsvc" + "github.com/gogf/gf/v2/util/grand" ) func init() { @@ -21,3 +23,15 @@ func init() { gsvc.SetRegistry(registry) gsel.SetBuilder(gsel.NewBuilderRoundRobin()) } +func GetInstanceAddr(ctx context.Context, name string) (addr string, err error) { + watch, err := gsvc.GetRegistry().Watch(ctx, name) + service, err := watch.Proceed() + if err != nil || service == nil { + err = errors.New("获取customerService服务实例失败!") + return + } + maxService := grand.N(0, len(service)-1) + maxEndpoints := grand.N(0, len(service[maxService].GetEndpoints())-1) + addr = service[maxService].GetEndpoints()[maxEndpoints].String() + return +} diff --git a/http/http.go b/http/http.go index afcad70..6a18287 100644 --- a/http/http.go +++ b/http/http.go @@ -55,7 +55,7 @@ func init() { - - - - `) + Httpserver.SetDumpRouterMap(true) //关闭打印路由注册信息 Httpserver.BindMiddlewareDefault(ghttp.MiddlewareHandlerResponse) go Httpserver.Run() Httpclient.SetDiscovery(gsvc.GetRegistry()) diff --git a/swagger/swagger.go b/swagger/swagger.go index bae67f8..f9981fa 100644 --- a/swagger/swagger.go +++ b/swagger/swagger.go @@ -48,7 +48,33 @@ func updateRefsRecursively(data interface{}, serviceName string) { } } } -func Aggregated() { +func init() { + http.Httpserver.SetOpenApiPath("/api.json") + http.Httpserver.SetSwaggerPath("/docs") //api文档访问路径 + http.Httpserver.SetSwaggerUITemplate(` + + + + + + + SwaggerUI + + + +
+ + + + + `) http.Httpserver.BindHandler("/swagger", func(r *ghttp.Request) { services, _ := g.Cfg().Get(r.GetCtx(), "consul.services") aggregatedDoc := &swaggerDoc{ From ab7d3de666035a2ddb598e1c74c4ba694afb0807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 28 Nov 2025 11:16:21 +0800 Subject: [PATCH 024/225] =?UTF-8?q?swagger=E8=81=9A=E5=90=88=E5=B0=81?= =?UTF-8?q?=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 1 + swagger/swagger.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/http/http.go b/http/http.go index 30c041d..757ae57 100644 --- a/http/http.go +++ b/http/http.go @@ -36,6 +36,7 @@ var Httpclient = g.Client() func init() { //s.Use(common.Cors) //中间件验证 //s.EnablePProf() //启用性能分析 + Httpserver.SetOpenApiPath("/api.json") Httpserver.SetDumpRouterMap(true) //关闭打印路由注册信息 Httpserver.BindMiddlewareDefault(ghttp.MiddlewareHandlerResponse) go Httpserver.Run() diff --git a/swagger/swagger.go b/swagger/swagger.go index f9981fa..e988705 100644 --- a/swagger/swagger.go +++ b/swagger/swagger.go @@ -49,7 +49,6 @@ func updateRefsRecursively(data interface{}, serviceName string) { } } func init() { - http.Httpserver.SetOpenApiPath("/api.json") http.Httpserver.SetSwaggerPath("/docs") //api文档访问路径 http.Httpserver.SetSwaggerUITemplate(` From 7e2884414dd002968e27802cc4e4a85f8aebce0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 28 Nov 2025 14:01:08 +0800 Subject: [PATCH 025/225] =?UTF-8?q?=E8=8E=B7=E5=8F=96=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E4=BB=A3=E7=A0=81=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/consul.go | 6 +----- jaeger/jaeger.go | 13 ++++--------- middleware/middleware.go | 6 +++--- mongo/mongo.go | 7 +++---- swagger/swagger.go | 4 ++-- 5 files changed, 13 insertions(+), 23 deletions(-) diff --git a/consul/consul.go b/consul/consul.go index 699f582..ba019c8 100644 --- a/consul/consul.go +++ b/consul/consul.go @@ -11,11 +11,7 @@ import ( ) func init() { - consulCfg, err := g.Cfg().Get(context.Background(), "consul.address") - if err != nil { - panic(err) - } - consulAddr := consulCfg.String() + consulAddr := g.Cfg().MustGet(context.Background(), "consul.address").String() registry, err := consul.New(consul.WithAddress(consulAddr)) if err != nil { panic(err) diff --git a/jaeger/jaeger.go b/jaeger/jaeger.go index ea8fa94..aeb5041 100644 --- a/jaeger/jaeger.go +++ b/jaeger/jaeger.go @@ -14,18 +14,13 @@ import ( var ShutDown func(ctx context.Context) func init() { - jaegerAgent, err := g.Cfg().Get(context.Background(), "jaeger.addr") - if err != nil { - panic(err) - } - serverName, err := g.Cfg().Get(context.Background(), "server.Name") - if err != nil { - panic(err) - } - ShutDown, err = otlphttp.Init(serverName.String(), jaegerAgent.String(), "/v1/traces") + jaegerAgent := g.Cfg().MustGet(context.Background(), "jaeger.addr").String() + serverName := g.Cfg().MustGet(context.Background(), "server.Name").String() + shutdown, err := otlphttp.Init(serverName, jaegerAgent, "/v1/traces") if err != nil { panic(err) } + ShutDown = shutdown } func NewTracer(r *ghttp.Request) { _, span := gtrace.NewSpan(r.Context(), r.GetServeHandler().GetMetaTag("summary")) diff --git a/middleware/middleware.go b/middleware/middleware.go index 593c25c..2bbf766 100644 --- a/middleware/middleware.go +++ b/middleware/middleware.go @@ -23,9 +23,9 @@ func Logger(r *ghttp.Request) { ) } -var rateLimit, _ = g.Cfg().Get(context.TODO(), "rate.limit") -var rateBurst, _ = g.Cfg().Get(context.TODO(), "rate.burst") -var limiter = rate.NewLimiter(rate.Limit(rateLimit.Int()), rateBurst.Int()) +var rateLimit = g.Cfg().MustGet(context.TODO(), "rate.limit").Int() +var rateBurst = g.Cfg().MustGet(context.TODO(), "rate.burst").Int() +var limiter = rate.NewLimiter(rate.Limit(rateLimit), rateBurst) func Limiter(r *ghttp.Request) { if !limiter.Allow() { diff --git a/mongo/mongo.go b/mongo/mongo.go index f0a8eb2..380fbad 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -20,8 +20,7 @@ var db = new(mongo.Database) func init() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - link, _ := g.Cfg().Get(context.Background(), "mongo.address") - mongoAddr := link.String() + mongoAddr := g.Cfg().MustGet(context.Background(), "mongo.address").String() opt := options.Client().ApplyURI(mongoAddr) client, err := mongo.Connect(opt) if err != nil { @@ -95,9 +94,9 @@ func Insert(ctx context.Context, documents []interface{}, collection string, opt return } -// Count 查询总数 +// Count 查询总数 func Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { // 调用驱动的 CountDocuments,在数据库端执行的 count, err = db.Collection(collection).CountDocuments(ctx, filter) return -} \ No newline at end of file +} diff --git a/swagger/swagger.go b/swagger/swagger.go index e988705..40157da 100644 --- a/swagger/swagger.go +++ b/swagger/swagger.go @@ -75,7 +75,7 @@ func init() { `) http.Httpserver.BindHandler("/swagger", func(r *ghttp.Request) { - services, _ := g.Cfg().Get(r.GetCtx(), "consul.services") + services := g.Cfg().MustGet(r.GetCtx(), "consul.services").Slice() aggregatedDoc := &swaggerDoc{ OpenAPI: "3.0.0", // 假设统一为 3.0.0 版本 Info: map[string]interface{}{ @@ -86,7 +86,7 @@ func init() { Paths: make(map[string]interface{}), Components: make(map[string]interface{}), } - for _, v := range services.Slice() { + for _, v := range services { serviceName := gconv.String(gconv.Map(v)["name"]) instanceAddr, err := consul.GetInstanceAddr(r.GetCtx(), serviceName) if err != nil || instanceAddr == "" { From 302336a769d2f3c25ca4a2ef14806619e5354d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 28 Nov 2025 15:19:02 +0800 Subject: [PATCH 026/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0Mongo=E5=9F=BA?= =?UTF-8?q?=E7=A1=80=E5=AD=97=E6=AE=B5=E7=BB=93=E6=9E=84=E4=BD=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- do/base_do.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 do/base_do.go diff --git a/do/base_do.go b/do/base_do.go new file mode 100644 index 0000000..4d7f45e --- /dev/null +++ b/do/base_do.go @@ -0,0 +1,14 @@ +package do + +import ( + "go.mongodb.org/mongo-driver/v2/bson" + "time" +) + +type MongoBaseDO struct { + Creator bson.ObjectID `bson:"creator" json:"creator"` + CreatedAt time.Time `bson:"createdAt" json:"createdAt"` + Updater bson.ObjectID `bson:"updater" json:"updater"` + UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"` + TenantId bson.ObjectID `bson:"tenantId" json:"tenantId"` +} From 64b107d48b36bebbaf0ae0ce19caaa60bc418612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 28 Nov 2025 15:21:53 +0800 Subject: [PATCH 027/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0Mongo=E5=9F=BA?= =?UTF-8?q?=E7=A1=80=E5=AD=97=E6=AE=B5=E7=BB=93=E6=9E=84=E4=BD=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- do/base_do.go | 1 + 1 file changed, 1 insertion(+) diff --git a/do/base_do.go b/do/base_do.go index 4d7f45e..e80b00e 100644 --- a/do/base_do.go +++ b/do/base_do.go @@ -6,6 +6,7 @@ import ( ) type MongoBaseDO struct { + Id bson.ObjectID `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID Creator bson.ObjectID `bson:"creator" json:"creator"` CreatedAt time.Time `bson:"createdAt" json:"createdAt"` Updater bson.ObjectID `bson:"updater" json:"updater"` From 52944f1c1cff0997be754f42c7139fe1274c1cd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 28 Nov 2025 15:41:35 +0800 Subject: [PATCH 028/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0Mongo=E5=9F=BA?= =?UTF-8?q?=E7=A1=80=E5=AD=97=E6=AE=B5=E7=BB=93=E6=9E=84=E4=BD=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- do/base_do.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/do/base_do.go b/do/base_do.go index e80b00e..803850a 100644 --- a/do/base_do.go +++ b/do/base_do.go @@ -7,9 +7,10 @@ import ( type MongoBaseDO struct { Id bson.ObjectID `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID - Creator bson.ObjectID `bson:"creator" json:"creator"` - CreatedAt time.Time `bson:"createdAt" json:"createdAt"` - Updater bson.ObjectID `bson:"updater" json:"updater"` - UpdatedAt time.Time `bson:"updatedAt" json:"updatedAt"` - TenantId bson.ObjectID `bson:"tenantId" json:"tenantId"` + Creator bson.ObjectID `bson:"creator,omitempty" json:"creator"` + CreatedAt time.Time `bson:"createdAt,omitempty" json:"createdAt"` + Updater bson.ObjectID `bson:"updater,omitempty" json:"updater"` + UpdatedAt time.Time `bson:"updatedAt,omitempty" json:"updatedAt"` + TenantId bson.ObjectID `bson:"tenantId" json:"tenantId"` // 租户ID + IsDeleted int `bson:"isDeleted" json:"isDeleted"` // 是否删除,1已删除,0未删除 } From c38961e908cba003ce3d086d3912acc1e55ac28c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 28 Nov 2025 17:13:16 +0800 Subject: [PATCH 029/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0Mongo=E5=9F=BA?= =?UTF-8?q?=E7=A1=80=E5=AD=97=E6=AE=B5=E7=BB=93=E6=9E=84=E4=BD=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 3 +-- mongo/mongo.go | 28 ++++++++++++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/http/http.go b/http/http.go index 757ae57..2b95a47 100644 --- a/http/http.go +++ b/http/http.go @@ -6,7 +6,6 @@ import ( "fmt" _ "gitee.com/red-future---jilin-g/common/consul" "gitee.com/red-future---jilin-g/common/jaeger" - "gitee.com/red-future---jilin-g/common/middleware" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" @@ -50,7 +49,7 @@ func RouteRegister(controllers []interface{}) { return fmt.Sprintf("/%s", strings.ToLower(s)) }) Httpserver.Group(convertedStr, func(group *ghttp.RouterGroup) { - group.Middleware(middleware.Limiter, jaeger.NewTracer) + group.Middleware(jaeger.NewTracer) group.Bind(t) }) } diff --git a/mongo/mongo.go b/mongo/mongo.go index 380fbad..3bfa69a 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -40,6 +40,7 @@ func Find(ctx context.Context, filter bson.M, result interface{}, collection str if err = utils.ValidStructPtr(result); err != nil { return } + //filter["isDeleted"] = 0 cur, err := db.Collection(collection).Find(ctx, filter, opts...) if err != nil { return @@ -50,13 +51,14 @@ func Find(ctx context.Context, filter bson.M, result interface{}, collection str // FindOne 查询1条记录 func FindOne(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOneOptions]) (err error) { - if len(filter) == 0 { - err = gerror.New("缺少查询条件") - return - } + //if len(filter) == 0 { + // err = gerror.New("缺少查询条件") + // return + //} if err = utils.ValidStructPtr(result); err != nil { return } + //filter["isDeleted"] = 0 cur := db.Collection(collection).FindOne(ctx, filter, opts...) err = cur.Decode(result) if err == mongo.ErrNoDocuments { @@ -67,6 +69,11 @@ func FindOne(ctx context.Context, filter bson.M, result interface{}, collection // Delete 删除记录 func Delete(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.DeleteManyOptions]) (count int64, err error) { + if len(filter) == 0 { + err = gerror.New("缺少查询条件") + return + } + //filter["isDeleted"] = 0 r, err := db.Collection(collection).DeleteMany(ctx, filter, opts...) if err != nil { return @@ -77,6 +84,11 @@ func Delete(ctx context.Context, filter bson.M, collection string, opts ...optio // Update 修改记录 func Update(ctx context.Context, filter bson.M, update interface{}, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { + if len(filter) == 0 { + err = gerror.New("缺少查询条件") + return + } + //filter["isDeleted"] = 0 result, err = db.Collection(collection).UpdateMany(ctx, filter, update, opts...) if err != nil { return @@ -86,6 +98,9 @@ func Update(ctx context.Context, filter bson.M, update interface{}, collection s // Insert 插入多条记录 func Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { + //for _, document := range documents { + // gconv.Map(document)["isDeleted"] = 0 + //} r, err := db.Collection(collection).InsertMany(ctx, documents, opts...) if err != nil { return @@ -96,6 +111,11 @@ func Insert(ctx context.Context, documents []interface{}, collection string, opt // Count 查询总数 func Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { + //if len(filter) == 0 { + // err = gerror.New("缺少查询条件") + // return + //} + //filter["isDeleted"] = 0 // 调用驱动的 CountDocuments,在数据库端执行的 count, err = db.Collection(collection).CountDocuments(ctx, filter) return From 0a8d1f27336937486036e1e2f66f496c4b52cab2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 1 Dec 2025 10:06:11 +0800 Subject: [PATCH 030/225] =?UTF-8?q?=E8=AE=BE=E7=BD=AE=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E6=97=B6=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/http/http.go b/http/http.go index 2b95a47..5997f90 100644 --- a/http/http.go +++ b/http/http.go @@ -11,6 +11,7 @@ import ( "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/net/gsvc" "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/util/gconv" "net/http" "reflect" @@ -33,6 +34,10 @@ var Httpserver = g.Server() var Httpclient = g.Client() func init() { + err := gtime.SetTimeZone("Asia/Shanghai") + if err != nil { + panic("设置时区失败") + } //s.Use(common.Cors) //中间件验证 //s.EnablePProf() //启用性能分析 Httpserver.SetOpenApiPath("/api.json") From 8ed282877ea289539fc1aa9e3ec6d959773105c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 1 Dec 2025 14:09:04 +0800 Subject: [PATCH 031/225] =?UTF-8?q?base=E5=AE=9E=E4=BD=93=E7=B1=BBIsDelete?= =?UTF-8?q?d=E5=AD=97=E6=AE=B5=E7=B1=BB=E5=9E=8B=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- do/base_do.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/do/base_do.go b/do/base_do.go index 803850a..842334d 100644 --- a/do/base_do.go +++ b/do/base_do.go @@ -11,6 +11,6 @@ type MongoBaseDO struct { CreatedAt time.Time `bson:"createdAt,omitempty" json:"createdAt"` Updater bson.ObjectID `bson:"updater,omitempty" json:"updater"` UpdatedAt time.Time `bson:"updatedAt,omitempty" json:"updatedAt"` - TenantId bson.ObjectID `bson:"tenantId" json:"tenantId"` // 租户ID - IsDeleted int `bson:"isDeleted" json:"isDeleted"` // 是否删除,1已删除,0未删除 + TenantId bson.ObjectID `bson:"tenantId" json:"tenantId" default:"1"` // 租户ID + IsDeleted bool `bson:"isDeleted" json:"isDeleted" default:"false"` } From cd3571554fe512d688ea4dfadf60e0d89cdbc435 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 2 Dec 2025 09:07:21 +0800 Subject: [PATCH 032/225] =?UTF-8?q?1.=E4=BF=AE=E6=94=B9basedo=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E7=B1=BB=E5=9E=8B=202=E3=80=81=E6=8F=90=E4=BE=9B?= =?UTF-8?q?=E8=8E=B7=E5=8F=96authen=E4=B8=AD=E7=94=A8=E6=88=B7=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E7=9A=84=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- do/base_do.go | 13 +++++++++---- middleware/middleware.go | 2 ++ utils/utils.go | 23 +++++++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/do/base_do.go b/do/base_do.go index 842334d..37c5067 100644 --- a/do/base_do.go +++ b/do/base_do.go @@ -1,16 +1,21 @@ package do import ( - "go.mongodb.org/mongo-driver/v2/bson" "time" + + "go.mongodb.org/mongo-driver/v2/bson" ) type MongoBaseDO struct { Id bson.ObjectID `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID - Creator bson.ObjectID `bson:"creator,omitempty" json:"creator"` + Creator interface{} `bson:"creator,omitempty" json:"creator"` CreatedAt time.Time `bson:"createdAt,omitempty" json:"createdAt"` - Updater bson.ObjectID `bson:"updater,omitempty" json:"updater"` + Updater interface{} `bson:"updater,omitempty" json:"updater"` UpdatedAt time.Time `bson:"updatedAt,omitempty" json:"updatedAt"` - TenantId bson.ObjectID `bson:"tenantId" json:"tenantId" default:"1"` // 租户ID + TenantId interface{} `bson:"tenantId" json:"tenantId" default:"1"` // 租户ID IsDeleted bool `bson:"isDeleted" json:"isDeleted" default:"false"` } +type User struct { + Id interface{} `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID + TenantId interface{} `bson:"tenantId" json:"tenantId" default:"1"` // 租户ID +} diff --git a/middleware/middleware.go b/middleware/middleware.go index 2bbf766..d37ba83 100644 --- a/middleware/middleware.go +++ b/middleware/middleware.go @@ -2,6 +2,7 @@ package middleware import ( "context" + "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/os/gtime" @@ -35,6 +36,7 @@ func Limiter(r *ghttp.Request) { r.Middleware.Next() } func Auth(r *ghttp.Request) { + //utils.GetUserInfo(r.GetCtx()) token := r.Header.Get("Authorization") if token == "" || !gstr.HasPrefix(token, "Bearer ") { r.Response.WriteStatusExit(401, "Unauthorized") diff --git a/utils/utils.go b/utils/utils.go index 5c24731..7e319fb 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1,9 +1,15 @@ package utils import ( + "context" "fmt" + "gitee.com/red-future---jilin-g/common/do" + "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/errors/gcode" "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" + "github.com/tiger1103/gfast-token/gftoken" "reflect" "time" ) @@ -51,3 +57,20 @@ func GetMonthToday(t time.Time, month int) time.Time { } return target.AddDate(0, 0, t.Day()-1) } +func GetUserInfo(ctx context.Context) (user do.User, err error) { + redisAddr := g.Cfg().MustGet(ctx, "redis.default.address").String() + gft := gftoken.NewGfToken( + gftoken.WithCacheKey("gfToken:"), + gftoken.WithTimeout(20), + gftoken.WithMaxRefresh(10), + gftoken.WithMultiLogin(true), + //gftoken.WithExcludePaths(g.SliceStr{"/excludeDemo"}), + gftoken.WithGRedisConfig(&gredis.Config{ + Address: redisAddr, + Db: 1, + })) + data, _ := gft.ParseToken(g.RequestFromCtx(ctx)) + user.Id = gconv.Map(data.Data)["id"] + user.TenantId = gconv.Map(data.Data)["tenantId"] + return +} From 5d4c8c8711e6cf71a0d393b9f05f252e2c8b7842 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 2 Dec 2025 14:59:07 +0800 Subject: [PATCH 033/225] =?UTF-8?q?=E4=BF=AE=E6=94=B9rag=E7=9A=84=E5=AE=9E?= =?UTF-8?q?=E4=BD=93=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/agent.go | 24 ++++++++++++------- ragflow/chat.go | 8 +++---- ragflow/chunk.go | 11 +++++---- ragflow/dataset.go | 9 +++---- ragflow/document.go | 58 +++++++++++++++++++++++++++++++++------------ ragflow/session.go | 8 +++---- 6 files changed, 78 insertions(+), 40 deletions(-) diff --git a/ragflow/agent.go b/ragflow/agent.go index 7e0ed1e..4ce263a 100644 --- a/ragflow/agent.go +++ b/ragflow/agent.go @@ -10,10 +10,17 @@ import ( // Agent Agent 结构体 type Agent struct { - ID string `json:"id"` - Title string `json:"title"` - Description string `json:"description"` - DSL map[string]interface{} `json:"dsl"` // Canvas DSL 对象 + ID string `json:"id"` // Agent ID + Title string `json:"title"` // Agent 标题 + Description string `json:"description"` // Agent 描述 + Avatar string `json:"avatar"` // 头像(Base64 编码) + CanvasType string `json:"canvas_type"` // 画布类型 + CreateDate string `json:"create_date"` // 创建日期(格式化字符串) + CreateTime int64 `json:"create_time"` // 创建时间(Unix 时间戳) + UpdateDate string `json:"update_date"` // 更新日期(格式化字符串) + UpdateTime int64 `json:"update_time"` // 更新时间(Unix 时间戳) + UserID string `json:"user_id"` // 用户 ID + DSL map[string]interface{} `json:"dsl"` // Canvas DSL 对象,定义 Agent 的工作流 } // CreateAgentReq 创建 Agent 请求 @@ -41,10 +48,10 @@ type ListAgentsReq struct { } // ListAgentsRes 列出 Agent 响应 +// 注意:API 不返回 total 字段,仅返回 data 数组 type ListAgentsRes struct { - Code int `json:"code"` - Data []*Agent `json:"data"` - Total int `json:"total"` + Code int `json:"code"` // 状态码,0 表示成功 + Data []*Agent `json:"data"` // Agent 列表 } // CreateAgent 创建 Agent @@ -79,7 +86,8 @@ func (c *Client) UpdateAgent(ctx context.Context, agentID string, req *UpdateAge func (c *Client) DeleteAgent(ctx context.Context, agentID string) error { path := fmt.Sprintf("/api/v1/agents/%s", agentID) var res CommonResponse - if err := c.request(ctx, "DELETE", path, nil, &res); err != nil { + // 官方文档要求传空对象,不是 nil + if err := c.request(ctx, "DELETE", path, map[string]interface{}{}, &res); err != nil { return fmt.Errorf("delete agent failed: %w", err) } if !res.IsSuccess() { diff --git a/ragflow/chat.go b/ragflow/chat.go index bb19251..6f65335 100644 --- a/ragflow/chat.go +++ b/ragflow/chat.go @@ -82,10 +82,10 @@ type ListChatsReq struct { } // ListChatsRes 列出聊天助手响应 +// 注意:API 不返回 total 字段,仅返回 data 数组 type ListChatsRes struct { - Code int `json:"code"` - Data []*Chat `json:"data"` - Total int `json:"total"` // API 文档中未明确 total 字段,但通常列表接口会有 + Code int `json:"code"` // 状态码,0 表示成功 + Data []*Chat `json:"data"` // 聊天助手列表 } // DeleteChatsReq 删除聊天助手请求 @@ -111,7 +111,7 @@ func (c *Client) CreateChat(ctx context.Context, req *CreateChatReq) (*Chat, err // ListChats 列出聊天助手 func (c *Client) ListChats(ctx context.Context, req *ListChatsReq) (*ListChatsRes, error) { - path := "/api/v1/chats?" + path := "/api/v1/chats" params := map[string]interface{}{} if req.Page > 0 { params["page"] = req.Page diff --git a/ragflow/chunk.go b/ragflow/chunk.go index dad9e59..7e0b66e 100644 --- a/ragflow/chunk.go +++ b/ragflow/chunk.go @@ -39,12 +39,13 @@ type ListChunksReq struct { } // ListChunksRes 列出知识块响应 +// 注意:响应结构包含 chunks(知识块列表)、doc(关联文档信息)和 total(总数) type ListChunksRes struct { - Code int `json:"code"` + Code int `json:"code"` // 状态码,0 表示成功 Data struct { - Chunks []*Chunk `json:"chunks"` - Doc interface{} `json:"doc"` // 文档信息,暂时用 interface{} - Total int `json:"total"` + Chunks []*Chunk `json:"chunks"` // 知识块列表 + Doc interface{} `json:"doc"` // 关联文档信息(完整的 Document 对象) + Total int `json:"total"` // 知识块总数 } `json:"data"` } @@ -108,7 +109,7 @@ func (c *Client) AddChunk(ctx context.Context, datasetId, documentId string, req // ListChunks 列出知识块 func (c *Client) ListChunks(ctx context.Context, datasetId, documentId string, req *ListChunksReq) (*ListChunksRes, error) { - path := fmt.Sprintf("/api/v1/datasets/%s/documents/%s/chunks?", datasetId, documentId) + path := fmt.Sprintf("/api/v1/datasets/%s/documents/%s/chunks", datasetId, documentId) params := map[string]interface{}{} if req.Keywords != "" { params["keywords"] = req.Keywords diff --git a/ragflow/dataset.go b/ragflow/dataset.go index 0301d70..83162b7 100644 --- a/ragflow/dataset.go +++ b/ragflow/dataset.go @@ -67,10 +67,11 @@ type ListDatasetsReq struct { } // ListDatasetsRes 列出数据集响应 +// 注意:与 Agent/Chat 等接口不同,Dataset API 会返回 total 字段 type ListDatasetsRes struct { - Code int `json:"code"` - Data []*Dataset `json:"data"` - Total int `json:"total"` + Code int `json:"code"` // 状态码,0 表示成功 + Data []*Dataset `json:"data"` // 数据集列表 + Total int `json:"total"` // 总数据集数 } // DeleteDatasetsReq 删除数据集请求 @@ -97,7 +98,7 @@ func (c *Client) CreateDataset(ctx context.Context, req *CreateDatasetReq) (*Dat // ListDatasets 列出数据集 func (c *Client) ListDatasets(ctx context.Context, req *ListDatasetsReq) (*ListDatasetsRes, error) { // 构建查询参数 - path := "/api/v1/datasets?" + path := "/api/v1/datasets" params := map[string]interface{}{} if req.Page > 0 { params["page"] = req.Page diff --git a/ragflow/document.go b/ragflow/document.go index 44827ca..60d6192 100644 --- a/ragflow/document.go +++ b/ragflow/document.go @@ -3,12 +3,13 @@ package ragflow import ( "context" "fmt" + "strings" ) // 数据集内文件管理 // 参考: https://ragflow.com.cn/docs/dev/http_api_reference#数据集内文件管理 -// Document 文档结构体 +// ... (rest of the code remains the same) type Document struct { Id string `json:"id"` DatasetId string `json:"dataset_id"` @@ -39,22 +40,27 @@ type UploadDocumentReq struct { // ListDocumentsReq 列出文档请求 type ListDocumentsReq struct { - Page int `json:"page,omitempty"` - PageSize int `json:"page_size,omitempty"` - OrderBy string `json:"orderby,omitempty"` - Desc bool `json:"desc,omitempty"` - Keywords string `json:"keywords,omitempty"` - Id string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - CreateTimeFrom int64 `json:"create_time_from,omitempty"` - CreateTimeTo int64 `json:"create_time_to,omitempty"` + Page int `json:"page,omitempty"` // 页码,默认 1 + PageSize int `json:"page_size,omitempty"` // 每页数量,默认 30 + OrderBy string `json:"orderby,omitempty"` // 排序字段:create_time(默认)或 update_time + Desc bool `json:"desc,omitempty"` // 是否降序,默认 true + Keywords string `json:"keywords,omitempty"` // 关键词过滤(匹配文档标题) + Id string `json:"id,omitempty"` // 文档 ID 过滤 + Name string `json:"name,omitempty"` // 文档名称过滤 + CreateTimeFrom int64 `json:"create_time_from,omitempty"` // 创建时间起始(Unix 时间戳),0 表示无限制 + CreateTimeTo int64 `json:"create_time_to,omitempty"` // 创建时间截止(Unix 时间戳),0 表示无限制 + Suffix []string `json:"suffix,omitempty"` // 文件后缀过滤,如 ["pdf", "txt", "docx"] + Run []string `json:"run,omitempty"` // 处理状态过滤,支持 ["UNSTART", "RUNNING", "CANCEL", "DONE", "FAIL"] 或数字格式 ["0", "1", "2", "3", "4"] } // ListDocumentsRes 列出文档响应 +// 注意:响应结构与其他 List 接口不同,data 是一个对象而非数组 type ListDocumentsRes struct { - Code int `json:"code"` - Data []*Document `json:"data"` - Total int `json:"total"` + Code int `json:"code"` // 状态码,0 表示成功 + Data struct { + Docs []*Document `json:"docs"` // 文档列表 + TotalDatasets int `json:"total_datasets"` // 总文档数 + } `json:"data"` } // DeleteDocumentsReq 删除文档请求 @@ -64,7 +70,7 @@ type DeleteDocumentsReq struct { // ListDocuments 列出文档 func (c *Client) ListDocuments(ctx context.Context, datasetId string, req *ListDocumentsReq) (*ListDocumentsRes, error) { - path := fmt.Sprintf("/api/v1/datasets/%s/documents?", datasetId) + path := fmt.Sprintf("/api/v1/datasets/%s/documents", datasetId) params := map[string]interface{}{} if req.Page > 0 { params["page"] = req.Page @@ -96,11 +102,33 @@ func (c *Client) ListDocuments(ctx context.Context, datasetId string, req *ListD params["create_time_to"] = req.CreateTimeTo } + // 构造查询字符串 query := buildQueryString(params) + var queryParts []string if query != "" { - path += "?" + query + queryParts = append(queryParts, query) } + // 处理数组参数:suffix(文件后缀过滤) + // API 要求多个值时重复参数名,如:suffix=pdf&suffix=txt + // 这里使用 fmt.Sprintf 来构造每个参数值 + for _, suffix := range req.Suffix { + queryParts = append(queryParts, fmt.Sprintf("suffix=%s", suffix)) + } + + // 处理数组参数:run(处理状态过滤) + // 支持数字格式("0"-"4")或文本格式("UNSTART", "RUNNING", "CANCEL", "DONE", "FAIL") + // 这里使用 fmt.Sprintf 来构造每个参数值 + for _, run := range req.Run { + queryParts = append(queryParts, fmt.Sprintf("run=%s", run)) + } + + // 构造最终请求路径 + if len(queryParts) > 0 { + path += "?" + strings.Join(queryParts, "&") + } + + // 发送请求并处理响应 var res ListDocumentsRes if err := c.request(ctx, "GET", path, nil, &res); err != nil { return nil, err diff --git a/ragflow/session.go b/ragflow/session.go index 3f8a240..832082a 100644 --- a/ragflow/session.go +++ b/ragflow/session.go @@ -43,10 +43,10 @@ type ListSessionsReq struct { } // ListSessionsRes 列出会话响应 +// 注意:API 不返回 total 字段,仅返回 data 数组 type ListSessionsRes struct { - Code int `json:"code"` - Data []*Session `json:"data"` - Total int `json:"total"` // API 文档未明确 + Code int `json:"code"` // 状态码,0 表示成功 + Data []*Session `json:"data"` // 会话列表 } // DeleteSessionsReq 删除会话请求 @@ -93,7 +93,7 @@ func (c *Client) CreateSession(ctx context.Context, chatId string, req *CreateSe // ListSessions 列出会话 func (c *Client) ListSessions(ctx context.Context, chatId string, req *ListSessionsReq) (*ListSessionsRes, error) { - path := fmt.Sprintf("/api/v1/chats/%s/sessions?", chatId) + path := fmt.Sprintf("/api/v1/chats/%s/sessions", chatId) params := map[string]interface{}{} if req.Page > 0 { params["page"] = req.Page From 161a0a43a92011e47c69bed89881fff7d8491c05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 2 Dec 2025 11:27:35 +0800 Subject: [PATCH 034/225] =?UTF-8?q?1.=E4=BF=AE=E6=94=B9basedo=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E7=B1=BB=E5=9E=8B=202=E3=80=81=E6=8F=90=E4=BE=9B?= =?UTF-8?q?=E8=8E=B7=E5=8F=96authen=E4=B8=AD=E7=94=A8=E6=88=B7=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E7=9A=84=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- do/base_do.go | 4 +-- mongo/mongo.go | 69 +++++++++++++++++++++++++++++++++++++------------- utils/utils.go | 5 ++-- 3 files changed, 56 insertions(+), 22 deletions(-) diff --git a/do/base_do.go b/do/base_do.go index 37c5067..5c4001f 100644 --- a/do/base_do.go +++ b/do/base_do.go @@ -16,6 +16,6 @@ type MongoBaseDO struct { IsDeleted bool `bson:"isDeleted" json:"isDeleted" default:"false"` } type User struct { - Id interface{} `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID - TenantId interface{} `bson:"tenantId" json:"tenantId" default:"1"` // 租户ID + UserName interface{} `bson:"userName" json:"userName"` // MongoDB 默认 ID + TenantId interface{} `bson:"tenantId" json:"tenantId"` // 租户ID } diff --git a/mongo/mongo.go b/mongo/mongo.go index 3bfa69a..1318dfb 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -9,7 +9,9 @@ import ( "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/text/gstr" + "github.com/gogf/gf/v2/util/gconv" "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/mongo" "go.mongodb.org/mongo-driver/v2/mongo/options" @@ -40,7 +42,12 @@ func Find(ctx context.Context, filter bson.M, result interface{}, collection str if err = utils.ValidStructPtr(result); err != nil { return } - //filter["isDeleted"] = 0 + filter["isDeleted"] = false + user, err := utils.GetUserInfo(ctx) + if err != nil { + return + } + filter["tenantId"] = user.TenantId cur, err := db.Collection(collection).Find(ctx, filter, opts...) if err != nil { return @@ -51,14 +58,19 @@ func Find(ctx context.Context, filter bson.M, result interface{}, collection str // FindOne 查询1条记录 func FindOne(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOneOptions]) (err error) { - //if len(filter) == 0 { - // err = gerror.New("缺少查询条件") - // return - //} + if len(filter) == 0 { + err = gerror.New("缺少查询条件") + return + } if err = utils.ValidStructPtr(result); err != nil { return } - //filter["isDeleted"] = 0 + filter["isDeleted"] = false + user, err := utils.GetUserInfo(ctx) + if err != nil { + return + } + filter["tenantId"] = user.TenantId cur := db.Collection(collection).FindOne(ctx, filter, opts...) err = cur.Decode(result) if err == mongo.ErrNoDocuments { @@ -73,7 +85,11 @@ func Delete(ctx context.Context, filter bson.M, collection string, opts ...optio err = gerror.New("缺少查询条件") return } - //filter["isDeleted"] = 0 + user, err := utils.GetUserInfo(ctx) + if err != nil { + return + } + filter["tenantId"] = user.TenantId r, err := db.Collection(collection).DeleteMany(ctx, filter, opts...) if err != nil { return @@ -88,8 +104,16 @@ func Update(ctx context.Context, filter bson.M, update interface{}, collection s err = gerror.New("缺少查询条件") return } - //filter["isDeleted"] = 0 - result, err = db.Collection(collection).UpdateMany(ctx, filter, update, opts...) + filter["isDeleted"] = false + user, err := utils.GetUserInfo(ctx) + if err != nil { + return + } + filter["tenantId"] = user.TenantId + doc := gconv.Map(update) + doc["updater"] = user.UserName + doc["updatedAt"] = gtime.Now().Time + result, err = db.Collection(collection).UpdateMany(ctx, filter, doc, opts...) if err != nil { return } @@ -98,10 +122,23 @@ func Update(ctx context.Context, filter bson.M, update interface{}, collection s // Insert 插入多条记录 func Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { - //for _, document := range documents { - // gconv.Map(document)["isDeleted"] = 0 - //} - r, err := db.Collection(collection).InsertMany(ctx, documents, opts...) + user, err := utils.GetUserInfo(ctx) + if err != nil { + return + } + docs := make([]interface{}, 0, len(documents)) + for _, document := range documents { + doc := gconv.Map(document) + delete(doc, "id") + doc["creator"] = user.UserName + doc["createdAt"] = gtime.Now().Time + doc["updater"] = user.UserName + doc["updatedAt"] = gtime.Now().Time + doc["tenantId"] = user.TenantId + doc["isDeleted"] = false + docs = append(docs, doc) + } + r, err := db.Collection(collection).InsertMany(ctx, docs, opts...) if err != nil { return } @@ -111,11 +148,7 @@ func Insert(ctx context.Context, documents []interface{}, collection string, opt // Count 查询总数 func Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { - //if len(filter) == 0 { - // err = gerror.New("缺少查询条件") - // return - //} - //filter["isDeleted"] = 0 + filter["isDeleted"] = false // 调用驱动的 CountDocuments,在数据库端执行的 count, err = db.Collection(collection).CountDocuments(ctx, filter) return diff --git a/utils/utils.go b/utils/utils.go index 7e319fb..c2d049f 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -70,7 +70,8 @@ func GetUserInfo(ctx context.Context) (user do.User, err error) { Db: 1, })) data, _ := gft.ParseToken(g.RequestFromCtx(ctx)) - user.Id = gconv.Map(data.Data)["id"] - user.TenantId = gconv.Map(data.Data)["tenantId"] + dataMap := gconv.Map(data.Data) + user.UserName = dataMap["userName"] + user.TenantId = dataMap["tenantId"] return } From 358e175799b0b677451225b4f2783de09f289f08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 2 Dec 2025 15:56:57 +0800 Subject: [PATCH 035/225] =?UTF-8?q?mongo=20update=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 1318dfb..92f0c2a 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -99,7 +99,7 @@ func Delete(ctx context.Context, filter bson.M, collection string, opts ...optio } // Update 修改记录 -func Update(ctx context.Context, filter bson.M, update interface{}, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { +func Update(ctx context.Context, filter bson.M, update bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { if len(filter) == 0 { err = gerror.New("缺少查询条件") return @@ -110,10 +110,11 @@ func Update(ctx context.Context, filter bson.M, update interface{}, collection s return } filter["tenantId"] = user.TenantId - doc := gconv.Map(update) - doc["updater"] = user.UserName - doc["updatedAt"] = gtime.Now().Time - result, err = db.Collection(collection).UpdateMany(ctx, filter, doc, opts...) + setDoc := update["$set"].(bson.M) + setDoc["updater"] = user.UserName + setDoc["updatedAt"] = gtime.Now().Time + update = bson.M{"$set": setDoc} + result, err = db.Collection(collection).UpdateMany(ctx, filter, update, opts...) if err != nil { return } From 62d47ebde75b8f83e8e2d1b40e6f1c537fd59a8e Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Wed, 3 Dec 2025 09:59:40 +0800 Subject: [PATCH 036/225] =?UTF-8?q?RAGflow=E7=9A=84init=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=92=8Cmq?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rabbitmq/client.go | 179 ++++++++++++++++++++++++++++++++ rabbitmq/consumer.go | 165 ++++++++++++++++++++++++++++++ rabbitmq/publisher.go | 147 +++++++++++++++++++++++++++ rabbitmq/setup.go | 231 ++++++++++++++++++++++++++++++++++++++++++ ragflow/client.go | 80 ++++++++++++--- ragflow/config.yaml | 10 ++ 6 files changed, 798 insertions(+), 14 deletions(-) create mode 100644 rabbitmq/client.go create mode 100644 rabbitmq/consumer.go create mode 100644 rabbitmq/publisher.go create mode 100644 rabbitmq/setup.go create mode 100644 ragflow/config.yaml diff --git a/rabbitmq/client.go b/rabbitmq/client.go new file mode 100644 index 0000000..7738082 --- /dev/null +++ b/rabbitmq/client.go @@ -0,0 +1,179 @@ +package rabbitmq + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/gogf/gf/v2/frame/g" + amqp "github.com/rabbitmq/amqp091-go" +) + +var ( + conn *amqp.Connection + channel *amqp.Channel + once sync.Once + mu sync.RWMutex +) + +// Config RabbitMQ 配置 +type Config struct { + Host string + Port int + Username string + Password string + VHost string +} + +// Init 初始化 RabbitMQ 连接 +func Init(ctx context.Context, cfg *Config) error { + var err error + once.Do(func() { + // 构建连接字符串 + url := fmt.Sprintf("amqp://%s:%s@%s:%d/%s", + cfg.Username, + cfg.Password, + cfg.Host, + cfg.Port, + cfg.VHost, + ) + + // 创建连接 + conn, err = amqp.Dial(url) + if err != nil { + g.Log().Errorf(ctx, "RabbitMQ 连接失败: %v", err) + return + } + + // 创建 Channel + channel, err = conn.Channel() + if err != nil { + g.Log().Errorf(ctx, "创建 RabbitMQ Channel 失败: %v", err) + return + } + + // 监听连接关闭 + go handleConnectionClose(ctx) + + g.Log().Info(ctx, "RabbitMQ 连接成功") + }) + + return err +} + +// InitFromConfig 从配置文件初始化 +func InitFromConfig(ctx context.Context) error { + cfg := &Config{ + Host: g.Cfg().MustGet(ctx, "rabbitmq.host").String(), + Port: g.Cfg().MustGet(ctx, "rabbitmq.port").Int(), + Username: g.Cfg().MustGet(ctx, "rabbitmq.username").String(), + Password: g.Cfg().MustGet(ctx, "rabbitmq.password").String(), + VHost: g.Cfg().MustGet(ctx, "rabbitmq.vhost", "/").String(), + } + + return Init(ctx, cfg) +} + +// GetChannel 获取 Channel +func GetChannel() (*amqp.Channel, error) { + mu.RLock() + defer mu.RUnlock() + + if channel == nil || channel.IsClosed() { + return nil, fmt.Errorf("RabbitMQ Channel 未初始化或已关闭") + } + + return channel, nil +} + +// GetConnection 获取连接 +func GetConnection() (*amqp.Connection, error) { + mu.RLock() + defer mu.RUnlock() + + if conn == nil || conn.IsClosed() { + return nil, fmt.Errorf("RabbitMQ 连接未初始化或已关闭") + } + + return conn, nil +} + +// handleConnectionClose 监听连接关闭并重连 +func handleConnectionClose(ctx context.Context) { + closeErr := make(chan *amqp.Error) + conn.NotifyClose(closeErr) + + err := <-closeErr + if err != nil { + g.Log().Errorf(ctx, "RabbitMQ 连接关闭: %v,尝试重连...", err) + reconnect(ctx) + } +} + +// reconnect 重新连接 +func reconnect(ctx context.Context) { + mu.Lock() + defer mu.Unlock() + + for i := 0; i < 10; i++ { + time.Sleep(time.Duration(i+1) * time.Second) + + cfg := &Config{ + Host: g.Cfg().MustGet(ctx, "rabbitmq.host").String(), + Port: g.Cfg().MustGet(ctx, "rabbitmq.port").Int(), + Username: g.Cfg().MustGet(ctx, "rabbitmq.username").String(), + Password: g.Cfg().MustGet(ctx, "rabbitmq.password").String(), + VHost: g.Cfg().MustGet(ctx, "rabbitmq.vhost", "/").String(), + } + + url := fmt.Sprintf("amqp://%s:%s@%s:%d/%s", + cfg.Username, + cfg.Password, + cfg.Host, + cfg.Port, + cfg.VHost, + ) + + var err error + conn, err = amqp.Dial(url) + if err != nil { + g.Log().Errorf(ctx, "重连失败 (尝试 %d/10): %v", i+1, err) + continue + } + + channel, err = conn.Channel() + if err != nil { + g.Log().Errorf(ctx, "创建 Channel 失败 (尝试 %d/10): %v", i+1, err) + continue + } + + g.Log().Info(ctx, "RabbitMQ 重连成功") + go handleConnectionClose(ctx) + return + } + + g.Log().Fatal(ctx, "RabbitMQ 重连失败,已达到最大重试次数") +} + +// Close 关闭连接 +func Close(ctx context.Context) error { + mu.Lock() + defer mu.Unlock() + + if channel != nil { + if err := channel.Close(); err != nil { + g.Log().Errorf(ctx, "关闭 RabbitMQ Channel 失败: %v", err) + } + } + + if conn != nil { + if err := conn.Close(); err != nil { + g.Log().Errorf(ctx, "关闭 RabbitMQ 连接失败: %v", err) + return err + } + } + + g.Log().Info(ctx, "RabbitMQ 连接已关闭") + return nil +} diff --git a/rabbitmq/consumer.go b/rabbitmq/consumer.go new file mode 100644 index 0000000..f9fe3ae --- /dev/null +++ b/rabbitmq/consumer.go @@ -0,0 +1,165 @@ +package rabbitmq + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/gogf/gf/v2/frame/g" + amqp "github.com/rabbitmq/amqp091-go" +) + +// MessageHandler 消息处理函数 +type MessageHandler func(ctx context.Context, body []byte) error + +// Consumer 消费者 +type Consumer struct { + queue string + consumerTag string + prefetchCount int // QoS: 预取数量(并发控制) + autoAck bool // 是否自动确认 + handler MessageHandler + workerCount int // worker 数量 +} + +// ConsumerOption 消费者配置选项 +type ConsumerOption func(*Consumer) + +// WithPrefetchCount 设置预取数量(并发控制) +func WithPrefetchCount(count int) ConsumerOption { + return func(c *Consumer) { + c.prefetchCount = count + } +} + +// WithAutoAck 设置自动确认 +func WithAutoAck(autoAck bool) ConsumerOption { + return func(c *Consumer) { + c.autoAck = autoAck + } +} + +// WithWorkerCount 设置 worker 数量 +func WithWorkerCount(count int) ConsumerOption { + return func(c *Consumer) { + c.workerCount = count + } +} + +// WithConsumerTag 设置消费者标签 +func WithConsumerTag(tag string) ConsumerOption { + return func(c *Consumer) { + c.consumerTag = tag + } +} + +// NewConsumer 创建消费者 +func NewConsumer(queue string, handler MessageHandler, opts ...ConsumerOption) *Consumer { + c := &Consumer{ + queue: queue, + consumerTag: "", + prefetchCount: 1, // 默认 1 个 + autoAck: false, // 默认手动确认 + handler: handler, + workerCount: 1, // 默认 1 个 worker + } + + // 应用选项 + for _, opt := range opts { + opt(c) + } + + return c +} + +// Start 启动消费者 +func (c *Consumer) Start(ctx context.Context) error { + ch, err := GetChannel() + if err != nil { + return err + } + + // 设置 QoS(并发控制) + err = ch.Qos( + c.prefetchCount, // prefetchCount: 每个 consumer 最多同时处理的消息数 + 0, // prefetchSize: 0 表示不限制 + false, // global: false 表示仅应用于当前 channel + ) + if err != nil { + return fmt.Errorf("设置 QoS 失败: %v", err) + } + + // 开始消费 + msgs, err := ch.Consume( + c.queue, // queue + c.consumerTag, // consumer tag + c.autoAck, // auto-ack + false, // exclusive + false, // no-local + false, // no-wait + nil, // args + ) + if err != nil { + return fmt.Errorf("开始消费失败: %v", err) + } + + g.Log().Infof(ctx, "消费者已启动: queue=%s, prefetch=%d, workers=%d", + c.queue, c.prefetchCount, c.workerCount) + + // 启动多个 worker + for i := 0; i < c.workerCount; i++ { + go c.worker(ctx, i, msgs) + } + + return nil +} + +// worker 工作协程 +func (c *Consumer) worker(ctx context.Context, workerID int, msgs <-chan amqp.Delivery) { + g.Log().Debugf(ctx, "Worker %d 已启动", workerID) + + for msg := range msgs { + // 处理消息 + err := c.handler(ctx, msg.Body) + + if err != nil { + g.Log().Errorf(ctx, "Worker %d 处理消息失败: %v", workerID, err) + + // 如果不是自动确认,需要手动 Nack + if !c.autoAck { + // requeue=false: 不重新入队,进入死信队列 + msg.Nack(false, false) + } + } else { + // 处理成功,手动确认 + if !c.autoAck { + msg.Ack(false) + } + + g.Log().Debugf(ctx, "Worker %d 处理消息成功", workerID) + } + } + + g.Log().Debugf(ctx, "Worker %d 已停止", workerID) +} + +// StartTypedConsumer 启动类型化消费者(自动反序列化) +func StartTypedConsumer[T any]( + ctx context.Context, + queue string, + handler func(ctx context.Context, msg *T) error, + opts ...ConsumerOption, +) error { + // 包装处理函数 + wrappedHandler := func(ctx context.Context, body []byte) error { + var msg T + if err := json.Unmarshal(body, &msg); err != nil { + return fmt.Errorf("反序列化消息失败: %v", err) + } + + return handler(ctx, &msg) + } + + consumer := NewConsumer(queue, wrappedHandler, opts...) + return consumer.Start(ctx) +} diff --git a/rabbitmq/publisher.go b/rabbitmq/publisher.go new file mode 100644 index 0000000..096744b --- /dev/null +++ b/rabbitmq/publisher.go @@ -0,0 +1,147 @@ +package rabbitmq + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/gogf/gf/v2/frame/g" + amqp "github.com/rabbitmq/amqp091-go" +) + +// Publisher 消息发布器 +type Publisher struct { + exchange string + routingKey string +} + +// NewPublisher 创建发布器 +func NewPublisher(exchange, routingKey string) *Publisher { + return &Publisher{ + exchange: exchange, + routingKey: routingKey, + } +} + +// Publish 发布消息 +func (p *Publisher) Publish(ctx context.Context, message interface{}) error { + ch, err := GetChannel() + if err != nil { + return err + } + + // 序列化消息 + body, err := json.Marshal(message) + if err != nil { + return fmt.Errorf("消息序列化失败: %v", err) + } + + // 发布消息 + err = ch.PublishWithContext( + ctx, + p.exchange, // exchange + p.routingKey, // routing key + false, // mandatory + false, // immediate + amqp.Publishing{ + DeliveryMode: amqp.Persistent, // 持久化 + ContentType: "application/json", + Body: body, + }, + ) + + if err != nil { + g.Log().Errorf(ctx, "发布消息失败: exchange=%s, routingKey=%s, err=%v", + p.exchange, p.routingKey, err) + return err + } + + g.Log().Debugf(ctx, "消息发布成功: exchange=%s, routingKey=%s", + p.exchange, p.routingKey) + + return nil +} + +// PublishDelayed 发布延时消息 +// delaySeconds: 延时秒数 +func (p *Publisher) PublishDelayed(ctx context.Context, message interface{}, delaySeconds int) error { + ch, err := GetChannel() + if err != nil { + return err + } + + // 序列化消息 + body, err := json.Marshal(message) + if err != nil { + return fmt.Errorf("消息序列化失败: %v", err) + } + + // 发布延时消息(需要 rabbitmq_delayed_message_exchange 插件) + err = ch.PublishWithContext( + ctx, + p.exchange, // exchange(必须是 x-delayed-message 类型) + p.routingKey, // routing key + false, // mandatory + false, // immediate + amqp.Publishing{ + DeliveryMode: amqp.Persistent, + ContentType: "application/json", + Body: body, + Headers: amqp.Table{ + "x-delay": delaySeconds * 1000, // 延时(毫秒) + }, + }, + ) + + if err != nil { + g.Log().Errorf(ctx, "发布延时消息失败: exchange=%s, routingKey=%s, delay=%ds, err=%v", + p.exchange, p.routingKey, delaySeconds, err) + return err + } + + g.Log().Debugf(ctx, "延时消息发布成功: exchange=%s, routingKey=%s, delay=%ds", + p.exchange, p.routingKey, delaySeconds) + + return nil +} + +// PublishBatch 批量发布消息 +func (p *Publisher) PublishBatch(ctx context.Context, messages []interface{}) error { + if len(messages) == 0 { + return nil + } + + ch, err := GetChannel() + if err != nil { + return err + } + + for i, message := range messages { + body, err := json.Marshal(message) + if err != nil { + g.Log().Errorf(ctx, "消息 %d 序列化失败: %v", i, err) + continue + } + + err = ch.PublishWithContext( + ctx, + p.exchange, + p.routingKey, + false, + false, + amqp.Publishing{ + DeliveryMode: amqp.Persistent, + ContentType: "application/json", + Body: body, + }, + ) + + if err != nil { + g.Log().Errorf(ctx, "消息 %d 发布失败: %v", i, err) + continue + } + } + + g.Log().Infof(ctx, "批量发布完成: 共 %d 条消息", len(messages)) + return nil +} diff --git a/rabbitmq/setup.go b/rabbitmq/setup.go new file mode 100644 index 0000000..bc8bf59 --- /dev/null +++ b/rabbitmq/setup.go @@ -0,0 +1,231 @@ +package rabbitmq + +import ( + "context" + "fmt" + + "github.com/gogf/gf/v2/frame/g" + amqp "github.com/rabbitmq/amqp091-go" +) + +// QueueConfig 队列配置 +type QueueConfig struct { + Name string + Durable bool // 持久化 + AutoDelete bool // 自动删除 + Exclusive bool // 排他 + Args amqp.Table // 额外参数 +} + +// ExchangeConfig Exchange 配置 +type ExchangeConfig struct { + Name string + Type string // direct/topic/fanout/x-delayed-message + Durable bool + AutoDelete bool + Args amqp.Table +} + +// BindingConfig 绑定配置 +type BindingConfig struct { + Queue string + Exchange string + RoutingKey string + Args amqp.Table +} + +// DeclareQueue 声明队列 +func DeclareQueue(ctx context.Context, cfg *QueueConfig) error { + ch, err := GetChannel() + if err != nil { + return err + } + + _, err = ch.QueueDeclare( + cfg.Name, + cfg.Durable, + cfg.AutoDelete, + cfg.Exclusive, + false, // no-wait + cfg.Args, + ) + + if err != nil { + g.Log().Errorf(ctx, "声明队列失败: %s, err=%v", cfg.Name, err) + return err + } + + g.Log().Infof(ctx, "队列声明成功: %s", cfg.Name) + return nil +} + +// DeclareExchange 声明 Exchange +func DeclareExchange(ctx context.Context, cfg *ExchangeConfig) error { + ch, err := GetChannel() + if err != nil { + return err + } + + err = ch.ExchangeDeclare( + cfg.Name, + cfg.Type, + cfg.Durable, + cfg.AutoDelete, + false, // internal + false, // no-wait + cfg.Args, + ) + + if err != nil { + g.Log().Errorf(ctx, "声明 Exchange 失败: %s, err=%v", cfg.Name, err) + return err + } + + g.Log().Infof(ctx, "Exchange 声明成功: %s (type=%s)", cfg.Name, cfg.Type) + return nil +} + +// BindQueue 绑定队列到 Exchange +func BindQueue(ctx context.Context, cfg *BindingConfig) error { + ch, err := GetChannel() + if err != nil { + return err + } + + err = ch.QueueBind( + cfg.Queue, + cfg.RoutingKey, + cfg.Exchange, + false, // no-wait + cfg.Args, + ) + + if err != nil { + g.Log().Errorf(ctx, "绑定队列失败: queue=%s, exchange=%s, routingKey=%s, err=%v", + cfg.Queue, cfg.Exchange, cfg.RoutingKey, err) + return err + } + + g.Log().Infof(ctx, "队列绑定成功: queue=%s → exchange=%s (routingKey=%s)", + cfg.Queue, cfg.Exchange, cfg.RoutingKey) + return nil +} + +// SetupDelayExchange 设置延时 Exchange(需要 rabbitmq_delayed_message_exchange 插件) +func SetupDelayExchange(ctx context.Context, exchangeName string) error { + return DeclareExchange(ctx, &ExchangeConfig{ + Name: exchangeName, + Type: "x-delayed-message", + Durable: true, + Args: amqp.Table{ + "x-delayed-type": "direct", + }, + }) +} + +// SetupDeadLetterQueue 设置死信队列 +func SetupDeadLetterQueue(ctx context.Context, queueName, exchangeName string) error { + // 1. 声明死信 Exchange + err := DeclareExchange(ctx, &ExchangeConfig{ + Name: exchangeName, + Type: "direct", + Durable: true, + }) + if err != nil { + return err + } + + // 2. 声明死信队列 + err = DeclareQueue(ctx, &QueueConfig{ + Name: queueName, + Durable: true, + }) + if err != nil { + return err + } + + // 3. 绑定 + return BindQueue(ctx, &BindingConfig{ + Queue: queueName, + Exchange: exchangeName, + RoutingKey: queueName, + }) +} + +// SetupQueueWithDLX 创建带死信队列的普通队列 +func SetupQueueWithDLX(ctx context.Context, queueName, dlxExchange, dlxRoutingKey string) error { + return DeclareQueue(ctx, &QueueConfig{ + Name: queueName, + Durable: true, + Args: amqp.Table{ + "x-dead-letter-exchange": dlxExchange, + "x-dead-letter-routing-key": dlxRoutingKey, + }, + }) +} + +// SetupBasicTopology 设置基础拓扑(适用于小红书客服场景) +func SetupBasicTopology(ctx context.Context) error { + // 1. 声明普通 Exchange + err := DeclareExchange(ctx, &ExchangeConfig{ + Name: "ragflow_exchange", + Type: "direct", + Durable: true, + }) + if err != nil { + return err + } + + // 2. 声明延时 Exchange + err = SetupDelayExchange(ctx, "delay_exchange") + if err != nil { + return fmt.Errorf("延时 Exchange 声明失败(可能未安装插件): %v", err) + } + + // 3. 声明死信队列 + err = SetupDeadLetterQueue(ctx, "dead_letter_queue", "dlx_exchange") + if err != nil { + return err + } + + // 4. 声明业务队列 + queues := []struct { + name string + dlx bool // 是否需要死信队列 + }{ + {"ragflow_request_queue", true}, + {"follow_up_queue", true}, + {"archive_queue", true}, + } + + for _, q := range queues { + if q.dlx { + err = SetupQueueWithDLX(ctx, q.name, "dlx_exchange", "dead_letter_queue") + } else { + err = DeclareQueue(ctx, &QueueConfig{ + Name: q.name, + Durable: true, + }) + } + if err != nil { + return err + } + } + + // 5. 绑定队列 + bindings := []BindingConfig{ + {Queue: "ragflow_request_queue", Exchange: "ragflow_exchange", RoutingKey: "ragflow_request_queue"}, + {Queue: "follow_up_queue", Exchange: "delay_exchange", RoutingKey: "follow_up_queue"}, + {Queue: "archive_queue", Exchange: "delay_exchange", RoutingKey: "archive_queue"}, + } + + for _, b := range bindings { + err = BindQueue(ctx, &b) + if err != nil { + return err + } + } + + g.Log().Info(ctx, "RabbitMQ 拓扑结构设置完成") + return nil +} diff --git a/ragflow/client.go b/ragflow/client.go index c340f73..bdbcd39 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -9,27 +9,79 @@ import ( "net/url" "strings" + "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/gclient" + "github.com/gogf/gf/v2/os/gcfg" ) +var ( + // globalClient 全局 RAGFlow 客户端(单例,自动初始化) + globalClient *Client +) + +// init 包初始化时自动创建全局客户端 +func init() { + ctx := context.Background() + + // 读取配置 + baseURL, apiKey := loadConfig(ctx) + + // 如果配置不完整,跳过初始化 + if baseURL == "" || apiKey == "" { + g.Log().Warning(ctx, "⚠️ RAGFlow 配置未找到,请在 common/ragflow/config.yaml 中配置") + return + } + + // 初始化全局客户端 + httpClient := gclient.New() + httpClient.SetHeader("Authorization", fmt.Sprintf("Bearer %s", apiKey)) + httpClient.SetHeader("Content-Type", "application/json") + + globalClient = &Client{ + BaseURL: strings.TrimSuffix(baseURL, "/"), + APIKey: apiKey, + HTTPClient: httpClient, + } + + g.Log().Infof(ctx, "✅ RAGFlow 全局客户端初始化成功: baseURL=%s", baseURL) +} + +// loadConfig 从配置文件加载 RAGFlow 配置 +func loadConfig(ctx context.Context) (baseURL, apiKey string) { + // 创建配置实例 + cfg, err := gcfg.New() + if err != nil { + g.Log().Debugf(ctx, "创建配置实例失败: %v", err) + return "", "" + } + + // 设置配置文件 + adapter, ok := cfg.GetAdapter().(*gcfg.AdapterFile) + if !ok { + g.Log().Debug(ctx, "配置适配器类型不匹配") + return "", "" + } + + adapter.SetFileName("config.yaml") + + // 读取配置项 + baseURL = cfg.MustGet(ctx, "ragflow.base_url").String() + apiKey = cfg.MustGet(ctx, "ragflow.api_key").String() + + return baseURL, apiKey +} + +// GetGlobalClient 获取全局客户端 +// 使用示例:client := ragflow.GetGlobalClient() +func GetGlobalClient() *Client { + return globalClient +} + // Client RAGFlow API 客户端 type Client struct { BaseURL string APIKey string - HTTPClient *gclient.Client -} - -// NewClient 创建新的 RAGFlow 客户端 -func NewClient(baseURL, apiKey string) *Client { - client := gclient.New() - client.SetHeader("Authorization", fmt.Sprintf("Bearer %s", apiKey)) - client.SetHeader("Content-Type", "application/json") - - return &Client{ - BaseURL: strings.TrimSuffix(baseURL, "/"), - APIKey: apiKey, - HTTPClient: client, - } + HTTPClient *gclient.Client // HTTP 客户端 } // CommonResponse 通用响应结构 diff --git a/ragflow/config.yaml b/ragflow/config.yaml new file mode 100644 index 0000000..95c7e13 --- /dev/null +++ b/ragflow/config.yaml @@ -0,0 +1,10 @@ +# RAGFlow 配置文件 +# 用于全局客户端自动初始化 + +ragflow: + # RAGFlow 服务地址 + base_url: "http://localhost:9380" + + # RAGFlow API Key + # 获取方式:登录 RAGFlow 管理界面 -> 设置 -> API Keys + api_key: "ragflow-your-api-key-here" From ff8257c6052120aecf022b858eb0ff21c2886a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 2 Dec 2025 18:25:48 +0800 Subject: [PATCH 037/225] =?UTF-8?q?jaeger=E4=BF=AE=E6=94=B9=E8=AF=BB?= =?UTF-8?q?=E5=8F=96config=E5=AD=97=E6=AE=B5=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jaeger/jaeger.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jaeger/jaeger.go b/jaeger/jaeger.go index aeb5041..f0cdbab 100644 --- a/jaeger/jaeger.go +++ b/jaeger/jaeger.go @@ -3,6 +3,7 @@ package jaeger import ( "context" "encoding/json" + "gitee.com/red-future---jilin-g/common/http" "github.com/gogf/gf/contrib/trace/otlphttp/v2" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" @@ -15,8 +16,7 @@ var ShutDown func(ctx context.Context) func init() { jaegerAgent := g.Cfg().MustGet(context.Background(), "jaeger.addr").String() - serverName := g.Cfg().MustGet(context.Background(), "server.Name").String() - shutdown, err := otlphttp.Init(serverName, jaegerAgent, "/v1/traces") + shutdown, err := otlphttp.Init(http.Httpserver.GetName(), jaegerAgent, "/v1/traces") if err != nil { panic(err) } From 4581e74807708a3ef93e6ec18bf7faa987c143e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 2 Dec 2025 18:36:51 +0800 Subject: [PATCH 038/225] =?UTF-8?q?jaeger=E4=BF=AE=E6=94=B9=E8=AF=BB?= =?UTF-8?q?=E5=8F=96config=E5=AD=97=E6=AE=B5=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jaeger/jaeger.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/jaeger/jaeger.go b/jaeger/jaeger.go index f0cdbab..9116cc4 100644 --- a/jaeger/jaeger.go +++ b/jaeger/jaeger.go @@ -3,20 +3,21 @@ package jaeger import ( "context" "encoding/json" - "gitee.com/red-future---jilin-g/common/http" + "strconv" + "github.com/gogf/gf/contrib/trace/otlphttp/v2" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/net/gtrace" "go.opentelemetry.io/otel/attribute" - "strconv" ) var ShutDown func(ctx context.Context) func init() { jaegerAgent := g.Cfg().MustGet(context.Background(), "jaeger.addr").String() - shutdown, err := otlphttp.Init(http.Httpserver.GetName(), jaegerAgent, "/v1/traces") + serverName := g.Cfg().MustGet(context.Background(), "server.name").String() + shutdown, err := otlphttp.Init(serverName, jaegerAgent, "/v1/traces") if err != nil { panic(err) } From 15b781e2e4f10d68b44bcdbe3e0a327e6113851d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 3 Dec 2025 09:03:02 +0800 Subject: [PATCH 039/225] =?UTF-8?q?consul=E5=9C=B0=E5=9D=80=E5=88=86?= =?UTF-8?q?=E9=85=8D=E4=BC=98=E5=85=88=E8=B0=83=E7=94=A8gateway=E5=90=8CIP?= =?UTF-8?q?=E7=9A=84=E5=BE=AE=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/consul.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/consul/consul.go b/consul/consul.go index ba019c8..8b1c00a 100644 --- a/consul/consul.go +++ b/consul/consul.go @@ -3,6 +3,9 @@ package consul import ( "context" "errors" + "fmt" + "net" + "github.com/gogf/gf/contrib/registry/consul/v2" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/gsel" @@ -19,6 +22,26 @@ func init() { gsvc.SetRegistry(registry) gsel.SetBuilder(gsel.NewBuilderRoundRobin()) } +func getLocalIP() (string, error) { + // 获取本机所有网络接口 + addrs, err := net.InterfaceAddrs() + if err != nil { + return "", err + } + + for _, addr := range addrs { + // 检查是否是IP地址 + if ipNet, ok := addr.(*net.IPNet); ok && !ipNet.IP.IsLoopback() { + if ipNet.IP.To4() != nil { + // 返回第一个非回环的IPv4地址 + return ipNet.IP.String(), nil + } + } + } + + return "", fmt.Errorf("无法找到本地IP地址") +} + func GetInstanceAddr(ctx context.Context, name string) (addr string, err error) { watch, err := gsvc.GetRegistry().Watch(ctx, name) service, err := watch.Proceed() @@ -28,6 +51,17 @@ func GetInstanceAddr(ctx context.Context, name string) (addr string, err error) } maxService := grand.N(0, len(service)-1) maxEndpoints := grand.N(0, len(service[maxService].GetEndpoints())-1) + //优先使用gateway同IP的服务实例 + addr, err = getLocalIP() + if err != nil { + return + } + for _, point := range service[maxService].GetEndpoints() { + if point.Host() == addr { + addr = point.String() + return + } + } addr = service[maxService].GetEndpoints()[maxEndpoints].String() return } From 90780f636213165ec2ead3320579210a8274f85c Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Wed, 3 Dec 2025 10:09:00 +0800 Subject: [PATCH 040/225] =?UTF-8?q?ragflow=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/client.go | 25 ++++--------------------- ragflow/config.yaml | 10 ---------- 2 files changed, 4 insertions(+), 31 deletions(-) delete mode 100644 ragflow/config.yaml diff --git a/ragflow/client.go b/ragflow/client.go index bdbcd39..95ef503 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -11,7 +11,6 @@ import ( "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/gclient" - "github.com/gogf/gf/v2/os/gcfg" ) var ( @@ -28,7 +27,7 @@ func init() { // 如果配置不完整,跳过初始化 if baseURL == "" || apiKey == "" { - g.Log().Warning(ctx, "⚠️ RAGFlow 配置未找到,请在 common/ragflow/config.yaml 中配置") + g.Log().Warning(ctx, "⚠️ RAGFlow 配置未找到,请在项目 config.yml 中添加 ragflow.base_url 和 ragflow.api_key") return } @@ -48,25 +47,9 @@ func init() { // loadConfig 从配置文件加载 RAGFlow 配置 func loadConfig(ctx context.Context) (baseURL, apiKey string) { - // 创建配置实例 - cfg, err := gcfg.New() - if err != nil { - g.Log().Debugf(ctx, "创建配置实例失败: %v", err) - return "", "" - } - - // 设置配置文件 - adapter, ok := cfg.GetAdapter().(*gcfg.AdapterFile) - if !ok { - g.Log().Debug(ctx, "配置适配器类型不匹配") - return "", "" - } - - adapter.SetFileName("config.yaml") - - // 读取配置项 - baseURL = cfg.MustGet(ctx, "ragflow.base_url").String() - apiKey = cfg.MustGet(ctx, "ragflow.api_key").String() + // 使用 GoFrame 全局配置(从项目的 config.yml 读取) + baseURL = g.Cfg().MustGet(ctx, "ragflow.base_url", "").String() + apiKey = g.Cfg().MustGet(ctx, "ragflow.api_key", "").String() return baseURL, apiKey } diff --git a/ragflow/config.yaml b/ragflow/config.yaml deleted file mode 100644 index 95c7e13..0000000 --- a/ragflow/config.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# RAGFlow 配置文件 -# 用于全局客户端自动初始化 - -ragflow: - # RAGFlow 服务地址 - base_url: "http://localhost:9380" - - # RAGFlow API Key - # 获取方式:登录 RAGFlow 管理界面 -> 设置 -> API Keys - api_key: "ragflow-your-api-key-here" From 0738f6f9570613037c38ca2c441e6e6480837064 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Wed, 3 Dec 2025 15:33:32 +0800 Subject: [PATCH 041/225] goroutine --- mongo/mongo.go | 14 +++++++++ rabbitmq/client.go | 68 +++++++++++++++++++++++++++++++++++--------- rabbitmq/consumer.go | 66 ++++++++++++++++++++++++++++-------------- 3 files changed, 114 insertions(+), 34 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 92f0c2a..b7928ff 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -56,6 +56,20 @@ func Find(ctx context.Context, filter bson.M, result interface{}, collection str return } +// FindWithoutTenant 查询多条记录(不过滤租户,用于导出等场景) +func FindWithoutTenant(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOptions]) (err error) { + if err = utils.ValidStructPtr(result); err != nil { + return + } + // 不添加 tenantId 过滤条件 + cur, err := db.Collection(collection).Find(ctx, filter, opts...) + if err != nil { + return + } + err = cur.All(ctx, result) + return +} + // FindOne 查询1条记录 func FindOne(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOneOptions]) (err error) { if len(filter) == 0 { diff --git a/rabbitmq/client.go b/rabbitmq/client.go index 7738082..4c0a05f 100644 --- a/rabbitmq/client.go +++ b/rabbitmq/client.go @@ -11,10 +11,12 @@ import ( ) var ( - conn *amqp.Connection - channel *amqp.Channel - once sync.Once - mu sync.RWMutex + conn *amqp.Connection + channel *amqp.Channel + once sync.Once + mu sync.RWMutex + closeWatcher chan struct{} // 用于停止监听 goroutine + watcherStarted bool // 防止重复启动监听 ) // Config RabbitMQ 配置 @@ -53,8 +55,14 @@ func Init(ctx context.Context, cfg *Config) error { return } - // 监听连接关闭 - go handleConnectionClose(ctx) + // 初始化关闭监听器 + closeWatcher = make(chan struct{}) + + // 监听连接关闭(只启动一次) + if !watcherStarted { + go handleConnectionClose(ctx) + watcherStarted = true + } g.Log().Info(ctx, "RabbitMQ 连接成功") }) @@ -101,13 +109,38 @@ func GetConnection() (*amqp.Connection, error) { // handleConnectionClose 监听连接关闭并重连 func handleConnectionClose(ctx context.Context) { - closeErr := make(chan *amqp.Error) - conn.NotifyClose(closeErr) + for { + // 检查是否需要停止监听 + select { + case <-closeWatcher: + g.Log().Info(ctx, "停止监听 RabbitMQ 连接状态") + return + default: + } - err := <-closeErr - if err != nil { - g.Log().Errorf(ctx, "RabbitMQ 连接关闭: %v,尝试重连...", err) - reconnect(ctx) + mu.RLock() + currentConn := conn + mu.RUnlock() + + if currentConn == nil { + return + } + + // 创建关闭通知 channel + closeErr := make(chan *amqp.Error, 1) + currentConn.NotifyClose(closeErr) + + // 等待连接关闭或停止信号 + select { + case err := <-closeErr: + if err != nil { + g.Log().Errorf(ctx, "RabbitMQ 连接关闭: %v,尝试重连...", err) + reconnect(ctx) + } + case <-closeWatcher: + g.Log().Info(ctx, "停止监听 RabbitMQ 连接状态") + return + } } } @@ -149,7 +182,7 @@ func reconnect(ctx context.Context) { } g.Log().Info(ctx, "RabbitMQ 重连成功") - go handleConnectionClose(ctx) + // 不再重复启动监听 goroutine return } @@ -161,10 +194,17 @@ func Close(ctx context.Context) error { mu.Lock() defer mu.Unlock() + // 停止监听 goroutine + if closeWatcher != nil { + close(closeWatcher) + closeWatcher = nil + } + if channel != nil { if err := channel.Close(); err != nil { g.Log().Errorf(ctx, "关闭 RabbitMQ Channel 失败: %v", err) } + channel = nil } if conn != nil { @@ -172,8 +212,10 @@ func Close(ctx context.Context) error { g.Log().Errorf(ctx, "关闭 RabbitMQ 连接失败: %v", err) return err } + conn = nil } + watcherStarted = false g.Log().Info(ctx, "RabbitMQ 连接已关闭") return nil } diff --git a/rabbitmq/consumer.go b/rabbitmq/consumer.go index f9fe3ae..1b0746d 100644 --- a/rabbitmq/consumer.go +++ b/rabbitmq/consumer.go @@ -19,7 +19,8 @@ type Consumer struct { prefetchCount int // QoS: 预取数量(并发控制) autoAck bool // 是否自动确认 handler MessageHandler - workerCount int // worker 数量 + workerCount int // worker 数量 + cancel context.CancelFunc // 用于停止 worker } // ConsumerOption 消费者配置选项 @@ -74,6 +75,9 @@ func NewConsumer(queue string, handler MessageHandler, opts ...ConsumerOption) * // Start 启动消费者 func (c *Consumer) Start(ctx context.Context) error { + // 创建可取消的 context + workerCtx, cancel := context.WithCancel(ctx) + c.cancel = cancel ch, err := GetChannel() if err != nil { return err @@ -108,7 +112,7 @@ func (c *Consumer) Start(ctx context.Context) error { // 启动多个 worker for i := 0; i < c.workerCount; i++ { - go c.worker(ctx, i, msgs) + go c.worker(workerCtx, i, msgs) } return nil @@ -118,29 +122,40 @@ func (c *Consumer) Start(ctx context.Context) error { func (c *Consumer) worker(ctx context.Context, workerID int, msgs <-chan amqp.Delivery) { g.Log().Debugf(ctx, "Worker %d 已启动", workerID) - for msg := range msgs { - // 处理消息 - err := c.handler(ctx, msg.Body) - - if err != nil { - g.Log().Errorf(ctx, "Worker %d 处理消息失败: %v", workerID, err) - - // 如果不是自动确认,需要手动 Nack - if !c.autoAck { - // requeue=false: 不重新入队,进入死信队列 - msg.Nack(false, false) - } - } else { - // 处理成功,手动确认 - if !c.autoAck { - msg.Ack(false) + for { + select { + case <-ctx.Done(): + // Context 取消,退出 + g.Log().Infof(ctx, "Worker %d 收到停止信号,正在退出", workerID) + return + case msg, ok := <-msgs: + if !ok { + // Channel 关闭,退出 + g.Log().Infof(ctx, "Worker %d 消息通道已关闭,退出", workerID) + return } - g.Log().Debugf(ctx, "Worker %d 处理消息成功", workerID) + // 处理消息 + err := c.handler(ctx, msg.Body) + + if err != nil { + g.Log().Errorf(ctx, "Worker %d 处理消息失败: %v", workerID, err) + + // 如果不是自动确认,需要手动 Nack + if !c.autoAck { + // requeue=false: 不重新入队,进入死信队列 + msg.Nack(false, false) + } + } else { + // 处理成功,手动确认 + if !c.autoAck { + msg.Ack(false) + } + + g.Log().Debugf(ctx, "Worker %d 处理消息成功", workerID) + } } } - - g.Log().Debugf(ctx, "Worker %d 已停止", workerID) } // StartTypedConsumer 启动类型化消费者(自动反序列化) @@ -163,3 +178,12 @@ func StartTypedConsumer[T any]( consumer := NewConsumer(queue, wrappedHandler, opts...) return consumer.Start(ctx) } + +// Stop 停止消费者 +func (c *Consumer) Stop(ctx context.Context) { + if c.cancel != nil { + g.Log().Infof(ctx, "正在停止消费者: queue=%s", c.queue) + c.cancel() + c.cancel = nil + } +} From 17cc8a371ae9e7d82b124b1d7437ba348d283adb Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Wed, 3 Dec 2025 16:39:55 +0800 Subject: [PATCH 042/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=AD=97=E7=AC=A6?= =?UTF-8?q?=E4=B8=B2=E6=B8=85=E7=90=86=E5=92=8Ctoken=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jaeger/jaeger.go | 13 ++++++++++++- mongo/mongo.go | 14 -------------- utils/utils.go | 23 ++++++++++++++++++++--- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/jaeger/jaeger.go b/jaeger/jaeger.go index 9116cc4..31e50e4 100644 --- a/jaeger/jaeger.go +++ b/jaeger/jaeger.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "strconv" + "strings" "github.com/gogf/gf/contrib/trace/otlphttp/v2" "github.com/gogf/gf/v2/frame/g" @@ -28,7 +29,17 @@ func NewTracer(r *ghttp.Request) { defer span.End() span.SetAttributes(attribute.String("request", getParams(r))) r.Middleware.Next() - span.SetAttributes(attribute.String("response", r.Response.BufferString())) + + // 清理响应字符串,确保 UTF-8 有效(处理二进制数据如 ZIP 文件) + response := r.Response.BufferString() + cleanResponse := strings.ToValidUTF8(response, "") + + // 如果响应太大(如文件下载),只记录前 1000 字符 + if len(cleanResponse) > 1000 { + cleanResponse = cleanResponse[:1000] + "... (truncated)" + } + + span.SetAttributes(attribute.String("response", cleanResponse)) } func getParams(r *ghttp.Request) string { params := map[string]interface{}{} diff --git a/mongo/mongo.go b/mongo/mongo.go index b7928ff..92f0c2a 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -56,20 +56,6 @@ func Find(ctx context.Context, filter bson.M, result interface{}, collection str return } -// FindWithoutTenant 查询多条记录(不过滤租户,用于导出等场景) -func FindWithoutTenant(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOptions]) (err error) { - if err = utils.ValidStructPtr(result); err != nil { - return - } - // 不添加 tenantId 过滤条件 - cur, err := db.Collection(collection).Find(ctx, filter, opts...) - if err != nil { - return - } - err = cur.All(ctx, result) - return -} - // FindOne 查询1条记录 func FindOne(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOneOptions]) (err error) { if len(filter) == 0 { diff --git a/utils/utils.go b/utils/utils.go index c2d049f..1e45148 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -3,6 +3,9 @@ package utils import ( "context" "fmt" + "reflect" + "time" + "gitee.com/red-future---jilin-g/common/do" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/errors/gcode" @@ -10,8 +13,6 @@ import ( "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/util/gconv" "github.com/tiger1103/gfast-token/gftoken" - "reflect" - "time" ) // ValidStructPtr 验证是否为结构体指针 @@ -69,7 +70,23 @@ func GetUserInfo(ctx context.Context) (user do.User, err error) { Address: redisAddr, Db: 1, })) - data, _ := gft.ParseToken(g.RequestFromCtx(ctx)) + + // 解析 token,不再忽略错误 + data, err := gft.ParseToken(g.RequestFromCtx(ctx)) + if err != nil { + return user, gerror.Wrap(err, "token 解析失败") + } + + // 检查 data 是否为 nil + if data == nil { + return user, gerror.New("token 数据为空") + } + + // 检查 data.Data 是否为 nil + if data.Data == nil { + return user, gerror.New("用户信息为空") + } + dataMap := gconv.Map(data.Data) user.UserName = dataMap["userName"] user.TenantId = dataMap["tenantId"] From 0541ac776a0613c2f5596b9ef7c1414ace6c0633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 3 Dec 2025 12:04:20 +0800 Subject: [PATCH 043/225] =?UTF-8?q?consul=E5=9C=B0=E5=9D=80=E5=88=86?= =?UTF-8?q?=E9=85=8D=E4=BC=98=E5=85=88=E8=B0=83=E7=94=A8gateway=E5=90=8CIP?= =?UTF-8?q?=E7=9A=84=E5=BE=AE=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/consul.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/consul/consul.go b/consul/consul.go index 8b1c00a..a2a61cf 100644 --- a/consul/consul.go +++ b/consul/consul.go @@ -49,19 +49,18 @@ func GetInstanceAddr(ctx context.Context, name string) (addr string, err error) err = errors.New("获取customerService服务实例失败!") return } - maxService := grand.N(0, len(service)-1) - maxEndpoints := grand.N(0, len(service[maxService].GetEndpoints())-1) //优先使用gateway同IP的服务实例 addr, err = getLocalIP() if err != nil { return } - for _, point := range service[maxService].GetEndpoints() { - if point.Host() == addr { - addr = point.String() + for _, s := range service { + if s.GetEndpoints()[0].Host() == addr { + addr = s.GetEndpoints()[0].String() return } } - addr = service[maxService].GetEndpoints()[maxEndpoints].String() + maxService := grand.N(0, len(service)-1) + addr = service[maxService].GetEndpoints()[0].String() return } From eed57aa41a41aff684220630ee02d838abd9d9ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 3 Dec 2025 13:58:37 +0800 Subject: [PATCH 044/225] =?UTF-8?q?token=E5=A4=B1=E6=95=88=E9=94=99?= =?UTF-8?q?=E8=AF=AF=E8=BF=94=E5=9B=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 redis/redis.go diff --git a/redis/redis.go b/redis/redis.go new file mode 100644 index 0000000..5f5b962 --- /dev/null +++ b/redis/redis.go @@ -0,0 +1,12 @@ +package redis + +import ( + "github.com/gogf/gf/v2/database/gredis" + "github.com/gogf/gf/v2/frame/g" +) + +var RedisClient *gredis.Redis + +func init() { + RedisClient = g.Redis() +} From 20f9fe12b042969c78d60c876646b42f4a8856c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 3 Dec 2025 14:34:27 +0800 Subject: [PATCH 045/225] =?UTF-8?q?gateway=E5=88=86=E9=85=8D=E5=AE=9E?= =?UTF-8?q?=E4=BE=8B=E7=AD=96=E7=95=A5=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/consul.go | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/consul/consul.go b/consul/consul.go index a2a61cf..638ec2f 100644 --- a/consul/consul.go +++ b/consul/consul.go @@ -41,7 +41,15 @@ func getLocalIP() (string, error) { return "", fmt.Errorf("无法找到本地IP地址") } - +func getInstanceAddrByIp(ctx context.Context, ip string, services []gsvc.Service) (addr string) { + for _, s := range services { + if s.GetEndpoints()[0].Host() == addr { + addr = s.GetEndpoints()[0].String() + return + } + } + return +} func GetInstanceAddr(ctx context.Context, name string) (addr string, err error) { watch, err := gsvc.GetRegistry().Watch(ctx, name) service, err := watch.Proceed() @@ -49,17 +57,21 @@ func GetInstanceAddr(ctx context.Context, name string) (addr string, err error) err = errors.New("获取customerService服务实例失败!") return } - //优先使用gateway同IP的服务实例 + //优先使用客户端IP获取实例(前后端在同一台机器调试) + addr = getInstanceAddrByIp(ctx, g.RequestFromCtx(ctx).GetRemoteIp(), service) + if !g.IsEmpty(addr) { + return + } + //优先使用gateway同IP的服务实例(前后端不同机器调试) addr, err = getLocalIP() if err != nil { return } - for _, s := range service { - if s.GetEndpoints()[0].Host() == addr { - addr = s.GetEndpoints()[0].String() - return - } + addr = getInstanceAddrByIp(ctx, addr, service) + if !g.IsEmpty(addr) { + return } + //随机获取一个服务实例 maxService := grand.N(0, len(service)-1) addr = service[maxService].GetEndpoints()[0].String() return From b5fa17e58dcde7f3b6b4587f407080ce70aaac6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 3 Dec 2025 14:40:03 +0800 Subject: [PATCH 046/225] =?UTF-8?q?gateway=E5=88=86=E9=85=8D=E5=AE=9E?= =?UTF-8?q?=E4=BE=8B=E7=AD=96=E7=95=A5=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/consul.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consul/consul.go b/consul/consul.go index 638ec2f..415a137 100644 --- a/consul/consul.go +++ b/consul/consul.go @@ -58,7 +58,7 @@ func GetInstanceAddr(ctx context.Context, name string) (addr string, err error) return } //优先使用客户端IP获取实例(前后端在同一台机器调试) - addr = getInstanceAddrByIp(ctx, g.RequestFromCtx(ctx).GetRemoteIp(), service) + addr = getInstanceAddrByIp(ctx, g.RequestFromCtx(ctx).GetClientIp(), service) if !g.IsEmpty(addr) { return } From 4300257195f655c8a8eb71f4c00e8f1136b825c5 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Wed, 3 Dec 2025 17:22:38 +0800 Subject: [PATCH 047/225] =?UTF-8?q?=E6=8F=90=E4=BA=A4=20utils=20,=20=20?= =?UTF-8?q?=E5=8E=BB=E6=8E=89=E8=B0=83=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/utils.go b/utils/utils.go index 1e45148..78a176a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -71,7 +71,7 @@ func GetUserInfo(ctx context.Context) (user do.User, err error) { Db: 1, })) - // 解析 token,不再忽略错误 + // 解析 token data, err := gft.ParseToken(g.RequestFromCtx(ctx)) if err != nil { return user, gerror.Wrap(err, "token 解析失败") From 3f4ac12f52848505bc9157f53487f8d4667f2364 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Thu, 4 Dec 2025 17:39:31 +0800 Subject: [PATCH 048/225] redis --- http/http.go | 9 ++- ragflow/worker_pool.go | 165 +++++++++++++++++++++++++++++++++++++++++ redis/redis.go | 162 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 332 insertions(+), 4 deletions(-) create mode 100644 ragflow/worker_pool.go diff --git a/http/http.go b/http/http.go index 5997f90..0023e64 100644 --- a/http/http.go +++ b/http/http.go @@ -4,6 +4,11 @@ import ( "context" "errors" "fmt" + "net/http" + "reflect" + "regexp" + "strings" + _ "gitee.com/red-future---jilin-g/common/consul" "gitee.com/red-future---jilin-g/common/jaeger" "gitee.com/red-future---jilin-g/common/utils" @@ -13,10 +18,6 @@ import ( "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/util/gconv" - "net/http" - "reflect" - "regexp" - "strings" ) type ResponseEmpty struct { diff --git a/ragflow/worker_pool.go b/ragflow/worker_pool.go new file mode 100644 index 0000000..682786f --- /dev/null +++ b/ragflow/worker_pool.go @@ -0,0 +1,165 @@ +package ragflow + +import ( + "context" + + "gitee.com/red-future---jilin-g/common/redis" + "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/os/grpool" +) + +// WorkerPool RAGFlow 请求处理协程池 +type WorkerPool struct { + pool *grpool.Pool + size int +} + +// NewWorkerPool 创建协程池 +// 参数: +// - size: 协程池大小,建议设置为 CPU 核心数的 2-4 倍 +// +// 返回: +// - *WorkerPool: 协程池实例 +// - error: 创建失败时返回错误 +func NewWorkerPool(size int) (*WorkerPool, error) { + if size <= 0 { + return nil, gerror.New("协程池大小必须大于0") + } + + pool := grpool.New(size) + + return &WorkerPool{ + pool: pool, + size: size, + }, nil +} + +// Submit 提交任务到协程池 +// 参数: +// - ctx: 上下文 +// - task: 要执行的任务函数 +// +// 返回:error 提交失败时返回错误 +func (w *WorkerPool) Submit(ctx context.Context, task func(ctx context.Context)) error { + return w.pool.Add(ctx, func(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + glog.Errorf(ctx, "协程池任务执行 panic: %v", r) + } + }() + + task(ctx) + }) +} + +// Size 获取协程池大小 +func (w *WorkerPool) Size() int { + return w.size +} + +// Jobs 获取当前等待执行的任务数量 +func (w *WorkerPool) Jobs() int { + return w.pool.Jobs() +} + +// Close 关闭协程池 +func (w *WorkerPool) Close() { + w.pool.Close() +} + +// WorkerStats 协程池统计信息 +type WorkerStats struct { + PoolSize int // 协程池大小 + Jobs int // 等待执行的任务数 +} + +// Stats 获取协程池统计信息 +func (w *WorkerPool) Stats() WorkerStats { + return WorkerStats{ + PoolSize: w.size, + Jobs: w.pool.Jobs(), + } +} + +// PrintStats 打印协程池统计信息 +func (w *WorkerPool) PrintStats(ctx context.Context) { + stats := w.Stats() + glog.Infof(ctx, "协程池统计 - 池大小: %d, 等待任务: %d", stats.PoolSize, stats.Jobs) +} + +// QueueProcessor 队列处理器,从 Redis 队列中取出任务并提交到协程池 +type QueueProcessor struct { + pool *WorkerPool + queueKey string + timeout int + stopChan chan struct{} + handleFunc func(ctx context.Context, message string) error +} + +// NewQueueProcessor 创建队列处理器 +// 参数: +// - pool: 协程池 +// - queueKey: Redis 队列键名 +// - timeout: 从队列取消息的超时时间(秒) +// - handleFunc: 消息处理函数 +func NewQueueProcessor(pool *WorkerPool, queueKey string, timeout int, handleFunc func(ctx context.Context, message string) error) *QueueProcessor { + return &QueueProcessor{ + pool: pool, + queueKey: queueKey, + timeout: timeout, + stopChan: make(chan struct{}), + handleFunc: handleFunc, + } +} + +// Start 启动队列处理器 +// 会阻塞运行,持续从 Redis 队列中取出消息并提交到协程池处理 +func (q *QueueProcessor) Start(ctx context.Context) error { + glog.Infof(ctx, "队列处理器启动 - 队列: %s, 超时: %ds", q.queueKey, q.timeout) + + for { + select { + case <-q.stopChan: + glog.Info(ctx, "队列处理器收到停止信号") + return nil + default: + // 从 Redis 队列中取出消息 + message, err := q.fetchMessage(ctx) + if err != nil { + glog.Errorf(ctx, "从队列取消息失败: %v", err) + continue + } + + // 队列为空,继续等待 + if message == "" { + continue + } + + // 提交到协程池处理 + if err := q.submitTask(ctx, message); err != nil { + glog.Errorf(ctx, "提交任务到协程池失败: %v", err) + } + } + } +} + +// Stop 停止队列处理器 +func (q *QueueProcessor) Stop() { + close(q.stopChan) +} + +// fetchMessage 从 Redis 队列中取出消息 +func (q *QueueProcessor) fetchMessage(ctx context.Context) (string, error) { + // 调用 Redis 队列的 PopFromQueue 方法从队列中取出消息 + return redis.PopFromQueue(ctx, q.queueKey, q.timeout) +} + +// submitTask 将消息处理任务提交到协程池 +func (q *QueueProcessor) submitTask(ctx context.Context, message string) error { + return q.pool.Submit(ctx, func(ctx context.Context) { + if err := q.handleFunc(ctx, message); err != nil { + glog.Errorf(ctx, "处理消息失败: %v, 消息: %s", err, message) + } + }) +} diff --git a/redis/redis.go b/redis/redis.go index 5f5b962..fb197dd 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -1,6 +1,9 @@ package redis import ( + "context" + "time" + "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" ) @@ -10,3 +13,162 @@ var RedisClient *gredis.Redis func init() { RedisClient = g.Redis() } + +// 队列操作常量 +const ( + // RAGFlow 请求队列 Key + RAGFlowRequestQueueKey = "ragflow:request:queue" + // 会话最后活跃时间 Key 前缀 + SessionLastActiveKeyPrefix = "ragflow:session:" +) + +// PushToQueue 将消息推入队列(LPUSH从左侧入队) +// 用于 Controller 层将 RAGFlow 请求推入队列,由后台 Goroutine 处理 +// 参数: +// - queueKey: 队列键名 +// - message: 要推入的消息内容 +// +// 返回:error 推入失败时返回错误 +func PushToQueue(ctx context.Context, queueKey string, message interface{}) error { + _, err := RedisClient.Do(ctx, "LPUSH", queueKey, message) + return err +} + +// PopFromQueue 从队列中阻塞取出消息(BRPOP从右侧出队) +// 后台 Goroutine 使用此方法从队列中取出请求进行处理 +// 参数: +// - queueKey: 队列键名 +// - timeout: 阻塞超时时间(秒),0表示永久阻塞 +// +// 返回: +// - string: 取出的消息内容,超时或队列为空返回空字符串 +// - error: 操作失败时返回错误 +func PopFromQueue(ctx context.Context, queueKey string, timeout int) (string, error) { + result, err := RedisClient.Do(ctx, "BRPOP", queueKey, timeout) + if err != nil { + return "", err + } + + // BRPOP 返回 [key, value],我们需要取 value + if result == nil { + return "", nil // 超时返回空 + } + + // GoFrame gredis 返回的是 *gvar.Var 类型 + arr := result.Strings() + if len(arr) >= 2 { + return arr[1], nil // arr[0] 是 key,arr[1] 是 value + } + + return "", nil +} + +// GetQueueLength 获取队列当前长度 +// 用于监控队列积压情况 +// 参数: +// - queueKey: 队列键名 +// +// 返回: +// - int64: 队列中消息数量 +// - error: 操作失败时返回错误 +func GetQueueLength(ctx context.Context, queueKey string) (int64, error) { + result, err := RedisClient.Do(ctx, "LLEN", queueKey) + if err != nil { + return 0, err + } + return result.Int64(), nil +} + +// SetSessionLastActive 设置用户最后活跃时间 +// 用于控制是否发送追问:用户回复后更新活跃时间,避免重复追问 +// 过期时间:2小时,超过2小时未活跃的记录会自动删除 +// 参数: +// - userId: 用户ID +// +// 返回:error 设置失败时返回错误 +func SetSessionLastActive(ctx context.Context, userId string) error { + key := SessionLastActiveKeyPrefix + userId + ":last_active" + timestamp := time.Now().Unix() + + // 设置过期时间为 2 小时 + return RedisClient.SetEX(ctx, key, timestamp, 7200) +} + +// GetSessionLastActive 获取用户最后活跃时间 +// 参数: +// - userId: 用户ID +// +// 返回: +// - int64: Unix时间戳,未找到返回0 +// - error: 操作失败时返回错误 +func GetSessionLastActive(ctx context.Context, userId string) (int64, error) { + key := SessionLastActiveKeyPrefix + userId + ":last_active" + result, err := RedisClient.Get(ctx, key) + if err != nil { + return 0, err + } + + if result.IsNil() { + return 0, nil // 未找到返回 0 + } + + return result.Int64(), nil +} + +// IsUserActive 检查用户是否在指定时间范围内活跃过 +// 用于追问逻辑:如果用户最近活跃过,则不发送追问消息 +// 参数: +// - userId: 用户ID +// - seconds: 时间范围(秒),例如传入300表示检查5分钟内是否活跃 +// +// 返回: +// - bool: true表示用户在指定时间内活跃过 +// - error: 操作失败时返回错误 +func IsUserActive(ctx context.Context, userId string, seconds int64) (bool, error) { + lastActive, err := GetSessionLastActive(ctx, userId) + if err != nil { + return false, err + } + + if lastActive == 0 { + return false, nil // 未找到记录,视为不活跃 + } + + now := time.Now().Unix() + return (now - lastActive) < seconds, nil +} + +// SetSessionCache 缓存用户的 RAGFlow Session ID +// 避免每次请求都创建新 Session,提高性能 +// 过期时间:7天,超过7天未使用的Session会自动清理 +// 参数: +// - userId: 用户ID +// - sessionId: RAGFlow返回的Session ID +// +// 返回:error 设置失败时返回错误 +func SetSessionCache(ctx context.Context, userId, sessionId string) error { + key := SessionLastActiveKeyPrefix + userId + ":session_id" + return RedisClient.SetEX(ctx, key, sessionId, 7*24*3600) +} + +// GetSessionCache 获取缓存的 RAGFlow Session ID +// 如果缓存中存在则直接使用,不存在则需要创建新Session +// 参数: +// - userId: 用户ID +// +// 返回: +// - string: Session ID,未找到返回空字符串 +// - error: 操作失败时返回错误 +func GetSessionCache(ctx context.Context, userId string) (string, error) { + key := SessionLastActiveKeyPrefix + userId + ":session_id" + result, err := RedisClient.Get(ctx, key) + if err != nil { + return "", err + } + + if result.IsNil() { + return "", nil + } + + return result.String(), nil +} From 25452e04b7a1c1cf4a6428883ce0fa4dc7d314ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 4 Dec 2025 17:38:34 +0800 Subject: [PATCH 049/225] =?UTF-8?q?mongodb=E5=9F=BA=E7=A1=80=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E6=96=B9=E6=B3=95=E5=A2=9E=E5=8A=A0redis=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consts/redis_key.go | 7 +++ mongo/mongo.go | 135 +++++++++++++++++++++++++++++++++++++++++++- utils/utils.go | 20 +++++++ 3 files changed, 159 insertions(+), 3 deletions(-) create mode 100644 consts/redis_key.go diff --git a/consts/redis_key.go b/consts/redis_key.go new file mode 100644 index 0000000..aef970f --- /dev/null +++ b/consts/redis_key.go @@ -0,0 +1,7 @@ +package consts + +const CleanList = "list:tenantId-%v:collection-%s:*" +const CleanCount = "count:tenantId-%v:collection-%s:*" +const List = "list:tenantId-%v:collection-%s:filter:%s:options:%s" +const Count = "count:tenantId-%v:collection-%s:filter:%s" +const One = "one:tenantId-%v:collection-%s:filter:%s:*" diff --git a/mongo/mongo.go b/mongo/mongo.go index 92f0c2a..834223c 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -2,9 +2,13 @@ package mongo import ( "context" + "errors" + "fmt" "strings" "time" + "gitee.com/red-future---jilin-g/common/consts" + "gitee.com/red-future---jilin-g/common/redis" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" @@ -36,23 +40,88 @@ func init() { } db = client.Database(dbName) } +func listOptionsToMap(ctx context.Context, opts ...options.Lister[options.FindOptions]) (m map[string]interface{}) { + // 输出opts参数中的值 + m = make(map[string]interface{}) + for _, opt := range opts { + var findOpts options.FindOptions + optFuncs := opt.List() + for _, fn := range optFuncs { + fn(&findOpts) + } + if findOpts.Limit != nil { + m["limit"] = *findOpts.Limit + } + if findOpts.Skip != nil { + m["skip"] = *findOpts.Skip + } + if findOpts.Sort != nil { + m["sort"] = findOpts.Sort + } + if findOpts.Projection != nil { + m["projection"] = findOpts.Projection + } + } + m = utils.OrderMap(m) + return +} +func oneOptionsToMap(ctx context.Context, opts ...options.Lister[options.FindOneOptions]) (m map[string]interface{}) { + // 输出opts参数中的值 + m = make(map[string]interface{}) + for _, opt := range opts { + var findOpts options.FindOneOptions + optFuncs := opt.List() + for _, fn := range optFuncs { + fn(&findOpts) + } + if findOpts.Skip != nil { + m["skip"] = *findOpts.Skip + } + if findOpts.Sort != nil { + m["sort"] = findOpts.Sort + } + if findOpts.Projection != nil { + m["projection"] = findOpts.Projection + } + } + m = utils.OrderMap(m) + return +} // Find 查询多条记录 func Find(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOptions]) (err error) { if err = utils.ValidStructPtr(result); err != nil { return } - filter["isDeleted"] = false user, err := utils.GetUserInfo(ctx) if err != nil { return } + filter["isDeleted"] = false filter["tenantId"] = user.TenantId + filterMap := utils.OrderMap(filter) + optsMap := listOptionsToMap(ctx, opts...) + redisKey := fmt.Sprintf(consts.List, user.TenantId, collection, gconv.String(filterMap), gconv.String(optsMap)) + resultStr, err := redis.RedisClient.Get(ctx, redisKey) + if err != nil { + return + } + if !g.IsEmpty(resultStr) { + err = gconv.Scan(resultStr, result) + if err != nil { + return err + } + return + } cur, err := db.Collection(collection).Find(ctx, filter, opts...) if err != nil { return } err = cur.All(ctx, result) + err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) + if err != nil { + return err + } return } @@ -65,17 +134,35 @@ func FindOne(ctx context.Context, filter bson.M, result interface{}, collection if err = utils.ValidStructPtr(result); err != nil { return } - filter["isDeleted"] = false user, err := utils.GetUserInfo(ctx) if err != nil { return } + filter["isDeleted"] = false filter["tenantId"] = user.TenantId + filterMap := utils.OrderMap(filter) + optsMap := oneOptionsToMap(ctx, opts...) + redisKey := fmt.Sprintf(consts.One, user.TenantId, collection, gconv.String(filterMap), gconv.String(optsMap)) + resultStr, err := redis.RedisClient.Get(ctx, redisKey) + if err != nil { + return + } + if !g.IsEmpty(resultStr) { + err = gconv.Scan(resultStr, result) + if err != nil { + return err + } + return + } cur := db.Collection(collection).FindOne(ctx, filter, opts...) err = cur.Decode(result) - if err == mongo.ErrNoDocuments { + if errors.Is(err, mongo.ErrNoDocuments) { err = nil } + err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) + if err != nil { + return err + } return } @@ -95,6 +182,14 @@ func Delete(ctx context.Context, filter bson.M, collection string, opts ...optio return } count = r.DeletedCount + _, err = redis.RedisClient.Del(ctx, fmt.Sprintf(consts.CleanList, user.TenantId, collection)) + if err != nil { + return + } + _, err = redis.RedisClient.Del(ctx, fmt.Sprintf(consts.CleanCount, user.TenantId, collection)) + if err != nil { + return + } return } @@ -118,6 +213,14 @@ func Update(ctx context.Context, filter bson.M, update bson.M, collection string if err != nil { return } + _, err = redis.RedisClient.Del(ctx, fmt.Sprintf(consts.CleanList, user.TenantId, collection)) + if err != nil { + return + } + _, err = redis.RedisClient.Del(ctx, fmt.Sprintf(consts.CleanCount, user.TenantId, collection)) + if err != nil { + return + } return } @@ -144,13 +247,39 @@ func Insert(ctx context.Context, documents []interface{}, collection string, opt return } ids = r.InsertedIDs + _, err = redis.RedisClient.Del(ctx, fmt.Sprintf(consts.CleanList, user.TenantId, collection)) + if err != nil { + return + } + _, err = redis.RedisClient.Del(ctx, fmt.Sprintf(consts.CleanCount, user.TenantId, collection)) + if err != nil { + return + } return } // Count 查询总数 func Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { + user, err := utils.GetUserInfo(ctx) + if err != nil { + return + } filter["isDeleted"] = false + filterMap := utils.OrderMap(filter) + redisKey := fmt.Sprintf(consts.Count, user.TenantId, collection, gconv.String(filterMap)) + resultStr, err := redis.RedisClient.Get(ctx, redisKey) + if err != nil { + return + } + if !g.IsEmpty(resultStr) { + count = gconv.Int64(resultStr) + return + } // 调用驱动的 CountDocuments,在数据库端执行的 count, err = db.Collection(collection).CountDocuments(ctx, filter) + err = redis.RedisClient.SetEX(ctx, redisKey, count, int64(time.Hour)) + if err != nil { + return + } return } diff --git a/utils/utils.go b/utils/utils.go index 78a176a..d539f81 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "reflect" + "sort" "time" "gitee.com/red-future---jilin-g/common/do" @@ -92,3 +93,22 @@ func GetUserInfo(ctx context.Context) (user do.User, err error) { user.TenantId = dataMap["tenantId"] return } +func OrderMap(m map[string]interface{}) map[string]interface{} { + // 提取所有key + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + + // 使用标准排序算法对key进行排序 + // 使用strings.Sort确保排序结果永远一致 + sort.Strings(keys) + + // 创建有序map + orderedMap := make(map[string]interface{}, len(m)) + for _, k := range keys { + orderedMap[k] = m[k] + } + + return orderedMap +} From a78bf2676bc3689de532799f87ad7f8a8644c851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 5 Dec 2025 09:51:36 +0800 Subject: [PATCH 050/225] =?UTF-8?q?mongodb=E5=9F=BA=E7=A1=80=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E6=96=B9=E6=B3=95=E5=A2=9E=E5=8A=A0redis=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 57 +++++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 834223c..8a43cf2 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -98,7 +98,6 @@ func Find(ctx context.Context, filter bson.M, result interface{}, collection str return } filter["isDeleted"] = false - filter["tenantId"] = user.TenantId filterMap := utils.OrderMap(filter) optsMap := listOptionsToMap(ctx, opts...) redisKey := fmt.Sprintf(consts.List, user.TenantId, collection, gconv.String(filterMap), gconv.String(optsMap)) @@ -113,6 +112,8 @@ func Find(ctx context.Context, filter bson.M, result interface{}, collection str } return } + + filter["tenantId"] = user.TenantId cur, err := db.Collection(collection).Find(ctx, filter, opts...) if err != nil { return @@ -139,7 +140,6 @@ func FindOne(ctx context.Context, filter bson.M, result interface{}, collection return } filter["isDeleted"] = false - filter["tenantId"] = user.TenantId filterMap := utils.OrderMap(filter) optsMap := oneOptionsToMap(ctx, opts...) redisKey := fmt.Sprintf(consts.One, user.TenantId, collection, gconv.String(filterMap), gconv.String(optsMap)) @@ -154,6 +154,7 @@ func FindOne(ctx context.Context, filter bson.M, result interface{}, collection } return } + filter["tenantId"] = user.TenantId cur := db.Collection(collection).FindOne(ctx, filter, opts...) err = cur.Decode(result) if errors.Is(err, mongo.ErrNoDocuments) { @@ -165,6 +166,31 @@ func FindOne(ctx context.Context, filter bson.M, result interface{}, collection } return } +func cleanRedis(ctx context.Context, tenantId interface{}, collection string) (err error) { + listKeys := fmt.Sprintf(consts.CleanList, tenantId, collection) + keys, err := redis.RedisClient.Keys(ctx, listKeys) + if err != nil { + return + } + for _, key := range keys { + _, err = redis.RedisClient.Del(ctx, key) + if err != nil { + return + } + } + countKeys := fmt.Sprintf(consts.CleanCount, tenantId, collection) + keys, err = redis.RedisClient.Keys(ctx, countKeys) + if err != nil { + return + } + for _, key := range keys { + _, err = redis.RedisClient.Del(ctx, key) + if err != nil { + return + } + } + return +} // Delete 删除记录 func Delete(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.DeleteManyOptions]) (count int64, err error) { @@ -182,14 +208,7 @@ func Delete(ctx context.Context, filter bson.M, collection string, opts ...optio return } count = r.DeletedCount - _, err = redis.RedisClient.Del(ctx, fmt.Sprintf(consts.CleanList, user.TenantId, collection)) - if err != nil { - return - } - _, err = redis.RedisClient.Del(ctx, fmt.Sprintf(consts.CleanCount, user.TenantId, collection)) - if err != nil { - return - } + err = cleanRedis(ctx, user.TenantId, collection) return } @@ -213,14 +232,7 @@ func Update(ctx context.Context, filter bson.M, update bson.M, collection string if err != nil { return } - _, err = redis.RedisClient.Del(ctx, fmt.Sprintf(consts.CleanList, user.TenantId, collection)) - if err != nil { - return - } - _, err = redis.RedisClient.Del(ctx, fmt.Sprintf(consts.CleanCount, user.TenantId, collection)) - if err != nil { - return - } + err = cleanRedis(ctx, user.TenantId, collection) return } @@ -247,14 +259,7 @@ func Insert(ctx context.Context, documents []interface{}, collection string, opt return } ids = r.InsertedIDs - _, err = redis.RedisClient.Del(ctx, fmt.Sprintf(consts.CleanList, user.TenantId, collection)) - if err != nil { - return - } - _, err = redis.RedisClient.Del(ctx, fmt.Sprintf(consts.CleanCount, user.TenantId, collection)) - if err != nil { - return - } + err = cleanRedis(ctx, user.TenantId, collection) return } From 01999b56a4329dff832da3c0c162cb7baac45aa7 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Fri, 5 Dec 2025 11:44:07 +0800 Subject: [PATCH 051/225] =?UTF-8?q?redis=E4=BD=BF=E7=94=A8stream=E6=96=B9?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/worker_pool.go | 90 ++++++++----- redis/redis.go | 286 +++++++++++++++++++++++++++++++++-------- 2 files changed, 285 insertions(+), 91 deletions(-) diff --git a/ragflow/worker_pool.go b/ragflow/worker_pool.go index 682786f..8c46c47 100644 --- a/ragflow/worker_pool.go +++ b/ragflow/worker_pool.go @@ -88,57 +88,70 @@ func (w *WorkerPool) PrintStats(ctx context.Context) { glog.Infof(ctx, "协程池统计 - 池大小: %d, 等待任务: %d", stats.PoolSize, stats.Jobs) } -// QueueProcessor 队列处理器,从 Redis 队列中取出任务并提交到协程池 +// QueueProcessor Stream 处理器,从 Redis Stream 中取出任务并提交到协程池 type QueueProcessor struct { - pool *WorkerPool - queueKey string - timeout int - stopChan chan struct{} - handleFunc func(ctx context.Context, message string) error + pool *WorkerPool + streamKey string // Stream 键名 + groupName string // 消费者组名称 + consumerName string // 消费者名称 + timeout int64 // 阻塞超时时间(毫秒) + batchSize int64 // 每次读取的消息数量 + stopChan chan struct{} + handleFunc func(ctx context.Context, message map[string]interface{}) error } -// NewQueueProcessor 创建队列处理器 +// NewQueueProcessor 创建 Stream 处理器 // 参数: // - pool: 协程池 -// - queueKey: Redis 队列键名 -// - timeout: 从队列取消息的超时时间(秒) +// - streamKey: Redis Stream 键名 +// - groupName: 消费者组名称 +// - consumerName: 消费者名称(唯一标识) +// - timeout: 从 Stream 取消息的超时时间(毫秒) +// - batchSize: 每次读取的消息数量 // - handleFunc: 消息处理函数 -func NewQueueProcessor(pool *WorkerPool, queueKey string, timeout int, handleFunc func(ctx context.Context, message string) error) *QueueProcessor { +func NewQueueProcessor(pool *WorkerPool, streamKey, groupName, consumerName string, timeout int64, batchSize int64, handleFunc func(ctx context.Context, message map[string]interface{}) error) *QueueProcessor { return &QueueProcessor{ - pool: pool, - queueKey: queueKey, - timeout: timeout, - stopChan: make(chan struct{}), - handleFunc: handleFunc, + pool: pool, + streamKey: streamKey, + groupName: groupName, + consumerName: consumerName, + timeout: timeout, + batchSize: batchSize, + stopChan: make(chan struct{}), + handleFunc: handleFunc, } } -// Start 启动队列处理器 -// 会阻塞运行,持续从 Redis 队列中取出消息并提交到协程池处理 +// Start 启动 Stream 处理器 +// 会阻塞运行,持续从 Redis Stream 中取出消息并提交到协程池处理 func (q *QueueProcessor) Start(ctx context.Context) error { - glog.Infof(ctx, "队列处理器启动 - 队列: %s, 超时: %ds", q.queueKey, q.timeout) + glog.Infof(ctx, "Stream 处理器启动 - Stream: %s, 消费者组: %s, 消费者: %s, 超时: %dms", + q.streamKey, q.groupName, q.consumerName, q.timeout) for { select { case <-q.stopChan: - glog.Info(ctx, "队列处理器收到停止信号") + glog.Info(ctx, "Stream 处理器收到停止信号") return nil default: - // 从 Redis 队列中取出消息 - message, err := q.fetchMessage(ctx) + // 从 Redis Stream 中读取消息 + messages, err := q.fetchMessages(ctx) if err != nil { - glog.Errorf(ctx, "从队列取消息失败: %v", err) + glog.Errorf(ctx, "从 Stream 读取消息失败: %v", err) continue } - // 队列为空,继续等待 - if message == "" { + // 没有新消息,继续等待 + if len(messages) == 0 { continue } - // 提交到协程池处理 - if err := q.submitTask(ctx, message); err != nil { - glog.Errorf(ctx, "提交任务到协程池失败: %v", err) + // 处理每条消息 + for _, msg := range messages { + // 提交到协程池处理 + if err := q.submitTask(ctx, msg); err != nil { + glog.Errorf(ctx, "提交任务到协程池失败: %v, 消息ID: %s", err, msg.ID) + } } } } @@ -149,17 +162,26 @@ func (q *QueueProcessor) Stop() { close(q.stopChan) } -// fetchMessage 从 Redis 队列中取出消息 -func (q *QueueProcessor) fetchMessage(ctx context.Context) (string, error) { - // 调用 Redis 队列的 PopFromQueue 方法从队列中取出消息 - return redis.PopFromQueue(ctx, q.queueKey, q.timeout) +// fetchMessages 从 Redis Stream 中读取消息 +func (q *QueueProcessor) fetchMessages(ctx context.Context) ([]redis.StreamMessage, error) { + // 从消费者组读取消息 + return redis.ReadFromStream(ctx, q.streamKey, q.groupName, q.consumerName, q.batchSize, q.timeout) } // submitTask 将消息处理任务提交到协程池 -func (q *QueueProcessor) submitTask(ctx context.Context, message string) error { +func (q *QueueProcessor) submitTask(ctx context.Context, message redis.StreamMessage) error { return q.pool.Submit(ctx, func(ctx context.Context) { - if err := q.handleFunc(ctx, message); err != nil { - glog.Errorf(ctx, "处理消息失败: %v, 消息: %s", err, message) + // 处理消息 + if err := q.handleFunc(ctx, message.Values); err != nil { + glog.Errorf(ctx, "处理消息失败: %v, 消息ID: %s", err, message.ID) + return + } + + // 处理成功后确认消息 + if err := redis.AckMessage(ctx, q.streamKey, q.groupName, message.ID); err != nil { + glog.Errorf(ctx, "确认消息失败: %v, 消息ID: %s", err, message.ID) + } else { + glog.Debugf(ctx, "消息处理完成并已确认: %s", message.ID) } }) } diff --git a/redis/redis.go b/redis/redis.go index fb197dd..4e339dd 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -2,81 +2,250 @@ package redis import ( "context" + "strconv" "time" - "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" + "github.com/redis/go-redis/v9" ) -var RedisClient *gredis.Redis +var RedisClient *redis.Client func init() { - RedisClient = g.Redis() + // 从 GoFrame 配置读取 Redis 配置 + ctx := context.Background() + + // 读取 Redis 配置 + addr := g.Cfg().MustGet(ctx, "redis.default.address").String() + + password := g.Cfg().MustGet(ctx, "redis.default.pass", "").String() + db := g.Cfg().MustGet(ctx, "redis.default.db", 0).Int() + + // 读取超时配置 + dialTimeout := g.Cfg().MustGet(ctx, "redis.default.dialTimeout", "30s").Duration() + readTimeout := g.Cfg().MustGet(ctx, "redis.default.readTimeout", "30s").Duration() + writeTimeout := g.Cfg().MustGet(ctx, "redis.default.writeTimeout", "30s").Duration() + + // 创建 Redis 客户端 + RedisClient = redis.NewClient(&redis.Options{ + Addr: addr, + Password: password, + DB: db, + DialTimeout: dialTimeout, + ReadTimeout: readTimeout, + WriteTimeout: writeTimeout, + // 不设置 Protocol(让 go-redis 自动协商) + // Protocol: 2, + }) } -// 队列操作常量 +// Stream 和消费者组常量 const ( - // RAGFlow 请求队列 Key - RAGFlowRequestQueueKey = "ragflow:request:queue" + // RAGFlow 请求 Stream Key + RAGFlowRequestStreamKey = "ragflow:request:stream" + // RAGFlow 消费者组名称 + RAGFlowConsumerGroup = "ragflow:consumer:group" // 会话最后活跃时间 Key 前缀 SessionLastActiveKeyPrefix = "ragflow:session:" ) -// PushToQueue 将消息推入队列(LPUSH从左侧入队) -// 用于 Controller 层将 RAGFlow 请求推入队列,由后台 Goroutine 处理 -// 参数: -// - queueKey: 队列键名 -// - message: 要推入的消息内容 -// -// 返回:error 推入失败时返回错误 -func PushToQueue(ctx context.Context, queueKey string, message interface{}) error { - _, err := RedisClient.Do(ctx, "LPUSH", queueKey, message) - return err +// StreamMessage Redis Stream 消息结构 +type StreamMessage struct { + ID string // 消息ID(自动生成) + Values map[string]interface{} // 消息内容 } -// PopFromQueue 从队列中阻塞取出消息(BRPOP从右侧出队) -// 后台 Goroutine 使用此方法从队列中取出请求进行处理 +// InitStreamGroup 初始化 Stream 和消费者组 +// 在应用启动时调用一次,创建 Stream 和消费者组 +// 使用 GoFrame Do() 方法执行 XGROUP CREATE 命令 // 参数: -// - queueKey: 队列键名 -// - timeout: 阻塞超时时间(秒),0表示永久阻塞 +// - streamKey: Stream 键名 +// - groupName: 消费者组名称 +// +// 返回:error 初始化失败时返回错误 +func InitStreamGroup(ctx context.Context, streamKey, groupName string) error { + // 使用 XGroupCreateMkStream 创建消费者组 + // 如果 Stream 不存在会自动创建 (MKSTREAM) + // "0": 从 Stream 开头开始消费 + err := RedisClient.XGroupCreateMkStream(ctx, streamKey, groupName, "0").Err() + if err != nil { + // 如果组已存在,忽略 BUSYGROUP 错误 + if err.Error() == "BUSYGROUP Consumer Group name already exists" { + return nil + } + return err + } + return nil +} + +// AddToStream 将消息添加到 Stream +// 用于 Controller 层将 RAGFlow 请求推入 Stream +// 参数: +// - streamKey: Stream 键名 +// - values: 消息内容(键值对) // // 返回: -// - string: 取出的消息内容,超时或队列为空返回空字符串 -// - error: 操作失败时返回错误 -func PopFromQueue(ctx context.Context, queueKey string, timeout int) (string, error) { - result, err := RedisClient.Do(ctx, "BRPOP", queueKey, timeout) +// - string: 消息ID +// - error: 添加失败时返回错误 +func AddToStream(ctx context.Context, streamKey string, values map[string]interface{}) (string, error) { + // 使用 XAdd 添加消息到 Stream + messageID, err := RedisClient.XAdd(ctx, &redis.XAddArgs{ + Stream: streamKey, + Values: values, + }).Result() if err != nil { return "", err } - - // BRPOP 返回 [key, value],我们需要取 value - if result == nil { - return "", nil // 超时返回空 - } - - // GoFrame gredis 返回的是 *gvar.Var 类型 - arr := result.Strings() - if len(arr) >= 2 { - return arr[1], nil // arr[0] 是 key,arr[1] 是 value - } - - return "", nil + return messageID, nil } -// GetQueueLength 获取队列当前长度 -// 用于监控队列积压情况 +// ReadFromStream 从 Stream 读取消息(消费者组模式) +// 后台 Goroutine 使用此方法从 Stream 中取出请求进行处理 // 参数: -// - queueKey: 队列键名 +// - streamKey: Stream 键名 +// - groupName: 消费者组名称 +// - consumerName: 消费者名称(唯一标识) +// - count: 每次读取的消息数量 +// - blockMs: 阻塞时间(毫秒),0表示不阻塞 // // 返回: -// - int64: 队列中消息数量 +// - []StreamMessage: 消息列表 +// - error: 读取失败时返回错误 +func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName string, count int64, blockMs int64) ([]StreamMessage, error) { + // 使用 XReadGroup 从消费者组读取消息 + // ">" 表示读取未被消费的新消息(只获取新消息) + // 如果使用 "0" 或其他 ID,则返回 Pending 消息(未确认的消息) + streams, err := RedisClient.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: groupName, + Consumer: consumerName, + Streams: []string{streamKey, ">"}, // Stream名称 + 起始ID + Count: count, + Block: time.Duration(blockMs) * time.Millisecond, + }).Result() + + // 处理错误:超时或没有数据时返回 redis.Nil + if err != nil { + if err == redis.Nil { + // 超时或没有数据,返回空数组 + return []StreamMessage{}, nil + } + return nil, err + } + + // 解析返回的消息 + var messages []StreamMessage + for _, stream := range streams { + for _, msg := range stream.Messages { + messages = append(messages, StreamMessage{ + ID: msg.ID, + Values: msg.Values, + }) + } + } + + return messages, nil +} + +// AckMessage 确认消息已处理 +// 处理完消息后必须调用此方法确认,否则消息会保留在 Pending List (PEL) +// 确认后消息会从 PEL 中移除 +// 参数: +// - streamKey: Stream 键名 +// - groupName: 消费者组名称 +// - messageIDs: 要确认的消息ID列表 +// +// 返回:error 确认失败时返回错误 +func AckMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error { + // 使用 XAck 确认消息 + // 返回值是成功确认的消息数量 + count, err := RedisClient.XAck(ctx, streamKey, groupName, messageIDs...).Result() + if err != nil { + return err + } + // 可以检查 count 是否等于 len(messageIDs) + _ = count + return nil +} + +// GetStreamLength 获取 Stream 当前长度 +// 用于监控 Stream 消息积压情况 +// 参数: +// - streamKey: Stream 键名 +// +// 返回: +// - int64: Stream 中消息数量 // - error: 操作失败时返回错误 -func GetQueueLength(ctx context.Context, queueKey string) (int64, error) { - result, err := RedisClient.Do(ctx, "LLEN", queueKey) +func GetStreamLength(ctx context.Context, streamKey string) (int64, error) { + // 使用 XLen 获取 Stream 长度 + length, err := RedisClient.XLen(ctx, streamKey).Result() if err != nil { return 0, err } - return result.Int64(), nil + return length, nil +} + +// GetPendingMessages 获取待处理消息(未确认的消息) +// 用于监控和重试失败的消息 +// 参数: +// - streamKey: Stream 键名 +// - groupName: 消费者组名称 +// - start: 起始ID,"-" 表示最小ID +// - end: 结束ID,"+" 表示最大ID +// - count: 返回数量 +// +// 返回: +// - []redis.XPendingExt: Pending 消息列表 +// - error: 操作失败时返回错误 +func GetPendingMessages(ctx context.Context, streamKey, groupName string, start, end string, count int64) ([]redis.XPendingExt, error) { + // 使用 XPendingExt 获取详细的 Pending 消息 + pending, err := RedisClient.XPendingExt(ctx, &redis.XPendingExtArgs{ + Stream: streamKey, + Group: groupName, + Start: start, + End: end, + Count: count, + }).Result() + if err != nil { + return nil, err + } + return pending, nil +} + +// ClaimPendingMessage 认领超时的 Pending 消息 +// 当某个消费者故障后,其他消费者可以认领其未完成的消息 +// 参数: +// - streamKey: Stream 键名 +// - groupName: 消费者组名称 +// - consumerName: 新消费者名称 +// - minIdleTime: 消息空闲时间(毫秒),超过此时间才能被认领 +// - messageIDs: 要认领的消息ID列表 +// +// 返回: +// - []StreamMessage: 认领的消息列表 +// - error: 操作失败时返回错误 +func ClaimPendingMessage(ctx context.Context, streamKey, groupName, consumerName string, minIdleTime int64, messageIDs ...string) ([]StreamMessage, error) { + // 使用 XClaim 认领消息 + msgs, err := RedisClient.XClaim(ctx, &redis.XClaimArgs{ + Stream: streamKey, + Group: groupName, + Consumer: consumerName, + MinIdle: time.Duration(minIdleTime) * time.Millisecond, + Messages: messageIDs, + }).Result() + if err != nil { + return nil, err + } + + // 转换为 StreamMessage + var messages []StreamMessage + for _, msg := range msgs { + messages = append(messages, StreamMessage{ + ID: msg.ID, + Values: msg.Values, + }) + } + + return messages, nil } // SetSessionLastActive 设置用户最后活跃时间 @@ -91,7 +260,7 @@ func SetSessionLastActive(ctx context.Context, userId string) error { timestamp := time.Now().Unix() // 设置过期时间为 2 小时 - return RedisClient.SetEX(ctx, key, timestamp, 7200) + return RedisClient.Set(ctx, key, timestamp, 2*time.Hour).Err() } // GetSessionLastActive 获取用户最后活跃时间 @@ -103,16 +272,20 @@ func SetSessionLastActive(ctx context.Context, userId string) error { // - error: 操作失败时返回错误 func GetSessionLastActive(ctx context.Context, userId string) (int64, error) { key := SessionLastActiveKeyPrefix + userId + ":last_active" - result, err := RedisClient.Get(ctx, key) + result, err := RedisClient.Get(ctx, key).Result() + if err == redis.Nil { + return 0, nil // 未找到返回 0 + } if err != nil { return 0, err } - if result.IsNil() { - return 0, nil // 未找到返回 0 + // 将字符串转换为 int64 + timestamp, err := strconv.ParseInt(result, 10, 64) + if err != nil { + return 0, err } - - return result.Int64(), nil + return timestamp, nil } // IsUserActive 检查用户是否在指定时间范围内活跃过 @@ -148,7 +321,7 @@ func IsUserActive(ctx context.Context, userId string, seconds int64) (bool, erro // 返回:error 设置失败时返回错误 func SetSessionCache(ctx context.Context, userId, sessionId string) error { key := SessionLastActiveKeyPrefix + userId + ":session_id" - return RedisClient.SetEX(ctx, key, sessionId, 7*24*3600) + return RedisClient.Set(ctx, key, sessionId, 7*24*time.Hour).Err() } // GetSessionCache 获取缓存的 RAGFlow Session ID @@ -161,14 +334,13 @@ func SetSessionCache(ctx context.Context, userId, sessionId string) error { // - error: 操作失败时返回错误 func GetSessionCache(ctx context.Context, userId string) (string, error) { key := SessionLastActiveKeyPrefix + userId + ":session_id" - result, err := RedisClient.Get(ctx, key) + result, err := RedisClient.Get(ctx, key).Result() + if err == redis.Nil { + return "", nil // 未找到返回空字符串 + } if err != nil { return "", err } - if result.IsNil() { - return "", nil - } - - return result.String(), nil + return result, nil } From 70b8b7b1d0754a1892a010896a3604d823b350ec Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Fri, 5 Dec 2025 12:18:04 +0800 Subject: [PATCH 052/225] =?UTF-8?q?=E6=9B=B4=E6=94=B9=20redis=E6=96=B9?= =?UTF-8?q?=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 404 +++++++++++++++++++++++++------------------------ 1 file changed, 206 insertions(+), 198 deletions(-) diff --git a/redis/redis.go b/redis/redis.go index 4e339dd..2cd7840 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -2,41 +2,24 @@ package redis import ( "context" - "strconv" + "strings" "time" + "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" - "github.com/redis/go-redis/v9" + "github.com/gogf/gf/v2/util/gconv" ) -var RedisClient *redis.Client +// GRedisClient GoFrame gredis 客户端,统一使用 +var GRedisClient *gredis.Redis + +// RedisClient GRedisClient 的别名,保持向后兼容 +var RedisClient *gredis.Redis func init() { - // 从 GoFrame 配置读取 Redis 配置 - ctx := context.Background() - - // 读取 Redis 配置 - addr := g.Cfg().MustGet(ctx, "redis.default.address").String() - - password := g.Cfg().MustGet(ctx, "redis.default.pass", "").String() - db := g.Cfg().MustGet(ctx, "redis.default.db", 0).Int() - - // 读取超时配置 - dialTimeout := g.Cfg().MustGet(ctx, "redis.default.dialTimeout", "30s").Duration() - readTimeout := g.Cfg().MustGet(ctx, "redis.default.readTimeout", "30s").Duration() - writeTimeout := g.Cfg().MustGet(ctx, "redis.default.writeTimeout", "30s").Duration() - - // 创建 Redis 客户端 - RedisClient = redis.NewClient(&redis.Options{ - Addr: addr, - Password: password, - DB: db, - DialTimeout: dialTimeout, - ReadTimeout: readTimeout, - WriteTimeout: writeTimeout, - // 不设置 Protocol(让 go-redis 自动协商) - // Protocol: 2, - }) + // 初始化 GoFrame gredis 客户端 + GRedisClient = g.Redis() + RedisClient = GRedisClient // 别名指向同一个客户端 } // Stream 和消费者组常量 @@ -56,21 +39,14 @@ type StreamMessage struct { } // InitStreamGroup 初始化 Stream 和消费者组 -// 在应用启动时调用一次,创建 Stream 和消费者组 -// 使用 GoFrame Do() 方法执行 XGROUP CREATE 命令 -// 参数: -// - streamKey: Stream 键名 -// - groupName: 消费者组名称 -// -// 返回:error 初始化失败时返回错误 +// 使用 gredis Do() 方法执行 XGROUP CREATE 命令 func InitStreamGroup(ctx context.Context, streamKey, groupName string) error { - // 使用 XGroupCreateMkStream 创建消费者组 - // 如果 Stream 不存在会自动创建 (MKSTREAM) - // "0": 从 Stream 开头开始消费 - err := RedisClient.XGroupCreateMkStream(ctx, streamKey, groupName, "0").Err() + // XGROUP CREATE streamKey groupName 0 MKSTREAM + _, err := GRedisClient.Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") if err != nil { - // 如果组已存在,忽略 BUSYGROUP 错误 - if err.Error() == "BUSYGROUP Consumer Group name already exists" { + // 如果组已存在,忽略错误 + errStr := err.Error() + if strings.Contains(errStr, "BUSYGROUP") || strings.Contains(errStr, "already exists") { return nil } return err @@ -79,66 +55,94 @@ func InitStreamGroup(ctx context.Context, streamKey, groupName string) error { } // AddToStream 将消息添加到 Stream -// 用于 Controller 层将 RAGFlow 请求推入 Stream -// 参数: -// - streamKey: Stream 键名 -// - values: 消息内容(键值对) -// -// 返回: -// - string: 消息ID -// - error: 添加失败时返回错误 +// 使用 gredis Do() 方法执行 XADD 命令 func AddToStream(ctx context.Context, streamKey string, values map[string]interface{}) (string, error) { - // 使用 XAdd 添加消息到 Stream - messageID, err := RedisClient.XAdd(ctx, &redis.XAddArgs{ - Stream: streamKey, - Values: values, - }).Result() + // XADD streamKey * field1 value1 field2 value2 ... + args := []interface{}{streamKey, "*"} // "*" 自动生成ID + for key, val := range values { + args = append(args, key, val) + } + + result, err := GRedisClient.Do(ctx, "XADD", args...) if err != nil { return "", err } + + // 返回消息ID + messageID := result.String() return messageID, nil } // ReadFromStream 从 Stream 读取消息(消费者组模式) -// 后台 Goroutine 使用此方法从 Stream 中取出请求进行处理 -// 参数: -// - streamKey: Stream 键名 -// - groupName: 消费者组名称 -// - consumerName: 消费者名称(唯一标识) -// - count: 每次读取的消息数量 -// - blockMs: 阻塞时间(毫秒),0表示不阻塞 -// -// 返回: -// - []StreamMessage: 消息列表 -// - error: 读取失败时返回错误 +// 使用 gredis Do() 方法执行 XREADGROUP 命令 func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName string, count int64, blockMs int64) ([]StreamMessage, error) { - // 使用 XReadGroup 从消费者组读取消息 - // ">" 表示读取未被消费的新消息(只获取新消息) - // 如果使用 "0" 或其他 ID,则返回 Pending 消息(未确认的消息) - streams, err := RedisClient.XReadGroup(ctx, &redis.XReadGroupArgs{ - Group: groupName, - Consumer: consumerName, - Streams: []string{streamKey, ">"}, // Stream名称 + 起始ID - Count: count, - Block: time.Duration(blockMs) * time.Millisecond, - }).Result() + // XREADGROUP GROUP groupName consumerName COUNT count BLOCK blockMs STREAMS streamKey > + result, err := GRedisClient.Do(ctx, + "XREADGROUP", "GROUP", groupName, consumerName, + "COUNT", count, + "BLOCK", blockMs, + "STREAMS", streamKey, ">", + ) - // 处理错误:超时或没有数据时返回 redis.Nil if err != nil { - if err == redis.Nil { - // 超时或没有数据,返回空数组 - return []StreamMessage{}, nil - } return nil, err } - // 解析返回的消息 - var messages []StreamMessage - for _, stream := range streams { - for _, msg := range stream.Messages { + // 解析返回值 + // 格式: [[streamKey, [[msgID, [field1, value1, field2, value2, ...]], ...]]] + messages := []StreamMessage{} + + if result == nil { + // 超时或没有数据 + return messages, nil + } + + // 类型断言:result.Val() 返回 interface{} + streamsArray, ok := result.Val().([]interface{}) + if !ok || len(streamsArray) == 0 { + return messages, nil + } + + // 遍历每个 stream + for _, streamData := range streamsArray { + streamArray, ok := streamData.([]interface{}) + if !ok || len(streamArray) < 2 { + continue + } + + // streamArray[0] 是 streamKey, streamArray[1] 是消息数组 + messagesArray, ok := streamArray[1].([]interface{}) + if !ok { + continue + } + + // 解析每条消息 + for _, msgData := range messagesArray { + msgArray, ok := msgData.([]interface{}) + if !ok || len(msgArray) < 2 { + continue + } + + // msgArray[0] 是 ID, msgArray[1] 是字段数组 + msgID := gconv.String(msgArray[0]) + fieldsArray, ok := msgArray[1].([]interface{}) + if !ok { + continue + } + + // 解析字段为 map + values := make(map[string]interface{}) + for i := 0; i < len(fieldsArray); i += 2 { + if i+1 < len(fieldsArray) { + key := gconv.String(fieldsArray[i]) + val := fieldsArray[i+1] + values[key] = val + } + } + messages = append(messages, StreamMessage{ - ID: msg.ID, - Values: msg.Values, + ID: msgID, + Values: values, }) } } @@ -147,101 +151,125 @@ func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName stri } // AckMessage 确认消息已处理 -// 处理完消息后必须调用此方法确认,否则消息会保留在 Pending List (PEL) -// 确认后消息会从 PEL 中移除 -// 参数: -// - streamKey: Stream 键名 -// - groupName: 消费者组名称 -// - messageIDs: 要确认的消息ID列表 -// -// 返回:error 确认失败时返回错误 +// 使用 gredis Do() 方法执行 XACK 命令 func AckMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error { - // 使用 XAck 确认消息 - // 返回值是成功确认的消息数量 - count, err := RedisClient.XAck(ctx, streamKey, groupName, messageIDs...).Result() - if err != nil { - return err + // XACK streamKey groupName messageID1 messageID2 ... + args := []interface{}{streamKey, groupName} + for _, id := range messageIDs { + args = append(args, id) } - // 可以检查 count 是否等于 len(messageIDs) - _ = count - return nil + + _, err := GRedisClient.Do(ctx, "XACK", args...) + return err } // GetStreamLength 获取 Stream 当前长度 -// 用于监控 Stream 消息积压情况 -// 参数: -// - streamKey: Stream 键名 -// -// 返回: -// - int64: Stream 中消息数量 -// - error: 操作失败时返回错误 +// 使用 gredis Do() 方法执行 XLEN 命令 func GetStreamLength(ctx context.Context, streamKey string) (int64, error) { - // 使用 XLen 获取 Stream 长度 - length, err := RedisClient.XLen(ctx, streamKey).Result() + // XLEN streamKey + result, err := GRedisClient.Do(ctx, "XLEN", streamKey) if err != nil { return 0, err } + + length := gconv.Int64(result) return length, nil } -// GetPendingMessages 获取待处理消息(未确认的消息) -// 用于监控和重试失败的消息 -// 参数: -// - streamKey: Stream 键名 -// - groupName: 消费者组名称 -// - start: 起始ID,"-" 表示最小ID -// - end: 结束ID,"+" 表示最大ID -// - count: 返回数量 -// -// 返回: -// - []redis.XPendingExt: Pending 消息列表 -// - error: 操作失败时返回错误 -func GetPendingMessages(ctx context.Context, streamKey, groupName string, start, end string, count int64) ([]redis.XPendingExt, error) { - // 使用 XPendingExt 获取详细的 Pending 消息 - pending, err := RedisClient.XPendingExt(ctx, &redis.XPendingExtArgs{ - Stream: streamKey, - Group: groupName, - Start: start, - End: end, - Count: count, - }).Result() +// PendingMessage Pending 消息结构 +type PendingMessage struct { + ID string // 消息ID + Consumer string // 消费者名称 + Idle int64 // 空闲时间(毫秒) + RetryCount int64 // 重试次数 +} + +// GetPendingMessages 获取待处理消息 +// 使用 gredis Do() 方法执行 XPENDING 命令 +func GetPendingMessages(ctx context.Context, streamKey, groupName string, start, end string, count int64) ([]PendingMessage, error) { + // XPENDING streamKey groupName start end count + result, err := GRedisClient.Do(ctx, "XPENDING", streamKey, groupName, start, end, count) if err != nil { return nil, err } - return pending, nil + + if result == nil { + return []PendingMessage{}, nil + } + + // 解析返回值:[[ID, consumer, idle, retryCount], ...] + pendingArray, ok := result.Val().([]interface{}) + if !ok { + return []PendingMessage{}, nil + } + + var messages []PendingMessage + for _, item := range pendingArray { + itemArray, ok := item.([]interface{}) + if !ok || len(itemArray) < 4 { + continue + } + + messages = append(messages, PendingMessage{ + ID: gconv.String(itemArray[0]), + Consumer: gconv.String(itemArray[1]), + Idle: gconv.Int64(itemArray[2]), + RetryCount: gconv.Int64(itemArray[3]), + }) + } + + return messages, nil } // ClaimPendingMessage 认领超时的 Pending 消息 -// 当某个消费者故障后,其他消费者可以认领其未完成的消息 -// 参数: -// - streamKey: Stream 键名 -// - groupName: 消费者组名称 -// - consumerName: 新消费者名称 -// - minIdleTime: 消息空闲时间(毫秒),超过此时间才能被认领 -// - messageIDs: 要认领的消息ID列表 -// -// 返回: -// - []StreamMessage: 认领的消息列表 -// - error: 操作失败时返回错误 +// 使用 gredis Do() 方法执行 XCLAIM 命令 func ClaimPendingMessage(ctx context.Context, streamKey, groupName, consumerName string, minIdleTime int64, messageIDs ...string) ([]StreamMessage, error) { - // 使用 XClaim 认领消息 - msgs, err := RedisClient.XClaim(ctx, &redis.XClaimArgs{ - Stream: streamKey, - Group: groupName, - Consumer: consumerName, - MinIdle: time.Duration(minIdleTime) * time.Millisecond, - Messages: messageIDs, - }).Result() + // XCLAIM streamKey groupName consumerName minIdleTime messageID1 messageID2 ... + args := []interface{}{streamKey, groupName, consumerName, minIdleTime} + for _, id := range messageIDs { + args = append(args, id) + } + + result, err := GRedisClient.Do(ctx, "XCLAIM", args...) if err != nil { return nil, err } - // 转换为 StreamMessage + if result == nil { + return []StreamMessage{}, nil + } + + // 解析返回值:类似 XREADGROUP + messagesArray, ok := result.Val().([]interface{}) + if !ok { + return []StreamMessage{}, nil + } + var messages []StreamMessage - for _, msg := range msgs { + for _, msgData := range messagesArray { + msgArray, ok := msgData.([]interface{}) + if !ok || len(msgArray) < 2 { + continue + } + + msgID := gconv.String(msgArray[0]) + fieldsArray, ok := msgArray[1].([]interface{}) + if !ok { + continue + } + + values := make(map[string]interface{}) + for i := 0; i < len(fieldsArray); i += 2 { + if i+1 < len(fieldsArray) { + key := gconv.String(fieldsArray[i]) + val := fieldsArray[i+1] + values[key] = val + } + } + messages = append(messages, StreamMessage{ - ID: msg.ID, - Values: msg.Values, + ID: msgID, + Values: values, }) } @@ -249,42 +277,30 @@ func ClaimPendingMessage(ctx context.Context, streamKey, groupName, consumerName } // SetSessionLastActive 设置用户最后活跃时间 -// 用于控制是否发送追问:用户回复后更新活跃时间,避免重复追问 -// 过期时间:2小时,超过2小时未活跃的记录会自动删除 -// 参数: -// - userId: 用户ID -// -// 返回:error 设置失败时返回错误 +// 使用 gredis SetEX 方法 func SetSessionLastActive(ctx context.Context, userId string) error { key := SessionLastActiveKeyPrefix + userId + ":last_active" timestamp := time.Now().Unix() - // 设置过期时间为 2 小时 - return RedisClient.Set(ctx, key, timestamp, 2*time.Hour).Err() + // SETEX key 7200 value (7200秒 = 2小时) + _, err := GRedisClient.Do(ctx, "SETEX", key, 7200, timestamp) + return err } // GetSessionLastActive 获取用户最后活跃时间 -// 参数: -// - userId: 用户ID -// -// 返回: -// - int64: Unix时间戳,未找到返回0 -// - error: 操作失败时返回错误 +// 使用 gredis Get 方法 func GetSessionLastActive(ctx context.Context, userId string) (int64, error) { key := SessionLastActiveKeyPrefix + userId + ":last_active" - result, err := RedisClient.Get(ctx, key).Result() - if err == redis.Nil { - return 0, nil // 未找到返回 0 - } + result, err := GRedisClient.Get(ctx, key) if err != nil { return 0, err } - // 将字符串转换为 int64 - timestamp, err := strconv.ParseInt(result, 10, 64) - if err != nil { - return 0, err + if result.IsEmpty() { + return 0, nil } + + timestamp := gconv.Int64(result.Val()) return timestamp, nil } @@ -312,35 +328,27 @@ func IsUserActive(ctx context.Context, userId string, seconds int64) (bool, erro } // SetSessionCache 缓存用户的 RAGFlow Session ID -// 避免每次请求都创建新 Session,提高性能 -// 过期时间:7天,超过7天未使用的Session会自动清理 -// 参数: -// - userId: 用户ID -// - sessionId: RAGFlow返回的Session ID -// -// 返回:error 设置失败时返回错误 +// 使用 gredis SetEX 方法 func SetSessionCache(ctx context.Context, userId, sessionId string) error { key := SessionLastActiveKeyPrefix + userId + ":session_id" - return RedisClient.Set(ctx, key, sessionId, 7*24*time.Hour).Err() + + // SETEX key 604800 value (604800秒 = 7天) + _, err := GRedisClient.Do(ctx, "SETEX", key, 604800, sessionId) + return err } // GetSessionCache 获取缓存的 RAGFlow Session ID -// 如果缓存中存在则直接使用,不存在则需要创建新Session -// 参数: -// - userId: 用户ID -// -// 返回: -// - string: Session ID,未找到返回空字符串 -// - error: 操作失败时返回错误 +// 使用 gredis Get 方法 func GetSessionCache(ctx context.Context, userId string) (string, error) { key := SessionLastActiveKeyPrefix + userId + ":session_id" - result, err := RedisClient.Get(ctx, key).Result() - if err == redis.Nil { - return "", nil // 未找到返回空字符串 - } + result, err := GRedisClient.Get(ctx, key) if err != nil { return "", err } - return result, nil + if result.IsEmpty() { + return "", nil + } + + return result.String(), nil } From 15ce1cf34551311715ea99f8854c20a8a0433f18 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sat, 6 Dec 2025 09:36:10 +0800 Subject: [PATCH 053/225] =?UTF-8?q?=E4=BD=BF=E7=94=A8=20gtime=E8=AE=BE?= =?UTF-8?q?=E7=BD=AEredis=E6=97=B6=E9=97=B4=E6=88=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/redis/redis.go b/redis/redis.go index 2cd7840..ee17d3b 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -3,10 +3,10 @@ package redis import ( "context" "strings" - "time" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/util/gconv" ) @@ -280,7 +280,7 @@ func ClaimPendingMessage(ctx context.Context, streamKey, groupName, consumerName // 使用 gredis SetEX 方法 func SetSessionLastActive(ctx context.Context, userId string) error { key := SessionLastActiveKeyPrefix + userId + ":last_active" - timestamp := time.Now().Unix() + timestamp := gtime.Now().Timestamp() // SETEX key 7200 value (7200秒 = 2小时) _, err := GRedisClient.Do(ctx, "SETEX", key, 7200, timestamp) @@ -323,7 +323,7 @@ func IsUserActive(ctx context.Context, userId string, seconds int64) (bool, erro return false, nil // 未找到记录,视为不活跃 } - now := time.Now().Unix() + now := gtime.Now().Timestamp() return (now - lastActive) < seconds, nil } From 86661c687aca65838c680b0bf8f512778ab0f128 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sat, 6 Dec 2025 10:13:38 +0800 Subject: [PATCH 054/225] =?UTF-8?q?map=E9=A2=84=E5=88=86=E9=85=8D=E5=AE=B9?= =?UTF-8?q?=E9=87=8F=E9=81=BF=E5=85=8D=E5=8A=A8=E6=80=81=E6=89=A9=E5=AE=B9?= =?UTF-8?q?,=E4=BC=98=E5=8C=96=E4=BA=86=E8=BF=94=E5=9B=9E=E5=80=BC?= =?UTF-8?q?=E5=A4=8D=E7=94=A8=E6=83=85=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/client.go | 3 +-- redis/redis.go | 20 ++++++++++++-------- redis/types.go | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 10 deletions(-) create mode 100644 redis/types.go diff --git a/ragflow/client.go b/ragflow/client.go index 95ef503..0633dfc 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -50,8 +50,7 @@ func loadConfig(ctx context.Context) (baseURL, apiKey string) { // 使用 GoFrame 全局配置(从项目的 config.yml 读取) baseURL = g.Cfg().MustGet(ctx, "ragflow.base_url", "").String() apiKey = g.Cfg().MustGet(ctx, "ragflow.api_key", "").String() - - return baseURL, apiKey + return } // GetGlobalClient 获取全局客户端 diff --git a/redis/redis.go b/redis/redis.go index ee17d3b..7856a4b 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -90,7 +90,8 @@ func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName stri // 解析返回值 // 格式: [[streamKey, [[msgID, [field1, value1, field2, value2, ...]], ...]]] - messages := []StreamMessage{} + // 预分配容量,避免动态扩容 + messages := make([]StreamMessage, 0, int(count)) if result == nil { // 超时或没有数据 @@ -130,8 +131,8 @@ func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName stri continue } - // 解析字段为 map - values := make(map[string]interface{}) + // 解析字段为 map,预分配容量,避免动态扩容 + values := make(map[string]interface{}, len(fieldsArray)/2) for i := 0; i < len(fieldsArray); i += 2 { if i+1 < len(fieldsArray) { key := gconv.String(fieldsArray[i]) @@ -154,7 +155,9 @@ func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName stri // 使用 gredis Do() 方法执行 XACK 命令 func AckMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error { // XACK streamKey groupName messageID1 messageID2 ... - args := []interface{}{streamKey, groupName} + // 预分配容量,避免动态扩容 + args := make([]interface{}, 0, len(messageIDs)+2) + args = append(args, streamKey, groupName) for _, id := range messageIDs { args = append(args, id) } @@ -245,7 +248,8 @@ func ClaimPendingMessage(ctx context.Context, streamKey, groupName, consumerName return []StreamMessage{}, nil } - var messages []StreamMessage + // 预分配容量,避免动态扩容 + messages := make([]StreamMessage, 0, len(messagesArray)) for _, msgData := range messagesArray { msgArray, ok := msgData.([]interface{}) if !ok || len(msgArray) < 2 { @@ -258,12 +262,12 @@ func ClaimPendingMessage(ctx context.Context, streamKey, groupName, consumerName continue } - values := make(map[string]interface{}) + // 预分配 map 容量 ,避免动态扩容 + values := make(map[string]interface{}, len(fieldsArray)/2) for i := 0; i < len(fieldsArray); i += 2 { if i+1 < len(fieldsArray) { key := gconv.String(fieldsArray[i]) - val := fieldsArray[i+1] - values[key] = val + values[key] = fieldsArray[i+1] } } diff --git a/redis/types.go b/redis/types.go new file mode 100644 index 0000000..8671a1a --- /dev/null +++ b/redis/types.go @@ -0,0 +1,39 @@ +package redis + +// SendStreamMessage 发送到 Redis Stream 的消息结构 +type SendStreamMessage struct { + UserId string `json:"user_id"` // 用户ID + Content string `json:"content"` // 消息内容 + Timestamp int64 `json:"timestamp"` // 时间戳(秒) + MessageId string `json:"message_id"` // 消息唯一ID +} + +// ToMap 转换为 map[string]interface{} 用于 Stream 存储 +func (m *SendStreamMessage) ToMap() map[string]interface{} { + return map[string]interface{}{ + "user_id": m.UserId, + "content": m.Content, + "timestamp": m.Timestamp, + "message_id": m.MessageId, + } +} + +// BatchStreamMessage 批量消息结构 +type BatchStreamMessage struct { + UserId string `json:"user_id"` // 用户ID + Content string `json:"content"` // 消息内容 + Timestamp int64 `json:"timestamp"` // 时间戳(秒) + BatchId string `json:"batch_id"` // 批次ID + Index int `json:"index"` // 批次内序号 +} + +// ToMap 转换为 map[string]interface{} 用于 Stream 存储 +func (m *BatchStreamMessage) ToMap() map[string]interface{} { + return map[string]interface{}{ + "user_id": m.UserId, + "content": m.Content, + "timestamp": m.Timestamp, + "batch_id": m.BatchId, + "index": m.Index, + } +} From f7cb007491fd6f3238ac6dcbb8010ffab80a46d7 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sat, 6 Dec 2025 12:02:34 +0800 Subject: [PATCH 055/225] =?UTF-8?q?=E5=8D=8F=E7=A8=8B=E6=B1=A0=E5=8D=95?= =?UTF-8?q?=E4=BE=8B=E6=A8=A1=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/worker_pool.go | 41 +++++++++++++++++++++++++++++++---------- redis/redis.go | 35 +++++++++++++++++++---------------- 2 files changed, 50 insertions(+), 26 deletions(-) diff --git a/ragflow/worker_pool.go b/ragflow/worker_pool.go index 8c46c47..47767bf 100644 --- a/ragflow/worker_pool.go +++ b/ragflow/worker_pool.go @@ -2,6 +2,7 @@ package ragflow import ( "context" + "sync" "gitee.com/red-future---jilin-g/common/redis" "github.com/gogf/gf/v2/errors/gerror" @@ -15,24 +16,44 @@ type WorkerPool struct { size int } -// NewWorkerPool 创建协程池 +// 单例模式相关变量 +var ( + workerPoolInstance *WorkerPool + workerPoolOnce sync.Once +) + +// GetWorkerPoolWithSize 获取指定大小的协程池单例 +// 使用 sync.Once 确保只创建一次,size 仅首次调用生效 +func GetWorkerPoolWithSize(size int) *WorkerPool { + workerPoolOnce.Do(func() { + if size <= 0 { + size = 200 // 默认大小 + } + workerPoolInstance = &WorkerPool{ + pool: grpool.New(size), + size: size, + } + }) + return workerPoolInstance +} + +// GetWorkerPool 获取协程池单例(使用默认大小 200) +func GetWorkerPool() *WorkerPool { + return GetWorkerPoolWithSize(200) +} + +// NewWorkerPool 创建协程池(兼容旧代码,内部使用单例) // 参数: -// - size: 协程池大小,建议设置为 CPU 核心数的 2-4 倍 +// - size: 协程池大小,仅首次调用生效 // // 返回: -// - *WorkerPool: 协程池实例 +// - *WorkerPool: 协程池单例实例 // - error: 创建失败时返回错误 func NewWorkerPool(size int) (*WorkerPool, error) { if size <= 0 { return nil, gerror.New("协程池大小必须大于0") } - - pool := grpool.New(size) - - return &WorkerPool{ - pool: pool, - size: size, - }, nil + return GetWorkerPoolWithSize(size), nil } // Submit 提交任务到协程池 diff --git a/redis/redis.go b/redis/redis.go index 7856a4b..1e5ca10 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -10,16 +10,19 @@ import ( "github.com/gogf/gf/v2/util/gconv" ) -// GRedisClient GoFrame gredis 客户端,统一使用 +// GRedisClient GoFrame gredis 客户端,统一使用(懒加载) var GRedisClient *gredis.Redis // RedisClient GRedisClient 的别名,保持向后兼容 var RedisClient *gredis.Redis -func init() { - // 初始化 GoFrame gredis 客户端 - GRedisClient = g.Redis() - RedisClient = GRedisClient // 别名指向同一个客户端 +// GetRedisClient 获取 Redis 客户端(懒加载) +func GetRedisClient() *gredis.Redis { + if GRedisClient == nil { + GRedisClient = g.Redis() + RedisClient = GRedisClient + } + return GRedisClient } // Stream 和消费者组常量 @@ -42,7 +45,7 @@ type StreamMessage struct { // 使用 gredis Do() 方法执行 XGROUP CREATE 命令 func InitStreamGroup(ctx context.Context, streamKey, groupName string) error { // XGROUP CREATE streamKey groupName 0 MKSTREAM - _, err := GRedisClient.Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") + _, err := GetRedisClient().Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") if err != nil { // 如果组已存在,忽略错误 errStr := err.Error() @@ -63,7 +66,7 @@ func AddToStream(ctx context.Context, streamKey string, values map[string]interf args = append(args, key, val) } - result, err := GRedisClient.Do(ctx, "XADD", args...) + result, err := GetRedisClient().Do(ctx, "XADD", args...) if err != nil { return "", err } @@ -77,7 +80,7 @@ func AddToStream(ctx context.Context, streamKey string, values map[string]interf // 使用 gredis Do() 方法执行 XREADGROUP 命令 func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName string, count int64, blockMs int64) ([]StreamMessage, error) { // XREADGROUP GROUP groupName consumerName COUNT count BLOCK blockMs STREAMS streamKey > - result, err := GRedisClient.Do(ctx, + result, err := GetRedisClient().Do(ctx, "XREADGROUP", "GROUP", groupName, consumerName, "COUNT", count, "BLOCK", blockMs, @@ -162,7 +165,7 @@ func AckMessage(ctx context.Context, streamKey, groupName string, messageIDs ... args = append(args, id) } - _, err := GRedisClient.Do(ctx, "XACK", args...) + _, err := GetRedisClient().Do(ctx, "XACK", args...) return err } @@ -170,7 +173,7 @@ func AckMessage(ctx context.Context, streamKey, groupName string, messageIDs ... // 使用 gredis Do() 方法执行 XLEN 命令 func GetStreamLength(ctx context.Context, streamKey string) (int64, error) { // XLEN streamKey - result, err := GRedisClient.Do(ctx, "XLEN", streamKey) + result, err := GetRedisClient().Do(ctx, "XLEN", streamKey) if err != nil { return 0, err } @@ -191,7 +194,7 @@ type PendingMessage struct { // 使用 gredis Do() 方法执行 XPENDING 命令 func GetPendingMessages(ctx context.Context, streamKey, groupName string, start, end string, count int64) ([]PendingMessage, error) { // XPENDING streamKey groupName start end count - result, err := GRedisClient.Do(ctx, "XPENDING", streamKey, groupName, start, end, count) + result, err := GetRedisClient().Do(ctx, "XPENDING", streamKey, groupName, start, end, count) if err != nil { return nil, err } @@ -233,7 +236,7 @@ func ClaimPendingMessage(ctx context.Context, streamKey, groupName, consumerName args = append(args, id) } - result, err := GRedisClient.Do(ctx, "XCLAIM", args...) + result, err := GetRedisClient().Do(ctx, "XCLAIM", args...) if err != nil { return nil, err } @@ -287,7 +290,7 @@ func SetSessionLastActive(ctx context.Context, userId string) error { timestamp := gtime.Now().Timestamp() // SETEX key 7200 value (7200秒 = 2小时) - _, err := GRedisClient.Do(ctx, "SETEX", key, 7200, timestamp) + _, err := GetRedisClient().Do(ctx, "SETEX", key, 7200, timestamp) return err } @@ -295,7 +298,7 @@ func SetSessionLastActive(ctx context.Context, userId string) error { // 使用 gredis Get 方法 func GetSessionLastActive(ctx context.Context, userId string) (int64, error) { key := SessionLastActiveKeyPrefix + userId + ":last_active" - result, err := GRedisClient.Get(ctx, key) + result, err := GetRedisClient().Get(ctx, key) if err != nil { return 0, err } @@ -337,7 +340,7 @@ func SetSessionCache(ctx context.Context, userId, sessionId string) error { key := SessionLastActiveKeyPrefix + userId + ":session_id" // SETEX key 604800 value (604800秒 = 7天) - _, err := GRedisClient.Do(ctx, "SETEX", key, 604800, sessionId) + _, err := GetRedisClient().Do(ctx, "SETEX", key, 604800, sessionId) return err } @@ -345,7 +348,7 @@ func SetSessionCache(ctx context.Context, userId, sessionId string) error { // 使用 gredis Get 方法 func GetSessionCache(ctx context.Context, userId string) (string, error) { key := SessionLastActiveKeyPrefix + userId + ":session_id" - result, err := GRedisClient.Get(ctx, key) + result, err := GetRedisClient().Get(ctx, key) if err != nil { return "", err } From 4b2b5e6177c53bb296ef7723221b0731bc76bbbf Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sat, 6 Dec 2025 18:04:29 +0800 Subject: [PATCH 056/225] =?UTF-8?q?=E9=87=8D=E6=9E=84=E4=BA=86=E4=B8=80?= =?UTF-8?q?=E4=B8=8B=20rag=E7=9A=84=E6=96=B9=E6=B3=95,=20=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=20goframe=E7=9A=84=E6=A1=86=E6=9E=B6,=20=E8=BF=98?= =?UTF-8?q?=E6=9C=89redis=E8=BF=9E=E6=8E=A5=E9=83=A8=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rabbitmq/client.go | 33 +++---- rabbitmq/consumer.go | 16 ++-- rabbitmq/publisher.go | 28 +++--- rabbitmq/setup.go | 22 ++--- ragflow/agent.go | 41 ++++----- ragflow/chat.go | 29 +++--- ragflow/chunk.go | 37 ++++---- ragflow/client.go | 38 ++++---- ragflow/dataset.go | 29 +++--- ragflow/document.go | 31 +++---- ragflow/openai.go | 32 +++---- ragflow/session.go | 29 +++--- ragflow/system.go | 16 ++-- ragflow/worker_pool.go | 12 +++ redis/redis.go | 200 ++++++++++++++++++++++++++++------------- redis/types.go | 65 ++++++++++++++ 16 files changed, 398 insertions(+), 260 deletions(-) diff --git a/rabbitmq/client.go b/rabbitmq/client.go index 4c0a05f..68eda88 100644 --- a/rabbitmq/client.go +++ b/rabbitmq/client.go @@ -2,11 +2,12 @@ package rabbitmq import ( "context" - "fmt" "sync" "time" + "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" amqp "github.com/rabbitmq/amqp091-go" ) @@ -33,13 +34,7 @@ func Init(ctx context.Context, cfg *Config) error { var err error once.Do(func() { // 构建连接字符串 - url := fmt.Sprintf("amqp://%s:%s@%s:%d/%s", - cfg.Username, - cfg.Password, - cfg.Host, - cfg.Port, - cfg.VHost, - ) + url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost // 创建连接 conn, err = amqp.Dial(url) @@ -89,7 +84,7 @@ func GetChannel() (*amqp.Channel, error) { defer mu.RUnlock() if channel == nil || channel.IsClosed() { - return nil, fmt.Errorf("RabbitMQ Channel 未初始化或已关闭") + return nil, gerror.New("RabbitMQ Channel 未初始化或已关闭") } return channel, nil @@ -101,7 +96,7 @@ func GetConnection() (*amqp.Connection, error) { defer mu.RUnlock() if conn == nil || conn.IsClosed() { - return nil, fmt.Errorf("RabbitMQ 连接未初始化或已关闭") + return nil, gerror.New("RabbitMQ 连接未初始化或已关闭") } return conn, nil @@ -160,13 +155,7 @@ func reconnect(ctx context.Context) { VHost: g.Cfg().MustGet(ctx, "rabbitmq.vhost", "/").String(), } - url := fmt.Sprintf("amqp://%s:%s@%s:%d/%s", - cfg.Username, - cfg.Password, - cfg.Host, - cfg.Port, - cfg.VHost, - ) + url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost var err error conn, err = amqp.Dial(url) @@ -190,7 +179,7 @@ func reconnect(ctx context.Context) { } // Close 关闭连接 -func Close(ctx context.Context) error { +func Close(ctx context.Context) (err error) { mu.Lock() defer mu.Unlock() @@ -201,21 +190,21 @@ func Close(ctx context.Context) error { } if channel != nil { - if err := channel.Close(); err != nil { + if err = channel.Close(); err != nil { g.Log().Errorf(ctx, "关闭 RabbitMQ Channel 失败: %v", err) } channel = nil } if conn != nil { - if err := conn.Close(); err != nil { + if err = conn.Close(); err != nil { g.Log().Errorf(ctx, "关闭 RabbitMQ 连接失败: %v", err) - return err + return } conn = nil } watcherStarted = false g.Log().Info(ctx, "RabbitMQ 连接已关闭") - return nil + return } diff --git a/rabbitmq/consumer.go b/rabbitmq/consumer.go index 1b0746d..4fb5a2f 100644 --- a/rabbitmq/consumer.go +++ b/rabbitmq/consumer.go @@ -2,9 +2,9 @@ package rabbitmq import ( "context" - "encoding/json" - "fmt" + "github.com/gogf/gf/v2/encoding/gjson" + "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" amqp "github.com/rabbitmq/amqp091-go" ) @@ -74,7 +74,7 @@ func NewConsumer(queue string, handler MessageHandler, opts ...ConsumerOption) * } // Start 启动消费者 -func (c *Consumer) Start(ctx context.Context) error { +func (c *Consumer) Start(ctx context.Context) (err error) { // 创建可取消的 context workerCtx, cancel := context.WithCancel(ctx) c.cancel = cancel @@ -90,7 +90,7 @@ func (c *Consumer) Start(ctx context.Context) error { false, // global: false 表示仅应用于当前 channel ) if err != nil { - return fmt.Errorf("设置 QoS 失败: %v", err) + return gerror.Newf("设置 QoS 失败: %v", err) } // 开始消费 @@ -104,7 +104,7 @@ func (c *Consumer) Start(ctx context.Context) error { nil, // args ) if err != nil { - return fmt.Errorf("开始消费失败: %v", err) + return gerror.Newf("开始消费失败: %v", err) } g.Log().Infof(ctx, "消费者已启动: queue=%s, prefetch=%d, workers=%d", @@ -115,7 +115,7 @@ func (c *Consumer) Start(ctx context.Context) error { go c.worker(workerCtx, i, msgs) } - return nil + return } // worker 工作协程 @@ -168,8 +168,8 @@ func StartTypedConsumer[T any]( // 包装处理函数 wrappedHandler := func(ctx context.Context, body []byte) error { var msg T - if err := json.Unmarshal(body, &msg); err != nil { - return fmt.Errorf("反序列化消息失败: %v", err) + if err := gjson.DecodeTo(body, &msg); err != nil { + return gerror.Newf("反序列化消息失败: %v", err) } return handler(ctx, &msg) diff --git a/rabbitmq/publisher.go b/rabbitmq/publisher.go index 096744b..182622b 100644 --- a/rabbitmq/publisher.go +++ b/rabbitmq/publisher.go @@ -2,9 +2,9 @@ package rabbitmq import ( "context" - "encoding/json" - "fmt" + "github.com/gogf/gf/v2/encoding/gjson" + "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" amqp "github.com/rabbitmq/amqp091-go" ) @@ -24,16 +24,16 @@ func NewPublisher(exchange, routingKey string) *Publisher { } // Publish 发布消息 -func (p *Publisher) Publish(ctx context.Context, message interface{}) error { +func (p *Publisher) Publish(ctx context.Context, message interface{}) (err error) { ch, err := GetChannel() if err != nil { return err } // 序列化消息 - body, err := json.Marshal(message) + body, err := gjson.Encode(message) if err != nil { - return fmt.Errorf("消息序列化失败: %v", err) + return gerror.Newf("消息序列化失败: %v", err) } // 发布消息 @@ -59,21 +59,21 @@ func (p *Publisher) Publish(ctx context.Context, message interface{}) error { g.Log().Debugf(ctx, "消息发布成功: exchange=%s, routingKey=%s", p.exchange, p.routingKey) - return nil + return } // PublishDelayed 发布延时消息 // delaySeconds: 延时秒数 -func (p *Publisher) PublishDelayed(ctx context.Context, message interface{}, delaySeconds int) error { +func (p *Publisher) PublishDelayed(ctx context.Context, message interface{}, delaySeconds int) (err error) { ch, err := GetChannel() if err != nil { return err } // 序列化消息 - body, err := json.Marshal(message) + body, err := gjson.Encode(message) if err != nil { - return fmt.Errorf("消息序列化失败: %v", err) + return gerror.Newf("消息序列化失败: %v", err) } // 发布延时消息(需要 rabbitmq_delayed_message_exchange 插件) @@ -102,13 +102,13 @@ func (p *Publisher) PublishDelayed(ctx context.Context, message interface{}, del g.Log().Debugf(ctx, "延时消息发布成功: exchange=%s, routingKey=%s, delay=%ds", p.exchange, p.routingKey, delaySeconds) - return nil + return } // PublishBatch 批量发布消息 -func (p *Publisher) PublishBatch(ctx context.Context, messages []interface{}) error { +func (p *Publisher) PublishBatch(ctx context.Context, messages []interface{}) (err error) { if len(messages) == 0 { - return nil + return } ch, err := GetChannel() @@ -117,7 +117,7 @@ func (p *Publisher) PublishBatch(ctx context.Context, messages []interface{}) er } for i, message := range messages { - body, err := json.Marshal(message) + body, err := gjson.Encode(message) if err != nil { g.Log().Errorf(ctx, "消息 %d 序列化失败: %v", i, err) continue @@ -143,5 +143,5 @@ func (p *Publisher) PublishBatch(ctx context.Context, messages []interface{}) er } g.Log().Infof(ctx, "批量发布完成: 共 %d 条消息", len(messages)) - return nil + return } diff --git a/rabbitmq/setup.go b/rabbitmq/setup.go index bc8bf59..47793fd 100644 --- a/rabbitmq/setup.go +++ b/rabbitmq/setup.go @@ -2,8 +2,8 @@ package rabbitmq import ( "context" - "fmt" + "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" amqp "github.com/rabbitmq/amqp091-go" ) @@ -35,7 +35,7 @@ type BindingConfig struct { } // DeclareQueue 声明队列 -func DeclareQueue(ctx context.Context, cfg *QueueConfig) error { +func DeclareQueue(ctx context.Context, cfg *QueueConfig) (err error) { ch, err := GetChannel() if err != nil { return err @@ -56,11 +56,11 @@ func DeclareQueue(ctx context.Context, cfg *QueueConfig) error { } g.Log().Infof(ctx, "队列声明成功: %s", cfg.Name) - return nil + return } // DeclareExchange 声明 Exchange -func DeclareExchange(ctx context.Context, cfg *ExchangeConfig) error { +func DeclareExchange(ctx context.Context, cfg *ExchangeConfig) (err error) { ch, err := GetChannel() if err != nil { return err @@ -82,11 +82,11 @@ func DeclareExchange(ctx context.Context, cfg *ExchangeConfig) error { } g.Log().Infof(ctx, "Exchange 声明成功: %s (type=%s)", cfg.Name, cfg.Type) - return nil + return } // BindQueue 绑定队列到 Exchange -func BindQueue(ctx context.Context, cfg *BindingConfig) error { +func BindQueue(ctx context.Context, cfg *BindingConfig) (err error) { ch, err := GetChannel() if err != nil { return err @@ -108,7 +108,7 @@ func BindQueue(ctx context.Context, cfg *BindingConfig) error { g.Log().Infof(ctx, "队列绑定成功: queue=%s → exchange=%s (routingKey=%s)", cfg.Queue, cfg.Exchange, cfg.RoutingKey) - return nil + return } // SetupDelayExchange 设置延时 Exchange(需要 rabbitmq_delayed_message_exchange 插件) @@ -165,9 +165,9 @@ func SetupQueueWithDLX(ctx context.Context, queueName, dlxExchange, dlxRoutingKe } // SetupBasicTopology 设置基础拓扑(适用于小红书客服场景) -func SetupBasicTopology(ctx context.Context) error { +func SetupBasicTopology(ctx context.Context) (err error) { // 1. 声明普通 Exchange - err := DeclareExchange(ctx, &ExchangeConfig{ + err = DeclareExchange(ctx, &ExchangeConfig{ Name: "ragflow_exchange", Type: "direct", Durable: true, @@ -179,7 +179,7 @@ func SetupBasicTopology(ctx context.Context) error { // 2. 声明延时 Exchange err = SetupDelayExchange(ctx, "delay_exchange") if err != nil { - return fmt.Errorf("延时 Exchange 声明失败(可能未安装插件): %v", err) + return gerror.Newf("延时 Exchange 声明失败(可能未安装插件): %v", err) } // 3. 声明死信队列 @@ -227,5 +227,5 @@ func SetupBasicTopology(ctx context.Context) error { } g.Log().Info(ctx, "RabbitMQ 拓扑结构设置完成") - return nil + return } diff --git a/ragflow/agent.go b/ragflow/agent.go index 4ce263a..e389e59 100644 --- a/ragflow/agent.go +++ b/ragflow/agent.go @@ -2,7 +2,8 @@ package ragflow import ( "context" - "fmt" + + "github.com/gogf/gf/v2/errors/gerror" ) // Agent AGENT 管理 @@ -56,44 +57,44 @@ type ListAgentsRes struct { // CreateAgent 创建 Agent // POST /api/v1/agents -func (c *Client) CreateAgent(ctx context.Context, req *CreateAgentReq) error { +func (c *Client) CreateAgent(ctx context.Context, req *CreateAgentReq) (err error) { var res CommonResponse - if err := c.request(ctx, "POST", "/api/v1/agents", req, &res); err != nil { - return fmt.Errorf("create agent failed: %w", err) + if err = c.request(ctx, "POST", "/api/v1/agents", req, &res); err != nil { + return gerror.Newf("create agent failed: %v", err) } if !res.IsSuccess() { - return fmt.Errorf("create agent failed: %s", res.Message) + return gerror.Newf("create agent failed: %s", res.Message) } - return nil + return } // UpdateAgent 更新 Agent // PUT /api/v1/agents/{agent_id} -func (c *Client) UpdateAgent(ctx context.Context, agentID string, req *UpdateAgentReq) error { - path := fmt.Sprintf("/api/v1/agents/%s", agentID) +func (c *Client) UpdateAgent(ctx context.Context, agentID string, req *UpdateAgentReq) (err error) { + path := "/api/v1/agents/" + agentID var res CommonResponse - if err := c.request(ctx, "PUT", path, req, &res); err != nil { - return fmt.Errorf("update agent failed: %w", err) + if err = c.request(ctx, "PUT", path, req, &res); err != nil { + return gerror.Newf("update agent failed: %v", err) } if !res.IsSuccess() { - return fmt.Errorf("update agent failed: %s", res.Message) + return gerror.Newf("update agent failed: %s", res.Message) } - return nil + return } // DeleteAgent 删除 Agent // DELETE /api/v1/agents/{agent_id} -func (c *Client) DeleteAgent(ctx context.Context, agentID string) error { - path := fmt.Sprintf("/api/v1/agents/%s", agentID) +func (c *Client) DeleteAgent(ctx context.Context, agentID string) (err error) { + path := "/api/v1/agents/" + agentID var res CommonResponse // 官方文档要求传空对象,不是 nil - if err := c.request(ctx, "DELETE", path, map[string]interface{}{}, &res); err != nil { - return fmt.Errorf("delete agent failed: %w", err) + if err = c.request(ctx, "DELETE", path, map[string]interface{}{}, &res); err != nil { + return gerror.Newf("delete agent failed: %v", err) } if !res.IsSuccess() { - return fmt.Errorf("delete agent failed: %s", res.Message) + return gerror.Newf("delete agent failed: %s", res.Message) } - return nil + return } // ListAgents 列出 Agent @@ -131,10 +132,10 @@ func (c *Client) ListAgents(ctx context.Context, req *ListAgentsReq) (*ListAgent var res ListAgentsRes if err := c.request(ctx, "GET", path, nil, &res); err != nil { - return nil, fmt.Errorf("list agents failed: %w", err) + return nil, gerror.Newf("list agents failed: %v", err) } if res.Code != 0 { - return nil, fmt.Errorf("list agents failed: code=%d", res.Code) + return nil, gerror.Newf("list agents failed: code=%d", res.Code) } return &res, nil } diff --git a/ragflow/chat.go b/ragflow/chat.go index 6f65335..7f0acb5 100644 --- a/ragflow/chat.go +++ b/ragflow/chat.go @@ -2,7 +2,8 @@ package ragflow import ( "context" - "fmt" + + "github.com/gogf/gf/v2/errors/gerror" ) // 聊天助手管理 @@ -104,7 +105,7 @@ func (c *Client) CreateChat(ctx context.Context, req *CreateChatReq) (*Chat, err return nil, err } if res.Code != 0 { - return nil, fmt.Errorf("create chat failed: %s", res.Msg) + return nil, gerror.Newf("create chat failed: %s", res.Msg) } return res.Data, nil } @@ -144,33 +145,33 @@ func (c *Client) ListChats(ctx context.Context, req *ListChatsReq) (*ListChatsRe return nil, err } if res.Code != 0 { - return nil, fmt.Errorf("list chats failed: code=%d", res.Code) + return nil, gerror.Newf("list chats failed: code=%d", res.Code) } return &res, nil } // DeleteChats 删除聊天助手 -func (c *Client) DeleteChats(ctx context.Context, ids []string) error { +func (c *Client) DeleteChats(ctx context.Context, ids []string) (err error) { req := DeleteChatsReq{Ids: ids} var res CommonResponse - if err := c.request(ctx, "DELETE", "/api/v1/chats", req, &res); err != nil { - return err + if err = c.request(ctx, "DELETE", "/api/v1/chats", req, &res); err != nil { + return } if !res.IsSuccess() { - return fmt.Errorf("delete chats failed: %s", res.Message) + return gerror.Newf("delete chats failed: %s", res.Message) } - return nil + return } // UpdateChat 更新聊天助手 -func (c *Client) UpdateChat(ctx context.Context, id string, req *UpdateChatReq) error { +func (c *Client) UpdateChat(ctx context.Context, id string, req *UpdateChatReq) (err error) { var res CommonResponse - path := fmt.Sprintf("/api/v1/chats/%s", id) - if err := c.request(ctx, "PUT", path, req, &res); err != nil { - return err + path := "/api/v1/chats/" + id + if err = c.request(ctx, "PUT", path, req, &res); err != nil { + return } if !res.IsSuccess() { - return fmt.Errorf("update chat failed: %s", res.Message) + return gerror.Newf("update chat failed: %s", res.Message) } - return nil + return } diff --git a/ragflow/chunk.go b/ragflow/chunk.go index 7e0b66e..030d21a 100644 --- a/ragflow/chunk.go +++ b/ragflow/chunk.go @@ -2,7 +2,8 @@ package ragflow import ( "context" - "fmt" + + "github.com/gogf/gf/v2/errors/gerror" ) // 数据集内知识块管理 @@ -90,7 +91,7 @@ type RetrieveChunksRes struct { // AddChunk 添加知识块 func (c *Client) AddChunk(ctx context.Context, datasetId, documentId string, req *AddChunkReq) (*Chunk, error) { - path := fmt.Sprintf("/api/v1/datasets/%s/documents/%s/chunks", datasetId, documentId) + path := "/api/v1/datasets/" + datasetId + "/documents/" + documentId + "/chunks" var res struct { Code int `json:"code"` Data struct { @@ -102,14 +103,14 @@ func (c *Client) AddChunk(ctx context.Context, datasetId, documentId string, req return nil, err } if res.Code != 0 { - return nil, fmt.Errorf("add chunk failed: %s", res.Msg) + return nil, gerror.Newf("add chunk failed: %s", res.Msg) } return res.Data.Chunk, nil } // ListChunks 列出知识块 func (c *Client) ListChunks(ctx context.Context, datasetId, documentId string, req *ListChunksReq) (*ListChunksRes, error) { - path := fmt.Sprintf("/api/v1/datasets/%s/documents/%s/chunks", datasetId, documentId) + path := "/api/v1/datasets/" + datasetId + "/documents/" + documentId + "/chunks" params := map[string]interface{}{} if req.Keywords != "" { params["keywords"] = req.Keywords @@ -134,36 +135,36 @@ func (c *Client) ListChunks(ctx context.Context, datasetId, documentId string, r return nil, err } if res.Code != 0 { - return nil, fmt.Errorf("list chunks failed: code=%d", res.Code) + return nil, gerror.Newf("list chunks failed: code=%d", res.Code) } return &res, nil } // DeleteChunks 删除知识块 -func (c *Client) DeleteChunks(ctx context.Context, datasetId, documentId string, chunkIds []string) error { +func (c *Client) DeleteChunks(ctx context.Context, datasetId, documentId string, chunkIds []string) (err error) { req := DeleteChunksReq{ChunkIds: chunkIds} var res CommonResponse - path := fmt.Sprintf("/api/v1/datasets/%s/documents/%s/chunks", datasetId, documentId) - if err := c.request(ctx, "DELETE", path, req, &res); err != nil { - return err + path := "/api/v1/datasets/" + datasetId + "/documents/" + documentId + "/chunks" + if err = c.request(ctx, "DELETE", path, req, &res); err != nil { + return } if !res.IsSuccess() { - return fmt.Errorf("delete chunks failed: %s", res.Message) + return gerror.Newf("delete chunks failed: %s", res.Message) } - return nil + return } // UpdateChunk 更新知识块 -func (c *Client) UpdateChunk(ctx context.Context, datasetId, documentId, chunkId string, req *UpdateChunkReq) error { +func (c *Client) UpdateChunk(ctx context.Context, datasetId, documentId, chunkId string, req *UpdateChunkReq) (err error) { var res CommonResponse - path := fmt.Sprintf("/api/v1/datasets/%s/documents/%s/chunks/%s", datasetId, documentId, chunkId) - if err := c.request(ctx, "PUT", path, req, &res); err != nil { - return err + path := "/api/v1/datasets/" + datasetId + "/documents/" + documentId + "/chunks/" + chunkId + if err = c.request(ctx, "PUT", path, req, &res); err != nil { + return } if !res.IsSuccess() { - return fmt.Errorf("update chunk failed: %s", res.Message) + return gerror.Newf("update chunk failed: %s", res.Message) } - return nil + return } // RetrieveChunks 检索知识块 @@ -173,7 +174,7 @@ func (c *Client) RetrieveChunks(ctx context.Context, req *RetrieveChunksReq) (*R return nil, err } if res.Code != 0 { - return nil, fmt.Errorf("retrieve chunks failed: code=%d", res.Code) + return nil, gerror.Newf("retrieve chunks failed: code=%d", res.Code) } return &res, nil } diff --git a/ragflow/client.go b/ragflow/client.go index 0633dfc..6b4ccc5 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -2,13 +2,12 @@ package ragflow import ( "context" - "encoding/json" - "fmt" - "io" "net/http" "net/url" "strings" + "github.com/gogf/gf/v2/encoding/gjson" + "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/gclient" ) @@ -33,7 +32,7 @@ func init() { // 初始化全局客户端 httpClient := gclient.New() - httpClient.SetHeader("Authorization", fmt.Sprintf("Bearer %s", apiKey)) + httpClient.SetHeader("Authorization", "Bearer "+apiKey) httpClient.SetHeader("Content-Type", "application/json") globalClient = &Client{ @@ -79,20 +78,19 @@ func (r *CommonResponse) IsSuccess() bool { } // request 发送 HTTP 请求 -func (c *Client) request(ctx context.Context, method, path string, body interface{}, result interface{}) error { +func (c *Client) request(ctx context.Context, method, path string, body interface{}, result interface{}) (err error) { fullURL := c.BaseURL + path - var reqBody io.Reader + var reqBody string if body != nil { - jsonData, err := json.Marshal(body) + jsonData, err := gjson.Encode(body) if err != nil { - return fmt.Errorf("marshal request body failed: %w", err) + return gerror.Newf("marshal request body failed: %v", err) } - reqBody = strings.NewReader(string(jsonData)) + reqBody = string(jsonData) } var resp *gclient.Response - var err error switch method { case "GET": @@ -104,28 +102,24 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac case "DELETE": resp, err = c.HTTPClient.Delete(ctx, fullURL, reqBody) default: - return fmt.Errorf("unsupported method: %s", method) + return gerror.Newf("unsupported method: %s", method) } if err != nil { - return fmt.Errorf("http request failed: %w", err) + return gerror.Newf("http request failed: %v", err) } defer resp.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("http request failed with status: %d", resp.StatusCode) + return gerror.Newf("http request failed with status: %d", resp.StatusCode) } respBody := resp.ReadAll() - if err != nil { - return fmt.Errorf("read response body failed: %w", err) + if err = gjson.DecodeTo(respBody, result); err != nil { + return gerror.Newf("unmarshal response failed: %v", err) } - if err := json.Unmarshal(respBody, result); err != nil { - return fmt.Errorf("unmarshal response failed: %w", err) - } - - return nil + return } // buildQueryString 构建查询字符串 @@ -134,9 +128,9 @@ func buildQueryString(params map[string]interface{}) string { return "" } - var parts []string + parts := make([]string, 0, len(params)) for k, v := range params { - parts = append(parts, fmt.Sprintf("%s=%v", url.QueryEscape(k), url.QueryEscape(fmt.Sprintf("%v", v)))) + parts = append(parts, url.QueryEscape(k)+"="+url.QueryEscape(g.NewVar(v).String())) } return strings.Join(parts, "&") } diff --git a/ragflow/dataset.go b/ragflow/dataset.go index 83162b7..431b332 100644 --- a/ragflow/dataset.go +++ b/ragflow/dataset.go @@ -2,7 +2,8 @@ package ragflow import ( "context" - "fmt" + + "github.com/gogf/gf/v2/errors/gerror" ) // 数据集管理 @@ -90,7 +91,7 @@ func (c *Client) CreateDataset(ctx context.Context, req *CreateDatasetReq) (*Dat return nil, err } if res.Code != 0 { - return nil, fmt.Errorf("create dataset failed: %s", res.Msg) + return nil, gerror.Newf("create dataset failed: %s", res.Msg) } return res.Data, nil } @@ -134,33 +135,33 @@ func (c *Client) ListDatasets(ctx context.Context, req *ListDatasetsReq) (*ListD return nil, err } if res.Code != 0 { - return nil, fmt.Errorf("list datasets failed: code=%d", res.Code) + return nil, gerror.Newf("list datasets failed: code=%d", res.Code) } return &res, nil } // DeleteDataset 删除数据集 -func (c *Client) DeleteDataset(ctx context.Context, ids []string) error { +func (c *Client) DeleteDataset(ctx context.Context, ids []string) (err error) { req := DeleteDatasetsReq{Ids: ids} var res CommonResponse - if err := c.request(ctx, "DELETE", "/api/v1/datasets", req, &res); err != nil { - return err + if err = c.request(ctx, "DELETE", "/api/v1/datasets", req, &res); err != nil { + return } if !res.IsSuccess() { - return fmt.Errorf("delete dataset failed: %s", res.Message) + return gerror.Newf("delete dataset failed: %s", res.Message) } - return nil + return } // UpdateDataset 更新数据集 -func (c *Client) UpdateDataset(ctx context.Context, id string, req *UpdateDatasetReq) error { +func (c *Client) UpdateDataset(ctx context.Context, id string, req *UpdateDatasetReq) (err error) { var res CommonResponse - path := fmt.Sprintf("/api/v1/datasets/%s", id) - if err := c.request(ctx, "PUT", path, req, &res); err != nil { - return err + path := "/api/v1/datasets/" + id + if err = c.request(ctx, "PUT", path, req, &res); err != nil { + return } if !res.IsSuccess() { - return fmt.Errorf("update dataset failed: %s", res.Message) + return gerror.Newf("update dataset failed: %s", res.Message) } - return nil + return } diff --git a/ragflow/document.go b/ragflow/document.go index 60d6192..c08f32b 100644 --- a/ragflow/document.go +++ b/ragflow/document.go @@ -2,8 +2,9 @@ package ragflow import ( "context" - "fmt" "strings" + + "github.com/gogf/gf/v2/errors/gerror" ) // 数据集内文件管理 @@ -70,7 +71,7 @@ type DeleteDocumentsReq struct { // ListDocuments 列出文档 func (c *Client) ListDocuments(ctx context.Context, datasetId string, req *ListDocumentsReq) (*ListDocumentsRes, error) { - path := fmt.Sprintf("/api/v1/datasets/%s/documents", datasetId) + path := "/api/v1/datasets/" + datasetId + "/documents" params := map[string]interface{}{} if req.Page > 0 { params["page"] = req.Page @@ -111,16 +112,14 @@ func (c *Client) ListDocuments(ctx context.Context, datasetId string, req *ListD // 处理数组参数:suffix(文件后缀过滤) // API 要求多个值时重复参数名,如:suffix=pdf&suffix=txt - // 这里使用 fmt.Sprintf 来构造每个参数值 for _, suffix := range req.Suffix { - queryParts = append(queryParts, fmt.Sprintf("suffix=%s", suffix)) + queryParts = append(queryParts, "suffix="+suffix) } // 处理数组参数:run(处理状态过滤) // 支持数字格式("0"-"4")或文本格式("UNSTART", "RUNNING", "CANCEL", "DONE", "FAIL") - // 这里使用 fmt.Sprintf 来构造每个参数值 for _, run := range req.Run { - queryParts = append(queryParts, fmt.Sprintf("run=%s", run)) + queryParts = append(queryParts, "run="+run) } // 构造最终请求路径 @@ -134,7 +133,7 @@ func (c *Client) ListDocuments(ctx context.Context, datasetId string, req *ListD return nil, err } if res.Code != 0 { - return nil, fmt.Errorf("list documents failed: code=%d", res.Code) + return nil, gerror.Newf("list documents failed: code=%d", res.Code) } return &res, nil } @@ -142,23 +141,21 @@ func (c *Client) ListDocuments(ctx context.Context, datasetId string, req *ListD // UploadDocument 上传文档 // 注意:此方法需要特殊处理 multipart/form-data,目前的 request 方法可能不支持 // 我们需要扩展 request 方法或在此处单独实现 -func (c *Client) UploadDocument(ctx context.Context, datasetId string, filePaths []string) error { +func (c *Client) UploadDocument(ctx context.Context, datasetId string, filePaths []string) (err error) { // TODO: 实现文件上传逻辑,需要使用 gclient 的 UploadFile 功能 - // 由于 request 方法封装了 JSON 处理,这里可能需要绕过 request 方法直接使用 c.Client - // 暂时留空或仅做简单提示,待完善 Client 封装以支持文件上传 - return fmt.Errorf("upload document not implemented yet") + return gerror.New("upload document not implemented yet") } // DeleteDocument 删除文档 -func (c *Client) DeleteDocument(ctx context.Context, datasetId string, ids []string) error { +func (c *Client) DeleteDocument(ctx context.Context, datasetId string, ids []string) (err error) { req := DeleteDocumentsReq{Ids: ids} var res CommonResponse - path := fmt.Sprintf("/api/v1/datasets/%s/documents", datasetId) - if err := c.request(ctx, "DELETE", path, req, &res); err != nil { - return err + path := "/api/v1/datasets/" + datasetId + "/documents" + if err = c.request(ctx, "DELETE", path, req, &res); err != nil { + return } if !res.IsSuccess() { - return fmt.Errorf("delete document failed: %s", res.Message) + return gerror.Newf("delete document failed: %s", res.Message) } - return nil + return } diff --git a/ragflow/openai.go b/ragflow/openai.go index acaa2ff..4218592 100644 --- a/ragflow/openai.go +++ b/ragflow/openai.go @@ -2,8 +2,9 @@ package ragflow import ( "context" - "encoding/json" - "fmt" + + "github.com/gogf/gf/v2/encoding/gjson" + "github.com/gogf/gf/v2/errors/gerror" ) // OpenAICompatibleAPI 与 OpenAI 兼容的 API @@ -64,11 +65,11 @@ type ChatCompletionChunk struct { // CreateChatCompletion 创建聊天补全(与聊天助手) // POST /api/v1/chats_openai/{chat_id}/chat/completions func (c *Client) CreateChatCompletion(ctx context.Context, chatID string, req *ChatCompletionRequest) (*ChatCompletionResponse, error) { - path := fmt.Sprintf("/api/v1/chats_openai/%s/chat/completions", chatID) + path := "/api/v1/chats_openai/" + chatID + "/chat/completions" var resp ChatCompletionResponse if err := c.request(ctx, "POST", path, req, &resp); err != nil { - return nil, fmt.Errorf("create chat completion failed: %w", err) + return nil, gerror.Newf("create chat completion failed: %v", err) } return &resp, nil @@ -77,11 +78,11 @@ func (c *Client) CreateChatCompletion(ctx context.Context, chatID string, req *C // CreateAgentCompletion 创建 Agent 补全 // POST /api/v1/agents_openai/{agent_id}/chat/completions func (c *Client) CreateAgentCompletion(ctx context.Context, agentID string, req *ChatCompletionRequest) (*ChatCompletionResponse, error) { - path := fmt.Sprintf("/api/v1/agents_openai/%s/chat/completions", agentID) + path := "/api/v1/agents_openai/" + agentID + "/chat/completions" var resp ChatCompletionResponse if err := c.request(ctx, "POST", path, req, &resp); err != nil { - return nil, fmt.Errorf("create agent completion failed: %w", err) + return nil, gerror.Newf("create agent completion failed: %v", err) } return &resp, nil @@ -91,31 +92,26 @@ func (c *Client) CreateAgentCompletion(ctx context.Context, agentID string, req // 注意:流式响应需要特殊处理,这里返回一个可用于读取流的接口 func (c *Client) CreateChatCompletionStream(ctx context.Context, chatID string, req *ChatCompletionRequest) (*StreamReader, error) { req.Stream = true - _ = fmt.Sprintf("/api/v1/chats_openai/%s/chat/completions", chatID) - // TODO: 实现流式读取逻辑 - return nil, fmt.Errorf("stream mode not implemented yet") + return nil, gerror.New("stream mode not implemented yet") } // StreamReader 流式响应读取器 type StreamReader struct { - decoder *json.Decoder - close func() error + _ *gjson.Json // TODO: 实现流式读取时使用 + close func() error } // ReadChunk 读取下一个响应块 +// TODO: 实现流式读取逻辑 func (sr *StreamReader) ReadChunk() (*ChatCompletionChunk, error) { - var chunk ChatCompletionChunk - if err := sr.decoder.Decode(&chunk); err != nil { - return nil, err - } - return &chunk, nil + return nil, gerror.New("stream mode not implemented yet") } // Close 关闭流 -func (sr *StreamReader) Close() error { +func (sr *StreamReader) Close() (err error) { if sr.close != nil { return sr.close() } - return nil + return } diff --git a/ragflow/session.go b/ragflow/session.go index 832082a..ee82b5d 100644 --- a/ragflow/session.go +++ b/ragflow/session.go @@ -2,7 +2,8 @@ package ragflow import ( "context" - "fmt" + + "github.com/gogf/gf/v2/errors/gerror" ) // 会话管理 @@ -76,7 +77,7 @@ type ChatCompletionRes struct { // CreateSession 创建会话 func (c *Client) CreateSession(ctx context.Context, chatId string, req *CreateSessionReq) (*Session, error) { - path := fmt.Sprintf("/api/v1/chats/%s/sessions", chatId) + path := "/api/v1/chats/" + chatId + "/sessions" var res struct { Code int `json:"code"` Data *Session `json:"data"` @@ -86,14 +87,14 @@ func (c *Client) CreateSession(ctx context.Context, chatId string, req *CreateSe return nil, err } if res.Code != 0 { - return nil, fmt.Errorf("create session failed: %s", res.Msg) + return nil, gerror.Newf("create session failed: %s", res.Msg) } return res.Data, nil } // ListSessions 列出会话 func (c *Client) ListSessions(ctx context.Context, chatId string, req *ListSessionsReq) (*ListSessionsRes, error) { - path := fmt.Sprintf("/api/v1/chats/%s/sessions", chatId) + path := "/api/v1/chats/" + chatId + "/sessions" params := map[string]interface{}{} if req.Page > 0 { params["page"] = req.Page @@ -129,40 +130,40 @@ func (c *Client) ListSessions(ctx context.Context, chatId string, req *ListSessi return nil, err } if res.Code != 0 { - return nil, fmt.Errorf("list sessions failed: code=%d", res.Code) + return nil, gerror.Newf("list sessions failed: code=%d", res.Code) } return &res, nil } // DeleteSessions 删除会话 -func (c *Client) DeleteSessions(ctx context.Context, chatId string, ids []string) error { +func (c *Client) DeleteSessions(ctx context.Context, chatId string, ids []string) (err error) { req := DeleteSessionsReq{Ids: ids} var res CommonResponse - path := fmt.Sprintf("/api/v1/chats/%s/sessions", chatId) - if err := c.request(ctx, "DELETE", path, req, &res); err != nil { - return err + path := "/api/v1/chats/" + chatId + "/sessions" + if err = c.request(ctx, "DELETE", path, req, &res); err != nil { + return } if !res.IsSuccess() { - return fmt.Errorf("delete sessions failed: %s", res.Message) + return gerror.Newf("delete sessions failed: %s", res.Message) } - return nil + return } // ChatCompletion 对话 (目前仅支持非流式) func (c *Client) ChatCompletion(ctx context.Context, chatId string, req *ChatCompletionReq) (*ChatCompletionRes, error) { - path := fmt.Sprintf("/api/v1/chats/%s/completions", chatId) + path := "/api/v1/chats/" + chatId + "/completions" var res ChatCompletionRes // 如果需要流式支持,需要使用 gclient 的流式处理能力,这里暂只实现非流式 if req.Stream { - return nil, fmt.Errorf("stream mode not supported yet") + return nil, gerror.New("stream mode not supported yet") } if err := c.request(ctx, "POST", path, req, &res); err != nil { return nil, err } if res.Code != 0 { - return nil, fmt.Errorf("chat completion failed: code=%d", res.Code) + return nil, gerror.Newf("chat completion failed: code=%d", res.Code) } return &res, nil } diff --git a/ragflow/system.go b/ragflow/system.go index f294c91..1a7da73 100644 --- a/ragflow/system.go +++ b/ragflow/system.go @@ -2,7 +2,8 @@ package ragflow import ( "context" - "fmt" + + "github.com/gogf/gf/v2/errors/gerror" ) // System 系统管理 @@ -10,11 +11,11 @@ import ( // HealthStatus 健康状态 type HealthStatus struct { - DB string `json:"db"` // "ok" 或 "nok" - Redis string `json:"redis"` // "ok" 或 "nok" - DocEngine string `json:"doc_engine"` // "ok" 或 "nok" - Storage string `json:"storage"` // "ok" 或 "nok" - Status string `json:"status"` // 整体状态: "ok" 或 "nok" + DB string `json:"db"` // "ok" 或 "nok" + Redis string `json:"redis"` // "ok" 或 "nok" + DocEngine string `json:"doc_engine"` // "ok" 或 "nok" + Storage string `json:"storage"` // "ok" 或 "nok" + Status string `json:"status"` // 整体状态: "ok" 或 "nok" Meta map[string]interface{} `json:"_meta,omitempty"` // 详细错误信息 } @@ -23,7 +24,7 @@ type HealthStatus struct { func (c *Client) CheckHealth(ctx context.Context) (*HealthStatus, error) { var status HealthStatus if err := c.request(ctx, "GET", "/v1/system/healthz", nil, &status); err != nil { - return nil, fmt.Errorf("check health failed: %w", err) + return nil, gerror.Newf("check health failed: %v", err) } return &status, nil } @@ -36,4 +37,3 @@ func (c *Client) IsHealthy(ctx context.Context) (bool, error) { } return status.Status == "ok", nil } - diff --git a/ragflow/worker_pool.go b/ragflow/worker_pool.go index 47767bf..b836ef8 100644 --- a/ragflow/worker_pool.go +++ b/ragflow/worker_pool.go @@ -149,12 +149,18 @@ func (q *QueueProcessor) Start(ctx context.Context) error { glog.Infof(ctx, "Stream 处理器启动 - Stream: %s, 消费者组: %s, 消费者: %s, 超时: %dms", q.streamKey, q.groupName, q.consumerName, q.timeout) + loopCount := 0 for { select { case <-q.stopChan: glog.Info(ctx, "Stream 处理器收到停止信号") return nil default: + loopCount++ + if loopCount%10 == 1 { + glog.Debugf(ctx, "[DEBUG] 第 %d 次循环,准备读取消息...", loopCount) + } + // 从 Redis Stream 中读取消息 messages, err := q.fetchMessages(ctx) if err != nil { @@ -164,11 +170,17 @@ func (q *QueueProcessor) Start(ctx context.Context) error { // 没有新消息,继续等待 if len(messages) == 0 { + if loopCount%10 == 1 { + glog.Debugf(ctx, "[DEBUG] 第 %d 次循环,无新消息", loopCount) + } continue } + glog.Infof(ctx, "[DEBUG] 收到 %d 条消息", len(messages)) + // 处理每条消息 for _, msg := range messages { + glog.Infof(ctx, "[DEBUG] 处理消息 ID: %s, Values: %+v", msg.ID, msg.Values) // 提交到协程池处理 if err := q.submitTask(ctx, msg); err != nil { glog.Errorf(ctx, "提交任务到协程池失败: %v, 消息ID: %s", err, msg.ID) diff --git a/redis/redis.go b/redis/redis.go index 1e5ca10..186d2ad 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -3,33 +3,50 @@ package redis import ( "context" "strings" + "sync" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/util/gconv" ) -// GRedisClient GoFrame gredis 客户端,统一使用(懒加载) -var GRedisClient *gredis.Redis +var ( + // redisClient 单例 Redis 客户端 + redisClient *gredis.Redis + // redisOnce 确保只初始化一次 + redisOnce sync.Once + // RedisClient 兼容导出(供 mongo.go 使用) + // 注意:这是一个指向单例的指针,首次调用 GetRedisClient() 后生效 + RedisClient *gredis.Redis +) -// RedisClient GRedisClient 的别名,保持向后兼容 -var RedisClient *gredis.Redis - -// GetRedisClient 获取 Redis 客户端(懒加载) +// GetRedisClient 获取 Redis 客户端(单例模式) func GetRedisClient() *gredis.Redis { - if GRedisClient == nil { - GRedisClient = g.Redis() - RedisClient = GRedisClient - } - return GRedisClient + redisOnce.Do(func() { + redisClient = g.Redis() + RedisClient = redisClient // 同步更新兼容导出 + }) + return redisClient +} + +// init 包初始化时自动初始化 Redis 客户端 +func init() { + GetRedisClient() } // Stream 和消费者组常量 const ( // RAGFlow 请求 Stream Key RAGFlowRequestStreamKey = "ragflow:request:stream" - // RAGFlow 消费者组名称 + // RAGFlow 响应 Stream Key + RAGFlowResponseStreamKey = "ragflow:response:stream" + // RAGFlow 请求消费者组名称 + RAGFlowRequestConsumerGroup = "ragflow:request:consumer:group" + // RAGFlow 响应消费者组名称 + RAGFlowResponseConsumerGroup = "ragflow:response:consumer:group" + // RAGFlow 消费者组名称(兼容旧代码) RAGFlowConsumerGroup = "ragflow:consumer:group" // 会话最后活跃时间 Key 前缀 SessionLastActiveKeyPrefix = "ragflow:session:" @@ -79,6 +96,9 @@ func AddToStream(ctx context.Context, streamKey string, values map[string]interf // ReadFromStream 从 Stream 读取消息(消费者组模式) // 使用 gredis Do() 方法执行 XREADGROUP 命令 func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName string, count int64, blockMs int64) ([]StreamMessage, error) { + glog.Debugf(ctx, "[DEBUG Redis] XREADGROUP GROUP %s %s COUNT %d BLOCK %d STREAMS %s >", + groupName, consumerName, count, blockMs, streamKey) + // XREADGROUP GROUP groupName consumerName COUNT count BLOCK blockMs STREAMS streamKey > result, err := GetRedisClient().Do(ctx, "XREADGROUP", "GROUP", groupName, consumerName, @@ -88,66 +108,89 @@ func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName stri ) if err != nil { + glog.Errorf(ctx, "[DEBUG Redis] XREADGROUP 错误: %v", err) return nil, err } - // 解析返回值 - // 格式: [[streamKey, [[msgID, [field1, value1, field2, value2, ...]], ...]]] + glog.Debugf(ctx, "[DEBUG Redis] XREADGROUP 返回: %+v", result) + // 预分配容量,避免动态扩容 messages := make([]StreamMessage, 0, int(count)) - if result == nil { + if result == nil || result.IsEmpty() { // 超时或没有数据 return messages, nil } - // 类型断言:result.Val() 返回 interface{} - streamsArray, ok := result.Val().([]interface{}) - if !ok || len(streamsArray) == 0 { - return messages, nil - } + // GoFrame gredis 返回格式: map[streamKey:[[msgID [field1 value1 field2 value2 ...]] ...]] + resultVal := result.Val() - // 遍历每个 stream - for _, streamData := range streamsArray { - streamArray, ok := streamData.([]interface{}) - if !ok || len(streamArray) < 2 { - continue - } - - // streamArray[0] 是 streamKey, streamArray[1] 是消息数组 - messagesArray, ok := streamArray[1].([]interface{}) - if !ok { - continue - } - - // 解析每条消息 - for _, msgData := range messagesArray { - msgArray, ok := msgData.([]interface{}) - if !ok || len(msgArray) < 2 { - continue - } - - // msgArray[0] 是 ID, msgArray[1] 是字段数组 - msgID := gconv.String(msgArray[0]) - fieldsArray, ok := msgArray[1].([]interface{}) + // 尝试 map 格式(GoFrame gredis 返回) + if streamsMap, ok := resultVal.(map[interface{}]interface{}); ok { + for _, streamMsgs := range streamsMap { + msgsArray, ok := streamMsgs.([]interface{}) if !ok { continue } - - // 解析字段为 map,预分配容量,避免动态扩容 - values := make(map[string]interface{}, len(fieldsArray)/2) - for i := 0; i < len(fieldsArray); i += 2 { - if i+1 < len(fieldsArray) { - key := gconv.String(fieldsArray[i]) - val := fieldsArray[i+1] - values[key] = val + for _, msgData := range msgsArray { + msgArray, ok := msgData.([]interface{}) + if !ok || len(msgArray) < 2 { + continue } + msgID := gconv.String(msgArray[0]) + fieldsArray, ok := msgArray[1].([]interface{}) + if !ok { + continue + } + values := make(map[string]interface{}, len(fieldsArray)/2) + for i := 0; i < len(fieldsArray); i += 2 { + if i+1 < len(fieldsArray) { + key := gconv.String(fieldsArray[i]) + values[key] = fieldsArray[i+1] + } + } + messages = append(messages, StreamMessage{ + ID: msgID, + Values: values, + }) } + } + return messages, nil + } - messages = append(messages, StreamMessage{ - ID: msgID, - Values: values, - }) + // 尝试数组格式(标准 Redis 返回) + if streamsArray, ok := resultVal.([]interface{}); ok && len(streamsArray) > 0 { + for _, streamData := range streamsArray { + streamArray, ok := streamData.([]interface{}) + if !ok || len(streamArray) < 2 { + continue + } + messagesArray, ok := streamArray[1].([]interface{}) + if !ok { + continue + } + for _, msgData := range messagesArray { + msgArray, ok := msgData.([]interface{}) + if !ok || len(msgArray) < 2 { + continue + } + msgID := gconv.String(msgArray[0]) + fieldsArray, ok := msgArray[1].([]interface{}) + if !ok { + continue + } + values := make(map[string]interface{}, len(fieldsArray)/2) + for i := 0; i < len(fieldsArray); i += 2 { + if i+1 < len(fieldsArray) { + key := gconv.String(fieldsArray[i]) + values[key] = fieldsArray[i+1] + } + } + messages = append(messages, StreamMessage{ + ID: msgID, + Values: values, + }) + } } } @@ -200,16 +243,16 @@ func GetPendingMessages(ctx context.Context, streamKey, groupName string, start, } if result == nil { - return []PendingMessage{}, nil + return nil, nil } // 解析返回值:[[ID, consumer, idle, retryCount], ...] pendingArray, ok := result.Val().([]interface{}) if !ok { - return []PendingMessage{}, nil + return nil, nil } - var messages []PendingMessage + messages := make([]PendingMessage, 0, len(pendingArray)) for _, item := range pendingArray { itemArray, ok := item.([]interface{}) if !ok || len(itemArray) < 4 { @@ -242,13 +285,13 @@ func ClaimPendingMessage(ctx context.Context, streamKey, groupName, consumerName } if result == nil { - return []StreamMessage{}, nil + return nil, nil } // 解析返回值:类似 XREADGROUP messagesArray, ok := result.Val().([]interface{}) if !ok { - return []StreamMessage{}, nil + return nil, nil } // 预分配容量,避免动态扩容 @@ -344,6 +387,43 @@ func SetSessionCache(ctx context.Context, userId, sessionId string) error { return err } +// 限流相关常量 +const ( + // RateLimitKeyPrefix 限流计数器 Key 前缀 + RateLimitKeyPrefix = "ragflow:ratelimit:" +) + +// IncrRateLimit 增加限流计数器,返回当前计数 +// windowSeconds: 时间窗口(秒) +func IncrRateLimit(ctx context.Context, key string, windowSeconds int64) (count int64, err error) { + fullKey := RateLimitKeyPrefix + key + result, err := GetRedisClient().Do(ctx, "INCR", fullKey) + if err != nil { + return + } + count = result.Int64() + + // 首次设置过期时间 + if count == 1 { + GetRedisClient().Do(ctx, "EXPIRE", fullKey, windowSeconds) + } + return +} + +// GetRateLimit 获取当前限流计数 +func GetRateLimit(ctx context.Context, key string) (count int64, err error) { + fullKey := RateLimitKeyPrefix + key + result, err := GetRedisClient().Get(ctx, fullKey) + if err != nil { + return + } + if result.IsEmpty() { + return 0, nil + } + count = result.Int64() + return +} + // GetSessionCache 获取缓存的 RAGFlow Session ID // 使用 gredis Get 方法 func GetSessionCache(ctx context.Context, userId string) (string, error) { diff --git a/redis/types.go b/redis/types.go index 8671a1a..ecdce57 100644 --- a/redis/types.go +++ b/redis/types.go @@ -37,3 +37,68 @@ func (m *BatchStreamMessage) ToMap() map[string]interface{} { "index": m.Index, } } + +// ResponseStreamMessage RAGFlow 响应消息结构(写入结果 Stream) +type ResponseStreamMessage struct { + UserId string `json:"user_id"` // 用户ID + Platform string `json:"platform"` // 平台标识 + Question string `json:"question"` // 用户问题 + Content string `json:"content"` // RAGFlow 回复内容 + SessionId string `json:"session_id"` // RAGFlow Session ID + Timestamp int64 `json:"timestamp"` // 时间戳(秒) + MessageId string `json:"message_id"` // 原始消息ID +} + +// ToMap 转换为 map[string]interface{} 用于 Stream 存储 +func (m *ResponseStreamMessage) ToMap() map[string]interface{} { + return map[string]interface{}{ + "user_id": m.UserId, + "platform": m.Platform, + "question": m.Question, + "content": m.Content, + "session_id": m.SessionId, + "timestamp": m.Timestamp, + "message_id": m.MessageId, + } +} + +// FollowUpMessage 追问消息结构(RabbitMQ 延时队列) +type FollowUpMessage struct { + UserId string `json:"user_id"` // 用户ID + Platform string `json:"platform"` // 平台标识 + Content string `json:"content"` // 追问内容 + FollowUpType int `json:"follow_up_type"` // 追问类型:1=30s, 2=60s, 3=180s + Timestamp int64 `json:"timestamp"` // 发送时间戳 +} + +// 追问话术常量 +const ( + FollowUpType1 = 1 // 30秒追问 + FollowUpType2 = 2 // 60秒追问 + FollowUpType3 = 3 // 180秒追问 +) + +// 追问话术内容 +var FollowUpContents = map[int]string{ + FollowUpType1: "还有其他问题吗?", + FollowUpType2: "如果需要帮助,随时告诉我~", + FollowUpType3: "我一直在线,有问题随时找我~", +} + +// 追问延时时间(秒) +var FollowUpDelays = map[int]int{ + FollowUpType1: 30, + FollowUpType2: 60, + FollowUpType3: 180, +} + +// ArchiveMessage 会话归档消息结构(RabbitMQ 延时队列) +type ArchiveMessage struct { + UserId string `json:"user_id"` // 用户ID + Platform string `json:"platform"` // 平台标识 + SessionId string `json:"session_id"` // RAGFlow Session ID + Timestamp int64 `json:"timestamp"` // 发送时间戳 +} + +// 归档延时时间(秒) +const ArchiveDelaySeconds = 3600 // 60分钟 From 87b3ac9878f9262d83dac5c75374c8db5ea2ddd3 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 9 Dec 2025 09:20:44 +0800 Subject: [PATCH 057/225] =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=B1=A0,redis,?= =?UTF-8?q?=E5=92=8C=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/constants.go | 34 +++++++++++++ rabbitmq/publisher.go | 19 ++++--- ragflow/client.go | 2 + ragflow/worker_pool.go | 52 +++++-------------- redis/redis.go | 80 +++++++++++++----------------- redis/types.go | 110 ++++++++++++++++++++--------------------- 6 files changed, 149 insertions(+), 148 deletions(-) create mode 100644 config/constants.go diff --git a/config/constants.go b/config/constants.go new file mode 100644 index 0000000..0cf2867 --- /dev/null +++ b/config/constants.go @@ -0,0 +1,34 @@ +package config + +// ==================== 可配置常量 ==================== +// 修改以下值来调整系统行为 + +// -------------------- 追问配置 -------------------- + +// FollowUpDelay1 第一次追问延时(秒) +var FollowUpDelay1 = 30 + +// FollowUpDelay2 第二次追问延时(秒) +var FollowUpDelay2 = 60 + +// FollowUpDelay3 第三次追问延时(秒) +var FollowUpDelay3 = 180 + +// FollowUpContent1 第一次追问话术 +var FollowUpContent1 = "还有其他问题吗?" + +// FollowUpContent2 第二次追问话术 +var FollowUpContent2 = "如果需要帮助,随时告诉我~" + +// FollowUpContent3 第三次追问话术 +var FollowUpContent3 = "我一直在线,有问题随时找我~" + +// -------------------- 归档配置 -------------------- + +// ArchiveDelay 归档延时(秒),默认 1 小时 +var ArchiveDelay = 3600 + +// -------------------- 历史上下文配置 -------------------- + +// HistoryContextLimit 读取历史对话轮数(用于新 Session 上下文注入) +var HistoryContextLimit int64 = 5 diff --git a/rabbitmq/publisher.go b/rabbitmq/publisher.go index 182622b..d9753b3 100644 --- a/rabbitmq/publisher.go +++ b/rabbitmq/publisher.go @@ -23,8 +23,13 @@ func NewPublisher(exchange, routingKey string) *Publisher { } } -// Publish 发布消息 +// Publish 发布消息(使用默认 routing key) func (p *Publisher) Publish(ctx context.Context, message interface{}) (err error) { + return p.PublishWithRoutingKey(ctx, p.routingKey, message) +} + +// PublishWithRoutingKey 发布消息(指定 routing key) +func (p *Publisher) PublishWithRoutingKey(ctx context.Context, routingKey string, message interface{}) (err error) { ch, err := GetChannel() if err != nil { return err @@ -39,10 +44,10 @@ func (p *Publisher) Publish(ctx context.Context, message interface{}) (err error // 发布消息 err = ch.PublishWithContext( ctx, - p.exchange, // exchange - p.routingKey, // routing key - false, // mandatory - false, // immediate + p.exchange, // exchange + routingKey, // routing key + false, // mandatory + false, // immediate amqp.Publishing{ DeliveryMode: amqp.Persistent, // 持久化 ContentType: "application/json", @@ -52,12 +57,12 @@ func (p *Publisher) Publish(ctx context.Context, message interface{}) (err error if err != nil { g.Log().Errorf(ctx, "发布消息失败: exchange=%s, routingKey=%s, err=%v", - p.exchange, p.routingKey, err) + p.exchange, routingKey, err) return err } g.Log().Debugf(ctx, "消息发布成功: exchange=%s, routingKey=%s", - p.exchange, p.routingKey) + p.exchange, routingKey) return } diff --git a/ragflow/client.go b/ragflow/client.go index 6b4ccc5..618d84b 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -5,6 +5,7 @@ import ( "net/http" "net/url" "strings" + "time" "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/errors/gerror" @@ -34,6 +35,7 @@ func init() { httpClient := gclient.New() httpClient.SetHeader("Authorization", "Bearer "+apiKey) httpClient.SetHeader("Content-Type", "application/json") + httpClient.SetTimeout(60 * time.Second) // RAGFlow AI 推理需要较长时间 globalClient = &Client{ BaseURL: strings.TrimSuffix(baseURL, "/"), diff --git a/ragflow/worker_pool.go b/ragflow/worker_pool.go index b836ef8..06f7878 100644 --- a/ragflow/worker_pool.go +++ b/ragflow/worker_pool.go @@ -2,58 +2,28 @@ package ragflow import ( "context" - "sync" "gitee.com/red-future---jilin-g/common/redis" - "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/os/grpool" ) -// WorkerPool RAGFlow 请求处理协程池 +// 默认协程池大小 +const defaultPoolSize = 200 + +// workerPool 协程池单例(grpool.New 是原型模式,需要变量引用) +var workerPool = grpool.New(defaultPoolSize) + +// WorkerPool RAGFlow 请求处理协程池(封装 grpool) type WorkerPool struct { pool *grpool.Pool size int } -// 单例模式相关变量 -var ( - workerPoolInstance *WorkerPool - workerPoolOnce sync.Once -) - -// GetWorkerPoolWithSize 获取指定大小的协程池单例 -// 使用 sync.Once 确保只创建一次,size 仅首次调用生效 -func GetWorkerPoolWithSize(size int) *WorkerPool { - workerPoolOnce.Do(func() { - if size <= 0 { - size = 200 // 默认大小 - } - workerPoolInstance = &WorkerPool{ - pool: grpool.New(size), - size: size, - } - }) - return workerPoolInstance -} - -// GetWorkerPool 获取协程池单例(使用默认大小 200) -func GetWorkerPool() *WorkerPool { - return GetWorkerPoolWithSize(200) -} - -// NewWorkerPool 创建协程池(兼容旧代码,内部使用单例) -// 参数: -// - size: 协程池大小,仅首次调用生效 -// -// 返回: -// - *WorkerPool: 协程池单例实例 -// - error: 创建失败时返回错误 -func NewWorkerPool(size int) (*WorkerPool, error) { - if size <= 0 { - return nil, gerror.New("协程池大小必须大于0") - } - return GetWorkerPoolWithSize(size), nil +// Pool 协程池单例实例(直接引用使用) +var Pool = &WorkerPool{ + pool: workerPool, + size: defaultPoolSize, } // Submit 提交任务到协程池 diff --git a/redis/redis.go b/redis/redis.go index 186d2ad..f90f3b1 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -3,38 +3,18 @@ package redis import ( "context" "strings" - "sync" - "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/util/gconv" ) -var ( - // redisClient 单例 Redis 客户端 - redisClient *gredis.Redis - // redisOnce 确保只初始化一次 - redisOnce sync.Once - // RedisClient 兼容导出(供 mongo.go 使用) - // 注意:这是一个指向单例的指针,首次调用 GetRedisClient() 后生效 - RedisClient *gredis.Redis -) +// redisClient 内部使用的 Redis 客户端(g.Redis() 是原型模式,需要变量引用) +var redisClient = g.Redis() -// GetRedisClient 获取 Redis 客户端(单例模式) -func GetRedisClient() *gredis.Redis { - redisOnce.Do(func() { - redisClient = g.Redis() - RedisClient = redisClient // 同步更新兼容导出 - }) - return redisClient -} - -// init 包初始化时自动初始化 Redis 客户端 -func init() { - GetRedisClient() -} +// RedisClient 导出的 Redis 客户端(供 mongo.go 使用) +var RedisClient = redisClient // Stream 和消费者组常量 const ( @@ -62,7 +42,7 @@ type StreamMessage struct { // 使用 gredis Do() 方法执行 XGROUP CREATE 命令 func InitStreamGroup(ctx context.Context, streamKey, groupName string) error { // XGROUP CREATE streamKey groupName 0 MKSTREAM - _, err := GetRedisClient().Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") + _, err := redisClient.Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") if err != nil { // 如果组已存在,忽略错误 errStr := err.Error() @@ -76,21 +56,25 @@ func InitStreamGroup(ctx context.Context, streamKey, groupName string) error { // AddToStream 将消息添加到 Stream // 使用 gredis Do() 方法执行 XADD 命令 -func AddToStream(ctx context.Context, streamKey string, values map[string]interface{}) (string, error) { +// msg 可以是结构体或 map,内部自动转换 +func AddToStream(ctx context.Context, streamKey string, msg interface{}) (messageID string, err error) { + // 将结构体转换为 map + values := gconv.Map(msg) + // XADD streamKey * field1 value1 field2 value2 ... - args := []interface{}{streamKey, "*"} // "*" 自动生成ID + args := make([]interface{}, 0, len(values)*2+2) + args = append(args, streamKey, "*") // "*" 自动生成ID for key, val := range values { args = append(args, key, val) } - result, err := GetRedisClient().Do(ctx, "XADD", args...) + result, err := redisClient.Do(ctx, "XADD", args...) if err != nil { - return "", err + return } - // 返回消息ID - messageID := result.String() - return messageID, nil + messageID = result.String() + return } // ReadFromStream 从 Stream 读取消息(消费者组模式) @@ -100,7 +84,7 @@ func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName stri groupName, consumerName, count, blockMs, streamKey) // XREADGROUP GROUP groupName consumerName COUNT count BLOCK blockMs STREAMS streamKey > - result, err := GetRedisClient().Do(ctx, + result, err := redisClient.Do(ctx, "XREADGROUP", "GROUP", groupName, consumerName, "COUNT", count, "BLOCK", blockMs, @@ -208,7 +192,7 @@ func AckMessage(ctx context.Context, streamKey, groupName string, messageIDs ... args = append(args, id) } - _, err := GetRedisClient().Do(ctx, "XACK", args...) + _, err := redisClient.Do(ctx, "XACK", args...) return err } @@ -216,7 +200,7 @@ func AckMessage(ctx context.Context, streamKey, groupName string, messageIDs ... // 使用 gredis Do() 方法执行 XLEN 命令 func GetStreamLength(ctx context.Context, streamKey string) (int64, error) { // XLEN streamKey - result, err := GetRedisClient().Do(ctx, "XLEN", streamKey) + result, err := redisClient.Do(ctx, "XLEN", streamKey) if err != nil { return 0, err } @@ -237,7 +221,7 @@ type PendingMessage struct { // 使用 gredis Do() 方法执行 XPENDING 命令 func GetPendingMessages(ctx context.Context, streamKey, groupName string, start, end string, count int64) ([]PendingMessage, error) { // XPENDING streamKey groupName start end count - result, err := GetRedisClient().Do(ctx, "XPENDING", streamKey, groupName, start, end, count) + result, err := redisClient.Do(ctx, "XPENDING", streamKey, groupName, start, end, count) if err != nil { return nil, err } @@ -279,7 +263,7 @@ func ClaimPendingMessage(ctx context.Context, streamKey, groupName, consumerName args = append(args, id) } - result, err := GetRedisClient().Do(ctx, "XCLAIM", args...) + result, err := redisClient.Do(ctx, "XCLAIM", args...) if err != nil { return nil, err } @@ -333,7 +317,7 @@ func SetSessionLastActive(ctx context.Context, userId string) error { timestamp := gtime.Now().Timestamp() // SETEX key 7200 value (7200秒 = 2小时) - _, err := GetRedisClient().Do(ctx, "SETEX", key, 7200, timestamp) + _, err := redisClient.Do(ctx, "SETEX", key, 7200, timestamp) return err } @@ -341,7 +325,7 @@ func SetSessionLastActive(ctx context.Context, userId string) error { // 使用 gredis Get 方法 func GetSessionLastActive(ctx context.Context, userId string) (int64, error) { key := SessionLastActiveKeyPrefix + userId + ":last_active" - result, err := GetRedisClient().Get(ctx, key) + result, err := redisClient.Get(ctx, key) if err != nil { return 0, err } @@ -383,7 +367,7 @@ func SetSessionCache(ctx context.Context, userId, sessionId string) error { key := SessionLastActiveKeyPrefix + userId + ":session_id" // SETEX key 604800 value (604800秒 = 7天) - _, err := GetRedisClient().Do(ctx, "SETEX", key, 604800, sessionId) + _, err := redisClient.Do(ctx, "SETEX", key, 604800, sessionId) return err } @@ -397,7 +381,7 @@ const ( // windowSeconds: 时间窗口(秒) func IncrRateLimit(ctx context.Context, key string, windowSeconds int64) (count int64, err error) { fullKey := RateLimitKeyPrefix + key - result, err := GetRedisClient().Do(ctx, "INCR", fullKey) + result, err := redisClient.Do(ctx, "INCR", fullKey) if err != nil { return } @@ -405,7 +389,7 @@ func IncrRateLimit(ctx context.Context, key string, windowSeconds int64) (count // 首次设置过期时间 if count == 1 { - GetRedisClient().Do(ctx, "EXPIRE", fullKey, windowSeconds) + redisClient.Do(ctx, "EXPIRE", fullKey, windowSeconds) } return } @@ -413,7 +397,7 @@ func IncrRateLimit(ctx context.Context, key string, windowSeconds int64) (count // GetRateLimit 获取当前限流计数 func GetRateLimit(ctx context.Context, key string) (count int64, err error) { fullKey := RateLimitKeyPrefix + key - result, err := GetRedisClient().Get(ctx, fullKey) + result, err := redisClient.Get(ctx, fullKey) if err != nil { return } @@ -425,10 +409,9 @@ func GetRateLimit(ctx context.Context, key string) (count int64, err error) { } // GetSessionCache 获取缓存的 RAGFlow Session ID -// 使用 gredis Get 方法 func GetSessionCache(ctx context.Context, userId string) (string, error) { key := SessionLastActiveKeyPrefix + userId + ":session_id" - result, err := GetRedisClient().Get(ctx, key) + result, err := redisClient.Get(ctx, key) if err != nil { return "", err } @@ -439,3 +422,10 @@ func GetSessionCache(ctx context.Context, userId string) (string, error) { return result.String(), nil } + +// DelSessionCache 删除缓存的 RAGFlow Session ID(归档时调用) +func DelSessionCache(ctx context.Context, userId string) error { + key := SessionLastActiveKeyPrefix + userId + ":session_id" + _, err := redisClient.Del(ctx, key) + return err +} diff --git a/redis/types.go b/redis/types.go index ecdce57..59bf280 100644 --- a/redis/types.go +++ b/redis/types.go @@ -1,21 +1,23 @@ package redis +import "gitee.com/red-future---jilin-g/common/config" + +// HistoryMessage 历史消息结构(用于上下文注入) +type HistoryMessage struct { + Question string `json:"question"` // 用户问题 + Answer string `json:"answer"` // AI 回复 +} + // SendStreamMessage 发送到 Redis Stream 的消息结构 type SendStreamMessage struct { - UserId string `json:"user_id"` // 用户ID - Content string `json:"content"` // 消息内容 - Timestamp int64 `json:"timestamp"` // 时间戳(秒) - MessageId string `json:"message_id"` // 消息唯一ID -} - -// ToMap 转换为 map[string]interface{} 用于 Stream 存储 -func (m *SendStreamMessage) ToMap() map[string]interface{} { - return map[string]interface{}{ - "user_id": m.UserId, - "content": m.Content, - "timestamp": m.Timestamp, - "message_id": m.MessageId, - } + UserId string `json:"user_id"` // 用户ID + Content string `json:"content"` // 消息内容 + Timestamp int64 `json:"timestamp"` // 时间戳(秒) + MessageId string `json:"message_id"` // 消息唯一ID + Platform string `json:"platform,omitempty"` // 平台标识 + AccountId string `json:"account_id,omitempty"` // 账号ID + TenantId string `json:"tenant_id,omitempty"` // 租户ID(数据隔离) + History []HistoryMessage `json:"history,omitempty"` // 历史对话(归档后恢复时携带) } // BatchStreamMessage 批量消息结构 @@ -27,21 +29,11 @@ type BatchStreamMessage struct { Index int `json:"index"` // 批次内序号 } -// ToMap 转换为 map[string]interface{} 用于 Stream 存储 -func (m *BatchStreamMessage) ToMap() map[string]interface{} { - return map[string]interface{}{ - "user_id": m.UserId, - "content": m.Content, - "timestamp": m.Timestamp, - "batch_id": m.BatchId, - "index": m.Index, - } -} - -// ResponseStreamMessage RAGFlow 响应消息结构(写入结果 Stream) +// ResponseStreamMessage RAGFlow 响应消息结构(MQ 消息) type ResponseStreamMessage struct { UserId string `json:"user_id"` // 用户ID Platform string `json:"platform"` // 平台标识 + TenantId string `json:"tenant_id"` // 租户ID Question string `json:"question"` // 用户问题 Content string `json:"content"` // RAGFlow 回复内容 SessionId string `json:"session_id"` // RAGFlow Session ID @@ -49,19 +41,6 @@ type ResponseStreamMessage struct { MessageId string `json:"message_id"` // 原始消息ID } -// ToMap 转换为 map[string]interface{} 用于 Stream 存储 -func (m *ResponseStreamMessage) ToMap() map[string]interface{} { - return map[string]interface{}{ - "user_id": m.UserId, - "platform": m.Platform, - "question": m.Question, - "content": m.Content, - "session_id": m.SessionId, - "timestamp": m.Timestamp, - "message_id": m.MessageId, - } -} - // FollowUpMessage 追问消息结构(RabbitMQ 延时队列) type FollowUpMessage struct { UserId string `json:"user_id"` // 用户ID @@ -71,25 +50,39 @@ type FollowUpMessage struct { Timestamp int64 `json:"timestamp"` // 发送时间戳 } -// 追问话术常量 +// 追问类型常量 const ( - FollowUpType1 = 1 // 30秒追问 - FollowUpType2 = 2 // 60秒追问 - FollowUpType3 = 3 // 180秒追问 + FollowUpType1 = 1 // 第一次追问 + FollowUpType2 = 2 // 第二次追问 + FollowUpType3 = 3 // 第三次追问 ) -// 追问话术内容 -var FollowUpContents = map[int]string{ - FollowUpType1: "还有其他问题吗?", - FollowUpType2: "如果需要帮助,随时告诉我~", - FollowUpType3: "我一直在线,有问题随时找我~", +// GetFollowUpContent 获取追问话术(从 config 包读取) +func GetFollowUpContent(followUpType int) string { + switch followUpType { + case FollowUpType1: + return config.FollowUpContent1 + case FollowUpType2: + return config.FollowUpContent2 + case FollowUpType3: + return config.FollowUpContent3 + default: + return "" + } } -// 追问延时时间(秒) -var FollowUpDelays = map[int]int{ - FollowUpType1: 30, - FollowUpType2: 60, - FollowUpType3: 180, +// GetFollowUpDelay 获取追问延时(从 config 包读取) +func GetFollowUpDelay(followUpType int) int { + switch followUpType { + case FollowUpType1: + return config.FollowUpDelay1 + case FollowUpType2: + return config.FollowUpDelay2 + case FollowUpType3: + return config.FollowUpDelay3 + default: + return 0 + } } // ArchiveMessage 会话归档消息结构(RabbitMQ 延时队列) @@ -100,5 +93,12 @@ type ArchiveMessage struct { Timestamp int64 `json:"timestamp"` // 发送时间戳 } -// 归档延时时间(秒) -const ArchiveDelaySeconds = 3600 // 60分钟 +// GetArchiveDelay 获取归档延时(从 config 包读取) +func GetArchiveDelay() int { + return config.ArchiveDelay +} + +// GetHistoryContextLimit 获取历史上下文轮数(从 config 包读取) +func GetHistoryContextLimit() int64 { + return config.HistoryContextLimit +} From 43a8834c5a121e0cf42c6f2fa74e94db948e0e86 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 9 Dec 2025 17:55:08 +0800 Subject: [PATCH 058/225] =?UTF-8?q?=E4=B8=8D=E5=90=8C=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E6=B3=A8=E5=86=8C=E4=B8=8D=E5=90=8C=E7=BB=84=E4=BB=B6=E6=A8=A1?= =?UTF-8?q?=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/consul.go | 31 +++++++--- elasticsearch/client.go | 116 ++++++++++++++++++++++++++++++++++++++ jaeger/jaeger.go | 39 ++++++++++--- ragflow/client.go | 51 +++++++++-------- ragflow/session.go | 7 ++- redis/redis.go | 26 +++++++-- startup/startup.go | 122 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 347 insertions(+), 45 deletions(-) create mode 100644 elasticsearch/client.go create mode 100644 startup/startup.go diff --git a/consul/consul.go b/consul/consul.go index 415a137..7bf8b7c 100644 --- a/consul/consul.go +++ b/consul/consul.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net" + "sync" "github.com/gogf/gf/contrib/registry/consul/v2" "github.com/gogf/gf/v2/frame/g" @@ -13,14 +14,30 @@ import ( "github.com/gogf/gf/v2/util/grand" ) +var initOnce sync.Once + +// Init 初始化 Consul 注册中心(延迟初始化,首次调用时执行) +func Init() { + initOnce.Do(func() { + consulAddr := g.Cfg().MustGet(context.Background(), "consul.address").String() + if consulAddr == "" { + g.Log().Warning(context.Background(), "⚠️ Consul 配置未找到,跳过初始化") + return + } + registry, err := consul.New(consul.WithAddress(consulAddr)) + if err != nil { + g.Log().Errorf(context.Background(), "Consul 初始化失败: %v", err) + return + } + gsvc.SetRegistry(registry) + gsel.SetBuilder(gsel.NewBuilderRoundRobin()) + g.Log().Infof(context.Background(), "✅ Consul 初始化成功: %s", consulAddr) + }) +} + func init() { - consulAddr := g.Cfg().MustGet(context.Background(), "consul.address").String() - registry, err := consul.New(consul.WithAddress(consulAddr)) - if err != nil { - panic(err) - } - gsvc.SetRegistry(registry) - gsel.SetBuilder(gsel.NewBuilderRoundRobin()) + // 默认自动初始化(保持向后兼容) + Init() } func getLocalIP() (string, error) { // 获取本机所有网络接口 diff --git a/elasticsearch/client.go b/elasticsearch/client.go new file mode 100644 index 0000000..ead9d1b --- /dev/null +++ b/elasticsearch/client.go @@ -0,0 +1,116 @@ +package elasticsearch + +import ( + "context" + "sync" + + "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" + "github.com/olivere/elastic/v7" +) + +var ( + client *elastic.Client + clientOnce sync.Once +) + +// Config ES 配置 +type Config struct { + Addresses []string // ES 地址列表 + Username string // 用户名 + Password string // 密码 +} + +// Init 初始化 ES 客户端(单例) +func Init(ctx context.Context) (err error) { + clientOnce.Do(func() { + addresses := g.Cfg().MustGet(ctx, "elasticsearch.addresses", []string{"http://localhost:9200"}).Strings() + username := g.Cfg().MustGet(ctx, "elasticsearch.username", "").String() + password := g.Cfg().MustGet(ctx, "elasticsearch.password", "").String() + + options := []elastic.ClientOptionFunc{ + elastic.SetURL(addresses...), + elastic.SetSniff(false), // 禁用嗅探,避免容器环境问题 + } + + if username != "" && password != "" { + options = append(options, elastic.SetBasicAuth(username, password)) + } + + client, err = elastic.NewClient(options...) + if err != nil { + glog.Errorf(ctx, "ES 客户端初始化失败: %v", err) + return + } + + // 测试连接 + info, code, err := client.Ping(addresses[0]).Do(ctx) + if err != nil { + glog.Errorf(ctx, "ES 连接测试失败: %v", err) + return + } + glog.Infof(ctx, "ES 连接成功 - 版本: %s, 状态码: %d", info.Version.Number, code) + }) + return +} + +// GetClient 获取 ES 客户端 +func GetClient() *elastic.Client { + return client +} + +// BulkIndex 批量写入文档 +func BulkIndex(ctx context.Context, indexName string, docs []interface{}) (err error) { + if client == nil { + return gerror.New("ES 客户端未初始化") + } + + bulk := client.Bulk().Index(indexName) + for _, doc := range docs { + bulk.Add(elastic.NewBulkIndexRequest().Doc(doc)) + } + + resp, err := bulk.Do(ctx) + if err != nil { + return + } + + if resp.Errors { + for _, item := range resp.Failed() { + glog.Errorf(ctx, "ES 写入失败 - Index: %s, Error: %s", item.Index, item.Error.Reason) + } + } + + glog.Infof(ctx, "ES 批量写入完成 - 索引: %s, 成功: %d, 失败: %d", + indexName, len(resp.Succeeded()), len(resp.Failed())) + return +} + +// CreateIndexIfNotExists 创建索引(如果不存在) +func CreateIndexIfNotExists(ctx context.Context, indexName, mapping string) (err error) { + if client == nil { + return gerror.New("ES 客户端未初始化") + } + + exists, err := client.IndexExists(indexName).Do(ctx) + if err != nil { + return + } + + if !exists { + _, err = client.CreateIndex(indexName).BodyString(mapping).Do(ctx) + if err != nil { + return + } + glog.Infof(ctx, "ES 索引创建成功: %s", indexName) + } + return +} + +// Close 关闭客户端 +func Close() { + if client != nil { + client.Stop() + } +} diff --git a/jaeger/jaeger.go b/jaeger/jaeger.go index 31e50e4..9acc586 100644 --- a/jaeger/jaeger.go +++ b/jaeger/jaeger.go @@ -5,6 +5,7 @@ import ( "encoding/json" "strconv" "strings" + "sync" "github.com/gogf/gf/contrib/trace/otlphttp/v2" "github.com/gogf/gf/v2/frame/g" @@ -13,16 +14,38 @@ import ( "go.opentelemetry.io/otel/attribute" ) -var ShutDown func(ctx context.Context) +var ( + ShutDown func(ctx context.Context) + initOnce sync.Once +) + +// Init 初始化 Jaeger 链路追踪(延迟初始化,首次调用时执行) +func Init() { + initOnce.Do(func() { + ctx := context.Background() + jaegerAgent := g.Cfg().MustGet(ctx, "jaeger.addr").String() + serverName := g.Cfg().MustGet(ctx, "server.name").String() + + if jaegerAgent == "" { + g.Log().Warning(ctx, "⚠️ Jaeger 配置未找到,跳过初始化") + ShutDown = func(ctx context.Context) {} // 空函数,避免 nil panic + return + } + + shutdown, err := otlphttp.Init(serverName, jaegerAgent, "/v1/traces") + if err != nil { + g.Log().Errorf(ctx, "Jaeger 初始化失败: %v", err) + ShutDown = func(ctx context.Context) {} + return + } + ShutDown = shutdown + g.Log().Infof(ctx, "✅ Jaeger 初始化成功: %s", jaegerAgent) + }) +} func init() { - jaegerAgent := g.Cfg().MustGet(context.Background(), "jaeger.addr").String() - serverName := g.Cfg().MustGet(context.Background(), "server.name").String() - shutdown, err := otlphttp.Init(serverName, jaegerAgent, "/v1/traces") - if err != nil { - panic(err) - } - ShutDown = shutdown + // 默认自动初始化(保持向后兼容) + Init() } func NewTracer(r *ghttp.Request) { _, span := gtrace.NewSpan(r.Context(), r.GetServeHandler().GetMetaTag("summary")) diff --git a/ragflow/client.go b/ragflow/client.go index 618d84b..e19bd94 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -5,6 +5,7 @@ import ( "net/http" "net/url" "strings" + "sync" "time" "github.com/gogf/gf/v2/encoding/gjson" @@ -14,36 +15,39 @@ import ( ) var ( - // globalClient 全局 RAGFlow 客户端(单例,自动初始化) + // globalClient 全局 RAGFlow 客户端(单例,延迟初始化) globalClient *Client + clientOnce sync.Once ) -// init 包初始化时自动创建全局客户端 -func init() { - ctx := context.Background() +// initClient 延迟初始化客户端 +func initClient() { + clientOnce.Do(func() { + ctx := context.Background() - // 读取配置 - baseURL, apiKey := loadConfig(ctx) + // 读取配置 + baseURL, apiKey := loadConfig(ctx) - // 如果配置不完整,跳过初始化 - if baseURL == "" || apiKey == "" { - g.Log().Warning(ctx, "⚠️ RAGFlow 配置未找到,请在项目 config.yml 中添加 ragflow.base_url 和 ragflow.api_key") - return - } + // 如果配置不完整,跳过初始化 + if baseURL == "" || apiKey == "" { + g.Log().Warning(ctx, "⚠️ RAGFlow 配置未找到,请在项目 config.yml 中添加 ragflow.base_url 和 ragflow.api_key") + return + } - // 初始化全局客户端 - httpClient := gclient.New() - httpClient.SetHeader("Authorization", "Bearer "+apiKey) - httpClient.SetHeader("Content-Type", "application/json") - httpClient.SetTimeout(60 * time.Second) // RAGFlow AI 推理需要较长时间 + // 初始化全局客户端 + httpClient := gclient.New() + httpClient.SetHeader("Authorization", "Bearer "+apiKey) + httpClient.SetHeader("Content-Type", "application/json") + httpClient.SetTimeout(180 * time.Second) // RAGFlow AI 推理需要较长时间 - globalClient = &Client{ - BaseURL: strings.TrimSuffix(baseURL, "/"), - APIKey: apiKey, - HTTPClient: httpClient, - } + globalClient = &Client{ + BaseURL: strings.TrimSuffix(baseURL, "/"), + APIKey: apiKey, + HTTPClient: httpClient, + } - g.Log().Infof(ctx, "✅ RAGFlow 全局客户端初始化成功: baseURL=%s", baseURL) + g.Log().Infof(ctx, "✅ RAGFlow 全局客户端初始化成功: baseURL=%s", baseURL) + }) } // loadConfig 从配置文件加载 RAGFlow 配置 @@ -54,9 +58,10 @@ func loadConfig(ctx context.Context) (baseURL, apiKey string) { return } -// GetGlobalClient 获取全局客户端 +// GetGlobalClient 获取全局客户端(延迟初始化) // 使用示例:client := ragflow.GetGlobalClient() func GetGlobalClient() *Client { + initClient() return globalClient } diff --git a/ragflow/session.go b/ragflow/session.go index ee82b5d..d534b04 100644 --- a/ragflow/session.go +++ b/ragflow/session.go @@ -65,8 +65,9 @@ type ChatCompletionReq struct { // ChatCompletionRes 对话响应 (非流式) type ChatCompletionRes struct { - Code int `json:"code"` - Data struct { + Code int `json:"code"` + Message string `json:"message"` // 错误信息 + Data struct { Answer string `json:"answer"` Reference interface{} `json:"reference"` AudioBinary interface{} `json:"audio_binary"` @@ -163,7 +164,7 @@ func (c *Client) ChatCompletion(ctx context.Context, chatId string, req *ChatCom return nil, err } if res.Code != 0 { - return nil, gerror.Newf("chat completion failed: code=%d", res.Code) + return nil, gerror.Newf("chat completion failed: code=%d, message=%s", res.Code, res.Message) } return &res, nil } diff --git a/redis/redis.go b/redis/redis.go index f90f3b1..a74df47 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -3,18 +3,36 @@ package redis import ( "context" "strings" + "sync" + "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/util/gconv" ) -// redisClient 内部使用的 Redis 客户端(g.Redis() 是原型模式,需要变量引用) -var redisClient = g.Redis() +var ( + // redisClient 内部使用的 Redis 客户端(单例模式) + redisClient *gredis.Redis + redisOnce sync.Once +) -// RedisClient 导出的 Redis 客户端(供 mongo.go 使用) -var RedisClient = redisClient +// getClient 获取 Redis 客户端(延迟初始化) +func getClient() *gredis.Redis { + redisOnce.Do(func() { + redisClient = g.Redis() + }) + return redisClient +} + +// GetRedisClient 获取 Redis 客户端(供外部使用) +func GetRedisClient() *gredis.Redis { + return getClient() +} + +// RedisClient 导出的 Redis 客户端(供 mongo.go 使用,兼容旧代码) +var RedisClient = getClient() // Stream 和消费者组常量 const ( diff --git a/startup/startup.go b/startup/startup.go new file mode 100644 index 0000000..2fc5882 --- /dev/null +++ b/startup/startup.go @@ -0,0 +1,122 @@ +// Package startup 提供服务启动时的组件初始化控制 +// 各服务可以按需初始化所需组件,避免不必要的资源占用 +package startup + +import ( + "context" + "sync" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" +) + +// Components 组件配置 +type Components struct { + Consul bool // Consul 服务注册发现(所有服务都需要) + Jaeger bool // Jaeger 链路追踪(所有服务都需要) + Redis bool // Redis 缓存 + RabbitMQ bool // RabbitMQ 消息队列 + MongoDB bool // MongoDB 数据库 + RAGFlow bool // RAGFlow AI 客户端 + ES bool // Elasticsearch +} + +var ( + initialized bool + initOnce sync.Once + components *Components +) + +// Init 初始化指定的组件 +// 示例: +// +// bootstrap.Init(ctx, &bootstrap.Components{ +// Consul: true, +// Jaeger: true, +// Redis: true, +// RabbitMQ: true, +// }) +func Init(ctx context.Context, c *Components) { + initOnce.Do(func() { + components = c + initialized = true + glog.Infof(ctx, "Bootstrap 初始化完成: %+v", c) + }) +} + +// IsInitialized 检查是否已初始化 +func IsInitialized() bool { + return initialized +} + +// GetComponents 获取组件配置 +func GetComponents() *Components { + if components == nil { + // 默认配置:从配置文件读取 + return loadFromConfig() + } + return components +} + +// NeedRedis 是否需要 Redis +func NeedRedis() bool { + c := GetComponents() + return c != nil && c.Redis +} + +// NeedRabbitMQ 是否需要 RabbitMQ +func NeedRabbitMQ() bool { + c := GetComponents() + return c != nil && c.RabbitMQ +} + +// NeedMongoDB 是否需要 MongoDB +func NeedMongoDB() bool { + c := GetComponents() + return c != nil && c.MongoDB +} + +// NeedRAGFlow 是否需要 RAGFlow +func NeedRAGFlow() bool { + c := GetComponents() + return c != nil && c.RAGFlow +} + +// NeedES 是否需要 Elasticsearch +func NeedES() bool { + c := GetComponents() + return c != nil && c.ES +} + +// loadFromConfig 从配置文件加载组件配置 +// 如果配置文件中没有 startup 配置,则默认全部启动 +func loadFromConfig() *Components { + ctx := context.Background() + + // 检查是否有 startup 配置节 + startupCfg := g.Cfg().MustGet(ctx, "startup") + if startupCfg.IsEmpty() { + // 没有配置 startup,默认全部启动 + glog.Debug(ctx, "未找到 startup 配置,默认启动所有组件") + return &Components{ + Consul: true, + Jaeger: true, + Redis: true, + RabbitMQ: true, + MongoDB: true, + RAGFlow: true, + ES: true, + } + } + + // 有配置则按配置来,未配置的项默认 true + return &Components{ + Consul: g.Cfg().MustGet(ctx, "startup.consul", true).Bool(), + Jaeger: g.Cfg().MustGet(ctx, "startup.jaeger", true).Bool(), + Redis: g.Cfg().MustGet(ctx, "startup.redis", true).Bool(), + RabbitMQ: g.Cfg().MustGet(ctx, "startup.rabbitmq", true).Bool(), + MongoDB: g.Cfg().MustGet(ctx, "startup.mongodb", true).Bool(), + RAGFlow: g.Cfg().MustGet(ctx, "startup.ragflow", true).Bool(), + ES: g.Cfg().MustGet(ctx, "startup.es", true).Bool(), + } +} From 1be0709580ac121d70d48616b18d86d12eb7bafc Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Wed, 10 Dec 2025 09:50:54 +0800 Subject: [PATCH 059/225] =?UTF-8?q?client=20=E8=B6=85=E6=97=B6=E6=97=B6?= =?UTF-8?q?=E9=95=BF=E4=BF=AE=E6=94=B9=20,?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + ragflow/client.go | 46 +++++++++++++++++++++++++++++++++++++++------- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 406a7f0..aeb617a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ rabbitmq/开发指南.md ragflow/agent文档.md ragflow/README_GLOBAL.md redis/stream使用示例.md +ragflow/client_http.go diff --git a/ragflow/client.go b/ragflow/client.go index e19bd94..612b2a7 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -2,6 +2,7 @@ package ragflow import ( "context" + "net" "net/http" "net/url" "strings" @@ -34,12 +35,32 @@ func initClient() { return } + // 自定义 Transport,增大连接池(解决并发连接不足导致的超时) + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 200, // 最大空闲连接数 + MaxIdleConnsPerHost: 100, // 每个 host 最大空闲连接数(关键!默认只有 2) + MaxConnsPerHost: 100, // 每个 host 最大连接数 + IdleConnTimeout: 90 * time.Second, // 空闲连接超时 + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + ResponseHeaderTimeout: 180 * time.Second, // 等待响应头超时(关键!) + } + // 初始化全局客户端 httpClient := gclient.New() + httpClient.SetBrowserMode(false) httpClient.SetHeader("Authorization", "Bearer "+apiKey) httpClient.SetHeader("Content-Type", "application/json") httpClient.SetTimeout(180 * time.Second) // RAGFlow AI 推理需要较长时间 + // 设置自定义 Transport + httpClient.Client.Transport = transport + globalClient = &Client{ BaseURL: strings.TrimSuffix(baseURL, "/"), APIKey: apiKey, @@ -97,32 +118,43 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac reqBody = string(jsonData) } + // 设置 180 秒超时(RAGFlow AI 推理需要较长时间) + reqCtx, cancel := context.WithTimeout(ctx, 180*time.Second) + defer cancel() + var resp *gclient.Response switch method { case "GET": - resp, err = c.HTTPClient.Get(ctx, fullURL) + resp, err = c.HTTPClient.Get(reqCtx, fullURL) case "POST": - resp, err = c.HTTPClient.Post(ctx, fullURL, reqBody) + resp, err = c.HTTPClient.Post(reqCtx, fullURL, reqBody) case "PUT": - resp, err = c.HTTPClient.Put(ctx, fullURL, reqBody) + resp, err = c.HTTPClient.Put(reqCtx, fullURL, reqBody) case "DELETE": - resp, err = c.HTTPClient.Delete(ctx, fullURL, reqBody) + resp, err = c.HTTPClient.Delete(reqCtx, fullURL, reqBody) default: return gerror.Newf("unsupported method: %s", method) } if err != nil { - return gerror.Newf("http request failed: %v", err) + g.Log().Errorf(ctx, "[RAGFlow HTTP] 请求失败: method=%s, url=%s, error=%v", method, fullURL, err) + return gerror.Newf("request failed: %v", err) } defer resp.Close() + respBody := resp.ReadAll() + + // 打印响应详情 + g.Log().Debugf(ctx, "[RAGFlow HTTP] 响应: status=%d, body=%s", resp.StatusCode, string(respBody)) + if resp.StatusCode != http.StatusOK { - return gerror.Newf("http request failed with status: %d", resp.StatusCode) + g.Log().Errorf(ctx, "[RAGFlow HTTP] 非200响应: status=%d, body=%s", resp.StatusCode, string(respBody)) + return gerror.Newf("http status %d: %s", resp.StatusCode, string(respBody)) } - respBody := resp.ReadAll() if err = gjson.DecodeTo(respBody, result); err != nil { + g.Log().Errorf(ctx, "[RAGFlow HTTP] 解析响应失败: body=%s, error=%v", string(respBody), err) return gerror.Newf("unmarshal response failed: %v", err) } From 5e3a7f30f7ace8d1909ed39a1b9f38714172b902 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Wed, 10 Dec 2025 18:02:31 +0800 Subject: [PATCH 060/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0es=E5=BD=92=E6=A1=A3?= =?UTF-8?q?=20=E5=88=86=E5=B8=83=E5=BC=8F=E5=92=8Cconstants=E5=8F=98?= =?UTF-8?q?=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 +- config/constants.go | 8 ++ mongo/mongo.go | 57 ++++++++++-- ragflow/client.go | 73 +++++++++------- ragflow/worker_pool.go | 192 ++++++++++++----------------------------- redis/redis.go | 28 ++++++ 6 files changed, 186 insertions(+), 174 deletions(-) diff --git a/.gitignore b/.gitignore index aeb617a..5e12a7a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ rabbitmq/开发指南.md ragflow/agent文档.md ragflow/README_GLOBAL.md redis/stream使用示例.md -ragflow/client_http.go +ragflow/client_backup.go.bak diff --git a/config/constants.go b/config/constants.go index 0cf2867..8963e8d 100644 --- a/config/constants.go +++ b/config/constants.go @@ -32,3 +32,11 @@ var ArchiveDelay = 3600 // HistoryContextLimit 读取历史对话轮数(用于新 Session 上下文注入) var HistoryContextLimit int64 = 5 + +// -------------------- Stream 消费配置 -------------------- + +// DefaultBatchSize 批量读取消息数量(削峰填谷) +var DefaultBatchSize int64 = 200 + +// DefaultBlockTimeout 阻塞超时时间(毫秒) +var DefaultBlockTimeout int64 = 2000 diff --git a/mongo/mongo.go b/mongo/mongo.go index 8a43cf2..f1f9eb9 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -8,6 +8,7 @@ import ( "time" "gitee.com/red-future---jilin-g/common/consts" + "gitee.com/red-future---jilin-g/common/do" "gitee.com/red-future---jilin-g/common/redis" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/errors/gerror" @@ -21,7 +22,12 @@ import ( "go.mongodb.org/mongo-driver/v2/mongo/options" ) -var db = new(mongo.Database) +var db *mongo.Database + +// GetDB 获取 MongoDB 数据库实例 +func GetDB() *mongo.Database { + return db +} func init() { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -88,12 +94,49 @@ func oneOptionsToMap(ctx context.Context, opts ...options.Lister[options.FindOne return } +// GetTenantInfo 获取租户信息 +// 优先从 token 获取,失败则从请求参数 customerServiceId 查询 customer_service_account 表 +func GetTenantInfo(ctx context.Context) (user do.User, err error) { + // 1. 优先从 token 获取 + user, err = utils.GetUserInfo(ctx) + if err == nil { + return + } + + // 2. token 获取失败,尝试从请求参数获取 customerServiceId + req := g.RequestFromCtx(ctx) + if req == nil { + return user, gerror.New("无法获取租户信息:无 token 且无 request") + } + + customerServiceId := req.Get("customerServiceId").String() + if customerServiceId == "" { + customerServiceId = req.Get("customer_service_id").String() + } + if customerServiceId == "" { + return user, gerror.New("无法获取租户信息:无 token 且无 customerServiceId 参数") + } + + // 3. 直接查询 customer_service_account 表获取 tenantId + filter := bson.M{"customerServiceId": customerServiceId, "isDeleted": false} + var account struct { + TenantId interface{} `bson:"tenantId"` + } + if findErr := db.Collection("customer_service_account").FindOne(ctx, filter).Decode(&account); findErr != nil { + return user, gerror.Newf("通过 customerServiceId 查询租户失败: %v", findErr) + } + + user.TenantId = account.TenantId + user.UserName = customerServiceId + return +} + // Find 查询多条记录 func Find(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOptions]) (err error) { if err = utils.ValidStructPtr(result); err != nil { return } - user, err := utils.GetUserInfo(ctx) + user, err := GetTenantInfo(ctx) if err != nil { return } @@ -135,7 +178,7 @@ func FindOne(ctx context.Context, filter bson.M, result interface{}, collection if err = utils.ValidStructPtr(result); err != nil { return } - user, err := utils.GetUserInfo(ctx) + user, err := GetTenantInfo(ctx) if err != nil { return } @@ -198,7 +241,7 @@ func Delete(ctx context.Context, filter bson.M, collection string, opts ...optio err = gerror.New("缺少查询条件") return } - user, err := utils.GetUserInfo(ctx) + user, err := GetTenantInfo(ctx) if err != nil { return } @@ -219,7 +262,7 @@ func Update(ctx context.Context, filter bson.M, update bson.M, collection string return } filter["isDeleted"] = false - user, err := utils.GetUserInfo(ctx) + user, err := GetTenantInfo(ctx) if err != nil { return } @@ -238,7 +281,7 @@ func Update(ctx context.Context, filter bson.M, update bson.M, collection string // Insert 插入多条记录 func Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { - user, err := utils.GetUserInfo(ctx) + user, err := GetTenantInfo(ctx) if err != nil { return } @@ -265,7 +308,7 @@ func Insert(ctx context.Context, documents []interface{}, collection string, opt // Count 查询总数 func Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { - user, err := utils.GetUserInfo(ctx) + user, err := GetTenantInfo(ctx) if err != nil { return } diff --git a/ragflow/client.go b/ragflow/client.go index 612b2a7..a5f64b2 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -1,7 +1,9 @@ package ragflow import ( + "bytes" "context" + "io" "net" "net/http" "net/url" @@ -12,9 +14,13 @@ import ( "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/net/gclient" ) +// gclient 完全不能用! +// 1. New() 默认 ResponseHeaderTimeout=30s +// 2. Clone() 内部调用 New(),链式调用会重置 Transport +// 3. 必须用原生 http.Client + var ( // globalClient 全局 RAGFlow 客户端(单例,延迟初始化) globalClient *Client @@ -51,15 +57,15 @@ func initClient() { ResponseHeaderTimeout: 180 * time.Second, // 等待响应头超时(关键!) } - // 初始化全局客户端 - httpClient := gclient.New() - httpClient.SetBrowserMode(false) - httpClient.SetHeader("Authorization", "Bearer "+apiKey) - httpClient.SetHeader("Content-Type", "application/json") - httpClient.SetTimeout(180 * time.Second) // RAGFlow AI 推理需要较长时间 + // 使用原生 http.Client(gclient 完全不能用,Clone() 内部调用 New() 会重置 Transport) + httpClient := &http.Client{ + Transport: transport, + Timeout: 0, // 不设置全局超时,由 context 控制 + } - // 设置自定义 Transport - httpClient.Client.Transport = transport + // 验证 Transport 设置 + g.Log().Infof(ctx, "✅ Transport 配置: ResponseHeaderTimeout=%v, MaxIdleConnsPerHost=%d, DisableKeepAlives=%v", + transport.ResponseHeaderTimeout, transport.MaxIdleConnsPerHost, transport.DisableKeepAlives) globalClient = &Client{ BaseURL: strings.TrimSuffix(baseURL, "/"), @@ -90,7 +96,7 @@ func GetGlobalClient() *Client { type Client struct { BaseURL string APIKey string - HTTPClient *gclient.Client // HTTP 客户端 + HTTPClient *http.Client // 原生 HTTP 客户端(gclient 不能用) } // CommonResponse 通用响应结构 @@ -118,39 +124,44 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac reqBody = string(jsonData) } - // 设置 180 秒超时(RAGFlow AI 推理需要较长时间) - reqCtx, cancel := context.WithTimeout(ctx, 180*time.Second) + // 使用独立的 context 设置 300 秒超时(RAGFlow 高并发时响应较慢) + reqCtx, cancel := context.WithTimeout(context.Background(), 300*time.Second) defer cancel() + startTime := time.Now() - var resp *gclient.Response - - switch method { - case "GET": - resp, err = c.HTTPClient.Get(reqCtx, fullURL) - case "POST": - resp, err = c.HTTPClient.Post(reqCtx, fullURL, reqBody) - case "PUT": - resp, err = c.HTTPClient.Put(reqCtx, fullURL, reqBody) - case "DELETE": - resp, err = c.HTTPClient.Delete(reqCtx, fullURL, reqBody) - default: - return gerror.Newf("unsupported method: %s", method) + // 创建请求 + req, err := http.NewRequestWithContext(reqCtx, method, fullURL, bytes.NewReader([]byte(reqBody))) + if err != nil { + return gerror.Newf("create request failed: %v", err) } + // 设置请求头 + req.Header.Set("Authorization", "Bearer "+c.APIKey) + req.Header.Set("Content-Type", "application/json") + + // 发送请求 + g.Log().Infof(ctx, "[RAGFlow HTTP] 发送请求: method=%s, url=%s", method, fullURL) + resp, err := c.HTTPClient.Do(req) + elapsed := time.Since(startTime) if err != nil { - g.Log().Errorf(ctx, "[RAGFlow HTTP] 请求失败: method=%s, url=%s, error=%v", method, fullURL, err) + g.Log().Errorf(ctx, "[RAGFlow HTTP] 请求失败(耗时 %v): method=%s, url=%s, error=%v", elapsed, method, fullURL, err) return gerror.Newf("request failed: %v", err) } - defer resp.Close() + g.Log().Infof(ctx, "[RAGFlow HTTP] 收到响应(耗时 %v): status=%d, url=%s", elapsed, resp.StatusCode, fullURL) + defer resp.Body.Close() - respBody := resp.ReadAll() + // 读取响应 + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return gerror.Newf("read response failed: %v", err) + } // 打印响应详情 - g.Log().Debugf(ctx, "[RAGFlow HTTP] 响应: status=%d, body=%s", resp.StatusCode, string(respBody)) + g.Log().Debugf(ctx, "[RAGFlow HTTP] 响应: status=%d, body=%s", resp.StatusCode, respBody) if resp.StatusCode != http.StatusOK { - g.Log().Errorf(ctx, "[RAGFlow HTTP] 非200响应: status=%d, body=%s", resp.StatusCode, string(respBody)) - return gerror.Newf("http status %d: %s", resp.StatusCode, string(respBody)) + g.Log().Errorf(ctx, "[RAGFlow HTTP] 非200响应: status=%d, body=%s", resp.StatusCode, respBody) + return gerror.Newf("http status %d: %s", resp.StatusCode, respBody) } if err = gjson.DecodeTo(respBody, result); err != nil { diff --git a/ragflow/worker_pool.go b/ragflow/worker_pool.go index 06f7878..98a1b48 100644 --- a/ragflow/worker_pool.go +++ b/ragflow/worker_pool.go @@ -2,189 +2,111 @@ package ragflow import ( "context" + "strings" + "time" "gitee.com/red-future---jilin-g/common/redis" "github.com/gogf/gf/v2/os/glog" - "github.com/gogf/gf/v2/os/grpool" ) -// 默认协程池大小 -const defaultPoolSize = 200 +// 默认批量大小(每次从 Redis 读取并发送的消息数) +const defaultBatchSize = 200 -// workerPool 协程池单例(grpool.New 是原型模式,需要变量引用) -var workerPool = grpool.New(defaultPoolSize) - -// WorkerPool RAGFlow 请求处理协程池(封装 grpool) -type WorkerPool struct { - pool *grpool.Pool - size int -} - -// Pool 协程池单例实例(直接引用使用) -var Pool = &WorkerPool{ - pool: workerPool, - size: defaultPoolSize, -} - -// Submit 提交任务到协程池 -// 参数: -// - ctx: 上下文 -// - task: 要执行的任务函数 -// -// 返回:error 提交失败时返回错误 -func (w *WorkerPool) Submit(ctx context.Context, task func(ctx context.Context)) error { - return w.pool.Add(ctx, func(ctx context.Context) { - defer func() { - if r := recover(); r != nil { - glog.Errorf(ctx, "协程池任务执行 panic: %v", r) - } - }() - - task(ctx) - }) -} - -// Size 获取协程池大小 -func (w *WorkerPool) Size() int { - return w.size -} - -// Jobs 获取当前等待执行的任务数量 -func (w *WorkerPool) Jobs() int { - return w.pool.Jobs() -} - -// Close 关闭协程池 -func (w *WorkerPool) Close() { - w.pool.Close() -} - -// WorkerStats 协程池统计信息 -type WorkerStats struct { - PoolSize int // 协程池大小 - Jobs int // 等待执行的任务数 -} - -// Stats 获取协程池统计信息 -func (w *WorkerPool) Stats() WorkerStats { - return WorkerStats{ - PoolSize: w.size, - Jobs: w.pool.Jobs(), - } -} - -// PrintStats 打印协程池统计信息 -func (w *WorkerPool) PrintStats(ctx context.Context) { - stats := w.Stats() - glog.Infof(ctx, "协程池统计 - 池大小: %d, 等待任务: %d", stats.PoolSize, stats.Jobs) -} - -// QueueProcessor Stream 处理器,从 Redis Stream 中取出任务并提交到协程池 +// QueueProcessor Stream 处理器,批量读取消息并发送到 RAGFlow type QueueProcessor struct { - pool *WorkerPool - streamKey string // Stream 键名 - groupName string // 消费者组名称 - consumerName string // 消费者名称 - timeout int64 // 阻塞超时时间(毫秒) - batchSize int64 // 每次读取的消息数量 - stopChan chan struct{} + streamKey string // Stream 键名 + groupName string // 消费者组名称 + consumerName string // 消费者名称 + timeout int64 // 阻塞超时时间(毫秒) + batchSize int64 // 最大并发数(信号量容量) + stopChan chan struct{} // 停止信号 + semaphore chan struct{} // 并发信号量(控制最大并发) handleFunc func(ctx context.Context, message map[string]interface{}) error } // NewQueueProcessor 创建 Stream 处理器 -// 参数: -// - pool: 协程池 -// - streamKey: Redis Stream 键名 -// - groupName: 消费者组名称 -// - consumerName: 消费者名称(唯一标识) -// - timeout: 从 Stream 取消息的超时时间(毫秒) -// - batchSize: 每次读取的消息数量 -// - handleFunc: 消息处理函数 -func NewQueueProcessor(pool *WorkerPool, streamKey, groupName, consumerName string, timeout int64, batchSize int64, handleFunc func(ctx context.Context, message map[string]interface{}) error) *QueueProcessor { +func NewQueueProcessor(streamKey, groupName, consumerName string, timeout, batchSize int64, handleFunc func(ctx context.Context, message map[string]interface{}) error) *QueueProcessor { return &QueueProcessor{ - pool: pool, streamKey: streamKey, groupName: groupName, consumerName: consumerName, timeout: timeout, batchSize: batchSize, stopChan: make(chan struct{}), + semaphore: make(chan struct{}, batchSize), // 信号量容量 = 最大并发数 handleFunc: handleFunc, } } // Start 启动 Stream 处理器 -// 会阻塞运行,持续从 Redis Stream 中取出消息并提交到协程池处理 +// 削峰填谷:每次读取 batchSize 条消息,并发发送,发完立刻读下一批 func (q *QueueProcessor) Start(ctx context.Context) error { - glog.Infof(ctx, "Stream 处理器启动 - Stream: %s, 消费者组: %s, 消费者: %s, 超时: %dms", - q.streamKey, q.groupName, q.consumerName, q.timeout) + glog.Infof(ctx, "Stream 处理器启动 - Stream: %s, 消费者组: %s, 消费者: %s, 批量大小: %d", + q.streamKey, q.groupName, q.consumerName, q.batchSize) + + // 确保 Consumer Group 存在(重试直到成功) + for { + if err := redis.CreateConsumerGroup(ctx, q.streamKey, q.groupName); err != nil { + // BUSYGROUP 表示已存在,不是错误 + if strings.Contains(err.Error(), "BUSYGROUP") { + glog.Debugf(ctx, "Consumer Group 已存在") + break + } + glog.Warningf(ctx, "创建 Consumer Group 失败: %v,1秒后重试", err) + time.Sleep(time.Second) + continue + } + glog.Infof(ctx, "Consumer Group 创建成功") + break + } - loopCount := 0 for { select { case <-q.stopChan: glog.Info(ctx, "Stream 处理器收到停止信号") return nil default: - loopCount++ - if loopCount%10 == 1 { - glog.Debugf(ctx, "[DEBUG] 第 %d 次循环,准备读取消息...", loopCount) - } - - // 从 Redis Stream 中读取消息 - messages, err := q.fetchMessages(ctx) + // 1. 从 Redis Stream 读取一批消息 + messages, err := redis.ReadFromStream(ctx, q.streamKey, q.groupName, q.consumerName, q.batchSize, q.timeout) if err != nil { glog.Errorf(ctx, "从 Stream 读取消息失败: %v", err) continue } - // 没有新消息,继续等待 if len(messages) == 0 { - if loopCount%10 == 1 { - glog.Debugf(ctx, "[DEBUG] 第 %d 次循环,无新消息", loopCount) - } continue } - glog.Infof(ctx, "[DEBUG] 收到 %d 条消息", len(messages)) + glog.Debugf(ctx, "读取 %d 条消息,开始发送", len(messages)) - // 处理每条消息 + // 2. 用信号量控制并发:获取信号量后发送,完成后释放 for _, msg := range messages { - glog.Infof(ctx, "[DEBUG] 处理消息 ID: %s, Values: %+v", msg.ID, msg.Values) - // 提交到协程池处理 - if err := q.submitTask(ctx, msg); err != nil { - glog.Errorf(ctx, "提交任务到协程池失败: %v, 消息ID: %s", err, msg.ID) - } + // 获取信号量(阻塞直到有空位) + q.semaphore <- struct{}{} + go func(m redis.StreamMessage) { + defer func() { <-q.semaphore }() // 完成后释放信号量 + q.processMessage(ctx, m) + }(msg) } + // 3. 立刻读下一批(不等待,信号量自动控制并发数) } } } +// processMessage 处理单条消息(异步执行) +func (q *QueueProcessor) processMessage(ctx context.Context, message redis.StreamMessage) { + // 调用处理函数发送到 RAGFlow + if err := q.handleFunc(ctx, message.Values); err != nil { + glog.Errorf(ctx, "消息处理失败: %v, 消息ID: %s", err, message.ID) + } + + // 无论成功失败都 ACK(避免重复消费) + if err := redis.AckMessage(ctx, q.streamKey, q.groupName, message.ID); err != nil { + glog.Errorf(ctx, "确认消息失败: %v, 消息ID: %s", err, message.ID) + } +} + // Stop 停止队列处理器 func (q *QueueProcessor) Stop() { close(q.stopChan) } - -// fetchMessages 从 Redis Stream 中读取消息 -func (q *QueueProcessor) fetchMessages(ctx context.Context) ([]redis.StreamMessage, error) { - // 从消费者组读取消息 - return redis.ReadFromStream(ctx, q.streamKey, q.groupName, q.consumerName, q.batchSize, q.timeout) -} - -// submitTask 将消息处理任务提交到协程池 -func (q *QueueProcessor) submitTask(ctx context.Context, message redis.StreamMessage) error { - return q.pool.Submit(ctx, func(ctx context.Context) { - // 处理消息 - if err := q.handleFunc(ctx, message.Values); err != nil { - glog.Errorf(ctx, "处理消息失败: %v, 消息ID: %s", err, message.ID) - return - } - - // 处理成功后确认消息 - if err := redis.AckMessage(ctx, q.streamKey, q.groupName, message.ID); err != nil { - glog.Errorf(ctx, "确认消息失败: %v, 消息ID: %s", err, message.ID) - } else { - glog.Debugf(ctx, "消息处理完成并已确认: %s", message.ID) - } - }) -} diff --git a/redis/redis.go b/redis/redis.go index a74df47..7156952 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -95,6 +95,13 @@ func AddToStream(ctx context.Context, streamKey string, msg interface{}) (messag return } +// CreateConsumerGroup 创建消费者组(如果不存在) +// XGROUP CREATE streamKey groupName $ MKSTREAM +func CreateConsumerGroup(ctx context.Context, streamKey, groupName string) error { + _, err := redisClient.Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "$", "MKSTREAM") + return err +} + // ReadFromStream 从 Stream 读取消息(消费者组模式) // 使用 gredis Do() 方法执行 XREADGROUP 命令 func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName string, count int64, blockMs int64) ([]StreamMessage, error) { @@ -447,3 +454,24 @@ func DelSessionCache(ctx context.Context, userId string) error { _, err := redisClient.Del(ctx, key) return err } + +// TryLock 尝试获取分布式锁(非阻塞) +// key: 锁的键名 +// expireSeconds: 锁的过期时间(秒),防止死锁 +// 返回 true 表示获取成功,false 表示锁已被其他节点持有 +func TryLock(ctx context.Context, key string, expireSeconds int) bool { + // SET key value NX EX expireSeconds + result, err := redisClient.Do(ctx, "SET", key, gtime.Now().String(), "NX", "EX", expireSeconds) + if err != nil { + glog.Errorf(ctx, "获取分布式锁失败: %v", err) + return false + } + return result.String() == "OK" +} + +// Unlock 释放分布式锁 +func Unlock(ctx context.Context, key string) { + if _, err := redisClient.Del(ctx, key); err != nil { + glog.Errorf(ctx, "释放分布式锁失败: %v", err) + } +} From 976e4c6f2c77a8ff6588c57c912d929255da0c4f Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Thu, 11 Dec 2025 17:19:33 +0800 Subject: [PATCH 061/225] =?UTF-8?q?=E6=9B=B4=E6=94=B9client,=20=E6=9B=B4?= =?UTF-8?q?=E6=94=B9=E4=B8=BAgclient=E6=96=B9=E6=B3=95,=E5=BC=83=E7=94=A8?= =?UTF-8?q?=E9=93=BE=E5=BC=8F=E8=B0=83=E7=94=A8=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 ++ ragflow/client.go | 89 ++++++++++++++++++----------------------------- 2 files changed, 36 insertions(+), 55 deletions(-) diff --git a/.gitignore b/.gitignore index 5e12a7a..591b05d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ ragflow/agent文档.md ragflow/README_GLOBAL.md redis/stream使用示例.md ragflow/client_backup.go.bak +ragflow/为什么不能使用gclient.md +ragflow/agent文档.md diff --git a/ragflow/client.go b/ragflow/client.go index a5f64b2..f6e450b 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -1,9 +1,7 @@ package ragflow import ( - "bytes" "context" - "io" "net" "net/http" "net/url" @@ -14,13 +12,9 @@ import ( "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/gclient" ) -// gclient 完全不能用! -// 1. New() 默认 ResponseHeaderTimeout=30s -// 2. Clone() 内部调用 New(),链式调用会重置 Transport -// 3. 必须用原生 http.Client - var ( // globalClient 全局 RAGFlow 客户端(单例,延迟初始化) globalClient *Client @@ -41,7 +35,7 @@ func initClient() { return } - // 自定义 Transport,增大连接池(解决并发连接不足导致的超时) + // 自定义 Transport(增大连接池,设置超时) transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ @@ -49,23 +43,18 @@ func initClient() { KeepAlive: 30 * time.Second, }).DialContext, MaxIdleConns: 200, // 最大空闲连接数 - MaxIdleConnsPerHost: 100, // 每个 host 最大空闲连接数(关键!默认只有 2) + MaxIdleConnsPerHost: 100, // 每个 host 最大空闲连接数 MaxConnsPerHost: 100, // 每个 host 最大连接数 IdleConnTimeout: 90 * time.Second, // 空闲连接超时 TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, - ResponseHeaderTimeout: 180 * time.Second, // 等待响应头超时(关键!) + ResponseHeaderTimeout: 180 * time.Second, // 等待响应头超时 } - // 使用原生 http.Client(gclient 完全不能用,Clone() 内部调用 New() 会重置 Transport) - httpClient := &http.Client{ - Transport: transport, - Timeout: 0, // 不设置全局超时,由 context 控制 - } - - // 验证 Transport 设置 - g.Log().Infof(ctx, "✅ Transport 配置: ResponseHeaderTimeout=%v, MaxIdleConnsPerHost=%d, DisableKeepAlives=%v", - transport.ResponseHeaderTimeout, transport.MaxIdleConnsPerHost, transport.DisableKeepAlives) + // 初始化 gclient(不使用链式调用,避免 Transport 被重置) + httpClient := gclient.New() + httpClient.Client.Transport = transport + httpClient.Client.Timeout = 180 * time.Second globalClient = &Client{ BaseURL: strings.TrimSuffix(baseURL, "/"), @@ -73,7 +62,7 @@ func initClient() { HTTPClient: httpClient, } - g.Log().Infof(ctx, "✅ RAGFlow 全局客户端初始化成功: baseURL=%s", baseURL) + g.Log().Infof(ctx, "✅ RAGFlow 客户端初始化成功: baseURL=%s, timeout=180s", baseURL) }) } @@ -96,7 +85,7 @@ func GetGlobalClient() *Client { type Client struct { BaseURL string APIKey string - HTTPClient *http.Client // 原生 HTTP 客户端(gclient 不能用) + HTTPClient *gclient.Client // HTTP 客户端 } // CommonResponse 通用响应结构 @@ -111,61 +100,51 @@ func (r *CommonResponse) IsSuccess() bool { return r.Code == 0 } -// request 发送 HTTP 请求 +// request 发送 HTTP 请求(不使用链式调用) func (c *Client) request(ctx context.Context, method, path string, body interface{}, result interface{}) (err error) { fullURL := c.BaseURL + path + // 序列化请求体 var reqBody string if body != nil { - jsonData, err := gjson.Encode(body) - if err != nil { - return gerror.Newf("marshal request body failed: %v", err) + jsonData, jsonErr := gjson.Encode(body) + if jsonErr != nil { + return gerror.Newf("marshal request body failed: %v", jsonErr) } reqBody = string(jsonData) } - // 使用独立的 context 设置 300 秒超时(RAGFlow 高并发时响应较慢) - reqCtx, cancel := context.WithTimeout(context.Background(), 300*time.Second) - defer cancel() - startTime := time.Now() - - // 创建请求 - req, err := http.NewRequestWithContext(reqCtx, method, fullURL, bytes.NewReader([]byte(reqBody))) - if err != nil { - return gerror.Newf("create request failed: %v", err) - } - // 设置请求头 - req.Header.Set("Authorization", "Bearer "+c.APIKey) - req.Header.Set("Content-Type", "application/json") + c.HTTPClient.SetHeader("Authorization", "Bearer "+c.APIKey) + c.HTTPClient.SetHeader("Content-Type", "application/json") // 发送请求 - g.Log().Infof(ctx, "[RAGFlow HTTP] 发送请求: method=%s, url=%s", method, fullURL) - resp, err := c.HTTPClient.Do(req) - elapsed := time.Since(startTime) + var resp *gclient.Response + switch method { + case "GET": + resp, err = c.HTTPClient.Get(ctx, fullURL) + case "POST": + resp, err = c.HTTPClient.Post(ctx, fullURL, reqBody) + case "PUT": + resp, err = c.HTTPClient.Put(ctx, fullURL, reqBody) + case "DELETE": + resp, err = c.HTTPClient.Delete(ctx, fullURL, reqBody) + default: + return gerror.Newf("unsupported method: %s", method) + } + if err != nil { - g.Log().Errorf(ctx, "[RAGFlow HTTP] 请求失败(耗时 %v): method=%s, url=%s, error=%v", elapsed, method, fullURL, err) return gerror.Newf("request failed: %v", err) } - g.Log().Infof(ctx, "[RAGFlow HTTP] 收到响应(耗时 %v): status=%d, url=%s", elapsed, resp.StatusCode, fullURL) - defer resp.Body.Close() + defer resp.Close() - // 读取响应 - respBody, err := io.ReadAll(resp.Body) - if err != nil { - return gerror.Newf("read response failed: %v", err) - } - - // 打印响应详情 - g.Log().Debugf(ctx, "[RAGFlow HTTP] 响应: status=%d, body=%s", resp.StatusCode, respBody) + respBody := resp.ReadAll() if resp.StatusCode != http.StatusOK { - g.Log().Errorf(ctx, "[RAGFlow HTTP] 非200响应: status=%d, body=%s", resp.StatusCode, respBody) - return gerror.Newf("http status %d: %s", resp.StatusCode, respBody) + return gerror.Newf("http status %d: %s", resp.StatusCode, string(respBody)) } if err = gjson.DecodeTo(respBody, result); err != nil { - g.Log().Errorf(ctx, "[RAGFlow HTTP] 解析响应失败: body=%s, error=%v", string(respBody), err) return gerror.Newf("unmarshal response failed: %v", err) } From e168325ee252ce80278d0102cf86e388ef4a87a8 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Thu, 11 Dec 2025 11:26:50 +0800 Subject: [PATCH 062/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0minio=E5=9B=BE?= =?UTF-8?q?=E7=89=87=E4=B8=8A=E4=BC=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- minio/minio.go | 110 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 minio/minio.go diff --git a/minio/minio.go b/minio/minio.go new file mode 100644 index 0000000..5b3b469 --- /dev/null +++ b/minio/minio.go @@ -0,0 +1,110 @@ +package minio + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/ghttp" + "github.com/google/uuid" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// MinIOConfig 映射 YAML 中的 minio 配置节点 +type MinIOConfig struct { + Endpoint string `yaml:"endpoint"` // MinIO API 地址 + AccessKey string `yaml:"accessKey"` // AK + SecretKey string `yaml:"secretKey"` // SK + Secure bool `yaml:"secure"` // 是否启用 SSL + Region string `yaml:"region"` // 区域 + BucketName string `yaml:"bucketName"` // 默认桶名 + PresignedExpire string `yaml:"presignedExpire"` // 预签名URL过期时间(如 5m、1h) +} + +// 全局 MinIO 客户端(初始化一次,避免重复创建) +var minioClient *minio.Client +var minioCfg MinIOConfig + +// initMinIO 初始化 MinIO 客户端 +func initMinIO(ctx context.Context) error { + var err error + // 加载 MinIO 配置(可从配置文件/环境变量读取,这里硬编码示例) + minioCfg = MinIOConfig{ + Endpoint: g.Cfg().MustGet(ctx, "minio.endpoint").String(), + AccessKey: g.Cfg().MustGet(ctx, "minio.accessKey").String(), + SecretKey: g.Cfg().MustGet(ctx, "minio.secretKey").String(), + Secure: g.Cfg().MustGet(ctx, "minio.secure").Bool(), + Region: g.Cfg().MustGet(ctx, "minio.region").String(), + BucketName: g.Cfg().MustGet(ctx, "minio.bucketName").String(), // 专门存储图片的桶 + } + // 创建 MinIO 客户端 + minioClient, err = minio.New(minioCfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(minioCfg.AccessKey, minioCfg.SecretKey, ""), + Secure: minioCfg.Secure, + Region: minioCfg.Region, + }) + if err != nil { + return fmt.Errorf("初始化 MinIO 客户端失败: %w", err) + } + // 检查/创建图片桶 + exists, err := minioClient.BucketExists(ctx, minioCfg.BucketName) + if err != nil { + return fmt.Errorf("检查桶是否存在失败: %w", err) + } + if !exists { + err = minioClient.MakeBucket(ctx, minioCfg.BucketName, minio.MakeBucketOptions{Region: minioCfg.Region}) + if err != nil { + return fmt.Errorf("创建桶失败: %w", err) + } + fmt.Printf("成功创建 MinIO 桶: %s\n", minioCfg.BucketName) + } + return nil +} + +func UploadImage(fileHeader *ghttp.UploadFile) (presignedUrl string, err error) { + ctx := context.Background() + // 初始化 MinIO 客户端 + err = initMinIO(ctx) + if err != nil { + return "", fmt.Errorf("初始化 MinIO 客户端失败: %w", err) + } + // 生成唯一的 MinIO 对象名(避免覆盖) + fileExt := filepath.Ext(fileHeader.Filename) // 原文件后缀(如 .jpg) + uniqueID := uuid.New().String()[:8] // 8位随机UUID + timestamp := time.Now().Format("20060102") // 日期目录(便于管理) + objectName := fmt.Sprintf("images/%s/%s%s", timestamp, uniqueID, fileExt) // 存储路径:images/20251209/abc12345.jpg + // 打开文件,获取 io.Reader(*os.File 实现了 io.Reader) + file, err := fileHeader.Open() + if err != nil { /* 处理错误 */ + return "", fmt.Errorf("打开文件失败: %w", err) + } + defer file.Close() // 必须关闭,避免文件句柄泄露 + // 设置存储桶公共读权限 + policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::` + minioCfg.BucketName + `/*"]}]}` + err = minioClient.SetBucketPolicy(context.Background(), minioCfg.BucketName, policy) + if err != nil { + return "", fmt.Errorf("设置存储桶权限失败: %w", err) + } + // 执行图片上传 + _, err = minioClient.PutObject( + context.Background(), + minioCfg.BucketName, + objectName, + file, + fileHeader.Size, + minio.PutObjectOptions{ + ContentType: "image/jpeg", // 关键:指定图片MIME类型,S3会根据此类型处理 + // 若需要图片可公开访问,添加如下配置(根据需求选择) + //ACL: minio.ACLPublicRead, + }, + ) + if err != nil { + return "", fmt.Errorf("上传图片失败: %w", err) + } + replace := strings.Replace(objectName, "images/", "/", 1) + return replace, err +} From 8830091e7d024163ded052e62978c96927172ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 11 Dec 2025 15:03:43 +0800 Subject: [PATCH 063/225] =?UTF-8?q?gomod=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/utils.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/utils.go b/utils/utils.go index d539f81..c4b8393 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -71,7 +71,10 @@ func GetUserInfo(ctx context.Context) (user do.User, err error) { Address: redisAddr, Db: 1, })) - + r := g.RequestFromCtx(ctx) + if r == nil { + return + } // 解析 token data, err := gft.ParseToken(g.RequestFromCtx(ctx)) if err != nil { From 8218487ed07bbd22cc79d91f0df31505289df797 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Mon, 15 Dec 2025 17:27:10 +0800 Subject: [PATCH 064/225] =?UTF-8?q?=E5=A4=8D=E7=94=A8http.go=E9=87=8C?= =?UTF-8?q?=E9=9D=A2=E7=9A=84=20g.client(),=E6=9B=B4=E6=96=B0config.yml?= =?UTF-8?q?=E7=9A=84=E6=B3=A8=E5=86=8C=E6=96=B9=E5=BC=8F,=20=E6=9B=B4?= =?UTF-8?q?=E6=96=B0jaeger=E9=93=BE=E8=B7=AF=E8=BF=BD=E8=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 2 +- jaeger/jaeger.go | 81 +++++++++++++++++++++++++++++++++++++++++++++- ragflow/client.go | 46 ++++++++------------------ startup/startup.go | 32 ++++-------------- 4 files changed, 102 insertions(+), 59 deletions(-) diff --git a/http/http.go b/http/http.go index 0023e64..b76621d 100644 --- a/http/http.go +++ b/http/http.go @@ -72,7 +72,7 @@ func doRequest(ctx context.Context, method string, url string, target any, data } defer func() { if err = response.Close(); err != nil { - glog.Errorf(ctx, `%+v`, err) + glog.Error(ctx, err) } }() result := response.ReadAll() diff --git a/jaeger/jaeger.go b/jaeger/jaeger.go index 9acc586..0132dd3 100644 --- a/jaeger/jaeger.go +++ b/jaeger/jaeger.go @@ -12,6 +12,8 @@ import ( "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/net/gtrace" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" ) var ( @@ -47,10 +49,75 @@ func init() { // 默认自动初始化(保持向后兼容) Init() } + +// NewSpan 创建新的链路追踪 Span +// spanName: Span 名称,用于在 Jaeger UI 中标识 +// 返回带有 Span 的 context 和 Span 对象,调用方需 defer span.End() +func NewSpan(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, *gtrace.Span) { + return gtrace.NewSpan(ctx, spanName, opts...) +} + +// RecordError 统一错误记录方法 +// 功能: +// 1. 控制台输出错误(带完整堆栈 %+v) +// 2. Jaeger 链路追踪记录错误 +// 3. 设置 Span 错误状态 +// +// 使用示例: +// +// jaeger.RecordError(ctx, err, "保存数据失败") +// +// 参数: +// - ctx: 包含 trace span 的上下文 +// - err: 错误对象(支持 gerror 堆栈) +// - msg: 可选的错误描述(用于日志和 Jaeger 显示) +func RecordError(ctx context.Context, err error, msg ...string) { + if err == nil { + return + } + // 1. 控制台输出(%+v 打印完整堆栈) + if len(msg) > 0 && msg[0] != "" { + g.Log().Errorf(ctx, "%s: %+v", msg[0], err) + } else { + g.Log().Errorf(ctx, "%+v", err) + } + // 2. Jaeger 记录(从 context 获取当前 span) + span := trace.SpanFromContext(ctx) + if span == nil || !span.IsRecording() { + return + } + // 3. 记录错误到 span + span.RecordError(err) + span.SetAttributes( + attribute.Bool("error", true), + attribute.String("error.message", err.Error()), + ) + // 4. 设置 span 状态为错误 + if len(msg) > 0 && msg[0] != "" { + span.SetAttributes(attribute.String("error.msg", msg[0])) + span.SetStatus(codes.Error, msg[0]+": "+err.Error()) + return + } + span.SetStatus(codes.Error, err.Error()) +} + +// NewTracer HTTP 请求链路追踪中间件 +// 功能: +// 1. 为每个 HTTP 请求创建 Span +// 2. 记录请求参数和响应内容 +// 3. 自动捕获错误并记录到 Jaeger +// +// 使用方式:在路由组中注册为中间件 +// +// group.Middleware(jaeger.NewTracer) func NewTracer(r *ghttp.Request) { - _, span := gtrace.NewSpan(r.Context(), r.GetServeHandler().GetMetaTag("summary")) + // 创建 Span(名称取自 controller 方法的 summary 标签) + ctx, span := gtrace.NewSpan(r.Context(), r.GetServeHandler().GetMetaTag("summary")) + r.SetCtx(ctx) defer span.End() + // 记录请求参数 span.SetAttributes(attribute.String("request", getParams(r))) + // 执行后续中间件和 handler r.Middleware.Next() // 清理响应字符串,确保 UTF-8 有效(处理二进制数据如 ZIP 文件) @@ -63,7 +130,19 @@ func NewTracer(r *ghttp.Request) { } span.SetAttributes(attribute.String("response", cleanResponse)) + span.SetAttributes(attribute.Int("http.status_code", r.Response.Status)) + + if err := r.GetError(); err != nil { + RecordError(ctx, err) + return + } + if r.Response.Status >= 500 { + span.SetAttributes(attribute.Bool("error", true)) + span.SetStatus(codes.Error, "http status "+strconv.Itoa(r.Response.Status)) + } } + +// getParams 提取请求参数(用于 Jaeger 记录) func getParams(r *ghttp.Request) string { params := map[string]interface{}{} if r.Method == "POST" { diff --git a/ragflow/client.go b/ragflow/client.go index f6e450b..7dab00a 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -2,13 +2,13 @@ package ragflow import ( "context" - "net" "net/http" "net/url" "strings" "sync" "time" + commonHttp "gitee.com/red-future---jilin-g/common/http" "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" @@ -35,34 +35,13 @@ func initClient() { return } - // 自定义 Transport(增大连接池,设置超时) - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }).DialContext, - MaxIdleConns: 200, // 最大空闲连接数 - MaxIdleConnsPerHost: 100, // 每个 host 最大空闲连接数 - MaxConnsPerHost: 100, // 每个 host 最大连接数 - IdleConnTimeout: 90 * time.Second, // 空闲连接超时 - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - ResponseHeaderTimeout: 180 * time.Second, // 等待响应头超时 - } - - // 初始化 gclient(不使用链式调用,避免 Transport 被重置) - httpClient := gclient.New() - httpClient.Client.Transport = transport - httpClient.Client.Timeout = 180 * time.Second - globalClient = &Client{ BaseURL: strings.TrimSuffix(baseURL, "/"), APIKey: apiKey, - HTTPClient: httpClient, + HTTPClient: commonHttp.Httpclient, } - g.Log().Infof(ctx, "✅ RAGFlow 客户端初始化成功: baseURL=%s, timeout=180s", baseURL) + g.Log().Infof(ctx, "✅ RAGFlow 客户端初始化成功: baseURL=%s", baseURL) }) } @@ -100,7 +79,7 @@ func (r *CommonResponse) IsSuccess() bool { return r.Code == 0 } -// request 发送 HTTP 请求(不使用链式调用) +// request 发送 HTTP 请求 func (c *Client) request(ctx context.Context, method, path string, body interface{}, result interface{}) (err error) { fullURL := c.BaseURL + path @@ -114,21 +93,24 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac reqBody = string(jsonData) } - // 设置请求头 - c.HTTPClient.SetHeader("Authorization", "Bearer "+c.APIKey) - c.HTTPClient.SetHeader("Content-Type", "application/json") + // 设置请求头和超时 + // 注意:使用 Chain 模式,避免修改全局 Httpclient + client := c.HTTPClient.Timeout(180 * time.Second).Header(map[string]string{ + "Authorization": "Bearer " + c.APIKey, + "Content-Type": "application/json", + }) // 发送请求 var resp *gclient.Response switch method { case "GET": - resp, err = c.HTTPClient.Get(ctx, fullURL) + resp, err = client.Get(ctx, fullURL) case "POST": - resp, err = c.HTTPClient.Post(ctx, fullURL, reqBody) + resp, err = client.Post(ctx, fullURL, reqBody) case "PUT": - resp, err = c.HTTPClient.Put(ctx, fullURL, reqBody) + resp, err = client.Put(ctx, fullURL, reqBody) case "DELETE": - resp, err = c.HTTPClient.Delete(ctx, fullURL, reqBody) + resp, err = client.Delete(ctx, fullURL, reqBody) default: return gerror.Newf("unsupported method: %s", method) } diff --git a/startup/startup.go b/startup/startup.go index 2fc5882..d927e43 100644 --- a/startup/startup.go +++ b/startup/startup.go @@ -92,31 +92,13 @@ func NeedES() bool { // 如果配置文件中没有 startup 配置,则默认全部启动 func loadFromConfig() *Components { ctx := context.Background() - - // 检查是否有 startup 配置节 - startupCfg := g.Cfg().MustGet(ctx, "startup") - if startupCfg.IsEmpty() { - // 没有配置 startup,默认全部启动 - glog.Debug(ctx, "未找到 startup 配置,默认启动所有组件") - return &Components{ - Consul: true, - Jaeger: true, - Redis: true, - RabbitMQ: true, - MongoDB: true, - RAGFlow: true, - ES: true, - } - } - - // 有配置则按配置来,未配置的项默认 true return &Components{ - Consul: g.Cfg().MustGet(ctx, "startup.consul", true).Bool(), - Jaeger: g.Cfg().MustGet(ctx, "startup.jaeger", true).Bool(), - Redis: g.Cfg().MustGet(ctx, "startup.redis", true).Bool(), - RabbitMQ: g.Cfg().MustGet(ctx, "startup.rabbitmq", true).Bool(), - MongoDB: g.Cfg().MustGet(ctx, "startup.mongodb", true).Bool(), - RAGFlow: g.Cfg().MustGet(ctx, "startup.ragflow", true).Bool(), - ES: g.Cfg().MustGet(ctx, "startup.es", true).Bool(), + Consul: !g.Cfg().MustGet(ctx, "consul").IsEmpty(), + Jaeger: !g.Cfg().MustGet(ctx, "jaeger").IsEmpty(), + Redis: !g.Cfg().MustGet(ctx, "redis").IsEmpty(), + RabbitMQ: !g.Cfg().MustGet(ctx, "rabbitmq").IsEmpty(), + MongoDB: !g.Cfg().MustGet(ctx, "mongo").IsEmpty(), + RAGFlow: !g.Cfg().MustGet(ctx, "ragflow").IsEmpty(), + ES: !g.Cfg().MustGet(ctx, "elasticsearch").IsEmpty(), } } From 795555eb2f1f2925dde18beae9fb4cfb9b5eb445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 15 Dec 2025 17:14:23 +0800 Subject: [PATCH 065/225] =?UTF-8?q?mongodb=E5=9F=BA=E7=A1=80=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E6=96=B9=E6=B3=95=E5=A2=9E=E5=8A=A0redis=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mongo/mongo.go b/mongo/mongo.go index f1f9eb9..cc1f8bc 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -232,6 +232,11 @@ func cleanRedis(ctx context.Context, tenantId interface{}, collection string) (e return } } + oneKey := fmt.Sprintf(consts.One, tenantId, collection) + _, err = redis.RedisClient.Del(ctx, oneKey) + if err != nil { + return + } return } From c4d232d6ff8289ea43dee6a998c40797aca6726f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 15 Dec 2025 17:14:45 +0800 Subject: [PATCH 066/225] =?UTF-8?q?mongodb=E5=9F=BA=E7=A1=80=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E6=96=B9=E6=B3=95=E5=A2=9E=E5=8A=A0redis=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 591b05d..ba979d1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ redis/stream使用示例.md ragflow/client_backup.go.bak ragflow/为什么不能使用gclient.md ragflow/agent文档.md +/.idea/MarsCodeWorkspaceAppSettings.xml From 03aef62184f971ba669fe5281cf162810350537b Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 16 Dec 2025 11:52:46 +0800 Subject: [PATCH 067/225] =?UTF-8?q?=E7=AC=AC=E4=BA=94=E6=AC=A1=E5=AF=B9?= =?UTF-8?q?=E8=AF=9D=E5=8F=91=E5=8D=A1=E7=89=87,=20=E7=94=A8redis=E8=AE=B0?= =?UTF-8?q?=E5=BD=95=E5=8D=A1=E7=89=87=E6=AC=A1=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/配置中心方案.md | 267 +++++++++++++++++++++++++++++++++++++++++ redis/redis.go | 46 +++++++ startup/startup.go | 104 ---------------- 3 files changed, 313 insertions(+), 104 deletions(-) create mode 100644 consul/配置中心方案.md delete mode 100644 startup/startup.go diff --git a/consul/配置中心方案.md b/consul/配置中心方案.md new file mode 100644 index 0000000..3c87785 --- /dev/null +++ b/consul/配置中心方案.md @@ -0,0 +1,267 @@ +# Consul KV 配置中心方案 + +**文档版本**:v1.0 +**创建日期**:2025-12-16 +**负责项目**:customerservice + message + gateway + +--- + +## 一、核心目标 + +| 目标 | 说明 | +|------|------| +| **安全性** | 敏感配置(连接地址/密码)不写在代码里,从 Consul KV 读取 | +| **回退机制** | Consul 不可用时,使用本地默认配置保证服务可启动 | +| **热更新** | Consul KV 变更后,服务无需重启立即生效 | +| **性能最优** | 请求路径只读内存(原子快照),不访问 Consul;单协程阻塞监听,不空转 | + +--- + +## 二、配置分类与热更新策略 + +### 2.1 配置分类 + +| 分类 | 配置项 | 热更新方式 | +|------|--------|------------| +| **连接池配置** | http 连接池数量、redis 连接池数量、mongo 连接池数量 | 平滑切换(创建新池→原子替换→延迟关闭旧池) | +| **协程池配置** | grpool worker 数量 | 动态调整(`grpool.SetSize()`) | +| **业务参数** | 追问延时、归档延时、批量大小、超时时间、卡片触发轮数 | 直接原子更新内存变量 | +| **开关/限流** | 功能开关、限流阈值、日志级别 | 直接原子更新内存变量 | +| **中间件地址** | Redis/MQ/Mongo/ES/Jaeger 地址 | 平滑切换连接池 | + +### 2.2 热更新策略矩阵 + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ 热更新策略矩阵 │ +├─────────────────────────────────────────────────────────────────┤ +│ 配置变更类型 │ 更新方式 │ 对请求影响 │ +├─────────────────────────────────────────────────────────────────┤ +│ 业务参数/开关 │ atomic.Value │ 无影响,立即生效 │ +│ 协程池数量 │ grpool.SetSize │ 无影响,渐进生效 │ +│ 连接池数量/地址 │ 平滑切换单例 │ 无影响,新请求用新池 │ +└─────────────────────────────────────────────────────────────────┘ +``` + +--- + +## 三、Consul KV 键名规范 + +``` +config/{service}/pool +config/{service}/business +config/{service}/middleware +``` + +### 3.1 示例 KV 结构 + +**Key**: `config/customerservice/pool` +```yaml +http: + maxConns: 100 # HTTP 连接池最大连接数 + maxIdleConns: 20 # HTTP 连接池最大空闲连接 +redis: + maxActive: 50 # Redis 连接池最大活跃连接 + maxIdle: 10 # Redis 连接池最大空闲连接 + idleTimeout: 300 # 空闲超时(秒) +grpool: + workerSize: 200 # 协程池 worker 数量 +``` + +**Key**: `config/customerservice/business` +```yaml +followUp: + delay1: 30 # 第一次追问延时(秒) + delay2: 60 # 第二次追问延时(秒) + delay3: 180 # 第三次追问延时(秒) +archive: + delay: 3600 # 归档延时(秒) +stream: + batchSize: 200 # 批量读取消息数量 + blockTimeout: 2000 # 阻塞超时(毫秒) +card: + triggerCount: 5 # 触发发送卡片的对话轮数 +``` + +**Key**: `config/customerservice/middleware` +```yaml +redis: + address: "192.168.3.200:6379" + password: "" + db: 0 +rabbitmq: + host: "192.168.3.200" + port: 5672 + username: "guest" + password: "guest" + vhost: "/" +mongo: + uri: "mongodb://192.168.3.200:27017" + database: "customer_service" +``` + +--- + +## 四、架构设计 + +### 4.1 模块结构 + +``` +common/configcenter/ +├── configcenter.go # 核心:启动/停止/获取配置 +├── watcher.go # Consul KV 阻塞监听(单协程) +├── snapshot.go # 配置快照(atomic.Value) +├── hook.go # 变更回调分发 +└── types.go # 配置结构体定义 +``` + +### 4.2 核心接口 + +```go +// 启动配置中心(每个服务 main.go 调用一次) +func Start(ctx context.Context, opts Options) error + +// 获取当前配置快照(请求路径调用,O(1) 无锁) +func Get() *Snapshot + +// 注册变更回调(组件注册自己的重初始化逻辑) +func Subscribe(name string, handler func(old, new *Snapshot)) + +// 优雅停止 +func Stop() +``` + +### 4.3 数据流 + +``` +┌──────────────┐ 阻塞查询 ┌──────────────┐ +│ Consul KV │ ─────────────→ │ Watcher │ (单协程,不空转) +└──────────────┘ (长轮询) └──────┬───────┘ + │ 配置变更 + ▼ + ┌──────────────┐ + │ Snapshot │ (atomic.Value 原子替换) + └──────┬───────┘ + │ + ┌───────────────────────┼───────────────────────┐ + ▼ ▼ ▼ + ┌────────────────┐ ┌────────────────┐ ┌────────────────┐ + │ 业务代码读取 │ │ Hook 回调 │ │ 日志记录 │ + │ Get().Pool │ │ 重建连接池 │ │ 版本号变更 │ + └────────────────┘ └────────────────┘ └────────────────┘ +``` + +--- + +## 五、服务启动流程 + +```go +func main() { + ctx := context.Background() + + // 1. 启动配置中心(优先从 Consul 拉取,失败则用本地默认) + if err := configcenter.Start(ctx, configcenter.Options{ + ConsulAddr: "192.168.3.200:8500", + ServiceName: "customerservice", + DefaultPath: "config.yml", // 回退配置 + }); err != nil { + glog.Warningf(ctx, "配置中心启动失败,使用本地配置: %v", err) + } + + // 2. 注册组件热更新回调 + configcenter.Subscribe("redis", redis.OnConfigChange) + configcenter.Subscribe("grpool", ragflow.OnPoolSizeChange) + configcenter.Subscribe("rabbitmq", rabbitmq.OnConfigChange) + + // 3. 初始化组件(使用 configcenter.Get() 获取配置) + initComponents(ctx) + + // 4. 启动服务... + + // 5. 优雅退出 + defer configcenter.Stop() +} +``` + +--- + +## 六、性能保证 + +| 环节 | 性能策略 | +|------|----------| +| **请求路径** | 只读 `atomic.Value`,O(1),无锁,无网络 | +| **监听 Consul** | 阻塞查询(Blocking Query),无变更时不消耗 CPU | +| **配置解析** | 只在变更时解析一次 YAML → 结构体 | +| **连接池切换** | 原子替换指针,旧池延迟 30s 关闭(等待请求完成) | +| **协程池调整** | `grpool` 内置支持动态调整,无需重建 | + +--- + +## 七、回退与容错 + +| 场景 | 处理方式 | +|------|----------| +| **启动时 Consul 不可用** | 使用本地 `config.yml` 默认配置,服务正常启动 | +| **运行时 Consul 断连** | 保持最后一次成功配置,自动重试连接 | +| **配置格式错误** | 拒绝更新,保持旧配置,记录错误日志 | +| **连接池切换失败** | 保持旧连接池,记录错误,不影响服务 | + +--- + +## 八、Consul KV 操作示例 + +### 8.1 通过 Consul UI 操作 + +1. 访问 `http://192.168.3.200:8500/ui` +2. 点击 **Key/Value** 菜单 +3. 创建/编辑 Key,如 `config/customerservice/business` +4. 在 Value 中粘贴 YAML 配置内容 +5. 点击 **Save** + +### 8.2 通过 CLI 操作 + +```bash +# 写入配置 +consul kv put config/customerservice/business @business.yml + +# 读取配置 +consul kv get config/customerservice/business + +# 删除配置 +consul kv delete config/customerservice/business +``` + +### 8.3 通过 HTTP API 操作 + +```bash +# 写入配置 +curl --request PUT \ + --url http://192.168.3.200:8500/v1/kv/config/customerservice/business \ + --data-binary @business.yml + +# 读取配置(Base64 编码) +curl http://192.168.3.200:8500/v1/kv/config/customerservice/business + +# 阻塞查询(长轮询,等待变更) +curl "http://192.168.3.200:8500/v1/kv/config/customerservice/business?index=123&wait=5m" +``` + +--- + +## 九、实现进度 + +| 阶段 | 状态 | 说明 | +|------|------|------| +| 方案设计 | ✅ 完成 | 本文档 | +| common/configcenter 模块 | ⏳ 待实现 | 核心配置中心模块 | +| 各组件热更新回调 | ⏳ 待实现 | Redis/MQ/Mongo/ES 等 | +| 联调验证 | ⏳ 待实现 | KV 修改后无需重启生效 | + +--- + +## 十、注意事项 + +1. **安全性**:生产环境的 Consul 必须启用 ACL,限制 KV 访问权限 +2. **版本控制**:重要配置变更前,建议先备份旧配置 +3. **灰度发布**:可通过 `config/{env}/{service}` 实现多环境配置隔离 +4. **监控告警**:建议对配置变更事件添加日志和告警 diff --git a/redis/redis.go b/redis/redis.go index 7156952..1585f51 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -475,3 +475,49 @@ func Unlock(ctx context.Context, key string) { glog.Errorf(ctx, "释放分布式锁失败: %v", err) } } + +// ============== 对话计数相关(用于卡片触发)============== + +const ( + // ConversationCountKeyPrefix 对话计数 Key 前缀 + ConversationCountKeyPrefix = "ragflow:conversation:count:" +) + +// IncrConversationCount 增加用户对话计数,返回当前轮数 +// 用于判断是否触发发送卡片(如对话5轮后发送) +// expireSeconds: 过期时间(秒),建议与会话超时一致(如7200秒=2小时) +func IncrConversationCount(ctx context.Context, userId, platform string, expireSeconds int64) (count int64, err error) { + key := ConversationCountKeyPrefix + userId + "_" + platform + result, err := redisClient.Do(ctx, "INCR", key) + if err != nil { + return + } + count = result.Int64() + + // 首次设置过期时间 + if count == 1 { + redisClient.Do(ctx, "EXPIRE", key, expireSeconds) + } + return +} + +// GetConversationCount 获取用户当前对话轮数 +func GetConversationCount(ctx context.Context, userId, platform string) (count int64, err error) { + key := ConversationCountKeyPrefix + userId + "_" + platform + result, err := redisClient.Get(ctx, key) + if err != nil { + return + } + if result.IsEmpty() { + return 0, nil + } + count = result.Int64() + return +} + +// ResetConversationCount 重置用户对话计数(归档/卡片发送后调用) +func ResetConversationCount(ctx context.Context, userId, platform string) error { + key := ConversationCountKeyPrefix + userId + "_" + platform + _, err := redisClient.Del(ctx, key) + return err +} diff --git a/startup/startup.go b/startup/startup.go deleted file mode 100644 index d927e43..0000000 --- a/startup/startup.go +++ /dev/null @@ -1,104 +0,0 @@ -// Package startup 提供服务启动时的组件初始化控制 -// 各服务可以按需初始化所需组件,避免不必要的资源占用 -package startup - -import ( - "context" - "sync" - - "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/os/glog" -) - -// Components 组件配置 -type Components struct { - Consul bool // Consul 服务注册发现(所有服务都需要) - Jaeger bool // Jaeger 链路追踪(所有服务都需要) - Redis bool // Redis 缓存 - RabbitMQ bool // RabbitMQ 消息队列 - MongoDB bool // MongoDB 数据库 - RAGFlow bool // RAGFlow AI 客户端 - ES bool // Elasticsearch -} - -var ( - initialized bool - initOnce sync.Once - components *Components -) - -// Init 初始化指定的组件 -// 示例: -// -// bootstrap.Init(ctx, &bootstrap.Components{ -// Consul: true, -// Jaeger: true, -// Redis: true, -// RabbitMQ: true, -// }) -func Init(ctx context.Context, c *Components) { - initOnce.Do(func() { - components = c - initialized = true - glog.Infof(ctx, "Bootstrap 初始化完成: %+v", c) - }) -} - -// IsInitialized 检查是否已初始化 -func IsInitialized() bool { - return initialized -} - -// GetComponents 获取组件配置 -func GetComponents() *Components { - if components == nil { - // 默认配置:从配置文件读取 - return loadFromConfig() - } - return components -} - -// NeedRedis 是否需要 Redis -func NeedRedis() bool { - c := GetComponents() - return c != nil && c.Redis -} - -// NeedRabbitMQ 是否需要 RabbitMQ -func NeedRabbitMQ() bool { - c := GetComponents() - return c != nil && c.RabbitMQ -} - -// NeedMongoDB 是否需要 MongoDB -func NeedMongoDB() bool { - c := GetComponents() - return c != nil && c.MongoDB -} - -// NeedRAGFlow 是否需要 RAGFlow -func NeedRAGFlow() bool { - c := GetComponents() - return c != nil && c.RAGFlow -} - -// NeedES 是否需要 Elasticsearch -func NeedES() bool { - c := GetComponents() - return c != nil && c.ES -} - -// loadFromConfig 从配置文件加载组件配置 -// 如果配置文件中没有 startup 配置,则默认全部启动 -func loadFromConfig() *Components { - ctx := context.Background() - return &Components{ - Consul: !g.Cfg().MustGet(ctx, "consul").IsEmpty(), - Jaeger: !g.Cfg().MustGet(ctx, "jaeger").IsEmpty(), - Redis: !g.Cfg().MustGet(ctx, "redis").IsEmpty(), - RabbitMQ: !g.Cfg().MustGet(ctx, "rabbitmq").IsEmpty(), - MongoDB: !g.Cfg().MustGet(ctx, "mongo").IsEmpty(), - RAGFlow: !g.Cfg().MustGet(ctx, "ragflow").IsEmpty(), - ES: !g.Cfg().MustGet(ctx, "elasticsearch").IsEmpty(), - } -} From b0b4c37ed6c88e8b8f95faff77183ad3525de39b Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Mon, 15 Dec 2025 18:15:23 +0800 Subject: [PATCH 068/225] =?UTF-8?q?=E4=BF=AE=E6=94=B9-redis=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E5=8D=95=E6=9D=A1=E6=9F=A5=E8=AF=A2=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consts/redis_key.go | 2 +- mongo/mongo.go | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/consts/redis_key.go b/consts/redis_key.go index aef970f..183aa69 100644 --- a/consts/redis_key.go +++ b/consts/redis_key.go @@ -4,4 +4,4 @@ const CleanList = "list:tenantId-%v:collection-%s:*" const CleanCount = "count:tenantId-%v:collection-%s:*" const List = "list:tenantId-%v:collection-%s:filter:%s:options:%s" const Count = "count:tenantId-%v:collection-%s:filter:%s" -const One = "one:tenantId-%v:collection-%s:filter:%s:*" +const One = "one:tenantId-%v:collection-%s:filter:%s" diff --git a/mongo/mongo.go b/mongo/mongo.go index cc1f8bc..336a5ad 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -184,8 +184,7 @@ func FindOne(ctx context.Context, filter bson.M, result interface{}, collection } filter["isDeleted"] = false filterMap := utils.OrderMap(filter) - optsMap := oneOptionsToMap(ctx, opts...) - redisKey := fmt.Sprintf(consts.One, user.TenantId, collection, gconv.String(filterMap), gconv.String(optsMap)) + redisKey := fmt.Sprintf(consts.One, user.TenantId, collection, gconv.String(filterMap)) resultStr, err := redis.RedisClient.Get(ctx, redisKey) if err != nil { return @@ -209,7 +208,7 @@ func FindOne(ctx context.Context, filter bson.M, result interface{}, collection } return } -func cleanRedis(ctx context.Context, tenantId interface{}, collection string) (err error) { +func cleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collection string) (err error) { listKeys := fmt.Sprintf(consts.CleanList, tenantId, collection) keys, err := redis.RedisClient.Keys(ctx, listKeys) if err != nil { @@ -232,7 +231,11 @@ func cleanRedis(ctx context.Context, tenantId interface{}, collection string) (e return } } - oneKey := fmt.Sprintf(consts.One, tenantId, collection) + filter["isDeleted"] = false + delete(filter, "tenantId") + filterMap := utils.OrderMap(filter) + oneKey := fmt.Sprintf(consts.One, tenantId, collection, gconv.String(filterMap)) + fmt.Println(gconv.String(filterMap)) _, err = redis.RedisClient.Del(ctx, oneKey) if err != nil { return @@ -256,7 +259,7 @@ func Delete(ctx context.Context, filter bson.M, collection string, opts ...optio return } count = r.DeletedCount - err = cleanRedis(ctx, user.TenantId, collection) + err = cleanRedis(ctx, filter, user.TenantId, collection) return } @@ -280,7 +283,7 @@ func Update(ctx context.Context, filter bson.M, update bson.M, collection string if err != nil { return } - err = cleanRedis(ctx, user.TenantId, collection) + err = cleanRedis(ctx, filter, user.TenantId, collection) return } @@ -307,7 +310,7 @@ func Insert(ctx context.Context, documents []interface{}, collection string, opt return } ids = r.InsertedIDs - err = cleanRedis(ctx, user.TenantId, collection) + err = cleanRedis(ctx, bson.M{}, user.TenantId, collection) return } From e7c2fe280fc7fbb15796a42e4746c1d4c43aac1e Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Mon, 15 Dec 2025 23:04:00 +0800 Subject: [PATCH 069/225] =?UTF-8?q?=E4=BF=AE=E6=94=B9-redis=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E5=8D=95=E6=9D=A1=E6=9F=A5=E8=AF=A2=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 1 - 1 file changed, 1 deletion(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 336a5ad..d0fd7f0 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -235,7 +235,6 @@ func cleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collec delete(filter, "tenantId") filterMap := utils.OrderMap(filter) oneKey := fmt.Sprintf(consts.One, tenantId, collection, gconv.String(filterMap)) - fmt.Println(gconv.String(filterMap)) _, err = redis.RedisClient.Del(ctx, oneKey) if err != nil { return From 8823f543707affeb8c3e394cd3eaa6e24786cbd6 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 16 Dec 2025 15:20:16 +0800 Subject: [PATCH 070/225] =?UTF-8?q?=E8=AF=9D=E6=9C=AFredis=E6=9B=B4?= =?UTF-8?q?=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + redis/redis.go | 93 +++++++++++++++++++++++++++++++++++--------------- 2 files changed, 67 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index ba979d1..95c5e6f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ ragflow/client_backup.go.bak ragflow/为什么不能使用gclient.md ragflow/agent文档.md /.idea/MarsCodeWorkspaceAppSettings.xml +consul/配置中心方案.md diff --git a/redis/redis.go b/redis/redis.go index 1585f51..72b0ee5 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -479,45 +479,84 @@ func Unlock(ctx context.Context, key string) { // ============== 对话计数相关(用于卡片触发)============== const ( - // ConversationCountKeyPrefix 对话计数 Key 前缀 - ConversationCountKeyPrefix = "ragflow:conversation:count:" + // UserStateKeyPrefix 用户会话状态 Key 前缀(融合阶段+计数) + UserStateKeyPrefix = "ragflow:user:state:" + // UserStateExpireSeconds 用户状态过期时间(5分钟) + UserStateExpireSeconds = 300 ) -// IncrConversationCount 增加用户对话计数,返回当前轮数 -// 用于判断是否触发发送卡片(如对话5轮后发送) -// expireSeconds: 过期时间(秒),建议与会话超时一致(如7200秒=2小时) -func IncrConversationCount(ctx context.Context, userId, platform string, expireSeconds int64) (count int64, err error) { - key := ConversationCountKeyPrefix + userId + "_" + platform - result, err := redisClient.Do(ctx, "INCR", key) - if err != nil { - return - } - count = result.Int64() - - // 首次设置过期时间 - if count == 1 { - redisClient.Do(ctx, "EXPIRE", key, expireSeconds) - } - return +// UserState 用户会话状态(阶段+对话计数,统一5分钟过期) +type UserState struct { + Stage int `json:"stage"` // 用户阶段:0=AI模型 1=打招呼 2=业务 3=发卡片 + Count int64 `json:"count"` // 对话计数 } -// GetConversationCount 获取用户当前对话轮数 -func GetConversationCount(ctx context.Context, userId, platform string) (count int64, err error) { - key := ConversationCountKeyPrefix + userId + "_" + platform - result, err := redisClient.Get(ctx, key) +// GetUserState 获取用户状态(阶段+计数) +func GetUserState(ctx context.Context, userId, platform string) (state *UserState, err error) { + key := UserStateKeyPrefix + userId + "_" + platform + result, err := redisClient.Do(ctx, "HGETALL", key) if err != nil { return } + + state = &UserState{} if result.IsEmpty() { - return 0, nil + return // 返回默认值 stage=0, count=0 } - count = result.Int64() + + m := result.Map() + state.Stage = gconv.Int(m["stage"]) + state.Count = gconv.Int64(m["count"]) return } -// ResetConversationCount 重置用户对话计数(归档/卡片发送后调用) -func ResetConversationCount(ctx context.Context, userId, platform string) error { - key := ConversationCountKeyPrefix + userId + "_" + platform +// SetUserStage 设置用户阶段,并刷新过期时间 +func SetUserStage(ctx context.Context, userId, platform string, stage int) error { + key := UserStateKeyPrefix + userId + "_" + platform + _, err := redisClient.Do(ctx, "HSET", key, "stage", stage) + if err != nil { + return err + } + _, err = redisClient.Do(ctx, "EXPIRE", key, UserStateExpireSeconds) + return err +} + +// IncrUserCount 增加用户对话计数,返回当前轮数,并刷新过期时间 +func IncrUserCount(ctx context.Context, userId, platform string) (count int64, err error) { + key := UserStateKeyPrefix + userId + "_" + platform + result, err := redisClient.Do(ctx, "HINCRBY", key, "count", 1) + if err != nil { + return + } + count = result.Int64() + _, err = redisClient.Do(ctx, "EXPIRE", key, UserStateExpireSeconds) + return +} + +// ResetUserState 重置用户状态(归档时调用) +func ResetUserState(ctx context.Context, userId, platform string) error { + key := UserStateKeyPrefix + userId + "_" + platform _, err := redisClient.Del(ctx, key) return err } + +// ========== 以下为兼容旧接口(内部调用新实现)========== + +// IncrConversationCount 增加用户对话计数(兼容旧接口) +func IncrConversationCount(ctx context.Context, userId, platform string, _ int64) (count int64, err error) { + return IncrUserCount(ctx, userId, platform) +} + +// GetConversationCount 获取用户当前对话轮数(兼容旧接口) +func GetConversationCount(ctx context.Context, userId, platform string) (count int64, err error) { + state, err := GetUserState(ctx, userId, platform) + if err != nil { + return + } + return state.Count, nil +} + +// ResetConversationCount 重置用户对话计数(兼容旧接口) +func ResetConversationCount(ctx context.Context, userId, platform string) error { + return ResetUserState(ctx, userId, platform) +} From e22f7373e6dfddc97f6372f7fd93c578c81bc4ea Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 16 Dec 2025 18:00:07 +0800 Subject: [PATCH 071/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + consul/配置中心方案.md | 1 + 2 files changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 95c5e6f..4063493 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ ragflow/为什么不能使用gclient.md ragflow/agent文档.md /.idea/MarsCodeWorkspaceAppSettings.xml consul/配置中心方案.md +consul/配置中心方案.md diff --git a/consul/配置中心方案.md b/consul/配置中心方案.md index 3c87785..797279b 100644 --- a/consul/配置中心方案.md +++ b/consul/配置中心方案.md @@ -106,6 +106,7 @@ mongo: ### 4.1 模块结构 +consul api 里有个watch , 用watch去做 ``` common/configcenter/ ├── configcenter.go # 核心:启动/停止/获取配置 From f130c230e839519eae3000e49ea2cde0de232026 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 16 Dec 2025 18:00:16 +0800 Subject: [PATCH 072/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index d0fd7f0..394cec0 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -30,13 +30,19 @@ func GetDB() *mongo.Database { } func init() { + // 按需初始化:没有配置 mongo.address 则跳过 + mongoAddr := g.Cfg().MustGet(context.Background(), "mongo.address").String() + if mongoAddr == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - mongoAddr := g.Cfg().MustGet(context.Background(), "mongo.address").String() opt := options.Client().ApplyURI(mongoAddr) client, err := mongo.Connect(opt) if err != nil { glog.Error(ctx, "mongodb连接失败", err) + return } // 从连接串中解析数据库名 dbName := gstr.SubStr(mongoAddr, strings.LastIndex(mongoAddr, "/")+1, len(mongoAddr)) @@ -45,6 +51,7 @@ func init() { dbName = gstr.SubStr(dbName, 0, strings.Index(dbName, "?")) } db = client.Database(dbName) + glog.Info(ctx, "✅ MongoDB 初始化成功") } func listOptionsToMap(ctx context.Context, opts ...options.Lister[options.FindOptions]) (m map[string]interface{}) { // 输出opts参数中的值 From 53c339222701c18f9de566b22a2864ed38d6f887 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Thu, 18 Dec 2025 18:01:21 +0800 Subject: [PATCH 073/225] =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E5=99=A8=E5=87=86?= =?UTF-8?q?=E5=A4=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/directions.go | 236 +++++++++++++++++++++++++++++++++++ config/redis.go | 0 config/welcome.go | 40 ++++++ config/welcome_messages.yaml | 53 ++++++++ rabbitmq/consumer_manager.go | 171 +++++++++++++++++++++++++ rabbitmq/queue_setup.go | 111 ++++++++++++++++ redis/redis.go | 76 ++++++++++- 7 files changed, 682 insertions(+), 5 deletions(-) create mode 100644 config/directions.go create mode 100644 config/redis.go create mode 100644 config/welcome.go create mode 100644 config/welcome_messages.yaml create mode 100644 rabbitmq/consumer_manager.go create mode 100644 rabbitmq/queue_setup.go diff --git a/config/directions.go b/config/directions.go new file mode 100644 index 0000000..e0e2c15 --- /dev/null +++ b/config/directions.go @@ -0,0 +1,236 @@ +// Package config 提供全局配置管理和Consul监听 +// +// 本包实现了基于Consul的配置热更新机制,所有服务导入common包即可自动获得配置监听能力 +package config + +import ( + "context" + "encoding/json" + "sync" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" + "github.com/hashicorp/consul/api" +) + +// Direction 咨询方向配置 +type Direction struct { + Name string `json:"name"` // 方向名称(如:气血、减肥) + ChatId string `json:"chat_id"` // RAGFlow对话ID +} + +var ( + directionsCache []Direction // 本地缓存(内存读取,超快) + cacheMu sync.RWMutex // 读写锁(支持多goroutine并发读) + startOnce sync.Once // 确保只启动一次监听 + consulClient *api.Client // Consul客户端(复用连接) +) + +// init 包初始化函数(所有服务导入common包时自动执行) +// +// Fallback顺序:Consul → config.yml +func init() { + ctx := context.Background() + + // 检查Consul是否配置 + consulAddr := g.Cfg().MustGet(ctx, "consul.address").String() + if consulAddr == "" { + glog.Warning(ctx, "Consul未配置,使用本地配置") + loadDirectionsFromLocal(ctx) + return + } + + // 初始化Consul客户端 + config := api.DefaultConfig() + config.Address = consulAddr + client, err := api.NewClient(config) + if err != nil { + glog.Errorf(ctx, "Consul客户端初始化失败: %v,fallback到本地配置", err) + loadDirectionsFromLocal(ctx) + return + } + consulClient = client + + // 启动后台监听(单例,确保只启动一次) + startOnce.Do(func() { + go startConsulWatcher(ctx) + glog.Info(ctx, "Consul配置监听已启动") + }) +} + +// GetDirections 获取咨询方向配置(从内存缓存读取) +// +// 返回: +// +// []Direction: 方向列表 +// +// 特点: +// - 高性能:读内存缓存,无网络IO +// - 线程安全:使用读锁,支持并发读取 +// - 自动更新:后台监听Consul,配置变化时自动更新缓存 +// +// 使用示例: +// +// dirs := config.GetDirections() +// for _, dir := range dirs { +// fmt.Printf("%s -> %s\n", dir.Name, dir.ChatId) +// } +func GetDirections() []Direction { + cacheMu.RLock() + defer cacheMu.RUnlock() + + // 返回副本,避免外部修改缓存 + result := make([]Direction, len(directionsCache)) + copy(result, directionsCache) + return result +} + +// GetDirectionChatId 根据方向名称获取对应的ChatId +// +// 参数: +// +// name: 方向名称(如:"气血"、"减肥") +// +// 返回: +// +// chatId: 对应的RAGFlow对话ID,未找到返回空字符串 +// +// 使用示例: +// +// chatId := config.GetDirectionChatId("气血") +func GetDirectionChatId(name string) string { + cacheMu.RLock() + defer cacheMu.RUnlock() + + for _, dir := range directionsCache { + if dir.Name == name { + return dir.ChatId + } + } + return "" +} + +// startConsulWatcher 后台监听Consul配置变化(Blocking Query长连接) +// +// 工作原理: +// 1. 使用Consul Blocking Query API(长连接,只在变化时返回) +// 2. 收到变化通知后更新本地缓存 +// 3. 自动重连(网络异常时自动恢复) +// +// 资源消耗: +// - 一个长连接(保持5分钟) +// - 配置未变化时几乎不占用CPU和网络 +// - 对比轮询:节省99%资源 +// +// 注意: +// - 此函数在后台goroutine中运行 +// - 使用Blocking Query避免轮询 +func startConsulWatcher(ctx context.Context) { + const consulKey = "ragflow/directions" + kv := consulClient.KV() + var lastIndex uint64 + + // 初始化时先读取一次配置 + if err := loadDirectionsFromConsul(ctx, kv); err != nil { + glog.Warningf(ctx, "初始化加载Consul配置失败: %v", err) + } + + // 持续监听配置变化 + for { + // Consul Blocking Query(长连接模式) + // WaitIndex: 指定版本号,只在配置变化时返回 + // WaitTime: 最长等待时间(超时后返回,客户端重新请求) + pair, meta, err := kv.Get(consulKey, &api.QueryOptions{ + WaitIndex: lastIndex, + WaitTime: 5 * time.Minute, + }) + + if err != nil { + glog.Errorf(ctx, "Consul查询失败: %v", err) + time.Sleep(5 * time.Second) // 错误时等待5秒重试 + continue + } + + // 配置版本号变化,说明有更新 + if meta.LastIndex != lastIndex { + lastIndex = meta.LastIndex + + // 配置被删除 + if pair == nil { + glog.Warning(ctx, "Consul配置已删除: "+consulKey) + cacheMu.Lock() + directionsCache = []Direction{} + cacheMu.Unlock() + continue + } + + // 解析并更新缓存 + var dirs []Direction + if err := json.Unmarshal(pair.Value, &dirs); err != nil { + glog.Errorf(ctx, "解析Consul配置失败: %v", err) + continue + } + + cacheMu.Lock() + directionsCache = dirs + cacheMu.Unlock() + + glog.Infof(ctx, "Consul配置已更新: %d个方向", len(dirs)) + } + } +} + +// loadDirectionsFromConsul 从Consul加载配置(初始化时调用) +func loadDirectionsFromConsul(ctx context.Context, kv *api.KV) error { + const consulKey = "ragflow/directions" + + pair, _, err := kv.Get(consulKey, nil) + if err != nil { + // Consul查询失败,fallback到本地配置 + glog.Warningf(ctx, "Consul查询失败: %v,fallback到本地配置", err) + loadDirectionsFromLocal(ctx) + return err + } + + if pair == nil { + glog.Warning(ctx, "Consul中未找到配置: "+consulKey+",fallback到本地配置") + loadDirectionsFromLocal(ctx) + return nil + } + + var dirs []Direction + if err := json.Unmarshal(pair.Value, &dirs); err != nil { + glog.Errorf(ctx, "解析Consul配置失败: %v,fallback到本地配置", err) + loadDirectionsFromLocal(ctx) + return err + } + + cacheMu.Lock() + directionsCache = dirs + cacheMu.Unlock() + + glog.Infof(ctx, "已加载Consul配置: %d个方向", len(dirs)) + return nil +} + +// loadDirectionsFromLocal 从本地config.yml加载配置(fallback机制) +func loadDirectionsFromLocal(ctx context.Context) { + directionsConfig := g.Cfg().MustGet(ctx, "ragflow.directions") + if directionsConfig.IsEmpty() { + glog.Warning(ctx, "本地配置中也未找到 ragflow.directions") + return + } + + var dirs []Direction + if err := directionsConfig.Scan(&dirs); err != nil { + glog.Errorf(ctx, "解析本地配置失败: %v", err) + return + } + + cacheMu.Lock() + directionsCache = dirs + cacheMu.Unlock() + + glog.Infof(ctx, "已加载config.yml配置: %d个方向", len(dirs)) +} diff --git a/config/redis.go b/config/redis.go new file mode 100644 index 0000000..e69de29 diff --git a/config/welcome.go b/config/welcome.go new file mode 100644 index 0000000..6dfe133 --- /dev/null +++ b/config/welcome.go @@ -0,0 +1,40 @@ +package config + +import ( + "context" + "sync" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" +) + +var ( + welcomeCache map[string]string + welcomeMu sync.RWMutex + welcomeOnce sync.Once +) + +// initWelcomeMessages 初始化欢迎话术配置 +func initWelcomeMessages(ctx context.Context) { + welcomeOnce.Do(func() { + cfg := g.Cfg() + welcomeMap := cfg.MustGet(ctx, "welcomes").MapStrStr() + + welcomeMu.Lock() + welcomeCache = welcomeMap + welcomeMu.Unlock() + + glog.Infof(ctx, "已加载欢迎话术配置: %d个方向", len(welcomeMap)) + }) +} + +// GetWelcomeMessage 根据方向名称获取欢迎话术 +func GetWelcomeMessage(direction string) string { + ctx := context.Background() + initWelcomeMessages(ctx) + + welcomeMu.RLock() + defer welcomeMu.RUnlock() + + return welcomeCache[direction] +} diff --git a/config/welcome_messages.yaml b/config/welcome_messages.yaml new file mode 100644 index 0000000..0286482 --- /dev/null +++ b/config/welcome_messages.yaml @@ -0,0 +1,53 @@ +# 各咨询方向的欢迎话术配置 +# 当用户在状态5选择方向后,会自动发送对应的欢迎语 + +welcomes: + 乳腺贴: | + 🙋‍♀️ 欢迎姐妹们来到药济堂!我们是一个拥有10年经验的大健康专业团队🏅,专注于保守调理乳腺问题,已经成功帮助超过1万位姐妹轻松调理乳腺健康❤️。 + 如果您有结节或增生的困扰,欢迎随时咨询! + 请回复下面的数字,让我帮助您分析结节情况: + + 结节 + 增生 + 点击👇获取更快速的服务! + + 肝病: | + 你好,我是黄医生,有什么肝脏方面的问题我可以帮助您吗?无论是肝病、乙肝、丙肝,还是肝硬化腹水、脂肪肝、酒精肝,我都会为您提供专业的建议。😷 + 肝病的类型很多,常见症状有乏力、食欲减退和肝区不适等。为了更好地帮您,我需要了解一些详细的信息,比如: + + 您现在有腹胀或腹水的情况吗? + 是第一次出现还是反复出现呢? + 目前是早期还是中晚期呢? + 有没有病毒性肝炎的病史呢? + 请您留一下联系方式,我可以发送您的报告,并给您详细解读肝病治疗方案和成功案例。📋 + + 车膜: | + 🎉亲爱的车主,欢迎来到6膜王! + 🚗 我们在车膜行业深耕十年,拥有2家千平米的门店和30+专业团队,致力于为您提供最优质的服务! + ✨ 无论是隐形车衣还是改色膜,我们都能精准适配您的需求。留下您的【车型➕VX】,我们的资深顾问会立即为您匹配专属方案、膜材讲解以及报价!❤️ + + 毛孔: | + 啊啊~亲爱的姐妹们,最近有很多小伙伴在问我关于毛孔的问题!我之前也是毛孔大到妆容卡粉,真的是烦恼不断。😩 但在经过一段时间的摸索后,我终于找到了合适的方法,效果真的很好,差不多一个月就改善了许多!(亲测有效!)💖 + 如果需要帮助,可以随时告诉我,我非常乐意分享我的经验给你们哦! + + 免税店: | + Hi,长春的宝子们~💕欢迎锁定小红提免税集合店! + 这里有你想要的一切:美妆、香水、包包、首饰和大牌护肤品,正品保真不踩雷,价格特别美丽哦✨ + 全城顺丰包邮,购物更方便!期待你们来逛快来发现更多惊喜吧!💖 + + 门店地址:长春市绿园区皓月大路吾悦广场1楼 1036号,等你来哦!🌟 + + 减肥: | + 你好呀,姐妹~你也有肉肉的困扰吗?我现在已经掉了二十多斤了,至今都没有反弹! + 想要方法的话可以直接回复"1",我分享给你哦~ + (💗未成年发育期、哺乳期的姐妹我就不推荐啦) + + 气血: | + 亲爱的,欢迎光临!🌸 + 如果你有月经不调或气血不足的问题,随时可以问我哦! + + 停经闭经 + 痛经难忍 + 量少 + 经期不准 + 💗只需回复数字,我们会为你提供专业建议! 🌟如需更多帮助,点击下方咨询专业老师,我们一起寻求解决方案~ diff --git a/rabbitmq/consumer_manager.go b/rabbitmq/consumer_manager.go new file mode 100644 index 0000000..28695ca --- /dev/null +++ b/rabbitmq/consumer_manager.go @@ -0,0 +1,171 @@ +// Package rabbitmq 提供 RabbitMQ 消费者管理功能 +// +// 本文件实现消费者统一管理,简化业务层的启动逻辑 +package rabbitmq + +import ( + "context" + "sync" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" +) + +// ManagedConsumer 托管消费者(包含启动和停止函数) +type ManagedConsumer struct { + Name string // 消费者名称 + Start func(ctx context.Context) error // 启动函数 + Stop func(ctx context.Context) // 停止函数 +} + +// ConsumerManager RabbitMQ 消费者管理器 +// +// 职责: +// 1. 统一管理所有 RabbitMQ 消费者的生命周期 +// 2. 初始化 RabbitMQ 连接和队列 +// 3. 启动/停止所有消费者 +// 4. 协调消费者的优雅退出 +// +// 使用示例: +// +// mgr := rabbitmq.NewConsumerManager(ctx) +// mgr.Register("响应消费者", responseConsumer.Start, responseConsumer.Stop) +// mgr.Init() +// defer mgr.Stop() +type ConsumerManager struct { + ctx context.Context // 全局上下文 + consumers []*ManagedConsumer // 消费者列表 + wg sync.WaitGroup // 等待所有消费者协程退出 +} + +// NewConsumerManager 创建消费者管理器 +// +// 参数: +// +// ctx: 上下文 +// +// 返回: +// +// *ConsumerManager: 消费者管理器实例 +func NewConsumerManager(ctx context.Context) *ConsumerManager { + return &ConsumerManager{ + ctx: ctx, + consumers: make([]*ManagedConsumer, 0), + } +} + +// Register 注册消费者 +// +// 参数: +// +// name: 消费者名称(用于日志) +// startFunc: 启动函数 +// stopFunc: 停止函数 +// +// 使用示例: +// +// consumer := service.NewResponseConsumer(ctx) +// mgr.Register("响应消费者", consumer.Start, consumer.Stop) +func (cm *ConsumerManager) Register(name string, startFunc func(ctx context.Context) error, stopFunc func(ctx context.Context)) { + cm.consumers = append(cm.consumers, &ManagedConsumer{ + Name: name, + Start: startFunc, + Stop: stopFunc, + }) +} + +// Init 初始化并启动所有消费者 +// +// 执行流程: +// 1. 检查 RabbitMQ 配置(未配置则跳过) +// 2. 初始化 RabbitMQ 连接 +// 3. 声明并绑定队列(响应队列、延时落库队列) +// 4. 异步启动所有已注册的消费者 +// +// 返回: +// +// err: 错误信息,成功返回 nil +// +// 注意: +// - 如果 RabbitMQ 未配置,不会报错,只是跳过初始化 +// - 响应队列初始化失败会导致 Fatal 退出 +// - 延时落库队列失败只会 Warning,不影响主流程 +func (cm *ConsumerManager) Init() (err error) { + // 检查配置文件中是否配置了 RabbitMQ + if g.Cfg().MustGet(cm.ctx, "rabbitmq").IsEmpty() { + glog.Info(cm.ctx, "RabbitMQ未配置,跳过消费者初始化") + return + } + + // 初始化 RabbitMQ 连接(从 config.yml 读取配置) + if err = InitFromConfig(cm.ctx); err != nil { + glog.Fatalf(cm.ctx, "初始化 RabbitMQ 失败: %v", err) + return + } + glog.Info(cm.ctx, "RabbitMQ 连接已初始化") + + // 设置响应队列(RAGFlow 响应消息) + if err = SetupResponseQueue(cm.ctx); err != nil { + glog.Fatalf(cm.ctx, "设置响应队列失败: %v", err) + return + } + + // 设置延时落库队列(对话缓存兜底机制) + // 失败不影响主流程,只记录 Warning + if err = SetupDelayedFlushQueue(cm.ctx); err != nil { + glog.Warningf(cm.ctx, "设置延时落库队列失败: %v", err) + } + + // 异步启动所有已注册的消费者 + cm.startConsumers() + return +} + +// startConsumers 启动所有消费者(内部方法) +// +// 实现: +// 1. 遍历已注册的消费者 +// 2. 每个消费者在独立的 goroutine 中运行 +// 3. 使用 WaitGroup 追踪所有消费者协程 +func (cm *ConsumerManager) startConsumers() { + for _, c := range cm.consumers { + cm.wg.Add(1) + go func(consumer *ManagedConsumer) { + defer cm.wg.Done() + if err := consumer.Start(cm.ctx); err != nil { + glog.Errorf(cm.ctx, "%s启动失败: %v", consumer.Name, err) + } + }(c) + glog.Infof(cm.ctx, "%s已启动", c.Name) + } +} + +// Stop 停止所有消费者(优雅退出) +// +// 执行流程: +// 1. 依次停止所有消费者(调用各自的 Stop 方法) +// 2. 等待所有消费者协程退出(WaitGroup.Wait) +// 3. 关闭 RabbitMQ 连接 +// +// 使用场景: +// - 收到 SIGINT/SIGTERM 信号时 +// - 程序正常退出时 +// - defer mgr.Stop() +// +// 注意: +// - Stop 方法会阻塞直到所有消费者完全退出 +// - 确保消费者能正确响应 Stop 信号 +func (cm *ConsumerManager) Stop() { + // 依次停止所有消费者 + for _, c := range cm.consumers { + c.Stop(cm.ctx) + glog.Infof(cm.ctx, "%s已停止", c.Name) + } + + // 等待所有消费者协程退出 + cm.wg.Wait() + + // 关闭 RabbitMQ 连接 + Close(cm.ctx) + glog.Info(cm.ctx, "所有消费者已停止,RabbitMQ连接已关闭") +} diff --git a/rabbitmq/queue_setup.go b/rabbitmq/queue_setup.go new file mode 100644 index 0000000..dbb3130 --- /dev/null +++ b/rabbitmq/queue_setup.go @@ -0,0 +1,111 @@ +// Package rabbitmq 提供 RabbitMQ 队列初始化的封装方法 +// +// 本文件包含常用队列的声明和绑定逻辑,简化业务层的队列配置代码 +package rabbitmq + +import ( + "context" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" +) + +// SetupResponseQueue 初始化 RAGFlow 响应队列 +// +// 功能: +// 1. 声明持久化队列(从配置文件读取队列名,默认 ragflow.response.queue) +// 2. 绑定到 ragflow.response Exchange(Topic 类型) +// 3. 使用通配符 # 匹配所有 routing key(userId) +// +// 参数: +// +// ctx: 上下文 +// +// 返回: +// +// err: 错误信息,成功返回 nil +// +// 配置示例(config.yml): +// +// rabbitmq: +// responseQueue: "ragflow.response.queue" # 可选,默认值 +func SetupResponseQueue(ctx context.Context) (err error) { + // 从配置文件读取队列名(支持每个开发者配置独立队列名) + responseQueue := g.Cfg().MustGet(ctx, "rabbitmq.responseQueue", "ragflow.response.queue").String() + + // 声明持久化队列(服务器重启后队列仍存在) + if err = DeclareQueue(ctx, &QueueConfig{ + Name: responseQueue, + Durable: true, // 持久化,防止数据丢失 + }); err != nil { + glog.Errorf(ctx, "声明响应队列失败: %v", err) + return + } + + // 绑定队列到 Exchange + // Exchange 类型为 topic,routing key 使用通配符 # 匹配所有 userId + if err = BindQueue(ctx, &BindingConfig{ + Queue: responseQueue, + Exchange: "ragflow.response", // RAGFlow 响应 Exchange + RoutingKey: "#", // 通配符,匹配所有消息 + }); err != nil { + glog.Errorf(ctx, "绑定响应队列失败: %v", err) + return + } + + glog.Infof(ctx, "响应队列已绑定: %s -> ragflow.response (routingKey=#)", responseQueue) + return +} + +// SetupDelayedFlushQueue 初始化延时落库队列 +// +// 功能: +// 1. 声明延时 Exchange(x-delayed-message 插件) +// 2. 声明持久化队列 conversation.flush.queue +// 3. 绑定队列到延时 Exchange +// +// 用途: +// +// 对话缓存延时落库机制的兜底策略 +// 当对话少于5句时,10分钟后触发延时消息将缓存写入MongoDB +// +// 参数: +// +// ctx: 上下文 +// +// 返回: +// +// err: 错误信息,成功返回 nil +// +// 相关: +// - service/conversation_service.go: handleResponse() +// - service/conversation_service.go: handleDelayedFlush() +func SetupDelayedFlushQueue(ctx context.Context) (err error) { + // 声明延时 Exchange(需要 RabbitMQ 安装 x-delayed-message 插件) + if err = SetupDelayExchange(ctx, "conversation.flush.delayed"); err != nil { + glog.Warningf(ctx, "声明延时落库 Exchange 失败: %v", err) + return + } + + // 声明持久化队列 + if err = DeclareQueue(ctx, &QueueConfig{ + Name: "conversation.flush.queue", + Durable: true, // 持久化,防止延时消息丢失 + }); err != nil { + glog.Warningf(ctx, "声明延时落库 Queue 失败: %v", err) + return + } + + // 绑定队列到延时 Exchange + if err = BindQueue(ctx, &BindingConfig{ + Queue: "conversation.flush.queue", + Exchange: "conversation.flush.delayed", + RoutingKey: "flush", // 延时落库消息的 routing key + }); err != nil { + glog.Warningf(ctx, "绑定延时落库 Queue 失败: %v", err) + return + } + + glog.Info(ctx, "延时落库队列已配置") + return +} diff --git a/redis/redis.go b/redis/redis.go index 72b0ee5..69ff1c5 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -485,10 +485,11 @@ const ( UserStateExpireSeconds = 300 ) -// UserState 用户会话状态(阶段+对话计数,统一5分钟过期) +// UserState 用户会话状态(阶段+对话计数+咨询方向,统一5分钟过期) type UserState struct { - Stage int `json:"stage"` // 用户阶段:0=AI模型 1=打招呼 2=业务 3=发卡片 - Count int64 `json:"count"` // 对话计数 + Stage int `json:"stage"` // 用户阶段:5=未选择方向 0=AI模型 1=打招呼 2=业务 3=发卡片 + Count int64 `json:"count"` // 对话计数 + Direction string `json:"direction"` // 用户选择的咨询方向(如:产品咨询、售后服务) } // GetUserState 获取用户状态(阶段+计数) @@ -499,14 +500,15 @@ func GetUserState(ctx context.Context, userId, platform string) (state *UserStat return } - state = &UserState{} + state = &UserState{Stage: 5} // 默认状态5(未选择方向) if result.IsEmpty() { - return // 返回默认值 stage=0, count=0 + return } m := result.Map() state.Stage = gconv.Int(m["stage"]) state.Count = gconv.Int64(m["count"]) + state.Direction = gconv.String(m["direction"]) return } @@ -521,6 +523,17 @@ func SetUserStage(ctx context.Context, userId, platform string, stage int) error return err } +// SetUserDirection 设置用户选择的咨询方向,并刷新过期时间 +func SetUserDirection(ctx context.Context, userId, platform, direction string) error { + key := UserStateKeyPrefix + userId + "_" + platform + _, err := redisClient.Do(ctx, "HSET", key, "direction", direction) + if err != nil { + return err + } + _, err = redisClient.Do(ctx, "EXPIRE", key, UserStateExpireSeconds) + return err +} + // IncrUserCount 增加用户对话计数,返回当前轮数,并刷新过期时间 func IncrUserCount(ctx context.Context, userId, platform string) (count int64, err error) { key := UserStateKeyPrefix + userId + "_" + platform @@ -540,6 +553,59 @@ func ResetUserState(ctx context.Context, userId, platform string) error { return err } +// ============== 对话缓存相关(5句落库)============== + +const ( + // ConversationCacheKeyPrefix 对话缓存 Key 前缀 + ConversationCacheKeyPrefix = "ragflow:conversation:cache:" + // ConversationCacheExpireSeconds 对话缓存过期时间(10分钟) + ConversationCacheExpireSeconds = 600 +) + +// CacheConversation 缓存单条对话到Redis List +func CacheConversation(ctx context.Context, userId, platform string, data []byte) error { + key := ConversationCacheKeyPrefix + userId + "_" + platform + _, err := redisClient.Do(ctx, "RPUSH", key, string(data)) + if err != nil { + return err + } + _, err = redisClient.Do(ctx, "EXPIRE", key, ConversationCacheExpireSeconds) + return err +} + +// GetCachedConversations 获取缓存的对话列表并清空 +func GetCachedConversations(ctx context.Context, userId, platform string) (list []string, err error) { + key := ConversationCacheKeyPrefix + userId + "_" + platform + result, err := redisClient.Do(ctx, "LRANGE", key, 0, -1) + if err != nil { + return + } + if result.IsEmpty() { + return + } + list = result.Strings() + // 清空缓存 + redisClient.Del(ctx, key) + return +} + +// GetCachedConversationCount 获取缓存的对话数量 +func GetCachedConversationCount(ctx context.Context, userId, platform string) (count int64, err error) { + key := ConversationCacheKeyPrefix + userId + "_" + platform + result, err := redisClient.Do(ctx, "LLEN", key) + if err != nil { + return + } + return result.Int64(), nil +} + +// ClearCachedConversations 清空对话缓存(归档时调用) +func ClearCachedConversations(ctx context.Context, userId, platform string) error { + key := ConversationCacheKeyPrefix + userId + "_" + platform + _, err := redisClient.Del(ctx, key) + return err +} + // ========== 以下为兼容旧接口(内部调用新实现)========== // IncrConversationCount 增加用户对话计数(兼容旧接口) From d82025e854dfa7ba6046cb5f47a9d268bcdb31d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 18 Dec 2025 16:43:05 +0800 Subject: [PATCH 074/225] =?UTF-8?q?gomod=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/http/http.go b/http/http.go index b76621d..09f2641 100644 --- a/http/http.go +++ b/http/http.go @@ -92,3 +92,11 @@ func Post(ctx context.Context, url string, target any, data ...any) (err error) err = doRequest(ctx, http.MethodPost, url, target, data) return } +func Put(ctx context.Context, url string, target any, data ...any) (err error) { + err = doRequest(ctx, http.MethodPut, url, target, data) + return +} +func Delete(ctx context.Context, url string, target any, data ...any) (err error) { + err = doRequest(ctx, http.MethodPut, url, target, data) + return +} From 5bc580b3b8f6cd2e2fb514b1ebedbda8a4baeacb Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Fri, 19 Dec 2025 09:18:38 +0800 Subject: [PATCH 075/225] =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E5=99=A8=E6=9B=B4?= =?UTF-8?q?=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/redis.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 config/redis.go diff --git a/config/redis.go b/config/redis.go deleted file mode 100644 index e69de29..0000000 From 7381d57b77e7a7c63a4fa89fdc5036dc0331d0e4 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Fri, 19 Dec 2025 15:02:05 +0800 Subject: [PATCH 076/225] =?UTF-8?q?=E5=AE=8C=E6=88=90websocket=E5=AF=B9?= =?UTF-8?q?=E8=AF=9D=E6=BC=94=E7=A4=BA=E5=92=8Cmain=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E7=9A=84=E5=8A=9F=E8=83=BD=E6=8A=BD=E7=A6=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/welcome.go | 4 ++-- rabbitmq/consumer.go | 28 ++++++++++++++++++++++++++++ rabbitmq/instance.go | 41 +++++++++++++++++++++++++++++++++++++++++ redis/redis.go | 5 +++++ redis/types.go | 17 +++++++++-------- 5 files changed, 85 insertions(+), 10 deletions(-) create mode 100644 rabbitmq/instance.go diff --git a/config/welcome.go b/config/welcome.go index 6dfe133..4e4bb76 100644 --- a/config/welcome.go +++ b/config/welcome.go @@ -17,8 +17,8 @@ var ( // initWelcomeMessages 初始化欢迎话术配置 func initWelcomeMessages(ctx context.Context) { welcomeOnce.Do(func() { - cfg := g.Cfg() - welcomeMap := cfg.MustGet(ctx, "welcomes").MapStrStr() + // 从默认配置文件(config.yml)读取 welcomes 配置 + welcomeMap := g.Cfg().MustGet(ctx, "welcomes").MapStrStr() welcomeMu.Lock() welcomeCache = welcomeMap diff --git a/rabbitmq/consumer.go b/rabbitmq/consumer.go index 4fb5a2f..0ddca60 100644 --- a/rabbitmq/consumer.go +++ b/rabbitmq/consumer.go @@ -83,6 +83,34 @@ func (c *Consumer) Start(ctx context.Context) (err error) { return err } + // 声明队列(如果不存在则创建) + _, err = ch.QueueDeclare( + c.queue, // name + true, // durable(持久化) + false, // autoDelete(不自动删除) + false, // exclusive(非独占) + false, // noWait + nil, // arguments + ) + if err != nil { + return gerror.Newf("声明队列失败: %v", err) + } + + // TODO: 队列绑定逻辑暂时注释,避免重复binding导致消息重复投递 + // 绑定队列到Exchange(使用队列名作为routingKey,支持多租户) + // Exchange类型应该是topic,绑定模式为 #(接收所有消息) + // err = ch.QueueBind( + // c.queue, // queue name + // "#", // routing key(通配符,接收所有消息) + // "ragflow.response", // exchange name + // false, // noWait + // nil, // arguments + // ) + // if err != nil { + // g.Log().Warningf(ctx, "绑定队列到Exchange失败(可能Exchange不存在或类型不匹配): %v", err) + // // 不返回错误,继续启动消费者(可能是direct exchange或队列已绑定) + // } + // 设置 QoS(并发控制) err = ch.Qos( c.prefetchCount, // prefetchCount: 每个 consumer 最多同时处理的消息数 diff --git a/rabbitmq/instance.go b/rabbitmq/instance.go new file mode 100644 index 0000000..330c6d5 --- /dev/null +++ b/rabbitmq/instance.go @@ -0,0 +1,41 @@ +package rabbitmq + +import ( + "fmt" + "os" + "sync" + + "github.com/gogf/gf/v2/util/guid" +) + +var ( + instanceId string + instanceOnce sync.Once +) + +// getInstanceId 获取当前实例的唯一标识(单例) +// 格式:{hostname}.{uuid8} +func getInstanceId() string { + instanceOnce.Do(func() { + // 获取主机名 + hostname, err := os.Hostname() + if err != nil || hostname == "" { + hostname = "unknown" + } + + // 生成8位UUID + uuid := guid.S()[:8] + + instanceId = fmt.Sprintf("%s.%s", hostname, uuid) + }) + return instanceId +} + +// GetInstanceQueueName 获取当前实例的响应队列名 +// 格式:{baseQueue}.{hostname}.{uuid8} +func GetInstanceQueueName(baseQueue string) string { + if baseQueue == "" { + baseQueue = "ragflow.response" + } + return fmt.Sprintf("%s.%s", baseQueue, getInstanceId()) +} diff --git a/redis/redis.go b/redis/redis.go index 69ff1c5..072fb1b 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -502,6 +502,11 @@ func GetUserState(ctx context.Context, userId, platform string) (state *UserStat state = &UserState{Stage: 5} // 默认状态5(未选择方向) if result.IsEmpty() { + // Redis为空,初始化默认状态 + if initErr := SetUserStage(ctx, userId, platform, 5); initErr != nil { + err = initErr + return + } return } diff --git a/redis/types.go b/redis/types.go index 59bf280..17c571f 100644 --- a/redis/types.go +++ b/redis/types.go @@ -10,14 +10,15 @@ type HistoryMessage struct { // SendStreamMessage 发送到 Redis Stream 的消息结构 type SendStreamMessage struct { - UserId string `json:"user_id"` // 用户ID - Content string `json:"content"` // 消息内容 - Timestamp int64 `json:"timestamp"` // 时间戳(秒) - MessageId string `json:"message_id"` // 消息唯一ID - Platform string `json:"platform,omitempty"` // 平台标识 - AccountId string `json:"account_id,omitempty"` // 账号ID - TenantId string `json:"tenant_id,omitempty"` // 租户ID(数据隔离) - History []HistoryMessage `json:"history,omitempty"` // 历史对话(归档后恢复时携带) + UserId string `json:"user_id"` // 用户ID + Content string `json:"content"` // 消息内容 + Timestamp int64 `json:"timestamp"` // 时间戳(秒) + MessageId string `json:"message_id"` // 消息唯一ID + Platform string `json:"platform,omitempty"` // 平台标识 + AccountId string `json:"account_id,omitempty"` // 账号ID + TenantId string `json:"tenant_id,omitempty"` // 租户ID(数据隔离) + ReplyQueue string `json:"reply_queue,omitempty"` // 响应队列名称(支持多实例独立队列) + History []HistoryMessage `json:"history,omitempty"` // 历史对话(归档后恢复时携带) } // BatchStreamMessage 批量消息结构 From 8850ed283e2a3fba555dc1b4d4c4c8acfdd03658 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Fri, 19 Dec 2025 15:23:20 +0800 Subject: [PATCH 077/225] =?UTF-8?q?=E7=94=B1message=E7=BB=91=E5=AE=9Aqueue?= =?UTF-8?q?=E5=88=B0exchange?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rabbitmq/consumer.go | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/rabbitmq/consumer.go b/rabbitmq/consumer.go index 0ddca60..305b3f7 100644 --- a/rabbitmq/consumer.go +++ b/rabbitmq/consumer.go @@ -84,6 +84,7 @@ func (c *Consumer) Start(ctx context.Context) (err error) { } // 声明队列(如果不存在则创建) + // 注意:Queue到Exchange的绑定应由message服务在发送响应时动态创建,或通过运维工具提前配置 _, err = ch.QueueDeclare( c.queue, // name true, // durable(持久化) @@ -96,21 +97,6 @@ func (c *Consumer) Start(ctx context.Context) (err error) { return gerror.Newf("声明队列失败: %v", err) } - // TODO: 队列绑定逻辑暂时注释,避免重复binding导致消息重复投递 - // 绑定队列到Exchange(使用队列名作为routingKey,支持多租户) - // Exchange类型应该是topic,绑定模式为 #(接收所有消息) - // err = ch.QueueBind( - // c.queue, // queue name - // "#", // routing key(通配符,接收所有消息) - // "ragflow.response", // exchange name - // false, // noWait - // nil, // arguments - // ) - // if err != nil { - // g.Log().Warningf(ctx, "绑定队列到Exchange失败(可能Exchange不存在或类型不匹配): %v", err) - // // 不返回错误,继续启动消费者(可能是direct exchange或队列已绑定) - // } - // 设置 QoS(并发控制) err = ch.Qos( c.prefetchCount, // prefetchCount: 每个 consumer 最多同时处理的消息数 From 4380527eff141129b3da671622c9af5bde7a7212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 19 Dec 2025 12:42:37 +0800 Subject: [PATCH 078/225] =?UTF-8?q?gomod=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 156 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 143 insertions(+), 13 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 394cec0..8bb52c8 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "sync" "time" "gitee.com/red-future---jilin-g/common/consts" @@ -22,37 +23,166 @@ import ( "go.mongodb.org/mongo-driver/v2/mongo/options" ) -var db *mongo.Database +var ( + db *mongo.Database + client *mongo.Client + isConnected bool + mu sync.RWMutex + mongoAddr string + dbName string + healthCtx context.Context + healthCancel context.CancelFunc +) // GetDB 获取 MongoDB 数据库实例 func GetDB() *mongo.Database { + mu.RLock() + defer mu.RUnlock() return db } +// connect 建立MongoDB连接 +func connect() error { + mu.Lock() + defer mu.Unlock() + + if client != nil { + err := client.Disconnect(context.Background()) + if err != nil { + return err + } + } + + // 创建连接选项 + opt := options.Client(). + ApplyURI(mongoAddr). + SetMaxPoolSize(100). + SetMinPoolSize(10). + SetMaxConnecting(10). + SetConnectTimeout(10 * time.Second) + + var err error + client, err = mongo.Connect(opt) + if err != nil { + isConnected = false + glog.Error(context.Background(), "MongoDB连接失败", err) + return err + } + + // 测试连接 + testCtx, testCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer testCancel() + + err = client.Ping(testCtx, nil) + if err != nil { + isConnected = false + glog.Error(testCtx, "MongoDB连接测试失败", err) + return err + } + + db = client.Database(dbName) + isConnected = true + glog.Info(context.Background(), "✅ MongoDB连接成功") + return nil +} + +// healthCheck 健康检查协程 +func healthCheck() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-healthCtx.Done(): + return + case <-ticker.C: + mu.RLock() + currentConnected := isConnected + currentClient := client + mu.RUnlock() + + if !currentConnected || currentClient == nil { + glog.Warning(context.Background(), "MongoDB连接断开,尝试重连") + if err := reconnect(); err != nil { + glog.Error(context.Background(), "MongoDB重连失败", err) + } + continue + } + + // 测试连接状态 + testCtx, testCancel := context.WithTimeout(context.Background(), 5*time.Second) + err := currentClient.Ping(testCtx, nil) + testCancel() + + if err != nil { + mu.Lock() + isConnected = false + mu.Unlock() + glog.Warning(context.Background(), "MongoDB连接健康检查失败", err) + + // 尝试重连 + if err := reconnect(); err != nil { + glog.Error(context.Background(), "MongoDB重连失败", err) + } + } else { + glog.Debug(context.Background(), "MongoDB连接健康检查通过") + } + } + } +} + +// reconnect 重连函数 +func reconnect() error { + maxRetries := 3 + retryDelay := 2 * time.Second + + for i := 0; i < maxRetries; i++ { + glog.Info(context.Background(), fmt.Sprintf("尝试第%d次重连MongoDB", i+1)) + + if err := connect(); err == nil { + glog.Info(context.Background(), "MongoDB重连成功") + return nil + } + + if i < maxRetries-1 { + time.Sleep(retryDelay) + retryDelay *= 2 // 指数退避 + } + } + + return gerror.New("MongoDB重连失败,已达到最大重试次数") +} + +// init 初始化MongoDB连接 func init() { // 按需初始化:没有配置 mongo.address 则跳过 - mongoAddr := g.Cfg().MustGet(context.Background(), "mongo.address").String() + mongoAddr = g.Cfg().MustGet(context.Background(), "mongo.address").String() if mongoAddr == "" { return } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - opt := options.Client().ApplyURI(mongoAddr) - client, err := mongo.Connect(opt) - if err != nil { - glog.Error(ctx, "mongodb连接失败", err) - return - } + // 创建健康检查上下文 + healthCtx, healthCancel = context.WithCancel(context.Background()) + // 从连接串中解析数据库名 - dbName := gstr.SubStr(mongoAddr, strings.LastIndex(mongoAddr, "/")+1, len(mongoAddr)) + dbName = gstr.SubStr(mongoAddr, strings.LastIndex(mongoAddr, "/")+1, len(mongoAddr)) // 如果连接串带有参数(如 ?retryWrites=true),需要去掉参数部分 if strings.Contains(dbName, "?") { dbName = gstr.SubStr(dbName, 0, strings.Index(dbName, "?")) } - db = client.Database(dbName) - glog.Info(ctx, "✅ MongoDB 初始化成功") + + // 初始连接 + if err := connect(); err != nil { + glog.Error(context.Background(), "MongoDB初始连接失败", err) + return + } + + // 启动健康检查协程 + go healthCheck() + + glog.Info(context.Background(), "✅ MongoDB初始化完成,连接健康检查已启动") } + func listOptionsToMap(ctx context.Context, opts ...options.Lister[options.FindOptions]) (m map[string]interface{}) { // 输出opts参数中的值 m = make(map[string]interface{}) From 2b1ed8e8a0fa6fc131124b0d92745fb2b2a4a974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 19 Dec 2025 14:58:00 +0800 Subject: [PATCH 079/225] =?UTF-8?q?mongodb=E5=9F=BA=E7=A1=80=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E6=96=B9=E6=B3=95=E5=A2=9E=E5=8A=A0redis=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 156 +++++-------------------------------------------- 1 file changed, 13 insertions(+), 143 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 8bb52c8..394cec0 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "strings" - "sync" "time" "gitee.com/red-future---jilin-g/common/consts" @@ -23,166 +22,37 @@ import ( "go.mongodb.org/mongo-driver/v2/mongo/options" ) -var ( - db *mongo.Database - client *mongo.Client - isConnected bool - mu sync.RWMutex - mongoAddr string - dbName string - healthCtx context.Context - healthCancel context.CancelFunc -) +var db *mongo.Database // GetDB 获取 MongoDB 数据库实例 func GetDB() *mongo.Database { - mu.RLock() - defer mu.RUnlock() return db } -// connect 建立MongoDB连接 -func connect() error { - mu.Lock() - defer mu.Unlock() - - if client != nil { - err := client.Disconnect(context.Background()) - if err != nil { - return err - } - } - - // 创建连接选项 - opt := options.Client(). - ApplyURI(mongoAddr). - SetMaxPoolSize(100). - SetMinPoolSize(10). - SetMaxConnecting(10). - SetConnectTimeout(10 * time.Second) - - var err error - client, err = mongo.Connect(opt) - if err != nil { - isConnected = false - glog.Error(context.Background(), "MongoDB连接失败", err) - return err - } - - // 测试连接 - testCtx, testCancel := context.WithTimeout(context.Background(), 5*time.Second) - defer testCancel() - - err = client.Ping(testCtx, nil) - if err != nil { - isConnected = false - glog.Error(testCtx, "MongoDB连接测试失败", err) - return err - } - - db = client.Database(dbName) - isConnected = true - glog.Info(context.Background(), "✅ MongoDB连接成功") - return nil -} - -// healthCheck 健康检查协程 -func healthCheck() { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - - for { - select { - case <-healthCtx.Done(): - return - case <-ticker.C: - mu.RLock() - currentConnected := isConnected - currentClient := client - mu.RUnlock() - - if !currentConnected || currentClient == nil { - glog.Warning(context.Background(), "MongoDB连接断开,尝试重连") - if err := reconnect(); err != nil { - glog.Error(context.Background(), "MongoDB重连失败", err) - } - continue - } - - // 测试连接状态 - testCtx, testCancel := context.WithTimeout(context.Background(), 5*time.Second) - err := currentClient.Ping(testCtx, nil) - testCancel() - - if err != nil { - mu.Lock() - isConnected = false - mu.Unlock() - glog.Warning(context.Background(), "MongoDB连接健康检查失败", err) - - // 尝试重连 - if err := reconnect(); err != nil { - glog.Error(context.Background(), "MongoDB重连失败", err) - } - } else { - glog.Debug(context.Background(), "MongoDB连接健康检查通过") - } - } - } -} - -// reconnect 重连函数 -func reconnect() error { - maxRetries := 3 - retryDelay := 2 * time.Second - - for i := 0; i < maxRetries; i++ { - glog.Info(context.Background(), fmt.Sprintf("尝试第%d次重连MongoDB", i+1)) - - if err := connect(); err == nil { - glog.Info(context.Background(), "MongoDB重连成功") - return nil - } - - if i < maxRetries-1 { - time.Sleep(retryDelay) - retryDelay *= 2 // 指数退避 - } - } - - return gerror.New("MongoDB重连失败,已达到最大重试次数") -} - -// init 初始化MongoDB连接 func init() { // 按需初始化:没有配置 mongo.address 则跳过 - mongoAddr = g.Cfg().MustGet(context.Background(), "mongo.address").String() + mongoAddr := g.Cfg().MustGet(context.Background(), "mongo.address").String() if mongoAddr == "" { return } - // 创建健康检查上下文 - healthCtx, healthCancel = context.WithCancel(context.Background()) - + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + opt := options.Client().ApplyURI(mongoAddr) + client, err := mongo.Connect(opt) + if err != nil { + glog.Error(ctx, "mongodb连接失败", err) + return + } // 从连接串中解析数据库名 - dbName = gstr.SubStr(mongoAddr, strings.LastIndex(mongoAddr, "/")+1, len(mongoAddr)) + dbName := gstr.SubStr(mongoAddr, strings.LastIndex(mongoAddr, "/")+1, len(mongoAddr)) // 如果连接串带有参数(如 ?retryWrites=true),需要去掉参数部分 if strings.Contains(dbName, "?") { dbName = gstr.SubStr(dbName, 0, strings.Index(dbName, "?")) } - - // 初始连接 - if err := connect(); err != nil { - glog.Error(context.Background(), "MongoDB初始连接失败", err) - return - } - - // 启动健康检查协程 - go healthCheck() - - glog.Info(context.Background(), "✅ MongoDB初始化完成,连接健康检查已启动") + db = client.Database(dbName) + glog.Info(ctx, "✅ MongoDB 初始化成功") } - func listOptionsToMap(ctx context.Context, opts ...options.Lister[options.FindOptions]) (m map[string]interface{}) { // 输出opts参数中的值 m = make(map[string]interface{}) From 5d22d07a3b4958d4982e06c417820256a2ffe6f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 19 Dec 2025 15:17:19 +0800 Subject: [PATCH 080/225] =?UTF-8?q?gomod=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 177 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 164 insertions(+), 13 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 394cec0..35b8f3e 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "sync" "time" "gitee.com/red-future---jilin-g/common/consts" @@ -22,36 +23,186 @@ import ( "go.mongodb.org/mongo-driver/v2/mongo/options" ) -var db *mongo.Database +var ( + db *mongo.Database + client *mongo.Client + isConnected bool + mu sync.RWMutex + mongoAddr string + dbName string + healthCtx context.Context + healthCancel context.CancelFunc +) // GetDB 获取 MongoDB 数据库实例 func GetDB() *mongo.Database { + mu.RLock() + defer mu.RUnlock() return db } +// IsConnected 检查连接状态 +func IsConnected() bool { + mu.RLock() + defer mu.RUnlock() + return isConnected +} + +// connect 建立MongoDB连接 +func connect() error { + mu.Lock() + defer mu.Unlock() + + if client != nil { + client.Disconnect(context.Background()) + } + + // 创建连接选项 + opt := options.Client(). + ApplyURI(mongoAddr). + SetMaxPoolSize(100). + SetMinPoolSize(10). + SetMaxConnecting(10). + SetConnectTimeout(10 * time.Second) + + var err error + client, err = mongo.Connect(opt) + if err != nil { + isConnected = false + glog.Error(context.Background(), "MongoDB连接失败", err) + return err + } + + // 测试连接 + testCtx, testCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer testCancel() + + err = client.Ping(testCtx, nil) + if err != nil { + isConnected = false + glog.Error(testCtx, "MongoDB连接测试失败", err) + return err + } + + db = client.Database(dbName) + isConnected = true + glog.Info(context.Background(), "✅ MongoDB连接成功") + return nil +} + +// healthCheck 健康检查协程 +func healthCheck() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-healthCtx.Done(): + return + case <-ticker.C: + mu.RLock() + currentConnected := isConnected + currentClient := client + mu.RUnlock() + + if !currentConnected || currentClient == nil { + glog.Warning(context.Background(), "MongoDB连接断开,尝试重连") + if err := reconnect(); err != nil { + glog.Error(context.Background(), "MongoDB重连失败", err) + } + continue + } + + // 测试连接状态 + testCtx, testCancel := context.WithTimeout(context.Background(), 5*time.Second) + err := currentClient.Ping(testCtx, nil) + testCancel() + + if err != nil { + mu.Lock() + isConnected = false + mu.Unlock() + glog.Warning(context.Background(), "MongoDB连接健康检查失败", err) + + // 尝试重连 + if err := reconnect(); err != nil { + glog.Error(context.Background(), "MongoDB重连失败", err) + } + } else { + glog.Debug(context.Background(), "MongoDB连接健康检查通过") + } + } + } +} + +// reconnect 重连函数 +func reconnect() error { + maxRetries := 3 + retryDelay := 2 * time.Second + + for i := 0; i < maxRetries; i++ { + glog.Info(context.Background(), fmt.Sprintf("尝试第%d次重连MongoDB", i+1)) + + if err := connect(); err == nil { + glog.Info(context.Background(), "MongoDB重连成功") + return nil + } + + if i < maxRetries-1 { + time.Sleep(retryDelay) + retryDelay *= 2 // 指数退避 + } + } + + return gerror.New("MongoDB重连失败,已达到最大重试次数") +} + +// init 初始化MongoDB连接 func init() { // 按需初始化:没有配置 mongo.address 则跳过 - mongoAddr := g.Cfg().MustGet(context.Background(), "mongo.address").String() + mongoAddr = g.Cfg().MustGet(context.Background(), "mongo.address").String() if mongoAddr == "" { return } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - opt := options.Client().ApplyURI(mongoAddr) - client, err := mongo.Connect(opt) - if err != nil { - glog.Error(ctx, "mongodb连接失败", err) - return - } + // 创建健康检查上下文 + healthCtx, healthCancel = context.WithCancel(context.Background()) + // 从连接串中解析数据库名 - dbName := gstr.SubStr(mongoAddr, strings.LastIndex(mongoAddr, "/")+1, len(mongoAddr)) + dbName = gstr.SubStr(mongoAddr, strings.LastIndex(mongoAddr, "/")+1, len(mongoAddr)) // 如果连接串带有参数(如 ?retryWrites=true),需要去掉参数部分 if strings.Contains(dbName, "?") { dbName = gstr.SubStr(dbName, 0, strings.Index(dbName, "?")) } - db = client.Database(dbName) - glog.Info(ctx, "✅ MongoDB 初始化成功") + go func() { + // 初始连接 + if err := connect(); err != nil { + glog.Error(context.Background(), "MongoDB初始连接失败", err) + return + } + }() + + // 启动健康检查协程 + go healthCheck() +} + +// Close 关闭MongoDB连接 +func Close() { + if healthCancel != nil { + healthCancel() + } + + mu.Lock() + defer mu.Unlock() + + if client != nil { + disconnectCtx, disconnectCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer disconnectCancel() + client.Disconnect(disconnectCtx) + } + + isConnected = false + glog.Info(context.Background(), "MongoDB连接已关闭") } func listOptionsToMap(ctx context.Context, opts ...options.Lister[options.FindOptions]) (m map[string]interface{}) { // 输出opts参数中的值 From 2a31e3faacf86ff7436d37b0a151d62dee9cf96e Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Fri, 19 Dec 2025 17:57:46 +0800 Subject: [PATCH 081/225] =?UTF-8?q?=E5=B0=86go=20Httpserver.Run()=20?= =?UTF-8?q?=E6=8C=AA=E5=88=B0=20routeregister=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 2 +- redis/redis.go | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/http/http.go b/http/http.go index 09f2641..67bdc51 100644 --- a/http/http.go +++ b/http/http.go @@ -44,7 +44,6 @@ func init() { Httpserver.SetOpenApiPath("/api.json") Httpserver.SetDumpRouterMap(true) //关闭打印路由注册信息 Httpserver.BindMiddlewareDefault(ghttp.MiddlewareHandlerResponse) - go Httpserver.Run() Httpclient.SetDiscovery(gsvc.GetRegistry()) } func RouteRegister(controllers []interface{}) { @@ -59,6 +58,7 @@ func RouteRegister(controllers []interface{}) { group.Bind(t) }) } + go Httpserver.Run() } func doRequest(ctx context.Context, method string, url string, target any, data ...any) (err error) { err = utils.ValidStructPtr(target) diff --git a/redis/redis.go b/redis/redis.go index 072fb1b..8d393b2 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -567,9 +567,9 @@ const ( ConversationCacheExpireSeconds = 600 ) -// CacheConversation 缓存单条对话到Redis List -func CacheConversation(ctx context.Context, userId, platform string, data []byte) error { - key := ConversationCacheKeyPrefix + userId + "_" + platform +// CacheConversation 缓存单条对话到Redis List(按sessionId存储) +func CacheConversation(ctx context.Context, sessionId string, data []byte) error { + key := ConversationCacheKeyPrefix + sessionId _, err := redisClient.Do(ctx, "RPUSH", key, string(data)) if err != nil { return err @@ -578,9 +578,9 @@ func CacheConversation(ctx context.Context, userId, platform string, data []byte return err } -// GetCachedConversations 获取缓存的对话列表并清空 -func GetCachedConversations(ctx context.Context, userId, platform string) (list []string, err error) { - key := ConversationCacheKeyPrefix + userId + "_" + platform +// GetCachedConversations 获取缓存的对话列表并清空(按sessionId查询) +func GetCachedConversations(ctx context.Context, sessionId string) (list []string, err error) { + key := ConversationCacheKeyPrefix + sessionId result, err := redisClient.Do(ctx, "LRANGE", key, 0, -1) if err != nil { return @@ -594,9 +594,9 @@ func GetCachedConversations(ctx context.Context, userId, platform string) (list return } -// GetCachedConversationCount 获取缓存的对话数量 -func GetCachedConversationCount(ctx context.Context, userId, platform string) (count int64, err error) { - key := ConversationCacheKeyPrefix + userId + "_" + platform +// GetCachedConversationCount 获取缓存的对话数量(按sessionId查询) +func GetCachedConversationCount(ctx context.Context, sessionId string) (count int64, err error) { + key := ConversationCacheKeyPrefix + sessionId result, err := redisClient.Do(ctx, "LLEN", key) if err != nil { return @@ -604,9 +604,9 @@ func GetCachedConversationCount(ctx context.Context, userId, platform string) (c return result.Int64(), nil } -// ClearCachedConversations 清空对话缓存(归档时调用) -func ClearCachedConversations(ctx context.Context, userId, platform string) error { - key := ConversationCacheKeyPrefix + userId + "_" + platform +// ClearCachedConversations 清空对话缓存(归档时调用,按sessionId) +func ClearCachedConversations(ctx context.Context, sessionId string) error { + key := ConversationCacheKeyPrefix + sessionId _, err := redisClient.Del(ctx, key) return err } From 4b9491cf2033074fb7f895a988153a1cae63b23a Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sat, 20 Dec 2025 11:27:10 +0800 Subject: [PATCH 082/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0es=E6=8A=A5=E9=94=99?= =?UTF-8?q?=E9=99=8D=E7=BA=A7=E4=B8=BA=E8=AD=A6=E5=91=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- elasticsearch/client.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/elasticsearch/client.go b/elasticsearch/client.go index ead9d1b..4159f31 100644 --- a/elasticsearch/client.go +++ b/elasticsearch/client.go @@ -45,9 +45,11 @@ func Init(ctx context.Context) (err error) { } // 测试连接 - info, code, err := client.Ping(addresses[0]).Do(ctx) - if err != nil { - glog.Errorf(ctx, "ES 连接测试失败: %v", err) + info, code, testErr := client.Ping(addresses[0]).Do(ctx) + if testErr != nil { + glog.Warningf(ctx, "ES 连接测试失败(月度归档功能不可用): %v", testErr) + client = nil // 清空客户端 + err = testErr return } glog.Infof(ctx, "ES 连接成功 - 版本: %s, 状态码: %d", info.Version.Number, code) From 134df0699c4f787a424dbe55cfa62e20525c4eb3 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sat, 20 Dec 2025 13:26:44 +0800 Subject: [PATCH 083/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E8=BF=BD=E9=97=AE=E6=B6=88=E6=81=AF=E5=86=85=E7=A7=9F=E6=88=B7?= =?UTF-8?q?id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/types.go | 1 + 1 file changed, 1 insertion(+) diff --git a/redis/types.go b/redis/types.go index 17c571f..a6a3e36 100644 --- a/redis/types.go +++ b/redis/types.go @@ -44,6 +44,7 @@ type ResponseStreamMessage struct { // FollowUpMessage 追问消息结构(RabbitMQ 延时队列) type FollowUpMessage struct { + TenantId string `json:"tenant_id"` // 租户ID UserId string `json:"user_id"` // 用户ID Platform string `json:"platform"` // 平台标识 Content string `json:"content"` // 追问内容 From 7c103a1d1d258efcc2f9cd2fa0fdcd964ce879ff Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sat, 20 Dec 2025 14:50:41 +0800 Subject: [PATCH 084/225] =?UTF-8?q?redis=20=E7=BC=93=E5=AD=98=E6=96=B9?= =?UTF-8?q?=E5=BC=8F=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/redis/redis.go b/redis/redis.go index 8d393b2..8209d79 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -382,21 +382,13 @@ func IsUserActive(ctx context.Context, userId string, seconds int64) (bool, erro return false, nil // 未找到记录,视为不活跃 } + // 检查时间差 now := gtime.Now().Timestamp() return (now - lastActive) < seconds, nil } -// SetSessionCache 缓存用户的 RAGFlow Session ID -// 使用 gredis SetEX 方法 -func SetSessionCache(ctx context.Context, userId, sessionId string) error { - key := SessionLastActiveKeyPrefix + userId + ":session_id" +// ============== 限流相关 ============== - // SETEX key 604800 value (604800秒 = 7天) - _, err := redisClient.Do(ctx, "SETEX", key, 604800, sessionId) - return err -} - -// 限流相关常量 const ( // RateLimitKeyPrefix 限流计数器 Key 前缀 RateLimitKeyPrefix = "ragflow:ratelimit:" @@ -433,9 +425,17 @@ func GetRateLimit(ctx context.Context, key string) (count int64, err error) { return } -// GetSessionCache 获取缓存的 RAGFlow Session ID -func GetSessionCache(ctx context.Context, userId string) (string, error) { - key := SessionLastActiveKeyPrefix + userId + ":session_id" +// SetSessionCache 缓存 RAGFlow Session ID(租户+用户+方向隔离) +func SetSessionCache(ctx context.Context, tenantId, userId, chatId, sessionId string) error { + key := SessionLastActiveKeyPrefix + tenantId + ":" + userId + ":" + chatId + ":session_id" + // SETEX key 7200 value (7200秒 = 2小时,与last_active保持一致) + _, err := redisClient.Do(ctx, "SETEX", key, 7200, sessionId) + return err +} + +// GetSessionCache 获取缓存的 RAGFlow Session ID(租户+用户+方向隔离) +func GetSessionCache(ctx context.Context, tenantId, userId, chatId string) (string, error) { + key := SessionLastActiveKeyPrefix + tenantId + ":" + userId + ":" + chatId + ":session_id" result, err := redisClient.Get(ctx, key) if err != nil { return "", err @@ -448,9 +448,9 @@ func GetSessionCache(ctx context.Context, userId string) (string, error) { return result.String(), nil } -// DelSessionCache 删除缓存的 RAGFlow Session ID(归档时调用) -func DelSessionCache(ctx context.Context, userId string) error { - key := SessionLastActiveKeyPrefix + userId + ":session_id" +// DelSessionCache 删除缓存的 RAGFlow Session ID(归档时调用,租户+用户+方向隔离) +func DelSessionCache(ctx context.Context, tenantId, userId, chatId string) error { + key := SessionLastActiveKeyPrefix + tenantId + ":" + userId + ":" + chatId + ":session_id" _, err := redisClient.Del(ctx, key) return err } From 3683aae9afcd073eafad4491e54c2eaf9060e997 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sat, 20 Dec 2025 15:04:13 +0800 Subject: [PATCH 085/225] =?UTF-8?q?=20session=E7=BC=93=E5=AD=98=E6=96=B9?= =?UTF-8?q?=E6=B3=95=EF=BC=88tenantId+userId=E9=9A=94=E7=A6=BB=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/redis/redis.go b/redis/redis.go index 8209d79..1ebfe1c 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -425,17 +425,17 @@ func GetRateLimit(ctx context.Context, key string) (count int64, err error) { return } -// SetSessionCache 缓存 RAGFlow Session ID(租户+用户+方向隔离) -func SetSessionCache(ctx context.Context, tenantId, userId, chatId, sessionId string) error { - key := SessionLastActiveKeyPrefix + tenantId + ":" + userId + ":" + chatId + ":session_id" +// SetSessionCache 缓存 RAGFlow Session ID(租户+用户隔离) +func SetSessionCache(ctx context.Context, tenantId, userId, sessionId string) error { + key := SessionLastActiveKeyPrefix + tenantId + ":" + userId + ":session_id" // SETEX key 7200 value (7200秒 = 2小时,与last_active保持一致) _, err := redisClient.Do(ctx, "SETEX", key, 7200, sessionId) return err } -// GetSessionCache 获取缓存的 RAGFlow Session ID(租户+用户+方向隔离) -func GetSessionCache(ctx context.Context, tenantId, userId, chatId string) (string, error) { - key := SessionLastActiveKeyPrefix + tenantId + ":" + userId + ":" + chatId + ":session_id" +// GetSessionCache 获取缓存的 RAGFlow Session ID(租户+用户隔离) +func GetSessionCache(ctx context.Context, tenantId, userId string) (string, error) { + key := SessionLastActiveKeyPrefix + tenantId + ":" + userId + ":session_id" result, err := redisClient.Get(ctx, key) if err != nil { return "", err @@ -448,9 +448,9 @@ func GetSessionCache(ctx context.Context, tenantId, userId, chatId string) (stri return result.String(), nil } -// DelSessionCache 删除缓存的 RAGFlow Session ID(归档时调用,租户+用户+方向隔离) -func DelSessionCache(ctx context.Context, tenantId, userId, chatId string) error { - key := SessionLastActiveKeyPrefix + tenantId + ":" + userId + ":" + chatId + ":session_id" +// DelSessionCache 删除缓存的 RAGFlow Session ID(归档时调用,租户+用户隔离) +func DelSessionCache(ctx context.Context, tenantId, userId string) error { + key := SessionLastActiveKeyPrefix + tenantId + ":" + userId + ":session_id" _, err := redisClient.Del(ctx, key) return err } From 08f8b2248a163fa2658b37c1d1504f31bf1201e5 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Mon, 22 Dec 2025 17:50:53 +0800 Subject: [PATCH 086/225] =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=E9=9A=94?= =?UTF-8?q?=E7=A6=BB=E4=B8=8E=E8=AF=9D=E6=9C=AF=E7=AE=A1=E7=90=86=E6=96=B9?= =?UTF-8?q?=E6=A1=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rabbitmq/delay.go | 95 +++++++++++++++++++++++++++++++++++++++++++++ ragflow/chat.go | 49 ++++++++++++++--------- ragflow/document.go | 65 ++++++++++++++++++++++++++++--- 3 files changed, 186 insertions(+), 23 deletions(-) create mode 100644 rabbitmq/delay.go diff --git a/rabbitmq/delay.go b/rabbitmq/delay.go new file mode 100644 index 0000000..82faa66 --- /dev/null +++ b/rabbitmq/delay.go @@ -0,0 +1,95 @@ +// Package rabbitmq - RabbitMQ延时消息发布 +package rabbitmq + +import ( + "context" + "time" + + "github.com/gogf/gf/v2/encoding/gjson" + "github.com/gogf/gf/v2/errors/gerror" + amqp "github.com/rabbitmq/amqp091-go" +) + +// PublishWithDelay 发布延时消息到RabbitMQ +// delaySeconds: 延时秒数 +func PublishWithDelay(ctx context.Context, routingKey string, message interface{}, delaySeconds int) error { + ch, err := GetChannel() + if err != nil { + return gerror.Wrap(err, "获取RabbitMQ通道失败") + } + if ch == nil { + return gerror.New("RabbitMQ通道未初始化") + } + + // 序列化消息 + body, err := gjson.Encode(message) + if err != nil { + return gerror.Wrapf(err, "序列化消息失败") + } + + // 声明延时交换机(x-delayed-message类型) + // 注意:需要RabbitMQ安装延时插件 rabbitmq-plugins enable rabbitmq_delayed_message_exchange + exchangeName := "delayed.exchange" + err = ch.ExchangeDeclare( + exchangeName, + "x-delayed-message", // 延时交换机类型 + true, // durable + false, // auto-deleted + false, // internal + false, // no-wait + amqp.Table{ + "x-delayed-type": "direct", // 底层交换机类型 + }, + ) + if err != nil { + return gerror.Wrapf(err, "声明延时交换机失败") + } + + // 声明队列 + queue, err := ch.QueueDeclare( + routingKey, // 队列名使用routingKey + true, // durable + false, // delete when unused + false, // exclusive + false, // no-wait + nil, + ) + if err != nil { + return gerror.Wrapf(err, "声明队列失败") + } + + // 绑定队列到交换机 + err = ch.QueueBind( + queue.Name, // queue name + routingKey, // routing key + exchangeName, // exchange + false, + nil, + ) + if err != nil { + return gerror.Wrapf(err, "绑定队列失败") + } + + // 发布延时消息 + err = ch.PublishWithContext( + ctx, + exchangeName, // exchange + routingKey, // routing key + false, // mandatory + false, // immediate + amqp.Publishing{ + ContentType: "application/json", + Body: body, + DeliveryMode: amqp.Persistent, // 持久化消息 + Headers: amqp.Table{ + "x-delay": delaySeconds * 1000, // 延时时间(毫秒) + }, + Timestamp: time.Now(), + }, + ) + if err != nil { + return gerror.Wrapf(err, "发布延时消息失败") + } + + return nil +} diff --git a/ragflow/chat.go b/ragflow/chat.go index 7f0acb5..1bf69c1 100644 --- a/ragflow/chat.go +++ b/ragflow/chat.go @@ -6,6 +6,37 @@ import ( "github.com/gogf/gf/v2/errors/gerror" ) +// CreateChatReq 创建对话配置请求 +type CreateChatReq struct { + Name string `json:"name"` // 对话配置名称 + DatasetIds []string `json:"dataset_ids"` // 关联的知识库ID列表 + Prompt *PromptConfig `json:"prompt"` // 提示词配置 +} + +// PromptConfig 提示词配置 +type PromptConfig struct { + Prompt string `json:"prompt"` // 提示词内容 + SimilarityThreshold float64 `json:"similarity_threshold"` // 相似度阈值 + KeywordsSimilarityWeight float64 `json:"keywords_similarity_weight"` // 关键词相似度权重 + TopN int `json:"top_n"` // 返回顶部N个chunk + EmptyResponse string `json:"empty_response"` // 无匹配时回复 + Opener string `json:"opener"` // 开场白 + ShowQuote bool `json:"show_quote"` // 是否显示引用 + Variables []map[string]interface{} `json:"variables"` // 变量列表 +} + +// CreateChatRes 创建对话配置响应 +type CreateChatRes struct { + ChatId string `json:"id"` // 对话配置ID +} + +// UpdateChatReq 更新对话配置请求 +type UpdateChatReq struct { + Name string `json:"name,omitempty"` // 对话配置名称 + DatasetIds []string `json:"dataset_ids,omitempty"` // 关联的知识库ID列表 + Prompt *PromptConfig `json:"prompt,omitempty"` // 提示词配置 +} + // 聊天助手管理 // 参考: https://ragflow.com.cn/docs/dev/http_api_reference#聊天助手管理 @@ -54,24 +85,6 @@ type Variable struct { Optional bool `json:"optional"` } -// CreateChatReq 创建聊天助手请求 -type CreateChatReq struct { - Name string `json:"name"` - Avatar string `json:"avatar,omitempty"` - DatasetIds []string `json:"dataset_ids,omitempty"` - Llm *Llm `json:"llm,omitempty"` - Prompt *Prompt `json:"prompt,omitempty"` -} - -// UpdateChatReq 更新聊天助手请求 -type UpdateChatReq struct { - Name string `json:"name,omitempty"` - Avatar string `json:"avatar,omitempty"` - DatasetIds []string `json:"dataset_ids,omitempty"` - Llm *Llm `json:"llm,omitempty"` - Prompt *Prompt `json:"prompt,omitempty"` -} - // ListChatsReq 列出聊天助手请求 type ListChatsReq struct { Page int `json:"page,omitempty"` diff --git a/ragflow/document.go b/ragflow/document.go index c08f32b..2fee990 100644 --- a/ragflow/document.go +++ b/ragflow/document.go @@ -1,10 +1,15 @@ +// Package ragflow - RAGFlow文档管理 +// 功能:RAGFlow知识库文档的上传、列表、删除操作 package ragflow import ( + "bytes" "context" "strings" + "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" ) // 数据集内文件管理 @@ -138,12 +143,62 @@ func (c *Client) ListDocuments(ctx context.Context, datasetId string, req *ListD return &res, nil } -// UploadDocument 上传文档 -// 注意:此方法需要特殊处理 multipart/form-data,目前的 request 方法可能不支持 -// 我们需要扩展 request 方法或在此处单独实现 +// UploadDocumentFromText 上传文本内容作为文档 +func (c *Client) UploadDocumentFromText(ctx context.Context, datasetId, content, filename string) (documentId string, err error) { + if datasetId == "" { + return "", gerror.New("datasetId不能为空") + } + if content == "" { + return "", gerror.New("文档内容不能为空") + } + if filename == "" { + filename = "document.txt" + } + + // 构造URL + url := c.BaseURL + "/api/v1/datasets/" + datasetId + "/documents" + + // 使用gclient上传文本作为文件 + client := c.HTTPClient.Clone() + client.SetHeader("Authorization", "Bearer "+c.APIKey) + + // 使用ContentType方法上传multipart表单 + resp, err := client.Post(ctx, url, g.Map{ + "file": bytes.NewReader([]byte(content)), + }) + if err != nil { + return "", err + } + defer resp.Close() + + // 解析响应 + var result struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Id string `json:"id"` + } `json:"data"` + } + + bodyBytes := resp.ReadAll() + if err = gjson.DecodeTo(bodyBytes, &result); err != nil { + return "", err + } + + if result.Code != 0 { + return "", gerror.Newf("上传文档失败 (code=%d): %s", result.Code, result.Message) + } + + if result.Data.Id == "" { + return "", gerror.New("上传成功但未返回文档ID") + } + + return result.Data.Id, nil +} + +// UploadDocument 上传文档(保留兼容) func (c *Client) UploadDocument(ctx context.Context, datasetId string, filePaths []string) (err error) { - // TODO: 实现文件上传逻辑,需要使用 gclient 的 UploadFile 功能 - return gerror.New("upload document not implemented yet") + return gerror.New("upload document from file not implemented yet, use UploadDocumentFromText instead") } // DeleteDocument 删除文档 From f3193327512314fd271f6ec3bf45d62ac340db6b Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Mon, 22 Dec 2025 15:11:07 +0800 Subject: [PATCH 087/225] =?UTF-8?q?minio=E4=BF=AE=E6=94=B9=E5=9B=BE?= =?UTF-8?q?=E7=89=87=E4=B8=8A=E4=BC=A0-=E6=94=AF=E6=8C=81=E5=A4=9A?= =?UTF-8?q?=E5=9B=BE=E4=B8=8A=E4=BC=A0=20mongo=E5=A2=9E=E5=8A=A0=E7=B1=BB?= =?UTF-8?q?=E5=9E=8B=E8=BD=AC=E6=8D=A2-=E5=B0=86entity=20=E8=BD=AC?= =?UTF-8?q?=E6=8D=A2=E4=B8=BA=20bson.M?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- minio/minio.go | 26 +++++++++++++++++++++++--- mongo/mongo.go | 22 ++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/minio/minio.go b/minio/minio.go index 5b3b469..91b21ed 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -65,13 +65,33 @@ func initMinIO(ctx context.Context) error { return nil } -func UploadImage(fileHeader *ghttp.UploadFile) (presignedUrl string, err error) { - ctx := context.Background() +func UploadImages(ctx context.Context, fileHeaders []*ghttp.UploadFile) (imagesUrls []string, err error) { + imageUrls := make([]string, 0, len(fileHeaders)) // 存储上传后的图片访问地址 + for _, fileHeader := range fileHeaders { + var imageUrl string + imageUrl, err = uploadImg(ctx, fileHeader) + if err != nil { + return nil, err + } + imageUrls = append(imageUrls, imageUrl) + } + return imageUrls, err +} + +func UploadImage(ctx context.Context, fileHeader *ghttp.UploadFile) (imagesUrl string, err error) { // 初始化 MinIO 客户端 err = initMinIO(ctx) if err != nil { return "", fmt.Errorf("初始化 MinIO 客户端失败: %w", err) } + imageUrl, err := uploadImg(ctx, fileHeader) + if err != nil { + return "", err + } + return imageUrl, err +} + +func uploadImg(ctx context.Context, fileHeader *ghttp.UploadFile) (imagesUrl string, err error) { // 生成唯一的 MinIO 对象名(避免覆盖) fileExt := filepath.Ext(fileHeader.Filename) // 原文件后缀(如 .jpg) uniqueID := uuid.New().String()[:8] // 8位随机UUID @@ -91,7 +111,7 @@ func UploadImage(fileHeader *ghttp.UploadFile) (presignedUrl string, err error) } // 执行图片上传 _, err = minioClient.PutObject( - context.Background(), + ctx, minioCfg.BucketName, objectName, file, diff --git a/mongo/mongo.go b/mongo/mongo.go index 35b8f3e..baabbe5 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -496,3 +496,25 @@ func Count(ctx context.Context, filter bson.M, collection string) (count int64, } return } + +// EntityToBSONM 将 *entity/entity 转换为 bson.M +// 支持传入值类型或指针类型,返回 bson.M 和错误信息 +func EntityToBSONM(entity interface{}) (bson.M, error) { + // 第一步:判断入参是否为 nil 或无效类型 + if entity == nil { + return nil, fmt.Errorf("传入的 entity 实例为 nil") + } + // 第二步:将 entity 序列化为 BSON 字节流 + // bson.Marshal 支持值类型和指针类型,会自动解析结构体的 bson 标签 + bsonBytes, err := bson.Marshal(entity) + if err != nil { + return nil, fmt.Errorf("entity 序列化为 BSON 字节流失败:%w", err) + } + // 第三步:将 BSON 字节流反序列化为 bson.M + var bsonMap bson.M + err = bson.Unmarshal(bsonBytes, &bsonMap) + if err != nil { + return nil, fmt.Errorf("BSON 字节流反序列化为 bson.M 失败:%w", err) + } + return bsonMap, nil +} From b6409898a66847b4d9b4761f7791323d1893b887 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Mon, 22 Dec 2025 20:05:01 +0800 Subject: [PATCH 088/225] =?UTF-8?q?=E7=9F=A5=E8=AF=86=E5=BA=93=E8=B0=83?= =?UTF-8?q?=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/document.go | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/ragflow/document.go b/ragflow/document.go index 2fee990..823f26e 100644 --- a/ragflow/document.go +++ b/ragflow/document.go @@ -5,11 +5,11 @@ package ragflow import ( "bytes" "context" + "mime/multipart" "strings" "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/errors/gerror" - "github.com/gogf/gf/v2/frame/g" ) // 数据集内文件管理 @@ -158,16 +158,34 @@ func (c *Client) UploadDocumentFromText(ctx context.Context, datasetId, content, // 构造URL url := c.BaseURL + "/api/v1/datasets/" + datasetId + "/documents" - // 使用gclient上传文本作为文件 + // 创建multipart writer + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + // 添加文件字段 + part, err := writer.CreateFormFile("file", filename) + if err != nil { + return "", gerror.Wrap(err, "创建form file失败") + } + + // 写入内容 + if _, err = part.Write([]byte(content)); err != nil { + return "", gerror.Wrap(err, "写入文件内容失败") + } + + // 关闭multipart writer + if err = writer.Close(); err != nil { + return "", gerror.Wrap(err, "关闭multipart writer失败") + } + + // 发送请求 client := c.HTTPClient.Clone() client.SetHeader("Authorization", "Bearer "+c.APIKey) + client.SetHeader("Content-Type", writer.FormDataContentType()) - // 使用ContentType方法上传multipart表单 - resp, err := client.Post(ctx, url, g.Map{ - "file": bytes.NewReader([]byte(content)), - }) + resp, err := client.Post(ctx, url, body.Bytes()) if err != nil { - return "", err + return "", gerror.Wrap(err, "上传文档请求失败") } defer resp.Close() From 61223b0546a924917693ce5c7ac7225d9fa4a438 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 23 Dec 2025 17:03:58 +0800 Subject: [PATCH 089/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/document.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/ragflow/document.go b/ragflow/document.go index 823f26e..5913572 100644 --- a/ragflow/document.go +++ b/ragflow/document.go @@ -219,6 +219,32 @@ func (c *Client) UploadDocument(ctx context.Context, datasetId string, filePaths return gerror.New("upload document from file not implemented yet, use UploadDocumentFromText instead") } +// ParseDocumentsReq 解析文档请求 +type ParseDocumentsReq struct { + DocumentIds []string `json:"document_ids"` // 要解析的文档ID列表 +} + +// ParseDocuments 解析文档(上传后必须调用此接口才会开始解析) +func (c *Client) ParseDocuments(ctx context.Context, datasetId string, documentIds []string) error { + if datasetId == "" { + return gerror.New("datasetId不能为空") + } + if len(documentIds) == 0 { + return gerror.New("documentIds不能为空") + } + + req := ParseDocumentsReq{DocumentIds: documentIds} + var res CommonResponse + path := "/api/v1/datasets/" + datasetId + "/chunks" + if err := c.request(ctx, "POST", path, req, &res); err != nil { + return err + } + if !res.IsSuccess() { + return gerror.Newf("解析文档失败: %s", res.Message) + } + return nil +} + // DeleteDocument 删除文档 func (c *Client) DeleteDocument(ctx context.Context, datasetId string, ids []string) (err error) { req := DeleteDocumentsReq{Ids: ids} From 3dc3ebc9ad7a2d57a5d62771f3014b1ef9a6299b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 23 Dec 2025 16:33:57 +0800 Subject: [PATCH 090/225] =?UTF-8?q?common/http=E5=A2=9E=E5=8A=A0headers?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/http/http.go b/http/http.go index 67bdc51..2fcfdbc 100644 --- a/http/http.go +++ b/http/http.go @@ -15,7 +15,6 @@ import ( "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/net/gsvc" - "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/util/gconv" ) @@ -60,21 +59,18 @@ func RouteRegister(controllers []interface{}) { } go Httpserver.Run() } -func doRequest(ctx context.Context, method string, url string, target any, data ...any) (err error) { +func doRequest(ctx context.Context, method string, url string, headers map[string]string, target any, data ...any) (err error) { err = utils.ValidStructPtr(target) if err != nil { return } + Httpclient.SetHeaderMap(headers) Httpclient.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) response, err := Httpclient.DoRequest(ctx, method, url, data) + defer response.Close() if err != nil { return } - defer func() { - if err = response.Close(); err != nil { - glog.Error(ctx, err) - } - }() result := response.ReadAll() resultStrut := &ghttp.DefaultHandlerResponse{} if gconv.Struct(result, &resultStrut); resultStrut.Code != 200 { @@ -84,19 +80,19 @@ func doRequest(ctx context.Context, method string, url string, target any, data } return } -func Get(ctx context.Context, url string, target any, data ...any) (err error) { - err = doRequest(ctx, http.MethodGet, url, target, data) +func Get(ctx context.Context, url string, headers map[string]string, target any, data ...any) (err error) { + err = doRequest(ctx, http.MethodGet, url, headers, target, data) return } -func Post(ctx context.Context, url string, target any, data ...any) (err error) { - err = doRequest(ctx, http.MethodPost, url, target, data) +func Post(ctx context.Context, url string, headers map[string]string, target any, data ...any) (err error) { + err = doRequest(ctx, http.MethodPost, url, headers, target, data) return } -func Put(ctx context.Context, url string, target any, data ...any) (err error) { - err = doRequest(ctx, http.MethodPut, url, target, data) +func Put(ctx context.Context, url string, headers map[string]string, target any, data ...any) (err error) { + err = doRequest(ctx, http.MethodPut, url, headers, target, data) return } -func Delete(ctx context.Context, url string, target any, data ...any) (err error) { - err = doRequest(ctx, http.MethodPut, url, target, data) +func Delete(ctx context.Context, url string, headers map[string]string, target any, data ...any) (err error) { + err = doRequest(ctx, http.MethodPut, url, headers, target, data) return } From ecf3c297592de58461a8689a175c705af10296ba Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Wed, 24 Dec 2025 18:33:11 +0800 Subject: [PATCH 091/225] =?UTF-8?q?=E6=8F=90=E4=BA=A45=E4=B8=AA=E6=96=87?= =?UTF-8?q?=E4=BB=B6,=20=E4=BF=AE=E6=94=B9=E4=BA=86mongo=E6=96=B9=E6=B3=95?= =?UTF-8?q?,=20=E5=A6=82=E6=9E=9Ctoken=E8=8E=B7=E5=8F=96=E9=94=99=E8=AF=AF?= =?UTF-8?q?,=20=E7=84=B6=E5=90=8E=E8=8E=B7=E5=8F=96=E5=AE=8C=E7=A7=9F?= =?UTF-8?q?=E6=88=B7id=E5=90=8E,=20=E6=8A=8A=E9=94=99=E8=AF=AF=E6=B8=85?= =?UTF-8?q?=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 49 ++++++++++++++++++++++++++++-------- rabbitmq/consumer_manager.go | 10 +++++--- ragflow/document.go | 32 ++++++++++++----------- redis/redis.go | 18 ++++++++++--- redis/types.go | 20 ++++++++------- 5 files changed, 88 insertions(+), 41 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index baabbe5..d02c9ad 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -261,31 +261,58 @@ func GetTenantInfo(ctx context.Context) (user do.User, err error) { return } - // 2. token 获取失败,尝试从请求参数获取 customerServiceId + // 2. token 获取失败,尝试从请求参数或context获取 accountName + var accountName string + + // 2.1 尝试从request获取(HTTP请求场景) req := g.RequestFromCtx(ctx) - if req == nil { - return user, gerror.New("无法获取租户信息:无 token 且无 request") + if req != nil { + accountName = req.Get("accountName").String() + if accountName == "" { + accountName = req.Get("account_name").String() + } + // 兼容旧参数名 + if accountName == "" { + accountName = req.Get("customerServiceId").String() + } + if accountName == "" { + accountName = req.Get("customer_service_id").String() + } } - customerServiceId := req.Get("customerServiceId").String() - if customerServiceId == "" { - customerServiceId = req.Get("customer_service_id").String() + // 2.2 request不存在或未获取到,尝试从context.Value获取(WebSocket场景) + if accountName == "" { + if val := ctx.Value("accountName"); val != nil { + if str, ok := val.(string); ok { + accountName = str + } + } + // 兼容旧参数名 + if accountName == "" { + if val := ctx.Value("customerServiceId"); val != nil { + if str, ok := val.(string); ok { + accountName = str + } + } + } } - if customerServiceId == "" { - return user, gerror.New("无法获取租户信息:无 token 且无 customerServiceId 参数") + + if accountName == "" { + return user, gerror.New("无法获取租户信息:无 token 且无 accountName 参数") } // 3. 直接查询 customer_service_account 表获取 tenantId - filter := bson.M{"customerServiceId": customerServiceId, "isDeleted": false} + filter := bson.M{"accountName": accountName, "isDeleted": false} var account struct { TenantId interface{} `bson:"tenantId"` } if findErr := db.Collection("customer_service_account").FindOne(ctx, filter).Decode(&account); findErr != nil { - return user, gerror.Newf("通过 customerServiceId 查询租户失败: %v", findErr) + return user, gerror.Newf("通过 accountName 查询租户失败: %v", findErr) } user.TenantId = account.TenantId - user.UserName = customerServiceId + user.UserName = accountName + err = nil // 清空之前从token获取时的错误 return } diff --git a/rabbitmq/consumer_manager.go b/rabbitmq/consumer_manager.go index 28695ca..54648ce 100644 --- a/rabbitmq/consumer_manager.go +++ b/rabbitmq/consumer_manager.go @@ -104,9 +104,13 @@ func (cm *ConsumerManager) Init() (err error) { } glog.Info(cm.ctx, "RabbitMQ 连接已初始化") - // 设置响应队列(RAGFlow 响应消息) - if err = SetupResponseQueue(cm.ctx); err != nil { - glog.Fatalf(cm.ctx, "设置响应队列失败: %v", err) + // 声明响应Exchange(队列由各消费者自己声明和绑定) + if err = DeclareExchange(cm.ctx, &ExchangeConfig{ + Name: "ragflow.response", + Type: "topic", + Durable: true, + }); err != nil { + glog.Fatalf(cm.ctx, "声明响应Exchange失败: %v", err) return } diff --git a/ragflow/document.go b/ragflow/document.go index 5913572..0d819a7 100644 --- a/ragflow/document.go +++ b/ragflow/document.go @@ -5,10 +5,10 @@ package ragflow import ( "bytes" "context" + "encoding/json" "mime/multipart" "strings" - "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/errors/gerror" ) @@ -44,6 +44,11 @@ type UploadDocumentReq struct { FilePaths []string // 本地文件路径列表 } +// UploadDocumentRes 上传文档响应 +type UploadDocumentRes struct { + Id string `json:"id"` // 文档ID +} + // ListDocumentsReq 列出文档请求 type ListDocumentsReq struct { Page int `json:"page,omitempty"` // 页码,默认 1 @@ -190,28 +195,25 @@ func (c *Client) UploadDocumentFromText(ctx context.Context, datasetId, content, defer resp.Close() // 解析响应 - var result struct { - Code int `json:"code"` - Message string `json:"message"` - Data struct { - Id string `json:"id"` - } `json:"data"` + var response struct { + Code int `json:"code"` + Message string `json:"message"` + Data []UploadDocumentRes `json:"data"` // RAGFlow返回数组 } - bodyBytes := resp.ReadAll() - if err = gjson.DecodeTo(bodyBytes, &result); err != nil { - return "", err + if err := json.Unmarshal(resp.ReadAll(), &response); err != nil { + return "", gerror.Newf("json Decode failed: %v", err) } - if result.Code != 0 { - return "", gerror.Newf("上传文档失败 (code=%d): %s", result.Code, result.Message) + if len(response.Data) == 0 { + return "", gerror.New("上传文档返回data为空") } - if result.Data.Id == "" { - return "", gerror.New("上传成功但未返回文档ID") + if response.Code != 0 { + return "", gerror.Newf("上传文档失败 (code=%d): %s", response.Code, response.Message) } - return result.Data.Id, nil + return response.Data[0].Id, nil } // UploadDocument 上传文档(保留兼容) diff --git a/redis/redis.go b/redis/redis.go index 1ebfe1c..278f912 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -487,9 +487,10 @@ const ( // UserState 用户会话状态(阶段+对话计数+咨询方向,统一5分钟过期) type UserState struct { - Stage int `json:"stage"` // 用户阶段:5=未选择方向 0=AI模型 1=打招呼 2=业务 3=发卡片 - Count int64 `json:"count"` // 对话计数 - Direction string `json:"direction"` // 用户选择的咨询方向(如:产品咨询、售后服务) + Stage int `json:"stage"` // 当前阶段 + Direction string `json:"direction"` // 咨询方向 + Count int64 `json:"count"` // 对话计数(v5.2卡片触发) + CustomerServiceId string `json:"customerServiceId"` // 用户选择的方向对应的客服账号ID } // GetUserState 获取用户状态(阶段+计数) @@ -528,6 +529,17 @@ func SetUserStage(ctx context.Context, userId, platform string, stage int) error return err } +// SetUserCustomerServiceId 设置用户对应的客服账号ID,并刷新过期时间 +func SetUserCustomerServiceId(ctx context.Context, userId, platform, customerServiceId string) error { + key := UserStateKeyPrefix + userId + "_" + platform + _, err := redisClient.Do(ctx, "HSET", key, "customerServiceId", customerServiceId) + if err != nil { + return err + } + _, err = redisClient.Do(ctx, "EXPIRE", key, UserStateExpireSeconds) + return err +} + // SetUserDirection 设置用户选择的咨询方向,并刷新过期时间 func SetUserDirection(ctx context.Context, userId, platform, direction string) error { key := UserStateKeyPrefix + userId + "_" + platform diff --git a/redis/types.go b/redis/types.go index a6a3e36..1ceab65 100644 --- a/redis/types.go +++ b/redis/types.go @@ -10,15 +10,17 @@ type HistoryMessage struct { // SendStreamMessage 发送到 Redis Stream 的消息结构 type SendStreamMessage struct { - UserId string `json:"user_id"` // 用户ID - Content string `json:"content"` // 消息内容 - Timestamp int64 `json:"timestamp"` // 时间戳(秒) - MessageId string `json:"message_id"` // 消息唯一ID - Platform string `json:"platform,omitempty"` // 平台标识 - AccountId string `json:"account_id,omitempty"` // 账号ID - TenantId string `json:"tenant_id,omitempty"` // 租户ID(数据隔离) - ReplyQueue string `json:"reply_queue,omitempty"` // 响应队列名称(支持多实例独立队列) - History []HistoryMessage `json:"history,omitempty"` // 历史对话(归档后恢复时携带) + UserId string `json:"user_id"` // 用户ID + Content string `json:"content"` // 消息内容 + Timestamp int64 `json:"timestamp"` // 时间戳(秒) + MessageId string `json:"message_id"` // 消息唯一ID + Platform string `json:"platform,omitempty"` // 平台标识 + AccountId string `json:"account_id,omitempty"` // 账号ID + TenantId string `json:"tenant_id,omitempty"` // 租户ID(数据隔离) + CustomerServiceId string `json:"customer_service_id,omitempty"` // 客服账号ID + ChatId string `json:"chat_id,omitempty"` // RAGFlow Chat ID(从ragflow_config查询) + ReplyQueue string `json:"reply_queue,omitempty"` // 响应队列名称(支持多实例独立队列) + History []HistoryMessage `json:"history,omitempty"` // 历史对话(归档后恢复时携带) } // BatchStreamMessage 批量消息结构 From 177438ec331f9d9dfdbc7901cb1724c75da8ef33 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Thu, 25 Dec 2025 15:46:19 +0800 Subject: [PATCH 092/225] =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/chat.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ragflow/chat.go b/ragflow/chat.go index 1bf69c1..31e1b4b 100644 --- a/ragflow/chat.go +++ b/ragflow/chat.go @@ -8,9 +8,11 @@ import ( // CreateChatReq 创建对话配置请求 type CreateChatReq struct { - Name string `json:"name"` // 对话配置名称 - DatasetIds []string `json:"dataset_ids"` // 关联的知识库ID列表 - Prompt *PromptConfig `json:"prompt"` // 提示词配置 + Name string `json:"name"` // 对话配置名称(助理姓名) + Description string `json:"description,omitempty"` // 助理描述 + DatasetIds []string `json:"dataset_ids"` // 关联的知识库ID列表 + Prompt *PromptConfig `json:"prompt"` // 提示词配置 + Llm *Llm `json:"llm,omitempty"` // LLM配置 } // PromptConfig 提示词配置 @@ -19,10 +21,10 @@ type PromptConfig struct { SimilarityThreshold float64 `json:"similarity_threshold"` // 相似度阈值 KeywordsSimilarityWeight float64 `json:"keywords_similarity_weight"` // 关键词相似度权重 TopN int `json:"top_n"` // 返回顶部N个chunk - EmptyResponse string `json:"empty_response"` // 无匹配时回复 - Opener string `json:"opener"` // 开场白 - ShowQuote bool `json:"show_quote"` // 是否显示引用 - Variables []map[string]interface{} `json:"variables"` // 变量列表 + EmptyResponse string `json:"empty_response"` // 无匹配时回复(必须显式传入空字符串才能让LLM自由发挥,不传入会使用RAGFlow默认提示词) + Opener string `json:"opener,omitempty"` // 开场白 + ShowQuote bool `json:"show_quote,omitempty"` // 是否显示引用 + Variables []map[string]interface{} `json:"variables,omitempty"` // 变量列表 } // CreateChatRes 创建对话配置响应 From c3f935fb73bbfca172f3f8b44dc517f5cb5f7506 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 23 Dec 2025 17:19:08 +0800 Subject: [PATCH 093/225] =?UTF-8?q?common/http=E5=A2=9E=E5=8A=A0headers?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/http.go b/http/http.go index 2fcfdbc..b6504e0 100644 --- a/http/http.go +++ b/http/http.go @@ -64,8 +64,8 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin if err != nil { return } - Httpclient.SetHeaderMap(headers) Httpclient.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) + Httpclient.SetHeaderMap(headers) response, err := Httpclient.DoRequest(ctx, method, url, data) defer response.Close() if err != nil { From 7695752e1ccee2c30035d08dc6a094f52fd27120 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Wed, 24 Dec 2025 13:55:05 +0800 Subject: [PATCH 094/225] =?UTF-8?q?minio=E4=BF=AE=E6=94=B9=E5=9B=BE?= =?UTF-8?q?=E7=89=87=E4=B8=8A=E4=BC=A0-=E5=A2=9E=E5=8A=A0=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E5=9B=BE=E7=89=87=E7=BB=9F=E4=B8=80=E8=AE=BF=E9=97=AE?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- minio/minio.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/minio/minio.go b/minio/minio.go index 91b21ed..7ffcf96 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -66,6 +66,11 @@ func initMinIO(ctx context.Context) error { } func UploadImages(ctx context.Context, fileHeaders []*ghttp.UploadFile) (imagesUrls []string, err error) { + // 初始化 MinIO 客户端 + err = initMinIO(ctx) + if err != nil { + return nil, fmt.Errorf("初始化 MinIO 客户端失败: %w", err) + } imageUrls := make([]string, 0, len(fileHeaders)) // 存储上传后的图片访问地址 for _, fileHeader := range fileHeaders { var imageUrl string @@ -128,3 +133,15 @@ func uploadImg(ctx context.Context, fileHeader *ghttp.UploadFile) (imagesUrl str replace := strings.Replace(objectName, "images/", "/", 1) return replace, err } + +// GetImgAddressPrefix 拼接图片前缀地址 +func GetImgAddressPrefix(ctx context.Context) (imageUrl string) { + // 拼接图片前缀地址 + secure := g.Cfg().MustGet(ctx, "minio.secure").Bool() + var url = "http://" + if secure { + url = "https://" + } + imgAddressPrefix := url + g.Cfg().MustGet(ctx, "minio.endpoint").String() + "/" + g.Cfg().MustGet(ctx, "minio.bucketName").String() + "/images" + return imgAddressPrefix +} From 7e3b0362d9b783adc1f1e4b8952e10babd7058f1 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Fri, 26 Dec 2025 15:13:07 +0800 Subject: [PATCH 095/225] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=BD=AE=E8=AF=A2?= =?UTF-8?q?=E4=B8=BA=E9=98=BB=E5=A1=9E=E6=B6=88=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 23 +++++++++++++++-------- redis/types.go | 22 +++++++++++----------- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/redis/redis.go b/redis/redis.go index 278f912..12d92b6 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -108,8 +108,15 @@ func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName stri glog.Debugf(ctx, "[DEBUG Redis] XREADGROUP GROUP %s %s COUNT %d BLOCK %d STREAMS %s >", groupName, consumerName, count, blockMs, streamKey) + // 检查是否需要记录trace(避免轮询产生大量trace) + execCtx := ctx + if !g.Cfg().MustGet(ctx, "jaeger.traceStream", true).Bool() { + // 不记录trace:使用background context(不继承span) + execCtx = context.Background() + } + // XREADGROUP GROUP groupName consumerName COUNT count BLOCK blockMs STREAMS streamKey > - result, err := redisClient.Do(ctx, + result, err := redisClient.Do(execCtx, "XREADGROUP", "GROUP", groupName, consumerName, "COUNT", count, "BLOCK", blockMs, @@ -487,10 +494,10 @@ const ( // UserState 用户会话状态(阶段+对话计数+咨询方向,统一5分钟过期) type UserState struct { - Stage int `json:"stage"` // 当前阶段 - Direction string `json:"direction"` // 咨询方向 - Count int64 `json:"count"` // 对话计数(v5.2卡片触发) - CustomerServiceId string `json:"customerServiceId"` // 用户选择的方向对应的客服账号ID + Stage int `json:"stage"` // 当前阶段 + Direction string `json:"direction"` // 咨询方向 + Count int64 `json:"count"` // 对话计数(v5.2卡片触发) + AccountName string `json:"accountName"` // 用户选择的方向对应的客服账号名称 } // GetUserState 获取用户状态(阶段+计数) @@ -529,10 +536,10 @@ func SetUserStage(ctx context.Context, userId, platform string, stage int) error return err } -// SetUserCustomerServiceId 设置用户对应的客服账号ID,并刷新过期时间 -func SetUserCustomerServiceId(ctx context.Context, userId, platform, customerServiceId string) error { +// SetUserAccountName 设置用户对应的客服账号名称,并刷新过期时间 +func SetUserAccountName(ctx context.Context, userId, platform, accountName string) error { key := UserStateKeyPrefix + userId + "_" + platform - _, err := redisClient.Do(ctx, "HSET", key, "customerServiceId", customerServiceId) + _, err := redisClient.Do(ctx, "HSET", key, "accountName", accountName) if err != nil { return err } diff --git a/redis/types.go b/redis/types.go index 1ceab65..5c82526 100644 --- a/redis/types.go +++ b/redis/types.go @@ -10,17 +10,17 @@ type HistoryMessage struct { // SendStreamMessage 发送到 Redis Stream 的消息结构 type SendStreamMessage struct { - UserId string `json:"user_id"` // 用户ID - Content string `json:"content"` // 消息内容 - Timestamp int64 `json:"timestamp"` // 时间戳(秒) - MessageId string `json:"message_id"` // 消息唯一ID - Platform string `json:"platform,omitempty"` // 平台标识 - AccountId string `json:"account_id,omitempty"` // 账号ID - TenantId string `json:"tenant_id,omitempty"` // 租户ID(数据隔离) - CustomerServiceId string `json:"customer_service_id,omitempty"` // 客服账号ID - ChatId string `json:"chat_id,omitempty"` // RAGFlow Chat ID(从ragflow_config查询) - ReplyQueue string `json:"reply_queue,omitempty"` // 响应队列名称(支持多实例独立队列) - History []HistoryMessage `json:"history,omitempty"` // 历史对话(归档后恢复时携带) + UserId string `json:"user_id"` // 用户ID + Content string `json:"content"` // 消息内容 + Timestamp int64 `json:"timestamp"` // 时间戳(秒) + MessageId string `json:"message_id"` // 消息唯一ID + Platform string `json:"platform,omitempty"` // 平台标识 + AccountId string `json:"account_id,omitempty"` // 账号ID + TenantId string `json:"tenant_id,omitempty"` // 租户ID(数据隔离) + AccountName string `json:"account_name,omitempty"` // 客服账号名称 + ChatId string `json:"chat_id,omitempty"` // RAGFlow Chat ID(从ragflow_config查询) + ReplyQueue string `json:"reply_queue,omitempty"` // 响应队列名称(支持多实例独立队列) + History []HistoryMessage `json:"history,omitempty"` // 历史对话(归档后恢复时携带) } // BatchStreamMessage 批量消息结构 From aa863a5e10229ab12da45a6f3d4cb1fc82faf139 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Fri, 26 Dec 2025 14:02:25 +0800 Subject: [PATCH 096/225] =?UTF-8?q?mongo=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=EF=BC=9ASaveOrUpdate=20=E6=89=B9=E9=87=8F?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=88=96=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 102 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 97 insertions(+), 5 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index d02c9ad..e1e1b91 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -458,9 +458,13 @@ func Update(ctx context.Context, filter bson.M, update bson.M, collection string if err != nil { return } - filter["tenantId"] = user.TenantId + if !g.IsEmpty(user.TenantId) { + filter["tenantId"] = user.TenantId + } setDoc := update["$set"].(bson.M) - setDoc["updater"] = user.UserName + if !g.IsEmpty(user.UserName) { + setDoc["updater"] = user.UserName + } setDoc["updatedAt"] = gtime.Now().Time update = bson.M{"$set": setDoc} result, err = db.Collection(collection).UpdateMany(ctx, filter, update, opts...) @@ -471,6 +475,88 @@ func Update(ctx context.Context, filter bson.M, update bson.M, collection string return } +// SaveOrUpdate 批量增加或修改 +func SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.BulkWriteResult, err error) { + if len(filter) == 0 || len(update) == 0 { + err = gerror.New("缺少查询条件或更新数据") + return + } + + if len(filter) != len(update) { + err = gerror.New("查询条件和更新数据的数量必须一致") + return + } + user, err := GetTenantInfo(ctx) + if err != nil { + return + } + + // 构建批量操作模型 + var models []mongo.WriteModel + + for i := 0; i < len(filter); i++ { + // 处理过滤器 + filter[i]["isDeleted"] = false + if !g.IsEmpty(user.TenantId) { + filter[i]["tenantId"] = user.TenantId + } + + // 处理更新数据 + if setDoc, exists := update[i]["$set"].(bson.M); exists { + if !g.IsEmpty(user.UserName) { + setDoc["updater"] = user.UserName + } + setDoc["updatedAt"] = gtime.Now().Time + } else { + // 如果没有$set字段,则创建一个 + setDoc := bson.M{} + if !g.IsEmpty(user.UserName) { + setDoc["updater"] = user.UserName + } + setDoc["updatedAt"] = gtime.Now().Time + update[i]["$set"] = setDoc + } + + // 创建更新操作模型 + updateModel := mongo.NewUpdateOneModel() + updateModel.SetFilter(filter[i]) + updateModel.SetUpdate(update[i]) + updateModel.SetUpsert(true) // 默认不插入新文档 + + // 处理选项参数 + if len(opts) > 0 { + for _, opt := range opts { + var updateOpts options.UpdateManyOptions + optFuncs := opt.List() + for _, fn := range optFuncs { + fn(&updateOpts) + } + if updateOpts.Upsert != nil { + updateModel.SetUpsert(*updateOpts.Upsert) + } + } + } + + models = append(models, updateModel) + } + + // 执行批量操作,无序执行提高性能 + bulkOpts := options.BulkWrite().SetOrdered(false) + bulkResult, err := db.Collection(collection).BulkWrite(ctx, models, bulkOpts) + if err != nil { + return nil, err + } + + // 清理相关缓存 + for _, filterItem := range filter { + err = cleanRedis(ctx, filterItem, user.TenantId, collection) + if err != nil { + glog.Warning(ctx, "清理Redis缓存失败:", err) + } + } + return bulkResult, nil +} + // Insert 插入多条记录 func Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { user, err := GetTenantInfo(ctx) @@ -481,11 +567,17 @@ func Insert(ctx context.Context, documents []interface{}, collection string, opt for _, document := range documents { doc := gconv.Map(document) delete(doc, "id") - doc["creator"] = user.UserName + if !g.IsEmpty(user.UserName) { + doc["creator"] = user.UserName + } + if !g.IsEmpty(user.UserName) { + doc["updater"] = user.UserName + } + if !g.IsEmpty(user.TenantId) { + doc["tenantId"] = user.TenantId + } doc["createdAt"] = gtime.Now().Time - doc["updater"] = user.UserName doc["updatedAt"] = gtime.Now().Time - doc["tenantId"] = user.TenantId doc["isDeleted"] = false docs = append(docs, doc) } From 5f232b0ebddb42dac1f4f96297e8978277a289ee Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Fri, 26 Dec 2025 18:11:00 +0800 Subject: [PATCH 097/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0goroutine=E6=96=B9?= =?UTF-8?q?=E5=BC=8F=E5=A4=84=E7=90=86for=E5=BE=AA=E7=8E=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/worker_pool.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/ragflow/worker_pool.go b/ragflow/worker_pool.go index 98a1b48..f7635b2 100644 --- a/ragflow/worker_pool.go +++ b/ragflow/worker_pool.go @@ -7,6 +7,7 @@ import ( "gitee.com/red-future---jilin-g/common/redis" "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/os/grpool" ) // 默认批量大小(每次从 Redis 读取并发送的消息数) @@ -18,14 +19,17 @@ type QueueProcessor struct { groupName string // 消费者组名称 consumerName string // 消费者名称 timeout int64 // 阻塞超时时间(毫秒) - batchSize int64 // 最大并发数(信号量容量) + batchSize int64 // 最大并发数(协程池大小) stopChan chan struct{} // 停止信号 - semaphore chan struct{} // 并发信号量(控制最大并发) + pool *grpool.Pool // GoFrame协程池 handleFunc func(ctx context.Context, message map[string]interface{}) error } // NewQueueProcessor 创建 Stream 处理器 func NewQueueProcessor(streamKey, groupName, consumerName string, timeout, batchSize int64, handleFunc func(ctx context.Context, message map[string]interface{}) error) *QueueProcessor { + // 创建协程池:固定大小,避免频繁创建销毁goroutine + pool := grpool.New(int(batchSize)) + return &QueueProcessor{ streamKey: streamKey, groupName: groupName, @@ -33,7 +37,7 @@ func NewQueueProcessor(streamKey, groupName, consumerName string, timeout, batch timeout: timeout, batchSize: batchSize, stopChan: make(chan struct{}), - semaphore: make(chan struct{}, batchSize), // 信号量容量 = 最大并发数 + pool: pool, // 使用GoFrame协程池 handleFunc: handleFunc, } } @@ -79,16 +83,15 @@ func (q *QueueProcessor) Start(ctx context.Context) error { glog.Debugf(ctx, "读取 %d 条消息,开始发送", len(messages)) - // 2. 用信号量控制并发:获取信号量后发送,完成后释放 + // 2. 使用协程池提交任务:复用goroutine,避免频繁创建销毁 for _, msg := range messages { - // 获取信号量(阻塞直到有空位) - q.semaphore <- struct{}{} - go func(m redis.StreamMessage) { - defer func() { <-q.semaphore }() // 完成后释放信号量 + m := msg // 捕获循环变量 + // 提交到协程池,池满时会阻塞等待空闲worker + q.pool.Add(ctx, func(ctx context.Context) { q.processMessage(ctx, m) - }(msg) + }) } - // 3. 立刻读下一批(不等待,信号量自动控制并发数) + // 3. 立刻读下一批(不等待,协程池自动控制并发数) } } } @@ -109,4 +112,6 @@ func (q *QueueProcessor) processMessage(ctx context.Context, message redis.Strea // Stop 停止队列处理器 func (q *QueueProcessor) Stop() { close(q.stopChan) + // 关闭协程池,等待所有任务完成 + q.pool.Close() } From c4b4cfe50fe3f8670c54b568113cfb4da86faee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 26 Dec 2025 17:12:10 +0800 Subject: [PATCH 098/225] =?UTF-8?q?common/redis=E5=A2=9E=E5=8A=A0Lock?= =?UTF-8?q?=E6=96=B9=E6=B3=95=EF=BC=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/redis/redis.go b/redis/redis.go index 12d92b6..b7297dd 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -4,6 +4,7 @@ import ( "context" "strings" "sync" + "time" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" @@ -31,6 +32,36 @@ func GetRedisClient() *gredis.Redis { return getClient() } +// Lock 分布式锁 +func Lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context)) { + limit := 3 +LOOP: + if limit < 0 { + return + } + limit-- + if val, err := RedisClient.Set(ctx, key, true, gredis.SetOption{ + TTLOption: gredis.TTLOption{ + EX: &expireSeconds, + }, + NX: true, + }); err != nil { + glog.Errorf(ctx, "RedisClient.Lock error: %v", err) + } else { + if val.Bool() { + defer func(RedisClient *gredis.Redis, ctx context.Context, key string) { + if _, err = RedisClient.Del(ctx, key); err != nil { + glog.Errorf(ctx, "RedisClient.Del error: %v", err) + } + }(RedisClient, ctx, key) + fn(ctx) + } else { + time.Sleep(time.Second) + goto LOOP + } + } +} + // RedisClient 导出的 Redis 客户端(供 mongo.go 使用,兼容旧代码) var RedisClient = getClient() From a464e4187a18731401fe77f04bf4ccf744ccd28c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 26 Dec 2025 17:18:31 +0800 Subject: [PATCH 099/225] =?UTF-8?q?common/redis=E5=A2=9E=E5=8A=A0Lock?= =?UTF-8?q?=E6=96=B9=E6=B3=95=EF=BC=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/redis/redis.go b/redis/redis.go index b7297dd..09c60c2 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -33,11 +33,11 @@ func GetRedisClient() *gredis.Redis { } // Lock 分布式锁 -func Lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context)) { +func Lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context) error) error { limit := 3 LOOP: if limit < 0 { - return + return nil } limit-- if val, err := RedisClient.Set(ctx, key, true, gredis.SetOption{ @@ -47,6 +47,7 @@ LOOP: NX: true, }); err != nil { glog.Errorf(ctx, "RedisClient.Lock error: %v", err) + return nil } else { if val.Bool() { defer func(RedisClient *gredis.Redis, ctx context.Context, key string) { @@ -54,7 +55,7 @@ LOOP: glog.Errorf(ctx, "RedisClient.Del error: %v", err) } }(RedisClient, ctx, key) - fn(ctx) + return fn(ctx) } else { time.Sleep(time.Second) goto LOOP From 6c9e8b125a9b38d15416a1cbc77d456f2e2f79ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 26 Dec 2025 17:35:52 +0800 Subject: [PATCH 100/225] =?UTF-8?q?common/redis=E5=A2=9E=E5=8A=A0Lock?= =?UTF-8?q?=E6=96=B9=E6=B3=95=EF=BC=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/redis/redis.go b/redis/redis.go index 09c60c2..3e8a5b0 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -33,11 +33,11 @@ func GetRedisClient() *gredis.Redis { } // Lock 分布式锁 -func Lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context) error) error { +func Lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context) error) (success bool, err error) { limit := 3 LOOP: if limit < 0 { - return nil + return } limit-- if val, err := RedisClient.Set(ctx, key, true, gredis.SetOption{ @@ -46,8 +46,7 @@ LOOP: }, NX: true, }); err != nil { - glog.Errorf(ctx, "RedisClient.Lock error: %v", err) - return nil + return } else { if val.Bool() { defer func(RedisClient *gredis.Redis, ctx context.Context, key string) { @@ -55,7 +54,10 @@ LOOP: glog.Errorf(ctx, "RedisClient.Del error: %v", err) } }(RedisClient, ctx, key) - return fn(ctx) + if err = fn(ctx); err != nil { + return false, err + } + return true, nil } else { time.Sleep(time.Second) goto LOOP From c342345c0bd102222b870d9b9c5cf57fd2b74fa3 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Fri, 26 Dec 2025 17:45:11 +0800 Subject: [PATCH 101/225] =?UTF-8?q?redis=20Lock=E6=96=B9=E6=B3=95=20?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0-=E9=94=81=E9=87=8D=E8=AF=95=E6=AC=A1?= =?UTF-8?q?=E6=95=B0=E8=80=97=E5=B0=BD=E8=BF=94=E5=9B=9E=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/redis/redis.go b/redis/redis.go index 3e8a5b0..df2dacb 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -2,6 +2,7 @@ package redis import ( "context" + "errors" "strings" "sync" "time" @@ -37,7 +38,7 @@ func Lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx cont limit := 3 LOOP: if limit < 0 { - return + return false, errors.New("锁重试次数耗尽") } limit-- if val, err := RedisClient.Set(ctx, key, true, gredis.SetOption{ @@ -46,7 +47,7 @@ LOOP: }, NX: true, }); err != nil { - return + return false, err } else { if val.Bool() { defer func(RedisClient *gredis.Redis, ctx context.Context, key string) { From 63c19d0727d403576bbcdf953e9bcc4b9e36e561 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Mon, 29 Dec 2025 11:33:37 +0800 Subject: [PATCH 102/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=BA=86=E4=B8=80?= =?UTF-8?q?=E4=B8=8Bmq=E7=BB=91=E5=AE=9A=E9=98=9F=E5=88=97=E5=90=8D,?= =?UTF-8?q?=E4=BD=BF=E7=94=A8os.hostname?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rabbitmq/instance.go | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/rabbitmq/instance.go b/rabbitmq/instance.go index 330c6d5..8ac30dc 100644 --- a/rabbitmq/instance.go +++ b/rabbitmq/instance.go @@ -1,10 +1,12 @@ package rabbitmq import ( + "context" "fmt" "os" "sync" + "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/util/guid" ) @@ -14,19 +16,35 @@ var ( ) // getInstanceId 获取当前实例的唯一标识(单例) -// 格式:{hostname}.{uuid8} +// 优先级:配置文件 > 环境变量 > 容器名/主机名 > 随机UUID func getInstanceId() string { instanceOnce.Do(func() { - // 获取主机名 + ctx := context.Background() + + // 1. 优先从配置文件读取(手动指定,最高优先级) + instanceId = g.Cfg().MustGet(ctx, "rabbitmq.instanceName").String() + if instanceId != "" { + return + } + + // 2. 读取环境变量(Docker/K8s部署时设置) + instanceId = os.Getenv("INSTANCE_NAME") + if instanceId != "" { + return + } + + // 3. 使用主机名(Docker容器名/主机名) hostname, err := os.Hostname() if err != nil || hostname == "" { hostname = "unknown" } - // 生成8位UUID - uuid := guid.S()[:8] - - instanceId = fmt.Sprintf("%s.%s", hostname, uuid) + // 4. 如果主机名是默认值(本地开发),添加随机后缀避免冲突 + if hostname == "localhost" || hostname == "unknown" { + instanceId = hostname + "." + guid.S()[:4] + } else { + instanceId = hostname + } }) return instanceId } From c80ecb19e446b806283e45f6dfa41b1f53027384 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Mon, 29 Dec 2025 14:46:39 +0800 Subject: [PATCH 103/225] =?UTF-8?q?mongo=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=98=AF=E5=90=A6=E9=87=8D=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=95=B0=E6=8D=AE=EF=BC=8C=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E9=9A=8F=E6=9C=BA=E4=BF=AE=E6=94=B9=E5=88=A0=E9=99=A4=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E6=8E=A5=E5=8F=A3=EF=BC=9Bredis=E5=A2=9E=E5=8A=A0-?= =?UTF-8?q?=E9=98=9F=E5=88=97=E6=B6=88=E6=81=AF=E8=AF=BB=E5=8F=96=E5=B0=81?= =?UTF-8?q?=E8=A3=85=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 158 ++++++++++++++++++++++++++++++++--------------- redis/message.go | 14 +++++ redis/redis.go | 57 +++++++++++++++-- 3 files changed, 175 insertions(+), 54 deletions(-) create mode 100644 redis/message.go diff --git a/mongo/mongo.go b/mongo/mongo.go index e1e1b91..711b0c0 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/gogf/gf/v2/container/gvar" "strings" "sync" "time" @@ -317,7 +318,7 @@ func GetTenantInfo(ctx context.Context) (user do.User, err error) { } // Find 查询多条记录 -func Find(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOptions]) (err error) { +func Find(ctx context.Context, NoCache bool, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOptions]) (err error) { if err = utils.ValidStructPtr(result); err != nil { return } @@ -328,34 +329,39 @@ func Find(ctx context.Context, filter bson.M, result interface{}, collection str filter["isDeleted"] = false filterMap := utils.OrderMap(filter) optsMap := listOptionsToMap(ctx, opts...) - redisKey := fmt.Sprintf(consts.List, user.TenantId, collection, gconv.String(filterMap), gconv.String(optsMap)) - resultStr, err := redis.RedisClient.Get(ctx, redisKey) - if err != nil { - return - } - if !g.IsEmpty(resultStr) { - err = gconv.Scan(resultStr, result) + redisKey := "" + if !NoCache { + redisKey = fmt.Sprintf(consts.List, user.TenantId, collection, gconv.String(filterMap), gconv.String(optsMap)) + var resultStr *gvar.Var + resultStr, err = redis.RedisClient.Get(ctx, redisKey) if err != nil { - return err + return + } + if !g.IsEmpty(resultStr) { + err = gconv.Scan(resultStr, result) + if err != nil { + return err + } + return } - return } - filter["tenantId"] = user.TenantId cur, err := db.Collection(collection).Find(ctx, filter, opts...) if err != nil { return } err = cur.All(ctx, result) - err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) - if err != nil { - return err + if !NoCache { + err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) + if err != nil { + return err + } } return } // FindOne 查询1条记录 -func FindOne(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOneOptions]) (err error) { +func FindOne(ctx context.Context, NoCache bool, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOneOptions]) (err error) { if len(filter) == 0 { err = gerror.New("缺少查询条件") return @@ -369,27 +375,35 @@ func FindOne(ctx context.Context, filter bson.M, result interface{}, collection } filter["isDeleted"] = false filterMap := utils.OrderMap(filter) - redisKey := fmt.Sprintf(consts.One, user.TenantId, collection, gconv.String(filterMap)) - resultStr, err := redis.RedisClient.Get(ctx, redisKey) - if err != nil { - return - } - if !g.IsEmpty(resultStr) { - err = gconv.Scan(resultStr, result) + redisKey := "" + if !NoCache { + redisKey := fmt.Sprintf(consts.One, user.TenantId, collection, gconv.String(filterMap)) + var resultStr *gvar.Var + resultStr, err = redis.RedisClient.Get(ctx, redisKey) if err != nil { - return err + return + } + if !g.IsEmpty(resultStr) { + err = gconv.Scan(resultStr, result) + if err != nil { + return err + } + return } - return } - filter["tenantId"] = user.TenantId + if !g.IsEmpty(user.TenantId) { + filter["tenantId"] = user.TenantId + } cur := db.Collection(collection).FindOne(ctx, filter, opts...) err = cur.Decode(result) if errors.Is(err, mongo.ErrNoDocuments) { err = nil } - err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) - if err != nil { - return err + if !NoCache { + err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) + if err != nil { + return err + } } return } @@ -475,13 +489,61 @@ func Update(ctx context.Context, filter bson.M, update bson.M, collection string return } +// RandomSoftDelete 随机软删除个文档的 _id +func RandomSoftDelete(ctx context.Context, limit int, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { + // 步骤 1: 使用聚合管道的 $sample 操作符随机抽取5个文档的 _id + pipeline := mongo.Pipeline{ + // 阶段1: 为每个文档添加一个 0-1 之间的随机数字段 'random' + bson.D{{Key: "$addFields", Value: bson.D{{Key: "random", Value: bson.M{"$rand": bson.M{}}}}}}, + // 阶段1: 匹配所有未删除的文档 + bson.D{{Key: "$match", Value: bson.D{{Key: "isDeleted", Value: false}}}}, + // 阶段2: 按随机数降序排序 + bson.D{{Key: "$sort", Value: bson.D{{Key: "random", Value: -1}}}}, + // 阶段3: 只取前5个 + bson.D{{Key: "$limit", Value: limit}}, + // 阶段4: 只投影 _id + bson.D{{Key: "$project", Value: bson.D{{Key: "_id", Value: 1}}}}, + } + cursor, err := db.Collection(collection).Aggregate(ctx, pipeline) + if err != nil { + return + } + defer cursor.Close(ctx) + // 步骤 2: 从聚合结果中提取 _id 到一个切片中 + var idsToUpdate []bson.ObjectID + for cursor.Next(ctx) { + var result bson.M + if err := cursor.Decode(&result); err != nil { + return nil, err + } + // 将 bson.M 中的 _id 断言为 primitive.ObjectID + id := result["_id"].(bson.ObjectID) + idsToUpdate = append(idsToUpdate, id) + } + if err := cursor.Err(); err != nil { + return nil, err + } + fmt.Printf("准备更新的随机文档ID: %v\n", idsToUpdate) + // 步骤 3: 使用 $in 操作符和 UpdateMany 批量更新选定的文档 + if len(idsToUpdate) > 0 { + // 过滤条件:匹配 idsToUpdate 切片中的任意一个 _id + filter := bson.D{{Key: "_id", Value: bson.D{{Key: "$in", Value: idsToUpdate}}}} + // 更新操作:使用 $set 修改字段 + update := bson.D{{Key: "$set", Value: bson.D{{Key: "isDeleted", Value: true}}}} + _, err = db.Collection(collection).UpdateMany(ctx, filter, update) + if err != nil { + return + } + } + return +} + // SaveOrUpdate 批量增加或修改 func SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.BulkWriteResult, err error) { if len(filter) == 0 || len(update) == 0 { err = gerror.New("缺少查询条件或更新数据") return } - if len(filter) != len(update) { err = gerror.New("查询条件和更新数据的数量必须一致") return @@ -490,17 +552,14 @@ func SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collect if err != nil { return } - // 构建批量操作模型 var models []mongo.WriteModel - for i := 0; i < len(filter); i++ { // 处理过滤器 filter[i]["isDeleted"] = false if !g.IsEmpty(user.TenantId) { filter[i]["tenantId"] = user.TenantId } - // 处理更新数据 if setDoc, exists := update[i]["$set"].(bson.M); exists { if !g.IsEmpty(user.UserName) { @@ -516,13 +575,11 @@ func SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collect setDoc["updatedAt"] = gtime.Now().Time update[i]["$set"] = setDoc } - // 创建更新操作模型 updateModel := mongo.NewUpdateOneModel() updateModel.SetFilter(filter[i]) updateModel.SetUpdate(update[i]) updateModel.SetUpsert(true) // 默认不插入新文档 - // 处理选项参数 if len(opts) > 0 { for _, opt := range opts { @@ -536,17 +593,14 @@ func SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collect } } } - models = append(models, updateModel) } - // 执行批量操作,无序执行提高性能 bulkOpts := options.BulkWrite().SetOrdered(false) bulkResult, err := db.Collection(collection).BulkWrite(ctx, models, bulkOpts) if err != nil { return nil, err } - // 清理相关缓存 for _, filterItem := range filter { err = cleanRedis(ctx, filterItem, user.TenantId, collection) @@ -591,27 +645,33 @@ func Insert(ctx context.Context, documents []interface{}, collection string, opt } // Count 查询总数 -func Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { +func Count(ctx context.Context, NoCache bool, filter bson.M, collection string) (count int64, err error) { user, err := GetTenantInfo(ctx) if err != nil { return } filter["isDeleted"] = false filterMap := utils.OrderMap(filter) - redisKey := fmt.Sprintf(consts.Count, user.TenantId, collection, gconv.String(filterMap)) - resultStr, err := redis.RedisClient.Get(ctx, redisKey) - if err != nil { - return - } - if !g.IsEmpty(resultStr) { - count = gconv.Int64(resultStr) - return + redisKey := "" + if !NoCache { + redisKey = fmt.Sprintf(consts.Count, user.TenantId, collection, gconv.String(filterMap)) + var resultStr *gvar.Var + resultStr, err = redis.RedisClient.Get(ctx, redisKey) + if err != nil { + return + } + if !g.IsEmpty(resultStr) { + count = gconv.Int64(resultStr) + return + } } // 调用驱动的 CountDocuments,在数据库端执行的 count, err = db.Collection(collection).CountDocuments(ctx, filter) - err = redis.RedisClient.SetEX(ctx, redisKey, count, int64(time.Hour)) - if err != nil { - return + if !NoCache { + err = redis.RedisClient.SetEX(ctx, redisKey, count, int64(time.Hour)) + if err != nil { + return + } } return } diff --git a/redis/message.go b/redis/message.go new file mode 100644 index 0000000..e7e6cb0 --- /dev/null +++ b/redis/message.go @@ -0,0 +1,14 @@ +package redis + +import "context" + +type QueueMessage struct { + StreamKey string // Stream 键名 + GroupName string // 消费者组名称 + ConsumerName string // 消费者名称 + Timeout int64 // 阻塞超时时间(毫秒) + BatchSize int64 // 最大并发数(信号量容量) + BlockMs int64 + Block bool + HandleFunc func(ctx context.Context, message map[string]interface{}) error +} diff --git a/redis/redis.go b/redis/redis.go index df2dacb..552d273 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -3,6 +3,7 @@ package redis import ( "context" "errors" + "fmt" "strings" "sync" "time" @@ -33,6 +34,9 @@ func GetRedisClient() *gredis.Redis { return getClient() } +// RedisClient 导出的 Redis 客户端(供 mongo.go 使用,兼容旧代码) +var RedisClient = getClient() + // Lock 分布式锁 func Lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context) error) (success bool, err error) { limit := 3 @@ -66,8 +70,52 @@ LOOP: } } -// RedisClient 导出的 Redis 客户端(供 mongo.go 使用,兼容旧代码) -var RedisClient = getClient() +func GetReadStream(ctx context.Context, msg ...QueueMessage) error { + for _, t := range msg { + err := GetReadFromStream(ctx, t.StreamKey, t.GroupName, t.ConsumerName, t.BatchSize, t.BlockMs, t.Block, t.HandleFunc) + if err != nil { + return err + } + } + return nil +} + +// GetReadFromStream 读取ReadFromStream数据 +func GetReadFromStream(ctx context.Context, streamKey, groupName, consumerName string, count, blockMs int64, Block bool, fn func(ctx context.Context, message map[string]interface{}) error) (err error) { + glog.Infof(ctx, "初始化 Stream: %s, 消费者组: %s", streamKey, groupName) + err = InitStreamGroup(ctx, streamKey, groupName) + if err != nil { + return err + } + for { + // 从 Redis Stream 读取一批消息 + messages, err := ReadFromStream(ctx, streamKey, groupName, consumerName, count, blockMs) + if err != nil { + glog.Errorf(ctx, "[DEBUG Redis] XREADGROUP 错误: %v", err) + return err + } + // 处理消息 + for _, msg := range messages { + fmt.Printf("消费者 '%s' -> 接收到消息 ID: %s, 内容: %v\n", consumerName, msg.ID, msg.Values) + // 业务处理 + if err = fn(ctx, msg.Values); err != nil { + return err + } + // 确认消息 (ACK) + if Block { + // 处理成功后,必须调用 XAck,否则消息会一直留在 PEL 中 + err = AckMessage(ctx, streamKey, groupName, msg.ID) + if err != nil { + glog.Infof(ctx, "消费者 '%s' 确认消息 ID %s 失败: %v\n", consumerName, msg.ID, err) + } else { + glog.Infof(ctx, "消费者 '%s' -> 已确认消息 ID: %s\n", consumerName, msg.ID) + } + } + + } + } + return +} // Stream 和消费者组常量 const ( @@ -150,6 +198,7 @@ func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName stri execCtx = context.Background() } +RECONNECT: // XREADGROUP GROUP groupName consumerName COUNT count BLOCK blockMs STREAMS streamKey > result, err := redisClient.Do(execCtx, "XREADGROUP", "GROUP", groupName, consumerName, @@ -157,10 +206,8 @@ func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName stri "BLOCK", blockMs, "STREAMS", streamKey, ">", ) - if err != nil { - glog.Errorf(ctx, "[DEBUG Redis] XREADGROUP 错误: %v", err) - return nil, err + goto RECONNECT } glog.Debugf(ctx, "[DEBUG Redis] XREADGROUP 返回: %+v", result) From cc67dd2485b3fc427faeafd44e599c349cff6b96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 29 Dec 2025 14:44:32 +0800 Subject: [PATCH 104/225] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=99=90=E6=B5=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consts/redis_key.go | 26 +++- middleware/middleware.go | 18 --- middleware/rate_limiter.go | 300 +++++++++++++++++++++++++++++++++++++ redis/redis.go | 11 +- 4 files changed, 325 insertions(+), 30 deletions(-) create mode 100644 middleware/rate_limiter.go diff --git a/consts/redis_key.go b/consts/redis_key.go index 183aa69..3f85dff 100644 --- a/consts/redis_key.go +++ b/consts/redis_key.go @@ -1,7 +1,23 @@ package consts -const CleanList = "list:tenantId-%v:collection-%s:*" -const CleanCount = "count:tenantId-%v:collection-%s:*" -const List = "list:tenantId-%v:collection-%s:filter:%s:options:%s" -const Count = "count:tenantId-%v:collection-%s:filter:%s" -const One = "one:tenantId-%v:collection-%s:filter:%s" +// Redis 数据缓存 Key 常量 +const ( + CleanList = "list:tenantId-%v:collection-%s:*" // 清理列表Key + CleanCount = "count:tenantId-%v:collection-%s:*" // 清理计数Key + List = "list:tenantId-%v:collection-%s:filter:%s:options:%s" // 列表查询Key + Count = "count:tenantId-%v:collection-%s:filter:%s" // 计数查询Key + One = "one:tenantId-%v:collection-%s:filter:%s" // 单条查询Key +) + +// 限流 Redis Key 常量 +const ( + RateLimitKeyPrefix = "ragflow:ratelimit:" // 限流Key前缀 + RateLimitKeyIP = "ip:%s" // IP限流: ip:192.168.1.1 + RateLimitKeyUser = "user:%s" // 用户限流: user:123 或 user:anon:192.168.1.1 + RateLimitKeyService = "service:%s" // 服务限流: service:customerService + RateLimitKeyGlobal = "global:requests" // 全局限流: global:requests + RateLimitKeyOrder = "order:create:%s" // 订单创建限流: order:create:123 + RateLimitKeyTransfer = "wallet:transfer:%s" // 钱包转账限流: wallet:transfer:123 + RateLimitKeyMessage = "cs:message:%s" // 客服消息限流: cs:message:123 + RateLimitKeyUpload = "oss:upload:%s" // 文件上传限流: oss:upload:123 +) diff --git a/middleware/middleware.go b/middleware/middleware.go index d37ba83..9178ddc 100644 --- a/middleware/middleware.go +++ b/middleware/middleware.go @@ -1,13 +1,10 @@ package middleware import ( - "context" - "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/text/gstr" - "golang.org/x/time/rate" ) // Logger 中间件 @@ -24,17 +21,6 @@ func Logger(r *ghttp.Request) { ) } -var rateLimit = g.Cfg().MustGet(context.TODO(), "rate.limit").Int() -var rateBurst = g.Cfg().MustGet(context.TODO(), "rate.burst").Int() -var limiter = rate.NewLimiter(rate.Limit(rateLimit), rateBurst) - -func Limiter(r *ghttp.Request) { - if !limiter.Allow() { - r.Response.WriteStatusExit(429) // Return 429 Too Many Requests - r.ExitAll() - } - r.Middleware.Next() -} func Auth(r *ghttp.Request) { //utils.GetUserInfo(r.GetCtx()) token := r.Header.Get("Authorization") @@ -51,7 +37,3 @@ func Auth(r *ghttp.Request) { r.Middleware.Next() } -func validateToken(token string) bool { - // 实现 token 验证逻辑 - return token == "valid-token" -} diff --git a/middleware/rate_limiter.go b/middleware/rate_limiter.go new file mode 100644 index 0000000..f8c9bc6 --- /dev/null +++ b/middleware/rate_limiter.go @@ -0,0 +1,300 @@ +package middleware + +import ( + "fmt" + "strings" + + "gitee.com/red-future---jilin-g/common/consts" + "gitee.com/red-future---jilin-g/common/redis" + "gitee.com/red-future---jilin-g/common/utils" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/ghttp" + "github.com/gogf/gf/v2/text/gstr" + "github.com/gogf/gf/v2/util/gconv" +) + +// GlobalLimiter 全局限流中间件(使用Redis分布式控制) +func GlobalLimiter(r *ghttp.Request) { + // 从配置文件读取全局限流参数 + globalLimit := g.Cfg().MustGet(r.GetCtx(), "rate.limit", 800).Int64() + + key := consts.RateLimitKeyGlobal + + // 使用Redis计数器进行全局限流 + count, err := redis.IncrRateLimit(r.GetCtx(), key, 1) // 1秒窗口 + if err != nil { + g.Log().Errorf(r.GetCtx(), "全局限流Redis错误: %v", err) + r.Middleware.Next() + return + } + + if count > globalLimit { + g.Log().Warningf(r.GetCtx(), "全局限流触发: count: %d, limit: %d", count, globalLimit) + r.Response.WriteStatusExit(429, "系统当前繁忙,请稍后再试") + return + } + + r.Middleware.Next() +} + +// IPLimiter IP限流中间件(防DDoS) +func IPLimiter(r *ghttp.Request) { + ip := r.GetClientIp() + key := fmt.Sprintf(consts.RateLimitKeyIP, ip) + + // 从配置文件读取IP限流参数 + ipLimit := g.Cfg().MustGet(r.GetCtx(), "rate.ip.limit", 100).Int64() + + // 使用Redis计数器 + count, err := redis.IncrRateLimit(r.GetCtx(), key, 1) // 1秒窗口 + if err != nil { + g.Log().Errorf(r.GetCtx(), "IP限流Redis错误: %v", err) + r.Middleware.Next() + return + } + + if count > ipLimit { + g.Log().Warningf(r.GetCtx(), "IP限流触发: %s, count: %d, limit: %d", ip, count, ipLimit) + r.Response.WriteStatusExit(429, "请求过于频繁,请稍后再试") + return + } + + r.Middleware.Next() +} + +// UserLimiter 用户维度限流中间件(防止单用户滥用) +func UserLimiter(r *ghttp.Request) { + // 从JWT获取用户ID(如果已登录) + var userId string + var isAuth bool = false + + if token := r.Header.Get("Authorization"); token != "" && gstr.HasPrefix(token, "Bearer ") { + // 这里应该解析JWT获取用户ID,简化示例中直接使用token + tokenStr := gstr.SubStrFrom(token, "7") + if tokenStr != "" && validateToken(tokenStr) { + userId = tokenStr + isAuth = true + } + } + + // 如果没有userId,使用IP作为标识 + if userId == "" { + userId = "anon:" + r.GetClientIp() + } + + // 从配置文件读取用户限流参数 + var userLimit int64 + if isAuth { + userLimit = g.Cfg().MustGet(r.GetCtx(), "rate.user.authenticated.limit", 50).Int64() + } else { + userLimit = g.Cfg().MustGet(r.GetCtx(), "rate.user.anonymous.limit", 20).Int64() + } + + key := fmt.Sprintf(consts.RateLimitKeyUser, userId) + count, err := redis.IncrRateLimit(r.GetCtx(), key, 1) + if err != nil { + g.Log().Errorf(r.GetCtx(), "用户限流Redis错误: %v", err) + r.Middleware.Next() + return + } + + if count > userLimit { + userType := "已登录" + if !isAuth { + userType = "未登录" + } + g.Log().Warningf(r.GetCtx(), "用户限流触发: %s, count: %d, limit: %d, type: %s", userId, count, userLimit, userType) + r.Response.WriteStatusExit(429, "您的请求过于频繁,请稍后再试") + return + } + + r.Middleware.Next() +} + +// ServiceLimiter 服务维度限流中间件(保护微服务) +func ServiceLimiter(r *ghttp.Request) { + // 从URL路径提取服务名: /customerService/xxx -> customerService + pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + if len(pathParts) == 0 { + r.Middleware.Next() + return + } + + serverName := pathParts[0] + + // 从配置文件读取服务限流参数 + serviceLimitKey := fmt.Sprintf("rate.services.%s.limit", serverName) + limit := g.Cfg().MustGet(r.GetCtx(), serviceLimitKey, 0).Int64() + + // 如果配置为0,说明该服务没有限流配置,跳过限流 + if limit == 0 { + r.Middleware.Next() + return + } + + key := fmt.Sprintf(consts.RateLimitKeyService, serverName) + count, err := redis.IncrRateLimit(r.GetCtx(), key, 1) + if err != nil { + g.Log().Errorf(r.GetCtx(), "服务限流Redis错误: %v", err) + r.Middleware.Next() + return + } + + if count > limit { + g.Log().Warningf(r.GetCtx(), "服务限流触发: %s, count: %d, limit: %d", serverName, count, limit) + r.Response.WriteStatusExit(429, fmt.Sprintf("服务 '%s' 当前繁忙,请稍后再试", serverName)) + return + } + + r.Middleware.Next() +} + +// OrderCreateLimiter 订单创建限流中间件 +// 限制: 每个用户每分钟最多创建10个订单 +func OrderCreateLimiter(r *ghttp.Request) { + userId := getUserIdFromContext(r) // 从context获取用户ID + if userId == "" { + // 如果无法获取用户信息,跳过限流检查 + r.Middleware.Next() + return + } + + key := fmt.Sprintf(consts.RateLimitKeyOrder, userId) + + // 限制: 每个用户每分钟最多创建10个订单 + count, err := redis.IncrRateLimit(r.GetCtx(), key, 60) // 60秒窗口 + if err != nil { + g.Log().Errorf(r.GetCtx(), "订单创建限流Redis错误: %v", err) + r.Middleware.Next() + return + } + + if count > 10 { + g.Log().Warningf(r.GetCtx(), "订单创建限流触发: %s, count: %d", userId, count) + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 429, + Message: "下单过于频繁,请稍后再试", + }) + return + } + + r.Middleware.Next() +} + +// WalletTransferLimiter 钱包转账限流中间件 +// 限制: 每个用户每分钟最多转账5次 +func WalletTransferLimiter(r *ghttp.Request) { + userId := getUserIdFromContext(r) // 从context获取用户ID + if userId == "" { + r.Middleware.Next() + return + } + + key := fmt.Sprintf(consts.RateLimitKeyTransfer, userId) + + // 限制: 每个用户每分钟最多转账5次 + count, err := redis.IncrRateLimit(r.GetCtx(), key, 60) // 60秒窗口 + if err != nil { + g.Log().Errorf(r.GetCtx(), "钱包转账限流Redis错误: %v", err) + r.Middleware.Next() + return + } + + if count > 5 { + g.Log().Warningf(r.GetCtx(), "钱包转账限流触发: %s, count: %d", userId, count) + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 429, + Message: "转账操作过于频繁,请稍后再试", + }) + return + } + + r.Middleware.Next() +} + +// CSMessageLimiter 客服消息限流中间件 +// 限制: 每个用户每分钟最多发送30条消息 +func CSMessageLimiter(r *ghttp.Request) { + userId := getUserIdFromContext(r) // 从context获取用户ID + if userId == "" { + r.Middleware.Next() + return + } + + key := fmt.Sprintf(consts.RateLimitKeyMessage, userId) + + // 限制: 每个用户每分钟最多发送30条消息 + count, err := redis.IncrRateLimit(r.GetCtx(), key, 60) // 60秒窗口 + if err != nil { + g.Log().Errorf(r.GetCtx(), "客服消息限流Redis错误: %v", err) + r.Middleware.Next() + return + } + + if count > 30 { + g.Log().Warningf(r.GetCtx(), "客服消息限流触发: %s, count: %d", userId, count) + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 429, + Message: "消息发送过于频繁,请稍后再试", + }) + return + } + + r.Middleware.Next() +} + +// OSSUploadLimiter 文件上传限流中间件 +// 限制: 每个用户每分钟最多上传10个文件 +func OSSUploadLimiter(r *ghttp.Request) { + userId := getUserIdFromContext(r) // 从context获取用户ID + if userId == "" { + r.Middleware.Next() + return + } + + key := fmt.Sprintf(consts.RateLimitKeyUpload, userId) + + // 限制: 每个用户每分钟最多上传10个文件 + count, err := redis.IncrRateLimit(r.GetCtx(), key, 60) // 60秒窗口 + if err != nil { + g.Log().Errorf(r.GetCtx(), "文件上传限流Redis错误: %v", err) + r.Middleware.Next() + return + } + + if count > 10 { + g.Log().Warningf(r.GetCtx(), "文件上传限流触发: %s, count: %d", userId, count) + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 429, + Message: "文件上传过于频繁,请稍后再试", + }) + return + } + + r.Middleware.Next() +} + +// getUserIdFromContext 从请求上下文中获取用户ID +// 使用项目中已有的utils.GetUserInfo方法 +func getUserIdFromContext(r *ghttp.Request) string { + // 使用项目中已有的utils.GetUserInfo方法获取用户信息 + user, err := utils.GetUserInfo(r.GetCtx()) + if err != nil { + // 如果获取用户信息失败,返回空字符串 + return "" + } + + // 在这个项目中,UserName就是用来标识用户的ID + // 转换为字符串类型 + if user.UserName != nil { + return gconv.String(user.UserName) + } + + return "" +} + +// validateToken 验证token有效性 +func validateToken(token string) bool { + // 实现 token 验证逻辑 + return token == "valid-token" +} diff --git a/redis/redis.go b/redis/redis.go index 552d273..aa4e947 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "gitee.com/red-future---jilin-g/common/consts" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/glog" @@ -478,15 +479,11 @@ func IsUserActive(ctx context.Context, userId string, seconds int64) (bool, erro // ============== 限流相关 ============== -const ( - // RateLimitKeyPrefix 限流计数器 Key 前缀 - RateLimitKeyPrefix = "ragflow:ratelimit:" -) - // IncrRateLimit 增加限流计数器,返回当前计数 +// key: 限流key(需要包含完整路径,如 "ip:192.168.1.1") // windowSeconds: 时间窗口(秒) func IncrRateLimit(ctx context.Context, key string, windowSeconds int64) (count int64, err error) { - fullKey := RateLimitKeyPrefix + key + fullKey := consts.RateLimitKeyPrefix + key result, err := redisClient.Do(ctx, "INCR", fullKey) if err != nil { return @@ -502,7 +499,7 @@ func IncrRateLimit(ctx context.Context, key string, windowSeconds int64) (count // GetRateLimit 获取当前限流计数 func GetRateLimit(ctx context.Context, key string) (count int64, err error) { - fullKey := RateLimitKeyPrefix + key + fullKey := consts.RateLimitKeyPrefix + key result, err := redisClient.Get(ctx, fullKey) if err != nil { return From 836460306eeafc57efe6a637de0de613c51fa1db Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Mon, 29 Dec 2025 14:52:31 +0800 Subject: [PATCH 105/225] =?UTF-8?q?mongo=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=98=AF=E5=90=A6=E9=87=8D=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E6=95=B0=E6=8D=AE=EF=BC=8C=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E9=9A=8F=E6=9C=BA=E4=BF=AE=E6=94=B9=E5=88=A0=E9=99=A4=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E6=8E=A5=E5=8F=A3=EF=BC=9Bredis=E5=A2=9E=E5=8A=A0-?= =?UTF-8?q?=E9=98=9F=E5=88=97=E6=B6=88=E6=81=AF=E8=AF=BB=E5=8F=96=E5=B0=81?= =?UTF-8?q?=E8=A3=85=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis/redis.go b/redis/redis.go index aa4e947..c338e75 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -81,7 +81,7 @@ func GetReadStream(ctx context.Context, msg ...QueueMessage) error { return nil } -// GetReadFromStream 读取ReadFromStream数据 +// GetReadFromStream 读取ReadFromStream数据 func GetReadFromStream(ctx context.Context, streamKey, groupName, consumerName string, count, blockMs int64, Block bool, fn func(ctx context.Context, message map[string]interface{}) error) (err error) { glog.Infof(ctx, "初始化 Stream: %s, 消费者组: %s", streamKey, groupName) err = InitStreamGroup(ctx, streamKey, groupName) From 55e8a829de89dd615a0aa305ebb47ab22f711962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 30 Dec 2025 09:29:36 +0800 Subject: [PATCH 106/225] =?UTF-8?q?mongo.go=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 88 ++++++++++++++++++++++++++------------------------ 1 file changed, 46 insertions(+), 42 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 711b0c0..4c3855f 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -4,11 +4,12 @@ import ( "context" "errors" "fmt" - "github.com/gogf/gf/v2/container/gvar" "strings" "sync" "time" + "github.com/gogf/gf/v2/container/gvar" + "gitee.com/red-future---jilin-g/common/consts" "gitee.com/red-future---jilin-g/common/do" "gitee.com/red-future---jilin-g/common/redis" @@ -24,6 +25,16 @@ import ( "go.mongodb.org/mongo-driver/v2/mongo/options" ) +type MongoDB struct { + Cache bool +} + +func DB(cache bool) *MongoDB { + return &MongoDB{ + Cache: cache, + } +} + var ( db *mongo.Database client *mongo.Client @@ -35,15 +46,8 @@ var ( healthCancel context.CancelFunc ) -// GetDB 获取 MongoDB 数据库实例 -func GetDB() *mongo.Database { - mu.RLock() - defer mu.RUnlock() - return db -} - -// IsConnected 检查连接状态 -func IsConnected() bool { +// checkConnected 检查连接状态 +func checkConnected() bool { mu.RLock() defer mu.RUnlock() return isConnected @@ -187,8 +191,8 @@ func init() { go healthCheck() } -// Close 关闭MongoDB连接 -func Close() { +// close 关闭MongoDB连接 +func close() { if healthCancel != nil { healthCancel() } @@ -253,9 +257,9 @@ func oneOptionsToMap(ctx context.Context, opts ...options.Lister[options.FindOne return } -// GetTenantInfo 获取租户信息 +// getTenantInfo 获取租户信息 // 优先从 token 获取,失败则从请求参数 customerServiceId 查询 customer_service_account 表 -func GetTenantInfo(ctx context.Context) (user do.User, err error) { +func getTenantInfo(ctx context.Context) (user do.User, err error) { // 1. 优先从 token 获取 user, err = utils.GetUserInfo(ctx) if err == nil { @@ -318,20 +322,19 @@ func GetTenantInfo(ctx context.Context) (user do.User, err error) { } // Find 查询多条记录 -func Find(ctx context.Context, NoCache bool, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOptions]) (err error) { +func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOptions]) (err error) { if err = utils.ValidStructPtr(result); err != nil { return } - user, err := GetTenantInfo(ctx) + user, err := getTenantInfo(ctx) if err != nil { return } filter["isDeleted"] = false filterMap := utils.OrderMap(filter) optsMap := listOptionsToMap(ctx, opts...) - redisKey := "" - if !NoCache { - redisKey = fmt.Sprintf(consts.List, user.TenantId, collection, gconv.String(filterMap), gconv.String(optsMap)) + redisKey := fmt.Sprintf(consts.List, user.TenantId, collection, gconv.String(filterMap), gconv.String(optsMap)) + if m.Cache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) if err != nil { @@ -350,8 +353,11 @@ func Find(ctx context.Context, NoCache bool, filter bson.M, result interface{}, if err != nil { return } - err = cur.All(ctx, result) - if !NoCache { + defer cur.Close(ctx) + if err = cur.All(ctx, result); err != nil { + return + } + if m.Cache { err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) if err != nil { return err @@ -361,7 +367,7 @@ func Find(ctx context.Context, NoCache bool, filter bson.M, result interface{}, } // FindOne 查询1条记录 -func FindOne(ctx context.Context, NoCache bool, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOneOptions]) (err error) { +func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOneOptions]) (err error) { if len(filter) == 0 { err = gerror.New("缺少查询条件") return @@ -369,15 +375,14 @@ func FindOne(ctx context.Context, NoCache bool, filter bson.M, result interface{ if err = utils.ValidStructPtr(result); err != nil { return } - user, err := GetTenantInfo(ctx) + user, err := getTenantInfo(ctx) if err != nil { return } filter["isDeleted"] = false filterMap := utils.OrderMap(filter) - redisKey := "" - if !NoCache { - redisKey := fmt.Sprintf(consts.One, user.TenantId, collection, gconv.String(filterMap)) + redisKey := fmt.Sprintf(consts.One, user.TenantId, collection, gconv.String(filterMap)) + if m.Cache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) if err != nil { @@ -399,7 +404,7 @@ func FindOne(ctx context.Context, NoCache bool, filter bson.M, result interface{ if errors.Is(err, mongo.ErrNoDocuments) { err = nil } - if !NoCache { + if m.Cache { err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) if err != nil { return err @@ -442,12 +447,12 @@ func cleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collec } // Delete 删除记录 -func Delete(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.DeleteManyOptions]) (count int64, err error) { +func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.DeleteManyOptions]) (count int64, err error) { if len(filter) == 0 { err = gerror.New("缺少查询条件") return } - user, err := GetTenantInfo(ctx) + user, err := getTenantInfo(ctx) if err != nil { return } @@ -462,13 +467,13 @@ func Delete(ctx context.Context, filter bson.M, collection string, opts ...optio } // Update 修改记录 -func Update(ctx context.Context, filter bson.M, update bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { +func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { if len(filter) == 0 { err = gerror.New("缺少查询条件") return } filter["isDeleted"] = false - user, err := GetTenantInfo(ctx) + user, err := getTenantInfo(ctx) if err != nil { return } @@ -490,7 +495,7 @@ func Update(ctx context.Context, filter bson.M, update bson.M, collection string } // RandomSoftDelete 随机软删除个文档的 _id -func RandomSoftDelete(ctx context.Context, limit int, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { +func (m *MongoDB) RandomSoftDelete(ctx context.Context, limit int, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { // 步骤 1: 使用聚合管道的 $sample 操作符随机抽取5个文档的 _id pipeline := mongo.Pipeline{ // 阶段1: 为每个文档添加一个 0-1 之间的随机数字段 'random' @@ -539,7 +544,7 @@ func RandomSoftDelete(ctx context.Context, limit int, collection string, opts .. } // SaveOrUpdate 批量增加或修改 -func SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.BulkWriteResult, err error) { +func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.BulkWriteResult, err error) { if len(filter) == 0 || len(update) == 0 { err = gerror.New("缺少查询条件或更新数据") return @@ -548,7 +553,7 @@ func SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collect err = gerror.New("查询条件和更新数据的数量必须一致") return } - user, err := GetTenantInfo(ctx) + user, err := getTenantInfo(ctx) if err != nil { return } @@ -612,8 +617,8 @@ func SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collect } // Insert 插入多条记录 -func Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { - user, err := GetTenantInfo(ctx) +func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { + user, err := getTenantInfo(ctx) if err != nil { return } @@ -645,16 +650,15 @@ func Insert(ctx context.Context, documents []interface{}, collection string, opt } // Count 查询总数 -func Count(ctx context.Context, NoCache bool, filter bson.M, collection string) (count int64, err error) { - user, err := GetTenantInfo(ctx) +func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { + user, err := getTenantInfo(ctx) if err != nil { return } filter["isDeleted"] = false filterMap := utils.OrderMap(filter) - redisKey := "" - if !NoCache { - redisKey = fmt.Sprintf(consts.Count, user.TenantId, collection, gconv.String(filterMap)) + redisKey := fmt.Sprintf(consts.Count, user.TenantId, collection, gconv.String(filterMap)) + if m.Cache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) if err != nil { @@ -667,7 +671,7 @@ func Count(ctx context.Context, NoCache bool, filter bson.M, collection string) } // 调用驱动的 CountDocuments,在数据库端执行的 count, err = db.Collection(collection).CountDocuments(ctx, filter) - if !NoCache { + if m.Cache { err = redis.RedisClient.SetEX(ctx, redisKey, count, int64(time.Hour)) if err != nil { return From 7d0538710468a4b777d2d8cb2a1ac60503de5a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 30 Dec 2025 09:53:06 +0800 Subject: [PATCH 107/225] =?UTF-8?q?mongo.go=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 4c3855f..d0bff5f 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -29,9 +29,13 @@ type MongoDB struct { Cache bool } -func DB(cache bool) *MongoDB { +func DB(cache ...bool) *MongoDB { + b := true + if len(cache) > 0 { + b = cache[0] + } return &MongoDB{ - Cache: cache, + Cache: b, } } From f06e050d78484183d6e5d245918aec9a8c19ecc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 30 Dec 2025 10:52:12 +0800 Subject: [PATCH 108/225] =?UTF-8?q?mongo.go=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 86 ++++++++------------------------------------------ utils/utils.go | 64 +++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 102 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index d0bff5f..101f2d1 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -11,7 +11,6 @@ import ( "github.com/gogf/gf/v2/container/gvar" "gitee.com/red-future---jilin-g/common/consts" - "gitee.com/red-future---jilin-g/common/do" "gitee.com/red-future---jilin-g/common/redis" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/errors/gerror" @@ -99,6 +98,13 @@ func connect() error { return nil } +// GetDB 获取 MongoDB 数据库实例 +func GetDB() *mongo.Database { + mu.RLock() + defer mu.RUnlock() + return db +} + // healthCheck 健康检查协程 func healthCheck() { ticker := time.NewTicker(30 * time.Second) @@ -261,76 +267,12 @@ func oneOptionsToMap(ctx context.Context, opts ...options.Lister[options.FindOne return } -// getTenantInfo 获取租户信息 -// 优先从 token 获取,失败则从请求参数 customerServiceId 查询 customer_service_account 表 -func getTenantInfo(ctx context.Context) (user do.User, err error) { - // 1. 优先从 token 获取 - user, err = utils.GetUserInfo(ctx) - if err == nil { - return - } - - // 2. token 获取失败,尝试从请求参数或context获取 accountName - var accountName string - - // 2.1 尝试从request获取(HTTP请求场景) - req := g.RequestFromCtx(ctx) - if req != nil { - accountName = req.Get("accountName").String() - if accountName == "" { - accountName = req.Get("account_name").String() - } - // 兼容旧参数名 - if accountName == "" { - accountName = req.Get("customerServiceId").String() - } - if accountName == "" { - accountName = req.Get("customer_service_id").String() - } - } - - // 2.2 request不存在或未获取到,尝试从context.Value获取(WebSocket场景) - if accountName == "" { - if val := ctx.Value("accountName"); val != nil { - if str, ok := val.(string); ok { - accountName = str - } - } - // 兼容旧参数名 - if accountName == "" { - if val := ctx.Value("customerServiceId"); val != nil { - if str, ok := val.(string); ok { - accountName = str - } - } - } - } - - if accountName == "" { - return user, gerror.New("无法获取租户信息:无 token 且无 accountName 参数") - } - - // 3. 直接查询 customer_service_account 表获取 tenantId - filter := bson.M{"accountName": accountName, "isDeleted": false} - var account struct { - TenantId interface{} `bson:"tenantId"` - } - if findErr := db.Collection("customer_service_account").FindOne(ctx, filter).Decode(&account); findErr != nil { - return user, gerror.Newf("通过 accountName 查询租户失败: %v", findErr) - } - - user.TenantId = account.TenantId - user.UserName = accountName - err = nil // 清空之前从token获取时的错误 - return -} - // Find 查询多条记录 func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOptions]) (err error) { if err = utils.ValidStructPtr(result); err != nil { return } - user, err := getTenantInfo(ctx) + user, err := utils.GetUserInfo(ctx) if err != nil { return } @@ -379,7 +321,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} if err = utils.ValidStructPtr(result); err != nil { return } - user, err := getTenantInfo(ctx) + user, err := utils.GetUserInfo(ctx) if err != nil { return } @@ -456,7 +398,7 @@ func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, err = gerror.New("缺少查询条件") return } - user, err := getTenantInfo(ctx) + user, err := utils.GetUserInfo(ctx) if err != nil { return } @@ -477,7 +419,7 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll return } filter["isDeleted"] = false - user, err := getTenantInfo(ctx) + user, err := utils.GetUserInfo(ctx) if err != nil { return } @@ -557,7 +499,7 @@ func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bs err = gerror.New("查询条件和更新数据的数量必须一致") return } - user, err := getTenantInfo(ctx) + user, err := utils.GetUserInfo(ctx) if err != nil { return } @@ -622,7 +564,7 @@ func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bs // Insert 插入多条记录 func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { - user, err := getTenantInfo(ctx) + user, err := utils.GetUserInfo(ctx) if err != nil { return } @@ -655,7 +597,7 @@ func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collectio // Count 查询总数 func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { - user, err := getTenantInfo(ctx) + user, err := utils.GetUserInfo(ctx) if err != nil { return } diff --git a/utils/utils.go b/utils/utils.go index c4b8393..b8ca4ca 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -60,40 +60,44 @@ func GetMonthToday(t time.Time, month int) time.Time { return target.AddDate(0, 0, t.Day()-1) } func GetUserInfo(ctx context.Context) (user do.User, err error) { - redisAddr := g.Cfg().MustGet(ctx, "redis.default.address").String() - gft := gftoken.NewGfToken( - gftoken.WithCacheKey("gfToken:"), - gftoken.WithTimeout(20), - gftoken.WithMaxRefresh(10), - gftoken.WithMultiLogin(true), - //gftoken.WithExcludePaths(g.SliceStr{"/excludeDemo"}), - gftoken.WithGRedisConfig(&gredis.Config{ - Address: redisAddr, - Db: 1, - })) r := g.RequestFromCtx(ctx) - if r == nil { - return - } - // 解析 token - data, err := gft.ParseToken(g.RequestFromCtx(ctx)) - if err != nil { - return user, gerror.Wrap(err, "token 解析失败") - } + if r != nil { + redisAddr := g.Cfg().MustGet(ctx, "redis.default.address").String() + gft := gftoken.NewGfToken( + gftoken.WithCacheKey("gfToken:"), + gftoken.WithTimeout(20), + gftoken.WithMaxRefresh(10), + gftoken.WithMultiLogin(true), + //gftoken.WithExcludePaths(g.SliceStr{"/excludeDemo"}), + gftoken.WithGRedisConfig(&gredis.Config{ + Address: redisAddr, + Db: 1, + })) + // 解析 token + data, err := gft.ParseToken(g.RequestFromCtx(ctx)) + if err != nil { + return user, gerror.Wrap(err, "token 解析失败") + } - // 检查 data 是否为 nil - if data == nil { - return user, gerror.New("token 数据为空") - } + // 检查 data 是否为 nil + if data == nil { + return user, gerror.New("token 数据为空") + } - // 检查 data.Data 是否为 nil - if data.Data == nil { - return user, gerror.New("用户信息为空") - } + // 检查 data.Data 是否为 nil + if data.Data == nil { + return user, gerror.New("用户信息为空") + } - dataMap := gconv.Map(data.Data) - user.UserName = dataMap["userName"] - user.TenantId = dataMap["tenantId"] + dataMap := gconv.Map(data.Data) + user.UserName = dataMap["userName"] + user.TenantId = dataMap["tenantId"] + } else { + user.TenantId = ctx.Value("tenantId") + } + if user.TenantId == nil { + return user, gerror.New("租户信息为空") + } return } func OrderMap(m map[string]interface{}) map[string]interface{} { From 0f235ad52d933914d063b94a9081cf33a73ce741 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 30 Dec 2025 13:46:02 +0800 Subject: [PATCH 109/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0mq=E7=8B=AC=E7=AB=8Bc?= =?UTF-8?q?hannel=E9=81=BF=E5=85=8D=E5=B9=B6=E5=8F=91=E4=BA=89=E5=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rabbitmq/consumer.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/rabbitmq/consumer.go b/rabbitmq/consumer.go index 305b3f7..160d4bd 100644 --- a/rabbitmq/consumer.go +++ b/rabbitmq/consumer.go @@ -21,6 +21,7 @@ type Consumer struct { handler MessageHandler workerCount int // worker 数量 cancel context.CancelFunc // 用于停止 worker + channel *amqp.Channel // 独立Channel(避免并发冲突) } // ConsumerOption 消费者配置选项 @@ -78,10 +79,17 @@ func (c *Consumer) Start(ctx context.Context) (err error) { // 创建可取消的 context workerCtx, cancel := context.WithCancel(ctx) c.cancel = cancel - ch, err := GetChannel() + + // 为每个消费者创建独立Channel(避免并发冲突) + conn, err := GetConnection() if err != nil { - return err + return gerror.Wrap(err, "获取RabbitMQ连接失败") } + c.channel, err = conn.Channel() + if err != nil { + return gerror.Wrap(err, "创建独立Channel失败") + } + ch := c.channel // 声明队列(如果不存在则创建) // 注意:Queue到Exchange的绑定应由message服务在发送响应时动态创建,或通过运维工具提前配置 @@ -196,8 +204,13 @@ func StartTypedConsumer[T any]( // Stop 停止消费者 func (c *Consumer) Stop(ctx context.Context) { if c.cancel != nil { - g.Log().Infof(ctx, "正在停止消费者: queue=%s", c.queue) c.cancel() - c.cancel = nil } + // 关闭独立Channel + if c.channel != nil && !c.channel.IsClosed() { + c.channel.Close() + g.Log().Debugf(ctx, "消费者Channel已关闭: queue=%s", c.queue) + } + g.Log().Infof(ctx, "正在停止消费者: queue=%s", c.queue) + c.cancel = nil } From b30be00e8baef1309094ff16238106301cba7785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 30 Dec 2025 12:22:43 +0800 Subject: [PATCH 110/225] =?UTF-8?q?mongo.go=E9=87=8D=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/配置中心方案.md | 268 ----------------------------------------- 1 file changed, 268 deletions(-) delete mode 100644 consul/配置中心方案.md diff --git a/consul/配置中心方案.md b/consul/配置中心方案.md deleted file mode 100644 index 797279b..0000000 --- a/consul/配置中心方案.md +++ /dev/null @@ -1,268 +0,0 @@ -# Consul KV 配置中心方案 - -**文档版本**:v1.0 -**创建日期**:2025-12-16 -**负责项目**:customerservice + message + gateway - ---- - -## 一、核心目标 - -| 目标 | 说明 | -|------|------| -| **安全性** | 敏感配置(连接地址/密码)不写在代码里,从 Consul KV 读取 | -| **回退机制** | Consul 不可用时,使用本地默认配置保证服务可启动 | -| **热更新** | Consul KV 变更后,服务无需重启立即生效 | -| **性能最优** | 请求路径只读内存(原子快照),不访问 Consul;单协程阻塞监听,不空转 | - ---- - -## 二、配置分类与热更新策略 - -### 2.1 配置分类 - -| 分类 | 配置项 | 热更新方式 | -|------|--------|------------| -| **连接池配置** | http 连接池数量、redis 连接池数量、mongo 连接池数量 | 平滑切换(创建新池→原子替换→延迟关闭旧池) | -| **协程池配置** | grpool worker 数量 | 动态调整(`grpool.SetSize()`) | -| **业务参数** | 追问延时、归档延时、批量大小、超时时间、卡片触发轮数 | 直接原子更新内存变量 | -| **开关/限流** | 功能开关、限流阈值、日志级别 | 直接原子更新内存变量 | -| **中间件地址** | Redis/MQ/Mongo/ES/Jaeger 地址 | 平滑切换连接池 | - -### 2.2 热更新策略矩阵 - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ 热更新策略矩阵 │ -├─────────────────────────────────────────────────────────────────┤ -│ 配置变更类型 │ 更新方式 │ 对请求影响 │ -├─────────────────────────────────────────────────────────────────┤ -│ 业务参数/开关 │ atomic.Value │ 无影响,立即生效 │ -│ 协程池数量 │ grpool.SetSize │ 无影响,渐进生效 │ -│ 连接池数量/地址 │ 平滑切换单例 │ 无影响,新请求用新池 │ -└─────────────────────────────────────────────────────────────────┘ -``` - ---- - -## 三、Consul KV 键名规范 - -``` -config/{service}/pool -config/{service}/business -config/{service}/middleware -``` - -### 3.1 示例 KV 结构 - -**Key**: `config/customerservice/pool` -```yaml -http: - maxConns: 100 # HTTP 连接池最大连接数 - maxIdleConns: 20 # HTTP 连接池最大空闲连接 -redis: - maxActive: 50 # Redis 连接池最大活跃连接 - maxIdle: 10 # Redis 连接池最大空闲连接 - idleTimeout: 300 # 空闲超时(秒) -grpool: - workerSize: 200 # 协程池 worker 数量 -``` - -**Key**: `config/customerservice/business` -```yaml -followUp: - delay1: 30 # 第一次追问延时(秒) - delay2: 60 # 第二次追问延时(秒) - delay3: 180 # 第三次追问延时(秒) -archive: - delay: 3600 # 归档延时(秒) -stream: - batchSize: 200 # 批量读取消息数量 - blockTimeout: 2000 # 阻塞超时(毫秒) -card: - triggerCount: 5 # 触发发送卡片的对话轮数 -``` - -**Key**: `config/customerservice/middleware` -```yaml -redis: - address: "192.168.3.200:6379" - password: "" - db: 0 -rabbitmq: - host: "192.168.3.200" - port: 5672 - username: "guest" - password: "guest" - vhost: "/" -mongo: - uri: "mongodb://192.168.3.200:27017" - database: "customer_service" -``` - ---- - -## 四、架构设计 - -### 4.1 模块结构 - -consul api 里有个watch , 用watch去做 -``` -common/configcenter/ -├── configcenter.go # 核心:启动/停止/获取配置 -├── watcher.go # Consul KV 阻塞监听(单协程) -├── snapshot.go # 配置快照(atomic.Value) -├── hook.go # 变更回调分发 -└── types.go # 配置结构体定义 -``` - -### 4.2 核心接口 - -```go -// 启动配置中心(每个服务 main.go 调用一次) -func Start(ctx context.Context, opts Options) error - -// 获取当前配置快照(请求路径调用,O(1) 无锁) -func Get() *Snapshot - -// 注册变更回调(组件注册自己的重初始化逻辑) -func Subscribe(name string, handler func(old, new *Snapshot)) - -// 优雅停止 -func Stop() -``` - -### 4.3 数据流 - -``` -┌──────────────┐ 阻塞查询 ┌──────────────┐ -│ Consul KV │ ─────────────→ │ Watcher │ (单协程,不空转) -└──────────────┘ (长轮询) └──────┬───────┘ - │ 配置变更 - ▼ - ┌──────────────┐ - │ Snapshot │ (atomic.Value 原子替换) - └──────┬───────┘ - │ - ┌───────────────────────┼───────────────────────┐ - ▼ ▼ ▼ - ┌────────────────┐ ┌────────────────┐ ┌────────────────┐ - │ 业务代码读取 │ │ Hook 回调 │ │ 日志记录 │ - │ Get().Pool │ │ 重建连接池 │ │ 版本号变更 │ - └────────────────┘ └────────────────┘ └────────────────┘ -``` - ---- - -## 五、服务启动流程 - -```go -func main() { - ctx := context.Background() - - // 1. 启动配置中心(优先从 Consul 拉取,失败则用本地默认) - if err := configcenter.Start(ctx, configcenter.Options{ - ConsulAddr: "192.168.3.200:8500", - ServiceName: "customerservice", - DefaultPath: "config.yml", // 回退配置 - }); err != nil { - glog.Warningf(ctx, "配置中心启动失败,使用本地配置: %v", err) - } - - // 2. 注册组件热更新回调 - configcenter.Subscribe("redis", redis.OnConfigChange) - configcenter.Subscribe("grpool", ragflow.OnPoolSizeChange) - configcenter.Subscribe("rabbitmq", rabbitmq.OnConfigChange) - - // 3. 初始化组件(使用 configcenter.Get() 获取配置) - initComponents(ctx) - - // 4. 启动服务... - - // 5. 优雅退出 - defer configcenter.Stop() -} -``` - ---- - -## 六、性能保证 - -| 环节 | 性能策略 | -|------|----------| -| **请求路径** | 只读 `atomic.Value`,O(1),无锁,无网络 | -| **监听 Consul** | 阻塞查询(Blocking Query),无变更时不消耗 CPU | -| **配置解析** | 只在变更时解析一次 YAML → 结构体 | -| **连接池切换** | 原子替换指针,旧池延迟 30s 关闭(等待请求完成) | -| **协程池调整** | `grpool` 内置支持动态调整,无需重建 | - ---- - -## 七、回退与容错 - -| 场景 | 处理方式 | -|------|----------| -| **启动时 Consul 不可用** | 使用本地 `config.yml` 默认配置,服务正常启动 | -| **运行时 Consul 断连** | 保持最后一次成功配置,自动重试连接 | -| **配置格式错误** | 拒绝更新,保持旧配置,记录错误日志 | -| **连接池切换失败** | 保持旧连接池,记录错误,不影响服务 | - ---- - -## 八、Consul KV 操作示例 - -### 8.1 通过 Consul UI 操作 - -1. 访问 `http://192.168.3.200:8500/ui` -2. 点击 **Key/Value** 菜单 -3. 创建/编辑 Key,如 `config/customerservice/business` -4. 在 Value 中粘贴 YAML 配置内容 -5. 点击 **Save** - -### 8.2 通过 CLI 操作 - -```bash -# 写入配置 -consul kv put config/customerservice/business @business.yml - -# 读取配置 -consul kv get config/customerservice/business - -# 删除配置 -consul kv delete config/customerservice/business -``` - -### 8.3 通过 HTTP API 操作 - -```bash -# 写入配置 -curl --request PUT \ - --url http://192.168.3.200:8500/v1/kv/config/customerservice/business \ - --data-binary @business.yml - -# 读取配置(Base64 编码) -curl http://192.168.3.200:8500/v1/kv/config/customerservice/business - -# 阻塞查询(长轮询,等待变更) -curl "http://192.168.3.200:8500/v1/kv/config/customerservice/business?index=123&wait=5m" -``` - ---- - -## 九、实现进度 - -| 阶段 | 状态 | 说明 | -|------|------|------| -| 方案设计 | ✅ 完成 | 本文档 | -| common/configcenter 模块 | ⏳ 待实现 | 核心配置中心模块 | -| 各组件热更新回调 | ⏳ 待实现 | Redis/MQ/Mongo/ES 等 | -| 联调验证 | ⏳ 待实现 | KV 修改后无需重启生效 | - ---- - -## 十、注意事项 - -1. **安全性**:生产环境的 Consul 必须启用 ACL,限制 KV 访问权限 -2. **版本控制**:重要配置变更前,建议先备份旧配置 -3. **灰度发布**:可通过 `config/{env}/{service}` 实现多环境配置隔离 -4. **监控告警**:建议对配置变更事件添加日志和告警 From 342a7b00cba686352c2027d6c41803d2cfb13b0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 30 Dec 2025 16:14:15 +0800 Subject: [PATCH 111/225] =?UTF-8?q?common=E5=A2=9E=E5=8A=A0=E9=80=9A?= =?UTF-8?q?=E7=94=A8log=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- beans/beans.go | 9 ++ log/consts/log_const.go | 15 ++++ log/controller/log_controller.go | 76 +++++++++++++++++ log/dao/log_dao.go | 142 +++++++++++++++++++++++++++++++ log/model/dto/log_dto.go | 96 +++++++++++++++++++++ log/model/entity/log.go | 24 ++++++ log/service/log_service.go | 121 ++++++++++++++++++++++++++ 7 files changed, 483 insertions(+) create mode 100644 beans/beans.go create mode 100644 log/consts/log_const.go create mode 100644 log/controller/log_controller.go create mode 100644 log/dao/log_dao.go create mode 100644 log/model/dto/log_dto.go create mode 100644 log/model/entity/log.go create mode 100644 log/service/log_service.go diff --git a/beans/beans.go b/beans/beans.go new file mode 100644 index 0000000..5254575 --- /dev/null +++ b/beans/beans.go @@ -0,0 +1,9 @@ +package beans + +type ResponseEmpty struct { +} +type Page struct { + PageNum int `p:"pageNum"` //当前页码 + PageSize int `p:"pageSize"` //每页数 + Total int //总页数 +} diff --git a/log/consts/log_const.go b/log/consts/log_const.go new file mode 100644 index 0000000..ca33ca3 --- /dev/null +++ b/log/consts/log_const.go @@ -0,0 +1,15 @@ +package consts + +// OperationType 操作类型常量 +type OperationType string + +const ( + OperationCreate OperationType = "create" // 创建 + OperationUpdate OperationType = "update" // 更新 + OperationDelete OperationType = "delete" // 删除 +) + +// OperationLogCollection 操作日志集合名称常量 +const ( + OperationLogCollection = "operation_logs" // 操作日志集合名称 +) diff --git a/log/controller/log_controller.go b/log/controller/log_controller.go new file mode 100644 index 0000000..e9cb10c --- /dev/null +++ b/log/controller/log_controller.go @@ -0,0 +1,76 @@ +package controller + +import ( + "context" + "strings" + + "gitee.com/red-future---jilin-g/common/beans" + "gitee.com/red-future---jilin-g/common/log/model/dto" + "gitee.com/red-future---jilin-g/common/log/service" +) + +type operationLog struct{} + +// OperationLog 操作日志控制器 +var OperationLog = new(operationLog) + +// GetByID 根据ID获取操作日志 +// @Summary 获取操作日志详情 +// @Description 根据日志ID获取操作日志的详细信息 +func (c *operationLog) GetByID(ctx context.Context, req *dto.GetLogReq) (res *dto.GetLogResp, err error) { + logInfo, err := service.OperationLog.GetByID(ctx, req.ID) + if err != nil { + return + } + + res = &dto.GetLogResp{ + OperationLogInfo: *logInfo, + } + return +} + +// List 查询操作日志列表(通用方法,支持根据不同条件动态查询) +// @Summary 查询操作日志列表 +// @Description 根据多个条件查询操作日志列表 +func (c *operationLog) List(ctx context.Context, req *dto.ListLogsReq) (res *dto.ListLogsResp, err error) { + // 处理排序字段 + var sortFields []string + if req.SortFields != "" { + sortFields = strings.Split(req.SortFields, ",") + } + + logs, total, err := service.OperationLog.List(ctx, req, sortFields...) + if err != nil { + return + } + + res = &dto.ListLogsResp{ + Logs: logs, + Total: total, + } + return +} + +// RecordCreate 记录创建操作日志 +// @Summary 记录创建操作日志 +// @Description 记录数据创建操作的行为日志 +func (c *operationLog) RecordCreate(ctx context.Context, req *dto.RecordCreateLogReq) (res *beans.ResponseEmpty, err error) { + err = service.OperationLog.RecordCreate(ctx, req.Module, req.Service, req.Resource, req.ResourceID, req.Description, req.AfterData) + return +} + +// RecordUpdate 记录更新操作日志 +// @Summary 记录更新操作日志 +// @Description 记录数据更新操作的行为日志 +func (c *operationLog) RecordUpdate(ctx context.Context, req *dto.RecordUpdateLogReq) (res *beans.ResponseEmpty, err error) { + err = service.OperationLog.RecordUpdate(ctx, req.Module, req.Service, req.Resource, req.ResourceID, req.Description, req.BeforeData, req.AfterData) + return +} + +// RecordDelete 记录删除操作日志 +// @Summary 记录删除操作日志 +// @Description 记录数据删除操作的行为日志 +func (c *operationLog) RecordDelete(ctx context.Context, req *dto.RecordDeleteLogReq) (res *beans.ResponseEmpty, err error) { + err = service.OperationLog.RecordDelete(ctx, req.Module, req.Service, req.Resource, req.ResourceID, req.Description, req.BeforeData) + return +} diff --git a/log/dao/log_dao.go b/log/dao/log_dao.go new file mode 100644 index 0000000..4b75dec --- /dev/null +++ b/log/dao/log_dao.go @@ -0,0 +1,142 @@ +package dao + +import ( + "context" + "time" + + "gitee.com/red-future---jilin-g/common/log/consts" + "gitee.com/red-future---jilin-g/common/log/model/dto" + "gitee.com/red-future---jilin-g/common/log/model/entity" + "gitee.com/red-future---jilin-g/common/mongo" + "go.mongodb.org/mongo-driver/v2/bson" + "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +type log struct{} + +// Log 日志数据访问对象 +var Log = &log{} + +// Create 创建日志记录 +func (d *log) Create(ctx context.Context, log *entity.OperationLog) error { + _, err := mongo.DB().Insert(ctx, []interface{}{log}, consts.OperationLogCollection) + return err +} + +// CreateBatch 批量创建日志记录 +func (d *log) CreateBatch(ctx context.Context, logs []*entity.OperationLog) error { + if len(logs) == 0 { + return nil + } + + documents := make([]interface{}, len(logs)) + for i, log := range logs { + documents[i] = log + } + + _, err := mongo.DB().Insert(ctx, documents, consts.OperationLogCollection) + return err +} + +// GetByID 根据ID获取日志 +func (d *log) GetByID(ctx context.Context, id string) (*entity.OperationLog, error) { + objectID, err := bson.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + + filter := bson.M{"_id": objectID} + var log entity.OperationLog + err = mongo.DB().FindOne(ctx, filter, &log, consts.OperationLogCollection) + if err != nil { + return nil, err + } + return &log, nil +} + +// List 查询日志列表(通用方法,通过filter动态拼接查询条件) +func (d *log) List(ctx context.Context, filter *dto.ListLogsReq, sortFields ...string) ([]*entity.OperationLog, int64, error) { + bsonFilter := buildFilter(filter) + + total, err := mongo.DB().Count(ctx, bsonFilter, consts.OperationLogCollection) + if err != nil { + return nil, 0, err + } + + var findOptions []options.Lister[options.FindOptions] + if filter.PageNum > 0 && filter.PageSize > 0 { + findOptions = append(findOptions, options.Find().SetSkip(int64((filter.PageNum-1)*filter.PageSize)).SetLimit(int64(filter.PageSize))) + } + + if len(sortFields) > 0 { + sort := bson.D{} + for _, field := range sortFields { + var order int + if len(field) > 0 && field[0] == '-' { + order = -1 + field = field[1:] + } else { + order = 1 + } + sort = append(sort, bson.E{Key: field, Value: order}) + } + findOptions = append(findOptions, options.Find().SetSort(sort)) + } else { + findOptions = append(findOptions, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) + } + + var logs []*entity.OperationLog + err = mongo.DB().Find(ctx, bsonFilter, &logs, consts.OperationLogCollection, findOptions...) + if err != nil { + return nil, 0, err + } + + return logs, total, nil +} + +// buildFilter 构建MongoDB查询过滤器 +func buildFilter(filter interface{}) bson.M { + bsonFilter := make(bson.M) + + // 从ListLogsReq结构体中提取字段值 + if req, ok := filter.(*dto.ListLogsReq); ok { + if req.Module != "" { + bsonFilter["module"] = req.Module + } + if req.Service != "" { + bsonFilter["service"] = req.Service + } + if req.Operation != "" { + bsonFilter["operation"] = req.Operation + } + if req.Resource != "" { + bsonFilter["resource"] = req.Resource + } + if req.ResourceID != "" { + bsonFilter["resource_id"] = req.ResourceID + } + if req.UserID != "" { + bsonFilter["user_id"] = req.UserID + } + + // 处理时间范围字段 + if req.StartTime != "" || req.EndTime != "" { + timeFilter := bson.M{} + if req.StartTime != "" { + if startTime, err := time.Parse("2006-01-02 15:04:05", req.StartTime); err == nil { + timeFilter["$gte"] = startTime + } + } + if req.EndTime != "" { + if endTime, err := time.Parse("2006-01-02 15:04:05", req.EndTime); err == nil { + timeFilter["$lte"] = endTime + } + } + if len(timeFilter) > 0 { + bsonFilter["createdAt"] = timeFilter + } + } + } + + return bsonFilter +} diff --git a/log/model/dto/log_dto.go b/log/model/dto/log_dto.go new file mode 100644 index 0000000..148e141 --- /dev/null +++ b/log/model/dto/log_dto.go @@ -0,0 +1,96 @@ +package dto + +import ( + "gitee.com/red-future---jilin-g/common/beans" + "github.com/gogf/gf/v2/frame/g" +) + +// ========== 操作日志查询相关DTO ========== + +// GetLogReq 获取操作日志请求 +type GetLogReq struct { + g.Meta `path:"/getLog" method:"get" tags:"操作日志" summary:"获取操作日志详情" dc:"根据日志ID获取操作日志的详细信息"` + ID string `json:"id" v:"required" dc:"日志ID"` +} + +// GetLogResp 获取操作日志响应 +type GetLogResp struct { + OperationLogInfo +} + +// OperationLogInfo 操作日志信息 +type OperationLogInfo struct { + ID string `json:"id" dc:"日志ID"` + Module string `json:"module" dc:"模块名"` + Service string `json:"service" dc:"服务名"` + Operation string `json:"operation" dc:"操作类型"` + Resource string `json:"resource" dc:"资源类型"` + ResourceID string `json:"resource_id" dc:"资源ID"` + UserID interface{} `json:"user_id" dc:"操作人ID"` + UserName string `json:"user_name" dc:"操作人名称"` + IPAddress string `json:"ip_address" dc:"操作IP地址"` + UserAgent string `json:"user_agent" dc:"用户代理"` + Description string `json:"description" dc:"操作描述"` + BeforeData map[string]interface{} `json:"before_data" dc:"操作前的数据"` + AfterData map[string]interface{} `json:"after_data" dc:"操作后的数据"` + ExtraData map[string]interface{} `json:"extra_data" dc:"额外数据"` + CreatedAt string `json:"created_at" dc:"创建时间"` + UpdatedAt string `json:"updated_at" dc:"更新时间"` +} + +// ListLogsReq 查询操作日志列表请求(通用方法,支持根据不同条件动态查询) +type ListLogsReq struct { + g.Meta `path:"/listLogs" method:"get" tags:"操作日志" summary:"查询操作日志列表" dc:"根据多个条件查询操作日志列表"` + beans.Page + Module string `json:"module" dc:"模块名(可选)"` + Service string `json:"service" dc:"服务名(可选)"` + Operation string `json:"operation" dc:"操作类型(可选)"` + Resource string `json:"resource" dc:"资源类型(可选)"` + ResourceID string `json:"resource_id" dc:"资源ID(可选)"` + UserID string `json:"user_id" dc:"用户ID(可选)"` + StartTime string `json:"start_time" dc:"开始时间(可选)"` + EndTime string `json:"end_time" dc:"结束时间(可选)"` + SortFields string `json:"sort_fields" dc:"排序字段,多个用逗号分隔,如:-createdAt,module(可选)"` +} + +// ListLogsResp 查询操作日志列表响应 +type ListLogsResp struct { + Logs []OperationLogInfo `json:"logs" dc:"日志列表"` + Total int64 `json:"total" dc:"总数"` +} + +// ========== 记录操作日志DTO ========== + +// RecordCreateLogReq 记录创建操作日志请求 +type RecordCreateLogReq struct { + g.Meta `path:"/recordCreateLog" method:"post" tags:"操作日志" summary:"记录创建操作日志" dc:"记录数据创建操作的行为日志"` + Module string `json:"module" v:"required" dc:"模块名"` + Service string `json:"service" v:"required" dc:"服务名"` + Resource string `json:"resource" v:"required" dc:"资源类型"` + ResourceID string `json:"resource_id" v:"required" dc:"资源ID"` + Description string `json:"description" dc:"操作描述"` + AfterData map[string]interface{} `json:"after_data" dc:"操作后的数据"` +} + +// RecordUpdateLogReq 记录更新操作日志请求 +type RecordUpdateLogReq struct { + g.Meta `path:"/recordUpdateLog" method:"post" tags:"操作日志" summary:"记录更新操作日志" dc:"记录数据更新操作的行为日志"` + Module string `json:"module" v:"required" dc:"模块名"` + Service string `json:"service" v:"required" dc:"服务名"` + Resource string `json:"resource" v:"required" dc:"资源类型"` + ResourceID string `json:"resource_id" v:"required" dc:"资源ID"` + Description string `json:"description" dc:"操作描述"` + BeforeData map[string]interface{} `json:"before_data" dc:"操作前的数据"` + AfterData map[string]interface{} `json:"after_data" dc:"操作后的数据"` +} + +// RecordDeleteLogReq 记录删除操作日志请求 +type RecordDeleteLogReq struct { + g.Meta `path:"/recordDeleteLog" method:"post" tags:"操作日志" summary:"记录删除操作日志" dc:"记录数据删除操作的行为日志"` + Module string `json:"module" v:"required" dc:"模块名"` + Service string `json:"service" v:"required" dc:"服务名"` + Resource string `json:"resource" v:"required" dc:"资源类型"` + ResourceID string `json:"resource_id" v:"required" dc:"资源ID"` + Description string `json:"description" dc:"操作描述"` + BeforeData map[string]interface{} `json:"before_data" dc:"操作前的数据"` +} diff --git a/log/model/entity/log.go b/log/model/entity/log.go new file mode 100644 index 0000000..e180a77 --- /dev/null +++ b/log/model/entity/log.go @@ -0,0 +1,24 @@ +package entity + +import ( + "gitee.com/red-future---jilin-g/common/do" +) + +// OperationLog 操作日志实体 - 用于记录数据增删改操作行为 +type OperationLog struct { + do.MongoBaseDO `bson:",inline"` + + Module string `bson:"module" json:"module"` // 模块名:如 order, wallet, market 等 + Service string `bson:"service" json:"service"` // 服务名:具体的微服务名称 + Operation string `bson:"operation" json:"operation"` // 操作类型:create, update, delete + Resource string `bson:"resource" json:"resource"` // 资源类型:如 order, wallet, product 等 + ResourceID string `bson:"resource_id" json:"resource_id"` // 资源ID:具体操作的数据ID,如订单号、钱包ID等 + UserID interface{} `bson:"user_id" json:"user_id"` // 操作人ID + UserName string `bson:"user_name" json:"user_name"` // 操作人名称 + IPAddress string `bson:"ip_address" json:"ip_address"` // 操作IP地址 + UserAgent string `bson:"user_agent" json:"user_agent"` // 用户代理 + Description string `bson:"description" json:"description"` // 操作描述 + BeforeData map[string]interface{} `bson:"before_data,omitempty" json:"before_data"` // 操作前的数据(用于update/delete) + AfterData map[string]interface{} `bson:"after_data,omitempty" json:"after_data"` // 操作后的数据(用于create/update) + ExtraData map[string]interface{} `bson:"extra_data,omitempty" json:"extra_data"` // 额外数据 +} diff --git a/log/service/log_service.go b/log/service/log_service.go new file mode 100644 index 0000000..ba99f15 --- /dev/null +++ b/log/service/log_service.go @@ -0,0 +1,121 @@ +package service + +import ( + "context" + + "gitee.com/red-future---jilin-g/common/log/consts" + "gitee.com/red-future---jilin-g/common/log/dao" + "gitee.com/red-future---jilin-g/common/log/model/dto" + logEntity "gitee.com/red-future---jilin-g/common/log/model/entity" + "gitee.com/red-future---jilin-g/common/utils" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/gtime" + "github.com/gogf/gf/v2/util/gconv" +) + +type operationLog struct{} + +// OperationLog 操作日志服务 +var OperationLog = &operationLog{} + +// RecordCreate 记录创建操作 +func (s *operationLog) RecordCreate(ctx context.Context, module, service, resource, resourceID, description string, afterData map[string]interface{}) error { + return s.record(ctx, module, service, string(consts.OperationCreate), resource, resourceID, description, nil, afterData, nil) +} + +// RecordUpdate 记录更新操作 +func (s *operationLog) RecordUpdate(ctx context.Context, module, service, resource, resourceID, description string, beforeData, afterData map[string]interface{}) error { + return s.record(ctx, module, service, string(consts.OperationUpdate), resource, resourceID, description, beforeData, afterData, nil) +} + +// RecordDelete 记录删除操作 +func (s *operationLog) RecordDelete(ctx context.Context, module, service, resource, resourceID, description string, beforeData map[string]interface{}) error { + return s.record(ctx, module, service, string(consts.OperationDelete), resource, resourceID, description, beforeData, nil, nil) +} + +// BatchRecordCreate 批量记录创建操作 +func (s *operationLog) BatchRecordCreate(ctx context.Context, logs []*logEntity.OperationLog) error { + return dao.Log.CreateBatch(ctx, logs) +} + +// GetByID 根据ID获取操作日志 +func (s *operationLog) GetByID(ctx context.Context, id string) (*dto.OperationLogInfo, error) { + log, err := dao.Log.GetByID(ctx, id) + if err != nil { + return nil, err + } + + var logInfo dto.OperationLogInfo + gconv.Struct(log, &logInfo) + logInfo.ID = log.Id.Hex() + logInfo.CreatedAt = gtime.New(log.CreatedAt).Format("Y-m-d H:i:s") + logInfo.UpdatedAt = gtime.New(log.UpdatedAt).Format("Y-m-d H:i:s") + return &logInfo, nil +} + +// List 查询操作日志列表 +func (s *operationLog) List(ctx context.Context, filter interface{}, sortFields ...string) ([]dto.OperationLogInfo, int64, error) { + logs, total, err := dao.Log.List(ctx, filter, sortFields...) + if err != nil { + return nil, 0, err + } + + var logInfos []dto.OperationLogInfo + err = gconv.Slice(logs, &logInfos) + if err != nil { + return nil, 0, err + } + + // 处理特殊字段 + for i, log := range logs { + logInfos[i].ID = log.Id.Hex() + logInfos[i].CreatedAt = gtime.New(log.CreatedAt).Format("Y-m-d H:i:s") + logInfos[i].UpdatedAt = gtime.New(log.UpdatedAt).Format("Y-m-d H:i:s") + } + + return logInfos, total, nil +} + +// record 记录操作日志的通用方法 +func (s *operationLog) record(ctx context.Context, module, service, operation, resource, resourceID, description string, beforeData, afterData, extraData map[string]interface{}) error { + // 获取用户信息 + user, err := utils.GetUserInfo(ctx) + if err != nil { + return err + } + + // 获取请求信息 + ipAddress, userAgent := getHTTPRequestInfo(ctx) + + var userName string + if user.UserName != nil { + userName = gconv.String(user.UserName) + } + + log := &logEntity.OperationLog{ + Module: module, + Service: service, + Operation: operation, + Resource: resource, + ResourceID: resourceID, + UserID: user.UserName, + UserName: userName, + IPAddress: ipAddress, + UserAgent: userAgent, + Description: description, + BeforeData: beforeData, + AfterData: afterData, + ExtraData: extraData, + } + + return dao.Log.Create(ctx, log) +} + +// getHTTPRequestInfo 从上下文中获取HTTP请求信息 +func getHTTPRequestInfo(ctx context.Context) (ipAddress, userAgent string) { + request := g.RequestFromCtx(ctx) + if request != nil { + return request.GetClientIp(), request.Header.Get("User-Agent") + } + return "", "" +} From 5ded2b08a236e61fe77a9810c721b28a724f2645 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Tue, 30 Dec 2025 16:17:57 +0800 Subject: [PATCH 112/225] =?UTF-8?q?minio=E6=96=87=E4=BB=B6=E4=B8=8A?= =?UTF-8?q?=E4=BC=A0=E5=AD=98=E5=82=A8=E8=B7=AF=E5=BE=84=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- minio/minio.go | 185 ++++++++++++++++++++++++------------------------- 1 file changed, 91 insertions(+), 94 deletions(-) diff --git a/minio/minio.go b/minio/minio.go index 7ffcf96..03df1d4 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -3,8 +3,11 @@ package minio import ( "context" "fmt" + "gitee.com/red-future---jilin-g/common/utils" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/util/gconv" + "net/http" "path/filepath" - "strings" "time" "github.com/gogf/gf/v2/frame/g" @@ -14,134 +17,128 @@ import ( "github.com/minio/minio-go/v7/pkg/credentials" ) -// MinIOConfig 映射 YAML 中的 minio 配置节点 -type MinIOConfig struct { - Endpoint string `yaml:"endpoint"` // MinIO API 地址 - AccessKey string `yaml:"accessKey"` // AK - SecretKey string `yaml:"secretKey"` // SK - Secure bool `yaml:"secure"` // 是否启用 SSL - Region string `yaml:"region"` // 区域 - BucketName string `yaml:"bucketName"` // 默认桶名 - PresignedExpire string `yaml:"presignedExpire"` // 预签名URL过期时间(如 5m、1h) +// IoConfig 映射 YAML 中的 minio 配置节点 +type IoConfig struct { + Endpoint string `yaml:"endpoint"` // MinIO API 地址 + AccessKey string `yaml:"accessKey"` // AK + SecretKey string `yaml:"secretKey"` // SK + Secure bool `yaml:"secure"` // 是否启用 SSL + Region string `yaml:"region"` // 区域 } // 全局 MinIO 客户端(初始化一次,避免重复创建) var minioClient *minio.Client -var minioCfg MinIOConfig +var minioCfg IoConfig -// initMinIO 初始化 MinIO 客户端 -func initMinIO(ctx context.Context) error { - var err error - // 加载 MinIO 配置(可从配置文件/环境变量读取,这里硬编码示例) - minioCfg = MinIOConfig{ - Endpoint: g.Cfg().MustGet(ctx, "minio.endpoint").String(), - AccessKey: g.Cfg().MustGet(ctx, "minio.accessKey").String(), - SecretKey: g.Cfg().MustGet(ctx, "minio.secretKey").String(), - Secure: g.Cfg().MustGet(ctx, "minio.secure").Bool(), - Region: g.Cfg().MustGet(ctx, "minio.region").String(), - BucketName: g.Cfg().MustGet(ctx, "minio.bucketName").String(), // 专门存储图片的桶 +// initMinIO 初始化 MinIO 客户端。 +func init() { + ctx := context.Background() + if !g.Cfg().MustGet(ctx, "minio").IsEmpty() { + // 加载 MinIO 配置(可从配置文件/环境变量读取,这里硬编码示例) + minioCfg = IoConfig{ + Endpoint: g.Cfg().MustGet(ctx, "minio.endpoint").String(), + AccessKey: g.Cfg().MustGet(ctx, "minio.accessKey").String(), + SecretKey: g.Cfg().MustGet(ctx, "minio.secretKey").String(), + Secure: g.Cfg().MustGet(ctx, "minio.secure").Bool(), + Region: g.Cfg().MustGet(ctx, "minio.region").String(), + } + // 创建 MinIO 客户端 + var err error + if minioClient, err = minio.New(minioCfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(minioCfg.AccessKey, minioCfg.SecretKey, ""), + Secure: minioCfg.Secure, + Region: minioCfg.Region, + }); err != nil { + glog.Errorf(ctx, "初始化 MinIO 客户端失败: %v", err) + } } - // 创建 MinIO 客户端 - minioClient, err = minio.New(minioCfg.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(minioCfg.AccessKey, minioCfg.SecretKey, ""), - Secure: minioCfg.Secure, - Region: minioCfg.Region, - }) +} + +func UploadFile(ctx context.Context, fileHeader *ghttp.UploadFile) (imagesUrl string, err error) { + return uploadFile(ctx, getBucketName(ctx), fileHeader) +} + +func uploadFile(ctx context.Context, bucketName string, fileHeader *ghttp.UploadFile) (imagesUrl string, err error) { + // 检查/创建桶 + exists, err := minioClient.BucketExists(ctx, bucketName) if err != nil { - return fmt.Errorf("初始化 MinIO 客户端失败: %w", err) - } - // 检查/创建图片桶 - exists, err := minioClient.BucketExists(ctx, minioCfg.BucketName) - if err != nil { - return fmt.Errorf("检查桶是否存在失败: %w", err) + glog.Errorf(ctx, "检查桶是否存在失败: %v", err) + return } if !exists { - err = minioClient.MakeBucket(ctx, minioCfg.BucketName, minio.MakeBucketOptions{Region: minioCfg.Region}) - if err != nil { - return fmt.Errorf("创建桶失败: %w", err) + if err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: minioCfg.Region}); err != nil { + glog.Errorf(ctx, "创建桶失败: %v", err) + return } - fmt.Printf("成功创建 MinIO 桶: %s\n", minioCfg.BucketName) + glog.Infof(ctx, "成功创建 MinIO 桶: %s", bucketName) } - return nil -} - -func UploadImages(ctx context.Context, fileHeaders []*ghttp.UploadFile) (imagesUrls []string, err error) { - // 初始化 MinIO 客户端 - err = initMinIO(ctx) - if err != nil { - return nil, fmt.Errorf("初始化 MinIO 客户端失败: %w", err) - } - imageUrls := make([]string, 0, len(fileHeaders)) // 存储上传后的图片访问地址 - for _, fileHeader := range fileHeaders { - var imageUrl string - imageUrl, err = uploadImg(ctx, fileHeader) - if err != nil { - return nil, err - } - imageUrls = append(imageUrls, imageUrl) - } - return imageUrls, err -} - -func UploadImage(ctx context.Context, fileHeader *ghttp.UploadFile) (imagesUrl string, err error) { - // 初始化 MinIO 客户端 - err = initMinIO(ctx) - if err != nil { - return "", fmt.Errorf("初始化 MinIO 客户端失败: %w", err) - } - imageUrl, err := uploadImg(ctx, fileHeader) - if err != nil { - return "", err - } - return imageUrl, err -} - -func uploadImg(ctx context.Context, fileHeader *ghttp.UploadFile) (imagesUrl string, err error) { - // 生成唯一的 MinIO 对象名(避免覆盖) - fileExt := filepath.Ext(fileHeader.Filename) // 原文件后缀(如 .jpg) - uniqueID := uuid.New().String()[:8] // 8位随机UUID - timestamp := time.Now().Format("20060102") // 日期目录(便于管理) - objectName := fmt.Sprintf("images/%s/%s%s", timestamp, uniqueID, fileExt) // 存储路径:images/20251209/abc12345.jpg // 打开文件,获取 io.Reader(*os.File 实现了 io.Reader) file, err := fileHeader.Open() - if err != nil { /* 处理错误 */ - return "", fmt.Errorf("打开文件失败: %w", err) + if err != nil { + glog.Errorf(ctx, "打开文件失败: %v", err) + return } defer file.Close() // 必须关闭,避免文件句柄泄露 - // 设置存储桶公共读权限 - policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::` + minioCfg.BucketName + `/*"]}]}` - err = minioClient.SetBucketPolicy(context.Background(), minioCfg.BucketName, policy) + // 获取文件类型 + buffer := make([]byte, 512) + _, err = file.Read(buffer) if err != nil { - return "", fmt.Errorf("设置存储桶权限失败: %w", err) + glog.Errorf(ctx, "读取文件头失败: %v", err) + return + } + contentType := http.DetectContentType(buffer) + // 重置文件读取位置,否则后续 PutObject 会从第512字节开始上传 + if _, err = file.Seek(0, 0); err != nil { + glog.Errorf(ctx, "重置文件读取位置失败: %v", err) + return + } + // 生成唯一的 MinIO 对象名(避免覆盖) + fileExt := filepath.Ext(fileHeader.Filename) // 原文件后缀(如 .jpg) + uniqueID := uuid.New().String()[:32] // 32位随机UUID + timestamp := time.Now().Format("2006-01-02") // 日期目录(便于管理) + objectName := fmt.Sprintf("/%s/%s%s", timestamp, uniqueID, fileExt) // 存储路径:20251209/abc12345.jpg + // 设置存储桶公共读权限 + policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::` + bucketName + `/*"]}]}` + if err = minioClient.SetBucketPolicy(ctx, bucketName, policy); err != nil { + glog.Errorf(ctx, "设置存储桶权限失败: %v", err) + return } // 执行图片上传 _, err = minioClient.PutObject( ctx, - minioCfg.BucketName, + bucketName, objectName, file, fileHeader.Size, minio.PutObjectOptions{ - ContentType: "image/jpeg", // 关键:指定图片MIME类型,S3会根据此类型处理 + ContentType: contentType, // 关键:指定图片MIME类型,S3会根据此类型处理 // 若需要图片可公开访问,添加如下配置(根据需求选择) //ACL: minio.ACLPublicRead, }, ) if err != nil { - return "", fmt.Errorf("上传图片失败: %w", err) + glog.Errorf(ctx, "上传图片失败: %v", err) + return } - replace := strings.Replace(objectName, "images/", "/", 1) - return replace, err + return objectName, err } -// GetImgAddressPrefix 拼接图片前缀地址 -func GetImgAddressPrefix(ctx context.Context) (imageUrl string) { +// GetIFileAddressPrefix 拼接图片前缀地址 +func GetIFileAddressPrefix(ctx context.Context) (imageUrl string) { // 拼接图片前缀地址 - secure := g.Cfg().MustGet(ctx, "minio.secure").Bool() var url = "http://" - if secure { + if minioCfg.Secure { url = "https://" } - imgAddressPrefix := url + g.Cfg().MustGet(ctx, "minio.endpoint").String() + "/" + g.Cfg().MustGet(ctx, "minio.bucketName").String() + "/images" + imgAddressPrefix := url + minioCfg.Endpoint + "/" + getBucketName(ctx) return imgAddressPrefix } + +func getBucketName(ctx context.Context) (bucketName string) { + user, err := utils.GetUserInfo(ctx) + if err != nil { + glog.Errorf(ctx, "获取用户信息失败: %v", err) + return + } + return "tenantid-" + gconv.String(user.TenantId) +} From 8c4918e4edd7435db04b173d4a4942d3192a8201 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 30 Dec 2025 18:18:56 +0800 Subject: [PATCH 113/225] =?UTF-8?q?mongo=E5=BC=80=E5=8F=91=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E7=B1=BB=E5=A2=9E=E5=8A=A0=E5=A2=9E=E5=88=A0=E6=94=B9?= =?UTF-8?q?=E6=93=8D=E4=BD=9C=E6=97=A5=E5=BF=97=E5=86=99=E5=85=A5redis?= =?UTF-8?q?=E6=B6=88=E6=81=AF=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- log/controller/log_controller.go | 25 ----------- log/dao/log_dao.go | 20 +++------ log/model/dto/log_dto.go | 73 ++++++++------------------------ log/model/entity/log.go | 20 +++------ log/service/log_service.go | 55 ++++++++++-------------- mongo/mongo.go | 51 ++++++++++++++++++++++ 6 files changed, 106 insertions(+), 138 deletions(-) diff --git a/log/controller/log_controller.go b/log/controller/log_controller.go index e9cb10c..996293d 100644 --- a/log/controller/log_controller.go +++ b/log/controller/log_controller.go @@ -4,7 +4,6 @@ import ( "context" "strings" - "gitee.com/red-future---jilin-g/common/beans" "gitee.com/red-future---jilin-g/common/log/model/dto" "gitee.com/red-future---jilin-g/common/log/service" ) @@ -50,27 +49,3 @@ func (c *operationLog) List(ctx context.Context, req *dto.ListLogsReq) (res *dto } return } - -// RecordCreate 记录创建操作日志 -// @Summary 记录创建操作日志 -// @Description 记录数据创建操作的行为日志 -func (c *operationLog) RecordCreate(ctx context.Context, req *dto.RecordCreateLogReq) (res *beans.ResponseEmpty, err error) { - err = service.OperationLog.RecordCreate(ctx, req.Module, req.Service, req.Resource, req.ResourceID, req.Description, req.AfterData) - return -} - -// RecordUpdate 记录更新操作日志 -// @Summary 记录更新操作日志 -// @Description 记录数据更新操作的行为日志 -func (c *operationLog) RecordUpdate(ctx context.Context, req *dto.RecordUpdateLogReq) (res *beans.ResponseEmpty, err error) { - err = service.OperationLog.RecordUpdate(ctx, req.Module, req.Service, req.Resource, req.ResourceID, req.Description, req.BeforeData, req.AfterData) - return -} - -// RecordDelete 记录删除操作日志 -// @Summary 记录删除操作日志 -// @Description 记录数据删除操作的行为日志 -func (c *operationLog) RecordDelete(ctx context.Context, req *dto.RecordDeleteLogReq) (res *beans.ResponseEmpty, err error) { - err = service.OperationLog.RecordDelete(ctx, req.Module, req.Service, req.Resource, req.ResourceID, req.Description, req.BeforeData) - return -} diff --git a/log/dao/log_dao.go b/log/dao/log_dao.go index 4b75dec..c459117 100644 --- a/log/dao/log_dao.go +++ b/log/dao/log_dao.go @@ -100,24 +100,18 @@ func buildFilter(filter interface{}) bson.M { // 从ListLogsReq结构体中提取字段值 if req, ok := filter.(*dto.ListLogsReq); ok { - if req.Module != "" { - bsonFilter["module"] = req.Module + if req.ServiceName != "" { + bsonFilter["service_name"] = req.ServiceName } - if req.Service != "" { - bsonFilter["service"] = req.Service + if req.Collection != "" { + bsonFilter["collection"] = req.Collection + } + if req.CollectionID != "" { + bsonFilter["collection_id"] = req.CollectionID } if req.Operation != "" { bsonFilter["operation"] = req.Operation } - if req.Resource != "" { - bsonFilter["resource"] = req.Resource - } - if req.ResourceID != "" { - bsonFilter["resource_id"] = req.ResourceID - } - if req.UserID != "" { - bsonFilter["user_id"] = req.UserID - } // 处理时间范围字段 if req.StartTime != "" || req.EndTime != "" { diff --git a/log/model/dto/log_dto.go b/log/model/dto/log_dto.go index 148e141..81e8b29 100644 --- a/log/model/dto/log_dto.go +++ b/log/model/dto/log_dto.go @@ -20,37 +20,27 @@ type GetLogResp struct { // OperationLogInfo 操作日志信息 type OperationLogInfo struct { - ID string `json:"id" dc:"日志ID"` - Module string `json:"module" dc:"模块名"` - Service string `json:"service" dc:"服务名"` - Operation string `json:"operation" dc:"操作类型"` - Resource string `json:"resource" dc:"资源类型"` - ResourceID string `json:"resource_id" dc:"资源ID"` - UserID interface{} `json:"user_id" dc:"操作人ID"` - UserName string `json:"user_name" dc:"操作人名称"` - IPAddress string `json:"ip_address" dc:"操作IP地址"` - UserAgent string `json:"user_agent" dc:"用户代理"` - Description string `json:"description" dc:"操作描述"` - BeforeData map[string]interface{} `json:"before_data" dc:"操作前的数据"` - AfterData map[string]interface{} `json:"after_data" dc:"操作后的数据"` - ExtraData map[string]interface{} `json:"extra_data" dc:"额外数据"` - CreatedAt string `json:"created_at" dc:"创建时间"` - UpdatedAt string `json:"updated_at" dc:"更新时间"` + ID string `json:"id" dc:"日志ID"` + ServiceName string `json:"service_name" dc:"服务名"` + Collection string `json:"collection" dc:"数据所在集合名称"` + CollectionID string `json:"collection_id" dc:"数据ID"` + Operation string `json:"operation" dc:"操作类型"` + UserName string `json:"user_name" dc:"操作人名称"` + IPAddress string `json:"ip_address" dc:"操作IP地址"` + Data map[string]interface{} `json:"data" dc:"当前数据"` } // ListLogsReq 查询操作日志列表请求(通用方法,支持根据不同条件动态查询) type ListLogsReq struct { g.Meta `path:"/listLogs" method:"get" tags:"操作日志" summary:"查询操作日志列表" dc:"根据多个条件查询操作日志列表"` beans.Page - Module string `json:"module" dc:"模块名(可选)"` - Service string `json:"service" dc:"服务名(可选)"` - Operation string `json:"operation" dc:"操作类型(可选)"` - Resource string `json:"resource" dc:"资源类型(可选)"` - ResourceID string `json:"resource_id" dc:"资源ID(可选)"` - UserID string `json:"user_id" dc:"用户ID(可选)"` - StartTime string `json:"start_time" dc:"开始时间(可选)"` - EndTime string `json:"end_time" dc:"结束时间(可选)"` - SortFields string `json:"sort_fields" dc:"排序字段,多个用逗号分隔,如:-createdAt,module(可选)"` + ServiceName string `json:"service_name" dc:"服务名(可选)"` + Collection string `json:"collection" dc:"数据所在集合名称(可选)"` + CollectionID string `json:"collection_id" dc:"数据ID(可选)"` + Operation string `json:"operation" dc:"操作类型(可选)"` + StartTime string `json:"start_time" dc:"开始时间(可选)"` + EndTime string `json:"end_time" dc:"结束时间(可选)"` + SortFields string `json:"sort_fields" dc:"排序字段,多个用逗号分隔,如:-createdAt"` } // ListLogsResp 查询操作日志列表响应 @@ -63,34 +53,7 @@ type ListLogsResp struct { // RecordCreateLogReq 记录创建操作日志请求 type RecordCreateLogReq struct { - g.Meta `path:"/recordCreateLog" method:"post" tags:"操作日志" summary:"记录创建操作日志" dc:"记录数据创建操作的行为日志"` - Module string `json:"module" v:"required" dc:"模块名"` - Service string `json:"service" v:"required" dc:"服务名"` - Resource string `json:"resource" v:"required" dc:"资源类型"` - ResourceID string `json:"resource_id" v:"required" dc:"资源ID"` - Description string `json:"description" dc:"操作描述"` - AfterData map[string]interface{} `json:"after_data" dc:"操作后的数据"` -} - -// RecordUpdateLogReq 记录更新操作日志请求 -type RecordUpdateLogReq struct { - g.Meta `path:"/recordUpdateLog" method:"post" tags:"操作日志" summary:"记录更新操作日志" dc:"记录数据更新操作的行为日志"` - Module string `json:"module" v:"required" dc:"模块名"` - Service string `json:"service" v:"required" dc:"服务名"` - Resource string `json:"resource" v:"required" dc:"资源类型"` - ResourceID string `json:"resource_id" v:"required" dc:"资源ID"` - Description string `json:"description" dc:"操作描述"` - BeforeData map[string]interface{} `json:"before_data" dc:"操作前的数据"` - AfterData map[string]interface{} `json:"after_data" dc:"操作后的数据"` -} - -// RecordDeleteLogReq 记录删除操作日志请求 -type RecordDeleteLogReq struct { - g.Meta `path:"/recordDeleteLog" method:"post" tags:"操作日志" summary:"记录删除操作日志" dc:"记录数据删除操作的行为日志"` - Module string `json:"module" v:"required" dc:"模块名"` - Service string `json:"service" v:"required" dc:"服务名"` - Resource string `json:"resource" v:"required" dc:"资源类型"` - ResourceID string `json:"resource_id" v:"required" dc:"资源ID"` - Description string `json:"description" dc:"操作描述"` - BeforeData map[string]interface{} `json:"before_data" dc:"操作前的数据"` + ServiceName string `json:"service_name" v:"required" dc:"服务名"` + Collection string `json:"collection" v:"required" dc:"数据所在集合名称"` + Data []interface{} `json:"data" dc:"当前数据"` } diff --git a/log/model/entity/log.go b/log/model/entity/log.go index e180a77..cfa8f28 100644 --- a/log/model/entity/log.go +++ b/log/model/entity/log.go @@ -8,17 +8,11 @@ import ( type OperationLog struct { do.MongoBaseDO `bson:",inline"` - Module string `bson:"module" json:"module"` // 模块名:如 order, wallet, market 等 - Service string `bson:"service" json:"service"` // 服务名:具体的微服务名称 - Operation string `bson:"operation" json:"operation"` // 操作类型:create, update, delete - Resource string `bson:"resource" json:"resource"` // 资源类型:如 order, wallet, product 等 - ResourceID string `bson:"resource_id" json:"resource_id"` // 资源ID:具体操作的数据ID,如订单号、钱包ID等 - UserID interface{} `bson:"user_id" json:"user_id"` // 操作人ID - UserName string `bson:"user_name" json:"user_name"` // 操作人名称 - IPAddress string `bson:"ip_address" json:"ip_address"` // 操作IP地址 - UserAgent string `bson:"user_agent" json:"user_agent"` // 用户代理 - Description string `bson:"description" json:"description"` // 操作描述 - BeforeData map[string]interface{} `bson:"before_data,omitempty" json:"before_data"` // 操作前的数据(用于update/delete) - AfterData map[string]interface{} `bson:"after_data,omitempty" json:"after_data"` // 操作后的数据(用于create/update) - ExtraData map[string]interface{} `bson:"extra_data,omitempty" json:"extra_data"` // 额外数据 + ServiceName string `bson:"service_name" json:"service_name"` // 服务名:具体的微服务名称 + Collection string `bson:"collection" json:"collection"` // 集合名:数据所在的集合名称 + CollectionID string `bson:"collection_id" json:"collection_id"` // 数据ID:具体操作的数据ID,如订单号、钱包ID等 + Operation string `bson:"operation" json:"operation"` // 操作类型:create, update, delete + UserName string `bson:"user_name" json:"user_name"` // 操作人名称 + IPAddress string `bson:"ip_address" json:"ip_address"` // 操作IP地址 + Data map[string]interface{} `bson:"data,omitempty" json:"data"` // 当前数据:操作时的数据状态 } diff --git a/log/service/log_service.go b/log/service/log_service.go index ba99f15..cc9ea4e 100644 --- a/log/service/log_service.go +++ b/log/service/log_service.go @@ -9,7 +9,6 @@ import ( logEntity "gitee.com/red-future---jilin-g/common/log/model/entity" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/util/gconv" ) @@ -19,18 +18,18 @@ type operationLog struct{} var OperationLog = &operationLog{} // RecordCreate 记录创建操作 -func (s *operationLog) RecordCreate(ctx context.Context, module, service, resource, resourceID, description string, afterData map[string]interface{}) error { - return s.record(ctx, module, service, string(consts.OperationCreate), resource, resourceID, description, nil, afterData, nil) +func (s *operationLog) RecordCreate(ctx context.Context, serviceName, collection, collectionID string, data map[string]interface{}) error { + return s.record(ctx, serviceName, collection, collectionID, string(consts.OperationCreate), data) } // RecordUpdate 记录更新操作 -func (s *operationLog) RecordUpdate(ctx context.Context, module, service, resource, resourceID, description string, beforeData, afterData map[string]interface{}) error { - return s.record(ctx, module, service, string(consts.OperationUpdate), resource, resourceID, description, beforeData, afterData, nil) +func (s *operationLog) RecordUpdate(ctx context.Context, serviceName, collection, collectionID string, data map[string]interface{}) error { + return s.record(ctx, serviceName, collection, collectionID, string(consts.OperationUpdate), data) } // RecordDelete 记录删除操作 -func (s *operationLog) RecordDelete(ctx context.Context, module, service, resource, resourceID, description string, beforeData map[string]interface{}) error { - return s.record(ctx, module, service, string(consts.OperationDelete), resource, resourceID, description, beforeData, nil, nil) +func (s *operationLog) RecordDelete(ctx context.Context, serviceName, collection, collectionID string, data map[string]interface{}) error { + return s.record(ctx, serviceName, collection, collectionID, string(consts.OperationDelete), data) } // BatchRecordCreate 批量记录创建操作 @@ -46,22 +45,22 @@ func (s *operationLog) GetByID(ctx context.Context, id string) (*dto.OperationLo } var logInfo dto.OperationLogInfo - gconv.Struct(log, &logInfo) + if err := gconv.Struct(log, &logInfo); err != nil { + return nil, err + } logInfo.ID = log.Id.Hex() - logInfo.CreatedAt = gtime.New(log.CreatedAt).Format("Y-m-d H:i:s") - logInfo.UpdatedAt = gtime.New(log.UpdatedAt).Format("Y-m-d H:i:s") return &logInfo, nil } // List 查询操作日志列表 -func (s *operationLog) List(ctx context.Context, filter interface{}, sortFields ...string) ([]dto.OperationLogInfo, int64, error) { +func (s *operationLog) List(ctx context.Context, filter *dto.ListLogsReq, sortFields ...string) ([]dto.OperationLogInfo, int64, error) { logs, total, err := dao.Log.List(ctx, filter, sortFields...) if err != nil { return nil, 0, err } var logInfos []dto.OperationLogInfo - err = gconv.Slice(logs, &logInfos) + err = gconv.Structs(logs, &logInfos) if err != nil { return nil, 0, err } @@ -69,15 +68,13 @@ func (s *operationLog) List(ctx context.Context, filter interface{}, sortFields // 处理特殊字段 for i, log := range logs { logInfos[i].ID = log.Id.Hex() - logInfos[i].CreatedAt = gtime.New(log.CreatedAt).Format("Y-m-d H:i:s") - logInfos[i].UpdatedAt = gtime.New(log.UpdatedAt).Format("Y-m-d H:i:s") } return logInfos, total, nil } // record 记录操作日志的通用方法 -func (s *operationLog) record(ctx context.Context, module, service, operation, resource, resourceID, description string, beforeData, afterData, extraData map[string]interface{}) error { +func (s *operationLog) record(ctx context.Context, serviceName, collection, collectionID, operation string, data map[string]interface{}) error { // 获取用户信息 user, err := utils.GetUserInfo(ctx) if err != nil { @@ -85,7 +82,7 @@ func (s *operationLog) record(ctx context.Context, module, service, operation, r } // 获取请求信息 - ipAddress, userAgent := getHTTPRequestInfo(ctx) + ipAddress := getHTTPRequestInfo(ctx) var userName string if user.UserName != nil { @@ -93,29 +90,23 @@ func (s *operationLog) record(ctx context.Context, module, service, operation, r } log := &logEntity.OperationLog{ - Module: module, - Service: service, - Operation: operation, - Resource: resource, - ResourceID: resourceID, - UserID: user.UserName, - UserName: userName, - IPAddress: ipAddress, - UserAgent: userAgent, - Description: description, - BeforeData: beforeData, - AfterData: afterData, - ExtraData: extraData, + ServiceName: serviceName, + Collection: collection, + CollectionID: collectionID, + Operation: operation, + UserName: userName, + IPAddress: ipAddress, + Data: data, } return dao.Log.Create(ctx, log) } // getHTTPRequestInfo 从上下文中获取HTTP请求信息 -func getHTTPRequestInfo(ctx context.Context) (ipAddress, userAgent string) { +func getHTTPRequestInfo(ctx context.Context) string { request := g.RequestFromCtx(ctx) if request != nil { - return request.GetClientIp(), request.Header.Get("User-Agent") + return request.GetClientIp() } - return "", "" + return "" } diff --git a/mongo/mongo.go b/mongo/mongo.go index 101f2d1..466021a 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "gitee.com/red-future---jilin-g/common/log/model/dto" "github.com/gogf/gf/v2/container/gvar" "gitee.com/red-future---jilin-g/common/consts" @@ -403,12 +404,25 @@ func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, return } filter["tenantId"] = user.TenantId + var rows []interface{} + if err = m.Find(ctx, filter, &rows, collection); err != nil { + return + } r, err := db.Collection(collection).DeleteMany(ctx, filter, opts...) if err != nil { return } count = r.DeletedCount err = cleanRedis(ctx, filter, user.TenantId, collection) + serverName := g.Cfg().MustGet(ctx, "server.name").String() + logRedisKey := fmt.Sprintf("log:%s", serverName) + if _, err = redis.AddToStream(ctx, logRedisKey, &dto.RecordCreateLogReq{ + ServiceName: serverName, + Collection: collection, + Data: rows, + }); err != nil { + glog.Error(ctx, "mongoLog-AddToStream err: %v", err) + } return } @@ -432,11 +446,24 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll } setDoc["updatedAt"] = gtime.Now().Time update = bson.M{"$set": setDoc} + var rows []interface{} + if err = m.Find(ctx, filter, &rows, collection); err != nil { + return + } result, err = db.Collection(collection).UpdateMany(ctx, filter, update, opts...) if err != nil { return } err = cleanRedis(ctx, filter, user.TenantId, collection) + serverName := g.Cfg().MustGet(ctx, "server.name").String() + logRedisKey := fmt.Sprintf("log:%s", serverName) + if _, err = redis.AddToStream(ctx, logRedisKey, &dto.RecordCreateLogReq{ + ServiceName: serverName, + Collection: collection, + Data: rows, + }); err != nil { + glog.Error(ctx, "mongoLog-AddToStream err: %v", err) + } return } @@ -592,6 +619,30 @@ func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collectio } ids = r.InsertedIDs err = cleanRedis(ctx, bson.M{}, user.TenantId, collection) + //写日志 + serverName := g.Cfg().MustGet(ctx, "server.name").String() + logRedisKey := fmt.Sprintf("log:%s", serverName) + if len(ids) == 0 { + return + } + rows := make([]interface{}, 0, len(ids)) + if len(ids) == 1 { + doc := gconv.Map(documents[0]) + doc["id"] = ids[0] + rows = append(rows, doc) + } else { + filter := bson.M{"_id": bson.M{"$in": ids}} + if err = m.Find(ctx, filter, &rows, collection); err != nil { + return + } + } + if _, err = redis.AddToStream(ctx, logRedisKey, &dto.RecordCreateLogReq{ + ServiceName: serverName, + Collection: collection, + Data: rows, + }); err != nil { + glog.Error(ctx, "mongoLog-AddToStream err: %v", err) + } return } From 636e0ec4f2b0a6622b3552da460c680b46adbcec Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Tue, 30 Dec 2025 18:21:50 +0800 Subject: [PATCH 114/225] =?UTF-8?q?mongo=E4=B8=AD=E6=B8=85=E9=99=A4?= =?UTF-8?q?=E7=BC=93=E5=AD=98CleanRedis=E6=96=B9=E6=B3=95=E6=94=B9?= =?UTF-8?q?=E6=88=90=E5=85=AC=E5=85=B1=E7=9A=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 466021a..777b2e1 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -359,7 +359,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} } return } -func cleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collection string) (err error) { +func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collection string) (err error) { listKeys := fmt.Sprintf(consts.CleanList, tenantId, collection) keys, err := redis.RedisClient.Keys(ctx, listKeys) if err != nil { @@ -413,7 +413,7 @@ func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, return } count = r.DeletedCount - err = cleanRedis(ctx, filter, user.TenantId, collection) + err = m.CleanRedis(ctx, filter, user.TenantId, collection) serverName := g.Cfg().MustGet(ctx, "server.name").String() logRedisKey := fmt.Sprintf("log:%s", serverName) if _, err = redis.AddToStream(ctx, logRedisKey, &dto.RecordCreateLogReq{ @@ -454,7 +454,7 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll if err != nil { return } - err = cleanRedis(ctx, filter, user.TenantId, collection) + err = m.CleanRedis(ctx, filter, user.TenantId, collection) serverName := g.Cfg().MustGet(ctx, "server.name").String() logRedisKey := fmt.Sprintf("log:%s", serverName) if _, err = redis.AddToStream(ctx, logRedisKey, &dto.RecordCreateLogReq{ @@ -581,7 +581,7 @@ func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bs } // 清理相关缓存 for _, filterItem := range filter { - err = cleanRedis(ctx, filterItem, user.TenantId, collection) + err = m.CleanRedis(ctx, filterItem, user.TenantId, collection) if err != nil { glog.Warning(ctx, "清理Redis缓存失败:", err) } @@ -618,7 +618,7 @@ func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collectio return } ids = r.InsertedIDs - err = cleanRedis(ctx, bson.M{}, user.TenantId, collection) + err = m.CleanRedis(ctx, bson.M{}, user.TenantId, collection) //写日志 serverName := g.Cfg().MustGet(ctx, "server.name").String() logRedisKey := fmt.Sprintf("log:%s", serverName) From 65acd04e1a4f23e2064077b62031b8a84d53947e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 30 Dec 2025 18:27:09 +0800 Subject: [PATCH 115/225] =?UTF-8?q?http=E5=B7=A5=E5=85=B7=E5=8C=85?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E6=B3=A8=E5=86=8Clog=E8=B7=AF=E7=94=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/http/http.go b/http/http.go index b6504e0..6cfd395 100644 --- a/http/http.go +++ b/http/http.go @@ -11,6 +11,7 @@ import ( _ "gitee.com/red-future---jilin-g/common/consul" "gitee.com/red-future---jilin-g/common/jaeger" + "gitee.com/red-future---jilin-g/common/log/controller" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" @@ -46,6 +47,10 @@ func init() { Httpclient.SetDiscovery(gsvc.GetRegistry()) } func RouteRegister(controllers []interface{}) { + Httpserver.Group("/log", func(group *ghttp.RouterGroup) { + group.Middleware(jaeger.NewTracer) + group.Bind(controller.OperationLog) + }) re := regexp.MustCompile("[A-Z]") for _, t := range controllers { sName := reflect.ValueOf(t).Elem().Type().Name() From 38ae9edd54f229c2d12b4c5e675fbdb151d3ff47 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Wed, 31 Dec 2025 10:46:39 +0800 Subject: [PATCH 116/225] =?UTF-8?q?redis=E6=B6=88=E8=B4=B9=E9=98=9F?= =?UTF-8?q?=E5=88=97=E6=8E=A5=E5=8F=A3=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/message.go | 4 ++-- redis/redis.go | 15 ++++++++------- utils/utils.go | 2 ++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/redis/message.go b/redis/message.go index e7e6cb0..a20f019 100644 --- a/redis/message.go +++ b/redis/message.go @@ -8,7 +8,7 @@ type QueueMessage struct { ConsumerName string // 消费者名称 Timeout int64 // 阻塞超时时间(毫秒) BatchSize int64 // 最大并发数(信号量容量) - BlockMs int64 - Block bool + BlockMs int64 // 阻塞时间 + AutoAck bool //ACK确认,true自动确认,false手动确认 HandleFunc func(ctx context.Context, message map[string]interface{}) error } diff --git a/redis/redis.go b/redis/redis.go index c338e75..f1af3ac 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -3,7 +3,6 @@ package redis import ( "context" "errors" - "fmt" "strings" "sync" "time" @@ -73,16 +72,17 @@ LOOP: func GetReadStream(ctx context.Context, msg ...QueueMessage) error { for _, t := range msg { - err := GetReadFromStream(ctx, t.StreamKey, t.GroupName, t.ConsumerName, t.BatchSize, t.BlockMs, t.Block, t.HandleFunc) + err := GetReadFromStream(ctx, t.StreamKey, t.GroupName, t.ConsumerName, t.BatchSize, t.BlockMs, t.AutoAck, t.HandleFunc) if err != nil { - return err + glog.Infof(ctx, "读取ReadFromStream数据失败-> 键名: %s, 消费者组: %s, 消费者名称%v\n, 失败err:%v\n", t.StreamKey, t.GroupName, t.ConsumerName, err) + continue } } return nil } // GetReadFromStream 读取ReadFromStream数据 -func GetReadFromStream(ctx context.Context, streamKey, groupName, consumerName string, count, blockMs int64, Block bool, fn func(ctx context.Context, message map[string]interface{}) error) (err error) { +func GetReadFromStream(ctx context.Context, streamKey, groupName, consumerName string, count, blockMs int64, autoAck bool, fn func(ctx context.Context, message map[string]interface{}) error) (err error) { glog.Infof(ctx, "初始化 Stream: %s, 消费者组: %s", streamKey, groupName) err = InitStreamGroup(ctx, streamKey, groupName) if err != nil { @@ -97,13 +97,14 @@ func GetReadFromStream(ctx context.Context, streamKey, groupName, consumerName s } // 处理消息 for _, msg := range messages { - fmt.Printf("消费者 '%s' -> 接收到消息 ID: %s, 内容: %v\n", consumerName, msg.ID, msg.Values) + glog.Infof(ctx, "消费者 '%s' -> 接收到消息 ID: %s, 内容: %v\n", consumerName, msg.ID, msg.Values) // 业务处理 if err = fn(ctx, msg.Values); err != nil { - return err + glog.Infof(ctx, "业务处理失败-> err:%v\n", err) + continue } // 确认消息 (ACK) - if Block { + if autoAck { // 处理成功后,必须调用 XAck,否则消息会一直留在 PEL 中 err = AckMessage(ctx, streamKey, groupName, msg.ID) if err != nil { diff --git a/utils/utils.go b/utils/utils.go index b8ca4ca..c18874d 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -94,6 +94,8 @@ func GetUserInfo(ctx context.Context) (user do.User, err error) { user.TenantId = dataMap["tenantId"] } else { user.TenantId = ctx.Value("tenantId") + user.UserName = ctx.Value("userName") + fmt.Println("user.UserName==================", user.UserName) } if user.TenantId == nil { return user, gerror.New("租户信息为空") From 183ce252b7fe1f96e4983492dffa9da9bb56227f Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Wed, 31 Dec 2025 10:57:57 +0800 Subject: [PATCH 117/225] =?UTF-8?q?redis=E6=B6=88=E8=B4=B9=E9=98=9F?= =?UTF-8?q?=E5=88=97=E6=8E=A5=E5=8F=A3=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/message.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/redis/message.go b/redis/message.go index a20f019..a9cfcbe 100644 --- a/redis/message.go +++ b/redis/message.go @@ -6,9 +6,8 @@ type QueueMessage struct { StreamKey string // Stream 键名 GroupName string // 消费者组名称 ConsumerName string // 消费者名称 - Timeout int64 // 阻塞超时时间(毫秒) BatchSize int64 // 最大并发数(信号量容量) BlockMs int64 // 阻塞时间 - AutoAck bool //ACK确认,true自动确认,false手动确认 + AutoAck bool // ACK确认,true自动确认,false手动确认 HandleFunc func(ctx context.Context, message map[string]interface{}) error } From e51ee8c0bf96cc4823f4542bd69a37be2dfa06a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 31 Dec 2025 11:03:21 +0800 Subject: [PATCH 118/225] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E5=85=AC=E5=85=B1beans=E5=8C=85=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- beans/beans.go | 22 ++++++++++++++++++++++ do/base_do.go | 21 --------------------- log/model/entity/log.go | 4 ++-- utils/utils.go | 4 ++-- 4 files changed, 26 insertions(+), 25 deletions(-) delete mode 100644 do/base_do.go diff --git a/beans/beans.go b/beans/beans.go index 5254575..53edaa7 100644 --- a/beans/beans.go +++ b/beans/beans.go @@ -1,9 +1,31 @@ package beans +import ( + "time" + + "go.mongodb.org/mongo-driver/v2/bson" +) + type ResponseEmpty struct { } + type Page struct { PageNum int `p:"pageNum"` //当前页码 PageSize int `p:"pageSize"` //每页数 Total int //总页数 } + +type MongoBaseDO struct { + Id bson.ObjectID `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID + Creator interface{} `bson:"creator,omitempty" json:"creator"` + CreatedAt time.Time `bson:"createdAt,omitempty" json:"createdAt"` + Updater interface{} `bson:"updater,omitempty" json:"updater"` + UpdatedAt time.Time `bson:"updatedAt,omitempty" json:"updatedAt"` + TenantId interface{} `bson:"tenantId" json:"tenantId" default:"1"` // 租户ID + IsDeleted bool `bson:"isDeleted" json:"isDeleted" default:"false"` +} + +type User struct { + UserName interface{} `bson:"userName" json:"userName"` // MongoDB 默认 ID + TenantId interface{} `bson:"tenantId" json:"tenantId"` // 租户ID +} diff --git a/do/base_do.go b/do/base_do.go deleted file mode 100644 index 5c4001f..0000000 --- a/do/base_do.go +++ /dev/null @@ -1,21 +0,0 @@ -package do - -import ( - "time" - - "go.mongodb.org/mongo-driver/v2/bson" -) - -type MongoBaseDO struct { - Id bson.ObjectID `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID - Creator interface{} `bson:"creator,omitempty" json:"creator"` - CreatedAt time.Time `bson:"createdAt,omitempty" json:"createdAt"` - Updater interface{} `bson:"updater,omitempty" json:"updater"` - UpdatedAt time.Time `bson:"updatedAt,omitempty" json:"updatedAt"` - TenantId interface{} `bson:"tenantId" json:"tenantId" default:"1"` // 租户ID - IsDeleted bool `bson:"isDeleted" json:"isDeleted" default:"false"` -} -type User struct { - UserName interface{} `bson:"userName" json:"userName"` // MongoDB 默认 ID - TenantId interface{} `bson:"tenantId" json:"tenantId"` // 租户ID -} diff --git a/log/model/entity/log.go b/log/model/entity/log.go index cfa8f28..ca59c9b 100644 --- a/log/model/entity/log.go +++ b/log/model/entity/log.go @@ -1,12 +1,12 @@ package entity import ( - "gitee.com/red-future---jilin-g/common/do" + "gitee.com/red-future---jilin-g/common/beans" ) // OperationLog 操作日志实体 - 用于记录数据增删改操作行为 type OperationLog struct { - do.MongoBaseDO `bson:",inline"` + beans.MongoBaseDO `bson:",inline"` ServiceName string `bson:"service_name" json:"service_name"` // 服务名:具体的微服务名称 Collection string `bson:"collection" json:"collection"` // 集合名:数据所在的集合名称 diff --git a/utils/utils.go b/utils/utils.go index c18874d..68f8c87 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -7,7 +7,7 @@ import ( "sort" "time" - "gitee.com/red-future---jilin-g/common/do" + "gitee.com/red-future---jilin-g/common/beans" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/errors/gcode" "github.com/gogf/gf/v2/errors/gerror" @@ -59,7 +59,7 @@ func GetMonthToday(t time.Time, month int) time.Time { } return target.AddDate(0, 0, t.Day()-1) } -func GetUserInfo(ctx context.Context) (user do.User, err error) { +func GetUserInfo(ctx context.Context) (user beans.User, err error) { r := g.RequestFromCtx(ctx) if r != nil { redisAddr := g.Cfg().MustGet(ctx, "redis.default.address").String() From 3b8980bb2494cf8329f089fac7fbab659399618e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 31 Dec 2025 14:03:37 +0800 Subject: [PATCH 119/225] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E5=85=AC=E5=85=B1beans=E5=8C=85=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/http/http.go b/http/http.go index 6cfd395..1f3463d 100644 --- a/http/http.go +++ b/http/http.go @@ -72,10 +72,10 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin Httpclient.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) Httpclient.SetHeaderMap(headers) response, err := Httpclient.DoRequest(ctx, method, url, data) - defer response.Close() if err != nil { return } + defer response.Close() result := response.ReadAll() resultStrut := &ghttp.DefaultHandlerResponse{} if gconv.Struct(result, &resultStrut); resultStrut.Code != 200 { From 4269b4fd793a8efaf8563ce44f046802c24a096e Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Wed, 31 Dec 2025 16:10:23 +0800 Subject: [PATCH 120/225] =?UTF-8?q?feat:=20HTTP=E6=A8=A1=E5=BC=8F=E6=94=AF?= =?UTF-8?q?=E6=8C=81=20-=20=E4=BC=98=E5=8C=96Stream=E6=B6=88=E8=B4=B9?= =?UTF-8?q?=E9=80=BB=E8=BE=91=EF=BC=8C=E6=B7=BB=E5=8A=A0pending=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E4=BC=98=E5=85=88=E8=AF=BB=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/worker_pool.go | 30 +++++++++++++++++++++++++++--- redis/redis.go | 28 ++++++++++++++++++++++------ 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/ragflow/worker_pool.go b/ragflow/worker_pool.go index f7635b2..32b4b67 100644 --- a/ragflow/worker_pool.go +++ b/ragflow/worker_pool.go @@ -2,6 +2,7 @@ package ragflow import ( "context" + "runtime/debug" "strings" "time" @@ -81,11 +82,14 @@ func (q *QueueProcessor) Start(ctx context.Context) error { continue } - glog.Debugf(ctx, "读取 %d 条消息,开始发送", len(messages)) + glog.Infof(ctx, "✅ 从Stream读取到 %d 条消息,开始处理", len(messages)) // 2. 使用协程池提交任务:复用goroutine,避免频繁创建销毁 - for _, msg := range messages { + for i, msg := range messages { m := msg // 捕获循环变量 + msgIndex := i + 1 + glog.Infof(ctx, "📨 准备处理第 %d/%d 条消息 - ID: %s", msgIndex, len(messages), m.ID) + // 提交到协程池,池满时会阻塞等待空闲worker q.pool.Add(ctx, func(ctx context.Context) { q.processMessage(ctx, m) @@ -98,9 +102,29 @@ func (q *QueueProcessor) Start(ctx context.Context) error { // processMessage 处理单条消息(异步执行) func (q *QueueProcessor) processMessage(ctx context.Context, message redis.StreamMessage) { + // 捕获panic,防止协程崩溃 + defer func() { + if r := recover(); r != nil { + glog.Errorf(ctx, "❌ PANIC: 消息处理发生panic - 消息ID: %s, panic内容: %v\n堆栈:\n%s", + message.ID, r, debug.Stack()) + } + }() + + glog.Infof(ctx, "🔄 开始处理消息 - ID: %s", message.ID) + + // 打印实际字段名(调试用) + var fieldNames []string + for key := range message.Values { + fieldNames = append(fieldNames, key) + } + glog.Infof(ctx, "📋 消息字段名列表: %v", fieldNames) + glog.Infof(ctx, "📦 消息完整内容: %+v", message.Values) + // 调用处理函数发送到 RAGFlow if err := q.handleFunc(ctx, message.Values); err != nil { - glog.Errorf(ctx, "消息处理失败: %v, 消息ID: %s", err, message.ID) + glog.Errorf(ctx, "❌ 消息处理失败: %v, 消息ID: %s", err, message.ID) + } else { + glog.Infof(ctx, "✅ 消息处理成功 - ID: %s", message.ID) } // 无论成功失败都 ACK(避免重复消费) diff --git a/redis/redis.go b/redis/redis.go index f1af3ac..c112d41 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -190,9 +190,6 @@ func CreateConsumerGroup(ctx context.Context, streamKey, groupName string) error // ReadFromStream 从 Stream 读取消息(消费者组模式) // 使用 gredis Do() 方法执行 XREADGROUP 命令 func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName string, count int64, blockMs int64) ([]StreamMessage, error) { - glog.Debugf(ctx, "[DEBUG Redis] XREADGROUP GROUP %s %s COUNT %d BLOCK %d STREAMS %s >", - groupName, consumerName, count, blockMs, streamKey) - // 检查是否需要记录trace(避免轮询产生大量trace) execCtx := ctx if !g.Cfg().MustGet(ctx, "jaeger.traceStream", true).Bool() { @@ -201,17 +198,36 @@ func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName stri } RECONNECT: - // XREADGROUP GROUP groupName consumerName COUNT count BLOCK blockMs STREAMS streamKey > + // 先尝试读取pending消息(ID=0),处理积压 + glog.Debugf(ctx, "[DEBUG Redis] XREADGROUP GROUP %s %s COUNT %d BLOCK 0 STREAMS %s 0", + groupName, consumerName, count, streamKey) + result, err := redisClient.Do(execCtx, "XREADGROUP", "GROUP", groupName, consumerName, "COUNT", count, - "BLOCK", blockMs, - "STREAMS", streamKey, ">", + "BLOCK", 0, // 不阻塞,立即返回 + "STREAMS", streamKey, "0", // ID=0 读取pending消息 ) if err != nil { goto RECONNECT } + // 如果没有pending消息,读取新消息 + if result == nil || result.IsEmpty() { + glog.Debugf(ctx, "[DEBUG Redis] 无pending消息,读取新消息 XREADGROUP GROUP %s %s COUNT %d BLOCK %d STREAMS %s >", + groupName, consumerName, count, blockMs, streamKey) + + result, err = redisClient.Do(execCtx, + "XREADGROUP", "GROUP", groupName, consumerName, + "COUNT", count, + "BLOCK", blockMs, + "STREAMS", streamKey, ">", + ) + if err != nil { + goto RECONNECT + } + } + glog.Debugf(ctx, "[DEBUG Redis] XREADGROUP 返回: %+v", result) // 预分配容量,避免动态扩容 From bf235c709d2642afe62a687cdb851d035daa04dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 31 Dec 2025 23:38:33 +0800 Subject: [PATCH 121/225] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E5=88=86?= =?UTF-8?q?=E5=B8=83=E5=BC=8F=E7=86=94=E6=96=AD=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 508 ++++++++++++++++++++++++++++++++++ 1 file changed, 508 insertions(+) create mode 100644 middleware/circuit_breaker.go diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go new file mode 100644 index 0000000..7678220 --- /dev/null +++ b/middleware/circuit_breaker.go @@ -0,0 +1,508 @@ +package middleware + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "gitee.com/red-future---jilin-g/common/jaeger" + "gitee.com/red-future---jilin-g/common/redis" + "github.com/gogf/gf/v2/database/gredis" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/ghttp" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/util/gconv" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// 熔断器状态 +type CircuitState int + +const ( + StateClosed CircuitState = iota // 关闭:正常请求 + StateOpen // 开启:熔断,拒绝请求 + StateHalfOpen // 半开:尝试恢复 +) + +func (s CircuitState) String() string { + switch s { + case StateClosed: + return "closed" + case StateOpen: + return "open" + case StateHalfOpen: + return "half_open" + default: + return "unknown" + } +} + +// 熔断器配置 +type CircuitBreakerConfig struct { + MaxFailures int // 最大失败次数 + Timeout time.Duration // 熔断超时时间(多久后尝试恢复) + HalfOpenSuccess int // 半开状态连续成功次数 + EnableDistributed bool // 是否启用分布式熔断(Redis) +} + +// 熔断器 +type CircuitBreaker struct { + mu sync.RWMutex + state CircuitState + failures int + halfOpenSuccess int + lastFailureTime time.Time + config CircuitBreakerConfig + serviceName string +} + +// Redis Key 前缀 +const ( + CircuitBreakerStateKeyPrefix = "circuit:breaker:%s:state" // 熔断状态 + CircuitBreakerFailuresKeyPrefix = "circuit:breaker:%s:failures" // 失败计数 + CircuitBreakerLastFailKeyPrefix = "circuit:breaker:%s:last_fail" // 最后失败时间 + CircuitBreakerHalfOpenKeyPrefix = "circuit:breaker:%s:half_open_success" // 半开成功计数 +) + +var ( + circuitBreakers = make(map[string]*CircuitBreaker) + circuitMu sync.RWMutex +) + +// GetOrCreateCircuitBreaker 获取或创建熔断器 +func GetOrCreateCircuitBreaker(serviceName string, config CircuitBreakerConfig) *CircuitBreaker { + circuitMu.RLock() + cb, exists := circuitBreakers[serviceName] + circuitMu.RUnlock() + + if exists { + return cb + } + + circuitMu.Lock() + defer circuitMu.Unlock() + + // 双重检查 + if cb, exists := circuitBreakers[serviceName]; exists { + return cb + } + + cb = &CircuitBreaker{ + state: StateClosed, + config: config, + serviceName: serviceName, + } + circuitBreakers[serviceName] = cb + + glog.Infof(context.Background(), "✅ 熔断器已初始化 - 服务: %s, 配置: MaxFailures=%d, Timeout=%v", + serviceName, config.MaxFailures, config.Timeout) + + return cb +} + +// AllowRequest 判断是否允许请求通过 +func (cb *CircuitBreaker) AllowRequest(ctx context.Context) bool { + // 分布式模式:从 Redis 获取全局熔断状态 + if cb.config.EnableDistributed { + redisKey := fmt.Sprintf(CircuitBreakerStateKeyPrefix, cb.serviceName) + state, err := redis.RedisClient.Get(ctx, redisKey) + + if err == nil && !state.IsEmpty() { + stateStr := state.String() + if stateStr == "open" { + // 检查是否超时(进入半开状态) + lastFailKey := fmt.Sprintf(CircuitBreakerLastFailKeyPrefix, cb.serviceName) + lastFail, _ := redis.RedisClient.Get(ctx, lastFailKey) + + if !lastFail.IsEmpty() { + lastFailTime := gconv.Int64(lastFail.Val()) + now := time.Now().Unix() + if (now - lastFailTime) >= int64(cb.config.Timeout.Seconds()) { + glog.Debugf(ctx, "🔓 熔断器进入半开状态 - 服务: %s", cb.serviceName) + return true + } + } + glog.Debugf(ctx, "🔒 熔断器已开启 - 服务: %s, 拒绝请求", cb.serviceName) + return false + } + } + } + + cb.mu.RLock() + localState := cb.state + cb.mu.RUnlock() + + switch localState { + case StateClosed: + return true + case StateOpen: + // 检查是否进入半开状态 + if time.Since(cb.lastFailureTime) > cb.config.Timeout { + glog.Debugf(ctx, "🔓 熔断器进入半开状态 - 服务: %s", cb.serviceName) + return true + } + glog.Debugf(ctx, "🔒 熔断器已开启 - 服务: %s, 拒绝请求", cb.serviceName) + return false + case StateHalfOpen: + return true + } + + return true +} + +// RecordSuccess 记录成功 +func (cb *CircuitBreaker) RecordSuccess(ctx context.Context) { + cb.mu.Lock() + defer cb.mu.Unlock() + + cb.failures = 0 + + if cb.state == StateHalfOpen { + cb.halfOpenSuccess++ + + if cb.halfOpenSuccess >= cb.config.HalfOpenSuccess { + oldState := cb.state + cb.state = StateClosed + cb.halfOpenSuccess = 0 + + glog.Infof(ctx, "✅ 熔断器已恢复 - 服务: %s, 状态: %s -> %s", + cb.serviceName, oldState, cb.state) + + // 记录恢复事件到 Jaeger + _, span := jaeger.NewSpan(ctx, "circuit_breaker_recovered") + span.SetAttributes( + attribute.String("service", cb.serviceName), + attribute.String("old_state", oldState.String()), + attribute.String("new_state", cb.state.String()), + ) + span.End() + + // 分布式模式:清除 Redis 熔断状态 + if cb.config.EnableDistributed { + go cb.clearRedisState(ctx) + } + } else { + glog.Debugf(ctx, "🔼 半开状态成功计数: %d/%d - 服务: %s", + cb.halfOpenSuccess, cb.config.HalfOpenSuccess, cb.serviceName) + } + } +} + +// RecordFailure 记录失败 +func (cb *CircuitBreaker) RecordFailure(ctx context.Context) { + cb.mu.Lock() + defer cb.mu.Unlock() + + cb.failures++ + cb.lastFailureTime = time.Now() + + oldState := cb.state + if cb.failures >= cb.config.MaxFailures { + cb.state = StateOpen + + glog.Warningf(ctx, "⚠️ 熔断器已触发 - 服务: %s, 状态: %s -> %s, 失败次数: %d/%d", + cb.serviceName, oldState, cb.state, cb.failures, cb.config.MaxFailures) + + // 记录熔断事件到 Jaeger + ctx2, span := jaeger.NewSpan(ctx, "circuit_breaker_triggered") + span.SetAttributes( + attribute.String("service", cb.serviceName), + attribute.Int("failures", cb.failures), + attribute.String("old_state", oldState.String()), + attribute.String("new_state", cb.state.String()), + ) + span.End() + trace.SpanFromContext(ctx2).End() + + // 分布式模式:将熔断状态写入 Redis + if cb.config.EnableDistributed { + go cb.setRedisState(ctx) + } + } else { + glog.Debugf(ctx, "📊 熔断器失败计数: %d/%d - 服务: %s", + cb.failures, cb.config.MaxFailures, cb.serviceName) + } +} + +// setRedisState 将熔断状态写入 Redis(分布式同步) +func (cb *CircuitBreaker) setRedisState(ctx context.Context) { + stateKey := fmt.Sprintf(CircuitBreakerStateKeyPrefix, cb.serviceName) + failuresKey := fmt.Sprintf(CircuitBreakerFailuresKeyPrefix, cb.serviceName) + lastFailKey := fmt.Sprintf(CircuitBreakerLastFailKeyPrefix, cb.serviceName) + + // 设置熔断状态 + ttl := int64(cb.config.Timeout.Seconds()) + _, err := redis.RedisClient.Set(ctx, stateKey, "open", gredis.SetOption{ + TTLOption: gredis.TTLOption{EX: &ttl}, + }) + if err != nil { + glog.Errorf(ctx, "设置熔断状态到 Redis 失败: %v", err) + } + + // 设置失败计数 + _, err = redis.RedisClient.Set(ctx, failuresKey, cb.failures, gredis.SetOption{ + TTLOption: gredis.TTLOption{EX: &ttl}, + }) + if err != nil { + glog.Errorf(ctx, "设置失败计数到 Redis 失败: %v", err) + } + + // 设置最后失败时间 + _, err = redis.RedisClient.Set(ctx, lastFailKey, time.Now().Unix(), gredis.SetOption{ + TTLOption: gredis.TTLOption{EX: &ttl}, + }) + if err != nil { + glog.Errorf(ctx, "设置最后失败时间到 Redis 失败: %v", err) + } + + glog.Infof(ctx, "📡 熔断状态已同步到 Redis - 服务: %s", cb.serviceName) +} + +// clearRedisState 清除 Redis 中的熔断状态(分布式恢复) +func (cb *CircuitBreaker) clearRedisState(ctx context.Context) { + stateKey := fmt.Sprintf(CircuitBreakerStateKeyPrefix, cb.serviceName) + failuresKey := fmt.Sprintf(CircuitBreakerFailuresKeyPrefix, cb.serviceName) + lastFailKey := fmt.Sprintf(CircuitBreakerLastFailKeyPrefix, cb.serviceName) + + // 删除熔断状态 + _, err := redis.RedisClient.Del(ctx, stateKey) + if err != nil { + glog.Errorf(ctx, "删除熔断状态从 Redis 失败: %v", err) + } + + // 删除失败计数 + _, err = redis.RedisClient.Del(ctx, failuresKey) + if err != nil { + glog.Errorf(ctx, "删除失败计数从 Redis 失败: %v", err) + } + + // 删除最后失败时间 + _, err = redis.RedisClient.Del(ctx, lastFailKey) + if err != nil { + glog.Errorf(ctx, "删除最后失败时间从 Redis 失败: %v", err) + } + + glog.Infof(ctx, "📡 熔断状态已从 Redis 清除 - 服务: %s", cb.serviceName) +} + +// GetState 获取熔断器当前状态 +func (cb *CircuitBreaker) GetState() CircuitState { + cb.mu.RLock() + defer cb.mu.RUnlock() + return cb.state +} + +// GetFailures 获取当前失败次数 +func (cb *CircuitBreaker) GetFailures() int { + cb.mu.RLock() + defer cb.mu.RUnlock() + return cb.failures +} + +// Reset 重置熔断器(手动恢复) +func (cb *CircuitBreaker) Reset(ctx context.Context) { + cb.mu.Lock() + defer cb.mu.Unlock() + + oldState := cb.state + cb.state = StateClosed + cb.failures = 0 + cb.halfOpenSuccess = 0 + + glog.Infof(ctx, "🔄 熔断器已手动重置 - 服务: %s, 状态: %s -> %s", + cb.serviceName, oldState, cb.state) + + // 清除 Redis 状态 + if cb.config.EnableDistributed { + go cb.clearRedisState(ctx) + } +} + +// GetAllBreakers 获取所有熔断器状态 +func GetAllBreakers() map[string]map[string]interface{} { + circuitMu.RLock() + defer circuitMu.RUnlock() + + result := make(map[string]map[string]interface{}) + for name, cb := range circuitBreakers { + cb.mu.RLock() + result[name] = map[string]interface{}{ + "state": cb.state.String(), + "failures": cb.failures, + "halfOpenSuccess": cb.halfOpenSuccess, + "lastFailureTime": cb.lastFailureTime.Format("2006-01-02 15:04:05"), + } + cb.mu.RUnlock() + } + return result +} + +// getCircuitBreakerConfig 从配置文件读取熔断器配置 +func getCircuitBreakerConfig(ctx context.Context, serviceName string) CircuitBreakerConfig { + return CircuitBreakerConfig{ + MaxFailures: g.Cfg().MustGet(ctx, fmt.Sprintf("circuitBreaker.%s.maxFailures", serviceName), 5).Int(), + Timeout: g.Cfg().MustGet(ctx, fmt.Sprintf("circuitBreaker.%s.timeout", serviceName), "30s").Duration(), + HalfOpenSuccess: g.Cfg().MustGet(ctx, fmt.Sprintf("circuitBreaker.%s.halfOpenSuccess", serviceName), 2).Int(), + EnableDistributed: g.Cfg().MustGet(ctx, "circuitBreaker.enableDistributed", true).Bool(), + } +} + +// CircuitBreakerMiddleware Gateway 熔断中间件 +func CircuitBreakerMiddleware(r *ghttp.Request) { + // 从 URL 提取服务名 + pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + if len(pathParts) == 0 { + r.Middleware.Next() + return + } + + serviceName := pathParts[0] + + // 跳过非微服务路径 + if serviceName == "health" || serviceName == "metrics" || serviceName == "swagger" { + r.Middleware.Next() + return + } + + // 获取熔断配置 + config := getCircuitBreakerConfig(r.GetCtx(), serviceName) + + // 获取或创建熔断器 + cb := GetOrCreateCircuitBreaker(serviceName, config) + + // 判断是否允许请求通过 + if !cb.AllowRequest(r.GetCtx()) { + // 熔断开启,返回降级响应 + glog.Warningf(r.GetCtx(), "⛔ 服务 %s 熔断中,触发降级", serviceName) + + // 获取降级响应 + fallbackResponse := getFallbackResponse(serviceName) + + // 记录降级事件到 Jaeger + ctx2, span := jaeger.NewSpan(r.GetCtx(), "circuit_breaker_fallback") + span.SetAttributes( + attribute.String("service", serviceName), + attribute.String("fallback_type", "circuit_breaker_open"), + ) + span.End() + trace.SpanFromContext(ctx2).End() + + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 503, + Message: "服务暂时不可用,已自动降级", + Data: fallbackResponse, + }) + return + } + + // 记录请求开始时间 + startTime := time.Now() + + // 执行后续处理(包括代理转发) + r.Middleware.Next() + + // 根据响应状态记录成功/失败 + elapsed := time.Since(startTime) + if r.Response.Status >= 500 { + cb.RecordFailure(r.GetCtx()) + glog.Warningf(r.GetCtx(), "❌ 服务 %s 请求失败 - 状态码: %d, 耗时: %v", + serviceName, r.Response.Status, elapsed) + } else { + cb.RecordSuccess(r.GetCtx()) + glog.Debugf(r.GetCtx(), "✅ 服务 %s 请求成功 - 状态码: %d, 耗时: %v", + serviceName, r.Response.Status, elapsed) + } +} + +// getFallbackResponse 获取降级响应 +func getFallbackResponse(serviceName string) interface{} { + switch serviceName { + case "customerService": + // 客服服务降级:返回固定话术 + return map[string]interface{}{ + "message": "当前客服系统繁忙,智能助手暂时离线。如需帮助,可拨打:400-xxx-xxxx", + "fallback": true, + "type": "fixed_response", + } + case "order": + // 订单服务降级:返回排队提示 + return map[string]interface{}{ + "message": "订单系统繁忙,您的请求已排队,请稍后刷新查看", + "fallback": true, + "type": "queue_hint", + } + case "assets": + // 资产服务降级:返回缓存数据 + return map[string]interface{}{ + "message": "资产数据正在更新中,显示的是5分钟前的缓存数据", + "fallback": true, + "type": "cached_data", + } + case "wallet": + // 钱包服务降级:返回只读数据 + return map[string]interface{}{ + "message": "钱包服务暂时不可用,无法进行转账操作,查询功能正常", + "fallback": true, + "type": "read_only", + } + case "market": + // 市场服务降级:返回推荐数据 + return map[string]interface{}{ + "message": "市场数据加载中,为您推荐以下热门商品", + "fallback": true, + "type": "recommended", + } + default: + // 默认降级响应 + return map[string]interface{}{ + "message": "服务暂时不可用,请稍后再试", + "fallback": true, + "type": "default", + } + } +} + +// CircuitBreakerHealthCheckHandler 熔断器健康检查接口 +func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { + breakers := GetAllBreakers() + + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 200, + Message: "熔断器状态查询成功", + Data: breakers, + }) +} + +// CircuitBreakerResetHandler 熔断器手动重置接口 +func CircuitBreakerResetHandler(r *ghttp.Request) { + serviceName := r.Get("serviceName").String() + if serviceName == "" { + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 400, + Message: "服务名不能为空", + }) + return + } + + circuitMu.RLock() + cb, exists := circuitBreakers[serviceName] + circuitMu.RUnlock() + + if !exists { + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 404, + Message: fmt.Sprintf("服务 %s 的熔断器不存在", serviceName), + }) + return + } + + cb.Reset(r.GetCtx()) + + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 200, + Message: fmt.Sprintf("服务 %s 的熔断器已重置", serviceName), + }) +} From 75d9348896015d56449df88e9614955cd7140135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 00:04:43 +0800 Subject: [PATCH 122/225] =?UTF-8?q?fix:=20=E8=A1=A5=E5=85=85=E5=8D=8A?= =?UTF-8?q?=E5=BC=80=E7=8A=B6=E6=80=81=E7=9A=84=E5=88=86=E5=B8=83=E5=BC=8F?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 37 +++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 7678220..73d67ba 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -129,6 +129,15 @@ func (cb *CircuitBreaker) AllowRequest(ctx context.Context) bool { return false } } + + // 从 Redis 同步半开成功计数 + halfOpenKey := fmt.Sprintf(CircuitBreakerHalfOpenKeyPrefix, cb.serviceName) + halfOpenValue, err := redis.RedisClient.Get(ctx, halfOpenKey) + if err == nil && !halfOpenValue.IsEmpty() { + cb.mu.Lock() + cb.halfOpenSuccess = gconv.Int(halfOpenValue.Val()) + cb.mu.Unlock() + } } cb.mu.RLock() @@ -163,6 +172,18 @@ func (cb *CircuitBreaker) RecordSuccess(ctx context.Context) { if cb.state == StateHalfOpen { cb.halfOpenSuccess++ + // 分布式模式:同步半开成功计数到 Redis + if cb.config.EnableDistributed { + halfOpenKey := fmt.Sprintf(CircuitBreakerHalfOpenKeyPrefix, cb.serviceName) + ttl := int64(cb.config.Timeout.Seconds()) + _, err := redis.RedisClient.Set(ctx, halfOpenKey, cb.halfOpenSuccess, gredis.SetOption{ + TTLOption: gredis.TTLOption{EX: &ttl}, + }) + if err != nil { + glog.Errorf(ctx, "同步半开成功计数到 Redis 失败: %v", err) + } + } + if cb.halfOpenSuccess >= cb.config.HalfOpenSuccess { oldState := cb.state cb.state = StateClosed @@ -232,6 +253,7 @@ func (cb *CircuitBreaker) setRedisState(ctx context.Context) { stateKey := fmt.Sprintf(CircuitBreakerStateKeyPrefix, cb.serviceName) failuresKey := fmt.Sprintf(CircuitBreakerFailuresKeyPrefix, cb.serviceName) lastFailKey := fmt.Sprintf(CircuitBreakerLastFailKeyPrefix, cb.serviceName) + halfOpenKey := fmt.Sprintf(CircuitBreakerHalfOpenKeyPrefix, cb.serviceName) // 设置熔断状态 ttl := int64(cb.config.Timeout.Seconds()) @@ -258,6 +280,14 @@ func (cb *CircuitBreaker) setRedisState(ctx context.Context) { glog.Errorf(ctx, "设置最后失败时间到 Redis 失败: %v", err) } + // 重置半开成功计数 + _, err = redis.RedisClient.Set(ctx, halfOpenKey, 0, gredis.SetOption{ + TTLOption: gredis.TTLOption{EX: &ttl}, + }) + if err != nil { + glog.Errorf(ctx, "设置半开成功计数到 Redis 失败: %v", err) + } + glog.Infof(ctx, "📡 熔断状态已同步到 Redis - 服务: %s", cb.serviceName) } @@ -266,6 +296,7 @@ func (cb *CircuitBreaker) clearRedisState(ctx context.Context) { stateKey := fmt.Sprintf(CircuitBreakerStateKeyPrefix, cb.serviceName) failuresKey := fmt.Sprintf(CircuitBreakerFailuresKeyPrefix, cb.serviceName) lastFailKey := fmt.Sprintf(CircuitBreakerLastFailKeyPrefix, cb.serviceName) + halfOpenKey := fmt.Sprintf(CircuitBreakerHalfOpenKeyPrefix, cb.serviceName) // 删除熔断状态 _, err := redis.RedisClient.Del(ctx, stateKey) @@ -285,6 +316,12 @@ func (cb *CircuitBreaker) clearRedisState(ctx context.Context) { glog.Errorf(ctx, "删除最后失败时间从 Redis 失败: %v", err) } + // 删除半开成功计数 + _, err = redis.RedisClient.Del(ctx, halfOpenKey) + if err != nil { + glog.Errorf(ctx, "删除半开成功计数从 Redis 失败: %v", err) + } + glog.Infof(ctx, "📡 熔断状态已从 Redis 清除 - 服务: %s", cb.serviceName) } From 9940788b65c8383606323dc2a4850a9806e96b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 01:33:59 +0800 Subject: [PATCH 123/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 768 ++++++++++++++-------------------- 1 file changed, 321 insertions(+), 447 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 73d67ba..b28c414 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -7,389 +7,217 @@ import ( "sync" "time" - "gitee.com/red-future---jilin-g/common/jaeger" - "gitee.com/red-future---jilin-g/common/redis" - "github.com/gogf/gf/v2/database/gredis" + "github.com/alibaba/sentinel-golang/api" + "github.com/alibaba/sentinel-golang/core/circuitbreaker" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" - "github.com/gogf/gf/v2/os/glog" - "github.com/gogf/gf/v2/util/gconv" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" ) -// 熔断器状态 -type CircuitState int +// CircuitBreakerState 熔断器状态 +type CircuitBreakerState string const ( - StateClosed CircuitState = iota // 关闭:正常请求 - StateOpen // 开启:熔断,拒绝请求 - StateHalfOpen // 半开:尝试恢复 + StateClosed CircuitBreakerState = "closed" // 关闭:正常状态 + StateOpen CircuitBreakerState = "open" // 开启:熔断状态 + StateHalfOpen CircuitBreakerState = "half-open" // 半开:尝试恢复状态 ) -func (s CircuitState) String() string { - switch s { - case StateClosed: - return "closed" - case StateOpen: - return "open" - case StateHalfOpen: - return "half_open" - default: - return "unknown" - } -} - -// 熔断器配置 +// CircuitBreakerConfig 熔断器配置 type CircuitBreakerConfig struct { - MaxFailures int // 最大失败次数 - Timeout time.Duration // 熔断超时时间(多久后尝试恢复) - HalfOpenSuccess int // 半开状态连续成功次数 - EnableDistributed bool // 是否启用分布式熔断(Redis) + MaxFailures int // 连续失败次数 + Timeout string // 熔断超时时间 + HalfOpenSuccess int // 半开状态连续成功次数 + SuccessStatusCodes []int // 视为成功的HTTP状态码 + SlowRequestThreshold string // 慢请求阈值 + HalfOpenRequestSampleRate float64 // 半开状态请求采样率 + Dimension string // 熔断器维度: service/ip/user + EnableSlidingWindow bool // 是否启用滑动窗口 + SlidingWindowSize string // 滑动窗口大小 + FailureRateThreshold float64 // 失败率阈值 } -// 熔断器 -type CircuitBreaker struct { - mu sync.RWMutex - state CircuitState - failures int - halfOpenSuccess int - lastFailureTime time.Time - config CircuitBreakerConfig - serviceName string +// CircuitBreakerInfo 熔断器信息 +type CircuitBreakerInfo struct { + ResourceName string `json:"resourceName"` // 资源名称 + State CircuitBreakerState `json:"state"` // 当前状态 + Config *CircuitBreakerConfig `json:"config"` // 配置信息 + FailCount int64 `json:"failCount"` // 失败次数 + TotalCount int64 `json:"totalCount"` // 总请求数 + LastOpenTime time.Time `json:"lastOpenTime"` // 上次熔断时间 + NextRetryTime time.Time `json:"nextRetryTime"` // 下次重试时间 } -// Redis Key 前缀 -const ( - CircuitBreakerStateKeyPrefix = "circuit:breaker:%s:state" // 熔断状态 - CircuitBreakerFailuresKeyPrefix = "circuit:breaker:%s:failures" // 失败计数 - CircuitBreakerLastFailKeyPrefix = "circuit:breaker:%s:last_fail" // 最后失败时间 - CircuitBreakerHalfOpenKeyPrefix = "circuit:breaker:%s:half_open_success" // 半开成功计数 -) - var ( - circuitBreakers = make(map[string]*CircuitBreaker) - circuitMu sync.RWMutex + // circuitBreakers 存储所有熔断器状态(用于健康检查) + circuitBreakers sync.Map + // enableDistributed 是否启用分布式熔断 + enableDistributed = false + // circuitBreakerConfigs 熔断器配置缓存 + circuitBreakerConfigs sync.Map ) -// GetOrCreateCircuitBreaker 获取或创建熔断器 -func GetOrCreateCircuitBreaker(serviceName string, config CircuitBreakerConfig) *CircuitBreaker { - circuitMu.RLock() - cb, exists := circuitBreakers[serviceName] - circuitMu.RUnlock() +// InitCircuitBreaker 初始化Sentinel熔断器 +func InitCircuitBreaker() error { + ctx := context.Background() + // 从配置文件读取是否启用分布式熔断 + enableDistributed = g.Cfg().MustGet(ctx, "circuitBreaker.enableDistributed", false).Bool() - if exists { - return cb + // 初始化Sentinel + err := api.InitDefault() + if err != nil { + return fmt.Errorf("Sentinel初始化失败: %v", err) } - circuitMu.Lock() - defer circuitMu.Unlock() + g.Log().Infof(ctx, "Sentinel熔断器初始化成功,分布式熔断: %v", enableDistributed) - // 双重检查 - if cb, exists := circuitBreakers[serviceName]; exists { - return cb + // 加载所有服务的熔断器配置 + loadCircuitBreakerConfigs() + + // 为每个服务创建熔断器 + services := []string{ + "customerService", "order", "assets", "cid", "oss", + "wallet", "market", "knapsack", } - cb = &CircuitBreaker{ - state: StateClosed, - config: config, - serviceName: serviceName, + for _, service := range services { + serviceConfig := loadServiceCircuitBreakerConfig(service) + if serviceConfig != nil { + circuitBreakerConfigs.Store(service, serviceConfig) + initErr := initServiceCircuitBreaker(service, serviceConfig) + if initErr != nil { + g.Log().Errorf(ctx, "服务 %s 熔断器初始化失败: %v", service, initErr) + } + } } - circuitBreakers[serviceName] = cb - glog.Infof(context.Background(), "✅ 熔断器已初始化 - 服务: %s, 配置: MaxFailures=%d, Timeout=%v", - serviceName, config.MaxFailures, config.Timeout) - - return cb + return nil } -// AllowRequest 判断是否允许请求通过 -func (cb *CircuitBreaker) AllowRequest(ctx context.Context) bool { - // 分布式模式:从 Redis 获取全局熔断状态 - if cb.config.EnableDistributed { - redisKey := fmt.Sprintf(CircuitBreakerStateKeyPrefix, cb.serviceName) - state, err := redis.RedisClient.Get(ctx, redisKey) - - if err == nil && !state.IsEmpty() { - stateStr := state.String() - if stateStr == "open" { - // 检查是否超时(进入半开状态) - lastFailKey := fmt.Sprintf(CircuitBreakerLastFailKeyPrefix, cb.serviceName) - lastFail, _ := redis.RedisClient.Get(ctx, lastFailKey) - - if !lastFail.IsEmpty() { - lastFailTime := gconv.Int64(lastFail.Val()) - now := time.Now().Unix() - if (now - lastFailTime) >= int64(cb.config.Timeout.Seconds()) { - glog.Debugf(ctx, "🔓 熔断器进入半开状态 - 服务: %s", cb.serviceName) - return true - } - } - glog.Debugf(ctx, "🔒 熔断器已开启 - 服务: %s, 拒绝请求", cb.serviceName) - return false - } - } - - // 从 Redis 同步半开成功计数 - halfOpenKey := fmt.Sprintf(CircuitBreakerHalfOpenKeyPrefix, cb.serviceName) - halfOpenValue, err := redis.RedisClient.Get(ctx, halfOpenKey) - if err == nil && !halfOpenValue.IsEmpty() { - cb.mu.Lock() - cb.halfOpenSuccess = gconv.Int(halfOpenValue.Val()) - cb.mu.Unlock() - } +// loadCircuitBreakerConfigs 加载熔断器配置 +func loadCircuitBreakerConfigs() { + services := []string{ + "customerService", "order", "assets", "cid", "oss", + "wallet", "market", "knapsack", } - - cb.mu.RLock() - localState := cb.state - cb.mu.RUnlock() - - switch localState { - case StateClosed: - return true - case StateOpen: - // 检查是否进入半开状态 - if time.Since(cb.lastFailureTime) > cb.config.Timeout { - glog.Debugf(ctx, "🔓 熔断器进入半开状态 - 服务: %s", cb.serviceName) - return true - } - glog.Debugf(ctx, "🔒 熔断器已开启 - 服务: %s, 拒绝请求", cb.serviceName) - return false - case StateHalfOpen: - return true - } - - return true -} - -// RecordSuccess 记录成功 -func (cb *CircuitBreaker) RecordSuccess(ctx context.Context) { - cb.mu.Lock() - defer cb.mu.Unlock() - - cb.failures = 0 - - if cb.state == StateHalfOpen { - cb.halfOpenSuccess++ - - // 分布式模式:同步半开成功计数到 Redis - if cb.config.EnableDistributed { - halfOpenKey := fmt.Sprintf(CircuitBreakerHalfOpenKeyPrefix, cb.serviceName) - ttl := int64(cb.config.Timeout.Seconds()) - _, err := redis.RedisClient.Set(ctx, halfOpenKey, cb.halfOpenSuccess, gredis.SetOption{ - TTLOption: gredis.TTLOption{EX: &ttl}, - }) - if err != nil { - glog.Errorf(ctx, "同步半开成功计数到 Redis 失败: %v", err) - } - } - - if cb.halfOpenSuccess >= cb.config.HalfOpenSuccess { - oldState := cb.state - cb.state = StateClosed - cb.halfOpenSuccess = 0 - - glog.Infof(ctx, "✅ 熔断器已恢复 - 服务: %s, 状态: %s -> %s", - cb.serviceName, oldState, cb.state) - - // 记录恢复事件到 Jaeger - _, span := jaeger.NewSpan(ctx, "circuit_breaker_recovered") - span.SetAttributes( - attribute.String("service", cb.serviceName), - attribute.String("old_state", oldState.String()), - attribute.String("new_state", cb.state.String()), - ) - span.End() - - // 分布式模式:清除 Redis 熔断状态 - if cb.config.EnableDistributed { - go cb.clearRedisState(ctx) - } - } else { - glog.Debugf(ctx, "🔼 半开状态成功计数: %d/%d - 服务: %s", - cb.halfOpenSuccess, cb.config.HalfOpenSuccess, cb.serviceName) + for _, service := range services { + config := loadServiceCircuitBreakerConfig(service) + if config != nil { + circuitBreakerConfigs.Store(service, config) } } } -// RecordFailure 记录失败 -func (cb *CircuitBreaker) RecordFailure(ctx context.Context) { - cb.mu.Lock() - defer cb.mu.Unlock() +// loadServiceCircuitBreakerConfig 加载单个服务的熔断器配置 +func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { + ctx := context.Background() + key := fmt.Sprintf("circuitBreaker.%s", serviceName) - cb.failures++ - cb.lastFailureTime = time.Now() + maxFailures := g.Cfg().MustGet(ctx, key+".maxFailures", 5).Int() + timeout := g.Cfg().MustGet(ctx, key+".timeout", "60s").String() + halfOpenSuccess := g.Cfg().MustGet(ctx, key+".halfOpenSuccess", 2).Int() + slowRequestThreshold := g.Cfg().MustGet(ctx, key+".slowRequestThreshold", "3s").String() + dimension := g.Cfg().MustGet(ctx, key+".dimension", "service").String() + enableSlidingWindow := g.Cfg().MustGet(ctx, key+".enableSlidingWindow", false).Bool() + slidingWindowSize := g.Cfg().MustGet(ctx, key+".slidingWindowSize", "60s").String() + failureRateThreshold := g.Cfg().MustGet(ctx, key+".failureRateThreshold", 0.5).Float64() + halfOpenRequestSampleRate := g.Cfg().MustGet(ctx, key+".halfOpenRequestSampleRate", 1.0).Float64() - oldState := cb.state - if cb.failures >= cb.config.MaxFailures { - cb.state = StateOpen + // 解析成功状态码 + successCodes := g.Cfg().MustGet(ctx, key+".successStatusCodes", "200,201,204").String() + statusCodes := parseStatusCodes(successCodes) - glog.Warningf(ctx, "⚠️ 熔断器已触发 - 服务: %s, 状态: %s -> %s, 失败次数: %d/%d", - cb.serviceName, oldState, cb.state, cb.failures, cb.config.MaxFailures) + return &CircuitBreakerConfig{ + MaxFailures: maxFailures, + Timeout: timeout, + HalfOpenSuccess: halfOpenSuccess, + SuccessStatusCodes: statusCodes, + SlowRequestThreshold: slowRequestThreshold, + HalfOpenRequestSampleRate: halfOpenRequestSampleRate, + Dimension: dimension, + EnableSlidingWindow: enableSlidingWindow, + SlidingWindowSize: slidingWindowSize, + FailureRateThreshold: failureRateThreshold, + } +} - // 记录熔断事件到 Jaeger - ctx2, span := jaeger.NewSpan(ctx, "circuit_breaker_triggered") - span.SetAttributes( - attribute.String("service", cb.serviceName), - attribute.Int("failures", cb.failures), - attribute.String("old_state", oldState.String()), - attribute.String("new_state", cb.state.String()), - ) - span.End() - trace.SpanFromContext(ctx2).End() +// parseStatusCodes 解析HTTP状态码 +func parseStatusCodes(str string) []int { + parts := strings.Split(str, ",") + codes := make([]int, 0, len(parts)) + for _, part := range parts { + var code int + if _, err := fmt.Sscanf(strings.TrimSpace(part), "%d", &code); err == nil { + codes = append(codes, code) + } + } + return codes +} - // 分布式模式:将熔断状态写入 Redis - if cb.config.EnableDistributed { - go cb.setRedisState(ctx) +// initServiceCircuitBreaker 初始化服务熔断器 +func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) error { + timeout, _ := time.ParseDuration(config.Timeout) + slowRequestThreshold, _ := time.ParseDuration(config.SlowRequestThreshold) + _, _ = time.ParseDuration(config.SlidingWindowSize) + + resourceName := fmt.Sprintf("service:%s", serviceName) + + var rule []*circuitbreaker.Rule + if config.EnableSlidingWindow { + // 使用滑动窗口统计(更精确)- 慢调用比例策略 + rule = []*circuitbreaker.Rule{ + { + Resource: resourceName, + Strategy: circuitbreaker.SlowRequestRatio, + RetryTimeoutMs: uint32(timeout.Milliseconds()), + MinRequestAmount: uint64(config.MaxFailures), + StatIntervalMs: 1000, + StatSlidingWindowBucketCount: 10, + MaxAllowedRtMs: uint64(slowRequestThreshold.Milliseconds()), + Threshold: config.FailureRateThreshold, + }, } } else { - glog.Debugf(ctx, "📊 熔断器失败计数: %d/%d - 服务: %s", - cb.failures, cb.config.MaxFailures, cb.serviceName) - } -} - -// setRedisState 将熔断状态写入 Redis(分布式同步) -func (cb *CircuitBreaker) setRedisState(ctx context.Context) { - stateKey := fmt.Sprintf(CircuitBreakerStateKeyPrefix, cb.serviceName) - failuresKey := fmt.Sprintf(CircuitBreakerFailuresKeyPrefix, cb.serviceName) - lastFailKey := fmt.Sprintf(CircuitBreakerLastFailKeyPrefix, cb.serviceName) - halfOpenKey := fmt.Sprintf(CircuitBreakerHalfOpenKeyPrefix, cb.serviceName) - - // 设置熔断状态 - ttl := int64(cb.config.Timeout.Seconds()) - _, err := redis.RedisClient.Set(ctx, stateKey, "open", gredis.SetOption{ - TTLOption: gredis.TTLOption{EX: &ttl}, - }) - if err != nil { - glog.Errorf(ctx, "设置熔断状态到 Redis 失败: %v", err) - } - - // 设置失败计数 - _, err = redis.RedisClient.Set(ctx, failuresKey, cb.failures, gredis.SetOption{ - TTLOption: gredis.TTLOption{EX: &ttl}, - }) - if err != nil { - glog.Errorf(ctx, "设置失败计数到 Redis 失败: %v", err) - } - - // 设置最后失败时间 - _, err = redis.RedisClient.Set(ctx, lastFailKey, time.Now().Unix(), gredis.SetOption{ - TTLOption: gredis.TTLOption{EX: &ttl}, - }) - if err != nil { - glog.Errorf(ctx, "设置最后失败时间到 Redis 失败: %v", err) - } - - // 重置半开成功计数 - _, err = redis.RedisClient.Set(ctx, halfOpenKey, 0, gredis.SetOption{ - TTLOption: gredis.TTLOption{EX: &ttl}, - }) - if err != nil { - glog.Errorf(ctx, "设置半开成功计数到 Redis 失败: %v", err) - } - - glog.Infof(ctx, "📡 熔断状态已同步到 Redis - 服务: %s", cb.serviceName) -} - -// clearRedisState 清除 Redis 中的熔断状态(分布式恢复) -func (cb *CircuitBreaker) clearRedisState(ctx context.Context) { - stateKey := fmt.Sprintf(CircuitBreakerStateKeyPrefix, cb.serviceName) - failuresKey := fmt.Sprintf(CircuitBreakerFailuresKeyPrefix, cb.serviceName) - lastFailKey := fmt.Sprintf(CircuitBreakerLastFailKeyPrefix, cb.serviceName) - halfOpenKey := fmt.Sprintf(CircuitBreakerHalfOpenKeyPrefix, cb.serviceName) - - // 删除熔断状态 - _, err := redis.RedisClient.Del(ctx, stateKey) - if err != nil { - glog.Errorf(ctx, "删除熔断状态从 Redis 失败: %v", err) - } - - // 删除失败计数 - _, err = redis.RedisClient.Del(ctx, failuresKey) - if err != nil { - glog.Errorf(ctx, "删除失败计数从 Redis 失败: %v", err) - } - - // 删除最后失败时间 - _, err = redis.RedisClient.Del(ctx, lastFailKey) - if err != nil { - glog.Errorf(ctx, "删除最后失败时间从 Redis 失败: %v", err) - } - - // 删除半开成功计数 - _, err = redis.RedisClient.Del(ctx, halfOpenKey) - if err != nil { - glog.Errorf(ctx, "删除半开成功计数从 Redis 失败: %v", err) - } - - glog.Infof(ctx, "📡 熔断状态已从 Redis 清除 - 服务: %s", cb.serviceName) -} - -// GetState 获取熔断器当前状态 -func (cb *CircuitBreaker) GetState() CircuitState { - cb.mu.RLock() - defer cb.mu.RUnlock() - return cb.state -} - -// GetFailures 获取当前失败次数 -func (cb *CircuitBreaker) GetFailures() int { - cb.mu.RLock() - defer cb.mu.RUnlock() - return cb.failures -} - -// Reset 重置熔断器(手动恢复) -func (cb *CircuitBreaker) Reset(ctx context.Context) { - cb.mu.Lock() - defer cb.mu.Unlock() - - oldState := cb.state - cb.state = StateClosed - cb.failures = 0 - cb.halfOpenSuccess = 0 - - glog.Infof(ctx, "🔄 熔断器已手动重置 - 服务: %s, 状态: %s -> %s", - cb.serviceName, oldState, cb.state) - - // 清除 Redis 状态 - if cb.config.EnableDistributed { - go cb.clearRedisState(ctx) - } -} - -// GetAllBreakers 获取所有熔断器状态 -func GetAllBreakers() map[string]map[string]interface{} { - circuitMu.RLock() - defer circuitMu.RUnlock() - - result := make(map[string]map[string]interface{}) - for name, cb := range circuitBreakers { - cb.mu.RLock() - result[name] = map[string]interface{}{ - "state": cb.state.String(), - "failures": cb.failures, - "halfOpenSuccess": cb.halfOpenSuccess, - "lastFailureTime": cb.lastFailureTime.Format("2006-01-02 15:04:05"), + // 使用连续失败计数(更简单快速)- 异常数策略 + rule = []*circuitbreaker.Rule{ + { + Resource: resourceName, + Strategy: circuitbreaker.ErrorCount, + RetryTimeoutMs: uint32(timeout.Milliseconds()), + MinRequestAmount: uint64(config.MaxFailures), + StatIntervalMs: 1000, // 1秒统计窗口 + Threshold: float64(config.MaxFailures), + }, } - cb.mu.RUnlock() } - return result + + // 加载规则到Sentinel + _, err := circuitbreaker.LoadRules(rule) + if err != nil { + return fmt.Errorf("加载熔断规则失败: %v", err) + } + + // 初始化熔断器信息 + cbInfo := &CircuitBreakerInfo{ + ResourceName: resourceName, + State: StateClosed, + Config: config, + } + circuitBreakers.Store(serviceName, cbInfo) + + strategy := "error_count" + if config.EnableSlidingWindow { + strategy = "slow_ratio" + } + + g.Log().Infof(context.Background(), "服务 %s 熔断器初始化成功: resource=%s, strategy=%s, timeout=%v", + serviceName, resourceName, strategy, timeout) + + return nil } -// getCircuitBreakerConfig 从配置文件读取熔断器配置 -func getCircuitBreakerConfig(ctx context.Context, serviceName string) CircuitBreakerConfig { - return CircuitBreakerConfig{ - MaxFailures: g.Cfg().MustGet(ctx, fmt.Sprintf("circuitBreaker.%s.maxFailures", serviceName), 5).Int(), - Timeout: g.Cfg().MustGet(ctx, fmt.Sprintf("circuitBreaker.%s.timeout", serviceName), "30s").Duration(), - HalfOpenSuccess: g.Cfg().MustGet(ctx, fmt.Sprintf("circuitBreaker.%s.halfOpenSuccess", serviceName), 2).Int(), - EnableDistributed: g.Cfg().MustGet(ctx, "circuitBreaker.enableDistributed", true).Bool(), - } -} - -// CircuitBreakerMiddleware Gateway 熔断中间件 +// CircuitBreakerMiddleware 熔断降级中间件(使用阿里Sentinel) func CircuitBreakerMiddleware(r *ghttp.Request) { - // 从 URL 提取服务名 + // 从URL路径提取服务名 pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") if len(pathParts) == 0 { r.Middleware.Next() @@ -397,149 +225,195 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { } serviceName := pathParts[0] + resourceName := fmt.Sprintf("service:%s", serviceName) - // 跳过非微服务路径 - if serviceName == "health" || serviceName == "metrics" || serviceName == "swagger" { - r.Middleware.Next() + // 检查是否启用分布式熔断 + if enableDistributed { + // 检查Redis中的熔断状态 + if isCircuitBreakerOpenInDistributed(r.GetCtx(), resourceName) { + g.Log().Warningf(r.GetCtx(), "分布式熔断触发: %s", resourceName) + r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 熔断中,请稍后再试", serviceName)) + return + } + } + + // 使用Sentinel进行熔断保护 + entry, blockError := api.Entry(resourceName) + if blockError != nil { + // 被熔断拦截 + g.Log().Warningf(r.GetCtx(), "熔断触发: %s, reason: %v", resourceName, blockError) + + // 更新熔断器状态 + if val, ok := circuitBreakers.Load(serviceName); ok { + cbInfo := val.(*CircuitBreakerInfo) + cbInfo.State = StateOpen + cbInfo.LastOpenTime = time.Now() + if timeout, err := time.ParseDuration(cbInfo.Config.Timeout); err == nil { + cbInfo.NextRetryTime = time.Now().Add(timeout) + } + circuitBreakers.Store(serviceName, cbInfo) + } + + r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 暂时不可用,请稍后再试", serviceName)) return } - // 获取熔断配置 - config := getCircuitBreakerConfig(r.GetCtx(), serviceName) - - // 获取或创建熔断器 - cb := GetOrCreateCircuitBreaker(serviceName, config) - - // 判断是否允许请求通过 - if !cb.AllowRequest(r.GetCtx()) { - // 熔断开启,返回降级响应 - glog.Warningf(r.GetCtx(), "⛔ 服务 %s 熔断中,触发降级", serviceName) - - // 获取降级响应 - fallbackResponse := getFallbackResponse(serviceName) - - // 记录降级事件到 Jaeger - ctx2, span := jaeger.NewSpan(r.GetCtx(), "circuit_breaker_fallback") - span.SetAttributes( - attribute.String("service", serviceName), - attribute.String("fallback_type", "circuit_breaker_open"), - ) - span.End() - trace.SpanFromContext(ctx2).End() - - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 503, - Message: "服务暂时不可用,已自动降级", - Data: fallbackResponse, - }) - return - } - - // 记录请求开始时间 - startTime := time.Now() - - // 执行后续处理(包括代理转发) + // 执行后续中间件和业务逻辑 r.Middleware.Next() - // 根据响应状态记录成功/失败 - elapsed := time.Since(startTime) - if r.Response.Status >= 500 { - cb.RecordFailure(r.GetCtx()) - glog.Warningf(r.GetCtx(), "❌ 服务 %s 请求失败 - 状态码: %d, 耗时: %v", - serviceName, r.Response.Status, elapsed) - } else { - cb.RecordSuccess(r.GetCtx()) - glog.Debugf(r.GetCtx(), "✅ 服务 %s 请求成功 - 状态码: %d, 耗时: %v", - serviceName, r.Response.Status, elapsed) + // 记录请求结果(基于HTTP状态码) + statusCode := r.Response.Status + if !isSuccessStatusCode(resourceName, statusCode) { + // 记录异常 + api.TraceError(entry, fmt.Errorf("request failed with status: %d", statusCode)) } + + // 退出Sentinel资源 + entry.Exit() } -// getFallbackResponse 获取降级响应 -func getFallbackResponse(serviceName string) interface{} { - switch serviceName { - case "customerService": - // 客服服务降级:返回固定话术 - return map[string]interface{}{ - "message": "当前客服系统繁忙,智能助手暂时离线。如需帮助,可拨打:400-xxx-xxxx", - "fallback": true, - "type": "fixed_response", - } - case "order": - // 订单服务降级:返回排队提示 - return map[string]interface{}{ - "message": "订单系统繁忙,您的请求已排队,请稍后刷新查看", - "fallback": true, - "type": "queue_hint", - } - case "assets": - // 资产服务降级:返回缓存数据 - return map[string]interface{}{ - "message": "资产数据正在更新中,显示的是5分钟前的缓存数据", - "fallback": true, - "type": "cached_data", - } - case "wallet": - // 钱包服务降级:返回只读数据 - return map[string]interface{}{ - "message": "钱包服务暂时不可用,无法进行转账操作,查询功能正常", - "fallback": true, - "type": "read_only", - } - case "market": - // 市场服务降级:返回推荐数据 - return map[string]interface{}{ - "message": "市场数据加载中,为您推荐以下热门商品", - "fallback": true, - "type": "recommended", - } - default: - // 默认降级响应 - return map[string]interface{}{ - "message": "服务暂时不可用,请稍后再试", - "fallback": true, - "type": "default", - } +// isSuccessStatusCode 判断HTTP状态码是否成功 +func isSuccessStatusCode(resourceName string, statusCode int) bool { + serviceName := strings.TrimPrefix(resourceName, "service:") + if serviceName == "" { + // 默认只认为2xx是成功 + return statusCode >= 200 && statusCode < 300 } + + // 从配置中获取成功状态码列表 + var serviceConfig *CircuitBreakerConfig + if val, ok := circuitBreakerConfigs.Load(serviceName); ok { + serviceConfig = val.(*CircuitBreakerConfig) + } + + if serviceConfig != nil && len(serviceConfig.SuccessStatusCodes) > 0 { + for _, code := range serviceConfig.SuccessStatusCodes { + if statusCode == code { + return true + } + } + return false + } + + // 默认:2xx状态码为成功 + return statusCode >= 200 && statusCode < 300 +} + +// isCircuitBreakerOpenInDistributed 检查分布式熔断状态 +func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) bool { + key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) + value, err := g.Redis().Get(ctx, key) + if err != nil || value.IsNil() { + return false + } + state := value.String() + return state == "open" } // CircuitBreakerHealthCheckHandler 熔断器健康检查接口 func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { - breakers := GetAllBreakers() + status := make(map[string]interface{}) + + // 遍历所有熔断器 + circuitBreakers.Range(func(key, value interface{}) bool { + serviceName := key.(string) + cbInfo := value.(*CircuitBreakerInfo) + + // 获取Sentinel中的实际状态 + rules := circuitbreaker.GetRulesOfResource(cbInfo.ResourceName) + var stateStr string + if len(rules) > 0 { + stateStr = string(cbInfo.State) + } else { + stateStr = "unknown" + } + + status[serviceName] = map[string]interface{}{ + "resource": cbInfo.ResourceName, + "state": stateStr, + "config": cbInfo.Config, + "lastOpenTime": cbInfo.LastOpenTime, + "nextRetryTime": cbInfo.NextRetryTime, + } + return true + }) r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ Code: 200, - Message: "熔断器状态查询成功", - Data: breakers, + Message: "熔断器状态", + Data: status, }) } -// CircuitBreakerResetHandler 熔断器手动重置接口 +// getSentinelStateString 转换Sentinel状态为字符串 +func getSentinelStateString(state circuitbreaker.State) string { + switch state { + case circuitbreaker.Closed: + return string(StateClosed) + case circuitbreaker.Open: + return string(StateOpen) + case circuitbreaker.HalfOpen: + return string(StateHalfOpen) + default: + return "unknown" + } +} + +// CircuitBreakerResetHandler 熔断器手动重置接口(仅限管理后台调用) func CircuitBreakerResetHandler(r *ghttp.Request) { - serviceName := r.Get("serviceName").String() + serviceName := r.Get("service").String() if serviceName == "" { r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ Code: 400, - Message: "服务名不能为空", + Message: "缺少service参数", }) return } - circuitMu.RLock() - cb, exists := circuitBreakers[serviceName] - circuitMu.RUnlock() + resourceName := fmt.Sprintf("service:%s", serviceName) - if !exists { + // 重置Sentinel规则 - 清空现有规则 + _, err := circuitbreaker.LoadRules([]*circuitbreaker.Rule{}) + if err != nil { r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 404, - Message: fmt.Sprintf("服务 %s 的熔断器不存在", serviceName), + Code: 500, + Message: fmt.Sprintf("重置熔断器失败: %v", err), }) return } - cb.Reset(r.GetCtx()) + // 重新加载规则 + if val, ok := circuitBreakerConfigs.Load(serviceName); ok { + config := val.(*CircuitBreakerConfig) + err = initServiceCircuitBreaker(serviceName, config) + if err != nil { + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 500, + Message: fmt.Sprintf("重置熔断器失败: %v", err), + }) + return + } + } + + // 更新内存状态 + if val, ok := circuitBreakers.Load(serviceName); ok { + cbInfo := val.(*CircuitBreakerInfo) + cbInfo.State = StateClosed + cbInfo.LastOpenTime = time.Time{} + cbInfo.NextRetryTime = time.Time{} + circuitBreakers.Store(serviceName, cbInfo) + } + + // 重置分布式状态(如果启用) + if enableDistributed { + key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) + _, _ = g.Redis().Del(r.GetCtx(), key) + } + + g.Log().Infof(r.GetCtx(), "熔断器已手动重置: %s", resourceName) r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ Code: 200, - Message: fmt.Sprintf("服务 %s 的熔断器已重置", serviceName), + Message: fmt.Sprintf("服务 '%s' 的熔断器已重置", serviceName), }) } From 274bab258bffe89f242288c281882adc4fef8054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 07:38:00 +0800 Subject: [PATCH 124/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 266 ++++++++++++++++++++++++++-------- 1 file changed, 209 insertions(+), 57 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index b28c414..688560f 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "time" "github.com/alibaba/sentinel-golang/api" @@ -34,17 +35,30 @@ type CircuitBreakerConfig struct { EnableSlidingWindow bool // 是否启用滑动窗口 SlidingWindowSize string // 滑动窗口大小 FailureRateThreshold float64 // 失败率阈值 + EnableFallback bool // 是否启用降级 + FallbackMessage string // 降级提示消息 +} + +// CircuitBreakerMetrics 熔断器指标 +type CircuitBreakerMetrics struct { + TotalRequests atomic.Int64 // 总请求数 + PassRequests atomic.Int64 // 通过请求数 + BlockRequests atomic.Int64 // 阻塞请求数 + FailureRequests atomic.Int64 // 失败请求数 + OpenCount atomic.Int64 // 熔断开启次数 } // CircuitBreakerInfo 熔断器信息 type CircuitBreakerInfo struct { - ResourceName string `json:"resourceName"` // 资源名称 - State CircuitBreakerState `json:"state"` // 当前状态 - Config *CircuitBreakerConfig `json:"config"` // 配置信息 - FailCount int64 `json:"failCount"` // 失败次数 - TotalCount int64 `json:"totalCount"` // 总请求数 - LastOpenTime time.Time `json:"lastOpenTime"` // 上次熔断时间 - NextRetryTime time.Time `json:"nextRetryTime"` // 下次重试时间 + ResourceName string `json:"resourceName"` // 资源名称 + State CircuitBreakerState `json:"state"` // 当前状态 + Config *CircuitBreakerConfig `json:"config"` // 配置信息 + FailCount int64 `json:"failCount"` // 失败次数 + TotalCount int64 `json:"totalCount"` // 总请求数 + LastOpenTime time.Time `json:"lastOpenTime"` // 上次熔断时间 + NextRetryTime time.Time `json:"nextRetryTime"` // 下次重试时间 + Metrics *CircuitBreakerMetrics `json:"metrics"` // 指标统计 + mu sync.RWMutex // 保护状态更新 } var ( @@ -54,6 +68,8 @@ var ( enableDistributed = false // circuitBreakerConfigs 熔断器配置缓存 circuitBreakerConfigs sync.Map + // distributedSyncLock 分布式同步锁 + distributedSyncLock sync.Mutex ) // InitCircuitBreaker 初始化Sentinel熔断器 @@ -70,15 +86,17 @@ func InitCircuitBreaker() error { g.Log().Infof(ctx, "Sentinel熔断器初始化成功,分布式熔断: %v", enableDistributed) - // 加载所有服务的熔断器配置 - loadCircuitBreakerConfigs() - - // 为每个服务创建熔断器 - services := []string{ + // 动态从配置文件读取服务列表 + services := g.Cfg().MustGet(ctx, "circuitBreaker.services", []string{ "customerService", "order", "assets", "cid", "oss", "wallet", "market", "knapsack", + }).Strings() + + if len(services) == 0 { + return fmt.Errorf("未配置熔断器服务列表") } + // 为每个服务创建熔断器 for _, service := range services { serviceConfig := loadServiceCircuitBreakerConfig(service) if serviceConfig != nil { @@ -86,25 +104,37 @@ func InitCircuitBreaker() error { initErr := initServiceCircuitBreaker(service, serviceConfig) if initErr != nil { g.Log().Errorf(ctx, "服务 %s 熔断器初始化失败: %v", service, initErr) + } else { + g.Log().Infof(ctx, "服务 %s 熔断器初始化成功", service) } } } + g.Log().Infof(ctx, "共初始化 %d 个服务熔断器", len(services)) return nil } -// loadCircuitBreakerConfigs 加载熔断器配置 -func loadCircuitBreakerConfigs() { - services := []string{ - "customerService", "order", "assets", "cid", "oss", - "wallet", "market", "knapsack", +// ReloadCircuitBreakerConfig 动态重新加载熔断器配置 +func ReloadCircuitBreakerConfig(serviceName string) error { + ctx := context.Background() + + // 重新加载配置 + serviceConfig := loadServiceCircuitBreakerConfig(serviceName) + if serviceConfig == nil { + return fmt.Errorf("未找到服务 %s 的配置", serviceName) } - for _, service := range services { - config := loadServiceCircuitBreakerConfig(service) - if config != nil { - circuitBreakerConfigs.Store(service, config) - } + + // 更新配置缓存 + circuitBreakerConfigs.Store(serviceName, serviceConfig) + + // 重新初始化熔断器 + err := initServiceCircuitBreaker(serviceName, serviceConfig) + if err != nil { + return fmt.Errorf("重新初始化熔断器失败: %v", err) } + + g.Log().Infof(ctx, "服务 %s 熔断器配置重新加载成功", serviceName) + return nil } // loadServiceCircuitBreakerConfig 加载单个服务的熔断器配置 @@ -121,6 +151,8 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { slidingWindowSize := g.Cfg().MustGet(ctx, key+".slidingWindowSize", "60s").String() failureRateThreshold := g.Cfg().MustGet(ctx, key+".failureRateThreshold", 0.5).Float64() halfOpenRequestSampleRate := g.Cfg().MustGet(ctx, key+".halfOpenRequestSampleRate", 1.0).Float64() + enableFallback := g.Cfg().MustGet(ctx, key+".enableFallback", false).Bool() + fallbackMessage := g.Cfg().MustGet(ctx, key+".fallbackMessage", "").String() // 解析成功状态码 successCodes := g.Cfg().MustGet(ctx, key+".successStatusCodes", "200,201,204").String() @@ -137,6 +169,8 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { EnableSlidingWindow: enableSlidingWindow, SlidingWindowSize: slidingWindowSize, FailureRateThreshold: failureRateThreshold, + EnableFallback: enableFallback, + FallbackMessage: fallbackMessage, } } @@ -201,6 +235,7 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) ResourceName: resourceName, State: StateClosed, Config: config, + Metrics: &CircuitBreakerMetrics{}, } circuitBreakers.Store(serviceName, cbInfo) @@ -217,6 +252,8 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) // CircuitBreakerMiddleware 熔断降级中间件(使用阿里Sentinel) func CircuitBreakerMiddleware(r *ghttp.Request) { + startTime := time.Now() + // 从URL路径提取服务名 pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") if len(pathParts) == 0 { @@ -227,12 +264,23 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { serviceName := pathParts[0] resourceName := fmt.Sprintf("service:%s", serviceName) + // 获取熔断器信息 + val, ok := circuitBreakers.Load(serviceName) + if !ok { + // 未配置熔断器,直接放行 + r.Middleware.Next() + return + } + + cbInfo := val.(*CircuitBreakerInfo) + cbInfo.Metrics.TotalRequests.Add(1) + // 检查是否启用分布式熔断 if enableDistributed { - // 检查Redis中的熔断状态 if isCircuitBreakerOpenInDistributed(r.GetCtx(), resourceName) { + cbInfo.Metrics.BlockRequests.Add(1) g.Log().Warningf(r.GetCtx(), "分布式熔断触发: %s", resourceName) - r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 熔断中,请稍后再试", serviceName)) + sendFallbackResponse(r, serviceName, cbInfo.Config) return } } @@ -241,20 +289,25 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { entry, blockError := api.Entry(resourceName) if blockError != nil { // 被熔断拦截 + cbInfo.Metrics.BlockRequests.Add(1) + cbInfo.Metrics.OpenCount.Add(1) g.Log().Warningf(r.GetCtx(), "熔断触发: %s, reason: %v", resourceName, blockError) // 更新熔断器状态 - if val, ok := circuitBreakers.Load(serviceName); ok { - cbInfo := val.(*CircuitBreakerInfo) - cbInfo.State = StateOpen - cbInfo.LastOpenTime = time.Now() - if timeout, err := time.ParseDuration(cbInfo.Config.Timeout); err == nil { - cbInfo.NextRetryTime = time.Now().Add(timeout) - } - circuitBreakers.Store(serviceName, cbInfo) + cbInfo.mu.Lock() + cbInfo.State = StateOpen + cbInfo.LastOpenTime = time.Now() + if timeout, err := time.ParseDuration(cbInfo.Config.Timeout); err == nil { + cbInfo.NextRetryTime = time.Now().Add(timeout) + } + cbInfo.mu.Unlock() + + // 同步到分布式存储 + if enableDistributed { + syncCircuitBreakerStateToDistributed(r.GetCtx(), resourceName, "open") } - r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 暂时不可用,请稍后再试", serviceName)) + sendFallbackResponse(r, serviceName, cbInfo.Config) return } @@ -263,15 +316,32 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { // 记录请求结果(基于HTTP状态码) statusCode := r.Response.Status + duration := time.Since(startTime) + if !isSuccessStatusCode(resourceName, statusCode) { // 记录异常 + cbInfo.Metrics.FailureRequests.Add(1) api.TraceError(entry, fmt.Errorf("request failed with status: %d", statusCode)) + g.Log().Debugf(r.GetCtx(), "服务 %s 请求失败: status=%d, duration=%v", serviceName, statusCode, duration) + } else { + cbInfo.Metrics.PassRequests.Add(1) } // 退出Sentinel资源 entry.Exit() } +// sendFallbackResponse 发送降级响应 +func sendFallbackResponse(r *ghttp.Request, serviceName string, config *CircuitBreakerConfig) { + if config.EnableFallback && config.FallbackMessage != "" { + // 自定义降级消息 + r.Response.WriteStatusExit(503, config.FallbackMessage) + } else { + // 默认消息 + r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 暂时不可用,请稍后再试", serviceName)) + } +} + // isSuccessStatusCode 判断HTTP状态码是否成功 func isSuccessStatusCode(resourceName string, statusCode int) bool { serviceName := strings.TrimPrefix(resourceName, "service:") @@ -310,38 +380,67 @@ func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) return state == "open" } +// syncCircuitBreakerStateToDistributed 同步熔断器状态到分布式存储 +func syncCircuitBreakerStateToDistributed(ctx context.Context, resourceName, state string) { + distributedSyncLock.Lock() + defer distributedSyncLock.Unlock() + + key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) + // 设置过期时间为5分钟,使用SetEX + _, err := g.Redis().Do(ctx, "SETEX", key, 300, state) + if err != nil { + g.Log().Errorf(ctx, "同步熔断状态到Redis失败: %v", err) + } +} + // CircuitBreakerHealthCheckHandler 熔断器健康检查接口 func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { status := make(map[string]interface{}) + totalServices := 0 + openServices := 0 // 遍历所有熔断器 circuitBreakers.Range(func(key, value interface{}) bool { serviceName := key.(string) cbInfo := value.(*CircuitBreakerInfo) - // 获取Sentinel中的实际状态 - rules := circuitbreaker.GetRulesOfResource(cbInfo.ResourceName) - var stateStr string - if len(rules) > 0 { - stateStr = string(cbInfo.State) - } else { - stateStr = "unknown" + totalServices++ + cbInfo.mu.RLock() + isOpen := cbInfo.State == StateOpen + if isOpen { + openServices++ } status[serviceName] = map[string]interface{}{ - "resource": cbInfo.ResourceName, - "state": stateStr, - "config": cbInfo.Config, - "lastOpenTime": cbInfo.LastOpenTime, - "nextRetryTime": cbInfo.NextRetryTime, + "resource": cbInfo.ResourceName, + "state": string(cbInfo.State), + "lastOpenTime": cbInfo.LastOpenTime, + "nextRetryTime": cbInfo.NextRetryTime, + "totalRequests": cbInfo.Metrics.TotalRequests.Load(), + "passRequests": cbInfo.Metrics.PassRequests.Load(), + "blockRequests": cbInfo.Metrics.BlockRequests.Load(), + "failureRequests": cbInfo.Metrics.FailureRequests.Load(), + "openCount": cbInfo.Metrics.OpenCount.Load(), } + cbInfo.mu.RUnlock() + return true }) + summary := map[string]interface{}{ + "totalServices": totalServices, + "openServices": openServices, + "closedServices": totalServices - openServices, + "distributed": enableDistributed, + } + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ Code: 200, Message: "熔断器状态", - Data: status, + Data: map[string]interface{}{ + "summary": summary, + "services": status, + }, }) } @@ -372,20 +471,25 @@ func CircuitBreakerResetHandler(r *ghttp.Request) { resourceName := fmt.Sprintf("service:%s", serviceName) - // 重置Sentinel规则 - 清空现有规则 - _, err := circuitbreaker.LoadRules([]*circuitbreaker.Rule{}) - if err != nil { - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 500, - Message: fmt.Sprintf("重置熔断器失败: %v", err), - }) - return + // 获取当前服务的所有规则 + currentRules := circuitbreaker.GetRulesOfResource(resourceName) + + // 只删除当前服务的规则 + if len(currentRules) > 0 { + _, err := circuitbreaker.LoadRulesOfResource(resourceName, []*circuitbreaker.Rule{}) + if err != nil { + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 500, + Message: fmt.Sprintf("重置熔断器失败: %v", err), + }) + return + } } - // 重新加载规则 + // 重新加载该服务的规则 if val, ok := circuitBreakerConfigs.Load(serviceName); ok { config := val.(*CircuitBreakerConfig) - err = initServiceCircuitBreaker(serviceName, config) + err := initServiceCircuitBreaker(serviceName, config) if err != nil { r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ Code: 500, @@ -398,10 +502,11 @@ func CircuitBreakerResetHandler(r *ghttp.Request) { // 更新内存状态 if val, ok := circuitBreakers.Load(serviceName); ok { cbInfo := val.(*CircuitBreakerInfo) + cbInfo.mu.Lock() cbInfo.State = StateClosed cbInfo.LastOpenTime = time.Time{} cbInfo.NextRetryTime = time.Time{} - circuitBreakers.Store(serviceName, cbInfo) + cbInfo.mu.Unlock() } // 重置分布式状态(如果启用) @@ -417,3 +522,50 @@ func CircuitBreakerResetHandler(r *ghttp.Request) { Message: fmt.Sprintf("服务 '%s' 的熔断器已重置", serviceName), }) } + +// CircuitBreakerReloadHandler 熔断器配置重载接口 +func CircuitBreakerReloadHandler(r *ghttp.Request) { + serviceName := r.Get("service").String() + + if serviceName == "" { + // 重载所有服务 + services := g.Cfg().MustGet(r.GetCtx(), "circuitBreaker.services", []string{}).Strings() + successCount := 0 + failCount := 0 + + for _, service := range services { + err := ReloadCircuitBreakerConfig(service) + if err != nil { + g.Log().Errorf(r.GetCtx(), "服务 %s 配置重载失败: %v", service, err) + failCount++ + } else { + successCount++ + } + } + + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 200, + Message: fmt.Sprintf("配置重载完成: 成功 %d, 失败 %d", successCount, failCount), + Data: map[string]interface{}{ + "success": successCount, + "failed": failCount, + }, + }) + return + } + + // 重载单个服务 + err := ReloadCircuitBreakerConfig(serviceName) + if err != nil { + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 500, + Message: fmt.Sprintf("重载失败: %v", err), + }) + return + } + + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 200, + Message: fmt.Sprintf("服务 '%s' 的熔断器配置已重载", serviceName), + }) +} From 60902a9839cf65702fca062cce0f75da8168f5c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 10:37:01 +0800 Subject: [PATCH 125/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 327 +++++++++++++++++++++++----------- 1 file changed, 225 insertions(+), 102 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 688560f..e025b56 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -25,18 +25,17 @@ const ( // CircuitBreakerConfig 熔断器配置 type CircuitBreakerConfig struct { - MaxFailures int // 连续失败次数 - Timeout string // 熔断超时时间 - HalfOpenSuccess int // 半开状态连续成功次数 - SuccessStatusCodes []int // 视为成功的HTTP状态码 - SlowRequestThreshold string // 慢请求阈值 - HalfOpenRequestSampleRate float64 // 半开状态请求采样率 - Dimension string // 熔断器维度: service/ip/user - EnableSlidingWindow bool // 是否启用滑动窗口 - SlidingWindowSize string // 滑动窗口大小 - FailureRateThreshold float64 // 失败率阈值 - EnableFallback bool // 是否启用降级 - FallbackMessage string // 降级提示消息 + Enabled bool // 是否启用熔断器 + MaxFailures int // 连续失败次数 + Timeout string // 熔断超时时间 + SuccessStatusCodes []int // 视为成功的HTTP状态码 + SlowRequestThreshold string // 慢请求阈值 + EnableSlidingWindow bool // 是否启用滑动窗口 + FailureRateThreshold float64 // 失败率阈值 + EnableFallback bool // 是否启用降级 + FallbackMessage string // 降级提示消息 + RequestTimeout int // 请求超时时间(毫秒) + DistributedTTL int // 分布式熔断状态TTL(秒) } // CircuitBreakerMetrics 熔断器指标 @@ -64,19 +63,17 @@ type CircuitBreakerInfo struct { var ( // circuitBreakers 存储所有熔断器状态(用于健康检查) circuitBreakers sync.Map - // enableDistributed 是否启用分布式熔断 - enableDistributed = false // circuitBreakerConfigs 熔断器配置缓存 circuitBreakerConfigs sync.Map // distributedSyncLock 分布式同步锁 distributedSyncLock sync.Mutex + // stateChangeListeners 状态变化监听器 + stateChangeListeners sync.Map ) // InitCircuitBreaker 初始化Sentinel熔断器 func InitCircuitBreaker() error { ctx := context.Background() - // 从配置文件读取是否启用分布式熔断 - enableDistributed = g.Cfg().MustGet(ctx, "circuitBreaker.enableDistributed", false).Bool() // 初始化Sentinel err := api.InitDefault() @@ -84,33 +81,46 @@ func InitCircuitBreaker() error { return fmt.Errorf("Sentinel初始化失败: %v", err) } - g.Log().Infof(ctx, "Sentinel熔断器初始化成功,分布式熔断: %v", enableDistributed) + // 注册熔断器状态变化监听器 + registerStateChangeListeners() - // 动态从配置文件读取服务列表 - services := g.Cfg().MustGet(ctx, "circuitBreaker.services", []string{ - "customerService", "order", "assets", "cid", "oss", - "wallet", "market", "knapsack", - }).Strings() + g.Log().Infof(ctx, "Sentinel熔断器初始化成功") - if len(services) == 0 { - return fmt.Errorf("未配置熔断器服务列表") - } + // 扫描配置文件中所有配置了熔断器的服务 + services := g.Cfg().MustGet(ctx, "circuitBreaker").Map() - // 为每个服务创建熔断器 - for _, service := range services { - serviceConfig := loadServiceCircuitBreakerConfig(service) - if serviceConfig != nil { - circuitBreakerConfigs.Store(service, serviceConfig) - initErr := initServiceCircuitBreaker(service, serviceConfig) - if initErr != nil { - g.Log().Errorf(ctx, "服务 %s 熔断器初始化失败: %v", service, initErr) - } else { - g.Log().Infof(ctx, "服务 %s 熔断器初始化成功", service) - } + // 过滤掉非服务配置的key + serviceNames := make([]string, 0) + for key := range services { + if key != "services" && key != "enableDistributed" && key != "requestTimeout" && key != "distributedTTL" { + serviceNames = append(serviceNames, key) } } - g.Log().Infof(ctx, "共初始化 %d 个服务熔断器", len(services)) + if len(serviceNames) == 0 { + g.Log().Infof(ctx, "未配置任何服务熔断器") + return nil + } + + // 为每个服务创建熔断器 + enabledCount := 0 + for _, serviceName := range serviceNames { + serviceConfig := loadServiceCircuitBreakerConfig(serviceName) + if serviceConfig != nil && serviceConfig.Enabled { + circuitBreakerConfigs.Store(serviceName, serviceConfig) + initErr := initServiceCircuitBreaker(serviceName, serviceConfig) + if initErr != nil { + g.Log().Errorf(ctx, "服务 %s 熔断器初始化失败: %v", serviceName, initErr) + } else { + g.Log().Infof(ctx, "服务 %s 熔断器初始化成功", serviceName) + enabledCount++ + } + } else { + g.Log().Infof(ctx, "服务 %s 熔断器未启用", serviceName) + } + } + + g.Log().Infof(ctx, "共初始化 %d 个服务熔断器,其中 %d 个已启用", len(serviceNames), enabledCount) return nil } @@ -142,35 +152,33 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { ctx := context.Background() key := fmt.Sprintf("circuitBreaker.%s", serviceName) + enabled := g.Cfg().MustGet(ctx, key+".enabled", true).Bool() maxFailures := g.Cfg().MustGet(ctx, key+".maxFailures", 5).Int() timeout := g.Cfg().MustGet(ctx, key+".timeout", "60s").String() - halfOpenSuccess := g.Cfg().MustGet(ctx, key+".halfOpenSuccess", 2).Int() slowRequestThreshold := g.Cfg().MustGet(ctx, key+".slowRequestThreshold", "3s").String() - dimension := g.Cfg().MustGet(ctx, key+".dimension", "service").String() enableSlidingWindow := g.Cfg().MustGet(ctx, key+".enableSlidingWindow", false).Bool() - slidingWindowSize := g.Cfg().MustGet(ctx, key+".slidingWindowSize", "60s").String() failureRateThreshold := g.Cfg().MustGet(ctx, key+".failureRateThreshold", 0.5).Float64() - halfOpenRequestSampleRate := g.Cfg().MustGet(ctx, key+".halfOpenRequestSampleRate", 1.0).Float64() enableFallback := g.Cfg().MustGet(ctx, key+".enableFallback", false).Bool() fallbackMessage := g.Cfg().MustGet(ctx, key+".fallbackMessage", "").String() + requestTimeout := g.Cfg().MustGet(ctx, key+".requestTimeout", 30000).Int() + distributedTTL := g.Cfg().MustGet(ctx, key+".distributedTTL", 300).Int() // 解析成功状态码 successCodes := g.Cfg().MustGet(ctx, key+".successStatusCodes", "200,201,204").String() statusCodes := parseStatusCodes(successCodes) return &CircuitBreakerConfig{ - MaxFailures: maxFailures, - Timeout: timeout, - HalfOpenSuccess: halfOpenSuccess, - SuccessStatusCodes: statusCodes, - SlowRequestThreshold: slowRequestThreshold, - HalfOpenRequestSampleRate: halfOpenRequestSampleRate, - Dimension: dimension, - EnableSlidingWindow: enableSlidingWindow, - SlidingWindowSize: slidingWindowSize, - FailureRateThreshold: failureRateThreshold, - EnableFallback: enableFallback, - FallbackMessage: fallbackMessage, + Enabled: enabled, + MaxFailures: maxFailures, + Timeout: timeout, + SuccessStatusCodes: statusCodes, + SlowRequestThreshold: slowRequestThreshold, + EnableSlidingWindow: enableSlidingWindow, + FailureRateThreshold: failureRateThreshold, + EnableFallback: enableFallback, + FallbackMessage: fallbackMessage, + RequestTimeout: requestTimeout, + DistributedTTL: distributedTTL, } } @@ -189,9 +197,20 @@ func parseStatusCodes(str string) []int { // initServiceCircuitBreaker 初始化服务熔断器 func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) error { - timeout, _ := time.ParseDuration(config.Timeout) - slowRequestThreshold, _ := time.ParseDuration(config.SlowRequestThreshold) - _, _ = time.ParseDuration(config.SlidingWindowSize) + // 验证配置参数 + if err := validateCircuitBreakerConfig(config); err != nil { + return fmt.Errorf("配置验证失败: %v", err) + } + + timeout, err := time.ParseDuration(config.Timeout) + if err != nil { + return fmt.Errorf("解析超时时间失败: %v", err) + } + + slowRequestThreshold, err := time.ParseDuration(config.SlowRequestThreshold) + if err != nil { + return fmt.Errorf("解析慢请求阈值失败: %v", err) + } resourceName := fmt.Sprintf("service:%s", serviceName) @@ -225,7 +244,7 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) } // 加载规则到Sentinel - _, err := circuitbreaker.LoadRules(rule) + _, err = circuitbreaker.LoadRules(rule) if err != nil { return fmt.Errorf("加载熔断规则失败: %v", err) } @@ -244,8 +263,8 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) strategy = "slow_ratio" } - g.Log().Infof(context.Background(), "服务 %s 熔断器初始化成功: resource=%s, strategy=%s, timeout=%v", - serviceName, resourceName, strategy, timeout) + g.Log().Infof(context.Background(), "服务 %s 熔断器初始化成功: resource=%s, strategy=%s, timeout=%v, threshold=%.2f", + serviceName, resourceName, strategy, timeout, rule[0].Threshold) return nil } @@ -253,6 +272,7 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) // CircuitBreakerMiddleware 熔断降级中间件(使用阿里Sentinel) func CircuitBreakerMiddleware(r *ghttp.Request) { startTime := time.Now() + ctx := r.GetCtx() // 从URL路径提取服务名 pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") @@ -262,25 +282,47 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { } serviceName := pathParts[0] - resourceName := fmt.Sprintf("service:%s", serviceName) - // 获取熔断器信息 - val, ok := circuitBreakers.Load(serviceName) + // 获取熔断器配置 + val, ok := circuitBreakerConfigs.Load(serviceName) if !ok { // 未配置熔断器,直接放行 r.Middleware.Next() return } - cbInfo := val.(*CircuitBreakerInfo) + config := val.(*CircuitBreakerConfig) + if !config.Enabled { + // 熔断器未启用,直接放行 + r.Middleware.Next() + return + } + + // 获取熔断器信息 + cbInfoVal, ok := circuitBreakers.Load(serviceName) + if !ok { + r.Middleware.Next() + return + } + cbInfo := cbInfoVal.(*CircuitBreakerInfo) cbInfo.Metrics.TotalRequests.Add(1) + // 设置请求超时(使用服务独立配置) + if config.RequestTimeout > 0 { + ctx, cancel := context.WithTimeout(ctx, time.Duration(config.RequestTimeout)*time.Millisecond) + r.SetCtx(ctx) + defer cancel() + } + + resourceName := fmt.Sprintf("service:%s", serviceName) + // 检查是否启用分布式熔断 - if enableDistributed { - if isCircuitBreakerOpenInDistributed(r.GetCtx(), resourceName) { + if config.DistributedTTL > 0 { + if isCircuitBreakerOpenInDistributed(ctx, resourceName) { cbInfo.Metrics.BlockRequests.Add(1) - g.Log().Warningf(r.GetCtx(), "分布式熔断触发: %s", resourceName) - sendFallbackResponse(r, serviceName, cbInfo.Config) + g.Log().Warningf(ctx, "分布式熔断触发: %s", resourceName) + notifyStateChange(serviceName, StateOpen, StateOpen) + sendFallbackResponse(r, serviceName, config, "distributed") return } } @@ -291,23 +333,29 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { // 被熔断拦截 cbInfo.Metrics.BlockRequests.Add(1) cbInfo.Metrics.OpenCount.Add(1) - g.Log().Warningf(r.GetCtx(), "熔断触发: %s, reason: %v", resourceName, blockError) + g.Log().Warningf(ctx, "熔断触发: %s, reason: %v", resourceName, blockError) // 更新熔断器状态 cbInfo.mu.Lock() + oldState := cbInfo.State cbInfo.State = StateOpen cbInfo.LastOpenTime = time.Now() - if timeout, err := time.ParseDuration(cbInfo.Config.Timeout); err == nil { + if timeout, err := time.ParseDuration(config.Timeout); err == nil { cbInfo.NextRetryTime = time.Now().Add(timeout) } cbInfo.mu.Unlock() - // 同步到分布式存储 - if enableDistributed { - syncCircuitBreakerStateToDistributed(r.GetCtx(), resourceName, "open") + // 通知状态变化(如果状态改变) + if oldState != StateOpen { + notifyStateChange(serviceName, oldState, StateOpen) } - sendFallbackResponse(r, serviceName, cbInfo.Config) + // 同步到分布式存储 + if config.DistributedTTL > 0 { + syncCircuitBreakerStateToDistributed(ctx, resourceName, "open", config.DistributedTTL) + } + + sendFallbackResponse(r, serviceName, config, "blocked") return } @@ -322,9 +370,17 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { // 记录异常 cbInfo.Metrics.FailureRequests.Add(1) api.TraceError(entry, fmt.Errorf("request failed with status: %d", statusCode)) - g.Log().Debugf(r.GetCtx(), "服务 %s 请求失败: status=%d, duration=%v", serviceName, statusCode, duration) + g.Log().Debugf(ctx, "服务 %s 请求失败: status=%d, duration=%v", serviceName, statusCode, duration) } else { cbInfo.Metrics.PassRequests.Add(1) + // 更新状态为关闭(如果之前是开启状态) + cbInfo.mu.Lock() + oldState := cbInfo.State + if cbInfo.State != StateClosed { + cbInfo.State = StateClosed + notifyStateChange(serviceName, oldState, StateClosed) + } + cbInfo.mu.Unlock() } // 退出Sentinel资源 @@ -332,13 +388,22 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { } // sendFallbackResponse 发送降级响应 -func sendFallbackResponse(r *ghttp.Request, serviceName string, config *CircuitBreakerConfig) { +func sendFallbackResponse(r *ghttp.Request, serviceName string, config *CircuitBreakerConfig, reason string) { if config.EnableFallback && config.FallbackMessage != "" { // 自定义降级消息 r.Response.WriteStatusExit(503, config.FallbackMessage) } else { - // 默认消息 - r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 暂时不可用,请稍后再试", serviceName)) + // 根据原因返回不同的状态码和消息 + switch reason { + case "timeout": + r.Response.WriteStatusExit(504, fmt.Sprintf("服务 '%s' 响应超时", serviceName)) + case "blocked": + r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 熔断保护中,请稍后再试", serviceName)) + case "distributed": + r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 分布式熔断中", serviceName)) + default: + r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 暂时不可用,请稍后再试", serviceName)) + } } } @@ -372,7 +437,13 @@ func isSuccessStatusCode(resourceName string, statusCode int) bool { // isCircuitBreakerOpenInDistributed 检查分布式熔断状态 func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) bool { key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) - value, err := g.Redis().Get(ctx, key) + + redis := g.Redis() + if redis == nil { + return false + } + + value, err := redis.Get(ctx, key) if err != nil || value.IsNil() { return false } @@ -381,18 +452,70 @@ func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) } // syncCircuitBreakerStateToDistributed 同步熔断器状态到分布式存储 -func syncCircuitBreakerStateToDistributed(ctx context.Context, resourceName, state string) { +func syncCircuitBreakerStateToDistributed(ctx context.Context, resourceName, state string, ttl int) { distributedSyncLock.Lock() defer distributedSyncLock.Unlock() key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) - // 设置过期时间为5分钟,使用SetEX - _, err := g.Redis().Do(ctx, "SETEX", key, 300, state) + + redis := g.Redis() + if redis == nil { + g.Log().Errorf(ctx, "Redis客户端未初始化,无法同步熔断状态") + return + } + + _, err := redis.Do(ctx, "SETEX", key, ttl, state) if err != nil { g.Log().Errorf(ctx, "同步熔断状态到Redis失败: %v", err) } } +// validateCircuitBreakerConfig 验证熔断器配置 +func validateCircuitBreakerConfig(config *CircuitBreakerConfig) error { + if config.MaxFailures <= 0 { + return fmt.Errorf("maxFailures必须大于0") + } + if config.FailureRateThreshold < 0 || config.FailureRateThreshold > 1 { + return fmt.Errorf("failureRateThreshold必须在0.0-1.0之间") + } + if len(config.SuccessStatusCodes) == 0 { + return fmt.Errorf("successStatusCodes不能为空") + } + return nil +} + +// registerStateChangeListeners 注册状态变化监听器 +func registerStateChangeListeners() { + // 示例:注册默认监听器 + RegisterStateChangeListener("default", func(serviceName string, fromState, toState CircuitBreakerState) { + g.Log().Infof(context.Background(), "熔断器状态变化: service=%s, %s -> %s", + serviceName, fromState, toState) + }) +} + +// StateChangeListener 状态变化监听器类型 +type StateChangeListener func(serviceName string, fromState, toState CircuitBreakerState) + +// RegisterStateChangeListener 注册状态变化监听器 +func RegisterStateChangeListener(name string, listener StateChangeListener) { + stateChangeListeners.Store(name, listener) +} + +// UnregisterStateChangeListener 取消注册状态变化监听器 +func UnregisterStateChangeListener(name string) { + stateChangeListeners.Delete(name) +} + +// notifyStateChange 通知所有监听器状态变化 +func notifyStateChange(serviceName string, fromState, toState CircuitBreakerState) { + stateChangeListeners.Range(func(key, value interface{}) bool { + if listener, ok := value.(StateChangeListener); ok { + listener(serviceName, fromState, toState) + } + return true + }) +} + // CircuitBreakerHealthCheckHandler 熔断器健康检查接口 func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { status := make(map[string]interface{}) @@ -431,7 +554,6 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { "totalServices": totalServices, "openServices": openServices, "closedServices": totalServices - openServices, - "distributed": enableDistributed, } r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ @@ -444,20 +566,6 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { }) } -// getSentinelStateString 转换Sentinel状态为字符串 -func getSentinelStateString(state circuitbreaker.State) string { - switch state { - case circuitbreaker.Closed: - return string(StateClosed) - case circuitbreaker.Open: - return string(StateOpen) - case circuitbreaker.HalfOpen: - return string(StateHalfOpen) - default: - return "unknown" - } -} - // CircuitBreakerResetHandler 熔断器手动重置接口(仅限管理后台调用) func CircuitBreakerResetHandler(r *ghttp.Request) { serviceName := r.Get("service").String() @@ -510,9 +618,15 @@ func CircuitBreakerResetHandler(r *ghttp.Request) { } // 重置分布式状态(如果启用) - if enableDistributed { - key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) - _, _ = g.Redis().Del(r.GetCtx(), key) + if val, ok := circuitBreakerConfigs.Load(serviceName); ok { + config := val.(*CircuitBreakerConfig) + if config.DistributedTTL > 0 { + key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) + redis := g.Redis() + if redis != nil { + _, _ = redis.Del(r.GetCtx(), key) + } + } } g.Log().Infof(r.GetCtx(), "熔断器已手动重置: %s", resourceName) @@ -528,12 +642,21 @@ func CircuitBreakerReloadHandler(r *ghttp.Request) { serviceName := r.Get("service").String() if serviceName == "" { - // 重载所有服务 - services := g.Cfg().MustGet(r.GetCtx(), "circuitBreaker.services", []string{}).Strings() + // 重载所有服务 - 扫描配置文件中所有服务 + services := g.Cfg().MustGet(r.GetCtx(), "circuitBreaker").Map() + + // 过滤出服务名 + serviceNames := make([]string, 0) + for key := range services { + if key != "services" && key != "enableDistributed" && key != "requestTimeout" && key != "distributedTTL" { + serviceNames = append(serviceNames, key) + } + } + successCount := 0 failCount := 0 - for _, service := range services { + for _, service := range serviceNames { err := ReloadCircuitBreakerConfig(service) if err != nil { g.Log().Errorf(r.GetCtx(), "服务 %s 配置重载失败: %v", service, err) From 456bee2cae955930dc6739fc129eae6d20e70e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 10:48:47 +0800 Subject: [PATCH 126/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 168 ++++++++++++++++++++++------------ 1 file changed, 107 insertions(+), 61 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index e025b56..58a2ee9 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -18,24 +18,25 @@ import ( type CircuitBreakerState string const ( - StateClosed CircuitBreakerState = "closed" // 关闭:正常状态 - StateOpen CircuitBreakerState = "open" // 开启:熔断状态 - StateHalfOpen CircuitBreakerState = "half-open" // 半开:尝试恢复状态 + StateClosed CircuitBreakerState = "closed" // 关闭:正常状态 + StateOpen CircuitBreakerState = "open" // 开启:熔断状态 ) // CircuitBreakerConfig 熔断器配置 type CircuitBreakerConfig struct { - Enabled bool // 是否启用熔断器 - MaxFailures int // 连续失败次数 - Timeout string // 熔断超时时间 - SuccessStatusCodes []int // 视为成功的HTTP状态码 - SlowRequestThreshold string // 慢请求阈值 - EnableSlidingWindow bool // 是否启用滑动窗口 - FailureRateThreshold float64 // 失败率阈值 - EnableFallback bool // 是否启用降级 - FallbackMessage string // 降级提示消息 - RequestTimeout int // 请求超时时间(毫秒) - DistributedTTL int // 分布式熔断状态TTL(秒) + Enabled bool // 是否启用熔断器 + MaxFailures int // 连续失败次数 + Timeout string // 熔断超时时间 + TimeoutParsed time.Duration // 缓存的超时时间(性能优化) + SuccessStatusCodes []int // 视为成功的HTTP状态码 + SlowRequestThreshold string // 慢请求阈值 + SlowRequestThresholdParsed time.Duration // 缓存的慢请求阈值(性能优化) + EnableSlidingWindow bool // 是否启用滑动窗口 + FailureRateThreshold float64 // 失败率阈值 + EnableFallback bool // 是否启用降级 + FallbackMessage string // 降级提示消息 + RequestTimeout int // 请求超时时间(毫秒) + DistributedTTL int // 分布式熔断状态TTL(秒) } // CircuitBreakerMetrics 熔断器指标 @@ -52,8 +53,6 @@ type CircuitBreakerInfo struct { ResourceName string `json:"resourceName"` // 资源名称 State CircuitBreakerState `json:"state"` // 当前状态 Config *CircuitBreakerConfig `json:"config"` // 配置信息 - FailCount int64 `json:"failCount"` // 失败次数 - TotalCount int64 `json:"totalCount"` // 总请求数 LastOpenTime time.Time `json:"lastOpenTime"` // 上次熔断时间 NextRetryTime time.Time `json:"nextRetryTime"` // 下次重试时间 Metrics *CircuitBreakerMetrics `json:"metrics"` // 指标统计 @@ -65,10 +64,12 @@ var ( circuitBreakers sync.Map // circuitBreakerConfigs 熔断器配置缓存 circuitBreakerConfigs sync.Map - // distributedSyncLock 分布式同步锁 - distributedSyncLock sync.Mutex + // distributedSyncLocks 分布式同步锁(按服务名分片) + distributedSyncLocks sync.Map // stateChangeListeners 状态变化监听器 stateChangeListeners sync.Map + // stateChangeListenersRegistered 默认监听器是否已注册 + stateChangeListenersRegistered sync.Map ) // InitCircuitBreaker 初始化Sentinel熔断器 @@ -78,7 +79,7 @@ func InitCircuitBreaker() error { // 初始化Sentinel err := api.InitDefault() if err != nil { - return fmt.Errorf("Sentinel初始化失败: %v", err) + return fmt.Errorf("sentinel初始化失败: %v", err) } // 注册熔断器状态变化监听器 @@ -90,12 +91,7 @@ func InitCircuitBreaker() error { services := g.Cfg().MustGet(ctx, "circuitBreaker").Map() // 过滤掉非服务配置的key - serviceNames := make([]string, 0) - for key := range services { - if key != "services" && key != "enableDistributed" && key != "requestTimeout" && key != "distributedTTL" { - serviceNames = append(serviceNames, key) - } - } + serviceNames := filterServiceNames(services) if len(serviceNames) == 0 { g.Log().Infof(ctx, "未配置任何服务熔断器") @@ -167,18 +163,33 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { successCodes := g.Cfg().MustGet(ctx, key+".successStatusCodes", "200,201,204").String() statusCodes := parseStatusCodes(successCodes) + // 解析时间(缓存结果,性能优化) + timeoutParsed, err := time.ParseDuration(timeout) + if err != nil { + timeoutParsed = 60 * time.Second + g.Log().Warningf(ctx, "服务 %s 的 timeout 解析失败,使用默认值: %v", serviceName, err) + } + + slowRequestThresholdParsed, err := time.ParseDuration(slowRequestThreshold) + if err != nil { + slowRequestThresholdParsed = 3 * time.Second + g.Log().Warningf(ctx, "服务 %s 的 slowRequestThreshold 解析失败,使用默认值: %v", serviceName, err) + } + return &CircuitBreakerConfig{ - Enabled: enabled, - MaxFailures: maxFailures, - Timeout: timeout, - SuccessStatusCodes: statusCodes, - SlowRequestThreshold: slowRequestThreshold, - EnableSlidingWindow: enableSlidingWindow, - FailureRateThreshold: failureRateThreshold, - EnableFallback: enableFallback, - FallbackMessage: fallbackMessage, - RequestTimeout: requestTimeout, - DistributedTTL: distributedTTL, + Enabled: enabled, + MaxFailures: maxFailures, + Timeout: timeout, + TimeoutParsed: timeoutParsed, + SuccessStatusCodes: statusCodes, + SlowRequestThreshold: slowRequestThreshold, + SlowRequestThresholdParsed: slowRequestThresholdParsed, + EnableSlidingWindow: enableSlidingWindow, + FailureRateThreshold: failureRateThreshold, + EnableFallback: enableFallback, + FallbackMessage: fallbackMessage, + RequestTimeout: requestTimeout, + DistributedTTL: distributedTTL, } } @@ -195,6 +206,24 @@ func parseStatusCodes(str string) []int { return codes } +// filterServiceNames 过滤服务名(排除非服务配置的key) +func filterServiceNames(services map[string]interface{}) []string { + excludeKeys := map[string]bool{ + "services": true, + "enableDistributed": true, + "requestTimeout": true, + "distributedTTL": true, + } + + serviceNames := make([]string, 0, len(services)) + for key := range services { + if !excludeKeys[key] { + serviceNames = append(serviceNames, key) + } + } + return serviceNames +} + // initServiceCircuitBreaker 初始化服务熔断器 func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) error { // 验证配置参数 @@ -202,15 +231,9 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) return fmt.Errorf("配置验证失败: %v", err) } - timeout, err := time.ParseDuration(config.Timeout) - if err != nil { - return fmt.Errorf("解析超时时间失败: %v", err) - } - - slowRequestThreshold, err := time.ParseDuration(config.SlowRequestThreshold) - if err != nil { - return fmt.Errorf("解析慢请求阈值失败: %v", err) - } + // 使用缓存的时间值(性能优化) + timeout := config.TimeoutParsed + slowRequestThreshold := config.SlowRequestThresholdParsed resourceName := fmt.Sprintf("service:%s", serviceName) @@ -243,8 +266,11 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) } } - // 加载规则到Sentinel - _, err = circuitbreaker.LoadRules(rule) + // 先清理旧规则(健壮性改进) + _, _ = circuitbreaker.LoadRulesOfResource(resourceName, []*circuitbreaker.Rule{}) + + // 加载新规则到Sentinel + _, err := circuitbreaker.LoadRules(rule) if err != nil { return fmt.Errorf("加载熔断规则失败: %v", err) } @@ -340,9 +366,8 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { oldState := cbInfo.State cbInfo.State = StateOpen cbInfo.LastOpenTime = time.Now() - if timeout, err := time.ParseDuration(config.Timeout); err == nil { - cbInfo.NextRetryTime = time.Now().Add(timeout) - } + // 使用缓存的时间值(性能优化) + cbInfo.NextRetryTime = time.Now().Add(config.TimeoutParsed) cbInfo.mu.Unlock() // 通知状态变化(如果状态改变) @@ -395,8 +420,6 @@ func sendFallbackResponse(r *ghttp.Request, serviceName string, config *CircuitB } else { // 根据原因返回不同的状态码和消息 switch reason { - case "timeout": - r.Response.WriteStatusExit(504, fmt.Sprintf("服务 '%s' 响应超时", serviceName)) case "blocked": r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 熔断保护中,请稍后再试", serviceName)) case "distributed": @@ -451,10 +474,19 @@ func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) return state == "open" } +// getDistributedLock 获取分布式锁(按服务名分片) +func getDistributedLock(serviceName string) *sync.Mutex { + lock, _ := distributedSyncLocks.LoadOrStore(serviceName, &sync.Mutex{}) + return lock.(*sync.Mutex) +} + // syncCircuitBreakerStateToDistributed 同步熔断器状态到分布式存储 func syncCircuitBreakerStateToDistributed(ctx context.Context, resourceName, state string, ttl int) { - distributedSyncLock.Lock() - defer distributedSyncLock.Unlock() + // 提取服务名用于锁分片 + serviceName := strings.TrimPrefix(resourceName, "service:") + lock := getDistributedLock(serviceName) + lock.Lock() + defer lock.Unlock() key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) @@ -481,12 +513,30 @@ func validateCircuitBreakerConfig(config *CircuitBreakerConfig) error { if len(config.SuccessStatusCodes) == 0 { return fmt.Errorf("successStatusCodes不能为空") } + if config.RequestTimeout < 0 || config.RequestTimeout > 300000 { + return fmt.Errorf("requestTimeout必须在0-300000毫秒之间") + } + if config.DistributedTTL < 0 || config.DistributedTTL > 3600 { + return fmt.Errorf("distributedTTL必须在0-3600秒之间") + } + // 验证时间字符串格式(如果缓存为空,说明解析失败) + if config.TimeoutParsed == 0 { + return fmt.Errorf("timeout格式错误,应为有效的时间字符串(如30s, 1m)") + } + if config.SlowRequestThresholdParsed == 0 { + return fmt.Errorf("slowRequestThreshold格式错误,应为有效的时间字符串(如3s, 1m)") + } return nil } // registerStateChangeListeners 注册状态变化监听器 func registerStateChangeListeners() { - // 示例:注册默认监听器 + // 检查是否已注册,防止重复注册(健壮性改进) + if _, exists := stateChangeListenersRegistered.LoadOrStore("default", true); exists { + return + } + + // 注册默认监听器 RegisterStateChangeListener("default", func(serviceName string, fromState, toState CircuitBreakerState) { g.Log().Infof(context.Background(), "熔断器状态变化: service=%s, %s -> %s", serviceName, fromState, toState) @@ -534,6 +584,7 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { openServices++ } + // 从Metrics中读取数据(修复数据准确性问题) status[serviceName] = map[string]interface{}{ "resource": cbInfo.ResourceName, "state": string(cbInfo.State), @@ -646,12 +697,7 @@ func CircuitBreakerReloadHandler(r *ghttp.Request) { services := g.Cfg().MustGet(r.GetCtx(), "circuitBreaker").Map() // 过滤出服务名 - serviceNames := make([]string, 0) - for key := range services { - if key != "services" && key != "enableDistributed" && key != "requestTimeout" && key != "distributedTTL" { - serviceNames = append(serviceNames, key) - } - } + serviceNames := filterServiceNames(services) successCount := 0 failCount := 0 From b1f52bb3288405ad2a5b645784a11154bbac6ff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 11:08:55 +0800 Subject: [PATCH 127/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 144 ++++++++++++++++++++++++++++------ 1 file changed, 118 insertions(+), 26 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 58a2ee9..703e236 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -37,6 +37,7 @@ type CircuitBreakerConfig struct { FallbackMessage string // 降级提示消息 RequestTimeout int // 请求超时时间(毫秒) DistributedTTL int // 分布式熔断状态TTL(秒) + AdminIPs []string // 允许重置熔断器的管理员IP列表 } // CircuitBreakerMetrics 熔断器指标 @@ -46,6 +47,7 @@ type CircuitBreakerMetrics struct { BlockRequests atomic.Int64 // 阻塞请求数 FailureRequests atomic.Int64 // 失败请求数 OpenCount atomic.Int64 // 熔断开启次数 + LastResetTime atomic.Int64 // 上次重置时间(Unix时间戳) } // CircuitBreakerInfo 熔断器信息 @@ -158,6 +160,7 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { fallbackMessage := g.Cfg().MustGet(ctx, key+".fallbackMessage", "").String() requestTimeout := g.Cfg().MustGet(ctx, key+".requestTimeout", 30000).Int() distributedTTL := g.Cfg().MustGet(ctx, key+".distributedTTL", 300).Int() + adminIPs := g.Cfg().MustGet(ctx, key+".adminIPs", "").String() // 解析成功状态码 successCodes := g.Cfg().MustGet(ctx, key+".successStatusCodes", "200,201,204").String() @@ -176,6 +179,9 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { g.Log().Warningf(ctx, "服务 %s 的 slowRequestThreshold 解析失败,使用默认值: %v", serviceName, err) } + // 解析管理员IP列表 + adminIPList := parseAdminIPs(adminIPs) + return &CircuitBreakerConfig{ Enabled: enabled, MaxFailures: maxFailures, @@ -190,6 +196,7 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { FallbackMessage: fallbackMessage, RequestTimeout: requestTimeout, DistributedTTL: distributedTTL, + AdminIPs: adminIPList, } } @@ -206,6 +213,22 @@ func parseStatusCodes(str string) []int { return codes } +// parseAdminIPs 解析管理员IP列表 +func parseAdminIPs(str string) []string { + if str == "" { + return nil + } + parts := strings.Split(str, ",") + ips := make([]string, 0, len(parts)) + for _, part := range parts { + ip := strings.TrimSpace(part) + if ip != "" { + ips = append(ips, ip) + } + } + return ips +} + // filterServiceNames 过滤服务名(排除非服务配置的key) func filterServiceNames(services map[string]interface{}) []string { excludeKeys := map[string]bool{ @@ -282,6 +305,7 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) Config: config, Metrics: &CircuitBreakerMetrics{}, } + cbInfo.Metrics.LastResetTime.Store(time.Now().Unix()) circuitBreakers.Store(serviceName, cbInfo) strategy := "error_count" @@ -300,15 +324,13 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { startTime := time.Now() ctx := r.GetCtx() - // 从URL路径提取服务名 - pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") - if len(pathParts) == 0 { + // 从URL路径提取服务名(改进提取逻辑) + serviceName := extractServiceName(r.URL.Path) + if serviceName == "" { r.Middleware.Next() return } - serviceName := pathParts[0] - // 获取熔断器配置 val, ok := circuitBreakerConfigs.Load(serviceName) if !ok { @@ -391,7 +413,8 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { statusCode := r.Response.Status duration := time.Since(startTime) - if !isSuccessStatusCode(resourceName, statusCode) { + // 使用提前获取的config判断状态码(性能优化) + if !isSuccessStatusCode(config, statusCode) { // 记录异常 cbInfo.Metrics.FailureRequests.Add(1) api.TraceError(entry, fmt.Errorf("request failed with status: %d", statusCode)) @@ -431,32 +454,39 @@ func sendFallbackResponse(r *ghttp.Request, serviceName string, config *CircuitB } // isSuccessStatusCode 判断HTTP状态码是否成功 -func isSuccessStatusCode(resourceName string, statusCode int) bool { - serviceName := strings.TrimPrefix(resourceName, "service:") - if serviceName == "" { - // 默认只认为2xx是成功 - return statusCode >= 200 && statusCode < 300 - } - - // 从配置中获取成功状态码列表 - var serviceConfig *CircuitBreakerConfig - if val, ok := circuitBreakerConfigs.Load(serviceName); ok { - serviceConfig = val.(*CircuitBreakerConfig) - } - - if serviceConfig != nil && len(serviceConfig.SuccessStatusCodes) > 0 { - for _, code := range serviceConfig.SuccessStatusCodes { +func isSuccessStatusCode(config *CircuitBreakerConfig, statusCode int) bool { + if len(config.SuccessStatusCodes) > 0 { + for _, code := range config.SuccessStatusCodes { if statusCode == code { return true } } return false } - // 默认:2xx状态码为成功 return statusCode >= 200 && statusCode < 300 } +// extractServiceName 从URL路径提取服务名(改进提取逻辑) +func extractServiceName(path string) string { + // 去除首尾斜杠并分割 + path = strings.Trim(path, "/") + if path == "" { + return "" + } + parts := strings.Split(path, "/") + if len(parts) == 0 { + return "" + } + serviceName := parts[0] + + // 验证服务名是否在已配置的熔断器中 + if _, ok := circuitBreakerConfigs.Load(serviceName); ok { + return serviceName + } + return "" +} + // isCircuitBreakerOpenInDistributed 检查分布式熔断状态 func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) bool { key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) @@ -536,10 +566,16 @@ func registerStateChangeListeners() { return } - // 注册默认监听器 + // 注册默认监听器(区分日志级别) RegisterStateChangeListener("default", func(serviceName string, fromState, toState CircuitBreakerState) { - g.Log().Infof(context.Background(), "熔断器状态变化: service=%s, %s -> %s", - serviceName, fromState, toState) + // Open状态使用Warning级别,Closed状态使用Info级别 + if toState == StateOpen { + g.Log().Warningf(context.Background(), "熔断器状态变化: service=%s, %s -> %s", + serviceName, fromState, toState) + } else { + g.Log().Infof(context.Background(), "熔断器状态变化: service=%s, %s -> %s", + serviceName, fromState, toState) + } }) } @@ -585,6 +621,12 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { } // 从Metrics中读取数据(修复数据准确性问题) + lastResetTime := cbInfo.Metrics.LastResetTime.Load() + var lastResetTimeStr string + if lastResetTime > 0 { + lastResetTimeStr = time.Unix(lastResetTime, 0).Format("2006-01-02 15:04:05") + } + status[serviceName] = map[string]interface{}{ "resource": cbInfo.ResourceName, "state": string(cbInfo.State), @@ -595,6 +637,7 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { "blockRequests": cbInfo.Metrics.BlockRequests.Load(), "failureRequests": cbInfo.Metrics.FailureRequests.Load(), "openCount": cbInfo.Metrics.OpenCount.Load(), + "lastResetTime": lastResetTimeStr, } cbInfo.mu.RUnlock() @@ -617,6 +660,39 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { }) } +// isAdminIP 检查请求IP是否在管理员白名单中 +func isAdminIP(r *ghttp.Request) bool { + clientIP := r.GetClientIp() + if clientIP == "" { + return false + } + + // 检查所有服务的adminIPs配置 + var allowedIPs []string + circuitBreakerConfigs.Range(func(key, value interface{}) bool { + config := value.(*CircuitBreakerConfig) + if len(config.AdminIPs) > 0 { + allowedIPs = append(allowedIPs, config.AdminIPs...) + } + return true + }) + + // 如果没有配置白名单,允许所有IP(向后兼容) + if len(allowedIPs) == 0 { + return true + } + + // 检查IP是否在白名单中 + for _, allowedIP := range allowedIPs { + if clientIP == allowedIP { + return true + } + } + + g.Log().Warningf(r.GetCtx(), "熔断器重置请求被拒绝,IP不在白名单中: %s", clientIP) + return false +} + // CircuitBreakerResetHandler 熔断器手动重置接口(仅限管理后台调用) func CircuitBreakerResetHandler(r *ghttp.Request) { serviceName := r.Get("service").String() @@ -628,6 +704,15 @@ func CircuitBreakerResetHandler(r *ghttp.Request) { return } + // 权限验证:检查IP是否在白名单中 + if !isAdminIP(r) { + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 403, + Message: "权限不足,禁止访问", + }) + return + } + resourceName := fmt.Sprintf("service:%s", serviceName) // 获取当前服务的所有规则 @@ -658,13 +743,20 @@ func CircuitBreakerResetHandler(r *ghttp.Request) { } } - // 更新内存状态 + // 更新内存状态并重置指标 if val, ok := circuitBreakers.Load(serviceName); ok { cbInfo := val.(*CircuitBreakerInfo) cbInfo.mu.Lock() cbInfo.State = StateClosed cbInfo.LastOpenTime = time.Time{} cbInfo.NextRetryTime = time.Time{} + // 重置指标 + cbInfo.Metrics.TotalRequests.Store(0) + cbInfo.Metrics.PassRequests.Store(0) + cbInfo.Metrics.BlockRequests.Store(0) + cbInfo.Metrics.FailureRequests.Store(0) + cbInfo.Metrics.OpenCount.Store(0) + cbInfo.Metrics.LastResetTime.Store(time.Now().Unix()) cbInfo.mu.Unlock() } From 1e88347e2c859fa341c0613f270dd6b0957ac264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 11:15:53 +0800 Subject: [PATCH 128/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 104 ++++++++++++++++++++++++++-------- 1 file changed, 80 insertions(+), 24 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 703e236..36c1832 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -38,6 +38,8 @@ type CircuitBreakerConfig struct { RequestTimeout int // 请求超时时间(毫秒) DistributedTTL int // 分布式熔断状态TTL(秒) AdminIPs []string // 允许重置熔断器的管理员IP列表 + StatIntervalMs int // 统计窗口时长(毫秒),默认1000ms + MinRequestAmount int // 最小请求数量,默认与MaxFailures相同 } // CircuitBreakerMetrics 熔断器指标 @@ -72,6 +74,10 @@ var ( stateChangeListeners sync.Map // stateChangeListenersRegistered 默认监听器是否已注册 stateChangeListenersRegistered sync.Map + // allowedAdminIPsCache 缓存的所有管理员IP白名单(性能优化) + allowedAdminIPsCache []string + // allowedAdminIPsCacheMutex 保护白名单缓存的并发访问 + allowedAdminIPsCacheMutex sync.RWMutex ) // InitCircuitBreaker 初始化Sentinel熔断器 @@ -89,6 +95,9 @@ func InitCircuitBreaker() error { g.Log().Infof(ctx, "Sentinel熔断器初始化成功") + // 更新管理员IP白名单缓存 + updateAdminIPsCache() + // 扫描配置文件中所有配置了熔断器的服务 services := g.Cfg().MustGet(ctx, "circuitBreaker").Map() @@ -161,6 +170,8 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { requestTimeout := g.Cfg().MustGet(ctx, key+".requestTimeout", 30000).Int() distributedTTL := g.Cfg().MustGet(ctx, key+".distributedTTL", 300).Int() adminIPs := g.Cfg().MustGet(ctx, key+".adminIPs", "").String() + statIntervalMs := g.Cfg().MustGet(ctx, key+".statIntervalMs", 1000).Int() + minRequestAmount := g.Cfg().MustGet(ctx, key+".minRequestAmount", 0).Int() // 解析成功状态码 successCodes := g.Cfg().MustGet(ctx, key+".successStatusCodes", "200,201,204").String() @@ -182,6 +193,11 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { // 解析管理员IP列表 adminIPList := parseAdminIPs(adminIPs) + // 如果minRequestAmount未配置,则使用maxFailures作为默认值 + if minRequestAmount == 0 { + minRequestAmount = maxFailures + } + return &CircuitBreakerConfig{ Enabled: enabled, MaxFailures: maxFailures, @@ -197,6 +213,8 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { RequestTimeout: requestTimeout, DistributedTTL: distributedTTL, AdminIPs: adminIPList, + StatIntervalMs: statIntervalMs, + MinRequestAmount: minRequestAmount, } } @@ -213,6 +231,31 @@ func parseStatusCodes(str string) []int { return codes } +// updateAdminIPsCache 更新管理员IP白名单缓存(性能优化) +func updateAdminIPsCache() { + var ipList []string + ipSet := make(map[string]bool) + + // 收集所有服务的adminIPs配置 + circuitBreakerConfigs.Range(func(key, value interface{}) bool { + config := value.(*CircuitBreakerConfig) + if len(config.AdminIPs) > 0 { + for _, ip := range config.AdminIPs { + if !ipSet[ip] { + ipSet[ip] = true + ipList = append(ipList, ip) + } + } + } + return true + }) + + // 更新缓存 + allowedAdminIPsCacheMutex.Lock() + allowedAdminIPsCache = ipList + allowedAdminIPsCacheMutex.Unlock() +} + // parseAdminIPs 解析管理员IP列表 func parseAdminIPs(str string) []string { if str == "" { @@ -268,8 +311,8 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) Resource: resourceName, Strategy: circuitbreaker.SlowRequestRatio, RetryTimeoutMs: uint32(timeout.Milliseconds()), - MinRequestAmount: uint64(config.MaxFailures), - StatIntervalMs: 1000, + MinRequestAmount: uint64(config.MinRequestAmount), + StatIntervalMs: uint32(config.StatIntervalMs), StatSlidingWindowBucketCount: 10, MaxAllowedRtMs: uint64(slowRequestThreshold.Milliseconds()), Threshold: config.FailureRateThreshold, @@ -282,8 +325,8 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) Resource: resourceName, Strategy: circuitbreaker.ErrorCount, RetryTimeoutMs: uint32(timeout.Milliseconds()), - MinRequestAmount: uint64(config.MaxFailures), - StatIntervalMs: 1000, // 1秒统计窗口 + MinRequestAmount: uint64(config.MinRequestAmount), + StatIntervalMs: uint32(config.StatIntervalMs), Threshold: float64(config.MaxFailures), }, } @@ -355,6 +398,9 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { cbInfo := cbInfoVal.(*CircuitBreakerInfo) cbInfo.Metrics.TotalRequests.Add(1) + // 提前构造resourceName(性能优化) + resourceName := fmt.Sprintf("service:%s", serviceName) + // 设置请求超时(使用服务独立配置) if config.RequestTimeout > 0 { ctx, cancel := context.WithTimeout(ctx, time.Duration(config.RequestTimeout)*time.Millisecond) @@ -362,14 +408,11 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { defer cancel() } - resourceName := fmt.Sprintf("service:%s", serviceName) - // 检查是否启用分布式熔断 if config.DistributedTTL > 0 { if isCircuitBreakerOpenInDistributed(ctx, resourceName) { cbInfo.Metrics.BlockRequests.Add(1) g.Log().Warningf(ctx, "分布式熔断触发: %s", resourceName) - notifyStateChange(serviceName, StateOpen, StateOpen) sendFallbackResponse(r, serviceName, config, "distributed") return } @@ -535,26 +578,32 @@ func syncCircuitBreakerStateToDistributed(ctx context.Context, resourceName, sta // validateCircuitBreakerConfig 验证熔断器配置 func validateCircuitBreakerConfig(config *CircuitBreakerConfig) error { if config.MaxFailures <= 0 { - return fmt.Errorf("maxFailures必须大于0") + return fmt.Errorf("maxFailures必须大于0,当前值: %d", config.MaxFailures) } if config.FailureRateThreshold < 0 || config.FailureRateThreshold > 1 { - return fmt.Errorf("failureRateThreshold必须在0.0-1.0之间") + return fmt.Errorf("failureRateThreshold必须在0.0-1.0之间,当前值: %.2f", config.FailureRateThreshold) } if len(config.SuccessStatusCodes) == 0 { return fmt.Errorf("successStatusCodes不能为空") } if config.RequestTimeout < 0 || config.RequestTimeout > 300000 { - return fmt.Errorf("requestTimeout必须在0-300000毫秒之间") + return fmt.Errorf("requestTimeout必须在0-300000毫秒之间,当前值: %d", config.RequestTimeout) } if config.DistributedTTL < 0 || config.DistributedTTL > 3600 { - return fmt.Errorf("distributedTTL必须在0-3600秒之间") + return fmt.Errorf("distributedTTL必须在0-3600秒之间,当前值: %d", config.DistributedTTL) + } + if config.StatIntervalMs < 100 || config.StatIntervalMs > 60000 { + return fmt.Errorf("statIntervalMs必须在100-60000毫秒之间,当前值: %d", config.StatIntervalMs) + } + if config.MinRequestAmount < 1 || config.MinRequestAmount > 10000 { + return fmt.Errorf("minRequestAmount必须在1-10000之间,当前值: %d", config.MinRequestAmount) } // 验证时间字符串格式(如果缓存为空,说明解析失败) if config.TimeoutParsed == 0 { - return fmt.Errorf("timeout格式错误,应为有效的时间字符串(如30s, 1m)") + return fmt.Errorf("timeout格式错误,应为有效的时间字符串(如30s, 1m),当前值: %s", config.Timeout) } if config.SlowRequestThresholdParsed == 0 { - return fmt.Errorf("slowRequestThreshold格式错误,应为有效的时间字符串(如3s, 1m)") + return fmt.Errorf("slowRequestThreshold格式错误,应为有效的时间字符串(如3s, 1m),当前值: %s", config.SlowRequestThreshold) } return nil } @@ -660,22 +709,17 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { }) } -// isAdminIP 检查请求IP是否在管理员白名单中 +// isAdminIP 检查请求IP是否在管理员白名单中(使用缓存优化性能) func isAdminIP(r *ghttp.Request) bool { clientIP := r.GetClientIp() if clientIP == "" { return false } - // 检查所有服务的adminIPs配置 - var allowedIPs []string - circuitBreakerConfigs.Range(func(key, value interface{}) bool { - config := value.(*CircuitBreakerConfig) - if len(config.AdminIPs) > 0 { - allowedIPs = append(allowedIPs, config.AdminIPs...) - } - return true - }) + // 读取缓存的白名单(性能优化) + allowedAdminIPsCacheMutex.RLock() + allowedIPs := allowedAdminIPsCache + allowedAdminIPsCacheMutex.RUnlock() // 如果没有配置白名单,允许所有IP(向后兼容) if len(allowedIPs) == 0 { @@ -689,7 +733,7 @@ func isAdminIP(r *ghttp.Request) bool { } } - g.Log().Warningf(r.GetCtx(), "熔断器重置请求被拒绝,IP不在白名单中: %s", clientIP) + g.Log().Warningf(r.GetCtx(), "熔断器操作请求被拒绝,IP不在白名单中: %s", clientIP) return false } @@ -784,6 +828,15 @@ func CircuitBreakerResetHandler(r *ghttp.Request) { func CircuitBreakerReloadHandler(r *ghttp.Request) { serviceName := r.Get("service").String() + // 权限验证:检查IP是否在白名单中(P0级别安全问题) + if !isAdminIP(r) { + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 403, + Message: "权限不足,禁止访问", + }) + return + } + if serviceName == "" { // 重载所有服务 - 扫描配置文件中所有服务 services := g.Cfg().MustGet(r.GetCtx(), "circuitBreaker").Map() @@ -804,6 +857,9 @@ func CircuitBreakerReloadHandler(r *ghttp.Request) { } } + // 更新管理员IP白名单缓存 + updateAdminIPsCache() + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ Code: 200, Message: fmt.Sprintf("配置重载完成: 成功 %d, 失败 %d", successCount, failCount), From a4a999b6449daad6761d4c70a08717002fa7fc4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 12:05:12 +0800 Subject: [PATCH 129/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 444 ++++++++++++++++++++++++++-------- 1 file changed, 343 insertions(+), 101 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 36c1832..4ca4f0e 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -3,6 +3,7 @@ package middleware import ( "context" "fmt" + "net" "strings" "sync" "sync/atomic" @@ -22,6 +23,35 @@ const ( StateOpen CircuitBreakerState = "open" // 开启:熔断状态 ) +// 熔断器状态常量(用于atomic.Int64) +const ( + stateClosed int64 = 0 + stateOpen int64 = 1 +) + +// getState 获取熔断器状态字符串 +func (cb *CircuitBreakerInfo) getState() CircuitBreakerState { + if cb.State.Load() == stateOpen { + return StateOpen + } + return StateClosed +} + +// setState 设置熔断器状态(atomic操作,无锁) +func (cb *CircuitBreakerInfo) setState(state CircuitBreakerState) CircuitBreakerState { + var newState int64 + if state == StateOpen { + newState = stateOpen + } else { + newState = stateClosed + } + oldState := cb.State.Swap(newState) + if oldState == stateOpen { + return StateOpen + } + return StateClosed +} + // CircuitBreakerConfig 熔断器配置 type CircuitBreakerConfig struct { Enabled bool // 是否启用熔断器 @@ -40,6 +70,9 @@ type CircuitBreakerConfig struct { AdminIPs []string // 允许重置熔断器的管理员IP列表 StatIntervalMs int // 统计窗口时长(毫秒),默认1000ms MinRequestAmount int // 最小请求数量,默认与MaxFailures相同 + AdminCIDRs []string // 允许重置熔断器的管理员CIDR列表(P0:支持IP段) + // P1:预编译的CIDR网络掩码(性能优化) + CIDRNetMasks []*net.IPNet } // CircuitBreakerMetrics 熔断器指标 @@ -48,19 +81,24 @@ type CircuitBreakerMetrics struct { PassRequests atomic.Int64 // 通过请求数 BlockRequests atomic.Int64 // 阻塞请求数 FailureRequests atomic.Int64 // 失败请求数 + SlowRequests atomic.Int64 // 慢请求数(P2:可观测性) OpenCount atomic.Int64 // 熔断开启次数 LastResetTime atomic.Int64 // 上次重置时间(Unix时间戳) + // 使用atomic.Int64实现简单的时间戳存储,避免使用mutex + LastOpenTime atomic.Int64 // 上次熔断时间(Unix时间戳) + NextRetryTime atomic.Int64 // 下次重试时间(Unix时间戳) } // CircuitBreakerInfo 熔断器信息 type CircuitBreakerInfo struct { - ResourceName string `json:"resourceName"` // 资源名称 - State CircuitBreakerState `json:"state"` // 当前状态 - Config *CircuitBreakerConfig `json:"config"` // 配置信息 - LastOpenTime time.Time `json:"lastOpenTime"` // 上次熔断时间 - NextRetryTime time.Time `json:"nextRetryTime"` // 下次重试时间 - Metrics *CircuitBreakerMetrics `json:"metrics"` // 指标统计 - mu sync.RWMutex // 保护状态更新 + ResourceName string `json:"resourceName"` // 资源名称 + State atomic.Int64 `json:"state"` // 当前状态(0:closed, 1:open),使用atomic避免mutex + Config *CircuitBreakerConfig `json:"config"` // 配置信息 + Metrics *CircuitBreakerMetrics `json:"metrics"` // 指标统计 + // 预编译的成功状态码集合(P1:性能优化) + SuccessCodeMap map[int]bool + // P1:预编译的CIDR网络掩码(避免重复解析) + CIDRNetMasks []*net.IPNet } var ( @@ -74,10 +112,16 @@ var ( stateChangeListeners sync.Map // stateChangeListenersRegistered 默认监听器是否已注册 stateChangeListenersRegistered sync.Map - // allowedAdminIPsCache 缓存的所有管理员IP白名单(性能优化) - allowedAdminIPsCache []string - // allowedAdminIPsCacheMutex 保护白名单缓存的并发访问 - allowedAdminIPsCacheMutex sync.RWMutex + // P1:使用map代替slice优化IP查找性能 + allowedAdminIPsMap map[string]bool + // allowedAdminIPsMutex 保护白名单缓存的并发访问 + allowedAdminIPsMutex sync.RWMutex + // P1:预编译的CIDR网络掩码列表 + allowedAdminCIDRs []*net.IPNet + // allowedAdminCIDRsMutex 保护CIDR列表的并发访问 + allowedAdminCIDRsMutex sync.RWMutex + // totalServicesCount 缓存总服务数(P1:性能优化) + totalServicesCount atomic.Int64 ) // InitCircuitBreaker 初始化Sentinel熔断器 @@ -95,9 +139,6 @@ func InitCircuitBreaker() error { g.Log().Infof(ctx, "Sentinel熔断器初始化成功") - // 更新管理员IP白名单缓存 - updateAdminIPsCache() - // 扫描配置文件中所有配置了熔断器的服务 services := g.Cfg().MustGet(ctx, "circuitBreaker").Map() @@ -109,6 +150,9 @@ func InitCircuitBreaker() error { return nil } + // P1:缓存总服务数 + totalServicesCount.Store(int64(len(serviceNames))) + // 为每个服务创建熔断器 enabledCount := 0 for _, serviceName := range serviceNames { @@ -127,6 +171,9 @@ func InitCircuitBreaker() error { } } + // P1:更新管理员IP白名单缓存(在所有服务配置加载完成后) + updateAdminIPsCache() + g.Log().Infof(ctx, "共初始化 %d 个服务熔断器,其中 %d 个已启用", len(serviceNames), enabledCount) return nil } @@ -170,12 +217,13 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { requestTimeout := g.Cfg().MustGet(ctx, key+".requestTimeout", 30000).Int() distributedTTL := g.Cfg().MustGet(ctx, key+".distributedTTL", 300).Int() adminIPs := g.Cfg().MustGet(ctx, key+".adminIPs", "").String() + adminCIDRs := g.Cfg().MustGet(ctx, key+".adminCIDRs", "").String() // P0:支持CIDR statIntervalMs := g.Cfg().MustGet(ctx, key+".statIntervalMs", 1000).Int() minRequestAmount := g.Cfg().MustGet(ctx, key+".minRequestAmount", 0).Int() - // 解析成功状态码 + // 解析成功状态码(使用map用于快速查找) successCodes := g.Cfg().MustGet(ctx, key+".successStatusCodes", "200,201,204").String() - statusCodes := parseStatusCodes(successCodes) + statusCodes := parseStatusCodesSlice(successCodes) // 解析时间(缓存结果,性能优化) timeoutParsed, err := time.ParseDuration(timeout) @@ -193,6 +241,13 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { // 解析管理员IP列表 adminIPList := parseAdminIPs(adminIPs) + // P1:预编译CIDR为net.IPNet(支持IPv4和IPv6) + cidrNets, err := parseAdminCIDRs(adminCIDRs) + if err != nil { + g.Log().Warningf(ctx, "服务 %s 的 adminCIDRs 解析失败: %v", serviceName, err) + cidrNets = nil + } + // 如果minRequestAmount未配置,则使用maxFailures作为默认值 if minRequestAmount == 0 { minRequestAmount = maxFailures @@ -213,13 +268,27 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { RequestTimeout: requestTimeout, DistributedTTL: distributedTTL, AdminIPs: adminIPList, + CIDRNetMasks: cidrNets, // P1:预编译的CIDR网络掩码 StatIntervalMs: statIntervalMs, MinRequestAmount: minRequestAmount, } } -// parseStatusCodes 解析HTTP状态码 -func parseStatusCodes(str string) []int { +// parseStatusCodes 解析HTTP状态码(返回map用于快速查找) +func parseStatusCodes(str string) map[int]bool { + parts := strings.Split(str, ",") + codeMap := make(map[int]bool, len(parts)) + for _, part := range parts { + var code int + if _, err := fmt.Sscanf(strings.TrimSpace(part), "%d", &code); err == nil { + codeMap[code] = true + } + } + return codeMap +} + +// parseStatusCodesSlice 解析HTTP状态码(返回切片用于配置) +func parseStatusCodesSlice(str string) []int { parts := strings.Split(str, ",") codes := make([]int, 0, len(parts)) for _, part := range parts { @@ -231,29 +300,68 @@ func parseStatusCodes(str string) []int { return codes } -// updateAdminIPsCache 更新管理员IP白名单缓存(性能优化) +// parseAdminCIDRs 解析管理员CIDR列表(P1:预编译为net.IPNet,P0:支持IPv6) +func parseAdminCIDRs(str string) ([]*net.IPNet, error) { + if str == "" { + return nil, nil + } + parts := strings.Split(str, ",") + nets := make([]*net.IPNet, 0, len(parts)) + for _, part := range parts { + cidr := strings.TrimSpace(part) + if cidr != "" { + // 使用net.ParseCIDR解析CIDR(支持IPv4和IPv6) + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + return nil, fmt.Errorf("解析CIDR失败: %s, error: %v", cidr, err) + } + nets = append(nets, ipNet) + } + } + return nets, nil +} + +// updateAdminIPsCache 更新管理员IP白名单缓存(P1:使用map优化性能,P1:预编译CIDR) func updateAdminIPsCache() { - var ipList []string - ipSet := make(map[string]bool) + ipMap := make(map[string]bool) + cidrNets := make([]*net.IPNet, 0) // 收集所有服务的adminIPs配置 circuitBreakerConfigs.Range(func(key, value interface{}) bool { config := value.(*CircuitBreakerConfig) if len(config.AdminIPs) > 0 { for _, ip := range config.AdminIPs { - if !ipSet[ip] { - ipSet[ip] = true - ipList = append(ipList, ip) + if !ipMap[ip] { + ipMap[ip] = true } } } + // P1:使用预编译的CIDR网络掩码 + if len(config.CIDRNetMasks) > 0 { + for _, cidrNet := range config.CIDRNetMasks { + cidrNets = append(cidrNets, cidrNet) + } + } return true }) // 更新缓存 - allowedAdminIPsCacheMutex.Lock() - allowedAdminIPsCache = ipList - allowedAdminIPsCacheMutex.Unlock() + allowedAdminIPsMutex.Lock() + allowedAdminIPsMap = ipMap + allowedAdminIPsMutex.Unlock() + + allowedAdminCIDRsMutex.Lock() + allowedAdminCIDRs = cidrNets + allowedAdminCIDRsMutex.Unlock() +} + +// isIPInCIDR 检查IP是否在CIDR范围内(P1:使用预编译的net.IPNet,P0:支持IPv6) +func isIPInCIDR(ipStr string, cidrNet *net.IPNet) bool { + ip := net.ParseIP(ipStr) + if ip == nil { + return false + } + return cidrNet.Contains(ip) } // parseAdminIPs 解析管理员IP列表 @@ -341,13 +449,20 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) return fmt.Errorf("加载熔断规则失败: %v", err) } - // 初始化熔断器信息 - cbInfo := &CircuitBreakerInfo{ - ResourceName: resourceName, - State: StateClosed, - Config: config, - Metrics: &CircuitBreakerMetrics{}, + // 初始化熔断器信息(P1:直接从slice构建map,避免重复解析) + successCodeMap := make(map[int]bool, len(config.SuccessStatusCodes)) + for _, code := range config.SuccessStatusCodes { + successCodeMap[code] = true } + + cbInfo := &CircuitBreakerInfo{ + ResourceName: resourceName, + Config: config, + Metrics: &CircuitBreakerMetrics{}, + SuccessCodeMap: successCodeMap, + CIDRNetMasks: config.CIDRNetMasks, + } + cbInfo.State.Store(stateClosed) cbInfo.Metrics.LastResetTime.Store(time.Now().Unix()) circuitBreakers.Store(serviceName, cbInfo) @@ -367,39 +482,32 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { startTime := time.Now() ctx := r.GetCtx() - // 从URL路径提取服务名(改进提取逻辑) + // 从URL路径提取服务名并获取配置(P1:合并重复验证) serviceName := extractServiceName(r.URL.Path) if serviceName == "" { r.Middleware.Next() return } - // 获取熔断器配置 - val, ok := circuitBreakerConfigs.Load(serviceName) + // 获取熔断器信息(包含配置) + cbInfoVal, ok := circuitBreakers.Load(serviceName) if !ok { // 未配置熔断器,直接放行 r.Middleware.Next() return } - config := val.(*CircuitBreakerConfig) + cbInfo := cbInfoVal.(*CircuitBreakerInfo) + config := cbInfo.Config if !config.Enabled { // 熔断器未启用,直接放行 r.Middleware.Next() return } - - // 获取熔断器信息 - cbInfoVal, ok := circuitBreakers.Load(serviceName) - if !ok { - r.Middleware.Next() - return - } - cbInfo := cbInfoVal.(*CircuitBreakerInfo) cbInfo.Metrics.TotalRequests.Add(1) // 提前构造resourceName(性能优化) - resourceName := fmt.Sprintf("service:%s", serviceName) + resourceName := cbInfo.ResourceName // 设置请求超时(使用服务独立配置) if config.RequestTimeout > 0 { @@ -426,18 +534,15 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { cbInfo.Metrics.OpenCount.Add(1) g.Log().Warningf(ctx, "熔断触发: %s, reason: %v", resourceName, blockError) - // 更新熔断器状态 - cbInfo.mu.Lock() - oldState := cbInfo.State - cbInfo.State = StateOpen - cbInfo.LastOpenTime = time.Now() - // 使用缓存的时间值(性能优化) - cbInfo.NextRetryTime = time.Now().Add(config.TimeoutParsed) - cbInfo.mu.Unlock() + // 使用atomic更新状态(无锁) + oldStateStr := cbInfo.setState(StateOpen) + now := time.Now() + cbInfo.Metrics.LastOpenTime.Store(now.Unix()) + cbInfo.Metrics.NextRetryTime.Store(now.Add(config.TimeoutParsed).Unix()) // 通知状态变化(如果状态改变) - if oldState != StateOpen { - notifyStateChange(serviceName, oldState, StateOpen) + if oldStateStr != StateOpen { + notifyStateChange(serviceName, oldStateStr, StateOpen) } // 同步到分布式存储 @@ -456,30 +561,37 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { statusCode := r.Response.Status duration := time.Since(startTime) - // 使用提前获取的config判断状态码(性能优化) - if !isSuccessStatusCode(config, statusCode) { + // 判断是否为慢请求(P2:可观测性) + if duration > config.SlowRequestThresholdParsed { + cbInfo.Metrics.SlowRequests.Add(1) + } + + // 使用cbInfo.SuccessCodeMap判断状态码(性能优化) + if !isSuccessStatusCode(cbInfo, statusCode) { // 记录异常 cbInfo.Metrics.FailureRequests.Add(1) api.TraceError(entry, fmt.Errorf("request failed with status: %d", statusCode)) g.Log().Debugf(ctx, "服务 %s 请求失败: status=%d, duration=%v", serviceName, statusCode, duration) } else { cbInfo.Metrics.PassRequests.Add(1) - // 更新状态为关闭(如果之前是开启状态) - cbInfo.mu.Lock() - oldState := cbInfo.State - if cbInfo.State != StateClosed { - cbInfo.State = StateClosed - notifyStateChange(serviceName, oldState, StateClosed) + // 更新状态为关闭(如果之前是开启状态,使用atomic操作) + if cbInfo.getState() != StateClosed { + oldStateStr := cbInfo.setState(StateClosed) + if oldStateStr != StateClosed { + notifyStateChange(serviceName, oldStateStr, StateClosed) + } } - cbInfo.mu.Unlock() } // 退出Sentinel资源 entry.Exit() } -// sendFallbackResponse 发送降级响应 +// sendFallbackResponse 发送降级响应(P0:添加日志记录) func sendFallbackResponse(r *ghttp.Request, serviceName string, config *CircuitBreakerConfig, reason string) { + // P0:记录降级日志,便于问题排查 + g.Log().Warningf(r.GetCtx(), "熔断器降级: service=%s, reason=%s, clientIP=%s", serviceName, reason, r.GetClientIp()) + if config.EnableFallback && config.FallbackMessage != "" { // 自定义降级消息 r.Response.WriteStatusExit(503, config.FallbackMessage) @@ -496,21 +608,16 @@ func sendFallbackResponse(r *ghttp.Request, serviceName string, config *CircuitB } } -// isSuccessStatusCode 判断HTTP状态码是否成功 -func isSuccessStatusCode(config *CircuitBreakerConfig, statusCode int) bool { - if len(config.SuccessStatusCodes) > 0 { - for _, code := range config.SuccessStatusCodes { - if statusCode == code { - return true - } - } - return false +// isSuccessStatusCode 判断HTTP状态码是否成功(使用cbInfo.SuccessCodeMap优化性能) +func isSuccessStatusCode(cbInfo *CircuitBreakerInfo, statusCode int) bool { + if cbInfo.SuccessCodeMap != nil && len(cbInfo.SuccessCodeMap) > 0 { + return cbInfo.SuccessCodeMap[statusCode] } // 默认:2xx状态码为成功 return statusCode >= 200 && statusCode < 300 } -// extractServiceName 从URL路径提取服务名(改进提取逻辑) +// extractServiceName 从URL路径提取服务名(P0:添加URL编码处理) func extractServiceName(path string) string { // 去除首尾斜杠并分割 path = strings.Trim(path, "/") @@ -523,6 +630,19 @@ func extractServiceName(path string) string { } serviceName := parts[0] + // P0:处理URL编码,将 %2F 等转义字符还原 + // 注意:在goframe的网关中间件中,路径通常已经被框架处理过 + // 但为了安全性,这里对包含%的情况进行简单处理 + if strings.Contains(serviceName, "%") { + // 尝试解码URL编码的字符串 + // 使用path.Unescape而不是url.QueryUnescape,因为我们处理的是路径片段 + decoded, err := pathUnescape(serviceName) + if err == nil { + serviceName = decoded + } + // 如果解码失败,继续使用原始serviceName + } + // 验证服务名是否在已配置的熔断器中 if _, ok := circuitBreakerConfigs.Load(serviceName); ok { return serviceName @@ -530,6 +650,56 @@ func extractServiceName(path string) string { return "" } +// pathUnescape 路径片段的URL解码(P0:安全性改进) +// 注意:Go 1.8+ 可以使用 path.Unescape,这里提供兼容实现 +func pathUnescape(s string) (string, error) { + // 使用strings.Builder优化性能 + var builder strings.Builder + builder.Grow(len(s)) + + for i := 0; i < len(s); i++ { + switch s[i] { + case '%': + // 处理百分号编码 + if i+2 >= len(s) { + // 不完整的编码,保留原样 + builder.WriteByte(s[i]) + continue + } + // 解析十六进制数字 + high := hexDigit(s[i+1]) + low := hexDigit(s[i+2]) + if high == 0xFF || low == 0xFF { + // 无效的十六进制,保留原样 + builder.WriteByte(s[i]) + continue + } + builder.WriteByte((high << 4) | low) + i += 2 + case '+': + // 路径片段中的+通常不需要解码为空格 + builder.WriteByte('+') + default: + builder.WriteByte(s[i]) + } + } + return builder.String(), nil +} + +// hexDigit 将十六进制字符转换为对应的数值 +func hexDigit(c byte) byte { + switch { + case '0' <= c && c <= '9': + return c - '0' + case 'a' <= c && c <= 'f': + return c - 'a' + 10 + case 'A' <= c && c <= 'F': + return c - 'A' + 10 + default: + return 0xFF // 无效字符 + } +} + // isCircuitBreakerOpenInDistributed 检查分布式熔断状态 func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) bool { key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) @@ -651,45 +821,95 @@ func notifyStateChange(serviceName string, fromState, toState CircuitBreakerStat }) } -// CircuitBreakerHealthCheckHandler 熔断器健康检查接口 +// CircuitBreakerHealthCheckHandler 熔断器健康检查接口(P0:添加IP白名单验证,P1:添加分页支持,P1:优化性能) func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { + // P0:权限验证 + if !isAdminIP(r) { + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ + Code: 403, + Message: "权限不足,禁止访问", + }) + return + } + status := make(map[string]interface{}) totalServices := 0 openServices := 0 - // 遍历所有熔断器 + // P1:分页参数 + page := r.Get("page").Int() + size := r.Get("size").Int() + if page < 0 { + page = 0 + } + if size <= 0 || size > 100 { + size = 20 // 默认20条,最多100条 + } + + // P1:使用缓存的totalServicesCount避免每次遍历 + total := int(totalServicesCount.Load()) + start := page * size + + // P1:只遍历分页范围内的服务(通过计数跳过) + end := start + size + if end > total { + end = total + } + + current := 0 circuitBreakers.Range(func(key, value interface{}) bool { + // 跳过前面的页 + if current < start { + current++ + return true + } + // 只处理当前页 + if current >= end { + return false + } + serviceName := key.(string) cbInfo := value.(*CircuitBreakerInfo) totalServices++ - cbInfo.mu.RLock() - isOpen := cbInfo.State == StateOpen + isOpen := cbInfo.getState() == StateOpen if isOpen { openServices++ } - // 从Metrics中读取数据(修复数据准确性问题) + // 从Metrics中读取数据(使用atomic) lastResetTime := cbInfo.Metrics.LastResetTime.Load() var lastResetTimeStr string if lastResetTime > 0 { lastResetTimeStr = time.Unix(lastResetTime, 0).Format("2006-01-02 15:04:05") } + lastOpenTime := cbInfo.Metrics.LastOpenTime.Load() + var lastOpenTimeStr string + if lastOpenTime > 0 { + lastOpenTimeStr = time.Unix(lastOpenTime, 0).Format("2006-01-02 15:04:05") + } + + nextRetryTime := cbInfo.Metrics.NextRetryTime.Load() + var nextRetryTimeStr string + if nextRetryTime > 0 { + nextRetryTimeStr = time.Unix(nextRetryTime, 0).Format("2006-01-02 15:04:05") + } + status[serviceName] = map[string]interface{}{ "resource": cbInfo.ResourceName, - "state": string(cbInfo.State), - "lastOpenTime": cbInfo.LastOpenTime, - "nextRetryTime": cbInfo.NextRetryTime, + "state": string(cbInfo.getState()), + "lastOpenTime": lastOpenTimeStr, + "nextRetryTime": nextRetryTimeStr, "totalRequests": cbInfo.Metrics.TotalRequests.Load(), "passRequests": cbInfo.Metrics.PassRequests.Load(), "blockRequests": cbInfo.Metrics.BlockRequests.Load(), "failureRequests": cbInfo.Metrics.FailureRequests.Load(), + "slowRequests": cbInfo.Metrics.SlowRequests.Load(), "openCount": cbInfo.Metrics.OpenCount.Load(), "lastResetTime": lastResetTimeStr, } - cbInfo.mu.RUnlock() - + current++ return true }) @@ -705,31 +925,54 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { Data: map[string]interface{}{ "summary": summary, "services": status, + "page": page, + "size": size, + "total": total, }, }) } -// isAdminIP 检查请求IP是否在管理员白名单中(使用缓存优化性能) +// isAdminIP 检查请求IP是否在管理员白名单中(P1:使用map优化性能,P0:支持IPv6 CIDR) func isAdminIP(r *ghttp.Request) bool { clientIP := r.GetClientIp() if clientIP == "" { return false } - // 读取缓存的白名单(性能优化) - allowedAdminIPsCacheMutex.RLock() - allowedIPs := allowedAdminIPsCache - allowedAdminIPsCacheMutex.RUnlock() + // 读取缓存的IP白名单(P1:使用map实现O(1)查找) + allowedAdminIPsMutex.RLock() + allowedIPs := allowedAdminIPsMap + allowedAdminIPsMutex.RUnlock() // 如果没有配置白名单,允许所有IP(向后兼容) if len(allowedIPs) == 0 { + allowedAdminCIDRsMutex.RLock() + hasCIDRs := len(allowedAdminCIDRs) > 0 + allowedAdminCIDRsMutex.RUnlock() + // 如果也没有CIDR,则允许所有IP + if !hasCIDRs { + return true + } + } + + // 精确IP匹配(P1:map查找O(1)) + if allowedIPs[clientIP] { return true } - // 检查IP是否在白名单中 - for _, allowedIP := range allowedIPs { - if clientIP == allowedIP { - return true + // P1:使用预编译的CIDR网络掩码匹配(支持IPv4和IPv6) + allowedAdminCIDRsMutex.RLock() + cidrNets := allowedAdminCIDRs + allowedAdminCIDRsMutex.RUnlock() + + if len(cidrNets) > 0 { + clientNetIP := net.ParseIP(clientIP) + if clientNetIP != nil { + for _, cidrNet := range cidrNets { + if cidrNet.Contains(clientNetIP) { + return true + } + } } } @@ -787,21 +1030,20 @@ func CircuitBreakerResetHandler(r *ghttp.Request) { } } - // 更新内存状态并重置指标 + // 更新内存状态并重置指标(使用atomic操作) if val, ok := circuitBreakers.Load(serviceName); ok { cbInfo := val.(*CircuitBreakerInfo) - cbInfo.mu.Lock() - cbInfo.State = StateClosed - cbInfo.LastOpenTime = time.Time{} - cbInfo.NextRetryTime = time.Time{} + cbInfo.State.Store(stateClosed) + cbInfo.Metrics.LastOpenTime.Store(0) + cbInfo.Metrics.NextRetryTime.Store(0) // 重置指标 cbInfo.Metrics.TotalRequests.Store(0) cbInfo.Metrics.PassRequests.Store(0) cbInfo.Metrics.BlockRequests.Store(0) cbInfo.Metrics.FailureRequests.Store(0) + cbInfo.Metrics.SlowRequests.Store(0) cbInfo.Metrics.OpenCount.Store(0) cbInfo.Metrics.LastResetTime.Store(time.Now().Unix()) - cbInfo.mu.Unlock() } // 重置分布式状态(如果启用) From d03665facc92a0affd48032b8e906debd60a1c92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 13:14:46 +0800 Subject: [PATCH 130/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 1542 ++++++++++++++++++--------------- 1 file changed, 852 insertions(+), 690 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 4ca4f0e..913c581 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net" + "strconv" "strings" "sync" "sync/atomic" @@ -19,130 +20,201 @@ import ( type CircuitBreakerState string const ( - StateClosed CircuitBreakerState = "closed" // 关闭:正常状态 - StateOpen CircuitBreakerState = "open" // 开启:熔断状态 + StateClosed CircuitBreakerState = "closed" + StateOpen CircuitBreakerState = "open" + StateHalfOpen CircuitBreakerState = "halfopen" ) // 熔断器状态常量(用于atomic.Int64) const ( - stateClosed int64 = 0 - stateOpen int64 = 1 + stateClosed int64 = 0 + stateOpen int64 = 1 + stateHalfOpen int64 = 2 ) -// getState 获取熔断器状态字符串 -func (cb *CircuitBreakerInfo) getState() CircuitBreakerState { - if cb.State.Load() == stateOpen { - return StateOpen - } - return StateClosed -} - -// setState 设置熔断器状态(atomic操作,无锁) -func (cb *CircuitBreakerInfo) setState(state CircuitBreakerState) CircuitBreakerState { - var newState int64 - if state == StateOpen { - newState = stateOpen - } else { - newState = stateClosed - } - oldState := cb.State.Swap(newState) - if oldState == stateOpen { - return StateOpen - } - return StateClosed -} - // CircuitBreakerConfig 熔断器配置 type CircuitBreakerConfig struct { - Enabled bool // 是否启用熔断器 - MaxFailures int // 连续失败次数 - Timeout string // 熔断超时时间 - TimeoutParsed time.Duration // 缓存的超时时间(性能优化) - SuccessStatusCodes []int // 视为成功的HTTP状态码 - SlowRequestThreshold string // 慢请求阈值 - SlowRequestThresholdParsed time.Duration // 缓存的慢请求阈值(性能优化) - EnableSlidingWindow bool // 是否启用滑动窗口 - FailureRateThreshold float64 // 失败率阈值 - EnableFallback bool // 是否启用降级 - FallbackMessage string // 降级提示消息 - RequestTimeout int // 请求超时时间(毫秒) - DistributedTTL int // 分布式熔断状态TTL(秒) - AdminIPs []string // 允许重置熔断器的管理员IP列表 - StatIntervalMs int // 统计窗口时长(毫秒),默认1000ms - MinRequestAmount int // 最小请求数量,默认与MaxFailures相同 - AdminCIDRs []string // 允许重置熔断器的管理员CIDR列表(P0:支持IP段) - // P1:预编译的CIDR网络掩码(性能优化) - CIDRNetMasks []*net.IPNet + Enabled bool + MaxFailures int + Timeout string + TimeoutParsed time.Duration + SuccessStatusCodes []int + SlowRequestThreshold string + SlowRequestThresholdParsed time.Duration + EnableSlidingWindow bool + FailureRateThreshold float64 + EnableFallback bool + FallbackMessage string + RequestTimeout int + DistributedTTL int + AdminIPs []string + StatIntervalMs int + MinRequestAmount int + AdminCIDRs []string + HalfOpenMaxRequests int + HalfOpenSuccessThreshold float64 + WarmupDuration string + WarmupDurationParsed time.Duration + EnableAdaptiveThreshold bool + AdaptiveMinThreshold float64 + AdaptiveMaxThreshold float64 + CIDRNetMasks []*net.IPNet } // CircuitBreakerMetrics 熔断器指标 type CircuitBreakerMetrics struct { - TotalRequests atomic.Int64 // 总请求数 - PassRequests atomic.Int64 // 通过请求数 - BlockRequests atomic.Int64 // 阻塞请求数 - FailureRequests atomic.Int64 // 失败请求数 - SlowRequests atomic.Int64 // 慢请求数(P2:可观测性) - OpenCount atomic.Int64 // 熔断开启次数 - LastResetTime atomic.Int64 // 上次重置时间(Unix时间戳) - // 使用atomic.Int64实现简单的时间戳存储,避免使用mutex - LastOpenTime atomic.Int64 // 上次熔断时间(Unix时间戳) - NextRetryTime atomic.Int64 // 下次重试时间(Unix时间戳) + // 请求统计 + TotalRequests atomic.Int64 + PassRequests atomic.Int64 + BlockRequests atomic.Int64 + FailureRequests atomic.Int64 + SlowRequests atomic.Int64 + + // 状态统计 + OpenCount atomic.Int64 + ClosedCount atomic.Int64 + HalfOpenCount atomic.Int64 + + // 时间戳 + LastResetTime atomic.Int64 + LastOpenTime atomic.Int64 + NextRetryTime atomic.Int64 + LastCloseTime atomic.Int64 + LastHalfOpenTime atomic.Int64 + + // 半开状态统计 + HalfOpenRequests atomic.Int64 + HalfOpenPassed atomic.Int64 + HalfOpenFailed atomic.Int64 + + // 性能指标 + TotalResponseTime atomic.Int64 // 总响应时间(纳秒) + MinResponseTime atomic.Int64 // 最小响应时间(纳秒) + MaxResponseTime atomic.Int64 // 最大响应时间(纳秒) + + // 窗口统计(用于计算成功率等) + WindowStartTime atomic.Int64 // 统计窗口开始时间 + WindowRequests atomic.Int64 // 窗口内请求总数 + WindowFailures atomic.Int64 // 窗口内失败数 } // CircuitBreakerInfo 熔断器信息 type CircuitBreakerInfo struct { - ResourceName string `json:"resourceName"` // 资源名称 - State atomic.Int64 `json:"state"` // 当前状态(0:closed, 1:open),使用atomic避免mutex - Config *CircuitBreakerConfig `json:"config"` // 配置信息 - Metrics *CircuitBreakerMetrics `json:"metrics"` // 指标统计 - // 预编译的成功状态码集合(P1:性能优化) - SuccessCodeMap map[int]bool - // P1:预编译的CIDR网络掩码(避免重复解析) - CIDRNetMasks []*net.IPNet + ResourceName string + State atomic.Int64 + Config *CircuitBreakerConfig + Metrics *CircuitBreakerMetrics + SuccessCodeMap map[int]bool + CIDRNetMasks []*net.IPNet + AdaptiveThreshold float64 + WarmupEndTime int64 } var ( - // circuitBreakers 存储所有熔断器状态(用于健康检查) - circuitBreakers sync.Map - // circuitBreakerConfigs 熔断器配置缓存 - circuitBreakerConfigs sync.Map - // distributedSyncLocks 分布式同步锁(按服务名分片) - distributedSyncLocks sync.Map - // stateChangeListeners 状态变化监听器 - stateChangeListeners sync.Map - // stateChangeListenersRegistered 默认监听器是否已注册 + circuitBreakers sync.Map + circuitBreakerConfigs sync.Map + stateChangeListeners sync.Map stateChangeListenersRegistered sync.Map - // P1:使用map代替slice优化IP查找性能 - allowedAdminIPsMap map[string]bool - // allowedAdminIPsMutex 保护白名单缓存的并发访问 - allowedAdminIPsMutex sync.RWMutex - // P1:预编译的CIDR网络掩码列表 - allowedAdminCIDRs []*net.IPNet - // allowedAdminCIDRsMutex 保护CIDR列表的并发访问 - allowedAdminCIDRsMutex sync.RWMutex - // totalServicesCount 缓存总服务数(P1:性能优化) - totalServicesCount atomic.Int64 + allowedAdminIPsMap map[string]bool + allowedAdminIPsMutex sync.RWMutex + allowedAdminCIDRs []*net.IPNet + allowedAdminCIDRsMutex sync.RWMutex + totalServicesCount atomic.Int64 + serviceNamesSlice []string + serviceNamesMutex sync.RWMutex ) +// 默认值常量 +const ( + defaultMaxFailures = 5 + defaultTimeout = "60s" + defaultSlowRequestThreshold = "3s" + defaultStatIntervalMs = 1000 + defaultRequestTimeout = 30000 + defaultDistributedTTL = 300 + defaultHalfOpenMaxRequests = 5 + defaultWarmupDuration = "10s" + defaultHalfOpenSuccessThreshold = 0.5 +) + +// getState 获取熔断器状态 +func (cb *CircuitBreakerInfo) getState() CircuitBreakerState { + switch cb.State.Load() { + case stateOpen: + return StateOpen + case stateHalfOpen: + return StateHalfOpen + default: + return StateClosed + } +} + +// setState 设置熔断器状态 +func (cb *CircuitBreakerInfo) setState(state CircuitBreakerState) CircuitBreakerState { + return cb.setStateWithMetrics(state, true) +} + +// setStateWithMetrics 设置熔断器状态并更新指标 +func (cb *CircuitBreakerInfo) setStateWithMetrics(state CircuitBreakerState, updateMetrics bool) CircuitBreakerState { + var newState int64 + switch state { + case StateOpen: + newState = stateOpen + case StateHalfOpen: + newState = stateHalfOpen + default: + newState = stateClosed + } + + oldState := cb.State.Swap(newState) + var oldStateEnum CircuitBreakerState + + switch oldState { + case stateOpen: + oldStateEnum = StateOpen + case stateHalfOpen: + oldStateEnum = StateHalfOpen + default: + oldStateEnum = StateClosed + } + + // 如果状态发生了变化且需要更新指标 + if oldStateEnum != state && updateMetrics { + now := time.Now().Unix() + + // 根据新状态更新计数器 + switch state { + case StateOpen: + cb.Metrics.OpenCount.Add(1) + cb.Metrics.LastOpenTime.Store(now) + case StateClosed: + cb.Metrics.ClosedCount.Add(1) + cb.Metrics.LastCloseTime.Store(now) + case StateHalfOpen: + cb.Metrics.HalfOpenCount.Add(1) + cb.Metrics.LastHalfOpenTime.Store(now) + } + + // 设置下一次重试时间(如果是打开状态) + if state == StateOpen { + cb.Metrics.NextRetryTime.Store(time.Now().Add(cb.Config.TimeoutParsed).Unix()) + } + } + + return oldStateEnum +} + // InitCircuitBreaker 初始化Sentinel熔断器 func InitCircuitBreaker() error { ctx := context.Background() - - // 初始化Sentinel - err := api.InitDefault() - if err != nil { + if err := api.InitDefault(); err != nil { return fmt.Errorf("sentinel初始化失败: %v", err) } - // 注册熔断器状态变化监听器 registerStateChangeListeners() - g.Log().Infof(ctx, "Sentinel熔断器初始化成功") - // 扫描配置文件中所有配置了熔断器的服务 services := g.Cfg().MustGet(ctx, "circuitBreaker").Map() - - // 过滤掉非服务配置的key serviceNames := filterServiceNames(services) if len(serviceNames) == 0 { @@ -150,402 +222,423 @@ func InitCircuitBreaker() error { return nil } - // P1:缓存总服务数 totalServicesCount.Store(int64(len(serviceNames))) + serviceNamesMutex.Lock() + serviceNamesSlice = serviceNames + serviceNamesMutex.Unlock() - // 为每个服务创建熔断器 enabledCount := 0 for _, serviceName := range serviceNames { - serviceConfig := loadServiceCircuitBreakerConfig(serviceName) - if serviceConfig != nil && serviceConfig.Enabled { - circuitBreakerConfigs.Store(serviceName, serviceConfig) - initErr := initServiceCircuitBreaker(serviceName, serviceConfig) - if initErr != nil { - g.Log().Errorf(ctx, "服务 %s 熔断器初始化失败: %v", serviceName, initErr) + config := loadServiceCircuitBreakerConfig(serviceName) + if config != nil && config.Enabled { + circuitBreakerConfigs.Store(serviceName, config) + if err := initServiceCircuitBreaker(serviceName, config); err != nil { + g.Log().Errorf(ctx, "服务 %s 熔断器初始化失败: %v", serviceName, err) } else { g.Log().Infof(ctx, "服务 %s 熔断器初始化成功", serviceName) enabledCount++ } - } else { - g.Log().Infof(ctx, "服务 %s 熔断器未启用", serviceName) } } - // P1:更新管理员IP白名单缓存(在所有服务配置加载完成后) updateAdminIPsCache() - g.Log().Infof(ctx, "共初始化 %d 个服务熔断器,其中 %d 个已启用", len(serviceNames), enabledCount) return nil } // ReloadCircuitBreakerConfig 动态重新加载熔断器配置 func ReloadCircuitBreakerConfig(serviceName string) error { - ctx := context.Background() - - // 重新加载配置 - serviceConfig := loadServiceCircuitBreakerConfig(serviceName) - if serviceConfig == nil { + config := loadServiceCircuitBreakerConfig(serviceName) + if config == nil { return fmt.Errorf("未找到服务 %s 的配置", serviceName) } + if err := validateCircuitBreakerConfig(config); err != nil { + return fmt.Errorf("配置验证失败: %v", err) + } - // 更新配置缓存 - circuitBreakerConfigs.Store(serviceName, serviceConfig) + oldConfig, _ := circuitBreakerConfigs.Load(serviceName) + circuitBreakerConfigs.Store(serviceName, config) - // 重新初始化熔断器 - err := initServiceCircuitBreaker(serviceName, serviceConfig) - if err != nil { + if err := initServiceCircuitBreaker(serviceName, config); err != nil { + if oldConfig != nil { + circuitBreakerConfigs.Store(serviceName, oldConfig) + } return fmt.Errorf("重新初始化熔断器失败: %v", err) } - g.Log().Infof(ctx, "服务 %s 熔断器配置重新加载成功", serviceName) + g.Log().Infof(context.Background(), "服务 %s 熔断器配置重新加载成功", serviceName) return nil } -// loadServiceCircuitBreakerConfig 加载单个服务的熔断器配置 +// loadServiceCircuitBreakerConfig 加载配置 func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { ctx := context.Background() - key := fmt.Sprintf("circuitBreaker.%s", serviceName) + key := "circuitBreaker." + serviceName - enabled := g.Cfg().MustGet(ctx, key+".enabled", true).Bool() - maxFailures := g.Cfg().MustGet(ctx, key+".maxFailures", 5).Int() - timeout := g.Cfg().MustGet(ctx, key+".timeout", "60s").String() - slowRequestThreshold := g.Cfg().MustGet(ctx, key+".slowRequestThreshold", "3s").String() - enableSlidingWindow := g.Cfg().MustGet(ctx, key+".enableSlidingWindow", false).Bool() - failureRateThreshold := g.Cfg().MustGet(ctx, key+".failureRateThreshold", 0.5).Float64() - enableFallback := g.Cfg().MustGet(ctx, key+".enableFallback", false).Bool() - fallbackMessage := g.Cfg().MustGet(ctx, key+".fallbackMessage", "").String() - requestTimeout := g.Cfg().MustGet(ctx, key+".requestTimeout", 30000).Int() - distributedTTL := g.Cfg().MustGet(ctx, key+".distributedTTL", 300).Int() - adminIPs := g.Cfg().MustGet(ctx, key+".adminIPs", "").String() - adminCIDRs := g.Cfg().MustGet(ctx, key+".adminCIDRs", "").String() // P0:支持CIDR - statIntervalMs := g.Cfg().MustGet(ctx, key+".statIntervalMs", 1000).Int() - minRequestAmount := g.Cfg().MustGet(ctx, key+".minRequestAmount", 0).Int() + config := &CircuitBreakerConfig{ + Enabled: g.Cfg().MustGet(ctx, key+".enabled", true).Bool(), + MaxFailures: g.Cfg().MustGet(ctx, key+".maxFailures", defaultMaxFailures).Int(), + Timeout: g.Cfg().MustGet(ctx, key+".timeout", defaultTimeout).String(), + SlowRequestThreshold: g.Cfg().MustGet(ctx, key+".slowRequestThreshold", defaultSlowRequestThreshold).String(), + EnableSlidingWindow: g.Cfg().MustGet(ctx, key+".enableSlidingWindow", false).Bool(), + FailureRateThreshold: g.Cfg().MustGet(ctx, key+".failureRateThreshold", 0.5).Float64(), + EnableFallback: g.Cfg().MustGet(ctx, key+".enableFallback", false).Bool(), + FallbackMessage: g.Cfg().MustGet(ctx, key+".fallbackMessage", "").String(), + RequestTimeout: g.Cfg().MustGet(ctx, key+".requestTimeout", defaultRequestTimeout).Int(), + DistributedTTL: g.Cfg().MustGet(ctx, key+".distributedTTL", defaultDistributedTTL).Int(), + StatIntervalMs: g.Cfg().MustGet(ctx, key+".statIntervalMs", defaultStatIntervalMs).Int(), + HalfOpenMaxRequests: g.Cfg().MustGet(ctx, key+".halfOpenMaxRequests", defaultHalfOpenMaxRequests).Int(), + HalfOpenSuccessThreshold: g.Cfg().MustGet(ctx, key+".halfOpenSuccessThreshold", defaultHalfOpenSuccessThreshold).Float64(), + WarmupDuration: g.Cfg().MustGet(ctx, key+".warmupDuration", defaultWarmupDuration).String(), + EnableAdaptiveThreshold: g.Cfg().MustGet(ctx, key+".enableAdaptiveThreshold", false).Bool(), + AdaptiveMinThreshold: g.Cfg().MustGet(ctx, key+".adaptiveMinThreshold", 0.3).Float64(), + AdaptiveMaxThreshold: g.Cfg().MustGet(ctx, key+".adaptiveMaxThreshold", 0.7).Float64(), + } - // 解析成功状态码(使用map用于快速查找) + config.MinRequestAmount = g.Cfg().MustGet(ctx, key+".minRequestAmount", 0).Int() + if config.MinRequestAmount == 0 { + config.MinRequestAmount = config.MaxFailures + } + + // 解析时间 - 使用默认值处理解析错误 + timeoutParsed, err := time.ParseDuration(config.Timeout) + if err != nil { + g.Log().Warningf(ctx, "解析timeout失败: %s, 使用默认值 %s, error: %v", config.Timeout, defaultTimeout, err) + timeoutParsed, _ = time.ParseDuration(defaultTimeout) + config.Timeout = defaultTimeout + } + config.TimeoutParsed = timeoutParsed + + slowThresholdParsed, err := time.ParseDuration(config.SlowRequestThreshold) + if err != nil { + g.Log().Warningf(ctx, "解析slowRequestThreshold失败: %s, 使用默认值 %s, error: %v", config.SlowRequestThreshold, defaultSlowRequestThreshold, err) + slowThresholdParsed, _ = time.ParseDuration(defaultSlowRequestThreshold) + config.SlowRequestThreshold = defaultSlowRequestThreshold + } + config.SlowRequestThresholdParsed = slowThresholdParsed + + warmupParsed, err := time.ParseDuration(config.WarmupDuration) + if err != nil { + g.Log().Warningf(ctx, "解析warmupDuration失败: %s, 使用默认值 %s, error: %v", config.WarmupDuration, defaultWarmupDuration, err) + warmupParsed, _ = time.ParseDuration(defaultWarmupDuration) + config.WarmupDuration = defaultWarmupDuration + } + config.WarmupDurationParsed = warmupParsed + + // 解析状态码 successCodes := g.Cfg().MustGet(ctx, key+".successStatusCodes", "200,201,204").String() - statusCodes := parseStatusCodesSlice(successCodes) + config.SuccessStatusCodes = parseIntSlice(successCodes) - // 解析时间(缓存结果,性能优化) - timeoutParsed, err := time.ParseDuration(timeout) - if err != nil { - timeoutParsed = 60 * time.Second - g.Log().Warningf(ctx, "服务 %s 的 timeout 解析失败,使用默认值: %v", serviceName, err) - } + // 解析IP和CIDR + config.AdminIPs = parseStrings(g.Cfg().MustGet(ctx, key+".adminIPs", "").String()) + config.AdminCIDRs = parseStrings(g.Cfg().MustGet(ctx, key+".adminCIDRs", "").String()) + config.CIDRNetMasks, _ = parseCIDRs(config.AdminCIDRs) - slowRequestThresholdParsed, err := time.ParseDuration(slowRequestThreshold) - if err != nil { - slowRequestThresholdParsed = 3 * time.Second - g.Log().Warningf(ctx, "服务 %s 的 slowRequestThreshold 解析失败,使用默认值: %v", serviceName, err) - } - - // 解析管理员IP列表 - adminIPList := parseAdminIPs(adminIPs) - - // P1:预编译CIDR为net.IPNet(支持IPv4和IPv6) - cidrNets, err := parseAdminCIDRs(adminCIDRs) - if err != nil { - g.Log().Warningf(ctx, "服务 %s 的 adminCIDRs 解析失败: %v", serviceName, err) - cidrNets = nil - } - - // 如果minRequestAmount未配置,则使用maxFailures作为默认值 - if minRequestAmount == 0 { - minRequestAmount = maxFailures - } - - return &CircuitBreakerConfig{ - Enabled: enabled, - MaxFailures: maxFailures, - Timeout: timeout, - TimeoutParsed: timeoutParsed, - SuccessStatusCodes: statusCodes, - SlowRequestThreshold: slowRequestThreshold, - SlowRequestThresholdParsed: slowRequestThresholdParsed, - EnableSlidingWindow: enableSlidingWindow, - FailureRateThreshold: failureRateThreshold, - EnableFallback: enableFallback, - FallbackMessage: fallbackMessage, - RequestTimeout: requestTimeout, - DistributedTTL: distributedTTL, - AdminIPs: adminIPList, - CIDRNetMasks: cidrNets, // P1:预编译的CIDR网络掩码 - StatIntervalMs: statIntervalMs, - MinRequestAmount: minRequestAmount, - } + return config } -// parseStatusCodes 解析HTTP状态码(返回map用于快速查找) -func parseStatusCodes(str string) map[int]bool { +// parseIntSlice 解析整数切片 +func parseIntSlice(str string) []int { parts := strings.Split(str, ",") - codeMap := make(map[int]bool, len(parts)) + result := make([]int, 0, len(parts)) for _, part := range parts { - var code int - if _, err := fmt.Sscanf(strings.TrimSpace(part), "%d", &code); err == nil { - codeMap[code] = true + if val, err := strconv.Atoi(strings.TrimSpace(part)); err == nil { + result = append(result, val) } } - return codeMap + return result } -// parseStatusCodesSlice 解析HTTP状态码(返回切片用于配置) -func parseStatusCodesSlice(str string) []int { - parts := strings.Split(str, ",") - codes := make([]int, 0, len(parts)) - for _, part := range parts { - var code int - if _, err := fmt.Sscanf(strings.TrimSpace(part), "%d", &code); err == nil { - codes = append(codes, code) - } - } - return codes -} - -// parseAdminCIDRs 解析管理员CIDR列表(P1:预编译为net.IPNet,P0:支持IPv6) -func parseAdminCIDRs(str string) ([]*net.IPNet, error) { +// parseStrings 解析字符串切片 +func parseStrings(str string) []string { if str == "" { - return nil, nil + return nil } parts := strings.Split(str, ",") - nets := make([]*net.IPNet, 0, len(parts)) + result := make([]string, 0, len(parts)) for _, part := range parts { - cidr := strings.TrimSpace(part) - if cidr != "" { - // 使用net.ParseCIDR解析CIDR(支持IPv4和IPv6) - _, ipNet, err := net.ParseCIDR(cidr) - if err != nil { - return nil, fmt.Errorf("解析CIDR失败: %s, error: %v", cidr, err) + if trimmed := strings.TrimSpace(part); trimmed != "" { + result = append(result, trimmed) + } + } + return result +} + +// parseCIDRs 解析CIDR列表 +func parseCIDRs(strs []string) ([]*net.IPNet, error) { + nets := make([]*net.IPNet, 0, len(strs)) + for _, s := range strs { + if s == "*" { + if _, ipv4Net, err := net.ParseCIDR("0.0.0.0/0"); err == nil { + nets = append(nets, ipv4Net) } + if _, ipv6Net, err := net.ParseCIDR("::/0"); err == nil { + nets = append(nets, ipv6Net) + } + continue + } + if _, ipNet, err := net.ParseCIDR(s); err == nil { nets = append(nets, ipNet) } } return nets, nil } -// updateAdminIPsCache 更新管理员IP白名单缓存(P1:使用map优化性能,P1:预编译CIDR) -func updateAdminIPsCache() { - ipMap := make(map[string]bool) - cidrNets := make([]*net.IPNet, 0) - - // 收集所有服务的adminIPs配置 - circuitBreakerConfigs.Range(func(key, value interface{}) bool { - config := value.(*CircuitBreakerConfig) - if len(config.AdminIPs) > 0 { - for _, ip := range config.AdminIPs { - if !ipMap[ip] { - ipMap[ip] = true - } - } - } - // P1:使用预编译的CIDR网络掩码 - if len(config.CIDRNetMasks) > 0 { - for _, cidrNet := range config.CIDRNetMasks { - cidrNets = append(cidrNets, cidrNet) - } - } - return true - }) - - // 更新缓存 - allowedAdminIPsMutex.Lock() - allowedAdminIPsMap = ipMap - allowedAdminIPsMutex.Unlock() - - allowedAdminCIDRsMutex.Lock() - allowedAdminCIDRs = cidrNets - allowedAdminCIDRsMutex.Unlock() +// newCircuitBreakerMetrics 创建并初始化熔断器指标 +func newCircuitBreakerMetrics() *CircuitBreakerMetrics { + metrics := &CircuitBreakerMetrics{ + MinResponseTime: atomic.Int64{}, + MaxResponseTime: atomic.Int64{}, + } + metrics.MinResponseTime.Store(1<<63 - 1) // 最大int64值作为初始最小值 + return metrics } -// isIPInCIDR 检查IP是否在CIDR范围内(P1:使用预编译的net.IPNet,P0:支持IPv6) -func isIPInCIDR(ipStr string, cidrNet *net.IPNet) bool { - ip := net.ParseIP(ipStr) - if ip == nil { - return false - } - return cidrNet.Contains(ip) -} +// updateWindowStats 更新窗口统计信息 +func (cb *CircuitBreakerInfo) updateWindowStats(isSuccess bool, ctx context.Context) { + now := time.Now().Unix() + windowStart := cb.Metrics.WindowStartTime.Load() -// parseAdminIPs 解析管理员IP列表 -func parseAdminIPs(str string) []string { - if str == "" { - return nil + // 默认窗口大小为60秒 + windowSize := int64(60) + + // 如果超过窗口大小,重置统计 + if now-windowStart >= windowSize { + // 使用原子操作重置窗口 + if cb.Metrics.WindowStartTime.CompareAndSwap(windowStart, now) { + cb.Metrics.WindowRequests.Store(0) + cb.Metrics.WindowFailures.Store(0) + } + // 重新获取最新的windowStart + windowStart = cb.Metrics.WindowStartTime.Load() } - parts := strings.Split(str, ",") - ips := make([]string, 0, len(parts)) - for _, part := range parts { - ip := strings.TrimSpace(part) - if ip != "" { - ips = append(ips, ip) + + // 原子更新窗口内请求总数 + cb.Metrics.WindowRequests.Add(1) + if !isSuccess { + cb.Metrics.WindowFailures.Add(1) + } + + // 计算当前窗口内的成功率 + total := cb.Metrics.WindowRequests.Load() + failures := cb.Metrics.WindowFailures.Load() + if total > 0 { + successRate := float64(total-failures) / float64(total) + if successRate < 0.5 && total >= 10 { // 如果成功率低于50%且有足够样本 + g.Log().Warningf(ctx, "熔断器 %s 窗口内成功率较低: %.2f%%, total=%d, failures=%d", + cb.ResourceName, successRate*100, total, failures) } } - return ips } -// filterServiceNames 过滤服务名(排除非服务配置的key) -func filterServiceNames(services map[string]interface{}) []string { - excludeKeys := map[string]bool{ - "services": true, - "enableDistributed": true, - "requestTimeout": true, - "distributedTTL": true, +// validateCircuitBreakerConfig 验证配置 +func validateCircuitBreakerConfig(config *CircuitBreakerConfig) error { + if config.MaxFailures <= 0 { + return fmt.Errorf("maxFailures必须大于0") } - - serviceNames := make([]string, 0, len(services)) - for key := range services { - if !excludeKeys[key] { - serviceNames = append(serviceNames, key) + if config.FailureRateThreshold < 0 || config.FailureRateThreshold > 1 { + return fmt.Errorf("failureRateThreshold必须在0.0-1.0之间") + } + if len(config.SuccessStatusCodes) == 0 { + return fmt.Errorf("successStatusCodes不能为空") + } + if config.RequestTimeout < 0 || config.RequestTimeout > 300000 { + return fmt.Errorf("requestTimeout必须在0-300000毫秒之间") + } + if config.DistributedTTL < 0 || config.DistributedTTL > 3600 { + return fmt.Errorf("distributedTTL必须在0-3600秒之间") + } + if config.StatIntervalMs < 100 || config.StatIntervalMs > 60000 { + return fmt.Errorf("statIntervalMs必须在100-60000毫秒之间") + } + if config.MinRequestAmount < 1 || config.MinRequestAmount > 10000 { + return fmt.Errorf("minRequestAmount必须在1-10000之间") + } + if config.HalfOpenMaxRequests < 1 || config.HalfOpenMaxRequests > 100 { + return fmt.Errorf("halfOpenMaxRequests必须在1-100之间") + } + if config.HalfOpenSuccessThreshold < 0 || config.HalfOpenSuccessThreshold > 1 { + return fmt.Errorf("halfOpenSuccessThreshold必须在0.0-1.0之间") + } + if config.EnableAdaptiveThreshold { + if config.AdaptiveMinThreshold < 0 || config.AdaptiveMinThreshold > 1 { + return fmt.Errorf("adaptiveMinThreshold必须在0.0-1.0之间") + } + if config.AdaptiveMaxThreshold < 0 || config.AdaptiveMaxThreshold > 1 { + return fmt.Errorf("adaptiveMaxThreshold必须在0.0-1.0之间") + } + if config.AdaptiveMinThreshold >= config.AdaptiveMaxThreshold { + return fmt.Errorf("adaptiveMinThreshold必须小于adaptiveMaxThreshold") } } - return serviceNames + return nil } // initServiceCircuitBreaker 初始化服务熔断器 func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) error { - // 验证配置参数 if err := validateCircuitBreakerConfig(config); err != nil { - return fmt.Errorf("配置验证失败: %v", err) + return err } - // 使用缓存的时间值(性能优化) - timeout := config.TimeoutParsed - slowRequestThreshold := config.SlowRequestThresholdParsed - - resourceName := fmt.Sprintf("service:%s", serviceName) + resourceName := "service:" + serviceName + threshold := config.FailureRateThreshold + if config.EnableAdaptiveThreshold { + threshold = (config.AdaptiveMinThreshold + config.AdaptiveMaxThreshold) / 2 + } var rule []*circuitbreaker.Rule if config.EnableSlidingWindow { - // 使用滑动窗口统计(更精确)- 慢调用比例策略 - rule = []*circuitbreaker.Rule{ - { - Resource: resourceName, - Strategy: circuitbreaker.SlowRequestRatio, - RetryTimeoutMs: uint32(timeout.Milliseconds()), - MinRequestAmount: uint64(config.MinRequestAmount), - StatIntervalMs: uint32(config.StatIntervalMs), - StatSlidingWindowBucketCount: 10, - MaxAllowedRtMs: uint64(slowRequestThreshold.Milliseconds()), - Threshold: config.FailureRateThreshold, - }, - } + rule = []*circuitbreaker.Rule{{ + Resource: resourceName, + Strategy: circuitbreaker.SlowRequestRatio, + RetryTimeoutMs: uint32(config.TimeoutParsed.Milliseconds()), + MinRequestAmount: uint64(config.MinRequestAmount), + StatIntervalMs: uint32(config.StatIntervalMs), + StatSlidingWindowBucketCount: 10, + MaxAllowedRtMs: uint64(config.SlowRequestThresholdParsed.Milliseconds()), + Threshold: threshold, + }} } else { - // 使用连续失败计数(更简单快速)- 异常数策略 - rule = []*circuitbreaker.Rule{ - { - Resource: resourceName, - Strategy: circuitbreaker.ErrorCount, - RetryTimeoutMs: uint32(timeout.Milliseconds()), - MinRequestAmount: uint64(config.MinRequestAmount), - StatIntervalMs: uint32(config.StatIntervalMs), - Threshold: float64(config.MaxFailures), - }, - } + rule = []*circuitbreaker.Rule{{ + Resource: resourceName, + Strategy: circuitbreaker.ErrorCount, + RetryTimeoutMs: uint32(config.TimeoutParsed.Milliseconds()), + MinRequestAmount: uint64(config.MinRequestAmount), + StatIntervalMs: uint32(config.StatIntervalMs), + Threshold: float64(config.MaxFailures), + }} } - // 先清理旧规则(健壮性改进) - _, _ = circuitbreaker.LoadRulesOfResource(resourceName, []*circuitbreaker.Rule{}) - - // 加载新规则到Sentinel - _, err := circuitbreaker.LoadRules(rule) - if err != nil { + if _, err := circuitbreaker.LoadRulesOfResource(resourceName, []*circuitbreaker.Rule{}); err != nil { + return fmt.Errorf("清空熔断规则失败: %v", err) + } + if _, err := circuitbreaker.LoadRules(rule); err != nil { return fmt.Errorf("加载熔断规则失败: %v", err) } - // 初始化熔断器信息(P1:直接从slice构建map,避免重复解析) successCodeMap := make(map[int]bool, len(config.SuccessStatusCodes)) for _, code := range config.SuccessStatusCodes { successCodeMap[code] = true } cbInfo := &CircuitBreakerInfo{ - ResourceName: resourceName, - Config: config, - Metrics: &CircuitBreakerMetrics{}, - SuccessCodeMap: successCodeMap, - CIDRNetMasks: config.CIDRNetMasks, + ResourceName: resourceName, + Config: config, + Metrics: newCircuitBreakerMetrics(), + SuccessCodeMap: successCodeMap, + CIDRNetMasks: config.CIDRNetMasks, + AdaptiveThreshold: threshold, + WarmupEndTime: time.Now().Add(config.WarmupDurationParsed).Unix(), } cbInfo.State.Store(stateClosed) cbInfo.Metrics.LastResetTime.Store(time.Now().Unix()) + cbInfo.Metrics.LastCloseTime.Store(time.Now().Unix()) + cbInfo.Metrics.WindowStartTime.Store(time.Now().Unix()) + + // 初始化响应时间统计 + cbInfo.Metrics.MinResponseTime.Store(1<<63 - 1) // 最大int64值作为初始最小值 circuitBreakers.Store(serviceName, cbInfo) strategy := "error_count" if config.EnableSlidingWindow { strategy = "slow_ratio" } - g.Log().Infof(context.Background(), "服务 %s 熔断器初始化成功: resource=%s, strategy=%s, timeout=%v, threshold=%.2f", - serviceName, resourceName, strategy, timeout, rule[0].Threshold) + serviceName, resourceName, strategy, config.TimeoutParsed, rule[0].Threshold) return nil } -// CircuitBreakerMiddleware 熔断降级中间件(使用阿里Sentinel) +// CircuitBreakerMiddleware 熔断降级中间件 func CircuitBreakerMiddleware(r *ghttp.Request) { startTime := time.Now() ctx := r.GetCtx() - // 从URL路径提取服务名并获取配置(P1:合并重复验证) serviceName := extractServiceName(r.URL.Path) if serviceName == "" { r.Middleware.Next() return } - // 获取熔断器信息(包含配置) cbInfoVal, ok := circuitBreakers.Load(serviceName) if !ok { - // 未配置熔断器,直接放行 r.Middleware.Next() return } - cbInfo := cbInfoVal.(*CircuitBreakerInfo) + cbInfo, ok := cbInfoVal.(*CircuitBreakerInfo) + if !ok { + r.Middleware.Next() + return + } config := cbInfo.Config if !config.Enabled { - // 熔断器未启用,直接放行 r.Middleware.Next() return } cbInfo.Metrics.TotalRequests.Add(1) - // 提前构造resourceName(性能优化) - resourceName := cbInfo.ResourceName - - // 设置请求超时(使用服务独立配置) - if config.RequestTimeout > 0 { - ctx, cancel := context.WithTimeout(ctx, time.Duration(config.RequestTimeout)*time.Millisecond) - r.SetCtx(ctx) - defer cancel() + // 预热期检查 + if time.Now().Unix() < cbInfo.WarmupEndTime { + r.Middleware.Next() + return } - // 检查是否启用分布式熔断 - if config.DistributedTTL > 0 { - if isCircuitBreakerOpenInDistributed(ctx, resourceName) { + resourceName := cbInfo.ResourceName + if config.RequestTimeout > 0 { + var ctxCancel context.CancelFunc + ctx, ctxCancel = context.WithTimeout(ctx, time.Duration(config.RequestTimeout)*time.Millisecond) + r.SetCtx(ctx) + defer ctxCancel() + } + + // 分布式熔断检查 + if config.DistributedTTL > 0 && isCircuitBreakerOpenInDistributed(ctx, resourceName) { + cbInfo.Metrics.BlockRequests.Add(1) + g.Log().Warningf(ctx, "分布式熔断触发: %s", resourceName) + sendFallbackResponse(r, serviceName, config, "distributed") + return + } + + // 半开状态处理 - 使用原子操作确保线程安全 + currentState := cbInfo.getState() + if currentState == StateHalfOpen { + // 使用原子操作安全地递增半开请求计数 + halfOpenRequests := cbInfo.Metrics.HalfOpenRequests.Add(1) + + // 如果超过最大半开请求数量,回滚并触发熔断 + if halfOpenRequests > int64(config.HalfOpenMaxRequests) { + // 原子递减回滚 + cbInfo.Metrics.HalfOpenRequests.Add(-1) cbInfo.Metrics.BlockRequests.Add(1) - g.Log().Warningf(ctx, "分布式熔断触发: %s", resourceName) - sendFallbackResponse(r, serviceName, config, "distributed") + + // 尝试转换为打开状态,如果成功则记录日志 + oldState := cbInfo.setState(StateOpen) + if oldState != StateOpen { + g.Log().Warningf(ctx, "半开状态试探请求超限,恢复熔断: %s", resourceName) + if config.DistributedTTL > 0 { + syncCircuitBreakerStateToDistributed(ctx, resourceName, "open", config.DistributedTTL) + } + } + + sendFallbackResponse(r, serviceName, config, "halfopen_limit") return } } - // 使用Sentinel进行熔断保护 entry, blockError := api.Entry(resourceName) if blockError != nil { - // 被熔断拦截 + if entry != nil { + entry.Exit() + } + cbInfo.Metrics.BlockRequests.Add(1) - cbInfo.Metrics.OpenCount.Add(1) - g.Log().Warningf(ctx, "熔断触发: %s, reason: %v", resourceName, blockError) - - // 使用atomic更新状态(无锁) - oldStateStr := cbInfo.setState(StateOpen) - now := time.Now() - cbInfo.Metrics.LastOpenTime.Store(now.Unix()) - cbInfo.Metrics.NextRetryTime.Store(now.Add(config.TimeoutParsed).Unix()) + oldState := cbInfo.setStateWithMetrics(StateOpen, true) - // 通知状态变化(如果状态改变) - if oldStateStr != StateOpen { - notifyStateChange(serviceName, oldStateStr, StateOpen) + if oldState != StateOpen { + notifyStateChange(serviceName, oldState, StateOpen) } - // 同步到分布式存储 if config.DistributedTTL > 0 { syncCircuitBreakerStateToDistributed(ctx, resourceName, "open", config.DistributedTTL) } @@ -554,72 +647,149 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { return } - // 执行后续中间件和业务逻辑 + if entry != nil { + defer entry.Exit() + } + r.Middleware.Next() - // 记录请求结果(基于HTTP状态码) statusCode := r.Response.Status + if statusCode < 100 || statusCode > 599 { + return + } duration := time.Since(startTime) - // 判断是否为慢请求(P2:可观测性) + // 记录响应时间统计 + durationNs := duration.Nanoseconds() + cbInfo.Metrics.TotalResponseTime.Add(durationNs) + + // 原子更新最小响应时间 + for { + currentMin := cbInfo.Metrics.MinResponseTime.Load() + if durationNs >= currentMin { + break + } + if cbInfo.Metrics.MinResponseTime.CompareAndSwap(currentMin, durationNs) { + break + } + } + + // 原子更新最大响应时间 + for { + currentMax := cbInfo.Metrics.MaxResponseTime.Load() + if durationNs <= currentMax { + break + } + if cbInfo.Metrics.MaxResponseTime.CompareAndSwap(currentMax, durationNs) { + break + } + } + if duration > config.SlowRequestThresholdParsed { cbInfo.Metrics.SlowRequests.Add(1) } - // 使用cbInfo.SuccessCodeMap判断状态码(性能优化) - if !isSuccessStatusCode(cbInfo, statusCode) { - // 记录异常 + isSuccess := isSuccessStatusCode(cbInfo, statusCode) + + // 更新窗口统计 + cbInfo.updateWindowStats(isSuccess, ctx) + + if !isSuccess { cbInfo.Metrics.FailureRequests.Add(1) - api.TraceError(entry, fmt.Errorf("request failed with status: %d", statusCode)) + if entry != nil { + api.TraceError(entry, fmt.Errorf("request failed with status: %d", statusCode)) + } g.Log().Debugf(ctx, "服务 %s 请求失败: status=%d, duration=%v", serviceName, statusCode, duration) + + // 重新获取当前状态,避免使用过期状态 + currentState := cbInfo.getState() + if currentState == StateHalfOpen { + cbInfo.Metrics.HalfOpenFailed.Add(1) + oldState := cbInfo.setStateWithMetrics(StateOpen, true) + if oldState == StateHalfOpen { + g.Log().Warningf(ctx, "半开状态请求失败,恢复熔断: %s", resourceName) + if config.DistributedTTL > 0 { + syncCircuitBreakerStateToDistributed(ctx, resourceName, "open", config.DistributedTTL) + } + } + } } else { cbInfo.Metrics.PassRequests.Add(1) - // 更新状态为关闭(如果之前是开启状态,使用atomic操作) - if cbInfo.getState() != StateClosed { - oldStateStr := cbInfo.setState(StateClosed) - if oldStateStr != StateClosed { - notifyStateChange(serviceName, oldStateStr, StateClosed) + + // 重新获取当前状态 + currentState := cbInfo.getState() + if currentState == StateHalfOpen { + // 原子递增成功计数 + halfOpenPassed := cbInfo.Metrics.HalfOpenPassed.Add(1) + totalRequests := cbInfo.Metrics.HalfOpenRequests.Load() + + // 计算成功率,确保分母不为零 + if totalRequests > 0 { + successRate := float64(halfOpenPassed) / float64(totalRequests) + + // 检查是否达到成功率阈值,如果达到则关闭熔断器 + if successRate >= config.HalfOpenSuccessThreshold { + // 原子设置状态为关闭,确保只有一个goroutine能成功转换 + oldState := cbInfo.setStateWithMetrics(StateClosed, true) + if oldState == StateHalfOpen { + // 重置半开统计 + cbInfo.Metrics.HalfOpenPassed.Store(0) + cbInfo.Metrics.HalfOpenRequests.Store(0) + cbInfo.Metrics.HalfOpenFailed.Store(0) + + g.Log().Infof(ctx, "半开状态成功,恢复关闭: %s, successRate=%.2f, total=%d, passed=%d", + resourceName, successRate, totalRequests, halfOpenPassed) + + // 同步分布式状态 + if config.DistributedTTL > 0 { + syncCircuitBreakerStateToDistributed(ctx, resourceName, "closed", config.DistributedTTL) + } + } + } + } + } else if currentState != StateClosed { + // 如果状态不是关闭但也不是半开,尝试重置为关闭状态 + oldState := cbInfo.setStateWithMetrics(StateClosed, true) + if oldState != StateClosed { + notifyStateChange(serviceName, oldState, StateClosed) } } } - - // 退出Sentinel资源 - entry.Exit() } -// sendFallbackResponse 发送降级响应(P0:添加日志记录) +// sendFallbackResponse 发送降级响应 func sendFallbackResponse(r *ghttp.Request, serviceName string, config *CircuitBreakerConfig, reason string) { - // P0:记录降级日志,便于问题排查 g.Log().Warningf(r.GetCtx(), "熔断器降级: service=%s, reason=%s, clientIP=%s", serviceName, reason, r.GetClientIp()) if config.EnableFallback && config.FallbackMessage != "" { - // 自定义降级消息 r.Response.WriteStatusExit(503, config.FallbackMessage) - } else { - // 根据原因返回不同的状态码和消息 - switch reason { - case "blocked": - r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 熔断保护中,请稍后再试", serviceName)) - case "distributed": - r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 分布式熔断中", serviceName)) - default: - r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 暂时不可用,请稍后再试", serviceName)) - } + return + } + + switch reason { + case "blocked": + r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 熔断保护中,请稍后再试", serviceName)) + case "distributed": + r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 分布式熔断中", serviceName)) + default: + r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 暂时不可用,请稍后再试", serviceName)) } } -// isSuccessStatusCode 判断HTTP状态码是否成功(使用cbInfo.SuccessCodeMap优化性能) +// isSuccessStatusCode 判断HTTP状态码是否成功 func isSuccessStatusCode(cbInfo *CircuitBreakerInfo, statusCode int) bool { - if cbInfo.SuccessCodeMap != nil && len(cbInfo.SuccessCodeMap) > 0 { + // 验证状态码范围 + if statusCode < 100 || statusCode > 599 { + return false + } + if len(cbInfo.SuccessCodeMap) > 0 { return cbInfo.SuccessCodeMap[statusCode] } - // 默认:2xx状态码为成功 return statusCode >= 200 && statusCode < 300 } -// extractServiceName 从URL路径提取服务名(P0:添加URL编码处理) +// extractServiceName 从URL路径提取服务名 func extractServiceName(path string) string { - // 去除首尾斜杠并分割 path = strings.Trim(path, "/") if path == "" { return "" @@ -630,55 +800,38 @@ func extractServiceName(path string) string { } serviceName := parts[0] - // P0:处理URL编码,将 %2F 等转义字符还原 - // 注意:在goframe的网关中间件中,路径通常已经被框架处理过 - // 但为了安全性,这里对包含%的情况进行简单处理 if strings.Contains(serviceName, "%") { - // 尝试解码URL编码的字符串 - // 使用path.Unescape而不是url.QueryUnescape,因为我们处理的是路径片段 - decoded, err := pathUnescape(serviceName) - if err == nil { + if decoded, err := pathUnescape(serviceName); err == nil { serviceName = decoded } - // 如果解码失败,继续使用原始serviceName } - // 验证服务名是否在已配置的熔断器中 if _, ok := circuitBreakerConfigs.Load(serviceName); ok { return serviceName } return "" } -// pathUnescape 路径片段的URL解码(P0:安全性改进) -// 注意:Go 1.8+ 可以使用 path.Unescape,这里提供兼容实现 +// pathUnescape 路径片段的URL解码 func pathUnescape(s string) (string, error) { - // 使用strings.Builder优化性能 var builder strings.Builder builder.Grow(len(s)) for i := 0; i < len(s); i++ { switch s[i] { case '%': - // 处理百分号编码 if i+2 >= len(s) { - // 不完整的编码,保留原样 builder.WriteByte(s[i]) continue } - // 解析十六进制数字 high := hexDigit(s[i+1]) low := hexDigit(s[i+2]) if high == 0xFF || low == 0xFF { - // 无效的十六进制,保留原样 builder.WriteByte(s[i]) - continue + } else { + builder.WriteByte((high << 4) | low) + i += 2 } - builder.WriteByte((high << 4) | low) - i += 2 - case '+': - // 路径片段中的+通常不需要解码为空格 - builder.WriteByte('+') default: builder.WriteByte(s[i]) } @@ -686,7 +839,6 @@ func pathUnescape(s string) (string, error) { return builder.String(), nil } -// hexDigit 将十六进制字符转换为对应的数值 func hexDigit(c byte) byte { switch { case '0' <= c && c <= '9': @@ -696,188 +848,199 @@ func hexDigit(c byte) byte { case 'A' <= c && c <= 'F': return c - 'A' + 10 default: - return 0xFF // 无效字符 + return 0xFF } } +// updateAdminIPsCache 更新管理员IP白名单缓存 +func updateAdminIPsCache() { + ipMap := make(map[string]bool) + cidrNets := make([]*net.IPNet, 0) + + circuitBreakerConfigs.Range(func(_, value interface{}) bool { + config, ok := value.(*CircuitBreakerConfig) + if !ok { + return true + } + for _, ip := range config.AdminIPs { + if !ipMap[ip] { + ipMap[ip] = true + } + } + cidrNets = append(cidrNets, config.CIDRNetMasks...) + return true + }) + + allowedAdminIPsMutex.Lock() + allowedAdminIPsMap = ipMap + allowedAdminIPsMutex.Unlock() + + allowedAdminCIDRsMutex.Lock() + allowedAdminCIDRs = cidrNets + allowedAdminCIDRsMutex.Unlock() +} + +// filterServiceNames 过滤服务名 +func filterServiceNames(services map[string]interface{}) []string { + excludeKeys := map[string]bool{"services": true, "enableDistributed": true, "requestTimeout": true, "distributedTTL": true} + result := make([]string, 0, len(services)) + for key := range services { + if !excludeKeys[key] { + result = append(result, key) + } + } + return result +} + // isCircuitBreakerOpenInDistributed 检查分布式熔断状态 func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) bool { - key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) - + key := "circuit_breaker:" + resourceName + ":state" redis := g.Redis() if redis == nil { return false } - value, err := redis.Get(ctx, key) if err != nil || value.IsNil() { return false } - state := value.String() - return state == "open" + return value.String() == "open" } -// getDistributedLock 获取分布式锁(按服务名分片) -func getDistributedLock(serviceName string) *sync.Mutex { - lock, _ := distributedSyncLocks.LoadOrStore(serviceName, &sync.Mutex{}) - return lock.(*sync.Mutex) -} - -// syncCircuitBreakerStateToDistributed 同步熔断器状态到分布式存储 +// syncCircuitBreakerStateToDistributed 同步熔断器状态到Redis func syncCircuitBreakerStateToDistributed(ctx context.Context, resourceName, state string, ttl int) { - // 提取服务名用于锁分片 - serviceName := strings.TrimPrefix(resourceName, "service:") - lock := getDistributedLock(serviceName) - lock.Lock() - defer lock.Unlock() - - key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) - + key := "circuit_breaker:" + resourceName + ":state" + lockKey := "circuit_breaker:" + resourceName + ":lock" redis := g.Redis() if redis == nil { - g.Log().Errorf(ctx, "Redis客户端未初始化,无法同步熔断状态") + g.Log().Warningf(ctx, "Redis未初始化,无法同步分布式熔断状态: %s", resourceName) return } - _, err := redis.Do(ctx, "SETEX", key, ttl, state) + lockValue := fmt.Sprintf("%d", time.Now().UnixNano()) + + // 获取分布式锁 + locked, err := redis.Do(ctx, "SET", lockKey, lockValue, "NX", "EX", 10) if err != nil { - g.Log().Errorf(ctx, "同步熔断状态到Redis失败: %v", err) + g.Log().Errorf(ctx, "获取分布式锁失败: %s, error: %v", lockKey, err) + return } -} -// validateCircuitBreakerConfig 验证熔断器配置 -func validateCircuitBreakerConfig(config *CircuitBreakerConfig) error { - if config.MaxFailures <= 0 { - return fmt.Errorf("maxFailures必须大于0,当前值: %d", config.MaxFailures) - } - if config.FailureRateThreshold < 0 || config.FailureRateThreshold > 1 { - return fmt.Errorf("failureRateThreshold必须在0.0-1.0之间,当前值: %.2f", config.FailureRateThreshold) - } - if len(config.SuccessStatusCodes) == 0 { - return fmt.Errorf("successStatusCodes不能为空") - } - if config.RequestTimeout < 0 || config.RequestTimeout > 300000 { - return fmt.Errorf("requestTimeout必须在0-300000毫秒之间,当前值: %d", config.RequestTimeout) - } - if config.DistributedTTL < 0 || config.DistributedTTL > 3600 { - return fmt.Errorf("distributedTTL必须在0-3600秒之间,当前值: %d", config.DistributedTTL) - } - if config.StatIntervalMs < 100 || config.StatIntervalMs > 60000 { - return fmt.Errorf("statIntervalMs必须在100-60000毫秒之间,当前值: %d", config.StatIntervalMs) - } - if config.MinRequestAmount < 1 || config.MinRequestAmount > 10000 { - return fmt.Errorf("minRequestAmount必须在1-10000之间,当前值: %d", config.MinRequestAmount) - } - // 验证时间字符串格式(如果缓存为空,说明解析失败) - if config.TimeoutParsed == 0 { - return fmt.Errorf("timeout格式错误,应为有效的时间字符串(如30s, 1m),当前值: %s", config.Timeout) - } - if config.SlowRequestThresholdParsed == 0 { - return fmt.Errorf("slowRequestThreshold格式错误,应为有效的时间字符串(如3s, 1m),当前值: %s", config.SlowRequestThreshold) - } + // 检查是否获取到锁 + var isLocked bool + if locked != nil && !locked.IsNil() { + isLocked = true + } else { + // 检查锁是否已过期 + currentLock, err := redis.Get(ctx, lockKey) + if err == nil && !currentLock.IsNil() { + lockTime, _ := strconv.ParseInt(currentLock.String(), 10, 64) + // 如果锁已经存在超过10秒(超时),强制获取 + if time.Now().UnixNano()-lockTime > 10*1e9 { + // 使用SETNX方式获取锁,使用Lua脚本保证原子性 + luaAcquire := ` +local current = redis.call("get", KEYS[1]) +if current and tonumber(current) then + local lockTime = tonumber(current) + if redis.call("TIME")[1] * 1000000000 + redis.call("TIME")[2] - lockTime > 10000000000 then + redis.call("del", KEYS[1]) + return redis.call("set", KEYS[1], ARGV[1], "EX", 10) + end return nil -} - -// registerStateChangeListeners 注册状态变化监听器 -func registerStateChangeListeners() { - // 检查是否已注册,防止重复注册(健壮性改进) - if _, exists := stateChangeListenersRegistered.LoadOrStore("default", true); exists { - return +else + return redis.call("set", KEYS[1], ARGV[1], "EX", 10) +end` + locked, err = redis.Do(ctx, "EVAL", luaAcquire, 1, lockKey, lockValue) + if err != nil { + g.Log().Errorf(ctx, "强制获取分布式锁失败: %s, error: %v", lockKey, err) + return + } + if locked != nil && !locked.IsNil() { + isLocked = true + } + } + } } - // 注册默认监听器(区分日志级别) - RegisterStateChangeListener("default", func(serviceName string, fromState, toState CircuitBreakerState) { - // Open状态使用Warning级别,Closed状态使用Info级别 - if toState == StateOpen { - g.Log().Warningf(context.Background(), "熔断器状态变化: service=%s, %s -> %s", - serviceName, fromState, toState) + if isLocked { + defer func() { + // 使用Lua脚本原子性地删除锁,只删除自己创建的锁 + luaScript := `if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end` + _, err := redis.Do(ctx, "EVAL", luaScript, 1, lockKey, lockValue) + if err != nil { + g.Log().Warningf(ctx, "释放分布式锁失败: %s, error: %v", lockKey, err) + } + }() + + // 设置状态 + _, err := redis.Do(ctx, "SETEX", key, ttl, state) + if err != nil { + g.Log().Errorf(ctx, "设置分布式熔断状态失败: %s=%s, error: %v", key, state, err) } else { - g.Log().Infof(context.Background(), "熔断器状态变化: service=%s, %s -> %s", - serviceName, fromState, toState) + g.Log().Debugf(ctx, "分布式熔断状态已同步: %s=%s (TTL: %d)", key, state, ttl) } - }) + } else { + g.Log().Debugf(ctx, "未获取到分布式锁,跳过状态同步: %s", lockKey) + } } -// StateChangeListener 状态变化监听器类型 -type StateChangeListener func(serviceName string, fromState, toState CircuitBreakerState) - -// RegisterStateChangeListener 注册状态变化监听器 -func RegisterStateChangeListener(name string, listener StateChangeListener) { - stateChangeListeners.Store(name, listener) -} - -// UnregisterStateChangeListener 取消注册状态变化监听器 -func UnregisterStateChangeListener(name string) { - stateChangeListeners.Delete(name) -} - -// notifyStateChange 通知所有监听器状态变化 -func notifyStateChange(serviceName string, fromState, toState CircuitBreakerState) { - stateChangeListeners.Range(func(key, value interface{}) bool { - if listener, ok := value.(StateChangeListener); ok { - listener(serviceName, fromState, toState) - } - return true - }) -} - -// CircuitBreakerHealthCheckHandler 熔断器健康检查接口(P0:添加IP白名单验证,P1:添加分页支持,P1:优化性能) +// CircuitBreakerHealthCheckHandler 健康检查接口 func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { - // P0:权限验证 if !isAdminIP(r) { - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 403, - Message: "权限不足,禁止访问", - }) + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 403, Message: "权限不足,禁止访问"}) return } - status := make(map[string]interface{}) - totalServices := 0 - openServices := 0 - - // P1:分页参数 page := r.Get("page").Int() size := r.Get("size").Int() if page < 0 { page = 0 } if size <= 0 || size > 100 { - size = 20 // 默认20条,最多100条 + size = 20 } - // P1:使用缓存的totalServicesCount避免每次遍历 - total := int(totalServicesCount.Load()) + serviceNamesMutex.RLock() + slice := serviceNamesSlice + serviceNamesMutex.RUnlock() + + total := len(slice) start := page * size + if start >= total { + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: "熔断器状态", + Data: map[string]interface{}{ + "summary": map[string]interface{}{"totalServices": 0, "openServices": 0, "closedServices": 0, "halfOpenServices": 0}, + "services": map[string]interface{}{}, "page": page, "size": size, "total": total}}) + return + } - // P1:只遍历分页范围内的服务(通过计数跳过) end := start + size if end > total { end = total } - current := 0 - circuitBreakers.Range(func(key, value interface{}) bool { - // 跳过前面的页 - if current < start { - current++ - return true - } - // 只处理当前页 - if current >= end { - return false - } - - serviceName := key.(string) - cbInfo := value.(*CircuitBreakerInfo) + status := make(map[string]interface{}) + totalServices := 0 + openServices := 0 + halfOpenServices := 0 + for i := start; i < end; i++ { + serviceName := slice[i] + cbInfoVal, ok := circuitBreakers.Load(serviceName) + if !ok { + continue + } + cbInfo, ok := cbInfoVal.(*CircuitBreakerInfo) + if !ok { + continue + } totalServices++ - isOpen := cbInfo.getState() == StateOpen - if isOpen { + state := cbInfo.getState() + if state == StateOpen { openServices++ + } else if state == StateHalfOpen { + halfOpenServices++ } - // 从Metrics中读取数据(使用atomic) lastResetTime := cbInfo.Metrics.LastResetTime.Load() var lastResetTimeStr string if lastResetTime > 0 { @@ -897,70 +1060,52 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { } status[serviceName] = map[string]interface{}{ - "resource": cbInfo.ResourceName, - "state": string(cbInfo.getState()), - "lastOpenTime": lastOpenTimeStr, - "nextRetryTime": nextRetryTimeStr, - "totalRequests": cbInfo.Metrics.TotalRequests.Load(), - "passRequests": cbInfo.Metrics.PassRequests.Load(), - "blockRequests": cbInfo.Metrics.BlockRequests.Load(), - "failureRequests": cbInfo.Metrics.FailureRequests.Load(), - "slowRequests": cbInfo.Metrics.SlowRequests.Load(), - "openCount": cbInfo.Metrics.OpenCount.Load(), - "lastResetTime": lastResetTimeStr, + "resource": cbInfo.ResourceName, + "state": string(state), + "lastOpenTime": lastOpenTimeStr, + "nextRetryTime": nextRetryTimeStr, + "totalRequests": cbInfo.Metrics.TotalRequests.Load(), + "passRequests": cbInfo.Metrics.PassRequests.Load(), + "blockRequests": cbInfo.Metrics.BlockRequests.Load(), + "failureRequests": cbInfo.Metrics.FailureRequests.Load(), + "slowRequests": cbInfo.Metrics.SlowRequests.Load(), + "openCount": cbInfo.Metrics.OpenCount.Load(), + "lastResetTime": lastResetTimeStr, + "halfOpenRequests": cbInfo.Metrics.HalfOpenRequests.Load(), + "halfOpenPassed": cbInfo.Metrics.HalfOpenPassed.Load(), } - current++ - return true - }) - - summary := map[string]interface{}{ - "totalServices": totalServices, - "openServices": openServices, - "closedServices": totalServices - openServices, } - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 200, - Message: "熔断器状态", + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: "熔断器状态", Data: map[string]interface{}{ - "summary": summary, - "services": status, - "page": page, - "size": size, - "total": total, - }, - }) + "summary": map[string]interface{}{"totalServices": totalServices, "openServices": openServices, "closedServices": totalServices - openServices - halfOpenServices, "halfOpenServices": halfOpenServices}, + "services": status, "page": page, "size": size, "total": total}}) } -// isAdminIP 检查请求IP是否在管理员白名单中(P1:使用map优化性能,P0:支持IPv6 CIDR) +// isAdminIP 检查IP是否在白名单中 func isAdminIP(r *ghttp.Request) bool { clientIP := r.GetClientIp() if clientIP == "" { return false } - // 读取缓存的IP白名单(P1:使用map实现O(1)查找) allowedAdminIPsMutex.RLock() allowedIPs := allowedAdminIPsMap allowedAdminIPsMutex.RUnlock() - // 如果没有配置白名单,允许所有IP(向后兼容) if len(allowedIPs) == 0 { allowedAdminCIDRsMutex.RLock() hasCIDRs := len(allowedAdminCIDRs) > 0 allowedAdminCIDRsMutex.RUnlock() - // 如果也没有CIDR,则允许所有IP if !hasCIDRs { return true } } - // 精确IP匹配(P1:map查找O(1)) if allowedIPs[clientIP] { return true } - // P1:使用预编译的CIDR网络掩码匹配(支持IPv4和IPv6) allowedAdminCIDRsMutex.RLock() cidrNets := allowedAdminCIDRs allowedAdminCIDRsMutex.RUnlock() @@ -980,151 +1125,168 @@ func isAdminIP(r *ghttp.Request) bool { return false } -// CircuitBreakerResetHandler 熔断器手动重置接口(仅限管理后台调用) +// CircuitBreakerResetHandler 重置熔断器 func CircuitBreakerResetHandler(r *ghttp.Request) { serviceName := r.Get("service").String() - if serviceName == "" { - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 400, - Message: "缺少service参数", - }) - return - } - // 权限验证:检查IP是否在白名单中 if !isAdminIP(r) { - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 403, - Message: "权限不足,禁止访问", - }) + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 403, Message: "权限不足,禁止访问"}) return } - resourceName := fmt.Sprintf("service:%s", serviceName) - - // 获取当前服务的所有规则 - currentRules := circuitbreaker.GetRulesOfResource(resourceName) - - // 只删除当前服务的规则 - if len(currentRules) > 0 { - _, err := circuitbreaker.LoadRulesOfResource(resourceName, []*circuitbreaker.Rule{}) - if err != nil { - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 500, - Message: fmt.Sprintf("重置熔断器失败: %v", err), - }) - return - } - } - - // 重新加载该服务的规则 - if val, ok := circuitBreakerConfigs.Load(serviceName); ok { - config := val.(*CircuitBreakerConfig) - err := initServiceCircuitBreaker(serviceName, config) - if err != nil { - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 500, - Message: fmt.Sprintf("重置熔断器失败: %v", err), - }) - return - } - } - - // 更新内存状态并重置指标(使用atomic操作) - if val, ok := circuitBreakers.Load(serviceName); ok { - cbInfo := val.(*CircuitBreakerInfo) - cbInfo.State.Store(stateClosed) - cbInfo.Metrics.LastOpenTime.Store(0) - cbInfo.Metrics.NextRetryTime.Store(0) - // 重置指标 - cbInfo.Metrics.TotalRequests.Store(0) - cbInfo.Metrics.PassRequests.Store(0) - cbInfo.Metrics.BlockRequests.Store(0) - cbInfo.Metrics.FailureRequests.Store(0) - cbInfo.Metrics.SlowRequests.Store(0) - cbInfo.Metrics.OpenCount.Store(0) - cbInfo.Metrics.LastResetTime.Store(time.Now().Unix()) - } - - // 重置分布式状态(如果启用) - if val, ok := circuitBreakerConfigs.Load(serviceName); ok { - config := val.(*CircuitBreakerConfig) - if config.DistributedTTL > 0 { - key := fmt.Sprintf("circuit_breaker:%s:state", resourceName) - redis := g.Redis() - if redis != nil { - _, _ = redis.Del(r.GetCtx(), key) - } - } - } - - g.Log().Infof(r.GetCtx(), "熔断器已手动重置: %s", resourceName) - - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 200, - Message: fmt.Sprintf("服务 '%s' 的熔断器已重置", serviceName), - }) -} - -// CircuitBreakerReloadHandler 熔断器配置重载接口 -func CircuitBreakerReloadHandler(r *ghttp.Request) { - serviceName := r.Get("service").String() - - // 权限验证:检查IP是否在白名单中(P0级别安全问题) - if !isAdminIP(r) { - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 403, - Message: "权限不足,禁止访问", - }) - return - } - - if serviceName == "" { - // 重载所有服务 - 扫描配置文件中所有服务 - services := g.Cfg().MustGet(r.GetCtx(), "circuitBreaker").Map() - - // 过滤出服务名 - serviceNames := filterServiceNames(services) + if serviceName == "" || serviceName == "*" { + serviceNamesMutex.RLock() + slice := serviceNamesSlice + serviceNamesMutex.RUnlock() successCount := 0 failCount := 0 - for _, service := range serviceNames { - err := ReloadCircuitBreakerConfig(service) - if err != nil { - g.Log().Errorf(r.GetCtx(), "服务 %s 配置重载失败: %v", service, err) + for _, name := range slice { + if err := resetSingleService(r, name); err != nil { + g.Log().Errorf(r.GetCtx(), "服务 %s 熔断器重置失败: %v", name, err) failCount++ } else { successCount++ } } - // 更新管理员IP白名单缓存 + g.Log().Infof(r.GetCtx(), "批量重置熔断器完成: 成功 %d, 失败 %d", successCount, failCount) + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: fmt.Sprintf("批量重置完成: 成功 %d, 失败 %d", successCount, failCount), + Data: map[string]interface{}{"success": successCount, "failed": failCount}}) + return + } + + if err := resetSingleService(r, serviceName); err != nil { + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 500, Message: fmt.Sprintf("重置熔断器失败: %v", err)}) + return + } + + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: fmt.Sprintf("服务 '%s' 的熔断器已重置", serviceName)}) +} + +// resetSingleService 重置单个服务 +func resetSingleService(r *ghttp.Request, serviceName string) error { + resourceName := "service:" + serviceName + + if rules := circuitbreaker.GetRulesOfResource(resourceName); len(rules) > 0 { + if _, err := circuitbreaker.LoadRulesOfResource(resourceName, []*circuitbreaker.Rule{}); err != nil { + return err + } + } + + if configVal, ok := circuitBreakerConfigs.Load(serviceName); ok { + if err := initServiceCircuitBreaker(serviceName, configVal.(*CircuitBreakerConfig)); err != nil { + return err + } + } + + if cbInfoVal, ok := circuitBreakers.Load(serviceName); ok { + cbInfo := cbInfoVal.(*CircuitBreakerInfo) + cbInfo.State.Store(stateClosed) + cbInfo.Metrics.LastOpenTime.Store(0) + cbInfo.Metrics.NextRetryTime.Store(0) + cbInfo.Metrics.TotalRequests.Store(0) + cbInfo.Metrics.PassRequests.Store(0) + cbInfo.Metrics.BlockRequests.Store(0) + cbInfo.Metrics.FailureRequests.Store(0) + cbInfo.Metrics.SlowRequests.Store(0) + cbInfo.Metrics.OpenCount.Store(0) + cbInfo.Metrics.HalfOpenRequests.Store(0) + cbInfo.Metrics.HalfOpenPassed.Store(0) + cbInfo.WarmupEndTime = time.Now().Add(cbInfo.Config.WarmupDurationParsed).Unix() + cbInfo.Metrics.LastResetTime.Store(time.Now().Unix()) + } + + if configVal, ok := circuitBreakerConfigs.Load(serviceName); ok { + config, ok := configVal.(*CircuitBreakerConfig) + if ok && config.DistributedTTL > 0 { + redis := g.Redis() + if redis != nil { + if _, err := redis.Del(r.GetCtx(), "circuit_breaker:"+resourceName+":state"); err != nil { + g.Log().Warningf(r.GetCtx(), "清除分布式熔断状态失败: %s, error: %v", resourceName, err) + } + } + } + } + + g.Log().Infof(r.GetCtx(), "熔断器已手动重置: %s", resourceName) + return nil +} + +// CircuitBreakerReloadHandler 配置重载接口 +func CircuitBreakerReloadHandler(r *ghttp.Request) { + serviceName := r.Get("service").String() + + if !isAdminIP(r) { + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 403, Message: "权限不足,禁止访问"}) + return + } + + if serviceName == "" || serviceName == "*" { + serviceNamesMutex.RLock() + slice := serviceNamesSlice + serviceNamesMutex.RUnlock() + + successCount := 0 + failCount := 0 + failures := make(map[string]string) + + for _, service := range slice { + if err := ReloadCircuitBreakerConfig(service); err != nil { + g.Log().Errorf(r.GetCtx(), "服务 %s 配置重载失败: %v", service, err) + failCount++ + failures[service] = err.Error() + } else { + successCount++ + } + } + updateAdminIPsCache() - - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 200, - Message: fmt.Sprintf("配置重载完成: 成功 %d, 失败 %d", successCount, failCount), - Data: map[string]interface{}{ - "success": successCount, - "failed": failCount, - }, - }) + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: fmt.Sprintf("配置重载完成: 成功 %d, 失败 %d", successCount, failCount), + Data: map[string]interface{}{"success": successCount, "failed": failCount, "failures": failures}}) return } - // 重载单个服务 - err := ReloadCircuitBreakerConfig(serviceName) - if err != nil { - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 500, - Message: fmt.Sprintf("重载失败: %v", err), - }) + if err := ReloadCircuitBreakerConfig(serviceName); err != nil { + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 500, Message: fmt.Sprintf("重载失败: %v", err)}) return } - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 200, - Message: fmt.Sprintf("服务 '%s' 的熔断器配置已重载", serviceName), + updateAdminIPsCache() + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: fmt.Sprintf("服务 '%s' 的熔断器配置已重载", serviceName)}) +} + +// StateChangeListener 状态变化监听器类型 +type StateChangeListener func(serviceName string, fromState, toState CircuitBreakerState) + +// RegisterStateChangeListener 注册监听器 +func RegisterStateChangeListener(name string, listener StateChangeListener) { + stateChangeListeners.Store(name, listener) +} + +// notifyStateChange 通知监听器 +func notifyStateChange(serviceName string, fromState, toState CircuitBreakerState) { + stateChangeListeners.Range(func(_, value interface{}) bool { + listener, ok := value.(StateChangeListener) + if ok { + listener(serviceName, fromState, toState) + } + return true + }) +} + +// registerStateChangeListeners 注册默认监听器 +func registerStateChangeListeners() { + if _, exists := stateChangeListenersRegistered.LoadOrStore("default", true); exists { + return + } + + RegisterStateChangeListener("default", func(serviceName string, fromState, toState CircuitBreakerState) { + level := "Info" + if toState == StateOpen { + level = "Warning" + } + g.Log().Print(context.Background(), level, fmt.Sprintf("熔断器状态变化: service=%s, %s -> %s", serviceName, fromState, toState)) }) } From cb6d0911d3796311532f2173f147a34fec61d83b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 13:39:51 +0800 Subject: [PATCH 131/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 413 ++++++++++++++++------------------ 1 file changed, 194 insertions(+), 219 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 913c581..8c615a0 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -10,6 +10,7 @@ import ( "sync/atomic" "time" + "gitee.com/red-future---jilin-g/common/redis" "github.com/alibaba/sentinel-golang/api" "github.com/alibaba/sentinel-golang/core/circuitbreaker" "github.com/gogf/gf/v2/frame/g" @@ -156,54 +157,70 @@ func (cb *CircuitBreakerInfo) setState(state CircuitBreakerState) CircuitBreaker // setStateWithMetrics 设置熔断器状态并更新指标 func (cb *CircuitBreakerInfo) setStateWithMetrics(state CircuitBreakerState, updateMetrics bool) CircuitBreakerState { - var newState int64 - switch state { - case StateOpen: - newState = stateOpen - case StateHalfOpen: - newState = stateHalfOpen - default: - newState = stateClosed - } - + newState := cb.stateToInt64(state) oldState := cb.State.Swap(newState) - var oldStateEnum CircuitBreakerState - - switch oldState { - case stateOpen: - oldStateEnum = StateOpen - case stateHalfOpen: - oldStateEnum = StateHalfOpen - default: - oldStateEnum = StateClosed - } + oldStateEnum := cb.int64ToState(oldState) // 如果状态发生了变化且需要更新指标 if oldStateEnum != state && updateMetrics { - now := time.Now().Unix() - - // 根据新状态更新计数器 - switch state { - case StateOpen: - cb.Metrics.OpenCount.Add(1) - cb.Metrics.LastOpenTime.Store(now) - case StateClosed: - cb.Metrics.ClosedCount.Add(1) - cb.Metrics.LastCloseTime.Store(now) - case StateHalfOpen: - cb.Metrics.HalfOpenCount.Add(1) - cb.Metrics.LastHalfOpenTime.Store(now) - } - - // 设置下一次重试时间(如果是打开状态) - if state == StateOpen { - cb.Metrics.NextRetryTime.Store(time.Now().Add(cb.Config.TimeoutParsed).Unix()) - } + cb.updateStateMetrics(state) } return oldStateEnum } +// init 初始化熔断器信息 +func (cb *CircuitBreakerInfo) init() { + cb.State.Store(stateClosed) + cb.Metrics.LastResetTime.Store(time.Now().Unix()) + cb.Metrics.LastCloseTime.Store(time.Now().Unix()) + cb.Metrics.WindowStartTime.Store(time.Now().Unix()) +} + +// stateToInt64 将CircuitBreakerState转换为int64状态 +func (cb *CircuitBreakerInfo) stateToInt64(state CircuitBreakerState) int64 { + switch state { + case StateOpen: + return stateOpen + case StateHalfOpen: + return stateHalfOpen + default: + return stateClosed + } +} + +// int64ToState 将int64状态转换为CircuitBreakerState +func (cb *CircuitBreakerInfo) int64ToState(state int64) CircuitBreakerState { + switch state { + case stateOpen: + return StateOpen + case stateHalfOpen: + return StateHalfOpen + default: + return StateClosed + } +} + +// updateStateMetrics 更新状态相关的指标 +func (cb *CircuitBreakerInfo) updateStateMetrics(state CircuitBreakerState) { + now := time.Now().Unix() + + // 根据新状态更新计数器 + switch state { + case StateOpen: + cb.Metrics.OpenCount.Add(1) + cb.Metrics.LastOpenTime.Store(now) + // 设置下一次重试时间 + cb.Metrics.NextRetryTime.Store(time.Now().Add(cb.Config.TimeoutParsed).Unix()) + case StateClosed: + cb.Metrics.ClosedCount.Add(1) + cb.Metrics.LastCloseTime.Store(now) + case StateHalfOpen: + cb.Metrics.HalfOpenCount.Add(1) + cb.Metrics.LastHalfOpenTime.Store(now) + } +} + // InitCircuitBreaker 初始化Sentinel熔断器 func InitCircuitBreaker() error { ctx := context.Background() @@ -301,29 +318,9 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { } // 解析时间 - 使用默认值处理解析错误 - timeoutParsed, err := time.ParseDuration(config.Timeout) - if err != nil { - g.Log().Warningf(ctx, "解析timeout失败: %s, 使用默认值 %s, error: %v", config.Timeout, defaultTimeout, err) - timeoutParsed, _ = time.ParseDuration(defaultTimeout) - config.Timeout = defaultTimeout - } - config.TimeoutParsed = timeoutParsed - - slowThresholdParsed, err := time.ParseDuration(config.SlowRequestThreshold) - if err != nil { - g.Log().Warningf(ctx, "解析slowRequestThreshold失败: %s, 使用默认值 %s, error: %v", config.SlowRequestThreshold, defaultSlowRequestThreshold, err) - slowThresholdParsed, _ = time.ParseDuration(defaultSlowRequestThreshold) - config.SlowRequestThreshold = defaultSlowRequestThreshold - } - config.SlowRequestThresholdParsed = slowThresholdParsed - - warmupParsed, err := time.ParseDuration(config.WarmupDuration) - if err != nil { - g.Log().Warningf(ctx, "解析warmupDuration失败: %s, 使用默认值 %s, error: %v", config.WarmupDuration, defaultWarmupDuration, err) - warmupParsed, _ = time.ParseDuration(defaultWarmupDuration) - config.WarmupDuration = defaultWarmupDuration - } - config.WarmupDurationParsed = warmupParsed + config.TimeoutParsed, config.Timeout = parseDurationWithDefault(ctx, config.Timeout, defaultTimeout, "timeout") + config.SlowRequestThresholdParsed, config.SlowRequestThreshold = parseDurationWithDefault(ctx, config.SlowRequestThreshold, defaultSlowRequestThreshold, "slowRequestThreshold") + config.WarmupDurationParsed, config.WarmupDuration = parseDurationWithDefault(ctx, config.WarmupDuration, defaultWarmupDuration, "warmupDuration") // 解析状态码 successCodes := g.Cfg().MustGet(ctx, key+".successStatusCodes", "200,201,204").String() @@ -364,6 +361,76 @@ func parseStrings(str string) []string { return result } +// parseDurationWithDefault 解析持续时间,失败时使用默认值 +func parseDurationWithDefault(ctx context.Context, durationStr, defaultStr, fieldName string) (time.Duration, string) { + durationParsed, err := time.ParseDuration(durationStr) + if err != nil { + g.Log().Warningf(ctx, "解析%s失败: %s, 使用默认值 %s, error: %v", fieldName, durationStr, defaultStr, err) + durationParsed, _ = time.ParseDuration(defaultStr) + return durationParsed, defaultStr + } + return durationParsed, durationStr +} + +// atomicUpdateMin 原子更新最小值 +func atomicUpdateMin(minValue *atomic.Int64, newValue int64) { + for { + currentMin := minValue.Load() + if newValue >= currentMin { + break + } + if minValue.CompareAndSwap(currentMin, newValue) { + break + } + } +} + +// atomicUpdateMax 原子更新最大值 +func atomicUpdateMax(maxValue *atomic.Int64, newValue int64) { + for { + currentMax := maxValue.Load() + if newValue <= currentMax { + break + } + if maxValue.CompareAndSwap(currentMax, newValue) { + break + } + } +} + +// getAllowedIPs 获取允许的IP列表(带锁保护) +func getAllowedIPs() map[string]bool { + allowedAdminIPsMutex.RLock() + defer allowedAdminIPsMutex.RUnlock() + return allowedAdminIPsMap +} + +// getAllowedCIDRs 获取允许的CIDR列表(带锁保护) +func getAllowedCIDRs() []*net.IPNet { + allowedAdminCIDRsMutex.RLock() + defer allowedAdminCIDRsMutex.RUnlock() + return allowedAdminCIDRs +} + +// reset 重置所有指标到初始状态 +func (m *CircuitBreakerMetrics) reset() { + m.TotalRequests.Store(0) + m.PassRequests.Store(0) + m.BlockRequests.Store(0) + m.FailureRequests.Store(0) + m.SlowRequests.Store(0) + m.OpenCount.Store(0) + m.HalfOpenRequests.Store(0) + m.HalfOpenPassed.Store(0) + m.HalfOpenFailed.Store(0) + m.TotalResponseTime.Store(0) + m.MinResponseTime.Store(1<<63 - 1) // 最大int64值作为初始最小值 + m.MaxResponseTime.Store(0) + m.WindowRequests.Store(0) + m.WindowFailures.Store(0) + // 时间戳相关字段不重置,LastResetTime在调用时单独设置 +} + // parseCIDRs 解析CIDR列表 func parseCIDRs(strs []string) ([]*net.IPNet, error) { nets := make([]*net.IPNet, 0, len(strs)) @@ -386,11 +453,8 @@ func parseCIDRs(strs []string) ([]*net.IPNet, error) { // newCircuitBreakerMetrics 创建并初始化熔断器指标 func newCircuitBreakerMetrics() *CircuitBreakerMetrics { - metrics := &CircuitBreakerMetrics{ - MinResponseTime: atomic.Int64{}, - MaxResponseTime: atomic.Int64{}, - } - metrics.MinResponseTime.Store(1<<63 - 1) // 最大int64值作为初始最小值 + metrics := &CircuitBreakerMetrics{} + metrics.reset() return metrics } @@ -530,13 +594,7 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) AdaptiveThreshold: threshold, WarmupEndTime: time.Now().Add(config.WarmupDurationParsed).Unix(), } - cbInfo.State.Store(stateClosed) - cbInfo.Metrics.LastResetTime.Store(time.Now().Unix()) - cbInfo.Metrics.LastCloseTime.Store(time.Now().Unix()) - cbInfo.Metrics.WindowStartTime.Store(time.Now().Unix()) - - // 初始化响应时间统计 - cbInfo.Metrics.MinResponseTime.Store(1<<63 - 1) // 最大int64值作为初始最小值 + cbInfo.init() circuitBreakers.Store(serviceName, cbInfo) strategy := "error_count" @@ -663,27 +721,9 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { durationNs := duration.Nanoseconds() cbInfo.Metrics.TotalResponseTime.Add(durationNs) - // 原子更新最小响应时间 - for { - currentMin := cbInfo.Metrics.MinResponseTime.Load() - if durationNs >= currentMin { - break - } - if cbInfo.Metrics.MinResponseTime.CompareAndSwap(currentMin, durationNs) { - break - } - } - - // 原子更新最大响应时间 - for { - currentMax := cbInfo.Metrics.MaxResponseTime.Load() - if durationNs <= currentMax { - break - } - if cbInfo.Metrics.MaxResponseTime.CompareAndSwap(currentMax, durationNs) { - break - } - } + // 原子更新最小和最大响应时间 + atomicUpdateMin(&cbInfo.Metrics.MinResponseTime, durationNs) + atomicUpdateMax(&cbInfo.Metrics.MaxResponseTime, durationNs) if duration > config.SlowRequestThresholdParsed { cbInfo.Metrics.SlowRequests.Add(1) @@ -908,77 +948,32 @@ func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) // syncCircuitBreakerStateToDistributed 同步熔断器状态到Redis func syncCircuitBreakerStateToDistributed(ctx context.Context, resourceName, state string, ttl int) { - key := "circuit_breaker:" + resourceName + ":state" + stateKey := "circuit_breaker:" + resourceName + ":state" lockKey := "circuit_breaker:" + resourceName + ":lock" - redis := g.Redis() - if redis == nil { + redisClient := g.Redis() + if redisClient == nil { g.Log().Warningf(ctx, "Redis未初始化,无法同步分布式熔断状态: %s", resourceName) return } - lockValue := fmt.Sprintf("%d", time.Now().UnixNano()) + // 使用common/redis中的Lock方法获取分布式锁 + success, err := redis.Lock(ctx, lockKey, 10, func(ctx context.Context) error { + // 设置熔断器状态 + _, err := redisClient.Do(ctx, "SETEX", stateKey, ttl, state) + if err != nil { + g.Log().Errorf(ctx, "设置分布式熔断状态失败: %s=%s, error: %v", stateKey, state, err) + } else { + g.Log().Debugf(ctx, "分布式熔断状态已同步: %s=%s (TTL: %d)", stateKey, state, ttl) + } + return nil + }) - // 获取分布式锁 - locked, err := redis.Do(ctx, "SET", lockKey, lockValue, "NX", "EX", 10) if err != nil { g.Log().Errorf(ctx, "获取分布式锁失败: %s, error: %v", lockKey, err) return } - // 检查是否获取到锁 - var isLocked bool - if locked != nil && !locked.IsNil() { - isLocked = true - } else { - // 检查锁是否已过期 - currentLock, err := redis.Get(ctx, lockKey) - if err == nil && !currentLock.IsNil() { - lockTime, _ := strconv.ParseInt(currentLock.String(), 10, 64) - // 如果锁已经存在超过10秒(超时),强制获取 - if time.Now().UnixNano()-lockTime > 10*1e9 { - // 使用SETNX方式获取锁,使用Lua脚本保证原子性 - luaAcquire := ` -local current = redis.call("get", KEYS[1]) -if current and tonumber(current) then - local lockTime = tonumber(current) - if redis.call("TIME")[1] * 1000000000 + redis.call("TIME")[2] - lockTime > 10000000000 then - redis.call("del", KEYS[1]) - return redis.call("set", KEYS[1], ARGV[1], "EX", 10) - end - return nil -else - return redis.call("set", KEYS[1], ARGV[1], "EX", 10) -end` - locked, err = redis.Do(ctx, "EVAL", luaAcquire, 1, lockKey, lockValue) - if err != nil { - g.Log().Errorf(ctx, "强制获取分布式锁失败: %s, error: %v", lockKey, err) - return - } - if locked != nil && !locked.IsNil() { - isLocked = true - } - } - } - } - - if isLocked { - defer func() { - // 使用Lua脚本原子性地删除锁,只删除自己创建的锁 - luaScript := `if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end` - _, err := redis.Do(ctx, "EVAL", luaScript, 1, lockKey, lockValue) - if err != nil { - g.Log().Warningf(ctx, "释放分布式锁失败: %s, error: %v", lockKey, err) - } - }() - - // 设置状态 - _, err := redis.Do(ctx, "SETEX", key, ttl, state) - if err != nil { - g.Log().Errorf(ctx, "设置分布式熔断状态失败: %s=%s, error: %v", key, state, err) - } else { - g.Log().Debugf(ctx, "分布式熔断状态已同步: %s=%s (TTL: %d)", key, state, ttl) - } - } else { + if !success { g.Log().Debugf(ctx, "未获取到分布式锁,跳过状态同步: %s", lockKey) } } @@ -1089,34 +1084,25 @@ func isAdminIP(r *ghttp.Request) bool { return false } - allowedAdminIPsMutex.RLock() - allowedIPs := allowedAdminIPsMap - allowedAdminIPsMutex.RUnlock() + allowedIPs := getAllowedIPs() + allowedCIDRs := getAllowedCIDRs() - if len(allowedIPs) == 0 { - allowedAdminCIDRsMutex.RLock() - hasCIDRs := len(allowedAdminCIDRs) > 0 - allowedAdminCIDRsMutex.RUnlock() - if !hasCIDRs { - return true - } + // 如果没有任何限制,允许访问 + if len(allowedIPs) == 0 && len(allowedCIDRs) == 0 { + return true } + // 检查IP白名单 if allowedIPs[clientIP] { return true } - allowedAdminCIDRsMutex.RLock() - cidrNets := allowedAdminCIDRs - allowedAdminCIDRsMutex.RUnlock() - - if len(cidrNets) > 0 { - clientNetIP := net.ParseIP(clientIP) - if clientNetIP != nil { - for _, cidrNet := range cidrNets { - if cidrNet.Contains(clientNetIP) { - return true - } + // 检查CIDR白名单 + clientNetIP := net.ParseIP(clientIP) + if clientNetIP != nil { + for _, cidrNet := range allowedCIDRs { + if cidrNet.Contains(clientNetIP) { + return true } } } @@ -1125,6 +1111,29 @@ func isAdminIP(r *ghttp.Request) bool { return false } +// batchProcessServices 批量处理服务 +func batchProcessServices(r *ghttp.Request, processFunc func(serviceName string) error) (int, int, map[string]string) { + successCount := 0 + failCount := 0 + failures := make(map[string]string) + + serviceNamesMutex.RLock() + slice := serviceNamesSlice + serviceNamesMutex.RUnlock() + + for _, serviceName := range slice { + if err := processFunc(serviceName); err != nil { + g.Log().Errorf(r.GetCtx(), "服务 %s 处理失败: %v", serviceName, err) + failCount++ + failures[serviceName] = err.Error() + } else { + successCount++ + } + } + + return successCount, failCount, failures +} + // CircuitBreakerResetHandler 重置熔断器 func CircuitBreakerResetHandler(r *ghttp.Request) { serviceName := r.Get("service").String() @@ -1135,25 +1144,13 @@ func CircuitBreakerResetHandler(r *ghttp.Request) { } if serviceName == "" || serviceName == "*" { - serviceNamesMutex.RLock() - slice := serviceNamesSlice - serviceNamesMutex.RUnlock() - - successCount := 0 - failCount := 0 - - for _, name := range slice { - if err := resetSingleService(r, name); err != nil { - g.Log().Errorf(r.GetCtx(), "服务 %s 熔断器重置失败: %v", name, err) - failCount++ - } else { - successCount++ - } - } + successCount, failCount, failures := batchProcessServices(r, func(name string) error { + return resetSingleService(r, name) + }) g.Log().Infof(r.GetCtx(), "批量重置熔断器完成: 成功 %d, 失败 %d", successCount, failCount) r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: fmt.Sprintf("批量重置完成: 成功 %d, 失败 %d", successCount, failCount), - Data: map[string]interface{}{"success": successCount, "failed": failCount}}) + Data: map[string]interface{}{"success": successCount, "failed": failCount, "failures": failures}}) return } @@ -1184,16 +1181,8 @@ func resetSingleService(r *ghttp.Request, serviceName string) error { if cbInfoVal, ok := circuitBreakers.Load(serviceName); ok { cbInfo := cbInfoVal.(*CircuitBreakerInfo) cbInfo.State.Store(stateClosed) - cbInfo.Metrics.LastOpenTime.Store(0) - cbInfo.Metrics.NextRetryTime.Store(0) - cbInfo.Metrics.TotalRequests.Store(0) - cbInfo.Metrics.PassRequests.Store(0) - cbInfo.Metrics.BlockRequests.Store(0) - cbInfo.Metrics.FailureRequests.Store(0) - cbInfo.Metrics.SlowRequests.Store(0) - cbInfo.Metrics.OpenCount.Store(0) - cbInfo.Metrics.HalfOpenRequests.Store(0) - cbInfo.Metrics.HalfOpenPassed.Store(0) + // 重置指标 + cbInfo.Metrics.reset() cbInfo.WarmupEndTime = time.Now().Add(cbInfo.Config.WarmupDurationParsed).Unix() cbInfo.Metrics.LastResetTime.Store(time.Now().Unix()) } @@ -1224,23 +1213,9 @@ func CircuitBreakerReloadHandler(r *ghttp.Request) { } if serviceName == "" || serviceName == "*" { - serviceNamesMutex.RLock() - slice := serviceNamesSlice - serviceNamesMutex.RUnlock() - - successCount := 0 - failCount := 0 - failures := make(map[string]string) - - for _, service := range slice { - if err := ReloadCircuitBreakerConfig(service); err != nil { - g.Log().Errorf(r.GetCtx(), "服务 %s 配置重载失败: %v", service, err) - failCount++ - failures[service] = err.Error() - } else { - successCount++ - } - } + successCount, failCount, failures := batchProcessServices(r, func(serviceName string) error { + return ReloadCircuitBreakerConfig(serviceName) + }) updateAdminIPsCache() r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: fmt.Sprintf("配置重载完成: 成功 %d, 失败 %d", successCount, failCount), From e03a4bfcff1ada8d07c01567bf152f647ab5ed4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 13:54:34 +0800 Subject: [PATCH 132/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 157 ++++++++++++++++++++-------------- 1 file changed, 93 insertions(+), 64 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 8c615a0..3520277 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -221,6 +221,43 @@ func (cb *CircuitBreakerInfo) updateStateMetrics(state CircuitBreakerState) { } } +// getCircuitBreakerInfoAndConfig 获取熔断器信息和配置 +func getCircuitBreakerInfoAndConfig(serviceName string) (*CircuitBreakerInfo, *CircuitBreakerConfig) { + cbInfoVal, ok := circuitBreakers.Load(serviceName) + if !ok { + return nil, nil + } + + cbInfo, ok := cbInfoVal.(*CircuitBreakerInfo) + if !ok { + return nil, nil + } + + return cbInfo, cbInfo.Config +} + +// updateResponseTimeStats 更新响应时间统计 +func updateResponseTimeStats(cbInfo *CircuitBreakerInfo, duration time.Duration, config *CircuitBreakerConfig) { + durationNs := duration.Nanoseconds() + cbInfo.Metrics.TotalResponseTime.Add(durationNs) + + // 原子更新最小和最大响应时间 + atomicUpdateMin(&cbInfo.Metrics.MinResponseTime, durationNs) + atomicUpdateMax(&cbInfo.Metrics.MaxResponseTime, durationNs) + + if duration > config.SlowRequestThresholdParsed { + cbInfo.Metrics.SlowRequests.Add(1) + } +} + +// formatUnixTime 格式化Unix时间戳 +func formatUnixTime(timestamp int64) string { + if timestamp > 0 { + return time.Unix(timestamp, 0).Format("2006-01-02 15:04:05") + } + return "" +} + // InitCircuitBreaker 初始化Sentinel熔断器 func InitCircuitBreaker() error { ctx := context.Background() @@ -398,14 +435,24 @@ func atomicUpdateMax(maxValue *atomic.Int64, newValue int64) { } } -// getAllowedIPs 获取允许的IP列表(带锁保护) +// getAllowedIPsAndCIDRs 获取允许的IP和CIDR列表(带锁保护) +func getAllowedIPsAndCIDRs() (map[string]bool, []*net.IPNet) { + allowedAdminIPsMutex.RLock() + allowedAdminCIDRsMutex.RLock() + defer allowedAdminIPsMutex.RUnlock() + defer allowedAdminCIDRsMutex.RUnlock() + + return allowedAdminIPsMap, allowedAdminCIDRs +} + +// getAllowedIPs 获取允许的IP列表(带锁保护,兼容旧代码) func getAllowedIPs() map[string]bool { allowedAdminIPsMutex.RLock() defer allowedAdminIPsMutex.RUnlock() return allowedAdminIPsMap } -// getAllowedCIDRs 获取允许的CIDR列表(带锁保护) +// getAllowedCIDRs 获取允许的CIDR列表(带锁保护,兼容旧代码) func getAllowedCIDRs() []*net.IPNet { allowedAdminCIDRsMutex.RLock() defer allowedAdminCIDRsMutex.RUnlock() @@ -495,41 +542,57 @@ func (cb *CircuitBreakerInfo) updateWindowStats(isSuccess bool, ctx context.Cont } } +// validateInRange 验证值是否在指定范围内 +func validateInRange(name string, value, min, max int) error { + if value < min || value > max { + return fmt.Errorf("%s必须在%d-%d之间", name, min, max) + } + return nil +} + +// validateFloatInRange 验证浮点数值是否在指定范围内 +func validateFloatInRange(name string, value, min, max float64) error { + if value < min || value > max { + return fmt.Errorf("%s必须在%.1f-%.1f之间", name, min, max) + } + return nil +} + // validateCircuitBreakerConfig 验证配置 func validateCircuitBreakerConfig(config *CircuitBreakerConfig) error { if config.MaxFailures <= 0 { return fmt.Errorf("maxFailures必须大于0") } - if config.FailureRateThreshold < 0 || config.FailureRateThreshold > 1 { - return fmt.Errorf("failureRateThreshold必须在0.0-1.0之间") + if err := validateFloatInRange("failureRateThreshold", config.FailureRateThreshold, 0.0, 1.0); err != nil { + return err } if len(config.SuccessStatusCodes) == 0 { return fmt.Errorf("successStatusCodes不能为空") } - if config.RequestTimeout < 0 || config.RequestTimeout > 300000 { - return fmt.Errorf("requestTimeout必须在0-300000毫秒之间") + if err := validateInRange("requestTimeout", config.RequestTimeout, 0, 300000); err != nil { + return err } - if config.DistributedTTL < 0 || config.DistributedTTL > 3600 { - return fmt.Errorf("distributedTTL必须在0-3600秒之间") + if err := validateInRange("distributedTTL", config.DistributedTTL, 0, 3600); err != nil { + return err } - if config.StatIntervalMs < 100 || config.StatIntervalMs > 60000 { - return fmt.Errorf("statIntervalMs必须在100-60000毫秒之间") + if err := validateInRange("statIntervalMs", config.StatIntervalMs, 100, 60000); err != nil { + return err } - if config.MinRequestAmount < 1 || config.MinRequestAmount > 10000 { - return fmt.Errorf("minRequestAmount必须在1-10000之间") + if err := validateInRange("minRequestAmount", config.MinRequestAmount, 1, 10000); err != nil { + return err } - if config.HalfOpenMaxRequests < 1 || config.HalfOpenMaxRequests > 100 { - return fmt.Errorf("halfOpenMaxRequests必须在1-100之间") + if err := validateInRange("halfOpenMaxRequests", config.HalfOpenMaxRequests, 1, 100); err != nil { + return err } - if config.HalfOpenSuccessThreshold < 0 || config.HalfOpenSuccessThreshold > 1 { - return fmt.Errorf("halfOpenSuccessThreshold必须在0.0-1.0之间") + if err := validateFloatInRange("halfOpenSuccessThreshold", config.HalfOpenSuccessThreshold, 0.0, 1.0); err != nil { + return err } if config.EnableAdaptiveThreshold { - if config.AdaptiveMinThreshold < 0 || config.AdaptiveMinThreshold > 1 { - return fmt.Errorf("adaptiveMinThreshold必须在0.0-1.0之间") + if err := validateFloatInRange("adaptiveMinThreshold", config.AdaptiveMinThreshold, 0.0, 1.0); err != nil { + return err } - if config.AdaptiveMaxThreshold < 0 || config.AdaptiveMaxThreshold > 1 { - return fmt.Errorf("adaptiveMaxThreshold必须在0.0-1.0之间") + if err := validateFloatInRange("adaptiveMaxThreshold", config.AdaptiveMaxThreshold, 0.0, 1.0); err != nil { + return err } if config.AdaptiveMinThreshold >= config.AdaptiveMaxThreshold { return fmt.Errorf("adaptiveMinThreshold必须小于adaptiveMaxThreshold") @@ -618,19 +681,8 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { return } - cbInfoVal, ok := circuitBreakers.Load(serviceName) - if !ok { - r.Middleware.Next() - return - } - - cbInfo, ok := cbInfoVal.(*CircuitBreakerInfo) - if !ok { - r.Middleware.Next() - return - } - config := cbInfo.Config - if !config.Enabled { + cbInfo, config := getCircuitBreakerInfoAndConfig(serviceName) + if cbInfo == nil || config == nil || !config.Enabled { r.Middleware.Next() return } @@ -718,16 +770,7 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { duration := time.Since(startTime) // 记录响应时间统计 - durationNs := duration.Nanoseconds() - cbInfo.Metrics.TotalResponseTime.Add(durationNs) - - // 原子更新最小和最大响应时间 - atomicUpdateMin(&cbInfo.Metrics.MinResponseTime, durationNs) - atomicUpdateMax(&cbInfo.Metrics.MaxResponseTime, durationNs) - - if duration > config.SlowRequestThresholdParsed { - cbInfo.Metrics.SlowRequests.Add(1) - } + updateResponseTimeStats(cbInfo, duration, config) isSuccess := isSuccessStatusCode(cbInfo, statusCode) @@ -1036,23 +1079,10 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { halfOpenServices++ } - lastResetTime := cbInfo.Metrics.LastResetTime.Load() - var lastResetTimeStr string - if lastResetTime > 0 { - lastResetTimeStr = time.Unix(lastResetTime, 0).Format("2006-01-02 15:04:05") - } - - lastOpenTime := cbInfo.Metrics.LastOpenTime.Load() - var lastOpenTimeStr string - if lastOpenTime > 0 { - lastOpenTimeStr = time.Unix(lastOpenTime, 0).Format("2006-01-02 15:04:05") - } - - nextRetryTime := cbInfo.Metrics.NextRetryTime.Load() - var nextRetryTimeStr string - if nextRetryTime > 0 { - nextRetryTimeStr = time.Unix(nextRetryTime, 0).Format("2006-01-02 15:04:05") - } + // 格式化时间字符串 + lastResetTimeStr := formatUnixTime(cbInfo.Metrics.LastResetTime.Load()) + lastOpenTimeStr := formatUnixTime(cbInfo.Metrics.LastOpenTime.Load()) + nextRetryTimeStr := formatUnixTime(cbInfo.Metrics.NextRetryTime.Load()) status[serviceName] = map[string]interface{}{ "resource": cbInfo.ResourceName, @@ -1084,8 +1114,8 @@ func isAdminIP(r *ghttp.Request) bool { return false } - allowedIPs := getAllowedIPs() - allowedCIDRs := getAllowedCIDRs() + // 一次性获取IP和CIDR列表,减少锁操作 + allowedIPs, allowedCIDRs := getAllowedIPsAndCIDRs() // 如果没有任何限制,允许访问 if len(allowedIPs) == 0 && len(allowedCIDRs) == 0 { @@ -1098,8 +1128,7 @@ func isAdminIP(r *ghttp.Request) bool { } // 检查CIDR白名单 - clientNetIP := net.ParseIP(clientIP) - if clientNetIP != nil { + if clientNetIP := net.ParseIP(clientIP); clientNetIP != nil { for _, cidrNet := range allowedCIDRs { if cidrNet.Contains(clientNetIP) { return true From 63191c147df882a1decead84edc5b7f588874333 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 14:07:14 +0800 Subject: [PATCH 133/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 130 ++++++++++++++-------------------- 1 file changed, 53 insertions(+), 77 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 3520277..c39b452 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -252,10 +252,10 @@ func updateResponseTimeStats(cbInfo *CircuitBreakerInfo, duration time.Duration, // formatUnixTime 格式化Unix时间戳 func formatUnixTime(timestamp int64) string { - if timestamp > 0 { - return time.Unix(timestamp, 0).Format("2006-01-02 15:04:05") + if timestamp <= 0 { + return "" } - return "" + return time.Unix(timestamp, 0).Format("2006-01-02 15:04:05") } // InitCircuitBreaker 初始化Sentinel熔断器 @@ -445,20 +445,6 @@ func getAllowedIPsAndCIDRs() (map[string]bool, []*net.IPNet) { return allowedAdminIPsMap, allowedAdminCIDRs } -// getAllowedIPs 获取允许的IP列表(带锁保护,兼容旧代码) -func getAllowedIPs() map[string]bool { - allowedAdminIPsMutex.RLock() - defer allowedAdminIPsMutex.RUnlock() - return allowedAdminIPsMap -} - -// getAllowedCIDRs 获取允许的CIDR列表(带锁保护,兼容旧代码) -func getAllowedCIDRs() []*net.IPNet { - allowedAdminCIDRsMutex.RLock() - defer allowedAdminCIDRsMutex.RUnlock() - return allowedAdminCIDRs -} - // reset 重置所有指标到初始状态 func (m *CircuitBreakerMetrics) reset() { m.TotalRequests.Store(0) @@ -533,9 +519,9 @@ func (cb *CircuitBreakerInfo) updateWindowStats(isSuccess bool, ctx context.Cont // 计算当前窗口内的成功率 total := cb.Metrics.WindowRequests.Load() failures := cb.Metrics.WindowFailures.Load() - if total > 0 { + if total >= 10 { // 有足够样本时才记录 successRate := float64(total-failures) / float64(total) - if successRate < 0.5 && total >= 10 { // 如果成功率低于50%且有足够样本 + if successRate < 0.5 { // 如果成功率低于50% g.Log().Warningf(ctx, "熔断器 %s 窗口内成功率较低: %.2f%%, total=%d, failures=%d", cb.ResourceName, successRate*100, total, failures) } @@ -614,28 +600,25 @@ func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) } var rule []*circuitbreaker.Rule - if config.EnableSlidingWindow { - rule = []*circuitbreaker.Rule{{ - Resource: resourceName, - Strategy: circuitbreaker.SlowRequestRatio, - RetryTimeoutMs: uint32(config.TimeoutParsed.Milliseconds()), - MinRequestAmount: uint64(config.MinRequestAmount), - StatIntervalMs: uint32(config.StatIntervalMs), - StatSlidingWindowBucketCount: 10, - MaxAllowedRtMs: uint64(config.SlowRequestThresholdParsed.Milliseconds()), - Threshold: threshold, - }} - } else { - rule = []*circuitbreaker.Rule{{ - Resource: resourceName, - Strategy: circuitbreaker.ErrorCount, - RetryTimeoutMs: uint32(config.TimeoutParsed.Milliseconds()), - MinRequestAmount: uint64(config.MinRequestAmount), - StatIntervalMs: uint32(config.StatIntervalMs), - Threshold: float64(config.MaxFailures), - }} + baseRule := &circuitbreaker.Rule{ + Resource: resourceName, + RetryTimeoutMs: uint32(config.TimeoutParsed.Milliseconds()), + MinRequestAmount: uint64(config.MinRequestAmount), + StatIntervalMs: uint32(config.StatIntervalMs), } + if config.EnableSlidingWindow { + baseRule.Strategy = circuitbreaker.SlowRequestRatio + baseRule.StatSlidingWindowBucketCount = 10 + baseRule.MaxAllowedRtMs = uint64(config.SlowRequestThresholdParsed.Milliseconds()) + baseRule.Threshold = threshold + } else { + baseRule.Strategy = circuitbreaker.ErrorCount + baseRule.Threshold = float64(config.MaxFailures) + } + + rule = []*circuitbreaker.Rule{baseRule} + if _, err := circuitbreaker.LoadRulesOfResource(resourceName, []*circuitbreaker.Rule{}); err != nil { return fmt.Errorf("清空熔断规则失败: %v", err) } @@ -849,14 +832,14 @@ func sendFallbackResponse(r *ghttp.Request, serviceName string, config *CircuitB return } + msg := fmt.Sprintf("服务 '%s' 暂时不可用,请稍后再试", serviceName) switch reason { case "blocked": - r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 熔断保护中,请稍后再试", serviceName)) + msg = fmt.Sprintf("服务 '%s' 熔断保护中,请稍后再试", serviceName) case "distributed": - r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 分布式熔断中", serviceName)) - default: - r.Response.WriteStatusExit(503, fmt.Sprintf("服务 '%s' 暂时不可用,请稍后再试", serviceName)) + msg = fmt.Sprintf("服务 '%s' 分布式熔断中", serviceName) } + r.Response.WriteStatusExit(503, msg) } // isSuccessStatusCode 判断HTTP状态码是否成功 @@ -877,16 +860,16 @@ func extractServiceName(path string) string { if path == "" { return "" } - parts := strings.Split(path, "/") - if len(parts) == 0 { - return "" - } - serviceName := parts[0] + // 获取第一个路径段 + if idx := strings.Index(path, "/"); idx > 0 { + path = path[:idx] + } + + // 解码URL编码(简化版) + serviceName := path if strings.Contains(serviceName, "%") { - if decoded, err := pathUnescape(serviceName); err == nil { - serviceName = decoded - } + serviceName = urlDecode(serviceName) } if _, ok := circuitBreakerConfigs.Load(serviceName); ok { @@ -895,31 +878,24 @@ func extractServiceName(path string) string { return "" } -// pathUnescape 路径片段的URL解码 -func pathUnescape(s string) (string, error) { - var builder strings.Builder - builder.Grow(len(s)) +// urlDecode 简单的URL解码 +func urlDecode(s string) string { + result := make([]byte, 0, len(s)) for i := 0; i < len(s); i++ { - switch s[i] { - case '%': - if i+2 >= len(s) { - builder.WriteByte(s[i]) - continue + if s[i] == '%' && i+2 < len(s) { + if high := hexDigit(s[i+1]); high != 0xFF { + if low := hexDigit(s[i+2]); low != 0xFF { + result = append(result, (high<<4)|low) + i += 2 + continue + } } - high := hexDigit(s[i+1]) - low := hexDigit(s[i+2]) - if high == 0xFF || low == 0xFF { - builder.WriteByte(s[i]) - } else { - builder.WriteByte((high << 4) | low) - i += 2 - } - default: - builder.WriteByte(s[i]) } + result = append(result, s[i]) } - return builder.String(), nil + + return string(result) } func hexDigit(c byte) byte { @@ -978,11 +954,11 @@ func filterServiceNames(services map[string]interface{}) []string { // isCircuitBreakerOpenInDistributed 检查分布式熔断状态 func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) bool { key := "circuit_breaker:" + resourceName + ":state" - redis := g.Redis() - if redis == nil { + redisClient := g.Redis() + if redisClient == nil { return false } - value, err := redis.Get(ctx, key) + value, err := redisClient.Get(ctx, key) if err != nil || value.IsNil() { return false } @@ -1219,9 +1195,9 @@ func resetSingleService(r *ghttp.Request, serviceName string) error { if configVal, ok := circuitBreakerConfigs.Load(serviceName); ok { config, ok := configVal.(*CircuitBreakerConfig) if ok && config.DistributedTTL > 0 { - redis := g.Redis() - if redis != nil { - if _, err := redis.Del(r.GetCtx(), "circuit_breaker:"+resourceName+":state"); err != nil { + redisClient := g.Redis() + if redisClient != nil { + if _, err := redisClient.Del(r.GetCtx(), "circuit_breaker:"+resourceName+":state"); err != nil { g.Log().Warningf(r.GetCtx(), "清除分布式熔断状态失败: %s, error: %v", resourceName, err) } } From 8ac1630514d64d497cdb71c8d9965d1fdf66c021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 1 Jan 2026 15:13:36 +0800 Subject: [PATCH 134/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=86=94=E6=96=AD?= =?UTF-8?q?=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 183 ++++++++++++++++++++++++++++------ 1 file changed, 151 insertions(+), 32 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index c39b452..71b3c2c 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -99,6 +99,126 @@ type CircuitBreakerMetrics struct { WindowFailures atomic.Int64 // 窗口内失败数 } +// 实现HalfOpenMetrics接口 +func (m *CircuitBreakerMetrics) GetHalfOpenRequests() *atomic.Int64 { + return &m.HalfOpenRequests +} + +func (m *CircuitBreakerMetrics) GetHalfOpenPassed() *atomic.Int64 { + return &m.HalfOpenPassed +} + +func (m *CircuitBreakerMetrics) GetHalfOpenFailed() *atomic.Int64 { + return &m.HalfOpenFailed +} + +func (m *CircuitBreakerMetrics) AddHalfOpenRequests(delta int64) { + m.HalfOpenRequests.Add(delta) +} + +func (m *CircuitBreakerMetrics) AddHalfOpenPassed(delta int64) { + m.HalfOpenPassed.Add(delta) +} + +func (m *CircuitBreakerMetrics) AddHalfOpenFailed(delta int64) { + m.HalfOpenFailed.Add(delta) +} + +// 半开状态指标接口,定义半开状态管理需要的原子操作 +type HalfOpenMetrics interface { + GetHalfOpenRequests() *atomic.Int64 + GetHalfOpenPassed() *atomic.Int64 + GetHalfOpenFailed() *atomic.Int64 + AddHalfOpenRequests(delta int64) + AddHalfOpenPassed(delta int64) + AddHalfOpenFailed(delta int64) +} + +// HalfOpenManager 半开状态管理器 +// 解决多个goroutine同时操作半开状态时可能出现的竞态条件和数据不一致问题 +type HalfOpenManager struct { + mu sync.RWMutex +} + +// NewHalfOpenManager 创建半开状态管理器实例 +func NewHalfOpenManager() *HalfOpenManager { + return &HalfOpenManager{} +} + +var ( + halfOpenManagerInstance *HalfOpenManager + halfOpenManagerOnce sync.Once +) + +// GetHalfOpenManager 获取半开状态管理器单例 +func GetHalfOpenManager() *HalfOpenManager { + halfOpenManagerOnce.Do(func() { + halfOpenManagerInstance = NewHalfOpenManager() + }) + return halfOpenManagerInstance +} + +// TryAcquireHalfOpenSlot 尝试获取半开状态的请求槽位 +func (m *HalfOpenManager) TryAcquireHalfOpenSlot(metrics HalfOpenMetrics, maxRequests int) (bool, int) { + if maxRequests <= 0 { + return false, 0 + } + + m.mu.Lock() + defer m.mu.Unlock() + + currentRequests := int(metrics.GetHalfOpenRequests().Load()) + if currentRequests >= maxRequests { + return false, currentRequests + } + + metrics.AddHalfOpenRequests(1) + metrics.AddHalfOpenPassed(1) + return true, currentRequests + 1 +} + +// RecordHalfOpenResult 记录半开状态请求结果,并检查是否达到成功阈值 +func (m *HalfOpenManager) RecordHalfOpenResult(metrics HalfOpenMetrics, isSuccess bool, successThreshold float64) bool { + if successThreshold < 0 || successThreshold > 1 { + successThreshold = 0.5 + } + + m.mu.Lock() + defer m.mu.Unlock() + + metrics.AddHalfOpenRequests(-1) + if isSuccess { + metrics.AddHalfOpenPassed(1) + } else { + metrics.AddHalfOpenFailed(1) + } + + return m.checkHalfOpenSuccessThreshold(metrics, successThreshold) +} + +// checkHalfOpenSuccessThreshold 检查半开状态的成功率是否达到阈值 +func (m *HalfOpenManager) checkHalfOpenSuccessThreshold(metrics HalfOpenMetrics, successThreshold float64) bool { + totalRequests := metrics.GetHalfOpenPassed().Load() + metrics.GetHalfOpenFailed().Load() + passedRequests := metrics.GetHalfOpenPassed().Load() + + if totalRequests == 0 { + return false + } + + successRate := float64(passedRequests) / float64(totalRequests) + return successRate >= successThreshold +} + +// ResetHalfOpenStats 重置半开状态统计 +func (m *HalfOpenManager) ResetHalfOpenStats(metrics HalfOpenMetrics) { + m.mu.Lock() + defer m.mu.Unlock() + + metrics.GetHalfOpenRequests().Store(0) + metrics.GetHalfOpenPassed().Store(0) + metrics.GetHalfOpenFailed().Store(0) +} + // CircuitBreakerInfo 熔断器信息 type CircuitBreakerInfo struct { ResourceName string @@ -693,16 +813,13 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { return } - // 半开状态处理 - 使用原子操作确保线程安全 + // 半开状态处理 - 使用HalfOpenManager确保线程安全 currentState := cbInfo.getState() if currentState == StateHalfOpen { - // 使用原子操作安全地递增半开请求计数 - halfOpenRequests := cbInfo.Metrics.HalfOpenRequests.Add(1) + manager := GetHalfOpenManager() + acquired, _ := manager.TryAcquireHalfOpenSlot(cbInfo.Metrics, config.HalfOpenMaxRequests) - // 如果超过最大半开请求数量,回滚并触发熔断 - if halfOpenRequests > int64(config.HalfOpenMaxRequests) { - // 原子递减回滚 - cbInfo.Metrics.HalfOpenRequests.Add(-1) + if !acquired { cbInfo.Metrics.BlockRequests.Add(1) // 尝试转换为打开状态,如果成功则记录日志 @@ -785,31 +902,20 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { // 重新获取当前状态 currentState := cbInfo.getState() if currentState == StateHalfOpen { - // 原子递增成功计数 - halfOpenPassed := cbInfo.Metrics.HalfOpenPassed.Add(1) - totalRequests := cbInfo.Metrics.HalfOpenRequests.Load() + manager := GetHalfOpenManager() + // 使用HalfOpenManager记录结果并检查是否达到阈值 + if manager.RecordHalfOpenResult(cbInfo.Metrics, true, config.HalfOpenSuccessThreshold) { + // 达到成功阈值,关闭熔断器 + oldState := cbInfo.setStateWithMetrics(StateClosed, true) + if oldState == StateHalfOpen { + // 重置半开统计 + manager.ResetHalfOpenStats(cbInfo.Metrics) - // 计算成功率,确保分母不为零 - if totalRequests > 0 { - successRate := float64(halfOpenPassed) / float64(totalRequests) + g.Log().Infof(ctx, "半开状态成功,恢复关闭: %s", resourceName) - // 检查是否达到成功率阈值,如果达到则关闭熔断器 - if successRate >= config.HalfOpenSuccessThreshold { - // 原子设置状态为关闭,确保只有一个goroutine能成功转换 - oldState := cbInfo.setStateWithMetrics(StateClosed, true) - if oldState == StateHalfOpen { - // 重置半开统计 - cbInfo.Metrics.HalfOpenPassed.Store(0) - cbInfo.Metrics.HalfOpenRequests.Store(0) - cbInfo.Metrics.HalfOpenFailed.Store(0) - - g.Log().Infof(ctx, "半开状态成功,恢复关闭: %s, successRate=%.2f, total=%d, passed=%d", - resourceName, successRate, totalRequests, halfOpenPassed) - - // 同步分布式状态 - if config.DistributedTTL > 0 { - syncCircuitBreakerStateToDistributed(ctx, resourceName, "closed", config.DistributedTTL) - } + // 同步分布式状态 + if config.DistributedTTL > 0 { + syncCircuitBreakerStateToDistributed(ctx, resourceName, "closed", config.DistributedTTL) } } } @@ -1197,8 +1303,21 @@ func resetSingleService(r *ghttp.Request, serviceName string) error { if ok && config.DistributedTTL > 0 { redisClient := g.Redis() if redisClient != nil { - if _, err := redisClient.Del(r.GetCtx(), "circuit_breaker:"+resourceName+":state"); err != nil { - g.Log().Warningf(r.GetCtx(), "清除分布式熔断状态失败: %s, error: %v", resourceName, err) + // 使用common/redis中的Lock方法确保分布式一致性 + lockKey := "circuit_breaker:" + resourceName + ":lock" + success, err := redis.Lock(r.GetCtx(), lockKey, 10, func(ctx context.Context) error { + _, err := redisClient.Del(ctx, "circuit_breaker:"+resourceName+":state") + if err != nil { + g.Log().Warningf(ctx, "清除分布式熔断状态失败: %s, error: %v", resourceName, err) + } + return nil + }) + + if err != nil { + g.Log().Errorf(r.GetCtx(), "获取分布式锁失败: %s, error: %v", lockKey, err) + } + if !success { + g.Log().Debugf(r.GetCtx(), "未获取到分布式锁,跳过状态清除: %s", lockKey) } } } From cfb7c6b011e72b143277b964a1cb21d040aec6fe Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sun, 4 Jan 2026 09:20:53 +0800 Subject: [PATCH 135/225] =?UTF-8?q?=E7=BB=9F=E4=B8=80MongoDB=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E5=91=BD=E5=90=8D=E4=B8=BAsnake=5Fcase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- beans/beans.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/beans/beans.go b/beans/beans.go index 53edaa7..07d8340 100644 --- a/beans/beans.go +++ b/beans/beans.go @@ -18,14 +18,14 @@ type Page struct { type MongoBaseDO struct { Id bson.ObjectID `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID Creator interface{} `bson:"creator,omitempty" json:"creator"` - CreatedAt time.Time `bson:"createdAt,omitempty" json:"createdAt"` + CreatedAt time.Time `bson:"created_at,omitempty" json:"createdAt"` Updater interface{} `bson:"updater,omitempty" json:"updater"` - UpdatedAt time.Time `bson:"updatedAt,omitempty" json:"updatedAt"` - TenantId interface{} `bson:"tenantId" json:"tenantId" default:"1"` // 租户ID - IsDeleted bool `bson:"isDeleted" json:"isDeleted" default:"false"` + UpdatedAt time.Time `bson:"updated_at,omitempty" json:"updatedAt"` + TenantId interface{} `bson:"tenant_id" json:"tenantId" default:"1"` // 租户ID + IsDeleted bool `bson:"is_deleted" json:"isDeleted" default:"false"` } type User struct { - UserName interface{} `bson:"userName" json:"userName"` // MongoDB 默认 ID - TenantId interface{} `bson:"tenantId" json:"tenantId"` // 租户ID + UserName interface{} `bson:"user_name" json:"userName"` // MongoDB 默认 ID + TenantId interface{} `bson:"tenant_id" json:"tenantId"` // 租户ID } From 27314840d1b7265d25b3c148cd8dfd07ed73074b Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sun, 4 Jan 2026 10:42:49 +0800 Subject: [PATCH 136/225] =?UTF-8?q?=E7=BB=9F=E4=B8=80Redis=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E7=BB=93=E6=9E=84=E4=BD=93=E5=AD=97=E6=AE=B5=E4=B8=BA?= =?UTF-8?q?=E9=A9=BC=E5=B3=B0camelCase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/types.go | 62 +++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/redis/types.go b/redis/types.go index 5c82526..3e7b1e0 100644 --- a/redis/types.go +++ b/redis/types.go @@ -10,48 +10,48 @@ type HistoryMessage struct { // SendStreamMessage 发送到 Redis Stream 的消息结构 type SendStreamMessage struct { - UserId string `json:"user_id"` // 用户ID - Content string `json:"content"` // 消息内容 - Timestamp int64 `json:"timestamp"` // 时间戳(秒) - MessageId string `json:"message_id"` // 消息唯一ID - Platform string `json:"platform,omitempty"` // 平台标识 - AccountId string `json:"account_id,omitempty"` // 账号ID - TenantId string `json:"tenant_id,omitempty"` // 租户ID(数据隔离) - AccountName string `json:"account_name,omitempty"` // 客服账号名称 - ChatId string `json:"chat_id,omitempty"` // RAGFlow Chat ID(从ragflow_config查询) - ReplyQueue string `json:"reply_queue,omitempty"` // 响应队列名称(支持多实例独立队列) - History []HistoryMessage `json:"history,omitempty"` // 历史对话(归档后恢复时携带) + UserId string `json:"userId"` // 用户ID + Content string `json:"content"` // 消息内容 + Timestamp int64 `json:"timestamp"` // 时间戳(秒) + MessageId string `json:"messageId"` // 消息唯一ID + Platform string `json:"platform,omitempty"` // 平台标识 + AccountId string `json:"accountId,omitempty"` // 账号ID + TenantId string `json:"tenantId,omitempty"` // 租户ID(数据隔离) + AccountName string `json:"accountName,omitempty"` // 客服账号名称 + ChatId string `json:"chatId,omitempty"` // RAGFlow Chat ID(从ragflow_config查询) + ReplyQueue string `json:"replyQueue,omitempty"` // 响应队列名称(支持多实例独立队列) + History []HistoryMessage `json:"history,omitempty"` // 历史对话(归档后恢复时携带) } // BatchStreamMessage 批量消息结构 type BatchStreamMessage struct { - UserId string `json:"user_id"` // 用户ID + UserId string `json:"userId"` // 用户ID Content string `json:"content"` // 消息内容 Timestamp int64 `json:"timestamp"` // 时间戳(秒) - BatchId string `json:"batch_id"` // 批次ID + BatchId string `json:"batchId"` // 批次ID Index int `json:"index"` // 批次内序号 } // ResponseStreamMessage RAGFlow 响应消息结构(MQ 消息) type ResponseStreamMessage struct { - UserId string `json:"user_id"` // 用户ID - Platform string `json:"platform"` // 平台标识 - TenantId string `json:"tenant_id"` // 租户ID - Question string `json:"question"` // 用户问题 - Content string `json:"content"` // RAGFlow 回复内容 - SessionId string `json:"session_id"` // RAGFlow Session ID - Timestamp int64 `json:"timestamp"` // 时间戳(秒) - MessageId string `json:"message_id"` // 原始消息ID + UserId string `json:"userId"` // 用户ID + Platform string `json:"platform"` // 平台标识 + TenantId string `json:"tenantId"` // 租户ID + Question string `json:"question"` // 用户问题 + Content string `json:"content"` // RAGFlow 回复内容 + SessionId string `json:"sessionId"` // RAGFlow Session ID + Timestamp int64 `json:"timestamp"` // 时间戳(秒) + MessageId string `json:"messageId"` // 原始消息ID } // FollowUpMessage 追问消息结构(RabbitMQ 延时队列) type FollowUpMessage struct { - TenantId string `json:"tenant_id"` // 租户ID - UserId string `json:"user_id"` // 用户ID - Platform string `json:"platform"` // 平台标识 - Content string `json:"content"` // 追问内容 - FollowUpType int `json:"follow_up_type"` // 追问类型:1=30s, 2=60s, 3=180s - Timestamp int64 `json:"timestamp"` // 发送时间戳 + TenantId string `json:"tenantId"` // 租户ID + UserId string `json:"userId"` // 用户ID + Platform string `json:"platform"` // 平台标识 + Content string `json:"content"` // 追问内容 + FollowUpType int `json:"followUpType"` // 追问类型:1=30s, 2=60s, 3=180s + Timestamp int64 `json:"timestamp"` // 发送时间戳 } // 追问类型常量 @@ -91,10 +91,10 @@ func GetFollowUpDelay(followUpType int) int { // ArchiveMessage 会话归档消息结构(RabbitMQ 延时队列) type ArchiveMessage struct { - UserId string `json:"user_id"` // 用户ID - Platform string `json:"platform"` // 平台标识 - SessionId string `json:"session_id"` // RAGFlow Session ID - Timestamp int64 `json:"timestamp"` // 发送时间戳 + UserId string `json:"userId"` // 用户ID + Platform string `json:"platform"` // 平台标识 + SessionId string `json:"sessionId"` // RAGFlow Session ID + Timestamp int64 `json:"timestamp"` // 发送时间戳 } // GetArchiveDelay 获取归档延时(从 config 包读取) From 41703aea3a06df1cd1435368eabdc56ab16414f7 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sun, 4 Jan 2026 13:54:14 +0800 Subject: [PATCH 137/225] =?UTF-8?q?=E4=BF=AE=E6=94=B9common/redis/types.go?= =?UTF-8?q?=20-=20=E7=A7=BB=E9=99=A4config=E5=8C=85=E4=BE=9D=E8=B5=96?= =?UTF-8?q?=EF=BC=8C=E6=94=B9=E4=B8=BA=E4=BB=8Econfig.yml=E8=AF=BB?= =?UTF-8?q?=E5=8F=96=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/types.go | 56 ++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/redis/types.go b/redis/types.go index 3e7b1e0..f76d97d 100644 --- a/redis/types.go +++ b/redis/types.go @@ -1,6 +1,10 @@ package redis -import "gitee.com/red-future---jilin-g/common/config" +import ( + "context" + + "github.com/gogf/gf/v2/frame/g" +) // HistoryMessage 历史消息结构(用于上下文注入) type HistoryMessage struct { @@ -61,32 +65,34 @@ const ( FollowUpType3 = 3 // 第三次追问 ) -// GetFollowUpContent 获取追问话术(从 config 包读取) +// GetFollowUpContent 获取追问话术(从 config.yml 读取) func GetFollowUpContent(followUpType int) string { - switch followUpType { - case FollowUpType1: - return config.FollowUpContent1 - case FollowUpType2: - return config.FollowUpContent2 - case FollowUpType3: - return config.FollowUpContent3 - default: + ctx := context.Background() + contents := g.Cfg().MustGet(ctx, "followUp.contents").Strings() + if len(contents) == 0 { return "" } + // followUpType: 1,2,3 对应数组索引 0,1,2 + index := followUpType - 1 + if index >= 0 && index < len(contents) { + return contents[index] + } + return "" } -// GetFollowUpDelay 获取追问延时(从 config 包读取) +// GetFollowUpDelay 获取追问延时(从 config.yml 读取) func GetFollowUpDelay(followUpType int) int { - switch followUpType { - case FollowUpType1: - return config.FollowUpDelay1 - case FollowUpType2: - return config.FollowUpDelay2 - case FollowUpType3: - return config.FollowUpDelay3 - default: - return 0 + ctx := context.Background() + delays := g.Cfg().MustGet(ctx, "followUp.delays").Ints() + if len(delays) == 0 { + return 30 // 默认30秒 } + // followUpType: 1,2,3 对应数组索引 0,1,2 + index := followUpType - 1 + if index >= 0 && index < len(delays) { + return delays[index] + } + return 30 } // ArchiveMessage 会话归档消息结构(RabbitMQ 延时队列) @@ -97,12 +103,14 @@ type ArchiveMessage struct { Timestamp int64 `json:"timestamp"` // 发送时间戳 } -// GetArchiveDelay 获取归档延时(从 config 包读取) +// GetArchiveDelay 获取归档延时(从 config.yml 读取) func GetArchiveDelay() int { - return config.ArchiveDelay + ctx := context.Background() + return g.Cfg().MustGet(ctx, "archive.delay", 3600).Int() // 默认3600秒(1小时) } -// GetHistoryContextLimit 获取历史上下文轮数(从 config 包读取) +// GetHistoryContextLimit 获取历史上下文轮数(从 config.yml 读取) func GetHistoryContextLimit() int64 { - return config.HistoryContextLimit + ctx := context.Background() + return g.Cfg().MustGet(ctx, "history.contextLimit", 5).Int64() // 默认5轮对话 } From c2574b0f9c4c01aeb1037bc1bf0bfb8a2ef445a4 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sun, 4 Jan 2026 14:13:14 +0800 Subject: [PATCH 138/225] =?UTF-8?q?=E5=88=A0=E9=99=A4common/config?= =?UTF-8?q?=E5=8C=85=20-=20=E9=85=8D=E7=BD=AE=E5=B7=B2=E8=BF=81=E7=A7=BB?= =?UTF-8?q?=E5=88=B0customerservice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/constants.go | 42 ------- config/directions.go | 236 ----------------------------------- config/welcome.go | 40 ------ config/welcome_messages.yaml | 53 -------- 4 files changed, 371 deletions(-) delete mode 100644 config/constants.go delete mode 100644 config/directions.go delete mode 100644 config/welcome.go delete mode 100644 config/welcome_messages.yaml diff --git a/config/constants.go b/config/constants.go deleted file mode 100644 index 8963e8d..0000000 --- a/config/constants.go +++ /dev/null @@ -1,42 +0,0 @@ -package config - -// ==================== 可配置常量 ==================== -// 修改以下值来调整系统行为 - -// -------------------- 追问配置 -------------------- - -// FollowUpDelay1 第一次追问延时(秒) -var FollowUpDelay1 = 30 - -// FollowUpDelay2 第二次追问延时(秒) -var FollowUpDelay2 = 60 - -// FollowUpDelay3 第三次追问延时(秒) -var FollowUpDelay3 = 180 - -// FollowUpContent1 第一次追问话术 -var FollowUpContent1 = "还有其他问题吗?" - -// FollowUpContent2 第二次追问话术 -var FollowUpContent2 = "如果需要帮助,随时告诉我~" - -// FollowUpContent3 第三次追问话术 -var FollowUpContent3 = "我一直在线,有问题随时找我~" - -// -------------------- 归档配置 -------------------- - -// ArchiveDelay 归档延时(秒),默认 1 小时 -var ArchiveDelay = 3600 - -// -------------------- 历史上下文配置 -------------------- - -// HistoryContextLimit 读取历史对话轮数(用于新 Session 上下文注入) -var HistoryContextLimit int64 = 5 - -// -------------------- Stream 消费配置 -------------------- - -// DefaultBatchSize 批量读取消息数量(削峰填谷) -var DefaultBatchSize int64 = 200 - -// DefaultBlockTimeout 阻塞超时时间(毫秒) -var DefaultBlockTimeout int64 = 2000 diff --git a/config/directions.go b/config/directions.go deleted file mode 100644 index e0e2c15..0000000 --- a/config/directions.go +++ /dev/null @@ -1,236 +0,0 @@ -// Package config 提供全局配置管理和Consul监听 -// -// 本包实现了基于Consul的配置热更新机制,所有服务导入common包即可自动获得配置监听能力 -package config - -import ( - "context" - "encoding/json" - "sync" - "time" - - "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/os/glog" - "github.com/hashicorp/consul/api" -) - -// Direction 咨询方向配置 -type Direction struct { - Name string `json:"name"` // 方向名称(如:气血、减肥) - ChatId string `json:"chat_id"` // RAGFlow对话ID -} - -var ( - directionsCache []Direction // 本地缓存(内存读取,超快) - cacheMu sync.RWMutex // 读写锁(支持多goroutine并发读) - startOnce sync.Once // 确保只启动一次监听 - consulClient *api.Client // Consul客户端(复用连接) -) - -// init 包初始化函数(所有服务导入common包时自动执行) -// -// Fallback顺序:Consul → config.yml -func init() { - ctx := context.Background() - - // 检查Consul是否配置 - consulAddr := g.Cfg().MustGet(ctx, "consul.address").String() - if consulAddr == "" { - glog.Warning(ctx, "Consul未配置,使用本地配置") - loadDirectionsFromLocal(ctx) - return - } - - // 初始化Consul客户端 - config := api.DefaultConfig() - config.Address = consulAddr - client, err := api.NewClient(config) - if err != nil { - glog.Errorf(ctx, "Consul客户端初始化失败: %v,fallback到本地配置", err) - loadDirectionsFromLocal(ctx) - return - } - consulClient = client - - // 启动后台监听(单例,确保只启动一次) - startOnce.Do(func() { - go startConsulWatcher(ctx) - glog.Info(ctx, "Consul配置监听已启动") - }) -} - -// GetDirections 获取咨询方向配置(从内存缓存读取) -// -// 返回: -// -// []Direction: 方向列表 -// -// 特点: -// - 高性能:读内存缓存,无网络IO -// - 线程安全:使用读锁,支持并发读取 -// - 自动更新:后台监听Consul,配置变化时自动更新缓存 -// -// 使用示例: -// -// dirs := config.GetDirections() -// for _, dir := range dirs { -// fmt.Printf("%s -> %s\n", dir.Name, dir.ChatId) -// } -func GetDirections() []Direction { - cacheMu.RLock() - defer cacheMu.RUnlock() - - // 返回副本,避免外部修改缓存 - result := make([]Direction, len(directionsCache)) - copy(result, directionsCache) - return result -} - -// GetDirectionChatId 根据方向名称获取对应的ChatId -// -// 参数: -// -// name: 方向名称(如:"气血"、"减肥") -// -// 返回: -// -// chatId: 对应的RAGFlow对话ID,未找到返回空字符串 -// -// 使用示例: -// -// chatId := config.GetDirectionChatId("气血") -func GetDirectionChatId(name string) string { - cacheMu.RLock() - defer cacheMu.RUnlock() - - for _, dir := range directionsCache { - if dir.Name == name { - return dir.ChatId - } - } - return "" -} - -// startConsulWatcher 后台监听Consul配置变化(Blocking Query长连接) -// -// 工作原理: -// 1. 使用Consul Blocking Query API(长连接,只在变化时返回) -// 2. 收到变化通知后更新本地缓存 -// 3. 自动重连(网络异常时自动恢复) -// -// 资源消耗: -// - 一个长连接(保持5分钟) -// - 配置未变化时几乎不占用CPU和网络 -// - 对比轮询:节省99%资源 -// -// 注意: -// - 此函数在后台goroutine中运行 -// - 使用Blocking Query避免轮询 -func startConsulWatcher(ctx context.Context) { - const consulKey = "ragflow/directions" - kv := consulClient.KV() - var lastIndex uint64 - - // 初始化时先读取一次配置 - if err := loadDirectionsFromConsul(ctx, kv); err != nil { - glog.Warningf(ctx, "初始化加载Consul配置失败: %v", err) - } - - // 持续监听配置变化 - for { - // Consul Blocking Query(长连接模式) - // WaitIndex: 指定版本号,只在配置变化时返回 - // WaitTime: 最长等待时间(超时后返回,客户端重新请求) - pair, meta, err := kv.Get(consulKey, &api.QueryOptions{ - WaitIndex: lastIndex, - WaitTime: 5 * time.Minute, - }) - - if err != nil { - glog.Errorf(ctx, "Consul查询失败: %v", err) - time.Sleep(5 * time.Second) // 错误时等待5秒重试 - continue - } - - // 配置版本号变化,说明有更新 - if meta.LastIndex != lastIndex { - lastIndex = meta.LastIndex - - // 配置被删除 - if pair == nil { - glog.Warning(ctx, "Consul配置已删除: "+consulKey) - cacheMu.Lock() - directionsCache = []Direction{} - cacheMu.Unlock() - continue - } - - // 解析并更新缓存 - var dirs []Direction - if err := json.Unmarshal(pair.Value, &dirs); err != nil { - glog.Errorf(ctx, "解析Consul配置失败: %v", err) - continue - } - - cacheMu.Lock() - directionsCache = dirs - cacheMu.Unlock() - - glog.Infof(ctx, "Consul配置已更新: %d个方向", len(dirs)) - } - } -} - -// loadDirectionsFromConsul 从Consul加载配置(初始化时调用) -func loadDirectionsFromConsul(ctx context.Context, kv *api.KV) error { - const consulKey = "ragflow/directions" - - pair, _, err := kv.Get(consulKey, nil) - if err != nil { - // Consul查询失败,fallback到本地配置 - glog.Warningf(ctx, "Consul查询失败: %v,fallback到本地配置", err) - loadDirectionsFromLocal(ctx) - return err - } - - if pair == nil { - glog.Warning(ctx, "Consul中未找到配置: "+consulKey+",fallback到本地配置") - loadDirectionsFromLocal(ctx) - return nil - } - - var dirs []Direction - if err := json.Unmarshal(pair.Value, &dirs); err != nil { - glog.Errorf(ctx, "解析Consul配置失败: %v,fallback到本地配置", err) - loadDirectionsFromLocal(ctx) - return err - } - - cacheMu.Lock() - directionsCache = dirs - cacheMu.Unlock() - - glog.Infof(ctx, "已加载Consul配置: %d个方向", len(dirs)) - return nil -} - -// loadDirectionsFromLocal 从本地config.yml加载配置(fallback机制) -func loadDirectionsFromLocal(ctx context.Context) { - directionsConfig := g.Cfg().MustGet(ctx, "ragflow.directions") - if directionsConfig.IsEmpty() { - glog.Warning(ctx, "本地配置中也未找到 ragflow.directions") - return - } - - var dirs []Direction - if err := directionsConfig.Scan(&dirs); err != nil { - glog.Errorf(ctx, "解析本地配置失败: %v", err) - return - } - - cacheMu.Lock() - directionsCache = dirs - cacheMu.Unlock() - - glog.Infof(ctx, "已加载config.yml配置: %d个方向", len(dirs)) -} diff --git a/config/welcome.go b/config/welcome.go deleted file mode 100644 index 4e4bb76..0000000 --- a/config/welcome.go +++ /dev/null @@ -1,40 +0,0 @@ -package config - -import ( - "context" - "sync" - - "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/os/glog" -) - -var ( - welcomeCache map[string]string - welcomeMu sync.RWMutex - welcomeOnce sync.Once -) - -// initWelcomeMessages 初始化欢迎话术配置 -func initWelcomeMessages(ctx context.Context) { - welcomeOnce.Do(func() { - // 从默认配置文件(config.yml)读取 welcomes 配置 - welcomeMap := g.Cfg().MustGet(ctx, "welcomes").MapStrStr() - - welcomeMu.Lock() - welcomeCache = welcomeMap - welcomeMu.Unlock() - - glog.Infof(ctx, "已加载欢迎话术配置: %d个方向", len(welcomeMap)) - }) -} - -// GetWelcomeMessage 根据方向名称获取欢迎话术 -func GetWelcomeMessage(direction string) string { - ctx := context.Background() - initWelcomeMessages(ctx) - - welcomeMu.RLock() - defer welcomeMu.RUnlock() - - return welcomeCache[direction] -} diff --git a/config/welcome_messages.yaml b/config/welcome_messages.yaml deleted file mode 100644 index 0286482..0000000 --- a/config/welcome_messages.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# 各咨询方向的欢迎话术配置 -# 当用户在状态5选择方向后,会自动发送对应的欢迎语 - -welcomes: - 乳腺贴: | - 🙋‍♀️ 欢迎姐妹们来到药济堂!我们是一个拥有10年经验的大健康专业团队🏅,专注于保守调理乳腺问题,已经成功帮助超过1万位姐妹轻松调理乳腺健康❤️。 - 如果您有结节或增生的困扰,欢迎随时咨询! - 请回复下面的数字,让我帮助您分析结节情况: - - 结节 - 增生 - 点击👇获取更快速的服务! - - 肝病: | - 你好,我是黄医生,有什么肝脏方面的问题我可以帮助您吗?无论是肝病、乙肝、丙肝,还是肝硬化腹水、脂肪肝、酒精肝,我都会为您提供专业的建议。😷 - 肝病的类型很多,常见症状有乏力、食欲减退和肝区不适等。为了更好地帮您,我需要了解一些详细的信息,比如: - - 您现在有腹胀或腹水的情况吗? - 是第一次出现还是反复出现呢? - 目前是早期还是中晚期呢? - 有没有病毒性肝炎的病史呢? - 请您留一下联系方式,我可以发送您的报告,并给您详细解读肝病治疗方案和成功案例。📋 - - 车膜: | - 🎉亲爱的车主,欢迎来到6膜王! - 🚗 我们在车膜行业深耕十年,拥有2家千平米的门店和30+专业团队,致力于为您提供最优质的服务! - ✨ 无论是隐形车衣还是改色膜,我们都能精准适配您的需求。留下您的【车型➕VX】,我们的资深顾问会立即为您匹配专属方案、膜材讲解以及报价!❤️ - - 毛孔: | - 啊啊~亲爱的姐妹们,最近有很多小伙伴在问我关于毛孔的问题!我之前也是毛孔大到妆容卡粉,真的是烦恼不断。😩 但在经过一段时间的摸索后,我终于找到了合适的方法,效果真的很好,差不多一个月就改善了许多!(亲测有效!)💖 - 如果需要帮助,可以随时告诉我,我非常乐意分享我的经验给你们哦! - - 免税店: | - Hi,长春的宝子们~💕欢迎锁定小红提免税集合店! - 这里有你想要的一切:美妆、香水、包包、首饰和大牌护肤品,正品保真不踩雷,价格特别美丽哦✨ - 全城顺丰包邮,购物更方便!期待你们来逛快来发现更多惊喜吧!💖 - - 门店地址:长春市绿园区皓月大路吾悦广场1楼 1036号,等你来哦!🌟 - - 减肥: | - 你好呀,姐妹~你也有肉肉的困扰吗?我现在已经掉了二十多斤了,至今都没有反弹! - 想要方法的话可以直接回复"1",我分享给你哦~ - (💗未成年发育期、哺乳期的姐妹我就不推荐啦) - - 气血: | - 亲爱的,欢迎光临!🌸 - 如果你有月经不调或气血不足的问题,随时可以问我哦! - - 停经闭经 - 痛经难忍 - 量少 - 经期不准 - 💗只需回复数字,我们会为你提供专业建议! 🌟如需更多帮助,点击下方咨询专业老师,我们一起寻求解决方案~ From ae28ac14979013cb1f47f4b98c7bbdaf6b43872c Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sun, 4 Jan 2026 14:24:45 +0800 Subject: [PATCH 139/225] =?UTF-8?q?=E5=88=A0=E9=99=A4common/consts?= =?UTF-8?q?=E5=8C=85=20-=20=E5=B0=86Redis=20Key=E5=B8=B8=E9=87=8F=E8=BF=81?= =?UTF-8?q?=E7=A7=BB=E5=88=B0redis/keys.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/rate_limiter.go | 4 ++-- mongo/mongo.go | 13 ++++++------- consts/redis_key.go => redis/keys.go | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) rename consts/redis_key.go => redis/keys.go (98%) diff --git a/middleware/rate_limiter.go b/middleware/rate_limiter.go index f8c9bc6..3f7df11 100644 --- a/middleware/rate_limiter.go +++ b/middleware/rate_limiter.go @@ -190,7 +190,7 @@ func WalletTransferLimiter(r *ghttp.Request) { return } - key := fmt.Sprintf(consts.RateLimitKeyTransfer, userId) + key := fmt.Sprintf(redis.RateLimitKeyTransfer, userId) // 限制: 每个用户每分钟最多转账5次 count, err := redis.IncrRateLimit(r.GetCtx(), key, 60) // 60秒窗口 @@ -252,7 +252,7 @@ func OSSUploadLimiter(r *ghttp.Request) { return } - key := fmt.Sprintf(consts.RateLimitKeyUpload, userId) + key := fmt.Sprintf(redis.RateLimitKeyUpload, userId) // 限制: 每个用户每分钟最多上传10个文件 count, err := redis.IncrRateLimit(r.GetCtx(), key, 60) // 60秒窗口 diff --git a/mongo/mongo.go b/mongo/mongo.go index 777b2e1..3df3396 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -11,7 +11,6 @@ import ( "gitee.com/red-future---jilin-g/common/log/model/dto" "github.com/gogf/gf/v2/container/gvar" - "gitee.com/red-future---jilin-g/common/consts" "gitee.com/red-future---jilin-g/common/redis" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/errors/gerror" @@ -280,7 +279,7 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c filter["isDeleted"] = false filterMap := utils.OrderMap(filter) optsMap := listOptionsToMap(ctx, opts...) - redisKey := fmt.Sprintf(consts.List, user.TenantId, collection, gconv.String(filterMap), gconv.String(optsMap)) + redisKey := fmt.Sprintf(redis.List, user.TenantId, collection, gconv.String(filterMap), gconv.String(optsMap)) if m.Cache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) @@ -328,7 +327,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} } filter["isDeleted"] = false filterMap := utils.OrderMap(filter) - redisKey := fmt.Sprintf(consts.One, user.TenantId, collection, gconv.String(filterMap)) + redisKey := fmt.Sprintf(redis.One, user.TenantId, collection, gconv.String(filterMap)) if m.Cache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) @@ -360,7 +359,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} return } func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collection string) (err error) { - listKeys := fmt.Sprintf(consts.CleanList, tenantId, collection) + listKeys := fmt.Sprintf(redis.CleanList, tenantId, collection) keys, err := redis.RedisClient.Keys(ctx, listKeys) if err != nil { return @@ -371,7 +370,7 @@ func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interf return } } - countKeys := fmt.Sprintf(consts.CleanCount, tenantId, collection) + countKeys := fmt.Sprintf(redis.CleanCount, tenantId, collection) keys, err = redis.RedisClient.Keys(ctx, countKeys) if err != nil { return @@ -385,7 +384,7 @@ func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interf filter["isDeleted"] = false delete(filter, "tenantId") filterMap := utils.OrderMap(filter) - oneKey := fmt.Sprintf(consts.One, tenantId, collection, gconv.String(filterMap)) + oneKey := fmt.Sprintf(redis.One, tenantId, collection, gconv.String(filterMap)) _, err = redis.RedisClient.Del(ctx, oneKey) if err != nil { return @@ -654,7 +653,7 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( } filter["isDeleted"] = false filterMap := utils.OrderMap(filter) - redisKey := fmt.Sprintf(consts.Count, user.TenantId, collection, gconv.String(filterMap)) + redisKey := fmt.Sprintf(redis.Count, user.TenantId, collection, gconv.String(filterMap)) if m.Cache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) diff --git a/consts/redis_key.go b/redis/keys.go similarity index 98% rename from consts/redis_key.go rename to redis/keys.go index 3f85dff..d438fe6 100644 --- a/consts/redis_key.go +++ b/redis/keys.go @@ -1,4 +1,4 @@ -package consts +package redis // Redis 数据缓存 Key 常量 const ( From c7f3a4d38d9b81b2a1b687724fa129a24bb20685 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sun, 4 Jan 2026 14:47:52 +0800 Subject: [PATCH 140/225] =?UTF-8?q?=E4=BF=AE=E5=A4=8Drate=5Flimiter.go?= =?UTF-8?q?=E4=B8=AD=E6=AE=8B=E7=95=99=E7=9A=84consts=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/rate_limiter.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/middleware/rate_limiter.go b/middleware/rate_limiter.go index 3f7df11..ba0e745 100644 --- a/middleware/rate_limiter.go +++ b/middleware/rate_limiter.go @@ -4,7 +4,6 @@ import ( "fmt" "strings" - "gitee.com/red-future---jilin-g/common/consts" "gitee.com/red-future---jilin-g/common/redis" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/frame/g" @@ -18,7 +17,7 @@ func GlobalLimiter(r *ghttp.Request) { // 从配置文件读取全局限流参数 globalLimit := g.Cfg().MustGet(r.GetCtx(), "rate.limit", 800).Int64() - key := consts.RateLimitKeyGlobal + key := redis.RateLimitKeyGlobal // 使用Redis计数器进行全局限流 count, err := redis.IncrRateLimit(r.GetCtx(), key, 1) // 1秒窗口 @@ -40,7 +39,7 @@ func GlobalLimiter(r *ghttp.Request) { // IPLimiter IP限流中间件(防DDoS) func IPLimiter(r *ghttp.Request) { ip := r.GetClientIp() - key := fmt.Sprintf(consts.RateLimitKeyIP, ip) + key := fmt.Sprintf(redis.RateLimitKeyIP, ip) // 从配置文件读取IP限流参数 ipLimit := g.Cfg().MustGet(r.GetCtx(), "rate.ip.limit", 100).Int64() @@ -90,7 +89,7 @@ func UserLimiter(r *ghttp.Request) { userLimit = g.Cfg().MustGet(r.GetCtx(), "rate.user.anonymous.limit", 20).Int64() } - key := fmt.Sprintf(consts.RateLimitKeyUser, userId) + key := fmt.Sprintf(redis.RateLimitKeyUser, userId) count, err := redis.IncrRateLimit(r.GetCtx(), key, 1) if err != nil { g.Log().Errorf(r.GetCtx(), "用户限流Redis错误: %v", err) @@ -132,7 +131,7 @@ func ServiceLimiter(r *ghttp.Request) { return } - key := fmt.Sprintf(consts.RateLimitKeyService, serverName) + key := fmt.Sprintf(redis.RateLimitKeyService, serverName) count, err := redis.IncrRateLimit(r.GetCtx(), key, 1) if err != nil { g.Log().Errorf(r.GetCtx(), "服务限流Redis错误: %v", err) @@ -159,7 +158,7 @@ func OrderCreateLimiter(r *ghttp.Request) { return } - key := fmt.Sprintf(consts.RateLimitKeyOrder, userId) + key := fmt.Sprintf(redis.RateLimitKeyOrder, userId) // 限制: 每个用户每分钟最多创建10个订单 count, err := redis.IncrRateLimit(r.GetCtx(), key, 60) // 60秒窗口 @@ -221,7 +220,7 @@ func CSMessageLimiter(r *ghttp.Request) { return } - key := fmt.Sprintf(consts.RateLimitKeyMessage, userId) + key := fmt.Sprintf(redis.RateLimitKeyMessage, userId) // 限制: 每个用户每分钟最多发送30条消息 count, err := redis.IncrRateLimit(r.GetCtx(), key, 60) // 60秒窗口 From a068e95017f351a74f741611e799965637adf8e8 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sun, 4 Jan 2026 14:53:53 +0800 Subject: [PATCH 141/225] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dredis.go=E4=B8=AD?= =?UTF-8?q?=E6=AE=8B=E7=95=99=E7=9A=84consts=E5=BC=95=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/redis/redis.go b/redis/redis.go index c112d41..38b372a 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -7,7 +7,6 @@ import ( "sync" "time" - "gitee.com/red-future---jilin-g/common/consts" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/glog" @@ -500,7 +499,7 @@ func IsUserActive(ctx context.Context, userId string, seconds int64) (bool, erro // key: 限流key(需要包含完整路径,如 "ip:192.168.1.1") // windowSeconds: 时间窗口(秒) func IncrRateLimit(ctx context.Context, key string, windowSeconds int64) (count int64, err error) { - fullKey := consts.RateLimitKeyPrefix + key + fullKey := RateLimitKeyPrefix + key result, err := redisClient.Do(ctx, "INCR", fullKey) if err != nil { return @@ -516,7 +515,7 @@ func IncrRateLimit(ctx context.Context, key string, windowSeconds int64) (count // GetRateLimit 获取当前限流计数 func GetRateLimit(ctx context.Context, key string) (count int64, err error) { - fullKey := consts.RateLimitKeyPrefix + key + fullKey := RateLimitKeyPrefix + key result, err := redisClient.Get(ctx, fullKey) if err != nil { return From cf27eee0ea8cb176bf11ba89f6d38c5e99c70f5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Sun, 4 Jan 2026 10:35:15 +0800 Subject: [PATCH 142/225] =?UTF-8?q?=E8=B5=84=E4=BA=A7=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=89=B9=E6=AC=A1=E5=BA=93=E5=AD=98=E7=AE=A1=E7=90=86=E6=A8=A1?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- utils/utils.go | 149 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/utils/utils.go b/utils/utils.go index 68f8c87..8d9efa4 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -3,8 +3,12 @@ package utils import ( "context" "fmt" + "net" "reflect" "sort" + "strconv" + "strings" + "sync/atomic" "time" "gitee.com/red-future---jilin-g/common/beans" @@ -59,6 +63,7 @@ func GetMonthToday(t time.Time, month int) time.Time { } return target.AddDate(0, 0, t.Day()-1) } + func GetUserInfo(ctx context.Context) (user beans.User, err error) { r := g.RequestFromCtx(ctx) if r != nil { @@ -102,6 +107,7 @@ func GetUserInfo(ctx context.Context) (user beans.User, err error) { } return } + func OrderMap(m map[string]interface{}) map[string]interface{} { // 提取所有key keys := make([]string, 0, len(m)) @@ -121,3 +127,146 @@ func OrderMap(m map[string]interface{}) map[string]interface{} { return orderedMap } + +// ParseIntSlice 解析整数切片 - 通用字符串处理工具 +func ParseIntSlice(str string) []int { + parts := strings.Split(str, ",") + result := make([]int, 0, len(parts)) + for _, part := range parts { + if val, err := strconv.Atoi(strings.TrimSpace(part)); err == nil { + result = append(result, val) + } + } + return result +} + +// ParseStrings 解析字符串切片 - 通用字符串处理工具 +func ParseStrings(str string) []string { + if str == "" { + return nil + } + parts := strings.Split(str, ",") + result := make([]string, 0, len(parts)) + for _, part := range parts { + if trimmed := strings.TrimSpace(part); trimmed != "" { + result = append(result, trimmed) + } + } + return result +} + +// FilterServiceNames 过滤服务名 - 通用映射处理工具 +func FilterServiceNames(services map[string]interface{}, excludeKeys ...string) []string { + excludeMap := make(map[string]bool) + for _, key := range excludeKeys { + excludeMap[key] = true + } + result := make([]string, 0, len(services)) + for key := range services { + if !excludeMap[key] { + result = append(result, key) + } + } + return result +} + +// FormatUnixTime 格式化Unix时间戳 - 通用时间处理工具 +func FormatUnixTime(timestamp int64) string { + if timestamp <= 0 { + return "" + } + return time.Unix(timestamp, 0).Format("2006-01-02 15:04:05") +} + +// ParseDurationWithDefault 解析持续时间,失败时使用默认值 - 通用时间处理工具 +func ParseDurationWithDefault(ctx context.Context, durationStr, defaultStr, fieldName string) (time.Duration, string) { + durationParsed, err := time.ParseDuration(durationStr) + if err != nil { + // 这里不能直接使用g.Log(),因为这是utils包,没有直接的日志访问 + // 调用方应该处理日志 + // g.Log().Warningf(ctx, "解析%s失败: %s, 使用默认值 %s, error: %v", fieldName, durationStr, defaultStr, err) + durationParsed, _ = time.ParseDuration(defaultStr) + return durationParsed, defaultStr + } + return durationParsed, durationStr +} + +// AtomicUpdateMin 原子更新最小值 - 通用数值处理工具 +func AtomicUpdateMin(minValue *atomic.Int64, newValue int64) { + for { + currentMin := minValue.Load() + if newValue >= currentMin { + break + } + if minValue.CompareAndSwap(currentMin, newValue) { + break + } + } +} + +// AtomicUpdateMax 原子更新最大值 - 通用数值处理工具 +func AtomicUpdateMax(maxValue *atomic.Int64, newValue int64) { + for { + currentMax := maxValue.Load() + if newValue <= currentMax { + break + } + if maxValue.CompareAndSwap(currentMax, newValue) { + break + } + } +} + +// ParseCIDRs 解析CIDR列表 - 通用网络处理工具 +func ParseCIDRs(strs []string) ([]*net.IPNet, error) { + nets := make([]*net.IPNet, 0, len(strs)) + for _, s := range strs { + if s == "*" { + if _, ipv4Net, err := net.ParseCIDR("0.0.0.0/0"); err == nil { + nets = append(nets, ipv4Net) + } + if _, ipv6Net, err := net.ParseCIDR("::/0"); err == nil { + nets = append(nets, ipv6Net) + } + continue + } + if _, ipNet, err := net.ParseCIDR(s); err == nil { + nets = append(nets, ipNet) + } + } + return nets, nil +} + +// UrlDecode 简单的URL解码 - 通用编码解码工具 +func UrlDecode(s string) string { + result := make([]byte, 0, len(s)) + + for i := 0; i < len(s); i++ { + if s[i] == '%' && i+2 < len(s) { + if high := HexDigit(s[i+1]); high != 0xFF { + if low := HexDigit(s[i+2]); low != 0xFF { + result = append(result, (high<<4)|low) + i += 2 + continue + } + } + } + result = append(result, s[i]) + } + + return string(result) +} + +// HexDigit 十六进制字符转数字 - 通用编码解码工具 +func HexDigit(c byte) byte { + switch { + case '0' <= c && c <= '9': + return c - '0' + case 'a' <= c && c <= 'f': + return c - 'a' + 10 + case 'A' <= c && c <= 'F': + return c - 'A' + 10 + default: + return 0xFF + } +} From d8e848a23867c4d94bb5ef98c4c8029a3406e03b Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 6 Jan 2026 17:23:03 +0800 Subject: [PATCH 143/225] =?UTF-8?q?=E5=A2=9E=E5=BC=BAStream=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E5=A4=84=E7=90=86=E6=97=A5=E5=BF=97=EF=BC=9A=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E6=8E=A5=E6=94=B6/=E5=A4=84=E7=90=86/panic=E8=AF=A6?= =?UTF-8?q?=E7=BB=86=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- consul/consul.go | 2 +- ragflow/document.go | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/consul/consul.go b/consul/consul.go index 7bf8b7c..b5e79cf 100644 --- a/consul/consul.go +++ b/consul/consul.go @@ -60,7 +60,7 @@ func getLocalIP() (string, error) { } func getInstanceAddrByIp(ctx context.Context, ip string, services []gsvc.Service) (addr string) { for _, s := range services { - if s.GetEndpoints()[0].Host() == addr { + if s.GetEndpoints()[0].Host() == ip { addr = s.GetEndpoints()[0].String() return } diff --git a/ragflow/document.go b/ragflow/document.go index 0d819a7..af39527 100644 --- a/ragflow/document.go +++ b/ragflow/document.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" ) // 数据集内文件管理 @@ -201,16 +202,23 @@ func (c *Client) UploadDocumentFromText(ctx context.Context, datasetId, content, Data []UploadDocumentRes `json:"data"` // RAGFlow返回数组 } - if err := json.Unmarshal(resp.ReadAll(), &response); err != nil { + respBody := resp.ReadAll() + g.Log().Debugf(ctx, "RAGFlow上传文档响应: %s", string(respBody)) + + if err := json.Unmarshal(respBody, &response); err != nil { + g.Log().Errorf(ctx, "解析RAGFlow响应失败: %v, 原始响应: %s", err, string(respBody)) return "", gerror.Newf("json Decode failed: %v", err) } - if len(response.Data) == 0 { - return "", gerror.New("上传文档返回data为空") + // 先检查code,再检查data + if response.Code != 0 { + g.Log().Errorf(ctx, "RAGFlow返回错误: code=%d, message=%s", response.Code, response.Message) + return "", gerror.Newf("上传文档失败 (code=%d): %s", response.Code, response.Message) } - if response.Code != 0 { - return "", gerror.Newf("上传文档失败 (code=%d): %s", response.Code, response.Message) + if len(response.Data) == 0 { + g.Log().Errorf(ctx, "RAGFlow返回data为空, 完整响应: %s", string(respBody)) + return "", gerror.New("上传文档返回data为空") } return response.Data[0].Id, nil From 270bf3b6bb3b450c9b24d7791b8460eae640c3cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 5 Jan 2026 15:08:08 +0800 Subject: [PATCH 144/225] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=99=90=E6=B5=81?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/rate_limiter.go | 190 ++----------------------------------- 1 file changed, 8 insertions(+), 182 deletions(-) diff --git a/middleware/rate_limiter.go b/middleware/rate_limiter.go index ba0e745..70dc61e 100644 --- a/middleware/rate_limiter.go +++ b/middleware/rate_limiter.go @@ -8,7 +8,6 @@ import ( "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" - "github.com/gogf/gf/v2/text/gstr" "github.com/gogf/gf/v2/util/gconv" ) @@ -63,46 +62,23 @@ func IPLimiter(r *ghttp.Request) { // UserLimiter 用户维度限流中间件(防止单用户滥用) func UserLimiter(r *ghttp.Request) { - // 从JWT获取用户ID(如果已登录) - var userId string - var isAuth bool = false - - if token := r.Header.Get("Authorization"); token != "" && gstr.HasPrefix(token, "Bearer ") { - // 这里应该解析JWT获取用户ID,简化示例中直接使用token - tokenStr := gstr.SubStrFrom(token, "7") - if tokenStr != "" && validateToken(tokenStr) { - userId = tokenStr - isAuth = true - } + var userName string + user, err := utils.GetUserInfo(r.GetCtx()) + if err != nil { + r.Response.WriteStatusExit(429, err.Error()) + return } - - // 如果没有userId,使用IP作为标识 - if userId == "" { - userId = "anon:" + r.GetClientIp() - } - + userName = gconv.String(user.UserName) // 从配置文件读取用户限流参数 - var userLimit int64 - if isAuth { - userLimit = g.Cfg().MustGet(r.GetCtx(), "rate.user.authenticated.limit", 50).Int64() - } else { - userLimit = g.Cfg().MustGet(r.GetCtx(), "rate.user.anonymous.limit", 20).Int64() - } - - key := fmt.Sprintf(redis.RateLimitKeyUser, userId) + userLimit := g.Cfg().MustGet(r.GetCtx(), "rate.user.limit", 50).Int64() + key := fmt.Sprintf(redis.RateLimitKeyUser, userName) count, err := redis.IncrRateLimit(r.GetCtx(), key, 1) if err != nil { g.Log().Errorf(r.GetCtx(), "用户限流Redis错误: %v", err) - r.Middleware.Next() return } if count > userLimit { - userType := "已登录" - if !isAuth { - userType = "未登录" - } - g.Log().Warningf(r.GetCtx(), "用户限流触发: %s, count: %d, limit: %d, type: %s", userId, count, userLimit, userType) r.Response.WriteStatusExit(429, "您的请求过于频繁,请稍后再试") return } @@ -147,153 +123,3 @@ func ServiceLimiter(r *ghttp.Request) { r.Middleware.Next() } - -// OrderCreateLimiter 订单创建限流中间件 -// 限制: 每个用户每分钟最多创建10个订单 -func OrderCreateLimiter(r *ghttp.Request) { - userId := getUserIdFromContext(r) // 从context获取用户ID - if userId == "" { - // 如果无法获取用户信息,跳过限流检查 - r.Middleware.Next() - return - } - - key := fmt.Sprintf(redis.RateLimitKeyOrder, userId) - - // 限制: 每个用户每分钟最多创建10个订单 - count, err := redis.IncrRateLimit(r.GetCtx(), key, 60) // 60秒窗口 - if err != nil { - g.Log().Errorf(r.GetCtx(), "订单创建限流Redis错误: %v", err) - r.Middleware.Next() - return - } - - if count > 10 { - g.Log().Warningf(r.GetCtx(), "订单创建限流触发: %s, count: %d", userId, count) - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 429, - Message: "下单过于频繁,请稍后再试", - }) - return - } - - r.Middleware.Next() -} - -// WalletTransferLimiter 钱包转账限流中间件 -// 限制: 每个用户每分钟最多转账5次 -func WalletTransferLimiter(r *ghttp.Request) { - userId := getUserIdFromContext(r) // 从context获取用户ID - if userId == "" { - r.Middleware.Next() - return - } - - key := fmt.Sprintf(redis.RateLimitKeyTransfer, userId) - - // 限制: 每个用户每分钟最多转账5次 - count, err := redis.IncrRateLimit(r.GetCtx(), key, 60) // 60秒窗口 - if err != nil { - g.Log().Errorf(r.GetCtx(), "钱包转账限流Redis错误: %v", err) - r.Middleware.Next() - return - } - - if count > 5 { - g.Log().Warningf(r.GetCtx(), "钱包转账限流触发: %s, count: %d", userId, count) - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 429, - Message: "转账操作过于频繁,请稍后再试", - }) - return - } - - r.Middleware.Next() -} - -// CSMessageLimiter 客服消息限流中间件 -// 限制: 每个用户每分钟最多发送30条消息 -func CSMessageLimiter(r *ghttp.Request) { - userId := getUserIdFromContext(r) // 从context获取用户ID - if userId == "" { - r.Middleware.Next() - return - } - - key := fmt.Sprintf(redis.RateLimitKeyMessage, userId) - - // 限制: 每个用户每分钟最多发送30条消息 - count, err := redis.IncrRateLimit(r.GetCtx(), key, 60) // 60秒窗口 - if err != nil { - g.Log().Errorf(r.GetCtx(), "客服消息限流Redis错误: %v", err) - r.Middleware.Next() - return - } - - if count > 30 { - g.Log().Warningf(r.GetCtx(), "客服消息限流触发: %s, count: %d", userId, count) - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 429, - Message: "消息发送过于频繁,请稍后再试", - }) - return - } - - r.Middleware.Next() -} - -// OSSUploadLimiter 文件上传限流中间件 -// 限制: 每个用户每分钟最多上传10个文件 -func OSSUploadLimiter(r *ghttp.Request) { - userId := getUserIdFromContext(r) // 从context获取用户ID - if userId == "" { - r.Middleware.Next() - return - } - - key := fmt.Sprintf(redis.RateLimitKeyUpload, userId) - - // 限制: 每个用户每分钟最多上传10个文件 - count, err := redis.IncrRateLimit(r.GetCtx(), key, 60) // 60秒窗口 - if err != nil { - g.Log().Errorf(r.GetCtx(), "文件上传限流Redis错误: %v", err) - r.Middleware.Next() - return - } - - if count > 10 { - g.Log().Warningf(r.GetCtx(), "文件上传限流触发: %s, count: %d", userId, count) - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{ - Code: 429, - Message: "文件上传过于频繁,请稍后再试", - }) - return - } - - r.Middleware.Next() -} - -// getUserIdFromContext 从请求上下文中获取用户ID -// 使用项目中已有的utils.GetUserInfo方法 -func getUserIdFromContext(r *ghttp.Request) string { - // 使用项目中已有的utils.GetUserInfo方法获取用户信息 - user, err := utils.GetUserInfo(r.GetCtx()) - if err != nil { - // 如果获取用户信息失败,返回空字符串 - return "" - } - - // 在这个项目中,UserName就是用来标识用户的ID - // 转换为字符串类型 - if user.UserName != nil { - return gconv.String(user.UserName) - } - - return "" -} - -// validateToken 验证token有效性 -func validateToken(token string) bool { - // 实现 token 验证逻辑 - return token == "valid-token" -} From 122e52a03df62562887ebcbdaab8f59cc9b16950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 5 Jan 2026 15:59:51 +0800 Subject: [PATCH 145/225] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=99=90=E6=B5=81?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 662 +++++++++++++--------------------- 1 file changed, 251 insertions(+), 411 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 71b3c2c..43378af 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -3,7 +3,6 @@ package middleware import ( "context" "fmt" - "net" "strconv" "strings" "sync" @@ -48,10 +47,8 @@ type CircuitBreakerConfig struct { FallbackMessage string RequestTimeout int DistributedTTL int - AdminIPs []string StatIntervalMs int MinRequestAmount int - AdminCIDRs []string HalfOpenMaxRequests int HalfOpenSuccessThreshold float64 WarmupDuration string @@ -59,7 +56,6 @@ type CircuitBreakerConfig struct { EnableAdaptiveThreshold bool AdaptiveMinThreshold float64 AdaptiveMaxThreshold float64 - CIDRNetMasks []*net.IPNet } // CircuitBreakerMetrics 熔断器指标 @@ -226,23 +222,14 @@ type CircuitBreakerInfo struct { Config *CircuitBreakerConfig Metrics *CircuitBreakerMetrics SuccessCodeMap map[int]bool - CIDRNetMasks []*net.IPNet AdaptiveThreshold float64 WarmupEndTime int64 } var ( circuitBreakers sync.Map - circuitBreakerConfigs sync.Map stateChangeListeners sync.Map stateChangeListenersRegistered sync.Map - allowedAdminIPsMap map[string]bool - allowedAdminIPsMutex sync.RWMutex - allowedAdminCIDRs []*net.IPNet - allowedAdminCIDRsMutex sync.RWMutex - totalServicesCount atomic.Int64 - serviceNamesSlice []string - serviceNamesMutex sync.RWMutex ) // 默认值常量 @@ -341,19 +328,29 @@ func (cb *CircuitBreakerInfo) updateStateMetrics(state CircuitBreakerState) { } } -// getCircuitBreakerInfoAndConfig 获取熔断器信息和配置 -func getCircuitBreakerInfoAndConfig(serviceName string) (*CircuitBreakerInfo, *CircuitBreakerConfig) { - cbInfoVal, ok := circuitBreakers.Load(serviceName) - if !ok { - return nil, nil +// getCircuitBreakerInfoByResource 根据资源名获取熔断器信息 +// 支持精确匹配和前缀匹配 +func getCircuitBreakerInfoByResource(resourceName string) (*CircuitBreakerInfo, *CircuitBreakerConfig) { + // 先尝试精确匹配 + if cbInfoVal, ok := circuitBreakers.Load(resourceName); ok { + cbInfo, ok := cbInfoVal.(*CircuitBreakerInfo) + if ok { + return cbInfo, cbInfo.Config + } } - cbInfo, ok := cbInfoVal.(*CircuitBreakerInfo) - if !ok { - return nil, nil + // 尝试前缀匹配:去掉查询参数部分 + if idx := strings.Index(resourceName, "?"); idx > 0 { + prefix := resourceName[:idx] + if cbInfoVal, ok := circuitBreakers.Load(prefix); ok { + cbInfo, ok := cbInfoVal.(*CircuitBreakerInfo) + if ok { + return cbInfo, cbInfo.Config + } + } } - return cbInfo, cbInfo.Config + return nil, nil } // updateResponseTimeStats 更新响应时间统计 @@ -388,88 +385,63 @@ func InitCircuitBreaker() error { registerStateChangeListeners() g.Log().Infof(ctx, "Sentinel熔断器初始化成功") - services := g.Cfg().MustGet(ctx, "circuitBreaker").Map() - serviceNames := filterServiceNames(services) - - if len(serviceNames) == 0 { - g.Log().Infof(ctx, "未配置任何服务熔断器") + // 加载接口级别的熔断器配置 + configs := g.Cfg().MustGet(ctx, "circuitBreaker.interfaces").Map() + if len(configs) == 0 { + g.Log().Infof(ctx, "未配置任何接口熔断器") return nil } - totalServicesCount.Store(int64(len(serviceNames))) - serviceNamesMutex.Lock() - serviceNamesSlice = serviceNames - serviceNamesMutex.Unlock() - enabledCount := 0 - for _, serviceName := range serviceNames { - config := loadServiceCircuitBreakerConfig(serviceName) + for resourcePattern, configData := range configs { + config, err := loadInterfaceCircuitBreakerConfig(ctx, resourcePattern, configData) + if err != nil { + g.Log().Errorf(ctx, "加载接口 %s 熔断器配置失败: %v", resourcePattern, err) + continue + } if config != nil && config.Enabled { - circuitBreakerConfigs.Store(serviceName, config) - if err := initServiceCircuitBreaker(serviceName, config); err != nil { - g.Log().Errorf(ctx, "服务 %s 熔断器初始化失败: %v", serviceName, err) + if err := initInterfaceCircuitBreaker(resourcePattern, config); err != nil { + g.Log().Errorf(ctx, "接口 %s 熔断器初始化失败: %v", resourcePattern, err) } else { - g.Log().Infof(ctx, "服务 %s 熔断器初始化成功", serviceName) + g.Log().Infof(ctx, "接口 %s 熔断器初始化成功", resourcePattern) enabledCount++ } } } - updateAdminIPsCache() - g.Log().Infof(ctx, "共初始化 %d 个服务熔断器,其中 %d 个已启用", len(serviceNames), enabledCount) + g.Log().Infof(ctx, "共初始化 %d 个接口熔断器,其中 %d 个已启用", len(configs), enabledCount) return nil } // ReloadCircuitBreakerConfig 动态重新加载熔断器配置 -func ReloadCircuitBreakerConfig(serviceName string) error { - config := loadServiceCircuitBreakerConfig(serviceName) - if config == nil { - return fmt.Errorf("未找到服务 %s 的配置", serviceName) +// loadInterfaceCircuitBreakerConfig 加载接口级别的熔断器配置 +func loadInterfaceCircuitBreakerConfig(ctx context.Context, resourcePattern string, configData interface{}) (*CircuitBreakerConfig, error) { + configMap, ok := configData.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("接口 %s 配置格式错误: %v", resourcePattern, configData) } - if err := validateCircuitBreakerConfig(config); err != nil { - return fmt.Errorf("配置验证失败: %v", err) - } - - oldConfig, _ := circuitBreakerConfigs.Load(serviceName) - circuitBreakerConfigs.Store(serviceName, config) - - if err := initServiceCircuitBreaker(serviceName, config); err != nil { - if oldConfig != nil { - circuitBreakerConfigs.Store(serviceName, oldConfig) - } - return fmt.Errorf("重新初始化熔断器失败: %v", err) - } - - g.Log().Infof(context.Background(), "服务 %s 熔断器配置重新加载成功", serviceName) - return nil -} - -// loadServiceCircuitBreakerConfig 加载配置 -func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { - ctx := context.Background() - key := "circuitBreaker." + serviceName config := &CircuitBreakerConfig{ - Enabled: g.Cfg().MustGet(ctx, key+".enabled", true).Bool(), - MaxFailures: g.Cfg().MustGet(ctx, key+".maxFailures", defaultMaxFailures).Int(), - Timeout: g.Cfg().MustGet(ctx, key+".timeout", defaultTimeout).String(), - SlowRequestThreshold: g.Cfg().MustGet(ctx, key+".slowRequestThreshold", defaultSlowRequestThreshold).String(), - EnableSlidingWindow: g.Cfg().MustGet(ctx, key+".enableSlidingWindow", false).Bool(), - FailureRateThreshold: g.Cfg().MustGet(ctx, key+".failureRateThreshold", 0.5).Float64(), - EnableFallback: g.Cfg().MustGet(ctx, key+".enableFallback", false).Bool(), - FallbackMessage: g.Cfg().MustGet(ctx, key+".fallbackMessage", "").String(), - RequestTimeout: g.Cfg().MustGet(ctx, key+".requestTimeout", defaultRequestTimeout).Int(), - DistributedTTL: g.Cfg().MustGet(ctx, key+".distributedTTL", defaultDistributedTTL).Int(), - StatIntervalMs: g.Cfg().MustGet(ctx, key+".statIntervalMs", defaultStatIntervalMs).Int(), - HalfOpenMaxRequests: g.Cfg().MustGet(ctx, key+".halfOpenMaxRequests", defaultHalfOpenMaxRequests).Int(), - HalfOpenSuccessThreshold: g.Cfg().MustGet(ctx, key+".halfOpenSuccessThreshold", defaultHalfOpenSuccessThreshold).Float64(), - WarmupDuration: g.Cfg().MustGet(ctx, key+".warmupDuration", defaultWarmupDuration).String(), - EnableAdaptiveThreshold: g.Cfg().MustGet(ctx, key+".enableAdaptiveThreshold", false).Bool(), - AdaptiveMinThreshold: g.Cfg().MustGet(ctx, key+".adaptiveMinThreshold", 0.3).Float64(), - AdaptiveMaxThreshold: g.Cfg().MustGet(ctx, key+".adaptiveMaxThreshold", 0.7).Float64(), + Enabled: getBoolFromMap(configMap, "enabled", true), + MaxFailures: getIntFromMap(configMap, "maxFailures", defaultMaxFailures), + Timeout: getStringFromMap(configMap, "timeout", defaultTimeout), + SlowRequestThreshold: getStringFromMap(configMap, "slowRequestThreshold", defaultSlowRequestThreshold), + EnableSlidingWindow: getBoolFromMap(configMap, "enableSlidingWindow", false), + FailureRateThreshold: getFloatFromMap(configMap, "failureRateThreshold", 0.5), + EnableFallback: getBoolFromMap(configMap, "enableFallback", false), + FallbackMessage: getStringFromMap(configMap, "fallbackMessage", ""), + RequestTimeout: getIntFromMap(configMap, "requestTimeout", defaultRequestTimeout), + DistributedTTL: getIntFromMap(configMap, "distributedTTL", defaultDistributedTTL), + StatIntervalMs: getIntFromMap(configMap, "statIntervalMs", defaultStatIntervalMs), + HalfOpenMaxRequests: getIntFromMap(configMap, "halfOpenMaxRequests", defaultHalfOpenMaxRequests), + HalfOpenSuccessThreshold: getFloatFromMap(configMap, "halfOpenSuccessThreshold", defaultHalfOpenSuccessThreshold), + WarmupDuration: getStringFromMap(configMap, "warmupDuration", defaultWarmupDuration), + EnableAdaptiveThreshold: getBoolFromMap(configMap, "enableAdaptiveThreshold", false), + AdaptiveMinThreshold: getFloatFromMap(configMap, "adaptiveMinThreshold", 0.3), + AdaptiveMaxThreshold: getFloatFromMap(configMap, "adaptiveMaxThreshold", 0.7), } - config.MinRequestAmount = g.Cfg().MustGet(ctx, key+".minRequestAmount", 0).Int() + config.MinRequestAmount = getIntFromMap(configMap, "minRequestAmount", 0) if config.MinRequestAmount == 0 { config.MinRequestAmount = config.MaxFailures } @@ -480,15 +452,125 @@ func loadServiceCircuitBreakerConfig(serviceName string) *CircuitBreakerConfig { config.WarmupDurationParsed, config.WarmupDuration = parseDurationWithDefault(ctx, config.WarmupDuration, defaultWarmupDuration, "warmupDuration") // 解析状态码 - successCodes := g.Cfg().MustGet(ctx, key+".successStatusCodes", "200,201,204").String() + successCodes := getStringFromMap(configMap, "successStatusCodes", "200,201,204") config.SuccessStatusCodes = parseIntSlice(successCodes) - // 解析IP和CIDR - config.AdminIPs = parseStrings(g.Cfg().MustGet(ctx, key+".adminIPs", "").String()) - config.AdminCIDRs = parseStrings(g.Cfg().MustGet(ctx, key+".adminCIDRs", "").String()) - config.CIDRNetMasks, _ = parseCIDRs(config.AdminCIDRs) + return config, nil +} - return config +// 辅助函数:从map中获取值 +func getBoolFromMap(m map[string]interface{}, key string, defaultValue bool) bool { + if val, ok := m[key]; ok { + if b, ok := val.(bool); ok { + return b + } + } + return defaultValue +} + +func getIntFromMap(m map[string]interface{}, key string, defaultValue int) int { + if val, ok := m[key]; ok { + switch v := val.(type) { + case int: + return v + case float64: + return int(v) + case string: + if i, err := strconv.Atoi(v); err == nil { + return i + } + } + } + return defaultValue +} + +func getFloatFromMap(m map[string]interface{}, key string, defaultValue float64) float64 { + if val, ok := m[key]; ok { + switch v := val.(type) { + case float64: + return v + case int: + return float64(v) + case string: + if f, err := strconv.ParseFloat(v, 64); err == nil { + return f + } + } + } + return defaultValue +} + +func getStringFromMap(m map[string]interface{}, key string, defaultValue string) string { + if val, ok := m[key]; ok { + if s, ok := val.(string); ok { + return s + } + } + return defaultValue +} + +// initInterfaceCircuitBreaker 初始化接口级别的熔断器 +func initInterfaceCircuitBreaker(resourcePattern string, config *CircuitBreakerConfig) error { + if err := validateCircuitBreakerConfig(config); err != nil { + return err + } + + threshold := config.FailureRateThreshold + if config.EnableAdaptiveThreshold { + threshold = (config.AdaptiveMinThreshold + config.AdaptiveMaxThreshold) / 2 + } + + var rule []*circuitbreaker.Rule + baseRule := &circuitbreaker.Rule{ + Resource: resourcePattern, + RetryTimeoutMs: uint32(config.TimeoutParsed.Milliseconds()), + MinRequestAmount: uint64(config.MinRequestAmount), + StatIntervalMs: uint32(config.StatIntervalMs), + } + + if config.EnableSlidingWindow { + baseRule.Strategy = circuitbreaker.SlowRequestRatio + baseRule.StatSlidingWindowBucketCount = 10 + baseRule.MaxAllowedRtMs = uint64(config.SlowRequestThresholdParsed.Milliseconds()) + baseRule.Threshold = threshold + } else { + baseRule.Strategy = circuitbreaker.ErrorCount + baseRule.Threshold = float64(config.MaxFailures) + } + + rule = []*circuitbreaker.Rule{baseRule} + + if _, err := circuitbreaker.LoadRulesOfResource(resourcePattern, []*circuitbreaker.Rule{}); err != nil { + return fmt.Errorf("清空熔断规则失败: %v", err) + } + if _, err := circuitbreaker.LoadRules(rule); err != nil { + return fmt.Errorf("加载熔断规则失败: %v", err) + } + + successCodeMap := make(map[int]bool, len(config.SuccessStatusCodes)) + for _, code := range config.SuccessStatusCodes { + successCodeMap[code] = true + } + + cbInfo := &CircuitBreakerInfo{ + ResourceName: resourcePattern, + Config: config, + Metrics: newCircuitBreakerMetrics(), + SuccessCodeMap: successCodeMap, + AdaptiveThreshold: threshold, + WarmupEndTime: time.Now().Add(config.WarmupDurationParsed).Unix(), + } + cbInfo.init() + circuitBreakers.Store(resourcePattern, cbInfo) + + strategy := "error_count" + if config.EnableSlidingWindow { + strategy = "slow_ratio" + } + g.Log().Infof(context.Background(), "接口 %s 熔断器初始化成功: resource=%s, strategy=%s, timeout=%v, threshold=%.2f", + resourcePattern, resourcePattern, strategy, config.TimeoutParsed, rule[0].Threshold) + + return nil } // parseIntSlice 解析整数切片 @@ -503,21 +585,6 @@ func parseIntSlice(str string) []int { return result } -// parseStrings 解析字符串切片 -func parseStrings(str string) []string { - if str == "" { - return nil - } - parts := strings.Split(str, ",") - result := make([]string, 0, len(parts)) - for _, part := range parts { - if trimmed := strings.TrimSpace(part); trimmed != "" { - result = append(result, trimmed) - } - } - return result -} - // parseDurationWithDefault 解析持续时间,失败时使用默认值 func parseDurationWithDefault(ctx context.Context, durationStr, defaultStr, fieldName string) (time.Duration, string) { durationParsed, err := time.ParseDuration(durationStr) @@ -555,16 +622,6 @@ func atomicUpdateMax(maxValue *atomic.Int64, newValue int64) { } } -// getAllowedIPsAndCIDRs 获取允许的IP和CIDR列表(带锁保护) -func getAllowedIPsAndCIDRs() (map[string]bool, []*net.IPNet) { - allowedAdminIPsMutex.RLock() - allowedAdminCIDRsMutex.RLock() - defer allowedAdminIPsMutex.RUnlock() - defer allowedAdminCIDRsMutex.RUnlock() - - return allowedAdminIPsMap, allowedAdminCIDRs -} - // reset 重置所有指标到初始状态 func (m *CircuitBreakerMetrics) reset() { m.TotalRequests.Store(0) @@ -584,26 +641,6 @@ func (m *CircuitBreakerMetrics) reset() { // 时间戳相关字段不重置,LastResetTime在调用时单独设置 } -// parseCIDRs 解析CIDR列表 -func parseCIDRs(strs []string) ([]*net.IPNet, error) { - nets := make([]*net.IPNet, 0, len(strs)) - for _, s := range strs { - if s == "*" { - if _, ipv4Net, err := net.ParseCIDR("0.0.0.0/0"); err == nil { - nets = append(nets, ipv4Net) - } - if _, ipv6Net, err := net.ParseCIDR("::/0"); err == nil { - nets = append(nets, ipv6Net) - } - continue - } - if _, ipNet, err := net.ParseCIDR(s); err == nil { - nets = append(nets, ipNet) - } - } - return nets, nil -} - // newCircuitBreakerMetrics 创建并初始化熔断器指标 func newCircuitBreakerMetrics() *CircuitBreakerMetrics { metrics := &CircuitBreakerMetrics{} @@ -707,84 +744,20 @@ func validateCircuitBreakerConfig(config *CircuitBreakerConfig) error { return nil } -// initServiceCircuitBreaker 初始化服务熔断器 -func initServiceCircuitBreaker(serviceName string, config *CircuitBreakerConfig) error { - if err := validateCircuitBreakerConfig(config); err != nil { - return err - } - - resourceName := "service:" + serviceName - threshold := config.FailureRateThreshold - if config.EnableAdaptiveThreshold { - threshold = (config.AdaptiveMinThreshold + config.AdaptiveMaxThreshold) / 2 - } - - var rule []*circuitbreaker.Rule - baseRule := &circuitbreaker.Rule{ - Resource: resourceName, - RetryTimeoutMs: uint32(config.TimeoutParsed.Milliseconds()), - MinRequestAmount: uint64(config.MinRequestAmount), - StatIntervalMs: uint32(config.StatIntervalMs), - } - - if config.EnableSlidingWindow { - baseRule.Strategy = circuitbreaker.SlowRequestRatio - baseRule.StatSlidingWindowBucketCount = 10 - baseRule.MaxAllowedRtMs = uint64(config.SlowRequestThresholdParsed.Milliseconds()) - baseRule.Threshold = threshold - } else { - baseRule.Strategy = circuitbreaker.ErrorCount - baseRule.Threshold = float64(config.MaxFailures) - } - - rule = []*circuitbreaker.Rule{baseRule} - - if _, err := circuitbreaker.LoadRulesOfResource(resourceName, []*circuitbreaker.Rule{}); err != nil { - return fmt.Errorf("清空熔断规则失败: %v", err) - } - if _, err := circuitbreaker.LoadRules(rule); err != nil { - return fmt.Errorf("加载熔断规则失败: %v", err) - } - - successCodeMap := make(map[int]bool, len(config.SuccessStatusCodes)) - for _, code := range config.SuccessStatusCodes { - successCodeMap[code] = true - } - - cbInfo := &CircuitBreakerInfo{ - ResourceName: resourceName, - Config: config, - Metrics: newCircuitBreakerMetrics(), - SuccessCodeMap: successCodeMap, - CIDRNetMasks: config.CIDRNetMasks, - AdaptiveThreshold: threshold, - WarmupEndTime: time.Now().Add(config.WarmupDurationParsed).Unix(), - } - cbInfo.init() - circuitBreakers.Store(serviceName, cbInfo) - - strategy := "error_count" - if config.EnableSlidingWindow { - strategy = "slow_ratio" - } - g.Log().Infof(context.Background(), "服务 %s 熔断器初始化成功: resource=%s, strategy=%s, timeout=%v, threshold=%.2f", - serviceName, resourceName, strategy, config.TimeoutParsed, rule[0].Threshold) - - return nil -} - // CircuitBreakerMiddleware 熔断降级中间件 func CircuitBreakerMiddleware(r *ghttp.Request) { startTime := time.Now() ctx := r.GetCtx() - serviceName := extractServiceName(r.URL.Path) - if serviceName == "" { + // 基于接口地址+请求参数生成熔断资源名 + resourceName := generateResourceName(r) + if resourceName == "" { r.Middleware.Next() return } - cbInfo, config := getCircuitBreakerInfoAndConfig(serviceName) + // 检查是否有该资源的熔断配置 + cbInfo, config := getCircuitBreakerInfoByResource(resourceName) if cbInfo == nil || config == nil || !config.Enabled { r.Middleware.Next() return @@ -797,7 +770,6 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { return } - resourceName := cbInfo.ResourceName if config.RequestTimeout > 0 { var ctxCancel context.CancelFunc ctx, ctxCancel = context.WithTimeout(ctx, time.Duration(config.RequestTimeout)*time.Millisecond) @@ -809,7 +781,7 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { if config.DistributedTTL > 0 && isCircuitBreakerOpenInDistributed(ctx, resourceName) { cbInfo.Metrics.BlockRequests.Add(1) g.Log().Warningf(ctx, "分布式熔断触发: %s", resourceName) - sendFallbackResponse(r, serviceName, config, "distributed") + sendFallbackResponse(r, resourceName, config, "distributed") return } @@ -831,7 +803,7 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { } } - sendFallbackResponse(r, serviceName, config, "halfopen_limit") + sendFallbackResponse(r, resourceName, config, "halfopen_limit") return } } @@ -846,14 +818,14 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { oldState := cbInfo.setStateWithMetrics(StateOpen, true) if oldState != StateOpen { - notifyStateChange(serviceName, oldState, StateOpen) + notifyStateChange(resourceName, oldState, StateOpen) } if config.DistributedTTL > 0 { syncCircuitBreakerStateToDistributed(ctx, resourceName, "open", config.DistributedTTL) } - sendFallbackResponse(r, serviceName, config, "blocked") + sendFallbackResponse(r, resourceName, config, "blocked") return } @@ -882,7 +854,7 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { if entry != nil { api.TraceError(entry, fmt.Errorf("request failed with status: %d", statusCode)) } - g.Log().Debugf(ctx, "服务 %s 请求失败: status=%d, duration=%v", serviceName, statusCode, duration) + g.Log().Debugf(ctx, "接口 %s 请求失败: status=%d, duration=%v", resourceName, statusCode, duration) // 重新获取当前状态,避免使用过期状态 currentState := cbInfo.getState() @@ -923,27 +895,27 @@ func CircuitBreakerMiddleware(r *ghttp.Request) { // 如果状态不是关闭但也不是半开,尝试重置为关闭状态 oldState := cbInfo.setStateWithMetrics(StateClosed, true) if oldState != StateClosed { - notifyStateChange(serviceName, oldState, StateClosed) + notifyStateChange(resourceName, oldState, StateClosed) } } } } // sendFallbackResponse 发送降级响应 -func sendFallbackResponse(r *ghttp.Request, serviceName string, config *CircuitBreakerConfig, reason string) { - g.Log().Warningf(r.GetCtx(), "熔断器降级: service=%s, reason=%s, clientIP=%s", serviceName, reason, r.GetClientIp()) +func sendFallbackResponse(r *ghttp.Request, resourceName string, config *CircuitBreakerConfig, reason string) { + g.Log().Warningf(r.GetCtx(), "熔断器降级: resource=%s, reason=%s", resourceName, reason) if config.EnableFallback && config.FallbackMessage != "" { r.Response.WriteStatusExit(503, config.FallbackMessage) return } - msg := fmt.Sprintf("服务 '%s' 暂时不可用,请稍后再试", serviceName) + msg := fmt.Sprintf("接口 '%s' 暂时不可用,请稍后再试", resourceName) switch reason { case "blocked": - msg = fmt.Sprintf("服务 '%s' 熔断保护中,请稍后再试", serviceName) + msg = fmt.Sprintf("接口 '%s' 熔断保护中,请稍后再试", resourceName) case "distributed": - msg = fmt.Sprintf("服务 '%s' 分布式熔断中", serviceName) + msg = fmt.Sprintf("接口 '%s' 分布式熔断中", resourceName) } r.Response.WriteStatusExit(503, msg) } @@ -960,101 +932,45 @@ func isSuccessStatusCode(cbInfo *CircuitBreakerInfo, statusCode int) bool { return statusCode >= 200 && statusCode < 300 } -// extractServiceName 从URL路径提取服务名 -func extractServiceName(path string) string { - path = strings.Trim(path, "/") - if path == "" { +// generateResourceName 基于接口地址+请求参数生成熔断资源名 +func generateResourceName(r *ghttp.Request) string { + method := r.Method + path := r.URL.Path + query := r.URL.Query().Encode() + + // 生成资源名:方法:路径?查询参数 + // 示例: GET:/api/users?userId=123 + resourceName := method + ":" + path + if query != "" { + // 对查询参数进行排序以确保相同的参数顺序生成相同的资源名 + sortedQuery := sortQueryString(query) + resourceName += "?" + sortedQuery + } + + return resourceName +} + +// sortQueryString 对查询字符串进行排序 +func sortQueryString(query string) string { + if query == "" { return "" } - // 获取第一个路径段 - if idx := strings.Index(path, "/"); idx > 0 { - path = path[:idx] + params := strings.Split(query, "&") + if len(params) == 0 { + return query } - // 解码URL编码(简化版) - serviceName := path - if strings.Contains(serviceName, "%") { - serviceName = urlDecode(serviceName) - } - - if _, ok := circuitBreakerConfigs.Load(serviceName); ok { - return serviceName - } - return "" -} - -// urlDecode 简单的URL解码 -func urlDecode(s string) string { - result := make([]byte, 0, len(s)) - - for i := 0; i < len(s); i++ { - if s[i] == '%' && i+2 < len(s) { - if high := hexDigit(s[i+1]); high != 0xFF { - if low := hexDigit(s[i+2]); low != 0xFF { - result = append(result, (high<<4)|low) - i += 2 - continue - } + // 简单的字符串排序 + for i := 0; i < len(params)-1; i++ { + for j := i + 1; j < len(params); j++ { + if params[i] > params[j] { + params[i], params[j] = params[j], params[i] } } - result = append(result, s[i]) } - return string(result) -} - -func hexDigit(c byte) byte { - switch { - case '0' <= c && c <= '9': - return c - '0' - case 'a' <= c && c <= 'f': - return c - 'a' + 10 - case 'A' <= c && c <= 'F': - return c - 'A' + 10 - default: - return 0xFF - } -} - -// updateAdminIPsCache 更新管理员IP白名单缓存 -func updateAdminIPsCache() { - ipMap := make(map[string]bool) - cidrNets := make([]*net.IPNet, 0) - - circuitBreakerConfigs.Range(func(_, value interface{}) bool { - config, ok := value.(*CircuitBreakerConfig) - if !ok { - return true - } - for _, ip := range config.AdminIPs { - if !ipMap[ip] { - ipMap[ip] = true - } - } - cidrNets = append(cidrNets, config.CIDRNetMasks...) - return true - }) - - allowedAdminIPsMutex.Lock() - allowedAdminIPsMap = ipMap - allowedAdminIPsMutex.Unlock() - - allowedAdminCIDRsMutex.Lock() - allowedAdminCIDRs = cidrNets - allowedAdminCIDRsMutex.Unlock() -} - -// filterServiceNames 过滤服务名 -func filterServiceNames(services map[string]interface{}) []string { - excludeKeys := map[string]bool{"services": true, "enableDistributed": true, "requestTimeout": true, "distributedTTL": true} - result := make([]string, 0, len(services)) - for key := range services { - if !excludeKeys[key] { - result = append(result, key) - } - } - return result + return strings.Join(params, "&") } // isCircuitBreakerOpenInDistributed 检查分布式熔断状态 @@ -1105,11 +1021,6 @@ func syncCircuitBreakerStateToDistributed(ctx context.Context, resourceName, sta // CircuitBreakerHealthCheckHandler 健康检查接口 func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { - if !isAdminIP(r) { - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 403, Message: "权限不足,禁止访问"}) - return - } - page := r.Get("page").Int() size := r.Get("size").Int() if page < 0 { @@ -1119,11 +1030,14 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { size = 20 } - serviceNamesMutex.RLock() - slice := serviceNamesSlice - serviceNamesMutex.RUnlock() + // 获取所有熔断器资源 + var resources []string + circuitBreakers.Range(func(key, value interface{}) bool { + resources = append(resources, key.(string)) + return true + }) - total := len(slice) + total := len(resources) start := page * size if start >= total { r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: "熔断器状态", @@ -1144,8 +1058,8 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { halfOpenServices := 0 for i := start; i < end; i++ { - serviceName := slice[i] - cbInfoVal, ok := circuitBreakers.Load(serviceName) + resourceName := resources[i] + cbInfoVal, ok := circuitBreakers.Load(resourceName) if !ok { continue } @@ -1166,7 +1080,7 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { lastOpenTimeStr := formatUnixTime(cbInfo.Metrics.LastOpenTime.Load()) nextRetryTimeStr := formatUnixTime(cbInfo.Metrics.NextRetryTime.Load()) - status[serviceName] = map[string]interface{}{ + status[resourceName] = map[string]interface{}{ "resource": cbInfo.ResourceName, "state": string(state), "lastOpenTime": lastOpenTimeStr, @@ -1189,74 +1103,34 @@ func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { "services": status, "page": page, "size": size, "total": total}}) } -// isAdminIP 检查IP是否在白名单中 -func isAdminIP(r *ghttp.Request) bool { - clientIP := r.GetClientIp() - if clientIP == "" { - return false - } - - // 一次性获取IP和CIDR列表,减少锁操作 - allowedIPs, allowedCIDRs := getAllowedIPsAndCIDRs() - - // 如果没有任何限制,允许访问 - if len(allowedIPs) == 0 && len(allowedCIDRs) == 0 { - return true - } - - // 检查IP白名单 - if allowedIPs[clientIP] { - return true - } - - // 检查CIDR白名单 - if clientNetIP := net.ParseIP(clientIP); clientNetIP != nil { - for _, cidrNet := range allowedCIDRs { - if cidrNet.Contains(clientNetIP) { - return true - } - } - } - - g.Log().Warningf(r.GetCtx(), "熔断器操作请求被拒绝,IP不在白名单中: %s", clientIP) - return false -} - -// batchProcessServices 批量处理服务 -func batchProcessServices(r *ghttp.Request, processFunc func(serviceName string) error) (int, int, map[string]string) { +// batchProcessResources 批量处理资源 +func batchProcessResources(r *ghttp.Request, processFunc func(resourceName string) error) (int, int, map[string]string) { successCount := 0 failCount := 0 failures := make(map[string]string) - serviceNamesMutex.RLock() - slice := serviceNamesSlice - serviceNamesMutex.RUnlock() - - for _, serviceName := range slice { - if err := processFunc(serviceName); err != nil { - g.Log().Errorf(r.GetCtx(), "服务 %s 处理失败: %v", serviceName, err) + circuitBreakers.Range(func(key, value interface{}) bool { + resourceName := key.(string) + if err := processFunc(resourceName); err != nil { + g.Log().Errorf(r.GetCtx(), "资源 %s 处理失败: %v", resourceName, err) failCount++ - failures[serviceName] = err.Error() + failures[resourceName] = err.Error() } else { successCount++ } - } + return true + }) return successCount, failCount, failures } // CircuitBreakerResetHandler 重置熔断器 func CircuitBreakerResetHandler(r *ghttp.Request) { - serviceName := r.Get("service").String() + resourceName := r.Get("resource").String() - if !isAdminIP(r) { - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 403, Message: "权限不足,禁止访问"}) - return - } - - if serviceName == "" || serviceName == "*" { - successCount, failCount, failures := batchProcessServices(r, func(name string) error { - return resetSingleService(r, name) + if resourceName == "" || resourceName == "*" { + successCount, failCount, failures := batchProcessResources(r, func(name string) error { + return resetSingleResource(r, name) }) g.Log().Infof(r.GetCtx(), "批量重置熔断器完成: 成功 %d, 失败 %d", successCount, failCount) @@ -1265,45 +1139,35 @@ func CircuitBreakerResetHandler(r *ghttp.Request) { return } - if err := resetSingleService(r, serviceName); err != nil { + if err := resetSingleResource(r, resourceName); err != nil { r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 500, Message: fmt.Sprintf("重置熔断器失败: %v", err)}) return } - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: fmt.Sprintf("服务 '%s' 的熔断器已重置", serviceName)}) + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: fmt.Sprintf("资源 '%s' 的熔断器已重置", resourceName)}) } -// resetSingleService 重置单个服务 -func resetSingleService(r *ghttp.Request, serviceName string) error { - resourceName := "service:" + serviceName - +// resetSingleResource 重置单个资源 +func resetSingleResource(r *ghttp.Request, resourceName string) error { if rules := circuitbreaker.GetRulesOfResource(resourceName); len(rules) > 0 { if _, err := circuitbreaker.LoadRulesOfResource(resourceName, []*circuitbreaker.Rule{}); err != nil { return err } } - if configVal, ok := circuitBreakerConfigs.Load(serviceName); ok { - if err := initServiceCircuitBreaker(serviceName, configVal.(*CircuitBreakerConfig)); err != nil { - return err - } - } - - if cbInfoVal, ok := circuitBreakers.Load(serviceName); ok { + if cbInfoVal, ok := circuitBreakers.Load(resourceName); ok { cbInfo := cbInfoVal.(*CircuitBreakerInfo) + config := cbInfo.Config cbInfo.State.Store(stateClosed) // 重置指标 cbInfo.Metrics.reset() - cbInfo.WarmupEndTime = time.Now().Add(cbInfo.Config.WarmupDurationParsed).Unix() + cbInfo.WarmupEndTime = time.Now().Add(config.WarmupDurationParsed).Unix() cbInfo.Metrics.LastResetTime.Store(time.Now().Unix()) - } - if configVal, ok := circuitBreakerConfigs.Load(serviceName); ok { - config, ok := configVal.(*CircuitBreakerConfig) - if ok && config.DistributedTTL > 0 { + // 清除分布式状态 + if config.DistributedTTL > 0 { redisClient := g.Redis() if redisClient != nil { - // 使用common/redis中的Lock方法确保分布式一致性 lockKey := "circuit_breaker:" + resourceName + ":lock" success, err := redis.Lock(r.GetCtx(), lockKey, 10, func(ctx context.Context) error { _, err := redisClient.Del(ctx, "circuit_breaker:"+resourceName+":state") @@ -1329,31 +1193,7 @@ func resetSingleService(r *ghttp.Request, serviceName string) error { // CircuitBreakerReloadHandler 配置重载接口 func CircuitBreakerReloadHandler(r *ghttp.Request) { - serviceName := r.Get("service").String() - - if !isAdminIP(r) { - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 403, Message: "权限不足,禁止访问"}) - return - } - - if serviceName == "" || serviceName == "*" { - successCount, failCount, failures := batchProcessServices(r, func(serviceName string) error { - return ReloadCircuitBreakerConfig(serviceName) - }) - - updateAdminIPsCache() - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: fmt.Sprintf("配置重载完成: 成功 %d, 失败 %d", successCount, failCount), - Data: map[string]interface{}{"success": successCount, "failed": failCount, "failures": failures}}) - return - } - - if err := ReloadCircuitBreakerConfig(serviceName); err != nil { - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 500, Message: fmt.Sprintf("重载失败: %v", err)}) - return - } - - updateAdminIPsCache() - r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 200, Message: fmt.Sprintf("服务 '%s' 的熔断器配置已重载", serviceName)}) + r.Response.WriteJsonExit(ghttp.DefaultHandlerResponse{Code: 501, Message: "基于接口的熔断器暂不支持配置重载"}) } // StateChangeListener 状态变化监听器类型 From e08103e097c94366ec741aa04aa424d216e407a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 5 Jan 2026 16:14:14 +0800 Subject: [PATCH 146/225] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=99=90=E6=B5=81?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 187 +++++++++++++++++++++++++++++----- 1 file changed, 159 insertions(+), 28 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 43378af..84aa185 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -168,6 +168,8 @@ func (m *HalfOpenManager) TryAcquireHalfOpenSlot(metrics HalfOpenMetrics, maxReq return false, currentRequests } + // 原子性保证:在一个CAS操作中增加计数 + // 这样可以确保HalfOpenRequests和HalfOpenPassed的一致性 metrics.AddHalfOpenRequests(1) metrics.AddHalfOpenPassed(1) return true, currentRequests + 1 @@ -182,20 +184,26 @@ func (m *HalfOpenManager) RecordHalfOpenResult(metrics HalfOpenMetrics, isSucces m.mu.Lock() defer m.mu.Unlock() + // 原子性:先减少请求计数 metrics.AddHalfOpenRequests(-1) + + // 记录结果 if isSuccess { metrics.AddHalfOpenPassed(1) } else { metrics.AddHalfOpenFailed(1) } + // 在锁保护下检查阈值,确保读取到一致的数据 return m.checkHalfOpenSuccessThreshold(metrics, successThreshold) } // checkHalfOpenSuccessThreshold 检查半开状态的成功率是否达到阈值 func (m *HalfOpenManager) checkHalfOpenSuccessThreshold(metrics HalfOpenMetrics, successThreshold float64) bool { - totalRequests := metrics.GetHalfOpenPassed().Load() + metrics.GetHalfOpenFailed().Load() + // 原子性:一次性读取所有计数器,避免读取到不一致的数据 passedRequests := metrics.GetHalfOpenPassed().Load() + failedRequests := metrics.GetHalfOpenFailed().Load() + totalRequests := passedRequests + failedRequests if totalRequests == 0 { return false @@ -312,13 +320,26 @@ func (cb *CircuitBreakerInfo) int64ToState(state int64) CircuitBreakerState { func (cb *CircuitBreakerInfo) updateStateMetrics(state CircuitBreakerState) { now := time.Now().Unix() + // 防护:确保时间戳在合理范围内 + // 32位系统上,Unix时间戳在2038年1月19日会溢出 + // 这里做一些防护,确保存储的时间戳是有效的 + if now < 0 || now > 1<<62 { + g.Log().Warningf(context.Background(), "检测到异常时间戳: %d, 将使用当前系统时间", now) + now = time.Now().Unix() + } + // 根据新状态更新计数器 switch state { case StateOpen: cb.Metrics.OpenCount.Add(1) cb.Metrics.LastOpenTime.Store(now) // 设置下一次重试时间 - cb.Metrics.NextRetryTime.Store(time.Now().Add(cb.Config.TimeoutParsed).Unix()) + nextRetry := time.Now().Add(cb.Config.TimeoutParsed).Unix() + if nextRetry < 0 || nextRetry > 1<<62 { + // 如果计算出异常时间,使用当前时间+超时秒数 + nextRetry = now + int64(cb.Config.TimeoutParsed.Seconds()) + } + cb.Metrics.NextRetryTime.Store(nextRetry) case StateClosed: cb.Metrics.ClosedCount.Add(1) cb.Metrics.LastCloseTime.Store(now) @@ -658,8 +679,10 @@ func (cb *CircuitBreakerInfo) updateWindowStats(isSuccess bool, ctx context.Cont // 如果超过窗口大小,重置统计 if now-windowStart >= windowSize { - // 使用原子操作重置窗口 + // 使用原子操作重置窗口(只有一个goroutine会成功) if cb.Metrics.WindowStartTime.CompareAndSwap(windowStart, now) { + // CAS成功的goroutine负责重置计数器 + // 注意:可能有一些请求的累加在重置之前完成,但不会丢失很多数据 cb.Metrics.WindowRequests.Store(0) cb.Metrics.WindowFailures.Store(0) } @@ -938,15 +961,37 @@ func generateResourceName(r *ghttp.Request) string { path := r.URL.Path query := r.URL.Query().Encode() + // 安全限制:防止资源名过长导致内存或存储问题 + const maxResourceNameLength = 512 + // 生成资源名:方法:路径?查询参数 // 示例: GET:/api/users?userId=123 resourceName := method + ":" + path + + // 限制路径长度 + if len(resourceName) > maxResourceNameLength/2 { + // 截断路径,保留头部以便识别 + resourceName = resourceName[:maxResourceNameLength/2] + "..." + } + if query != "" { // 对查询参数进行排序以确保相同的参数顺序生成相同的资源名 sortedQuery := sortQueryString(query) + + // 限制查询参数长度 + maxQueryLength := maxResourceNameLength - len(resourceName) - 1 + if len(sortedQuery) > maxQueryLength { + // 截断查询参数 + sortedQuery = sortedQuery[:maxQueryLength] + "..." + } resourceName += "?" + sortedQuery } + // 最终长度检查 + if len(resourceName) > maxResourceNameLength { + resourceName = resourceName[:maxResourceNameLength] + } + return resourceName } @@ -961,18 +1006,42 @@ func sortQueryString(query string) string { return query } - // 简单的字符串排序 - for i := 0; i < len(params)-1; i++ { - for j := i + 1; j < len(params); j++ { - if params[i] > params[j] { - params[i], params[j] = params[j], params[i] - } - } + // 使用快速排序替代冒泡排序(O(n log n) vs O(n²)) + // 限制最大参数数量,防止DoS攻击 + const maxParams = 100 + if len(params) > maxParams { + params = params[:maxParams] } + // 简单的快速排序实现 + quickSortStrings(params, 0, len(params)-1) + return strings.Join(params, "&") } +// quickSortStrings 快速排序字符串切片 +func quickSortStrings(arr []string, low, high int) { + if low < high { + pivot := partitionStrings(arr, low, high) + quickSortStrings(arr, low, pivot-1) + quickSortStrings(arr, pivot+1, high) + } +} + +// partitionStrings 快速排序的分区函数 +func partitionStrings(arr []string, low, high int) int { + pivot := arr[high] + i := low - 1 + for j := low; j < high; j++ { + if arr[j] <= pivot { + i++ + arr[i], arr[j] = arr[j], arr[i] + } + } + arr[i+1], arr[high] = arr[high], arr[i+1] + return i + 1 +} + // isCircuitBreakerOpenInDistributed 检查分布式熔断状态 func isCircuitBreakerOpenInDistributed(ctx context.Context, resourceName string) bool { key := "circuit_breaker:" + resourceName + ":state" @@ -997,30 +1066,72 @@ func syncCircuitBreakerStateToDistributed(ctx context.Context, resourceName, sta return } - // 使用common/redis中的Lock方法获取分布式锁 - success, err := redis.Lock(ctx, lockKey, 10, func(ctx context.Context) error { - // 设置熔断器状态 - _, err := redisClient.Do(ctx, "SETEX", stateKey, ttl, state) - if err != nil { - g.Log().Errorf(ctx, "设置分布式熔断状态失败: %s=%s, error: %v", stateKey, state, err) - } else { - g.Log().Debugf(ctx, "分布式熔断状态已同步: %s=%s (TTL: %d)", stateKey, state, ttl) + // 使用更短的锁超时时间(3秒),避免死锁风险 + // 同时添加重试机制,确保最终一致性 + lockTimeout := int64(3) + maxRetries := 2 + var lastErr error + + for attempt := 0; attempt <= maxRetries; attempt++ { + if attempt > 0 { + // 短暂延迟后重试 + time.Sleep(time.Duration(attempt*50) * time.Millisecond) } - return nil - }) - if err != nil { - g.Log().Errorf(ctx, "获取分布式锁失败: %s, error: %v", lockKey, err) - return + // 使用common/redis中的Lock方法获取分布式锁 + success, err := redis.Lock(ctx, lockKey, lockTimeout, func(ctx context.Context) error { + // 设置熔断器状态 + _, err := redisClient.Do(ctx, "SETEX", stateKey, ttl, state) + if err != nil { + g.Log().Errorf(ctx, "设置分布式熔断状态失败: %s=%s, error: %v", stateKey, state, err) + return err + } + g.Log().Debugf(ctx, "分布式熔断状态已同步: %s=%s (TTL: %d)", stateKey, state, ttl) + return nil + }) + + if err != nil { + lastErr = err + g.Log().Errorf(ctx, "获取分布式锁失败 (尝试 %d/%d): %s, error: %v", attempt+1, maxRetries+1, lockKey, err) + continue + } + + if success { + // 成功获取锁并设置状态 + return + } } - if !success { - g.Log().Debugf(ctx, "未获取到分布式锁,跳过状态同步: %s", lockKey) - } + // 所有尝试都失败 + g.Log().Warningf(ctx, "分布式熔断状态同步失败,跳过: %s, 最后错误: %v", lockKey, lastErr) } // CircuitBreakerHealthCheckHandler 健康检查接口 func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { + // 添加认证检查:使用JWT Token或API Key + // 从Header中获取认证信息 + authToken := r.Header.Get("Authorization") + if authToken == "" { + // 尝试从查询参数获取(仅用于开发环境) + authToken = r.Get("authToken").String() + } + + // 简单的Token验证(生产环境应使用更严格的认证) + // 建议使用JWT或其他安全的认证机制 + if authToken == "" { + g.Log().Warningf(r.GetCtx(), "熔断器健康检查被拒绝:缺少认证信息,IP=%s", r.GetClientIp()) + r.Response.WriteStatusExit(401, "Unauthorized: Missing authentication token") + return + } + + // TODO: 在这里添加真正的Token验证逻辑 + // 示例:使用JWT验证 + // claims, err := jwt.ParseWithClaims(authToken, &MyClaims{}) + // if err != nil { + // r.Response.WriteStatusExit(401, "Unauthorized: Invalid token") + // return + // } + page := r.Get("page").Int() size := r.Get("size").Int() if page < 0 { @@ -1126,6 +1237,20 @@ func batchProcessResources(r *ghttp.Request, processFunc func(resourceName strin // CircuitBreakerResetHandler 重置熔断器 func CircuitBreakerResetHandler(r *ghttp.Request) { + // 添加认证检查(与健康检查接口相同) + authToken := r.Header.Get("Authorization") + if authToken == "" { + authToken = r.Get("authToken").String() + } + + if authToken == "" { + g.Log().Warningf(r.GetCtx(), "熔断器重置被拒绝:缺少认证信息,IP=%s", r.GetClientIp()) + r.Response.WriteStatusExit(401, "Unauthorized: Missing authentication token") + return + } + + // TODO: 添加真正的Token验证逻辑 + resourceName := r.Get("resource").String() if resourceName == "" || resourceName == "*" { @@ -1161,7 +1286,12 @@ func resetSingleResource(r *ghttp.Request, resourceName string) error { cbInfo.State.Store(stateClosed) // 重置指标 cbInfo.Metrics.reset() - cbInfo.WarmupEndTime = time.Now().Add(config.WarmupDurationParsed).Unix() + warmupEndTime := time.Now().Add(config.WarmupDurationParsed).Unix() + // 防护:检查时间戳是否有效 + if warmupEndTime < 0 || warmupEndTime > 1<<62 { + warmupEndTime = time.Now().Unix() + int64(config.WarmupDurationParsed.Seconds()) + } + cbInfo.WarmupEndTime = warmupEndTime cbInfo.Metrics.LastResetTime.Store(time.Now().Unix()) // 清除分布式状态 @@ -1169,7 +1299,8 @@ func resetSingleResource(r *ghttp.Request, resourceName string) error { redisClient := g.Redis() if redisClient != nil { lockKey := "circuit_breaker:" + resourceName + ":lock" - success, err := redis.Lock(r.GetCtx(), lockKey, 10, func(ctx context.Context) error { + // 使用较短的锁超时时间 + success, err := redis.Lock(r.GetCtx(), lockKey, int64(3), func(ctx context.Context) error { _, err := redisClient.Del(ctx, "circuit_breaker:"+resourceName+":state") if err != nil { g.Log().Warningf(ctx, "清除分布式熔断状态失败: %s, error: %v", resourceName, err) From 4f4816e093417670494d3d1fe393c4314037efca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 5 Jan 2026 16:28:29 +0800 Subject: [PATCH 147/225] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=99=90=E6=B5=81?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/circuit_breaker.go | 83 +++++++++++++++++++++++++---------- middleware/middleware.go | 44 +++++++++++++++++++ redis/keys.go | 14 +++--- 3 files changed, 108 insertions(+), 33 deletions(-) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 84aa185..82af81a 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -209,6 +209,8 @@ func (m *HalfOpenManager) checkHalfOpenSuccessThreshold(metrics HalfOpenMetrics, return false } + // 使用浮点数除法计算成功率,避免整数除法精度丢失问题 + // 例如: passedRequests=1, failedRequests=2, 则 successRate = 0.333... 而不是 0 successRate := float64(passedRequests) / float64(totalRequests) return successRate >= successThreshold } @@ -1106,32 +1108,74 @@ func syncCircuitBreakerStateToDistributed(ctx context.Context, resourceName, sta g.Log().Warningf(ctx, "分布式熔断状态同步失败,跳过: %s, 最后错误: %v", lockKey, lastErr) } -// CircuitBreakerHealthCheckHandler 健康检查接口 -func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { - // 添加认证检查:使用JWT Token或API Key +// checkCircuitBreakerAuthToken 验证熔断器管理接口的认证Token +// 统一的认证逻辑,用于健康检查、重置等管理接口 +// 返回值: +// - true: 认证通过 +// - false: 认证失败,已发送401响应 +func checkCircuitBreakerAuthToken(r *ghttp.Request) bool { // 从Header中获取认证信息 authToken := r.Header.Get("Authorization") + + // 如果Header中没有,尝试从查询参数获取(仅用于开发/测试环境) + // 生产环境应禁用此方式,仅支持Header认证 if authToken == "" { - // 尝试从查询参数获取(仅用于开发环境) authToken = r.Get("authToken").String() } - // 简单的Token验证(生产环境应使用更严格的认证) - // 建议使用JWT或其他安全的认证机制 + // 检查Token是否为空 if authToken == "" { - g.Log().Warningf(r.GetCtx(), "熔断器健康检查被拒绝:缺少认证信息,IP=%s", r.GetClientIp()) + g.Log().Warningf(r.GetCtx(), "熔断器管理接口访问被拒绝:缺少认证Token, IP=%s, Path=%s", + r.GetClientIp(), r.URL.Path) r.Response.WriteStatusExit(401, "Unauthorized: Missing authentication token") - return + return false } - // TODO: 在这里添加真正的Token验证逻辑 - // 示例:使用JWT验证 - // claims, err := jwt.ParseWithClaims(authToken, &MyClaims{}) - // if err != nil { + // 支持Bearer Token格式 + if strings.HasPrefix(authToken, "Bearer ") { + authToken = strings.TrimPrefix(authToken, "Bearer ") + } + + // TODO: 实现完整的Token验证逻辑 + // 建议使用JWT或其他安全机制: + // + // 1. 使用gogf/gf/v2/os/gjwt进行JWT验证: + // token, err := gjwt.ParseAndVerify(authToken, []byte(secret)) + // if err != nil { + // return false + // } + // + // 2. 或使用其他JWT库(如github.com/golang-jwt/jwt/v5): + // claims := &MyClaims{} + // token, err := jwt.ParseWithClaims(authToken, claims, ...) + // + // 3. 验证Token的: + // - 签名有效性 + // - 过期时间(exp) + // - 签发者(iss) + // - 权限范围(scope/roles) + // + // 4. 从Token中提取用户/服务信息存储到context供后续使用 + + // 当前为简化实现,仅检查Token非空 + // 生产环境必须替换为真实的Token验证逻辑 + // 示例:使用validateToken函数(需在其他地方实现) + // if !validateToken(authToken) { + // g.Log().Warningf(r.GetCtx(), "熔断器管理接口访问被拒绝:无效的认证Token, IP=%s", r.GetClientIp()) // r.Response.WriteStatusExit(401, "Unauthorized: Invalid token") - // return + // return false // } + return true +} + +// CircuitBreakerHealthCheckHandler 健康检查接口 +func CircuitBreakerHealthCheckHandler(r *ghttp.Request) { + // 认证检查:使用Token验证 + if !checkCircuitBreakerAuthToken(r) { + return + } + page := r.Get("page").Int() size := r.Get("size").Int() if page < 0 { @@ -1237,20 +1281,11 @@ func batchProcessResources(r *ghttp.Request, processFunc func(resourceName strin // CircuitBreakerResetHandler 重置熔断器 func CircuitBreakerResetHandler(r *ghttp.Request) { - // 添加认证检查(与健康检查接口相同) - authToken := r.Header.Get("Authorization") - if authToken == "" { - authToken = r.Get("authToken").String() - } - - if authToken == "" { - g.Log().Warningf(r.GetCtx(), "熔断器重置被拒绝:缺少认证信息,IP=%s", r.GetClientIp()) - r.Response.WriteStatusExit(401, "Unauthorized: Missing authentication token") + // 认证检查 + if !checkCircuitBreakerAuthToken(r) { return } - // TODO: 添加真正的Token验证逻辑 - resourceName := r.Get("resource").String() if resourceName == "" || resourceName == "*" { diff --git a/middleware/middleware.go b/middleware/middleware.go index 9178ddc..4554720 100644 --- a/middleware/middleware.go +++ b/middleware/middleware.go @@ -30,6 +30,9 @@ func Auth(r *ghttp.Request) { } // 验证 token + // TODO: 实现完整的JWT验证逻辑 + // 当前为占位实现,实际使用时应替换为真实的token验证 + // 例如:使用gogf/gf/v2/os/gjwt或其他JWT库进行验证 if !validateToken(gstr.SubStrFrom(token, "7")) { r.Response.WriteStatusExit(401, "Unauthorized") return @@ -37,3 +40,44 @@ func Auth(r *ghttp.Request) { r.Middleware.Next() } + +// validateToken 验证Token有效性 +// 当前为简化实现,实际生产环境应使用JWT或其他安全机制进行验证 +// 示例: +// - 使用gogf/gf/v2/os/gjwt库解析和验证JWT token +// - 验证token签名、过期时间、签发者等 +// - 从token中提取用户信息并存储到context +// +// 返回值: +// - true: token有效 +// - false: token无效或过期 +func validateToken(token string) bool { + // TODO: 实现真实的token验证逻辑 + // 当前为占位实现,返回true以允许基本功能运行 + // 生产环境必须替换为真实的验证逻辑 + + // 简单的非空检查 + if token == "" { + return false + } + + // 建议的JWT验证示例(需要引入jwt库): + /* + claims := &jwt.MapClaims{} + t, err := jwt.ParseWithClaims(token, claims, func(token *jwt.Token) (interface{}, error) { + return []byte("your-secret-key"), nil + }) + if err != nil || !t.Valid { + return false + } + // 检查过期时间 + if exp, ok := (*claims)["exp"].(float64); ok { + if time.Now().Unix() > int64(exp) { + return false + } + } + */ + + // 临时返回true,实际使用时应实现完整验证 + return true +} diff --git a/redis/keys.go b/redis/keys.go index d438fe6..ac62939 100644 --- a/redis/keys.go +++ b/redis/keys.go @@ -11,13 +11,9 @@ const ( // 限流 Redis Key 常量 const ( - RateLimitKeyPrefix = "ragflow:ratelimit:" // 限流Key前缀 - RateLimitKeyIP = "ip:%s" // IP限流: ip:192.168.1.1 - RateLimitKeyUser = "user:%s" // 用户限流: user:123 或 user:anon:192.168.1.1 - RateLimitKeyService = "service:%s" // 服务限流: service:customerService - RateLimitKeyGlobal = "global:requests" // 全局限流: global:requests - RateLimitKeyOrder = "order:create:%s" // 订单创建限流: order:create:123 - RateLimitKeyTransfer = "wallet:transfer:%s" // 钱包转账限流: wallet:transfer:123 - RateLimitKeyMessage = "cs:message:%s" // 客服消息限流: cs:message:123 - RateLimitKeyUpload = "oss:upload:%s" // 文件上传限流: oss:upload:123 + RateLimitKeyPrefix = "ragflow:ratelimit:" // 限流Key前缀 + RateLimitKeyIP = "ip:%s" // IP限流: ip:192.168.1.1 + RateLimitKeyUser = "user:%s" // 用户限流: user:123 或 user:anon:192.168.1.1 + RateLimitKeyService = "service:%s" // 服务限流: service:customerService + RateLimitKeyGlobal = "global:requests" // 全局限流: global:requests ) From a43da6e06ebf2b272743832cf4a198cbc49ecec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 6 Jan 2026 09:43:33 +0800 Subject: [PATCH 148/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=88=86=E7=B1=BB?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E5=B1=9E=E6=80=A7=EF=BC=8C=E6=96=B9=E4=BE=BF?= =?UTF-8?q?C=E7=AB=AF=E6=A3=80=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/http/http.go b/http/http.go index 1f3463d..f03e3c3 100644 --- a/http/http.go +++ b/http/http.go @@ -20,17 +20,8 @@ import ( "github.com/gogf/gf/v2/util/gconv" ) -type ResponseEmpty struct { -} - const PageSize = 20 -type Page struct { - PageNum int `p:"pageNum"` //当前页码 - PageSize int `p:"pageSize"` //每页数 - Total int //总页数 -} - var Httpserver = g.Server() var Httpclient = g.Client() From 213ad9579136dfb23ff7440c4b00c78ad9a6b556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 6 Jan 2026 17:01:10 +0800 Subject: [PATCH 149/225] =?UTF-8?q?=E4=BC=98=E5=8C=96mongo=EF=BC=8C?= =?UTF-8?q?=E5=B0=81=E8=A3=85count=E9=80=BB=E8=BE=91=EF=BC=8C=E5=A4=84?= =?UTF-8?q?=E7=90=86objectId?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- beans/beans.go | 17 +++- http/http.go | 2 - log/controller/log_controller.go | 9 +- log/dao/log_dao.go | 2 +- log/model/dto/log_dto.go | 14 ++-- log/service/log_service.go | 4 +- mongo/mongo.go | 139 ++++++++++++++++--------------- utils/utils.go | 27 ++++++ 8 files changed, 123 insertions(+), 91 deletions(-) diff --git a/beans/beans.go b/beans/beans.go index 07d8340..a86f531 100644 --- a/beans/beans.go +++ b/beans/beans.go @@ -10,11 +10,22 @@ type ResponseEmpty struct { } type Page struct { - PageNum int `p:"pageNum"` //当前页码 - PageSize int `p:"pageSize"` //每页数 - Total int //总页数 + PageNum int64 `p:"pageNum"` //当前页码 + PageSize int64 `p:"pageSize"` //每页数 + Total int64 //总页数 } +type OrderEnum string + +const ( + Asc OrderEnum = "asc" // 正序 + Desc OrderEnum = "desc" // 倒序 +) + +type OrderBy struct { + Field string `p:"field"` //排序字段 + Order OrderEnum `p:"order"` //排序方式 +} type MongoBaseDO struct { Id bson.ObjectID `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID Creator interface{} `bson:"creator,omitempty" json:"creator"` diff --git a/http/http.go b/http/http.go index f03e3c3..45be505 100644 --- a/http/http.go +++ b/http/http.go @@ -20,8 +20,6 @@ import ( "github.com/gogf/gf/v2/util/gconv" ) -const PageSize = 20 - var Httpserver = g.Server() var Httpclient = g.Client() diff --git a/log/controller/log_controller.go b/log/controller/log_controller.go index 996293d..9b0ec56 100644 --- a/log/controller/log_controller.go +++ b/log/controller/log_controller.go @@ -2,7 +2,6 @@ package controller import ( "context" - "strings" "gitee.com/red-future---jilin-g/common/log/model/dto" "gitee.com/red-future---jilin-g/common/log/service" @@ -32,13 +31,7 @@ func (c *operationLog) GetByID(ctx context.Context, req *dto.GetLogReq) (res *dt // @Summary 查询操作日志列表 // @Description 根据多个条件查询操作日志列表 func (c *operationLog) List(ctx context.Context, req *dto.ListLogsReq) (res *dto.ListLogsResp, err error) { - // 处理排序字段 - var sortFields []string - if req.SortFields != "" { - sortFields = strings.Split(req.SortFields, ",") - } - - logs, total, err := service.OperationLog.List(ctx, req, sortFields...) + logs, total, err := service.OperationLog.List(ctx, req) if err != nil { return } diff --git a/log/dao/log_dao.go b/log/dao/log_dao.go index c459117..7e21a18 100644 --- a/log/dao/log_dao.go +++ b/log/dao/log_dao.go @@ -86,7 +86,7 @@ func (d *log) List(ctx context.Context, filter *dto.ListLogsReq, sortFields ...s } var logs []*entity.OperationLog - err = mongo.DB().Find(ctx, bsonFilter, &logs, consts.OperationLogCollection, findOptions...) + err = mongo.DB().Find(ctx, bsonFilter, &logs, consts.OperationLogCollection, nil, nil) if err != nil { return nil, 0, err } diff --git a/log/model/dto/log_dto.go b/log/model/dto/log_dto.go index 81e8b29..862e91c 100644 --- a/log/model/dto/log_dto.go +++ b/log/model/dto/log_dto.go @@ -34,13 +34,13 @@ type OperationLogInfo struct { type ListLogsReq struct { g.Meta `path:"/listLogs" method:"get" tags:"操作日志" summary:"查询操作日志列表" dc:"根据多个条件查询操作日志列表"` beans.Page - ServiceName string `json:"service_name" dc:"服务名(可选)"` - Collection string `json:"collection" dc:"数据所在集合名称(可选)"` - CollectionID string `json:"collection_id" dc:"数据ID(可选)"` - Operation string `json:"operation" dc:"操作类型(可选)"` - StartTime string `json:"start_time" dc:"开始时间(可选)"` - EndTime string `json:"end_time" dc:"结束时间(可选)"` - SortFields string `json:"sort_fields" dc:"排序字段,多个用逗号分隔,如:-createdAt"` + ServiceName string `json:"service_name" dc:"服务名(可选)"` + Collection string `json:"collection" dc:"数据所在集合名称(可选)"` + CollectionID string `json:"collection_id" dc:"数据ID(可选)"` + Operation string `json:"operation" dc:"操作类型(可选)"` + StartTime string `json:"start_time" dc:"开始时间(可选)"` + EndTime string `json:"end_time" dc:"结束时间(可选)"` + OrderBy []beans.OrderBy `json:"orderBy" dc:"排序字段"` } // ListLogsResp 查询操作日志列表响应 diff --git a/log/service/log_service.go b/log/service/log_service.go index cc9ea4e..70fcac4 100644 --- a/log/service/log_service.go +++ b/log/service/log_service.go @@ -53,8 +53,8 @@ func (s *operationLog) GetByID(ctx context.Context, id string) (*dto.OperationLo } // List 查询操作日志列表 -func (s *operationLog) List(ctx context.Context, filter *dto.ListLogsReq, sortFields ...string) ([]dto.OperationLogInfo, int64, error) { - logs, total, err := dao.Log.List(ctx, filter, sortFields...) +func (s *operationLog) List(ctx context.Context, req *dto.ListLogsReq) ([]dto.OperationLogInfo, int64, error) { + logs, total, err := dao.Log.List(ctx, req) if err != nil { return nil, 0, err } diff --git a/mongo/mongo.go b/mongo/mongo.go index 3df3396..2f75dc4 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -4,10 +4,12 @@ import ( "context" "errors" "fmt" + "reflect" "strings" "sync" "time" + "gitee.com/red-future---jilin-g/common/beans" "gitee.com/red-future---jilin-g/common/log/model/dto" "github.com/gogf/gf/v2/container/gvar" @@ -219,56 +221,11 @@ func close() { isConnected = false glog.Info(context.Background(), "MongoDB连接已关闭") } -func listOptionsToMap(ctx context.Context, opts ...options.Lister[options.FindOptions]) (m map[string]interface{}) { - // 输出opts参数中的值 - m = make(map[string]interface{}) - for _, opt := range opts { - var findOpts options.FindOptions - optFuncs := opt.List() - for _, fn := range optFuncs { - fn(&findOpts) - } - if findOpts.Limit != nil { - m["limit"] = *findOpts.Limit - } - if findOpts.Skip != nil { - m["skip"] = *findOpts.Skip - } - if findOpts.Sort != nil { - m["sort"] = findOpts.Sort - } - if findOpts.Projection != nil { - m["projection"] = findOpts.Projection - } - } - m = utils.OrderMap(m) - return -} -func oneOptionsToMap(ctx context.Context, opts ...options.Lister[options.FindOneOptions]) (m map[string]interface{}) { - // 输出opts参数中的值 - m = make(map[string]interface{}) - for _, opt := range opts { - var findOpts options.FindOneOptions - optFuncs := opt.List() - for _, fn := range optFuncs { - fn(&findOpts) - } - if findOpts.Skip != nil { - m["skip"] = *findOpts.Skip - } - if findOpts.Sort != nil { - m["sort"] = findOpts.Sort - } - if findOpts.Projection != nil { - m["projection"] = findOpts.Projection - } - } - m = utils.OrderMap(m) - return -} + +const PageSize = 20 // Find 查询多条记录 -func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOptions]) (err error) { +func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, collection string, page *beans.Page, orderBy []beans.OrderBy) (err error) { if err = utils.ValidStructPtr(result); err != nil { return } @@ -277,34 +234,79 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c return } filter["isDeleted"] = false - filterMap := utils.OrderMap(filter) - optsMap := listOptionsToMap(ctx, opts...) - redisKey := fmt.Sprintf(redis.List, user.TenantId, collection, gconv.String(filterMap), gconv.String(optsMap)) + filterKey := fmt.Sprintf("%+v", filter) + optionsKey := fmt.Sprintf("%+v%+v", page, orderBy) + redisKey := fmt.Sprintf(redis.List, user.TenantId, collection, filterKey, optionsKey) + resultValue := reflect.ValueOf(result) + resultValue = resultValue.Elem() + listField := resultValue.FieldByName("List") if m.Cache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) if err != nil { return } - if !g.IsEmpty(resultStr) { - err = gconv.Scan(resultStr, result) - if err != nil { - return err - } + if !resultStr.IsEmpty() { + resultStr.Structs(listField.Addr().Interface()) return } } filter["tenantId"] = user.TenantId - cur, err := db.Collection(collection).Find(ctx, filter, opts...) + + // 分页参数处理 + limit := int64(PageSize) + skip := int64(0) + if page != nil { + limit = page.PageSize + skip = (page.PageNum - 1) * limit + if skip <= 0 { + skip = 0 + } + } + total, err := m.Count(ctx, filter, collection) + if err != nil || total == 0 { + return + } + utils.SetValue(ctx, result, "Total", total) + opt := options.Find().SetSkip(skip) + if limit != -1 { + opt.SetLimit(limit) + } + if orderBy == nil { + opt.SetSort(bson.M{"createdAt": -1}) + } else { + orderBson := bson.M{} + for _, v := range orderBy { + if v.Order == beans.Asc { + orderBson[v.Field] = 1 + } else { + orderBson[v.Field] = -1 + } + } + opt.SetSort(orderBson) + } + + cur, err := db.Collection(collection).Find(ctx, filter, opt) if err != nil { return } defer cur.Close(ctx) - if err = cur.All(ctx, result); err != nil { + + // 先解码到[]bson.M,再转换到目标类型,处理datetime到string的转换 + var docs []bson.M + if err = cur.All(ctx, &docs); err != nil { + return + } + for _, v := range docs { + v["id"] = v["_id"].(bson.ObjectID).Hex() + delete(v, "_id") + } + // 使用gconv转换,处理类型转换 + if err = gconv.Structs(docs, listField.Addr().Interface()); err != nil { return } if m.Cache { - err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) + err = redis.RedisClient.SetEX(ctx, redisKey, docs, int64(time.Hour)) if err != nil { return err } @@ -326,8 +328,8 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} return } filter["isDeleted"] = false - filterMap := utils.OrderMap(filter) - redisKey := fmt.Sprintf(redis.One, user.TenantId, collection, gconv.String(filterMap)) + filterKey := fmt.Sprintf("%+v", filter) + redisKey := fmt.Sprintf(redis.One, user.TenantId, collection, filterKey) if m.Cache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) @@ -383,8 +385,8 @@ func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interf } filter["isDeleted"] = false delete(filter, "tenantId") - filterMap := utils.OrderMap(filter) - oneKey := fmt.Sprintf(redis.One, tenantId, collection, gconv.String(filterMap)) + filterKey := fmt.Sprintf("%+v", filter) + oneKey := fmt.Sprintf(redis.One, tenantId, collection, filterKey) _, err = redis.RedisClient.Del(ctx, oneKey) if err != nil { return @@ -404,7 +406,7 @@ func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, } filter["tenantId"] = user.TenantId var rows []interface{} - if err = m.Find(ctx, filter, &rows, collection); err != nil { + if err = m.Find(ctx, filter, &rows, collection, nil, nil); err != nil { return } r, err := db.Collection(collection).DeleteMany(ctx, filter, opts...) @@ -446,7 +448,7 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll setDoc["updatedAt"] = gtime.Now().Time update = bson.M{"$set": setDoc} var rows []interface{} - if err = m.Find(ctx, filter, &rows, collection); err != nil { + if err = m.Find(ctx, filter, &rows, collection, nil, nil); err != nil { return } result, err = db.Collection(collection).UpdateMany(ctx, filter, update, opts...) @@ -631,7 +633,7 @@ func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collectio rows = append(rows, doc) } else { filter := bson.M{"_id": bson.M{"$in": ids}} - if err = m.Find(ctx, filter, &rows, collection); err != nil { + if err = m.Find(ctx, filter, &rows, collection, nil, nil); err != nil { return } } @@ -652,8 +654,9 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( return } filter["isDeleted"] = false - filterMap := utils.OrderMap(filter) - redisKey := fmt.Sprintf(redis.Count, user.TenantId, collection, gconv.String(filterMap)) + delete(filter, "tenantId") + filterKey := fmt.Sprintf("%+v", filter) + redisKey := fmt.Sprintf(redis.Count, user.TenantId, collection, filterKey) if m.Cache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) diff --git a/utils/utils.go b/utils/utils.go index 8d9efa4..0915ca9 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -107,7 +107,34 @@ func GetUserInfo(ctx context.Context) (user beans.User, err error) { } return } +func SetValue(ctx context.Context, result any, key string, value any) { + // 检查context是否已取消 + select { + case <-ctx.Done(): + return + default: + // 使用反射设置result的Total属性 + resultValue := reflect.ValueOf(result) + if resultValue.Kind() == reflect.Ptr { + resultValue = resultValue.Elem() + totalField := resultValue.FieldByName(key) + if totalField.IsValid() && totalField.CanSet() { + totalField.Set(reflect.ValueOf(value)) + } + } + return + } +} +func Struts(ctx context.Context, pointer any, mapping ...map[string]string) { + // 检查context是否已取消 + select { + case <-ctx.Done(): + return + default: + return + } +} func OrderMap(m map[string]interface{}) map[string]interface{} { // 提取所有key keys := make([]string, 0, len(m)) From 40c5d5de79cea56c2cca3206f163296dd6fe95b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 6 Jan 2026 17:41:47 +0800 Subject: [PATCH 150/225] =?UTF-8?q?=E4=BC=98=E5=8C=96mongo=EF=BC=8C?= =?UTF-8?q?=E5=B0=81=E8=A3=85count=E9=80=BB=E8=BE=91=EF=BC=8C=E5=A4=84?= =?UTF-8?q?=E7=90=86objectId?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mongo/mongo.go b/mongo/mongo.go index 2f75dc4..fcef51c 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -238,6 +238,9 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c optionsKey := fmt.Sprintf("%+v%+v", page, orderBy) redisKey := fmt.Sprintf(redis.List, user.TenantId, collection, filterKey, optionsKey) resultValue := reflect.ValueOf(result) + if resultValue.IsNil() || resultValue.IsZero() { + return errors.New("result不能为空") + } resultValue = resultValue.Elem() listField := resultValue.FieldByName("List") if m.Cache { From a1f9e627e4312badf2ae21129e693a5e631d68bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 7 Jan 2026 16:19:28 +0800 Subject: [PATCH 151/225] =?UTF-8?q?=E4=BC=98=E5=8C=96mongo=EF=BC=8C?= =?UTF-8?q?=E5=B0=81=E8=A3=85count=E9=80=BB=E8=BE=91=EF=BC=8C=E5=A4=84?= =?UTF-8?q?=E7=90=86objectId?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- beans/beans.go | 18 +++++------ log/dao/log_dao.go | 2 +- mongo/mongo.go | 79 ++++++++++++++++++++++------------------------ 3 files changed, 47 insertions(+), 52 deletions(-) diff --git a/beans/beans.go b/beans/beans.go index a86f531..5c65a77 100644 --- a/beans/beans.go +++ b/beans/beans.go @@ -27,16 +27,16 @@ type OrderBy struct { Order OrderEnum `p:"order"` //排序方式 } type MongoBaseDO struct { - Id bson.ObjectID `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID - Creator interface{} `bson:"creator,omitempty" json:"creator"` - CreatedAt time.Time `bson:"created_at,omitempty" json:"createdAt"` - Updater interface{} `bson:"updater,omitempty" json:"updater"` - UpdatedAt time.Time `bson:"updated_at,omitempty" json:"updatedAt"` - TenantId interface{} `bson:"tenant_id" json:"tenantId" default:"1"` // 租户ID - IsDeleted bool `bson:"is_deleted" json:"isDeleted" default:"false"` + Id *bson.ObjectID `bson:"_id,omitempty" json:"id"` // MongoDB 默认 ID + Creator interface{} `bson:"creator,omitempty" json:"creator"` + CreatedAt *time.Time `bson:"createdAt,omitempty" json:"createdAt"` + Updater interface{} `bson:"updater,omitempty" json:"updater"` + UpdatedAt *time.Time `bson:"updatedAt,omitempty" json:"updatedAt"` + TenantId interface{} `bson:"tenantId" json:"tenantId" default:"1"` // 租户ID + IsDeleted bool `bson:"isDeleted" json:"isDeleted" default:"false"` } type User struct { - UserName interface{} `bson:"user_name" json:"userName"` // MongoDB 默认 ID - TenantId interface{} `bson:"tenant_id" json:"tenantId"` // 租户ID + UserName interface{} `bson:"userName" json:"userName"` // MongoDB 默认 ID + TenantId interface{} `bson:"tenantId" json:"tenantId"` // 租户ID } diff --git a/log/dao/log_dao.go b/log/dao/log_dao.go index 7e21a18..c209a45 100644 --- a/log/dao/log_dao.go +++ b/log/dao/log_dao.go @@ -86,7 +86,7 @@ func (d *log) List(ctx context.Context, filter *dto.ListLogsReq, sortFields ...s } var logs []*entity.OperationLog - err = mongo.DB().Find(ctx, bsonFilter, &logs, consts.OperationLogCollection, nil, nil) + _, err = mongo.DB().Find(ctx, bsonFilter, &logs, consts.OperationLogCollection, nil, nil) if err != nil { return nil, 0, err } diff --git a/mongo/mongo.go b/mongo/mongo.go index fcef51c..5e92b80 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "reflect" "strings" "sync" "time" @@ -12,6 +11,7 @@ import ( "gitee.com/red-future---jilin-g/common/beans" "gitee.com/red-future---jilin-g/common/log/model/dto" "github.com/gogf/gf/v2/container/gvar" + "github.com/gogf/gf/v2/os/grpool" "gitee.com/red-future---jilin-g/common/redis" "gitee.com/red-future---jilin-g/common/utils" @@ -174,8 +174,11 @@ func reconnect() error { return gerror.New("MongoDB重连失败,已达到最大重试次数") } +var logPool *grpool.Pool + // init 初始化MongoDB连接 func init() { + logPool = grpool.New(10) // 按需初始化:没有配置 mongo.address 则跳过 mongoAddr = g.Cfg().MustGet(context.Background(), "mongo.address").String() if mongoAddr == "" { @@ -225,10 +228,10 @@ func close() { const PageSize = 20 // Find 查询多条记录 -func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, collection string, page *beans.Page, orderBy []beans.OrderBy) (err error) { - if err = utils.ValidStructPtr(result); err != nil { - return - } +func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, collection string, page *beans.Page, orderBy []beans.OrderBy) (total int64, err error) { + //if err = utils.ValidStructPtr(result); err != nil { + // return + //} user, err := utils.GetUserInfo(ctx) if err != nil { return @@ -237,12 +240,6 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c filterKey := fmt.Sprintf("%+v", filter) optionsKey := fmt.Sprintf("%+v%+v", page, orderBy) redisKey := fmt.Sprintf(redis.List, user.TenantId, collection, filterKey, optionsKey) - resultValue := reflect.ValueOf(result) - if resultValue.IsNil() || resultValue.IsZero() { - return errors.New("result不能为空") - } - resultValue = resultValue.Elem() - listField := resultValue.FieldByName("List") if m.Cache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) @@ -250,7 +247,10 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c return } if !resultStr.IsEmpty() { - resultStr.Structs(listField.Addr().Interface()) + if err = resultStr.Structs(result); err != nil { + return + } + total = int64(len(resultStr.Array())) return } } @@ -261,19 +261,20 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c skip := int64(0) if page != nil { limit = page.PageSize - skip = (page.PageNum - 1) * limit - if skip <= 0 { + if limit == -1 { skip = 0 + } else { + skip = (page.PageNum - 1) * limit } } - total, err := m.Count(ctx, filter, collection) - if err != nil || total == 0 { - return - } - utils.SetValue(ctx, result, "Total", total) opt := options.Find().SetSkip(skip) if limit != -1 { opt.SetLimit(limit) + } else { + total, err = m.Count(ctx, filter, collection) + if err != nil || total == 0 { + return + } } if orderBy == nil { opt.SetSort(bson.M{"createdAt": -1}) @@ -293,25 +294,17 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c if err != nil { return } + if limit == -1 { + total = int64(cur.RemainingBatchLength()) + } defer cur.Close(ctx) - - // 先解码到[]bson.M,再转换到目标类型,处理datetime到string的转换 - var docs []bson.M - if err = cur.All(ctx, &docs); err != nil { - return - } - for _, v := range docs { - v["id"] = v["_id"].(bson.ObjectID).Hex() - delete(v, "_id") - } - // 使用gconv转换,处理类型转换 - if err = gconv.Structs(docs, listField.Addr().Interface()); err != nil { + if err = cur.All(ctx, result); err != nil { return } if m.Cache { - err = redis.RedisClient.SetEX(ctx, redisKey, docs, int64(time.Hour)) + err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) if err != nil { - return err + return } } return @@ -408,16 +401,17 @@ func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, return } filter["tenantId"] = user.TenantId - var rows []interface{} - if err = m.Find(ctx, filter, &rows, collection, nil, nil); err != nil { - return - } r, err := db.Collection(collection).DeleteMany(ctx, filter, opts...) if err != nil { return } count = r.DeletedCount err = m.CleanRedis(ctx, filter, user.TenantId, collection) + //写日志 + var rows []interface{} + if _, err = m.Find(ctx, filter, &rows, collection, nil, nil); err != nil { + return + } serverName := g.Cfg().MustGet(ctx, "server.name").String() logRedisKey := fmt.Sprintf("log:%s", serverName) if _, err = redis.AddToStream(ctx, logRedisKey, &dto.RecordCreateLogReq{ @@ -450,17 +444,18 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll } setDoc["updatedAt"] = gtime.Now().Time update = bson.M{"$set": setDoc} - var rows []interface{} - if err = m.Find(ctx, filter, &rows, collection, nil, nil); err != nil { - return - } result, err = db.Collection(collection).UpdateMany(ctx, filter, update, opts...) if err != nil { return } err = m.CleanRedis(ctx, filter, user.TenantId, collection) + //写日志 serverName := g.Cfg().MustGet(ctx, "server.name").String() logRedisKey := fmt.Sprintf("log:%s", serverName) + var rows []interface{} + if _, err = m.Find(ctx, filter, &rows, collection, nil, nil); err != nil { + return + } if _, err = redis.AddToStream(ctx, logRedisKey, &dto.RecordCreateLogReq{ ServiceName: serverName, Collection: collection, @@ -636,7 +631,7 @@ func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collectio rows = append(rows, doc) } else { filter := bson.M{"_id": bson.M{"$in": ids}} - if err = m.Find(ctx, filter, &rows, collection, nil, nil); err != nil { + if _, err = m.Find(ctx, filter, &rows, collection, nil, nil); err != nil { return } } From 07b4240226ca1b075b3a0a10501ae7771ffee6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 7 Jan 2026 17:44:15 +0800 Subject: [PATCH 152/225] =?UTF-8?q?=E4=BC=98=E5=8C=96mongo=EF=BC=8C?= =?UTF-8?q?=E5=B0=81=E8=A3=85count=E9=80=BB=E8=BE=91=EF=BC=8C=E5=A4=84?= =?UTF-8?q?=E7=90=86objectId?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 5e92b80..3f2141a 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -229,9 +229,9 @@ const PageSize = 20 // Find 查询多条记录 func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, collection string, page *beans.Page, orderBy []beans.OrderBy) (total int64, err error) { - //if err = utils.ValidStructPtr(result); err != nil { - // return - //} + if err = utils.ValidStructPtr(result); err != nil { + return + } user, err := utils.GetUserInfo(ctx) if err != nil { return From b5697d72fff1e3c2ad0cfac537e3904cdb455946 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 7 Jan 2026 18:27:43 +0800 Subject: [PATCH 153/225] =?UTF-8?q?=E4=BC=98=E5=8C=96mongo=EF=BC=8C?= =?UTF-8?q?=E5=B0=81=E8=A3=85count=E9=80=BB=E8=BE=91=EF=BC=8C=E5=A4=84?= =?UTF-8?q?=E7=90=86objectId?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- log/model/dto/log_dto.go | 9 ----- log/model/entity/log.go | 13 +++--- log/service/log_service.go | 13 ------ mongo/mongo.go | 83 +++++++++++++++----------------------- 4 files changed, 38 insertions(+), 80 deletions(-) diff --git a/log/model/dto/log_dto.go b/log/model/dto/log_dto.go index 862e91c..fab47af 100644 --- a/log/model/dto/log_dto.go +++ b/log/model/dto/log_dto.go @@ -48,12 +48,3 @@ type ListLogsResp struct { Logs []OperationLogInfo `json:"logs" dc:"日志列表"` Total int64 `json:"total" dc:"总数"` } - -// ========== 记录操作日志DTO ========== - -// RecordCreateLogReq 记录创建操作日志请求 -type RecordCreateLogReq struct { - ServiceName string `json:"service_name" v:"required" dc:"服务名"` - Collection string `json:"collection" v:"required" dc:"数据所在集合名称"` - Data []interface{} `json:"data" dc:"当前数据"` -} diff --git a/log/model/entity/log.go b/log/model/entity/log.go index ca59c9b..ac566a4 100644 --- a/log/model/entity/log.go +++ b/log/model/entity/log.go @@ -8,11 +8,10 @@ import ( type OperationLog struct { beans.MongoBaseDO `bson:",inline"` - ServiceName string `bson:"service_name" json:"service_name"` // 服务名:具体的微服务名称 - Collection string `bson:"collection" json:"collection"` // 集合名:数据所在的集合名称 - CollectionID string `bson:"collection_id" json:"collection_id"` // 数据ID:具体操作的数据ID,如订单号、钱包ID等 - Operation string `bson:"operation" json:"operation"` // 操作类型:create, update, delete - UserName string `bson:"user_name" json:"user_name"` // 操作人名称 - IPAddress string `bson:"ip_address" json:"ip_address"` // 操作IP地址 - Data map[string]interface{} `bson:"data,omitempty" json:"data"` // 当前数据:操作时的数据状态 + ServiceName string `bson:"service_name" json:"service_name"` // 服务名:具体的微服务名称 + Collection string `bson:"collection" json:"collection"` // 集合名:数据所在的集合名称 + CollectionID string `bson:"collection_id" json:"collection_id"` // 数据ID:具体操作的数据ID,如订单号、钱包ID等 + Operation string `bson:"operation" json:"operation"` // 操作类型:create, update, delete + IPAddress string `bson:"ip_address" json:"ip_address"` // 操作IP地址 + Data interface{} `bson:"data,omitempty" json:"data"` // 当前数据:操作时的数据状态 } diff --git a/log/service/log_service.go b/log/service/log_service.go index 70fcac4..73a92d6 100644 --- a/log/service/log_service.go +++ b/log/service/log_service.go @@ -7,7 +7,6 @@ import ( "gitee.com/red-future---jilin-g/common/log/dao" "gitee.com/red-future---jilin-g/common/log/model/dto" logEntity "gitee.com/red-future---jilin-g/common/log/model/entity" - "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/util/gconv" ) @@ -75,26 +74,14 @@ func (s *operationLog) List(ctx context.Context, req *dto.ListLogsReq) ([]dto.Op // record 记录操作日志的通用方法 func (s *operationLog) record(ctx context.Context, serviceName, collection, collectionID, operation string, data map[string]interface{}) error { - // 获取用户信息 - user, err := utils.GetUserInfo(ctx) - if err != nil { - return err - } - // 获取请求信息 ipAddress := getHTTPRequestInfo(ctx) - var userName string - if user.UserName != nil { - userName = gconv.String(user.UserName) - } - log := &logEntity.OperationLog{ ServiceName: serviceName, Collection: collection, CollectionID: collectionID, Operation: operation, - UserName: userName, IPAddress: ipAddress, Data: data, } diff --git a/mongo/mongo.go b/mongo/mongo.go index 3f2141a..773bd10 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -9,7 +9,7 @@ import ( "time" "gitee.com/red-future---jilin-g/common/beans" - "gitee.com/red-future---jilin-g/common/log/model/dto" + "gitee.com/red-future---jilin-g/common/log/model/entity" "github.com/gogf/gf/v2/container/gvar" "github.com/gogf/gf/v2/os/grpool" @@ -178,7 +178,7 @@ var logPool *grpool.Pool // init 初始化MongoDB连接 func init() { - logPool = grpool.New(10) + logPool = grpool.New(1) // 按需初始化:没有配置 mongo.address 则跳过 mongoAddr = g.Cfg().MustGet(context.Background(), "mongo.address").String() if mongoAddr == "" { @@ -390,6 +390,33 @@ func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interf return } +var serverName = g.Cfg().MustGet(context.TODO(), "server.name").String() +var logRedisKey = fmt.Sprintf("log:%s", serverName) + +func (m *MongoDB) log(ctx context.Context, filter bson.M, collection string, data interface{}, userName, tenantId interface{}, operationType string) { + _ = logPool.AddWithRecover(ctx, func(ctx context.Context) { + log := &entity.OperationLog{ + ServiceName: serverName, + Collection: collection, + CollectionID: filter["_id"].(string), + Operation: operationType, + IPAddress: g.RequestFromCtx(ctx).GetClientIp(), + Data: data, + } + log.Creator = userName + now := >ime.Now().Time + log.CreatedAt = now + log.UpdatedAt = now + log.TenantId = tenantId + if _, err := redis.AddToStream(ctx, logRedisKey, log); err != nil { + glog.Error(ctx, "mongoLog-AddToStream err: %v", err) + } + }, func(ctx context.Context, exception error) { + glog.Error(ctx, "mongoLog-AddWithRecover err: %v", exception) + }) + return +} + // Delete 删除记录 func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.DeleteManyOptions]) (count int64, err error) { if len(filter) == 0 { @@ -408,19 +435,7 @@ func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, count = r.DeletedCount err = m.CleanRedis(ctx, filter, user.TenantId, collection) //写日志 - var rows []interface{} - if _, err = m.Find(ctx, filter, &rows, collection, nil, nil); err != nil { - return - } - serverName := g.Cfg().MustGet(ctx, "server.name").String() - logRedisKey := fmt.Sprintf("log:%s", serverName) - if _, err = redis.AddToStream(ctx, logRedisKey, &dto.RecordCreateLogReq{ - ServiceName: serverName, - Collection: collection, - Data: rows, - }); err != nil { - glog.Error(ctx, "mongoLog-AddToStream err: %v", err) - } + m.log(ctx, filter, collection, nil, user.UserName, user.TenantId, "delete") return } @@ -450,19 +465,7 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll } err = m.CleanRedis(ctx, filter, user.TenantId, collection) //写日志 - serverName := g.Cfg().MustGet(ctx, "server.name").String() - logRedisKey := fmt.Sprintf("log:%s", serverName) - var rows []interface{} - if _, err = m.Find(ctx, filter, &rows, collection, nil, nil); err != nil { - return - } - if _, err = redis.AddToStream(ctx, logRedisKey, &dto.RecordCreateLogReq{ - ServiceName: serverName, - Collection: collection, - Data: rows, - }); err != nil { - glog.Error(ctx, "mongoLog-AddToStream err: %v", err) - } + m.log(ctx, filter, collection, update, user.UserName, user.TenantId, "update") return } @@ -619,29 +622,7 @@ func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collectio ids = r.InsertedIDs err = m.CleanRedis(ctx, bson.M{}, user.TenantId, collection) //写日志 - serverName := g.Cfg().MustGet(ctx, "server.name").String() - logRedisKey := fmt.Sprintf("log:%s", serverName) - if len(ids) == 0 { - return - } - rows := make([]interface{}, 0, len(ids)) - if len(ids) == 1 { - doc := gconv.Map(documents[0]) - doc["id"] = ids[0] - rows = append(rows, doc) - } else { - filter := bson.M{"_id": bson.M{"$in": ids}} - if _, err = m.Find(ctx, filter, &rows, collection, nil, nil); err != nil { - return - } - } - if _, err = redis.AddToStream(ctx, logRedisKey, &dto.RecordCreateLogReq{ - ServiceName: serverName, - Collection: collection, - Data: rows, - }); err != nil { - glog.Error(ctx, "mongoLog-AddToStream err: %v", err) - } + m.log(ctx, nil, collection, ids, user.UserName, user.TenantId, "insert") return } From 3b9f6125421192fafc87d764408d9ba6cf2d8280 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Thu, 8 Jan 2026 09:39:23 +0800 Subject: [PATCH 154/225] =?UTF-8?q?http=20doRequest=20=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E7=A0=81=E5=88=A4=E6=96=AD=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?resultStrut.Code=20=3D=3D=200?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/http/http.go b/http/http.go index 45be505..1d0edad 100644 --- a/http/http.go +++ b/http/http.go @@ -67,9 +67,9 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin defer response.Close() result := response.ReadAll() resultStrut := &ghttp.DefaultHandlerResponse{} - if gconv.Struct(result, &resultStrut); resultStrut.Code != 200 { + if err = gconv.Struct(result, &resultStrut); err != nil { err = errors.New(resultStrut.Message) - } else { + } else if resultStrut.Code == 200 || resultStrut.Code == 0 { gconv.Struct(resultStrut.Data, target) } return From 865cae5e9a88c3daf201530df4dd2bdec6b37a70 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Thu, 8 Jan 2026 15:55:44 +0800 Subject: [PATCH 155/225] =?UTF-8?q?=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/client.go | 57 ++++++++++----------------------------------- ragflow/document.go | 3 ++- 2 files changed, 14 insertions(+), 46 deletions(-) diff --git a/ragflow/client.go b/ragflow/client.go index 7dab00a..c0ee6c2 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -2,17 +2,13 @@ package ragflow import ( "context" - "net/http" "net/url" "strings" "sync" - "time" commonHttp "gitee.com/red-future---jilin-g/common/http" - "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/net/gclient" ) var ( @@ -36,9 +32,8 @@ func initClient() { } globalClient = &Client{ - BaseURL: strings.TrimSuffix(baseURL, "/"), - APIKey: apiKey, - HTTPClient: commonHttp.Httpclient, + BaseURL: strings.TrimSuffix(baseURL, "/"), + APIKey: apiKey, } g.Log().Infof(ctx, "✅ RAGFlow 客户端初始化成功: baseURL=%s", baseURL) @@ -47,14 +42,12 @@ func initClient() { // loadConfig 从配置文件加载 RAGFlow 配置 func loadConfig(ctx context.Context) (baseURL, apiKey string) { - // 使用 GoFrame 全局配置(从项目的 config.yml 读取) baseURL = g.Cfg().MustGet(ctx, "ragflow.base_url", "").String() apiKey = g.Cfg().MustGet(ctx, "ragflow.api_key", "").String() return } // GetGlobalClient 获取全局客户端(延迟初始化) -// 使用示例:client := ragflow.GetGlobalClient() func GetGlobalClient() *Client { initClient() return globalClient @@ -62,9 +55,8 @@ func GetGlobalClient() *Client { // Client RAGFlow API 客户端 type Client struct { - BaseURL string - APIKey string - HTTPClient *gclient.Client // HTTP 客户端 + BaseURL string + APIKey string } // CommonResponse 通用响应结构 @@ -79,55 +71,30 @@ func (r *CommonResponse) IsSuccess() bool { return r.Code == 0 } -// request 发送 HTTP 请求 +// request 发送 HTTP 请求(使用统一的common/http包) func (c *Client) request(ctx context.Context, method, path string, body interface{}, result interface{}) (err error) { fullURL := c.BaseURL + path - // 序列化请求体 - var reqBody string - if body != nil { - jsonData, jsonErr := gjson.Encode(body) - if jsonErr != nil { - return gerror.Newf("marshal request body failed: %v", jsonErr) - } - reqBody = string(jsonData) - } - - // 设置请求头和超时 - // 注意:使用 Chain 模式,避免修改全局 Httpclient - client := c.HTTPClient.Timeout(180 * time.Second).Header(map[string]string{ + headers := map[string]string{ "Authorization": "Bearer " + c.APIKey, "Content-Type": "application/json", - }) + } - // 发送请求 - var resp *gclient.Response switch method { case "GET": - resp, err = client.Get(ctx, fullURL) + err = commonHttp.Get(ctx, fullURL, headers, result, body) case "POST": - resp, err = client.Post(ctx, fullURL, reqBody) + err = commonHttp.Post(ctx, fullURL, headers, result, body) case "PUT": - resp, err = client.Put(ctx, fullURL, reqBody) + err = commonHttp.Put(ctx, fullURL, headers, result, body) case "DELETE": - resp, err = client.Delete(ctx, fullURL, reqBody) + err = commonHttp.Delete(ctx, fullURL, headers, result, body) default: return gerror.Newf("unsupported method: %s", method) } if err != nil { - return gerror.Newf("request failed: %v", err) - } - defer resp.Close() - - respBody := resp.ReadAll() - - if resp.StatusCode != http.StatusOK { - return gerror.Newf("http status %d: %s", resp.StatusCode, string(respBody)) - } - - if err = gjson.DecodeTo(respBody, result); err != nil { - return gerror.Newf("unmarshal response failed: %v", err) + return gerror.Newf("RAGFlow API request failed: %v", err) } return diff --git a/ragflow/document.go b/ragflow/document.go index af39527..f263c9b 100644 --- a/ragflow/document.go +++ b/ragflow/document.go @@ -9,6 +9,7 @@ import ( "mime/multipart" "strings" + commonHttp "gitee.com/red-future---jilin-g/common/http" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" ) @@ -185,7 +186,7 @@ func (c *Client) UploadDocumentFromText(ctx context.Context, datasetId, content, } // 发送请求 - client := c.HTTPClient.Clone() + client := commonHttp.Httpclient.Clone() client.SetHeader("Authorization", "Bearer "+c.APIKey) client.SetHeader("Content-Type", writer.FormDataContentType()) From 9b900c1159f49f58d5ee39213835f6316dfcdadc Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Fri, 9 Jan 2026 17:57:14 +0800 Subject: [PATCH 156/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0consul=E9=83=A8?= =?UTF-8?q?=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/client.go | 77 ++++++++++++++++++++++++++++++++++++--------- ragflow/document.go | 8 +++-- redis/types.go | 18 ++++++----- 3 files changed, 79 insertions(+), 24 deletions(-) diff --git a/ragflow/client.go b/ragflow/client.go index c0ee6c2..d5958bf 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -5,6 +5,7 @@ import ( "net/url" "strings" "sync" + "sync/atomic" commonHttp "gitee.com/red-future---jilin-g/common/http" "github.com/gogf/gf/v2/errors/gerror" @@ -23,27 +24,57 @@ func initClient() { ctx := context.Background() // 读取配置 - baseURL, apiKey := loadConfig(ctx) + endpoints, apiKey := loadConfig(ctx) // 如果配置不完整,跳过初始化 - if baseURL == "" || apiKey == "" { - g.Log().Warning(ctx, "⚠️ RAGFlow 配置未找到,请在项目 config.yml 中添加 ragflow.base_url 和 ragflow.api_key") + if len(endpoints) == 0 || apiKey == "" { + g.Log().Warning(ctx, "⚠️ RAGFlow 配置未找到,请在 config.yml 中添加 ragflow.base_url 或在 Consul 中配置 ragflow.endpoints") return } globalClient = &Client{ - BaseURL: strings.TrimSuffix(baseURL, "/"), - APIKey: apiKey, + Endpoints: endpoints, + APIKey: apiKey, } - g.Log().Infof(ctx, "✅ RAGFlow 客户端初始化成功: baseURL=%s", baseURL) + if len(endpoints) == 1 { + g.Log().Infof(ctx, "✅ RAGFlow 客户端初始化成功: endpoint=%s", endpoints[0]) + } else { + g.Log().Infof(ctx, "✅ RAGFlow 客户端初始化成功: endpoints=%v (负载均衡)", endpoints) + } }) } -// loadConfig 从配置文件加载 RAGFlow 配置 -func loadConfig(ctx context.Context) (baseURL, apiKey string) { - baseURL = g.Cfg().MustGet(ctx, "ragflow.base_url", "").String() +// loadConfig 从配置加载 RAGFlow 配置(支持实例级配置) +// 优先级: +// 1. Consul实例级配置 ragflow.endpoints (数组) +// 2. Consul全局配置 ragflow.endpoints (数组) +// 3. config.yml的 ragflow.base_url (单个URL,向后兼容) +func loadConfig(ctx context.Context) (endpoints []string, apiKey string) { + // 尝试从Consul读取endpoints(支持实例级配置) + // 注意:这里不能直接导入customerService/service包,会造成循环依赖 + // 所以只能从config.yml读取,Consul配置需要在customerservice层面调用时传入 + + // 读取API Key apiKey = g.Cfg().MustGet(ctx, "ragflow.api_key", "").String() + + // 尝试读取endpoints数组(从config.yml或Consul同步的配置) + endpointsConfig := g.Cfg().MustGet(ctx, "ragflow.endpoints") + if !endpointsConfig.IsEmpty() { + endpoints = endpointsConfig.Strings() + // 去除尾部斜杠 + for i := range endpoints { + endpoints[i] = strings.TrimSuffix(endpoints[i], "/") + } + return + } + + // Fallback到单个base_url(向后兼容) + baseURL := g.Cfg().MustGet(ctx, "ragflow.base_url", "").String() + if baseURL != "" { + endpoints = []string{strings.TrimSuffix(baseURL, "/")} + } + return } @@ -53,10 +84,24 @@ func GetGlobalClient() *Client { return globalClient } -// Client RAGFlow API 客户端 +// Client RAGFlow API 客户端(支持负载均衡) type Client struct { - BaseURL string - APIKey string + Endpoints []string // RAGFlow实例列表 + APIKey string // API密钥 + currentIndex atomic.Uint64 // 当前轮询索引(原子操作) +} + +// getNextEndpoint 获取下一个endpoint(轮询算法) +func (c *Client) getNextEndpoint() string { + if len(c.Endpoints) == 0 { + return "" + } + if len(c.Endpoints) == 1 { + return c.Endpoints[0] + } + // 原子递增并取模,实现轮询 + idx := c.currentIndex.Add(1) % uint64(len(c.Endpoints)) + return c.Endpoints[idx] } // CommonResponse 通用响应结构 @@ -71,9 +116,13 @@ func (r *CommonResponse) IsSuccess() bool { return r.Code == 0 } -// request 发送 HTTP 请求(使用统一的common/http包) +// request 发送 HTTP 请求(使用统一的common/http包,支持负载均衡) func (c *Client) request(ctx context.Context, method, path string, body interface{}, result interface{}) (err error) { - fullURL := c.BaseURL + path + endpoint := c.getNextEndpoint() + if endpoint == "" { + return gerror.New("RAGFlow endpoints not configured") + } + fullURL := endpoint + path headers := map[string]string{ "Authorization": "Bearer " + c.APIKey, diff --git a/ragflow/document.go b/ragflow/document.go index f263c9b..c6e39e0 100644 --- a/ragflow/document.go +++ b/ragflow/document.go @@ -162,8 +162,12 @@ func (c *Client) UploadDocumentFromText(ctx context.Context, datasetId, content, filename = "document.txt" } - // 构造URL - url := c.BaseURL + "/api/v1/datasets/" + datasetId + "/documents" + // 构造URL(使用负载均衡) + endpoint := c.getNextEndpoint() + if endpoint == "" { + return "", gerror.New("RAGFlow endpoints not configured") + } + url := endpoint + "/api/v1/datasets/" + datasetId + "/documents" // 创建multipart writer body := &bytes.Buffer{} diff --git a/redis/types.go b/redis/types.go index f76d97d..2ccfbde 100644 --- a/redis/types.go +++ b/redis/types.go @@ -38,14 +38,16 @@ type BatchStreamMessage struct { // ResponseStreamMessage RAGFlow 响应消息结构(MQ 消息) type ResponseStreamMessage struct { - UserId string `json:"userId"` // 用户ID - Platform string `json:"platform"` // 平台标识 - TenantId string `json:"tenantId"` // 租户ID - Question string `json:"question"` // 用户问题 - Content string `json:"content"` // RAGFlow 回复内容 - SessionId string `json:"sessionId"` // RAGFlow Session ID - Timestamp int64 `json:"timestamp"` // 时间戳(秒) - MessageId string `json:"messageId"` // 原始消息ID + UserId string `json:"userId"` // 用户ID + Platform string `json:"platform"` // 平台标识 + TenantId string `json:"tenantId"` // 租户ID + AccountId string `json:"accountId,omitempty"` // 账号ID + AccountName string `json:"accountName,omitempty"` // 客服账号名称 + Question string `json:"question"` // 用户问题 + Content string `json:"content"` // RAGFlow 回复内容 + SessionId string `json:"sessionId"` // RAGFlow Session ID + Timestamp int64 `json:"timestamp"` // 时间戳(秒) + MessageId string `json:"messageId"` // 原始消息ID } // FollowUpMessage 追问消息结构(RabbitMQ 延时队列) From 91dd762cc65d2bbb53acf88a1f04c849e9fd5141 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Thu, 8 Jan 2026 19:09:47 +0800 Subject: [PATCH 157/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E7=A7=9F=E6=88=B7=E6=A8=A1=E5=9D=97=E5=BC=80=E9=80=9A=E7=8A=B6?= =?UTF-8?q?=E6=80=81=E7=9A=84=E4=B8=AD=E9=97=B4=E4=BB=B6=E3=80=82=E4=BF=AE?= =?UTF-8?q?=E6=94=B9mongo=E6=9F=A5=E8=AF=A2list=E4=B8=AD=E5=A4=9A=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E6=8E=92=E5=BA=8F=E3=80=82utils=E4=B8=AD=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=E7=BB=93=E6=9E=84=E4=BD=93=E8=BD=AC=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- beans/module_tenant.go | 31 +++++++++ middleware/module_tenant_check.go | 108 ++++++++++++++++++++++++++++++ mongo/mongo.go | 6 +- utils/utils.go | 14 +++- 4 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 beans/module_tenant.go create mode 100644 middleware/module_tenant_check.go diff --git a/beans/module_tenant.go b/beans/module_tenant.go new file mode 100644 index 0000000..19df2ed --- /dev/null +++ b/beans/module_tenant.go @@ -0,0 +1,31 @@ +package beans + +import ( + "github.com/gogf/gf/v2/os/gtime" +) + +type ModuleTenantCheckReq struct { + ModuleKey string `p:"moduleKey" v:"required#模块Key不能为空"` + TenantId uint64 `p:"tenantId" v:"required#租户ID不能为空"` +} + +// ModuleTenantCheckRes 调用admin-go设置模块租户关系的响应 +type ModuleTenantCheckRes struct { + Status string `json:"status"` // 开通状态:activated(已开通)、expired(已到期)、not_activated(未开通) + Message string `json:"message"` // 状态描述 + OpenStatus bool `json:"openStatus"` // 开通状态 +} + +// ModuleTenant 模块租户关系实体(引用自admin-go) +type ModuleTenant struct { + Id uint64 `json:"id" description:""` + CreateBy uint64 `json:"createBy" description:"创建者"` + UpdateBy uint64 `json:"updateBy" description:"更新者"` + CreatedAt *gtime.Time `json:"createdAt" description:"创建时间"` + UpdatedAt *gtime.Time `json:"updatedAt" description:"更新时间"` + ModuleKey string `json:"moduleKey" description:"模块Key"` + TenantId uint64 `json:"tenantId" description:"租户ID"` + ExpireAt *gtime.Time `json:"expireAt" description:"到期时间"` + AssetId string `json:"assetId" description:"资产ID"` + AssetSkuId string `json:"assetSkuId" description:"资产SKU ID"` +} diff --git a/middleware/module_tenant_check.go b/middleware/module_tenant_check.go new file mode 100644 index 0000000..49bb307 --- /dev/null +++ b/middleware/module_tenant_check.go @@ -0,0 +1,108 @@ +package middleware + +import ( + "context" + "encoding/json" + "fmt" + "gitee.com/red-future---jilin-g/common/beans" + "gitee.com/red-future---jilin-g/common/http" + "gitee.com/red-future---jilin-g/common/message" + "gitee.com/red-future---jilin-g/common/utils" + "github.com/gogf/gf/v2/database/gredis" + "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/ghttp" + "github.com/gogf/gf/v2/os/gtime" + "github.com/gogf/gf/v2/util/gconv" + "time" +) + +type moduleTenant struct{} + +var ModuleTenant = new(moduleTenant) + +// ModuleTenantInfo 别名,引用admin-go的entity.ModuleTenant +type ModuleTenantInfo = beans.ModuleTenant + +func (s *moduleTenant) ModuleTenantCheck(r *ghttp.Request) { + getUserInfo, err := utils.GetUserInfo(r.Context()) + if err != nil { + r.Response.WriteJson(err) + r.Exit() + } + exit := gconv.Int64(time.Minute * 1) + getEX, err := message.GetRedisClientTest("test").GetEX(r.Context(), fmt.Sprintf("module_tenant:tenantId-%v", getUserInfo.TenantId), gredis.GetEXOption{ + TTLOption: gredis.TTLOption{ + EX: &exit, + }, + }) + if err != nil { + r.Response.WriteJson(err) + r.Exit() + } + // 获取模块key + moduleKey := g.Cfg().MustGet(context.Background(), "server.name") + if !g.IsEmpty(getEX.String()) { + list := make([]ModuleTenantInfo, 0) + if err = json.Unmarshal([]byte(getEX.String()), &list); err != nil { + r.Response.WriteJson(err) + r.Exit() + } + var expireAt *gtime.Time + for _, value := range list { + if value.ModuleKey == moduleKey.String() { + expireAt = value.ExpireAt + break + } + } + // 缓存中有数据,检查是否过期 + if !g.IsEmpty(expireAt) { + gt1 := gtime.New(time.Now()) + gt2 := gtime.New(expireAt) + if !gt1.Before(gt2) { + r.Response.WriteJson(gerror.New("您访问的模块已过期,请续期后再使用")) + r.Exit() + } + } else { + r.Response.WriteJson(gerror.New("您未开通此模块,请开通后再使用")) + r.Exit() + } + } else { + //将 http.Header 转换为 map[string]string + headers := make(map[string]string) + for k, v := range r.Request.Header { + if len(v) > 0 { + headers[k] = v[0] + } + } + // 缓存为空,调用admin-go的Check接口检查模块开通状态 + res, err := s.Check(r.Context(), headers, beans.ModuleTenantCheckReq{ + ModuleKey: moduleKey.String(), + TenantId: gconv.Uint64(getUserInfo.TenantId), + }) + if err != nil { + r.Response.WriteJson(err) + r.Exit() + } + // 根据检查结果判断是否允许访问 + if res.Status == "not_activated" { + r.Response.WriteJson(gerror.New("您未开通此模块,请开通后再使用")) + r.Exit() + } else if res.Status == "expired" { + r.Response.WriteJson(gerror.New("您访问的模块已过期,请续期后再使用")) + r.Exit() + } + } + r.Middleware.Next() // 继续执行后续中间件和路由处理 +} + +// Check 调用admin-go服务检查模块开通状态 +func (s *moduleTenant) Check(ctx context.Context, headers map[string]string, req beans.ModuleTenantCheckReq) (res *beans.ModuleTenantCheckRes, err error) { + if err = http.Get(ctx, "admin-go/api/v1/system/moduleTenant/check", headers, &res, + "moduleKey", req.ModuleKey, + "tenantId", req.TenantId, + ); err != nil { + return + } + return +} diff --git a/mongo/mongo.go b/mongo/mongo.go index 773bd10..59f51f9 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -279,12 +279,12 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c if orderBy == nil { opt.SetSort(bson.M{"createdAt": -1}) } else { - orderBson := bson.M{} + orderBson := bson.D{} for _, v := range orderBy { if v.Order == beans.Asc { - orderBson[v.Field] = 1 + orderBson = append(orderBson, bson.E{Key: v.Field, Value: 1}) // 1 表示升序 } else { - orderBson[v.Field] = -1 + orderBson = append(orderBson, bson.E{Key: v.Field, Value: -1}) // -1 表示降序 } } opt.SetSort(orderBson) diff --git a/utils/utils.go b/utils/utils.go index 0915ca9..c9b6f74 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -2,6 +2,7 @@ package utils import ( "context" + "encoding/json" "fmt" "net" "reflect" @@ -100,7 +101,6 @@ func GetUserInfo(ctx context.Context) (user beans.User, err error) { } else { user.TenantId = ctx.Value("tenantId") user.UserName = ctx.Value("userName") - fmt.Println("user.UserName==================", user.UserName) } if user.TenantId == nil { return user, gerror.New("租户信息为空") @@ -297,3 +297,15 @@ func HexDigit(c byte) byte { return 0xFF } } + +func Struct(params any, pointer any) error { + b, err := json.Marshal(params) + if err != nil { + return err + } + err = json.Unmarshal(b, &pointer) + if err != nil { + return err + } + return nil +} From 845fe0a324ddab24abbd6cd48588aea2280c3058 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Fri, 9 Jan 2026 10:06:50 +0800 Subject: [PATCH 158/225] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=20EntityToBsonWithFi?= =?UTF-8?q?lter=20=E5=B0=86=20*entity/entity=20=E8=BD=AC=E6=8D=A2=E4=B8=BA?= =?UTF-8?q?=20bson.M=EF=BC=8C=E5=B9=B6=E5=8F=AF=E9=80=89=E6=8B=A9=E6=98=AF?= =?UTF-8?q?=E5=90=A6=E8=BF=87=E6=BB=A4=E7=A9=BA=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 59f51f9..edc8781 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -435,7 +435,7 @@ func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, count = r.DeletedCount err = m.CleanRedis(ctx, filter, user.TenantId, collection) //写日志 - m.log(ctx, filter, collection, nil, user.UserName, user.TenantId, "delete") + //m.log(ctx, filter, collection, nil, user.UserName, user.TenantId, "delete") return } @@ -465,7 +465,7 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll } err = m.CleanRedis(ctx, filter, user.TenantId, collection) //写日志 - m.log(ctx, filter, collection, update, user.UserName, user.TenantId, "update") + //m.log(ctx, filter, collection, update, user.UserName, user.TenantId, "update") return } @@ -622,7 +622,7 @@ func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collectio ids = r.InsertedIDs err = m.CleanRedis(ctx, bson.M{}, user.TenantId, collection) //写日志 - m.log(ctx, nil, collection, ids, user.UserName, user.TenantId, "insert") + //m.log(ctx, nil, collection, ids, user.UserName, user.TenantId, "insert") return } @@ -658,9 +658,15 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( return } -// EntityToBSONM 将 *entity/entity 转换为 bson.M +// EntityToBson 将 *entity/entity 转换为 bson.M // 支持传入值类型或指针类型,返回 bson.M 和错误信息 -func EntityToBSONM(entity interface{}) (bson.M, error) { +func EntityToBson(entity interface{}) (bson.M, error) { + return EntityToBsonWithFilter(entity, false) +} + +// EntityToBsonWithFilter 将 *entity/entity 转换为 bson.M,并可选择是否过滤空值 +// filterEmpty: 为 true 时会过滤掉空值字段(nil、空字符串、空切片、空map等) +func EntityToBsonWithFilter(entity interface{}, filterEmpty bool) (bson.M, error) { // 第一步:判断入参是否为 nil 或无效类型 if entity == nil { return nil, fmt.Errorf("传入的 entity 实例为 nil") @@ -677,5 +683,13 @@ func EntityToBSONM(entity interface{}) (bson.M, error) { if err != nil { return nil, fmt.Errorf("BSON 字节流反序列化为 bson.M 失败:%w", err) } + // 如果需要过滤空值 + if filterEmpty { + for key, value := range bsonMap { + if g.IsEmpty(value) { + delete(bsonMap, key) + } + } + } return bsonMap, nil } From d44ed8bf35bb2141505671a3c319749e7089372e Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Fri, 9 Jan 2026 10:19:31 +0800 Subject: [PATCH 159/225] =?UTF-8?q?redis=20mq=20=E6=B6=88=E6=81=AF?= =?UTF-8?q?=E5=B0=81=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- message/message.go | 179 +++++++++++++++++++++++ message/rabbit.go | 351 +++++++++++++++++++++++++++++++++++++++++++++ message/redis.go | 232 ++++++++++++++++++++++++++++++ 3 files changed, 762 insertions(+) create mode 100644 message/message.go create mode 100644 message/rabbit.go create mode 100644 message/redis.go diff --git a/message/message.go b/message/message.go new file mode 100644 index 0000000..e2c72c1 --- /dev/null +++ b/message/message.go @@ -0,0 +1,179 @@ +package message + +import ( + "context" + "github.com/gogf/gf/v2/database/gredis" + + "github.com/gogf/gf/v2/errors/gerror" +) + +// GetRedisClient 获取 Redis 客户端(供外部使用) +func GetRedisClient() *gredis.Redis { + return getRedisClient() +} + +func GetRedisClientTest(name string) *gredis.Redis { + return getRedisClientTest(name) +} + +// GetLock 获取分布式锁 +func GetLock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context) error) (success bool, err error) { + return lock(ctx, key, expireSeconds, fn) +} + +// MessageConfig 消息配置接口 +type MessageConfig interface { + start(ctx context.Context) error + publish(ctx context.Context, data interface{}, options ...map[string]interface{}) (messageID string, err error) +} + +// RedisMessageConfig Redis Stream 消息配置 +type RedisMessageConfig struct { + StreamKey string // Stream 键名 + GroupName string // 消费者组名称 + ConsumerName string // 消费者名称 + BatchSize int64 // 最大并发数(信号量容量) + AutoAck bool // ACK确认,true自动确认,false手动确认 + HandleFunc func(ctx context.Context, message map[string]interface{}) error +} + +func (r *RedisMessageConfig) start(ctx context.Context) error { + return readFromStream(ctx, QueueMessage{ + StreamKey: r.StreamKey, + GroupName: r.GroupName, + ConsumerName: r.ConsumerName, + BatchSize: r.BatchSize, + AutoAck: r.AutoAck, + HandleFunc: r.HandleFunc, + }) +} + +func (r *RedisMessageConfig) publish(ctx context.Context, data interface{}, options ...map[string]interface{}) (messageID string, err error) { + return publishToRedis(ctx, r.StreamKey, data) +} + +// RabbitMQMessageConfig RabbitMQ 消息配置 +type RabbitMQMessageConfig struct { + Queue string // 队列名称 + Exchange string // 交换器名称 + RoutingKey string // 路由键 + PrefetchCount int // QoS: 预取数量(并发控制) + WorkerCount int // worker 数量 + ConsumerTag string // 消费者标签 + HandleFunc func(ctx context.Context, message map[string]interface{}) error +} + +func (r *RabbitMQMessageConfig) start(ctx context.Context) error { + return startRabbitMQConsumer(ctx, QueueMessage{ + Queue: r.Queue, + Exchange: r.Exchange, + RoutingKey: r.RoutingKey, + PrefetchCount: r.PrefetchCount, + WorkerCount: r.WorkerCount, + ConsumerTag: r.ConsumerTag, + AutoAck: true, + HandleFunc: r.HandleFunc, + }) +} + +func (r *RabbitMQMessageConfig) publish(ctx context.Context, data interface{}, options ...map[string]interface{}) (messageID string, err error) { + opts := make(map[string]interface{}) + if len(options) > 0 { + opts = options[0] + } + exchange := r.Exchange + routingKey := r.RoutingKey + delay := 0 + + if v, ok := opts["exchange"].(string); ok { + exchange = v + } + if v, ok := opts["routingKey"].(string); ok { + routingKey = v + } + if v, ok := opts["delay"].(int); ok { + delay = v + } + + if delay > 0 { + return publishDelayedToRabbitMQ(ctx, exchange, routingKey, data, delay) + } + return publishToRabbitMQ(ctx, exchange, routingKey, data) +} + +// QueueMessage 统一消息队列配置结构体(内部使用) +type QueueMessage struct { + // Redis Stream 配置 + StreamKey string + GroupName string + ConsumerName string + BatchSize int64 + AutoAck bool + HandleFunc func(ctx context.Context, message map[string]interface{}) error + + // RabbitMQ 配置 + Queue string + Exchange string + RoutingKey string + PrefetchCount int + WorkerCount int + ConsumerTag string +} + +// StartConsumers 启动消息消费者(统一入口) +// 支持同时启动多个消费者,包括 Redis Stream 和 RabbitMQ +func StartConsumers(ctx context.Context, configs ...MessageConfig) error { + for _, cfg := range configs { + if err := cfg.start(ctx); err != nil { + return gerror.Wrap(err, "启动消费者失败") + } + } + return nil +} + +// PublishMessage 发布消息(统一入口) +// 根据配置类型选择发布到 Redis Stream 或 RabbitMQ +func PublishMessage(ctx context.Context, cfg MessageConfig, data interface{}, options ...map[string]interface{}) (messageID string, err error) { + return cfg.publish(ctx, data, options...) +} + +// ========== Redis Stream 公共方法(方便迁移) ========== + +// AddToStream 将消息添加到 Redis Stream +//func AddToStream(ctx context.Context, streamKey string, msg interface{}) (messageID string, err error) { +// return addToStream(ctx, streamKey, msg) +//} + +// ReadFromStream 从 Redis Stream 读取消息(已废弃) +// 请使用 RedisMessageConfig.StartConsumers 启动消费者 +// 此方法保留用于向后兼容,但实际不会返回消息(异步消费模式) +func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName string, count, blockMs int64) ([]StreamMessage, error) { + return nil, gerror.New("ReadFromStream 已废弃,请使用 RedisMessageConfig.StartConsumers 启动消费者") +} + +// AckMessage 确认 Redis Stream 消息 +func AckMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error { + return ackMessage(ctx, streamKey, groupName, messageIDs...) +} + +// InitStreamGroup 初始化 Redis Stream 消费者组 +func InitStreamGroup(ctx context.Context, streamKey, groupName string) error { + return initStreamGroup(ctx, streamKey, groupName) +} + +// ========== RabbitMQ 公共方法(方便迁移) ========== + +// InitRabbitMQ 初始化 RabbitMQ 连接 +func InitRabbitMQ(ctx context.Context) error { + return initRabbitMQ(ctx) +} + +// PublishToRabbitMQ 发布消息到 RabbitMQ +//func PublishToRabbitMQ(ctx context.Context, exchange, routingKey string, message interface{}) error { +// return publishToRabbitMQ(ctx, exchange, routingKey, message) +//} + +// PublishDelayedToRabbitMQ 发布延时消息到 RabbitMQ +//func PublishDelayedToRabbitMQ(ctx context.Context, exchange, routingKey string, message interface{}, delaySeconds int) error { +// return publishDelayedToRabbitMQ(ctx, exchange, routingKey, message, delaySeconds) +//} diff --git a/message/rabbit.go b/message/rabbit.go new file mode 100644 index 0000000..bd6e010 --- /dev/null +++ b/message/rabbit.go @@ -0,0 +1,351 @@ +package message + +import ( + "context" + "sync" + "time" + + "github.com/gogf/gf/v2/encoding/gjson" + "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" + amqp "github.com/rabbitmq/amqp091-go" +) + +var ( + rabbitConn *amqp.Connection + rabbitChannel *amqp.Channel + rabbitOnce sync.Once + rabbitMu sync.RWMutex + rabbitCloseWatcher chan struct{} + rabbitWatcherStarted bool +) + +// Config RabbitMQ 配置 +type RabbitMQConfig struct { + Host string + Port int + Username string + Password string + VHost string +} + +// rabbitMQConfig 默认配置 +func getRabbitMQConfig() *RabbitMQConfig { + return &RabbitMQConfig{ + Host: g.Cfg().MustGet(context.Background(), "rabbitmq.host").String(), + Port: g.Cfg().MustGet(context.Background(), "rabbitmq.port").Int(), + Username: g.Cfg().MustGet(context.Background(), "rabbitmq.username").String(), + Password: g.Cfg().MustGet(context.Background(), "rabbitmq.password").String(), + VHost: g.Cfg().MustGet(context.Background(), "rabbitmq.vhost", "/").String(), + } +} + +// initRabbitMQ 初始化 RabbitMQ 连接 +func initRabbitMQ(ctx context.Context) error { + var err error + rabbitOnce.Do(func() { + cfg := getRabbitMQConfig() + url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost + + rabbitConn, err = amqp.Dial(url) + if err != nil { + g.Log().Errorf(ctx, "RabbitMQ 连接失败: %v", err) + return + } + + rabbitChannel, err = rabbitConn.Channel() + if err != nil { + g.Log().Errorf(ctx, "创建 RabbitMQ Channel 失败: %v", err) + return + } + + rabbitCloseWatcher = make(chan struct{}) + + if !rabbitWatcherStarted { + go handleRabbitMQConnectionClose(ctx) + rabbitWatcherStarted = true + } + + g.Log().Info(ctx, "RabbitMQ 连接成功") + }) + + return err +} + +// getRabbitMQChannel 获取 RabbitMQ Channel +func getRabbitMQChannel() (*amqp.Channel, error) { + rabbitMu.RLock() + defer rabbitMu.RUnlock() + + if rabbitChannel == nil || rabbitChannel.IsClosed() { + return nil, gerror.New("RabbitMQ Channel 未初始化或已关闭") + } + + return rabbitChannel, nil +} + +// getRabbitMQConnection 获取 RabbitMQ 连接 +func getRabbitMQConnection() (*amqp.Connection, error) { + rabbitMu.RLock() + defer rabbitMu.RUnlock() + + if rabbitConn == nil || rabbitConn.IsClosed() { + return nil, gerror.New("RabbitMQ 连接未初始化或已关闭") + } + + return rabbitConn, nil +} + +// handleRabbitMQConnectionClose 监听连接关闭并重连 +func handleRabbitMQConnectionClose(ctx context.Context) { + for { + select { + case <-rabbitCloseWatcher: + g.Log().Info(ctx, "停止监听 RabbitMQ 连接状态") + return + default: + } + + rabbitMu.RLock() + currentConn := rabbitConn + rabbitMu.RUnlock() + + if currentConn == nil { + return + } + + closeErr := make(chan *amqp.Error, 1) + currentConn.NotifyClose(closeErr) + + select { + case err := <-closeErr: + if err != nil { + g.Log().Errorf(ctx, "RabbitMQ 连接关闭: %v,尝试重连...", err) + reconnectRabbitMQ(ctx) + } + case <-rabbitCloseWatcher: + g.Log().Info(ctx, "停止监听 RabbitMQ 连接状态") + return + } + } +} + +// reconnectRabbitMQ 重新连接 +func reconnectRabbitMQ(ctx context.Context) { + rabbitMu.Lock() + defer rabbitMu.Unlock() + + for i := 0; i < 10; i++ { + time.Sleep(time.Duration(i+1) * time.Second) + + cfg := getRabbitMQConfig() + url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost + + var err error + rabbitConn, err = amqp.Dial(url) + if err != nil { + g.Log().Errorf(ctx, "重连失败 (尝试 %d/10): %v", i+1, err) + continue + } + + rabbitChannel, err = rabbitConn.Channel() + if err != nil { + g.Log().Errorf(ctx, "创建 Channel 失败 (尝试 %d/10): %v", i+1, err) + continue + } + + g.Log().Info(ctx, "RabbitMQ 重连成功") + return + } + + g.Log().Fatal(ctx, "RabbitMQ 重连失败,已达到最大重试次数") +} + +// startRabbitMQConsumer 启动 RabbitMQ 消费者 +func startRabbitMQConsumer(ctx context.Context, msg QueueMessage) error { + // 初始化连接 + if err := initRabbitMQ(ctx); err != nil { + return gerror.Wrap(err, "初始化 RabbitMQ 连接失败") + } + + // 创建独立 Channel(避免并发冲突) + conn, err := getRabbitMQConnection() + if err != nil { + return gerror.Wrap(err, "获取RabbitMQ连接失败") + } + + ch, err := conn.Channel() + if err != nil { + return gerror.Wrap(err, "创建独立Channel失败") + } + + // 声明队列 + _, err = ch.QueueDeclare( + msg.Queue, // name + true, // durable + false, // autoDelete + false, // exclusive + false, // noWait + nil, // arguments + ) + if err != nil { + return gerror.Newf("声明队列失败: %v", err) + } + + // 设置 QoS(并发控制) + prefetchCount := msg.PrefetchCount + if prefetchCount == 0 { + prefetchCount = 1 + } + err = ch.Qos( + prefetchCount, // prefetchCount + 0, // prefetchSize + false, // global + ) + if err != nil { + return gerror.Newf("设置 QoS 失败: %v", err) + } + + // 开始消费 + msgs, err := ch.Consume( + msg.Queue, // queue + msg.ConsumerTag, // consumer tag + msg.AutoAck, // auto-ack + false, // exclusive + false, // no-local + false, // no-wait + nil, // args + ) + if err != nil { + return gerror.Newf("开始消费失败: %v", err) + } + + workerCount := msg.WorkerCount + if workerCount == 0 { + workerCount = 1 + } + + g.Log().Infof(ctx, "RabbitMQ 消费者已启动: queue=%s, prefetch=%d, workers=%d", + msg.Queue, prefetchCount, workerCount) + + // 启动多个 worker + for i := 0; i < workerCount; i++ { + go rabbitMQWorker(ctx, i, msgs, msg) + } + + return nil +} + +// rabbitMQWorker RabbitMQ 工作协程 +func rabbitMQWorker(ctx context.Context, workerID int, msgs <-chan amqp.Delivery, msg QueueMessage) { + g.Log().Debugf(ctx, "RabbitMQ Worker %d 已启动", workerID) + + for { + select { + case <-ctx.Done(): + g.Log().Infof(ctx, "RabbitMQ Worker %d 收到停止信号,正在退出", workerID) + return + case delivery, ok := <-msgs: + if !ok { + g.Log().Infof(ctx, "RabbitMQ Worker %d 消息通道已关闭,退出", workerID) + return + } + + // 反序列化消息 + var message map[string]interface{} + if err := gjson.DecodeTo(delivery.Body, &message); err != nil { + g.Log().Errorf(ctx, "RabbitMQ Worker %d 反序列化消息失败: %v", workerID, err) + if !msg.AutoAck { + delivery.Nack(false, false) + } + continue + } + + // 处理消息 + err := msg.HandleFunc(ctx, message) + if err != nil { + g.Log().Errorf(ctx, "RabbitMQ Worker %d 处理消息失败: %v", workerID, err) + if !msg.AutoAck { + delivery.Nack(false, false) + } + } else { + if !msg.AutoAck { + delivery.Ack(false) + } + g.Log().Debugf(ctx, "RabbitMQ Worker %d 处理消息成功", workerID) + } + } + } +} + +// publishToRabbitMQ 发布消息到 RabbitMQ +func publishToRabbitMQ(ctx context.Context, exchange, routingKey string, message interface{}) (messageID string, err error) { + ch, err := getRabbitMQChannel() + if err != nil { + return + } + + body, err := gjson.Encode(message) + if err != nil { + return "", gerror.Newf("消息序列化失败: %v", err) + } + + err = ch.PublishWithContext( + ctx, + exchange, // exchange + routingKey, // routing key + false, // mandatory + false, // immediate + amqp.Publishing{ + DeliveryMode: amqp.Persistent, + ContentType: "application/json", + Body: body, + }, + ) + + if err != nil { + g.Log().Errorf(ctx, "发布消息失败: exchange=%s, routingKey=%s, err=%v", exchange, routingKey, err) + return + } + + g.Log().Debugf(ctx, "消息发布成功: exchange=%s, routingKey=%s", exchange, routingKey) + return messageID, nil +} + +// publishDelayedToRabbitMQ 发布延时消息到 RabbitMQ +func publishDelayedToRabbitMQ(ctx context.Context, exchange, routingKey string, message interface{}, delaySeconds int) (messageID string, err error) { + ch, err := getRabbitMQChannel() + if err != nil { + return + } + + body, err := gjson.Encode(message) + if err != nil { + return "", gerror.Newf("消息序列化失败: %v", err) + } + + err = ch.PublishWithContext( + ctx, + exchange, // exchange(必须是 x-delayed-message 类型) + routingKey, // routing key + false, // mandatory + false, // immediate + amqp.Publishing{ + DeliveryMode: amqp.Persistent, + ContentType: "application/json", + Body: body, + Headers: amqp.Table{ + "x-delay": delaySeconds * 1000, // 延时(毫秒) + }, + }, + ) + + if err != nil { + g.Log().Errorf(ctx, "发布延时消息失败: exchange=%s, routingKey=%s, delay=%ds, err=%v", exchange, routingKey, delaySeconds, err) + return + } + + g.Log().Debugf(ctx, "延时消息发布成功: exchange=%s, routingKey=%s, delay=%ds", exchange, routingKey, delaySeconds) + return messageID, nil +} diff --git a/message/redis.go b/message/redis.go new file mode 100644 index 0000000..9f28ebe --- /dev/null +++ b/message/redis.go @@ -0,0 +1,232 @@ +package message + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/gogf/gf/v2/database/gredis" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/util/gconv" +) + +// StreamMessage Redis Stream 消息结构 +type StreamMessage struct { + ID string // 消息ID(自动生成) + Values map[string]interface{} // 消息内容 +} + +// getClient 获取 Redis 客户端 +func getRedisClient() *gredis.Redis { + return g.Redis() +} + +// getClient 获取 Redis 客户端 +func getRedisClientTest(name string) *gredis.Redis { + return g.Redis(name) +} + +// getRedisClientByDB 根据DB获取Redis客户端,如果db<=0则返回默认客户端 +func getRedisClientByDB(db int) *gredis.Redis { + if db <= 0 { + return g.Redis() + } + // 创建连接到指定DB的Redis客户端 + client, err := gredis.New(&gredis.Config{ + Address: g.Cfg().MustGet(context.Background(), "redis.default.address").String(), + Db: db, + }) + if err != nil { + glog.Errorf(context.Background(), "创建Redis客户端失败: %v", err) + return g.Redis() + } + return client +} + +// lock 分布式锁 +func lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context) error) (success bool, err error) { + limit := 3 +LOOP: + if limit < 0 { + return false, errors.New("锁重试次数耗尽") + } + limit-- + if val, err := getRedisClient().Set(ctx, key, true, gredis.SetOption{ + TTLOption: gredis.TTLOption{ + EX: &expireSeconds, + }, + NX: true, + }); err != nil { + return false, err + } else { + if val.Bool() { + defer func(RedisClient *gredis.Redis, ctx context.Context, key string) { + if _, err = RedisClient.Del(ctx, key); err != nil { + glog.Errorf(ctx, "RedisClient.Del error: %v", err) + } + }(getRedisClient(), ctx, key) + if err = fn(ctx); err != nil { + return false, err + } + return true, nil + } else { + time.Sleep(time.Second) + goto LOOP + } + } +} + +// publishToRedis 将消息添加到 Redis Stream +func publishToRedis(ctx context.Context, streamKey string, msg interface{}) (messageID string, err error) { + values := gconv.Map(msg) + args := make([]interface{}, 0, len(values)*2+2) + args = append(args, streamKey, "*") + for key, val := range values { + args = append(args, key, val) + } + result, err := getRedisClient().Do(ctx, "XADD", args...) + if err != nil { + return + } + messageID = result.String() + return +} + +// initStreamGroup 初始化 Stream 和消费者组 +func initStreamGroup(ctx context.Context, streamKey, groupName string) error { + _, err := getRedisClient().Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") + if err != nil { + // 如果组已存在,忽略错误 + errStr := err.Error() + // 检查错误是否是 "BUSYGROUP Consumer Group name already exists" + if strings.Contains(errStr, "BUSYGROUP") || strings.Contains(errStr, "already exists") { + // 这是一个预期的情况,说明消费者组已经存在,无需处理 + return nil + } + // 这是一个真正的错误,需要记录或处理 + return err + } + return nil +} + +// readFromStream 从 Stream 读取消息 +func readFromStream(ctx context.Context, msg QueueMessage) error { + // 初始化 Stream 和消费者组 + if err := initStreamGroup(ctx, msg.StreamKey, msg.GroupName); err != nil { + return err + } + go func() { + RECONNECT: + for { + result, err := getRedisClient().Do(ctx, "XREADGROUP", "GROUP", msg.GroupName, msg.ConsumerName, "COUNT", msg.BatchSize, "BLOCK", 0, "STREAMS", msg.StreamKey, ">") + if err != nil { + select { + case <-ctx.Done(): + return + } + time.Sleep(time.Second) + goto RECONNECT + } + // 检查返回结果是否为空 + if result == nil || result.IsEmpty() { + continue + } + messages := make([]StreamMessage, 0, int(msg.BatchSize)) + // 尝试 map 格式(GoFrame gredis 返回) + if streamsMap, ok := result.Val().(map[interface{}]interface{}); ok { + for _, streamMsgs := range streamsMap { + msgsArray, ok := streamMsgs.([]interface{}) + if !ok { + continue + } + for _, msgData := range msgsArray { + msgArray, ok := msgData.([]interface{}) + if !ok || len(msgArray) < 2 { + continue + } + msgID := gconv.String(msgArray[0]) + fieldsArray, ok := msgArray[1].([]interface{}) + if !ok { + continue + } + values := make(map[string]interface{}, len(fieldsArray)/2) + for i := 0; i < len(fieldsArray); i += 2 { + if i+1 < len(fieldsArray) { + key := gconv.String(fieldsArray[i]) + values[key] = fieldsArray[i+1] + } + } + messages = append(messages, StreamMessage{ + ID: msgID, + Values: values, + }) + } + } + } + // 尝试数组格式(标准 Redis 返回) + if streamsArray, ok := result.Val().([]interface{}); ok && len(streamsArray) > 0 { + for _, streamData := range streamsArray { + streamArray, ok := streamData.([]interface{}) + if !ok || len(streamArray) < 2 { + continue + } + messagesArray, ok := streamArray[1].([]interface{}) + if !ok { + continue + } + for _, msgData := range messagesArray { + msgArray, ok := msgData.([]interface{}) + if !ok || len(msgArray) < 2 { + continue + } + msgID := gconv.String(msgArray[0]) + fieldsArray, ok := msgArray[1].([]interface{}) + if !ok { + continue + } + values := make(map[string]interface{}, len(fieldsArray)/2) + for i := 0; i < len(fieldsArray); i += 2 { + if i+1 < len(fieldsArray) { + key := gconv.String(fieldsArray[i]) + values[key] = fieldsArray[i+1] + } + } + messages = append(messages, StreamMessage{ + ID: msgID, + Values: values, + }) + } + } + } + // 处理消息 + for _, streamMsg := range messages { + // 业务处理 + if err := msg.HandleFunc(ctx, streamMsg.Values); err != nil { + glog.Infof(ctx, "业务处理失败-> err:%v\n", err) + continue + } + // 确认消息 + if msg.AutoAck { + err := ackMessage(ctx, msg.StreamKey, msg.GroupName, streamMsg.ID) + if err != nil { + glog.Infof(ctx, "消费者 '%s' 确认消息 ID %s 失败: %v\n", msg.ConsumerName, streamMsg.ID, err) + } + } + } + } + }() + return nil +} + +// ackMessage 确认消息已处理 +func ackMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error { + args := make([]interface{}, 0, len(messageIDs)+2) + args = append(args, streamKey, groupName) + for _, id := range messageIDs { + args = append(args, id) + } + _, err := getRedisClient().Do(ctx, "XACK", args...) + return err +} From 2ef4db0101e28b6e05bc20da39c89103e1b43b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 9 Jan 2026 15:24:43 +0800 Subject: [PATCH 160/225] =?UTF-8?q?common=E5=A2=9E=E5=8A=A0nats=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E9=98=9F=E5=88=97=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nats/nats.go | 631 ++++++++++++++++++++++++++++++++++++++++++++++ nats/nats_test.go | 140 ++++++++++ 2 files changed, 771 insertions(+) create mode 100644 nats/nats.go create mode 100644 nats/nats_test.go diff --git a/nats/nats.go b/nats/nats.go new file mode 100644 index 0000000..d1ec88e --- /dev/null +++ b/nats/nats.go @@ -0,0 +1,631 @@ +package nats + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +var ( + nc *nats.Conn + js jetstream.JetStream + inited bool + mu sync.RWMutex + natsURL string + healthCtx context.Context + healthCancel context.CancelFunc + connected bool + reconnectChan chan struct{} + + // 连接状态变化监听器 + connStateListeners []ConnStateListener + connListenersMu sync.RWMutex + + // 监控指标 + metrics Metrics +) + +// Metrics 监控指标 +type Metrics struct { + PublishCount atomic.Int64 + PublishError atomic.Int64 + SubscribeCount atomic.Int64 + RequestCount atomic.Int64 + RequestError atomic.Int64 + ConsumeCount atomic.Int64 + ConsumeError atomic.Int64 +} + +// ConnState 连接状态 +type ConnState int + +const ( + ConnStateDisconnected ConnState = iota + ConnStateConnecting + ConnStateConnected + ConnStateReconnecting + ConnStateClosed +) + +// ConnStateListener 连接状态监听器 +type ConnStateListener func(state ConnState, err error) + +// GetMetrics 获取监控指标 +func GetMetrics() Metrics { + return metrics +} + +// RegisterConnStateListener 注册连接状态监听器 +func RegisterConnStateListener(listener ConnStateListener) { + connListenersMu.Lock() + defer connListenersMu.Unlock() + connStateListeners = append(connStateListeners, listener) +} + +// UnregisterConnStateListener 取消注册连接状态监听器 +func UnregisterConnStateListener(listener ConnStateListener) { + connListenersMu.Lock() + defer connListenersMu.Unlock() + for i, l := range connStateListeners { + if l != nil && &l == &listener { + connStateListeners = append(connStateListeners[:i], connStateListeners[i+1:]...) + break + } + } +} + +// notifyConnState 通知所有监听器连接状态变化 +func notifyConnState(state ConnState, err error) { + connListenersMu.RLock() + listeners := make([]ConnStateListener, len(connStateListeners)) + copy(listeners, connStateListeners) + connListenersMu.RUnlock() + + for _, listener := range listeners { + if listener != nil { + listener(state, err) + } + } +} + +// init 初始化 NATS 连接 +func init() { + // 从配置文件读取 NATS 地址 + natsURL = g.Cfg().MustGet(context.Background(), "nats.url").String() + if natsURL == "" { + // 默认使用本地地址 + natsURL = nats.DefaultURL + } + + // 创建健康检查上下文 + healthCtx, healthCancel = context.WithCancel(context.Background()) + + // 创建重连通知通道(增大缓冲区避免丢失通知) + reconnectChan = make(chan struct{}, 10) + + // 启动连接 + go initConnection() + + // 启动健康检查协程 + go healthCheck() +} + +// initConnection 初始化连接 +func initConnection() { + ctx := context.Background() + notifyConnState(ConnStateConnecting, nil) + if err := connect(ctx); err != nil { + g.Log().Errorf(ctx, "NATS 初始连接失败: %v", err) + notifyConnState(ConnStateDisconnected, err) + } +} + +// connect 建立 NATS 连接 +func connect(ctx context.Context) error { + mu.Lock() + defer mu.Unlock() + + if nc != nil && !nc.IsClosed() { + nc.Close() + } + + // 连接选项配置 + opts := []nats.Option{ + nats.Name("goframe-nats-client"), + nats.ReconnectWait(2 * time.Second), + nats.MaxReconnects(-1), // 无限重连 + nats.PingInterval(10 * time.Second), + nats.MaxPingsOutstanding(5), + nats.ReconnectHandler(func(nc *nats.Conn) { + g.Log().Infof(ctx, "✅ NATS 重连成功: %s", nc.ConnectedUrl()) + connected = true + + // 重新创建 JetStream 实例 + if newJS, err := jetstream.New(nc); err == nil { + js = newJS + } + + // 通知重连成功 + notifyConnState(ConnStateConnected, nil) + + // 使用非阻塞发送避免阻塞 + select { + case reconnectChan <- struct{}{}: + default: + // 通道已满,丢弃通知 + } + }), + nats.DisconnectErrHandler(func(nc *nats.Conn, err error) { + g.Log().Warningf(ctx, "⚠️ NATS 连接断开: %v, 准备重连...", err) + connected = false + notifyConnState(ConnStateReconnecting, err) + }), + nats.ClosedHandler(func(nc *nats.Conn) { + g.Log().Infof(ctx, "NATS 连接已关闭: %s", nc.ConnectedUrl()) + connected = false + notifyConnState(ConnStateClosed, nil) + }), + nats.ErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { + g.Log().Errorf(ctx, "NATS 错误: %v", err) + }), + } + + var err error + nc, err = nats.Connect(natsURL, opts...) + if err != nil { + return fmt.Errorf("NATS 连接失败: %w", err) + } + + // 等待连接就绪 + if nc.Status() != nats.CONNECTED { + select { + case <-time.After(5 * time.Second): + notifyConnState(ConnStateDisconnected, fmt.Errorf("连接超时")) + return fmt.Errorf("NATS 连接超时") + case <-nc.StatusChanged(nats.CONNECTED): + } + } + + // 创建 JetStream 实例 + js, err = jetstream.New(nc) + if err != nil { + return fmt.Errorf("创建 JetStream 失败: %w", err) + } + + connected = true + inited = true + g.Log().Infof(ctx, "✅ NATS 连接成功: %s", nc.ConnectedUrl()) + notifyConnState(ConnStateConnected, nil) + return nil +} + +// healthCheck 健康检查协程(仅作为备用检查) +func healthCheck() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-healthCtx.Done(): + return + case <-ticker.C: + mu.RLock() + currentConnected := connected + currentConn := nc + mu.RUnlock() + + if !currentConnected || currentConn == nil || currentConn.IsClosed() { + // 仅记录日志,不尝试重连(NATS 已有自动重连机制) + g.Log().Warning(context.Background(), "NATS 连接断开,等待 NATS 自动重连...") + } + case <-reconnectChan: + // 重连成功的通知(仅记录日志) + g.Log().Info(context.Background(), "收到重连成功通知") + } + } +} + +// checkConnected 检查连接状态 +func checkConnected() bool { + mu.RLock() + defer mu.RUnlock() + return connected && nc != nil && !nc.IsClosed() +} + +// IsConnected 检查 NATS 是否已连接 +func IsConnected() bool { + return checkConnected() +} + +// GetConnState 获取当前连接状态 +func GetConnState() ConnState { + mu.RLock() + defer mu.RUnlock() + + if nc == nil { + return ConnStateDisconnected + } + + if nc.IsClosed() { + return ConnStateClosed + } + + if connected { + return ConnStateConnected + } + + return ConnStateDisconnected +} + +// CreateTaskStream 创建任务消息队列流 +// 存储策略: 文件存储 +// 工作队列模式: 工作队列策略 +func CreateTaskStream(ctx context.Context, streamName string, subjects []string) error { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + stream, err := js.Stream(ctx, streamName) + if err == nil { + // 流已存在,更新配置 + _, err = js.UpdateStream(ctx, jetstream.StreamConfig{ + Name: streamName, + Subjects: subjects, + Storage: jetstream.FileStorage, + Retention: jetstream.WorkQueuePolicy, + }) + if err != nil { + return fmt.Errorf("更新任务流失败: %w", err) + } + g.Log().Infof(ctx, "✅ 任务消息队列流已更新: %s", stream.CachedInfo().Config.Name) + return nil + } + + // 创建新流 + stream, err = js.CreateStream(ctx, jetstream.StreamConfig{ + Name: streamName, + Subjects: subjects, + Storage: jetstream.FileStorage, + Retention: jetstream.WorkQueuePolicy, + }) + if err != nil { + return fmt.Errorf("创建任务流失败: %w", err) + } + + g.Log().Infof(ctx, "✅ 任务消息队列流创建成功: %s (文件存储+工作队列策略)", stream.CachedInfo().Config.Name) + return nil +} + +// CreateLogStream 创建日志流 +// 存储策略: 内存存储 +// 副本数: 单副本 (1) +// 消息留存: 短时留存 (1小时) +func CreateLogStream(ctx context.Context, streamName string, subjects []string) error { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + maxAge := 1 * time.Hour + + stream, err := js.Stream(ctx, streamName) + if err == nil { + // 流已存在,更新配置 + _, err = js.UpdateStream(ctx, jetstream.StreamConfig{ + Name: streamName, + Subjects: subjects, + Storage: jetstream.MemoryStorage, + Replicas: 1, + MaxAge: maxAge, + }) + if err != nil { + return fmt.Errorf("更新日志流失败: %w", err) + } + g.Log().Infof(ctx, "✅ 日志流已更新: %s", stream.CachedInfo().Config.Name) + return nil + } + + // 创建新流 + stream, err = js.CreateStream(ctx, jetstream.StreamConfig{ + Name: streamName, + Subjects: subjects, + Storage: jetstream.MemoryStorage, + Replicas: 1, + MaxAge: maxAge, + }) + if err != nil { + return fmt.Errorf("创建日志流失败: %w", err) + } + + g.Log().Infof(ctx, "✅ 日志流创建成功: %s (内存存储+单副本+短时留存1小时)", stream.CachedInfo().Config.Name) + return nil +} + +// CreateTradeStream 创建交易业务流 +// 存储策略: 文件存储 +// 副本数: 3副本 +// 同步刷盘: 启用 +func CreateTradeStream(ctx context.Context, streamName string, subjects []string) error { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + stream, err := js.Stream(ctx, streamName) + if err == nil { + // 流已存在,更新配置 + _, err = js.UpdateStream(ctx, jetstream.StreamConfig{ + Name: streamName, + Subjects: subjects, + Storage: jetstream.FileStorage, + Replicas: 3, + RePublish: nil, + Duplicates: 0, + }) + if err != nil { + return fmt.Errorf("更新交易流失败: %w", err) + } + g.Log().Infof(ctx, "✅ 交易业务流已更新: %s", stream.CachedInfo().Config.Name) + return nil + } + + // 创建新流 + stream, err = js.CreateStream(ctx, jetstream.StreamConfig{ + Name: streamName, + Subjects: subjects, + Storage: jetstream.FileStorage, + Replicas: 3, + RePublish: nil, + Duplicates: 0, + }) + if err != nil { + return fmt.Errorf("创建交易流失败: %w", err) + } + + g.Log().Infof(ctx, "✅ 交易业务流创建成功: %s (文件存储+3副本+同步刷盘)", stream.CachedInfo().Config.Name) + return nil +} + +// Publish 发布消息到指定主题 +func Publish(ctx context.Context, subject string, data []byte) error { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + metrics.PublishCount.Add(1) + _, err := js.Publish(ctx, subject, data) + if err != nil { + metrics.PublishError.Add(1) + return fmt.Errorf("发布消息失败: %w", err) + } + + return nil +} + +// GetStream 获取流信息 +func GetStream(ctx context.Context, streamName string) (*jetstream.StreamInfo, error) { + if !checkConnected() { + return nil, fmt.Errorf("NATS 未连接") + } + + stream, err := js.Stream(ctx, streamName) + if err != nil { + return nil, fmt.Errorf("获取流失败: %w", err) + } + + info, err := stream.Info(ctx) + if err != nil { + return nil, fmt.Errorf("获取流信息失败: %w", err) + } + + return info, nil +} + +// ListStreams 列出所有流(简化实现) +// 注意:由于 API 限制,此方法可能需要根据实际需求进一步实现 +func ListStreams(ctx context.Context) ([]string, error) { + if !checkConnected() { + return nil, fmt.Errorf("NATS 未连接") + } + + // TODO: 根据实际 NATS 版本实现完整的流列表功能 + return []string{}, nil +} + +// DeleteStream 删除流 +func DeleteStream(ctx context.Context, streamName string) error { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + if err := js.DeleteStream(ctx, streamName); err != nil { + return fmt.Errorf("删除流失败: %w", err) + } + + g.Log().Infof(ctx, "✅ 流已删除: %s", streamName) + return nil +} + +// GetConsumer 获取消费者信息 +func GetConsumer(ctx context.Context, streamName, consumerName string) (*jetstream.ConsumerInfo, error) { + if !checkConnected() { + return nil, fmt.Errorf("NATS 未连接") + } + + consumer, err := js.Consumer(ctx, streamName, consumerName) + if err != nil { + return nil, fmt.Errorf("获取消费者失败: %w", err) + } + + info, err := consumer.Info(ctx) + if err != nil { + return nil, fmt.Errorf("获取消费者信息失败: %w", err) + } + + return info, nil +} + +// ListConsumers 列出指定流的所有消费者(简化实现) +// 注意:由于 API 限制,此方法可能需要根据实际需求进一步实现 +func ListConsumers(ctx context.Context, streamName string) ([]string, error) { + if !checkConnected() { + return nil, fmt.Errorf("NATS 未连接") + } + + // TODO: 根据实际 NATS 版本实现完整的消费者列表功能 + return []string{}, nil +} + +// DeleteConsumer 删除消费者 +func DeleteConsumer(ctx context.Context, streamName, consumerName string) error { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + if err := js.DeleteConsumer(ctx, streamName, consumerName); err != nil { + return fmt.Errorf("删除消费者失败: %w", err) + } + + g.Log().Infof(ctx, "✅ 消费者已删除: %s/%s", streamName, consumerName) + return nil +} + +// CreateConsumer 创建消费者 +func CreateConsumer(ctx context.Context, streamName, consumerName string, config jetstream.ConsumerConfig) (jetstream.Consumer, error) { + if !checkConnected() { + return nil, fmt.Errorf("NATS 未连接") + } + + // 尝试获取现有消费者 + consumer, err := js.Consumer(ctx, streamName, consumerName) + if err == nil { + return consumer, nil + } + + // 创建新消费者 + consumer, err = js.CreateConsumer(ctx, streamName, config) + if err != nil { + return nil, fmt.Errorf("创建消费者失败: %w", err) + } + + return consumer, nil +} + +// SubscribeRequest 订阅 RPC 请求 +// B服务作为服务提供者,订阅主题并响应请求时使用此方法 +// subject: 订阅的主题名,与 Request 调用时使用相同的 subject +func SubscribeRequest(subject string, handler func(subject string, data []byte) ([]byte, error)) (*nats.Subscription, error) { + if !checkConnected() { + return nil, fmt.Errorf("NATS 未连接") + } + + sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { + // 处理请求 + response, err := handler(msg.Subject, msg.Data) + if err != nil { + // 处理错误,发送错误响应 + errMsg := fmt.Sprintf("处理失败: %v", err) + if err = msg.Respond([]byte(errMsg)); err != nil { + g.Log().Errorf(context.Background(), "RPC 错误响应失败: %v", err) + } + return + } + + // 发送成功响应 + if err = msg.Respond(response); err != nil { + g.Log().Errorf(context.Background(), "RPC 响应失败: %v", err) + } + }) + + if err != nil { + return nil, fmt.Errorf("订阅 RPC 请求失败: %w", err) + } + + return sub, nil +} + +// SubscribeQueueRequest 订阅队列模式的 RPC 请求(负载均衡) +// 多个服务实例订阅同一主题,实现负载均衡 +// subject: 订阅的主题名,与 Request 调用时使用相同的 subject +// queueName: 队列组名,同一队列组的实例之间实现负载均衡 +func SubscribeQueueRequest(subject, queueName string, handler func(subject string, data []byte) ([]byte, error)) (*nats.Subscription, error) { + if !checkConnected() { + return nil, fmt.Errorf("NATS 未连接") + } + + sub, err := nc.QueueSubscribe(subject, queueName, func(msg *nats.Msg) { + // 处理请求 + response, err := handler(msg.Subject, msg.Data) + if err != nil { + // 处理错误,发送错误响应 + errMsg := fmt.Sprintf("处理失败: %v", err) + if err = msg.Respond([]byte(errMsg)); err != nil { + g.Log().Errorf(context.Background(), "RPC 错误响应失败: %v", err) + } + return + } + + // 发送成功响应 + if err := msg.Respond(response); err != nil { + g.Log().Errorf(context.Background(), "RPC 响应失败: %v", err) + } + }) + + if err != nil { + return nil, fmt.Errorf("订阅队列 RPC 请求失败: %w", err) + } + + return sub, nil +} + +// Request RPC 请求-响应模式 +// A服务调用B服务查询接口时使用此方法 +func Request(ctx context.Context, subject string, data []byte, timeout time.Duration) ([]byte, error) { + if !checkConnected() { + return nil, fmt.Errorf("NATS 未连接") + } + + metrics.RequestCount.Add(1) + + // 使用 timeout 参数创建超时上下文 + timeoutCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + msg, err := nc.RequestWithContext(timeoutCtx, subject, data) + if err != nil { + metrics.RequestError.Add(1) + return nil, fmt.Errorf("RPC 请求失败: %w", err) + } + + if msg == nil { + metrics.RequestError.Add(1) + return nil, fmt.Errorf("RPC 响应为空") + } + + return msg.Data, nil +} + +// Close 关闭 NATS 连接 +func Close() error { + mu.Lock() + defer mu.Unlock() + + // 停止健康检查协程 + if healthCancel != nil { + healthCancel() + } + + // 关闭连接 + if nc != nil && !nc.IsClosed() { + nc.Close() + connected = false + inited = false + g.Log().Info(context.Background(), "NATS 连接已关闭") + } + + return nil +} diff --git a/nats/nats_test.go b/nats/nats_test.go new file mode 100644 index 0000000..e5a16ec --- /dev/null +++ b/nats/nats_test.go @@ -0,0 +1,140 @@ +package nats + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// TestNatsBasicOperations 测试基础操作 +func TestNatsBasicOperations(t *testing.T) { + // 测试连接状态 + if !IsConnected() { + t.Log("NATS 未连接") + } + + // 测试连接状态获取 + state := GetConnState() + t.Logf("当前连接状态: %d", state) +} + +// TestNatsMetrics 测试监控指标 +func TestNatsMetrics(t *testing.T) { + metrics := GetMetrics() + t.Logf("发布计数: %d", metrics.PublishCount.Load()) + t.Logf("发布错误: %d", metrics.PublishError.Load()) + t.Logf("请求计数: %d", metrics.RequestCount.Load()) +} + +// TestNatsConnStateListener 测试连接状态监听 +func TestNatsConnStateListener(t *testing.T) { + listener := func(state ConnState, err error) { + fmt.Printf("连接状态变化: %d, 错误: %v\n", state, err) + } + + RegisterConnStateListener(listener) + defer UnregisterConnStateListener(listener) + + time.Sleep(1 * time.Second) +} + +// TestNatsStreamOperations 测试流操作 +func TestNatsStreamOperations(t *testing.T) { + ctx := context.Background() + + // 创建任务流 + err := CreateTaskStream(ctx, "test_tasks", []string{"test.task.>"}) + if err != nil { + t.Logf("创建任务流失败: %v", err) + } + + // 获取流信息 + info, err := GetStream(ctx, "test_tasks") + if err != nil { + t.Logf("获取流信息失败: %v", err) + } else { + t.Logf("流信息: %s", info.Config.Name) + } + + // 列出所有流 + streams, err := ListStreams(ctx) + if err != nil { + t.Logf("列出流失败: %v", err) + } else { + t.Logf("流列表: %v", streams) + } + + // 删除流 + err = DeleteStream(ctx, "test_tasks") + if err != nil { + t.Logf("删除流失败: %v", err) + } +} + +// TestNatsConsumerOperations 测试消费者操作 +func TestNatsConsumerOperations(t *testing.T) { + ctx := context.Background() + + // 创建测试流 + err := CreateTaskStream(ctx, "test_consumer", []string{"test.consumer.>"}) + if err != nil { + t.Logf("创建流失败: %v", err) + } + + // 创建消费者 + consumerConfig := jetstream.ConsumerConfig{ + Name: "test_consumer", + Durable: "test_consumer", + } + _, err = CreateConsumer(ctx, "test_consumer", "test_consumer", consumerConfig) + if err != nil { + t.Logf("创建消费者失败: %v", err) + } + + // 获取消费者信息 + info, err := GetConsumer(ctx, "test_consumer", "test_consumer") + if err != nil { + t.Logf("获取消费者信息失败: %v", err) + } else { + t.Logf("消费者信息: %s", info.Name) + } + + // 列出消费者 + consumers, err := ListConsumers(ctx, "test_consumer") + if err != nil { + t.Logf("列出消费者失败: %v", err) + } else { + t.Logf("消费者列表: %v", consumers) + } + + // 删除消费者 + err = DeleteConsumer(ctx, "test_consumer", "test_consumer") + if err != nil { + t.Logf("删除消费者失败: %v", err) + } + + // 清理流 + _ = DeleteStream(ctx, "test_consumer") +} + +// TestNatsPublishRequest 测试发布和请求 +func TestNatsPublishRequest(t *testing.T) { + ctx := context.Background() + + // 发布消息 + err := Publish(ctx, "test.publish", []byte("hello")) + if err != nil { + t.Logf("发布消息失败: %v", err) + } + + // RPC 请求 + response, err := Request(ctx, "test.request", []byte("request"), 5*time.Second) + if err != nil { + t.Logf("RPC 请求失败: %v", err) + } else { + t.Logf("RPC 响应: %s", string(response)) + } +} From a3ad38e8f655b42745afab54ab46c1bab857e524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 9 Jan 2026 16:46:35 +0800 Subject: [PATCH 161/225] =?UTF-8?q?common=E5=A2=9E=E5=8A=A0nats=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E9=98=9F=E5=88=97=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nats/nats.go | 401 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 362 insertions(+), 39 deletions(-) diff --git a/nats/nats.go b/nats/nats.go index d1ec88e..f04bab1 100644 --- a/nats/nats.go +++ b/nats/nats.go @@ -2,7 +2,9 @@ package nats import ( "context" + "encoding/json" "fmt" + "reflect" "sync" "sync/atomic" "time" @@ -515,102 +517,424 @@ func CreateConsumer(ctx context.Context, streamName, consumerName string, config return consumer, nil } -// SubscribeRequest 订阅 RPC 请求 -// B服务作为服务提供者,订阅主题并响应请求时使用此方法 -// subject: 订阅的主题名,与 Request 调用时使用相同的 subject -func SubscribeRequest(subject string, handler func(subject string, data []byte) ([]byte, error)) (*nats.Subscription, error) { +// ============ RPC 服务封装 ============ +// 以下方法提供了完全抽象的 RPC 调用接口 +// 调用方和响应方完全不需要知道底层使用的是 NATS 的发布订阅模式 + +// RPC 服务注册表 +var ( + rpcServices map[string]RPCHandler + rpcSubs map[string]*nats.Subscription // 服务名 -> 订阅 + rpcServicesMu sync.RWMutex + queueRPCServices map[string]map[string]RPCHandler // queueName -> subject -> handler + queueRPCSubs map[string]map[string]*nats.Subscription // queueName -> serviceName -> 订阅 + queueRPCMu sync.RWMutex +) + +// RPCHandler RPC 处理函数类型 +// 实现方只需要关注请求参数和返回值,无需了解底层 NATS 实现 +type RPCHandler func(ctx context.Context, req []byte) ([]byte, error) + +// RegisterRPCService 注册 RPC 服务(单实例) +// serviceName: 服务名称,调用方通过此名称调用服务 +// handler: 服务处理函数,接收请求并返回响应 +func RegisterRPCService(serviceName string, handler RPCHandler) error { if !checkConnected() { - return nil, fmt.Errorf("NATS 未连接") + return fmt.Errorf("NATS 未连接") } + rpcServicesMu.Lock() + if rpcServices == nil { + rpcServices = make(map[string]RPCHandler) + } + if rpcSubs == nil { + rpcSubs = make(map[string]*nats.Subscription) + } + + // 如果已存在该服务,先取消之前的订阅 + if oldSub, exists := rpcSubs[serviceName]; exists { + oldSub.Unsubscribe() + } + + rpcServices[serviceName] = handler + rpcServicesMu.Unlock() + + // 订阅服务主题 + subject := fmt.Sprintf("rpc.%s", serviceName) sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { - // 处理请求 - response, err := handler(msg.Subject, msg.Data) + ctx := context.Background() + response, err := handler(ctx, msg.Data) if err != nil { - // 处理错误,发送错误响应 errMsg := fmt.Sprintf("处理失败: %v", err) if err = msg.Respond([]byte(errMsg)); err != nil { - g.Log().Errorf(context.Background(), "RPC 错误响应失败: %v", err) + g.Log().Errorf(ctx, "RPC 错误响应失败: %v", err) } return } - // 发送成功响应 if err = msg.Respond(response); err != nil { - g.Log().Errorf(context.Background(), "RPC 响应失败: %v", err) + g.Log().Errorf(ctx, "RPC 响应失败: %v", err) } }) if err != nil { - return nil, fmt.Errorf("订阅 RPC 请求失败: %w", err) + return fmt.Errorf("注册 RPC 服务失败: %w", err) } - return sub, nil + rpcSubs[serviceName] = sub + metrics.SubscribeCount.Add(1) + g.Log().Infof(context.Background(), "✅ RPC 服务已注册: %s", serviceName) + return nil } -// SubscribeQueueRequest 订阅队列模式的 RPC 请求(负载均衡) -// 多个服务实例订阅同一主题,实现负载均衡 -// subject: 订阅的主题名,与 Request 调用时使用相同的 subject -// queueName: 队列组名,同一队列组的实例之间实现负载均衡 -func SubscribeQueueRequest(subject, queueName string, handler func(subject string, data []byte) ([]byte, error)) (*nats.Subscription, error) { +// RegisterQueueRPCService 注册 RPC 服务(集群模式) +// 多个服务实例注册同一服务时,请求会自动负载均衡 +// serviceName: 服务名称 +// queueName: 队列组名,同一队列组的实例共享请求 +// handler: 服务处理函数 +func RegisterQueueRPCService(serviceName, queueName string, handler RPCHandler) error { if !checkConnected() { - return nil, fmt.Errorf("NATS 未连接") + return fmt.Errorf("NATS 未连接") } + queueRPCMu.Lock() + if queueRPCServices == nil { + queueRPCServices = make(map[string]map[string]RPCHandler) + } + if queueRPCSubs == nil { + queueRPCSubs = make(map[string]map[string]*nats.Subscription) + } + if queueRPCServices[queueName] == nil { + queueRPCServices[queueName] = make(map[string]RPCHandler) + } + if queueRPCSubs[queueName] == nil { + queueRPCSubs[queueName] = make(map[string]*nats.Subscription) + } + + // 如果已存在该服务,先取消之前的订阅 + if oldSub, exists := queueRPCSubs[queueName][serviceName]; exists { + oldSub.Unsubscribe() + } + + queueRPCServices[queueName][serviceName] = handler + queueRPCMu.Unlock() + + // 订阅服务主题(队列模式) + subject := fmt.Sprintf("rpc.%s", serviceName) sub, err := nc.QueueSubscribe(subject, queueName, func(msg *nats.Msg) { - // 处理请求 - response, err := handler(msg.Subject, msg.Data) + ctx := context.Background() + response, err := handler(ctx, msg.Data) if err != nil { - // 处理错误,发送错误响应 errMsg := fmt.Sprintf("处理失败: %v", err) if err = msg.Respond([]byte(errMsg)); err != nil { - g.Log().Errorf(context.Background(), "RPC 错误响应失败: %v", err) + g.Log().Errorf(ctx, "RPC 错误响应失败: %v", err) } return } - // 发送成功响应 - if err := msg.Respond(response); err != nil { - g.Log().Errorf(context.Background(), "RPC 响应失败: %v", err) + if err = msg.Respond(response); err != nil { + g.Log().Errorf(ctx, "RPC 响应失败: %v", err) } }) if err != nil { - return nil, fmt.Errorf("订阅队列 RPC 请求失败: %w", err) + return fmt.Errorf("注册队列 RPC 服务失败: %w", err) } - return sub, nil + queueRPCMu.Lock() + queueRPCSubs[queueName][serviceName] = sub + queueRPCMu.Unlock() + + metrics.SubscribeCount.Add(1) + g.Log().Infof(context.Background(), "✅ 队列 RPC 服务已注册: %s (队列组: %s)", serviceName, queueName) + return nil } -// Request RPC 请求-响应模式 -// A服务调用B服务查询接口时使用此方法 -func Request(ctx context.Context, subject string, data []byte, timeout time.Duration) ([]byte, error) { +// CallRPC 调用 RPC 服务 +// serviceName: 服务名称 +// req: 请求数据 +// timeout: 超时时间 +// 返回: 响应数据和错误 +func CallRPC(ctx context.Context, serviceName string, req []byte, timeout time.Duration) ([]byte, error) { if !checkConnected() { return nil, fmt.Errorf("NATS 未连接") } metrics.RequestCount.Add(1) - // 使用 timeout 参数创建超时上下文 + // 检查本地是否有注册的单实例服务,如果有则直接调用(优化性能) + rpcServicesMu.RLock() + if localHandler, exists := rpcServices[serviceName]; exists { + rpcServicesMu.RUnlock() + // 本地直接调用,避免网络开销 + response, err := localHandler(ctx, req) + if err != nil { + metrics.RequestError.Add(1) + return nil, fmt.Errorf("本地调用 RPC 服务失败 [%s]: %w", serviceName, err) + } + return response, nil + } + rpcServicesMu.RUnlock() + + // 通过 NATS 网络调用远程服务 timeoutCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() - msg, err := nc.RequestWithContext(timeoutCtx, subject, data) + subject := fmt.Sprintf("rpc.%s", serviceName) + msg, err := nc.RequestWithContext(timeoutCtx, subject, req) if err != nil { metrics.RequestError.Add(1) - return nil, fmt.Errorf("RPC 请求失败: %w", err) + return nil, fmt.Errorf("调用 RPC 服务失败 [%s]: %w", serviceName, err) } if msg == nil { metrics.RequestError.Add(1) - return nil, fmt.Errorf("RPC 响应为空") + return nil, fmt.Errorf("RPC 响应为空 [%s]", serviceName) } return msg.Data, nil } -// Close 关闭 NATS 连接 -func Close() error { +// RegisterServiceOption 注册选项类型 +type RegisterServiceOption func(*registerServiceConfig) + +type registerServiceConfig struct { + queueName string // 队列组名(用于集群模式) + excludeMethods []string +} + +// WithQueueGroup 设置队列组名(集群模式) +func WithQueueGroup(queueName string) RegisterServiceOption { + return func(cfg *registerServiceConfig) { + cfg.queueName = queueName + } +} + +// WithExcludeMethods 排除不需要注册的方法 +func WithExcludeMethods(methods ...string) RegisterServiceOption { + return func(cfg *registerServiceConfig) { + cfg.excludeMethods = append(cfg.excludeMethods, methods...) + } +} + +// registerService 注册单个服务的所有公开方法(内部函数) +func registerService(service interface{}, serviceNamePrefix string, options ...RegisterServiceOption) error { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + // 应用选项 + cfg := ®isterServiceConfig{} + for _, opt := range options { + opt(cfg) + } + + // 创建排除方法集合 + excludeSet := make(map[string]struct{}) + for _, method := range cfg.excludeMethods { + excludeSet[method] = struct{}{} + } + + // 获取 service 的类型 + serviceType := reflect.TypeOf(service) + + // 遍历所有方法 + registeredCount := 0 + for i := 0; i < serviceType.NumMethod(); i++ { + method := serviceType.Method(i) + + // 只注册导出方法(首字母大写) + if !method.IsExported() { + continue + } + + // 排除指定的方法 + if _, exists := excludeSet[method.Name]; exists { + continue + } + + // 检查方法签名:必须是 func(ctx context.Context, request) (response, error) + if method.Type.NumIn() < 2 { + g.Log().Warningf(context.Background(), "方法 %s 的参数数量不足,跳过注册", method.Name) + continue + } + + // 第一个参数必须是 context.Context + if !method.Type.In(0).Implements(reflect.TypeOf((*context.Context)(nil)).Elem()) { + g.Log().Warningf(context.Background(), "方法 %s 的第一个参数必须是 context.Context,跳过注册", method.Name) + continue + } + + // 返回值必须是 (result, error) 或 error + if method.Type.NumOut() < 1 || method.Type.NumOut() > 2 { + g.Log().Warningf(context.Background(), "方法 %s 的返回值数量不正确,跳过注册", method.Name) + continue + } + + if !method.Type.Out(method.Type.NumOut() - 1).Implements(reflect.TypeOf((*error)(nil)).Elem()) { + g.Log().Warningf(context.Background(), "方法 %s 的最后一个返回值必须是 error,跳过注册", method.Name) + continue + } + + // 生成服务名称:前缀.方法名(保持原始方法名) + serviceName := fmt.Sprintf("%s.%s", serviceNamePrefix, method.Name) + + // 创建 RPC handler + handler := func(ctx context.Context, req []byte) ([]byte, error) { + // 准备方法调用参数 + args := make([]reflect.Value, 2) + args[0] = reflect.ValueOf(ctx) + + // 解析请求参数 + if len(req) > 0 { + // 如果方法有第二个参数,尝试解析 JSON + if method.Type.NumIn() > 1 { + reqValuePtr := reflect.New(method.Type.In(1)) + if err := json.Unmarshal(req, reqValuePtr.Interface()); err != nil { + return nil, fmt.Errorf("解析请求参数失败: %w", err) + } + args[1] = reqValuePtr.Elem() + } + } else if method.Type.NumIn() > 1 { + // 如果方法需要参数但请求为空,创建零值 + args[1] = reflect.Zero(method.Type.In(1)) + } + + // 调用方法 + results := method.Func.Call(args) + + // 处理返回值 + var err error + var result interface{} + + if len(results) == 1 { + // 只有 error + if !results[0].IsNil() { + err = results[0].Interface().(error) + } + } else if len(results) == 2 { + // (result, error) + result = results[0].Interface() + if !results[1].IsNil() { + err = results[1].Interface().(error) + } + } + + if err != nil { + return nil, err + } + + // 序列化返回值 + if result == nil || (reflect.ValueOf(result).Kind() == reflect.Ptr && reflect.ValueOf(result).IsNil()) { + return []byte("{}"), nil + } + + return json.Marshal(result) + } + + // 注册 RPC 服务 + var err error + if cfg.queueName != "" { + err = RegisterQueueRPCService(serviceName, cfg.queueName, handler) + } else { + err = RegisterRPCService(serviceName, handler) + } + + if err != nil { + g.Log().Errorf(context.Background(), "注册服务 %s 失败: %v", serviceName, err) + continue + } + + registeredCount++ + g.Log().Infof(context.Background(), "✅ 已自动注册 RPC 服务: %s -> %s", serviceName, method.Name) + } + + if registeredCount == 0 { + g.Log().Warningf(context.Background(), "未注册任何方法,请检查 %v 的方法签名", serviceNamePrefix) + return fmt.Errorf("未找到可注册的方法") + } + + g.Log().Infof(context.Background(), "✅ Service %v 共注册了 %d 个 RPC 方法", serviceNamePrefix, registeredCount) + return nil +} + +// AutoRegisterServices 自动注册多个服务的所有公开方法 +// serviceInstances: map[包名]service实例,如 map[string]interface{}{"user": userService, "order": orderService} +// options: 注册选项(可选) +// 示例: +// +// AutoRegisterServices(map[string]interface{}{ +// "user": userService, +// "order": orderService, +// }) +// 或 +// AutoRegisterServices(map[string]interface{}{ +// "order": orderService, +// }, WithQueueGroup("order-group")) +func AutoRegisterServices(serviceInstances map[string]interface{}, options ...RegisterServiceOption) error { + if len(serviceInstances) == 0 { + return fmt.Errorf("service 实例列表不能为空") + } + + totalRegistered := 0 + + // 遍历每个 service 实例 + for pkgName, serviceInstance := range serviceInstances { + // 注册服务 + err := registerService(serviceInstance, pkgName, options...) + if err != nil { + g.Log().Errorf(context.Background(), "注册 %s 服务失败: %v", pkgName, err) + continue + } + + totalRegistered++ + g.Log().Infof(context.Background(), "✅ %s 服务已自动注册", pkgName) + } + + if totalRegistered == 0 { + return fmt.Errorf("未能注册任何服务") + } + + g.Log().Infof(context.Background(), "✅ 共自动注册了 %d 个服务", totalRegistered) + return nil +} + +// Shutdown 优雅关闭:自动注销所有已注册的服务并关闭 NATS 连接 +func Shutdown() error { + ctx := context.Background() + g.Log().Info(ctx, "开始优雅关闭 NATS RPC 服务...") + + // 注销所有单实例服务 + rpcServicesMu.Lock() + singleServiceCount := len(rpcServices) + for serviceName := range rpcServices { + if sub, exists := rpcSubs[serviceName]; exists { + if err := sub.Unsubscribe(); err != nil { + g.Log().Errorf(ctx, "注销服务 %s 失败: %v", serviceName, err) + } + } + delete(rpcSubs, serviceName) + delete(rpcServices, serviceName) + } + rpcServicesMu.Unlock() + + // 注销所有队列服务 + queueRPCMu.Lock() + queueServiceCount := 0 + for queueName, servicesMap := range queueRPCServices { + queueServiceCount += len(servicesMap) + for serviceName, sub := range queueRPCSubs[queueName] { + if err := sub.Unsubscribe(); err != nil { + g.Log().Errorf(ctx, "注销队列服务 %s (队列: %s) 失败: %v", serviceName, queueName, err) + } + } + delete(queueRPCSubs, queueName) + delete(queueRPCServices, queueName) + } + queueRPCMu.Unlock() + + g.Log().Infof(ctx, "已注销 %d 个单实例服务和 %d 个队列服务", singleServiceCount, queueServiceCount) + mu.Lock() defer mu.Unlock() @@ -624,8 +948,7 @@ func Close() error { nc.Close() connected = false inited = false - g.Log().Info(context.Background(), "NATS 连接已关闭") } - + g.Log().Info(ctx, "NATS RPC 服务已优雅关闭") return nil } From 0254a59c73cdbc359bcfd5831b9ab9d02579762c Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Fri, 9 Jan 2026 18:12:57 +0800 Subject: [PATCH 162/225] =?UTF-8?q?isEmptyWithZero=20=E5=88=A4=E6=96=AD?= =?UTF-8?q?=E6=98=AF=E5=90=A6=E4=B8=BA=E7=A9=BA=E5=80=BC=EF=BC=8C=E4=BD=86?= =?UTF-8?q?=E4=BF=9D=E7=95=99=20int=20=E7=B1=BB=E5=9E=8B=E7=9A=84=200=20?= =?UTF-8?q?=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index edc8781..b1c8609 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "reflect" "strings" "sync" "time" @@ -471,6 +472,7 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll // RandomSoftDelete 随机软删除个文档的 _id func (m *MongoDB) RandomSoftDelete(ctx context.Context, limit int, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { + _ = opts // 步骤 1: 使用聚合管道的 $sample 操作符随机抽取5个文档的 _id pipeline := mongo.Pipeline{ // 阶段1: 为每个文档添加一个 0-1 之间的随机数字段 'random' @@ -665,7 +667,11 @@ func EntityToBson(entity interface{}) (bson.M, error) { } // EntityToBsonWithFilter 将 *entity/entity 转换为 bson.M,并可选择是否过滤空值 -// filterEmpty: 为 true 时会过滤掉空值字段(nil、空字符串、空切片、空map等) +// filterEmpty: 为 true 时会过滤掉空值字段(nil、空字符串、空切片、空map等),但保留 int 类型的 0 值 +// 支持: +// - 未传值的指针(如 *consts.AssetStatus(nil))会被过滤 +// - 传值为0的指针(如 *consts.AssetStatus(0))不会被过滤 +// - 非0的整数值不会被过滤 func EntityToBsonWithFilter(entity interface{}, filterEmpty bool) (bson.M, error) { // 第一步:判断入参是否为 nil 或无效类型 if entity == nil { @@ -686,10 +692,43 @@ func EntityToBsonWithFilter(entity interface{}, filterEmpty bool) (bson.M, error // 如果需要过滤空值 if filterEmpty { for key, value := range bsonMap { - if g.IsEmpty(value) { + // 判断是否为空值,但保留 int 类型的 0 值 + if isEmptyWithZero(value) { delete(bsonMap, key) } } } return bsonMap, nil } + +// isEmptyWithZero 判断是否为空值,但保留 int 类型的 0 值 +// 支持区分"未传值"和"传值为0"的情况: +// - *int(nil) 或 *consts.AssetStatus(nil) → 返回 true(过滤掉) +// - *int(0) 或 *consts.AssetStatus(0) → 返回 false(保留) +// - int(0) 或 consts.AssetStatus(0) → 返回 false(保留) +func isEmptyWithZero(value interface{}) bool { + // 先检查 value 是否为 nil + if value == nil { + return true + } + rv := reflect.ValueOf(value) + kind := rv.Kind() + // 处理 nil 指针 + if kind == reflect.Ptr { + if rv.IsNil() { + return true + } + // 判断时如果是指针,需要获取指向的值的类型 + kind = rv.Elem().Kind() + } + // 数字类型(int/uint/float)都保留,包括 0 值 + switch kind { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + reflect.Float32, reflect.Float64: + return false + default: + // 其他类型使用 g.IsEmpty 判断 + return g.IsEmpty(value) + } +} From 78ca473e9e6fa9ff5a9143ab7fa52b6f5ee61cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 9 Jan 2026 19:45:50 +0800 Subject: [PATCH 163/225] =?UTF-8?q?common=E5=A2=9E=E5=8A=A0nats=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E9=98=9F=E5=88=97=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 749 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 507 insertions(+), 242 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index b1c8609..3a04d72 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -1,24 +1,32 @@ +// ============================================================================= +// MongoDB 多数据源支持 +// 支持多数据源配置、自动重连、优雅关闭 +// 向后兼容原有的单数据源API +// ============================================================================= + package mongo import ( "context" "errors" "fmt" + "os" + "os/signal" "reflect" "strings" "sync" + "syscall" "time" "gitee.com/red-future---jilin-g/common/beans" "gitee.com/red-future---jilin-g/common/log/model/entity" - "github.com/gogf/gf/v2/container/gvar" - "github.com/gogf/gf/v2/os/grpool" - "gitee.com/red-future---jilin-g/common/redis" "gitee.com/red-future---jilin-g/common/utils" + "github.com/gogf/gf/v2/container/gvar" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/os/grpool" "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/text/gstr" "github.com/gogf/gf/v2/util/gconv" @@ -27,8 +35,367 @@ import ( "go.mongodb.org/mongo-driver/v2/mongo/options" ) +// ============================================================================= +// 数据源配置结构 +// ============================================================================= + +type DataSourceConfig struct { + Name string `json:"name"` + Address string `json:"address"` + Database string `json:"database"` + MaxPoolSize int32 `json:"maxPoolSize"` + MinPoolSize int32 `json:"minPoolSize"` + ConnectTimeout time.Duration `json:"connectTimeout"` +} + +// ============================================================================= +// 单个数据源接口 +// ============================================================================= + +type DataSource interface { + Name() string + Database() *mongo.Database + Client() *mongo.Client + IsConnected() bool + Connect(ctx context.Context) error + Reconnect(ctx context.Context) error + Close(ctx context.Context) error +} + +// ============================================================================= +// 数据源实现 +// ============================================================================= + +type BaseDataSource struct { + config *DataSourceConfig + client *mongo.Client + database *mongo.Database + isConnected bool + mu sync.RWMutex + lastError error + lastErrorTime time.Time +} + +func NewBaseDataSource(config *DataSourceConfig) *BaseDataSource { + return &BaseDataSource{ + config: config, + isConnected: false, + } +} + +func (d *BaseDataSource) Name() string { + return d.config.Name +} + +func (d *BaseDataSource) Database() *mongo.Database { + d.mu.RLock() + defer d.mu.RUnlock() + return d.database +} + +func (d *BaseDataSource) Client() *mongo.Client { + d.mu.RLock() + defer d.mu.RUnlock() + return d.client +} + +func (d *BaseDataSource) IsConnected() bool { + d.mu.RLock() + defer d.mu.RUnlock() + return d.isConnected && d.client != nil +} + +func (d *BaseDataSource) Connect(ctx context.Context) error { + d.mu.Lock() + defer d.mu.Unlock() + + if d.client != nil { + d.client.Disconnect(ctx) + } + + // 解析数据库名 + dbName := d.config.Database + if strings.Contains(dbName, "?") { + dbName = gstr.SubStr(dbName, 0, strings.Index(dbName, "?")) + } + + // 构建连接选项 + opt := options.Client(). + ApplyURI(d.config.Address). + SetMaxPoolSize(uint64(d.config.MaxPoolSize)). + SetMinPoolSize(uint64(d.config.MinPoolSize)). + SetConnectTimeout(d.config.ConnectTimeout). + SetMaxConnecting(10). + SetServerSelectionTimeout(10 * time.Second). + SetHeartbeatInterval(10 * time.Second). + SetMaxConnIdleTime(60 * time.Second). + SetRetryWrites(true). + SetRetryReads(true) + + var err error + d.client, err = mongo.Connect(opt) + if err != nil { + d.isConnected = false + d.lastError = err + d.lastErrorTime = time.Now() + return fmt.Errorf("datasource [%s] connection failed: %w", d.config.Name, err) + } + + // 测试连接 + pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err = d.client.Ping(pingCtx, nil); err != nil { + d.isConnected = false + d.lastError = err + d.lastErrorTime = time.Now() + return fmt.Errorf("datasource [%s] ping failed: %w", d.config.Name, err) + } + + d.database = d.client.Database(dbName) + d.isConnected = true + d.lastError = nil + glog.Infof(ctx, "✅ datasource [%s] connected successfully", d.config.Name) + return nil +} + +func (d *BaseDataSource) Reconnect(ctx context.Context) error { + glog.Infof(ctx, "🔄 reconnecting datasource [%s]", d.config.Name) + return d.Connect(ctx) +} + +func (d *BaseDataSource) Close(ctx context.Context) error { + d.mu.Lock() + defer d.mu.Unlock() + + if d.client != nil { + disconnectCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := d.client.Disconnect(disconnectCtx); err != nil { + return fmt.Errorf("datasource [%s] close failed: %w", d.config.Name, err) + } + } + + d.isConnected = false + glog.Infof(ctx, "datasource [%s] closed", d.config.Name) + return nil +} + +// ============================================================================= +// 多数据源管理器 +// ============================================================================= + +type DataSourceManager struct { + sources map[string]DataSource + mu sync.RWMutex + ctx context.Context + cancel context.CancelFunc + started bool + maxRetries int +} + +var ( + globalManager *DataSourceManager + managerOnce sync.Once +) + +// GetManager 获取全局管理器 +func GetManager() *DataSourceManager { + managerOnce.Do(func() { + ctx, cancel := context.WithCancel(context.Background()) + globalManager = &DataSourceManager{ + sources: make(map[string]DataSource), + ctx: ctx, + cancel: cancel, + started: false, + maxRetries: 3, + } + }) + return globalManager +} + +// RegisterDataSource 注册数据源 +func (m *DataSourceManager) RegisterDataSource(config *DataSourceConfig) error { + m.mu.Lock() + defer m.mu.Unlock() + + if _, exists := m.sources[config.Name]; exists { + return fmt.Errorf("datasource [%s] already exists", config.Name) + } + + source := NewBaseDataSource(config) + m.sources[config.Name] = source + return nil +} + +// GetDataSource 获取数据源 +func (m *DataSourceManager) GetDataSource(name string) (DataSource, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + source, exists := m.sources[name] + if !exists { + return nil, fmt.Errorf("datasource [%s] not found", name) + } + return source, nil +} + +// GetAllDataSourceNames 获取所有数据源名称 +func (m *DataSourceManager) GetAllDataSourceNames() []string { + m.mu.RLock() + defer m.mu.RUnlock() + + names := make([]string, 0, len(m.sources)) + for name := range m.sources { + names = append(names, name) + } + return names +} + +// InitializeFromConfig 从配置初始化数据源 +// 动态读取 config.yml 中 mongo 下的所有配置项 +func (m *DataSourceManager) InitializeFromConfig(ctx context.Context) error { + var firstErr error + + // 获取 mongo 配置下的所有子键 + mongoConfig := g.Cfg().MustGet(ctx, "mongo") + if mongoConfig.IsNil() { + glog.Warningf(ctx, "no mongo configuration found in config.yml") + return nil + } + + // 将配置转换为 map + configMap := mongoConfig.Map() + if configMap == nil { + glog.Warningf(ctx, "mongo configuration is not a map") + return nil + } + + // 遍历所有 mongo 子配置 + for name, subConfig := range configMap { + // 跳过非对象类型的配置 + subMap, ok := subConfig.(map[string]interface{}) + if !ok { + continue + } + + // 检查是否有 address 配置 + address, hasAddress := subMap["address"] + if !hasAddress || gconv.String(address) == "" { + continue + } + + // 构建数据源配置 + config := &DataSourceConfig{ + Name: name, + Address: gconv.String(address), + Database: gconv.String(subMap["database"]), + MaxPoolSize: int32(gconv.Int(subMap["maxPoolSize"])), + MinPoolSize: int32(gconv.Int(subMap["minPoolSize"])), + ConnectTimeout: gconv.Duration(subMap["connectTimeout"]), + } + + // 设置默认值 + if config.MaxPoolSize == 0 { + config.MaxPoolSize = 100 + } + if config.MinPoolSize == 0 { + config.MinPoolSize = 10 + } + if config.ConnectTimeout == 0 { + config.ConnectTimeout = 10 * time.Second + } + + // 注册数据源 + if err := m.RegisterDataSource(config); err != nil { + glog.Errorf(ctx, "failed to register datasource [%s]: %v", name, err) + if firstErr == nil { + firstErr = err + } + continue + } + + // 连接数据源 + source, _ := m.GetDataSource(name) + if err := source.Connect(ctx); err != nil { + glog.Errorf(ctx, "failed to initialize datasource [%s]: %v", name, err) + if firstErr == nil { + firstErr = err + } + } + } + + return firstErr +} + +// StartHealthCheck 启动健康检查 +func (m *DataSourceManager) StartHealthCheck() { + if m.started { + return + } + m.started = true + go m.healthCheckLoop() +} + +// healthCheckLoop 健康检查循环 +func (m *DataSourceManager) healthCheckLoop() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-m.ctx.Done(): + return + case <-ticker.C: + m.checkAndReconnect() + } + } +} + +// checkAndReconnect 检查并重新连接 +func (m *DataSourceManager) checkAndReconnect() { + m.mu.RLock() + defer m.mu.RUnlock() + + for name, source := range m.sources { + if !source.IsConnected() { + glog.Warningf(context.Background(), "datasource [%s] disconnected, attempting reconnect", name) + + reconnectCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := source.Reconnect(reconnectCtx); err != nil { + glog.Errorf(reconnectCtx, "datasource [%s] reconnect failed: %v", name, err) + } else { + glog.Infof(reconnectCtx, "✅ datasource [%s] reconnected successfully", name) + } + } + } +} + +// CloseAll 关闭所有数据源 +func (m *DataSourceManager) CloseAll(ctx context.Context) error { + m.cancel() + + m.mu.RLock() + defer m.mu.RUnlock() + + var lastErr error + for name, source := range m.sources { + if err := source.Close(ctx); err != nil { + glog.Errorf(ctx, "failed to close datasource [%s]: %v", name, err) + lastErr = err + } + } + return lastErr +} + +// ============================================================================= +// 向后兼容的MongoDB结构体 +// ============================================================================= + type MongoDB struct { - Cache bool + Cache bool + dataSource string // 数据源名称,默认为 "default" } func DB(cache ...bool) *MongoDB { @@ -37,199 +404,100 @@ func DB(cache ...bool) *MongoDB { b = cache[0] } return &MongoDB{ - Cache: b, + Cache: b, + dataSource: "default", } } +// WithDataSource 指定使用的数据源 +func (m *MongoDB) WithDataSource(name string) *MongoDB { + m.dataSource = name + return m +} + +// ============================================================================= +// 向后兼容的全局变量和方法 +// ============================================================================= + var ( - db *mongo.Database - client *mongo.Client - isConnected bool - mu sync.RWMutex - mongoAddr string - dbName string - healthCtx context.Context - healthCancel context.CancelFunc + manager = GetManager() + logPool *grpool.Pool + serverName string + logRedisKey string ) -// checkConnected 检查连接状态 -func checkConnected() bool { - mu.RLock() - defer mu.RUnlock() - return isConnected -} - -// connect 建立MongoDB连接 -func connect() error { - mu.Lock() - defer mu.Unlock() - - if client != nil { - client.Disconnect(context.Background()) - } - - // 创建连接选项 - opt := options.Client(). - ApplyURI(mongoAddr). - SetMaxPoolSize(100). - SetMinPoolSize(10). - SetMaxConnecting(10). - SetConnectTimeout(10 * time.Second) - - var err error - client, err = mongo.Connect(opt) - if err != nil { - isConnected = false - glog.Error(context.Background(), "MongoDB连接失败", err) - return err - } - - // 测试连接 - testCtx, testCancel := context.WithTimeout(context.Background(), 5*time.Second) - defer testCancel() - - err = client.Ping(testCtx, nil) - if err != nil { - isConnected = false - glog.Error(testCtx, "MongoDB连接测试失败", err) - return err - } - - db = client.Database(dbName) - isConnected = true - glog.Info(context.Background(), "✅ MongoDB连接成功") - return nil -} - -// GetDB 获取 MongoDB 数据库实例 -func GetDB() *mongo.Database { - mu.RLock() - defer mu.RUnlock() - return db -} - -// healthCheck 健康检查协程 -func healthCheck() { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - - for { - select { - case <-healthCtx.Done(): - return - case <-ticker.C: - mu.RLock() - currentConnected := isConnected - currentClient := client - mu.RUnlock() - - if !currentConnected || currentClient == nil { - glog.Warning(context.Background(), "MongoDB连接断开,尝试重连") - if err := reconnect(); err != nil { - glog.Error(context.Background(), "MongoDB重连失败", err) - } - continue - } - - // 测试连接状态 - testCtx, testCancel := context.WithTimeout(context.Background(), 5*time.Second) - err := currentClient.Ping(testCtx, nil) - testCancel() - - if err != nil { - mu.Lock() - isConnected = false - mu.Unlock() - glog.Warning(context.Background(), "MongoDB连接健康检查失败", err) - - // 尝试重连 - if err := reconnect(); err != nil { - glog.Error(context.Background(), "MongoDB重连失败", err) - } - } else { - glog.Debug(context.Background(), "MongoDB连接健康检查通过") - } - } - } -} - -// reconnect 重连函数 -func reconnect() error { - maxRetries := 3 - retryDelay := 2 * time.Second - - for i := 0; i < maxRetries; i++ { - glog.Info(context.Background(), fmt.Sprintf("尝试第%d次重连MongoDB", i+1)) - - if err := connect(); err == nil { - glog.Info(context.Background(), "MongoDB重连成功") - return nil - } - - if i < maxRetries-1 { - time.Sleep(retryDelay) - retryDelay *= 2 // 指数退避 - } - } - - return gerror.New("MongoDB重连失败,已达到最大重试次数") -} - -var logPool *grpool.Pool - -// init 初始化MongoDB连接 -func init() { - logPool = grpool.New(1) - // 按需初始化:没有配置 mongo.address 则跳过 - mongoAddr = g.Cfg().MustGet(context.Background(), "mongo.address").String() - if mongoAddr == "" { - return - } - - // 创建健康检查上下文 - healthCtx, healthCancel = context.WithCancel(context.Background()) - - // 从连接串中解析数据库名 - dbName = gstr.SubStr(mongoAddr, strings.LastIndex(mongoAddr, "/")+1, len(mongoAddr)) - // 如果连接串带有参数(如 ?retryWrites=true),需要去掉参数部分 - if strings.Contains(dbName, "?") { - dbName = gstr.SubStr(dbName, 0, strings.Index(dbName, "?")) - } - go func() { - // 初始连接 - if err := connect(); err != nil { - glog.Error(context.Background(), "MongoDB初始连接失败", err) - return - } - }() - - // 启动健康检查协程 - go healthCheck() -} - -// close 关闭MongoDB连接 -func close() { - if healthCancel != nil { - healthCancel() - } - - mu.Lock() - defer mu.Unlock() - - if client != nil { - disconnectCtx, disconnectCancel := context.WithTimeout(context.Background(), 5*time.Second) - defer disconnectCancel() - client.Disconnect(disconnectCtx) - } - - isConnected = false - glog.Info(context.Background(), "MongoDB连接已关闭") -} - const PageSize = 20 +// GetDB 获取默认数据源的数据库实例(向后兼容) +func GetDB() *mongo.Database { + source, err := manager.GetDataSource("default") + if err != nil { + return nil + } + return source.Database() +} + +// init 初始化多数据源 +func init() { + logPool = grpool.New(1) + serverName = g.Cfg().MustGet(context.TODO(), "server.name").String() + logRedisKey = fmt.Sprintf("log:%s", serverName) + + ctx := context.Background() + + // 从配置初始化多数据源 + if err := manager.InitializeFromConfig(ctx); err != nil { + glog.Errorf(ctx, "❌ Failed to initialize MongoDB datasources: %v", err) + } else { + glog.Infof(ctx, "✅ MongoDB datasources initialized: %v", manager.GetAllDataSourceNames()) + } + + // 启动健康检查 + manager.StartHealthCheck() + + // 设置优雅关闭 + setupGracefulShutdown() +} + +// setupGracefulShutdown 设置优雅关闭 +func setupGracefulShutdown() { + go func() { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + <-sigCh + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + glog.Info(ctx, "🔄 Shutting down MongoDB connections...") + if err := manager.CloseAll(ctx); err != nil { + glog.Errorf(ctx, "❌ Failed to close MongoDB connections: %v", err) + } else { + glog.Info(ctx, "✅ MongoDB connections closed successfully") + } + }() +} + +// ============================================================================= +// MongoDB 操作方法(支持多数据源) +// ============================================================================= + +// getDataSource 获取当前使用的数据源 +func (m *MongoDB) getDataSource() (DataSource, error) { + if m.dataSource == "" { + m.dataSource = "default" + } + return manager.GetDataSource(m.dataSource) +} + // Find 查询多条记录 func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, collection string, page *beans.Page, orderBy []beans.OrderBy) (total int64, err error) { + source, err := m.getDataSource() + if err != nil { + return 0, err + } + db := source.Database() + if err = utils.ValidStructPtr(result); err != nil { return } @@ -257,7 +525,6 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c } filter["tenantId"] = user.TenantId - // 分页参数处理 limit := int64(PageSize) skip := int64(0) if page != nil { @@ -283,9 +550,9 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c orderBson := bson.D{} for _, v := range orderBy { if v.Order == beans.Asc { - orderBson = append(orderBson, bson.E{Key: v.Field, Value: 1}) // 1 表示升序 + orderBson = append(orderBson, bson.E{Key: v.Field, Value: 1}) } else { - orderBson = append(orderBson, bson.E{Key: v.Field, Value: -1}) // -1 表示降序 + orderBson = append(orderBson, bson.E{Key: v.Field, Value: -1}) } } opt.SetSort(orderBson) @@ -313,6 +580,12 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c // FindOne 查询1条记录 func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOneOptions]) (err error) { + source, err := m.getDataSource() + if err != nil { + return err + } + db := source.Database() + if len(filter) == 0 { err = gerror.New("缺少查询条件") return @@ -357,6 +630,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} } return } + func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collection string) (err error) { listKeys := fmt.Sprintf(redis.CleanList, tenantId, collection) keys, err := redis.RedisClient.Keys(ctx, listKeys) @@ -391,9 +665,6 @@ func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interf return } -var serverName = g.Cfg().MustGet(context.TODO(), "server.name").String() -var logRedisKey = fmt.Sprintf("log:%s", serverName) - func (m *MongoDB) log(ctx context.Context, filter bson.M, collection string, data interface{}, userName, tenantId interface{}, operationType string) { _ = logPool.AddWithRecover(ctx, func(ctx context.Context) { log := &entity.OperationLog{ @@ -420,6 +691,12 @@ func (m *MongoDB) log(ctx context.Context, filter bson.M, collection string, dat // Delete 删除记录 func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.DeleteManyOptions]) (count int64, err error) { + source, err := m.getDataSource() + if err != nil { + return 0, err + } + db := source.Database() + if len(filter) == 0 { err = gerror.New("缺少查询条件") return @@ -435,13 +712,17 @@ func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, } count = r.DeletedCount err = m.CleanRedis(ctx, filter, user.TenantId, collection) - //写日志 - //m.log(ctx, filter, collection, nil, user.UserName, user.TenantId, "delete") return } // Update 修改记录 func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { + source, err := m.getDataSource() + if err != nil { + return nil, err + } + db := source.Database() + if len(filter) == 0 { err = gerror.New("缺少查询条件") return @@ -465,25 +746,23 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll return } err = m.CleanRedis(ctx, filter, user.TenantId, collection) - //写日志 - //m.log(ctx, filter, collection, update, user.UserName, user.TenantId, "update") return } // RandomSoftDelete 随机软删除个文档的 _id func (m *MongoDB) RandomSoftDelete(ctx context.Context, limit int, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { + source, err := m.getDataSource() + if err != nil { + return nil, err + } + db := source.Database() + _ = opts - // 步骤 1: 使用聚合管道的 $sample 操作符随机抽取5个文档的 _id pipeline := mongo.Pipeline{ - // 阶段1: 为每个文档添加一个 0-1 之间的随机数字段 'random' bson.D{{Key: "$addFields", Value: bson.D{{Key: "random", Value: bson.M{"$rand": bson.M{}}}}}}, - // 阶段1: 匹配所有未删除的文档 bson.D{{Key: "$match", Value: bson.D{{Key: "isDeleted", Value: false}}}}, - // 阶段2: 按随机数降序排序 bson.D{{Key: "$sort", Value: bson.D{{Key: "random", Value: -1}}}}, - // 阶段3: 只取前5个 bson.D{{Key: "$limit", Value: limit}}, - // 阶段4: 只投影 _id bson.D{{Key: "$project", Value: bson.D{{Key: "_id", Value: 1}}}}, } cursor, err := db.Collection(collection).Aggregate(ctx, pipeline) @@ -491,14 +770,13 @@ func (m *MongoDB) RandomSoftDelete(ctx context.Context, limit int, collection st return } defer cursor.Close(ctx) - // 步骤 2: 从聚合结果中提取 _id 到一个切片中 + var idsToUpdate []bson.ObjectID for cursor.Next(ctx) { var result bson.M if err := cursor.Decode(&result); err != nil { return nil, err } - // 将 bson.M 中的 _id 断言为 primitive.ObjectID id := result["_id"].(bson.ObjectID) idsToUpdate = append(idsToUpdate, id) } @@ -506,11 +784,9 @@ func (m *MongoDB) RandomSoftDelete(ctx context.Context, limit int, collection st return nil, err } fmt.Printf("准备更新的随机文档ID: %v\n", idsToUpdate) - // 步骤 3: 使用 $in 操作符和 UpdateMany 批量更新选定的文档 + if len(idsToUpdate) > 0 { - // 过滤条件:匹配 idsToUpdate 切片中的任意一个 _id filter := bson.D{{Key: "_id", Value: bson.D{{Key: "$in", Value: idsToUpdate}}}} - // 更新操作:使用 $set 修改字段 update := bson.D{{Key: "$set", Value: bson.D{{Key: "isDeleted", Value: true}}}} _, err = db.Collection(collection).UpdateMany(ctx, filter, update) if err != nil { @@ -522,6 +798,12 @@ func (m *MongoDB) RandomSoftDelete(ctx context.Context, limit int, collection st // SaveOrUpdate 批量增加或修改 func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.BulkWriteResult, err error) { + source, err := m.getDataSource() + if err != nil { + return nil, err + } + db := source.Database() + if len(filter) == 0 || len(update) == 0 { err = gerror.New("缺少查询条件或更新数据") return @@ -534,22 +816,19 @@ func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bs if err != nil { return } - // 构建批量操作模型 + var models []mongo.WriteModel for i := 0; i < len(filter); i++ { - // 处理过滤器 filter[i]["isDeleted"] = false if !g.IsEmpty(user.TenantId) { filter[i]["tenantId"] = user.TenantId } - // 处理更新数据 if setDoc, exists := update[i]["$set"].(bson.M); exists { if !g.IsEmpty(user.UserName) { setDoc["updater"] = user.UserName } setDoc["updatedAt"] = gtime.Now().Time } else { - // 如果没有$set字段,则创建一个 setDoc := bson.M{} if !g.IsEmpty(user.UserName) { setDoc["updater"] = user.UserName @@ -557,12 +836,10 @@ func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bs setDoc["updatedAt"] = gtime.Now().Time update[i]["$set"] = setDoc } - // 创建更新操作模型 updateModel := mongo.NewUpdateOneModel() updateModel.SetFilter(filter[i]) updateModel.SetUpdate(update[i]) - updateModel.SetUpsert(true) // 默认不插入新文档 - // 处理选项参数 + updateModel.SetUpsert(true) if len(opts) > 0 { for _, opt := range opts { var updateOpts options.UpdateManyOptions @@ -577,13 +854,13 @@ func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bs } models = append(models, updateModel) } - // 执行批量操作,无序执行提高性能 + bulkOpts := options.BulkWrite().SetOrdered(false) bulkResult, err := db.Collection(collection).BulkWrite(ctx, models, bulkOpts) if err != nil { return nil, err } - // 清理相关缓存 + for _, filterItem := range filter { err = m.CleanRedis(ctx, filterItem, user.TenantId, collection) if err != nil { @@ -595,6 +872,12 @@ func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bs // Insert 插入多条记录 func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { + source, err := m.getDataSource() + if err != nil { + return nil, err + } + db := source.Database() + user, err := utils.GetUserInfo(ctx) if err != nil { return @@ -623,13 +906,17 @@ func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collectio } ids = r.InsertedIDs err = m.CleanRedis(ctx, bson.M{}, user.TenantId, collection) - //写日志 - //m.log(ctx, nil, collection, ids, user.UserName, user.TenantId, "insert") return } -// Count 查询总数 +// Count 查询总数 func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { + source, err := m.getDataSource() + if err != nil { + return 0, err + } + db := source.Database() + user, err := utils.GetUserInfo(ctx) if err != nil { return @@ -649,7 +936,6 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( return } } - // 调用驱动的 CountDocuments,在数据库端执行的 count, err = db.Collection(collection).CountDocuments(ctx, filter) if m.Cache { err = redis.RedisClient.SetEX(ctx, redisKey, count, int64(time.Hour)) @@ -661,38 +947,26 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( } // EntityToBson 将 *entity/entity 转换为 bson.M -// 支持传入值类型或指针类型,返回 bson.M 和错误信息 func EntityToBson(entity interface{}) (bson.M, error) { return EntityToBsonWithFilter(entity, false) } // EntityToBsonWithFilter 将 *entity/entity 转换为 bson.M,并可选择是否过滤空值 -// filterEmpty: 为 true 时会过滤掉空值字段(nil、空字符串、空切片、空map等),但保留 int 类型的 0 值 -// 支持: -// - 未传值的指针(如 *consts.AssetStatus(nil))会被过滤 -// - 传值为0的指针(如 *consts.AssetStatus(0))不会被过滤 -// - 非0的整数值不会被过滤 func EntityToBsonWithFilter(entity interface{}, filterEmpty bool) (bson.M, error) { - // 第一步:判断入参是否为 nil 或无效类型 if entity == nil { return nil, fmt.Errorf("传入的 entity 实例为 nil") } - // 第二步:将 entity 序列化为 BSON 字节流 - // bson.Marshal 支持值类型和指针类型,会自动解析结构体的 bson 标签 bsonBytes, err := bson.Marshal(entity) if err != nil { return nil, fmt.Errorf("entity 序列化为 BSON 字节流失败:%w", err) } - // 第三步:将 BSON 字节流反序列化为 bson.M var bsonMap bson.M err = bson.Unmarshal(bsonBytes, &bsonMap) if err != nil { return nil, fmt.Errorf("BSON 字节流反序列化为 bson.M 失败:%w", err) } - // 如果需要过滤空值 if filterEmpty { for key, value := range bsonMap { - // 判断是否为空值,但保留 int 类型的 0 值 if isEmptyWithZero(value) { delete(bsonMap, key) } @@ -702,33 +976,24 @@ func EntityToBsonWithFilter(entity interface{}, filterEmpty bool) (bson.M, error } // isEmptyWithZero 判断是否为空值,但保留 int 类型的 0 值 -// 支持区分"未传值"和"传值为0"的情况: -// - *int(nil) 或 *consts.AssetStatus(nil) → 返回 true(过滤掉) -// - *int(0) 或 *consts.AssetStatus(0) → 返回 false(保留) -// - int(0) 或 consts.AssetStatus(0) → 返回 false(保留) func isEmptyWithZero(value interface{}) bool { - // 先检查 value 是否为 nil if value == nil { return true } rv := reflect.ValueOf(value) kind := rv.Kind() - // 处理 nil 指针 if kind == reflect.Ptr { if rv.IsNil() { return true } - // 判断时如果是指针,需要获取指向的值的类型 kind = rv.Elem().Kind() } - // 数字类型(int/uint/float)都保留,包括 0 值 switch kind { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Float32, reflect.Float64: return false default: - // 其他类型使用 g.IsEmpty 判断 return g.IsEmpty(value) } } From 65dc3d525dcebee5914f4fccdb021317bc8f5c42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 9 Jan 2026 21:08:31 +0800 Subject: [PATCH 164/225] =?UTF-8?q?mongo=E5=A4=9A=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E6=BA=90=E5=A2=9E=E5=BC=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/link.go | 420 +++++++++++++++++++++++++++++++++++++++++++++++++ mongo/mongo.go | 406 +---------------------------------------------- 2 files changed, 422 insertions(+), 404 deletions(-) create mode 100644 mongo/link.go diff --git a/mongo/link.go b/mongo/link.go new file mode 100644 index 0000000..77f1cec --- /dev/null +++ b/mongo/link.go @@ -0,0 +1,420 @@ +// ============================================================================= +// MongoDB 数据源连接管理 +// 负责数据源的连接、重连、健康检查和优雅关闭 +// ============================================================================= + +package mongo + +import ( + "context" + "fmt" + "os" + "os/signal" + "strings" + "sync" + "syscall" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/os/grpool" + "github.com/gogf/gf/v2/text/gstr" + "github.com/gogf/gf/v2/util/gconv" + "go.mongodb.org/mongo-driver/v2/mongo" + "go.mongodb.org/mongo-driver/v2/mongo/options" +) + +// ============================================================================= +// 数据源配置结构 +// ============================================================================= + +type DataSourceConfig struct { + Name string `json:"name"` + Address string `json:"address"` + Database string `json:"database"` + MaxPoolSize int32 `json:"maxPoolSize"` + MinPoolSize int32 `json:"minPoolSize"` + ConnectTimeout time.Duration `json:"connectTimeout"` +} + +// ============================================================================= +// 单个数据源接口 +// ============================================================================= + +type DataSource interface { + Name() string + Database() *mongo.Database + Client() *mongo.Client + IsConnected() bool + Connect(ctx context.Context) error + Reconnect(ctx context.Context) error + Close(ctx context.Context) error +} + +// ============================================================================= +// 数据源实现 +// ============================================================================= + +type BaseDataSource struct { + config *DataSourceConfig + client *mongo.Client + database *mongo.Database + isConnected bool + mu sync.RWMutex + lastError error + lastErrorTime time.Time +} + +func NewBaseDataSource(config *DataSourceConfig) *BaseDataSource { + return &BaseDataSource{ + config: config, + isConnected: false, + } +} + +func (d *BaseDataSource) Name() string { + return d.config.Name +} + +func (d *BaseDataSource) Database() *mongo.Database { + d.mu.RLock() + defer d.mu.RUnlock() + return d.database +} + +func (d *BaseDataSource) Client() *mongo.Client { + d.mu.RLock() + defer d.mu.RUnlock() + return d.client +} + +func (d *BaseDataSource) IsConnected() bool { + d.mu.RLock() + defer d.mu.RUnlock() + return d.isConnected && d.client != nil +} + +func (d *BaseDataSource) Connect(ctx context.Context) error { + d.mu.Lock() + defer d.mu.Unlock() + + if d.client != nil { + d.client.Disconnect(ctx) + } + + // 解析数据库名 + dbName := d.config.Database + if strings.Contains(dbName, "?") { + dbName = gstr.SubStr(dbName, 0, strings.Index(dbName, "?")) + } + + // 构建连接选项 + opt := options.Client(). + ApplyURI(d.config.Address). + SetMaxPoolSize(uint64(d.config.MaxPoolSize)). + SetMinPoolSize(uint64(d.config.MinPoolSize)). + SetConnectTimeout(d.config.ConnectTimeout). + SetMaxConnecting(10). + SetServerSelectionTimeout(10 * time.Second). + SetHeartbeatInterval(10 * time.Second). + SetMaxConnIdleTime(60 * time.Second). + SetRetryWrites(true). + SetRetryReads(true) + + var err error + d.client, err = mongo.Connect(opt) + if err != nil { + d.isConnected = false + d.lastError = err + d.lastErrorTime = time.Now() + return fmt.Errorf("datasource [%s] connection failed: %w", d.config.Name, err) + } + + // 测试连接 + pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err = d.client.Ping(pingCtx, nil); err != nil { + d.isConnected = false + d.lastError = err + d.lastErrorTime = time.Now() + return fmt.Errorf("datasource [%s] ping failed: %w", d.config.Name, err) + } + + d.database = d.client.Database(dbName) + d.isConnected = true + d.lastError = nil + glog.Infof(ctx, "✅ datasource [%s] connected successfully", d.config.Name) + return nil +} + +func (d *BaseDataSource) Reconnect(ctx context.Context) error { + glog.Infof(ctx, "🔄 reconnecting datasource [%s]", d.config.Name) + return d.Connect(ctx) +} + +func (d *BaseDataSource) Close(ctx context.Context) error { + d.mu.Lock() + defer d.mu.Unlock() + + if d.client != nil { + disconnectCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := d.client.Disconnect(disconnectCtx); err != nil { + return fmt.Errorf("datasource [%s] close failed: %w", d.config.Name, err) + } + } + + d.isConnected = false + glog.Infof(ctx, "datasource [%s] closed", d.config.Name) + return nil +} + +// ============================================================================= +// 多数据源管理器 +// ============================================================================= + +type DataSourceManager struct { + sources map[string]DataSource + mu sync.RWMutex + ctx context.Context + cancel context.CancelFunc + started bool + maxRetries int +} + +var ( + globalManager *DataSourceManager + managerOnce sync.Once +) + +// GetManager 获取全局管理器 +func GetManager() *DataSourceManager { + managerOnce.Do(func() { + ctx, cancel := context.WithCancel(context.Background()) + globalManager = &DataSourceManager{ + sources: make(map[string]DataSource), + ctx: ctx, + cancel: cancel, + started: false, + maxRetries: 3, + } + }) + return globalManager +} + +// RegisterDataSource 注册数据源 +func (m *DataSourceManager) RegisterDataSource(config *DataSourceConfig) error { + m.mu.Lock() + defer m.mu.Unlock() + + if _, exists := m.sources[config.Name]; exists { + return fmt.Errorf("datasource [%s] already exists", config.Name) + } + + source := NewBaseDataSource(config) + m.sources[config.Name] = source + return nil +} + +// GetDataSource 获取数据源 +func (m *DataSourceManager) GetDataSource(name string) (DataSource, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + source, exists := m.sources[name] + if !exists { + return nil, fmt.Errorf("datasource [%s] not found", name) + } + return source, nil +} + +// GetAllDataSourceNames 获取所有数据源名称 +func (m *DataSourceManager) GetAllDataSourceNames() []string { + m.mu.RLock() + defer m.mu.RUnlock() + + names := make([]string, 0, len(m.sources)) + for name := range m.sources { + names = append(names, name) + } + return names +} + +// init 初始化多数据源 +func init() { + logPool = grpool.New(1) + serverName = g.Cfg().MustGet(context.TODO(), "server.name").String() + logRedisKey = fmt.Sprintf("log:%s", serverName) + + ctx := context.Background() + + // 从配置初始化多数据源 + if err := manager.InitializeFromConfig(ctx); err != nil { + glog.Errorf(ctx, "❌ Failed to initialize MongoDB datasources: %v", err) + } else { + glog.Infof(ctx, "✅ MongoDB datasources initialized: %v", manager.GetAllDataSourceNames()) + } + + // 启动健康检查 + manager.StartHealthCheck() + + // 设置优雅关闭 + setupGracefulShutdown() +} + +// InitializeFromConfig 从配置初始化数据源 +// 动态读取 config.yml 中 mongo 下的所有配置项 +func (m *DataSourceManager) InitializeFromConfig(ctx context.Context) error { + var firstErr error + + // 获取 mongo 配置下的所有子键 + mongoConfig := g.Cfg().MustGet(ctx, "mongo") + if mongoConfig.IsNil() { + glog.Warningf(ctx, "no mongo configuration found in config.yml") + return nil + } + + // 将配置转换为 map + configMap := mongoConfig.Map() + if configMap == nil { + glog.Warningf(ctx, "mongo configuration is not a map") + return nil + } + + // 遍历所有 mongo 子配置 + for name, subConfig := range configMap { + // 跳过非对象类型的配置 + subMap, ok := subConfig.(map[string]interface{}) + if !ok { + continue + } + + // 检查是否有 address 配置 + address, hasAddress := subMap["address"] + if !hasAddress || gconv.String(address) == "" { + continue + } + + // 构建数据源配置 + config := &DataSourceConfig{ + Name: name, + Address: gconv.String(address), + Database: gconv.String(subMap["database"]), + MaxPoolSize: int32(gconv.Int(subMap["maxPoolSize"])), + MinPoolSize: int32(gconv.Int(subMap["minPoolSize"])), + ConnectTimeout: gconv.Duration(subMap["connectTimeout"]), + } + + // 设置默认值 + if config.MaxPoolSize == 0 { + config.MaxPoolSize = 100 + } + if config.MinPoolSize == 0 { + config.MinPoolSize = 10 + } + if config.ConnectTimeout == 0 { + config.ConnectTimeout = 10 * time.Second + } + + // 注册数据源 + if err := m.RegisterDataSource(config); err != nil { + glog.Errorf(ctx, "failed to register datasource [%s]: %v", name, err) + if firstErr == nil { + firstErr = err + } + continue + } + + // 连接数据源 + source, _ := m.GetDataSource(name) + if err := source.Connect(ctx); err != nil { + glog.Errorf(ctx, "failed to initialize datasource [%s]: %v", name, err) + if firstErr == nil { + firstErr = err + } + } + } + + return firstErr +} + +// StartHealthCheck 启动健康检查 +func (m *DataSourceManager) StartHealthCheck() { + if m.started { + return + } + m.started = true + go m.healthCheckLoop() +} + +// healthCheckLoop 健康检查循环 +func (m *DataSourceManager) healthCheckLoop() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-m.ctx.Done(): + return + case <-ticker.C: + m.checkAndReconnect() + } + } +} + +// checkAndReconnect 检查并重新连接 +func (m *DataSourceManager) checkAndReconnect() { + m.mu.RLock() + defer m.mu.RUnlock() + + for name, source := range m.sources { + if !source.IsConnected() { + glog.Warningf(context.Background(), "datasource [%s] disconnected, attempting reconnect", name) + + reconnectCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := source.Reconnect(reconnectCtx); err != nil { + glog.Errorf(reconnectCtx, "datasource [%s] reconnect failed: %v", name, err) + } else { + glog.Infof(reconnectCtx, "✅ datasource [%s] reconnected successfully", name) + } + } + } +} + +// CloseAll 关闭所有数据源 +func (m *DataSourceManager) CloseAll(ctx context.Context) error { + m.cancel() + + m.mu.RLock() + defer m.mu.RUnlock() + + var lastErr error + for name, source := range m.sources { + if err := source.Close(ctx); err != nil { + glog.Errorf(ctx, "failed to close datasource [%s]: %v", name, err) + lastErr = err + } + } + return lastErr +} + +// setupGracefulShutdown 设置优雅关闭 +func setupGracefulShutdown() { + go func() { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + <-sigCh + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + glog.Info(ctx, "🔄 Shutting down MongoDB connections...") + if err := manager.CloseAll(ctx); err != nil { + glog.Errorf(ctx, "❌ Failed to close MongoDB connections: %v", err) + } else { + glog.Info(ctx, "✅ MongoDB connections closed successfully") + } + }() +} diff --git a/mongo/mongo.go b/mongo/mongo.go index 3a04d72..c510ea5 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -1,7 +1,6 @@ // ============================================================================= -// MongoDB 多数据源支持 -// 支持多数据源配置、自动重连、优雅关闭 -// 向后兼容原有的单数据源API +// MongoDB 业务操作封装 +// 提供向后兼容的CRUD操作方法,支持多数据源 // ============================================================================= package mongo @@ -10,12 +9,7 @@ import ( "context" "errors" "fmt" - "os" - "os/signal" "reflect" - "strings" - "sync" - "syscall" "time" "gitee.com/red-future---jilin-g/common/beans" @@ -28,367 +22,12 @@ import ( "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/os/grpool" "github.com/gogf/gf/v2/os/gtime" - "github.com/gogf/gf/v2/text/gstr" "github.com/gogf/gf/v2/util/gconv" "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/mongo" "go.mongodb.org/mongo-driver/v2/mongo/options" ) -// ============================================================================= -// 数据源配置结构 -// ============================================================================= - -type DataSourceConfig struct { - Name string `json:"name"` - Address string `json:"address"` - Database string `json:"database"` - MaxPoolSize int32 `json:"maxPoolSize"` - MinPoolSize int32 `json:"minPoolSize"` - ConnectTimeout time.Duration `json:"connectTimeout"` -} - -// ============================================================================= -// 单个数据源接口 -// ============================================================================= - -type DataSource interface { - Name() string - Database() *mongo.Database - Client() *mongo.Client - IsConnected() bool - Connect(ctx context.Context) error - Reconnect(ctx context.Context) error - Close(ctx context.Context) error -} - -// ============================================================================= -// 数据源实现 -// ============================================================================= - -type BaseDataSource struct { - config *DataSourceConfig - client *mongo.Client - database *mongo.Database - isConnected bool - mu sync.RWMutex - lastError error - lastErrorTime time.Time -} - -func NewBaseDataSource(config *DataSourceConfig) *BaseDataSource { - return &BaseDataSource{ - config: config, - isConnected: false, - } -} - -func (d *BaseDataSource) Name() string { - return d.config.Name -} - -func (d *BaseDataSource) Database() *mongo.Database { - d.mu.RLock() - defer d.mu.RUnlock() - return d.database -} - -func (d *BaseDataSource) Client() *mongo.Client { - d.mu.RLock() - defer d.mu.RUnlock() - return d.client -} - -func (d *BaseDataSource) IsConnected() bool { - d.mu.RLock() - defer d.mu.RUnlock() - return d.isConnected && d.client != nil -} - -func (d *BaseDataSource) Connect(ctx context.Context) error { - d.mu.Lock() - defer d.mu.Unlock() - - if d.client != nil { - d.client.Disconnect(ctx) - } - - // 解析数据库名 - dbName := d.config.Database - if strings.Contains(dbName, "?") { - dbName = gstr.SubStr(dbName, 0, strings.Index(dbName, "?")) - } - - // 构建连接选项 - opt := options.Client(). - ApplyURI(d.config.Address). - SetMaxPoolSize(uint64(d.config.MaxPoolSize)). - SetMinPoolSize(uint64(d.config.MinPoolSize)). - SetConnectTimeout(d.config.ConnectTimeout). - SetMaxConnecting(10). - SetServerSelectionTimeout(10 * time.Second). - SetHeartbeatInterval(10 * time.Second). - SetMaxConnIdleTime(60 * time.Second). - SetRetryWrites(true). - SetRetryReads(true) - - var err error - d.client, err = mongo.Connect(opt) - if err != nil { - d.isConnected = false - d.lastError = err - d.lastErrorTime = time.Now() - return fmt.Errorf("datasource [%s] connection failed: %w", d.config.Name, err) - } - - // 测试连接 - pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - if err = d.client.Ping(pingCtx, nil); err != nil { - d.isConnected = false - d.lastError = err - d.lastErrorTime = time.Now() - return fmt.Errorf("datasource [%s] ping failed: %w", d.config.Name, err) - } - - d.database = d.client.Database(dbName) - d.isConnected = true - d.lastError = nil - glog.Infof(ctx, "✅ datasource [%s] connected successfully", d.config.Name) - return nil -} - -func (d *BaseDataSource) Reconnect(ctx context.Context) error { - glog.Infof(ctx, "🔄 reconnecting datasource [%s]", d.config.Name) - return d.Connect(ctx) -} - -func (d *BaseDataSource) Close(ctx context.Context) error { - d.mu.Lock() - defer d.mu.Unlock() - - if d.client != nil { - disconnectCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - if err := d.client.Disconnect(disconnectCtx); err != nil { - return fmt.Errorf("datasource [%s] close failed: %w", d.config.Name, err) - } - } - - d.isConnected = false - glog.Infof(ctx, "datasource [%s] closed", d.config.Name) - return nil -} - -// ============================================================================= -// 多数据源管理器 -// ============================================================================= - -type DataSourceManager struct { - sources map[string]DataSource - mu sync.RWMutex - ctx context.Context - cancel context.CancelFunc - started bool - maxRetries int -} - -var ( - globalManager *DataSourceManager - managerOnce sync.Once -) - -// GetManager 获取全局管理器 -func GetManager() *DataSourceManager { - managerOnce.Do(func() { - ctx, cancel := context.WithCancel(context.Background()) - globalManager = &DataSourceManager{ - sources: make(map[string]DataSource), - ctx: ctx, - cancel: cancel, - started: false, - maxRetries: 3, - } - }) - return globalManager -} - -// RegisterDataSource 注册数据源 -func (m *DataSourceManager) RegisterDataSource(config *DataSourceConfig) error { - m.mu.Lock() - defer m.mu.Unlock() - - if _, exists := m.sources[config.Name]; exists { - return fmt.Errorf("datasource [%s] already exists", config.Name) - } - - source := NewBaseDataSource(config) - m.sources[config.Name] = source - return nil -} - -// GetDataSource 获取数据源 -func (m *DataSourceManager) GetDataSource(name string) (DataSource, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - source, exists := m.sources[name] - if !exists { - return nil, fmt.Errorf("datasource [%s] not found", name) - } - return source, nil -} - -// GetAllDataSourceNames 获取所有数据源名称 -func (m *DataSourceManager) GetAllDataSourceNames() []string { - m.mu.RLock() - defer m.mu.RUnlock() - - names := make([]string, 0, len(m.sources)) - for name := range m.sources { - names = append(names, name) - } - return names -} - -// InitializeFromConfig 从配置初始化数据源 -// 动态读取 config.yml 中 mongo 下的所有配置项 -func (m *DataSourceManager) InitializeFromConfig(ctx context.Context) error { - var firstErr error - - // 获取 mongo 配置下的所有子键 - mongoConfig := g.Cfg().MustGet(ctx, "mongo") - if mongoConfig.IsNil() { - glog.Warningf(ctx, "no mongo configuration found in config.yml") - return nil - } - - // 将配置转换为 map - configMap := mongoConfig.Map() - if configMap == nil { - glog.Warningf(ctx, "mongo configuration is not a map") - return nil - } - - // 遍历所有 mongo 子配置 - for name, subConfig := range configMap { - // 跳过非对象类型的配置 - subMap, ok := subConfig.(map[string]interface{}) - if !ok { - continue - } - - // 检查是否有 address 配置 - address, hasAddress := subMap["address"] - if !hasAddress || gconv.String(address) == "" { - continue - } - - // 构建数据源配置 - config := &DataSourceConfig{ - Name: name, - Address: gconv.String(address), - Database: gconv.String(subMap["database"]), - MaxPoolSize: int32(gconv.Int(subMap["maxPoolSize"])), - MinPoolSize: int32(gconv.Int(subMap["minPoolSize"])), - ConnectTimeout: gconv.Duration(subMap["connectTimeout"]), - } - - // 设置默认值 - if config.MaxPoolSize == 0 { - config.MaxPoolSize = 100 - } - if config.MinPoolSize == 0 { - config.MinPoolSize = 10 - } - if config.ConnectTimeout == 0 { - config.ConnectTimeout = 10 * time.Second - } - - // 注册数据源 - if err := m.RegisterDataSource(config); err != nil { - glog.Errorf(ctx, "failed to register datasource [%s]: %v", name, err) - if firstErr == nil { - firstErr = err - } - continue - } - - // 连接数据源 - source, _ := m.GetDataSource(name) - if err := source.Connect(ctx); err != nil { - glog.Errorf(ctx, "failed to initialize datasource [%s]: %v", name, err) - if firstErr == nil { - firstErr = err - } - } - } - - return firstErr -} - -// StartHealthCheck 启动健康检查 -func (m *DataSourceManager) StartHealthCheck() { - if m.started { - return - } - m.started = true - go m.healthCheckLoop() -} - -// healthCheckLoop 健康检查循环 -func (m *DataSourceManager) healthCheckLoop() { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - - for { - select { - case <-m.ctx.Done(): - return - case <-ticker.C: - m.checkAndReconnect() - } - } -} - -// checkAndReconnect 检查并重新连接 -func (m *DataSourceManager) checkAndReconnect() { - m.mu.RLock() - defer m.mu.RUnlock() - - for name, source := range m.sources { - if !source.IsConnected() { - glog.Warningf(context.Background(), "datasource [%s] disconnected, attempting reconnect", name) - - reconnectCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - if err := source.Reconnect(reconnectCtx); err != nil { - glog.Errorf(reconnectCtx, "datasource [%s] reconnect failed: %v", name, err) - } else { - glog.Infof(reconnectCtx, "✅ datasource [%s] reconnected successfully", name) - } - } - } -} - -// CloseAll 关闭所有数据源 -func (m *DataSourceManager) CloseAll(ctx context.Context) error { - m.cancel() - - m.mu.RLock() - defer m.mu.RUnlock() - - var lastErr error - for name, source := range m.sources { - if err := source.Close(ctx); err != nil { - glog.Errorf(ctx, "failed to close datasource [%s]: %v", name, err) - lastErr = err - } - } - return lastErr -} - // ============================================================================= // 向后兼容的MongoDB结构体 // ============================================================================= @@ -437,47 +76,6 @@ func GetDB() *mongo.Database { return source.Database() } -// init 初始化多数据源 -func init() { - logPool = grpool.New(1) - serverName = g.Cfg().MustGet(context.TODO(), "server.name").String() - logRedisKey = fmt.Sprintf("log:%s", serverName) - - ctx := context.Background() - - // 从配置初始化多数据源 - if err := manager.InitializeFromConfig(ctx); err != nil { - glog.Errorf(ctx, "❌ Failed to initialize MongoDB datasources: %v", err) - } else { - glog.Infof(ctx, "✅ MongoDB datasources initialized: %v", manager.GetAllDataSourceNames()) - } - - // 启动健康检查 - manager.StartHealthCheck() - - // 设置优雅关闭 - setupGracefulShutdown() -} - -// setupGracefulShutdown 设置优雅关闭 -func setupGracefulShutdown() { - go func() { - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - <-sigCh - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - glog.Info(ctx, "🔄 Shutting down MongoDB connections...") - if err := manager.CloseAll(ctx); err != nil { - glog.Errorf(ctx, "❌ Failed to close MongoDB connections: %v", err) - } else { - glog.Info(ctx, "✅ MongoDB connections closed successfully") - } - }() -} - // ============================================================================= // MongoDB 操作方法(支持多数据源) // ============================================================================= From bf8b220a99b30cbcd1d556921081b221752db319 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Mon, 12 Jan 2026 17:36:14 +0800 Subject: [PATCH 165/225] =?UTF-8?q?=E5=85=BC=E5=AE=B9=E6=96=B0=E7=9A=84mon?= =?UTF-8?q?go=E6=96=B9=E6=B3=95=E6=A0=BC=E5=BC=8F,=E5=B9=B6=E4=B8=94?= =?UTF-8?q?=E4=BF=AE=E5=A4=8Dclient=E7=9A=84apitoken=E8=A6=86=E7=9B=96?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/chat.go | 12 ++++++--- ragflow/client.go | 62 ++++++++++++++++++++++++++++++++++++++-------- ragflow/dataset.go | 30 ++++++++++++++++++++-- ragflow/session.go | 5 ++++ 4 files changed, 94 insertions(+), 15 deletions(-) diff --git a/ragflow/chat.go b/ragflow/chat.go index 31e1b4b..bdce50b 100644 --- a/ragflow/chat.go +++ b/ragflow/chat.go @@ -34,9 +34,10 @@ type CreateChatRes struct { // UpdateChatReq 更新对话配置请求 type UpdateChatReq struct { - Name string `json:"name,omitempty"` // 对话配置名称 - DatasetIds []string `json:"dataset_ids,omitempty"` // 关联的知识库ID列表 - Prompt *PromptConfig `json:"prompt,omitempty"` // 提示词配置 + Name string `json:"name,omitempty"` // 对话配置名称 + Description string `json:"description,omitempty"` // 对话描述 + DatasetIds []string `json:"dataset_ids,omitempty"` // 关联的知识库ID列表(RAGFlow API使用下划线格式) + Prompt *PromptConfig `json:"prompt,omitempty"` // 提示词配置 } // 聊天助手管理 @@ -122,6 +123,11 @@ func (c *Client) CreateChat(ctx context.Context, req *CreateChatReq) (*Chat, err if res.Code != 0 { return nil, gerror.Newf("create chat failed: %s", res.Msg) } + // 检查响应数据是否为空:防止RAGFlow API返回 {"code":0, "data":null} + // 如果不检查直接返回,调用方会收到 (nil, nil),导致空指针异常 + if res.Data == nil { + return nil, gerror.Newf("create chat returned null data: %s", res.Msg) + } return res.Data, nil } diff --git a/ragflow/client.go b/ragflow/client.go index d5958bf..24a426a 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -7,9 +7,10 @@ import ( "sync" "sync/atomic" - commonHttp "gitee.com/red-future---jilin-g/common/http" + "github.com/gogf/gf/v2/encoding/gjson" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/gclient" ) var ( @@ -116,7 +117,23 @@ func (r *CommonResponse) IsSuccess() bool { return r.Code == 0 } -// request 发送 HTTP 请求(使用统一的common/http包,支持负载均衡) +// request 发送 HTTP 请求 +// +// 为什么不使用 common/http 包: +// 1. common/http/http.go:61 会用内部请求的Authorization覆盖RAGFlow API key: +// Httpclient.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) +// 这会导致RAGFlow API认证失败,因为内部token不是RAGFlow的API key +// +// 2. common/http/http.go:69-74 强制解析为内部API响应格式(ghttp.DefaultHandlerResponse): +// resultStrut := &ghttp.DefaultHandlerResponse{} +// if err = gconv.Struct(result, &resultStrut); err != nil { +// err = errors.New(resultStrut.Message) +// } else if resultStrut.Code == 200 || resultStrut.Code == 0 { +// gconv.Struct(resultStrut.Data, target) +// } +// RAGFlow API返回格式与内部API不同,会导致解析失败 +// +// 因此直接使用 g.Client() 调用第三方API,避免上述问题 func (c *Client) request(ctx context.Context, method, path string, body interface{}, result interface{}) (err error) { endpoint := c.getNextEndpoint() if endpoint == "" { @@ -124,26 +141,51 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac } fullURL := endpoint + path - headers := map[string]string{ - "Authorization": "Bearer " + c.APIKey, - "Content-Type": "application/json", + // 添加详细日志:请求信息 + g.Log().Infof(ctx, "RAGFlow请求: %s %s", method, fullURL) + if body != nil { + bodyJSON := g.NewVar(body).String() + g.Log().Infof(ctx, "RAGFlow请求体: %s", bodyJSON) } + // 创建新的HTTP客户端实例(避免共享状态) + client := g.Client() + client.SetHeader("Authorization", "Bearer "+c.APIKey) + client.SetHeader("Content-Type", "application/json") + + var response *gclient.Response switch method { case "GET": - err = commonHttp.Get(ctx, fullURL, headers, result, body) + response, err = client.Get(ctx, fullURL, body) case "POST": - err = commonHttp.Post(ctx, fullURL, headers, result, body) + response, err = client.Post(ctx, fullURL, body) case "PUT": - err = commonHttp.Put(ctx, fullURL, headers, result, body) + response, err = client.Put(ctx, fullURL, body) case "DELETE": - err = commonHttp.Delete(ctx, fullURL, headers, result, body) + response, err = client.Delete(ctx, fullURL, body) default: return gerror.Newf("unsupported method: %s", method) } if err != nil { - return gerror.Newf("RAGFlow API request failed: %v", err) + g.Log().Errorf(ctx, "RAGFlow HTTP请求失败: %v", err) + return gerror.Wrapf(err, "HTTP request to RAGFlow failed") + } + + if response == nil { + return gerror.New("HTTP response is nil") + } + + defer response.Close() + + // 读取响应体 + respBytes := response.ReadAll() + g.Log().Infof(ctx, "RAGFlow响应: %s", string(respBytes)) + + // 解析JSON到result + if err = gjson.DecodeTo(respBytes, result); err != nil { + g.Log().Errorf(ctx, "RAGFlow响应解析失败: %v, 原始响应: %s", err, string(respBytes)) + return gerror.Wrapf(err, "failed to decode RAGFlow response") } return diff --git a/ragflow/dataset.go b/ragflow/dataset.go index 431b332..b74970b 100644 --- a/ragflow/dataset.go +++ b/ragflow/dataset.go @@ -4,6 +4,7 @@ import ( "context" "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" ) // 数据集管理 @@ -82,17 +83,38 @@ type DeleteDatasetsReq struct { // CreateDataset 创建数据集 func (c *Client) CreateDataset(ctx context.Context, req *CreateDatasetReq) (*Dataset, error) { + g.Log().Infof(ctx, "CreateDataset请求: name=%s, description=%s, embedding_model=%s", req.Name, req.Description, req.EmbeddingModel) + var res struct { Code int `json:"code"` Data *Dataset `json:"data"` Msg string `json:"message"` } if err := c.request(ctx, "POST", "/api/v1/datasets", req, &res); err != nil { + g.Log().Errorf(ctx, "CreateDataset请求失败: %v", err) return nil, err } - if res.Code != 0 { - return nil, gerror.Newf("create dataset failed: %s", res.Msg) + + g.Log().Infof(ctx, "CreateDataset响应: code=%d, msg=%s, data_is_nil=%v", res.Code, res.Msg, res.Data == nil) + + // code=101表示dataset名称已存在(正常业务场景,不是错误) + // 调用方应该通过ListDatasets查找已有dataset并复用 + if res.Code == 101 { + return nil, gerror.Newf("Dataset名称已存在: %s", res.Msg) } + + // 其他非0的code表示真正的错误 + if res.Code != 0 { + return nil, gerror.Newf("创建知识库失败(code=%d): %s", res.Code, res.Msg) + } + + // code=0但data=null,表示创建异常(可能是RAGFlow配置问题,如embedding模型不可用、权限不足等) + // 这不是正常状态,应该返回错误而不是(nil, nil) + if res.Data == nil { + return nil, gerror.Newf("创建知识库返回空数据(code=0,data=null),可能是RAGFlow配置问题: %s", res.Msg) + } + + g.Log().Infof(ctx, "CreateDataset成功: id=%s, name=%s", res.Data.Id, res.Data.Name) return res.Data, nil } @@ -134,6 +156,10 @@ func (c *Client) ListDatasets(ctx context.Context, req *ListDatasetsReq) (*ListD if err := c.request(ctx, "GET", path, nil, &res); err != nil { return nil, err } + + // 添加调试日志 + g.Log().Infof(ctx, "ListDatasets原始响应: code=%d, total=%d, data_len=%d", res.Code, res.Total, len(res.Data)) + if res.Code != 0 { return nil, gerror.Newf("list datasets failed: code=%d", res.Code) } diff --git a/ragflow/session.go b/ragflow/session.go index d534b04..7659eb2 100644 --- a/ragflow/session.go +++ b/ragflow/session.go @@ -90,6 +90,11 @@ func (c *Client) CreateSession(ctx context.Context, chatId string, req *CreateSe if res.Code != 0 { return nil, gerror.Newf("create session failed: %s", res.Msg) } + // 检查响应数据是否为空:防止RAGFlow API返回 {"code":0, "data":null} + // 如果不检查直接返回,调用方会收到 (nil, nil),导致空指针异常 + if res.Data == nil { + return nil, gerror.Newf("create session returned null data: %s", res.Msg) + } return res.Data, nil } From 830d07c23dc722557b63bc9e27f2fe53c8ab482f Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 13 Jan 2026 10:16:35 +0800 Subject: [PATCH 166/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ragflow/client.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ragflow/client.go b/ragflow/client.go index 24a426a..41c7681 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -120,6 +120,7 @@ func (r *CommonResponse) IsSuccess() bool { // request 发送 HTTP 请求 // // 为什么不使用 common/http 包: +// // 1. common/http/http.go:61 会用内部请求的Authorization覆盖RAGFlow API key: // Httpclient.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) // 这会导致RAGFlow API认证失败,因为内部token不是RAGFlow的API key @@ -162,7 +163,12 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac case "PUT": response, err = client.Put(ctx, fullURL, body) case "DELETE": - response, err = client.Delete(ctx, fullURL, body) + // DELETE请求需要明确使用ContentJson发送body + if body != nil { + response, err = client.ContentJson().Delete(ctx, fullURL, body) + } else { + response, err = client.Delete(ctx, fullURL) + } default: return gerror.Newf("unsupported method: %s", method) } From a101d60df85cc2d142dbf61e5012c5cf1635d0b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 13 Jan 2026 11:08:17 +0800 Subject: [PATCH 167/225] =?UTF-8?q?ragflow=20http=E8=AF=B7=E6=B1=82=20head?= =?UTF-8?q?er=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 22 +++++++++++++++------- ragflow/client.go | 45 ++++++++++----------------------------------- 2 files changed, 25 insertions(+), 42 deletions(-) diff --git a/http/http.go b/http/http.go index 1d0edad..8f6f8af 100644 --- a/http/http.go +++ b/http/http.go @@ -58,19 +58,27 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin if err != nil { return } - Httpclient.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) - Httpclient.SetHeaderMap(headers) - response, err := Httpclient.DoRequest(ctx, method, url, data) + client := Httpclient + if len(data) > 0 { + client = Httpclient.Clone() + } + if len(headers) > 0 { + client.SetHeaderMap(headers) + } else { + client.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) + } + response, err := client.DoRequest(ctx, method, url, data...) if err != nil { return } defer response.Close() result := response.ReadAll() resultStrut := &ghttp.DefaultHandlerResponse{} - if err = gconv.Struct(result, &resultStrut); err != nil { - err = errors.New(resultStrut.Message) - } else if resultStrut.Code == 200 || resultStrut.Code == 0 { + gconv.Struct(result, &resultStrut) + if resultStrut.Code == 200 || resultStrut.Code == 0 { gconv.Struct(resultStrut.Data, target) + } else { + err = errors.New(resultStrut.Message) } return } @@ -87,6 +95,6 @@ func Put(ctx context.Context, url string, headers map[string]string, target any, return } func Delete(ctx context.Context, url string, headers map[string]string, target any, data ...any) (err error) { - err = doRequest(ctx, http.MethodPut, url, headers, target, data) + err = doRequest(ctx, http.MethodDelete, url, headers, target, data) return } diff --git a/ragflow/client.go b/ragflow/client.go index 41c7681..91601ad 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -7,10 +7,9 @@ import ( "sync" "sync/atomic" - "github.com/gogf/gf/v2/encoding/gjson" + "gitee.com/red-future---jilin-g/common/http" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/net/gclient" ) var ( @@ -150,50 +149,26 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac } // 创建新的HTTP客户端实例(避免共享状态) - client := g.Client() - client.SetHeader("Authorization", "Bearer "+c.APIKey) - client.SetHeader("Content-Type", "application/json") - - var response *gclient.Response + var headers = make(map[string]string) + headers["Authorization"] = "Bearer " + c.APIKey + headers["Content-Type"] = "application/json" switch method { case "GET": - response, err = client.Get(ctx, fullURL, body) + err = http.Get(ctx, fullURL, headers, result, body) case "POST": - response, err = client.Post(ctx, fullURL, body) + err = http.Post(ctx, fullURL, headers, result, body) case "PUT": - response, err = client.Put(ctx, fullURL, body) + err = http.Put(ctx, fullURL, headers, result, body) case "DELETE": - // DELETE请求需要明确使用ContentJson发送body + if body != nil { - response, err = client.ContentJson().Delete(ctx, fullURL, body) + err = http.Delete(ctx, fullURL, headers, result, body) } else { - response, err = client.Delete(ctx, fullURL) + err = http.Delete(ctx, fullURL, headers, result) } default: return gerror.Newf("unsupported method: %s", method) } - - if err != nil { - g.Log().Errorf(ctx, "RAGFlow HTTP请求失败: %v", err) - return gerror.Wrapf(err, "HTTP request to RAGFlow failed") - } - - if response == nil { - return gerror.New("HTTP response is nil") - } - - defer response.Close() - - // 读取响应体 - respBytes := response.ReadAll() - g.Log().Infof(ctx, "RAGFlow响应: %s", string(respBytes)) - - // 解析JSON到result - if err = gjson.DecodeTo(respBytes, result); err != nil { - g.Log().Errorf(ctx, "RAGFlow响应解析失败: %v, 原始响应: %s", err, string(respBytes)) - return gerror.Wrapf(err, "failed to decode RAGFlow response") - } - return } From e7e450f7cf04db569591ba4edc4efc5710145b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 13 Jan 2026 11:20:14 +0800 Subject: [PATCH 168/225] =?UTF-8?q?ragflow=20http=E8=AF=B7=E6=B1=82=20head?= =?UTF-8?q?er=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 1 + ragflow/client.go | 1 + 2 files changed, 2 insertions(+) diff --git a/http/http.go b/http/http.go index 8f6f8af..a63eb8a 100644 --- a/http/http.go +++ b/http/http.go @@ -74,6 +74,7 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin defer response.Close() result := response.ReadAll() resultStrut := &ghttp.DefaultHandlerResponse{} + gconv.Struct(result, &resultStrut) if resultStrut.Code == 200 || resultStrut.Code == 0 { gconv.Struct(resultStrut.Data, target) diff --git a/ragflow/client.go b/ragflow/client.go index 91601ad..2fb04fe 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -139,6 +139,7 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac if endpoint == "" { return gerror.New("RAGFlow endpoints not configured") } + fullURL := endpoint + path // 添加详细日志:请求信息 From aa3063c1ded289150b42875ec78c46acd0db05ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 13 Jan 2026 11:25:15 +0800 Subject: [PATCH 169/225] =?UTF-8?q?mongodb=E5=9F=BA=E7=A1=80=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E6=96=B9=E6=B3=95=E5=A2=9E=E5=8A=A0redis=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/http/http.go b/http/http.go index a63eb8a..52497b4 100644 --- a/http/http.go +++ b/http/http.go @@ -59,10 +59,8 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin return } client := Httpclient - if len(data) > 0 { - client = Httpclient.Clone() - } if len(headers) > 0 { + client = Httpclient.Clone() client.SetHeaderMap(headers) } else { client.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) From 1fe3f127c350c3294efa4bd1e6b6f9e4d93a53f3 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 13 Jan 2026 17:44:14 +0800 Subject: [PATCH 170/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0redis=E9=87=8D?= =?UTF-8?q?=E8=AF=95=E4=B8=89=E6=AC=A1,http=E6=96=B9=E6=B3=95=E5=85=BC?= =?UTF-8?q?=E5=AE=B9ragflow=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 13 +++++++++-- mongo/mongo.go | 22 +++++++++---------- redis/redis.go | 58 +++++++++++--------------------------------------- 3 files changed, 34 insertions(+), 59 deletions(-) diff --git a/http/http.go b/http/http.go index 52497b4..4378b21 100644 --- a/http/http.go +++ b/http/http.go @@ -53,6 +53,8 @@ func RouteRegister(controllers []interface{}) { } go Httpserver.Run() } + +// doRequest 统一HTTP请求处理(DELETE用ContentJson发送body,gconv.Struct增加err检查) func doRequest(ctx context.Context, method string, url string, headers map[string]string, target any, data ...any) (err error) { err = utils.ValidStructPtr(target) if err != nil { @@ -65,6 +67,9 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin } else { client.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) } + if method == http.MethodDelete && len(data) > 0 { // DELETE请求显式用ContentJson序列化body + client = client.ContentJson() + } response, err := client.DoRequest(ctx, method, url, data...) if err != nil { return @@ -73,9 +78,13 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin result := response.ReadAll() resultStrut := &ghttp.DefaultHandlerResponse{} - gconv.Struct(result, &resultStrut) + if err = gconv.Struct(result, &resultStrut); err != nil { // 修复:增加err检查 + return errors.New("响应解析失败: " + err.Error()) + } if resultStrut.Code == 200 || resultStrut.Code == 0 { - gconv.Struct(resultStrut.Data, target) + if err = gconv.Struct(resultStrut.Data, target); err != nil { // 修复:增加err检查 + return errors.New("数据解析失败: " + err.Error()) + } } else { err = errors.New(resultStrut.Message) } diff --git a/mongo/mongo.go b/mongo/mongo.go index c510ea5..22dc1a4 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -109,7 +109,7 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c redisKey := fmt.Sprintf(redis.List, user.TenantId, collection, filterKey, optionsKey) if m.Cache { var resultStr *gvar.Var - resultStr, err = redis.RedisClient.Get(ctx, redisKey) + resultStr, err = redis.RedisClient().Get(ctx, redisKey) if err != nil { return } @@ -168,7 +168,7 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c return } if m.Cache { - err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) + err = redis.RedisClient().SetEX(ctx, redisKey, result, int64(time.Hour)) if err != nil { return } @@ -200,7 +200,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} redisKey := fmt.Sprintf(redis.One, user.TenantId, collection, filterKey) if m.Cache { var resultStr *gvar.Var - resultStr, err = redis.RedisClient.Get(ctx, redisKey) + resultStr, err = redis.RedisClient().Get(ctx, redisKey) if err != nil { return } @@ -221,7 +221,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} err = nil } if m.Cache { - err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) + err = redis.RedisClient().SetEX(ctx, redisKey, result, int64(time.Hour)) if err != nil { return err } @@ -231,23 +231,23 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collection string) (err error) { listKeys := fmt.Sprintf(redis.CleanList, tenantId, collection) - keys, err := redis.RedisClient.Keys(ctx, listKeys) + keys, err := redis.RedisClient().Keys(ctx, listKeys) if err != nil { return } for _, key := range keys { - _, err = redis.RedisClient.Del(ctx, key) + _, err = redis.RedisClient().Del(ctx, key) if err != nil { return } } countKeys := fmt.Sprintf(redis.CleanCount, tenantId, collection) - keys, err = redis.RedisClient.Keys(ctx, countKeys) + keys, err = redis.RedisClient().Keys(ctx, countKeys) if err != nil { return } for _, key := range keys { - _, err = redis.RedisClient.Del(ctx, key) + _, err = redis.RedisClient().Del(ctx, key) if err != nil { return } @@ -256,7 +256,7 @@ func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interf delete(filter, "tenantId") filterKey := fmt.Sprintf("%+v", filter) oneKey := fmt.Sprintf(redis.One, tenantId, collection, filterKey) - _, err = redis.RedisClient.Del(ctx, oneKey) + _, err = redis.RedisClient().Del(ctx, oneKey) if err != nil { return } @@ -525,7 +525,7 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( redisKey := fmt.Sprintf(redis.Count, user.TenantId, collection, filterKey) if m.Cache { var resultStr *gvar.Var - resultStr, err = redis.RedisClient.Get(ctx, redisKey) + resultStr, err = redis.RedisClient().Get(ctx, redisKey) if err != nil { return } @@ -536,7 +536,7 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( } count, err = db.Collection(collection).CountDocuments(ctx, filter) if m.Cache { - err = redis.RedisClient.SetEX(ctx, redisKey, count, int64(time.Hour)) + err = redis.RedisClient().SetEX(ctx, redisKey, count, int64(time.Hour)) if err != nil { return } diff --git a/redis/redis.go b/redis/redis.go index 38b372a..aae782b 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -2,7 +2,6 @@ package redis import ( "context" - "errors" "strings" "sync" "time" @@ -15,60 +14,27 @@ import ( ) var ( - // redisClient 内部使用的 Redis 客户端(单例模式) redisClient *gredis.Redis redisOnce sync.Once ) -// getClient 获取 Redis 客户端(延迟初始化) -func getClient() *gredis.Redis { +// RedisClient 获取Redis客户端(支持重试3次,每次间隔2秒) +func RedisClient() *gredis.Redis { redisOnce.Do(func() { - redisClient = g.Redis() + for i := 0; i < 3; i++ { + redisClient = g.Redis() + if redisClient != nil { + ctx := context.Background() + if _, err := redisClient.Do(ctx, "PING"); err == nil { + return + } + } + time.Sleep(2 * time.Second) + } }) return redisClient } -// GetRedisClient 获取 Redis 客户端(供外部使用) -func GetRedisClient() *gredis.Redis { - return getClient() -} - -// RedisClient 导出的 Redis 客户端(供 mongo.go 使用,兼容旧代码) -var RedisClient = getClient() - -// Lock 分布式锁 -func Lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context) error) (success bool, err error) { - limit := 3 -LOOP: - if limit < 0 { - return false, errors.New("锁重试次数耗尽") - } - limit-- - if val, err := RedisClient.Set(ctx, key, true, gredis.SetOption{ - TTLOption: gredis.TTLOption{ - EX: &expireSeconds, - }, - NX: true, - }); err != nil { - return false, err - } else { - if val.Bool() { - defer func(RedisClient *gredis.Redis, ctx context.Context, key string) { - if _, err = RedisClient.Del(ctx, key); err != nil { - glog.Errorf(ctx, "RedisClient.Del error: %v", err) - } - }(RedisClient, ctx, key) - if err = fn(ctx); err != nil { - return false, err - } - return true, nil - } else { - time.Sleep(time.Second) - goto LOOP - } - } -} - func GetReadStream(ctx context.Context, msg ...QueueMessage) error { for _, t := range msg { err := GetReadFromStream(ctx, t.StreamKey, t.GroupName, t.ConsumerName, t.BatchSize, t.BlockMs, t.AutoAck, t.HandleFunc) From 138b81640ce0def168b10a702f15d7c33d4a6cff Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Fri, 16 Jan 2026 13:42:15 +0800 Subject: [PATCH 171/225] --- mongo/mongo.go | 22 ++++++++--------- redis/redis.go | 66 +++++++++++++++++++++++++++++++++++--------------- redis/types.go | 10 ++++++++ 3 files changed, 67 insertions(+), 31 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index 22dc1a4..c510ea5 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -109,7 +109,7 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c redisKey := fmt.Sprintf(redis.List, user.TenantId, collection, filterKey, optionsKey) if m.Cache { var resultStr *gvar.Var - resultStr, err = redis.RedisClient().Get(ctx, redisKey) + resultStr, err = redis.RedisClient.Get(ctx, redisKey) if err != nil { return } @@ -168,7 +168,7 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c return } if m.Cache { - err = redis.RedisClient().SetEX(ctx, redisKey, result, int64(time.Hour)) + err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) if err != nil { return } @@ -200,7 +200,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} redisKey := fmt.Sprintf(redis.One, user.TenantId, collection, filterKey) if m.Cache { var resultStr *gvar.Var - resultStr, err = redis.RedisClient().Get(ctx, redisKey) + resultStr, err = redis.RedisClient.Get(ctx, redisKey) if err != nil { return } @@ -221,7 +221,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} err = nil } if m.Cache { - err = redis.RedisClient().SetEX(ctx, redisKey, result, int64(time.Hour)) + err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) if err != nil { return err } @@ -231,23 +231,23 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collection string) (err error) { listKeys := fmt.Sprintf(redis.CleanList, tenantId, collection) - keys, err := redis.RedisClient().Keys(ctx, listKeys) + keys, err := redis.RedisClient.Keys(ctx, listKeys) if err != nil { return } for _, key := range keys { - _, err = redis.RedisClient().Del(ctx, key) + _, err = redis.RedisClient.Del(ctx, key) if err != nil { return } } countKeys := fmt.Sprintf(redis.CleanCount, tenantId, collection) - keys, err = redis.RedisClient().Keys(ctx, countKeys) + keys, err = redis.RedisClient.Keys(ctx, countKeys) if err != nil { return } for _, key := range keys { - _, err = redis.RedisClient().Del(ctx, key) + _, err = redis.RedisClient.Del(ctx, key) if err != nil { return } @@ -256,7 +256,7 @@ func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interf delete(filter, "tenantId") filterKey := fmt.Sprintf("%+v", filter) oneKey := fmt.Sprintf(redis.One, tenantId, collection, filterKey) - _, err = redis.RedisClient().Del(ctx, oneKey) + _, err = redis.RedisClient.Del(ctx, oneKey) if err != nil { return } @@ -525,7 +525,7 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( redisKey := fmt.Sprintf(redis.Count, user.TenantId, collection, filterKey) if m.Cache { var resultStr *gvar.Var - resultStr, err = redis.RedisClient().Get(ctx, redisKey) + resultStr, err = redis.RedisClient.Get(ctx, redisKey) if err != nil { return } @@ -536,7 +536,7 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( } count, err = db.Collection(collection).CountDocuments(ctx, filter) if m.Cache { - err = redis.RedisClient().SetEX(ctx, redisKey, count, int64(time.Hour)) + err = redis.RedisClient.SetEX(ctx, redisKey, count, int64(time.Hour)) if err != nil { return } diff --git a/redis/redis.go b/redis/redis.go index aae782b..648bd99 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -2,6 +2,7 @@ package redis import ( "context" + "errors" "strings" "sync" "time" @@ -14,27 +15,60 @@ import ( ) var ( + // redisClient 内部使用的 Redis 客户端(单例模式) redisClient *gredis.Redis redisOnce sync.Once ) -// RedisClient 获取Redis客户端(支持重试3次,每次间隔2秒) -func RedisClient() *gredis.Redis { +// getClient 获取 Redis 客户端(延迟初始化) +func getClient() *gredis.Redis { redisOnce.Do(func() { - for i := 0; i < 3; i++ { - redisClient = g.Redis() - if redisClient != nil { - ctx := context.Background() - if _, err := redisClient.Do(ctx, "PING"); err == nil { - return - } - } - time.Sleep(2 * time.Second) - } + redisClient = g.Redis() }) return redisClient } +// GetRedisClient 获取 Redis 客户端(供外部使用) +func GetRedisClient() *gredis.Redis { + return getClient() +} + +// RedisClient 导出的 Redis 客户端(供 mongo.go 使用,兼容旧代码) +var RedisClient = getClient() + +// Lock 分布式锁 +func Lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context) error) (success bool, err error) { + limit := 3 +LOOP: + if limit < 0 { + return false, errors.New("锁重试次数耗尽") + } + limit-- + if val, err := RedisClient.Set(ctx, key, true, gredis.SetOption{ + TTLOption: gredis.TTLOption{ + EX: &expireSeconds, + }, + NX: true, + }); err != nil { + return false, err + } else { + if val.Bool() { + defer func(RedisClient *gredis.Redis, ctx context.Context, key string) { + if _, err = RedisClient.Del(ctx, key); err != nil { + glog.Errorf(ctx, "RedisClient.Del error: %v", err) + } + }(RedisClient, ctx, key) + if err = fn(ctx); err != nil { + return false, err + } + return true, nil + } else { + time.Sleep(time.Second) + goto LOOP + } + } +} + func GetReadStream(ctx context.Context, msg ...QueueMessage) error { for _, t := range msg { err := GetReadFromStream(ctx, t.StreamKey, t.GroupName, t.ConsumerName, t.BatchSize, t.BlockMs, t.AutoAck, t.HandleFunc) @@ -164,9 +198,6 @@ func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName stri RECONNECT: // 先尝试读取pending消息(ID=0),处理积压 - glog.Debugf(ctx, "[DEBUG Redis] XREADGROUP GROUP %s %s COUNT %d BLOCK 0 STREAMS %s 0", - groupName, consumerName, count, streamKey) - result, err := redisClient.Do(execCtx, "XREADGROUP", "GROUP", groupName, consumerName, "COUNT", count, @@ -179,9 +210,6 @@ RECONNECT: // 如果没有pending消息,读取新消息 if result == nil || result.IsEmpty() { - glog.Debugf(ctx, "[DEBUG Redis] 无pending消息,读取新消息 XREADGROUP GROUP %s %s COUNT %d BLOCK %d STREAMS %s >", - groupName, consumerName, count, blockMs, streamKey) - result, err = redisClient.Do(execCtx, "XREADGROUP", "GROUP", groupName, consumerName, "COUNT", count, @@ -193,8 +221,6 @@ RECONNECT: } } - glog.Debugf(ctx, "[DEBUG Redis] XREADGROUP 返回: %+v", result) - // 预分配容量,避免动态扩容 messages := make([]StreamMessage, 0, int(count)) diff --git a/redis/types.go b/redis/types.go index 2ccfbde..f3dc0c7 100644 --- a/redis/types.go +++ b/redis/types.go @@ -116,3 +116,13 @@ func GetHistoryContextLimit() int64 { ctx := context.Background() return g.Cfg().MustGet(ctx, "history.contextLimit", 5).Int64() // 默认5轮对话 } + +// DocSyncMessage 文档同步消息结构(RAGFlow与MongoDB同步) +type DocSyncMessage struct { + DocId string `json:"docId"` // MongoDB文档ID + RagflowDocId string `json:"ragflowDocId"` // RAGFlow文档ID + TenantId string `json:"tenantId"` // 租户ID + DocType string `json:"docType"` // 文档类型:speechcraft/product + Action string `json:"action"` // 操作类型:sync_ragflow_id + Timestamp int64 `json:"timestamp"` // 时间戳 +} From 02e75e57da7f12d2ed7a8fe1e0cd5006b7c72c0b Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Sat, 17 Jan 2026 17:58:13 +0800 Subject: [PATCH 172/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 56 +++++++++++++++++++++++++++++++++++++-------- ragflow/client.go | 3 +-- ragflow/dataset.go | 3 --- ragflow/document.go | 1 - redis/types.go | 1 + 5 files changed, 49 insertions(+), 15 deletions(-) diff --git a/http/http.go b/http/http.go index 4378b21..47c110f 100644 --- a/http/http.go +++ b/http/http.go @@ -14,6 +14,7 @@ import ( "gitee.com/red-future---jilin-g/common/log/controller" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/gclient" "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/net/gsvc" "github.com/gogf/gf/v2/os/gtime" @@ -60,49 +61,86 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin if err != nil { return } - client := Httpclient + client := Httpclient.Clone() + + // POST/PUT/DELETE请求都需要显式用ContentJson序列化body + if (method == http.MethodPost || method == http.MethodPut || method == http.MethodDelete) && len(data) > 0 { + client = client.ContentJson() + } + + // 最后设置headers,确保不会被ContentJson覆盖 if len(headers) > 0 { - client = Httpclient.Clone() client.SetHeaderMap(headers) } else { client.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) } - if method == http.MethodDelete && len(data) > 0 { // DELETE请求显式用ContentJson序列化body - client = client.ContentJson() + + // 修复:避免data...展开导致的双重包装问题 + // 当只有一个元素时,直接传递该元素,避免被包装成数组 + var response *gclient.Response + if len(data) == 1 { + response, err = client.DoRequest(ctx, method, url, data[0]) + } else { + response, err = client.DoRequest(ctx, method, url, data...) } - response, err := client.DoRequest(ctx, method, url, data...) if err != nil { return } defer response.Close() result := response.ReadAll() + + // 添加调试日志:打印原始响应 + g.Log().Debugf(ctx, "[HTTP] 原始响应: %s", string(result)) + + // 第三方API特例:RAGFlow等第三方API响应格式为{code,data,message}一层结构,直接解析原始JSON到target + // 内部API格式为{code:200,message:"",data:{...}}两层结构,需经过DefaultHandlerResponse二次解析 + // 判断依据:URL包含/api/v1/(不影响内部API调用) + isThirdPartyAPI := strings.Contains(url, "/api/v1/") + + if isThirdPartyAPI { + // 第三方API特例:直接解析原始JSON到target,不经过DefaultHandlerResponse + if err = gconv.Struct(result, target); err != nil { + return errors.New("第三方API响应解析失败: " + err.Error()) + } + g.Log().Debugf(ctx, "[HTTP] 第三方API直接解析target: %+v", target) + return + } + + // 内部API:保持原有逻辑,先解析为DefaultHandlerResponse,再提取data字段 resultStrut := &ghttp.DefaultHandlerResponse{} if err = gconv.Struct(result, &resultStrut); err != nil { // 修复:增加err检查 return errors.New("响应解析失败: " + err.Error()) } + + // 添加调试日志:打印解析后的结构 + g.Log().Debugf(ctx, "[HTTP] 解析后结构: Code=%d, Message=%s, Data类型=%T, Data值=%+v", + resultStrut.Code, resultStrut.Message, resultStrut.Data, resultStrut.Data) + if resultStrut.Code == 200 || resultStrut.Code == 0 { if err = gconv.Struct(resultStrut.Data, target); err != nil { // 修复:增加err检查 return errors.New("数据解析失败: " + err.Error()) } + // 添加调试日志:打印最终的target + g.Log().Debugf(ctx, "[HTTP] 最终target: %+v", target) } else { err = errors.New(resultStrut.Message) } return } func Get(ctx context.Context, url string, headers map[string]string, target any, data ...any) (err error) { - err = doRequest(ctx, http.MethodGet, url, headers, target, data) + err = doRequest(ctx, http.MethodGet, url, headers, target, data...) return } func Post(ctx context.Context, url string, headers map[string]string, target any, data ...any) (err error) { - err = doRequest(ctx, http.MethodPost, url, headers, target, data) + err = doRequest(ctx, http.MethodPost, url, headers, target, data...) return } func Put(ctx context.Context, url string, headers map[string]string, target any, data ...any) (err error) { - err = doRequest(ctx, http.MethodPut, url, headers, target, data) + err = doRequest(ctx, http.MethodPut, url, headers, target, data...) return } func Delete(ctx context.Context, url string, headers map[string]string, target any, data ...any) (err error) { - err = doRequest(ctx, http.MethodDelete, url, headers, target, data) + err = doRequest(ctx, http.MethodDelete, url, headers, target, data...) return } diff --git a/ragflow/client.go b/ragflow/client.go index 2fb04fe..128f2fb 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -149,7 +149,7 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac g.Log().Infof(ctx, "RAGFlow请求体: %s", bodyJSON) } - // 创建新的HTTP客户端实例(避免共享状态) + // 使用common/http包 var headers = make(map[string]string) headers["Authorization"] = "Bearer " + c.APIKey headers["Content-Type"] = "application/json" @@ -161,7 +161,6 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac case "PUT": err = http.Put(ctx, fullURL, headers, result, body) case "DELETE": - if body != nil { err = http.Delete(ctx, fullURL, headers, result, body) } else { diff --git a/ragflow/dataset.go b/ragflow/dataset.go index b74970b..c5ac88b 100644 --- a/ragflow/dataset.go +++ b/ragflow/dataset.go @@ -157,9 +157,6 @@ func (c *Client) ListDatasets(ctx context.Context, req *ListDatasetsReq) (*ListD return nil, err } - // 添加调试日志 - g.Log().Infof(ctx, "ListDatasets原始响应: code=%d, total=%d, data_len=%d", res.Code, res.Total, len(res.Data)) - if res.Code != 0 { return nil, gerror.Newf("list datasets failed: code=%d", res.Code) } diff --git a/ragflow/document.go b/ragflow/document.go index c6e39e0..4158ea1 100644 --- a/ragflow/document.go +++ b/ragflow/document.go @@ -208,7 +208,6 @@ func (c *Client) UploadDocumentFromText(ctx context.Context, datasetId, content, } respBody := resp.ReadAll() - g.Log().Debugf(ctx, "RAGFlow上传文档响应: %s", string(respBody)) if err := json.Unmarshal(respBody, &response); err != nil { g.Log().Errorf(ctx, "解析RAGFlow响应失败: %v, 原始响应: %s", err, string(respBody)) diff --git a/redis/types.go b/redis/types.go index f3dc0c7..1e5a83f 100644 --- a/redis/types.go +++ b/redis/types.go @@ -102,6 +102,7 @@ type ArchiveMessage struct { UserId string `json:"userId"` // 用户ID Platform string `json:"platform"` // 平台标识 SessionId string `json:"sessionId"` // RAGFlow Session ID + TenantId string `json:"tenantId"` // 租户ID Timestamp int64 `json:"timestamp"` // 发送时间戳 } From 4d6aa1f38458633fb771fa7d7742a5af9a60f779 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Wed, 21 Jan 2026 10:20:32 +0800 Subject: [PATCH 173/225] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dredis=E5=92=8Cqueuepr?= =?UTF-8?q?ocessor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 4 ---- ragflow/client.go | 7 ------ ragflow/session.go | 3 +++ ragflow/worker_pool.go | 39 ++++++++++++++++++++++------------ redis/redis.go | 48 ++++++++++++++++++++++++++++++++++++------ 5 files changed, 71 insertions(+), 30 deletions(-) diff --git a/http/http.go b/http/http.go index 47c110f..9f0efd7 100644 --- a/http/http.go +++ b/http/http.go @@ -89,9 +89,6 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin defer response.Close() result := response.ReadAll() - // 添加调试日志:打印原始响应 - g.Log().Debugf(ctx, "[HTTP] 原始响应: %s", string(result)) - // 第三方API特例:RAGFlow等第三方API响应格式为{code,data,message}一层结构,直接解析原始JSON到target // 内部API格式为{code:200,message:"",data:{...}}两层结构,需经过DefaultHandlerResponse二次解析 // 判断依据:URL包含/api/v1/(不影响内部API调用) @@ -102,7 +99,6 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin if err = gconv.Struct(result, target); err != nil { return errors.New("第三方API响应解析失败: " + err.Error()) } - g.Log().Debugf(ctx, "[HTTP] 第三方API直接解析target: %+v", target) return } diff --git a/ragflow/client.go b/ragflow/client.go index 128f2fb..502a8c2 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -142,13 +142,6 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac fullURL := endpoint + path - // 添加详细日志:请求信息 - g.Log().Infof(ctx, "RAGFlow请求: %s %s", method, fullURL) - if body != nil { - bodyJSON := g.NewVar(body).String() - g.Log().Infof(ctx, "RAGFlow请求体: %s", bodyJSON) - } - // 使用common/http包 var headers = make(map[string]string) headers["Authorization"] = "Bearer " + c.APIKey diff --git a/ragflow/session.go b/ragflow/session.go index 7659eb2..966d0e8 100644 --- a/ragflow/session.go +++ b/ragflow/session.go @@ -4,6 +4,7 @@ import ( "context" "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" ) // 会话管理 @@ -85,9 +86,11 @@ func (c *Client) CreateSession(ctx context.Context, chatId string, req *CreateSe Msg string `json:"message"` } if err := c.request(ctx, "POST", path, req, &res); err != nil { + g.Log().Errorf(ctx, "❌ CreateSession请求失败: chatId=%s, req=%+v, error=%v", chatId, req, err) return nil, err } if res.Code != 0 { + g.Log().Errorf(ctx, "❌ CreateSession返回失败: chatId=%s, req=%+v, code=%d, msg=%s", chatId, req, res.Code, res.Msg) return nil, gerror.Newf("create session failed: %s", res.Msg) } // 检查响应数据是否为空:防止RAGFlow API返回 {"code":0, "data":null} diff --git a/ragflow/worker_pool.go b/ragflow/worker_pool.go index 32b4b67..608dd81 100644 --- a/ragflow/worker_pool.go +++ b/ragflow/worker_pool.go @@ -4,6 +4,7 @@ import ( "context" "runtime/debug" "strings" + "sync" "time" "gitee.com/red-future---jilin-g/common/redis" @@ -16,14 +17,15 @@ const defaultBatchSize = 200 // QueueProcessor Stream 处理器,批量读取消息并发送到 RAGFlow type QueueProcessor struct { - streamKey string // Stream 键名 - groupName string // 消费者组名称 - consumerName string // 消费者名称 - timeout int64 // 阻塞超时时间(毫秒) - batchSize int64 // 最大并发数(协程池大小) - stopChan chan struct{} // 停止信号 - pool *grpool.Pool // GoFrame协程池 - handleFunc func(ctx context.Context, message map[string]interface{}) error + streamKey string // Stream 键名 + groupName string // 消费者组名称 + consumerName string // 消费者名称 + timeout int64 // 阻塞超时时间(毫秒) + batchSize int64 // 最大并发数(协程池大小) + stopChan chan struct{} // 停止信号 + pool *grpool.Pool // GoFrame协程池 + handleFunc func(ctx context.Context, message map[string]interface{}) error + processingMsgs sync.Map // 正在处理的消息ID(去重用) } // NewQueueProcessor 创建 Stream 处理器 @@ -84,14 +86,27 @@ func (q *QueueProcessor) Start(ctx context.Context) error { glog.Infof(ctx, "✅ 从Stream读取到 %d 条消息,开始处理", len(messages)) - // 2. 使用协程池提交任务:复用goroutine,避免频繁创建销毁 + // 2. 去重+立即ACK:对话场景优先实时性,失败不重试 for i, msg := range messages { m := msg // 捕获循环变量 msgIndex := i + 1 + + // 去重:如果消息正在处理,跳过 + if _, exists := q.processingMsgs.LoadOrStore(m.ID, true); exists { + glog.Debugf(ctx, "⏭️ 跳过正在处理的消息 - ID: %s", m.ID) + continue + } + + // 立即ACK:对话场景不需要重试,避免重复消费 + if err := redis.AckMessage(ctx, q.streamKey, q.groupName, m.ID); err != nil { + glog.Errorf(ctx, "确认消息失败: %v, 消息ID: %s", err, m.ID) + } + glog.Infof(ctx, "📨 准备处理第 %d/%d 条消息 - ID: %s", msgIndex, len(messages), m.ID) // 提交到协程池,池满时会阻塞等待空闲worker q.pool.Add(ctx, func(ctx context.Context) { + defer q.processingMsgs.Delete(m.ID) // 处理完成后移除标记 q.processMessage(ctx, m) }) } @@ -127,10 +142,8 @@ func (q *QueueProcessor) processMessage(ctx context.Context, message redis.Strea glog.Infof(ctx, "✅ 消息处理成功 - ID: %s", message.ID) } - // 无论成功失败都 ACK(避免重复消费) - if err := redis.AckMessage(ctx, q.streamKey, q.groupName, message.ID); err != nil { - glog.Errorf(ctx, "确认消息失败: %v, 消息ID: %s", err, message.ID) - } + // ACK已在读取后立即执行,此处无需重复ACK + // 对话场景:失败直接丢弃,不重试(实时性优先) } // Stop 停止队列处理器 diff --git a/redis/redis.go b/redis/redis.go index 648bd99..b4556a6 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -180,9 +180,10 @@ func AddToStream(ctx context.Context, streamKey string, msg interface{}) (messag } // CreateConsumerGroup 创建消费者组(如果不存在) -// XGROUP CREATE streamKey groupName $ MKSTREAM +// XGROUP CREATE streamKey groupName 0 MKSTREAM +// 使用0作为起始ID,从Stream开头读取所有未消费消息 func CreateConsumerGroup(ctx context.Context, streamKey, groupName string) error { - _, err := redisClient.Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "$", "MKSTREAM") + _, err := redisClient.Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") return err } @@ -205,11 +206,29 @@ RECONNECT: "STREAMS", streamKey, "0", // ID=0 读取pending消息 ) if err != nil { + g.Log().Errorf(ctx, "❌ XREADGROUP读取pending失败: stream=%s, error=%v", streamKey, err) + time.Sleep(time.Second) goto RECONNECT } + // 检查pending结果是否为空(需要检查消息数组是否为空) + hasPending := false + if result != nil && !result.IsEmpty() { + // 尝试解析map格式 + if resultVal := result.Val(); resultVal != nil { + if streamsMap, ok := resultVal.(map[interface{}]interface{}); ok { + for _, streamMsgs := range streamsMap { + if msgsArray, ok := streamMsgs.([]interface{}); ok && len(msgsArray) > 0 { + hasPending = true + break + } + } + } + } + } + // 如果没有pending消息,读取新消息 - if result == nil || result.IsEmpty() { + if !hasPending { result, err = redisClient.Do(execCtx, "XREADGROUP", "GROUP", groupName, consumerName, "COUNT", count, @@ -217,6 +236,8 @@ RECONNECT: "STREAMS", streamKey, ">", ) if err != nil { + g.Log().Errorf(ctx, "❌ XREADGROUP读取新消息失败: stream=%s, error=%v", streamKey, err) + time.Sleep(time.Second) goto RECONNECT } } @@ -234,19 +255,26 @@ RECONNECT: // 尝试 map 格式(GoFrame gredis 返回) if streamsMap, ok := resultVal.(map[interface{}]interface{}); ok { - for _, streamMsgs := range streamsMap { + for streamKey, streamMsgs := range streamsMap { msgsArray, ok := streamMsgs.([]interface{}) if !ok { + g.Log().Errorf(ctx, "❌ streamMsgs类型转换失败: streamKey=%v, 实际类型=%T", streamKey, streamMsgs) continue } - for _, msgData := range msgsArray { + for i, msgData := range msgsArray { msgArray, ok := msgData.([]interface{}) - if !ok || len(msgArray) < 2 { + if !ok { + g.Log().Errorf(ctx, "❌ msgData类型转换失败: index=%d, 实际类型=%T", i, msgData) + continue + } + if len(msgArray) < 2 { + g.Log().Errorf(ctx, "❌ msgArray长度不足: index=%d, len=%d", i, len(msgArray)) continue } msgID := gconv.String(msgArray[0]) fieldsArray, ok := msgArray[1].([]interface{}) if !ok { + g.Log().Errorf(ctx, "❌ fieldsArray类型转换失败: msgID=%s, msgArray[1]类型=%T", msgID, msgArray[1]) continue } values := make(map[string]interface{}, len(fieldsArray)/2) @@ -262,6 +290,9 @@ RECONNECT: }) } } + if len(messages) == 0 { + g.Log().Errorf(ctx, "❌ [ReadFromStream] map格式解析失败: streamsMap长度=%d, 但未提取到消息", len(streamsMap)) + } return messages, nil } @@ -299,8 +330,13 @@ RECONNECT: }) } } + if len(messages) == 0 { + g.Log().Errorf(ctx, "❌ [ReadFromStream] 数组格式解析失败: streamsArray长度=%d, 但未提取到消息", len(streamsArray)) + } + return messages, nil } + g.Log().Errorf(ctx, "❌ [ReadFromStream] 无法识别的result格式, resultVal类型: %T, 值: %+v", resultVal, resultVal) return messages, nil } From 86cb177625fcad55f9bd615d8b222390420d67b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 13 Jan 2026 14:15:45 +0800 Subject: [PATCH 174/225] =?UTF-8?q?nats=E8=BF=9E=E6=8E=A5=E4=B8=8E?= =?UTF-8?q?=E6=89=A7=E8=A1=8C=E6=96=B9=E6=B3=95=E5=88=86=E7=A6=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/{link.go => connection.go} | 0 nats/connection.go | 318 ++++++++++++++++++++++++++++++ nats/nats.go | 326 +------------------------------ nats/nats_test.go | 2 +- 4 files changed, 329 insertions(+), 317 deletions(-) rename mongo/{link.go => connection.go} (100%) create mode 100644 nats/connection.go diff --git a/mongo/link.go b/mongo/connection.go similarity index 100% rename from mongo/link.go rename to mongo/connection.go diff --git a/nats/connection.go b/nats/connection.go new file mode 100644 index 0000000..eb8078a --- /dev/null +++ b/nats/connection.go @@ -0,0 +1,318 @@ +package nats + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +var ( + nc *nats.Conn + js jetstream.JetStream + inited bool + mu sync.RWMutex + natsURL string + healthCtx context.Context + healthCancel context.CancelFunc + connected bool + reconnectChan chan struct{} + + // 连接状态变化监听器 + connStateListeners []ConnStateListener + connListenersMu sync.RWMutex + + // 监控指标 + metrics Metrics +) + +// Metrics 监控指标 +type Metrics struct { + PublishCount atomic.Int64 + PublishError atomic.Int64 + SubscribeCount atomic.Int64 + RequestCount atomic.Int64 + RequestError atomic.Int64 + ConsumeCount atomic.Int64 + ConsumeError atomic.Int64 +} + +// ConnState 连接状态 +type ConnState int + +const ( + ConnStateDisconnected ConnState = iota + ConnStateConnecting + ConnStateConnected + ConnStateReconnecting + ConnStateClosed +) + +// ConnStateListener 连接状态监听器 +type ConnStateListener func(state ConnState, err error) + +// GetMetrics 获取监控指标 +func GetMetrics() Metrics { + return metrics +} + +// RegisterConnStateListener 注册连接状态监听器 +func RegisterConnStateListener(listener ConnStateListener) { + connListenersMu.Lock() + defer connListenersMu.Unlock() + connStateListeners = append(connStateListeners, listener) +} + +// UnregisterConnStateListener 取消注册连接状态监听器 +func UnregisterConnStateListener(listener ConnStateListener) { + connListenersMu.Lock() + defer connListenersMu.Unlock() + for i, l := range connStateListeners { + if l != nil && &l == &listener { + connStateListeners = append(connStateListeners[:i], connStateListeners[i+1:]...) + break + } + } +} + +// notifyConnState 通知所有监听器连接状态变化 +func notifyConnState(state ConnState, err error) { + connListenersMu.RLock() + listeners := make([]ConnStateListener, len(connStateListeners)) + copy(listeners, connStateListeners) + connListenersMu.RUnlock() + + for _, listener := range listeners { + if listener != nil { + listener(state, err) + } + } +} + +// init 初始化 NATS 连接 +func init() { + // 从配置文件读取 NATS 地址 + natsURL = g.Cfg().MustGet(context.Background(), "nats.url").String() + if natsURL == "" { + // 默认使用本地地址 + natsURL = nats.DefaultURL + } + + // 创建健康检查上下文 + healthCtx, healthCancel = context.WithCancel(context.Background()) + + // 创建重连通知通道(增大缓冲区避免丢失通知) + reconnectChan = make(chan struct{}, 10) + + // 启动连接 + go initConnection() + + // 启动健康检查协程 + go healthCheck() +} + +// initConnection 初始化连接 +func initConnection() { + ctx := context.Background() + notifyConnState(ConnStateConnecting, nil) + if err := connect(ctx); err != nil { + g.Log().Errorf(ctx, "NATS 初始连接失败: %v", err) + notifyConnState(ConnStateDisconnected, err) + } +} + +// connect 建立 NATS 连接 +func connect(ctx context.Context) error { + mu.Lock() + defer mu.Unlock() + + if nc != nil && !nc.IsClosed() { + nc.Close() + } + + // 连接选项配置 + opts := []nats.Option{ + nats.Name("goframe-nats-client"), + nats.ReconnectWait(2 * time.Second), + nats.MaxReconnects(-1), // 无限重连 + nats.PingInterval(10 * time.Second), + nats.MaxPingsOutstanding(5), + nats.ReconnectHandler(func(nc *nats.Conn) { + g.Log().Infof(ctx, "✅ NATS 重连成功: %s", nc.ConnectedUrl()) + connected = true + + // 重新创建 JetStream 实例 + if newJS, err := jetstream.New(nc); err == nil { + js = newJS + } + + // 通知重连成功 + notifyConnState(ConnStateConnected, nil) + + // 使用非阻塞发送避免阻塞 + select { + case reconnectChan <- struct{}{}: + default: + // 通道已满,丢弃通知 + } + }), + nats.DisconnectErrHandler(func(nc *nats.Conn, err error) { + g.Log().Warningf(ctx, "⚠️ NATS 连接断开: %v, 准备重连...", err) + connected = false + notifyConnState(ConnStateReconnecting, err) + }), + nats.ClosedHandler(func(nc *nats.Conn) { + g.Log().Infof(ctx, "NATS 连接已关闭: %s", nc.ConnectedUrl()) + connected = false + notifyConnState(ConnStateClosed, nil) + }), + nats.ErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { + g.Log().Errorf(ctx, "NATS 错误: %v", err) + }), + } + + var err error + nc, err = nats.Connect(natsURL, opts...) + if err != nil { + return fmt.Errorf("NATS 连接失败: %w", err) + } + + // 等待连接就绪 + if nc.Status() != nats.CONNECTED { + select { + case <-time.After(5 * time.Second): + notifyConnState(ConnStateDisconnected, fmt.Errorf("连接超时")) + return fmt.Errorf("NATS 连接超时") + case <-nc.StatusChanged(nats.CONNECTED): + } + } + + // 创建 JetStream 实例 + js, err = jetstream.New(nc) + if err != nil { + return fmt.Errorf("创建 JetStream 失败: %w", err) + } + + connected = true + inited = true + g.Log().Infof(ctx, "✅ NATS 连接成功: %s", nc.ConnectedUrl()) + notifyConnState(ConnStateConnected, nil) + return nil +} + +// healthCheck 健康检查协程(仅作为备用检查) +func healthCheck() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-healthCtx.Done(): + return + case <-ticker.C: + mu.RLock() + currentConnected := connected + currentConn := nc + mu.RUnlock() + + if !currentConnected || currentConn == nil || currentConn.IsClosed() { + // 仅记录日志,不尝试重连(NATS 已有自动重连机制) + g.Log().Warning(context.Background(), "NATS 连接断开,等待 NATS 自动重连...") + } + case <-reconnectChan: + // 重连成功的通知(仅记录日志) + g.Log().Info(context.Background(), "收到重连成功通知") + } + } +} + +// checkConnected 检查连接状态 +func checkConnected() bool { + mu.RLock() + defer mu.RUnlock() + return connected && nc != nil && !nc.IsClosed() +} + +// IsConnected 检查 NATS 是否已连接 +func IsConnected() bool { + return checkConnected() +} + +// GetConnState 获取当前连接状态 +func GetConnState() ConnState { + mu.RLock() + defer mu.RUnlock() + + if nc == nil { + return ConnStateDisconnected + } + + if nc.IsClosed() { + return ConnStateClosed + } + + if connected { + return ConnStateConnected + } + + return ConnStateDisconnected +} + +// Shutdown 优雅关闭:自动注销所有已注册的服务并关闭 NATS 连接 +func Shutdown() error { + ctx := context.Background() + g.Log().Info(ctx, "开始优雅关闭 NATS RPC 服务...") + + // 注销所有单实例服务 + rpcServicesMu.Lock() + singleServiceCount := len(rpcServices) + for serviceName := range rpcServices { + if sub, exists := rpcSubs[serviceName]; exists { + if err := sub.Unsubscribe(); err != nil { + g.Log().Errorf(ctx, "注销服务 %s 失败: %v", serviceName, err) + } + } + delete(rpcSubs, serviceName) + delete(rpcServices, serviceName) + } + rpcServicesMu.Unlock() + + // 注销所有队列服务 + queueRPCMu.Lock() + queueServiceCount := 0 + for queueName, servicesMap := range queueRPCServices { + queueServiceCount += len(servicesMap) + for serviceName, sub := range queueRPCSubs[queueName] { + if err := sub.Unsubscribe(); err != nil { + g.Log().Errorf(ctx, "注销队列服务 %s (队列: %s) 失败: %v", serviceName, queueName, err) + } + } + delete(queueRPCSubs, queueName) + delete(queueRPCServices, queueName) + } + queueRPCMu.Unlock() + + g.Log().Infof(ctx, "已注销 %d 个单实例服务和 %d 个队列服务", singleServiceCount, queueServiceCount) + + mu.Lock() + defer mu.Unlock() + + // 停止健康检查协程 + if healthCancel != nil { + healthCancel() + } + + // 关闭连接 + if nc != nil && !nc.IsClosed() { + nc.Close() + connected = false + inited = false + } + g.Log().Info(ctx, "NATS RPC 服务已优雅关闭") + return nil +} diff --git a/nats/nats.go b/nats/nats.go index f04bab1..bcc325f 100644 --- a/nats/nats.go +++ b/nats/nats.go @@ -6,7 +6,6 @@ import ( "fmt" "reflect" "sync" - "sync/atomic" "time" "github.com/gogf/gf/v2/frame/g" @@ -14,256 +13,19 @@ import ( "github.com/nats-io/nats.go/jetstream" ) +// RPC 服务注册表 var ( - nc *nats.Conn - js jetstream.JetStream - inited bool - mu sync.RWMutex - natsURL string - healthCtx context.Context - healthCancel context.CancelFunc - connected bool - reconnectChan chan struct{} - - // 连接状态变化监听器 - connStateListeners []ConnStateListener - connListenersMu sync.RWMutex - - // 监控指标 - metrics Metrics + rpcServices map[string]RPCHandler + rpcSubs map[string]*nats.Subscription // 服务名 -> 订阅 + rpcServicesMu sync.RWMutex + queueRPCServices map[string]map[string]RPCHandler // queueName -> subject -> handler + queueRPCSubs map[string]map[string]*nats.Subscription // queueName -> serviceName -> 订阅 + queueRPCMu sync.RWMutex ) -// Metrics 监控指标 -type Metrics struct { - PublishCount atomic.Int64 - PublishError atomic.Int64 - SubscribeCount atomic.Int64 - RequestCount atomic.Int64 - RequestError atomic.Int64 - ConsumeCount atomic.Int64 - ConsumeError atomic.Int64 -} - -// ConnState 连接状态 -type ConnState int - -const ( - ConnStateDisconnected ConnState = iota - ConnStateConnecting - ConnStateConnected - ConnStateReconnecting - ConnStateClosed -) - -// ConnStateListener 连接状态监听器 -type ConnStateListener func(state ConnState, err error) - -// GetMetrics 获取监控指标 -func GetMetrics() Metrics { - return metrics -} - -// RegisterConnStateListener 注册连接状态监听器 -func RegisterConnStateListener(listener ConnStateListener) { - connListenersMu.Lock() - defer connListenersMu.Unlock() - connStateListeners = append(connStateListeners, listener) -} - -// UnregisterConnStateListener 取消注册连接状态监听器 -func UnregisterConnStateListener(listener ConnStateListener) { - connListenersMu.Lock() - defer connListenersMu.Unlock() - for i, l := range connStateListeners { - if l != nil && &l == &listener { - connStateListeners = append(connStateListeners[:i], connStateListeners[i+1:]...) - break - } - } -} - -// notifyConnState 通知所有监听器连接状态变化 -func notifyConnState(state ConnState, err error) { - connListenersMu.RLock() - listeners := make([]ConnStateListener, len(connStateListeners)) - copy(listeners, connStateListeners) - connListenersMu.RUnlock() - - for _, listener := range listeners { - if listener != nil { - listener(state, err) - } - } -} - -// init 初始化 NATS 连接 -func init() { - // 从配置文件读取 NATS 地址 - natsURL = g.Cfg().MustGet(context.Background(), "nats.url").String() - if natsURL == "" { - // 默认使用本地地址 - natsURL = nats.DefaultURL - } - - // 创建健康检查上下文 - healthCtx, healthCancel = context.WithCancel(context.Background()) - - // 创建重连通知通道(增大缓冲区避免丢失通知) - reconnectChan = make(chan struct{}, 10) - - // 启动连接 - go initConnection() - - // 启动健康检查协程 - go healthCheck() -} - -// initConnection 初始化连接 -func initConnection() { - ctx := context.Background() - notifyConnState(ConnStateConnecting, nil) - if err := connect(ctx); err != nil { - g.Log().Errorf(ctx, "NATS 初始连接失败: %v", err) - notifyConnState(ConnStateDisconnected, err) - } -} - -// connect 建立 NATS 连接 -func connect(ctx context.Context) error { - mu.Lock() - defer mu.Unlock() - - if nc != nil && !nc.IsClosed() { - nc.Close() - } - - // 连接选项配置 - opts := []nats.Option{ - nats.Name("goframe-nats-client"), - nats.ReconnectWait(2 * time.Second), - nats.MaxReconnects(-1), // 无限重连 - nats.PingInterval(10 * time.Second), - nats.MaxPingsOutstanding(5), - nats.ReconnectHandler(func(nc *nats.Conn) { - g.Log().Infof(ctx, "✅ NATS 重连成功: %s", nc.ConnectedUrl()) - connected = true - - // 重新创建 JetStream 实例 - if newJS, err := jetstream.New(nc); err == nil { - js = newJS - } - - // 通知重连成功 - notifyConnState(ConnStateConnected, nil) - - // 使用非阻塞发送避免阻塞 - select { - case reconnectChan <- struct{}{}: - default: - // 通道已满,丢弃通知 - } - }), - nats.DisconnectErrHandler(func(nc *nats.Conn, err error) { - g.Log().Warningf(ctx, "⚠️ NATS 连接断开: %v, 准备重连...", err) - connected = false - notifyConnState(ConnStateReconnecting, err) - }), - nats.ClosedHandler(func(nc *nats.Conn) { - g.Log().Infof(ctx, "NATS 连接已关闭: %s", nc.ConnectedUrl()) - connected = false - notifyConnState(ConnStateClosed, nil) - }), - nats.ErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { - g.Log().Errorf(ctx, "NATS 错误: %v", err) - }), - } - - var err error - nc, err = nats.Connect(natsURL, opts...) - if err != nil { - return fmt.Errorf("NATS 连接失败: %w", err) - } - - // 等待连接就绪 - if nc.Status() != nats.CONNECTED { - select { - case <-time.After(5 * time.Second): - notifyConnState(ConnStateDisconnected, fmt.Errorf("连接超时")) - return fmt.Errorf("NATS 连接超时") - case <-nc.StatusChanged(nats.CONNECTED): - } - } - - // 创建 JetStream 实例 - js, err = jetstream.New(nc) - if err != nil { - return fmt.Errorf("创建 JetStream 失败: %w", err) - } - - connected = true - inited = true - g.Log().Infof(ctx, "✅ NATS 连接成功: %s", nc.ConnectedUrl()) - notifyConnState(ConnStateConnected, nil) - return nil -} - -// healthCheck 健康检查协程(仅作为备用检查) -func healthCheck() { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - - for { - select { - case <-healthCtx.Done(): - return - case <-ticker.C: - mu.RLock() - currentConnected := connected - currentConn := nc - mu.RUnlock() - - if !currentConnected || currentConn == nil || currentConn.IsClosed() { - // 仅记录日志,不尝试重连(NATS 已有自动重连机制) - g.Log().Warning(context.Background(), "NATS 连接断开,等待 NATS 自动重连...") - } - case <-reconnectChan: - // 重连成功的通知(仅记录日志) - g.Log().Info(context.Background(), "收到重连成功通知") - } - } -} - -// checkConnected 检查连接状态 -func checkConnected() bool { - mu.RLock() - defer mu.RUnlock() - return connected && nc != nil && !nc.IsClosed() -} - -// IsConnected 检查 NATS 是否已连接 -func IsConnected() bool { - return checkConnected() -} - -// GetConnState 获取当前连接状态 -func GetConnState() ConnState { - mu.RLock() - defer mu.RUnlock() - - if nc == nil { - return ConnStateDisconnected - } - - if nc.IsClosed() { - return ConnStateClosed - } - - if connected { - return ConnStateConnected - } - - return ConnStateDisconnected -} +// RPCHandler RPC 处理函数类型 +// 实现方只需要关注请求参数和返回值,无需了解底层 NATS 实现 +type RPCHandler func(ctx context.Context, req []byte) ([]byte, error) // CreateTaskStream 创建任务消息队列流 // 存储策略: 文件存储 @@ -521,20 +283,6 @@ func CreateConsumer(ctx context.Context, streamName, consumerName string, config // 以下方法提供了完全抽象的 RPC 调用接口 // 调用方和响应方完全不需要知道底层使用的是 NATS 的发布订阅模式 -// RPC 服务注册表 -var ( - rpcServices map[string]RPCHandler - rpcSubs map[string]*nats.Subscription // 服务名 -> 订阅 - rpcServicesMu sync.RWMutex - queueRPCServices map[string]map[string]RPCHandler // queueName -> subject -> handler - queueRPCSubs map[string]map[string]*nats.Subscription // queueName -> serviceName -> 订阅 - queueRPCMu sync.RWMutex -) - -// RPCHandler RPC 处理函数类型 -// 实现方只需要关注请求参数和返回值,无需了解底层 NATS 实现 -type RPCHandler func(ctx context.Context, req []byte) ([]byte, error) - // RegisterRPCService 注册 RPC 服务(单实例) // serviceName: 服务名称,调用方通过此名称调用服务 // handler: 服务处理函数,接收请求并返回响应 @@ -898,57 +646,3 @@ func AutoRegisterServices(serviceInstances map[string]interface{}, options ...Re g.Log().Infof(context.Background(), "✅ 共自动注册了 %d 个服务", totalRegistered) return nil } - -// Shutdown 优雅关闭:自动注销所有已注册的服务并关闭 NATS 连接 -func Shutdown() error { - ctx := context.Background() - g.Log().Info(ctx, "开始优雅关闭 NATS RPC 服务...") - - // 注销所有单实例服务 - rpcServicesMu.Lock() - singleServiceCount := len(rpcServices) - for serviceName := range rpcServices { - if sub, exists := rpcSubs[serviceName]; exists { - if err := sub.Unsubscribe(); err != nil { - g.Log().Errorf(ctx, "注销服务 %s 失败: %v", serviceName, err) - } - } - delete(rpcSubs, serviceName) - delete(rpcServices, serviceName) - } - rpcServicesMu.Unlock() - - // 注销所有队列服务 - queueRPCMu.Lock() - queueServiceCount := 0 - for queueName, servicesMap := range queueRPCServices { - queueServiceCount += len(servicesMap) - for serviceName, sub := range queueRPCSubs[queueName] { - if err := sub.Unsubscribe(); err != nil { - g.Log().Errorf(ctx, "注销队列服务 %s (队列: %s) 失败: %v", serviceName, queueName, err) - } - } - delete(queueRPCSubs, queueName) - delete(queueRPCServices, queueName) - } - queueRPCMu.Unlock() - - g.Log().Infof(ctx, "已注销 %d 个单实例服务和 %d 个队列服务", singleServiceCount, queueServiceCount) - - mu.Lock() - defer mu.Unlock() - - // 停止健康检查协程 - if healthCancel != nil { - healthCancel() - } - - // 关闭连接 - if nc != nil && !nc.IsClosed() { - nc.Close() - connected = false - inited = false - } - g.Log().Info(ctx, "NATS RPC 服务已优雅关闭") - return nil -} diff --git a/nats/nats_test.go b/nats/nats_test.go index e5a16ec..7cb5851 100644 --- a/nats/nats_test.go +++ b/nats/nats_test.go @@ -131,7 +131,7 @@ func TestNatsPublishRequest(t *testing.T) { } // RPC 请求 - response, err := Request(ctx, "test.request", []byte("request"), 5*time.Second) + response, err := CallRPC(ctx, "test.request", []byte("request"), 5*time.Second) if err != nil { t.Logf("RPC 请求失败: %v", err) } else { From cc940c27b7928cca4ef591c1d6b5fa29d0967f87 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Wed, 14 Jan 2026 18:34:56 +0800 Subject: [PATCH 175/225] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=E7=A7=9F=E6=88=B7=E6=A3=80=E6=9F=A5=E4=B8=AD=E9=97=B4=E4=BB=B6?= =?UTF-8?q?=EF=BC=8C=E9=87=8D=E6=9E=84=E4=B8=BA=E5=87=BD=E6=95=B0=E5=BC=8F?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E5=B9=B6=E6=B7=BB=E5=8A=A0=E8=B6=85=E7=BA=A7?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E5=91=98=E6=A3=80=E6=9F=A5=EF=BC=9B=E6=96=B0?= =?UTF-8?q?=E5=A2=9EMongoDB=E6=9B=B4=E6=96=B0=E8=BF=87=E6=BB=A4=E6=9E=84?= =?UTF-8?q?=E5=BB=BA=E5=B7=A5=E5=85=B7=EF=BC=9B=E4=BF=AE=E5=A4=8DRedis?= =?UTF-8?q?=E6=B6=88=E6=81=AF=E6=B6=88=E8=B4=B9=E9=94=99=E8=AF=AF=E5=A4=84?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- beans/user.go | 5 ++ message/redis.go | 8 +-- middleware/module_tenant_check.go | 86 ++++++++++++++++++------------- mongo/mongo.go | 12 +++++ 4 files changed, 71 insertions(+), 40 deletions(-) create mode 100644 beans/user.go diff --git a/beans/user.go b/beans/user.go new file mode 100644 index 0000000..3170e5b --- /dev/null +++ b/beans/user.go @@ -0,0 +1,5 @@ +package beans + +type IsSuperAdminRes struct { + IsSuperAdmin bool `p:"isSuperAdmin"` +} diff --git a/message/redis.go b/message/redis.go index 9f28ebe..7d5c76b 100644 --- a/message/redis.go +++ b/message/redis.go @@ -122,10 +122,10 @@ func readFromStream(ctx context.Context, msg QueueMessage) error { for { result, err := getRedisClient().Do(ctx, "XREADGROUP", "GROUP", msg.GroupName, msg.ConsumerName, "COUNT", msg.BatchSize, "BLOCK", 0, "STREAMS", msg.StreamKey, ">") if err != nil { - select { - case <-ctx.Done(): - return - } + //select { + //case <-ctx.Done(): + // return + //} time.Sleep(time.Second) goto RECONNECT } diff --git a/middleware/module_tenant_check.go b/middleware/module_tenant_check.go index 49bb307..f3c22fa 100644 --- a/middleware/module_tenant_check.go +++ b/middleware/module_tenant_check.go @@ -9,7 +9,6 @@ import ( "gitee.com/red-future---jilin-g/common/message" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/database/gredis" - "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/os/gtime" @@ -17,18 +16,27 @@ import ( "time" ) -type moduleTenant struct{} - -var ModuleTenant = new(moduleTenant) - -// ModuleTenantInfo 别名,引用admin-go的entity.ModuleTenant -type ModuleTenantInfo = beans.ModuleTenant - -func (s *moduleTenant) ModuleTenantCheck(r *ghttp.Request) { +func ModuleTenantCheck(r *ghttp.Request) { + //将 http.Header 转换为 map[string]string + headers := make(map[string]string) + for k, v := range r.Request.Header { + if len(v) > 0 { + headers[k] = v[0] + } + } + // 检查是否是超级管理员 + IsSuperAdmin, err := IsSuperAdmin(r.Context(), headers) + if err != nil { + SetResponseInfo(r.Context(), r, err) + } + // 如果是超级管理员,则不进行模块租户检查 + if IsSuperAdmin.IsSuperAdmin { + r.Middleware.Next() + return + } getUserInfo, err := utils.GetUserInfo(r.Context()) if err != nil { - r.Response.WriteJson(err) - r.Exit() + SetResponseInfo(r.Context(), r, err) } exit := gconv.Int64(time.Minute * 1) getEX, err := message.GetRedisClientTest("test").GetEX(r.Context(), fmt.Sprintf("module_tenant:tenantId-%v", getUserInfo.TenantId), gredis.GetEXOption{ @@ -37,16 +45,14 @@ func (s *moduleTenant) ModuleTenantCheck(r *ghttp.Request) { }, }) if err != nil { - r.Response.WriteJson(err) - r.Exit() + SetResponseInfo(r.Context(), r, err) } // 获取模块key moduleKey := g.Cfg().MustGet(context.Background(), "server.name") if !g.IsEmpty(getEX.String()) { - list := make([]ModuleTenantInfo, 0) + list := make([]beans.ModuleTenant, 0) if err = json.Unmarshal([]byte(getEX.String()), &list); err != nil { - r.Response.WriteJson(err) - r.Exit() + SetResponseInfo(r.Context(), r, err) } var expireAt *gtime.Time for _, value := range list { @@ -60,45 +66,45 @@ func (s *moduleTenant) ModuleTenantCheck(r *ghttp.Request) { gt1 := gtime.New(time.Now()) gt2 := gtime.New(expireAt) if !gt1.Before(gt2) { - r.Response.WriteJson(gerror.New("您访问的模块已过期,请续期后再使用")) - r.Exit() + SetResponseInfo(r.Context(), r, "您访问的模块已过期,请续期后再使用") } } else { - r.Response.WriteJson(gerror.New("您未开通此模块,请开通后再使用")) - r.Exit() + SetResponseInfo(r.Context(), r, "您未开通此模块,请开通后再使用") } } else { - //将 http.Header 转换为 map[string]string - headers := make(map[string]string) - for k, v := range r.Request.Header { - if len(v) > 0 { - headers[k] = v[0] - } - } // 缓存为空,调用admin-go的Check接口检查模块开通状态 - res, err := s.Check(r.Context(), headers, beans.ModuleTenantCheckReq{ + res, err := Check(r.Context(), headers, beans.ModuleTenantCheckReq{ ModuleKey: moduleKey.String(), TenantId: gconv.Uint64(getUserInfo.TenantId), }) if err != nil { - r.Response.WriteJson(err) - r.Exit() + SetResponseInfo(r.Context(), r, err) } // 根据检查结果判断是否允许访问 if res.Status == "not_activated" { - r.Response.WriteJson(gerror.New("您未开通此模块,请开通后再使用")) - r.Exit() + SetResponseInfo(r.Context(), r, "您未开通此模块,请开通后再使用") } else if res.Status == "expired" { - r.Response.WriteJson(gerror.New("您访问的模块已过期,请续期后再使用")) - r.Exit() + SetResponseInfo(r.Context(), r, "您访问的模块已过期,请续期后再使用") } } r.Middleware.Next() // 继续执行后续中间件和路由处理 } +// SetResponseInfo 设置响应信息 +func SetResponseInfo(ctx context.Context, r *ghttp.Request, message any) { + _ = ctx + r.Response.Status = 503 + r.Response.WriteJsonExit(map[string]interface{}{ + "success": false, + "code": 503, + "message": fmt.Sprintf("服务不可用:%s", message), + }) + r.Exit() +} + // Check 调用admin-go服务检查模块开通状态 -func (s *moduleTenant) Check(ctx context.Context, headers map[string]string, req beans.ModuleTenantCheckReq) (res *beans.ModuleTenantCheckRes, err error) { - if err = http.Get(ctx, "admin-go/api/v1/system/moduleTenant/check", headers, &res, +func Check(ctx context.Context, headerMap map[string]string, req beans.ModuleTenantCheckReq) (res *beans.ModuleTenantCheckRes, err error) { + if err = http.Get(ctx, "admin-go/api/v1/system/moduleTenant/check", headerMap, &res, "moduleKey", req.ModuleKey, "tenantId", req.TenantId, ); err != nil { @@ -106,3 +112,11 @@ func (s *moduleTenant) Check(ctx context.Context, headers map[string]string, req } return } + +// IsSuperAdmin 调用admin-go服务检查是否是超级管理员 +func IsSuperAdmin(ctx context.Context, headerMap map[string]string) (res *beans.IsSuperAdminRes, err error) { + if err = http.Get(ctx, "admin-go/api/v1/system/user/checkIsSuperAdmin", headerMap, &res); err != nil { + return + } + return +} diff --git a/mongo/mongo.go b/mongo/mongo.go index c510ea5..4cc2291 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -544,6 +544,18 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( return } +func BuildUpdateFilter(ctx context.Context, req interface{}) (filter bson.M, err error) { + _ = ctx + filter = bson.M{} + reqMap := gconv.Map(req) + for mk, mv := range reqMap { + if mk != "id" && !g.IsEmpty(mv) { + filter[mk] = mv + } + } + return +} + // EntityToBson 将 *entity/entity 转换为 bson.M func EntityToBson(entity interface{}) (bson.M, error) { return EntityToBsonWithFilter(entity, false) From f0e6bdd37cccdb27655d00a4857a5e287bedac64 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Fri, 16 Jan 2026 16:55:32 +0800 Subject: [PATCH 176/225] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=EF=BC=9A=E9=87=8D=E6=9E=84=E6=93=8D=E4=BD=9C?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=AE=B0=E5=BD=95=E4=B8=8E=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=EF=BC=8C=E6=94=AF=E6=8C=81=E6=89=B9=E9=87=8F?= =?UTF-8?q?=E6=93=8D=E4=BD=9C=E6=97=A5=E5=BF=97=E8=AE=B0=E5=BD=95=EF=BC=8C?= =?UTF-8?q?=E5=AE=8C=E5=96=84=E6=97=A5=E5=BF=97=E6=9F=A5=E8=AF=A2=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=EF=BC=8C=E5=A2=9E=E5=8A=A0=E8=BD=AF=E5=88=A0=E9=99=A4?= =?UTF-8?q?=E6=93=8D=E4=BD=9C=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- log/consts/log_const.go | 14 +- log/controller/log_controller.go | 32 +--- log/dao/log_dao.go | 79 +-------- log/model/dto/log_dto.go | 41 ++--- log/model/entity/log.go | 2 +- log/service/log_service.go | 109 +++++------- mongo/connection.go | 3 +- mongo/mongo.go | 280 ++++++++++++++++++++----------- 8 files changed, 260 insertions(+), 300 deletions(-) diff --git a/log/consts/log_const.go b/log/consts/log_const.go index ca33ca3..7f4e5fb 100644 --- a/log/consts/log_const.go +++ b/log/consts/log_const.go @@ -4,12 +4,20 @@ package consts type OperationType string const ( - OperationCreate OperationType = "create" // 创建 - OperationUpdate OperationType = "update" // 更新 - OperationDelete OperationType = "delete" // 删除 + OperationInsert OperationType = "insert" // 创建 + OperationUpdate OperationType = "update" // 更新 + OperationDelete OperationType = "delete" // 删除 + OperationDeleteSoft OperationType = "delete_soft" // 软删除 ) // OperationLogCollection 操作日志集合名称常量 const ( OperationLogCollection = "operation_logs" // 操作日志集合名称 ) + +// 消费者配置(从 Redis Stream 消费请求) +const StreamKey = "log:%s" // 请求 Stream 键名(与发消息的key一致) +const GroupName = "log:consumer:group" // 消费者组名 +const ConsumerName = "message-consumer-1" // 消费者名称(唯一标识) +const BatchSize = 1 // 批处理大小(每次读取1条) +const AutoAck = true // ACK是否自动确认(true自动确认,false不确认) diff --git a/log/controller/log_controller.go b/log/controller/log_controller.go index 9b0ec56..3485423 100644 --- a/log/controller/log_controller.go +++ b/log/controller/log_controller.go @@ -12,33 +12,7 @@ type operationLog struct{} // OperationLog 操作日志控制器 var OperationLog = new(operationLog) -// GetByID 根据ID获取操作日志 -// @Summary 获取操作日志详情 -// @Description 根据日志ID获取操作日志的详细信息 -func (c *operationLog) GetByID(ctx context.Context, req *dto.GetLogReq) (res *dto.GetLogResp, err error) { - logInfo, err := service.OperationLog.GetByID(ctx, req.ID) - if err != nil { - return - } - - res = &dto.GetLogResp{ - OperationLogInfo: *logInfo, - } - return -} - -// List 查询操作日志列表(通用方法,支持根据不同条件动态查询) -// @Summary 查询操作日志列表 -// @Description 根据多个条件查询操作日志列表 -func (c *operationLog) List(ctx context.Context, req *dto.ListLogsReq) (res *dto.ListLogsResp, err error) { - logs, total, err := service.OperationLog.List(ctx, req) - if err != nil { - return - } - - res = &dto.ListLogsResp{ - Logs: logs, - Total: total, - } - return +// GetByCollectionId 根据collectionId获取操作日志列表 +func (c *operationLog) GetByCollectionId(ctx context.Context, req *dto.ListLogsReq) (res *dto.ListLogsResp, err error) { + return service.OperationLog.GetByCollectionId(ctx, req) } diff --git a/log/dao/log_dao.go b/log/dao/log_dao.go index c209a45..7f435de 100644 --- a/log/dao/log_dao.go +++ b/log/dao/log_dao.go @@ -2,6 +2,8 @@ package dao import ( "context" + "gitee.com/red-future---jilin-g/common/beans" + "strings" "time" "gitee.com/red-future---jilin-g/common/log/consts" @@ -9,7 +11,6 @@ import ( "gitee.com/red-future---jilin-g/common/log/model/entity" "gitee.com/red-future---jilin-g/common/mongo" "go.mongodb.org/mongo-driver/v2/bson" - "go.mongodb.org/mongo-driver/v2/mongo/options" ) type log struct{} @@ -23,75 +24,14 @@ func (d *log) Create(ctx context.Context, log *entity.OperationLog) error { return err } -// CreateBatch 批量创建日志记录 -func (d *log) CreateBatch(ctx context.Context, logs []*entity.OperationLog) error { - if len(logs) == 0 { - return nil - } - - documents := make([]interface{}, len(logs)) - for i, log := range logs { - documents[i] = log - } - - _, err := mongo.DB().Insert(ctx, documents, consts.OperationLogCollection) - return err -} - -// GetByID 根据ID获取日志 -func (d *log) GetByID(ctx context.Context, id string) (*entity.OperationLog, error) { - objectID, err := bson.ObjectIDFromHex(id) - if err != nil { - return nil, err - } - - filter := bson.M{"_id": objectID} - var log entity.OperationLog - err = mongo.DB().FindOne(ctx, filter, &log, consts.OperationLogCollection) - if err != nil { - return nil, err - } - return &log, nil -} - // List 查询日志列表(通用方法,通过filter动态拼接查询条件) -func (d *log) List(ctx context.Context, filter *dto.ListLogsReq, sortFields ...string) ([]*entity.OperationLog, int64, error) { - bsonFilter := buildFilter(filter) - - total, err := mongo.DB().Count(ctx, bsonFilter, consts.OperationLogCollection) - if err != nil { - return nil, 0, err +func (d *log) List(ctx context.Context, req *dto.ListLogsReq) (res []*entity.OperationLog, total int64, err error) { + filter := buildFilter(req) + req.OrderBy = []beans.OrderBy{ + {Field: "createdAt", Order: beans.Desc}, } - - var findOptions []options.Lister[options.FindOptions] - if filter.PageNum > 0 && filter.PageSize > 0 { - findOptions = append(findOptions, options.Find().SetSkip(int64((filter.PageNum-1)*filter.PageSize)).SetLimit(int64(filter.PageSize))) - } - - if len(sortFields) > 0 { - sort := bson.D{} - for _, field := range sortFields { - var order int - if len(field) > 0 && field[0] == '-' { - order = -1 - field = field[1:] - } else { - order = 1 - } - sort = append(sort, bson.E{Key: field, Value: order}) - } - findOptions = append(findOptions, options.Find().SetSort(sort)) - } else { - findOptions = append(findOptions, options.Find().SetSort(bson.D{{Key: "createdAt", Value: -1}})) - } - - var logs []*entity.OperationLog - _, err = mongo.DB().Find(ctx, bsonFilter, &logs, consts.OperationLogCollection, nil, nil) - if err != nil { - return nil, 0, err - } - - return logs, total, nil + total, err = mongo.DB().Find(ctx, filter, &res, consts.OperationLogCollection, req.Page, req.OrderBy) + return } // buildFilter 构建MongoDB查询过滤器 @@ -107,12 +47,11 @@ func buildFilter(filter interface{}) bson.M { bsonFilter["collection"] = req.Collection } if req.CollectionID != "" { - bsonFilter["collection_id"] = req.CollectionID + bsonFilter["collection_id"] = bson.M{"$in": strings.Split(req.CollectionID, ",")} } if req.Operation != "" { bsonFilter["operation"] = req.Operation } - // 处理时间范围字段 if req.StartTime != "" || req.EndTime != "" { timeFilter := bson.M{} diff --git a/log/model/dto/log_dto.go b/log/model/dto/log_dto.go index fab47af..ba12fda 100644 --- a/log/model/dto/log_dto.go +++ b/log/model/dto/log_dto.go @@ -3,37 +3,13 @@ package dto import ( "gitee.com/red-future---jilin-g/common/beans" "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/gtime" ) -// ========== 操作日志查询相关DTO ========== - -// GetLogReq 获取操作日志请求 -type GetLogReq struct { - g.Meta `path:"/getLog" method:"get" tags:"操作日志" summary:"获取操作日志详情" dc:"根据日志ID获取操作日志的详细信息"` - ID string `json:"id" v:"required" dc:"日志ID"` -} - -// GetLogResp 获取操作日志响应 -type GetLogResp struct { - OperationLogInfo -} - -// OperationLogInfo 操作日志信息 -type OperationLogInfo struct { - ID string `json:"id" dc:"日志ID"` - ServiceName string `json:"service_name" dc:"服务名"` - Collection string `json:"collection" dc:"数据所在集合名称"` - CollectionID string `json:"collection_id" dc:"数据ID"` - Operation string `json:"operation" dc:"操作类型"` - UserName string `json:"user_name" dc:"操作人名称"` - IPAddress string `json:"ip_address" dc:"操作IP地址"` - Data map[string]interface{} `json:"data" dc:"当前数据"` -} - // ListLogsReq 查询操作日志列表请求(通用方法,支持根据不同条件动态查询) type ListLogsReq struct { g.Meta `path:"/listLogs" method:"get" tags:"操作日志" summary:"查询操作日志列表" dc:"根据多个条件查询操作日志列表"` - beans.Page + *beans.Page ServiceName string `json:"service_name" dc:"服务名(可选)"` Collection string `json:"collection" dc:"数据所在集合名称(可选)"` CollectionID string `json:"collection_id" dc:"数据ID(可选)"` @@ -48,3 +24,16 @@ type ListLogsResp struct { Logs []OperationLogInfo `json:"logs" dc:"日志列表"` Total int64 `json:"total" dc:"总数"` } + +// OperationLogInfo 操作日志信息 +type OperationLogInfo struct { + ID string `json:"id" dc:"日志ID"` + ServiceName string `json:"service_name" dc:"服务名"` + Collection string `json:"collection" dc:"数据所在集合名称"` + CollectionID interface{} `json:"collection_id" dc:"数据ID"` + Operation string `json:"operation" dc:"操作类型"` + Creator string `json:"creator" dc:"操作人名称"` + CreatedAt *gtime.Time `json:"createdAt" dc:"创建时间"` + Data interface{} `json:"data" dc:"当前数据"` + IPAddress string `json:"ip_address" dc:"操作IP地址"` +} diff --git a/log/model/entity/log.go b/log/model/entity/log.go index ac566a4..463926f 100644 --- a/log/model/entity/log.go +++ b/log/model/entity/log.go @@ -10,7 +10,7 @@ type OperationLog struct { ServiceName string `bson:"service_name" json:"service_name"` // 服务名:具体的微服务名称 Collection string `bson:"collection" json:"collection"` // 集合名:数据所在的集合名称 - CollectionID string `bson:"collection_id" json:"collection_id"` // 数据ID:具体操作的数据ID,如订单号、钱包ID等 + CollectionID interface{} `bson:"collection_id" json:"collection_id"` // 数据ID:具体操作的数据ID,如订单号、钱包ID等 Operation string `bson:"operation" json:"operation"` // 操作类型:create, update, delete IPAddress string `bson:"ip_address" json:"ip_address"` // 操作IP地址 Data interface{} `bson:"data,omitempty" json:"data"` // 当前数据:操作时的数据状态 diff --git a/log/service/log_service.go b/log/service/log_service.go index 73a92d6..c16e8cc 100644 --- a/log/service/log_service.go +++ b/log/service/log_service.go @@ -2,12 +2,11 @@ package service import ( "context" - - "gitee.com/red-future---jilin-g/common/log/consts" + "gitee.com/red-future---jilin-g/common/beans" "gitee.com/red-future---jilin-g/common/log/dao" "gitee.com/red-future---jilin-g/common/log/model/dto" logEntity "gitee.com/red-future---jilin-g/common/log/model/entity" - "github.com/gogf/gf/v2/frame/g" + "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/util/gconv" ) @@ -16,84 +15,52 @@ type operationLog struct{} // OperationLog 操作日志服务 var OperationLog = &operationLog{} -// RecordCreate 记录创建操作 -func (s *operationLog) RecordCreate(ctx context.Context, serviceName, collection, collectionID string, data map[string]interface{}) error { - return s.record(ctx, serviceName, collection, collectionID, string(consts.OperationCreate), data) -} - -// RecordUpdate 记录更新操作 -func (s *operationLog) RecordUpdate(ctx context.Context, serviceName, collection, collectionID string, data map[string]interface{}) error { - return s.record(ctx, serviceName, collection, collectionID, string(consts.OperationUpdate), data) -} - -// RecordDelete 记录删除操作 -func (s *operationLog) RecordDelete(ctx context.Context, serviceName, collection, collectionID string, data map[string]interface{}) error { - return s.record(ctx, serviceName, collection, collectionID, string(consts.OperationDelete), data) -} - -// BatchRecordCreate 批量记录创建操作 -func (s *operationLog) BatchRecordCreate(ctx context.Context, logs []*logEntity.OperationLog) error { - return dao.Log.CreateBatch(ctx, logs) -} - -// GetByID 根据ID获取操作日志 -func (s *operationLog) GetByID(ctx context.Context, id string) (*dto.OperationLogInfo, error) { - log, err := dao.Log.GetByID(ctx, id) - if err != nil { - return nil, err - } - - var logInfo dto.OperationLogInfo - if err := gconv.Struct(log, &logInfo); err != nil { - return nil, err - } - logInfo.ID = log.Id.Hex() - return &logInfo, nil -} - -// List 查询操作日志列表 -func (s *operationLog) List(ctx context.Context, req *dto.ListLogsReq) ([]dto.OperationLogInfo, int64, error) { - logs, total, err := dao.Log.List(ctx, req) - if err != nil { - return nil, 0, err - } - - var logInfos []dto.OperationLogInfo - err = gconv.Structs(logs, &logInfos) - if err != nil { - return nil, 0, err - } - - // 处理特殊字段 - for i, log := range logs { - logInfos[i].ID = log.Id.Hex() - } - - return logInfos, total, nil -} - -// record 记录操作日志的通用方法 -func (s *operationLog) record(ctx context.Context, serviceName, collection, collectionID, operation string, data map[string]interface{}) error { - // 获取请求信息 - ipAddress := getHTTPRequestInfo(ctx) +func (s *operationLog) AddOperationLog(ctx context.Context, msg map[string]interface{}) error { + serviceName := gconv.String(msg["service_name"]) + collection := gconv.String(msg["collection"]) + collectionId := gconv.Strings(msg["collection_id"]) + operation := gconv.String(msg["operation"]) + ipAddress := gconv.String(msg["ip_address"]) + data := gconv.Maps(msg["data"]) + creator := gconv.String(msg["creator"]) + createdAt := gconv.Time(msg["createdAt"]) + updater := gconv.String(msg["updater"]) + updatedAt := gconv.Time(msg["updatedAt"]) + tenantId := gconv.Float64(msg["tenantId"]) + // 设置 userId 和 tenantId 到 ctx + ctx = context.WithValue(ctx, "userName", creator) + ctx = context.WithValue(ctx, "tenantId", tenantId) log := &logEntity.OperationLog{ + MongoBaseDO: beans.MongoBaseDO{ + Creator: creator, + CreatedAt: &createdAt, + Updater: updater, + UpdatedAt: &updatedAt, + TenantId: tenantId, + }, ServiceName: serviceName, Collection: collection, - CollectionID: collectionID, + CollectionID: collectionId, Operation: operation, IPAddress: ipAddress, Data: data, } - return dao.Log.Create(ctx, log) } -// getHTTPRequestInfo 从上下文中获取HTTP请求信息 -func getHTTPRequestInfo(ctx context.Context) string { - request := g.RequestFromCtx(ctx) - if request != nil { - return request.GetClientIp() +// GetByCollectionId 根据集合ID获取操作日志 +func (s *operationLog) GetByCollectionId(ctx context.Context, req *dto.ListLogsReq) (res *dto.ListLogsResp, err error) { + logs, total, err := dao.Log.List(ctx, req) + if err != nil { + return } - return "" + res = &dto.ListLogsResp{ + Total: total, + } + err = utils.Struct(logs, &res.Logs) + if err != nil { + return + } + return } diff --git a/mongo/connection.go b/mongo/connection.go index 77f1cec..a46a559 100644 --- a/mongo/connection.go +++ b/mongo/connection.go @@ -8,6 +8,7 @@ package mongo import ( "context" "fmt" + "gitee.com/red-future---jilin-g/common/log/consts" "os" "os/signal" "strings" @@ -244,7 +245,7 @@ func (m *DataSourceManager) GetAllDataSourceNames() []string { func init() { logPool = grpool.New(1) serverName = g.Cfg().MustGet(context.TODO(), "server.name").String() - logRedisKey = fmt.Sprintf("log:%s", serverName) + LogRedisKey = fmt.Sprintf(consts.StreamKey, serverName) ctx := context.Background() diff --git a/mongo/mongo.go b/mongo/mongo.go index 4cc2291..2164546 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "gitee.com/red-future---jilin-g/common/log/consts" "reflect" "time" @@ -62,9 +63,15 @@ var ( manager = GetManager() logPool *grpool.Pool serverName string - logRedisKey string + LogRedisKey string ) +// FieldInfo 定义字段信息结构体 +type FieldInfo struct { + FieldName string + FieldValue interface{} +} + const PageSize = 20 // GetDB 获取默认数据源的数据库实例(向后兼容) @@ -88,6 +95,43 @@ func (m *MongoDB) getDataSource() (DataSource, error) { return manager.GetDataSource(m.dataSource) } +// Count 查询总数 +func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { + source, err := m.getDataSource() + if err != nil { + return 0, err + } + db := source.Database() + + user, err := utils.GetUserInfo(ctx) + if err != nil { + return + } + filter["isDeleted"] = false + delete(filter, "tenantId") + filterKey := fmt.Sprintf("%+v", filter) + redisKey := fmt.Sprintf(redis.Count, user.TenantId, collection, filterKey) + if m.Cache { + var resultStr *gvar.Var + resultStr, err = redis.RedisClient.Get(ctx, redisKey) + if err != nil { + return + } + if !g.IsEmpty(resultStr) { + count = gconv.Int64(resultStr) + return + } + } + count, err = db.Collection(collection).CountDocuments(ctx, filter) + if m.Cache { + err = redis.RedisClient.SetEX(ctx, redisKey, count, int64(time.Hour)) + if err != nil { + return + } + } + return +} + // Find 查询多条记录 func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, collection string, page *beans.Page, orderBy []beans.OrderBy) (total int64, err error) { source, err := m.getDataSource() @@ -103,7 +147,9 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c if err != nil { return } - filter["isDeleted"] = false + if g.IsEmpty(filter["isDeleted"]) { + filter["isDeleted"] = false + } filterKey := fmt.Sprintf("%+v", filter) optionsKey := fmt.Sprintf("%+v%+v", page, orderBy) redisKey := fmt.Sprintf(redis.List, user.TenantId, collection, filterKey, optionsKey) @@ -125,7 +171,7 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c limit := int64(PageSize) skip := int64(0) - if page != nil { + if page != nil && !g.IsEmpty(page.PageNum) && !g.IsEmpty(page.PageSize) { limit = page.PageSize if limit == -1 { skip = 0 @@ -229,6 +275,17 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} return } +// getDeletedData 获取要删除的数据 +func (m *MongoDB) getDeletedData(ctx context.Context, filter bson.M, collection string) (deletedIDs []bson.ObjectID, deletedData []bson.M, err error) { + // 查询要删除的数据 + _, err = m.Find(ctx, filter, &deletedData, collection, nil, nil) + // 从查询结果中获取 _id + for _, doc := range deletedData { + deletedIDs = append(deletedIDs, doc["_id"].(bson.ObjectID)) + } + return +} + func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collection string) (err error) { listKeys := fmt.Sprintf(redis.CleanList, tenantId, collection) keys, err := redis.RedisClient.Keys(ctx, listKeys) @@ -263,27 +320,92 @@ func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interf return } -func (m *MongoDB) log(ctx context.Context, filter bson.M, collection string, data interface{}, userName, tenantId interface{}, operationType string) { - _ = logPool.AddWithRecover(ctx, func(ctx context.Context) { - log := &entity.OperationLog{ - ServiceName: serverName, - Collection: collection, - CollectionID: filter["_id"].(string), - Operation: operationType, - IPAddress: g.RequestFromCtx(ctx).GetClientIp(), - Data: data, +func (m *MongoDB) log(ctx context.Context, ids []bson.ObjectID, filter bson.M, collection string, data interface{}, userName, tenantId interface{}, operationType consts.OperationType) { + // 提前获取 IP 地址,避免异步任务执行时请求已结束 + var ipAddress string + if request := g.RequestFromCtx(ctx); request != nil { + ipAddress = request.GetClientIp() + } + if operationType != consts.OperationInsert && operationType != consts.OperationDelete { + if !g.IsEmpty(filter["_id"]) { + objectID := filter["_id"].(*bson.ObjectID) + ids = append(ids, *objectID) + } else { + var err error + if ids, _, err = m.getDeletedData(ctx, filter, collection); err != nil { + return + } } - log.Creator = userName - now := >ime.Now().Time - log.CreatedAt = now - log.UpdatedAt = now - log.TenantId = tenantId - if _, err := redis.AddToStream(ctx, logRedisKey, log); err != nil { - glog.Error(ctx, "mongoLog-AddToStream err: %v", err) + } + log := &entity.OperationLog{ + ServiceName: serverName, + Collection: collection, + CollectionID: ids, + Operation: string(operationType), + IPAddress: ipAddress, + Data: data, + } + log.Creator = userName + log.Updater = userName + now := >ime.Now().Time + log.CreatedAt = now + log.UpdatedAt = now + log.TenantId = tenantId + // 使用新的 context 进行 Redis 操作 + if _, err := redis.AddToStream(ctx, LogRedisKey, log); err != nil { + glog.Error(ctx, "mongoLog-AddToStream err: %v", err) + } + return +} + +// Insert 插入多条记录 +func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { + source, err := m.getDataSource() + if err != nil { + return nil, err + } + db := source.Database() + + user, err := utils.GetUserInfo(ctx) + if err != nil { + return + } + docs := make([]interface{}, 0, len(documents)) + for _, document := range documents { + doc := gconv.Map(document) + delete(doc, "id") + if !g.IsEmpty(user.UserName) && g.IsEmpty(doc["creator"]) { + doc["creator"] = user.UserName } - }, func(ctx context.Context, exception error) { - glog.Error(ctx, "mongoLog-AddWithRecover err: %v", exception) - }) + if !g.IsEmpty(user.UserName) && g.IsEmpty(doc["updater"]) { + doc["updater"] = user.UserName + } + if !g.IsEmpty(user.TenantId) && g.IsEmpty(doc["tenantId"]) { + doc["tenantId"] = user.TenantId + } + if g.IsEmpty(doc["createdAt"]) { + doc["createdAt"] = gtime.Now().Time + } + if g.IsEmpty(doc["updatedAt"]) { + doc["updatedAt"] = gtime.Now().Time + } + doc["isDeleted"] = false + docs = append(docs, doc) + } + r, err := db.Collection(collection).InsertMany(ctx, docs, opts...) + if err != nil { + return + } + ids = r.InsertedIDs + err = m.CleanRedis(ctx, bson.M{}, user.TenantId, collection) + //写日志 + if collection != consts.OperationLogCollection { + objectIds := make([]bson.ObjectID, 0) + for _, id := range ids { + objectIds = append(objectIds, id.(bson.ObjectID)) + } + m.log(ctx, objectIds, nil, collection, nil, user.UserName, user.TenantId, consts.OperationInsert) + } return } @@ -304,15 +426,30 @@ func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, return } filter["tenantId"] = user.TenantId + // 获取要删除的数据 + ds, ms, err := m.getDeletedData(ctx, filter, collection) + if err != nil { + return + } + // 执行删除操作 r, err := db.Collection(collection).DeleteMany(ctx, filter, opts...) if err != nil { return } count = r.DeletedCount + // 清理redis err = m.CleanRedis(ctx, filter, user.TenantId, collection) + // 写日志 + m.log(ctx, ds, nil, collection, ms, user.UserName, user.TenantId, consts.OperationDelete) return } +// DeleteSoft 假删除记录 +func (m *MongoDB) DeleteSoft(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { + update := bson.M{"$set": bson.M{"isDeleted": true}} + return m.Update(ctx, filter, update, collection, opts...) +} + // Update 修改记录 func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { source, err := m.getDataSource() @@ -333,17 +470,38 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll if !g.IsEmpty(user.TenantId) { filter["tenantId"] = user.TenantId } + // 遍历 update 中的所有操作符和字段,存放到 list 中 + fieldList := make([]FieldInfo, 0) + for _, doc := range update { + if m, ok := doc.(bson.M); ok { + for fieldName, fieldValue := range m { + // 获取到字段名和字段值 + fieldList = append(fieldList, FieldInfo{ + FieldName: fieldName, + FieldValue: fieldValue, + }) + } + } + } setDoc := update["$set"].(bson.M) if !g.IsEmpty(user.UserName) { setDoc["updater"] = user.UserName } setDoc["updatedAt"] = gtime.Now().Time - update = bson.M{"$set": setDoc} + update["$set"] = setDoc result, err = db.Collection(collection).UpdateMany(ctx, filter, update, opts...) if err != nil { return } + // 清理redis err = m.CleanRedis(ctx, filter, user.TenantId, collection) + // 写日志 + if !g.IsEmpty(setDoc["isDeleted"]) && gconv.Bool(setDoc["isDeleted"]) { + filter["isDeleted"] = true + m.log(ctx, nil, filter, collection, nil, user.UserName, user.TenantId, consts.OperationDeleteSoft) + } else { + m.log(ctx, nil, filter, collection, fieldList, user.UserName, user.TenantId, consts.OperationUpdate) + } return } @@ -468,82 +626,6 @@ func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bs return bulkResult, nil } -// Insert 插入多条记录 -func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { - source, err := m.getDataSource() - if err != nil { - return nil, err - } - db := source.Database() - - user, err := utils.GetUserInfo(ctx) - if err != nil { - return - } - docs := make([]interface{}, 0, len(documents)) - for _, document := range documents { - doc := gconv.Map(document) - delete(doc, "id") - if !g.IsEmpty(user.UserName) { - doc["creator"] = user.UserName - } - if !g.IsEmpty(user.UserName) { - doc["updater"] = user.UserName - } - if !g.IsEmpty(user.TenantId) { - doc["tenantId"] = user.TenantId - } - doc["createdAt"] = gtime.Now().Time - doc["updatedAt"] = gtime.Now().Time - doc["isDeleted"] = false - docs = append(docs, doc) - } - r, err := db.Collection(collection).InsertMany(ctx, docs, opts...) - if err != nil { - return - } - ids = r.InsertedIDs - err = m.CleanRedis(ctx, bson.M{}, user.TenantId, collection) - return -} - -// Count 查询总数 -func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { - source, err := m.getDataSource() - if err != nil { - return 0, err - } - db := source.Database() - - user, err := utils.GetUserInfo(ctx) - if err != nil { - return - } - filter["isDeleted"] = false - delete(filter, "tenantId") - filterKey := fmt.Sprintf("%+v", filter) - redisKey := fmt.Sprintf(redis.Count, user.TenantId, collection, filterKey) - if m.Cache { - var resultStr *gvar.Var - resultStr, err = redis.RedisClient.Get(ctx, redisKey) - if err != nil { - return - } - if !g.IsEmpty(resultStr) { - count = gconv.Int64(resultStr) - return - } - } - count, err = db.Collection(collection).CountDocuments(ctx, filter) - if m.Cache { - err = redis.RedisClient.SetEX(ctx, redisKey, count, int64(time.Hour)) - if err != nil { - return - } - } - return -} - func BuildUpdateFilter(ctx context.Context, req interface{}) (filter bson.M, err error) { _ = ctx filter = bson.M{} From b303c1bfaa5fe31dd677fa1a7c890f56f1a94761 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Wed, 21 Jan 2026 16:33:37 +0800 Subject: [PATCH 177/225] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E5=92=8C=E6=A8=A1=E5=9D=97=E7=A7=9F=E6=88=B7?= =?UTF-8?q?=E6=A3=80=E6=9F=A5=E9=80=BB=E8=BE=91=EF=BC=8C=E6=96=B0=E5=A2=9E?= =?UTF-8?q?NATS=E6=B6=88=E6=81=AF=E9=85=8D=E7=BD=AE=E5=92=8CMongoDB?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E6=8E=A7=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- beans/beans.go | 3 +- beans/module_tenant.go | 48 +++ beans/user.go | 5 - log/consts/log_const.go | 2 + middleware/module_tenant_check.go | 47 +-- mongo/mongo.go | 47 ++- nats/msg.go | 65 +++ nats/nats.go | 440 ++----------------- nats/nats_consumer.go | 294 +++++++++++++ nats/nats_publish.go | 28 ++ nats/nats_rpc.go | 673 ++++++++++++++++++++++++++++++ nats/nats_task.go | 212 ++++++++++ nats/nats_test.go | 33 +- nats/task.go | 411 ++++++++++++++++++ nats/utils.go | 87 ++++ utils/utils.go | 56 ++- 16 files changed, 1954 insertions(+), 497 deletions(-) delete mode 100644 beans/user.go create mode 100644 nats/msg.go create mode 100644 nats/nats_consumer.go create mode 100644 nats/nats_publish.go create mode 100644 nats/nats_rpc.go create mode 100644 nats/nats_task.go create mode 100644 nats/task.go create mode 100644 nats/utils.go diff --git a/beans/beans.go b/beans/beans.go index 5c65a77..58c7281 100644 --- a/beans/beans.go +++ b/beans/beans.go @@ -37,6 +37,7 @@ type MongoBaseDO struct { } type User struct { - UserName interface{} `bson:"userName" json:"userName"` // MongoDB 默认 ID + UserId interface{} `bson:"userId" json:"userId"` // 用户ID + UserName interface{} `bson:"userName" json:"userName"` // 用户名 TenantId interface{} `bson:"tenantId" json:"tenantId"` // 租户ID } diff --git a/beans/module_tenant.go b/beans/module_tenant.go index 19df2ed..fb2f328 100644 --- a/beans/module_tenant.go +++ b/beans/module_tenant.go @@ -4,6 +4,54 @@ import ( "github.com/gogf/gf/v2/os/gtime" ) +// ModuleAssetId 模块资产ID映射(key-value结构) +// Key: 服务名,Value: 资产ID +var ModuleAssetId = map[string]string{ + "assets": "696b4acd1be1c8b76c4b4c15", // 资产模块 + "cid": "696f423705e496ba4ccbe665", // 广告模块 + "customerService": "696f421205e496ba4ccbe662", // AI客服模块 +} + +// 模块类型(值从ModuleAssetId map获取) +var ( + TenantModuleAssets = ModuleAssetId["assets"] // 资产模块 + TenantModuleAd = ModuleAssetId["cid"] // 广告模块 + TenantModuleAICs = ModuleAssetId["customerService"] // AI客服模块 +) + +// TenantModuleType 租户类型 +type TenantModuleType struct { + Key string + Value string +} + +// TenantModuleTypesAssets 资产模块租户类型 +var TenantModuleTypesAssets = []TenantModuleType{ + {Key: "private_cloud", Value: "私有云租户"}, + {Key: "supplier", Value: "供应商"}, + {Key: "small_shop", Value: "电商小店"}, +} + +// TenantModuleTypesAd 广告模块租户类型(待定) +var TenantModuleTypesAd []TenantModuleType + +// TenantModuleTypesAICs AI客服模块租户类型(待定) +var TenantModuleTypesAICs []TenantModuleType + +// GetTenantModuleTypes 获取模块的租户类型列表 +func GetTenantModuleTypes(module string) []TenantModuleType { + switch module { + case TenantModuleAssets: + return TenantModuleTypesAssets + case TenantModuleAd: + return TenantModuleTypesAd + case TenantModuleAICs: + return TenantModuleTypesAICs + default: + return []TenantModuleType{} + } +} + type ModuleTenantCheckReq struct { ModuleKey string `p:"moduleKey" v:"required#模块Key不能为空"` TenantId uint64 `p:"tenantId" v:"required#租户ID不能为空"` diff --git a/beans/user.go b/beans/user.go deleted file mode 100644 index 3170e5b..0000000 --- a/beans/user.go +++ /dev/null @@ -1,5 +0,0 @@ -package beans - -type IsSuperAdminRes struct { - IsSuperAdmin bool `p:"isSuperAdmin"` -} diff --git a/log/consts/log_const.go b/log/consts/log_const.go index 7f4e5fb..8e2d0cb 100644 --- a/log/consts/log_const.go +++ b/log/consts/log_const.go @@ -21,3 +21,5 @@ const GroupName = "log:consumer:group" // 消费者组名 const ConsumerName = "message-consumer-1" // 消费者名称(唯一标识) const BatchSize = 1 // 批处理大小(每次读取1条) const AutoAck = true // ACK是否自动确认(true自动确认,false不确认) + +const LogSubject = "log:subject" diff --git a/middleware/module_tenant_check.go b/middleware/module_tenant_check.go index f3c22fa..0fb32bb 100644 --- a/middleware/module_tenant_check.go +++ b/middleware/module_tenant_check.go @@ -5,8 +5,8 @@ import ( "encoding/json" "fmt" "gitee.com/red-future---jilin-g/common/beans" - "gitee.com/red-future---jilin-g/common/http" "gitee.com/red-future---jilin-g/common/message" + "gitee.com/red-future---jilin-g/common/nats" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" @@ -25,12 +25,12 @@ func ModuleTenantCheck(r *ghttp.Request) { } } // 检查是否是超级管理员 - IsSuperAdmin, err := IsSuperAdmin(r.Context(), headers) - if err != nil { + isSuperAdmin := false + if err := nats.CallRPC(r.Context(), "userService.IsSuperAdmin", nil, &isSuperAdmin); err != nil { SetResponseInfo(r.Context(), r, err) } // 如果是超级管理员,则不进行模块租户检查 - if IsSuperAdmin.IsSuperAdmin { + if isSuperAdmin || r.Request.RequestURI == "/asset/getAssetAndSku?assetId=696b4acd1be1c8b76c4b4c15" { r.Middleware.Next() return } @@ -48,7 +48,7 @@ func ModuleTenantCheck(r *ghttp.Request) { SetResponseInfo(r.Context(), r, err) } // 获取模块key - moduleKey := g.Cfg().MustGet(context.Background(), "server.name") + moduleKey := g.Cfg().MustGet(context.Background(), "server.name").String() if !g.IsEmpty(getEX.String()) { list := make([]beans.ModuleTenant, 0) if err = json.Unmarshal([]byte(getEX.String()), &list); err != nil { @@ -56,7 +56,7 @@ func ModuleTenantCheck(r *ghttp.Request) { } var expireAt *gtime.Time for _, value := range list { - if value.ModuleKey == moduleKey.String() { + if value.ModuleKey == moduleKey { expireAt = value.ExpireAt break } @@ -73,17 +73,19 @@ func ModuleTenantCheck(r *ghttp.Request) { } } else { // 缓存为空,调用admin-go的Check接口检查模块开通状态 - res, err := Check(r.Context(), headers, beans.ModuleTenantCheckReq{ - ModuleKey: moduleKey.String(), + checkRes := new(beans.ModuleTenantCheckRes) + checkReq := beans.ModuleTenantCheckReq{ + ModuleKey: moduleKey, TenantId: gconv.Uint64(getUserInfo.TenantId), - }) + } + err = nats.CallRPC(r.Context(), "moduleService.Check", &checkReq, checkRes) if err != nil { SetResponseInfo(r.Context(), r, err) } // 根据检查结果判断是否允许访问 - if res.Status == "not_activated" { + if checkRes.Status == "not_activated" { SetResponseInfo(r.Context(), r, "您未开通此模块,请开通后再使用") - } else if res.Status == "expired" { + } else if checkRes.Status == "expired" { SetResponseInfo(r.Context(), r, "您访问的模块已过期,请续期后再使用") } } @@ -93,30 +95,11 @@ func ModuleTenantCheck(r *ghttp.Request) { // SetResponseInfo 设置响应信息 func SetResponseInfo(ctx context.Context, r *ghttp.Request, message any) { _ = ctx - r.Response.Status = 503 + r.Response.Status = 402 r.Response.WriteJsonExit(map[string]interface{}{ "success": false, - "code": 503, + "code": 402, "message": fmt.Sprintf("服务不可用:%s", message), }) r.Exit() } - -// Check 调用admin-go服务检查模块开通状态 -func Check(ctx context.Context, headerMap map[string]string, req beans.ModuleTenantCheckReq) (res *beans.ModuleTenantCheckRes, err error) { - if err = http.Get(ctx, "admin-go/api/v1/system/moduleTenant/check", headerMap, &res, - "moduleKey", req.ModuleKey, - "tenantId", req.TenantId, - ); err != nil { - return - } - return -} - -// IsSuperAdmin 调用admin-go服务检查是否是超级管理员 -func IsSuperAdmin(ctx context.Context, headerMap map[string]string) (res *beans.IsSuperAdminRes, err error) { - if err = http.Get(ctx, "admin-go/api/v1/system/user/checkIsSuperAdmin", headerMap, &res); err != nil { - return - } - return -} diff --git a/mongo/mongo.go b/mongo/mongo.go index 2164546..f0b10ae 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -34,18 +34,16 @@ import ( // ============================================================================= type MongoDB struct { - Cache bool + noCache bool dataSource string // 数据源名称,默认为 "default" + noTenantId bool // 是否跳过租户过滤 } func DB(cache ...bool) *MongoDB { - b := true - if len(cache) > 0 { - b = cache[0] - } return &MongoDB{ - Cache: b, + noCache: false, dataSource: "default", + noTenantId: false, } } @@ -55,6 +53,18 @@ func (m *MongoDB) WithDataSource(name string) *MongoDB { return m } +// NoCache 不使用缓存 +func (m *MongoDB) NoCache() *MongoDB { + m.noCache = true + return m +} + +// NoTenantId 不使用租户过滤 +func (m *MongoDB) NoTenantId() *MongoDB { + m.noTenantId = true + return m +} + // ============================================================================= // 向后兼容的全局变量和方法 // ============================================================================= @@ -111,7 +121,7 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( delete(filter, "tenantId") filterKey := fmt.Sprintf("%+v", filter) redisKey := fmt.Sprintf(redis.Count, user.TenantId, collection, filterKey) - if m.Cache { + if !m.noCache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) if err != nil { @@ -122,8 +132,12 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( return } } + // 如果没有调用 noTenantId,则添加 tenantId 过滤 + if !m.noTenantId && !g.IsEmpty(user.TenantId) { + filter["tenantId"] = user.TenantId + } count, err = db.Collection(collection).CountDocuments(ctx, filter) - if m.Cache { + if !m.noCache { err = redis.RedisClient.SetEX(ctx, redisKey, count, int64(time.Hour)) if err != nil { return @@ -153,7 +167,7 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c filterKey := fmt.Sprintf("%+v", filter) optionsKey := fmt.Sprintf("%+v%+v", page, orderBy) redisKey := fmt.Sprintf(redis.List, user.TenantId, collection, filterKey, optionsKey) - if m.Cache { + if !m.noCache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) if err != nil { @@ -167,8 +181,10 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c return } } - filter["tenantId"] = user.TenantId - + // 如果没有调用 noTenantId,则添加 tenantId 过滤 + if !m.noTenantId && !g.IsEmpty(user.TenantId) { + filter["tenantId"] = user.TenantId + } limit := int64(PageSize) skip := int64(0) if page != nil && !g.IsEmpty(page.PageNum) && !g.IsEmpty(page.PageSize) { @@ -213,7 +229,7 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c if err = cur.All(ctx, result); err != nil { return } - if m.Cache { + if !m.noCache { err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) if err != nil { return @@ -244,7 +260,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} filter["isDeleted"] = false filterKey := fmt.Sprintf("%+v", filter) redisKey := fmt.Sprintf(redis.One, user.TenantId, collection, filterKey) - if m.Cache { + if !m.noCache { var resultStr *gvar.Var resultStr, err = redis.RedisClient.Get(ctx, redisKey) if err != nil { @@ -258,7 +274,8 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} return } } - if !g.IsEmpty(user.TenantId) { + // 如果没有调用 noTenantId,则添加 tenantId 过滤 + if !m.noTenantId && !g.IsEmpty(user.TenantId) { filter["tenantId"] = user.TenantId } cur := db.Collection(collection).FindOne(ctx, filter, opts...) @@ -266,7 +283,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} if errors.Is(err, mongo.ErrNoDocuments) { err = nil } - if m.Cache { + if !m.noCache { err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) if err != nil { return err diff --git a/nats/msg.go b/nats/msg.go new file mode 100644 index 0000000..5929241 --- /dev/null +++ b/nats/msg.go @@ -0,0 +1,65 @@ +package nats + +import ( + "context" + "github.com/gogf/gf/v2/errors/gerror" +) + +// NatsMessageConfig nats Stream 消息配置 +type NatsMessageConfig struct { + CreateTaskStreamName string + CreateTaskSubjects []string + PublishSubject string + CreateTaskConsumerName string + MsgCount int + HandleFunc func(ctx context.Context, message map[string]interface{}) error +} + +// MessageConfig 消息配置接口 +type MessageConfig interface { + createTaskStream(ctx context.Context) error + publish(ctx context.Context, data interface{}) error + createTaskConsumer(ctx context.Context) error + //startConsumer(ctx context.Context, handleFunc func(ctx context.Context, msg *nats.Msg) error) error +} + +func (n *NatsMessageConfig) createTaskStream(ctx context.Context) error { + return createTaskStreamSimple(ctx, n.CreateTaskStreamName, n.CreateTaskSubjects) +} + +// CreateTaskStreamBatch 批量创建任务消息队列流 +func CreateTaskStreamBatch(ctx context.Context, configs ...MessageConfig) error { + for _, cfg := range configs { + if err := cfg.createTaskStream(ctx); err != nil { + return gerror.Wrap(err, "创建任务消息队列流失败") + } + } + return nil +} + +func (n *NatsMessageConfig) publish(ctx context.Context, data interface{}) error { + return publish(ctx, n.PublishSubject, data) +} + +// PublishMessage 发布消息(统一入口) +func PublishMessage(ctx context.Context, cfg MessageConfig, data interface{}) (err error) { + return cfg.publish(ctx, data) +} + +func (n *NatsMessageConfig) createTaskConsumer(ctx context.Context) error { + return CreateConsumerPushMode(ctx, n.CreateTaskStreamName, n.CreateTaskConsumerName, n.PublishSubject, n.MsgCount) +} + +// CreateTaskConsumerBatch 批量创建任务消息队列消费者 +func CreateTaskConsumerBatch(ctx context.Context, configs ...MessageConfig) error { + for _, cfg := range configs { + if err := cfg.createTaskConsumer(ctx); err != nil { + return gerror.Wrap(err, "创建任务消息队列流失败") + } + } + return nil +} + +//func (n *NatsMessageConfig) startConsumer(ctx context.Context, handleFunc func(ctx context.Context, msg *nats.Msg) error) error { +// return ConsumeMessages(ctx, n.CreateTaskStreamName, n.CreateTaskConsumerName, handleFunc) +//} diff --git a/nats/nats.go b/nats/nats.go index bcc325f..d01886e 100644 --- a/nats/nats.go +++ b/nats/nats.go @@ -4,43 +4,26 @@ import ( "context" "encoding/json" "fmt" - "reflect" - "sync" "time" "github.com/gogf/gf/v2/frame/g" - "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" ) -// RPC 服务注册表 -var ( - rpcServices map[string]RPCHandler - rpcSubs map[string]*nats.Subscription // 服务名 -> 订阅 - rpcServicesMu sync.RWMutex - queueRPCServices map[string]map[string]RPCHandler // queueName -> subject -> handler - queueRPCSubs map[string]map[string]*nats.Subscription // queueName -> serviceName -> 订阅 - queueRPCMu sync.RWMutex -) - -// RPCHandler RPC 处理函数类型 -// 实现方只需要关注请求参数和返回值,无需了解底层 NATS 实现 -type RPCHandler func(ctx context.Context, req []byte) ([]byte, error) - -// CreateTaskStream 创建任务消息队列流 +// createTaskStream 创建任务消息队列流(内部使用,兼容旧版本) // 存储策略: 文件存储 // 工作队列模式: 工作队列策略 -func CreateTaskStream(ctx context.Context, streamName string, subjects []string) error { - if !checkConnected() { +func CreateTaskStream(ctx context.Context, streamInfo TaskStreamConfig) error { + if !IsConnected() { return fmt.Errorf("NATS 未连接") } - stream, err := js.Stream(ctx, streamName) + stream, err := js.Stream(ctx, streamInfo.StreamName) if err == nil { // 流已存在,更新配置 _, err = js.UpdateStream(ctx, jetstream.StreamConfig{ - Name: streamName, - Subjects: subjects, + Name: streamInfo.StreamName, + Subjects: streamInfo.Subjects, Storage: jetstream.FileStorage, Retention: jetstream.WorkQueuePolicy, }) @@ -53,8 +36,8 @@ func CreateTaskStream(ctx context.Context, streamName string, subjects []string) // 创建新流 stream, err = js.CreateStream(ctx, jetstream.StreamConfig{ - Name: streamName, - Subjects: subjects, + Name: streamInfo.StreamName, + Subjects: streamInfo.Subjects, Storage: jetstream.FileStorage, Retention: jetstream.WorkQueuePolicy, }) @@ -71,7 +54,7 @@ func CreateTaskStream(ctx context.Context, streamName string, subjects []string) // 副本数: 单副本 (1) // 消息留存: 短时留存 (1小时) func CreateLogStream(ctx context.Context, streamName string, subjects []string) error { - if !checkConnected() { + if !IsConnected() { return fmt.Errorf("NATS 未连接") } @@ -115,7 +98,7 @@ func CreateLogStream(ctx context.Context, streamName string, subjects []string) // 副本数: 3副本 // 同步刷盘: 启用 func CreateTradeStream(ctx context.Context, streamName string, subjects []string) error { - if !checkConnected() { + if !IsConnected() { return fmt.Errorf("NATS 未连接") } @@ -154,25 +137,30 @@ func CreateTradeStream(ctx context.Context, streamName string, subjects []string return nil } -// Publish 发布消息到指定主题 -func Publish(ctx context.Context, subject string, data []byte) error { - if !checkConnected() { +// JsPublish 发布消息到指定主题 +func JsPublish(ctx context.Context, subject string, data any) (err error) { + if !IsConnected() { return fmt.Errorf("NATS 未连接") } - + // 序列化数据 + dataBytes, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("序列化数据失败: %w", err) + } + // 发布消息 metrics.PublishCount.Add(1) - _, err := js.Publish(ctx, subject, data) + _, err = js.Publish(ctx, subject, dataBytes) if err != nil { metrics.PublishError.Add(1) return fmt.Errorf("发布消息失败: %w", err) } - return nil + return } // GetStream 获取流信息 func GetStream(ctx context.Context, streamName string) (*jetstream.StreamInfo, error) { - if !checkConnected() { + if !IsConnected() { return nil, fmt.Errorf("NATS 未连接") } @@ -192,7 +180,7 @@ func GetStream(ctx context.Context, streamName string) (*jetstream.StreamInfo, e // ListStreams 列出所有流(简化实现) // 注意:由于 API 限制,此方法可能需要根据实际需求进一步实现 func ListStreams(ctx context.Context) ([]string, error) { - if !checkConnected() { + if !IsConnected() { return nil, fmt.Errorf("NATS 未连接") } @@ -202,7 +190,7 @@ func ListStreams(ctx context.Context) ([]string, error) { // DeleteStream 删除流 func DeleteStream(ctx context.Context, streamName string) error { - if !checkConnected() { + if !IsConnected() { return fmt.Errorf("NATS 未连接") } @@ -216,7 +204,7 @@ func DeleteStream(ctx context.Context, streamName string) error { // GetConsumer 获取消费者信息 func GetConsumer(ctx context.Context, streamName, consumerName string) (*jetstream.ConsumerInfo, error) { - if !checkConnected() { + if !IsConnected() { return nil, fmt.Errorf("NATS 未连接") } @@ -236,7 +224,7 @@ func GetConsumer(ctx context.Context, streamName, consumerName string) (*jetstre // ListConsumers 列出指定流的所有消费者(简化实现) // 注意:由于 API 限制,此方法可能需要根据实际需求进一步实现 func ListConsumers(ctx context.Context, streamName string) ([]string, error) { - if !checkConnected() { + if !IsConnected() { return nil, fmt.Errorf("NATS 未连接") } @@ -246,7 +234,7 @@ func ListConsumers(ctx context.Context, streamName string) ([]string, error) { // DeleteConsumer 删除消费者 func DeleteConsumer(ctx context.Context, streamName, consumerName string) error { - if !checkConnected() { + if !IsConnected() { return fmt.Errorf("NATS 未连接") } @@ -260,7 +248,7 @@ func DeleteConsumer(ctx context.Context, streamName, consumerName string) error // CreateConsumer 创建消费者 func CreateConsumer(ctx context.Context, streamName, consumerName string, config jetstream.ConsumerConfig) (jetstream.Consumer, error) { - if !checkConnected() { + if !IsConnected() { return nil, fmt.Errorf("NATS 未连接") } @@ -270,379 +258,11 @@ func CreateConsumer(ctx context.Context, streamName, consumerName string, config return consumer, nil } - // 创建新消费者 - consumer, err = js.CreateConsumer(ctx, streamName, config) + // 推荐:不存在则创建,存在则更新配置 + consumer, err = js.CreateOrUpdateConsumer(ctx, streamName, config) if err != nil { return nil, fmt.Errorf("创建消费者失败: %w", err) } return consumer, nil } - -// ============ RPC 服务封装 ============ -// 以下方法提供了完全抽象的 RPC 调用接口 -// 调用方和响应方完全不需要知道底层使用的是 NATS 的发布订阅模式 - -// RegisterRPCService 注册 RPC 服务(单实例) -// serviceName: 服务名称,调用方通过此名称调用服务 -// handler: 服务处理函数,接收请求并返回响应 -func RegisterRPCService(serviceName string, handler RPCHandler) error { - if !checkConnected() { - return fmt.Errorf("NATS 未连接") - } - - rpcServicesMu.Lock() - if rpcServices == nil { - rpcServices = make(map[string]RPCHandler) - } - if rpcSubs == nil { - rpcSubs = make(map[string]*nats.Subscription) - } - - // 如果已存在该服务,先取消之前的订阅 - if oldSub, exists := rpcSubs[serviceName]; exists { - oldSub.Unsubscribe() - } - - rpcServices[serviceName] = handler - rpcServicesMu.Unlock() - - // 订阅服务主题 - subject := fmt.Sprintf("rpc.%s", serviceName) - sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { - ctx := context.Background() - response, err := handler(ctx, msg.Data) - if err != nil { - errMsg := fmt.Sprintf("处理失败: %v", err) - if err = msg.Respond([]byte(errMsg)); err != nil { - g.Log().Errorf(ctx, "RPC 错误响应失败: %v", err) - } - return - } - - if err = msg.Respond(response); err != nil { - g.Log().Errorf(ctx, "RPC 响应失败: %v", err) - } - }) - - if err != nil { - return fmt.Errorf("注册 RPC 服务失败: %w", err) - } - - rpcSubs[serviceName] = sub - metrics.SubscribeCount.Add(1) - g.Log().Infof(context.Background(), "✅ RPC 服务已注册: %s", serviceName) - return nil -} - -// RegisterQueueRPCService 注册 RPC 服务(集群模式) -// 多个服务实例注册同一服务时,请求会自动负载均衡 -// serviceName: 服务名称 -// queueName: 队列组名,同一队列组的实例共享请求 -// handler: 服务处理函数 -func RegisterQueueRPCService(serviceName, queueName string, handler RPCHandler) error { - if !checkConnected() { - return fmt.Errorf("NATS 未连接") - } - - queueRPCMu.Lock() - if queueRPCServices == nil { - queueRPCServices = make(map[string]map[string]RPCHandler) - } - if queueRPCSubs == nil { - queueRPCSubs = make(map[string]map[string]*nats.Subscription) - } - if queueRPCServices[queueName] == nil { - queueRPCServices[queueName] = make(map[string]RPCHandler) - } - if queueRPCSubs[queueName] == nil { - queueRPCSubs[queueName] = make(map[string]*nats.Subscription) - } - - // 如果已存在该服务,先取消之前的订阅 - if oldSub, exists := queueRPCSubs[queueName][serviceName]; exists { - oldSub.Unsubscribe() - } - - queueRPCServices[queueName][serviceName] = handler - queueRPCMu.Unlock() - - // 订阅服务主题(队列模式) - subject := fmt.Sprintf("rpc.%s", serviceName) - sub, err := nc.QueueSubscribe(subject, queueName, func(msg *nats.Msg) { - ctx := context.Background() - response, err := handler(ctx, msg.Data) - if err != nil { - errMsg := fmt.Sprintf("处理失败: %v", err) - if err = msg.Respond([]byte(errMsg)); err != nil { - g.Log().Errorf(ctx, "RPC 错误响应失败: %v", err) - } - return - } - - if err = msg.Respond(response); err != nil { - g.Log().Errorf(ctx, "RPC 响应失败: %v", err) - } - }) - - if err != nil { - return fmt.Errorf("注册队列 RPC 服务失败: %w", err) - } - - queueRPCMu.Lock() - queueRPCSubs[queueName][serviceName] = sub - queueRPCMu.Unlock() - - metrics.SubscribeCount.Add(1) - g.Log().Infof(context.Background(), "✅ 队列 RPC 服务已注册: %s (队列组: %s)", serviceName, queueName) - return nil -} - -// CallRPC 调用 RPC 服务 -// serviceName: 服务名称 -// req: 请求数据 -// timeout: 超时时间 -// 返回: 响应数据和错误 -func CallRPC(ctx context.Context, serviceName string, req []byte, timeout time.Duration) ([]byte, error) { - if !checkConnected() { - return nil, fmt.Errorf("NATS 未连接") - } - - metrics.RequestCount.Add(1) - - // 检查本地是否有注册的单实例服务,如果有则直接调用(优化性能) - rpcServicesMu.RLock() - if localHandler, exists := rpcServices[serviceName]; exists { - rpcServicesMu.RUnlock() - // 本地直接调用,避免网络开销 - response, err := localHandler(ctx, req) - if err != nil { - metrics.RequestError.Add(1) - return nil, fmt.Errorf("本地调用 RPC 服务失败 [%s]: %w", serviceName, err) - } - return response, nil - } - rpcServicesMu.RUnlock() - - // 通过 NATS 网络调用远程服务 - timeoutCtx, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - subject := fmt.Sprintf("rpc.%s", serviceName) - msg, err := nc.RequestWithContext(timeoutCtx, subject, req) - if err != nil { - metrics.RequestError.Add(1) - return nil, fmt.Errorf("调用 RPC 服务失败 [%s]: %w", serviceName, err) - } - - if msg == nil { - metrics.RequestError.Add(1) - return nil, fmt.Errorf("RPC 响应为空 [%s]", serviceName) - } - - return msg.Data, nil -} - -// RegisterServiceOption 注册选项类型 -type RegisterServiceOption func(*registerServiceConfig) - -type registerServiceConfig struct { - queueName string // 队列组名(用于集群模式) - excludeMethods []string -} - -// WithQueueGroup 设置队列组名(集群模式) -func WithQueueGroup(queueName string) RegisterServiceOption { - return func(cfg *registerServiceConfig) { - cfg.queueName = queueName - } -} - -// WithExcludeMethods 排除不需要注册的方法 -func WithExcludeMethods(methods ...string) RegisterServiceOption { - return func(cfg *registerServiceConfig) { - cfg.excludeMethods = append(cfg.excludeMethods, methods...) - } -} - -// registerService 注册单个服务的所有公开方法(内部函数) -func registerService(service interface{}, serviceNamePrefix string, options ...RegisterServiceOption) error { - if !checkConnected() { - return fmt.Errorf("NATS 未连接") - } - - // 应用选项 - cfg := ®isterServiceConfig{} - for _, opt := range options { - opt(cfg) - } - - // 创建排除方法集合 - excludeSet := make(map[string]struct{}) - for _, method := range cfg.excludeMethods { - excludeSet[method] = struct{}{} - } - - // 获取 service 的类型 - serviceType := reflect.TypeOf(service) - - // 遍历所有方法 - registeredCount := 0 - for i := 0; i < serviceType.NumMethod(); i++ { - method := serviceType.Method(i) - - // 只注册导出方法(首字母大写) - if !method.IsExported() { - continue - } - - // 排除指定的方法 - if _, exists := excludeSet[method.Name]; exists { - continue - } - - // 检查方法签名:必须是 func(ctx context.Context, request) (response, error) - if method.Type.NumIn() < 2 { - g.Log().Warningf(context.Background(), "方法 %s 的参数数量不足,跳过注册", method.Name) - continue - } - - // 第一个参数必须是 context.Context - if !method.Type.In(0).Implements(reflect.TypeOf((*context.Context)(nil)).Elem()) { - g.Log().Warningf(context.Background(), "方法 %s 的第一个参数必须是 context.Context,跳过注册", method.Name) - continue - } - - // 返回值必须是 (result, error) 或 error - if method.Type.NumOut() < 1 || method.Type.NumOut() > 2 { - g.Log().Warningf(context.Background(), "方法 %s 的返回值数量不正确,跳过注册", method.Name) - continue - } - - if !method.Type.Out(method.Type.NumOut() - 1).Implements(reflect.TypeOf((*error)(nil)).Elem()) { - g.Log().Warningf(context.Background(), "方法 %s 的最后一个返回值必须是 error,跳过注册", method.Name) - continue - } - - // 生成服务名称:前缀.方法名(保持原始方法名) - serviceName := fmt.Sprintf("%s.%s", serviceNamePrefix, method.Name) - - // 创建 RPC handler - handler := func(ctx context.Context, req []byte) ([]byte, error) { - // 准备方法调用参数 - args := make([]reflect.Value, 2) - args[0] = reflect.ValueOf(ctx) - - // 解析请求参数 - if len(req) > 0 { - // 如果方法有第二个参数,尝试解析 JSON - if method.Type.NumIn() > 1 { - reqValuePtr := reflect.New(method.Type.In(1)) - if err := json.Unmarshal(req, reqValuePtr.Interface()); err != nil { - return nil, fmt.Errorf("解析请求参数失败: %w", err) - } - args[1] = reqValuePtr.Elem() - } - } else if method.Type.NumIn() > 1 { - // 如果方法需要参数但请求为空,创建零值 - args[1] = reflect.Zero(method.Type.In(1)) - } - - // 调用方法 - results := method.Func.Call(args) - - // 处理返回值 - var err error - var result interface{} - - if len(results) == 1 { - // 只有 error - if !results[0].IsNil() { - err = results[0].Interface().(error) - } - } else if len(results) == 2 { - // (result, error) - result = results[0].Interface() - if !results[1].IsNil() { - err = results[1].Interface().(error) - } - } - - if err != nil { - return nil, err - } - - // 序列化返回值 - if result == nil || (reflect.ValueOf(result).Kind() == reflect.Ptr && reflect.ValueOf(result).IsNil()) { - return []byte("{}"), nil - } - - return json.Marshal(result) - } - - // 注册 RPC 服务 - var err error - if cfg.queueName != "" { - err = RegisterQueueRPCService(serviceName, cfg.queueName, handler) - } else { - err = RegisterRPCService(serviceName, handler) - } - - if err != nil { - g.Log().Errorf(context.Background(), "注册服务 %s 失败: %v", serviceName, err) - continue - } - - registeredCount++ - g.Log().Infof(context.Background(), "✅ 已自动注册 RPC 服务: %s -> %s", serviceName, method.Name) - } - - if registeredCount == 0 { - g.Log().Warningf(context.Background(), "未注册任何方法,请检查 %v 的方法签名", serviceNamePrefix) - return fmt.Errorf("未找到可注册的方法") - } - - g.Log().Infof(context.Background(), "✅ Service %v 共注册了 %d 个 RPC 方法", serviceNamePrefix, registeredCount) - return nil -} - -// AutoRegisterServices 自动注册多个服务的所有公开方法 -// serviceInstances: map[包名]service实例,如 map[string]interface{}{"user": userService, "order": orderService} -// options: 注册选项(可选) -// 示例: -// -// AutoRegisterServices(map[string]interface{}{ -// "user": userService, -// "order": orderService, -// }) -// 或 -// AutoRegisterServices(map[string]interface{}{ -// "order": orderService, -// }, WithQueueGroup("order-group")) -func AutoRegisterServices(serviceInstances map[string]interface{}, options ...RegisterServiceOption) error { - if len(serviceInstances) == 0 { - return fmt.Errorf("service 实例列表不能为空") - } - - totalRegistered := 0 - - // 遍历每个 service 实例 - for pkgName, serviceInstance := range serviceInstances { - // 注册服务 - err := registerService(serviceInstance, pkgName, options...) - if err != nil { - g.Log().Errorf(context.Background(), "注册 %s 服务失败: %v", pkgName, err) - continue - } - - totalRegistered++ - g.Log().Infof(context.Background(), "✅ %s 服务已自动注册", pkgName) - } - - if totalRegistered == 0 { - return fmt.Errorf("未能注册任何服务") - } - - g.Log().Infof(context.Background(), "✅ 共自动注册了 %d 个服务", totalRegistered) - return nil -} diff --git a/nats/nats_consumer.go b/nats/nats_consumer.go new file mode 100644 index 0000000..126000b --- /dev/null +++ b/nats/nats_consumer.go @@ -0,0 +1,294 @@ +package nats + +import ( + "context" + "fmt" + "github.com/gogf/gf/v2/frame/g" + "github.com/nats-io/nats.go/jetstream" +) + +// AckPolicy 确认策略 +type AckPolicy string + +const ( + AckPolicyExplicit AckPolicy = "explicit" // 显式确认(默认) + AckPolicyAll AckPolicy = "all" // 确认所有消息 + AckPolicyNone AckPolicy = "none" // 不需要确认 +) + +// DeliverPolicy 投递策略 +type DeliverPolicy string + +const ( + DeliverPolicyAll DeliverPolicy = "all" // 投递所有消息(包括已投递的) + DeliverPolicyLast DeliverPolicy = "last" // 从最后一条消息开始 + DeliverPolicyNew DeliverPolicy = "new" // 仅投递新消息(默认) + DeliverPolicyLastPerSubj DeliverPolicy = "lastpersubj" // 每个主题的最后一条 + DeliverPolicyByStartSeq DeliverPolicy = "by_start_sequence" // 按起始序列号 +) + +// ReplayPolicy 重放策略 +type ReplayPolicy string + +const ( + ReplayPolicyInstant ReplayPolicy = "instant" // 立即重放 + ReplayPolicyOriginal ReplayPolicy = "original" // 按原始顺序重放 +) + +// ConsumerConfig 消费者配置 +type ConsumerConfig struct { + DurableName string // 持久化名称(空表示临时消费者) + Description string // 描述信息 + AckPolicy AckPolicy // 确认策略 + AckWait int // 确认等待时间(秒) + MaxDeliver int // 最大投递次数 + FilterSubject string // 过滤主题(流内多主题时使用) + DeliverPolicy DeliverPolicy // 投递策略 + ReplayPolicy ReplayPolicy // 重放策略 + MaxWaiting int // 最大等待消息数 + MaxAckPending int // 最大待确认消息数 + OptStartTime int64 // 起始时间戳 + OptStartSeq uint64 // 起始序列号 + HeadersOnly bool // 仅消费消息头 + Backoff []int // 退避策略(秒数数组) + RateLimit uint64 // 消息速率限制(消息/秒) + Replica int // 副本数 + FlowControl bool // 启用流控 + Metadata map[string]string // 元数据 +} + +// parseAckPolicy 解析确认策略 +func parseAckPolicy(policy AckPolicy) jetstream.AckPolicy { + switch policy { + case AckPolicyAll: + return jetstream.AckAllPolicy + case AckPolicyNone: + return jetstream.AckNonePolicy + default: + return jetstream.AckExplicitPolicy + } +} + +// parseDeliverPolicy 解析投递策略 +func parseDeliverPolicy(policy DeliverPolicy) jetstream.DeliverPolicy { + switch policy { + case DeliverPolicyAll: + return jetstream.DeliverAllPolicy + case DeliverPolicyLast: + return jetstream.DeliverLastPolicy + case DeliverPolicyLastPerSubj: + return jetstream.DeliverLastPerSubjectPolicy + case DeliverPolicyByStartSeq: + return jetstream.DeliverByStartSequencePolicy + default: + return jetstream.DeliverNewPolicy + } +} + +// parseReplayPolicy 解析重放策略 +func parseReplayPolicy(policy ReplayPolicy) jetstream.ReplayPolicy { + switch policy { + case ReplayPolicyOriginal: + return jetstream.ReplayOriginalPolicy + default: + return jetstream.ReplayInstantPolicy + } +} + +// CreateTaskConsumer 创建任务消费者 +// 核心设计思路: +// 1. 显式确认:确保消息被正确处理后才确认 +// 2. 重试机制:通过 MaxDeliver 控制最大重试次数 +// 3. 持久化:DurableName 确保消费者状态持久化 +// 4. 流控:防止消费者过载 +func CreateTaskConsumer(ctx context.Context, streamName string, config ConsumerConfig) (jetstream.Consumer, error) { + if !IsConnected() { + return nil, fmt.Errorf("NATS 未连接") + } + + if streamName == "" { + return nil, fmt.Errorf("流名称不能为空") + } + + // 设置默认值 + if config.AckPolicy == "" { + config.AckPolicy = AckPolicyExplicit // 默认显式确认 + } + if config.AckWait == 0 { + config.AckWait = 30 // 默认30秒确认超时 + } + if config.MaxDeliver == 0 { + config.MaxDeliver = 3 // 默认最多投递3次 + } + if config.DeliverPolicy == "" { + config.DeliverPolicy = DeliverPolicyNew // 默认仅消费新消息 + } + if config.ReplayPolicy == "" { + config.ReplayPolicy = ReplayPolicyInstant // 默认立即重放 + } + if config.MaxAckPending == 0 { + config.MaxAckPending = 1000 // 默认最多1000条待确认消息 + } + + // 构建消费者配置 + jsConfig := jetstream.ConsumerConfig{ + Name: config.DurableName, + Description: config.Description, + AckPolicy: parseAckPolicy(config.AckPolicy), + AckWait: 0, + MaxDeliver: config.MaxDeliver, + FilterSubjects: []string{config.FilterSubject}, + DeliverPolicy: parseDeliverPolicy(config.DeliverPolicy), + ReplayPolicy: parseReplayPolicy(config.ReplayPolicy), + MaxWaiting: config.MaxWaiting, + MaxAckPending: config.MaxAckPending, + HeadersOnly: config.HeadersOnly, + RateLimit: config.RateLimit, + Replicas: config.Replica, + Metadata: config.Metadata, + } + + // 配置流控和心跳 + if config.FlowControl { + jsConfig.FlowControl = true + } + // 配置起始位置 + if config.OptStartSeq > 0 { + jsConfig.OptStartSeq = config.OptStartSeq + } + + // 创建新消费者 + consumer, err := js.CreateOrUpdateConsumer(ctx, streamName, jsConfig) + if err != nil { + return nil, fmt.Errorf("创建消费者失败: %w", err) + } + + // 记录配置信息 + configInfo := fmt.Sprintf("确认策略=%s, 最大投递=%d, 投递策略=%s", config.AckPolicy, config.MaxDeliver, config.DeliverPolicy) + if config.FilterSubject != "" { + configInfo += fmt.Sprintf(", 过滤主题=%s", config.FilterSubject) + } + g.Log().Infof(ctx, "✅ 任务消费者创建成功: %s/%s (%s)", streamName, config.DurableName, configInfo) + + return consumer, nil +} + +// CreateConsumerSimple 简化版创建消费者(适用于大多数场景) +// 只需提供流名称和消费者名称,其他使用默认配置 +func CreateConsumerSimple(ctx context.Context, streamName, durableName string) (err error) { + _, err = CreateTaskConsumer(ctx, streamName, ConsumerConfig{ + DurableName: durableName, + }) + return +} + +// CreateConsumerWithFilter 创建带主题过滤的消费者 +//func CreateConsumerWithFilter(ctx context.Context, streamName, durableName, filterSubject string) (jetstream.Consumer, error) { +// return CreateTaskConsumer(ctx, streamName, ConsumerConfig{ +// DurableName: durableName, +// FilterSubject: filterSubject, +// }) +//} + +// CreateConsumerEphemeral 创建临时消费者 +// 临时消费者没有持久化名称,连接断开后自动删除 +//func CreateConsumerEphemeral(ctx context.Context, streamName string) (jetstream.Consumer, error) { +// if !IsConnected() { +// return nil, fmt.Errorf("NATS 未连接") +// } +// +// jsConfig := jetstream.ConsumerConfig{ +// AckPolicy: jetstream.AckNonePolicy, +// AckWait: 0, +// MaxDeliver: 3, +// DeliverPolicy: jetstream.DeliverNewPolicy, +// ReplayPolicy: jetstream.ReplayInstantPolicy, +// MaxAckPending: 1000, +// } +// +// consumer, err := js.CreateConsumer(ctx, streamName, jsConfig) +// if err != nil { +// return nil, fmt.Errorf("创建临时消费者失败: %w", err) +// } +// +// g.Log().Infof(ctx, "✅ 临时消费者创建成功: %s", streamName) +// return consumer, nil +//} + +// CreateConsumerPushMode 创建推送模式消费者 +// 推送模式下,NATS 服务器主动将消息推送给消费者 +func CreateConsumerPushMode(ctx context.Context, streamName, durableName, subject string, msgCount int) (err error) { + _, err = CreateTaskConsumer(ctx, streamName, ConsumerConfig{ + DurableName: durableName, + FilterSubject: subject, + MaxAckPending: msgCount, + }) + return +} + +// CreateConsumerPullMode 创建拉取模式消费者 +// 拉取模式下,消费者主动从服务器拉取消息 +//func CreateConsumerPullMode(ctx context.Context, streamName, durableName string) (jetstream.Consumer, error) { +// return CreateTaskConsumer(ctx, streamName, ConsumerConfig{ +// DurableName: durableName, +// DeliverPolicy: DeliverPolicyAll, +// MaxAckPending: 500, // 拉取模式下待确认消息数可以设置小一些 +// }) +//} + +// ConsumeMessages 消费消息(推送模式) +func ConsumeMessages(ctx context.Context, streamName, consumerName string, handler jetstream.MessageHandler) error { + if !IsConnected() { + return fmt.Errorf("NATS 未连接") + } + // 获取消费者 + consumer, err := js.Consumer(ctx, streamName, consumerName) + if err != nil { + return fmt.Errorf("获取消费者失败: %w", err) + } + + // 业务处理 + //if err := handler(ctx, streamMsg.Values); err != nil { + // glog.Infof(ctx, "业务处理失败-> err:%v\n", err) + // continue + //} + //// 确认消息 + //if msg.AutoAck { + // err := ackMessage(ctx, msg.StreamKey, msg.GroupName, streamMsg.ID) + // if err != nil { + // glog.Infof(ctx, "消费者 '%s' 确认消息 ID %s 失败: %v\n", msg.ConsumerName, streamMsg.ID, err) + // } + //} + //// 创建消息处理函数 + //handler = func(msg jetstream.Msg) { + // // 解析消息 + // var task TaskMessage + // if err := json.Unmarshal(msg.Data(), &task); err != nil { + // g.Log().Errorf(ctx, "解析消息失败: %v", err) + // msg.Nak() // 拒绝消息,触发重试 + // return + // } + // + // // 处理业务逻辑 + // g.Log().Infof(ctx, "处理任务: %s", task.TaskID) + // + // // 处理成功,确认消息 + // msg.Ack() + //} + + // 开始消费 + _, err = consumer.Consume(handler) + if err != nil { + return fmt.Errorf("开始消费失败: %w", err) + } + + g.Log().Infof(ctx, "✅ 开始消费消息: %s/%s", streamName, consumerName) + return nil +} + +// 定义消息结构 +type TaskMessage struct { + TaskID string `json:"task_id"` + TaskType string `json:"task_type"` + Data string `json:"data"` +} diff --git a/nats/nats_publish.go b/nats/nats_publish.go new file mode 100644 index 0000000..32e3b20 --- /dev/null +++ b/nats/nats_publish.go @@ -0,0 +1,28 @@ +package nats + +import ( + "context" + "encoding/json" + "fmt" +) + +// publish 发布消息到指定主题 +func publish(ctx context.Context, subject string, data any) (err error) { + if !IsConnected() { + return fmt.Errorf("NATS 未连接") + } + // 序列化数据 + dataBytes, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("序列化数据失败: %w", err) + } + // 发布消息 + metrics.PublishCount.Add(1) + _, err = js.Publish(ctx, subject, dataBytes) + if err != nil { + metrics.PublishError.Add(1) + return fmt.Errorf("发布消息失败: %w", err) + } + + return +} diff --git a/nats/nats_rpc.go b/nats/nats_rpc.go new file mode 100644 index 0000000..8cbb8ba --- /dev/null +++ b/nats/nats_rpc.go @@ -0,0 +1,673 @@ +package nats + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/gogf/gf/v2/frame/g" + "github.com/nats-io/nats.go" + "reflect" + "sync" +) + +// ============ RPC 服务封装 ============ +// 以下方法提供了完全抽象的 RPC 调用接口 +// 调用方和响应方完全不需要知道底层使用的是 NATS 的发布订阅模式 + +// RPC 服务注册表 +var ( + rpcServices map[string]rpcHandler + rpcSubs map[string]*nats.Subscription // 服务名 -> 订阅 + rpcServicesMu sync.RWMutex + queueRPCServices map[string]map[string]rpcHandler // queueName -> subject -> handler + queueRPCSubs map[string]map[string]*nats.Subscription // queueName -> serviceName -> 订阅 + queueRPCMu sync.RWMutex + + // ============ TraceID 主动取消支持 ============ + // 全局映射表:TraceID -> CancelFunc,并发安全 + traceCancelMap map[string]context.CancelFunc + traceCancelMu sync.RWMutex + // 取消主题前缀 + cancelSubjectPrefix = "ctx.cancel.otel." +) + +// rpcHandler RPC 处理函数类型 +// 实现方只需要关注请求参数和返回值,无需了解底层 NATS 实现 +// 返回值可以是任意类型,会被自动序列化为 JSON +type rpcHandler func(ctx context.Context, req []byte) (any, error) + +// RegisterRPCService 注册 RPC 服务(单实例) +// serviceName: 服务名称,调用方通过此名称调用服务 +// handler: 服务处理函数,接收请求并返回响应 +func registerRPCService(serviceName string, handler rpcHandler) (err error) { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + rpcServicesMu.Lock() + if rpcServices == nil { + rpcServices = make(map[string]rpcHandler) + } + if rpcSubs == nil { + rpcSubs = make(map[string]*nats.Subscription) + } + + // 如果已存在该服务,先取消之前的订阅 + if oldSub, exists := rpcSubs[serviceName]; exists { + oldSub.Unsubscribe() + } + + rpcServices[serviceName] = handler + rpcServicesMu.Unlock() + + // 订阅服务主题 + subject := fmt.Sprintf("rpc.%s", serviceName) + sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { + // 执行处理函数 + executeHandler(handler, msg) + }) + + if err != nil { + return fmt.Errorf("注册 RPC 服务失败: %w", err) + } + + rpcSubs[serviceName] = sub + metrics.SubscribeCount.Add(1) + g.Log().Infof(context.Background(), "✅ RPC 服务已注册: %s", serviceName) + return nil +} + +// RegisterQueueRPCService 注册 RPC 服务(集群模式) +// 多个服务实例注册同一服务时,请求会自动负载均衡 +// serviceName: 服务名称 +// queueName: 队列组名,同一队列组的实例共享请求 +// handler: 服务处理函数 +func registerQueueRPCService(serviceName, queueName string, handler rpcHandler) (err error) { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + queueRPCMu.Lock() + if queueRPCServices == nil { + queueRPCServices = make(map[string]map[string]rpcHandler) + } + if queueRPCSubs == nil { + queueRPCSubs = make(map[string]map[string]*nats.Subscription) + } + if queueRPCServices[queueName] == nil { + queueRPCServices[queueName] = make(map[string]rpcHandler) + } + if queueRPCSubs[queueName] == nil { + queueRPCSubs[queueName] = make(map[string]*nats.Subscription) + } + + // 如果已存在该服务,先取消之前的订阅 + if oldSub, exists := queueRPCSubs[queueName][serviceName]; exists { + oldSub.Unsubscribe() + } + + queueRPCServices[queueName][serviceName] = handler + queueRPCMu.Unlock() + + // 订阅服务主题(队列模式) + subject := fmt.Sprintf("rpc.%s", serviceName) + sub, err := nc.QueueSubscribe(subject, queueName, func(msg *nats.Msg) { + // 执行处理函数 + executeHandler(handler, msg) + }) + + if err != nil { + return fmt.Errorf("注册队列 RPC 服务失败: %w", err) + } + + queueRPCMu.Lock() + queueRPCSubs[queueName][serviceName] = sub + queueRPCMu.Unlock() + + metrics.SubscribeCount.Add(1) + g.Log().Infof(context.Background(), "✅ 队列 RPC 服务已注册: %s (队列组: %s)", serviceName, queueName) + return nil +} + +// executeHandler 执行 RPC 处理函数 +func executeHandler(handler rpcHandler, msg *nats.Msg) { + // 响应 + var respData []byte + // 从消息头重建上下文 + ctx := headersToContext(context.Background(), msg.Header) + // 提取 TraceID,创建可取消的 context + ctx = createCancelContext(ctx, msg.Header.Get(TraceIDKey)) + // 检查 context 是否已取消(在调用 handler 之前) + select { + case <-ctx.Done(): + // context 已取消,返回取消错误 + g.Log().Infof(ctx, "RPC 请求已取消,traceID: %s", msg.Header.Get(TraceIDKey)) + // 仍然需要发送响应以避免客户端超时 + respData = []byte(`{"_err":"请求已取消"}`) + // 清理取消映射表 + cleanupTraceCancel(msg.Header.Get(TraceIDKey)) + return + default: + } + + // 执行业务处理 + response, err := handler(ctx, msg.Data) + + if err != nil { + // 错误时返回 {"_err": "错误信息"} + if respData, err = json.Marshal(map[string]any{"_err": err.Error()}); err != nil { + g.Log().Errorf(ctx, "RPC 错误响应序列化失败: %v", err) + respData = []byte(`{"_err":"错误响应序列化失败"}`) + } + } else if response == nil { + // 空响应时返回空对象(或 {"_err": ""}) + respData = []byte(`{}`) + } else { + // 成功时返回业务数据 + if respData, err = json.Marshal(response); err != nil { + g.Log().Errorf(ctx, "RPC 响应序列化失败: %v", err) + respData = []byte(`{"_err":"响应序列化失败"}`) + } + } + // 发送响应(必须执行) 如果客户端用 nc.Request(...) 发送消息 → 双向模式,服务端必须 msg.Respond + if err = msg.Respond(respData); err != nil { + g.Log().Errorf(ctx, "RPC 响应失败: %v", err) + } + // 请求结束,清理取消映射表 + cleanupTraceCancel(msg.Header.Get(TraceIDKey)) +} + +// createCancelContext 创建可取消的 context 并注册到取消映射表 +// 返回可取消的 context(如果 traceID 为空则返回原 context) +func createCancelContext(ctx context.Context, traceID string) context.Context { + if g.IsEmpty(traceID) { + return ctx + } + // 创建带取消功能的 context + taskCtx, cancel := context.WithCancel(ctx) + // 注册到取消映射表 + traceCancelMu.Lock() + if traceCancelMap == nil { + traceCancelMap = make(map[string]context.CancelFunc) + } + // 如果同一 TraceID 已有 CancelFunc,先调用它 + if oldCancel, exists := traceCancelMap[traceID]; exists { + oldCancel() + } + traceCancelMap[traceID] = cancel + traceCancelMu.Unlock() + + return taskCtx +} + +// ============ TraceID 主动取消功能 ============ +// 以下函数实现了基于 OpenTelemetry TraceID 的跨进程任务取消机制 + +// SetupCancelListener 设置取消监听器 +// 订阅取消主题,监听取消指令 +// 使用示例: +// +// sub, err := nats.SetupCancelListener(ctx) +func setupCancelListener(ctx context.Context) (*nats.Subscription, error) { + if !checkConnected() { + return nil, fmt.Errorf("NATS 未连接") + } + + if traceCancelMap == nil { + traceCancelMap = make(map[string]context.CancelFunc) + } + + // 修复问题3:订阅取消主题,格式: ctx.cancel.otel.* + // 使用 * 通配符而不是 >,因为 TraceID 是最后一部分 + cancelSubject := cancelSubjectPrefix + "*" + sub, err := nc.Subscribe(cancelSubject, func(msg *nats.Msg) { + // 从主题中解析 TraceID (去除前缀) + prefixLen := len(cancelSubjectPrefix) + if len(msg.Subject) <= prefixLen { + g.Log().Warningf(ctx, "取消消息主题格式错误: %s", msg.Subject) + return + } + traceID := msg.Subject[prefixLen:] + + if traceID == "" { + g.Log().Warning(ctx, "取消消息主题缺少 TraceID") + return + } + + // 从映射表获取 CancelFunc 并执行取消 + traceCancelMu.RLock() + cancel, ok := traceCancelMap[traceID] + traceCancelMu.RUnlock() + + if ok { + cancel() + g.Log().Infof(ctx, "📢 取消信号已发送,traceID: %s", traceID) + } else { + g.Log().Infof(ctx, "⚠️ 未找到对应的可取消任务,traceID: %s", traceID) + } + }) + + if err != nil { + return nil, fmt.Errorf("设置取消监听器失败: %w", err) + } + + metrics.SubscribeCount.Add(1) + g.Log().Infof(ctx, "✅ 取消监听器已设置: %s", cancelSubject) + return sub, nil +} + +// publishCancel 发布取消指令 +// 向指定 TraceID 发送取消信号 +// 使用示例: +// +// err := nats.publishCancel(ctx, traceID) +func publishCancel(ctx context.Context, traceID string) error { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + if traceID == "" { + return fmt.Errorf("TraceID 不能为空") + } + + cancelSubject := cancelSubjectPrefix + traceID + err := nc.Publish(cancelSubject, nil) + if err != nil { + return fmt.Errorf("发布取消信号失败: %w", err) + } + + g.Log().Infof(ctx, "📤 已发送取消信号,traceID: %s,主题: %s", traceID, cancelSubject) + return nil +} + +// cleanupTraceCancel 清理取消映射表中的条目 +// 任务取消/正常结束后必须调用此函数,避免内存泄漏 +// 使用示例: +// +// defer nats.cleanupTraceCancel(traceID) +func cleanupTraceCancel(traceID string) { + if traceID == "" { + return + } + + traceCancelMu.Lock() + defer traceCancelMu.Unlock() + + if _, ok := traceCancelMap[traceID]; ok { + delete(traceCancelMap, traceID) + g.Log().Infof(context.Background(), "✅ 已清理取消映射表,traceID: %s", traceID) + } +} + +// CallRPC 调用 RPC 服务 +// serviceName: 服务名称 +// req: 请求数据 +// 返回: 响应数据(任意类型)和错误 +func CallRPC(ctx context.Context, serviceName string, req any, resp any) (err error) { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + metrics.RequestCount.Add(1) + + // 验证 resp 必须是指针类型 + respValue := reflect.ValueOf(resp) + if respValue.Kind() != reflect.Ptr { + return fmt.Errorf("resp 参数必须是指针类型(当前类型: %T)", resp) + } + + // 构建请求体 + var reqBody []byte + if !g.IsEmpty(req) { + reqValue := reflect.ValueOf(req) + if !(reqValue.Kind() == reflect.Ptr && reqValue.IsNil()) && !reqValue.IsZero() { + reqData, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("序列化请求参数失败: %w", err) + } + reqBody = reqData + } + } + + // 检查本地是否有注册的单实例服务,如果有则直接调用(优化性能) + rpcServicesMu.RLock() + if localHandler, exists := rpcServices[serviceName]; exists { + rpcServicesMu.RUnlock() + + // 修复问题1:本地调用也需要处理取消机制 + var traceID string + if traceID, err = getTraceID(ctx); err != nil { + return err + } + // 提取 TraceID,创建可取消的 context + cancelCtx := createCancelContext(ctx, traceID) + // 执行本地调用 + var response interface{} + if response, err = localHandler(cancelCtx, reqBody); err != nil { + metrics.RequestError.Add(1) + return fmt.Errorf("本地调用 RPC 服务失败 [%s]: %w", serviceName, err) + } + + // 请求结束,清理取消映射表 + cleanupTraceCancel(traceID) + + // 检查是否为错误消息:尝试解析为 map,看是否包含 "_err" 字段 + var respMap map[string]any + if json.Unmarshal(response.([]byte), &respMap) == nil { + if errMsg, ok := respMap["_err"]; ok { + metrics.RequestError.Add(1) + return fmt.Errorf("%v", errMsg) + } + } + // 正常数据直接返回 + // responseMsg.Data 已经是 []byte 类型(来自 msg.Data),直接反序列化 + if err = json.Unmarshal(response.([]byte), resp); err != nil { + return fmt.Errorf("解析响应失败: %w (响应内容: %s)", err, response) + } + + return + } + rpcServicesMu.RUnlock() + + subject := fmt.Sprintf("rpc.%s", serviceName) + + // 创建消息并将上下文元数据写入消息头 + msg := nats.NewMsg(subject) + msg.Data = reqBody + headers, err := contextToHeaders(ctx) + if err != nil { + return fmt.Errorf("上下文转换失败: %w", err) + } + msg.Header = headers + + // 修复问题5:优化 go 协程避免资源泄漏 + // 使用 done channel 来确保 goroutine 能正确退出 + done := make(chan struct{}) + var closeDoneOnce sync.Once + closeDone := func() { + closeDoneOnce.Do(func() { + close(done) + }) + } + + if msg.Header.Get(TraceIDKey) != "" { + go func() { + defer closeDone() + select { + case <-ctx.Done(): + // context 被取消时,发送取消信号给服务端 + if errors.Is(ctx.Err(), context.Canceled) { + if err := publishCancel(context.Background(), msg.Header.Get(TraceIDKey)); err != nil { + g.Log().Errorf(ctx, "发送 RPC 取消信号失败: %v", err) + } else { + g.Log().Infof(ctx, "RPC 调用已取消,traceID: %s", msg.Header.Get(TraceIDKey)) + } + } + case <-done: + // 请求已完成,无需发送取消信号 + return + } + }() + } + + // 发送请求 + responseMsg, err := nc.RequestMsgWithContext(ctx, msg) + + // 关闭 done channel,通知 goroutine 退出 + closeDone() + + if err != nil { + metrics.RequestError.Add(1) + return fmt.Errorf("调用 RPC 服务失败 [%s]: %w", serviceName, err) + } + + if responseMsg == nil { + metrics.RequestError.Add(1) + return fmt.Errorf("RPC 响应为空 [%s]", serviceName) + } + + // 解析响应 + if len(responseMsg.Data) > 0 { + // 检查是否为错误消息:尝试解析为 map,看是否包含 "_err" 字段 + var respMap map[string]any + if json.Unmarshal(responseMsg.Data, &respMap) == nil { + if errMsg, ok := respMap["_err"]; ok { + metrics.RequestError.Add(1) + return fmt.Errorf("%v", errMsg) + } + } + // 正常数据直接返回 + // responseMsg.Data 已经是 []byte 类型(来自 msg.Data),直接反序列化 + if err = json.Unmarshal(responseMsg.Data, resp); err != nil { + return fmt.Errorf("解析响应失败: %w (响应内容: %s)", err, responseMsg.Data) + } + } + + return +} + +// RegisterServiceOption 注册选项类型 +type RegisterServiceOption func(*registerServiceConfig) + +type registerServiceConfig struct { + queueName string // 队列组名(用于集群模式) + excludeMethods []string +} + +// WithQueueGroup 设置队列组名(集群模式) +func WithQueueGroup(queueName string) RegisterServiceOption { + return func(cfg *registerServiceConfig) { + cfg.queueName = queueName + } +} + +// WithExcludeMethods 排除不需要注册的方法 +func WithExcludeMethods(methods ...string) RegisterServiceOption { + return func(cfg *registerServiceConfig) { + cfg.excludeMethods = append(cfg.excludeMethods, methods...) + } +} + +// AutoRegisterServices 自动注册多个服务的所有公开方法 +// serviceInstances: map[包名]service实例,如 map[string]interface{}{"user": userService, "order": orderService} +// options: 注册选项(可选) +// 示例: +// +// AutoRegisterServices(map[string]interface{}{ +// "user": userService, +// "order": orderService, +// }) +// 或 +// AutoRegisterServices(map[string]interface{}{ +// "order": orderService, +// }, WithQueueGroup("order-group")) +func AutoRegisterServices(ctx context.Context, serviceInstances map[string]interface{}, options ...RegisterServiceOption) error { + // 先注册 RPC 服务(如果 NATS 不可用则记录警告但不阻塞启动) + if !checkConnected() { + return fmt.Errorf("NATS 未连接,RPC 服务未注册") + } + + if len(serviceInstances) == 0 { + return fmt.Errorf("service 实例列表不能为空") + } + + totalRegistered := 0 + // 遍历每个 service 实例 + for pkgName, serviceInstance := range serviceInstances { + // 注册服务 + err := registerService(serviceInstance, pkgName, options...) + if err != nil { + g.Log().Errorf(ctx, "注册 %s 服务失败: %v", pkgName, err) + continue + } + totalRegistered++ + g.Log().Infof(ctx, "✅ %s 服务已自动注册", pkgName) + } + + if totalRegistered == 0 { + return fmt.Errorf("未能注册任何服务") + } + // 设置取消监听器(监听基于 TraceID 的取消请求) + //if _, err := setupCancelListener(ctx); err != nil { + // g.Log().Errorf(ctx, "设置取消监听器失败: %v", err) + //} else { + // g.Log().Infof(ctx, "✅ 取消监听器已自动设置") + //} + //g.Log().Infof(ctx, "✅ 共自动注册了 %d 个服务", totalRegistered) + + return nil +} + +// registerService 注册单个服务的所有公开方法(内部函数) +func registerService(service interface{}, serviceNamePrefix string, options ...RegisterServiceOption) (err error) { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + // 应用选项 + cfg := ®isterServiceConfig{} + for _, opt := range options { + opt(cfg) + } + + // 创建排除方法集合 + excludeSet := make(map[string]struct{}) + for _, method := range cfg.excludeMethods { + excludeSet[method] = struct{}{} + } + + // 获取 service 的类型 + serviceType := reflect.TypeOf(service) + + // 遍历所有方法 + registeredCount := 0 + for i := 0; i < serviceType.NumMethod(); i++ { + method := serviceType.Method(i) + + // 只注册导出方法(首字母大写) + if !method.IsExported() { + continue + } + + // 排除指定的方法 + if _, exists := excludeSet[method.Name]; exists { + continue + } + + // 检查方法签名:必须是 func(ctx context.Context, request) (response, error) + // 注意:method.Type.NumIn() 包含接收者,所以实际参数数量需要减去 1 + // 要求:接收者 + context.Context + request,总共3个参数 + if method.Type.NumIn() != 3 { + g.Log().Warningf(context.Background(), "方法 %s 必须有2个参数(context.Context 和请求参数),跳过注册", method.Name) + continue + } + + // 第一个参数(接收者之后的第一个参数)必须是 context.Context + // method.Type.In(0) 是接收者,method.Type.In(1) 才是第一个参数 + if !method.Type.In(1).Implements(reflect.TypeOf((*context.Context)(nil)).Elem()) { + g.Log().Warningf(context.Background(), "方法 %s 的第一个参数必须是 context.Context,跳过注册", method.Name) + continue + } + + // 第二个参数必须是结构体指针或数组 + reqType := method.Type.In(2) + if reqType.Kind() != reflect.Ptr && reqType.Kind() != reflect.Slice && reqType.Kind() != reflect.Array { + g.Log().Warningf(context.Background(), "方法 %s 的第二个参数必须是结构体指针或数组,跳过注册", method.Name) + continue + } + + // 返回值必须是 (result, error),即2个返回值 + if method.Type.NumOut() != 2 { + g.Log().Warningf(context.Background(), "方法 %s 必须有2个返回值(result 和 error),跳过注册", method.Name) + continue + } + + // 最后一个返回值必须是 error + if !method.Type.Out(1).Implements(reflect.TypeOf((*error)(nil)).Elem()) { + g.Log().Warningf(context.Background(), "方法 %s 的最后一个返回值必须是 error,跳过注册", method.Name) + continue + } + + // 生成服务名称:前缀.方法名(保持原始方法名) + serviceName := fmt.Sprintf("%s.%s", serviceNamePrefix, method.Name) + + // 创建 RPC handler + handler := func(ctx context.Context, req []byte) (any, error) { + // 准备方法调用参数 + // args[0] 是接收者, args[1] 是 ctx, args[2] 是请求参数 + args := make([]reflect.Value, 3) + args[0] = reflect.ValueOf(service) // 接收者 + args[1] = reflect.ValueOf(ctx) // context.Context + + // 解析请求参数 + if len(req) > 0 { + reqValuePtr := reflect.New(reqType) + + // 解析 JSON + if err := json.Unmarshal(req, reqValuePtr.Interface()); err != nil { + // 根据参数类型提供更友好的错误提示 + var typeHint string + if reqType.Kind() == reflect.Ptr { + typeHint = fmt.Sprintf("(期望类型: %s)", reqType.Elem().Name()) + } else { // reflect.Slice 或 reflect.Array + typeHint = fmt.Sprintf("(期望类型: %s,请确保客户端传递的是JSON数组格式)", reqType.String()) + } + return nil, fmt.Errorf("解析请求参数失败%s: %w", typeHint, err) + } + args[2] = reqValuePtr.Elem() + } else { + // 请求为空,创建零值 + args[2] = reflect.Zero(method.Type.In(2)) + } + + // 调用方法 + results := method.Func.Call(args) + + // 处理返回值 + var result any + + if len(results) == 1 { + // 只有 error + if !results[0].IsNil() { + err = results[0].Interface().(error) + } + } else if len(results) == 2 { + // (result, error) + result = results[0].Interface() + if !results[1].IsNil() { + err = results[1].Interface().(error) + } + } + if err != nil { + return nil, err + } + + return result, nil + } + + // 注册 RPC 服务 + var err error + if cfg.queueName != "" { + err = registerQueueRPCService(serviceName, cfg.queueName, handler) + } else { + err = registerRPCService(serviceName, handler) + } + + if err != nil { + g.Log().Errorf(context.Background(), "注册服务 %s 失败: %v", serviceName, err) + continue + } + + registeredCount++ + g.Log().Infof(context.Background(), "✅ 已自动注册 RPC 服务: %s -> %s", serviceName, method.Name) + } + + if registeredCount == 0 { + g.Log().Warningf(context.Background(), "未注册任何方法,请检查 %v 的方法签名", serviceNamePrefix) + return fmt.Errorf("未找到可注册的方法") + } + + g.Log().Infof(context.Background(), "✅ Service %v 共注册了 %d 个 RPC 方法", serviceNamePrefix, registeredCount) + return nil +} diff --git a/nats/nats_task.go b/nats/nats_task.go new file mode 100644 index 0000000..f6841ad --- /dev/null +++ b/nats/nats_task.go @@ -0,0 +1,212 @@ +package nats + +import ( + "context" + "fmt" + "github.com/gogf/gf/v2/frame/g" + "github.com/nats-io/nats.go/jetstream" + "time" +) + +// TaskStreamConfig 任务流配置 +type TaskStreamConfig struct { + StreamName string // 流名称 + Subjects []string // 主题数组(支持任务优先级,如 ["tasks.high","tasks.normal", "tasks.low"]) + PublishSubject string // 发布使用的主题(仅用于记录,不影响流配置) + Storage StorageType // 存储类型 + Retention RetentionType // 保留策略 + MaxAge time.Duration // 最大保留时间 + Duplicates time.Duration // 消息去重窗口时间 + Replicas int // 副本数 + MaxMsgSize int32 // 单条消息最大大小(字节) + MaxBytes int64 // 流最大存储大小(字节) + MaxMsgs int64 // 流中最大消息数 + MaxMsgsPerSubject int64 // 每个主题最大消息数 + MaxConsumers int // 最大消费者数量 + DenyPurge bool // 是否禁止清理流 + AllowRollup bool // 是否允许汇总消息 + DenyDelete bool // 是否禁止删除 + DiscardPerSubject bool // 是否按主题限制(工作队列模式) + Republish *RePublishConfig // 死信队列重新发布配置 +} + +// RePublishConfig 重新发布配置(用于死信队列) +type RePublishConfig struct { + Source string // 源主题 + Destination string // 目标主题 + HeadersOnly bool // 仅复制消息头 +} + +// StorageType 存储类型 +type StorageType string + +const ( + StorageFile StorageType = "file" // 文件存储(持久化) + StorageMemory StorageType = "memory" // 内存存储 +) + +// RetentionType 保留策略 +type RetentionType string + +const ( + RetentionLimit RetentionType = "limit" // 消息数量限制 + RetentionPolicy RetentionType = "interest" // 基于兴趣 + RetentionWorkQueue RetentionType = "workqueue" // 工作队列 +) + +// parseStorageType 解析存储类型 +func parseStorageType(st StorageType) jetstream.StorageType { + switch st { + case StorageMemory: + return jetstream.MemoryStorage + default: + return jetstream.FileStorage + } +} + +// parseRetentionType 解析保留策略 +func parseRetentionType(rt RetentionType) jetstream.RetentionPolicy { + switch rt { + case RetentionLimit: + return jetstream.LimitsPolicy + case RetentionPolicy: + return jetstream.InterestPolicy + default: + return jetstream.WorkQueuePolicy + } +} + +// createTaskStreamSimple 简化版创建任务流(适用于大多数场景) +// 只需提供流名称和主题数组,其他使用默认配置 +func createTaskStreamSimple(ctx context.Context, streamName string, subjects []string) error { + return createTaskStream(ctx, TaskStreamConfig{ + StreamName: streamName, + Subjects: subjects, + }) +} + +// createTaskStreamWithPriority 创建支持优先级的任务流 +func createTaskStreamWithPriority(ctx context.Context, streamPrefix string) error { + subjects := []string{ + fmt.Sprintf("%s.high.>", streamPrefix), + fmt.Sprintf("%s.normal.>", streamPrefix), + fmt.Sprintf("%s.low.>", streamPrefix), + } + return createTaskStream(ctx, TaskStreamConfig{ + StreamName: streamPrefix, + Subjects: subjects, + }) +} + +// CreateTaskStream 配置: 文件存储 + 工作队列策略 +// CreateTaskStream 创建任务消息队列流(JetStream 2.10+) +// 核心设计思路: +// 1. 严格持久化:使用文件存储,任务消息不会因为服务器重启而丢失 +// 2. 支持任务优先级:通过主题分级实现,如 ["tasks.high", "tasks.low"] +// 3. 死信队列支持:通过 RePublish 配置将失败任务路由到专门的 DLQ 流 +// 4. 灵活保留策略:根据任务重要性设置不同的保留时长(MaxAge) +// 5. 工作队列模式:确保每个任务只被一个消费者处理(DiscardPerSubject) +func createTaskStream(ctx context.Context, config TaskStreamConfig) error { + if !IsConnected() { + return fmt.Errorf("NATS 未连接") + } + + if g.IsNil(config.StreamName) { + return fmt.Errorf("流名称不能为空") + } + if len(config.Subjects) == 0 { + return fmt.Errorf("主题数组不能为空") + } + // 设置默认值 + if config.Storage == "" { + config.Storage = StorageFile // 默认文件存储 + } + if config.Retention == "" { + config.Retention = RetentionWorkQueue // 默认工作队列策略 + } + if config.MaxAge == 0 { + config.MaxAge = 24 * time.Hour // 默认保留24小时 + } + if config.Replicas == 0 { + config.Replicas = 1 // 默认单副本 + } + if config.MaxBytes == 0 { + config.MaxBytes = 10 * 1024 * 1024 * 1024 // 默认10GB + } + if config.MaxMsgs == 0 { + config.MaxMsgs = 100000 // 默认10万条消息 + } + if config.MaxMsgSize == 0 { + config.MaxMsgSize = 1024 * 1024 // 默认1MB + } + + if config.DiscardPerSubject { + config.DenyDelete = true // 工作队列模式下禁止删除 + } + + // 构建流配置 + jsConfig := jetstream.StreamConfig{ + Name: config.StreamName, + Subjects: config.Subjects, + Storage: parseStorageType(config.Storage), + Retention: parseRetentionType(config.Retention), + MaxAge: config.MaxAge, + Duplicates: config.Duplicates, + Replicas: config.Replicas, + MaxMsgSize: config.MaxMsgSize, + MaxBytes: config.MaxBytes, + MaxMsgs: config.MaxMsgs, + MaxMsgsPerSubject: config.MaxMsgsPerSubject, + MaxConsumers: config.MaxConsumers, + AllowRollup: config.AllowRollup, + DenyDelete: config.DenyDelete, + DenyPurge: config.DenyPurge, + Discard: jetstream.DiscardOld, // 默认删除旧消息 + DiscardNewPerSubject: config.DiscardPerSubject, + } + + // 配置死信队列重新发布(如果设置了) + if config.Republish != nil { + jsConfig.RePublish = &jetstream.RePublish{ + Source: config.Republish.Source, + Destination: config.Republish.Destination, + HeadersOnly: config.Republish.HeadersOnly, + } + } else { + // 使用固定的死信队列命名规范:{StreamName}.DLQ + dlqSubject := fmt.Sprintf("%s.DLQ", config.StreamName) + // 死信队列配置 + jsConfig.RePublish = &jetstream.RePublish{ + Source: ">", + Destination: dlqSubject, + HeadersOnly: true, + } + } + + // 检查流是否已存在 + stream, err := js.Stream(ctx, config.StreamName) + if err == nil { + // 流已存在,更新配置 + _, err = js.UpdateStream(ctx, jsConfig) + if err != nil { + return fmt.Errorf("更新任务流失败: %w", err) + } + g.Log().Infof(ctx, "✅ 任务消息队列流已更新: %s", stream.CachedInfo().Config.Name) + return nil + } + + // 创建新流 + stream, err = js.CreateStream(ctx, jsConfig) + if err != nil { + return fmt.Errorf("创建任务流失败: %w", err) + } + + // 记录配置信息 + configInfo := fmt.Sprintf("存储=%s, 策略=%s, 副本=%d, 保留=%v", config.Storage, config.Retention, config.Replicas, config.MaxAge) + if config.Republish != nil { + configInfo += fmt.Sprintf(", 死信队列=%s->%s", config.Republish.Source, config.Republish.Destination) + } + g.Log().Infof(ctx, "✅ 任务消息队列流创建成功: %s (%s)", stream.CachedInfo().Config.Name, configInfo) + + return nil +} diff --git a/nats/nats_test.go b/nats/nats_test.go index 7cb5851..94e1262 100644 --- a/nats/nats_test.go +++ b/nats/nats_test.go @@ -46,7 +46,12 @@ func TestNatsStreamOperations(t *testing.T) { ctx := context.Background() // 创建任务流 - err := CreateTaskStream(ctx, "test_tasks", []string{"test.task.>"}) + config := TaskStreamConfig{ + StreamName: "test_tasks", + Subjects: []string{"test.task.>"}, + //Subject: "test.task.process", + } + err := CreateTaskStream(ctx, config) if err != nil { t.Logf("创建任务流失败: %v", err) } @@ -79,7 +84,12 @@ func TestNatsConsumerOperations(t *testing.T) { ctx := context.Background() // 创建测试流 - err := CreateTaskStream(ctx, "test_consumer", []string{"test.consumer.>"}) + config := TaskStreamConfig{ + StreamName: "test_consumer", + Subjects: []string{"test.consumer.>"}, + //Subject: "test.consumer.process", + } + err := CreateTaskStream(ctx, config) if err != nil { t.Logf("创建流失败: %v", err) } @@ -119,22 +129,3 @@ func TestNatsConsumerOperations(t *testing.T) { // 清理流 _ = DeleteStream(ctx, "test_consumer") } - -// TestNatsPublishRequest 测试发布和请求 -func TestNatsPublishRequest(t *testing.T) { - ctx := context.Background() - - // 发布消息 - err := Publish(ctx, "test.publish", []byte("hello")) - if err != nil { - t.Logf("发布消息失败: %v", err) - } - - // RPC 请求 - response, err := CallRPC(ctx, "test.request", []byte("request"), 5*time.Second) - if err != nil { - t.Logf("RPC 请求失败: %v", err) - } else { - t.Logf("RPC 响应: %s", string(response)) - } -} diff --git a/nats/task.go b/nats/task.go new file mode 100644 index 0000000..996ded6 --- /dev/null +++ b/nats/task.go @@ -0,0 +1,411 @@ +package nats + +//import ( +// "context" +// "fmt" +// "time" +// +// "github.com/gogf/gf/v2/frame/g" +// "github.com/nats-io/nats.go/jetstream" +//) + +//// TaskPriority 任务优先级 +//type TaskPriority string +// +//const ( +// TaskPriorityHigh TaskPriority = "high" // 高优先级任务 +// TaskPriorityNormal TaskPriority = "normal" // 普通优先级任务 +// TaskPriorityLow TaskPriority = "low" // 低优先级任务 +//) +// +//// TaskStreamConfig 任务流配置 +//type TaskStreamConfig struct { +// StreamName string // 流名称 +// Subjects []string // 主题列表(支持优先级分级,如 tasks.high.>, tasks.normal.>, tasks.low.>) +// Subject string // 默认发布主题 +// Priority TaskPriority // 任务优先级 +// MaxAge time.Duration // 消息保留时长(根据任务重要性设置) +// MaxMsgsPerSub int64 // 每个订阅者最大消息数(防止内存溢出) +// Replicas int // 副本数(默认1,建议生产环境使用3) +// Duplicates time.Duration // 消息去重窗口(0表示不启用) +//} +// +//// TaskConsumerConfig 任务消费者配置 +//type TaskConsumerConfig struct { +// ConsumerName string // 消费者名称 +// AckPolicy *jetstream.AckPolicy +// MaxDeliveries int32 // 最大投递次数(用于重试控制) +// AckWait time.Duration // 等待ACK超时时间 +// Backoff []time.Duration // 重试退避策略 +// FilterSubject string // 过滤主题(可指定特定优先级任务) +// MaxAckPending int // 最大待确认消息数 +// MaxWaiting int // 最大等待消息数 +// ReplayPolicy *jetstream.ReplayPolicy // 重放策略 +//} + +// CreateTaskStream 创建任务流(基于 JetStream 2.10+ API) +// +// 核心设计思路: +// 1. 严格的持久化:使用文件存储(FileStorage)避免任务丢失 +// 2. 任务优先级:通过主题分级实现(tasks.high/tasks.normal/tasks.low) +// 3. 死信队列:配置死信队列处理失败任务 +// 4. 保留策略:按任务重要性设置不同的保留时长 +// 5. 工作队列策略:确保每条消息只被一个消费者处理 +// +// 参数: +// - ctx: 上下文 +// - config: 任务流配置 +// +// 返回: +// - error: 错误信息 +//func CreateTaskStream(ctx context.Context, config TaskStreamConfig) error { +// if !IsConnected() { +// return fmt.Errorf("NATS 未连接") +// } +// +// // 设置默认值 +// if config.MaxAge == 0 { +// config.MaxAge = 7 * 24 * time.Hour // 默认保留7天 +// } +// if config.MaxMsgsPerSub == 0 { +// config.MaxMsgsPerSub = 100000 // 默认每订阅者最多10万条消息 +// } +// if config.Replicas == 0 { +// config.Replicas = 1 // 默认单副本 +// } +// if config.Duplicates == 0 { +// config.Duplicates = 2 * time.Minute // 默认2分钟去重窗口 +// } +// +// // 验证主题配置 +// if len(config.Subjects) == 0 { +// return fmt.Errorf("任务流必须指定至少一个主题") +// } +// +// // 设置死信队列 +// // 使用固定的死信队列命名规范:{StreamName}.DLQ +// dlqSubject := fmt.Sprintf("%s.DLQ", config.StreamName) +// +// // 尝试获取现有流 +// stream, err := js.Stream(ctx, config.StreamName) +// if err == nil { +// // 流已存在,更新配置以适配任务流的特殊需求 +// _, err = js.UpdateStream(ctx, jetstream.StreamConfig{ +// Name: config.StreamName, +// Subjects: config.Subjects, +// Storage: jetstream.FileStorage, // 文件存储确保持久化 +// Retention: jetstream.WorkQueuePolicy, // 工作队列策略 +// MaxAge: config.MaxAge, +// MaxMsgs: config.MaxMsgsPerSub, +// Replicas: config.Replicas, +// Duplicates: config.Duplicates, +// // 死信队列配置 +// RePublish: &jetstream.RePublish{ +// Source: ">", // 匹配所有主题 +// Destination: dlqSubject, +// }, +// // 限制流大小(防止磁盘占用过多) +// MaxBytes: 10 * 1024 * 1024 * 1024, // 10GB +// }) +// if err != nil { +// return fmt.Errorf("更新任务流失败: %w", err) +// } +// g.Log().Infof(ctx, "✅ 任务流已更新: %s (优先级: %s, 保留: %v)", +// stream.CachedInfo().Config.Name, config.Priority, config.MaxAge) +// return nil +// } +// +// // 创建新任务流 +// streamConfig := jetstream.StreamConfig{ +// Name: config.StreamName, +// Subjects: config.Subjects, +// Storage: jetstream.FileStorage, // 文件存储确保持久化 +// Retention: jetstream.WorkQueuePolicy, // 工作队列策略 +// MaxAge: config.MaxAge, +// MaxMsgs: config.MaxMsgsPerSub, +// Replicas: config.Replicas, +// Duplicates: config.Duplicates, +// // 死信队列配置 +// RePublish: &jetstream.RePublish{ +// Source: ">", // 匹配所有主题 +// Destination: dlqSubject, +// }, +// // 限制流大小(防止磁盘占用过多) +// MaxBytes: 10 * 1024 * 1024 * 1024, // 10GB +// // 启用流清理 +// Discard: jetstream.DiscardOld, // 新消息替换旧消息 +// } +// +// stream, err = js.CreateStream(ctx, streamConfig) +// if err != nil { +// return fmt.Errorf("创建任务流失败: %w", err) +// } +// +// // 验证流是否创建成功 +// if stream == nil { +// return fmt.Errorf("创建任务流失败:流对象为空") +// } +// +// g.Log().Infof(ctx, "✅ 任务流创建成功: %s (文件存储+工作队列策略+死信队列, 优先级: %s, 保留: %v, 副本: %d)", +// stream.CachedInfo().Config.Name, config.Priority, config.MaxAge, config.Replicas) +// +// // 记录配置信息 +// g.Log().Infof(ctx, " - 主题列表: %v", config.Subjects) +// g.Log().Infof(ctx, " - 死信队列: %s", dlqSubject) +// g.Log().Infof(ctx, " - 最大消息数: %d", config.MaxMsgsPerSub) +// g.Log().Infof(ctx, " - 去重窗口: %v", config.Duplicates) +// +// return nil +//} +// +//// CreateOrUpdateTaskConsumer 创建或更新任务消费者(基于 JetStream 2.10+ API) +//// +//// 核心设计思路: +//// 1. 支持手动确认(AckExplicit)确保任务处理完成 +//// 2. 通过 Nack() 方法实现消息重试,超限后进入死信队列 +//// 3. 支持主题过滤,可订阅特定优先级任务 +//// 4. 限制待确认消息数,防止消费者过载 +//// 5. AckWait 设置消息处理超时时间 +//// +//// 参数: +//// - ctx: 上下文 +//// - streamName: 流名称 +//// - consumerConfig: 消费者配置 +//// +//// 返回: +//// - jetstream.Consumer: 消费者对象 +//// - error: 错误信息 +//func CreateOrUpdateTaskConsumer(ctx context.Context, streamName string, consumerConfig TaskConsumerConfig) (jetstream.Consumer, error) { +// if !IsConnected() { +// return nil, fmt.Errorf("NATS 未连接") +// } +// +// // 设置默认值 +// ackPolicy := jetstream.AckExplicitPolicy +// if consumerConfig.AckPolicy != nil { +// ackPolicy = *consumerConfig.AckPolicy +// } +// +// if consumerConfig.MaxDeliveries == 0 { +// consumerConfig.MaxDeliveries = 10 // 默认最多投递10次 +// } +// +// if consumerConfig.AckWait == 0 { +// consumerConfig.AckWait = 30 * time.Second // 默认30秒等待确认 +// } +// +// if consumerConfig.MaxAckPending == 0 { +// consumerConfig.MaxAckPending = 1000 // 默认最多1000条待确认消息 +// } +// +// if consumerConfig.MaxWaiting == 0 { +// consumerConfig.MaxWaiting = 512 // 默认最多512条等待消息 +// } +// +// replayPolicy := jetstream.ReplayInstantPolicy +// if consumerConfig.ReplayPolicy != nil { +// replayPolicy = *consumerConfig.ReplayPolicy +// } +// +// // 构建消费者配置 +// config := jetstream.ConsumerConfig{ +// Name: consumerConfig.ConsumerName, +// Durable: consumerConfig.ConsumerName, // 持久化消费者 +// AckPolicy: ackPolicy, +// AckWait: consumerConfig.AckWait, +// MaxAckPending: consumerConfig.MaxAckPending, +// MaxWaiting: consumerConfig.MaxWaiting, +// ReplayPolicy: replayPolicy, +// FilterSubject: consumerConfig.FilterSubject, +// } +// +// // 使用 CreateOrUpdateConsumer 创建或更新消费者 +// consumer, err := js.CreateOrUpdateConsumer(ctx, streamName, config) +// if err != nil { +// return nil, fmt.Errorf("创建任务消费者失败: %w", err) +// } +// +// g.Log().Infof(ctx, "✅ 任务消费者已创建/更新: %s/%s (等待确认: %v)", +// streamName, consumerConfig.ConsumerName, consumerConfig.AckWait) +// +// // 获取消费者信息并记录 +// info, err := consumer.Info(ctx) +// if err == nil { +// g.Log().Infof(ctx, " - 过滤主题: %s", info.Config.FilterSubject) +// g.Log().Infof(ctx, " - 最大待确认: %d", info.Config.MaxAckPending) +// g.Log().Infof(ctx, " - ACK策略: %s", info.Config.AckPolicy) +// } +// +// return consumer, nil +//} +// +//// CreateTaskStreamWithPriority 创建带优先级的任务流 +//// +//// 便捷方法,自动创建支持多优先级的任务流配置 +//// +//// 参数: +//// - ctx: 上下文 +//// - streamPrefix: 流名称前缀(如 "tasks") +//// - priority: 默认优先级 +//// +//// 返回: +//// - error: 错误信息 +//func CreateTaskStreamWithPriority(ctx context.Context, streamPrefix string, priority TaskPriority) error { +// if !IsConnected() { +// return fmt.Errorf("NATS 未连接") +// } +// +// // 构建支持多优先级的主题列表 +// subjects := []string{ +// fmt.Sprintf("%s.high.>", streamPrefix), // 高优先级任务 +// fmt.Sprintf("%s.normal.>", streamPrefix), // 普通优先级任务 +// fmt.Sprintf("%s.low.>", streamPrefix), // 低优先级任务 +// } +// +// // 根据优先级设置不同的保留时长 +// var maxAge time.Duration +// switch priority { +// case TaskPriorityHigh: +// maxAge = 30 * 24 * time.Hour // 高优先级保留30天 +// case TaskPriorityNormal: +// maxAge = 7 * 24 * time.Hour // 普通优先级保留7天 +// case TaskPriorityLow: +// maxAge = 24 * time.Hour // 低优先级保留1天 +// default: +// maxAge = 7 * 24 * time.Hour +// } +// +// config := TaskStreamConfig{ +// StreamName: streamPrefix, +// Subjects: subjects, +// Subject: fmt.Sprintf("%s.%s.>", streamPrefix, priority), +// Priority: priority, +// MaxAge: maxAge, +// MaxMsgsPerSub: 100000, +// Replicas: 1, +// Duplicates: 2 * time.Minute, +// } +// +// return CreateTaskStream(ctx, config) +//} +// +//// PublishTask 发布任务到指定流 +//// +//// 参数: +//// - ctx: 上下文 +//// - streamName: 流名称 +//// - task: 任务数据(会被JSON序列化) +//// +//// 返回: +//// - error: 错误信息 +//func PublishTask(ctx context.Context, streamName string, task interface{}) error { +// if !IsConnected() { +// return fmt.Errorf("NATS 未连接") +// } +// +// // 使用 JsPublish 发布消息 +// if err := JsPublish(ctx, streamName, task); err != nil { +// return fmt.Errorf("发布任务失败: %w", err) +// } +// +// return nil +//} +// +//// PublishTaskWithPriority 发布带优先级的任务 +//// +//// 参数: +//// - ctx: 上下文 +//// - streamPrefix: 流名称前缀 +//// - priority: 任务优先级 +//// - taskType: 任务类型 +//// - task: 任务数据(会被JSON序列化) +//// +//// 返回: +//// - error: 错误信息 +//func PublishTaskWithPriority(ctx context.Context, streamPrefix string, priority TaskPriority, taskType string, task interface{}) error { +// if !IsConnected() { +// return fmt.Errorf("NATS 未连接") +// } +// +// // 构建主题:{streamPrefix}.{priority}.{taskType} +// subject := fmt.Sprintf("%s.%s.%s", streamPrefix, priority, taskType) +// +// // 使用 JsPublish 发布消息 +// if err := JsPublish(ctx, subject, task); err != nil { +// return fmt.Errorf("发布任务失败: %w", err) +// } +// +// g.Log().Debugf(ctx, "任务已发布: %s (优先级: %s, 类型: %s)", subject, priority, taskType) +// +// return nil +//} +// +//// GetTaskStreamInfo 获取任务流信息 +//// +//// 参数: +//// - ctx: 上下文 +//// - streamName: 流名称 +//// +//// 返回: +//// - *jetstream.StreamInfo: 流信息 +//// - error: 错误信息 +//func GetTaskStreamInfo(ctx context.Context, streamName string) (*jetstream.StreamInfo, error) { +// if !IsConnected() { +// return nil, fmt.Errorf("NATS 未连接") +// } +// +// return GetStream(ctx, streamName) +//} +// +//// GetTaskConsumerInfo 获取任务消费者信息 +//// +//// 参数: +//// - ctx: 上下文 +//// - streamName: 流名称 +//// - consumerName: 消费者名称 +//// +//// 返回: +//// - *jetstream.ConsumerInfo: 消费者信息 +//// - error: 错误信息 +//func GetTaskConsumerInfo(ctx context.Context, streamName, consumerName string) (*jetstream.ConsumerInfo, error) { +// if !IsConnected() { +// return nil, fmt.Errorf("NATS 未连接") +// } +// +// return GetConsumer(ctx, streamName, consumerName) +//} +// +//// DeleteTaskStream 删除任务流 +//// +//// 注意:此操作会删除流及其所有消息,请谨慎使用 +//// +//// 参数: +//// - ctx: 上下文 +//// - streamName: 流名称 +//// +//// 返回: +//// - error: 错误信息 +//func DeleteTaskStream(ctx context.Context, streamName string) error { +// if !IsConnected() { +// return fmt.Errorf("NATS 未连接") +// } +// +// return DeleteStream(ctx, streamName) +//} +// +//// DeleteTaskConsumer 删除任务消费者 +//// +//// 参数: +//// - ctx: 上下文 +//// - streamName: 流名称 +//// - consumerName: 消费者名称 +//// +//// 返回: +//// - error: 错误信息 +//func DeleteTaskConsumer(ctx context.Context, streamName, consumerName string) error { +// if !IsConnected() { +// return fmt.Errorf("NATS 未连接") +// } +// +// return DeleteConsumer(ctx, streamName, consumerName) +//} diff --git a/nats/utils.go b/nats/utils.go new file mode 100644 index 0000000..1e43535 --- /dev/null +++ b/nats/utils.go @@ -0,0 +1,87 @@ +package nats + +import ( + "context" + "fmt" + "github.com/gogf/gf/v2/frame/g" + "github.com/nats-io/nats.go" + "go.opentelemetry.io/otel/trace" +) + +// ============ 上下文元数据工具函数 ============ +// 以下函数用于在 context 和 NATS 消息头之间互转元数据 + +// 定义常见的上下文元数据 key +const ( + TraceIDKey = "trace_id" + TokenKey = "token" +) + +func getTraceID(ctx context.Context) (traceID string, err error) { + // 提取 traceId:首先尝试从 OpenTelemetry Span 中提取,从 context 中提取 TraceID + span := trace.SpanFromContext(ctx) + if span != nil && span.SpanContext().HasTraceID() { + traceID = span.SpanContext().TraceID().String() + } else if tid := ctx.Value(TraceIDKey); tid != nil { + traceID = fmt.Sprintf("%v", tid) + } + if traceID == "" { + return traceID, fmt.Errorf("context 中没有 TraceID") + } + return +} + +// contextToHeaders 将 context 中的元数据转换为 NATS 消息头 +// 支持提取 user_id、tenant_id、trace_id、token 等常见字段 +func contextToHeaders(ctx context.Context) (nats.Header, error) { + headers := make(nats.Header) + + // 提取 traceId:首先尝试从 OpenTelemetry Span 中提取 + if traceID, err := getTraceID(ctx); err != nil { + return headers, err + } else { + headers.Set(TraceIDKey, traceID) + } + + // 提取 token(优先级:context value > HTTP Authorization header) + token := "" + if t := ctx.Value(TokenKey); t != nil { + token = fmt.Sprintf("%v", t) + } else if r := g.RequestFromCtx(ctx); r != nil { + // 从 HTTP 请求的 Authorization header 中提取 token + auth := r.GetHeader("Authorization") + if auth != "" { + // 移除 "Bearer " 前缀 + if len(auth) > 7 && auth[:7] == "Bearer " { + token = auth[7:] + } else { + token = auth + } + } + } + if token != "" { + headers.Set(TokenKey, token) + } + + return headers, nil +} + +// headersToContext 从 NATS 消息头重建 context +// 支持还原 user_id、tenant_id、trace_id、token 等字段 +func headersToContext(ctx context.Context, headers nats.Header) context.Context { + if headers == nil { + return ctx + } + + // 恢复 trace_id + if traceID := headers.Get(TraceIDKey); traceID != "" { + ctx = context.WithValue(ctx, TraceIDKey, traceID) + } + + // 恢复 token + if token := headers.Get(TokenKey); token != "" { + ctx = context.WithValue(ctx, TokenKey, token) + } + + return ctx +} diff --git a/utils/utils.go b/utils/utils.go index c9b6f74..0ae38d8 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -66,8 +66,18 @@ func GetMonthToday(t time.Time, month int) time.Time { } func GetUserInfo(ctx context.Context) (user beans.User, err error) { - r := g.RequestFromCtx(ctx) - if r != nil { + // 检查context是否已取消 + select { + case <-ctx.Done(): + return user, ctx.Err() + default: + } + + if !g.IsNil(ctx.Value("id")) || !g.IsNil(ctx.Value("userName")) || !g.IsNil(ctx.Value("tenantId")) { + user.UserId = ctx.Value("id") + user.UserName = ctx.Value("userName") + user.TenantId = ctx.Value("tenantId") + } else { redisAddr := g.Cfg().MustGet(ctx, "redis.default.address").String() gft := gftoken.NewGfToken( gftoken.WithCacheKey("gfToken:"), @@ -79,34 +89,47 @@ func GetUserInfo(ctx context.Context) (user beans.User, err error) { Address: redisAddr, Db: 1, })) - // 解析 token - data, err := gft.ParseToken(g.RequestFromCtx(ctx)) - if err != nil { - return user, gerror.Wrap(err, "token 解析失败") + var data *gftoken.CustomClaims + + if !g.IsNil(ctx.Value("token")) { + var tokenData *gftoken.TokenData + tokenData, _, err = gft.GetTokenData(ctx, ctx.Value("token").(string)) + if err != nil { + return user, gerror.Wrap(err, "token 解析失败") + } + var code int + if data, code = gft.IsNotExpired(tokenData.JwtToken); code != gftoken.JwtTokenOK { + return user, gerror.New("token jwt 解析失败") + } + } else if g.RequestFromCtx(ctx) != nil { + // 解析 token + data, err = gft.ParseToken(g.RequestFromCtx(ctx)) + if err != nil { + return user, gerror.Wrap(err, "token 解析失败") + } } // 检查 data 是否为 nil if data == nil { return user, gerror.New("token 数据为空") } - // 检查 data.Data 是否为 nil if data.Data == nil { + g.Log().Errorf(ctx, "data.Data 为空") return user, gerror.New("用户信息为空") } - dataMap := gconv.Map(data.Data) + user.UserId = dataMap["id"] user.UserName = dataMap["userName"] user.TenantId = dataMap["tenantId"] - } else { - user.TenantId = ctx.Value("tenantId") - user.UserName = ctx.Value("userName") } - if user.TenantId == nil { + + if g.IsNil(user.UserId) && g.IsNil(user.UserName) && g.IsNil(user.TenantId) { return user, gerror.New("租户信息为空") } return } + func SetValue(ctx context.Context, result any, key string, value any) { // 检查context是否已取消 select { @@ -207,6 +230,13 @@ func FormatUnixTime(timestamp int64) string { // ParseDurationWithDefault 解析持续时间,失败时使用默认值 - 通用时间处理工具 func ParseDurationWithDefault(ctx context.Context, durationStr, defaultStr, fieldName string) (time.Duration, string) { + // 检查context是否已取消 + select { + case <-ctx.Done(): + return 0, "" + default: + } + durationParsed, err := time.ParseDuration(durationStr) if err != nil { // 这里不能直接使用g.Log(),因为这是utils包,没有直接的日志访问 @@ -303,7 +333,7 @@ func Struct(params any, pointer any) error { if err != nil { return err } - err = json.Unmarshal(b, &pointer) + err = json.Unmarshal(b, pointer) if err != nil { return err } From b9f0360447bb3e98728035ad657b2fca84a9f523 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 22 Jan 2026 16:35:42 +0800 Subject: [PATCH 178/225] .gitignore --- rag/enio/base_task.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 rag/enio/base_task.go diff --git a/rag/enio/base_task.go b/rag/enio/base_task.go new file mode 100644 index 0000000..29623c0 --- /dev/null +++ b/rag/enio/base_task.go @@ -0,0 +1,27 @@ +package enio + +import ( + "gitee.com/red-future---jilin-g/common/beans" +) + +// BaseTask 任务基类 +type BaseTask struct { + beans.MongoBaseDO `bson:",inline"` + // 任务信息 + TaskType string `bson:"taskType" json:"taskType"` + Status string `bson:"status" json:"status"` + Priority int `bson:"priority,omitempty" json:"priority,omitempty"` + // 进度 + TotalItems int64 `bson:"totalItems" json:"totalItems"` + ProcessedItems int64 `bson:"processedItems" json:"processedItems"` + Progress float64 `bson:"progress" json:"progress"` + // 结果 + StartTime int64 `bson:"startTime" json:"startTime"` + EndTime int64 `bson:"endTime,omitempty" json:"endTime,omitempty"` + Duration int64 `bson:"duration,omitempty" json:"duration,omitempty"` + SuccessCount int64 `bson:"successCount" json:"successCount"` + FailCount int64 `bson:"failCount" json:"failCount"` + ErrorMsg string `bson:"errorMsg,omitempty" json:"errorMsg,omitempty"` + // 其他 + Executor string `bson:"executor,omitempty" json:"executor,omitempty"` +} From 8f6adbb16df3039667dd410f6f015742e0634500 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Thu, 22 Jan 2026 15:04:13 +0800 Subject: [PATCH 179/225] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=E7=A7=9F=E6=88=B7=E6=A3=80=E6=9F=A5=E9=80=BB=E8=BE=91=EF=BC=8C?= =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=95=B0=E6=8D=AE=E7=BB=93=E6=9E=84=E5=B9=B6?= =?UTF-8?q?=E7=AE=80=E5=8C=96=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- beans/module_tenant.go | 48 +++++++++++++------------ middleware/module_tenant_check.go | 58 +++++++++++++------------------ mongo/mongo.go | 55 +---------------------------- 3 files changed, 51 insertions(+), 110 deletions(-) diff --git a/beans/module_tenant.go b/beans/module_tenant.go index fb2f328..4b53f8b 100644 --- a/beans/module_tenant.go +++ b/beans/module_tenant.go @@ -19,27 +19,36 @@ var ( TenantModuleAICs = ModuleAssetId["customerService"] // AI客服模块 ) -// TenantModuleType 租户类型 -type TenantModuleType struct { +type TenantModuleType string + +const ( + TenantModuleTypePlatform TenantModuleType = "platform" + TenantModuleTypePrivate TenantModuleType = "private" + TenantModuleTypeSupplier TenantModuleType = "supplier" + TenantModuleTypeSmallShop TenantModuleType = "small_shop" +) + +// TenantModuleTypeKV 租户类型 +type TenantModuleTypeKV struct { Key string Value string } // TenantModuleTypesAssets 资产模块租户类型 -var TenantModuleTypesAssets = []TenantModuleType{ - {Key: "private_cloud", Value: "私有云租户"}, - {Key: "supplier", Value: "供应商"}, - {Key: "small_shop", Value: "电商小店"}, +var TenantModuleTypesAssets = []TenantModuleTypeKV{ + {Key: string(TenantModuleTypePrivate), Value: "私域租户"}, + {Key: string(TenantModuleTypeSupplier), Value: "供应商"}, + {Key: string(TenantModuleTypeSmallShop), Value: "电商小店"}, } // TenantModuleTypesAd 广告模块租户类型(待定) -var TenantModuleTypesAd []TenantModuleType +var TenantModuleTypesAd []TenantModuleTypeKV // TenantModuleTypesAICs AI客服模块租户类型(待定) -var TenantModuleTypesAICs []TenantModuleType +var TenantModuleTypesAICs []TenantModuleTypeKV // GetTenantModuleTypes 获取模块的租户类型列表 -func GetTenantModuleTypes(module string) []TenantModuleType { +func GetTenantModuleTypes(module string) []TenantModuleTypeKV { switch module { case TenantModuleAssets: return TenantModuleTypesAssets @@ -48,7 +57,7 @@ func GetTenantModuleTypes(module string) []TenantModuleType { case TenantModuleAICs: return TenantModuleTypesAICs default: - return []TenantModuleType{} + return []TenantModuleTypeKV{} } } @@ -59,21 +68,14 @@ type ModuleTenantCheckReq struct { // ModuleTenantCheckRes 调用admin-go设置模块租户关系的响应 type ModuleTenantCheckRes struct { - Status string `json:"status"` // 开通状态:activated(已开通)、expired(已到期)、not_activated(未开通) - Message string `json:"message"` // 状态描述 - OpenStatus bool `json:"openStatus"` // 开通状态 + Status bool `json:"status"` + CertificationStatus bool `json:"certificationStatus"` + Message string `json:"message"` // 状态描述 } // ModuleTenant 模块租户关系实体(引用自admin-go) type ModuleTenant struct { - Id uint64 `json:"id" description:""` - CreateBy uint64 `json:"createBy" description:"创建者"` - UpdateBy uint64 `json:"updateBy" description:"更新者"` - CreatedAt *gtime.Time `json:"createdAt" description:"创建时间"` - UpdatedAt *gtime.Time `json:"updatedAt" description:"更新时间"` - ModuleKey string `json:"moduleKey" description:"模块Key"` - TenantId uint64 `json:"tenantId" description:"租户ID"` - ExpireAt *gtime.Time `json:"expireAt" description:"到期时间"` - AssetId string `json:"assetId" description:"资产ID"` - AssetSkuId string `json:"assetSkuId" description:"资产SKU ID"` + ExpireAt *gtime.Time `json:"expireAt" description:"到期时间"` + TenantModuleType TenantModuleType `json:"tenantModuleType" description:"租户模块类型"` + CertificationStatus int `json:"certificationStatus" description:"认证状态"` } diff --git a/middleware/module_tenant_check.go b/middleware/module_tenant_check.go index 0fb32bb..1eff872 100644 --- a/middleware/module_tenant_check.go +++ b/middleware/module_tenant_check.go @@ -13,21 +13,15 @@ import ( "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/os/gtime" "github.com/gogf/gf/v2/util/gconv" + "net/http" "time" ) func ModuleTenantCheck(r *ghttp.Request) { - //将 http.Header 转换为 map[string]string - headers := make(map[string]string) - for k, v := range r.Request.Header { - if len(v) > 0 { - headers[k] = v[0] - } - } // 检查是否是超级管理员 isSuperAdmin := false if err := nats.CallRPC(r.Context(), "userService.IsSuperAdmin", nil, &isSuperAdmin); err != nil { - SetResponseInfo(r.Context(), r, err) + SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, err) } // 如果是超级管理员,则不进行模块租户检查 if isSuperAdmin || r.Request.RequestURI == "/asset/getAssetAndSku?assetId=696b4acd1be1c8b76c4b4c15" { @@ -36,7 +30,7 @@ func ModuleTenantCheck(r *ghttp.Request) { } getUserInfo, err := utils.GetUserInfo(r.Context()) if err != nil { - SetResponseInfo(r.Context(), r, err) + SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, err) } exit := gconv.Int64(time.Minute * 1) getEX, err := message.GetRedisClientTest("test").GetEX(r.Context(), fmt.Sprintf("module_tenant:tenantId-%v", getUserInfo.TenantId), gredis.GetEXOption{ @@ -45,34 +39,30 @@ func ModuleTenantCheck(r *ghttp.Request) { }, }) if err != nil { - SetResponseInfo(r.Context(), r, err) + SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, err) } // 获取模块key moduleKey := g.Cfg().MustGet(context.Background(), "server.name").String() if !g.IsEmpty(getEX.String()) { - list := make([]beans.ModuleTenant, 0) - if err = json.Unmarshal([]byte(getEX.String()), &list); err != nil { - SetResponseInfo(r.Context(), r, err) - } - var expireAt *gtime.Time - for _, value := range list { - if value.ModuleKey == moduleKey { - expireAt = value.ExpireAt - break - } + list := new(beans.ModuleTenant) + if err = json.Unmarshal(getEX.Bytes(), &list); err != nil { + SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, err) } // 缓存中有数据,检查是否过期 - if !g.IsEmpty(expireAt) { + if !g.IsEmpty(list.ExpireAt) { gt1 := gtime.New(time.Now()) - gt2 := gtime.New(expireAt) + gt2 := gtime.New(list.ExpireAt) if !gt1.Before(gt2) { - SetResponseInfo(r.Context(), r, "您访问的模块已过期,请续期后再使用") + SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, "功能模块已到期,请续期后再使用") + } else { + if list.CertificationStatus != 2 { + SetResponseInfo(r.Context(), r, http.StatusPreconditionRequired, "功能模块未认证通过,请认证后再使用") + } } } else { - SetResponseInfo(r.Context(), r, "您未开通此模块,请开通后再使用") + SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, "您未开通此功能模块,请开通后再使用") } } else { - // 缓存为空,调用admin-go的Check接口检查模块开通状态 checkRes := new(beans.ModuleTenantCheckRes) checkReq := beans.ModuleTenantCheckReq{ ModuleKey: moduleKey, @@ -80,25 +70,27 @@ func ModuleTenantCheck(r *ghttp.Request) { } err = nats.CallRPC(r.Context(), "moduleService.Check", &checkReq, checkRes) if err != nil { - SetResponseInfo(r.Context(), r, err) + SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, err) } // 根据检查结果判断是否允许访问 - if checkRes.Status == "not_activated" { - SetResponseInfo(r.Context(), r, "您未开通此模块,请开通后再使用") - } else if checkRes.Status == "expired" { - SetResponseInfo(r.Context(), r, "您访问的模块已过期,请续期后再使用") + if !checkRes.Status { + SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, checkRes.Message) + } else { + if !checkRes.CertificationStatus { + SetResponseInfo(r.Context(), r, http.StatusPreconditionRequired, checkRes.Message) + } } } + r.Middleware.Next() // 继续执行后续中间件和路由处理 } // SetResponseInfo 设置响应信息 -func SetResponseInfo(ctx context.Context, r *ghttp.Request, message any) { +func SetResponseInfo(ctx context.Context, r *ghttp.Request, code int, message any) { _ = ctx - r.Response.Status = 402 r.Response.WriteJsonExit(map[string]interface{}{ "success": false, - "code": 402, + "code": code, "message": fmt.Sprintf("服务不可用:%s", message), }) r.Exit() diff --git a/mongo/mongo.go b/mongo/mongo.go index f0b10ae..c25d86a 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -10,7 +10,6 @@ import ( "errors" "fmt" "gitee.com/red-future---jilin-g/common/log/consts" - "reflect" "time" "gitee.com/red-future---jilin-g/common/beans" @@ -643,7 +642,7 @@ func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bs return bulkResult, nil } -func BuildUpdateFilter(ctx context.Context, req interface{}) (filter bson.M, err error) { +func BuildUpdateData(ctx context.Context, req interface{}) (filter bson.M, err error) { _ = ctx filter = bson.M{} reqMap := gconv.Map(req) @@ -654,55 +653,3 @@ func BuildUpdateFilter(ctx context.Context, req interface{}) (filter bson.M, err } return } - -// EntityToBson 将 *entity/entity 转换为 bson.M -func EntityToBson(entity interface{}) (bson.M, error) { - return EntityToBsonWithFilter(entity, false) -} - -// EntityToBsonWithFilter 将 *entity/entity 转换为 bson.M,并可选择是否过滤空值 -func EntityToBsonWithFilter(entity interface{}, filterEmpty bool) (bson.M, error) { - if entity == nil { - return nil, fmt.Errorf("传入的 entity 实例为 nil") - } - bsonBytes, err := bson.Marshal(entity) - if err != nil { - return nil, fmt.Errorf("entity 序列化为 BSON 字节流失败:%w", err) - } - var bsonMap bson.M - err = bson.Unmarshal(bsonBytes, &bsonMap) - if err != nil { - return nil, fmt.Errorf("BSON 字节流反序列化为 bson.M 失败:%w", err) - } - if filterEmpty { - for key, value := range bsonMap { - if isEmptyWithZero(value) { - delete(bsonMap, key) - } - } - } - return bsonMap, nil -} - -// isEmptyWithZero 判断是否为空值,但保留 int 类型的 0 值 -func isEmptyWithZero(value interface{}) bool { - if value == nil { - return true - } - rv := reflect.ValueOf(value) - kind := rv.Kind() - if kind == reflect.Ptr { - if rv.IsNil() { - return true - } - kind = rv.Elem().Kind() - } - switch kind { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, - reflect.Float32, reflect.Float64: - return false - default: - return g.IsEmpty(value) - } -} From fd938ed980f8dbde2325ff40d26d29a27e2a2419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 22 Jan 2026 16:55:30 +0800 Subject: [PATCH 180/225] .gitignore --- rag/enio/task_enum.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 rag/enio/task_enum.go diff --git a/rag/enio/task_enum.go b/rag/enio/task_enum.go new file mode 100644 index 0000000..314b7e8 --- /dev/null +++ b/rag/enio/task_enum.go @@ -0,0 +1,21 @@ +package enio + +// TaskPriority 任务优先级 +type TaskPriority string + +const ( + TaskPriorityLow TaskPriority = "low" // 低优先级 + TaskPriorityMedium TaskPriority = "medium" // 中优先级 + TaskPriorityHigh TaskPriority = "high" // 高优先级 + TaskPriorityUrgent TaskPriority = "urgent" // 紧急 +) + +// TaskType 任务类型 +type TaskType string + +const ( + TaskTypeDocumentIngestion TaskType = "document_ingestion" // 文档摄入任务 + TaskTypeVectorIngestion TaskType = "vector_ingestion" // 向量摄入任务 + TaskTypeIndexCreation TaskType = "index_creation" // 索引创建任务 + TaskTypeQAProcessing TaskType = "qa_processing" // 问答处理任务 +) From 6a3c3c3ec83d891c94d21fd0f52e1a938620fe9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 22 Jan 2026 17:07:50 +0800 Subject: [PATCH 181/225] .gitignore --- rag/enio/base_task.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rag/enio/base_task.go b/rag/enio/base_task.go index 29623c0..0da9f59 100644 --- a/rag/enio/base_task.go +++ b/rag/enio/base_task.go @@ -8,9 +8,9 @@ import ( type BaseTask struct { beans.MongoBaseDO `bson:",inline"` // 任务信息 - TaskType string `bson:"taskType" json:"taskType"` - Status string `bson:"status" json:"status"` - Priority int `bson:"priority,omitempty" json:"priority,omitempty"` + TaskType string `bson:"taskType" json:"taskType"` + Status string `bson:"status" json:"status"` + Priority TaskPriority `bson:"priority,omitempty" json:"priority,omitempty"` // 进度 TotalItems int64 `bson:"totalItems" json:"totalItems"` ProcessedItems int64 `bson:"processedItems" json:"processedItems"` From 88d9d2b8a6dac9ab418c5b0a2f86551e8e44566f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 22 Jan 2026 19:52:54 +0800 Subject: [PATCH 182/225] .gitignore --- rag/enio/base_task.go | 2 +- rag/enio/task_status.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 rag/enio/task_status.go diff --git a/rag/enio/base_task.go b/rag/enio/base_task.go index 0da9f59..6364fba 100644 --- a/rag/enio/base_task.go +++ b/rag/enio/base_task.go @@ -9,7 +9,7 @@ type BaseTask struct { beans.MongoBaseDO `bson:",inline"` // 任务信息 TaskType string `bson:"taskType" json:"taskType"` - Status string `bson:"status" json:"status"` + Status TaskStatus `bson:"status" json:"status"` Priority TaskPriority `bson:"priority,omitempty" json:"priority,omitempty"` // 进度 TotalItems int64 `bson:"totalItems" json:"totalItems"` diff --git a/rag/enio/task_status.go b/rag/enio/task_status.go new file mode 100644 index 0000000..d6d2479 --- /dev/null +++ b/rag/enio/task_status.go @@ -0,0 +1,12 @@ +package enio + +// TaskStatus 任务状态 +type TaskStatus string + +const ( + TaskStatusPending TaskStatus = "pending" // 待处理 + TaskStatusRunning TaskStatus = "running" // 运行中 + TaskStatusCompleted TaskStatus = "completed" // 已完成 + TaskStatusFailed TaskStatus = "failed" // 失败 + TaskStatusCancelled TaskStatus = "cancelled" // 已取消 +) From 731a2a479dedf612d475b354837e29cee8b7e29b Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Thu, 22 Jan 2026 19:07:10 +0800 Subject: [PATCH 183/225] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=9C=AA=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E7=9A=84RandomSoftDelete=E6=96=B9=E6=B3=95=E5=B9=B6?= =?UTF-8?q?=E4=BF=AE=E5=A4=8Dlimit=E6=9D=A1=E4=BB=B6=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 48 ------------------------------------------------ 1 file changed, 48 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index c25d86a..c86cb62 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -197,7 +197,6 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c opt := options.Find().SetSkip(skip) if limit != -1 { opt.SetLimit(limit) - } else { total, err = m.Count(ctx, filter, collection) if err != nil || total == 0 { return @@ -521,53 +520,6 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll return } -// RandomSoftDelete 随机软删除个文档的 _id -func (m *MongoDB) RandomSoftDelete(ctx context.Context, limit int, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { - source, err := m.getDataSource() - if err != nil { - return nil, err - } - db := source.Database() - - _ = opts - pipeline := mongo.Pipeline{ - bson.D{{Key: "$addFields", Value: bson.D{{Key: "random", Value: bson.M{"$rand": bson.M{}}}}}}, - bson.D{{Key: "$match", Value: bson.D{{Key: "isDeleted", Value: false}}}}, - bson.D{{Key: "$sort", Value: bson.D{{Key: "random", Value: -1}}}}, - bson.D{{Key: "$limit", Value: limit}}, - bson.D{{Key: "$project", Value: bson.D{{Key: "_id", Value: 1}}}}, - } - cursor, err := db.Collection(collection).Aggregate(ctx, pipeline) - if err != nil { - return - } - defer cursor.Close(ctx) - - var idsToUpdate []bson.ObjectID - for cursor.Next(ctx) { - var result bson.M - if err := cursor.Decode(&result); err != nil { - return nil, err - } - id := result["_id"].(bson.ObjectID) - idsToUpdate = append(idsToUpdate, id) - } - if err := cursor.Err(); err != nil { - return nil, err - } - fmt.Printf("准备更新的随机文档ID: %v\n", idsToUpdate) - - if len(idsToUpdate) > 0 { - filter := bson.D{{Key: "_id", Value: bson.D{{Key: "$in", Value: idsToUpdate}}}} - update := bson.D{{Key: "$set", Value: bson.D{{Key: "isDeleted", Value: true}}}} - _, err = db.Collection(collection).UpdateMany(ctx, filter, update) - if err != nil { - return - } - } - return -} - // SaveOrUpdate 批量增加或修改 func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.BulkWriteResult, err error) { source, err := m.getDataSource() From 0036f232e13e24a5d19e6f33a4f8b7ddd2f36bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 22 Jan 2026 19:54:09 +0800 Subject: [PATCH 184/225] .gitignore --- rag/enio/{task_enum.go => priority_enum.go} | 0 rag/enio/{task_status.go => status_enum.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename rag/enio/{task_enum.go => priority_enum.go} (100%) rename rag/enio/{task_status.go => status_enum.go} (100%) diff --git a/rag/enio/task_enum.go b/rag/enio/priority_enum.go similarity index 100% rename from rag/enio/task_enum.go rename to rag/enio/priority_enum.go diff --git a/rag/enio/task_status.go b/rag/enio/status_enum.go similarity index 100% rename from rag/enio/task_status.go rename to rag/enio/status_enum.go From a0598451e6c5c78050cdfcc8e094e4bef84790a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 23 Jan 2026 23:52:31 +0800 Subject: [PATCH 185/225] .gitignore --- rag/enio/priority_enum.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/rag/enio/priority_enum.go b/rag/enio/priority_enum.go index 314b7e8..0fb371b 100644 --- a/rag/enio/priority_enum.go +++ b/rag/enio/priority_enum.go @@ -14,8 +14,11 @@ const ( type TaskType string const ( - TaskTypeDocumentIngestion TaskType = "document_ingestion" // 文档摄入任务 - TaskTypeVectorIngestion TaskType = "vector_ingestion" // 向量摄入任务 - TaskTypeIndexCreation TaskType = "index_creation" // 索引创建任务 - TaskTypeQAProcessing TaskType = "qa_processing" // 问答处理任务 + TaskTypeDocumentIngestion TaskType = "document_ingestion" // 文档摄入任务 + TaskTypeVectorIngestion TaskType = "vector_ingestion" // 向量摄入任务 + TaskTypeIndexCreation TaskType = "index_creation" // 索引创建任务 + TaskTypeQAProcessing TaskType = "qa_processing" // 问答处理任务 + TaskTypeKnowledgeConstruction TaskType = "knowledge_construction" // 知识库构建任务 + TaskTypeGraphBuilding TaskType = "graph_building" // 图谱构建任务 + TaskTypeKnowledgeSync TaskType = "knowledge_sync" // 知识同步任务 ) From 9c8d40d3baff737c1b2e41d3e3370d2e9fef1baa Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Fri, 23 Jan 2026 16:36:28 +0800 Subject: [PATCH 186/225] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20MongoDB=20?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=93=8D=E4=BD=9C=E4=B8=AD=20$set=20?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E4=B8=BA=E7=A9=BA=E6=97=B6=E7=9A=84=20panic?= =?UTF-8?q?=20=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index c86cb62..c262864 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -498,7 +498,10 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll } } } - setDoc := update["$set"].(bson.M) + setDoc := bson.M{} + if !g.IsEmpty(update["$set"]) { + setDoc = update["$set"].(bson.M) + } if !g.IsEmpty(user.UserName) { setDoc["updater"] = user.UserName } From 754ffbd84edf59e396f97242035b851855832da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 26 Jan 2026 09:02:49 +0800 Subject: [PATCH 187/225] .gitignore --- mongo/mongo.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index c262864..b4d2aac 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -9,9 +9,10 @@ import ( "context" "errors" "fmt" - "gitee.com/red-future---jilin-g/common/log/consts" "time" + "gitee.com/red-future---jilin-g/common/log/consts" + "gitee.com/red-future---jilin-g/common/beans" "gitee.com/red-future---jilin-g/common/log/model/entity" "gitee.com/red-future---jilin-g/common/redis" @@ -460,16 +461,16 @@ func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, } // DeleteSoft 假删除记录 -func (m *MongoDB) DeleteSoft(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { +func (m *MongoDB) DeleteSoft(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (modifiedCount int64, err error) { update := bson.M{"$set": bson.M{"isDeleted": true}} return m.Update(ctx, filter, update, collection, opts...) } // Update 修改记录 -func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.UpdateResult, err error) { +func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (modifiedCount int64, err error) { source, err := m.getDataSource() if err != nil { - return nil, err + return 0, err } db := source.Database() @@ -507,10 +508,11 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll } setDoc["updatedAt"] = gtime.Now().Time update["$set"] = setDoc - result, err = db.Collection(collection).UpdateMany(ctx, filter, update, opts...) + result, err := db.Collection(collection).UpdateMany(ctx, filter, update, opts...) if err != nil { return } + modifiedCount = result.ModifiedCount // 清理redis err = m.CleanRedis(ctx, filter, user.TenantId, collection) // 写日志 From c31848ae42dc05c38763201c0a10cbc00a0b700c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 26 Jan 2026 11:34:04 +0800 Subject: [PATCH 188/225] .gitignore --- rag/enio/priority_enum.go | 13 ------------- rag/enio/task_type.go | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 rag/enio/task_type.go diff --git a/rag/enio/priority_enum.go b/rag/enio/priority_enum.go index 0fb371b..365903a 100644 --- a/rag/enio/priority_enum.go +++ b/rag/enio/priority_enum.go @@ -9,16 +9,3 @@ const ( TaskPriorityHigh TaskPriority = "high" // 高优先级 TaskPriorityUrgent TaskPriority = "urgent" // 紧急 ) - -// TaskType 任务类型 -type TaskType string - -const ( - TaskTypeDocumentIngestion TaskType = "document_ingestion" // 文档摄入任务 - TaskTypeVectorIngestion TaskType = "vector_ingestion" // 向量摄入任务 - TaskTypeIndexCreation TaskType = "index_creation" // 索引创建任务 - TaskTypeQAProcessing TaskType = "qa_processing" // 问答处理任务 - TaskTypeKnowledgeConstruction TaskType = "knowledge_construction" // 知识库构建任务 - TaskTypeGraphBuilding TaskType = "graph_building" // 图谱构建任务 - TaskTypeKnowledgeSync TaskType = "knowledge_sync" // 知识同步任务 -) diff --git a/rag/enio/task_type.go b/rag/enio/task_type.go new file mode 100644 index 0000000..4dec33f --- /dev/null +++ b/rag/enio/task_type.go @@ -0,0 +1,14 @@ +package enio + +// TaskType 任务类型 +type TaskType string + +const ( + TaskTypeDocumentIngestion TaskType = "document_ingestion" // 文档摄入任务 + TaskTypeVectorIngestion TaskType = "vector_ingestion" // 向量摄入任务 + TaskTypeIndexCreation TaskType = "index_creation" // 索引创建任务 + TaskTypeQAProcessing TaskType = "qa_processing" // 问答处理任务 + TaskTypeKnowledgeConstruction TaskType = "knowledge_construction" // 知识库构建任务 + TaskTypeGraphBuilding TaskType = "graph_building" // 图谱构建任务 + TaskTypeKnowledgeSync TaskType = "knowledge_sync" // 知识同步任务 +) From 647e88b21a79acaffdeb1059fc0a3b65bb5385b9 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Mon, 26 Jan 2026 11:40:57 +0800 Subject: [PATCH 189/225] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20MinIO=20=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=89=8D=E7=BC=80=E9=85=8D=E7=BD=AE=E5=B9=B6=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E6=96=87=E4=BB=B6=E5=9C=B0=E5=9D=80=E6=8B=BC=E6=8E=A5?= =?UTF-8?q?=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- minio/minio.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/minio/minio.go b/minio/minio.go index 03df1d4..2b6a98c 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -19,11 +19,12 @@ import ( // IoConfig 映射 YAML 中的 minio 配置节点 type IoConfig struct { - Endpoint string `yaml:"endpoint"` // MinIO API 地址 - AccessKey string `yaml:"accessKey"` // AK - SecretKey string `yaml:"secretKey"` // SK - Secure bool `yaml:"secure"` // 是否启用 SSL - Region string `yaml:"region"` // 区域 + FilePrefix string `yaml:"filePrefix"` // 文件前缀 + Endpoint string `yaml:"endpoint"` // MinIO API 地址 + AccessKey string `yaml:"accessKey"` // AK + SecretKey string `yaml:"secretKey"` // SK + Secure bool `yaml:"secure"` // 是否启用 SSL + Region string `yaml:"region"` // 区域 } // 全局 MinIO 客户端(初始化一次,避免重复创建) @@ -36,11 +37,12 @@ func init() { if !g.Cfg().MustGet(ctx, "minio").IsEmpty() { // 加载 MinIO 配置(可从配置文件/环境变量读取,这里硬编码示例) minioCfg = IoConfig{ - Endpoint: g.Cfg().MustGet(ctx, "minio.endpoint").String(), - AccessKey: g.Cfg().MustGet(ctx, "minio.accessKey").String(), - SecretKey: g.Cfg().MustGet(ctx, "minio.secretKey").String(), - Secure: g.Cfg().MustGet(ctx, "minio.secure").Bool(), - Region: g.Cfg().MustGet(ctx, "minio.region").String(), + FilePrefix: g.Cfg().MustGet(ctx, "filePrefix.endpoint").String(), + Endpoint: g.Cfg().MustGet(ctx, "minio.endpoint").String(), + AccessKey: g.Cfg().MustGet(ctx, "minio.accessKey").String(), + SecretKey: g.Cfg().MustGet(ctx, "minio.secretKey").String(), + Secure: g.Cfg().MustGet(ctx, "minio.secure").Bool(), + Region: g.Cfg().MustGet(ctx, "minio.region").String(), } // 创建 MinIO 客户端 var err error @@ -123,14 +125,14 @@ func uploadFile(ctx context.Context, bucketName string, fileHeader *ghttp.Upload return objectName, err } -// GetIFileAddressPrefix 拼接图片前缀地址 -func GetIFileAddressPrefix(ctx context.Context) (imageUrl string) { +// GetFileAddressPrefix 拼接图片前缀地址 +func GetFileAddressPrefix(ctx context.Context) (imageUrl string) { // 拼接图片前缀地址 var url = "http://" if minioCfg.Secure { url = "https://" } - imgAddressPrefix := url + minioCfg.Endpoint + "/" + getBucketName(ctx) + imgAddressPrefix := url + minioCfg.FilePrefix + "/" + getBucketName(ctx) return imgAddressPrefix } From ca139c02bb6e2266043fbe2674a204dd8307754f Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Mon, 26 Jan 2026 11:52:28 +0800 Subject: [PATCH 190/225] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20MinIO=20=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E4=B8=AD=20filePrefix=20=E7=9A=84=E9=94=AE=E5=90=8D?= =?UTF-8?q?=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- minio/minio.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/minio/minio.go b/minio/minio.go index 2b6a98c..030d87f 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -37,7 +37,7 @@ func init() { if !g.Cfg().MustGet(ctx, "minio").IsEmpty() { // 加载 MinIO 配置(可从配置文件/环境变量读取,这里硬编码示例) minioCfg = IoConfig{ - FilePrefix: g.Cfg().MustGet(ctx, "filePrefix.endpoint").String(), + FilePrefix: g.Cfg().MustGet(ctx, "filePrefix").String(), Endpoint: g.Cfg().MustGet(ctx, "minio.endpoint").String(), AccessKey: g.Cfg().MustGet(ctx, "minio.accessKey").String(), SecretKey: g.Cfg().MustGet(ctx, "minio.secretKey").String(), From 17d6de4ffd6eb83282aaaeebca3b3859e8e0a0ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 26 Jan 2026 11:44:18 +0800 Subject: [PATCH 191/225] .gitignore --- rag/enio/base_task.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/rag/enio/base_task.go b/rag/enio/base_task.go index 6364fba..866d5e2 100644 --- a/rag/enio/base_task.go +++ b/rag/enio/base_task.go @@ -1,6 +1,8 @@ package enio import ( + "time" + "gitee.com/red-future---jilin-g/common/beans" ) @@ -8,7 +10,7 @@ import ( type BaseTask struct { beans.MongoBaseDO `bson:",inline"` // 任务信息 - TaskType string `bson:"taskType" json:"taskType"` + TaskType TaskType `bson:"taskType" json:"taskType"` Status TaskStatus `bson:"status" json:"status"` Priority TaskPriority `bson:"priority,omitempty" json:"priority,omitempty"` // 进度 @@ -16,12 +18,11 @@ type BaseTask struct { ProcessedItems int64 `bson:"processedItems" json:"processedItems"` Progress float64 `bson:"progress" json:"progress"` // 结果 - StartTime int64 `bson:"startTime" json:"startTime"` - EndTime int64 `bson:"endTime,omitempty" json:"endTime,omitempty"` - Duration int64 `bson:"duration,omitempty" json:"duration,omitempty"` - SuccessCount int64 `bson:"successCount" json:"successCount"` - FailCount int64 `bson:"failCount" json:"failCount"` - ErrorMsg string `bson:"errorMsg,omitempty" json:"errorMsg,omitempty"` + StartTime *time.Time `bson:"startTime" json:"startTime"` + EndTime *time.Time `bson:"endTime,omitempty" json:"endTime,omitempty"` + Duration int64 `bson:"duration,omitempty" json:"duration,omitempty"` + SuccessCount int64 `bson:"successCount" json:"successCount"` + FailCount int64 `bson:"failCount" json:"failCount"` // 其他 Executor string `bson:"executor,omitempty" json:"executor,omitempty"` } From e502aeea530f06455154972d4f0fcfa0525b9026 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Tue, 27 Jan 2026 17:54:58 +0800 Subject: [PATCH 192/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E4=B8=80=E4=B8=8Bhtt?= =?UTF-8?q?p.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 15 +---------- ragflow/client.go | 66 +++++++++++++++++++++++++++++------------------ 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/http/http.go b/http/http.go index 9f0efd7..285ceec 100644 --- a/http/http.go +++ b/http/http.go @@ -89,20 +89,7 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin defer response.Close() result := response.ReadAll() - // 第三方API特例:RAGFlow等第三方API响应格式为{code,data,message}一层结构,直接解析原始JSON到target - // 内部API格式为{code:200,message:"",data:{...}}两层结构,需经过DefaultHandlerResponse二次解析 - // 判断依据:URL包含/api/v1/(不影响内部API调用) - isThirdPartyAPI := strings.Contains(url, "/api/v1/") - - if isThirdPartyAPI { - // 第三方API特例:直接解析原始JSON到target,不经过DefaultHandlerResponse - if err = gconv.Struct(result, target); err != nil { - return errors.New("第三方API响应解析失败: " + err.Error()) - } - return - } - - // 内部API:保持原有逻辑,先解析为DefaultHandlerResponse,再提取data字段 + // 统一处理内部API响应格式:{code:200,message:"",data:{...}} resultStrut := &ghttp.DefaultHandlerResponse{} if err = gconv.Struct(result, &resultStrut); err != nil { // 修复:增加err检查 diff --git a/ragflow/client.go b/ragflow/client.go index 502a8c2..f88b0ba 100644 --- a/ragflow/client.go +++ b/ragflow/client.go @@ -2,14 +2,15 @@ package ragflow import ( "context" + "encoding/json" "net/url" "strings" "sync" "sync/atomic" - "gitee.com/red-future---jilin-g/common/http" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/gclient" ) var ( @@ -119,21 +120,9 @@ func (r *CommonResponse) IsSuccess() bool { // request 发送 HTTP 请求 // // 为什么不使用 common/http 包: -// -// 1. common/http/http.go:61 会用内部请求的Authorization覆盖RAGFlow API key: -// Httpclient.SetHeader("Authorization", g.RequestFromCtx(ctx).GetHeader("Authorization")) -// 这会导致RAGFlow API认证失败,因为内部token不是RAGFlow的API key -// -// 2. common/http/http.go:69-74 强制解析为内部API响应格式(ghttp.DefaultHandlerResponse): -// resultStrut := &ghttp.DefaultHandlerResponse{} -// if err = gconv.Struct(result, &resultStrut); err != nil { -// err = errors.New(resultStrut.Message) -// } else if resultStrut.Code == 200 || resultStrut.Code == 0 { -// gconv.Struct(resultStrut.Data, target) -// } -// RAGFlow API返回格式与内部API不同,会导致解析失败 -// -// 因此直接使用 g.Client() 调用第三方API,避免上述问题 +// common/http包统一处理内部API响应格式(ghttp.DefaultHandlerResponse), +// RAGFlow API返回格式为{code,data,message}一层结构,与内部API不同。 +// 因此直接使用 g.Client() 调用第三方API,在此处理RAGFlow特有的响应格式。 func (c *Client) request(ctx context.Context, method, path string, body interface{}, result interface{}) (err error) { endpoint := c.getNextEndpoint() if endpoint == "" { @@ -142,26 +131,53 @@ func (c *Client) request(ctx context.Context, method, path string, body interfac fullURL := endpoint + path - // 使用common/http包 - var headers = make(map[string]string) - headers["Authorization"] = "Bearer " + c.APIKey - headers["Content-Type"] = "application/json" + // 创建HTTP客户端并设置RAGFlow专用请求头 + client := g.Client() + client.SetHeader("Authorization", "Bearer "+c.APIKey) + client.SetHeader("Content-Type", "application/json") + + // 发送HTTP请求(避免data展开导致的双重包装) + var response *gclient.Response switch method { case "GET": - err = http.Get(ctx, fullURL, headers, result, body) + if body != nil { + response, err = client.Get(ctx, fullURL, body) + } else { + response, err = client.Get(ctx, fullURL) + } case "POST": - err = http.Post(ctx, fullURL, headers, result, body) + if body != nil { + response, err = client.Post(ctx, fullURL, body) + } else { + response, err = client.Post(ctx, fullURL) + } case "PUT": - err = http.Put(ctx, fullURL, headers, result, body) + if body != nil { + response, err = client.Put(ctx, fullURL, body) + } else { + response, err = client.Put(ctx, fullURL) + } case "DELETE": if body != nil { - err = http.Delete(ctx, fullURL, headers, result, body) + response, err = client.Delete(ctx, fullURL, body) } else { - err = http.Delete(ctx, fullURL, headers, result) + response, err = client.Delete(ctx, fullURL) } default: return gerror.Newf("unsupported method: %s", method) } + + if err != nil { + return + } + defer response.Close() + + // RAGFlow API响应格式:{code,data,message}一层结构,直接解析 + responseBody := response.ReadAll() + if err = json.Unmarshal(responseBody, result); err != nil { + return gerror.Newf("RAGFlow响应解析失败: %v, 原始响应: %s", err, string(responseBody)) + } + return } From d02da6eec15863a1680c8f3a4bb45a4bd8475b9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 27 Jan 2026 08:39:05 +0800 Subject: [PATCH 193/225] .gitignore --- http/http.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/http/http.go b/http/http.go index 285ceec..dc7ab36 100644 --- a/http/http.go +++ b/http/http.go @@ -41,6 +41,10 @@ func RouteRegister(controllers []interface{}) { group.Middleware(jaeger.NewTracer) group.Bind(controller.OperationLog) }) + Httpserver.BindHandler("/uploadConfig", func(r *ghttp.Request) { + config := g.Config().MustGet(r.GetCtx(), "upload") + r.Response.WriteJsonExit(config) + }) re := regexp.MustCompile("[A-Z]") for _, t := range controllers { sName := reflect.ValueOf(t).Elem().Type().Name() From 67fcfd2514722a97227846acfd89e490467d3ada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 27 Jan 2026 09:34:55 +0800 Subject: [PATCH 194/225] .gitignore --- http/http.go | 1 + 1 file changed, 1 insertion(+) diff --git a/http/http.go b/http/http.go index dc7ab36..4555b47 100644 --- a/http/http.go +++ b/http/http.go @@ -36,6 +36,7 @@ func init() { Httpserver.BindMiddlewareDefault(ghttp.MiddlewareHandlerResponse) Httpclient.SetDiscovery(gsvc.GetRegistry()) } + func RouteRegister(controllers []interface{}) { Httpserver.Group("/log", func(group *ghttp.RouterGroup) { group.Middleware(jaeger.NewTracer) From 1fcaeae3b0ccbbcbb73d13078f5cb42c066b96e4 Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Wed, 28 Jan 2026 10:17:03 +0800 Subject: [PATCH 195/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0redis=E8=B0=83?= =?UTF-8?q?=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mongo/mongo.go | 22 +++++++++++----------- redis/redis.go | 17 ++++++++++------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/mongo/mongo.go b/mongo/mongo.go index b4d2aac..b0426d7 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -123,7 +123,7 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( redisKey := fmt.Sprintf(redis.Count, user.TenantId, collection, filterKey) if !m.noCache { var resultStr *gvar.Var - resultStr, err = redis.RedisClient.Get(ctx, redisKey) + resultStr, err = redis.RedisClient().Get(ctx, redisKey) if err != nil { return } @@ -138,7 +138,7 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( } count, err = db.Collection(collection).CountDocuments(ctx, filter) if !m.noCache { - err = redis.RedisClient.SetEX(ctx, redisKey, count, int64(time.Hour)) + err = redis.RedisClient().SetEX(ctx, redisKey, count, int64(time.Hour)) if err != nil { return } @@ -169,7 +169,7 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c redisKey := fmt.Sprintf(redis.List, user.TenantId, collection, filterKey, optionsKey) if !m.noCache { var resultStr *gvar.Var - resultStr, err = redis.RedisClient.Get(ctx, redisKey) + resultStr, err = redis.RedisClient().Get(ctx, redisKey) if err != nil { return } @@ -229,7 +229,7 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c return } if !m.noCache { - err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) + err = redis.RedisClient().SetEX(ctx, redisKey, result, int64(time.Hour)) if err != nil { return } @@ -261,7 +261,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} redisKey := fmt.Sprintf(redis.One, user.TenantId, collection, filterKey) if !m.noCache { var resultStr *gvar.Var - resultStr, err = redis.RedisClient.Get(ctx, redisKey) + resultStr, err = redis.RedisClient().Get(ctx, redisKey) if err != nil { return } @@ -283,7 +283,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} err = nil } if !m.noCache { - err = redis.RedisClient.SetEX(ctx, redisKey, result, int64(time.Hour)) + err = redis.RedisClient().SetEX(ctx, redisKey, result, int64(time.Hour)) if err != nil { return err } @@ -304,23 +304,23 @@ func (m *MongoDB) getDeletedData(ctx context.Context, filter bson.M, collection func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collection string) (err error) { listKeys := fmt.Sprintf(redis.CleanList, tenantId, collection) - keys, err := redis.RedisClient.Keys(ctx, listKeys) + keys, err := redis.RedisClient().Keys(ctx, listKeys) if err != nil { return } for _, key := range keys { - _, err = redis.RedisClient.Del(ctx, key) + _, err = redis.RedisClient().Del(ctx, key) if err != nil { return } } countKeys := fmt.Sprintf(redis.CleanCount, tenantId, collection) - keys, err = redis.RedisClient.Keys(ctx, countKeys) + keys, err = redis.RedisClient().Keys(ctx, countKeys) if err != nil { return } for _, key := range keys { - _, err = redis.RedisClient.Del(ctx, key) + _, err = redis.RedisClient().Del(ctx, key) if err != nil { return } @@ -329,7 +329,7 @@ func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interf delete(filter, "tenantId") filterKey := fmt.Sprintf("%+v", filter) oneKey := fmt.Sprintf(redis.One, tenantId, collection, filterKey) - _, err = redis.RedisClient.Del(ctx, oneKey) + _, err = redis.RedisClient().Del(ctx, oneKey) if err != nil { return } diff --git a/redis/redis.go b/redis/redis.go index b4556a6..7b215d0 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -33,8 +33,10 @@ func GetRedisClient() *gredis.Redis { return getClient() } -// RedisClient 导出的 Redis 客户端(供 mongo.go 使用,兼容旧代码) -var RedisClient = getClient() +// RedisClient 获取 Redis 客户端(函数式,确保单例正确初始化) +func RedisClient() *gredis.Redis { + return getClient() +} // Lock 分布式锁 func Lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context) error) (success bool, err error) { @@ -44,7 +46,8 @@ LOOP: return false, errors.New("锁重试次数耗尽") } limit-- - if val, err := RedisClient.Set(ctx, key, true, gredis.SetOption{ + client := getClient() + if val, err := client.Set(ctx, key, true, gredis.SetOption{ TTLOption: gredis.TTLOption{ EX: &expireSeconds, }, @@ -53,11 +56,11 @@ LOOP: return false, err } else { if val.Bool() { - defer func(RedisClient *gredis.Redis, ctx context.Context, key string) { - if _, err = RedisClient.Del(ctx, key); err != nil { - glog.Errorf(ctx, "RedisClient.Del error: %v", err) + defer func(client *gredis.Redis, ctx context.Context, key string) { + if _, err = client.Del(ctx, key); err != nil { + glog.Errorf(ctx, "redis client Del error: %v", err) } - }(RedisClient, ctx, key) + }(client, ctx, key) if err = fn(ctx); err != nil { return false, err } From 65b231b88f6fcbe796510c73b5c9c1ad1d6a374d Mon Sep 17 00:00:00 2001 From: Cold <16419454+cold502@user.noreply.gitee.com> Date: Wed, 28 Jan 2026 15:27:32 +0800 Subject: [PATCH 196/225] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=AF=B9gateway?= =?UTF-8?q?=E7=9A=84=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- redis/redis.go | 70 +++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/redis/redis.go b/redis/redis.go index 7b215d0..d8dd359 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -147,7 +147,7 @@ type StreamMessage struct { // 使用 gredis Do() 方法执行 XGROUP CREATE 命令 func InitStreamGroup(ctx context.Context, streamKey, groupName string) error { // XGROUP CREATE streamKey groupName 0 MKSTREAM - _, err := redisClient.Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") + _, err := getClient().Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") if err != nil { // 如果组已存在,忽略错误 errStr := err.Error() @@ -173,7 +173,7 @@ func AddToStream(ctx context.Context, streamKey string, msg interface{}) (messag args = append(args, key, val) } - result, err := redisClient.Do(ctx, "XADD", args...) + result, err := getClient().Do(ctx, "XADD", args...) if err != nil { return } @@ -186,7 +186,7 @@ func AddToStream(ctx context.Context, streamKey string, msg interface{}) (messag // XGROUP CREATE streamKey groupName 0 MKSTREAM // 使用0作为起始ID,从Stream开头读取所有未消费消息 func CreateConsumerGroup(ctx context.Context, streamKey, groupName string) error { - _, err := redisClient.Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") + _, err := getClient().Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") return err } @@ -202,7 +202,7 @@ func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName stri RECONNECT: // 先尝试读取pending消息(ID=0),处理积压 - result, err := redisClient.Do(execCtx, + result, err := getClient().Do(execCtx, "XREADGROUP", "GROUP", groupName, consumerName, "COUNT", count, "BLOCK", 0, // 不阻塞,立即返回 @@ -232,7 +232,7 @@ RECONNECT: // 如果没有pending消息,读取新消息 if !hasPending { - result, err = redisClient.Do(execCtx, + result, err = getClient().Do(execCtx, "XREADGROUP", "GROUP", groupName, consumerName, "COUNT", count, "BLOCK", blockMs, @@ -354,7 +354,7 @@ func AckMessage(ctx context.Context, streamKey, groupName string, messageIDs ... args = append(args, id) } - _, err := redisClient.Do(ctx, "XACK", args...) + _, err := getClient().Do(ctx, "XACK", args...) return err } @@ -362,7 +362,7 @@ func AckMessage(ctx context.Context, streamKey, groupName string, messageIDs ... // 使用 gredis Do() 方法执行 XLEN 命令 func GetStreamLength(ctx context.Context, streamKey string) (int64, error) { // XLEN streamKey - result, err := redisClient.Do(ctx, "XLEN", streamKey) + result, err := getClient().Do(ctx, "XLEN", streamKey) if err != nil { return 0, err } @@ -383,7 +383,7 @@ type PendingMessage struct { // 使用 gredis Do() 方法执行 XPENDING 命令 func GetPendingMessages(ctx context.Context, streamKey, groupName string, start, end string, count int64) ([]PendingMessage, error) { // XPENDING streamKey groupName start end count - result, err := redisClient.Do(ctx, "XPENDING", streamKey, groupName, start, end, count) + result, err := getClient().Do(ctx, "XPENDING", streamKey, groupName, start, end, count) if err != nil { return nil, err } @@ -425,7 +425,7 @@ func ClaimPendingMessage(ctx context.Context, streamKey, groupName, consumerName args = append(args, id) } - result, err := redisClient.Do(ctx, "XCLAIM", args...) + result, err := getClient().Do(ctx, "XCLAIM", args...) if err != nil { return nil, err } @@ -479,7 +479,7 @@ func SetSessionLastActive(ctx context.Context, userId string) error { timestamp := gtime.Now().Timestamp() // SETEX key 7200 value (7200秒 = 2小时) - _, err := redisClient.Do(ctx, "SETEX", key, 7200, timestamp) + _, err := getClient().Do(ctx, "SETEX", key, 7200, timestamp) return err } @@ -487,7 +487,7 @@ func SetSessionLastActive(ctx context.Context, userId string) error { // 使用 gredis Get 方法 func GetSessionLastActive(ctx context.Context, userId string) (int64, error) { key := SessionLastActiveKeyPrefix + userId + ":last_active" - result, err := redisClient.Get(ctx, key) + result, err := getClient().Get(ctx, key) if err != nil { return 0, err } @@ -531,7 +531,7 @@ func IsUserActive(ctx context.Context, userId string, seconds int64) (bool, erro // windowSeconds: 时间窗口(秒) func IncrRateLimit(ctx context.Context, key string, windowSeconds int64) (count int64, err error) { fullKey := RateLimitKeyPrefix + key - result, err := redisClient.Do(ctx, "INCR", fullKey) + result, err := getClient().Do(ctx, "INCR", fullKey) if err != nil { return } @@ -539,7 +539,7 @@ func IncrRateLimit(ctx context.Context, key string, windowSeconds int64) (count // 首次设置过期时间 if count == 1 { - redisClient.Do(ctx, "EXPIRE", fullKey, windowSeconds) + getClient().Do(ctx, "EXPIRE", fullKey, windowSeconds) } return } @@ -547,7 +547,7 @@ func IncrRateLimit(ctx context.Context, key string, windowSeconds int64) (count // GetRateLimit 获取当前限流计数 func GetRateLimit(ctx context.Context, key string) (count int64, err error) { fullKey := RateLimitKeyPrefix + key - result, err := redisClient.Get(ctx, fullKey) + result, err := getClient().Get(ctx, fullKey) if err != nil { return } @@ -562,14 +562,14 @@ func GetRateLimit(ctx context.Context, key string) (count int64, err error) { func SetSessionCache(ctx context.Context, tenantId, userId, sessionId string) error { key := SessionLastActiveKeyPrefix + tenantId + ":" + userId + ":session_id" // SETEX key 7200 value (7200秒 = 2小时,与last_active保持一致) - _, err := redisClient.Do(ctx, "SETEX", key, 7200, sessionId) + _, err := getClient().Do(ctx, "SETEX", key, 7200, sessionId) return err } // GetSessionCache 获取缓存的 RAGFlow Session ID(租户+用户隔离) func GetSessionCache(ctx context.Context, tenantId, userId string) (string, error) { key := SessionLastActiveKeyPrefix + tenantId + ":" + userId + ":session_id" - result, err := redisClient.Get(ctx, key) + result, err := getClient().Get(ctx, key) if err != nil { return "", err } @@ -584,7 +584,7 @@ func GetSessionCache(ctx context.Context, tenantId, userId string) (string, erro // DelSessionCache 删除缓存的 RAGFlow Session ID(归档时调用,租户+用户隔离) func DelSessionCache(ctx context.Context, tenantId, userId string) error { key := SessionLastActiveKeyPrefix + tenantId + ":" + userId + ":session_id" - _, err := redisClient.Del(ctx, key) + _, err := getClient().Del(ctx, key) return err } @@ -594,7 +594,7 @@ func DelSessionCache(ctx context.Context, tenantId, userId string) error { // 返回 true 表示获取成功,false 表示锁已被其他节点持有 func TryLock(ctx context.Context, key string, expireSeconds int) bool { // SET key value NX EX expireSeconds - result, err := redisClient.Do(ctx, "SET", key, gtime.Now().String(), "NX", "EX", expireSeconds) + result, err := getClient().Do(ctx, "SET", key, gtime.Now().String(), "NX", "EX", expireSeconds) if err != nil { glog.Errorf(ctx, "获取分布式锁失败: %v", err) return false @@ -604,7 +604,7 @@ func TryLock(ctx context.Context, key string, expireSeconds int) bool { // Unlock 释放分布式锁 func Unlock(ctx context.Context, key string) { - if _, err := redisClient.Del(ctx, key); err != nil { + if _, err := getClient().Del(ctx, key); err != nil { glog.Errorf(ctx, "释放分布式锁失败: %v", err) } } @@ -629,7 +629,7 @@ type UserState struct { // GetUserState 获取用户状态(阶段+计数) func GetUserState(ctx context.Context, userId, platform string) (state *UserState, err error) { key := UserStateKeyPrefix + userId + "_" + platform - result, err := redisClient.Do(ctx, "HGETALL", key) + result, err := getClient().Do(ctx, "HGETALL", key) if err != nil { return } @@ -654,52 +654,52 @@ func GetUserState(ctx context.Context, userId, platform string) (state *UserStat // SetUserStage 设置用户阶段,并刷新过期时间 func SetUserStage(ctx context.Context, userId, platform string, stage int) error { key := UserStateKeyPrefix + userId + "_" + platform - _, err := redisClient.Do(ctx, "HSET", key, "stage", stage) + _, err := getClient().Do(ctx, "HSET", key, "stage", stage) if err != nil { return err } - _, err = redisClient.Do(ctx, "EXPIRE", key, UserStateExpireSeconds) + _, err = getClient().Do(ctx, "EXPIRE", key, UserStateExpireSeconds) return err } // SetUserAccountName 设置用户对应的客服账号名称,并刷新过期时间 func SetUserAccountName(ctx context.Context, userId, platform, accountName string) error { key := UserStateKeyPrefix + userId + "_" + platform - _, err := redisClient.Do(ctx, "HSET", key, "accountName", accountName) + _, err := getClient().Do(ctx, "HSET", key, "accountName", accountName) if err != nil { return err } - _, err = redisClient.Do(ctx, "EXPIRE", key, UserStateExpireSeconds) + _, err = getClient().Do(ctx, "EXPIRE", key, UserStateExpireSeconds) return err } // SetUserDirection 设置用户选择的咨询方向,并刷新过期时间 func SetUserDirection(ctx context.Context, userId, platform, direction string) error { key := UserStateKeyPrefix + userId + "_" + platform - _, err := redisClient.Do(ctx, "HSET", key, "direction", direction) + _, err := getClient().Do(ctx, "HSET", key, "direction", direction) if err != nil { return err } - _, err = redisClient.Do(ctx, "EXPIRE", key, UserStateExpireSeconds) + _, err = getClient().Do(ctx, "EXPIRE", key, UserStateExpireSeconds) return err } // IncrUserCount 增加用户对话计数,返回当前轮数,并刷新过期时间 func IncrUserCount(ctx context.Context, userId, platform string) (count int64, err error) { key := UserStateKeyPrefix + userId + "_" + platform - result, err := redisClient.Do(ctx, "HINCRBY", key, "count", 1) + result, err := getClient().Do(ctx, "HINCRBY", key, "count", 1) if err != nil { return } count = result.Int64() - _, err = redisClient.Do(ctx, "EXPIRE", key, UserStateExpireSeconds) + _, err = getClient().Do(ctx, "EXPIRE", key, UserStateExpireSeconds) return } // ResetUserState 重置用户状态(归档时调用) func ResetUserState(ctx context.Context, userId, platform string) error { key := UserStateKeyPrefix + userId + "_" + platform - _, err := redisClient.Del(ctx, key) + _, err := getClient().Del(ctx, key) return err } @@ -715,18 +715,18 @@ const ( // CacheConversation 缓存单条对话到Redis List(按sessionId存储) func CacheConversation(ctx context.Context, sessionId string, data []byte) error { key := ConversationCacheKeyPrefix + sessionId - _, err := redisClient.Do(ctx, "RPUSH", key, string(data)) + _, err := getClient().Do(ctx, "RPUSH", key, string(data)) if err != nil { return err } - _, err = redisClient.Do(ctx, "EXPIRE", key, ConversationCacheExpireSeconds) + _, err = getClient().Do(ctx, "EXPIRE", key, ConversationCacheExpireSeconds) return err } // GetCachedConversations 获取缓存的对话列表并清空(按sessionId查询) func GetCachedConversations(ctx context.Context, sessionId string) (list []string, err error) { key := ConversationCacheKeyPrefix + sessionId - result, err := redisClient.Do(ctx, "LRANGE", key, 0, -1) + result, err := getClient().Do(ctx, "LRANGE", key, 0, -1) if err != nil { return } @@ -735,14 +735,14 @@ func GetCachedConversations(ctx context.Context, sessionId string) (list []strin } list = result.Strings() // 清空缓存 - redisClient.Del(ctx, key) + getClient().Del(ctx, key) return } // GetCachedConversationCount 获取缓存的对话数量(按sessionId查询) func GetCachedConversationCount(ctx context.Context, sessionId string) (count int64, err error) { key := ConversationCacheKeyPrefix + sessionId - result, err := redisClient.Do(ctx, "LLEN", key) + result, err := getClient().Do(ctx, "LLEN", key) if err != nil { return } @@ -752,7 +752,7 @@ func GetCachedConversationCount(ctx context.Context, sessionId string) (count in // ClearCachedConversations 清空对话缓存(归档时调用,按sessionId) func ClearCachedConversations(ctx context.Context, sessionId string) error { key := ConversationCacheKeyPrefix + sessionId - _, err := redisClient.Del(ctx, key) + _, err := getClient().Del(ctx, key) return err } From a8993de6d546de1be8d40bb19e4c250de1be6c4d Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Thu, 29 Jan 2026 13:55:32 +0800 Subject: [PATCH 197/225] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=B6=88=E6=81=AF?= =?UTF-8?q?=E9=98=9F=E5=88=97=E6=A8=A1=E5=9D=97=EF=BC=8C=E7=BB=9F=E4=B8=80?= =?UTF-8?q?Redis/RabbitMQ/NATS=E5=AE=A2=E6=88=B7=E7=AB=AF=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=EF=BC=8C=E6=96=B0=E5=A2=9E=E6=B6=88=E6=81=AF=E4=BB=A3?= =?UTF-8?q?=E7=90=86=E6=8A=BD=E8=B1=A1=E5=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- message/message.go | 5 - message/msg_queue.go | 152 ++++++ nats/connection.go => message/nats_client.go | 93 ++-- {nats => message}/nats_rpc.go | 93 +++- message/rabbit.go | 6 +- .../client.go => message/rabbitmq_client.go | 28 +- message/redis.go | 111 +++-- message/redis_client.go | 468 ++++++++++++++++++ 8 files changed, 844 insertions(+), 112 deletions(-) create mode 100644 message/msg_queue.go rename nats/connection.go => message/nats_client.go (79%) rename {nats => message}/nats_rpc.go (89%) rename rabbitmq/client.go => message/rabbitmq_client.go (93%) create mode 100644 message/redis_client.go diff --git a/message/message.go b/message/message.go index e2c72c1..321fc5d 100644 --- a/message/message.go +++ b/message/message.go @@ -7,11 +7,6 @@ import ( "github.com/gogf/gf/v2/errors/gerror" ) -// GetRedisClient 获取 Redis 客户端(供外部使用) -func GetRedisClient() *gredis.Redis { - return getRedisClient() -} - func GetRedisClientTest(name string) *gredis.Redis { return getRedisClientTest(name) } diff --git a/message/msg_queue.go b/message/msg_queue.go new file mode 100644 index 0000000..be1f2da --- /dev/null +++ b/message/msg_queue.go @@ -0,0 +1,152 @@ +package message + +import ( + "context" + "fmt" +) + +type RedisConfig struct { + // Stream 名称 + Stream string + + // 消费者组名称 + Group string + + // 消费者名称 + Consumer string + + // 每次消费数量 + Count int64 + + // 是否自动 ACK + AutoAck bool + + // 处理函数 + HandleFunc func(ctx context.Context, message map[string]interface{}) error +} + +// RabbitMQConfig RabbitMQ 队列配置 +type RabbitMQConfig struct { + Mode string + Exchange string + Topic string + DelayMessage bool + + // 队列名称(必需) + Name string + + // 实际队列名(用于绑定) + Queue string + + // 是否持久化 + Durable bool + + // QoS 预取数量(每次推送的消息数量,默认10) + PrefetchCount int + + // 最大重试次数(默认3) + MaxRetry int + + // 是否自动 ACK + AutoAck bool + + // 处理函数 + HandleFunc func(ctx context.Context, message map[string]interface{}) error +} + +// NATSConfig NATS 队列配置 +type NATSConfig struct { + DelayMessage bool + // Stream 名称 + Stream string + + // 消费者名称 + Consumer string + + // 是否持久化 + Durable bool + + // 副本数 + Replicas int + // QoS 预取数量(每次推送的消息数量,默认10) + PrefetchCount int + + // 是否自动 ACK + AutoAck bool + + // 处理函数 + HandleFunc func(ctx context.Context, message map[string]interface{}) error +} + +// messageBroker 消息代理接口 +type messageBroker interface { + // StreamGroup 创建消费组(支持单个配置或批量配置) + streamGroup(ctx context.Context, configs ...interface{}) error + + // Publish 发布消息(支持单个配置或批量配置) + publish(ctx context.Context, config interface{}, data interface{}) error + + // PublishDelayed 发布延迟消息(支持单个配置或批量配置) + publishDelayed(ctx context.Context, config interface{}, data interface{}, delay int) error + + // Subscribe 订阅消息(支持单个配置或批量配置) + subscribe(ctx context.Context, configs ...interface{}) error +} + +type messageClientType string + +const ( + ClientTypeRedis messageClientType = "redis" + ClientTypeRabbitMQ messageClientType = "rabbitmq" + ClientTypeNATS messageClientType = "nats" +) + +// newMessageBroker 创建消息代理实例 +func newMessageBroker(ctx context.Context, clientType messageClientType) (messageBroker, error) { + switch clientType { + case ClientTypeRedis: + return &redisMessageClient{clientType: clientType}, nil + case ClientTypeRabbitMQ: + return &rabbitMQMessageClient{clientType: clientType}, nil + case ClientTypeNATS: + return &natsMessageClient{clientType: clientType}, nil + default: + return nil, fmt.Errorf("unknown client type: %s", clientType) + } +} + +// StreamGroup 直接创建消费组 +func StreamGroup(ctx context.Context, clientType messageClientType, configs ...interface{}) error { + broker, err := newMessageBroker(ctx, clientType) + if err != nil { + return err + } + return broker.streamGroup(ctx, configs...) +} + +// Publish 直接发布消息 +func Publish(ctx context.Context, clientType messageClientType, config interface{}, data interface{}) error { + broker, err := newMessageBroker(ctx, clientType) + if err != nil { + return err + } + return broker.publish(ctx, config, data) +} + +// PublishDelayed 直接发布延迟消息 +func PublishDelayed(ctx context.Context, clientType messageClientType, config interface{}, data interface{}, delay int) error { + broker, err := newMessageBroker(ctx, clientType) + if err != nil { + return err + } + return broker.publishDelayed(ctx, config, data, delay) +} + +// Subscribe 直接订阅消息 +func Subscribe(ctx context.Context, clientType messageClientType, configs ...interface{}) error { + broker, err := newMessageBroker(ctx, clientType) + if err != nil { + return err + } + return broker.subscribe(ctx, configs...) +} diff --git a/nats/connection.go b/message/nats_client.go similarity index 79% rename from nats/connection.go rename to message/nats_client.go index eb8078a..470bef7 100644 --- a/nats/connection.go +++ b/message/nats_client.go @@ -1,4 +1,4 @@ -package nats +package message import ( "context" @@ -16,7 +16,7 @@ var ( nc *nats.Conn js jetstream.JetStream inited bool - mu sync.RWMutex + natsMu sync.RWMutex natsURL string healthCtx context.Context healthCancel context.CancelFunc @@ -24,15 +24,15 @@ var ( reconnectChan chan struct{} // 连接状态变化监听器 - connStateListeners []ConnStateListener + connStateListeners []connStateListener connListenersMu sync.RWMutex // 监控指标 - metrics Metrics + metrics metricsCounter ) // Metrics 监控指标 -type Metrics struct { +type metricsCounter struct { PublishCount atomic.Int64 PublishError atomic.Int64 SubscribeCount atomic.Int64 @@ -43,33 +43,33 @@ type Metrics struct { } // ConnState 连接状态 -type ConnState int +type connState int const ( - ConnStateDisconnected ConnState = iota - ConnStateConnecting - ConnStateConnected - ConnStateReconnecting - ConnStateClosed + connStateDisconnected connState = iota + connStateConnecting + connStateConnected + connStateReconnecting + connStateClosed ) // ConnStateListener 连接状态监听器 -type ConnStateListener func(state ConnState, err error) +type connStateListener func(state connState, err error) // GetMetrics 获取监控指标 -func GetMetrics() Metrics { +func getMetrics() metricsCounter { return metrics } -// RegisterConnStateListener 注册连接状态监听器 -func RegisterConnStateListener(listener ConnStateListener) { +// registerConnStateListener 注册连接状态监听器 +func registerConnStateListener(listener connStateListener) { connListenersMu.Lock() defer connListenersMu.Unlock() connStateListeners = append(connStateListeners, listener) } -// UnregisterConnStateListener 取消注册连接状态监听器 -func UnregisterConnStateListener(listener ConnStateListener) { +// unregisterConnStateListener 取消注册连接状态监听器 +func unregisterConnStateListener(listener connStateListener) { connListenersMu.Lock() defer connListenersMu.Unlock() for i, l := range connStateListeners { @@ -81,9 +81,9 @@ func UnregisterConnStateListener(listener ConnStateListener) { } // notifyConnState 通知所有监听器连接状态变化 -func notifyConnState(state ConnState, err error) { +func notifyConnState(state connState, err error) { connListenersMu.RLock() - listeners := make([]ConnStateListener, len(connStateListeners)) + listeners := make([]connStateListener, len(connStateListeners)) copy(listeners, connStateListeners) connListenersMu.RUnlock() @@ -119,17 +119,17 @@ func init() { // initConnection 初始化连接 func initConnection() { ctx := context.Background() - notifyConnState(ConnStateConnecting, nil) + notifyConnState(connStateConnecting, nil) if err := connect(ctx); err != nil { g.Log().Errorf(ctx, "NATS 初始连接失败: %v", err) - notifyConnState(ConnStateDisconnected, err) + notifyConnState(connStateDisconnected, err) } } // connect 建立 NATS 连接 func connect(ctx context.Context) error { - mu.Lock() - defer mu.Unlock() + natsMu.Lock() + defer natsMu.Unlock() if nc != nil && !nc.IsClosed() { nc.Close() @@ -152,7 +152,7 @@ func connect(ctx context.Context) error { } // 通知重连成功 - notifyConnState(ConnStateConnected, nil) + notifyConnState(connStateConnected, nil) // 使用非阻塞发送避免阻塞 select { @@ -164,12 +164,12 @@ func connect(ctx context.Context) error { nats.DisconnectErrHandler(func(nc *nats.Conn, err error) { g.Log().Warningf(ctx, "⚠️ NATS 连接断开: %v, 准备重连...", err) connected = false - notifyConnState(ConnStateReconnecting, err) + notifyConnState(connStateReconnecting, err) }), nats.ClosedHandler(func(nc *nats.Conn) { g.Log().Infof(ctx, "NATS 连接已关闭: %s", nc.ConnectedUrl()) connected = false - notifyConnState(ConnStateClosed, nil) + notifyConnState(connStateClosed, nil) }), nats.ErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { g.Log().Errorf(ctx, "NATS 错误: %v", err) @@ -186,7 +186,7 @@ func connect(ctx context.Context) error { if nc.Status() != nats.CONNECTED { select { case <-time.After(5 * time.Second): - notifyConnState(ConnStateDisconnected, fmt.Errorf("连接超时")) + notifyConnState(connStateDisconnected, fmt.Errorf("连接超时")) return fmt.Errorf("NATS 连接超时") case <-nc.StatusChanged(nats.CONNECTED): } @@ -201,7 +201,7 @@ func connect(ctx context.Context) error { connected = true inited = true g.Log().Infof(ctx, "✅ NATS 连接成功: %s", nc.ConnectedUrl()) - notifyConnState(ConnStateConnected, nil) + notifyConnState(connStateConnected, nil) return nil } @@ -215,10 +215,10 @@ func healthCheck() { case <-healthCtx.Done(): return case <-ticker.C: - mu.RLock() + natsMu.RLock() currentConnected := connected currentConn := nc - mu.RUnlock() + natsMu.RUnlock() if !currentConnected || currentConn == nil || currentConn.IsClosed() { // 仅记录日志,不尝试重连(NATS 已有自动重连机制) @@ -233,38 +233,33 @@ func healthCheck() { // checkConnected 检查连接状态 func checkConnected() bool { - mu.RLock() - defer mu.RUnlock() + natsMu.RLock() + defer natsMu.RUnlock() return connected && nc != nil && !nc.IsClosed() } -// IsConnected 检查 NATS 是否已连接 -func IsConnected() bool { - return checkConnected() -} - -// GetConnState 获取当前连接状态 -func GetConnState() ConnState { - mu.RLock() - defer mu.RUnlock() +// getConnState 获取当前连接状态 +func getConnState() connState { + natsMu.RLock() + defer natsMu.RUnlock() if nc == nil { - return ConnStateDisconnected + return connStateDisconnected } if nc.IsClosed() { - return ConnStateClosed + return connStateClosed } if connected { - return ConnStateConnected + return connStateConnected } - return ConnStateDisconnected + return connStateDisconnected } -// Shutdown 优雅关闭:自动注销所有已注册的服务并关闭 NATS 连接 -func Shutdown() error { +// shutdown 优雅关闭:自动注销所有已注册的服务并关闭 NATS 连接 +func shutdown() error { ctx := context.Background() g.Log().Info(ctx, "开始优雅关闭 NATS RPC 服务...") @@ -299,8 +294,8 @@ func Shutdown() error { g.Log().Infof(ctx, "已注销 %d 个单实例服务和 %d 个队列服务", singleServiceCount, queueServiceCount) - mu.Lock() - defer mu.Unlock() + natsMu.Lock() + defer natsMu.Unlock() // 停止健康检查协程 if healthCancel != nil { diff --git a/nats/nats_rpc.go b/message/nats_rpc.go similarity index 89% rename from nats/nats_rpc.go rename to message/nats_rpc.go index 8cbb8ba..8644734 100644 --- a/nats/nats_rpc.go +++ b/message/nats_rpc.go @@ -1,4 +1,4 @@ -package nats +package message import ( "context" @@ -7,6 +7,7 @@ import ( "fmt" "github.com/gogf/gf/v2/frame/g" "github.com/nats-io/nats.go" + "go.opentelemetry.io/otel/trace" "reflect" "sync" ) @@ -509,12 +510,12 @@ func AutoRegisterServices(ctx context.Context, serviceInstances map[string]inter return fmt.Errorf("未能注册任何服务") } // 设置取消监听器(监听基于 TraceID 的取消请求) - //if _, err := setupCancelListener(ctx); err != nil { - // g.Log().Errorf(ctx, "设置取消监听器失败: %v", err) - //} else { - // g.Log().Infof(ctx, "✅ 取消监听器已自动设置") - //} - //g.Log().Infof(ctx, "✅ 共自动注册了 %d 个服务", totalRegistered) + if _, err := setupCancelListener(ctx); err != nil { + g.Log().Errorf(ctx, "设置取消监听器失败: %v", err) + } else { + g.Log().Infof(ctx, "✅ 取消监听器已自动设置") + } + g.Log().Infof(ctx, "✅ 共自动注册了 %d 个服务", totalRegistered) return nil } @@ -671,3 +672,81 @@ func registerService(service interface{}, serviceNamePrefix string, options ...R g.Log().Infof(context.Background(), "✅ Service %v 共注册了 %d 个 RPC 方法", serviceNamePrefix, registeredCount) return nil } + +// ============ 上下文元数据工具函数 ============ +// 以下函数用于在 context 和 NATS 消息头之间互转元数据 + +// 定义常见的上下文元数据 key +const ( + TraceIDKey = "trace_id" + TokenKey = "token" +) + +func getTraceID(ctx context.Context) (traceID string, err error) { + // 提取 traceId:首先尝试从 OpenTelemetry Span 中提取,从 context 中提取 TraceID + span := trace.SpanFromContext(ctx) + if span != nil && span.SpanContext().HasTraceID() { + traceID = span.SpanContext().TraceID().String() + } else if tid := ctx.Value(TraceIDKey); tid != nil { + traceID = fmt.Sprintf("%v", tid) + } + if traceID == "" { + return traceID, fmt.Errorf("context 中没有 TraceID") + } + return +} + +// contextToHeaders 将 context 中的元数据转换为 NATS 消息头 +// 支持提取 user_id、tenant_id、trace_id、token 等常见字段 +func contextToHeaders(ctx context.Context) (nats.Header, error) { + headers := make(nats.Header) + + // 提取 traceId:首先尝试从 OpenTelemetry Span 中提取 + if traceID, err := getTraceID(ctx); err != nil { + return headers, err + } else { + headers.Set(TraceIDKey, traceID) + } + + // 提取 token(优先级:context value > HTTP Authorization header) + token := "" + if t := ctx.Value(TokenKey); t != nil { + token = fmt.Sprintf("%v", t) + } else if r := g.RequestFromCtx(ctx); r != nil { + // 从 HTTP 请求的 Authorization header 中提取 token + auth := r.GetHeader("Authorization") + if auth != "" { + // 移除 "Bearer " 前缀 + if len(auth) > 7 && auth[:7] == "Bearer " { + token = auth[7:] + } else { + token = auth + } + } + } + if token != "" { + headers.Set(TokenKey, token) + } + + return headers, nil +} + +// headersToContext 从 NATS 消息头重建 context +// 支持还原 user_id、tenant_id、trace_id、token 等字段 +func headersToContext(ctx context.Context, headers nats.Header) context.Context { + if headers == nil { + return ctx + } + + // 恢复 trace_id + if traceID := headers.Get(TraceIDKey); traceID != "" { + ctx = context.WithValue(ctx, TraceIDKey, traceID) + } + + // 恢复 token + if token := headers.Get(TokenKey); token != "" { + ctx = context.WithValue(ctx, TokenKey, token) + } + + return ctx +} diff --git a/message/rabbit.go b/message/rabbit.go index bd6e010..1ef01fb 100644 --- a/message/rabbit.go +++ b/message/rabbit.go @@ -22,7 +22,7 @@ var ( ) // Config RabbitMQ 配置 -type RabbitMQConfig struct { +type RabbitMQConfig1 struct { Host string Port int Username string @@ -31,8 +31,8 @@ type RabbitMQConfig struct { } // rabbitMQConfig 默认配置 -func getRabbitMQConfig() *RabbitMQConfig { - return &RabbitMQConfig{ +func getRabbitMQConfig() *RabbitMQConfig1 { + return &RabbitMQConfig1{ Host: g.Cfg().MustGet(context.Background(), "rabbitmq.host").String(), Port: g.Cfg().MustGet(context.Background(), "rabbitmq.port").Int(), Username: g.Cfg().MustGet(context.Background(), "rabbitmq.username").String(), diff --git a/rabbitmq/client.go b/message/rabbitmq_client.go similarity index 93% rename from rabbitmq/client.go rename to message/rabbitmq_client.go index 68eda88..fc225f7 100644 --- a/rabbitmq/client.go +++ b/message/rabbitmq_client.go @@ -1,4 +1,4 @@ -package rabbitmq +package message import ( "context" @@ -14,8 +14,8 @@ import ( var ( conn *amqp.Connection channel *amqp.Channel - once sync.Once - mu sync.RWMutex + rabbitmqOnce sync.Once + rabbitmqMu sync.RWMutex closeWatcher chan struct{} // 用于停止监听 goroutine watcherStarted bool // 防止重复启动监听 ) @@ -32,7 +32,7 @@ type Config struct { // Init 初始化 RabbitMQ 连接 func Init(ctx context.Context, cfg *Config) error { var err error - once.Do(func() { + rabbitmqOnce.Do(func() { // 构建连接字符串 url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost @@ -80,8 +80,8 @@ func InitFromConfig(ctx context.Context) error { // GetChannel 获取 Channel func GetChannel() (*amqp.Channel, error) { - mu.RLock() - defer mu.RUnlock() + rabbitmqMu.RLock() + defer rabbitmqMu.RUnlock() if channel == nil || channel.IsClosed() { return nil, gerror.New("RabbitMQ Channel 未初始化或已关闭") @@ -92,8 +92,8 @@ func GetChannel() (*amqp.Channel, error) { // GetConnection 获取连接 func GetConnection() (*amqp.Connection, error) { - mu.RLock() - defer mu.RUnlock() + rabbitmqMu.RLock() + defer rabbitmqMu.RUnlock() if conn == nil || conn.IsClosed() { return nil, gerror.New("RabbitMQ 连接未初始化或已关闭") @@ -113,9 +113,9 @@ func handleConnectionClose(ctx context.Context) { default: } - mu.RLock() + rabbitmqMu.RLock() currentConn := conn - mu.RUnlock() + rabbitmqMu.RUnlock() if currentConn == nil { return @@ -141,8 +141,8 @@ func handleConnectionClose(ctx context.Context) { // reconnect 重新连接 func reconnect(ctx context.Context) { - mu.Lock() - defer mu.Unlock() + rabbitmqMu.Lock() + defer rabbitmqMu.Unlock() for i := 0; i < 10; i++ { time.Sleep(time.Duration(i+1) * time.Second) @@ -180,8 +180,8 @@ func reconnect(ctx context.Context) { // Close 关闭连接 func Close(ctx context.Context) (err error) { - mu.Lock() - defer mu.Unlock() + rabbitmqMu.Lock() + defer rabbitmqMu.Unlock() // 停止监听 goroutine if closeWatcher != nil { diff --git a/message/redis.go b/message/redis.go index 7d5c76b..67e88f1 100644 --- a/message/redis.go +++ b/message/redis.go @@ -3,6 +3,7 @@ package message import ( "context" "errors" + "fmt" "strings" "time" @@ -18,11 +19,6 @@ type StreamMessage struct { Values map[string]interface{} // 消息内容 } -// getClient 获取 Redis 客户端 -func getRedisClient() *gredis.Redis { - return g.Redis() -} - // getClient 获取 Redis 客户端 func getRedisClientTest(name string) *gredis.Redis { return g.Redis(name) @@ -47,46 +43,66 @@ func getRedisClientByDB(db int) *gredis.Redis { // lock 分布式锁 func lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context) error) (success bool, err error) { - limit := 3 -LOOP: - if limit < 0 { - return false, errors.New("锁重试次数耗尽") + ds, err := GetManager().GetDefaultDataSource() + if err != nil { + return false, fmt.Errorf("获取默认数据源失败: %w", err) } - limit-- - if val, err := getRedisClient().Set(ctx, key, true, gredis.SetOption{ - TTLOption: gredis.TTLOption{ - EX: &expireSeconds, - }, - NX: true, - }); err != nil { - return false, err - } else { - if val.Bool() { - defer func(RedisClient *gredis.Redis, ctx context.Context, key string) { - if _, err = RedisClient.Del(ctx, key); err != nil { - glog.Errorf(ctx, "RedisClient.Del error: %v", err) - } - }(getRedisClient(), ctx, key) - if err = fn(ctx); err != nil { - return false, err - } - return true, nil + + maxRetries := 3 + for i := 0; i < maxRetries; i++ { + if val, err := ds.Redis().Set(ctx, key, true, gredis.SetOption{ + TTLOption: gredis.TTLOption{ + EX: &expireSeconds, + }, + NX: true, + }); err != nil { + return false, err } else { - time.Sleep(time.Second) - goto LOOP + if val.Bool() { + defer func(redisClient *gredis.Redis, ctx context.Context, key string) { + if _, err = redisClient.Del(ctx, key); err != nil { + glog.Errorf(ctx, "RedisClient.Del error: %v", err) + } + }(ds.Redis(), ctx, key) + if err = fn(ctx); err != nil { + return false, err + } + return true, nil + } else { + // 检查上下文是否已取消 + if ctx.Err() != nil { + return false, ctx.Err() + } + // 非最后一次重试时才等待 + if i < maxRetries-1 { + time.Sleep(time.Second) + } + } } } + return false, errors.New("锁重试次数耗尽") } // publishToRedis 将消息添加到 Redis Stream func publishToRedis(ctx context.Context, streamKey string, msg interface{}) (messageID string, err error) { + ds, err := GetManager().GetDefaultDataSource() + if err != nil { + return "", fmt.Errorf("获取默认数据源失败: %w", err) + } + + if !ds.IsConnected() { + if err := ds.Reconnect(ctx); err != nil { + return "", fmt.Errorf("redis重连失败: %w", err) + } + } + values := gconv.Map(msg) args := make([]interface{}, 0, len(values)*2+2) args = append(args, streamKey, "*") for key, val := range values { args = append(args, key, val) } - result, err := getRedisClient().Do(ctx, "XADD", args...) + result, err := ds.Redis().Do(ctx, "XADD", args...) if err != nil { return } @@ -96,7 +112,18 @@ func publishToRedis(ctx context.Context, streamKey string, msg interface{}) (mes // initStreamGroup 初始化 Stream 和消费者组 func initStreamGroup(ctx context.Context, streamKey, groupName string) error { - _, err := getRedisClient().Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") + ds, err := GetManager().GetDefaultDataSource() + if err != nil { + return fmt.Errorf("获取默认数据源失败: %w", err) + } + + if !ds.IsConnected() { + if err := ds.Reconnect(ctx); err != nil { + return fmt.Errorf("redis重连失败: %w", err) + } + } + + _, err = ds.Redis().Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") if err != nil { // 如果组已存在,忽略错误 errStr := err.Error() @@ -113,6 +140,11 @@ func initStreamGroup(ctx context.Context, streamKey, groupName string) error { // readFromStream 从 Stream 读取消息 func readFromStream(ctx context.Context, msg QueueMessage) error { + ds, err := GetManager().GetDefaultDataSource() + if err != nil { + return fmt.Errorf("获取默认数据源失败: %w", err) + } + // 初始化 Stream 和消费者组 if err := initStreamGroup(ctx, msg.StreamKey, msg.GroupName); err != nil { return err @@ -120,7 +152,7 @@ func readFromStream(ctx context.Context, msg QueueMessage) error { go func() { RECONNECT: for { - result, err := getRedisClient().Do(ctx, "XREADGROUP", "GROUP", msg.GroupName, msg.ConsumerName, "COUNT", msg.BatchSize, "BLOCK", 0, "STREAMS", msg.StreamKey, ">") + result, err := ds.Redis().Do(ctx, "XREADGROUP", "GROUP", msg.GroupName, msg.ConsumerName, "COUNT", msg.BatchSize, "BLOCK", 0, "STREAMS", msg.StreamKey, ">") if err != nil { //select { //case <-ctx.Done(): @@ -222,11 +254,22 @@ func readFromStream(ctx context.Context, msg QueueMessage) error { // ackMessage 确认消息已处理 func ackMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error { + ds, err := GetManager().GetDefaultDataSource() + if err != nil { + return fmt.Errorf("获取默认数据源失败: %w", err) + } + + if !ds.IsConnected() { + if err := ds.Reconnect(ctx); err != nil { + return fmt.Errorf("redis重连失败: %w", err) + } + } + args := make([]interface{}, 0, len(messageIDs)+2) args = append(args, streamKey, groupName) for _, id := range messageIDs { args = append(args, id) } - _, err := getRedisClient().Do(ctx, "XACK", args...) + _, err = ds.Redis().Do(ctx, "XACK", args...) return err } diff --git a/message/redis_client.go b/message/redis_client.go new file mode 100644 index 0000000..2db78cb --- /dev/null +++ b/message/redis_client.go @@ -0,0 +1,468 @@ +// ============================================================================= +// Redis 数据源连接管理 +// 使用 GoFrame 框架自带的 Redis 客户端,负责数据源的连接、重连、健康检查和优雅关闭 +// ============================================================================= + +package message + +import ( + "context" + "fmt" + "os" + "os/signal" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/gogf/gf/v2/database/gredis" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/util/gconv" +) + +// ============================================================================= +// 数据源配置结构 +// ============================================================================= + +type RedisDataSourceConfig struct { + Name string `json:"name"` // 数据源名称 + Address string `json:"address"` // Redis 地址,如: 127.0.0.1:6379 + Db int `json:"db"` // 数据库编号 + Pass string `json:"pass"` // 密码 + Timeout time.Duration `json:"timeout"` // 连接超时 + MaxIdle int `json:"maxIdle"` // 最大空闲连接数 + MaxOpen int `json:"maxOpen"` // 最大活跃连接数 +} + +// ============================================================================= +// 单个数据源接口 +// ============================================================================= + +type DataSource interface { + Name() string + Redis() *gredis.Redis + IsConnected() bool + Connect(ctx context.Context) error + Reconnect(ctx context.Context) error + Close(ctx context.Context) error +} + +// ============================================================================= +// 数据源实现 +// ============================================================================= + +type BaseDataSource struct { + config *RedisDataSourceConfig + redis *gredis.Redis + isConnected bool + mu sync.RWMutex + lastError error + lastErrorTime time.Time + metrics RedisMetrics +} + +func NewBaseDataSource(config *RedisDataSourceConfig) *BaseDataSource { + return &BaseDataSource{ + config: config, + isConnected: false, + } +} + +func (d *BaseDataSource) Name() string { + return d.config.Name +} + +func (d *BaseDataSource) Redis() *gredis.Redis { + d.mu.RLock() + defer d.mu.RUnlock() + return d.redis +} + +func (d *BaseDataSource) IsConnected() bool { + d.mu.RLock() + defer d.mu.RUnlock() + return d.isConnected && d.redis != nil +} + +func (d *BaseDataSource) Connect(ctx context.Context) error { + d.mu.Lock() + defer d.mu.Unlock() + + // 设置默认值 + config := d.config + if config.Timeout == 0 { + config.Timeout = 10 * time.Second + } + if config.MaxIdle == 0 { + config.MaxIdle = 10 + } + if config.MaxOpen == 0 { + config.MaxOpen = 100 + } + + // 构建 GoFrame Redis 配置 + redisConfig := &gredis.Config{ + Address: config.Address, + Db: config.Db, + Pass: config.Pass, + } + + // 使用 GoFrame 的 Redis 连接 + redisObj, err := gredis.New(redisConfig) + if err != nil { + d.isConnected = false + d.lastError = err + d.lastErrorTime = time.Now() + d.metrics.PingError.Add(1) + return fmt.Errorf("datasource [%s] connection failed: %w", d.config.Name, err) + } + + d.redis = redisObj + + // 测试连接 + if err := d.Ping(ctx); err != nil { + d.isConnected = false + d.lastError = err + d.lastErrorTime = time.Now() + return fmt.Errorf("datasource [%s] ping failed: %w", d.config.Name, err) + } + + d.isConnected = true + d.lastError = nil + glog.Infof(ctx, "✅ datasource [%s] connected successfully", d.config.Name) + return nil +} + +func (d *BaseDataSource) Ping(ctx context.Context) error { + defer func() { + if r := recover(); r != nil { + d.metrics.PingError.Add(1) + glog.Errorf(ctx, "❌ datasource [%s] ping panic: %v", d.config.Name, r) + } + }() + + if d.redis == nil { + d.metrics.PingError.Add(1) + return fmt.Errorf("redis client is nil") + } + + _, err := d.redis.Do(ctx, "PING") + if err != nil { + d.metrics.PingError.Add(1) + return err + } + + d.metrics.PingCount.Add(1) + return nil +} + +func (d *BaseDataSource) Reconnect(ctx context.Context) error { + glog.Infof(ctx, "🔄 reconnecting datasource [%s]", d.config.Name) + return d.Connect(ctx) +} + +func (d *BaseDataSource) Close(ctx context.Context) error { + d.mu.Lock() + defer d.mu.Unlock() + + if d.redis != nil { + if err := d.redis.Close(ctx); err != nil { + return fmt.Errorf("datasource [%s] close failed: %w", d.config.Name, err) + } + } + + d.isConnected = false + d.redis = nil + glog.Infof(ctx, "datasource [%s] closed", d.config.Name) + return nil +} + +func (d *BaseDataSource) GetMetrics() RedisMetrics { + return d.metrics +} + +// ============================================================================= +// 监控指标 +// ============================================================================= + +type RedisMetrics struct { + PingCount atomic.Int64 + PingError atomic.Int64 + CommandCount atomic.Int64 + CommandError atomic.Int64 +} + +// GetPingMetrics 获取 Ping 相关指标 +func (m *RedisMetrics) GetPingMetrics() (int64, int64) { + return m.PingCount.Load(), m.PingError.Load() +} + +// GetCommandMetrics 获取命令相关指标 +func (m *RedisMetrics) GetCommandMetrics() (int64, int64) { + return m.CommandCount.Load(), m.CommandError.Load() +} + +// ============================================================================= +// 多数据源管理器 +// ============================================================================= + +type DataSourceManager struct { + sources map[string]DataSource + mu sync.RWMutex + ctx context.Context + cancel context.CancelFunc + started bool + maxRetries int + metrics RedisMetrics +} + +var ( + manager *DataSourceManager + once sync.Once +) + +// GetManager 获取全局管理器 +func GetManager() *DataSourceManager { + once.Do(func() { + ctx, cancel := context.WithCancel(context.Background()) + manager = &DataSourceManager{ + sources: make(map[string]DataSource), + ctx: ctx, + cancel: cancel, + started: false, + maxRetries: 3, + } + }) + return manager +} + +// RegisterDataSource 注册数据源 +func (m *DataSourceManager) RegisterDataSource(config *RedisDataSourceConfig) error { + m.mu.Lock() + defer m.mu.Unlock() + + if _, exists := m.sources[config.Name]; exists { + return fmt.Errorf("datasource [%s] already exists", config.Name) + } + + source := NewBaseDataSource(config) + m.sources[config.Name] = source + return nil +} + +// GetDataSource 获取数据源 +func (m *DataSourceManager) GetDataSource(name string) (DataSource, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + source, exists := m.sources[name] + if !exists { + return nil, fmt.Errorf("datasource [%s] not found", name) + } + return source, nil +} + +// GetAllDataSourceNames 获取所有数据源名称 +func (m *DataSourceManager) GetAllDataSourceNames() []string { + m.mu.RLock() + defer m.mu.RUnlock() + + names := make([]string, 0, len(m.sources)) + for name := range m.sources { + names = append(names, name) + } + return names +} + +// GetDefaultDataSource 获取默认数据源(第一个注册的数据源) +func (m *DataSourceManager) GetDefaultDataSource() (DataSource, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + for _, source := range m.sources { + return source, nil + } + return nil, fmt.Errorf("no datasource available") +} + +// GetMetrics 获取全局监控指标 +func (m *DataSourceManager) GetMetrics() RedisMetrics { + return m.metrics +} + +// init 初始化多数据源 +func init() { + ctx := context.Background() + + // 从配置初始化多数据源 + if err := GetManager().InitializeFromConfig(ctx); err != nil { + glog.Errorf(ctx, "❌ Failed to initialize Redis datasources: %v", err) + } else { + glog.Infof(ctx, "✅ Redis datasources initialized: %v", GetManager().GetAllDataSourceNames()) + } + + // 启动健康检查 + GetManager().StartHealthCheck() + + // 设置优雅关闭 + setupGracefulShutdown() +} + +// InitializeFromConfig 从配置初始化数据源 +// 动态读取 config.yml 中 redis 下的所有配置项 +func (m *DataSourceManager) InitializeFromConfig(ctx context.Context) error { + var firstErr error + + // 获取 redis 配置下的所有子键 + redisConfig := g.Cfg().MustGet(ctx, "redis") + if redisConfig.IsNil() { + glog.Warningf(ctx, "no redis configuration found in config.yml") + return nil + } + + // 将配置转换为 map + configMap := redisConfig.Map() + if configMap == nil { + glog.Warningf(ctx, "redis configuration is not a map") + return nil + } + + // 遍历所有 redis 子配置 + for name, subConfig := range configMap { + // 跳过非对象类型的配置 + subMap, ok := subConfig.(map[string]interface{}) + if !ok { + continue + } + + // 检查是否有 address 配置 + address, hasAddress := subMap["address"] + if !hasAddress || gconv.String(address) == "" { + continue + } + + // 构建数据源配置 + config := &RedisDataSourceConfig{ + Name: name, + Address: gconv.String(address), + Db: gconv.Int(subMap["db"]), + Pass: gconv.String(subMap["pass"]), + } + + // 设置默认值 + if config.Db == 0 { + config.Db = 0 + } + if config.Timeout == 0 { + config.Timeout = 10 * time.Second + } + if config.MaxIdle == 0 { + config.MaxIdle = 10 + } + if config.MaxOpen == 0 { + config.MaxOpen = 100 + } + + // 注册数据源 + if err := m.RegisterDataSource(config); err != nil { + glog.Errorf(ctx, "failed to register datasource [%s]: %v", name, err) + if firstErr == nil { + firstErr = err + } + continue + } + + // 连接数据源 + source, _ := m.GetDataSource(name) + if err := source.Connect(ctx); err != nil { + glog.Errorf(ctx, "failed to initialize datasource [%s]: %v", name, err) + if firstErr == nil { + firstErr = err + } + } + } + + return firstErr +} + +// StartHealthCheck 启动健康检查 +func (m *DataSourceManager) StartHealthCheck() { + if m.started { + return + } + m.started = true + go m.healthCheckLoop() +} + +// healthCheckLoop 健康检查循环 +func (m *DataSourceManager) healthCheckLoop() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-m.ctx.Done(): + return + case <-ticker.C: + m.checkAndReconnect() + } + } +} + +// checkAndReconnect 检查并重新连接 +func (m *DataSourceManager) checkAndReconnect() { + m.mu.RLock() + defer m.mu.RUnlock() + + for name, source := range m.sources { + if !source.IsConnected() { + glog.Warningf(context.Background(), "datasource [%s] disconnected, attempting reconnect", name) + + reconnectCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := source.Reconnect(reconnectCtx); err != nil { + glog.Errorf(reconnectCtx, "datasource [%s] reconnect failed: %v", name, err) + } else { + glog.Infof(reconnectCtx, "✅ datasource [%s] reconnected successfully", name) + } + } + } +} + +// CloseAll 关闭所有数据源 +func (m *DataSourceManager) CloseAll(ctx context.Context) error { + m.cancel() + + m.mu.RLock() + defer m.mu.RUnlock() + + var lastErr error + for name, source := range m.sources { + if err := source.Close(ctx); err != nil { + glog.Errorf(ctx, "failed to close datasource [%s]: %v", name, err) + lastErr = err + } + } + return lastErr +} + +// setupGracefulShutdown 设置优雅关闭 +func setupGracefulShutdown() { + go func() { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + <-sigCh + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + glog.Info(ctx, "🔄 Shutting down Redis connections...") + if err := GetManager().CloseAll(ctx); err != nil { + glog.Errorf(ctx, "❌ Failed to close Redis connections: %v", err) + } else { + glog.Info(ctx, "✅ Redis connections closed successfully") + } + }() +} From 5285767964f99fc46eb276ae98c76b9c7e661f2f Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Thu, 29 Jan 2026 13:57:50 +0800 Subject: [PATCH 198/225] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=B6=88=E6=81=AF?= =?UTF-8?q?=E9=98=9F=E5=88=97=E5=AE=9E=E7=8E=B0=EF=BC=9ANATS=E3=80=81Rabbi?= =?UTF-8?q?tMQ=20=E5=92=8C=20Redis=20=E7=9A=84=20Stream=20=E6=94=AF?= =?UTF-8?q?=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- message/nats_msg.go | 204 ++++++++++++++++++++++++ message/rabbitmq_msg.go | 286 +++++++++++++++++++++++++++++++++ message/redis_msg.go | 344 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 834 insertions(+) create mode 100644 message/nats_msg.go create mode 100644 message/rabbitmq_msg.go create mode 100644 message/redis_msg.go diff --git a/message/nats_msg.go b/message/nats_msg.go new file mode 100644 index 0000000..519de96 --- /dev/null +++ b/message/nats_msg.go @@ -0,0 +1,204 @@ +package message + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/nats-io/nats.go/jetstream" +) + +// natsMessageClient NATS 实现 +type natsMessageClient struct { + clientType messageClientType +} + +// StreamGroup 创建消费组(支持单个或批量) +func (q *natsMessageClient) streamGroup(ctx context.Context, configs ...interface{}) error { + if len(configs) == 0 { + return fmt.Errorf("配置不能为空") + } + for _, config := range configs { + cfg, ok := config.(*NATSConfig) + if !ok { + return fmt.Errorf("无效的 NATS 配置类型") + } + if err := q.createStreamGroup(ctx, cfg); err != nil { + return err + } + } + return nil +} + +// createStreamGroup 内部单个创建消费组 +func (q *natsMessageClient) createStreamGroup(ctx context.Context, cfg *NATSConfig) error { + // Stream 不存在,创建新的 + storage := jetstream.FileStorage + if !cfg.Durable { + storage = jetstream.MemoryStorage + } + if g.IsEmpty(cfg.Replicas) { + cfg.Replicas = 1 + } + // 构建流配置 + jsConfig := jetstream.StreamConfig{ + Name: cfg.Stream, + Subjects: []string{fmt.Sprintf("%s.>", cfg.Stream)}, + Replicas: cfg.Replicas, + NoAck: cfg.AutoAck, + AllowMsgSchedules: cfg.DelayMessage, // 延迟消息核心开关 + Storage: storage, + Discard: jetstream.DiscardOld, // 达到上限删除旧消息 + } + // 检查流是否已存在 + stream, err := js.Stream(ctx, cfg.Stream) + if err == nil { + // 流已存在,更新配置 + _, err = js.UpdateStream(ctx, jsConfig) + if err != nil { + return fmt.Errorf("更新任务流失败: %w", err) + } + g.Log().Infof(ctx, "任务流已更新: %s", stream.CachedInfo().Config.Name) + return nil + } + // 创建新流 + stream, err = js.CreateStream(ctx, jsConfig) + if err != nil { + return fmt.Errorf("创建任务流失败: %w", err) + } + + g.Log().Infof(ctx, "✅ NATS 队列初始化成功: stream=%s, consumer=%s", cfg.Stream, cfg.Consumer) + return nil +} + +// Publish 发布消息(支持单个或批量) +func (q *natsMessageClient) publish(ctx context.Context, config interface{}, data interface{}) error { + cfg, ok := config.(*NATSConfig) + if !ok { + return fmt.Errorf("无效的 NATS 配置类型") + } + payload, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("序列化数据失败: %w", err) + } + + // 发布消息到 JetStream + subject := fmt.Sprintf("%s.>", cfg.Stream) + _, err = js.Publish(ctx, subject, payload) + if err != nil { + g.Log().Errorf(ctx, "❌ NATS 发布消息失败: topic=%s, err=%v", cfg.Stream, err) + return err + } + + g.Log().Infof(ctx, "✅ NATS 发布消息成功: topic=%s", cfg.Stream) + return nil +} + +// PublishDelayed 发布延迟消息(支持单个或批量) +func (q *natsMessageClient) publishDelayed(ctx context.Context, config interface{}, data interface{}, delay int) error { + + cfg, ok := config.(*NATSConfig) + if !ok { + return fmt.Errorf("无效的 NATS 配置类型") + } + + payload, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("序列化数据失败: %w", err) + } + + // 使用 goroutine 实现简单的延迟发布 + go func() { + time.Sleep(time.Duration(delay)) + subject := fmt.Sprintf("%s.>", cfg.Stream) + if err := q.publishInternal(ctx, subject, payload); err != nil { + g.Log().Errorf(ctx, "❌ NATS 延迟消息发布失败: topic=%s, delay=%v, err=%v", cfg.Stream, delay, err) + } + }() + + g.Log().Infof(ctx, "✅ NATS 延迟消息已提交: topic=%s, delay=%v", cfg.Stream, delay) + + return nil +} + +// publishInternal 内部发布消息 +func (q *natsMessageClient) publishInternal(ctx context.Context, subject string, payload []byte) error { + _, err := js.Publish(ctx, subject, payload) + return err +} + +// Subscribe 订阅消息(支持单个或批量) +func (q *natsMessageClient) subscribe(ctx context.Context, configs ...interface{}) error { + if len(configs) == 0 { + return fmt.Errorf("配置不能为空") + } + + for _, config := range configs { + cfg, ok := config.(*NATSConfig) + if !ok { + return fmt.Errorf("无效的 NATS 配置类型") + } + handler := cfg.HandleFunc + if handler == nil { + return fmt.Errorf("必须提供处理函数") + } + if err := q.createSubscribe(ctx, cfg, handler); err != nil { + return err + } + } + return nil +} + +// subscribe 内部单个订阅消息 +func (q *natsMessageClient) createSubscribe(ctx context.Context, cfg *NATSConfig, handler func(ctx context.Context, message map[string]interface{}) error) error { + g.Log().Infof(ctx, "🔔 NATS 开始订阅: stream=%s, consumer=%s", cfg.Stream, cfg.Consumer) + // Stream 不存在,创建新的 + ackPolicy := jetstream.AckExplicitPolicy + if cfg.AutoAck { + ackPolicy = jetstream.AckNonePolicy + } + jsConfig := jetstream.ConsumerConfig{ + Name: cfg.Consumer, + Durable: cfg.Consumer, + AckPolicy: ackPolicy, + MaxDeliver: 3, + MaxAckPending: cfg.PrefetchCount, + } + // 创建新消费者 + consumer, err := js.CreateOrUpdateConsumer(ctx, cfg.Stream, jsConfig) + if err != nil { + return fmt.Errorf("创建消费者失败: %w", err) + } + // 创建消息处理函数 + msgHandler := func(msg jetstream.Msg) { + // 解析消息 + var data map[string]any + if err := json.Unmarshal(msg.Data(), &data); err != nil { + g.Log().Errorf(ctx, "解析消息失败: %v", err) + msg.Nak() + return + } + // 处理业务逻辑 + if err := handler(ctx, data); err != nil { + g.Log().Errorf(ctx, "处理消息失败: %v", err) + msg.Nak() + return + } + g.Log().Infof(ctx, "处理消息成功") + if !cfg.AutoAck { + msg.Ack() + } + } + + // 开始消费 + _, err = consumer.Consume(msgHandler) + if err != nil { + return fmt.Errorf("开始消费失败: %w", err) + } + + g.Log().Infof(ctx, "✅ 开始消费消息: %s/%s", cfg.Stream, cfg.Consumer) + + return nil +} diff --git a/message/rabbitmq_msg.go b/message/rabbitmq_msg.go new file mode 100644 index 0000000..8d49663 --- /dev/null +++ b/message/rabbitmq_msg.go @@ -0,0 +1,286 @@ +package message + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/gogf/gf/v2/frame/g" + amqp "github.com/rabbitmq/amqp091-go" +) + +// rabbitMQMessageClient RabbitMQ 实现 +type rabbitMQMessageClient struct { + clientType messageClientType +} + +// StreamGroup 创建消费组(支持单个或批量) +func (q *rabbitMQMessageClient) streamGroup(ctx context.Context, configs ...interface{}) error { + if len(configs) == 0 { + return fmt.Errorf("配置不能为空") + } + for _, config := range configs { + cfg, ok := config.(*RabbitMQConfig) + if !ok { + return fmt.Errorf("无效的 RabbitMQ 配置类型") + } + if err := q.setupQueue(ctx, channel, cfg, cfg.DelayMessage); err != nil { + return err + } + } + return nil +} + +// Publish 发布消息(支持单个或批量) +func (q *rabbitMQMessageClient) publish(ctx context.Context, config interface{}, data interface{}) error { + cfg, ok := config.(*RabbitMQConfig) + if !ok { + return fmt.Errorf("无效的 RabbitMQ 配置类型") + } + if err := q.publishMessage(ctx, cfg, "work", data, 0); err != nil { + g.Log().Errorf(ctx, "❌ RabbitMQ 发布消息失败: err=%v", err) + return err + } + return nil +} + +// PublishDelayed 发布延迟消息 +func (q *rabbitMQMessageClient) publishDelayed(ctx context.Context, config interface{}, data interface{}, delaySeconds int) error { + cfg, ok := config.(*RabbitMQConfig) + if !ok { + return fmt.Errorf("无效的 RabbitMQ 配置类型") + } + if err := q.publishMessage(ctx, cfg, "delayed", data, delaySeconds); err != nil { + g.Log().Errorf(ctx, "❌ RabbitMQ 发布延迟消息失败: err=%v", err) + return err + } + return nil +} + +func (q *rabbitMQMessageClient) publishMessage(ctx context.Context, cfg *RabbitMQConfig, mode string, data interface{}, delaySeconds int) error { + body, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("序列化数据失败: %w", err) + } + deliveryMode := amqp.Transient + if cfg.Durable { + deliveryMode = amqp.Persistent + } + publishing := amqp.Publishing{ + ContentType: "application/json", + Body: body, + DeliveryMode: deliveryMode, + Timestamp: time.Now(), + } + if delaySeconds > 0 { + publishing.Headers = amqp.Table{ + "x-delay": delaySeconds * 1000, // 延时时间(毫秒) + } + } + exchange, routingKey := q.parseExchangeAndRoutingKey(ctx, mode, cfg) + err = channel.PublishWithContext( + ctx, + exchange, + routingKey, + false, false, + publishing, + ) + return err +} + +func (q *rabbitMQMessageClient) parseExchangeAndRoutingKey(_ context.Context, mode string, cfg *RabbitMQConfig) (exchange, routingKey string) { + switch mode { + case "work", "": + exchange = "" // 默认交换机 + routingKey = cfg.Name // 队列名 + case "event", "topic": + exchange = cfg.Exchange + routingKey = cfg.Topic + case "broadcast": + exchange = cfg.Exchange + routingKey = "" // fanout忽略路由键 + case "delayed": + exchange = cfg.Exchange + ".delayed" + routingKey = cfg.Topic + default: + exchange = "" + routingKey = cfg.Name + } + return exchange, routingKey +} + +// setupQueue 统一的队列设置方法(声明 Exchange、队列、绑定、延迟 Exchange) +func (q *rabbitMQMessageClient) setupQueue(ctx context.Context, ch *amqp.Channel, cfg *RabbitMQConfig, delayMessage bool) error { + exchange, routingKey := q.parseExchangeAndRoutingKey(ctx, cfg.Mode, cfg) + + // 声明 Exchange + if err := ch.ExchangeDeclare(exchange, "topic", cfg.Durable, false, false, false, nil); err != nil { + return fmt.Errorf("声明 Exchange 失败: %w", err) + } + + // 声明队列 + if _, err := ch.QueueDeclare(cfg.Queue, cfg.Durable, false, false, false, nil); err != nil { + return fmt.Errorf("声明队列失败: %w", err) + } + + // 绑定队列 + if err := ch.QueueBind(cfg.Queue, routingKey, exchange, false, nil); err != nil { + return fmt.Errorf("绑定队列失败: %w", err) + } + + // 声明延迟 Exchange(如果需要) + if delayMessage { + if err := ch.ExchangeDeclare(exchange, "x-delayed-message", true, false, false, false, amqp.Table{"x-delayed-type": "direct"}); err != nil { + return fmt.Errorf("声明延迟 Exchange 失败: %w", err) + } + if err := ch.QueueBind(cfg.Name, routingKey, exchange, false, nil); err != nil { + return fmt.Errorf("绑定延迟队列失败: %w", err) + } + } + + return nil +} + +// Subscribe 订阅消息(支持单个或批量) +func (q *rabbitMQMessageClient) subscribe(ctx context.Context, configs ...interface{}) error { + if len(configs) == 0 { + return fmt.Errorf("配置不能为空") + } + + for _, config := range configs { + cfg, ok := config.(*RabbitMQConfig) + if !ok { + return fmt.Errorf("无效的 RabbitMQ 配置类型") + } + handler := cfg.HandleFunc + if handler == nil { + return fmt.Errorf("必须提供处理函数") + } + if err := q.createSubscribe(ctx, cfg, handler); err != nil { + return err + } + } + return nil +} + +// subscribe 内部单个订阅消息 +func (q *rabbitMQMessageClient) createSubscribe(ctx context.Context, cfg *RabbitMQConfig, handler func(ctx context.Context, message map[string]interface{}) error) error { + g.Log().Infof(ctx, "🔔 RabbitMQ 开始订阅: exchange=%s, queue=%s", cfg.Exchange, cfg.Queue) + + // 设置 Qos (预取数量),控制每次推送的消息数量 + // prefetchCount: 未 ACK 消息的最大数量 + // prefetchSize: 未 ACK 消息的总大小(0 表示不限制) + // global: false 表示仅应用于当前消费者 + prefetchCount := cfg.PrefetchCount + if prefetchCount <= 0 { + prefetchCount = 10 // 默认值为 10 + } + if err := channel.Qos(prefetchCount, 0, false); err != nil { + return fmt.Errorf("设置 Qos 失败: %w", err) + } + g.Log().Infof(ctx, "📊 设置 Prefetch Count: %d", prefetchCount) + + msg, err := channel.Consume( + cfg.Queue, // queue + cfg.Queue, // consumer + cfg.AutoAck, // auto-ack (根据配置决定) + false, // exclusive + false, // no-local + false, // no-wait + nil, // args + ) + if err != nil { + return fmt.Errorf("注册消费者失败: %w", err) + } + + go func() { + defer func() { + if r := recover(); r != nil { + g.Log().Errorf(ctx, "❌ RabbitMQ 消费者 panic: %v", r) + } + }() + + // 并发控制信号量 + semaphore := make(chan struct{}, 10) // 限制最大并发数为 10 + + for { + select { + case <-ctx.Done(): + g.Log().Infof(ctx, "🔕 RabbitMQ 消费者停止: queue=%s", cfg.Queue) + return + case msg, ok := <-msg: + if !ok { + g.Log().Warningf(ctx, "⚠️ RabbitMQ 消息通道关闭") + return + } + + // 获取并发控制槽位 + semaphore <- struct{}{} + + go func(m amqp.Delivery) { + defer func() { + <-semaphore // 释放槽位 + if r := recover(); r != nil { + g.Log().Errorf(ctx, "❌ 消息处理 panic: %v", r) + } + }() + + if err := q.handleMessageWithRetry(ctx, m, handler, cfg.MaxRetry); err != nil { + g.Log().Errorf(ctx, "❌ 消息处理失败(重试次数耗尽): %v", err) + + // 仅在手动 ACK 模式下拒绝消息 + if !cfg.AutoAck { + // 拒绝消息不再重新入队(避免死循环) + m.Nack(false, false) + } + return + } + + // 仅在手动 ACK 模式下确认消息 + if cfg.AutoAck { + if err := m.Ack(false); err != nil { + g.Log().Errorf(ctx, "❌ ACK 消息失败: %v", err) + } + } + }(msg) + } + } + }() + + return nil +} + +// handleMessageWithRetry 处理消息(支持重试) +func (q *rabbitMQMessageClient) handleMessageWithRetry(ctx context.Context, msg amqp.Delivery, handler func(ctx context.Context, message map[string]interface{}) error, maxRetry int) error { + var data map[string]interface{} + + if err := json.Unmarshal(msg.Body, &data); err != nil { + // 如果不是 JSON,直接使用原始内容 + data = map[string]interface{}{ + "data": string(msg.Body), + } + } + + // 重试逻辑 + for attempt := 0; attempt <= maxRetry; attempt++ { + if attempt > 0 { + g.Log().Infof(ctx, "🔄 消息处理重试 (第%d次)", attempt) + // 指数退避 + time.Sleep(time.Duration(attempt) * time.Second) + } + + err := handler(ctx, data) + if err == nil { + return nil // 成功 + } + + g.Log().Warningf(ctx, "⚠️ 消息处理失败 (第%d次): %v", attempt+1, err) + + if attempt == maxRetry { + return fmt.Errorf("达到最大重试次数 %d: %w", maxRetry, err) + } + } + + return nil +} diff --git a/message/redis_msg.go b/message/redis_msg.go new file mode 100644 index 0000000..135a78c --- /dev/null +++ b/message/redis_msg.go @@ -0,0 +1,344 @@ +package message + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/util/gconv" +) + +// redisMessageClient Redis 实现 +type redisMessageClient struct { + clientType messageClientType +} + +// RedisStreamMessage Redis Stream 消息结构 +type RedisStreamMessage struct { + ID string + Values map[string]interface{} +} + +// StreamGroup 创建消费组(支持单个或批量) +func (q *redisMessageClient) streamGroup(ctx context.Context, configs ...interface{}) error { + if len(configs) == 0 { + return fmt.Errorf("配置不能为空") + } + for _, config := range configs { + cfg, ok := config.(*RedisConfig) + if !ok { + return fmt.Errorf("无效的 Redis 配置类型") + } + if err := q.createStreamGroup(ctx, cfg); err != nil { + return err + } + } + return nil +} + +// streamGroup 内部单个创建消费组 +func (q *redisMessageClient) createStreamGroup(ctx context.Context, cfg *RedisConfig) error { + // 获取默认数据源 + ds, err := GetManager().GetDefaultDataSource() + if err != nil { + return fmt.Errorf("获取默认数据源失败: %w", err) + } + + // 检查连接状态,未连接则自动重连 + if !ds.IsConnected() { + if err := ds.Reconnect(ctx); err != nil { + return fmt.Errorf("redis重连失败: %w", err) + } + } + + _, err = ds.Redis().Do(ctx, "XGROUP", "CREATE", cfg.Stream, cfg.Group, "0", "MKSTREAM") + if err != nil { + errStr := err.Error() + if strings.Contains(errStr, "BUSYGROUP") && strings.Contains(errStr, "already exists") { + glog.Infof(ctx, "✅ Redis 消费者组已存在: %s", cfg.Group) + return nil + } + return fmt.Errorf("初始化消费者组失败: %w", err) + } + glog.Infof(ctx, "✅ Redis 消费者组创建成功: %s", cfg.Group) + return nil +} + +// Publish 内部单个发布消息 +func (q *redisMessageClient) publish(ctx context.Context, config interface{}, data interface{}) error { + ds, err := GetManager().GetDefaultDataSource() + if err != nil { + return fmt.Errorf("获取默认数据源失败: %w", err) + } + + if !ds.IsConnected() { + if err := ds.Reconnect(ctx); err != nil { + return fmt.Errorf("redis重连失败: %w", err) + } + } + + cfg, ok := config.(*RedisConfig) + if !ok { + return fmt.Errorf("无效的redis配置类型") + } + values := gconv.Map(data) + args := make([]interface{}, 0, len(values)*2+2) + args = append(args, cfg.Stream, "*") + for key, val := range values { + args = append(args, key, val) + } + result, err := ds.Redis().Do(ctx, "XADD", args...) + if err != nil { + g.Log().Errorf(ctx, "❌ Redis 发布消息失败: topic=%s, err=%v", cfg.Stream, err) + return err + } + g.Log().Infof(ctx, "✅ Redis 发布消息成功: topic=%s, messageID=%s", cfg.Stream, gconv.String(result)) + return nil +} + +// PublishDelayed 发布延迟消息(使用 ZSET) +func (q *redisMessageClient) publishDelayed(ctx context.Context, config interface{}, data interface{}, delay int) error { + ds, err := GetManager().GetDefaultDataSource() + if err != nil { + return fmt.Errorf("获取默认数据源失败: %w", err) + } + + if !ds.IsConnected() { + if err := ds.Reconnect(ctx); err != nil { + return fmt.Errorf("redis重连失败: %w", err) + } + } + + cfg, ok := config.(*RedisConfig) + if !ok { + return fmt.Errorf("无效的redis配置类型") + } + payload, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("序列化数据失败: %w", err) + } + score := float64(time.Now().Add(time.Duration(delay)).UnixMilli()) + delayedKey := fmt.Sprintf("delayed:%s", cfg.Stream) + + // ZADD delayedKey score payload + _, err = ds.Redis().Do(ctx, "ZADD", delayedKey, score, string(payload)) + if err != nil { + return err + } + + g.Log().Infof(ctx, "✅ Redis 延迟消息已发布: topic=%s, delay=%v", cfg.Stream, delay) + return nil +} + +// Subscribe 订阅消息(支持单个或批量) +func (q *redisMessageClient) subscribe(ctx context.Context, configs ...interface{}) error { + if len(configs) == 0 { + return fmt.Errorf("配置不能为空") + } + for _, config := range configs { + cfg, ok := config.(*RedisConfig) + if !ok { + return fmt.Errorf("无效的 Redis 配置类型") + } + handler := cfg.HandleFunc + if handler == nil { + return fmt.Errorf("必须提供处理函数") + } + if err := q.createSubscribe(ctx, cfg, handler); err != nil { + return err + } + } + return nil +} + +// subscribe 内部单个订阅消息 +func (q *redisMessageClient) createSubscribe(ctx context.Context, cfg *RedisConfig, handler func(ctx context.Context, message map[string]interface{}) error) error { + go func() { + defer func() { + if r := recover(); r != nil { + g.Log().Errorf(ctx, "❌ Redis 消费者 panic: %v", r) + } + }() + + retryTicker := time.NewTicker(time.Second) + defer retryTicker.Stop() + + // 重试计数器 + var consecutiveErrors int + const maxConsecutiveErrors = 3 + + for { + select { + case <-ctx.Done(): + g.Log().Infof(ctx, "🔕 Redis 消费者停止: topic=%s", cfg.Stream) + return + case <-retryTicker.C: + err := q.consumeMessages(ctx, cfg, handler) + if err != nil { + // 对于超时错误,返回nil继续循环,而不是返回错误 + if strings.Contains(err.Error(), "i/o timeout") || strings.Contains(err.Error(), "timeout") || + strings.Contains(err.Error(), "context deadline exceeded") || strings.Contains(err.Error(), "context canceled") { + + consecutiveErrors++ + if consecutiveErrors > maxConsecutiveErrors { + g.Log().Errorf(ctx, "Max retries exceeded, giving up") + return + } + backoffTime := 5 * time.Second + g.Log().Warningf(ctx, "⚠️ 等待 %v 后重试...", backoffTime) + + time.Sleep(backoffTime) + } else { + // 非超时错误(严重错误) + consecutiveErrors = 0 // 重置计数 + g.Log().Errorf(ctx, "严重错误,立即重试: %v", err) + + // 短暂等待后重试 + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + // 继续循环 + } + } + } else { + // 成功时重置错误计数器 + consecutiveErrors = 0 + } + } + } + }() + return nil +} + +// consumeMessages 消费消息 +func (q *redisMessageClient) consumeMessages(ctx context.Context, cfg *RedisConfig, handler func(ctx context.Context, message map[string]interface{}) error) error { + ds, err := GetManager().GetDefaultDataSource() + if err != nil { + return fmt.Errorf("获取默认数据源失败: %w", err) + } + + if !ds.IsConnected() { + if err := ds.Reconnect(ctx); err != nil { + return fmt.Errorf("redis重连失败: %w", err) + } + } + + // 检查消费者组是否存在 + if err := q.createStreamGroup(ctx, cfg); err != nil { + return fmt.Errorf("create stream group failed: %w", err) + } + + // 使用带重试的命令执行 + result, err := ds.Redis().Do(ctx, "XREADGROUP", "GROUP", cfg.Group, cfg.Consumer, "COUNT", cfg.Count, "BLOCK", 0, "STREAMS", cfg.Stream, ">") + if err != nil { + if strings.Contains(err.Error(), "i/o timeout") || strings.Contains(err.Error(), "timeout") || + strings.Contains(err.Error(), "context deadline exceeded") || strings.Contains(err.Error(), "context canceled") { + + } + return err + } + messages, err := q.parseStreamResult(result) + if err != nil { + return err + } + for _, msg := range messages { + // 处理消息 + if err := handler(ctx, msg.Values); err != nil { + g.Log().Errorf(ctx, "❌ 消息处理失败: messageID=%s, err=%v", msg.ID, err) + continue + } + + // ACK 消息 + if cfg.AutoAck { + if err := q.ackMessage(ctx, cfg.Stream, cfg.Group, msg.ID); err != nil { + g.Log().Errorf(ctx, "❌ ACK 消息失败: messageID=%s, err=%v", msg.ID, err) + } + } + } + + return nil +} + +// ackMessage ACK 消息 +func (q *redisMessageClient) ackMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error { + ds, err := GetManager().GetDefaultDataSource() + if err != nil { + return fmt.Errorf("获取默认数据源失败: %w", err) + } + + if !ds.IsConnected() { + if err := ds.Reconnect(ctx); err != nil { + return fmt.Errorf("redis重连失败: %w", err) + } + } + + args := make([]interface{}, 0, len(messageIDs)+2) + args = append(args, streamKey, groupName) + for _, id := range messageIDs { + args = append(args, id) + } + _, err = ds.Redis().Do(ctx, "XACK", args...) + return err +} + +// parseStreamResult 解析 Stream 结果 +func (q *redisMessageClient) parseStreamResult(result interface{}) ([]RedisStreamMessage, error) { + if result == nil { + return []RedisStreamMessage{}, nil + } + + var resultVal interface{} + + // 尝试获取 Val() 方法 + if valuer, ok := result.(interface{ Val() interface{} }); ok { + resultVal = valuer.Val() + } else { + resultVal = result + } + + // 检查是否为空 + if resultVal == nil { + return []RedisStreamMessage{}, nil + } + + // 预分配切片容量,避免多次扩容 + messages := make([]RedisStreamMessage, 0) + + if streamsMap, ok := resultVal.(map[interface{}]interface{}); ok { + for _, streamMsg := range streamsMap { + msgArray, ok := streamMsg.([]interface{}) + if !ok { + continue + } + for _, msgData := range msgArray { + msgArray, ok := msgData.([]interface{}) + if !ok || len(msgArray) < 2 { + continue + } + msgID := gconv.String(msgArray[0]) + fieldsArray, ok := msgArray[1].([]interface{}) + if !ok { + continue + } + values := make(map[string]interface{}, len(fieldsArray)/2) + for i := 0; i < len(fieldsArray); i += 2 { + if i+1 < len(fieldsArray) { + key := gconv.String(fieldsArray[i]) + values[key] = fieldsArray[i+1] + } + } + messages = append(messages, RedisStreamMessage{ + ID: msgID, + Values: values, + }) + } + } + } + + return messages, nil +} From d21061914679b64076406fd7b5c34bc762773568 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Thu, 29 Jan 2026 14:05:31 +0800 Subject: [PATCH 199/225] =?UTF-8?q?feat(nats):=20=E6=96=B0=E5=A2=9ENATS?= =?UTF-8?q?=E5=AE=A2=E6=88=B7=E7=AB=AF=E5=AE=9E=E7=8E=B0=EF=BC=8C=E6=94=AF?= =?UTF-8?q?=E6=8C=81RPC=E6=9C=8D=E5=8A=A1=E6=B3=A8=E5=86=8C=E3=80=81?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E5=92=8C=E8=B7=A8=E8=BF=9B=E7=A8=8B=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E5=8F=96=E6=B6=88=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nats/nats_client.go | 313 ++++++++++++++++++ nats/nats_rpc.go | 752 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1065 insertions(+) create mode 100644 nats/nats_client.go create mode 100644 nats/nats_rpc.go diff --git a/nats/nats_client.go b/nats/nats_client.go new file mode 100644 index 0000000..a4afebb --- /dev/null +++ b/nats/nats_client.go @@ -0,0 +1,313 @@ +package nats + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +var ( + nc *nats.Conn + js jetstream.JetStream + inited bool + natsMu sync.RWMutex + natsURL string + healthCtx context.Context + healthCancel context.CancelFunc + connected bool + reconnectChan chan struct{} + + // 连接状态变化监听器 + connStateListeners []connStateListener + connListenersMu sync.RWMutex + + // 监控指标 + metrics metricsCounter +) + +// Metrics 监控指标 +type metricsCounter struct { + PublishCount atomic.Int64 + PublishError atomic.Int64 + SubscribeCount atomic.Int64 + RequestCount atomic.Int64 + RequestError atomic.Int64 + ConsumeCount atomic.Int64 + ConsumeError atomic.Int64 +} + +// ConnState 连接状态 +type connState int + +const ( + connStateDisconnected connState = iota + connStateConnecting + connStateConnected + connStateReconnecting + connStateClosed +) + +// ConnStateListener 连接状态监听器 +type connStateListener func(state connState, err error) + +// GetMetrics 获取监控指标 +func getMetrics() metricsCounter { + return metrics +} + +// registerConnStateListener 注册连接状态监听器 +func registerConnStateListener(listener connStateListener) { + connListenersMu.Lock() + defer connListenersMu.Unlock() + connStateListeners = append(connStateListeners, listener) +} + +// unregisterConnStateListener 取消注册连接状态监听器 +func unregisterConnStateListener(listener connStateListener) { + connListenersMu.Lock() + defer connListenersMu.Unlock() + for i, l := range connStateListeners { + if l != nil && &l == &listener { + connStateListeners = append(connStateListeners[:i], connStateListeners[i+1:]...) + break + } + } +} + +// notifyConnState 通知所有监听器连接状态变化 +func notifyConnState(state connState, err error) { + connListenersMu.RLock() + listeners := make([]connStateListener, len(connStateListeners)) + copy(listeners, connStateListeners) + connListenersMu.RUnlock() + + for _, listener := range listeners { + if listener != nil { + listener(state, err) + } + } +} + +// init 初始化 NATS 连接 +func init() { + // 从配置文件读取 NATS 地址 + natsURL = g.Cfg().MustGet(context.Background(), "nats.url").String() + if natsURL == "" { + // 默认使用本地地址 + natsURL = nats.DefaultURL + } + + // 创建健康检查上下文 + healthCtx, healthCancel = context.WithCancel(context.Background()) + + // 创建重连通知通道(增大缓冲区避免丢失通知) + reconnectChan = make(chan struct{}, 10) + + // 启动连接 + go initConnection() + + // 启动健康检查协程 + go healthCheck() +} + +// initConnection 初始化连接 +func initConnection() { + ctx := context.Background() + notifyConnState(connStateConnecting, nil) + if err := connect(ctx); err != nil { + g.Log().Errorf(ctx, "NATS 初始连接失败: %v", err) + notifyConnState(connStateDisconnected, err) + } +} + +// connect 建立 NATS 连接 +func connect(ctx context.Context) error { + natsMu.Lock() + defer natsMu.Unlock() + + if nc != nil && !nc.IsClosed() { + nc.Close() + } + + // 连接选项配置 + opts := []nats.Option{ + nats.Name("goframe-nats-client"), + nats.ReconnectWait(2 * time.Second), + nats.MaxReconnects(-1), // 无限重连 + nats.PingInterval(10 * time.Second), + nats.MaxPingsOutstanding(5), + nats.ReconnectHandler(func(nc *nats.Conn) { + g.Log().Infof(ctx, "✅ NATS 重连成功: %s", nc.ConnectedUrl()) + connected = true + + // 重新创建 JetStream 实例 + if newJS, err := jetstream.New(nc); err == nil { + js = newJS + } + + // 通知重连成功 + notifyConnState(connStateConnected, nil) + + // 使用非阻塞发送避免阻塞 + select { + case reconnectChan <- struct{}{}: + default: + // 通道已满,丢弃通知 + } + }), + nats.DisconnectErrHandler(func(nc *nats.Conn, err error) { + g.Log().Warningf(ctx, "⚠️ NATS 连接断开: %v, 准备重连...", err) + connected = false + notifyConnState(connStateReconnecting, err) + }), + nats.ClosedHandler(func(nc *nats.Conn) { + g.Log().Infof(ctx, "NATS 连接已关闭: %s", nc.ConnectedUrl()) + connected = false + notifyConnState(connStateClosed, nil) + }), + nats.ErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { + g.Log().Errorf(ctx, "NATS 错误: %v", err) + }), + } + + var err error + nc, err = nats.Connect(natsURL, opts...) + if err != nil { + return fmt.Errorf("NATS 连接失败: %w", err) + } + + // 等待连接就绪 + if nc.Status() != nats.CONNECTED { + select { + case <-time.After(5 * time.Second): + notifyConnState(connStateDisconnected, fmt.Errorf("连接超时")) + return fmt.Errorf("NATS 连接超时") + case <-nc.StatusChanged(nats.CONNECTED): + } + } + + // 创建 JetStream 实例 + js, err = jetstream.New(nc) + if err != nil { + return fmt.Errorf("创建 JetStream 失败: %w", err) + } + + connected = true + inited = true + g.Log().Infof(ctx, "✅ NATS 连接成功: %s", nc.ConnectedUrl()) + notifyConnState(connStateConnected, nil) + return nil +} + +// healthCheck 健康检查协程(仅作为备用检查) +func healthCheck() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-healthCtx.Done(): + return + case <-ticker.C: + natsMu.RLock() + currentConnected := connected + currentConn := nc + natsMu.RUnlock() + + if !currentConnected || currentConn == nil || currentConn.IsClosed() { + // 仅记录日志,不尝试重连(NATS 已有自动重连机制) + g.Log().Warning(context.Background(), "NATS 连接断开,等待 NATS 自动重连...") + } + case <-reconnectChan: + // 重连成功的通知(仅记录日志) + g.Log().Info(context.Background(), "收到重连成功通知") + } + } +} + +// checkConnected 检查连接状态 +func checkConnected() bool { + natsMu.RLock() + defer natsMu.RUnlock() + return connected && nc != nil && !nc.IsClosed() +} + +// getConnState 获取当前连接状态 +func getConnState() connState { + natsMu.RLock() + defer natsMu.RUnlock() + + if nc == nil { + return connStateDisconnected + } + + if nc.IsClosed() { + return connStateClosed + } + + if connected { + return connStateConnected + } + + return connStateDisconnected +} + +// shutdown 优雅关闭:自动注销所有已注册的服务并关闭 NATS 连接 +func shutdown() error { + ctx := context.Background() + g.Log().Info(ctx, "开始优雅关闭 NATS RPC 服务...") + + // 注销所有单实例服务 + rpcServicesMu.Lock() + singleServiceCount := len(rpcServices) + for serviceName := range rpcServices { + if sub, exists := rpcSubs[serviceName]; exists { + if err := sub.Unsubscribe(); err != nil { + g.Log().Errorf(ctx, "注销服务 %s 失败: %v", serviceName, err) + } + } + delete(rpcSubs, serviceName) + delete(rpcServices, serviceName) + } + rpcServicesMu.Unlock() + + // 注销所有队列服务 + queueRPCMu.Lock() + queueServiceCount := 0 + for queueName, servicesMap := range queueRPCServices { + queueServiceCount += len(servicesMap) + for serviceName, sub := range queueRPCSubs[queueName] { + if err := sub.Unsubscribe(); err != nil { + g.Log().Errorf(ctx, "注销队列服务 %s (队列: %s) 失败: %v", serviceName, queueName, err) + } + } + delete(queueRPCSubs, queueName) + delete(queueRPCServices, queueName) + } + queueRPCMu.Unlock() + + g.Log().Infof(ctx, "已注销 %d 个单实例服务和 %d 个队列服务", singleServiceCount, queueServiceCount) + + natsMu.Lock() + defer natsMu.Unlock() + + // 停止健康检查协程 + if healthCancel != nil { + healthCancel() + } + + // 关闭连接 + if nc != nil && !nc.IsClosed() { + nc.Close() + connected = false + inited = false + } + g.Log().Info(ctx, "NATS RPC 服务已优雅关闭") + return nil +} diff --git a/nats/nats_rpc.go b/nats/nats_rpc.go new file mode 100644 index 0000000..e95514b --- /dev/null +++ b/nats/nats_rpc.go @@ -0,0 +1,752 @@ +package nats + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/gogf/gf/v2/frame/g" + "github.com/nats-io/nats.go" + "go.opentelemetry.io/otel/trace" + "reflect" + "sync" +) + +// ============ RPC 服务封装 ============ +// 以下方法提供了完全抽象的 RPC 调用接口 +// 调用方和响应方完全不需要知道底层使用的是 NATS 的发布订阅模式 + +// RPC 服务注册表 +var ( + rpcServices map[string]rpcHandler + rpcSubs map[string]*nats.Subscription // 服务名 -> 订阅 + rpcServicesMu sync.RWMutex + queueRPCServices map[string]map[string]rpcHandler // queueName -> subject -> handler + queueRPCSubs map[string]map[string]*nats.Subscription // queueName -> serviceName -> 订阅 + queueRPCMu sync.RWMutex + + // ============ TraceID 主动取消支持 ============ + // 全局映射表:TraceID -> CancelFunc,并发安全 + traceCancelMap map[string]context.CancelFunc + traceCancelMu sync.RWMutex + // 取消主题前缀 + cancelSubjectPrefix = "ctx.cancel.otel." +) + +// rpcHandler RPC 处理函数类型 +// 实现方只需要关注请求参数和返回值,无需了解底层 NATS 实现 +// 返回值可以是任意类型,会被自动序列化为 JSON +type rpcHandler func(ctx context.Context, req []byte) (any, error) + +// RegisterRPCService 注册 RPC 服务(单实例) +// serviceName: 服务名称,调用方通过此名称调用服务 +// handler: 服务处理函数,接收请求并返回响应 +func registerRPCService(serviceName string, handler rpcHandler) (err error) { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + rpcServicesMu.Lock() + if rpcServices == nil { + rpcServices = make(map[string]rpcHandler) + } + if rpcSubs == nil { + rpcSubs = make(map[string]*nats.Subscription) + } + + // 如果已存在该服务,先取消之前的订阅 + if oldSub, exists := rpcSubs[serviceName]; exists { + oldSub.Unsubscribe() + } + + rpcServices[serviceName] = handler + rpcServicesMu.Unlock() + + // 订阅服务主题 + subject := fmt.Sprintf("rpc.%s", serviceName) + sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { + // 执行处理函数 + executeHandler(handler, msg) + }) + + if err != nil { + return fmt.Errorf("注册 RPC 服务失败: %w", err) + } + + rpcSubs[serviceName] = sub + metrics.SubscribeCount.Add(1) + g.Log().Infof(context.Background(), "✅ RPC 服务已注册: %s", serviceName) + return nil +} + +// RegisterQueueRPCService 注册 RPC 服务(集群模式) +// 多个服务实例注册同一服务时,请求会自动负载均衡 +// serviceName: 服务名称 +// queueName: 队列组名,同一队列组的实例共享请求 +// handler: 服务处理函数 +func registerQueueRPCService(serviceName, queueName string, handler rpcHandler) (err error) { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + queueRPCMu.Lock() + if queueRPCServices == nil { + queueRPCServices = make(map[string]map[string]rpcHandler) + } + if queueRPCSubs == nil { + queueRPCSubs = make(map[string]map[string]*nats.Subscription) + } + if queueRPCServices[queueName] == nil { + queueRPCServices[queueName] = make(map[string]rpcHandler) + } + if queueRPCSubs[queueName] == nil { + queueRPCSubs[queueName] = make(map[string]*nats.Subscription) + } + + // 如果已存在该服务,先取消之前的订阅 + if oldSub, exists := queueRPCSubs[queueName][serviceName]; exists { + oldSub.Unsubscribe() + } + + queueRPCServices[queueName][serviceName] = handler + queueRPCMu.Unlock() + + // 订阅服务主题(队列模式) + subject := fmt.Sprintf("rpc.%s", serviceName) + sub, err := nc.QueueSubscribe(subject, queueName, func(msg *nats.Msg) { + // 执行处理函数 + executeHandler(handler, msg) + }) + + if err != nil { + return fmt.Errorf("注册队列 RPC 服务失败: %w", err) + } + + queueRPCMu.Lock() + queueRPCSubs[queueName][serviceName] = sub + queueRPCMu.Unlock() + + metrics.SubscribeCount.Add(1) + g.Log().Infof(context.Background(), "✅ 队列 RPC 服务已注册: %s (队列组: %s)", serviceName, queueName) + return nil +} + +// executeHandler 执行 RPC 处理函数 +func executeHandler(handler rpcHandler, msg *nats.Msg) { + // 响应 + var respData []byte + // 从消息头重建上下文 + ctx := headersToContext(context.Background(), msg.Header) + // 提取 TraceID,创建可取消的 context + ctx = createCancelContext(ctx, msg.Header.Get(TraceIDKey)) + // 检查 context 是否已取消(在调用 handler 之前) + select { + case <-ctx.Done(): + // context 已取消,返回取消错误 + g.Log().Infof(ctx, "RPC 请求已取消,traceID: %s", msg.Header.Get(TraceIDKey)) + // 仍然需要发送响应以避免客户端超时 + respData = []byte(`{"_err":"请求已取消"}`) + // 清理取消映射表 + cleanupTraceCancel(msg.Header.Get(TraceIDKey)) + return + default: + } + + // 执行业务处理 + response, err := handler(ctx, msg.Data) + + if err != nil { + // 错误时返回 {"_err": "错误信息"} + if respData, err = json.Marshal(map[string]any{"_err": err.Error()}); err != nil { + g.Log().Errorf(ctx, "RPC 错误响应序列化失败: %v", err) + respData = []byte(`{"_err":"错误响应序列化失败"}`) + } + } else if response == nil { + // 空响应时返回空对象(或 {"_err": ""}) + respData = []byte(`{}`) + } else { + // 成功时返回业务数据 + if respData, err = json.Marshal(response); err != nil { + g.Log().Errorf(ctx, "RPC 响应序列化失败: %v", err) + respData = []byte(`{"_err":"响应序列化失败"}`) + } + } + // 发送响应(必须执行) 如果客户端用 nc.Request(...) 发送消息 → 双向模式,服务端必须 msg.Respond + if err = msg.Respond(respData); err != nil { + g.Log().Errorf(ctx, "RPC 响应失败: %v", err) + } + // 请求结束,清理取消映射表 + cleanupTraceCancel(msg.Header.Get(TraceIDKey)) +} + +// createCancelContext 创建可取消的 context 并注册到取消映射表 +// 返回可取消的 context(如果 traceID 为空则返回原 context) +func createCancelContext(ctx context.Context, traceID string) context.Context { + if g.IsEmpty(traceID) { + return ctx + } + // 创建带取消功能的 context + taskCtx, cancel := context.WithCancel(ctx) + // 注册到取消映射表 + traceCancelMu.Lock() + if traceCancelMap == nil { + traceCancelMap = make(map[string]context.CancelFunc) + } + // 如果同一 TraceID 已有 CancelFunc,先调用它 + if oldCancel, exists := traceCancelMap[traceID]; exists { + oldCancel() + } + traceCancelMap[traceID] = cancel + traceCancelMu.Unlock() + + return taskCtx +} + +// ============ TraceID 主动取消功能 ============ +// 以下函数实现了基于 OpenTelemetry TraceID 的跨进程任务取消机制 + +// SetupCancelListener 设置取消监听器 +// 订阅取消主题,监听取消指令 +// 使用示例: +// +// sub, err := nats.SetupCancelListener(ctx) +func setupCancelListener(ctx context.Context) (*nats.Subscription, error) { + if !checkConnected() { + return nil, fmt.Errorf("NATS 未连接") + } + + if traceCancelMap == nil { + traceCancelMap = make(map[string]context.CancelFunc) + } + + // 修复问题3:订阅取消主题,格式: ctx.cancel.otel.* + // 使用 * 通配符而不是 >,因为 TraceID 是最后一部分 + cancelSubject := cancelSubjectPrefix + "*" + sub, err := nc.Subscribe(cancelSubject, func(msg *nats.Msg) { + // 从主题中解析 TraceID (去除前缀) + prefixLen := len(cancelSubjectPrefix) + if len(msg.Subject) <= prefixLen { + g.Log().Warningf(ctx, "取消消息主题格式错误: %s", msg.Subject) + return + } + traceID := msg.Subject[prefixLen:] + + if traceID == "" { + g.Log().Warning(ctx, "取消消息主题缺少 TraceID") + return + } + + // 从映射表获取 CancelFunc 并执行取消 + traceCancelMu.RLock() + cancel, ok := traceCancelMap[traceID] + traceCancelMu.RUnlock() + + if ok { + cancel() + g.Log().Infof(ctx, "📢 取消信号已发送,traceID: %s", traceID) + } else { + g.Log().Infof(ctx, "⚠️ 未找到对应的可取消任务,traceID: %s", traceID) + } + }) + + if err != nil { + return nil, fmt.Errorf("设置取消监听器失败: %w", err) + } + + metrics.SubscribeCount.Add(1) + g.Log().Infof(ctx, "✅ 取消监听器已设置: %s", cancelSubject) + return sub, nil +} + +// publishCancel 发布取消指令 +// 向指定 TraceID 发送取消信号 +// 使用示例: +// +// err := nats.publishCancel(ctx, traceID) +func publishCancel(ctx context.Context, traceID string) error { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + if traceID == "" { + return fmt.Errorf("TraceID 不能为空") + } + + cancelSubject := cancelSubjectPrefix + traceID + err := nc.Publish(cancelSubject, nil) + if err != nil { + return fmt.Errorf("发布取消信号失败: %w", err) + } + + g.Log().Infof(ctx, "📤 已发送取消信号,traceID: %s,主题: %s", traceID, cancelSubject) + return nil +} + +// cleanupTraceCancel 清理取消映射表中的条目 +// 任务取消/正常结束后必须调用此函数,避免内存泄漏 +// 使用示例: +// +// defer nats.cleanupTraceCancel(traceID) +func cleanupTraceCancel(traceID string) { + if traceID == "" { + return + } + + traceCancelMu.Lock() + defer traceCancelMu.Unlock() + + if _, ok := traceCancelMap[traceID]; ok { + delete(traceCancelMap, traceID) + g.Log().Infof(context.Background(), "✅ 已清理取消映射表,traceID: %s", traceID) + } +} + +// CallRPC 调用 RPC 服务 +// serviceName: 服务名称 +// req: 请求数据 +// 返回: 响应数据(任意类型)和错误 +func CallRPC(ctx context.Context, serviceName string, req any, resp any) (err error) { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + metrics.RequestCount.Add(1) + + // 验证 resp 必须是指针类型 + respValue := reflect.ValueOf(resp) + if respValue.Kind() != reflect.Ptr { + return fmt.Errorf("resp 参数必须是指针类型(当前类型: %T)", resp) + } + + // 构建请求体 + var reqBody []byte + if !g.IsEmpty(req) { + reqValue := reflect.ValueOf(req) + if !(reqValue.Kind() == reflect.Ptr && reqValue.IsNil()) && !reqValue.IsZero() { + reqData, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("序列化请求参数失败: %w", err) + } + reqBody = reqData + } + } + + // 检查本地是否有注册的单实例服务,如果有则直接调用(优化性能) + rpcServicesMu.RLock() + if localHandler, exists := rpcServices[serviceName]; exists { + rpcServicesMu.RUnlock() + + // 修复问题1:本地调用也需要处理取消机制 + var traceID string + if traceID, err = getTraceID(ctx); err != nil { + return err + } + // 提取 TraceID,创建可取消的 context + cancelCtx := createCancelContext(ctx, traceID) + // 执行本地调用 + var response interface{} + if response, err = localHandler(cancelCtx, reqBody); err != nil { + metrics.RequestError.Add(1) + return fmt.Errorf("本地调用 RPC 服务失败 [%s]: %w", serviceName, err) + } + + // 请求结束,清理取消映射表 + cleanupTraceCancel(traceID) + + // 检查是否为错误消息:尝试解析为 map,看是否包含 "_err" 字段 + var respMap map[string]any + if json.Unmarshal(response.([]byte), &respMap) == nil { + if errMsg, ok := respMap["_err"]; ok { + metrics.RequestError.Add(1) + return fmt.Errorf("%v", errMsg) + } + } + // 正常数据直接返回 + // responseMsg.Data 已经是 []byte 类型(来自 msg.Data),直接反序列化 + if err = json.Unmarshal(response.([]byte), resp); err != nil { + return fmt.Errorf("解析响应失败: %w (响应内容: %s)", err, response) + } + + return + } + rpcServicesMu.RUnlock() + + subject := fmt.Sprintf("rpc.%s", serviceName) + + // 创建消息并将上下文元数据写入消息头 + msg := nats.NewMsg(subject) + msg.Data = reqBody + headers, err := contextToHeaders(ctx) + if err != nil { + return fmt.Errorf("上下文转换失败: %w", err) + } + msg.Header = headers + + // 修复问题5:优化 go 协程避免资源泄漏 + // 使用 done channel 来确保 goroutine 能正确退出 + done := make(chan struct{}) + var closeDoneOnce sync.Once + closeDone := func() { + closeDoneOnce.Do(func() { + close(done) + }) + } + + if msg.Header.Get(TraceIDKey) != "" { + go func() { + defer closeDone() + select { + case <-ctx.Done(): + // context 被取消时,发送取消信号给服务端 + if errors.Is(ctx.Err(), context.Canceled) { + if err := publishCancel(context.Background(), msg.Header.Get(TraceIDKey)); err != nil { + g.Log().Errorf(ctx, "发送 RPC 取消信号失败: %v", err) + } else { + g.Log().Infof(ctx, "RPC 调用已取消,traceID: %s", msg.Header.Get(TraceIDKey)) + } + } + case <-done: + // 请求已完成,无需发送取消信号 + return + } + }() + } + + // 发送请求 + responseMsg, err := nc.RequestMsgWithContext(ctx, msg) + + // 关闭 done channel,通知 goroutine 退出 + closeDone() + + if err != nil { + metrics.RequestError.Add(1) + return fmt.Errorf("调用 RPC 服务失败 [%s]: %w", serviceName, err) + } + + if responseMsg == nil { + metrics.RequestError.Add(1) + return fmt.Errorf("RPC 响应为空 [%s]", serviceName) + } + + // 解析响应 + if len(responseMsg.Data) > 0 { + // 检查是否为错误消息:尝试解析为 map,看是否包含 "_err" 字段 + var respMap map[string]any + if json.Unmarshal(responseMsg.Data, &respMap) == nil { + if errMsg, ok := respMap["_err"]; ok { + metrics.RequestError.Add(1) + return fmt.Errorf("%v", errMsg) + } + } + // 正常数据直接返回 + // responseMsg.Data 已经是 []byte 类型(来自 msg.Data),直接反序列化 + if err = json.Unmarshal(responseMsg.Data, resp); err != nil { + return fmt.Errorf("解析响应失败: %w (响应内容: %s)", err, responseMsg.Data) + } + } + + return +} + +// RegisterServiceOption 注册选项类型 +type RegisterServiceOption func(*registerServiceConfig) + +type registerServiceConfig struct { + queueName string // 队列组名(用于集群模式) + excludeMethods []string +} + +// WithQueueGroup 设置队列组名(集群模式) +func WithQueueGroup(queueName string) RegisterServiceOption { + return func(cfg *registerServiceConfig) { + cfg.queueName = queueName + } +} + +// WithExcludeMethods 排除不需要注册的方法 +func WithExcludeMethods(methods ...string) RegisterServiceOption { + return func(cfg *registerServiceConfig) { + cfg.excludeMethods = append(cfg.excludeMethods, methods...) + } +} + +// AutoRegisterServices 自动注册多个服务的所有公开方法 +// serviceInstances: map[包名]service实例,如 map[string]interface{}{"user": userService, "order": orderService} +// options: 注册选项(可选) +// 示例: +// +// AutoRegisterServices(map[string]interface{}{ +// "user": userService, +// "order": orderService, +// }) +// 或 +// AutoRegisterServices(map[string]interface{}{ +// "order": orderService, +// }, WithQueueGroup("order-group")) +func AutoRegisterServices(ctx context.Context, serviceInstances map[string]interface{}, options ...RegisterServiceOption) error { + // 先注册 RPC 服务(如果 NATS 不可用则记录警告但不阻塞启动) + if !checkConnected() { + return fmt.Errorf("NATS 未连接,RPC 服务未注册") + } + + if len(serviceInstances) == 0 { + return fmt.Errorf("service 实例列表不能为空") + } + + totalRegistered := 0 + // 遍历每个 service 实例 + for pkgName, serviceInstance := range serviceInstances { + // 注册服务 + err := registerService(serviceInstance, pkgName, options...) + if err != nil { + g.Log().Errorf(ctx, "注册 %s 服务失败: %v", pkgName, err) + continue + } + totalRegistered++ + g.Log().Infof(ctx, "✅ %s 服务已自动注册", pkgName) + } + + if totalRegistered == 0 { + return fmt.Errorf("未能注册任何服务") + } + // 设置取消监听器(监听基于 TraceID 的取消请求) + if _, err := setupCancelListener(ctx); err != nil { + g.Log().Errorf(ctx, "设置取消监听器失败: %v", err) + } else { + g.Log().Infof(ctx, "✅ 取消监听器已自动设置") + } + g.Log().Infof(ctx, "✅ 共自动注册了 %d 个服务", totalRegistered) + + return nil +} + +// registerService 注册单个服务的所有公开方法(内部函数) +func registerService(service interface{}, serviceNamePrefix string, options ...RegisterServiceOption) (err error) { + if !checkConnected() { + return fmt.Errorf("NATS 未连接") + } + + // 应用选项 + cfg := ®isterServiceConfig{} + for _, opt := range options { + opt(cfg) + } + + // 创建排除方法集合 + excludeSet := make(map[string]struct{}) + for _, method := range cfg.excludeMethods { + excludeSet[method] = struct{}{} + } + + // 获取 service 的类型 + serviceType := reflect.TypeOf(service) + + // 遍历所有方法 + registeredCount := 0 + for i := 0; i < serviceType.NumMethod(); i++ { + method := serviceType.Method(i) + + // 只注册导出方法(首字母大写) + if !method.IsExported() { + continue + } + + // 排除指定的方法 + if _, exists := excludeSet[method.Name]; exists { + continue + } + + // 检查方法签名:必须是 func(ctx context.Context, request) (response, error) + // 注意:method.Type.NumIn() 包含接收者,所以实际参数数量需要减去 1 + // 要求:接收者 + context.Context + request,总共3个参数 + if method.Type.NumIn() != 3 { + g.Log().Warningf(context.Background(), "方法 %s 必须有2个参数(context.Context 和请求参数),跳过注册", method.Name) + continue + } + + // 第一个参数(接收者之后的第一个参数)必须是 context.Context + // method.Type.In(0) 是接收者,method.Type.In(1) 才是第一个参数 + if !method.Type.In(1).Implements(reflect.TypeOf((*context.Context)(nil)).Elem()) { + g.Log().Warningf(context.Background(), "方法 %s 的第一个参数必须是 context.Context,跳过注册", method.Name) + continue + } + + // 第二个参数必须是结构体指针或数组 + reqType := method.Type.In(2) + if reqType.Kind() != reflect.Ptr && reqType.Kind() != reflect.Slice && reqType.Kind() != reflect.Array { + g.Log().Warningf(context.Background(), "方法 %s 的第二个参数必须是结构体指针或数组,跳过注册", method.Name) + continue + } + + // 返回值必须是 (result, error),即2个返回值 + if method.Type.NumOut() != 2 { + g.Log().Warningf(context.Background(), "方法 %s 必须有2个返回值(result 和 error),跳过注册", method.Name) + continue + } + + // 最后一个返回值必须是 error + if !method.Type.Out(1).Implements(reflect.TypeOf((*error)(nil)).Elem()) { + g.Log().Warningf(context.Background(), "方法 %s 的最后一个返回值必须是 error,跳过注册", method.Name) + continue + } + + // 生成服务名称:前缀.方法名(保持原始方法名) + serviceName := fmt.Sprintf("%s.%s", serviceNamePrefix, method.Name) + + // 创建 RPC handler + handler := func(ctx context.Context, req []byte) (any, error) { + // 准备方法调用参数 + // args[0] 是接收者, args[1] 是 ctx, args[2] 是请求参数 + args := make([]reflect.Value, 3) + args[0] = reflect.ValueOf(service) // 接收者 + args[1] = reflect.ValueOf(ctx) // context.Context + + // 解析请求参数 + if len(req) > 0 { + reqValuePtr := reflect.New(reqType) + + // 解析 JSON + if err := json.Unmarshal(req, reqValuePtr.Interface()); err != nil { + // 根据参数类型提供更友好的错误提示 + var typeHint string + if reqType.Kind() == reflect.Ptr { + typeHint = fmt.Sprintf("(期望类型: %s)", reqType.Elem().Name()) + } else { // reflect.Slice 或 reflect.Array + typeHint = fmt.Sprintf("(期望类型: %s,请确保客户端传递的是JSON数组格式)", reqType.String()) + } + return nil, fmt.Errorf("解析请求参数失败%s: %w", typeHint, err) + } + args[2] = reqValuePtr.Elem() + } else { + // 请求为空,创建零值 + args[2] = reflect.Zero(method.Type.In(2)) + } + + // 调用方法 + results := method.Func.Call(args) + + // 处理返回值 + var result any + + if len(results) == 1 { + // 只有 error + if !results[0].IsNil() { + err = results[0].Interface().(error) + } + } else if len(results) == 2 { + // (result, error) + result = results[0].Interface() + if !results[1].IsNil() { + err = results[1].Interface().(error) + } + } + if err != nil { + return nil, err + } + + return result, nil + } + + // 注册 RPC 服务 + var err error + if cfg.queueName != "" { + err = registerQueueRPCService(serviceName, cfg.queueName, handler) + } else { + err = registerRPCService(serviceName, handler) + } + + if err != nil { + g.Log().Errorf(context.Background(), "注册服务 %s 失败: %v", serviceName, err) + continue + } + + registeredCount++ + g.Log().Infof(context.Background(), "✅ 已自动注册 RPC 服务: %s -> %s", serviceName, method.Name) + } + + if registeredCount == 0 { + g.Log().Warningf(context.Background(), "未注册任何方法,请检查 %v 的方法签名", serviceNamePrefix) + return fmt.Errorf("未找到可注册的方法") + } + + g.Log().Infof(context.Background(), "✅ Service %v 共注册了 %d 个 RPC 方法", serviceNamePrefix, registeredCount) + return nil +} + +// ============ 上下文元数据工具函数 ============ +// 以下函数用于在 context 和 NATS 消息头之间互转元数据 + +// 定义常见的上下文元数据 key +const ( + TraceIDKey = "trace_id" + TokenKey = "token" +) + +func getTraceID(ctx context.Context) (traceID string, err error) { + // 提取 traceId:首先尝试从 OpenTelemetry Span 中提取,从 context 中提取 TraceID + span := trace.SpanFromContext(ctx) + if span != nil && span.SpanContext().HasTraceID() { + traceID = span.SpanContext().TraceID().String() + } else if tid := ctx.Value(TraceIDKey); tid != nil { + traceID = fmt.Sprintf("%v", tid) + } + if traceID == "" { + return traceID, fmt.Errorf("context 中没有 TraceID") + } + return +} + +// contextToHeaders 将 context 中的元数据转换为 NATS 消息头 +// 支持提取 user_id、tenant_id、trace_id、token 等常见字段 +func contextToHeaders(ctx context.Context) (nats.Header, error) { + headers := make(nats.Header) + + // 提取 traceId:首先尝试从 OpenTelemetry Span 中提取 + if traceID, err := getTraceID(ctx); err != nil { + return headers, err + } else { + headers.Set(TraceIDKey, traceID) + } + + // 提取 token(优先级:context value > HTTP Authorization header) + token := "" + if t := ctx.Value(TokenKey); t != nil { + token = fmt.Sprintf("%v", t) + } else if r := g.RequestFromCtx(ctx); r != nil { + // 从 HTTP 请求的 Authorization header 中提取 token + auth := r.GetHeader("Authorization") + if auth != "" { + // 移除 "Bearer " 前缀 + if len(auth) > 7 && auth[:7] == "Bearer " { + token = auth[7:] + } else { + token = auth + } + } + } + if token != "" { + headers.Set(TokenKey, token) + } + + return headers, nil +} + +// headersToContext 从 NATS 消息头重建 context +// 支持还原 user_id、tenant_id、trace_id、token 等字段 +func headersToContext(ctx context.Context, headers nats.Header) context.Context { + if headers == nil { + return ctx + } + + // 恢复 trace_id + if traceID := headers.Get(TraceIDKey); traceID != "" { + ctx = context.WithValue(ctx, TraceIDKey, traceID) + } + + // 恢复 token + if token := headers.Get(TokenKey); token != "" { + ctx = context.WithValue(ctx, TokenKey, token) + } + + return ctx +} From 2cfbad55a3c41027eb004bd792fa7562eb0b200e Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Thu, 29 Jan 2026 14:05:59 +0800 Subject: [PATCH 200/225] =?UTF-8?q?=E6=96=B0=E5=A2=9E=20RabbitMQ=20?= =?UTF-8?q?=E5=AE=A2=E6=88=B7=E7=AB=AF=E5=AE=9E=E7=8E=B0=EF=BC=8C=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E8=BF=9E=E6=8E=A5=E7=AE=A1=E7=90=86=E3=80=81=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E9=87=8D=E8=BF=9E=E5=92=8C=E4=BC=98=E9=9B=85=E5=85=B3?= =?UTF-8?q?=E9=97=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rabbitmq/rabbitmq_client.go | 210 ++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 rabbitmq/rabbitmq_client.go diff --git a/rabbitmq/rabbitmq_client.go b/rabbitmq/rabbitmq_client.go new file mode 100644 index 0000000..d6a4585 --- /dev/null +++ b/rabbitmq/rabbitmq_client.go @@ -0,0 +1,210 @@ +package rabbitmq + +import ( + "context" + "sync" + "time" + + "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" + amqp "github.com/rabbitmq/amqp091-go" +) + +var ( + conn *amqp.Connection + channel *amqp.Channel + rabbitmqOnce sync.Once + rabbitmqMu sync.RWMutex + closeWatcher chan struct{} // 用于停止监听 goroutine + watcherStarted bool // 防止重复启动监听 +) + +// Config RabbitMQ 配置 +type Config struct { + Host string + Port int + Username string + Password string + VHost string +} + +// Init 初始化 RabbitMQ 连接 +func Init(ctx context.Context, cfg *Config) error { + var err error + rabbitmqOnce.Do(func() { + // 构建连接字符串 + url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost + + // 创建连接 + conn, err = amqp.Dial(url) + if err != nil { + g.Log().Errorf(ctx, "RabbitMQ 连接失败: %v", err) + return + } + + // 创建 Channel + channel, err = conn.Channel() + if err != nil { + g.Log().Errorf(ctx, "创建 RabbitMQ Channel 失败: %v", err) + return + } + + // 初始化关闭监听器 + closeWatcher = make(chan struct{}) + + // 监听连接关闭(只启动一次) + if !watcherStarted { + go handleConnectionClose(ctx) + watcherStarted = true + } + + g.Log().Info(ctx, "RabbitMQ 连接成功") + }) + + return err +} + +// InitFromConfig 从配置文件初始化 +func InitFromConfig(ctx context.Context) error { + cfg := &Config{ + Host: g.Cfg().MustGet(ctx, "rabbitmq.host").String(), + Port: g.Cfg().MustGet(ctx, "rabbitmq.port").Int(), + Username: g.Cfg().MustGet(ctx, "rabbitmq.username").String(), + Password: g.Cfg().MustGet(ctx, "rabbitmq.password").String(), + VHost: g.Cfg().MustGet(ctx, "rabbitmq.vhost", "/").String(), + } + + return Init(ctx, cfg) +} + +// GetChannel 获取 Channel +func GetChannel() (*amqp.Channel, error) { + rabbitmqMu.RLock() + defer rabbitmqMu.RUnlock() + + if channel == nil || channel.IsClosed() { + return nil, gerror.New("RabbitMQ Channel 未初始化或已关闭") + } + + return channel, nil +} + +// GetConnection 获取连接 +func GetConnection() (*amqp.Connection, error) { + rabbitmqMu.RLock() + defer rabbitmqMu.RUnlock() + + if conn == nil || conn.IsClosed() { + return nil, gerror.New("RabbitMQ 连接未初始化或已关闭") + } + + return conn, nil +} + +// handleConnectionClose 监听连接关闭并重连 +func handleConnectionClose(ctx context.Context) { + for { + // 检查是否需要停止监听 + select { + case <-closeWatcher: + g.Log().Info(ctx, "停止监听 RabbitMQ 连接状态") + return + default: + } + + rabbitmqMu.RLock() + currentConn := conn + rabbitmqMu.RUnlock() + + if currentConn == nil { + return + } + + // 创建关闭通知 channel + closeErr := make(chan *amqp.Error, 1) + currentConn.NotifyClose(closeErr) + + // 等待连接关闭或停止信号 + select { + case err := <-closeErr: + if err != nil { + g.Log().Errorf(ctx, "RabbitMQ 连接关闭: %v,尝试重连...", err) + reconnect(ctx) + } + case <-closeWatcher: + g.Log().Info(ctx, "停止监听 RabbitMQ 连接状态") + return + } + } +} + +// reconnect 重新连接 +func reconnect(ctx context.Context) { + rabbitmqMu.Lock() + defer rabbitmqMu.Unlock() + + for i := 0; i < 10; i++ { + time.Sleep(time.Duration(i+1) * time.Second) + + cfg := &Config{ + Host: g.Cfg().MustGet(ctx, "rabbitmq.host").String(), + Port: g.Cfg().MustGet(ctx, "rabbitmq.port").Int(), + Username: g.Cfg().MustGet(ctx, "rabbitmq.username").String(), + Password: g.Cfg().MustGet(ctx, "rabbitmq.password").String(), + VHost: g.Cfg().MustGet(ctx, "rabbitmq.vhost", "/").String(), + } + + url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost + + var err error + conn, err = amqp.Dial(url) + if err != nil { + g.Log().Errorf(ctx, "重连失败 (尝试 %d/10): %v", i+1, err) + continue + } + + channel, err = conn.Channel() + if err != nil { + g.Log().Errorf(ctx, "创建 Channel 失败 (尝试 %d/10): %v", i+1, err) + continue + } + + g.Log().Info(ctx, "RabbitMQ 重连成功") + // 不再重复启动监听 goroutine + return + } + + g.Log().Fatal(ctx, "RabbitMQ 重连失败,已达到最大重试次数") +} + +// Close 关闭连接 +func Close(ctx context.Context) (err error) { + rabbitmqMu.Lock() + defer rabbitmqMu.Unlock() + + // 停止监听 goroutine + if closeWatcher != nil { + close(closeWatcher) + closeWatcher = nil + } + + if channel != nil { + if err = channel.Close(); err != nil { + g.Log().Errorf(ctx, "关闭 RabbitMQ Channel 失败: %v", err) + } + channel = nil + } + + if conn != nil { + if err = conn.Close(); err != nil { + g.Log().Errorf(ctx, "关闭 RabbitMQ 连接失败: %v", err) + return + } + conn = nil + } + + watcherStarted = false + g.Log().Info(ctx, "RabbitMQ 连接已关闭") + return +} From 0798dcde3724e019eceb77fff1c88f049cb35be0 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Thu, 29 Jan 2026 14:23:50 +0800 Subject: [PATCH 201/225] =?UTF-8?q?=E5=9C=A8NATS=E6=B6=88=E6=81=AF?= =?UTF-8?q?=E5=8F=91=E5=B8=83=E5=89=8D=E6=B7=BB=E5=8A=A0=E6=B5=81=E7=BB=84?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- message/nats_msg.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/message/nats_msg.go b/message/nats_msg.go index 519de96..016917f 100644 --- a/message/nats_msg.go +++ b/message/nats_msg.go @@ -79,6 +79,10 @@ func (q *natsMessageClient) publish(ctx context.Context, config interface{}, dat if !ok { return fmt.Errorf("无效的 NATS 配置类型") } + err := q.createStreamGroup(ctx, cfg) + if err != nil { + return err + } payload, err := json.Marshal(data) if err != nil { return fmt.Errorf("序列化数据失败: %w", err) From 9445a42eabf97083c2135c86c5e24909510b90c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 30 Jan 2026 17:31:54 +0800 Subject: [PATCH 202/225] .gitignore --- rag/{enio => eino}/base_task.go | 2 +- rag/eino/embedding_qwen.go | 273 ++++++++++++++++++++++++++++ rag/{enio => eino}/priority_enum.go | 2 +- rag/{enio => eino}/status_enum.go | 2 +- rag/{enio => eino}/task_type.go | 2 +- 5 files changed, 277 insertions(+), 4 deletions(-) rename rag/{enio => eino}/base_task.go (98%) create mode 100644 rag/eino/embedding_qwen.go rename rag/{enio => eino}/priority_enum.go (95%) rename rag/{enio => eino}/status_enum.go (96%) rename rag/{enio => eino}/task_type.go (98%) diff --git a/rag/enio/base_task.go b/rag/eino/base_task.go similarity index 98% rename from rag/enio/base_task.go rename to rag/eino/base_task.go index 866d5e2..06d6a86 100644 --- a/rag/enio/base_task.go +++ b/rag/eino/base_task.go @@ -1,4 +1,4 @@ -package enio +package eino import ( "time" diff --git a/rag/eino/embedding_qwen.go b/rag/eino/embedding_qwen.go new file mode 100644 index 0000000..9496874 --- /dev/null +++ b/rag/eino/embedding_qwen.go @@ -0,0 +1,273 @@ +/* + * Copyright 2024 Red Future Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package eino + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + "github.com/cloudwego/eino/components/embedding" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/gclient" + "github.com/gogf/gf/v2/util/gconv" +) + +var ( + // 千问API默认配置 + defaultBaseURL = "https://dashscope.aliyuncs.com/api/v1/services/embeddings/text-embedding" + defaultTimeout = 10 * time.Minute + defaultRetryTimes = 2 +) + +type QwenEmbeddingConfig struct { + // Timeout specifies the maximum duration to wait for API responses + // Optional. Default: 10 minutes + Timeout *time.Duration `json:"timeout"` + + // HTTPClient specifies the client to send HTTP requests. + // Optional. Default &http.Client{Timeout: Timeout} + HTTPClient *http.Client `json:"http_client"` + + // RetryTimes specifies the number of retry attempts for failed API calls + // Optional. Default: 2 + RetryTimes *int `json:"retry_times"` + + // BaseURL specifies the base URL for Qwen DashScope service + // Optional. Default: "https://dashscope.aliyuncs.com/api/v1/services/embeddings/text-embedding" + BaseURL string `json:"base_url"` + + // APIKey specifies the API Key for authentication + // Required + APIKey string `json:"api_key"` + + // Model specifies the model name for Qwen embedding + // Required. Examples: "text-embedding-v2", "text-embedding-v3" + Model string `json:"model"` + + // TextType specifies the type of text: "document" or "query" + // Optional. Default: "document" + TextType string `json:"text_type"` + + // MaxConcurrentRequests specifies the maximum number of concurrent requests allowed + // Optional. Default: 5 + MaxConcurrentRequests *int `json:"max_concurrent_requests"` +} + +type QwenEmbedder struct { + client *gclient.Client + conf *QwenEmbeddingConfig +} + +// EmbeddingRequest 千问embedding请求结构 +type EmbeddingRequest struct { + Model string `json:"model"` + Input struct { + Texts []string `json:"texts"` + } `json:"input"` + Parameters struct { + TextType string `json:"text_type,omitempty"` + } `json:"parameters,omitempty"` +} + +// EmbeddingResponse 千问embedding响应结构 +type EmbeddingResponse struct { + Output struct { + Embeddings []struct { + TextIndex int `json:"text_index"` + Embedding []float64 `json:"embedding"` + } `json:"embeddings"` + } `json:"output"` + Usage struct { + TotalTokens int `json:"total_tokens"` + } `json:"usage"` + RequestID string `json:"request_id"` +} + +type APIError struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +func (e *APIError) Error() string { + return fmt.Sprintf("API Error: %s - %s (RequestID: %s)", e.Code, e.Message, e.RequestID) +} + +func buildQwenClient(config *QwenEmbeddingConfig) *gclient.Client { + if len(config.BaseURL) == 0 { + config.BaseURL = defaultBaseURL + } + if config.Timeout == nil { + config.Timeout = &defaultTimeout + } + if config.RetryTimes == nil { + defaultRetryTimes := 2 + config.RetryTimes = &defaultRetryTimes + } + if len(config.TextType) == 0 { + config.TextType = "document" + } + if config.MaxConcurrentRequests == nil { + defaultMaxConcurrentRequests := 5 + config.MaxConcurrentRequests = &defaultMaxConcurrentRequests + } + + client := g.Client() + client.SetTimeout(*config.Timeout) + + return client +} + +func NewQwenEmbedder(ctx context.Context, config *QwenEmbeddingConfig) (*QwenEmbedder, error) { + if len(config.APIKey) == 0 { + return nil, fmt.Errorf("[Qwen] APIKey is required") + } + if len(config.Model) == 0 { + return nil, fmt.Errorf("[Qwen] Model is required") + } + + client := buildQwenClient(config) + + return &QwenEmbedder{ + client: client, + conf: config, + }, nil +} + +func (e *QwenEmbedder) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ( + [][]float64, error) { + + if len(texts) == 0 { + return nil, fmt.Errorf("[Qwen] texts cannot be empty") + } + + options := embedding.GetCommonOptions(&embedding.Options{ + Model: &e.conf.Model, + }, opts...) + + conf := &embedding.Config{ + Model: dereferenceOrZero(options.Model), + } + + ctx = callbacks.EnsureRunInfo(ctx, e.GetType(), components.ComponentOfEmbedding) + ctx = callbacks.OnStart(ctx, &embedding.CallbackInput{ + Texts: texts, + Config: conf, + }) + defer func() { + if err := recover(); err != nil { + callbacks.OnError(ctx, fmt.Errorf("[Qwen] panic: %v", err)) + } + }() + + var usage *embedding.TokenUsage + var embeddings [][]float64 + var err error + + // 调用千问API获取embedding + embeddings, usage, err = e.callEmbeddingAPI(ctx, texts) + if err != nil { + callbacks.OnError(ctx, err) + return nil, err + } + + callbacks.OnEnd(ctx, &embedding.CallbackOutput{ + Embeddings: embeddings, + Config: conf, + TokenUsage: usage, + }) + + return embeddings, nil +} + +func (e *QwenEmbedder) callEmbeddingAPI(ctx context.Context, texts []string) ([][]float64, *embedding.TokenUsage, error) { + // 构建请求 + var req EmbeddingRequest + req.Model = e.conf.Model + req.Input.Texts = texts + req.Parameters.TextType = e.conf.TextType + + // 调用API + client := e.client.Clone() + client.SetHeader("Authorization", "Bearer "+e.conf.APIKey) + client.SetHeader("Content-Type", "application/json") + client.SetTimeout(*e.conf.Timeout) + + resp, err := client.Post(ctx, e.conf.BaseURL, req) + if err != nil { + return nil, nil, fmt.Errorf("[Qwen] HTTP request error: %w", err) + } + + defer resp.Close() + + // 检查状态码 + if resp.StatusCode != http.StatusOK { + var errResp APIError + result := resp.ReadAll() + if err = gconv.Struct(result, &errResp); err == nil && errResp.Code != "" { + return nil, nil, &errResp + } + return nil, nil, fmt.Errorf("[Qwen] HTTP status error: %d", resp.StatusCode) + } + + // 解析响应 + var apiResp EmbeddingResponse + result := resp.ReadAll() + if err = gconv.Struct(result, &apiResp); err != nil { + return nil, nil, fmt.Errorf("[Qwen] parse response error: %w", err) + } + + // 解析响应结果 + embeddings := make([][]float64, len(texts)) + for _, emb := range apiResp.Output.Embeddings { + if emb.TextIndex >= 0 && emb.TextIndex < len(embeddings) { + embeddings[emb.TextIndex] = emb.Embedding + } + } + + usage := &embedding.TokenUsage{ + TotalTokens: apiResp.Usage.TotalTokens, + } + + g.Log().Debugf(ctx, "[Qwen] Embedding success: request_id=%s, total_tokens=%d", apiResp.RequestID, usage.TotalTokens) + + return embeddings, usage, nil +} + +func (e *QwenEmbedder) GetType() string { + return getType() +} + +func (e *QwenEmbedder) IsCallbacksEnabled() bool { + return true +} + +func getType() string { + return "Qwen" +} + +func dereferenceOrZero[T any](v *T) T { + if v == nil { + var t T + return t + } + return *v +} diff --git a/rag/enio/priority_enum.go b/rag/eino/priority_enum.go similarity index 95% rename from rag/enio/priority_enum.go rename to rag/eino/priority_enum.go index 365903a..371706b 100644 --- a/rag/enio/priority_enum.go +++ b/rag/eino/priority_enum.go @@ -1,4 +1,4 @@ -package enio +package eino // TaskPriority 任务优先级 type TaskPriority string diff --git a/rag/enio/status_enum.go b/rag/eino/status_enum.go similarity index 96% rename from rag/enio/status_enum.go rename to rag/eino/status_enum.go index d6d2479..6e12daf 100644 --- a/rag/enio/status_enum.go +++ b/rag/eino/status_enum.go @@ -1,4 +1,4 @@ -package enio +package eino // TaskStatus 任务状态 type TaskStatus string diff --git a/rag/enio/task_type.go b/rag/eino/task_type.go similarity index 98% rename from rag/enio/task_type.go rename to rag/eino/task_type.go index 4dec33f..0ba5a64 100644 --- a/rag/enio/task_type.go +++ b/rag/eino/task_type.go @@ -1,4 +1,4 @@ -package enio +package eino // TaskType 任务类型 type TaskType string From 6acdbb6e8839f6a715110033d591d67ba9012365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 30 Jan 2026 17:33:03 +0800 Subject: [PATCH 203/225] .gitignore --- nats/msg.go | 65 ---- nats/nats.go | 268 --------------- nats/nats_client.go | 313 ------------------ nats/nats_consumer.go | 294 ----------------- nats/nats_publish.go | 28 -- nats/nats_rpc.go | 752 ------------------------------------------ nats/nats_task.go | 212 ------------ nats/nats_test.go | 131 -------- nats/task.go | 411 ----------------------- nats/utils.go | 87 ----- 10 files changed, 2561 deletions(-) delete mode 100644 nats/msg.go delete mode 100644 nats/nats.go delete mode 100644 nats/nats_client.go delete mode 100644 nats/nats_consumer.go delete mode 100644 nats/nats_publish.go delete mode 100644 nats/nats_rpc.go delete mode 100644 nats/nats_task.go delete mode 100644 nats/nats_test.go delete mode 100644 nats/task.go delete mode 100644 nats/utils.go diff --git a/nats/msg.go b/nats/msg.go deleted file mode 100644 index 5929241..0000000 --- a/nats/msg.go +++ /dev/null @@ -1,65 +0,0 @@ -package nats - -import ( - "context" - "github.com/gogf/gf/v2/errors/gerror" -) - -// NatsMessageConfig nats Stream 消息配置 -type NatsMessageConfig struct { - CreateTaskStreamName string - CreateTaskSubjects []string - PublishSubject string - CreateTaskConsumerName string - MsgCount int - HandleFunc func(ctx context.Context, message map[string]interface{}) error -} - -// MessageConfig 消息配置接口 -type MessageConfig interface { - createTaskStream(ctx context.Context) error - publish(ctx context.Context, data interface{}) error - createTaskConsumer(ctx context.Context) error - //startConsumer(ctx context.Context, handleFunc func(ctx context.Context, msg *nats.Msg) error) error -} - -func (n *NatsMessageConfig) createTaskStream(ctx context.Context) error { - return createTaskStreamSimple(ctx, n.CreateTaskStreamName, n.CreateTaskSubjects) -} - -// CreateTaskStreamBatch 批量创建任务消息队列流 -func CreateTaskStreamBatch(ctx context.Context, configs ...MessageConfig) error { - for _, cfg := range configs { - if err := cfg.createTaskStream(ctx); err != nil { - return gerror.Wrap(err, "创建任务消息队列流失败") - } - } - return nil -} - -func (n *NatsMessageConfig) publish(ctx context.Context, data interface{}) error { - return publish(ctx, n.PublishSubject, data) -} - -// PublishMessage 发布消息(统一入口) -func PublishMessage(ctx context.Context, cfg MessageConfig, data interface{}) (err error) { - return cfg.publish(ctx, data) -} - -func (n *NatsMessageConfig) createTaskConsumer(ctx context.Context) error { - return CreateConsumerPushMode(ctx, n.CreateTaskStreamName, n.CreateTaskConsumerName, n.PublishSubject, n.MsgCount) -} - -// CreateTaskConsumerBatch 批量创建任务消息队列消费者 -func CreateTaskConsumerBatch(ctx context.Context, configs ...MessageConfig) error { - for _, cfg := range configs { - if err := cfg.createTaskConsumer(ctx); err != nil { - return gerror.Wrap(err, "创建任务消息队列流失败") - } - } - return nil -} - -//func (n *NatsMessageConfig) startConsumer(ctx context.Context, handleFunc func(ctx context.Context, msg *nats.Msg) error) error { -// return ConsumeMessages(ctx, n.CreateTaskStreamName, n.CreateTaskConsumerName, handleFunc) -//} diff --git a/nats/nats.go b/nats/nats.go deleted file mode 100644 index d01886e..0000000 --- a/nats/nats.go +++ /dev/null @@ -1,268 +0,0 @@ -package nats - -import ( - "context" - "encoding/json" - "fmt" - "time" - - "github.com/gogf/gf/v2/frame/g" - "github.com/nats-io/nats.go/jetstream" -) - -// createTaskStream 创建任务消息队列流(内部使用,兼容旧版本) -// 存储策略: 文件存储 -// 工作队列模式: 工作队列策略 -func CreateTaskStream(ctx context.Context, streamInfo TaskStreamConfig) error { - if !IsConnected() { - return fmt.Errorf("NATS 未连接") - } - - stream, err := js.Stream(ctx, streamInfo.StreamName) - if err == nil { - // 流已存在,更新配置 - _, err = js.UpdateStream(ctx, jetstream.StreamConfig{ - Name: streamInfo.StreamName, - Subjects: streamInfo.Subjects, - Storage: jetstream.FileStorage, - Retention: jetstream.WorkQueuePolicy, - }) - if err != nil { - return fmt.Errorf("更新任务流失败: %w", err) - } - g.Log().Infof(ctx, "✅ 任务消息队列流已更新: %s", stream.CachedInfo().Config.Name) - return nil - } - - // 创建新流 - stream, err = js.CreateStream(ctx, jetstream.StreamConfig{ - Name: streamInfo.StreamName, - Subjects: streamInfo.Subjects, - Storage: jetstream.FileStorage, - Retention: jetstream.WorkQueuePolicy, - }) - if err != nil { - return fmt.Errorf("创建任务流失败: %w", err) - } - - g.Log().Infof(ctx, "✅ 任务消息队列流创建成功: %s (文件存储+工作队列策略)", stream.CachedInfo().Config.Name) - return nil -} - -// CreateLogStream 创建日志流 -// 存储策略: 内存存储 -// 副本数: 单副本 (1) -// 消息留存: 短时留存 (1小时) -func CreateLogStream(ctx context.Context, streamName string, subjects []string) error { - if !IsConnected() { - return fmt.Errorf("NATS 未连接") - } - - maxAge := 1 * time.Hour - - stream, err := js.Stream(ctx, streamName) - if err == nil { - // 流已存在,更新配置 - _, err = js.UpdateStream(ctx, jetstream.StreamConfig{ - Name: streamName, - Subjects: subjects, - Storage: jetstream.MemoryStorage, - Replicas: 1, - MaxAge: maxAge, - }) - if err != nil { - return fmt.Errorf("更新日志流失败: %w", err) - } - g.Log().Infof(ctx, "✅ 日志流已更新: %s", stream.CachedInfo().Config.Name) - return nil - } - - // 创建新流 - stream, err = js.CreateStream(ctx, jetstream.StreamConfig{ - Name: streamName, - Subjects: subjects, - Storage: jetstream.MemoryStorage, - Replicas: 1, - MaxAge: maxAge, - }) - if err != nil { - return fmt.Errorf("创建日志流失败: %w", err) - } - - g.Log().Infof(ctx, "✅ 日志流创建成功: %s (内存存储+单副本+短时留存1小时)", stream.CachedInfo().Config.Name) - return nil -} - -// CreateTradeStream 创建交易业务流 -// 存储策略: 文件存储 -// 副本数: 3副本 -// 同步刷盘: 启用 -func CreateTradeStream(ctx context.Context, streamName string, subjects []string) error { - if !IsConnected() { - return fmt.Errorf("NATS 未连接") - } - - stream, err := js.Stream(ctx, streamName) - if err == nil { - // 流已存在,更新配置 - _, err = js.UpdateStream(ctx, jetstream.StreamConfig{ - Name: streamName, - Subjects: subjects, - Storage: jetstream.FileStorage, - Replicas: 3, - RePublish: nil, - Duplicates: 0, - }) - if err != nil { - return fmt.Errorf("更新交易流失败: %w", err) - } - g.Log().Infof(ctx, "✅ 交易业务流已更新: %s", stream.CachedInfo().Config.Name) - return nil - } - - // 创建新流 - stream, err = js.CreateStream(ctx, jetstream.StreamConfig{ - Name: streamName, - Subjects: subjects, - Storage: jetstream.FileStorage, - Replicas: 3, - RePublish: nil, - Duplicates: 0, - }) - if err != nil { - return fmt.Errorf("创建交易流失败: %w", err) - } - - g.Log().Infof(ctx, "✅ 交易业务流创建成功: %s (文件存储+3副本+同步刷盘)", stream.CachedInfo().Config.Name) - return nil -} - -// JsPublish 发布消息到指定主题 -func JsPublish(ctx context.Context, subject string, data any) (err error) { - if !IsConnected() { - return fmt.Errorf("NATS 未连接") - } - // 序列化数据 - dataBytes, err := json.Marshal(data) - if err != nil { - return fmt.Errorf("序列化数据失败: %w", err) - } - // 发布消息 - metrics.PublishCount.Add(1) - _, err = js.Publish(ctx, subject, dataBytes) - if err != nil { - metrics.PublishError.Add(1) - return fmt.Errorf("发布消息失败: %w", err) - } - - return -} - -// GetStream 获取流信息 -func GetStream(ctx context.Context, streamName string) (*jetstream.StreamInfo, error) { - if !IsConnected() { - return nil, fmt.Errorf("NATS 未连接") - } - - stream, err := js.Stream(ctx, streamName) - if err != nil { - return nil, fmt.Errorf("获取流失败: %w", err) - } - - info, err := stream.Info(ctx) - if err != nil { - return nil, fmt.Errorf("获取流信息失败: %w", err) - } - - return info, nil -} - -// ListStreams 列出所有流(简化实现) -// 注意:由于 API 限制,此方法可能需要根据实际需求进一步实现 -func ListStreams(ctx context.Context) ([]string, error) { - if !IsConnected() { - return nil, fmt.Errorf("NATS 未连接") - } - - // TODO: 根据实际 NATS 版本实现完整的流列表功能 - return []string{}, nil -} - -// DeleteStream 删除流 -func DeleteStream(ctx context.Context, streamName string) error { - if !IsConnected() { - return fmt.Errorf("NATS 未连接") - } - - if err := js.DeleteStream(ctx, streamName); err != nil { - return fmt.Errorf("删除流失败: %w", err) - } - - g.Log().Infof(ctx, "✅ 流已删除: %s", streamName) - return nil -} - -// GetConsumer 获取消费者信息 -func GetConsumer(ctx context.Context, streamName, consumerName string) (*jetstream.ConsumerInfo, error) { - if !IsConnected() { - return nil, fmt.Errorf("NATS 未连接") - } - - consumer, err := js.Consumer(ctx, streamName, consumerName) - if err != nil { - return nil, fmt.Errorf("获取消费者失败: %w", err) - } - - info, err := consumer.Info(ctx) - if err != nil { - return nil, fmt.Errorf("获取消费者信息失败: %w", err) - } - - return info, nil -} - -// ListConsumers 列出指定流的所有消费者(简化实现) -// 注意:由于 API 限制,此方法可能需要根据实际需求进一步实现 -func ListConsumers(ctx context.Context, streamName string) ([]string, error) { - if !IsConnected() { - return nil, fmt.Errorf("NATS 未连接") - } - - // TODO: 根据实际 NATS 版本实现完整的消费者列表功能 - return []string{}, nil -} - -// DeleteConsumer 删除消费者 -func DeleteConsumer(ctx context.Context, streamName, consumerName string) error { - if !IsConnected() { - return fmt.Errorf("NATS 未连接") - } - - if err := js.DeleteConsumer(ctx, streamName, consumerName); err != nil { - return fmt.Errorf("删除消费者失败: %w", err) - } - - g.Log().Infof(ctx, "✅ 消费者已删除: %s/%s", streamName, consumerName) - return nil -} - -// CreateConsumer 创建消费者 -func CreateConsumer(ctx context.Context, streamName, consumerName string, config jetstream.ConsumerConfig) (jetstream.Consumer, error) { - if !IsConnected() { - return nil, fmt.Errorf("NATS 未连接") - } - - // 尝试获取现有消费者 - consumer, err := js.Consumer(ctx, streamName, consumerName) - if err == nil { - return consumer, nil - } - - // 推荐:不存在则创建,存在则更新配置 - consumer, err = js.CreateOrUpdateConsumer(ctx, streamName, config) - if err != nil { - return nil, fmt.Errorf("创建消费者失败: %w", err) - } - - return consumer, nil -} diff --git a/nats/nats_client.go b/nats/nats_client.go deleted file mode 100644 index a4afebb..0000000 --- a/nats/nats_client.go +++ /dev/null @@ -1,313 +0,0 @@ -package nats - -import ( - "context" - "fmt" - "sync" - "sync/atomic" - "time" - - "github.com/gogf/gf/v2/frame/g" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" -) - -var ( - nc *nats.Conn - js jetstream.JetStream - inited bool - natsMu sync.RWMutex - natsURL string - healthCtx context.Context - healthCancel context.CancelFunc - connected bool - reconnectChan chan struct{} - - // 连接状态变化监听器 - connStateListeners []connStateListener - connListenersMu sync.RWMutex - - // 监控指标 - metrics metricsCounter -) - -// Metrics 监控指标 -type metricsCounter struct { - PublishCount atomic.Int64 - PublishError atomic.Int64 - SubscribeCount atomic.Int64 - RequestCount atomic.Int64 - RequestError atomic.Int64 - ConsumeCount atomic.Int64 - ConsumeError atomic.Int64 -} - -// ConnState 连接状态 -type connState int - -const ( - connStateDisconnected connState = iota - connStateConnecting - connStateConnected - connStateReconnecting - connStateClosed -) - -// ConnStateListener 连接状态监听器 -type connStateListener func(state connState, err error) - -// GetMetrics 获取监控指标 -func getMetrics() metricsCounter { - return metrics -} - -// registerConnStateListener 注册连接状态监听器 -func registerConnStateListener(listener connStateListener) { - connListenersMu.Lock() - defer connListenersMu.Unlock() - connStateListeners = append(connStateListeners, listener) -} - -// unregisterConnStateListener 取消注册连接状态监听器 -func unregisterConnStateListener(listener connStateListener) { - connListenersMu.Lock() - defer connListenersMu.Unlock() - for i, l := range connStateListeners { - if l != nil && &l == &listener { - connStateListeners = append(connStateListeners[:i], connStateListeners[i+1:]...) - break - } - } -} - -// notifyConnState 通知所有监听器连接状态变化 -func notifyConnState(state connState, err error) { - connListenersMu.RLock() - listeners := make([]connStateListener, len(connStateListeners)) - copy(listeners, connStateListeners) - connListenersMu.RUnlock() - - for _, listener := range listeners { - if listener != nil { - listener(state, err) - } - } -} - -// init 初始化 NATS 连接 -func init() { - // 从配置文件读取 NATS 地址 - natsURL = g.Cfg().MustGet(context.Background(), "nats.url").String() - if natsURL == "" { - // 默认使用本地地址 - natsURL = nats.DefaultURL - } - - // 创建健康检查上下文 - healthCtx, healthCancel = context.WithCancel(context.Background()) - - // 创建重连通知通道(增大缓冲区避免丢失通知) - reconnectChan = make(chan struct{}, 10) - - // 启动连接 - go initConnection() - - // 启动健康检查协程 - go healthCheck() -} - -// initConnection 初始化连接 -func initConnection() { - ctx := context.Background() - notifyConnState(connStateConnecting, nil) - if err := connect(ctx); err != nil { - g.Log().Errorf(ctx, "NATS 初始连接失败: %v", err) - notifyConnState(connStateDisconnected, err) - } -} - -// connect 建立 NATS 连接 -func connect(ctx context.Context) error { - natsMu.Lock() - defer natsMu.Unlock() - - if nc != nil && !nc.IsClosed() { - nc.Close() - } - - // 连接选项配置 - opts := []nats.Option{ - nats.Name("goframe-nats-client"), - nats.ReconnectWait(2 * time.Second), - nats.MaxReconnects(-1), // 无限重连 - nats.PingInterval(10 * time.Second), - nats.MaxPingsOutstanding(5), - nats.ReconnectHandler(func(nc *nats.Conn) { - g.Log().Infof(ctx, "✅ NATS 重连成功: %s", nc.ConnectedUrl()) - connected = true - - // 重新创建 JetStream 实例 - if newJS, err := jetstream.New(nc); err == nil { - js = newJS - } - - // 通知重连成功 - notifyConnState(connStateConnected, nil) - - // 使用非阻塞发送避免阻塞 - select { - case reconnectChan <- struct{}{}: - default: - // 通道已满,丢弃通知 - } - }), - nats.DisconnectErrHandler(func(nc *nats.Conn, err error) { - g.Log().Warningf(ctx, "⚠️ NATS 连接断开: %v, 准备重连...", err) - connected = false - notifyConnState(connStateReconnecting, err) - }), - nats.ClosedHandler(func(nc *nats.Conn) { - g.Log().Infof(ctx, "NATS 连接已关闭: %s", nc.ConnectedUrl()) - connected = false - notifyConnState(connStateClosed, nil) - }), - nats.ErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { - g.Log().Errorf(ctx, "NATS 错误: %v", err) - }), - } - - var err error - nc, err = nats.Connect(natsURL, opts...) - if err != nil { - return fmt.Errorf("NATS 连接失败: %w", err) - } - - // 等待连接就绪 - if nc.Status() != nats.CONNECTED { - select { - case <-time.After(5 * time.Second): - notifyConnState(connStateDisconnected, fmt.Errorf("连接超时")) - return fmt.Errorf("NATS 连接超时") - case <-nc.StatusChanged(nats.CONNECTED): - } - } - - // 创建 JetStream 实例 - js, err = jetstream.New(nc) - if err != nil { - return fmt.Errorf("创建 JetStream 失败: %w", err) - } - - connected = true - inited = true - g.Log().Infof(ctx, "✅ NATS 连接成功: %s", nc.ConnectedUrl()) - notifyConnState(connStateConnected, nil) - return nil -} - -// healthCheck 健康检查协程(仅作为备用检查) -func healthCheck() { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - - for { - select { - case <-healthCtx.Done(): - return - case <-ticker.C: - natsMu.RLock() - currentConnected := connected - currentConn := nc - natsMu.RUnlock() - - if !currentConnected || currentConn == nil || currentConn.IsClosed() { - // 仅记录日志,不尝试重连(NATS 已有自动重连机制) - g.Log().Warning(context.Background(), "NATS 连接断开,等待 NATS 自动重连...") - } - case <-reconnectChan: - // 重连成功的通知(仅记录日志) - g.Log().Info(context.Background(), "收到重连成功通知") - } - } -} - -// checkConnected 检查连接状态 -func checkConnected() bool { - natsMu.RLock() - defer natsMu.RUnlock() - return connected && nc != nil && !nc.IsClosed() -} - -// getConnState 获取当前连接状态 -func getConnState() connState { - natsMu.RLock() - defer natsMu.RUnlock() - - if nc == nil { - return connStateDisconnected - } - - if nc.IsClosed() { - return connStateClosed - } - - if connected { - return connStateConnected - } - - return connStateDisconnected -} - -// shutdown 优雅关闭:自动注销所有已注册的服务并关闭 NATS 连接 -func shutdown() error { - ctx := context.Background() - g.Log().Info(ctx, "开始优雅关闭 NATS RPC 服务...") - - // 注销所有单实例服务 - rpcServicesMu.Lock() - singleServiceCount := len(rpcServices) - for serviceName := range rpcServices { - if sub, exists := rpcSubs[serviceName]; exists { - if err := sub.Unsubscribe(); err != nil { - g.Log().Errorf(ctx, "注销服务 %s 失败: %v", serviceName, err) - } - } - delete(rpcSubs, serviceName) - delete(rpcServices, serviceName) - } - rpcServicesMu.Unlock() - - // 注销所有队列服务 - queueRPCMu.Lock() - queueServiceCount := 0 - for queueName, servicesMap := range queueRPCServices { - queueServiceCount += len(servicesMap) - for serviceName, sub := range queueRPCSubs[queueName] { - if err := sub.Unsubscribe(); err != nil { - g.Log().Errorf(ctx, "注销队列服务 %s (队列: %s) 失败: %v", serviceName, queueName, err) - } - } - delete(queueRPCSubs, queueName) - delete(queueRPCServices, queueName) - } - queueRPCMu.Unlock() - - g.Log().Infof(ctx, "已注销 %d 个单实例服务和 %d 个队列服务", singleServiceCount, queueServiceCount) - - natsMu.Lock() - defer natsMu.Unlock() - - // 停止健康检查协程 - if healthCancel != nil { - healthCancel() - } - - // 关闭连接 - if nc != nil && !nc.IsClosed() { - nc.Close() - connected = false - inited = false - } - g.Log().Info(ctx, "NATS RPC 服务已优雅关闭") - return nil -} diff --git a/nats/nats_consumer.go b/nats/nats_consumer.go deleted file mode 100644 index 126000b..0000000 --- a/nats/nats_consumer.go +++ /dev/null @@ -1,294 +0,0 @@ -package nats - -import ( - "context" - "fmt" - "github.com/gogf/gf/v2/frame/g" - "github.com/nats-io/nats.go/jetstream" -) - -// AckPolicy 确认策略 -type AckPolicy string - -const ( - AckPolicyExplicit AckPolicy = "explicit" // 显式确认(默认) - AckPolicyAll AckPolicy = "all" // 确认所有消息 - AckPolicyNone AckPolicy = "none" // 不需要确认 -) - -// DeliverPolicy 投递策略 -type DeliverPolicy string - -const ( - DeliverPolicyAll DeliverPolicy = "all" // 投递所有消息(包括已投递的) - DeliverPolicyLast DeliverPolicy = "last" // 从最后一条消息开始 - DeliverPolicyNew DeliverPolicy = "new" // 仅投递新消息(默认) - DeliverPolicyLastPerSubj DeliverPolicy = "lastpersubj" // 每个主题的最后一条 - DeliverPolicyByStartSeq DeliverPolicy = "by_start_sequence" // 按起始序列号 -) - -// ReplayPolicy 重放策略 -type ReplayPolicy string - -const ( - ReplayPolicyInstant ReplayPolicy = "instant" // 立即重放 - ReplayPolicyOriginal ReplayPolicy = "original" // 按原始顺序重放 -) - -// ConsumerConfig 消费者配置 -type ConsumerConfig struct { - DurableName string // 持久化名称(空表示临时消费者) - Description string // 描述信息 - AckPolicy AckPolicy // 确认策略 - AckWait int // 确认等待时间(秒) - MaxDeliver int // 最大投递次数 - FilterSubject string // 过滤主题(流内多主题时使用) - DeliverPolicy DeliverPolicy // 投递策略 - ReplayPolicy ReplayPolicy // 重放策略 - MaxWaiting int // 最大等待消息数 - MaxAckPending int // 最大待确认消息数 - OptStartTime int64 // 起始时间戳 - OptStartSeq uint64 // 起始序列号 - HeadersOnly bool // 仅消费消息头 - Backoff []int // 退避策略(秒数数组) - RateLimit uint64 // 消息速率限制(消息/秒) - Replica int // 副本数 - FlowControl bool // 启用流控 - Metadata map[string]string // 元数据 -} - -// parseAckPolicy 解析确认策略 -func parseAckPolicy(policy AckPolicy) jetstream.AckPolicy { - switch policy { - case AckPolicyAll: - return jetstream.AckAllPolicy - case AckPolicyNone: - return jetstream.AckNonePolicy - default: - return jetstream.AckExplicitPolicy - } -} - -// parseDeliverPolicy 解析投递策略 -func parseDeliverPolicy(policy DeliverPolicy) jetstream.DeliverPolicy { - switch policy { - case DeliverPolicyAll: - return jetstream.DeliverAllPolicy - case DeliverPolicyLast: - return jetstream.DeliverLastPolicy - case DeliverPolicyLastPerSubj: - return jetstream.DeliverLastPerSubjectPolicy - case DeliverPolicyByStartSeq: - return jetstream.DeliverByStartSequencePolicy - default: - return jetstream.DeliverNewPolicy - } -} - -// parseReplayPolicy 解析重放策略 -func parseReplayPolicy(policy ReplayPolicy) jetstream.ReplayPolicy { - switch policy { - case ReplayPolicyOriginal: - return jetstream.ReplayOriginalPolicy - default: - return jetstream.ReplayInstantPolicy - } -} - -// CreateTaskConsumer 创建任务消费者 -// 核心设计思路: -// 1. 显式确认:确保消息被正确处理后才确认 -// 2. 重试机制:通过 MaxDeliver 控制最大重试次数 -// 3. 持久化:DurableName 确保消费者状态持久化 -// 4. 流控:防止消费者过载 -func CreateTaskConsumer(ctx context.Context, streamName string, config ConsumerConfig) (jetstream.Consumer, error) { - if !IsConnected() { - return nil, fmt.Errorf("NATS 未连接") - } - - if streamName == "" { - return nil, fmt.Errorf("流名称不能为空") - } - - // 设置默认值 - if config.AckPolicy == "" { - config.AckPolicy = AckPolicyExplicit // 默认显式确认 - } - if config.AckWait == 0 { - config.AckWait = 30 // 默认30秒确认超时 - } - if config.MaxDeliver == 0 { - config.MaxDeliver = 3 // 默认最多投递3次 - } - if config.DeliverPolicy == "" { - config.DeliverPolicy = DeliverPolicyNew // 默认仅消费新消息 - } - if config.ReplayPolicy == "" { - config.ReplayPolicy = ReplayPolicyInstant // 默认立即重放 - } - if config.MaxAckPending == 0 { - config.MaxAckPending = 1000 // 默认最多1000条待确认消息 - } - - // 构建消费者配置 - jsConfig := jetstream.ConsumerConfig{ - Name: config.DurableName, - Description: config.Description, - AckPolicy: parseAckPolicy(config.AckPolicy), - AckWait: 0, - MaxDeliver: config.MaxDeliver, - FilterSubjects: []string{config.FilterSubject}, - DeliverPolicy: parseDeliverPolicy(config.DeliverPolicy), - ReplayPolicy: parseReplayPolicy(config.ReplayPolicy), - MaxWaiting: config.MaxWaiting, - MaxAckPending: config.MaxAckPending, - HeadersOnly: config.HeadersOnly, - RateLimit: config.RateLimit, - Replicas: config.Replica, - Metadata: config.Metadata, - } - - // 配置流控和心跳 - if config.FlowControl { - jsConfig.FlowControl = true - } - // 配置起始位置 - if config.OptStartSeq > 0 { - jsConfig.OptStartSeq = config.OptStartSeq - } - - // 创建新消费者 - consumer, err := js.CreateOrUpdateConsumer(ctx, streamName, jsConfig) - if err != nil { - return nil, fmt.Errorf("创建消费者失败: %w", err) - } - - // 记录配置信息 - configInfo := fmt.Sprintf("确认策略=%s, 最大投递=%d, 投递策略=%s", config.AckPolicy, config.MaxDeliver, config.DeliverPolicy) - if config.FilterSubject != "" { - configInfo += fmt.Sprintf(", 过滤主题=%s", config.FilterSubject) - } - g.Log().Infof(ctx, "✅ 任务消费者创建成功: %s/%s (%s)", streamName, config.DurableName, configInfo) - - return consumer, nil -} - -// CreateConsumerSimple 简化版创建消费者(适用于大多数场景) -// 只需提供流名称和消费者名称,其他使用默认配置 -func CreateConsumerSimple(ctx context.Context, streamName, durableName string) (err error) { - _, err = CreateTaskConsumer(ctx, streamName, ConsumerConfig{ - DurableName: durableName, - }) - return -} - -// CreateConsumerWithFilter 创建带主题过滤的消费者 -//func CreateConsumerWithFilter(ctx context.Context, streamName, durableName, filterSubject string) (jetstream.Consumer, error) { -// return CreateTaskConsumer(ctx, streamName, ConsumerConfig{ -// DurableName: durableName, -// FilterSubject: filterSubject, -// }) -//} - -// CreateConsumerEphemeral 创建临时消费者 -// 临时消费者没有持久化名称,连接断开后自动删除 -//func CreateConsumerEphemeral(ctx context.Context, streamName string) (jetstream.Consumer, error) { -// if !IsConnected() { -// return nil, fmt.Errorf("NATS 未连接") -// } -// -// jsConfig := jetstream.ConsumerConfig{ -// AckPolicy: jetstream.AckNonePolicy, -// AckWait: 0, -// MaxDeliver: 3, -// DeliverPolicy: jetstream.DeliverNewPolicy, -// ReplayPolicy: jetstream.ReplayInstantPolicy, -// MaxAckPending: 1000, -// } -// -// consumer, err := js.CreateConsumer(ctx, streamName, jsConfig) -// if err != nil { -// return nil, fmt.Errorf("创建临时消费者失败: %w", err) -// } -// -// g.Log().Infof(ctx, "✅ 临时消费者创建成功: %s", streamName) -// return consumer, nil -//} - -// CreateConsumerPushMode 创建推送模式消费者 -// 推送模式下,NATS 服务器主动将消息推送给消费者 -func CreateConsumerPushMode(ctx context.Context, streamName, durableName, subject string, msgCount int) (err error) { - _, err = CreateTaskConsumer(ctx, streamName, ConsumerConfig{ - DurableName: durableName, - FilterSubject: subject, - MaxAckPending: msgCount, - }) - return -} - -// CreateConsumerPullMode 创建拉取模式消费者 -// 拉取模式下,消费者主动从服务器拉取消息 -//func CreateConsumerPullMode(ctx context.Context, streamName, durableName string) (jetstream.Consumer, error) { -// return CreateTaskConsumer(ctx, streamName, ConsumerConfig{ -// DurableName: durableName, -// DeliverPolicy: DeliverPolicyAll, -// MaxAckPending: 500, // 拉取模式下待确认消息数可以设置小一些 -// }) -//} - -// ConsumeMessages 消费消息(推送模式) -func ConsumeMessages(ctx context.Context, streamName, consumerName string, handler jetstream.MessageHandler) error { - if !IsConnected() { - return fmt.Errorf("NATS 未连接") - } - // 获取消费者 - consumer, err := js.Consumer(ctx, streamName, consumerName) - if err != nil { - return fmt.Errorf("获取消费者失败: %w", err) - } - - // 业务处理 - //if err := handler(ctx, streamMsg.Values); err != nil { - // glog.Infof(ctx, "业务处理失败-> err:%v\n", err) - // continue - //} - //// 确认消息 - //if msg.AutoAck { - // err := ackMessage(ctx, msg.StreamKey, msg.GroupName, streamMsg.ID) - // if err != nil { - // glog.Infof(ctx, "消费者 '%s' 确认消息 ID %s 失败: %v\n", msg.ConsumerName, streamMsg.ID, err) - // } - //} - //// 创建消息处理函数 - //handler = func(msg jetstream.Msg) { - // // 解析消息 - // var task TaskMessage - // if err := json.Unmarshal(msg.Data(), &task); err != nil { - // g.Log().Errorf(ctx, "解析消息失败: %v", err) - // msg.Nak() // 拒绝消息,触发重试 - // return - // } - // - // // 处理业务逻辑 - // g.Log().Infof(ctx, "处理任务: %s", task.TaskID) - // - // // 处理成功,确认消息 - // msg.Ack() - //} - - // 开始消费 - _, err = consumer.Consume(handler) - if err != nil { - return fmt.Errorf("开始消费失败: %w", err) - } - - g.Log().Infof(ctx, "✅ 开始消费消息: %s/%s", streamName, consumerName) - return nil -} - -// 定义消息结构 -type TaskMessage struct { - TaskID string `json:"task_id"` - TaskType string `json:"task_type"` - Data string `json:"data"` -} diff --git a/nats/nats_publish.go b/nats/nats_publish.go deleted file mode 100644 index 32e3b20..0000000 --- a/nats/nats_publish.go +++ /dev/null @@ -1,28 +0,0 @@ -package nats - -import ( - "context" - "encoding/json" - "fmt" -) - -// publish 发布消息到指定主题 -func publish(ctx context.Context, subject string, data any) (err error) { - if !IsConnected() { - return fmt.Errorf("NATS 未连接") - } - // 序列化数据 - dataBytes, err := json.Marshal(data) - if err != nil { - return fmt.Errorf("序列化数据失败: %w", err) - } - // 发布消息 - metrics.PublishCount.Add(1) - _, err = js.Publish(ctx, subject, dataBytes) - if err != nil { - metrics.PublishError.Add(1) - return fmt.Errorf("发布消息失败: %w", err) - } - - return -} diff --git a/nats/nats_rpc.go b/nats/nats_rpc.go deleted file mode 100644 index e95514b..0000000 --- a/nats/nats_rpc.go +++ /dev/null @@ -1,752 +0,0 @@ -package nats - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "github.com/gogf/gf/v2/frame/g" - "github.com/nats-io/nats.go" - "go.opentelemetry.io/otel/trace" - "reflect" - "sync" -) - -// ============ RPC 服务封装 ============ -// 以下方法提供了完全抽象的 RPC 调用接口 -// 调用方和响应方完全不需要知道底层使用的是 NATS 的发布订阅模式 - -// RPC 服务注册表 -var ( - rpcServices map[string]rpcHandler - rpcSubs map[string]*nats.Subscription // 服务名 -> 订阅 - rpcServicesMu sync.RWMutex - queueRPCServices map[string]map[string]rpcHandler // queueName -> subject -> handler - queueRPCSubs map[string]map[string]*nats.Subscription // queueName -> serviceName -> 订阅 - queueRPCMu sync.RWMutex - - // ============ TraceID 主动取消支持 ============ - // 全局映射表:TraceID -> CancelFunc,并发安全 - traceCancelMap map[string]context.CancelFunc - traceCancelMu sync.RWMutex - // 取消主题前缀 - cancelSubjectPrefix = "ctx.cancel.otel." -) - -// rpcHandler RPC 处理函数类型 -// 实现方只需要关注请求参数和返回值,无需了解底层 NATS 实现 -// 返回值可以是任意类型,会被自动序列化为 JSON -type rpcHandler func(ctx context.Context, req []byte) (any, error) - -// RegisterRPCService 注册 RPC 服务(单实例) -// serviceName: 服务名称,调用方通过此名称调用服务 -// handler: 服务处理函数,接收请求并返回响应 -func registerRPCService(serviceName string, handler rpcHandler) (err error) { - if !checkConnected() { - return fmt.Errorf("NATS 未连接") - } - - rpcServicesMu.Lock() - if rpcServices == nil { - rpcServices = make(map[string]rpcHandler) - } - if rpcSubs == nil { - rpcSubs = make(map[string]*nats.Subscription) - } - - // 如果已存在该服务,先取消之前的订阅 - if oldSub, exists := rpcSubs[serviceName]; exists { - oldSub.Unsubscribe() - } - - rpcServices[serviceName] = handler - rpcServicesMu.Unlock() - - // 订阅服务主题 - subject := fmt.Sprintf("rpc.%s", serviceName) - sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { - // 执行处理函数 - executeHandler(handler, msg) - }) - - if err != nil { - return fmt.Errorf("注册 RPC 服务失败: %w", err) - } - - rpcSubs[serviceName] = sub - metrics.SubscribeCount.Add(1) - g.Log().Infof(context.Background(), "✅ RPC 服务已注册: %s", serviceName) - return nil -} - -// RegisterQueueRPCService 注册 RPC 服务(集群模式) -// 多个服务实例注册同一服务时,请求会自动负载均衡 -// serviceName: 服务名称 -// queueName: 队列组名,同一队列组的实例共享请求 -// handler: 服务处理函数 -func registerQueueRPCService(serviceName, queueName string, handler rpcHandler) (err error) { - if !checkConnected() { - return fmt.Errorf("NATS 未连接") - } - - queueRPCMu.Lock() - if queueRPCServices == nil { - queueRPCServices = make(map[string]map[string]rpcHandler) - } - if queueRPCSubs == nil { - queueRPCSubs = make(map[string]map[string]*nats.Subscription) - } - if queueRPCServices[queueName] == nil { - queueRPCServices[queueName] = make(map[string]rpcHandler) - } - if queueRPCSubs[queueName] == nil { - queueRPCSubs[queueName] = make(map[string]*nats.Subscription) - } - - // 如果已存在该服务,先取消之前的订阅 - if oldSub, exists := queueRPCSubs[queueName][serviceName]; exists { - oldSub.Unsubscribe() - } - - queueRPCServices[queueName][serviceName] = handler - queueRPCMu.Unlock() - - // 订阅服务主题(队列模式) - subject := fmt.Sprintf("rpc.%s", serviceName) - sub, err := nc.QueueSubscribe(subject, queueName, func(msg *nats.Msg) { - // 执行处理函数 - executeHandler(handler, msg) - }) - - if err != nil { - return fmt.Errorf("注册队列 RPC 服务失败: %w", err) - } - - queueRPCMu.Lock() - queueRPCSubs[queueName][serviceName] = sub - queueRPCMu.Unlock() - - metrics.SubscribeCount.Add(1) - g.Log().Infof(context.Background(), "✅ 队列 RPC 服务已注册: %s (队列组: %s)", serviceName, queueName) - return nil -} - -// executeHandler 执行 RPC 处理函数 -func executeHandler(handler rpcHandler, msg *nats.Msg) { - // 响应 - var respData []byte - // 从消息头重建上下文 - ctx := headersToContext(context.Background(), msg.Header) - // 提取 TraceID,创建可取消的 context - ctx = createCancelContext(ctx, msg.Header.Get(TraceIDKey)) - // 检查 context 是否已取消(在调用 handler 之前) - select { - case <-ctx.Done(): - // context 已取消,返回取消错误 - g.Log().Infof(ctx, "RPC 请求已取消,traceID: %s", msg.Header.Get(TraceIDKey)) - // 仍然需要发送响应以避免客户端超时 - respData = []byte(`{"_err":"请求已取消"}`) - // 清理取消映射表 - cleanupTraceCancel(msg.Header.Get(TraceIDKey)) - return - default: - } - - // 执行业务处理 - response, err := handler(ctx, msg.Data) - - if err != nil { - // 错误时返回 {"_err": "错误信息"} - if respData, err = json.Marshal(map[string]any{"_err": err.Error()}); err != nil { - g.Log().Errorf(ctx, "RPC 错误响应序列化失败: %v", err) - respData = []byte(`{"_err":"错误响应序列化失败"}`) - } - } else if response == nil { - // 空响应时返回空对象(或 {"_err": ""}) - respData = []byte(`{}`) - } else { - // 成功时返回业务数据 - if respData, err = json.Marshal(response); err != nil { - g.Log().Errorf(ctx, "RPC 响应序列化失败: %v", err) - respData = []byte(`{"_err":"响应序列化失败"}`) - } - } - // 发送响应(必须执行) 如果客户端用 nc.Request(...) 发送消息 → 双向模式,服务端必须 msg.Respond - if err = msg.Respond(respData); err != nil { - g.Log().Errorf(ctx, "RPC 响应失败: %v", err) - } - // 请求结束,清理取消映射表 - cleanupTraceCancel(msg.Header.Get(TraceIDKey)) -} - -// createCancelContext 创建可取消的 context 并注册到取消映射表 -// 返回可取消的 context(如果 traceID 为空则返回原 context) -func createCancelContext(ctx context.Context, traceID string) context.Context { - if g.IsEmpty(traceID) { - return ctx - } - // 创建带取消功能的 context - taskCtx, cancel := context.WithCancel(ctx) - // 注册到取消映射表 - traceCancelMu.Lock() - if traceCancelMap == nil { - traceCancelMap = make(map[string]context.CancelFunc) - } - // 如果同一 TraceID 已有 CancelFunc,先调用它 - if oldCancel, exists := traceCancelMap[traceID]; exists { - oldCancel() - } - traceCancelMap[traceID] = cancel - traceCancelMu.Unlock() - - return taskCtx -} - -// ============ TraceID 主动取消功能 ============ -// 以下函数实现了基于 OpenTelemetry TraceID 的跨进程任务取消机制 - -// SetupCancelListener 设置取消监听器 -// 订阅取消主题,监听取消指令 -// 使用示例: -// -// sub, err := nats.SetupCancelListener(ctx) -func setupCancelListener(ctx context.Context) (*nats.Subscription, error) { - if !checkConnected() { - return nil, fmt.Errorf("NATS 未连接") - } - - if traceCancelMap == nil { - traceCancelMap = make(map[string]context.CancelFunc) - } - - // 修复问题3:订阅取消主题,格式: ctx.cancel.otel.* - // 使用 * 通配符而不是 >,因为 TraceID 是最后一部分 - cancelSubject := cancelSubjectPrefix + "*" - sub, err := nc.Subscribe(cancelSubject, func(msg *nats.Msg) { - // 从主题中解析 TraceID (去除前缀) - prefixLen := len(cancelSubjectPrefix) - if len(msg.Subject) <= prefixLen { - g.Log().Warningf(ctx, "取消消息主题格式错误: %s", msg.Subject) - return - } - traceID := msg.Subject[prefixLen:] - - if traceID == "" { - g.Log().Warning(ctx, "取消消息主题缺少 TraceID") - return - } - - // 从映射表获取 CancelFunc 并执行取消 - traceCancelMu.RLock() - cancel, ok := traceCancelMap[traceID] - traceCancelMu.RUnlock() - - if ok { - cancel() - g.Log().Infof(ctx, "📢 取消信号已发送,traceID: %s", traceID) - } else { - g.Log().Infof(ctx, "⚠️ 未找到对应的可取消任务,traceID: %s", traceID) - } - }) - - if err != nil { - return nil, fmt.Errorf("设置取消监听器失败: %w", err) - } - - metrics.SubscribeCount.Add(1) - g.Log().Infof(ctx, "✅ 取消监听器已设置: %s", cancelSubject) - return sub, nil -} - -// publishCancel 发布取消指令 -// 向指定 TraceID 发送取消信号 -// 使用示例: -// -// err := nats.publishCancel(ctx, traceID) -func publishCancel(ctx context.Context, traceID string) error { - if !checkConnected() { - return fmt.Errorf("NATS 未连接") - } - - if traceID == "" { - return fmt.Errorf("TraceID 不能为空") - } - - cancelSubject := cancelSubjectPrefix + traceID - err := nc.Publish(cancelSubject, nil) - if err != nil { - return fmt.Errorf("发布取消信号失败: %w", err) - } - - g.Log().Infof(ctx, "📤 已发送取消信号,traceID: %s,主题: %s", traceID, cancelSubject) - return nil -} - -// cleanupTraceCancel 清理取消映射表中的条目 -// 任务取消/正常结束后必须调用此函数,避免内存泄漏 -// 使用示例: -// -// defer nats.cleanupTraceCancel(traceID) -func cleanupTraceCancel(traceID string) { - if traceID == "" { - return - } - - traceCancelMu.Lock() - defer traceCancelMu.Unlock() - - if _, ok := traceCancelMap[traceID]; ok { - delete(traceCancelMap, traceID) - g.Log().Infof(context.Background(), "✅ 已清理取消映射表,traceID: %s", traceID) - } -} - -// CallRPC 调用 RPC 服务 -// serviceName: 服务名称 -// req: 请求数据 -// 返回: 响应数据(任意类型)和错误 -func CallRPC(ctx context.Context, serviceName string, req any, resp any) (err error) { - if !checkConnected() { - return fmt.Errorf("NATS 未连接") - } - - metrics.RequestCount.Add(1) - - // 验证 resp 必须是指针类型 - respValue := reflect.ValueOf(resp) - if respValue.Kind() != reflect.Ptr { - return fmt.Errorf("resp 参数必须是指针类型(当前类型: %T)", resp) - } - - // 构建请求体 - var reqBody []byte - if !g.IsEmpty(req) { - reqValue := reflect.ValueOf(req) - if !(reqValue.Kind() == reflect.Ptr && reqValue.IsNil()) && !reqValue.IsZero() { - reqData, err := json.Marshal(req) - if err != nil { - return fmt.Errorf("序列化请求参数失败: %w", err) - } - reqBody = reqData - } - } - - // 检查本地是否有注册的单实例服务,如果有则直接调用(优化性能) - rpcServicesMu.RLock() - if localHandler, exists := rpcServices[serviceName]; exists { - rpcServicesMu.RUnlock() - - // 修复问题1:本地调用也需要处理取消机制 - var traceID string - if traceID, err = getTraceID(ctx); err != nil { - return err - } - // 提取 TraceID,创建可取消的 context - cancelCtx := createCancelContext(ctx, traceID) - // 执行本地调用 - var response interface{} - if response, err = localHandler(cancelCtx, reqBody); err != nil { - metrics.RequestError.Add(1) - return fmt.Errorf("本地调用 RPC 服务失败 [%s]: %w", serviceName, err) - } - - // 请求结束,清理取消映射表 - cleanupTraceCancel(traceID) - - // 检查是否为错误消息:尝试解析为 map,看是否包含 "_err" 字段 - var respMap map[string]any - if json.Unmarshal(response.([]byte), &respMap) == nil { - if errMsg, ok := respMap["_err"]; ok { - metrics.RequestError.Add(1) - return fmt.Errorf("%v", errMsg) - } - } - // 正常数据直接返回 - // responseMsg.Data 已经是 []byte 类型(来自 msg.Data),直接反序列化 - if err = json.Unmarshal(response.([]byte), resp); err != nil { - return fmt.Errorf("解析响应失败: %w (响应内容: %s)", err, response) - } - - return - } - rpcServicesMu.RUnlock() - - subject := fmt.Sprintf("rpc.%s", serviceName) - - // 创建消息并将上下文元数据写入消息头 - msg := nats.NewMsg(subject) - msg.Data = reqBody - headers, err := contextToHeaders(ctx) - if err != nil { - return fmt.Errorf("上下文转换失败: %w", err) - } - msg.Header = headers - - // 修复问题5:优化 go 协程避免资源泄漏 - // 使用 done channel 来确保 goroutine 能正确退出 - done := make(chan struct{}) - var closeDoneOnce sync.Once - closeDone := func() { - closeDoneOnce.Do(func() { - close(done) - }) - } - - if msg.Header.Get(TraceIDKey) != "" { - go func() { - defer closeDone() - select { - case <-ctx.Done(): - // context 被取消时,发送取消信号给服务端 - if errors.Is(ctx.Err(), context.Canceled) { - if err := publishCancel(context.Background(), msg.Header.Get(TraceIDKey)); err != nil { - g.Log().Errorf(ctx, "发送 RPC 取消信号失败: %v", err) - } else { - g.Log().Infof(ctx, "RPC 调用已取消,traceID: %s", msg.Header.Get(TraceIDKey)) - } - } - case <-done: - // 请求已完成,无需发送取消信号 - return - } - }() - } - - // 发送请求 - responseMsg, err := nc.RequestMsgWithContext(ctx, msg) - - // 关闭 done channel,通知 goroutine 退出 - closeDone() - - if err != nil { - metrics.RequestError.Add(1) - return fmt.Errorf("调用 RPC 服务失败 [%s]: %w", serviceName, err) - } - - if responseMsg == nil { - metrics.RequestError.Add(1) - return fmt.Errorf("RPC 响应为空 [%s]", serviceName) - } - - // 解析响应 - if len(responseMsg.Data) > 0 { - // 检查是否为错误消息:尝试解析为 map,看是否包含 "_err" 字段 - var respMap map[string]any - if json.Unmarshal(responseMsg.Data, &respMap) == nil { - if errMsg, ok := respMap["_err"]; ok { - metrics.RequestError.Add(1) - return fmt.Errorf("%v", errMsg) - } - } - // 正常数据直接返回 - // responseMsg.Data 已经是 []byte 类型(来自 msg.Data),直接反序列化 - if err = json.Unmarshal(responseMsg.Data, resp); err != nil { - return fmt.Errorf("解析响应失败: %w (响应内容: %s)", err, responseMsg.Data) - } - } - - return -} - -// RegisterServiceOption 注册选项类型 -type RegisterServiceOption func(*registerServiceConfig) - -type registerServiceConfig struct { - queueName string // 队列组名(用于集群模式) - excludeMethods []string -} - -// WithQueueGroup 设置队列组名(集群模式) -func WithQueueGroup(queueName string) RegisterServiceOption { - return func(cfg *registerServiceConfig) { - cfg.queueName = queueName - } -} - -// WithExcludeMethods 排除不需要注册的方法 -func WithExcludeMethods(methods ...string) RegisterServiceOption { - return func(cfg *registerServiceConfig) { - cfg.excludeMethods = append(cfg.excludeMethods, methods...) - } -} - -// AutoRegisterServices 自动注册多个服务的所有公开方法 -// serviceInstances: map[包名]service实例,如 map[string]interface{}{"user": userService, "order": orderService} -// options: 注册选项(可选) -// 示例: -// -// AutoRegisterServices(map[string]interface{}{ -// "user": userService, -// "order": orderService, -// }) -// 或 -// AutoRegisterServices(map[string]interface{}{ -// "order": orderService, -// }, WithQueueGroup("order-group")) -func AutoRegisterServices(ctx context.Context, serviceInstances map[string]interface{}, options ...RegisterServiceOption) error { - // 先注册 RPC 服务(如果 NATS 不可用则记录警告但不阻塞启动) - if !checkConnected() { - return fmt.Errorf("NATS 未连接,RPC 服务未注册") - } - - if len(serviceInstances) == 0 { - return fmt.Errorf("service 实例列表不能为空") - } - - totalRegistered := 0 - // 遍历每个 service 实例 - for pkgName, serviceInstance := range serviceInstances { - // 注册服务 - err := registerService(serviceInstance, pkgName, options...) - if err != nil { - g.Log().Errorf(ctx, "注册 %s 服务失败: %v", pkgName, err) - continue - } - totalRegistered++ - g.Log().Infof(ctx, "✅ %s 服务已自动注册", pkgName) - } - - if totalRegistered == 0 { - return fmt.Errorf("未能注册任何服务") - } - // 设置取消监听器(监听基于 TraceID 的取消请求) - if _, err := setupCancelListener(ctx); err != nil { - g.Log().Errorf(ctx, "设置取消监听器失败: %v", err) - } else { - g.Log().Infof(ctx, "✅ 取消监听器已自动设置") - } - g.Log().Infof(ctx, "✅ 共自动注册了 %d 个服务", totalRegistered) - - return nil -} - -// registerService 注册单个服务的所有公开方法(内部函数) -func registerService(service interface{}, serviceNamePrefix string, options ...RegisterServiceOption) (err error) { - if !checkConnected() { - return fmt.Errorf("NATS 未连接") - } - - // 应用选项 - cfg := ®isterServiceConfig{} - for _, opt := range options { - opt(cfg) - } - - // 创建排除方法集合 - excludeSet := make(map[string]struct{}) - for _, method := range cfg.excludeMethods { - excludeSet[method] = struct{}{} - } - - // 获取 service 的类型 - serviceType := reflect.TypeOf(service) - - // 遍历所有方法 - registeredCount := 0 - for i := 0; i < serviceType.NumMethod(); i++ { - method := serviceType.Method(i) - - // 只注册导出方法(首字母大写) - if !method.IsExported() { - continue - } - - // 排除指定的方法 - if _, exists := excludeSet[method.Name]; exists { - continue - } - - // 检查方法签名:必须是 func(ctx context.Context, request) (response, error) - // 注意:method.Type.NumIn() 包含接收者,所以实际参数数量需要减去 1 - // 要求:接收者 + context.Context + request,总共3个参数 - if method.Type.NumIn() != 3 { - g.Log().Warningf(context.Background(), "方法 %s 必须有2个参数(context.Context 和请求参数),跳过注册", method.Name) - continue - } - - // 第一个参数(接收者之后的第一个参数)必须是 context.Context - // method.Type.In(0) 是接收者,method.Type.In(1) 才是第一个参数 - if !method.Type.In(1).Implements(reflect.TypeOf((*context.Context)(nil)).Elem()) { - g.Log().Warningf(context.Background(), "方法 %s 的第一个参数必须是 context.Context,跳过注册", method.Name) - continue - } - - // 第二个参数必须是结构体指针或数组 - reqType := method.Type.In(2) - if reqType.Kind() != reflect.Ptr && reqType.Kind() != reflect.Slice && reqType.Kind() != reflect.Array { - g.Log().Warningf(context.Background(), "方法 %s 的第二个参数必须是结构体指针或数组,跳过注册", method.Name) - continue - } - - // 返回值必须是 (result, error),即2个返回值 - if method.Type.NumOut() != 2 { - g.Log().Warningf(context.Background(), "方法 %s 必须有2个返回值(result 和 error),跳过注册", method.Name) - continue - } - - // 最后一个返回值必须是 error - if !method.Type.Out(1).Implements(reflect.TypeOf((*error)(nil)).Elem()) { - g.Log().Warningf(context.Background(), "方法 %s 的最后一个返回值必须是 error,跳过注册", method.Name) - continue - } - - // 生成服务名称:前缀.方法名(保持原始方法名) - serviceName := fmt.Sprintf("%s.%s", serviceNamePrefix, method.Name) - - // 创建 RPC handler - handler := func(ctx context.Context, req []byte) (any, error) { - // 准备方法调用参数 - // args[0] 是接收者, args[1] 是 ctx, args[2] 是请求参数 - args := make([]reflect.Value, 3) - args[0] = reflect.ValueOf(service) // 接收者 - args[1] = reflect.ValueOf(ctx) // context.Context - - // 解析请求参数 - if len(req) > 0 { - reqValuePtr := reflect.New(reqType) - - // 解析 JSON - if err := json.Unmarshal(req, reqValuePtr.Interface()); err != nil { - // 根据参数类型提供更友好的错误提示 - var typeHint string - if reqType.Kind() == reflect.Ptr { - typeHint = fmt.Sprintf("(期望类型: %s)", reqType.Elem().Name()) - } else { // reflect.Slice 或 reflect.Array - typeHint = fmt.Sprintf("(期望类型: %s,请确保客户端传递的是JSON数组格式)", reqType.String()) - } - return nil, fmt.Errorf("解析请求参数失败%s: %w", typeHint, err) - } - args[2] = reqValuePtr.Elem() - } else { - // 请求为空,创建零值 - args[2] = reflect.Zero(method.Type.In(2)) - } - - // 调用方法 - results := method.Func.Call(args) - - // 处理返回值 - var result any - - if len(results) == 1 { - // 只有 error - if !results[0].IsNil() { - err = results[0].Interface().(error) - } - } else if len(results) == 2 { - // (result, error) - result = results[0].Interface() - if !results[1].IsNil() { - err = results[1].Interface().(error) - } - } - if err != nil { - return nil, err - } - - return result, nil - } - - // 注册 RPC 服务 - var err error - if cfg.queueName != "" { - err = registerQueueRPCService(serviceName, cfg.queueName, handler) - } else { - err = registerRPCService(serviceName, handler) - } - - if err != nil { - g.Log().Errorf(context.Background(), "注册服务 %s 失败: %v", serviceName, err) - continue - } - - registeredCount++ - g.Log().Infof(context.Background(), "✅ 已自动注册 RPC 服务: %s -> %s", serviceName, method.Name) - } - - if registeredCount == 0 { - g.Log().Warningf(context.Background(), "未注册任何方法,请检查 %v 的方法签名", serviceNamePrefix) - return fmt.Errorf("未找到可注册的方法") - } - - g.Log().Infof(context.Background(), "✅ Service %v 共注册了 %d 个 RPC 方法", serviceNamePrefix, registeredCount) - return nil -} - -// ============ 上下文元数据工具函数 ============ -// 以下函数用于在 context 和 NATS 消息头之间互转元数据 - -// 定义常见的上下文元数据 key -const ( - TraceIDKey = "trace_id" - TokenKey = "token" -) - -func getTraceID(ctx context.Context) (traceID string, err error) { - // 提取 traceId:首先尝试从 OpenTelemetry Span 中提取,从 context 中提取 TraceID - span := trace.SpanFromContext(ctx) - if span != nil && span.SpanContext().HasTraceID() { - traceID = span.SpanContext().TraceID().String() - } else if tid := ctx.Value(TraceIDKey); tid != nil { - traceID = fmt.Sprintf("%v", tid) - } - if traceID == "" { - return traceID, fmt.Errorf("context 中没有 TraceID") - } - return -} - -// contextToHeaders 将 context 中的元数据转换为 NATS 消息头 -// 支持提取 user_id、tenant_id、trace_id、token 等常见字段 -func contextToHeaders(ctx context.Context) (nats.Header, error) { - headers := make(nats.Header) - - // 提取 traceId:首先尝试从 OpenTelemetry Span 中提取 - if traceID, err := getTraceID(ctx); err != nil { - return headers, err - } else { - headers.Set(TraceIDKey, traceID) - } - - // 提取 token(优先级:context value > HTTP Authorization header) - token := "" - if t := ctx.Value(TokenKey); t != nil { - token = fmt.Sprintf("%v", t) - } else if r := g.RequestFromCtx(ctx); r != nil { - // 从 HTTP 请求的 Authorization header 中提取 token - auth := r.GetHeader("Authorization") - if auth != "" { - // 移除 "Bearer " 前缀 - if len(auth) > 7 && auth[:7] == "Bearer " { - token = auth[7:] - } else { - token = auth - } - } - } - if token != "" { - headers.Set(TokenKey, token) - } - - return headers, nil -} - -// headersToContext 从 NATS 消息头重建 context -// 支持还原 user_id、tenant_id、trace_id、token 等字段 -func headersToContext(ctx context.Context, headers nats.Header) context.Context { - if headers == nil { - return ctx - } - - // 恢复 trace_id - if traceID := headers.Get(TraceIDKey); traceID != "" { - ctx = context.WithValue(ctx, TraceIDKey, traceID) - } - - // 恢复 token - if token := headers.Get(TokenKey); token != "" { - ctx = context.WithValue(ctx, TokenKey, token) - } - - return ctx -} diff --git a/nats/nats_task.go b/nats/nats_task.go deleted file mode 100644 index f6841ad..0000000 --- a/nats/nats_task.go +++ /dev/null @@ -1,212 +0,0 @@ -package nats - -import ( - "context" - "fmt" - "github.com/gogf/gf/v2/frame/g" - "github.com/nats-io/nats.go/jetstream" - "time" -) - -// TaskStreamConfig 任务流配置 -type TaskStreamConfig struct { - StreamName string // 流名称 - Subjects []string // 主题数组(支持任务优先级,如 ["tasks.high","tasks.normal", "tasks.low"]) - PublishSubject string // 发布使用的主题(仅用于记录,不影响流配置) - Storage StorageType // 存储类型 - Retention RetentionType // 保留策略 - MaxAge time.Duration // 最大保留时间 - Duplicates time.Duration // 消息去重窗口时间 - Replicas int // 副本数 - MaxMsgSize int32 // 单条消息最大大小(字节) - MaxBytes int64 // 流最大存储大小(字节) - MaxMsgs int64 // 流中最大消息数 - MaxMsgsPerSubject int64 // 每个主题最大消息数 - MaxConsumers int // 最大消费者数量 - DenyPurge bool // 是否禁止清理流 - AllowRollup bool // 是否允许汇总消息 - DenyDelete bool // 是否禁止删除 - DiscardPerSubject bool // 是否按主题限制(工作队列模式) - Republish *RePublishConfig // 死信队列重新发布配置 -} - -// RePublishConfig 重新发布配置(用于死信队列) -type RePublishConfig struct { - Source string // 源主题 - Destination string // 目标主题 - HeadersOnly bool // 仅复制消息头 -} - -// StorageType 存储类型 -type StorageType string - -const ( - StorageFile StorageType = "file" // 文件存储(持久化) - StorageMemory StorageType = "memory" // 内存存储 -) - -// RetentionType 保留策略 -type RetentionType string - -const ( - RetentionLimit RetentionType = "limit" // 消息数量限制 - RetentionPolicy RetentionType = "interest" // 基于兴趣 - RetentionWorkQueue RetentionType = "workqueue" // 工作队列 -) - -// parseStorageType 解析存储类型 -func parseStorageType(st StorageType) jetstream.StorageType { - switch st { - case StorageMemory: - return jetstream.MemoryStorage - default: - return jetstream.FileStorage - } -} - -// parseRetentionType 解析保留策略 -func parseRetentionType(rt RetentionType) jetstream.RetentionPolicy { - switch rt { - case RetentionLimit: - return jetstream.LimitsPolicy - case RetentionPolicy: - return jetstream.InterestPolicy - default: - return jetstream.WorkQueuePolicy - } -} - -// createTaskStreamSimple 简化版创建任务流(适用于大多数场景) -// 只需提供流名称和主题数组,其他使用默认配置 -func createTaskStreamSimple(ctx context.Context, streamName string, subjects []string) error { - return createTaskStream(ctx, TaskStreamConfig{ - StreamName: streamName, - Subjects: subjects, - }) -} - -// createTaskStreamWithPriority 创建支持优先级的任务流 -func createTaskStreamWithPriority(ctx context.Context, streamPrefix string) error { - subjects := []string{ - fmt.Sprintf("%s.high.>", streamPrefix), - fmt.Sprintf("%s.normal.>", streamPrefix), - fmt.Sprintf("%s.low.>", streamPrefix), - } - return createTaskStream(ctx, TaskStreamConfig{ - StreamName: streamPrefix, - Subjects: subjects, - }) -} - -// CreateTaskStream 配置: 文件存储 + 工作队列策略 -// CreateTaskStream 创建任务消息队列流(JetStream 2.10+) -// 核心设计思路: -// 1. 严格持久化:使用文件存储,任务消息不会因为服务器重启而丢失 -// 2. 支持任务优先级:通过主题分级实现,如 ["tasks.high", "tasks.low"] -// 3. 死信队列支持:通过 RePublish 配置将失败任务路由到专门的 DLQ 流 -// 4. 灵活保留策略:根据任务重要性设置不同的保留时长(MaxAge) -// 5. 工作队列模式:确保每个任务只被一个消费者处理(DiscardPerSubject) -func createTaskStream(ctx context.Context, config TaskStreamConfig) error { - if !IsConnected() { - return fmt.Errorf("NATS 未连接") - } - - if g.IsNil(config.StreamName) { - return fmt.Errorf("流名称不能为空") - } - if len(config.Subjects) == 0 { - return fmt.Errorf("主题数组不能为空") - } - // 设置默认值 - if config.Storage == "" { - config.Storage = StorageFile // 默认文件存储 - } - if config.Retention == "" { - config.Retention = RetentionWorkQueue // 默认工作队列策略 - } - if config.MaxAge == 0 { - config.MaxAge = 24 * time.Hour // 默认保留24小时 - } - if config.Replicas == 0 { - config.Replicas = 1 // 默认单副本 - } - if config.MaxBytes == 0 { - config.MaxBytes = 10 * 1024 * 1024 * 1024 // 默认10GB - } - if config.MaxMsgs == 0 { - config.MaxMsgs = 100000 // 默认10万条消息 - } - if config.MaxMsgSize == 0 { - config.MaxMsgSize = 1024 * 1024 // 默认1MB - } - - if config.DiscardPerSubject { - config.DenyDelete = true // 工作队列模式下禁止删除 - } - - // 构建流配置 - jsConfig := jetstream.StreamConfig{ - Name: config.StreamName, - Subjects: config.Subjects, - Storage: parseStorageType(config.Storage), - Retention: parseRetentionType(config.Retention), - MaxAge: config.MaxAge, - Duplicates: config.Duplicates, - Replicas: config.Replicas, - MaxMsgSize: config.MaxMsgSize, - MaxBytes: config.MaxBytes, - MaxMsgs: config.MaxMsgs, - MaxMsgsPerSubject: config.MaxMsgsPerSubject, - MaxConsumers: config.MaxConsumers, - AllowRollup: config.AllowRollup, - DenyDelete: config.DenyDelete, - DenyPurge: config.DenyPurge, - Discard: jetstream.DiscardOld, // 默认删除旧消息 - DiscardNewPerSubject: config.DiscardPerSubject, - } - - // 配置死信队列重新发布(如果设置了) - if config.Republish != nil { - jsConfig.RePublish = &jetstream.RePublish{ - Source: config.Republish.Source, - Destination: config.Republish.Destination, - HeadersOnly: config.Republish.HeadersOnly, - } - } else { - // 使用固定的死信队列命名规范:{StreamName}.DLQ - dlqSubject := fmt.Sprintf("%s.DLQ", config.StreamName) - // 死信队列配置 - jsConfig.RePublish = &jetstream.RePublish{ - Source: ">", - Destination: dlqSubject, - HeadersOnly: true, - } - } - - // 检查流是否已存在 - stream, err := js.Stream(ctx, config.StreamName) - if err == nil { - // 流已存在,更新配置 - _, err = js.UpdateStream(ctx, jsConfig) - if err != nil { - return fmt.Errorf("更新任务流失败: %w", err) - } - g.Log().Infof(ctx, "✅ 任务消息队列流已更新: %s", stream.CachedInfo().Config.Name) - return nil - } - - // 创建新流 - stream, err = js.CreateStream(ctx, jsConfig) - if err != nil { - return fmt.Errorf("创建任务流失败: %w", err) - } - - // 记录配置信息 - configInfo := fmt.Sprintf("存储=%s, 策略=%s, 副本=%d, 保留=%v", config.Storage, config.Retention, config.Replicas, config.MaxAge) - if config.Republish != nil { - configInfo += fmt.Sprintf(", 死信队列=%s->%s", config.Republish.Source, config.Republish.Destination) - } - g.Log().Infof(ctx, "✅ 任务消息队列流创建成功: %s (%s)", stream.CachedInfo().Config.Name, configInfo) - - return nil -} diff --git a/nats/nats_test.go b/nats/nats_test.go deleted file mode 100644 index 94e1262..0000000 --- a/nats/nats_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package nats - -import ( - "context" - "fmt" - "testing" - "time" - - "github.com/nats-io/nats.go/jetstream" -) - -// TestNatsBasicOperations 测试基础操作 -func TestNatsBasicOperations(t *testing.T) { - // 测试连接状态 - if !IsConnected() { - t.Log("NATS 未连接") - } - - // 测试连接状态获取 - state := GetConnState() - t.Logf("当前连接状态: %d", state) -} - -// TestNatsMetrics 测试监控指标 -func TestNatsMetrics(t *testing.T) { - metrics := GetMetrics() - t.Logf("发布计数: %d", metrics.PublishCount.Load()) - t.Logf("发布错误: %d", metrics.PublishError.Load()) - t.Logf("请求计数: %d", metrics.RequestCount.Load()) -} - -// TestNatsConnStateListener 测试连接状态监听 -func TestNatsConnStateListener(t *testing.T) { - listener := func(state ConnState, err error) { - fmt.Printf("连接状态变化: %d, 错误: %v\n", state, err) - } - - RegisterConnStateListener(listener) - defer UnregisterConnStateListener(listener) - - time.Sleep(1 * time.Second) -} - -// TestNatsStreamOperations 测试流操作 -func TestNatsStreamOperations(t *testing.T) { - ctx := context.Background() - - // 创建任务流 - config := TaskStreamConfig{ - StreamName: "test_tasks", - Subjects: []string{"test.task.>"}, - //Subject: "test.task.process", - } - err := CreateTaskStream(ctx, config) - if err != nil { - t.Logf("创建任务流失败: %v", err) - } - - // 获取流信息 - info, err := GetStream(ctx, "test_tasks") - if err != nil { - t.Logf("获取流信息失败: %v", err) - } else { - t.Logf("流信息: %s", info.Config.Name) - } - - // 列出所有流 - streams, err := ListStreams(ctx) - if err != nil { - t.Logf("列出流失败: %v", err) - } else { - t.Logf("流列表: %v", streams) - } - - // 删除流 - err = DeleteStream(ctx, "test_tasks") - if err != nil { - t.Logf("删除流失败: %v", err) - } -} - -// TestNatsConsumerOperations 测试消费者操作 -func TestNatsConsumerOperations(t *testing.T) { - ctx := context.Background() - - // 创建测试流 - config := TaskStreamConfig{ - StreamName: "test_consumer", - Subjects: []string{"test.consumer.>"}, - //Subject: "test.consumer.process", - } - err := CreateTaskStream(ctx, config) - if err != nil { - t.Logf("创建流失败: %v", err) - } - - // 创建消费者 - consumerConfig := jetstream.ConsumerConfig{ - Name: "test_consumer", - Durable: "test_consumer", - } - _, err = CreateConsumer(ctx, "test_consumer", "test_consumer", consumerConfig) - if err != nil { - t.Logf("创建消费者失败: %v", err) - } - - // 获取消费者信息 - info, err := GetConsumer(ctx, "test_consumer", "test_consumer") - if err != nil { - t.Logf("获取消费者信息失败: %v", err) - } else { - t.Logf("消费者信息: %s", info.Name) - } - - // 列出消费者 - consumers, err := ListConsumers(ctx, "test_consumer") - if err != nil { - t.Logf("列出消费者失败: %v", err) - } else { - t.Logf("消费者列表: %v", consumers) - } - - // 删除消费者 - err = DeleteConsumer(ctx, "test_consumer", "test_consumer") - if err != nil { - t.Logf("删除消费者失败: %v", err) - } - - // 清理流 - _ = DeleteStream(ctx, "test_consumer") -} diff --git a/nats/task.go b/nats/task.go deleted file mode 100644 index 996ded6..0000000 --- a/nats/task.go +++ /dev/null @@ -1,411 +0,0 @@ -package nats - -//import ( -// "context" -// "fmt" -// "time" -// -// "github.com/gogf/gf/v2/frame/g" -// "github.com/nats-io/nats.go/jetstream" -//) - -//// TaskPriority 任务优先级 -//type TaskPriority string -// -//const ( -// TaskPriorityHigh TaskPriority = "high" // 高优先级任务 -// TaskPriorityNormal TaskPriority = "normal" // 普通优先级任务 -// TaskPriorityLow TaskPriority = "low" // 低优先级任务 -//) -// -//// TaskStreamConfig 任务流配置 -//type TaskStreamConfig struct { -// StreamName string // 流名称 -// Subjects []string // 主题列表(支持优先级分级,如 tasks.high.>, tasks.normal.>, tasks.low.>) -// Subject string // 默认发布主题 -// Priority TaskPriority // 任务优先级 -// MaxAge time.Duration // 消息保留时长(根据任务重要性设置) -// MaxMsgsPerSub int64 // 每个订阅者最大消息数(防止内存溢出) -// Replicas int // 副本数(默认1,建议生产环境使用3) -// Duplicates time.Duration // 消息去重窗口(0表示不启用) -//} -// -//// TaskConsumerConfig 任务消费者配置 -//type TaskConsumerConfig struct { -// ConsumerName string // 消费者名称 -// AckPolicy *jetstream.AckPolicy -// MaxDeliveries int32 // 最大投递次数(用于重试控制) -// AckWait time.Duration // 等待ACK超时时间 -// Backoff []time.Duration // 重试退避策略 -// FilterSubject string // 过滤主题(可指定特定优先级任务) -// MaxAckPending int // 最大待确认消息数 -// MaxWaiting int // 最大等待消息数 -// ReplayPolicy *jetstream.ReplayPolicy // 重放策略 -//} - -// CreateTaskStream 创建任务流(基于 JetStream 2.10+ API) -// -// 核心设计思路: -// 1. 严格的持久化:使用文件存储(FileStorage)避免任务丢失 -// 2. 任务优先级:通过主题分级实现(tasks.high/tasks.normal/tasks.low) -// 3. 死信队列:配置死信队列处理失败任务 -// 4. 保留策略:按任务重要性设置不同的保留时长 -// 5. 工作队列策略:确保每条消息只被一个消费者处理 -// -// 参数: -// - ctx: 上下文 -// - config: 任务流配置 -// -// 返回: -// - error: 错误信息 -//func CreateTaskStream(ctx context.Context, config TaskStreamConfig) error { -// if !IsConnected() { -// return fmt.Errorf("NATS 未连接") -// } -// -// // 设置默认值 -// if config.MaxAge == 0 { -// config.MaxAge = 7 * 24 * time.Hour // 默认保留7天 -// } -// if config.MaxMsgsPerSub == 0 { -// config.MaxMsgsPerSub = 100000 // 默认每订阅者最多10万条消息 -// } -// if config.Replicas == 0 { -// config.Replicas = 1 // 默认单副本 -// } -// if config.Duplicates == 0 { -// config.Duplicates = 2 * time.Minute // 默认2分钟去重窗口 -// } -// -// // 验证主题配置 -// if len(config.Subjects) == 0 { -// return fmt.Errorf("任务流必须指定至少一个主题") -// } -// -// // 设置死信队列 -// // 使用固定的死信队列命名规范:{StreamName}.DLQ -// dlqSubject := fmt.Sprintf("%s.DLQ", config.StreamName) -// -// // 尝试获取现有流 -// stream, err := js.Stream(ctx, config.StreamName) -// if err == nil { -// // 流已存在,更新配置以适配任务流的特殊需求 -// _, err = js.UpdateStream(ctx, jetstream.StreamConfig{ -// Name: config.StreamName, -// Subjects: config.Subjects, -// Storage: jetstream.FileStorage, // 文件存储确保持久化 -// Retention: jetstream.WorkQueuePolicy, // 工作队列策略 -// MaxAge: config.MaxAge, -// MaxMsgs: config.MaxMsgsPerSub, -// Replicas: config.Replicas, -// Duplicates: config.Duplicates, -// // 死信队列配置 -// RePublish: &jetstream.RePublish{ -// Source: ">", // 匹配所有主题 -// Destination: dlqSubject, -// }, -// // 限制流大小(防止磁盘占用过多) -// MaxBytes: 10 * 1024 * 1024 * 1024, // 10GB -// }) -// if err != nil { -// return fmt.Errorf("更新任务流失败: %w", err) -// } -// g.Log().Infof(ctx, "✅ 任务流已更新: %s (优先级: %s, 保留: %v)", -// stream.CachedInfo().Config.Name, config.Priority, config.MaxAge) -// return nil -// } -// -// // 创建新任务流 -// streamConfig := jetstream.StreamConfig{ -// Name: config.StreamName, -// Subjects: config.Subjects, -// Storage: jetstream.FileStorage, // 文件存储确保持久化 -// Retention: jetstream.WorkQueuePolicy, // 工作队列策略 -// MaxAge: config.MaxAge, -// MaxMsgs: config.MaxMsgsPerSub, -// Replicas: config.Replicas, -// Duplicates: config.Duplicates, -// // 死信队列配置 -// RePublish: &jetstream.RePublish{ -// Source: ">", // 匹配所有主题 -// Destination: dlqSubject, -// }, -// // 限制流大小(防止磁盘占用过多) -// MaxBytes: 10 * 1024 * 1024 * 1024, // 10GB -// // 启用流清理 -// Discard: jetstream.DiscardOld, // 新消息替换旧消息 -// } -// -// stream, err = js.CreateStream(ctx, streamConfig) -// if err != nil { -// return fmt.Errorf("创建任务流失败: %w", err) -// } -// -// // 验证流是否创建成功 -// if stream == nil { -// return fmt.Errorf("创建任务流失败:流对象为空") -// } -// -// g.Log().Infof(ctx, "✅ 任务流创建成功: %s (文件存储+工作队列策略+死信队列, 优先级: %s, 保留: %v, 副本: %d)", -// stream.CachedInfo().Config.Name, config.Priority, config.MaxAge, config.Replicas) -// -// // 记录配置信息 -// g.Log().Infof(ctx, " - 主题列表: %v", config.Subjects) -// g.Log().Infof(ctx, " - 死信队列: %s", dlqSubject) -// g.Log().Infof(ctx, " - 最大消息数: %d", config.MaxMsgsPerSub) -// g.Log().Infof(ctx, " - 去重窗口: %v", config.Duplicates) -// -// return nil -//} -// -//// CreateOrUpdateTaskConsumer 创建或更新任务消费者(基于 JetStream 2.10+ API) -//// -//// 核心设计思路: -//// 1. 支持手动确认(AckExplicit)确保任务处理完成 -//// 2. 通过 Nack() 方法实现消息重试,超限后进入死信队列 -//// 3. 支持主题过滤,可订阅特定优先级任务 -//// 4. 限制待确认消息数,防止消费者过载 -//// 5. AckWait 设置消息处理超时时间 -//// -//// 参数: -//// - ctx: 上下文 -//// - streamName: 流名称 -//// - consumerConfig: 消费者配置 -//// -//// 返回: -//// - jetstream.Consumer: 消费者对象 -//// - error: 错误信息 -//func CreateOrUpdateTaskConsumer(ctx context.Context, streamName string, consumerConfig TaskConsumerConfig) (jetstream.Consumer, error) { -// if !IsConnected() { -// return nil, fmt.Errorf("NATS 未连接") -// } -// -// // 设置默认值 -// ackPolicy := jetstream.AckExplicitPolicy -// if consumerConfig.AckPolicy != nil { -// ackPolicy = *consumerConfig.AckPolicy -// } -// -// if consumerConfig.MaxDeliveries == 0 { -// consumerConfig.MaxDeliveries = 10 // 默认最多投递10次 -// } -// -// if consumerConfig.AckWait == 0 { -// consumerConfig.AckWait = 30 * time.Second // 默认30秒等待确认 -// } -// -// if consumerConfig.MaxAckPending == 0 { -// consumerConfig.MaxAckPending = 1000 // 默认最多1000条待确认消息 -// } -// -// if consumerConfig.MaxWaiting == 0 { -// consumerConfig.MaxWaiting = 512 // 默认最多512条等待消息 -// } -// -// replayPolicy := jetstream.ReplayInstantPolicy -// if consumerConfig.ReplayPolicy != nil { -// replayPolicy = *consumerConfig.ReplayPolicy -// } -// -// // 构建消费者配置 -// config := jetstream.ConsumerConfig{ -// Name: consumerConfig.ConsumerName, -// Durable: consumerConfig.ConsumerName, // 持久化消费者 -// AckPolicy: ackPolicy, -// AckWait: consumerConfig.AckWait, -// MaxAckPending: consumerConfig.MaxAckPending, -// MaxWaiting: consumerConfig.MaxWaiting, -// ReplayPolicy: replayPolicy, -// FilterSubject: consumerConfig.FilterSubject, -// } -// -// // 使用 CreateOrUpdateConsumer 创建或更新消费者 -// consumer, err := js.CreateOrUpdateConsumer(ctx, streamName, config) -// if err != nil { -// return nil, fmt.Errorf("创建任务消费者失败: %w", err) -// } -// -// g.Log().Infof(ctx, "✅ 任务消费者已创建/更新: %s/%s (等待确认: %v)", -// streamName, consumerConfig.ConsumerName, consumerConfig.AckWait) -// -// // 获取消费者信息并记录 -// info, err := consumer.Info(ctx) -// if err == nil { -// g.Log().Infof(ctx, " - 过滤主题: %s", info.Config.FilterSubject) -// g.Log().Infof(ctx, " - 最大待确认: %d", info.Config.MaxAckPending) -// g.Log().Infof(ctx, " - ACK策略: %s", info.Config.AckPolicy) -// } -// -// return consumer, nil -//} -// -//// CreateTaskStreamWithPriority 创建带优先级的任务流 -//// -//// 便捷方法,自动创建支持多优先级的任务流配置 -//// -//// 参数: -//// - ctx: 上下文 -//// - streamPrefix: 流名称前缀(如 "tasks") -//// - priority: 默认优先级 -//// -//// 返回: -//// - error: 错误信息 -//func CreateTaskStreamWithPriority(ctx context.Context, streamPrefix string, priority TaskPriority) error { -// if !IsConnected() { -// return fmt.Errorf("NATS 未连接") -// } -// -// // 构建支持多优先级的主题列表 -// subjects := []string{ -// fmt.Sprintf("%s.high.>", streamPrefix), // 高优先级任务 -// fmt.Sprintf("%s.normal.>", streamPrefix), // 普通优先级任务 -// fmt.Sprintf("%s.low.>", streamPrefix), // 低优先级任务 -// } -// -// // 根据优先级设置不同的保留时长 -// var maxAge time.Duration -// switch priority { -// case TaskPriorityHigh: -// maxAge = 30 * 24 * time.Hour // 高优先级保留30天 -// case TaskPriorityNormal: -// maxAge = 7 * 24 * time.Hour // 普通优先级保留7天 -// case TaskPriorityLow: -// maxAge = 24 * time.Hour // 低优先级保留1天 -// default: -// maxAge = 7 * 24 * time.Hour -// } -// -// config := TaskStreamConfig{ -// StreamName: streamPrefix, -// Subjects: subjects, -// Subject: fmt.Sprintf("%s.%s.>", streamPrefix, priority), -// Priority: priority, -// MaxAge: maxAge, -// MaxMsgsPerSub: 100000, -// Replicas: 1, -// Duplicates: 2 * time.Minute, -// } -// -// return CreateTaskStream(ctx, config) -//} -// -//// PublishTask 发布任务到指定流 -//// -//// 参数: -//// - ctx: 上下文 -//// - streamName: 流名称 -//// - task: 任务数据(会被JSON序列化) -//// -//// 返回: -//// - error: 错误信息 -//func PublishTask(ctx context.Context, streamName string, task interface{}) error { -// if !IsConnected() { -// return fmt.Errorf("NATS 未连接") -// } -// -// // 使用 JsPublish 发布消息 -// if err := JsPublish(ctx, streamName, task); err != nil { -// return fmt.Errorf("发布任务失败: %w", err) -// } -// -// return nil -//} -// -//// PublishTaskWithPriority 发布带优先级的任务 -//// -//// 参数: -//// - ctx: 上下文 -//// - streamPrefix: 流名称前缀 -//// - priority: 任务优先级 -//// - taskType: 任务类型 -//// - task: 任务数据(会被JSON序列化) -//// -//// 返回: -//// - error: 错误信息 -//func PublishTaskWithPriority(ctx context.Context, streamPrefix string, priority TaskPriority, taskType string, task interface{}) error { -// if !IsConnected() { -// return fmt.Errorf("NATS 未连接") -// } -// -// // 构建主题:{streamPrefix}.{priority}.{taskType} -// subject := fmt.Sprintf("%s.%s.%s", streamPrefix, priority, taskType) -// -// // 使用 JsPublish 发布消息 -// if err := JsPublish(ctx, subject, task); err != nil { -// return fmt.Errorf("发布任务失败: %w", err) -// } -// -// g.Log().Debugf(ctx, "任务已发布: %s (优先级: %s, 类型: %s)", subject, priority, taskType) -// -// return nil -//} -// -//// GetTaskStreamInfo 获取任务流信息 -//// -//// 参数: -//// - ctx: 上下文 -//// - streamName: 流名称 -//// -//// 返回: -//// - *jetstream.StreamInfo: 流信息 -//// - error: 错误信息 -//func GetTaskStreamInfo(ctx context.Context, streamName string) (*jetstream.StreamInfo, error) { -// if !IsConnected() { -// return nil, fmt.Errorf("NATS 未连接") -// } -// -// return GetStream(ctx, streamName) -//} -// -//// GetTaskConsumerInfo 获取任务消费者信息 -//// -//// 参数: -//// - ctx: 上下文 -//// - streamName: 流名称 -//// - consumerName: 消费者名称 -//// -//// 返回: -//// - *jetstream.ConsumerInfo: 消费者信息 -//// - error: 错误信息 -//func GetTaskConsumerInfo(ctx context.Context, streamName, consumerName string) (*jetstream.ConsumerInfo, error) { -// if !IsConnected() { -// return nil, fmt.Errorf("NATS 未连接") -// } -// -// return GetConsumer(ctx, streamName, consumerName) -//} -// -//// DeleteTaskStream 删除任务流 -//// -//// 注意:此操作会删除流及其所有消息,请谨慎使用 -//// -//// 参数: -//// - ctx: 上下文 -//// - streamName: 流名称 -//// -//// 返回: -//// - error: 错误信息 -//func DeleteTaskStream(ctx context.Context, streamName string) error { -// if !IsConnected() { -// return fmt.Errorf("NATS 未连接") -// } -// -// return DeleteStream(ctx, streamName) -//} -// -//// DeleteTaskConsumer 删除任务消费者 -//// -//// 参数: -//// - ctx: 上下文 -//// - streamName: 流名称 -//// - consumerName: 消费者名称 -//// -//// 返回: -//// - error: 错误信息 -//func DeleteTaskConsumer(ctx context.Context, streamName, consumerName string) error { -// if !IsConnected() { -// return fmt.Errorf("NATS 未连接") -// } -// -// return DeleteConsumer(ctx, streamName, consumerName) -//} diff --git a/nats/utils.go b/nats/utils.go deleted file mode 100644 index 1e43535..0000000 --- a/nats/utils.go +++ /dev/null @@ -1,87 +0,0 @@ -package nats - -import ( - "context" - "fmt" - "github.com/gogf/gf/v2/frame/g" - "github.com/nats-io/nats.go" - "go.opentelemetry.io/otel/trace" -) - -// ============ 上下文元数据工具函数 ============ -// 以下函数用于在 context 和 NATS 消息头之间互转元数据 - -// 定义常见的上下文元数据 key -const ( - TraceIDKey = "trace_id" - TokenKey = "token" -) - -func getTraceID(ctx context.Context) (traceID string, err error) { - // 提取 traceId:首先尝试从 OpenTelemetry Span 中提取,从 context 中提取 TraceID - span := trace.SpanFromContext(ctx) - if span != nil && span.SpanContext().HasTraceID() { - traceID = span.SpanContext().TraceID().String() - } else if tid := ctx.Value(TraceIDKey); tid != nil { - traceID = fmt.Sprintf("%v", tid) - } - if traceID == "" { - return traceID, fmt.Errorf("context 中没有 TraceID") - } - return -} - -// contextToHeaders 将 context 中的元数据转换为 NATS 消息头 -// 支持提取 user_id、tenant_id、trace_id、token 等常见字段 -func contextToHeaders(ctx context.Context) (nats.Header, error) { - headers := make(nats.Header) - - // 提取 traceId:首先尝试从 OpenTelemetry Span 中提取 - if traceID, err := getTraceID(ctx); err != nil { - return headers, err - } else { - headers.Set(TraceIDKey, traceID) - } - - // 提取 token(优先级:context value > HTTP Authorization header) - token := "" - if t := ctx.Value(TokenKey); t != nil { - token = fmt.Sprintf("%v", t) - } else if r := g.RequestFromCtx(ctx); r != nil { - // 从 HTTP 请求的 Authorization header 中提取 token - auth := r.GetHeader("Authorization") - if auth != "" { - // 移除 "Bearer " 前缀 - if len(auth) > 7 && auth[:7] == "Bearer " { - token = auth[7:] - } else { - token = auth - } - } - } - if token != "" { - headers.Set(TokenKey, token) - } - - return headers, nil -} - -// headersToContext 从 NATS 消息头重建 context -// 支持还原 user_id、tenant_id、trace_id、token 等字段 -func headersToContext(ctx context.Context, headers nats.Header) context.Context { - if headers == nil { - return ctx - } - - // 恢复 trace_id - if traceID := headers.Get(TraceIDKey); traceID != "" { - ctx = context.WithValue(ctx, TraceIDKey, traceID) - } - - // 恢复 token - if token := headers.Get(TokenKey); token != "" { - ctx = context.WithValue(ctx, TokenKey, token) - } - - return ctx -} From ee6d3c9033962f90e0f246e4b66ea03bf06d74b0 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Sat, 31 Jan 2026 05:17:14 +0800 Subject: [PATCH 204/225] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=B6=88=E6=81=AF?= =?UTF-8?q?=E9=98=9F=E5=88=97=E6=A8=A1=E5=9D=97=EF=BC=8C=E6=96=B0=E5=A2=9E?= =?UTF-8?q?NATS=E5=92=8CRabbitMQ=E8=BF=9E=E6=8E=A5=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=EF=BC=8C=E7=A7=BB=E9=99=A4=E6=97=A7=E7=89=88=E6=B6=88=E6=81=AF?= =?UTF-8?q?=E9=98=9F=E5=88=97=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- message/connection_nats.go | 144 +++++++++ message/connection_rabbitmq.go | 109 +++++++ message/message.go | 174 ----------- message/msg_interfaces.go | 26 ++ message/msg_plugin_manager.go | 133 +++++++++ message/msg_queue.go | 152 ---------- message/nats_client.go | 313 -------------------- message/nats_msg.go | 351 ++++++++++++++-------- message/nats_rpc.go | 74 ++--- message/rabbit.go | 351 ---------------------- message/rabbitmq_client.go | 210 -------------- message/rabbitmq_msg.go | 283 +++++++++--------- message/redis.go | 275 ------------------ message/redis_client.go | 468 ------------------------------ message/redis_msg.go | 280 +++++++++--------- middleware/module_tenant_check.go | 8 +- redis/redis.go | 6 +- 17 files changed, 966 insertions(+), 2391 deletions(-) create mode 100644 message/connection_nats.go create mode 100644 message/connection_rabbitmq.go delete mode 100644 message/message.go create mode 100644 message/msg_interfaces.go create mode 100644 message/msg_plugin_manager.go delete mode 100644 message/msg_queue.go delete mode 100644 message/nats_client.go delete mode 100644 message/rabbit.go delete mode 100644 message/rabbitmq_client.go delete mode 100644 message/redis.go delete mode 100644 message/redis_client.go diff --git a/message/connection_nats.go b/message/connection_nats.go new file mode 100644 index 0000000..e00f3f6 --- /dev/null +++ b/message/connection_nats.go @@ -0,0 +1,144 @@ +package message + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +var ( + nc *nats.Conn + js jetstream.JetStream + natsMu sync.RWMutex +) + +// natsConnect 建立 NATS 连接 +func natsConnect(ctx context.Context) error { + natsMu.Lock() + defer natsMu.Unlock() + + // 安全地关闭旧连接 + if oldConn := nc; oldConn != nil && !oldConn.IsClosed() { + oldConn.Close() + } + + // 从配置文件读取 NATS 地址 + natsURL := g.Cfg().MustGet(ctx, "nats.url").String() + if natsURL == "" { + // 默认使用本地地址 + natsURL = nats.DefaultURL + } + + // 使用独立的日志上下文,避免使用外部可能被取消的上下文 + logCtx := context.Background() + + // 连接选项配置 + opts := []nats.Option{ + nats.Name("goframe-nats-client"), + nats.ReconnectWait(2 * time.Second), + nats.MaxReconnects(-1), // 无限重连 + nats.PingInterval(10 * time.Second), + nats.MaxPingsOutstanding(5), + nats.ReconnectHandler(func(nc *nats.Conn) { + g.Log().Infof(logCtx, "✅ NATS 重连成功: %s", nc.ConnectedUrl()) + + natsMu.Lock() + defer natsMu.Unlock() + // 重新创建 JetStream 实例 + if newJS, err := jetstream.New(nc); err == nil { + js = newJS + } + }), + nats.DisconnectErrHandler(func(nc *nats.Conn, err error) { + g.Log().Warningf(logCtx, "⚠️ NATS 连接断开: %v, 准备重连...", err) + }), + nats.ClosedHandler(func(nc *nats.Conn) { + g.Log().Infof(logCtx, "NATS 连接已关闭: %s", nc.ConnectedUrl()) + }), + nats.ErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { + g.Log().Errorf(logCtx, "NATS 错误: %v", err) + }), + } + + var err error + nc, err = nats.Connect(natsURL, opts...) + if err != nil { + return fmt.Errorf("NATS 连接失败: %w", err) + } + + // 等待连接就绪 + if nc.Status() != nats.CONNECTED { + select { + case <-time.After(5 * time.Second): + // 连接超时,清理资源 + if nc != nil { + nc.Close() + } + return fmt.Errorf("NATS 连接超时") + case <-nc.StatusChanged(nats.CONNECTED): + // 连接成功 + case <-ctx.Done(): + // 外部上下文被取消,清理资源 + if nc != nil { + nc.Close() + } + return fmt.Errorf("NATS 连接被取消: %w", ctx.Err()) + } + } + + // 创建 JetStream 实例 + js, err = jetstream.New(nc) + if err != nil { + // 创建 JetStream 失败,清理连接 + if nc != nil { + nc.Close() + } + return fmt.Errorf("创建 JetStream 失败: %w", err) + } + + g.Log().Infof(ctx, "✅ NATS 连接成功: %s", nc.ConnectedUrl()) + return nil +} + +// natsPing 检测 NATS 连接状态 +func natsPing() bool { + natsMu.RLock() + defer natsMu.RUnlock() + + if nc == nil || nc.IsClosed() { + return false + } + + // 使用 NATS 的状态检查 + if nc.Status() != nats.CONNECTED { + return false + } + + return true +} + +// natsReconnect 重连 NATS +func natsReconnect(ctx context.Context) error { + if err := natsConnect(ctx); err != nil { + return fmt.Errorf("nats重连失败: %w", err) + } + return nil +} + +// natsClose 关闭 NATS 连接 +func natsClose(ctx context.Context) error { + natsMu.Lock() + defer natsMu.Unlock() + + if nc == nil || nc.IsClosed() { + return nil // 连接已经关闭或不存在 + } + nc.Close() + g.Log().Infof(ctx, "✅ NATS 连接已关闭") + return nil +} diff --git a/message/connection_rabbitmq.go b/message/connection_rabbitmq.go new file mode 100644 index 0000000..c98d2fe --- /dev/null +++ b/message/connection_rabbitmq.go @@ -0,0 +1,109 @@ +package message + +import ( + "context" + "fmt" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/util/gconv" + amqp "github.com/rabbitmq/amqp091-go" + "sync" + "time" +) + +var ( + conn *amqp.Connection + channel *amqp.Channel + rabbitmqMu sync.RWMutex +) + +// config RabbitMQ 配置 +type config struct { + Host string + Port int + Username string + Password string + VHost string +} + +func rabbitmqConnect(ctx context.Context) error { + rabbitmqMu.Lock() + defer rabbitmqMu.Unlock() + +LOOP: + cfg := &config{ + Host: g.Cfg().MustGet(ctx, "rabbitmq.host").String(), + Port: g.Cfg().MustGet(ctx, "rabbitmq.port").Int(), + Username: g.Cfg().MustGet(ctx, "rabbitmq.username").String(), + Password: g.Cfg().MustGet(ctx, "rabbitmq.password").String(), + VHost: g.Cfg().MustGet(ctx, "rabbitmq.vhost", "/").String(), + } + + url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost + + var err error + conn, err = amqp.Dial(url) + if err != nil { + g.Log().Errorf(ctx, "重连失败: %v", err) + + time.Sleep(2 * time.Second) + goto LOOP + } + + channel, err = conn.Channel() + if err != nil { + g.Log().Errorf(ctx, "创建 Channel 失败: %v", err) + + time.Sleep(2 * time.Second) + goto LOOP + } + + g.Log().Info(ctx, "RabbitMQ 重连成功") + return nil +} + +// rabbitmqReconnect 重新连接 +func rabbitmqReconnect(ctx context.Context) error { + if err := rabbitmqConnect(ctx); err != nil { + return fmt.Errorf("nats重连失败: %w", err) + } + return nil +} + +// rabbitmqPing 检测 RabbitMQ 连接状态 +func rabbitmqPing() bool { + rabbitmqMu.RLock() + defer rabbitmqMu.RUnlock() + + if conn == nil || conn.IsClosed() { + return false + } + + return true +} + +// rabbitmqClose 关闭连接 +func rabbitmqClose(ctx context.Context) error { + rabbitmqMu.Lock() + defer rabbitmqMu.Unlock() + + var lastErr error + + if channel != nil { + if err := channel.Close(); err != nil { + g.Log().Errorf(ctx, "关闭 RabbitMQ Channel 失败: %v", err) + lastErr = err + } + channel = nil + } + + if conn != nil { + if err := conn.Close(); err != nil { + g.Log().Errorf(ctx, "关闭 RabbitMQ 连接失败: %v", err) + lastErr = err + } + conn = nil + } + + g.Log().Info(ctx, "RabbitMQ 连接已关闭") + return lastErr +} diff --git a/message/message.go b/message/message.go deleted file mode 100644 index 321fc5d..0000000 --- a/message/message.go +++ /dev/null @@ -1,174 +0,0 @@ -package message - -import ( - "context" - "github.com/gogf/gf/v2/database/gredis" - - "github.com/gogf/gf/v2/errors/gerror" -) - -func GetRedisClientTest(name string) *gredis.Redis { - return getRedisClientTest(name) -} - -// GetLock 获取分布式锁 -func GetLock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context) error) (success bool, err error) { - return lock(ctx, key, expireSeconds, fn) -} - -// MessageConfig 消息配置接口 -type MessageConfig interface { - start(ctx context.Context) error - publish(ctx context.Context, data interface{}, options ...map[string]interface{}) (messageID string, err error) -} - -// RedisMessageConfig Redis Stream 消息配置 -type RedisMessageConfig struct { - StreamKey string // Stream 键名 - GroupName string // 消费者组名称 - ConsumerName string // 消费者名称 - BatchSize int64 // 最大并发数(信号量容量) - AutoAck bool // ACK确认,true自动确认,false手动确认 - HandleFunc func(ctx context.Context, message map[string]interface{}) error -} - -func (r *RedisMessageConfig) start(ctx context.Context) error { - return readFromStream(ctx, QueueMessage{ - StreamKey: r.StreamKey, - GroupName: r.GroupName, - ConsumerName: r.ConsumerName, - BatchSize: r.BatchSize, - AutoAck: r.AutoAck, - HandleFunc: r.HandleFunc, - }) -} - -func (r *RedisMessageConfig) publish(ctx context.Context, data interface{}, options ...map[string]interface{}) (messageID string, err error) { - return publishToRedis(ctx, r.StreamKey, data) -} - -// RabbitMQMessageConfig RabbitMQ 消息配置 -type RabbitMQMessageConfig struct { - Queue string // 队列名称 - Exchange string // 交换器名称 - RoutingKey string // 路由键 - PrefetchCount int // QoS: 预取数量(并发控制) - WorkerCount int // worker 数量 - ConsumerTag string // 消费者标签 - HandleFunc func(ctx context.Context, message map[string]interface{}) error -} - -func (r *RabbitMQMessageConfig) start(ctx context.Context) error { - return startRabbitMQConsumer(ctx, QueueMessage{ - Queue: r.Queue, - Exchange: r.Exchange, - RoutingKey: r.RoutingKey, - PrefetchCount: r.PrefetchCount, - WorkerCount: r.WorkerCount, - ConsumerTag: r.ConsumerTag, - AutoAck: true, - HandleFunc: r.HandleFunc, - }) -} - -func (r *RabbitMQMessageConfig) publish(ctx context.Context, data interface{}, options ...map[string]interface{}) (messageID string, err error) { - opts := make(map[string]interface{}) - if len(options) > 0 { - opts = options[0] - } - exchange := r.Exchange - routingKey := r.RoutingKey - delay := 0 - - if v, ok := opts["exchange"].(string); ok { - exchange = v - } - if v, ok := opts["routingKey"].(string); ok { - routingKey = v - } - if v, ok := opts["delay"].(int); ok { - delay = v - } - - if delay > 0 { - return publishDelayedToRabbitMQ(ctx, exchange, routingKey, data, delay) - } - return publishToRabbitMQ(ctx, exchange, routingKey, data) -} - -// QueueMessage 统一消息队列配置结构体(内部使用) -type QueueMessage struct { - // Redis Stream 配置 - StreamKey string - GroupName string - ConsumerName string - BatchSize int64 - AutoAck bool - HandleFunc func(ctx context.Context, message map[string]interface{}) error - - // RabbitMQ 配置 - Queue string - Exchange string - RoutingKey string - PrefetchCount int - WorkerCount int - ConsumerTag string -} - -// StartConsumers 启动消息消费者(统一入口) -// 支持同时启动多个消费者,包括 Redis Stream 和 RabbitMQ -func StartConsumers(ctx context.Context, configs ...MessageConfig) error { - for _, cfg := range configs { - if err := cfg.start(ctx); err != nil { - return gerror.Wrap(err, "启动消费者失败") - } - } - return nil -} - -// PublishMessage 发布消息(统一入口) -// 根据配置类型选择发布到 Redis Stream 或 RabbitMQ -func PublishMessage(ctx context.Context, cfg MessageConfig, data interface{}, options ...map[string]interface{}) (messageID string, err error) { - return cfg.publish(ctx, data, options...) -} - -// ========== Redis Stream 公共方法(方便迁移) ========== - -// AddToStream 将消息添加到 Redis Stream -//func AddToStream(ctx context.Context, streamKey string, msg interface{}) (messageID string, err error) { -// return addToStream(ctx, streamKey, msg) -//} - -// ReadFromStream 从 Redis Stream 读取消息(已废弃) -// 请使用 RedisMessageConfig.StartConsumers 启动消费者 -// 此方法保留用于向后兼容,但实际不会返回消息(异步消费模式) -func ReadFromStream(ctx context.Context, streamKey, groupName, consumerName string, count, blockMs int64) ([]StreamMessage, error) { - return nil, gerror.New("ReadFromStream 已废弃,请使用 RedisMessageConfig.StartConsumers 启动消费者") -} - -// AckMessage 确认 Redis Stream 消息 -func AckMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error { - return ackMessage(ctx, streamKey, groupName, messageIDs...) -} - -// InitStreamGroup 初始化 Redis Stream 消费者组 -func InitStreamGroup(ctx context.Context, streamKey, groupName string) error { - return initStreamGroup(ctx, streamKey, groupName) -} - -// ========== RabbitMQ 公共方法(方便迁移) ========== - -// InitRabbitMQ 初始化 RabbitMQ 连接 -func InitRabbitMQ(ctx context.Context) error { - return initRabbitMQ(ctx) -} - -// PublishToRabbitMQ 发布消息到 RabbitMQ -//func PublishToRabbitMQ(ctx context.Context, exchange, routingKey string, message interface{}) error { -// return publishToRabbitMQ(ctx, exchange, routingKey, message) -//} - -// PublishDelayedToRabbitMQ 发布延时消息到 RabbitMQ -//func PublishDelayedToRabbitMQ(ctx context.Context, exchange, routingKey string, message interface{}, delaySeconds int) error { -// return publishDelayedToRabbitMQ(ctx, exchange, routingKey, message, delaySeconds) -//} diff --git a/message/msg_interfaces.go b/message/msg_interfaces.go new file mode 100644 index 0000000..125cf81 --- /dev/null +++ b/message/msg_interfaces.go @@ -0,0 +1,26 @@ +package message + +import "context" + +type messagePublishConfig interface { + GetPublishMsgType() +} + +type messageSubscribeConfig interface { + GetSubscribeMsgType() +} + +// messageUtil 消息队列公共配置接口 +// 只暴露核心的发布/订阅功能,配置访问器方法不需要在公共接口中 +type messageUtil interface { + // Publish 发布消息 + Publish(ctx context.Context, msg messagePublishConfig) error + // Subscribe 订阅消息 + Subscribe(ctx context.Context, msg messageSubscribeConfig) error + // Ping 检测连接状态 + ping(ctx context.Context) bool + // Reconnect 重连 + reconnect(ctx context.Context) error + // Close 关闭连接 + close(ctx context.Context) error +} diff --git a/message/msg_plugin_manager.go b/message/msg_plugin_manager.go new file mode 100644 index 0000000..84c79eb --- /dev/null +++ b/message/msg_plugin_manager.go @@ -0,0 +1,133 @@ +package message + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/gogf/gf/v2/frame/g" +) + +// MessageType 消息队列类型 +type messageType string + +const ( + // MessageRedis Redis 消息队列 + MessageRedis messageType = "redis" + // MessageRabbitMQ RabbitMQ 消息队列 + MessageRabbitMQ messageType = "rabbitmq" + // MessageNATS NATS 消息队列 + MessageNATS messageType = "nats" +) + +// configFactory 消息队列配置工厂函数类型 +type configFactory func() messageUtil + +// PluginManager 消息队列插件管理器 +type pluginManager struct { + mu sync.RWMutex + instances map[messageType]messageUtil // 已连接的插件实例 +} + +var ( + defaultPluginManager = newPluginManager() + // 不再支持默认插件类型,必须显式指定类型 +) + +// newPluginManager 创建插件管理器 +func newPluginManager() *pluginManager { + return &pluginManager{ + instances: make(map[messageType]messageUtil), + } +} + +// RegisterPlugin 注册消息队列插件 +// 所有插件必须通过此方法注册,自动进行连接检测 +// 只有连接成功的插件才会被注册,连接失败的插件不会被注册 +// 异步无限重连,只有连接成功了才注册 +func registerPlugin(msgType messageType, factory configFactory) error { + if factory == nil { + return fmt.Errorf("factory cannot be nil") + } + + // 创建实例 + instance := factory() + ctx := context.Background() + + // 开启异步连接,无限重连直到成功 + go func() { + retryInterval := 2 * time.Second + maxInterval := 30 * time.Second + + for { + select { + case <-ctx.Done(): + g.Log().Errorf(ctx, "❌ [%s] 注册被取消", msgType) + return + default: + // 尝试连接(使用Reconnect方法) + if err := instance.reconnect(ctx); err == nil { + // 连接成功,注册插件 + if err := defaultPluginManager.register(msgType, instance); err != nil { + g.Log().Errorf(ctx, "❌ [%s] 注册插件失败: %v", msgType, err) + instance.close(ctx) + } else { + g.Log().Infof(ctx, "✅ [%s] 插件注册成功", msgType) + } + return + } + + // 连接失败,记录日志并等待重试 + g.Log().Warningf(ctx, "⚠️ [%s] 连接失败,%v 后重试...", msgType, retryInterval) + + select { + case <-time.After(retryInterval): + // 增加重试间隔,但不超过最大值 + retryInterval *= 2 + if retryInterval > maxInterval { + retryInterval = maxInterval + } + case <-ctx.Done(): + g.Log().Errorf(ctx, "❌ [%s] 注册被取消", msgType) + return + } + } + } + }() + + return nil +} + +// register 注册插件(内部方法) +func (m *pluginManager) register(msgType messageType, instance messageUtil) error { + m.mu.Lock() + defer m.mu.Unlock() + m.instances[msgType] = instance + return nil +} + +// GetMsgPlugin 获取消息队列插件 +func GetMsgPlugin(msgType messageType) (messageUtil, error) { + defaultPluginManager.mu.RLock() + instance, ok := defaultPluginManager.instances[msgType] + defaultPluginManager.mu.RUnlock() + + if !ok { + return nil, fmt.Errorf("unsupported message type: %s", msgType) + } + + return instance, nil +} + +// GetSupportedTypes 获取所有已注册的插件类型 +func GetSupportedTypes() []messageType { + defaultPluginManager.mu.RLock() + defer defaultPluginManager.mu.RUnlock() + + types := make([]messageType, 0, len(defaultPluginManager.instances)) + for t := range defaultPluginManager.instances { + types = append(types, t) + } + return types +} diff --git a/message/msg_queue.go b/message/msg_queue.go deleted file mode 100644 index be1f2da..0000000 --- a/message/msg_queue.go +++ /dev/null @@ -1,152 +0,0 @@ -package message - -import ( - "context" - "fmt" -) - -type RedisConfig struct { - // Stream 名称 - Stream string - - // 消费者组名称 - Group string - - // 消费者名称 - Consumer string - - // 每次消费数量 - Count int64 - - // 是否自动 ACK - AutoAck bool - - // 处理函数 - HandleFunc func(ctx context.Context, message map[string]interface{}) error -} - -// RabbitMQConfig RabbitMQ 队列配置 -type RabbitMQConfig struct { - Mode string - Exchange string - Topic string - DelayMessage bool - - // 队列名称(必需) - Name string - - // 实际队列名(用于绑定) - Queue string - - // 是否持久化 - Durable bool - - // QoS 预取数量(每次推送的消息数量,默认10) - PrefetchCount int - - // 最大重试次数(默认3) - MaxRetry int - - // 是否自动 ACK - AutoAck bool - - // 处理函数 - HandleFunc func(ctx context.Context, message map[string]interface{}) error -} - -// NATSConfig NATS 队列配置 -type NATSConfig struct { - DelayMessage bool - // Stream 名称 - Stream string - - // 消费者名称 - Consumer string - - // 是否持久化 - Durable bool - - // 副本数 - Replicas int - // QoS 预取数量(每次推送的消息数量,默认10) - PrefetchCount int - - // 是否自动 ACK - AutoAck bool - - // 处理函数 - HandleFunc func(ctx context.Context, message map[string]interface{}) error -} - -// messageBroker 消息代理接口 -type messageBroker interface { - // StreamGroup 创建消费组(支持单个配置或批量配置) - streamGroup(ctx context.Context, configs ...interface{}) error - - // Publish 发布消息(支持单个配置或批量配置) - publish(ctx context.Context, config interface{}, data interface{}) error - - // PublishDelayed 发布延迟消息(支持单个配置或批量配置) - publishDelayed(ctx context.Context, config interface{}, data interface{}, delay int) error - - // Subscribe 订阅消息(支持单个配置或批量配置) - subscribe(ctx context.Context, configs ...interface{}) error -} - -type messageClientType string - -const ( - ClientTypeRedis messageClientType = "redis" - ClientTypeRabbitMQ messageClientType = "rabbitmq" - ClientTypeNATS messageClientType = "nats" -) - -// newMessageBroker 创建消息代理实例 -func newMessageBroker(ctx context.Context, clientType messageClientType) (messageBroker, error) { - switch clientType { - case ClientTypeRedis: - return &redisMessageClient{clientType: clientType}, nil - case ClientTypeRabbitMQ: - return &rabbitMQMessageClient{clientType: clientType}, nil - case ClientTypeNATS: - return &natsMessageClient{clientType: clientType}, nil - default: - return nil, fmt.Errorf("unknown client type: %s", clientType) - } -} - -// StreamGroup 直接创建消费组 -func StreamGroup(ctx context.Context, clientType messageClientType, configs ...interface{}) error { - broker, err := newMessageBroker(ctx, clientType) - if err != nil { - return err - } - return broker.streamGroup(ctx, configs...) -} - -// Publish 直接发布消息 -func Publish(ctx context.Context, clientType messageClientType, config interface{}, data interface{}) error { - broker, err := newMessageBroker(ctx, clientType) - if err != nil { - return err - } - return broker.publish(ctx, config, data) -} - -// PublishDelayed 直接发布延迟消息 -func PublishDelayed(ctx context.Context, clientType messageClientType, config interface{}, data interface{}, delay int) error { - broker, err := newMessageBroker(ctx, clientType) - if err != nil { - return err - } - return broker.publishDelayed(ctx, config, data, delay) -} - -// Subscribe 直接订阅消息 -func Subscribe(ctx context.Context, clientType messageClientType, configs ...interface{}) error { - broker, err := newMessageBroker(ctx, clientType) - if err != nil { - return err - } - return broker.subscribe(ctx, configs...) -} diff --git a/message/nats_client.go b/message/nats_client.go deleted file mode 100644 index 470bef7..0000000 --- a/message/nats_client.go +++ /dev/null @@ -1,313 +0,0 @@ -package message - -import ( - "context" - "fmt" - "sync" - "sync/atomic" - "time" - - "github.com/gogf/gf/v2/frame/g" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" -) - -var ( - nc *nats.Conn - js jetstream.JetStream - inited bool - natsMu sync.RWMutex - natsURL string - healthCtx context.Context - healthCancel context.CancelFunc - connected bool - reconnectChan chan struct{} - - // 连接状态变化监听器 - connStateListeners []connStateListener - connListenersMu sync.RWMutex - - // 监控指标 - metrics metricsCounter -) - -// Metrics 监控指标 -type metricsCounter struct { - PublishCount atomic.Int64 - PublishError atomic.Int64 - SubscribeCount atomic.Int64 - RequestCount atomic.Int64 - RequestError atomic.Int64 - ConsumeCount atomic.Int64 - ConsumeError atomic.Int64 -} - -// ConnState 连接状态 -type connState int - -const ( - connStateDisconnected connState = iota - connStateConnecting - connStateConnected - connStateReconnecting - connStateClosed -) - -// ConnStateListener 连接状态监听器 -type connStateListener func(state connState, err error) - -// GetMetrics 获取监控指标 -func getMetrics() metricsCounter { - return metrics -} - -// registerConnStateListener 注册连接状态监听器 -func registerConnStateListener(listener connStateListener) { - connListenersMu.Lock() - defer connListenersMu.Unlock() - connStateListeners = append(connStateListeners, listener) -} - -// unregisterConnStateListener 取消注册连接状态监听器 -func unregisterConnStateListener(listener connStateListener) { - connListenersMu.Lock() - defer connListenersMu.Unlock() - for i, l := range connStateListeners { - if l != nil && &l == &listener { - connStateListeners = append(connStateListeners[:i], connStateListeners[i+1:]...) - break - } - } -} - -// notifyConnState 通知所有监听器连接状态变化 -func notifyConnState(state connState, err error) { - connListenersMu.RLock() - listeners := make([]connStateListener, len(connStateListeners)) - copy(listeners, connStateListeners) - connListenersMu.RUnlock() - - for _, listener := range listeners { - if listener != nil { - listener(state, err) - } - } -} - -// init 初始化 NATS 连接 -func init() { - // 从配置文件读取 NATS 地址 - natsURL = g.Cfg().MustGet(context.Background(), "nats.url").String() - if natsURL == "" { - // 默认使用本地地址 - natsURL = nats.DefaultURL - } - - // 创建健康检查上下文 - healthCtx, healthCancel = context.WithCancel(context.Background()) - - // 创建重连通知通道(增大缓冲区避免丢失通知) - reconnectChan = make(chan struct{}, 10) - - // 启动连接 - go initConnection() - - // 启动健康检查协程 - go healthCheck() -} - -// initConnection 初始化连接 -func initConnection() { - ctx := context.Background() - notifyConnState(connStateConnecting, nil) - if err := connect(ctx); err != nil { - g.Log().Errorf(ctx, "NATS 初始连接失败: %v", err) - notifyConnState(connStateDisconnected, err) - } -} - -// connect 建立 NATS 连接 -func connect(ctx context.Context) error { - natsMu.Lock() - defer natsMu.Unlock() - - if nc != nil && !nc.IsClosed() { - nc.Close() - } - - // 连接选项配置 - opts := []nats.Option{ - nats.Name("goframe-nats-client"), - nats.ReconnectWait(2 * time.Second), - nats.MaxReconnects(-1), // 无限重连 - nats.PingInterval(10 * time.Second), - nats.MaxPingsOutstanding(5), - nats.ReconnectHandler(func(nc *nats.Conn) { - g.Log().Infof(ctx, "✅ NATS 重连成功: %s", nc.ConnectedUrl()) - connected = true - - // 重新创建 JetStream 实例 - if newJS, err := jetstream.New(nc); err == nil { - js = newJS - } - - // 通知重连成功 - notifyConnState(connStateConnected, nil) - - // 使用非阻塞发送避免阻塞 - select { - case reconnectChan <- struct{}{}: - default: - // 通道已满,丢弃通知 - } - }), - nats.DisconnectErrHandler(func(nc *nats.Conn, err error) { - g.Log().Warningf(ctx, "⚠️ NATS 连接断开: %v, 准备重连...", err) - connected = false - notifyConnState(connStateReconnecting, err) - }), - nats.ClosedHandler(func(nc *nats.Conn) { - g.Log().Infof(ctx, "NATS 连接已关闭: %s", nc.ConnectedUrl()) - connected = false - notifyConnState(connStateClosed, nil) - }), - nats.ErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { - g.Log().Errorf(ctx, "NATS 错误: %v", err) - }), - } - - var err error - nc, err = nats.Connect(natsURL, opts...) - if err != nil { - return fmt.Errorf("NATS 连接失败: %w", err) - } - - // 等待连接就绪 - if nc.Status() != nats.CONNECTED { - select { - case <-time.After(5 * time.Second): - notifyConnState(connStateDisconnected, fmt.Errorf("连接超时")) - return fmt.Errorf("NATS 连接超时") - case <-nc.StatusChanged(nats.CONNECTED): - } - } - - // 创建 JetStream 实例 - js, err = jetstream.New(nc) - if err != nil { - return fmt.Errorf("创建 JetStream 失败: %w", err) - } - - connected = true - inited = true - g.Log().Infof(ctx, "✅ NATS 连接成功: %s", nc.ConnectedUrl()) - notifyConnState(connStateConnected, nil) - return nil -} - -// healthCheck 健康检查协程(仅作为备用检查) -func healthCheck() { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - - for { - select { - case <-healthCtx.Done(): - return - case <-ticker.C: - natsMu.RLock() - currentConnected := connected - currentConn := nc - natsMu.RUnlock() - - if !currentConnected || currentConn == nil || currentConn.IsClosed() { - // 仅记录日志,不尝试重连(NATS 已有自动重连机制) - g.Log().Warning(context.Background(), "NATS 连接断开,等待 NATS 自动重连...") - } - case <-reconnectChan: - // 重连成功的通知(仅记录日志) - g.Log().Info(context.Background(), "收到重连成功通知") - } - } -} - -// checkConnected 检查连接状态 -func checkConnected() bool { - natsMu.RLock() - defer natsMu.RUnlock() - return connected && nc != nil && !nc.IsClosed() -} - -// getConnState 获取当前连接状态 -func getConnState() connState { - natsMu.RLock() - defer natsMu.RUnlock() - - if nc == nil { - return connStateDisconnected - } - - if nc.IsClosed() { - return connStateClosed - } - - if connected { - return connStateConnected - } - - return connStateDisconnected -} - -// shutdown 优雅关闭:自动注销所有已注册的服务并关闭 NATS 连接 -func shutdown() error { - ctx := context.Background() - g.Log().Info(ctx, "开始优雅关闭 NATS RPC 服务...") - - // 注销所有单实例服务 - rpcServicesMu.Lock() - singleServiceCount := len(rpcServices) - for serviceName := range rpcServices { - if sub, exists := rpcSubs[serviceName]; exists { - if err := sub.Unsubscribe(); err != nil { - g.Log().Errorf(ctx, "注销服务 %s 失败: %v", serviceName, err) - } - } - delete(rpcSubs, serviceName) - delete(rpcServices, serviceName) - } - rpcServicesMu.Unlock() - - // 注销所有队列服务 - queueRPCMu.Lock() - queueServiceCount := 0 - for queueName, servicesMap := range queueRPCServices { - queueServiceCount += len(servicesMap) - for serviceName, sub := range queueRPCSubs[queueName] { - if err := sub.Unsubscribe(); err != nil { - g.Log().Errorf(ctx, "注销队列服务 %s (队列: %s) 失败: %v", serviceName, queueName, err) - } - } - delete(queueRPCSubs, queueName) - delete(queueRPCServices, queueName) - } - queueRPCMu.Unlock() - - g.Log().Infof(ctx, "已注销 %d 个单实例服务和 %d 个队列服务", singleServiceCount, queueServiceCount) - - natsMu.Lock() - defer natsMu.Unlock() - - // 停止健康检查协程 - if healthCancel != nil { - healthCancel() - } - - // 关闭连接 - if nc != nil && !nc.IsClosed() { - nc.Close() - connected = false - inited = false - } - g.Log().Info(ctx, "NATS RPC 服务已优雅关闭") - return nil -} diff --git a/message/nats_msg.go b/message/nats_msg.go index 016917f..205ac07 100644 --- a/message/nats_msg.go +++ b/message/nats_msg.go @@ -4,195 +4,274 @@ import ( "context" "encoding/json" "fmt" - "time" - "github.com/gogf/gf/v2/frame/g" + "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "time" ) -// natsMessageClient NATS 实现 -type natsMessageClient struct { - clientType messageClientType +type NatsPublishMsgConfig struct { + QueueName string + Durable bool + DelayTime int + Data any } -// StreamGroup 创建消费组(支持单个或批量) -func (q *natsMessageClient) streamGroup(ctx context.Context, configs ...interface{}) error { - if len(configs) == 0 { - return fmt.Errorf("配置不能为空") +type NatsSubscribeMsgConfig struct { + QueueName string + Durable bool + DelayTime int + ConsumerName string + AutoAck bool + PrefetchCount int + HandleFunc func(ctx context.Context, message map[string]interface{}) error +} + +func (*NatsPublishMsgConfig) GetPublishMsgType() { + +} + +func (*NatsSubscribeMsgConfig) GetSubscribeMsgType() { + +} + +func init() { + // 注册 Nats 插件,必须使用 RegisterPlugin 确保连接检测 + registerPlugin(MessageNATS, func() messageUtil { + return &natsMsg{} + }) +} + +type natsMsg struct{} + +// Ping 检测 NATS 连接状态 +func (c *natsMsg) ping(_ context.Context) bool { + return natsPing() +} + +// Reconnect 重连 NATS +func (c *natsMsg) reconnect(ctx context.Context) error { + return natsReconnect(ctx) +} + +// Close 关闭 NATS 连接 +func (c *natsMsg) close(ctx context.Context) error { + return natsClose(ctx) +} + +// Publish 发布消息 +func (c *natsMsg) Publish(ctx context.Context, msgConfig messagePublishConfig) error { + cfg, ok := msgConfig.(*NatsPublishMsgConfig) + if !ok { + return fmt.Errorf("无效的 NATS 配置类型") } - for _, config := range configs { - cfg, ok := config.(*NATSConfig) - if !ok { - return fmt.Errorf("无效的 NATS 配置类型") + if g.IsEmpty(cfg.QueueName) { + return fmt.Errorf("必须提供队列名称") + } + if g.IsEmpty(cfg.Data) { + return fmt.Errorf("必须提供数据") + } + return c.createPublish(ctx, cfg.QueueName, cfg.Durable, cfg.DelayTime, cfg.Data) +} + +// Publish 发布消息 +func (c *natsMsg) createPublish(ctx context.Context, subject string, durable bool, delayTime int, data any) error { + delayMsg := delayTime > 0 + if err := c.createStreamGroupInternal(ctx, subject, durable, delayMsg); err != nil { + return err + } + payload, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("序列化数据失败: %w", err) + } + + msg := &nats.Msg{ + Subject: subject, + Data: payload, + } + + if delayMsg { + // 计算目标投递时间 + targetTime := time.Now().Add(time.Duration(delayTime) * time.Second) + delayNs := time.Until(targetTime).Nanoseconds() + if delayNs < 0 { + delayNs = 0 } - if err := q.createStreamGroup(ctx, cfg); err != nil { - return err + + g.Log().Infof(ctx, "📅 NATS 延迟消息配置: DelayTime=%d秒, TargetTime=%v, DelayNs=%d纳秒(%.2f秒)", + delayTime, targetTime.Format("2006-01-02 15:04:05"), delayNs, float64(delayNs)/float64(time.Second.Nanoseconds())) + + // NATS JetStream 延迟消息使用 Nats-Msg-Delay Header(纳秒数) + msg.Header = nats.Header{ + "Nats-Msg-Delay": []string{fmt.Sprintf("%d", delayNs)}, + } + g.Log().Infof(ctx, "📅 NATS 延迟消息 Header: %v", msg.Header) + + // 获取 Stream 配置验证 + streamName, _ := getStreamInfo(durable, delayMsg) + stream, err := js.Stream(ctx, streamName) + if err == nil { + info, _ := stream.Info(ctx) + g.Log().Infof(ctx, "📅 Stream 配置: AllowMsgSchedules=%v, Storage=%v", + info.Config.AllowMsgSchedules, info.Config.Storage) + if !info.Config.AllowMsgSchedules { + g.Log().Errorf(ctx, "❌ Stream 不支持延迟消息!AllowMsgSchedules=false") + } } } + + // 发布消息到 JetStream + ack, err := js.PublishMsg(ctx, msg) + if err != nil { + g.Log().Errorf(ctx, "❌ NATS 发布消息失败: err=%v", err) + return err + } + + g.Log().Infof(ctx, "✅ NATS 发布消息成功: StreamSeq=%d, Domain=%s", ack.Sequence, ack.Domain) return nil } -// createStreamGroup 内部单个创建消费组 -func (q *natsMessageClient) createStreamGroup(ctx context.Context, cfg *NATSConfig) error { - // Stream 不存在,创建新的 - storage := jetstream.FileStorage - if !cfg.Durable { - storage = jetstream.MemoryStorage - } - if g.IsEmpty(cfg.Replicas) { - cfg.Replicas = 1 +// createStreamGroup 内部创建消费组 +func (c *natsMsg) createStreamGroupInternal(ctx context.Context, subject string, durable, delayMsg bool) error { + streamName, storage := getStreamInfo(durable, delayMsg) + + // 先检查 Stream 是否存在 + stream, err := js.Stream(ctx, streamName) + if err == nil { + // Stream 已存在,检查配置是否匹配 + info, _ := stream.Info(ctx) + if info.Config.AllowMsgSchedules != delayMsg || info.Config.Storage != storage { + g.Log().Infof(ctx, "🔄 Stream 配置不匹配,正在重新创建: stream=%s, 当前AllowMsgSchedules=%v, 需要%v", + streamName, info.Config.AllowMsgSchedules, delayMsg) + // 删除旧 Stream + if err := js.DeleteStream(ctx, streamName); err != nil { + g.Log().Warningf(ctx, "删除旧 Stream 失败: %v", err) + } + } else { + g.Log().Infof(ctx, "✅ Stream 已存在且配置正确: stream=%s", streamName) + return nil + } } + // 构建流配置 jsConfig := jetstream.StreamConfig{ - Name: cfg.Stream, - Subjects: []string{fmt.Sprintf("%s.>", cfg.Stream)}, - Replicas: cfg.Replicas, - NoAck: cfg.AutoAck, - AllowMsgSchedules: cfg.DelayMessage, // 延迟消息核心开关 + Name: streamName, + Subjects: []string{subject}, + AllowMsgSchedules: delayMsg, // 延迟消息核心开关 Storage: storage, Discard: jetstream.DiscardOld, // 达到上限删除旧消息 } - // 检查流是否已存在 - stream, err := js.Stream(ctx, cfg.Stream) - if err == nil { - // 流已存在,更新配置 - _, err = js.UpdateStream(ctx, jsConfig) - if err != nil { - return fmt.Errorf("更新任务流失败: %w", err) - } - g.Log().Infof(ctx, "任务流已更新: %s", stream.CachedInfo().Config.Name) - return nil - } - // 创建新流 + stream, err = js.CreateStream(ctx, jsConfig) if err != nil { return fmt.Errorf("创建任务流失败: %w", err) } - g.Log().Infof(ctx, "✅ NATS 队列初始化成功: stream=%s, consumer=%s", cfg.Stream, cfg.Consumer) + // 获取 Stream 信息验证配置 + info, err := stream.Info(ctx) + if err == nil { + g.Log().Infof(ctx, "✅ NATS 队列初始化成功: stream=%s, AllowMsgSchedules=%v, Storage=%v", + streamName, info.Config.AllowMsgSchedules, info.Config.Storage) + } + + g.Log().Infof(ctx, "✅ NATS 队列初始化成功: stream=%s", streamName) return nil } -// Publish 发布消息(支持单个或批量) -func (q *natsMessageClient) publish(ctx context.Context, config interface{}, data interface{}) error { - cfg, ok := config.(*NATSConfig) +// Subscribe 订阅消息 +func (c *natsMsg) Subscribe(ctx context.Context, msgConfig messageSubscribeConfig) error { + cfg, ok := msgConfig.(*NatsSubscribeMsgConfig) if !ok { return fmt.Errorf("无效的 NATS 配置类型") } - err := q.createStreamGroup(ctx, cfg) - if err != nil { - return err + if g.IsEmpty(cfg.QueueName) { + return fmt.Errorf("必须提供队列名称") } - payload, err := json.Marshal(data) - if err != nil { - return fmt.Errorf("序列化数据失败: %w", err) + if g.IsEmpty(cfg.ConsumerName) { + return fmt.Errorf("必须提供消费者名称") } - - // 发布消息到 JetStream - subject := fmt.Sprintf("%s.>", cfg.Stream) - _, err = js.Publish(ctx, subject, payload) - if err != nil { - g.Log().Errorf(ctx, "❌ NATS 发布消息失败: topic=%s, err=%v", cfg.Stream, err) - return err + if g.IsEmpty(cfg.HandleFunc) { + return fmt.Errorf("必须提供处理函数") } - - g.Log().Infof(ctx, "✅ NATS 发布消息成功: topic=%s", cfg.Stream) - return nil + if g.IsEmpty(cfg.PrefetchCount) { + cfg.PrefetchCount = 1 + } + return c.createSubscribeInternal(ctx, cfg.QueueName, cfg.ConsumerName, cfg.PrefetchCount, cfg.AutoAck, cfg.Durable, cfg.DelayTime, cfg.HandleFunc) } -// PublishDelayed 发布延迟消息(支持单个或批量) -func (q *natsMessageClient) publishDelayed(ctx context.Context, config interface{}, data interface{}, delay int) error { +// createSubscribe 内部订阅消息 +func (c *natsMsg) createSubscribeInternal(ctx context.Context, subject, consumerName string, prefetchCount int, autoAck, durable bool, delayTime int, handler func(ctx context.Context, message map[string]interface{}) error) error { + g.Log().Infof(ctx, "🔔 NATS 开始订阅: QueueName=%s, ConsumerName=%s", subject, consumerName) + delayMsg := delayTime > 0 + streamName, _ := getStreamInfo(durable, delayMsg) - cfg, ok := config.(*NATSConfig) - if !ok { - return fmt.Errorf("无效的 NATS 配置类型") + // 确保 Stream 存在,如果不存在则创建 + if err := c.createStreamGroupInternal(ctx, subject, durable, delayMsg); err != nil { + g.Log().Errorf(ctx, "创建 Stream 失败: %v", err) + return fmt.Errorf("创建 Stream 失败: %w", err) } - payload, err := json.Marshal(data) - if err != nil { - return fmt.Errorf("序列化数据失败: %w", err) - } - - // 使用 goroutine 实现简单的延迟发布 - go func() { - time.Sleep(time.Duration(delay)) - subject := fmt.Sprintf("%s.>", cfg.Stream) - if err := q.publishInternal(ctx, subject, payload); err != nil { - g.Log().Errorf(ctx, "❌ NATS 延迟消息发布失败: topic=%s, delay=%v, err=%v", cfg.Stream, delay, err) - } - }() - - g.Log().Infof(ctx, "✅ NATS 延迟消息已提交: topic=%s, delay=%v", cfg.Stream, delay) - - return nil -} - -// publishInternal 内部发布消息 -func (q *natsMessageClient) publishInternal(ctx context.Context, subject string, payload []byte) error { - _, err := js.Publish(ctx, subject, payload) - return err -} - -// Subscribe 订阅消息(支持单个或批量) -func (q *natsMessageClient) subscribe(ctx context.Context, configs ...interface{}) error { - if len(configs) == 0 { - return fmt.Errorf("配置不能为空") - } - - for _, config := range configs { - cfg, ok := config.(*NATSConfig) - if !ok { - return fmt.Errorf("无效的 NATS 配置类型") - } - handler := cfg.HandleFunc - if handler == nil { - return fmt.Errorf("必须提供处理函数") - } - if err := q.createSubscribe(ctx, cfg, handler); err != nil { - return err - } - } - return nil -} - -// subscribe 内部单个订阅消息 -func (q *natsMessageClient) createSubscribe(ctx context.Context, cfg *NATSConfig, handler func(ctx context.Context, message map[string]interface{}) error) error { - g.Log().Infof(ctx, "🔔 NATS 开始订阅: stream=%s, consumer=%s", cfg.Stream, cfg.Consumer) // Stream 不存在,创建新的 ackPolicy := jetstream.AckExplicitPolicy - if cfg.AutoAck { + if autoAck { ackPolicy = jetstream.AckNonePolicy } jsConfig := jetstream.ConsumerConfig{ - Name: cfg.Consumer, - Durable: cfg.Consumer, + Name: consumerName, + Durable: consumerName, + FilterSubject: subject, AckPolicy: ackPolicy, MaxDeliver: 3, - MaxAckPending: cfg.PrefetchCount, + MaxAckPending: prefetchCount, } // 创建新消费者 - consumer, err := js.CreateOrUpdateConsumer(ctx, cfg.Stream, jsConfig) + consumer, err := js.CreateOrUpdateConsumer(ctx, streamName, jsConfig) if err != nil { - return fmt.Errorf("创建消费者失败: %w", err) + g.Log().Errorf(ctx, "创建消费者失败: %v", err) + return err } + + // 获取消费者信息验证 + if cInfo, err := consumer.Info(ctx); err == nil { + g.Log().Infof(ctx, "🔔 消费者创建成功: %s, AckPolicy=%v, MaxAckPending=%d", + cInfo.Name, cInfo.Config.AckPolicy, cInfo.Config.MaxAckPending) + } + // 创建消息处理函数 msgHandler := func(msg jetstream.Msg) { + // 记录消息接收时间 + now := time.Now() + meta, err := msg.Metadata() + if err == nil { + g.Log().Infof(ctx, "📨 收到消息: StreamSeq=%d, Published=%v, Received=%v, 距离发布=%.2f秒", + meta.Sequence.Stream, + meta.Timestamp.Format("2006-01-02 15:04:05"), + now.Format("2006-01-02 15:04:05"), + now.Sub(meta.Timestamp).Seconds()) + } + // 解析消息 var data map[string]any if err := json.Unmarshal(msg.Data(), &data); err != nil { g.Log().Errorf(ctx, "解析消息失败: %v", err) - msg.Nak() + if err := msg.Nak(); err != nil { + g.Log().Errorf(ctx, "Nak 失败: %v", err) + } return } // 处理业务逻辑 if err := handler(ctx, data); err != nil { g.Log().Errorf(ctx, "处理消息失败: %v", err) - msg.Nak() + if err := msg.Nak(); err != nil { + g.Log().Errorf(ctx, "Nak 失败: %v", err) + } return } g.Log().Infof(ctx, "处理消息成功") - if !cfg.AutoAck { - msg.Ack() + if !autoAck { + if err := msg.Ack(); err != nil { + g.Log().Errorf(ctx, "Ack 失败: %v", err) + } } } @@ -202,7 +281,25 @@ func (q *natsMessageClient) createSubscribe(ctx context.Context, cfg *NATSConfig return fmt.Errorf("开始消费失败: %w", err) } - g.Log().Infof(ctx, "✅ 开始消费消息: %s/%s", cfg.Stream, cfg.Consumer) + g.Log().Infof(ctx, "✅ NATS 订阅成功") return nil } + +func getStreamInfo(durable, delayMsg bool) (string, jetstream.StorageType) { + // Stream 不存在,创建新的 + streamName := "ordinary_msg_memory" + storage := jetstream.MemoryStorage + + // 延迟消息必须使用 FileStorage(NATS 官方要求) + if delayMsg { + streamName = "delay_msg_file" + storage = jetstream.FileStorage + } else { + if durable { + streamName = "ordinary_msg_file" + storage = jetstream.FileStorage + } + } + return streamName, storage +} diff --git a/message/nats_rpc.go b/message/nats_rpc.go index 8644734..e204743 100644 --- a/message/nats_rpc.go +++ b/message/nats_rpc.go @@ -38,11 +38,11 @@ var ( // 返回值可以是任意类型,会被自动序列化为 JSON type rpcHandler func(ctx context.Context, req []byte) (any, error) -// RegisterRPCService 注册 RPC 服务(单实例) +// registerRPCService 注册 RPC 服务(单实例) // serviceName: 服务名称,调用方通过此名称调用服务 // handler: 服务处理函数,接收请求并返回响应 func registerRPCService(serviceName string, handler rpcHandler) (err error) { - if !checkConnected() { + if !natsPing() { return fmt.Errorf("NATS 未连接") } @@ -74,18 +74,17 @@ func registerRPCService(serviceName string, handler rpcHandler) (err error) { } rpcSubs[serviceName] = sub - metrics.SubscribeCount.Add(1) g.Log().Infof(context.Background(), "✅ RPC 服务已注册: %s", serviceName) return nil } -// RegisterQueueRPCService 注册 RPC 服务(集群模式) +// registerQueueRPCService 注册 RPC 服务(集群模式) // 多个服务实例注册同一服务时,请求会自动负载均衡 // serviceName: 服务名称 // queueName: 队列组名,同一队列组的实例共享请求 // handler: 服务处理函数 func registerQueueRPCService(serviceName, queueName string, handler rpcHandler) (err error) { - if !checkConnected() { + if !natsPing() { return fmt.Errorf("NATS 未连接") } @@ -126,7 +125,6 @@ func registerQueueRPCService(serviceName, queueName string, handler rpcHandler) queueRPCSubs[queueName][serviceName] = sub queueRPCMu.Unlock() - metrics.SubscribeCount.Add(1) g.Log().Infof(context.Background(), "✅ 队列 RPC 服务已注册: %s (队列组: %s)", serviceName, queueName) return nil } @@ -138,16 +136,16 @@ func executeHandler(handler rpcHandler, msg *nats.Msg) { // 从消息头重建上下文 ctx := headersToContext(context.Background(), msg.Header) // 提取 TraceID,创建可取消的 context - ctx = createCancelContext(ctx, msg.Header.Get(TraceIDKey)) + ctx = createCancelContext(ctx, msg.Header.Get(traceIDKey)) // 检查 context 是否已取消(在调用 handler 之前) select { case <-ctx.Done(): // context 已取消,返回取消错误 - g.Log().Infof(ctx, "RPC 请求已取消,traceID: %s", msg.Header.Get(TraceIDKey)) + g.Log().Infof(ctx, "RPC 请求已取消,traceID: %s", msg.Header.Get(traceIDKey)) // 仍然需要发送响应以避免客户端超时 respData = []byte(`{"_err":"请求已取消"}`) // 清理取消映射表 - cleanupTraceCancel(msg.Header.Get(TraceIDKey)) + cleanupTraceCancel(msg.Header.Get(traceIDKey)) return default: } @@ -176,7 +174,7 @@ func executeHandler(handler rpcHandler, msg *nats.Msg) { g.Log().Errorf(ctx, "RPC 响应失败: %v", err) } // 请求结束,清理取消映射表 - cleanupTraceCancel(msg.Header.Get(TraceIDKey)) + cleanupTraceCancel(msg.Header.Get(traceIDKey)) } // createCancelContext 创建可取消的 context 并注册到取消映射表 @@ -211,7 +209,7 @@ func createCancelContext(ctx context.Context, traceID string) context.Context { // // sub, err := nats.SetupCancelListener(ctx) func setupCancelListener(ctx context.Context) (*nats.Subscription, error) { - if !checkConnected() { + if !natsPing() { return nil, fmt.Errorf("NATS 未连接") } @@ -253,7 +251,6 @@ func setupCancelListener(ctx context.Context) (*nats.Subscription, error) { return nil, fmt.Errorf("设置取消监听器失败: %w", err) } - metrics.SubscribeCount.Add(1) g.Log().Infof(ctx, "✅ 取消监听器已设置: %s", cancelSubject) return sub, nil } @@ -264,7 +261,7 @@ func setupCancelListener(ctx context.Context) (*nats.Subscription, error) { // // err := nats.publishCancel(ctx, traceID) func publishCancel(ctx context.Context, traceID string) error { - if !checkConnected() { + if !natsPing() { return fmt.Errorf("NATS 未连接") } @@ -306,12 +303,10 @@ func cleanupTraceCancel(traceID string) { // req: 请求数据 // 返回: 响应数据(任意类型)和错误 func CallRPC(ctx context.Context, serviceName string, req any, resp any) (err error) { - if !checkConnected() { + if !natsPing() { return fmt.Errorf("NATS 未连接") } - metrics.RequestCount.Add(1) - // 验证 resp 必须是指针类型 respValue := reflect.ValueOf(resp) if respValue.Kind() != reflect.Ptr { @@ -346,7 +341,6 @@ func CallRPC(ctx context.Context, serviceName string, req any, resp any) (err er // 执行本地调用 var response interface{} if response, err = localHandler(cancelCtx, reqBody); err != nil { - metrics.RequestError.Add(1) return fmt.Errorf("本地调用 RPC 服务失败 [%s]: %w", serviceName, err) } @@ -357,7 +351,6 @@ func CallRPC(ctx context.Context, serviceName string, req any, resp any) (err er var respMap map[string]any if json.Unmarshal(response.([]byte), &respMap) == nil { if errMsg, ok := respMap["_err"]; ok { - metrics.RequestError.Add(1) return fmt.Errorf("%v", errMsg) } } @@ -392,17 +385,17 @@ func CallRPC(ctx context.Context, serviceName string, req any, resp any) (err er }) } - if msg.Header.Get(TraceIDKey) != "" { + if msg.Header.Get(traceIDKey) != "" { go func() { defer closeDone() select { case <-ctx.Done(): // context 被取消时,发送取消信号给服务端 if errors.Is(ctx.Err(), context.Canceled) { - if err := publishCancel(context.Background(), msg.Header.Get(TraceIDKey)); err != nil { + if err := publishCancel(context.Background(), msg.Header.Get(traceIDKey)); err != nil { g.Log().Errorf(ctx, "发送 RPC 取消信号失败: %v", err) } else { - g.Log().Infof(ctx, "RPC 调用已取消,traceID: %s", msg.Header.Get(TraceIDKey)) + g.Log().Infof(ctx, "RPC 调用已取消,traceID: %s", msg.Header.Get(traceIDKey)) } } case <-done: @@ -419,12 +412,10 @@ func CallRPC(ctx context.Context, serviceName string, req any, resp any) (err er closeDone() if err != nil { - metrics.RequestError.Add(1) return fmt.Errorf("调用 RPC 服务失败 [%s]: %w", serviceName, err) } if responseMsg == nil { - metrics.RequestError.Add(1) return fmt.Errorf("RPC 响应为空 [%s]", serviceName) } @@ -434,7 +425,6 @@ func CallRPC(ctx context.Context, serviceName string, req any, resp any) (err er var respMap map[string]any if json.Unmarshal(responseMsg.Data, &respMap) == nil { if errMsg, ok := respMap["_err"]; ok { - metrics.RequestError.Add(1) return fmt.Errorf("%v", errMsg) } } @@ -449,7 +439,7 @@ func CallRPC(ctx context.Context, serviceName string, req any, resp any) (err er } // RegisterServiceOption 注册选项类型 -type RegisterServiceOption func(*registerServiceConfig) +type registerServiceOption func(*registerServiceConfig) type registerServiceConfig struct { queueName string // 队列组名(用于集群模式) @@ -457,14 +447,14 @@ type registerServiceConfig struct { } // WithQueueGroup 设置队列组名(集群模式) -func WithQueueGroup(queueName string) RegisterServiceOption { +func WithQueueGroup(queueName string) registerServiceOption { return func(cfg *registerServiceConfig) { cfg.queueName = queueName } } // WithExcludeMethods 排除不需要注册的方法 -func WithExcludeMethods(methods ...string) RegisterServiceOption { +func WithExcludeMethods(methods ...string) registerServiceOption { return func(cfg *registerServiceConfig) { cfg.excludeMethods = append(cfg.excludeMethods, methods...) } @@ -483,9 +473,9 @@ func WithExcludeMethods(methods ...string) RegisterServiceOption { // AutoRegisterServices(map[string]interface{}{ // "order": orderService, // }, WithQueueGroup("order-group")) -func AutoRegisterServices(ctx context.Context, serviceInstances map[string]interface{}, options ...RegisterServiceOption) error { +func AutoRegisterServices(ctx context.Context, serviceInstances map[string]interface{}, options ...registerServiceOption) error { // 先注册 RPC 服务(如果 NATS 不可用则记录警告但不阻塞启动) - if !checkConnected() { + if !natsPing() { return fmt.Errorf("NATS 未连接,RPC 服务未注册") } @@ -521,8 +511,8 @@ func AutoRegisterServices(ctx context.Context, serviceInstances map[string]inter } // registerService 注册单个服务的所有公开方法(内部函数) -func registerService(service interface{}, serviceNamePrefix string, options ...RegisterServiceOption) (err error) { - if !checkConnected() { +func registerService(service interface{}, serviceNamePrefix string, options ...registerServiceOption) (err error) { + if !natsPing() { return fmt.Errorf("NATS 未连接") } @@ -676,10 +666,10 @@ func registerService(service interface{}, serviceNamePrefix string, options ...R // ============ 上下文元数据工具函数 ============ // 以下函数用于在 context 和 NATS 消息头之间互转元数据 -// 定义常见的上下文元数据 key +// 定义常见的上下文元数据 key(私有) const ( - TraceIDKey = "trace_id" - TokenKey = "token" + traceIDKey = "trace_id" + tokenKey = "token" ) func getTraceID(ctx context.Context) (traceID string, err error) { @@ -687,7 +677,7 @@ func getTraceID(ctx context.Context) (traceID string, err error) { span := trace.SpanFromContext(ctx) if span != nil && span.SpanContext().HasTraceID() { traceID = span.SpanContext().TraceID().String() - } else if tid := ctx.Value(TraceIDKey); tid != nil { + } else if tid := ctx.Value(traceIDKey); tid != nil { traceID = fmt.Sprintf("%v", tid) } if traceID == "" { @@ -705,12 +695,12 @@ func contextToHeaders(ctx context.Context) (nats.Header, error) { if traceID, err := getTraceID(ctx); err != nil { return headers, err } else { - headers.Set(TraceIDKey, traceID) + headers.Set(traceIDKey, traceID) } // 提取 token(优先级:context value > HTTP Authorization header) token := "" - if t := ctx.Value(TokenKey); t != nil { + if t := ctx.Value(tokenKey); t != nil { token = fmt.Sprintf("%v", t) } else if r := g.RequestFromCtx(ctx); r != nil { // 从 HTTP 请求的 Authorization header 中提取 token @@ -725,7 +715,7 @@ func contextToHeaders(ctx context.Context) (nats.Header, error) { } } if token != "" { - headers.Set(TokenKey, token) + headers.Set(tokenKey, token) } return headers, nil @@ -739,13 +729,13 @@ func headersToContext(ctx context.Context, headers nats.Header) context.Context } // 恢复 trace_id - if traceID := headers.Get(TraceIDKey); traceID != "" { - ctx = context.WithValue(ctx, TraceIDKey, traceID) + if traceID := headers.Get(traceIDKey); traceID != "" { + ctx = context.WithValue(ctx, traceIDKey, traceID) } // 恢复 token - if token := headers.Get(TokenKey); token != "" { - ctx = context.WithValue(ctx, TokenKey, token) + if token := headers.Get(tokenKey); token != "" { + ctx = context.WithValue(ctx, tokenKey, token) } return ctx diff --git a/message/rabbit.go b/message/rabbit.go deleted file mode 100644 index 1ef01fb..0000000 --- a/message/rabbit.go +++ /dev/null @@ -1,351 +0,0 @@ -package message - -import ( - "context" - "sync" - "time" - - "github.com/gogf/gf/v2/encoding/gjson" - "github.com/gogf/gf/v2/errors/gerror" - "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/util/gconv" - amqp "github.com/rabbitmq/amqp091-go" -) - -var ( - rabbitConn *amqp.Connection - rabbitChannel *amqp.Channel - rabbitOnce sync.Once - rabbitMu sync.RWMutex - rabbitCloseWatcher chan struct{} - rabbitWatcherStarted bool -) - -// Config RabbitMQ 配置 -type RabbitMQConfig1 struct { - Host string - Port int - Username string - Password string - VHost string -} - -// rabbitMQConfig 默认配置 -func getRabbitMQConfig() *RabbitMQConfig1 { - return &RabbitMQConfig1{ - Host: g.Cfg().MustGet(context.Background(), "rabbitmq.host").String(), - Port: g.Cfg().MustGet(context.Background(), "rabbitmq.port").Int(), - Username: g.Cfg().MustGet(context.Background(), "rabbitmq.username").String(), - Password: g.Cfg().MustGet(context.Background(), "rabbitmq.password").String(), - VHost: g.Cfg().MustGet(context.Background(), "rabbitmq.vhost", "/").String(), - } -} - -// initRabbitMQ 初始化 RabbitMQ 连接 -func initRabbitMQ(ctx context.Context) error { - var err error - rabbitOnce.Do(func() { - cfg := getRabbitMQConfig() - url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost - - rabbitConn, err = amqp.Dial(url) - if err != nil { - g.Log().Errorf(ctx, "RabbitMQ 连接失败: %v", err) - return - } - - rabbitChannel, err = rabbitConn.Channel() - if err != nil { - g.Log().Errorf(ctx, "创建 RabbitMQ Channel 失败: %v", err) - return - } - - rabbitCloseWatcher = make(chan struct{}) - - if !rabbitWatcherStarted { - go handleRabbitMQConnectionClose(ctx) - rabbitWatcherStarted = true - } - - g.Log().Info(ctx, "RabbitMQ 连接成功") - }) - - return err -} - -// getRabbitMQChannel 获取 RabbitMQ Channel -func getRabbitMQChannel() (*amqp.Channel, error) { - rabbitMu.RLock() - defer rabbitMu.RUnlock() - - if rabbitChannel == nil || rabbitChannel.IsClosed() { - return nil, gerror.New("RabbitMQ Channel 未初始化或已关闭") - } - - return rabbitChannel, nil -} - -// getRabbitMQConnection 获取 RabbitMQ 连接 -func getRabbitMQConnection() (*amqp.Connection, error) { - rabbitMu.RLock() - defer rabbitMu.RUnlock() - - if rabbitConn == nil || rabbitConn.IsClosed() { - return nil, gerror.New("RabbitMQ 连接未初始化或已关闭") - } - - return rabbitConn, nil -} - -// handleRabbitMQConnectionClose 监听连接关闭并重连 -func handleRabbitMQConnectionClose(ctx context.Context) { - for { - select { - case <-rabbitCloseWatcher: - g.Log().Info(ctx, "停止监听 RabbitMQ 连接状态") - return - default: - } - - rabbitMu.RLock() - currentConn := rabbitConn - rabbitMu.RUnlock() - - if currentConn == nil { - return - } - - closeErr := make(chan *amqp.Error, 1) - currentConn.NotifyClose(closeErr) - - select { - case err := <-closeErr: - if err != nil { - g.Log().Errorf(ctx, "RabbitMQ 连接关闭: %v,尝试重连...", err) - reconnectRabbitMQ(ctx) - } - case <-rabbitCloseWatcher: - g.Log().Info(ctx, "停止监听 RabbitMQ 连接状态") - return - } - } -} - -// reconnectRabbitMQ 重新连接 -func reconnectRabbitMQ(ctx context.Context) { - rabbitMu.Lock() - defer rabbitMu.Unlock() - - for i := 0; i < 10; i++ { - time.Sleep(time.Duration(i+1) * time.Second) - - cfg := getRabbitMQConfig() - url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost - - var err error - rabbitConn, err = amqp.Dial(url) - if err != nil { - g.Log().Errorf(ctx, "重连失败 (尝试 %d/10): %v", i+1, err) - continue - } - - rabbitChannel, err = rabbitConn.Channel() - if err != nil { - g.Log().Errorf(ctx, "创建 Channel 失败 (尝试 %d/10): %v", i+1, err) - continue - } - - g.Log().Info(ctx, "RabbitMQ 重连成功") - return - } - - g.Log().Fatal(ctx, "RabbitMQ 重连失败,已达到最大重试次数") -} - -// startRabbitMQConsumer 启动 RabbitMQ 消费者 -func startRabbitMQConsumer(ctx context.Context, msg QueueMessage) error { - // 初始化连接 - if err := initRabbitMQ(ctx); err != nil { - return gerror.Wrap(err, "初始化 RabbitMQ 连接失败") - } - - // 创建独立 Channel(避免并发冲突) - conn, err := getRabbitMQConnection() - if err != nil { - return gerror.Wrap(err, "获取RabbitMQ连接失败") - } - - ch, err := conn.Channel() - if err != nil { - return gerror.Wrap(err, "创建独立Channel失败") - } - - // 声明队列 - _, err = ch.QueueDeclare( - msg.Queue, // name - true, // durable - false, // autoDelete - false, // exclusive - false, // noWait - nil, // arguments - ) - if err != nil { - return gerror.Newf("声明队列失败: %v", err) - } - - // 设置 QoS(并发控制) - prefetchCount := msg.PrefetchCount - if prefetchCount == 0 { - prefetchCount = 1 - } - err = ch.Qos( - prefetchCount, // prefetchCount - 0, // prefetchSize - false, // global - ) - if err != nil { - return gerror.Newf("设置 QoS 失败: %v", err) - } - - // 开始消费 - msgs, err := ch.Consume( - msg.Queue, // queue - msg.ConsumerTag, // consumer tag - msg.AutoAck, // auto-ack - false, // exclusive - false, // no-local - false, // no-wait - nil, // args - ) - if err != nil { - return gerror.Newf("开始消费失败: %v", err) - } - - workerCount := msg.WorkerCount - if workerCount == 0 { - workerCount = 1 - } - - g.Log().Infof(ctx, "RabbitMQ 消费者已启动: queue=%s, prefetch=%d, workers=%d", - msg.Queue, prefetchCount, workerCount) - - // 启动多个 worker - for i := 0; i < workerCount; i++ { - go rabbitMQWorker(ctx, i, msgs, msg) - } - - return nil -} - -// rabbitMQWorker RabbitMQ 工作协程 -func rabbitMQWorker(ctx context.Context, workerID int, msgs <-chan amqp.Delivery, msg QueueMessage) { - g.Log().Debugf(ctx, "RabbitMQ Worker %d 已启动", workerID) - - for { - select { - case <-ctx.Done(): - g.Log().Infof(ctx, "RabbitMQ Worker %d 收到停止信号,正在退出", workerID) - return - case delivery, ok := <-msgs: - if !ok { - g.Log().Infof(ctx, "RabbitMQ Worker %d 消息通道已关闭,退出", workerID) - return - } - - // 反序列化消息 - var message map[string]interface{} - if err := gjson.DecodeTo(delivery.Body, &message); err != nil { - g.Log().Errorf(ctx, "RabbitMQ Worker %d 反序列化消息失败: %v", workerID, err) - if !msg.AutoAck { - delivery.Nack(false, false) - } - continue - } - - // 处理消息 - err := msg.HandleFunc(ctx, message) - if err != nil { - g.Log().Errorf(ctx, "RabbitMQ Worker %d 处理消息失败: %v", workerID, err) - if !msg.AutoAck { - delivery.Nack(false, false) - } - } else { - if !msg.AutoAck { - delivery.Ack(false) - } - g.Log().Debugf(ctx, "RabbitMQ Worker %d 处理消息成功", workerID) - } - } - } -} - -// publishToRabbitMQ 发布消息到 RabbitMQ -func publishToRabbitMQ(ctx context.Context, exchange, routingKey string, message interface{}) (messageID string, err error) { - ch, err := getRabbitMQChannel() - if err != nil { - return - } - - body, err := gjson.Encode(message) - if err != nil { - return "", gerror.Newf("消息序列化失败: %v", err) - } - - err = ch.PublishWithContext( - ctx, - exchange, // exchange - routingKey, // routing key - false, // mandatory - false, // immediate - amqp.Publishing{ - DeliveryMode: amqp.Persistent, - ContentType: "application/json", - Body: body, - }, - ) - - if err != nil { - g.Log().Errorf(ctx, "发布消息失败: exchange=%s, routingKey=%s, err=%v", exchange, routingKey, err) - return - } - - g.Log().Debugf(ctx, "消息发布成功: exchange=%s, routingKey=%s", exchange, routingKey) - return messageID, nil -} - -// publishDelayedToRabbitMQ 发布延时消息到 RabbitMQ -func publishDelayedToRabbitMQ(ctx context.Context, exchange, routingKey string, message interface{}, delaySeconds int) (messageID string, err error) { - ch, err := getRabbitMQChannel() - if err != nil { - return - } - - body, err := gjson.Encode(message) - if err != nil { - return "", gerror.Newf("消息序列化失败: %v", err) - } - - err = ch.PublishWithContext( - ctx, - exchange, // exchange(必须是 x-delayed-message 类型) - routingKey, // routing key - false, // mandatory - false, // immediate - amqp.Publishing{ - DeliveryMode: amqp.Persistent, - ContentType: "application/json", - Body: body, - Headers: amqp.Table{ - "x-delay": delaySeconds * 1000, // 延时(毫秒) - }, - }, - ) - - if err != nil { - g.Log().Errorf(ctx, "发布延时消息失败: exchange=%s, routingKey=%s, delay=%ds, err=%v", exchange, routingKey, delaySeconds, err) - return - } - - g.Log().Debugf(ctx, "延时消息发布成功: exchange=%s, routingKey=%s, delay=%ds", exchange, routingKey, delaySeconds) - return messageID, nil -} diff --git a/message/rabbitmq_client.go b/message/rabbitmq_client.go deleted file mode 100644 index fc225f7..0000000 --- a/message/rabbitmq_client.go +++ /dev/null @@ -1,210 +0,0 @@ -package message - -import ( - "context" - "sync" - "time" - - "github.com/gogf/gf/v2/errors/gerror" - "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/util/gconv" - amqp "github.com/rabbitmq/amqp091-go" -) - -var ( - conn *amqp.Connection - channel *amqp.Channel - rabbitmqOnce sync.Once - rabbitmqMu sync.RWMutex - closeWatcher chan struct{} // 用于停止监听 goroutine - watcherStarted bool // 防止重复启动监听 -) - -// Config RabbitMQ 配置 -type Config struct { - Host string - Port int - Username string - Password string - VHost string -} - -// Init 初始化 RabbitMQ 连接 -func Init(ctx context.Context, cfg *Config) error { - var err error - rabbitmqOnce.Do(func() { - // 构建连接字符串 - url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost - - // 创建连接 - conn, err = amqp.Dial(url) - if err != nil { - g.Log().Errorf(ctx, "RabbitMQ 连接失败: %v", err) - return - } - - // 创建 Channel - channel, err = conn.Channel() - if err != nil { - g.Log().Errorf(ctx, "创建 RabbitMQ Channel 失败: %v", err) - return - } - - // 初始化关闭监听器 - closeWatcher = make(chan struct{}) - - // 监听连接关闭(只启动一次) - if !watcherStarted { - go handleConnectionClose(ctx) - watcherStarted = true - } - - g.Log().Info(ctx, "RabbitMQ 连接成功") - }) - - return err -} - -// InitFromConfig 从配置文件初始化 -func InitFromConfig(ctx context.Context) error { - cfg := &Config{ - Host: g.Cfg().MustGet(ctx, "rabbitmq.host").String(), - Port: g.Cfg().MustGet(ctx, "rabbitmq.port").Int(), - Username: g.Cfg().MustGet(ctx, "rabbitmq.username").String(), - Password: g.Cfg().MustGet(ctx, "rabbitmq.password").String(), - VHost: g.Cfg().MustGet(ctx, "rabbitmq.vhost", "/").String(), - } - - return Init(ctx, cfg) -} - -// GetChannel 获取 Channel -func GetChannel() (*amqp.Channel, error) { - rabbitmqMu.RLock() - defer rabbitmqMu.RUnlock() - - if channel == nil || channel.IsClosed() { - return nil, gerror.New("RabbitMQ Channel 未初始化或已关闭") - } - - return channel, nil -} - -// GetConnection 获取连接 -func GetConnection() (*amqp.Connection, error) { - rabbitmqMu.RLock() - defer rabbitmqMu.RUnlock() - - if conn == nil || conn.IsClosed() { - return nil, gerror.New("RabbitMQ 连接未初始化或已关闭") - } - - return conn, nil -} - -// handleConnectionClose 监听连接关闭并重连 -func handleConnectionClose(ctx context.Context) { - for { - // 检查是否需要停止监听 - select { - case <-closeWatcher: - g.Log().Info(ctx, "停止监听 RabbitMQ 连接状态") - return - default: - } - - rabbitmqMu.RLock() - currentConn := conn - rabbitmqMu.RUnlock() - - if currentConn == nil { - return - } - - // 创建关闭通知 channel - closeErr := make(chan *amqp.Error, 1) - currentConn.NotifyClose(closeErr) - - // 等待连接关闭或停止信号 - select { - case err := <-closeErr: - if err != nil { - g.Log().Errorf(ctx, "RabbitMQ 连接关闭: %v,尝试重连...", err) - reconnect(ctx) - } - case <-closeWatcher: - g.Log().Info(ctx, "停止监听 RabbitMQ 连接状态") - return - } - } -} - -// reconnect 重新连接 -func reconnect(ctx context.Context) { - rabbitmqMu.Lock() - defer rabbitmqMu.Unlock() - - for i := 0; i < 10; i++ { - time.Sleep(time.Duration(i+1) * time.Second) - - cfg := &Config{ - Host: g.Cfg().MustGet(ctx, "rabbitmq.host").String(), - Port: g.Cfg().MustGet(ctx, "rabbitmq.port").Int(), - Username: g.Cfg().MustGet(ctx, "rabbitmq.username").String(), - Password: g.Cfg().MustGet(ctx, "rabbitmq.password").String(), - VHost: g.Cfg().MustGet(ctx, "rabbitmq.vhost", "/").String(), - } - - url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost - - var err error - conn, err = amqp.Dial(url) - if err != nil { - g.Log().Errorf(ctx, "重连失败 (尝试 %d/10): %v", i+1, err) - continue - } - - channel, err = conn.Channel() - if err != nil { - g.Log().Errorf(ctx, "创建 Channel 失败 (尝试 %d/10): %v", i+1, err) - continue - } - - g.Log().Info(ctx, "RabbitMQ 重连成功") - // 不再重复启动监听 goroutine - return - } - - g.Log().Fatal(ctx, "RabbitMQ 重连失败,已达到最大重试次数") -} - -// Close 关闭连接 -func Close(ctx context.Context) (err error) { - rabbitmqMu.Lock() - defer rabbitmqMu.Unlock() - - // 停止监听 goroutine - if closeWatcher != nil { - close(closeWatcher) - closeWatcher = nil - } - - if channel != nil { - if err = channel.Close(); err != nil { - g.Log().Errorf(ctx, "关闭 RabbitMQ Channel 失败: %v", err) - } - channel = nil - } - - if conn != nil { - if err = conn.Close(); err != nil { - g.Log().Errorf(ctx, "关闭 RabbitMQ 连接失败: %v", err) - return - } - conn = nil - } - - watcherStarted = false - g.Log().Info(ctx, "RabbitMQ 连接已关闭") - return -} diff --git a/message/rabbitmq_msg.go b/message/rabbitmq_msg.go index 8d49663..0a65e20 100644 --- a/message/rabbitmq_msg.go +++ b/message/rabbitmq_msg.go @@ -10,61 +10,129 @@ import ( amqp "github.com/rabbitmq/amqp091-go" ) -// rabbitMQMessageClient RabbitMQ 实现 -type rabbitMQMessageClient struct { - clientType messageClientType +type RabbitMQPublishMsgConfig struct { + QueueName string + Durable bool + DelayTime int + Data any } -// StreamGroup 创建消费组(支持单个或批量) -func (q *rabbitMQMessageClient) streamGroup(ctx context.Context, configs ...interface{}) error { - if len(configs) == 0 { - return fmt.Errorf("配置不能为空") - } - for _, config := range configs { - cfg, ok := config.(*RabbitMQConfig) - if !ok { - return fmt.Errorf("无效的 RabbitMQ 配置类型") - } - if err := q.setupQueue(ctx, channel, cfg, cfg.DelayMessage); err != nil { - return err - } - } - return nil +type RabbitMQSubscribeMsgConfig struct { + QueueName string + Durable bool + DelayTime int + ConsumerName string + AutoAck bool + PrefetchCount int + HandleFunc func(ctx context.Context, message map[string]interface{}) error } -// Publish 发布消息(支持单个或批量) -func (q *rabbitMQMessageClient) publish(ctx context.Context, config interface{}, data interface{}) error { - cfg, ok := config.(*RabbitMQConfig) +func (*RabbitMQPublishMsgConfig) GetPublishMsgType() { + +} + +func (*RabbitMQSubscribeMsgConfig) GetSubscribeMsgType() { + +} + +func init() { + // 注册 RabbitMQ 插件,必须使用 RegisterPlugin 确保连接检测 + //registerPlugin(MessageRabbitMQ, func() messageUtil { + // return &rabbitMQ{} + //}) +} + +type rabbitMQ struct{} + +// Ping 检测 RabbitMQ 连接状态 +func (c *rabbitMQ) ping(ctx context.Context) bool { + return rabbitmqPing() +} + +// Reconnect 重连 RabbitMQ +func (c *rabbitMQ) reconnect(ctx context.Context) error { + return rabbitmqReconnect(ctx) +} + +// Close 关闭 RabbitMQ 连接 +func (c *rabbitMQ) close(ctx context.Context) error { + return rabbitmqClose(ctx) +} + +// Publish 发布消息 +func (c *rabbitMQ) Publish(ctx context.Context, msgConfig messagePublishConfig) error { + cfg, ok := msgConfig.(*RabbitMQPublishMsgConfig) if !ok { return fmt.Errorf("无效的 RabbitMQ 配置类型") } - if err := q.publishMessage(ctx, cfg, "work", data, 0); err != nil { - g.Log().Errorf(ctx, "❌ RabbitMQ 发布消息失败: err=%v", err) - return err + if g.IsEmpty(cfg.QueueName) { + return fmt.Errorf("队列名称不能为空") } - return nil + if cfg.Data == nil { + return fmt.Errorf("数据不能为空") + } + return c.publishMessageInternal(ctx, cfg.QueueName, cfg.Durable, cfg.DelayTime, cfg.Data) } -// PublishDelayed 发布延迟消息 -func (q *rabbitMQMessageClient) publishDelayed(ctx context.Context, config interface{}, data interface{}, delaySeconds int) error { - cfg, ok := config.(*RabbitMQConfig) - if !ok { - return fmt.Errorf("无效的 RabbitMQ 配置类型") - } - if err := q.publishMessage(ctx, cfg, "delayed", data, delaySeconds); err != nil { - g.Log().Errorf(ctx, "❌ RabbitMQ 发布延迟消息失败: err=%v", err) - return err - } - return nil -} +// publishMessage 发布消息内部实现 +func (c *rabbitMQ) publishMessageInternal(ctx context.Context, queueName string, durable bool, delayTime int, data interface{}) error { + delayMsg := delayTime > 0 -func (q *rabbitMQMessageClient) publishMessage(ctx context.Context, cfg *RabbitMQConfig, mode string, data interface{}, delaySeconds int) error { + // 1. 决定 Exchange 类型 + exchangeType := "fanout" + exchangeName := queueName + routingKey := queueName + args := amqp.Table{} + if delayMsg { + exchangeType = "x-delayed-message" + exchangeName = queueName + ".delayed" + args["x-delayed-type"] = "fanout" // 底层用 topic + } + + // 2. 声明 Exchange(只声明一次) + if err := channel.ExchangeDeclare( + queueName, // exchange 交换机名称 + exchangeType, + durable, + false, // autoDelete + false, // internal + false, // noWait + args, + ); err != nil { + return fmt.Errorf("声明 Exchange 失败: %w", err) + } + + // 3. 声明队列 + if _, err := channel.QueueDeclare( + queueName, + durable, + false, // autoDelete + false, // exclusive + false, // noWait + nil, // args + ); err != nil { + return fmt.Errorf("声明队列失败: %w", err) + } + + // 4. 绑定队列 + if err := channel.QueueBind( + queueName, + routingKey, // routingKey 路由键 + exchangeName, // exchange 交换机名称 + false, // noWait + nil, // args + ); err != nil { + return fmt.Errorf("绑定队列失败: %w", err) + } + + // 5. 序列化数据 body, err := json.Marshal(data) if err != nil { return fmt.Errorf("序列化数据失败: %w", err) } + // 6. 发布消息 deliveryMode := amqp.Transient - if cfg.Durable { + if durable { deliveryMode = amqp.Persistent } publishing := amqp.Publishing{ @@ -73,15 +141,15 @@ func (q *rabbitMQMessageClient) publishMessage(ctx context.Context, cfg *RabbitM DeliveryMode: deliveryMode, Timestamp: time.Now(), } - if delaySeconds > 0 { + if delayMsg { + duration := time.Duration(delayTime) * time.Minute publishing.Headers = amqp.Table{ - "x-delay": delaySeconds * 1000, // 延时时间(毫秒) + "x-delay": duration, // 延迟时间(毫秒) } } - exchange, routingKey := q.parseExchangeAndRoutingKey(ctx, mode, cfg) err = channel.PublishWithContext( ctx, - exchange, + exchangeName, routingKey, false, false, publishing, @@ -89,106 +157,44 @@ func (q *rabbitMQMessageClient) publishMessage(ctx context.Context, cfg *RabbitM return err } -func (q *rabbitMQMessageClient) parseExchangeAndRoutingKey(_ context.Context, mode string, cfg *RabbitMQConfig) (exchange, routingKey string) { - switch mode { - case "work", "": - exchange = "" // 默认交换机 - routingKey = cfg.Name // 队列名 - case "event", "topic": - exchange = cfg.Exchange - routingKey = cfg.Topic - case "broadcast": - exchange = cfg.Exchange - routingKey = "" // fanout忽略路由键 - case "delayed": - exchange = cfg.Exchange + ".delayed" - routingKey = cfg.Topic - default: - exchange = "" - routingKey = cfg.Name +// Subscribe 订阅消息 +func (c *rabbitMQ) Subscribe(ctx context.Context, msgConfig messageSubscribeConfig) error { + cfg, ok := msgConfig.(*RabbitMQSubscribeMsgConfig) + if !ok { + return fmt.Errorf("无效的 RabbitMQ 配置类型") } - return exchange, routingKey + if g.IsEmpty(cfg.QueueName) { + return fmt.Errorf("队列名称不能为空") + } + if g.IsEmpty(cfg.ConsumerName) { + return fmt.Errorf("消费者名称不能为空") + } + if g.IsEmpty(cfg.PrefetchCount) { + cfg.PrefetchCount = 1 + } + if g.IsEmpty(cfg.HandleFunc) { + return fmt.Errorf("必须提供处理函数") + } + return c.createSubscribeInternal(ctx, cfg.QueueName, cfg.ConsumerName, cfg.PrefetchCount, cfg.AutoAck, cfg.HandleFunc) } -// setupQueue 统一的队列设置方法(声明 Exchange、队列、绑定、延迟 Exchange) -func (q *rabbitMQMessageClient) setupQueue(ctx context.Context, ch *amqp.Channel, cfg *RabbitMQConfig, delayMessage bool) error { - exchange, routingKey := q.parseExchangeAndRoutingKey(ctx, cfg.Mode, cfg) +// createSubscribe 内部订阅消息 +func (c *rabbitMQ) createSubscribeInternal(ctx context.Context, queueName, consumerName string, prefetchCount int, autoAck bool, handler func(ctx context.Context, message map[string]interface{}) error) error { + g.Log().Infof(ctx, "🔔 RabbitMQ 开始订阅: queueName=%s, consumerName=%s", queueName, consumerName) - // 声明 Exchange - if err := ch.ExchangeDeclare(exchange, "topic", cfg.Durable, false, false, false, nil); err != nil { - return fmt.Errorf("声明 Exchange 失败: %w", err) - } - - // 声明队列 - if _, err := ch.QueueDeclare(cfg.Queue, cfg.Durable, false, false, false, nil); err != nil { - return fmt.Errorf("声明队列失败: %w", err) - } - - // 绑定队列 - if err := ch.QueueBind(cfg.Queue, routingKey, exchange, false, nil); err != nil { - return fmt.Errorf("绑定队列失败: %w", err) - } - - // 声明延迟 Exchange(如果需要) - if delayMessage { - if err := ch.ExchangeDeclare(exchange, "x-delayed-message", true, false, false, false, amqp.Table{"x-delayed-type": "direct"}); err != nil { - return fmt.Errorf("声明延迟 Exchange 失败: %w", err) - } - if err := ch.QueueBind(cfg.Name, routingKey, exchange, false, nil); err != nil { - return fmt.Errorf("绑定延迟队列失败: %w", err) - } - } - - return nil -} - -// Subscribe 订阅消息(支持单个或批量) -func (q *rabbitMQMessageClient) subscribe(ctx context.Context, configs ...interface{}) error { - if len(configs) == 0 { - return fmt.Errorf("配置不能为空") - } - - for _, config := range configs { - cfg, ok := config.(*RabbitMQConfig) - if !ok { - return fmt.Errorf("无效的 RabbitMQ 配置类型") - } - handler := cfg.HandleFunc - if handler == nil { - return fmt.Errorf("必须提供处理函数") - } - if err := q.createSubscribe(ctx, cfg, handler); err != nil { - return err - } - } - return nil -} - -// subscribe 内部单个订阅消息 -func (q *rabbitMQMessageClient) createSubscribe(ctx context.Context, cfg *RabbitMQConfig, handler func(ctx context.Context, message map[string]interface{}) error) error { - g.Log().Infof(ctx, "🔔 RabbitMQ 开始订阅: exchange=%s, queue=%s", cfg.Exchange, cfg.Queue) - - // 设置 Qos (预取数量),控制每次推送的消息数量 - // prefetchCount: 未 ACK 消息的最大数量 - // prefetchSize: 未 ACK 消息的总大小(0 表示不限制) - // global: false 表示仅应用于当前消费者 - prefetchCount := cfg.PrefetchCount - if prefetchCount <= 0 { - prefetchCount = 10 // 默认值为 10 - } if err := channel.Qos(prefetchCount, 0, false); err != nil { return fmt.Errorf("设置 Qos 失败: %w", err) } g.Log().Infof(ctx, "📊 设置 Prefetch Count: %d", prefetchCount) msg, err := channel.Consume( - cfg.Queue, // queue - cfg.Queue, // consumer - cfg.AutoAck, // auto-ack (根据配置决定) - false, // exclusive - false, // no-local - false, // no-wait - nil, // args + queueName, // queue + consumerName, // consumer + autoAck, // auto-ack (根据配置决定) + false, // exclusive + false, // no-local + false, // no-wait + nil, // args ) if err != nil { return fmt.Errorf("注册消费者失败: %w", err) @@ -207,7 +213,7 @@ func (q *rabbitMQMessageClient) createSubscribe(ctx context.Context, cfg *Rabbit for { select { case <-ctx.Done(): - g.Log().Infof(ctx, "🔕 RabbitMQ 消费者停止: queue=%s", cfg.Queue) + g.Log().Infof(ctx, "🔕 RabbitMQ 消费者停止: queueName=%s, consumerName=%s", queueName, consumerName) return case msg, ok := <-msg: if !ok { @@ -226,11 +232,11 @@ func (q *rabbitMQMessageClient) createSubscribe(ctx context.Context, cfg *Rabbit } }() - if err := q.handleMessageWithRetry(ctx, m, handler, cfg.MaxRetry); err != nil { + if err := c.handleMessageWithRetryInternal(ctx, m, handler, autoAck); err != nil { g.Log().Errorf(ctx, "❌ 消息处理失败(重试次数耗尽): %v", err) // 仅在手动 ACK 模式下拒绝消息 - if !cfg.AutoAck { + if !autoAck { // 拒绝消息不再重新入队(避免死循环) m.Nack(false, false) } @@ -238,7 +244,7 @@ func (q *rabbitMQMessageClient) createSubscribe(ctx context.Context, cfg *Rabbit } // 仅在手动 ACK 模式下确认消息 - if cfg.AutoAck { + if autoAck { if err := m.Ack(false); err != nil { g.Log().Errorf(ctx, "❌ ACK 消息失败: %v", err) } @@ -252,7 +258,7 @@ func (q *rabbitMQMessageClient) createSubscribe(ctx context.Context, cfg *Rabbit } // handleMessageWithRetry 处理消息(支持重试) -func (q *rabbitMQMessageClient) handleMessageWithRetry(ctx context.Context, msg amqp.Delivery, handler func(ctx context.Context, message map[string]interface{}) error, maxRetry int) error { +func (c *rabbitMQ) handleMessageWithRetryInternal(ctx context.Context, msg amqp.Delivery, handler func(ctx context.Context, message map[string]interface{}) error, autoAck bool) error { var data map[string]interface{} if err := json.Unmarshal(msg.Body, &data); err != nil { @@ -263,6 +269,7 @@ func (q *rabbitMQMessageClient) handleMessageWithRetry(ctx context.Context, msg } // 重试逻辑 + const maxRetry = 3 for attempt := 0; attempt <= maxRetry; attempt++ { if attempt > 0 { g.Log().Infof(ctx, "🔄 消息处理重试 (第%d次)", attempt) diff --git a/message/redis.go b/message/redis.go deleted file mode 100644 index 67e88f1..0000000 --- a/message/redis.go +++ /dev/null @@ -1,275 +0,0 @@ -package message - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "github.com/gogf/gf/v2/database/gredis" - "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/os/glog" - "github.com/gogf/gf/v2/util/gconv" -) - -// StreamMessage Redis Stream 消息结构 -type StreamMessage struct { - ID string // 消息ID(自动生成) - Values map[string]interface{} // 消息内容 -} - -// getClient 获取 Redis 客户端 -func getRedisClientTest(name string) *gredis.Redis { - return g.Redis(name) -} - -// getRedisClientByDB 根据DB获取Redis客户端,如果db<=0则返回默认客户端 -func getRedisClientByDB(db int) *gredis.Redis { - if db <= 0 { - return g.Redis() - } - // 创建连接到指定DB的Redis客户端 - client, err := gredis.New(&gredis.Config{ - Address: g.Cfg().MustGet(context.Background(), "redis.default.address").String(), - Db: db, - }) - if err != nil { - glog.Errorf(context.Background(), "创建Redis客户端失败: %v", err) - return g.Redis() - } - return client -} - -// lock 分布式锁 -func lock(ctx context.Context, key string, expireSeconds int64, fn func(ctx context.Context) error) (success bool, err error) { - ds, err := GetManager().GetDefaultDataSource() - if err != nil { - return false, fmt.Errorf("获取默认数据源失败: %w", err) - } - - maxRetries := 3 - for i := 0; i < maxRetries; i++ { - if val, err := ds.Redis().Set(ctx, key, true, gredis.SetOption{ - TTLOption: gredis.TTLOption{ - EX: &expireSeconds, - }, - NX: true, - }); err != nil { - return false, err - } else { - if val.Bool() { - defer func(redisClient *gredis.Redis, ctx context.Context, key string) { - if _, err = redisClient.Del(ctx, key); err != nil { - glog.Errorf(ctx, "RedisClient.Del error: %v", err) - } - }(ds.Redis(), ctx, key) - if err = fn(ctx); err != nil { - return false, err - } - return true, nil - } else { - // 检查上下文是否已取消 - if ctx.Err() != nil { - return false, ctx.Err() - } - // 非最后一次重试时才等待 - if i < maxRetries-1 { - time.Sleep(time.Second) - } - } - } - } - return false, errors.New("锁重试次数耗尽") -} - -// publishToRedis 将消息添加到 Redis Stream -func publishToRedis(ctx context.Context, streamKey string, msg interface{}) (messageID string, err error) { - ds, err := GetManager().GetDefaultDataSource() - if err != nil { - return "", fmt.Errorf("获取默认数据源失败: %w", err) - } - - if !ds.IsConnected() { - if err := ds.Reconnect(ctx); err != nil { - return "", fmt.Errorf("redis重连失败: %w", err) - } - } - - values := gconv.Map(msg) - args := make([]interface{}, 0, len(values)*2+2) - args = append(args, streamKey, "*") - for key, val := range values { - args = append(args, key, val) - } - result, err := ds.Redis().Do(ctx, "XADD", args...) - if err != nil { - return - } - messageID = result.String() - return -} - -// initStreamGroup 初始化 Stream 和消费者组 -func initStreamGroup(ctx context.Context, streamKey, groupName string) error { - ds, err := GetManager().GetDefaultDataSource() - if err != nil { - return fmt.Errorf("获取默认数据源失败: %w", err) - } - - if !ds.IsConnected() { - if err := ds.Reconnect(ctx); err != nil { - return fmt.Errorf("redis重连失败: %w", err) - } - } - - _, err = ds.Redis().Do(ctx, "XGROUP", "CREATE", streamKey, groupName, "0", "MKSTREAM") - if err != nil { - // 如果组已存在,忽略错误 - errStr := err.Error() - // 检查错误是否是 "BUSYGROUP Consumer Group name already exists" - if strings.Contains(errStr, "BUSYGROUP") || strings.Contains(errStr, "already exists") { - // 这是一个预期的情况,说明消费者组已经存在,无需处理 - return nil - } - // 这是一个真正的错误,需要记录或处理 - return err - } - return nil -} - -// readFromStream 从 Stream 读取消息 -func readFromStream(ctx context.Context, msg QueueMessage) error { - ds, err := GetManager().GetDefaultDataSource() - if err != nil { - return fmt.Errorf("获取默认数据源失败: %w", err) - } - - // 初始化 Stream 和消费者组 - if err := initStreamGroup(ctx, msg.StreamKey, msg.GroupName); err != nil { - return err - } - go func() { - RECONNECT: - for { - result, err := ds.Redis().Do(ctx, "XREADGROUP", "GROUP", msg.GroupName, msg.ConsumerName, "COUNT", msg.BatchSize, "BLOCK", 0, "STREAMS", msg.StreamKey, ">") - if err != nil { - //select { - //case <-ctx.Done(): - // return - //} - time.Sleep(time.Second) - goto RECONNECT - } - // 检查返回结果是否为空 - if result == nil || result.IsEmpty() { - continue - } - messages := make([]StreamMessage, 0, int(msg.BatchSize)) - // 尝试 map 格式(GoFrame gredis 返回) - if streamsMap, ok := result.Val().(map[interface{}]interface{}); ok { - for _, streamMsgs := range streamsMap { - msgsArray, ok := streamMsgs.([]interface{}) - if !ok { - continue - } - for _, msgData := range msgsArray { - msgArray, ok := msgData.([]interface{}) - if !ok || len(msgArray) < 2 { - continue - } - msgID := gconv.String(msgArray[0]) - fieldsArray, ok := msgArray[1].([]interface{}) - if !ok { - continue - } - values := make(map[string]interface{}, len(fieldsArray)/2) - for i := 0; i < len(fieldsArray); i += 2 { - if i+1 < len(fieldsArray) { - key := gconv.String(fieldsArray[i]) - values[key] = fieldsArray[i+1] - } - } - messages = append(messages, StreamMessage{ - ID: msgID, - Values: values, - }) - } - } - } - // 尝试数组格式(标准 Redis 返回) - if streamsArray, ok := result.Val().([]interface{}); ok && len(streamsArray) > 0 { - for _, streamData := range streamsArray { - streamArray, ok := streamData.([]interface{}) - if !ok || len(streamArray) < 2 { - continue - } - messagesArray, ok := streamArray[1].([]interface{}) - if !ok { - continue - } - for _, msgData := range messagesArray { - msgArray, ok := msgData.([]interface{}) - if !ok || len(msgArray) < 2 { - continue - } - msgID := gconv.String(msgArray[0]) - fieldsArray, ok := msgArray[1].([]interface{}) - if !ok { - continue - } - values := make(map[string]interface{}, len(fieldsArray)/2) - for i := 0; i < len(fieldsArray); i += 2 { - if i+1 < len(fieldsArray) { - key := gconv.String(fieldsArray[i]) - values[key] = fieldsArray[i+1] - } - } - messages = append(messages, StreamMessage{ - ID: msgID, - Values: values, - }) - } - } - } - // 处理消息 - for _, streamMsg := range messages { - // 业务处理 - if err := msg.HandleFunc(ctx, streamMsg.Values); err != nil { - glog.Infof(ctx, "业务处理失败-> err:%v\n", err) - continue - } - // 确认消息 - if msg.AutoAck { - err := ackMessage(ctx, msg.StreamKey, msg.GroupName, streamMsg.ID) - if err != nil { - glog.Infof(ctx, "消费者 '%s' 确认消息 ID %s 失败: %v\n", msg.ConsumerName, streamMsg.ID, err) - } - } - } - } - }() - return nil -} - -// ackMessage 确认消息已处理 -func ackMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error { - ds, err := GetManager().GetDefaultDataSource() - if err != nil { - return fmt.Errorf("获取默认数据源失败: %w", err) - } - - if !ds.IsConnected() { - if err := ds.Reconnect(ctx); err != nil { - return fmt.Errorf("redis重连失败: %w", err) - } - } - - args := make([]interface{}, 0, len(messageIDs)+2) - args = append(args, streamKey, groupName) - for _, id := range messageIDs { - args = append(args, id) - } - _, err = ds.Redis().Do(ctx, "XACK", args...) - return err -} diff --git a/message/redis_client.go b/message/redis_client.go deleted file mode 100644 index 2db78cb..0000000 --- a/message/redis_client.go +++ /dev/null @@ -1,468 +0,0 @@ -// ============================================================================= -// Redis 数据源连接管理 -// 使用 GoFrame 框架自带的 Redis 客户端,负责数据源的连接、重连、健康检查和优雅关闭 -// ============================================================================= - -package message - -import ( - "context" - "fmt" - "os" - "os/signal" - "sync" - "sync/atomic" - "syscall" - "time" - - "github.com/gogf/gf/v2/database/gredis" - "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/os/glog" - "github.com/gogf/gf/v2/util/gconv" -) - -// ============================================================================= -// 数据源配置结构 -// ============================================================================= - -type RedisDataSourceConfig struct { - Name string `json:"name"` // 数据源名称 - Address string `json:"address"` // Redis 地址,如: 127.0.0.1:6379 - Db int `json:"db"` // 数据库编号 - Pass string `json:"pass"` // 密码 - Timeout time.Duration `json:"timeout"` // 连接超时 - MaxIdle int `json:"maxIdle"` // 最大空闲连接数 - MaxOpen int `json:"maxOpen"` // 最大活跃连接数 -} - -// ============================================================================= -// 单个数据源接口 -// ============================================================================= - -type DataSource interface { - Name() string - Redis() *gredis.Redis - IsConnected() bool - Connect(ctx context.Context) error - Reconnect(ctx context.Context) error - Close(ctx context.Context) error -} - -// ============================================================================= -// 数据源实现 -// ============================================================================= - -type BaseDataSource struct { - config *RedisDataSourceConfig - redis *gredis.Redis - isConnected bool - mu sync.RWMutex - lastError error - lastErrorTime time.Time - metrics RedisMetrics -} - -func NewBaseDataSource(config *RedisDataSourceConfig) *BaseDataSource { - return &BaseDataSource{ - config: config, - isConnected: false, - } -} - -func (d *BaseDataSource) Name() string { - return d.config.Name -} - -func (d *BaseDataSource) Redis() *gredis.Redis { - d.mu.RLock() - defer d.mu.RUnlock() - return d.redis -} - -func (d *BaseDataSource) IsConnected() bool { - d.mu.RLock() - defer d.mu.RUnlock() - return d.isConnected && d.redis != nil -} - -func (d *BaseDataSource) Connect(ctx context.Context) error { - d.mu.Lock() - defer d.mu.Unlock() - - // 设置默认值 - config := d.config - if config.Timeout == 0 { - config.Timeout = 10 * time.Second - } - if config.MaxIdle == 0 { - config.MaxIdle = 10 - } - if config.MaxOpen == 0 { - config.MaxOpen = 100 - } - - // 构建 GoFrame Redis 配置 - redisConfig := &gredis.Config{ - Address: config.Address, - Db: config.Db, - Pass: config.Pass, - } - - // 使用 GoFrame 的 Redis 连接 - redisObj, err := gredis.New(redisConfig) - if err != nil { - d.isConnected = false - d.lastError = err - d.lastErrorTime = time.Now() - d.metrics.PingError.Add(1) - return fmt.Errorf("datasource [%s] connection failed: %w", d.config.Name, err) - } - - d.redis = redisObj - - // 测试连接 - if err := d.Ping(ctx); err != nil { - d.isConnected = false - d.lastError = err - d.lastErrorTime = time.Now() - return fmt.Errorf("datasource [%s] ping failed: %w", d.config.Name, err) - } - - d.isConnected = true - d.lastError = nil - glog.Infof(ctx, "✅ datasource [%s] connected successfully", d.config.Name) - return nil -} - -func (d *BaseDataSource) Ping(ctx context.Context) error { - defer func() { - if r := recover(); r != nil { - d.metrics.PingError.Add(1) - glog.Errorf(ctx, "❌ datasource [%s] ping panic: %v", d.config.Name, r) - } - }() - - if d.redis == nil { - d.metrics.PingError.Add(1) - return fmt.Errorf("redis client is nil") - } - - _, err := d.redis.Do(ctx, "PING") - if err != nil { - d.metrics.PingError.Add(1) - return err - } - - d.metrics.PingCount.Add(1) - return nil -} - -func (d *BaseDataSource) Reconnect(ctx context.Context) error { - glog.Infof(ctx, "🔄 reconnecting datasource [%s]", d.config.Name) - return d.Connect(ctx) -} - -func (d *BaseDataSource) Close(ctx context.Context) error { - d.mu.Lock() - defer d.mu.Unlock() - - if d.redis != nil { - if err := d.redis.Close(ctx); err != nil { - return fmt.Errorf("datasource [%s] close failed: %w", d.config.Name, err) - } - } - - d.isConnected = false - d.redis = nil - glog.Infof(ctx, "datasource [%s] closed", d.config.Name) - return nil -} - -func (d *BaseDataSource) GetMetrics() RedisMetrics { - return d.metrics -} - -// ============================================================================= -// 监控指标 -// ============================================================================= - -type RedisMetrics struct { - PingCount atomic.Int64 - PingError atomic.Int64 - CommandCount atomic.Int64 - CommandError atomic.Int64 -} - -// GetPingMetrics 获取 Ping 相关指标 -func (m *RedisMetrics) GetPingMetrics() (int64, int64) { - return m.PingCount.Load(), m.PingError.Load() -} - -// GetCommandMetrics 获取命令相关指标 -func (m *RedisMetrics) GetCommandMetrics() (int64, int64) { - return m.CommandCount.Load(), m.CommandError.Load() -} - -// ============================================================================= -// 多数据源管理器 -// ============================================================================= - -type DataSourceManager struct { - sources map[string]DataSource - mu sync.RWMutex - ctx context.Context - cancel context.CancelFunc - started bool - maxRetries int - metrics RedisMetrics -} - -var ( - manager *DataSourceManager - once sync.Once -) - -// GetManager 获取全局管理器 -func GetManager() *DataSourceManager { - once.Do(func() { - ctx, cancel := context.WithCancel(context.Background()) - manager = &DataSourceManager{ - sources: make(map[string]DataSource), - ctx: ctx, - cancel: cancel, - started: false, - maxRetries: 3, - } - }) - return manager -} - -// RegisterDataSource 注册数据源 -func (m *DataSourceManager) RegisterDataSource(config *RedisDataSourceConfig) error { - m.mu.Lock() - defer m.mu.Unlock() - - if _, exists := m.sources[config.Name]; exists { - return fmt.Errorf("datasource [%s] already exists", config.Name) - } - - source := NewBaseDataSource(config) - m.sources[config.Name] = source - return nil -} - -// GetDataSource 获取数据源 -func (m *DataSourceManager) GetDataSource(name string) (DataSource, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - source, exists := m.sources[name] - if !exists { - return nil, fmt.Errorf("datasource [%s] not found", name) - } - return source, nil -} - -// GetAllDataSourceNames 获取所有数据源名称 -func (m *DataSourceManager) GetAllDataSourceNames() []string { - m.mu.RLock() - defer m.mu.RUnlock() - - names := make([]string, 0, len(m.sources)) - for name := range m.sources { - names = append(names, name) - } - return names -} - -// GetDefaultDataSource 获取默认数据源(第一个注册的数据源) -func (m *DataSourceManager) GetDefaultDataSource() (DataSource, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - for _, source := range m.sources { - return source, nil - } - return nil, fmt.Errorf("no datasource available") -} - -// GetMetrics 获取全局监控指标 -func (m *DataSourceManager) GetMetrics() RedisMetrics { - return m.metrics -} - -// init 初始化多数据源 -func init() { - ctx := context.Background() - - // 从配置初始化多数据源 - if err := GetManager().InitializeFromConfig(ctx); err != nil { - glog.Errorf(ctx, "❌ Failed to initialize Redis datasources: %v", err) - } else { - glog.Infof(ctx, "✅ Redis datasources initialized: %v", GetManager().GetAllDataSourceNames()) - } - - // 启动健康检查 - GetManager().StartHealthCheck() - - // 设置优雅关闭 - setupGracefulShutdown() -} - -// InitializeFromConfig 从配置初始化数据源 -// 动态读取 config.yml 中 redis 下的所有配置项 -func (m *DataSourceManager) InitializeFromConfig(ctx context.Context) error { - var firstErr error - - // 获取 redis 配置下的所有子键 - redisConfig := g.Cfg().MustGet(ctx, "redis") - if redisConfig.IsNil() { - glog.Warningf(ctx, "no redis configuration found in config.yml") - return nil - } - - // 将配置转换为 map - configMap := redisConfig.Map() - if configMap == nil { - glog.Warningf(ctx, "redis configuration is not a map") - return nil - } - - // 遍历所有 redis 子配置 - for name, subConfig := range configMap { - // 跳过非对象类型的配置 - subMap, ok := subConfig.(map[string]interface{}) - if !ok { - continue - } - - // 检查是否有 address 配置 - address, hasAddress := subMap["address"] - if !hasAddress || gconv.String(address) == "" { - continue - } - - // 构建数据源配置 - config := &RedisDataSourceConfig{ - Name: name, - Address: gconv.String(address), - Db: gconv.Int(subMap["db"]), - Pass: gconv.String(subMap["pass"]), - } - - // 设置默认值 - if config.Db == 0 { - config.Db = 0 - } - if config.Timeout == 0 { - config.Timeout = 10 * time.Second - } - if config.MaxIdle == 0 { - config.MaxIdle = 10 - } - if config.MaxOpen == 0 { - config.MaxOpen = 100 - } - - // 注册数据源 - if err := m.RegisterDataSource(config); err != nil { - glog.Errorf(ctx, "failed to register datasource [%s]: %v", name, err) - if firstErr == nil { - firstErr = err - } - continue - } - - // 连接数据源 - source, _ := m.GetDataSource(name) - if err := source.Connect(ctx); err != nil { - glog.Errorf(ctx, "failed to initialize datasource [%s]: %v", name, err) - if firstErr == nil { - firstErr = err - } - } - } - - return firstErr -} - -// StartHealthCheck 启动健康检查 -func (m *DataSourceManager) StartHealthCheck() { - if m.started { - return - } - m.started = true - go m.healthCheckLoop() -} - -// healthCheckLoop 健康检查循环 -func (m *DataSourceManager) healthCheckLoop() { - ticker := time.NewTicker(30 * time.Second) - defer ticker.Stop() - - for { - select { - case <-m.ctx.Done(): - return - case <-ticker.C: - m.checkAndReconnect() - } - } -} - -// checkAndReconnect 检查并重新连接 -func (m *DataSourceManager) checkAndReconnect() { - m.mu.RLock() - defer m.mu.RUnlock() - - for name, source := range m.sources { - if !source.IsConnected() { - glog.Warningf(context.Background(), "datasource [%s] disconnected, attempting reconnect", name) - - reconnectCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - if err := source.Reconnect(reconnectCtx); err != nil { - glog.Errorf(reconnectCtx, "datasource [%s] reconnect failed: %v", name, err) - } else { - glog.Infof(reconnectCtx, "✅ datasource [%s] reconnected successfully", name) - } - } - } -} - -// CloseAll 关闭所有数据源 -func (m *DataSourceManager) CloseAll(ctx context.Context) error { - m.cancel() - - m.mu.RLock() - defer m.mu.RUnlock() - - var lastErr error - for name, source := range m.sources { - if err := source.Close(ctx); err != nil { - glog.Errorf(ctx, "failed to close datasource [%s]: %v", name, err) - lastErr = err - } - } - return lastErr -} - -// setupGracefulShutdown 设置优雅关闭 -func setupGracefulShutdown() { - go func() { - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - <-sigCh - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - glog.Info(ctx, "🔄 Shutting down Redis connections...") - if err := GetManager().CloseAll(ctx); err != nil { - glog.Errorf(ctx, "❌ Failed to close Redis connections: %v", err) - } else { - glog.Info(ctx, "✅ Redis connections closed successfully") - } - }() -} diff --git a/message/redis_msg.go b/message/redis_msg.go index 135a78c..38dd331 100644 --- a/message/redis_msg.go +++ b/message/redis_msg.go @@ -2,7 +2,6 @@ package message import ( "context" - "encoding/json" "fmt" "strings" "time" @@ -12,151 +11,137 @@ import ( "github.com/gogf/gf/v2/util/gconv" ) -// redisMessageClient Redis 实现 -type redisMessageClient struct { - clientType messageClientType +type RedisPublishMsgConfig struct { + QueueName string + Data any } +type RedisSubscribeMsgConfig struct { + QueueName string + ConsumerName string + AutoAck bool + PrefetchCount int + HandleFunc func(ctx context.Context, message map[string]interface{}) error +} + +func (*RedisPublishMsgConfig) GetPublishMsgType() { + +} + +func (*RedisSubscribeMsgConfig) GetSubscribeMsgType() { + +} + +func init() { + // 注册 Redis 插件(连接由 RegisterPlugin 异步处理) + registerPlugin(MessageRedis, func() messageUtil { + return &redis{} + }) +} + +type redis struct{} + // RedisStreamMessage Redis Stream 消息结构 -type RedisStreamMessage struct { +type redisStreamMessage struct { ID string Values map[string]interface{} } -// StreamGroup 创建消费组(支持单个或批量) -func (q *redisMessageClient) streamGroup(ctx context.Context, configs ...interface{}) error { - if len(configs) == 0 { - return fmt.Errorf("配置不能为空") +// Ping 检测 Redis 连接状态 +func (c *redis) ping(ctx context.Context) bool { + conn, err := getDefaultDataSource() + if err != nil { + return false } - for _, config := range configs { - cfg, ok := config.(*RedisConfig) - if !ok { - return fmt.Errorf("无效的 Redis 配置类型") - } - if err := q.createStreamGroup(ctx, cfg); err != nil { - return err - } + return conn.redisPing(ctx) +} + +// Reconnect 重连 Redis +func (c *redis) reconnect(ctx context.Context) error { + conn, err := getDefaultDataSource() + if err != nil { + return fmt.Errorf("获取默认连接失败: %w", err) } + + if err := conn.redisReconnect(ctx); err != nil { + return fmt.Errorf("redis重连失败: %w", err) + } + return nil } -// streamGroup 内部单个创建消费组 -func (q *redisMessageClient) createStreamGroup(ctx context.Context, cfg *RedisConfig) error { - // 获取默认数据源 - ds, err := GetManager().GetDefaultDataSource() +// Close 关闭 Redis 连接 +func (c *redis) close(ctx context.Context) error { + conn, err := getDefaultDataSource() if err != nil { - return fmt.Errorf("获取默认数据源失败: %w", err) + return fmt.Errorf("获取默认连接失败: %w", err) } - // 检查连接状态,未连接则自动重连 - if !ds.IsConnected() { - if err := ds.Reconnect(ctx); err != nil { - return fmt.Errorf("redis重连失败: %w", err) - } + if err := conn.redisClose(ctx); err != nil { + return fmt.Errorf("关闭redis连接失败: %w", err) } - _, err = ds.Redis().Do(ctx, "XGROUP", "CREATE", cfg.Stream, cfg.Group, "0", "MKSTREAM") - if err != nil { - errStr := err.Error() - if strings.Contains(errStr, "BUSYGROUP") && strings.Contains(errStr, "already exists") { - glog.Infof(ctx, "✅ Redis 消费者组已存在: %s", cfg.Group) - return nil - } - return fmt.Errorf("初始化消费者组失败: %w", err) - } - glog.Infof(ctx, "✅ Redis 消费者组创建成功: %s", cfg.Group) return nil } -// Publish 内部单个发布消息 -func (q *redisMessageClient) publish(ctx context.Context, config interface{}, data interface{}) error { - ds, err := GetManager().GetDefaultDataSource() - if err != nil { - return fmt.Errorf("获取默认数据源失败: %w", err) - } - - if !ds.IsConnected() { - if err := ds.Reconnect(ctx); err != nil { - return fmt.Errorf("redis重连失败: %w", err) - } - } - - cfg, ok := config.(*RedisConfig) +// Publish 发布消息 +func (c *redis) Publish(ctx context.Context, msgConfig messagePublishConfig) error { + cfg, ok := msgConfig.(*RedisPublishMsgConfig) if !ok { - return fmt.Errorf("无效的redis配置类型") + return fmt.Errorf("无效的 Redis 配置类型") } - values := gconv.Map(data) + if g.IsEmpty(cfg.QueueName) { + return fmt.Errorf("队列名称不能为空") + } + if g.IsEmpty(cfg.Data) { + return fmt.Errorf("数据不能为空") + } + conn, err := getDefaultDataSource() + if err != nil { + return fmt.Errorf("获取默认连接失败: %w", err) + } + + if !conn.getIsConnected() { + if err := conn.redisReconnect(ctx); err != nil { + return fmt.Errorf("redis重连失败: %w", err) + } + } + + values := gconv.Map(cfg.Data) args := make([]interface{}, 0, len(values)*2+2) - args = append(args, cfg.Stream, "*") + args = append(args, cfg.QueueName, "*") for key, val := range values { args = append(args, key, val) } - result, err := ds.Redis().Do(ctx, "XADD", args...) + result, err := conn.getClient().Do(ctx, "XADD", args...) if err != nil { - g.Log().Errorf(ctx, "❌ Redis 发布消息失败: topic=%s, err=%v", cfg.Stream, err) + g.Log().Errorf(ctx, "❌ Redis 发布消息失败: key=%s, err=%v", cfg.QueueName, err) return err } - g.Log().Infof(ctx, "✅ Redis 发布消息成功: topic=%s, messageID=%s", cfg.Stream, gconv.String(result)) + g.Log().Infof(ctx, "✅ Redis 发布消息成功: key=%s, messageID=%s", cfg.QueueName, gconv.String(result)) return nil } -// PublishDelayed 发布延迟消息(使用 ZSET) -func (q *redisMessageClient) publishDelayed(ctx context.Context, config interface{}, data interface{}, delay int) error { - ds, err := GetManager().GetDefaultDataSource() - if err != nil { - return fmt.Errorf("获取默认数据源失败: %w", err) - } - - if !ds.IsConnected() { - if err := ds.Reconnect(ctx); err != nil { - return fmt.Errorf("redis重连失败: %w", err) - } - } - - cfg, ok := config.(*RedisConfig) +// Subscribe 订阅消息 +func (c *redis) Subscribe(ctx context.Context, msgConfig messageSubscribeConfig) error { + cfg, ok := msgConfig.(*RedisSubscribeMsgConfig) if !ok { - return fmt.Errorf("无效的redis配置类型") + return fmt.Errorf("无效的 Redis 配置类型") } - payload, err := json.Marshal(data) - if err != nil { - return fmt.Errorf("序列化数据失败: %w", err) + if g.IsEmpty(cfg.QueueName) { + return fmt.Errorf("队列名称不能为空") } - score := float64(time.Now().Add(time.Duration(delay)).UnixMilli()) - delayedKey := fmt.Sprintf("delayed:%s", cfg.Stream) - - // ZADD delayedKey score payload - _, err = ds.Redis().Do(ctx, "ZADD", delayedKey, score, string(payload)) - if err != nil { - return err + if g.IsEmpty(cfg.ConsumerName) { + return fmt.Errorf("消费者名称不能为空") } - - g.Log().Infof(ctx, "✅ Redis 延迟消息已发布: topic=%s, delay=%v", cfg.Stream, delay) - return nil + if g.IsEmpty(cfg.HandleFunc) { + return fmt.Errorf("处理函数不能为空") + } + return c.createSubscribe(ctx, cfg.QueueName, cfg.ConsumerName, cfg.PrefetchCount, cfg.AutoAck, cfg.HandleFunc) } -// Subscribe 订阅消息(支持单个或批量) -func (q *redisMessageClient) subscribe(ctx context.Context, configs ...interface{}) error { - if len(configs) == 0 { - return fmt.Errorf("配置不能为空") - } - for _, config := range configs { - cfg, ok := config.(*RedisConfig) - if !ok { - return fmt.Errorf("无效的 Redis 配置类型") - } - handler := cfg.HandleFunc - if handler == nil { - return fmt.Errorf("必须提供处理函数") - } - if err := q.createSubscribe(ctx, cfg, handler); err != nil { - return err - } - } - return nil -} - -// subscribe 内部单个订阅消息 -func (q *redisMessageClient) createSubscribe(ctx context.Context, cfg *RedisConfig, handler func(ctx context.Context, message map[string]interface{}) error) error { +// createSubscribe 内部订阅消息 +func (c *redis) createSubscribe(ctx context.Context, key, consumerName string, prefetchCount int, autoAck bool, handler func(ctx context.Context, message map[string]interface{}) error) error { go func() { defer func() { if r := recover(); r != nil { @@ -174,10 +159,10 @@ func (q *redisMessageClient) createSubscribe(ctx context.Context, cfg *RedisConf for { select { case <-ctx.Done(): - g.Log().Infof(ctx, "🔕 Redis 消费者停止: topic=%s", cfg.Stream) + g.Log().Infof(ctx, "🔕 Redis 消费者停止: topic=%s", key) return case <-retryTicker.C: - err := q.consumeMessages(ctx, cfg, handler) + err := c.consumeMessages(ctx, key, consumerName, prefetchCount, autoAck, handler) if err != nil { // 对于超时错误,返回nil继续循环,而不是返回错误 if strings.Contains(err.Error(), "i/o timeout") || strings.Contains(err.Error(), "timeout") || @@ -216,25 +201,25 @@ func (q *redisMessageClient) createSubscribe(ctx context.Context, cfg *RedisConf } // consumeMessages 消费消息 -func (q *redisMessageClient) consumeMessages(ctx context.Context, cfg *RedisConfig, handler func(ctx context.Context, message map[string]interface{}) error) error { - ds, err := GetManager().GetDefaultDataSource() +func (c *redis) consumeMessages(ctx context.Context, key, consumerName string, prefetchCount int, autoAck bool, handler func(ctx context.Context, message map[string]interface{}) error) error { + conn, err := getDefaultDataSource() if err != nil { - return fmt.Errorf("获取默认数据源失败: %w", err) + return fmt.Errorf("获取默认连接失败: %w", err) } - if !ds.IsConnected() { - if err := ds.Reconnect(ctx); err != nil { + if !conn.getIsConnected() { + if err := conn.redisReconnect(ctx); err != nil { return fmt.Errorf("redis重连失败: %w", err) } } // 检查消费者组是否存在 - if err := q.createStreamGroup(ctx, cfg); err != nil { + if err := c.createStreamGroup(ctx, key); err != nil { return fmt.Errorf("create stream group failed: %w", err) } // 使用带重试的命令执行 - result, err := ds.Redis().Do(ctx, "XREADGROUP", "GROUP", cfg.Group, cfg.Consumer, "COUNT", cfg.Count, "BLOCK", 0, "STREAMS", cfg.Stream, ">") + result, err := conn.getClient().Do(ctx, "XREADGROUP", "GROUP", "default", consumerName, "COUNT", prefetchCount, "BLOCK", 0, "STREAMS", key, ">") if err != nil { if strings.Contains(err.Error(), "i/o timeout") || strings.Contains(err.Error(), "timeout") || strings.Contains(err.Error(), "context deadline exceeded") || strings.Contains(err.Error(), "context canceled") { @@ -242,7 +227,7 @@ func (q *redisMessageClient) consumeMessages(ctx context.Context, cfg *RedisConf } return err } - messages, err := q.parseStreamResult(result) + messages, err := c.parseStreamResult(result) if err != nil { return err } @@ -254,8 +239,8 @@ func (q *redisMessageClient) consumeMessages(ctx context.Context, cfg *RedisConf } // ACK 消息 - if cfg.AutoAck { - if err := q.ackMessage(ctx, cfg.Stream, cfg.Group, msg.ID); err != nil { + if autoAck { + if err := c.ackMessage(ctx, key, "default", msg.ID); err != nil { g.Log().Errorf(ctx, "❌ ACK 消息失败: messageID=%s, err=%v", msg.ID, err) } } @@ -264,15 +249,42 @@ func (q *redisMessageClient) consumeMessages(ctx context.Context, cfg *RedisConf return nil } -// ackMessage ACK 消息 -func (q *redisMessageClient) ackMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error { - ds, err := GetManager().GetDefaultDataSource() +// createStreamGroup 内部单个创建消费组 +func (c *redis) createStreamGroup(ctx context.Context, key string) error { + conn, err := getDefaultDataSource() if err != nil { - return fmt.Errorf("获取默认数据源失败: %w", err) + return fmt.Errorf("获取默认连接失败: %w", err) } - if !ds.IsConnected() { - if err := ds.Reconnect(ctx); err != nil { + if !conn.getIsConnected() { + if err := conn.redisReconnect(ctx); err != nil { + return fmt.Errorf("redis重连失败: %w", err) + } + } + + groupName := "default" + _, err = conn.getClient().Do(ctx, "XGROUP", "CREATE", key, groupName, "0", "MKSTREAM") + if err != nil { + errStr := err.Error() + if strings.Contains(errStr, "BUSYGROUP") && strings.Contains(errStr, "already exists") { + glog.Infof(ctx, "✅ Redis 消费者组已存在: %s", groupName) + return nil + } + return fmt.Errorf("初始化消费者组失败: %w", err) + } + glog.Infof(ctx, "✅ Redis 消费者组创建成功: %s", groupName) + return nil +} + +// ackMessage ACK 消息 +func (c *redis) ackMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error { + conn, err := getDefaultDataSource() + if err != nil { + return fmt.Errorf("获取默认连接失败: %w", err) + } + + if !conn.getIsConnected() { + if err := conn.redisReconnect(ctx); err != nil { return fmt.Errorf("redis重连失败: %w", err) } } @@ -282,14 +294,14 @@ func (q *redisMessageClient) ackMessage(ctx context.Context, streamKey, groupNam for _, id := range messageIDs { args = append(args, id) } - _, err = ds.Redis().Do(ctx, "XACK", args...) + _, err = conn.getClient().Do(ctx, "XACK", args...) return err } // parseStreamResult 解析 Stream 结果 -func (q *redisMessageClient) parseStreamResult(result interface{}) ([]RedisStreamMessage, error) { +func (c *redis) parseStreamResult(result interface{}) ([]redisStreamMessage, error) { if result == nil { - return []RedisStreamMessage{}, nil + return []redisStreamMessage{}, nil } var resultVal interface{} @@ -303,15 +315,15 @@ func (q *redisMessageClient) parseStreamResult(result interface{}) ([]RedisStrea // 检查是否为空 if resultVal == nil { - return []RedisStreamMessage{}, nil + return []redisStreamMessage{}, nil } // 预分配切片容量,避免多次扩容 - messages := make([]RedisStreamMessage, 0) + messages := make([]redisStreamMessage, 0) if streamsMap, ok := resultVal.(map[interface{}]interface{}); ok { - for _, streamMsg := range streamsMap { - msgArray, ok := streamMsg.([]interface{}) + for _, streamData := range streamsMap { + msgArray, ok := streamData.([]interface{}) if !ok { continue } @@ -332,7 +344,7 @@ func (q *redisMessageClient) parseStreamResult(result interface{}) ([]RedisStrea values[key] = fieldsArray[i+1] } } - messages = append(messages, RedisStreamMessage{ + messages = append(messages, redisStreamMessage{ ID: msgID, Values: values, }) diff --git a/middleware/module_tenant_check.go b/middleware/module_tenant_check.go index 1eff872..2ba3518 100644 --- a/middleware/module_tenant_check.go +++ b/middleware/module_tenant_check.go @@ -6,7 +6,7 @@ import ( "fmt" "gitee.com/red-future---jilin-g/common/beans" "gitee.com/red-future---jilin-g/common/message" - "gitee.com/red-future---jilin-g/common/nats" + "gitee.com/red-future---jilin-g/common/redis" "gitee.com/red-future---jilin-g/common/utils" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" @@ -20,7 +20,7 @@ import ( func ModuleTenantCheck(r *ghttp.Request) { // 检查是否是超级管理员 isSuperAdmin := false - if err := nats.CallRPC(r.Context(), "userService.IsSuperAdmin", nil, &isSuperAdmin); err != nil { + if err := message.CallRPC(r.Context(), "userService.IsSuperAdmin", nil, &isSuperAdmin); err != nil { SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, err) } // 如果是超级管理员,则不进行模块租户检查 @@ -33,7 +33,7 @@ func ModuleTenantCheck(r *ghttp.Request) { SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, err) } exit := gconv.Int64(time.Minute * 1) - getEX, err := message.GetRedisClientTest("test").GetEX(r.Context(), fmt.Sprintf("module_tenant:tenantId-%v", getUserInfo.TenantId), gredis.GetEXOption{ + getEX, err := redis.GetRedisClientTest("test").GetEX(r.Context(), fmt.Sprintf("module_tenant:tenantId-%v", getUserInfo.TenantId), gredis.GetEXOption{ TTLOption: gredis.TTLOption{ EX: &exit, }, @@ -68,7 +68,7 @@ func ModuleTenantCheck(r *ghttp.Request) { ModuleKey: moduleKey, TenantId: gconv.Uint64(getUserInfo.TenantId), } - err = nats.CallRPC(r.Context(), "moduleService.Check", &checkReq, checkRes) + err = message.CallRPC(r.Context(), "moduleService.Check", &checkReq, checkRes) if err != nil { SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, err) } diff --git a/redis/redis.go b/redis/redis.go index d8dd359..8bcb742 100644 --- a/redis/redis.go +++ b/redis/redis.go @@ -28,9 +28,9 @@ func getClient() *gredis.Redis { return redisClient } -// GetRedisClient 获取 Redis 客户端(供外部使用) -func GetRedisClient() *gredis.Redis { - return getClient() +// getClient 获取 Redis 客户端 临时方法 +func GetRedisClientTest(name string) *gredis.Redis { + return g.Redis(name) } // RedisClient 获取 Redis 客户端(函数式,确保单例正确初始化) From 69d2ace17f361a9e9ecacfb7e0cfc6e9d42c2ed2 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Sat, 31 Jan 2026 07:35:41 +0800 Subject: [PATCH 205/225] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=B6=88=E6=81=AF?= =?UTF-8?q?=E9=98=9F=E5=88=97=E6=A8=A1=E5=9D=97=EF=BC=8C=E6=96=B0=E5=A2=9E?= =?UTF-8?q?NATS=E5=92=8CRabbitMQ=E8=BF=9E=E6=8E=A5=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=EF=BC=8C=E7=A7=BB=E9=99=A4=E6=97=A7=E7=89=88=E6=B6=88=E6=81=AF?= =?UTF-8?q?=E9=98=9F=E5=88=97=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- message/connection_redis.go | 485 ++++++++++++++++++++++++++++++++++++ 1 file changed, 485 insertions(+) create mode 100644 message/connection_redis.go diff --git a/message/connection_redis.go b/message/connection_redis.go new file mode 100644 index 0000000..465238f --- /dev/null +++ b/message/connection_redis.go @@ -0,0 +1,485 @@ +// ============================================================================= +// Redis 数据源连接管理 +// 负责 Redis 数据源的连接、重连、健康检查和优雅关闭 +// 支持多数据源和无限重连 +// ============================================================================= + +package message + +import ( + "context" + "fmt" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/gogf/gf/v2/database/gredis" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/util/gconv" +) + +// ============================================================================= +// Redis 数据源配置结构 +// ============================================================================= + +type redisDataSourceConfig struct { + name string // 数据源名称 + address string // Redis 地址,如: 127.0.0.1:6379 + db int // 数据库编号 + pass string // 密码 + maxRetries int // 最大重试次数,-1 表示无限重试 + retryInterval time.Duration // 重试间隔 +} + +// ============================================================================= +// Redis 数据源接口 +// ============================================================================= + +type redisDataSource interface { + name() string + getClient() *gredis.Redis + getIsConnected() bool + redisConnect(ctx context.Context) error + redisReconnect(ctx context.Context) error + redisClose(ctx context.Context) error + redisPing(ctx context.Context) bool +} + +// ============================================================================= +// Redis 数据源实现 +// ============================================================================= + +type baseRedisDataSource struct { + config *redisDataSourceConfig + client *gredis.Redis + isConnected bool + mu sync.RWMutex + lastError error + lastErrorTime time.Time + reconnectMu sync.Mutex +} + +func newBaseRedisDataSource(config *redisDataSourceConfig) *baseRedisDataSource { + return &baseRedisDataSource{ + config: config, + isConnected: false, + } +} + +func (d *baseRedisDataSource) name() string { + return d.config.name +} + +func (d *baseRedisDataSource) getClient() *gredis.Redis { + d.mu.RLock() + defer d.mu.RUnlock() + return d.client +} + +func (d *baseRedisDataSource) getIsConnected() bool { + d.mu.RLock() + defer d.mu.RUnlock() + return d.isConnected && d.client != nil +} + +func (d *baseRedisDataSource) redisConnect(ctx context.Context) error { + // 使用互斥锁防止并发重连 + d.reconnectMu.Lock() + defer d.reconnectMu.Unlock() + + d.mu.Lock() + if d.client != nil { + d.client.Close(ctx) + } + d.mu.Unlock() + + // 构建 GoFrame Redis 配置 + redisConfig := &gredis.Config{ + Address: d.config.address, + Db: d.config.db, + Pass: d.config.pass, + } + + // 使用 GoFrame 的 Redis 连接 + redisObj, err := gredis.New(redisConfig) + if err != nil { + d.mu.Lock() + d.isConnected = false + d.lastError = err + d.lastErrorTime = time.Now() + d.mu.Unlock() + return fmt.Errorf("datasource [%s] connection failed: %w", d.config.name, err) + } + + d.mu.Lock() + d.client = redisObj + d.mu.Unlock() + + // 测试连接 + if !d.redisPing(ctx) { + d.mu.Lock() + d.isConnected = false + d.lastError = err + d.lastErrorTime = time.Now() + d.mu.Unlock() + return fmt.Errorf("datasource [%s] ping failed: %w", d.config.name, err) + } + + d.mu.Lock() + d.isConnected = true + d.lastError = nil + d.mu.Unlock() + glog.Infof(ctx, "✅ datasource [%s] connected successfully", d.config.name) + return nil +} + +func (d *baseRedisDataSource) redisReconnect(ctx context.Context) error { + glog.Infof(ctx, "🔄 reconnecting datasource [%s]", d.config.name) + return d.redisConnect(ctx) +} + +func (d *baseRedisDataSource) redisClose(ctx context.Context) error { + d.mu.Lock() + defer d.mu.Unlock() + + if d.client != nil { + if err := d.client.Close(ctx); err != nil { + return fmt.Errorf("datasource [%s] close failed: %w", d.config.name, err) + } + } + + d.isConnected = false + glog.Infof(ctx, "datasource [%s] closed", d.config.name) + return nil +} + +func (d *baseRedisDataSource) redisPing(ctx context.Context) bool { + d.mu.RLock() + client := d.client + d.mu.RUnlock() + + if client == nil { + return false + } + + _, err := client.Do(ctx, "PING") + if err != nil { + return false + } + return true +} + +// ============================================================================= +// Redis 多数据源管理器 +// ============================================================================= + +type redisDataSourceManager struct { + sources map[string]redisDataSource + mu sync.RWMutex + ctx context.Context + cancel context.CancelFunc + started bool + maxRetries int + reconnectCh chan string +} + +var ( + globalRedisManager *redisDataSourceManager + redisManagerOnce sync.Once +) + +// getRedisManager 获取全局 Redis 管理器 +func getRedisManager() *redisDataSourceManager { + redisManagerOnce.Do(func() { + ctx, cancel := context.WithCancel(context.Background()) + globalRedisManager = &redisDataSourceManager{ + sources: make(map[string]redisDataSource), + ctx: ctx, + cancel: cancel, + started: false, + maxRetries: -1, // 默认无限重试 + reconnectCh: make(chan string, 100), + } + }) + return globalRedisManager +} + +// registerDataSource 注册 Redis 数据源 +func (m *redisDataSourceManager) registerDataSource(config *redisDataSourceConfig) error { + m.mu.Lock() + defer m.mu.Unlock() + + if _, exists := m.sources[config.name]; exists { + return fmt.Errorf("datasource [%s] already exists", config.name) + } + + source := newBaseRedisDataSource(config) + m.sources[config.name] = source + return nil +} + +// getDataSource 获取 Redis 数据源 +func (m *redisDataSourceManager) getDataSource(name string) (redisDataSource, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + source, exists := m.sources[name] + if !exists { + return nil, fmt.Errorf("datasource [%s] not found", name) + } + return source, nil +} + +// getAllDataSourceNames 获取所有 Redis 数据源名称 +func (m *redisDataSourceManager) getAllDataSourceNames() []string { + m.mu.RLock() + defer m.mu.RUnlock() + + names := make([]string, 0, len(m.sources)) + for name := range m.sources { + names = append(names, name) + } + return names +} + +// initializeFromConfig 从配置初始化 Redis 数据源 +func (m *redisDataSourceManager) initializeFromConfig(ctx context.Context) error { + var firstErr error + + // 获取 redis 配置下的所有子键 + redisConfig := g.Cfg().MustGet(ctx, "redis") + if redisConfig.IsNil() { + glog.Warningf(ctx, "no redis configuration found in config.yml") + return nil + } + + // 将配置转换为 map + configMap := redisConfig.Map() + if configMap == nil { + glog.Warningf(ctx, "redis configuration is not a map") + return nil + } + + // 遍历所有 redis 子配置 + for name, subConfig := range configMap { + // 跳过非对象类型的配置 + subMap, ok := subConfig.(map[string]interface{}) + if !ok { + continue + } + + // 检查是否有 address 配置 + address, hasAddress := subMap["address"] + if !hasAddress || gconv.String(address) == "" { + continue + } + + // 构建数据源配置 + config := &redisDataSourceConfig{ + name: name, + address: gconv.String(address), + db: gconv.Int(subMap["db"]), + pass: gconv.String(subMap["pass"]), + maxRetries: gconv.Int(subMap["maxRetries"]), + retryInterval: gconv.Duration(subMap["retryInterval"]), + } + + // 设置默认值 + if config.maxRetries == 0 { + config.maxRetries = -1 // 默认无限重试 + } + if config.retryInterval == 0 { + config.retryInterval = 5 * time.Second + } + + // 注册数据源 + if err := m.registerDataSource(config); err != nil { + glog.Errorf(ctx, "failed to register datasource [%s]: %v", name, err) + if firstErr == nil { + firstErr = err + } + continue + } + + // 连接数据源 + source, _ := m.getDataSource(name) + if err := source.redisConnect(ctx); err != nil { + glog.Errorf(ctx, "failed to initialize datasource [%s]: %v", name, err) + if firstErr == nil { + firstErr = err + } + } + } + + return firstErr +} + +// startHealthCheck 启动健康检查 +func (m *redisDataSourceManager) startHealthCheck() { + if m.started { + return + } + m.started = true + + // 启动健康检查循环 + go m.healthCheckLoop() + + // 启动重连处理循环 + go m.reconnectLoop() +} + +// healthCheckLoop 健康检查循环 +func (m *redisDataSourceManager) healthCheckLoop() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + + for { + select { + case <-m.ctx.Done(): + return + case <-ticker.C: + m.checkConnections() + } + } +} + +// reconnectLoop 重连处理循环 +func (m *redisDataSourceManager) reconnectLoop() { + reconnectCounts := make(map[string]int) + + for { + select { + case <-m.ctx.Done(): + return + case name := <-m.reconnectCh: + go func(dsName string) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + source, err := m.getDataSource(dsName) + if err != nil { + glog.Errorf(ctx, "datasource [%s] not found for reconnect", dsName) + return + } + + if err := source.redisReconnect(ctx); err != nil { + glog.Errorf(ctx, "datasource [%s] reconnect failed: %v", dsName, err) + + // 记录重连次数 + reconnectCounts[dsName]++ + + // 检查重连次数限制 + if m.maxRetries > 0 && reconnectCounts[dsName] > m.maxRetries { + glog.Errorf(ctx, "datasource [%s] reconnect count %d exceeds limit %d, stopping auto-reconnect", + dsName, reconnectCounts[dsName], m.maxRetries) + return + } + + // 延迟后重新放入重连队列 + time.Sleep(5 * time.Second) + select { + case m.reconnectCh <- dsName: + default: + // 通道已满,丢弃通知 + } + } else { + // 重连成功,重置计数器 + reconnectCounts[dsName] = 0 + } + }(name) + } + } +} + +// checkConnections 检查连接状态 +func (m *redisDataSourceManager) checkConnections() { + m.mu.RLock() + defer m.mu.RUnlock() + + for name, source := range m.sources { + if !source.getIsConnected() { + glog.Warningf(context.Background(), "datasource [%s] disconnected, queued for reconnect", name) + + // 发送到重连队列 + select { + case m.reconnectCh <- name: + default: + // 通道已满,丢弃通知 + } + } + } +} + +// closeAll 关闭所有 Redis 数据源 +func (m *redisDataSourceManager) closeAll(ctx context.Context) error { + m.cancel() + + m.mu.RLock() + defer m.mu.RUnlock() + + var lastErr error + for name, source := range m.sources { + if err := source.redisClose(ctx); err != nil { + glog.Errorf(ctx, "failed to close datasource [%s]: %v", name, err) + lastErr = err + } + } + return lastErr +} + +// ============================================================================= +// 全局初始化 +// ============================================================================= + +var ( + redisManager = getRedisManager() +) + +// init 初始化 Redis 数据源 +func init() { + ctx := context.Background() + + // 从配置初始化多数据源 + if err := redisManager.initializeFromConfig(ctx); err != nil { + glog.Errorf(ctx, "❌ Failed to initialize Redis datasources: %v", err) + } else { + glog.Infof(ctx, "✅ Redis datasources initialized: %v", redisManager.getAllDataSourceNames()) + } + + // 启动健康检查 + redisManager.startHealthCheck() + + // 设置优雅关闭 + setupGracefulShutdown() +} + +// setupGracefulShutdown 设置优雅关闭 +func setupGracefulShutdown() { + go func() { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + <-sigCh + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + glog.Info(ctx, "🔄 Shutting down Redis connections...") + if err := redisManager.closeAll(ctx); err != nil { + glog.Errorf(ctx, "❌ Failed to close Redis connections: %v", err) + } else { + glog.Info(ctx, "✅ Redis connections closed successfully") + } + }() +} + +// ============================================================================= +// 私有辅助函数 +// ============================================================================= + +// getDefaultDataSource 获取默认数据源 +func getDefaultDataSource() (redisDataSource, error) { + return redisManager.getDataSource("default") +} From 55a6ec037447a0094e108bcf48b27dca521934ab Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Wed, 4 Feb 2026 13:49:17 +0800 Subject: [PATCH 206/225] =?UTF-8?q?=E9=87=8D=E6=9E=84=E6=B6=88=E6=81=AF?= =?UTF-8?q?=E9=98=9F=E5=88=97=E8=BF=9E=E6=8E=A5=E7=AE=A1=E7=90=86=EF=BC=8C?= =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=A4=9A=E6=95=B0=E6=8D=AE=E6=BA=90=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 主要变更: 1. 重构NATS、RabbitMQ和Redis连接管理模块,支持多数据源配置 2. 统一连接管理接口,增加数据源名称参数 3. 优化连接状态检查和错误处理 4. 增加连接池管理和资源清理机制 5. 改进日志输出格式和内容 --- message/connection_nats.go | 171 +++++----- message/connection_rabbitmq.go | 169 ++++++---- message/connection_redis.go | 581 +++++++++------------------------ message/msg_interfaces.go | 14 +- message/msg_plugin_manager.go | 137 ++++---- message/nats_msg.go | 422 ++++++++++++++---------- message/nats_rpc.go | 42 ++- message/rabbitmq_msg.go | 246 +++++++------- message/reconnect.go | 73 +++++ message/redis_msg.go | 259 ++++++--------- message/store.go | 125 +++++++ message/stream.go | 212 ++++++++++++ 12 files changed, 1338 insertions(+), 1113 deletions(-) create mode 100644 message/reconnect.go create mode 100644 message/store.go create mode 100644 message/stream.go diff --git a/message/connection_nats.go b/message/connection_nats.go index e00f3f6..5b33021 100644 --- a/message/connection_nats.go +++ b/message/connection_nats.go @@ -8,137 +8,160 @@ import ( "github.com/gogf/gf/v2/frame/g" "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" ) var ( - nc *nats.Conn - js jetstream.JetStream - natsMu sync.RWMutex + muNats sync.RWMutex + natsConns map[string]*nats.Conn // key: 数据源名称, value: NATS 连接 + natsJS map[string]nats.JetStreamContext // key: 数据源名称, value: JetStream 上下文 ) -// natsConnect 建立 NATS 连接 -func natsConnect(ctx context.Context) error { - natsMu.Lock() - defer natsMu.Unlock() +func init() { + natsConns = make(map[string]*nats.Conn) + natsJS = make(map[string]nats.JetStreamContext) +} - // 安全地关闭旧连接 - if oldConn := nc; oldConn != nil && !oldConn.IsClosed() { +// natsConnect 建立 NATS 连接 +func natsConnect(ctx context.Context, name string) error { + + if g.Cfg().MustGet(ctx, "nats").IsEmpty() { + g.Log().Errorf(ctx, "❌ NATS 配置不存在") + return fmt.Errorf("NATS Configuration does not exist") + } + // 确定数据源名称 + dsName := "default" + if !g.IsEmpty(name) { + dsName = name + } + + g.Log().Infof(ctx, "🔔 NATS [%s] 开始创建连接", dsName) + muNats.Lock() + defer muNats.Unlock() + + // 安全地关闭旧连接(仅针对该数据源) + if oldConn, exists := natsConns[dsName]; exists && oldConn != nil && !oldConn.IsClosed() { oldConn.Close() + delete(natsConns, dsName) + delete(natsJS, dsName) } // 从配置文件读取 NATS 地址 - natsURL := g.Cfg().MustGet(ctx, "nats.url").String() + natsURL := g.Cfg().MustGet(ctx, fmt.Sprintf("nats.%s.url", dsName)).String() if natsURL == "" { // 默认使用本地地址 natsURL = nats.DefaultURL } - // 使用独立的日志上下文,避免使用外部可能被取消的上下文 - logCtx := context.Background() - // 连接选项配置 opts := []nats.Option{ - nats.Name("goframe-nats-client"), - nats.ReconnectWait(2 * time.Second), - nats.MaxReconnects(-1), // 无限重连 + nats.Name(fmt.Sprintf("goframe-nats-client-%s", dsName)), + nats.NoReconnect(), nats.PingInterval(10 * time.Second), nats.MaxPingsOutstanding(5), - nats.ReconnectHandler(func(nc *nats.Conn) { - g.Log().Infof(logCtx, "✅ NATS 重连成功: %s", nc.ConnectedUrl()) - - natsMu.Lock() - defer natsMu.Unlock() - // 重新创建 JetStream 实例 - if newJS, err := jetstream.New(nc); err == nil { - js = newJS - } - }), - nats.DisconnectErrHandler(func(nc *nats.Conn, err error) { - g.Log().Warningf(logCtx, "⚠️ NATS 连接断开: %v, 准备重连...", err) - }), nats.ClosedHandler(func(nc *nats.Conn) { - g.Log().Infof(logCtx, "NATS 连接已关闭: %s", nc.ConnectedUrl()) + g.Log().Infof(ctx, "NATS [%s] 连接已关闭: %s", dsName, nc.ConnectedUrl()) }), nats.ErrorHandler(func(nc *nats.Conn, sub *nats.Subscription, err error) { - g.Log().Errorf(logCtx, "NATS 错误: %v", err) + g.Log().Errorf(ctx, "❌ NATS [%s] 错误: %v", dsName, err) }), } - var err error - nc, err = nats.Connect(natsURL, opts...) + newConn, err := nats.Connect(natsURL, opts...) if err != nil { - return fmt.Errorf("NATS 连接失败: %w", err) + g.Log().Errorf(ctx, "❌ NATS [%s] 连接失败: %v", dsName, err) + return err } // 等待连接就绪 - if nc.Status() != nats.CONNECTED { + if newConn.Status() != nats.CONNECTED { select { case <-time.After(5 * time.Second): // 连接超时,清理资源 - if nc != nil { - nc.Close() - } + newConn.Close() + g.Log().Errorf(ctx, "❌ NATS [%s] 连接超时", dsName) return fmt.Errorf("NATS 连接超时") - case <-nc.StatusChanged(nats.CONNECTED): + case <-newConn.StatusChanged(nats.CONNECTED): // 连接成功 + g.Log().Infof(ctx, "✅ NATS [%s] 连接成功: %s", dsName, newConn.ConnectedUrl()) case <-ctx.Done(): // 外部上下文被取消,清理资源 - if nc != nil { - nc.Close() - } + newConn.Close() + g.Log().Errorf(ctx, "NATS [%s] 连接被取消: %v", dsName, ctx.Err()) return fmt.Errorf("NATS 连接被取消: %w", ctx.Err()) } } // 创建 JetStream 实例 - js, err = jetstream.New(nc) + newJS, err := newConn.JetStream(nats.MaxWait(10 * time.Second)) if err != nil { // 创建 JetStream 失败,清理连接 - if nc != nil { - nc.Close() - } - return fmt.Errorf("创建 JetStream 失败: %w", err) + newConn.Close() + g.Log().Errorf(ctx, "❌ NATS [%s] 创建 JetStream 失败: %v", dsName, err) + return err } - g.Log().Infof(ctx, "✅ NATS 连接成功: %s", nc.ConnectedUrl()) + // 保存连接和 JetStream 上下文 + natsConns[dsName] = newConn + natsJS[dsName] = newJS + return nil } // natsPing 检测 NATS 连接状态 -func natsPing() bool { - natsMu.RLock() - defer natsMu.RUnlock() - - if nc == nil || nc.IsClosed() { - return false +func natsPing(ctx context.Context, name string) bool { + // 确定数据源名称 + dsName := "default" + if !g.IsEmpty(name) { + dsName = name } - // 使用 NATS 的状态检查 - if nc.Status() != nats.CONNECTED { + muNats.RLock() + defer muNats.RUnlock() + + nc, exists := natsConns[dsName] + if !exists || nc == nil || nc.IsClosed() || nc.Status() != nats.CONNECTED { + g.Log().Errorf(ctx, "❌ NATS [%s] 连接已关闭或不可用", dsName) return false } - + g.Log().Infof(ctx, "📊 NATS [%s] 连接正常: %s", dsName, nc.ConnectedUrl()) return true } -// natsReconnect 重连 NATS -func natsReconnect(ctx context.Context) error { - if err := natsConnect(ctx); err != nil { - return fmt.Errorf("nats重连失败: %w", err) - } - return nil -} - // natsClose 关闭 NATS 连接 -func natsClose(ctx context.Context) error { - natsMu.Lock() - defer natsMu.Unlock() - - if nc == nil || nc.IsClosed() { - return nil // 连接已经关闭或不存在 +func natsClose(ctx context.Context, name string) error { + // 确定数据源名称 + dsName := "default" + if !g.IsEmpty(name) { + dsName = name } - nc.Close() - g.Log().Infof(ctx, "✅ NATS 连接已关闭") + + muNats.Lock() + defer muNats.Unlock() + + if nc, exists := natsConns[dsName]; exists && nc != nil && !nc.IsClosed() { + nc.Close() + } + delete(natsConns, dsName) + delete(natsJS, dsName) + + g.Log().Infof(ctx, "✅ NATS [%s] 连接已关闭", dsName) return nil } + +// getNatsConn 获取 NATS 连接(内部使用) +func getNatsConn(name string) *nats.Conn { + dsName := "default" + if !g.IsEmpty(name) { + dsName = name + } + return natsConns[dsName] +} + +// getNatsJS 获取 JetStream 上下文(内部使用) +func getNatsJS(name string) nats.JetStreamContext { + dsName := "default" + if !g.IsEmpty(name) { + dsName = name + } + return natsJS[dsName] +} diff --git a/message/connection_rabbitmq.go b/message/connection_rabbitmq.go index c98d2fe..8b8d47a 100644 --- a/message/connection_rabbitmq.go +++ b/message/connection_rabbitmq.go @@ -7,103 +7,158 @@ import ( "github.com/gogf/gf/v2/util/gconv" amqp "github.com/rabbitmq/amqp091-go" "sync" - "time" ) var ( - conn *amqp.Connection - channel *amqp.Channel - rabbitmqMu sync.RWMutex + muRabbitMQ sync.RWMutex + rabbitmqConns map[string]*amqp.Connection + rabbitmqChannels map[string]*amqp.Channel ) -// config RabbitMQ 配置 -type config struct { - Host string - Port int - Username string - Password string - VHost string +func init() { + rabbitmqConns = make(map[string]*amqp.Connection) + rabbitmqChannels = make(map[string]*amqp.Channel) } -func rabbitmqConnect(ctx context.Context) error { - rabbitmqMu.Lock() - defer rabbitmqMu.Unlock() - -LOOP: - cfg := &config{ - Host: g.Cfg().MustGet(ctx, "rabbitmq.host").String(), - Port: g.Cfg().MustGet(ctx, "rabbitmq.port").Int(), - Username: g.Cfg().MustGet(ctx, "rabbitmq.username").String(), - Password: g.Cfg().MustGet(ctx, "rabbitmq.password").String(), - VHost: g.Cfg().MustGet(ctx, "rabbitmq.vhost", "/").String(), +// rabbitmqConnect 建立 RabbitMQ 连接 +func rabbitmqConnect(ctx context.Context, name string) error { + if g.Cfg().MustGet(ctx, "rabbitmq").IsEmpty() { + g.Log().Errorf(ctx, "❌ RabbitMQ 配置不存在") + return fmt.Errorf("RabbitMQ Configuration does not exist") + } + // 确定数据源名称 + dsName := "default" + if !g.IsEmpty(name) { + dsName = name } - url := "amqp://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + gconv.String(cfg.Port) + "/" + cfg.VHost + g.Log().Infof(ctx, "🔔 RabbitMQ [%s] 开始创建连接", dsName) + muRabbitMQ.Lock() + defer muRabbitMQ.Unlock() - var err error - conn, err = amqp.Dial(url) + // 安全地关闭旧连接(仅针对该数据源) + if oldConn, exists := rabbitmqConns[dsName]; exists && oldConn != nil && !oldConn.IsClosed() { + oldConn.Close() + } + if oldChannel, exists := rabbitmqChannels[dsName]; exists && oldChannel != nil && !oldChannel.IsClosed() { + oldChannel.Close() + } + delete(rabbitmqConns, dsName) + delete(rabbitmqChannels, dsName) + + // 从配置文件读取 RabbitMQ 配置 + host := g.Cfg().MustGet(ctx, fmt.Sprintf("rabbitmq.%s.host", dsName)).String() + port := g.Cfg().MustGet(ctx, fmt.Sprintf("rabbitmq.%s.port", dsName)).Int() + username := g.Cfg().MustGet(ctx, fmt.Sprintf("rabbitmq.%s.username", dsName)).String() + password := g.Cfg().MustGet(ctx, fmt.Sprintf("rabbitmq.%s.password", dsName)).String() + vHost := g.Cfg().MustGet(ctx, fmt.Sprintf("rabbitmq.%s.vhost", dsName), "/").String() + if g.IsEmpty(host) { + return fmt.Errorf("❌ RabbitMQ 配置错误: host 不能为空 (数据源: %s)", dsName) + } + if g.IsEmpty(port) { + return fmt.Errorf("❌ RabbitMQ 配置错误: port 不能为空 (数据源: %s)", dsName) + } + if g.IsEmpty(username) { + return fmt.Errorf("❌ RabbitMQ 配置错误: username 不能为空 (数据源: %s)", dsName) + } + if g.IsEmpty(password) { + return fmt.Errorf("❌ RabbitMQ 配置错误: password 不能为空 (数据源: %s)", dsName) + } + // 构建连接 URL + url := "amqp://" + username + ":" + password + "@" + host + ":" + gconv.String(port) + "/" + vHost + + // 创建连接 + newConn, err := amqp.Dial(url) if err != nil { - g.Log().Errorf(ctx, "重连失败: %v", err) - - time.Sleep(2 * time.Second) - goto LOOP + g.Log().Errorf(ctx, "❌ RabbitMQ [%s] 连接失败: %v", dsName, err) + return err } - channel, err = conn.Channel() + // 创建 Channel + newChannel, err := newConn.Channel() if err != nil { - g.Log().Errorf(ctx, "创建 Channel 失败: %v", err) - - time.Sleep(2 * time.Second) - goto LOOP + g.Log().Errorf(ctx, "❌ RabbitMQ [%s] 创建 Channel 失败: %v", dsName, err) + newConn.Close() + return err } - g.Log().Info(ctx, "RabbitMQ 重连成功") - return nil -} + // 保存连接和 Channel + rabbitmqConns[dsName] = newConn + rabbitmqChannels[dsName] = newChannel -// rabbitmqReconnect 重新连接 -func rabbitmqReconnect(ctx context.Context) error { - if err := rabbitmqConnect(ctx); err != nil { - return fmt.Errorf("nats重连失败: %w", err) - } + g.Log().Infof(ctx, "✅ RabbitMQ [%s] 连接成功", dsName) return nil } // rabbitmqPing 检测 RabbitMQ 连接状态 -func rabbitmqPing() bool { - rabbitmqMu.RLock() - defer rabbitmqMu.RUnlock() +func rabbitmqPing(ctx context.Context, name string) bool { + // 确定数据源名称 + dsName := "default" + if !g.IsEmpty(name) { + dsName = name + } - if conn == nil || conn.IsClosed() { + muRabbitMQ.RLock() + defer muRabbitMQ.RUnlock() + + conn, exists := rabbitmqConns[dsName] + channel, channelExists := rabbitmqChannels[dsName] + if !exists || conn == nil || conn.IsClosed() || !channelExists || channel == nil || channel.IsClosed() { + g.Log().Errorf(ctx, "❌ RabbitMQ [%s] 连接已关闭或不可用", dsName) return false } + g.Log().Infof(ctx, "📊 RabbitMQ [%s] 连接正常", dsName) return true } -// rabbitmqClose 关闭连接 -func rabbitmqClose(ctx context.Context) error { - rabbitmqMu.Lock() - defer rabbitmqMu.Unlock() +// rabbitmqClose 关闭 RabbitMQ 连接 +func rabbitmqClose(ctx context.Context, name string) error { + // 确定数据源名称 + dsName := "default" + if !g.IsEmpty(name) { + dsName = name + } + + muRabbitMQ.Lock() + defer muRabbitMQ.Unlock() var lastErr error - if channel != nil { + if channel, exists := rabbitmqChannels[dsName]; exists && channel != nil && !channel.IsClosed() { if err := channel.Close(); err != nil { - g.Log().Errorf(ctx, "关闭 RabbitMQ Channel 失败: %v", err) + g.Log().Errorf(ctx, "❌ RabbitMQ [%s] 关闭 Channel 失败: %v", dsName, err) lastErr = err } - channel = nil } + delete(rabbitmqChannels, dsName) - if conn != nil { + if conn, exists := rabbitmqConns[dsName]; exists && conn != nil && !conn.IsClosed() { if err := conn.Close(); err != nil { - g.Log().Errorf(ctx, "关闭 RabbitMQ 连接失败: %v", err) + g.Log().Errorf(ctx, "❌ RabbitMQ [%s] 关闭连接失败: %v", dsName, err) lastErr = err } - conn = nil } + delete(rabbitmqConns, dsName) - g.Log().Info(ctx, "RabbitMQ 连接已关闭") + g.Log().Infof(ctx, "✅ RabbitMQ [%s] 连接已关闭", dsName) return lastErr } + +// getRabbitMQConn 获取 RabbitMQ 连接(内部使用) +func getRabbitMQConn(name string) *amqp.Connection { + dsName := "default" + if !g.IsEmpty(name) { + dsName = name + } + return rabbitmqConns[dsName] +} + +// getRabbitMQChannel 获取 RabbitMQ Channel(内部使用) +func getRabbitMQChannel(name string) *amqp.Channel { + dsName := "default" + if !g.IsEmpty(name) { + dsName = name + } + return rabbitmqChannels[dsName] +} diff --git a/message/connection_redis.go b/message/connection_redis.go index 465238f..528eca0 100644 --- a/message/connection_redis.go +++ b/message/connection_redis.go @@ -1,7 +1,6 @@ // ============================================================================= -// Redis 数据源连接管理 -// 负责 Redis 数据源的连接、重连、健康检查和优雅关闭 -// 支持多数据源和无限重连 +// Redis 连接管理 +// 负责 Redis 的连接、重连、健康检查和优雅关闭 // ============================================================================= package message @@ -9,477 +8,191 @@ package message import ( "context" "fmt" - "os" - "os/signal" "sync" - "syscall" "time" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/os/glog" - "github.com/gogf/gf/v2/util/gconv" ) -// ============================================================================= -// Redis 数据源配置结构 -// ============================================================================= +var ( + muRedis sync.RWMutex + redisConns map[string]*gredis.Redis + redisConfigs map[string]*gredis.Config +) -type redisDataSourceConfig struct { - name string // 数据源名称 - address string // Redis 地址,如: 127.0.0.1:6379 - db int // 数据库编号 - pass string // 密码 - maxRetries int // 最大重试次数,-1 表示无限重试 - retryInterval time.Duration // 重试间隔 +func init() { + redisConns = make(map[string]*gredis.Redis) + redisConfigs = make(map[string]*gredis.Config) } -// ============================================================================= -// Redis 数据源接口 -// ============================================================================= - -type redisDataSource interface { - name() string - getClient() *gredis.Redis - getIsConnected() bool - redisConnect(ctx context.Context) error - redisReconnect(ctx context.Context) error - redisClose(ctx context.Context) error - redisPing(ctx context.Context) bool -} - -// ============================================================================= -// Redis 数据源实现 -// ============================================================================= - -type baseRedisDataSource struct { - config *redisDataSourceConfig - client *gredis.Redis - isConnected bool - mu sync.RWMutex - lastError error - lastErrorTime time.Time - reconnectMu sync.Mutex -} - -func newBaseRedisDataSource(config *redisDataSourceConfig) *baseRedisDataSource { - return &baseRedisDataSource{ - config: config, - isConnected: false, +// redisConnect 建立 Redis 连接 +// name: 数据源名称,如果为空则使用默认数据源 +func redisConnect(ctx context.Context, name string) error { + if g.Cfg().MustGet(ctx, "redis").IsEmpty() { + g.Log().Errorf(ctx, "❌ Redis 配置不存在") + return fmt.Errorf("redis Configuration does not exist") } -} - -func (d *baseRedisDataSource) name() string { - return d.config.name -} - -func (d *baseRedisDataSource) getClient() *gredis.Redis { - d.mu.RLock() - defer d.mu.RUnlock() - return d.client -} - -func (d *baseRedisDataSource) getIsConnected() bool { - d.mu.RLock() - defer d.mu.RUnlock() - return d.isConnected && d.client != nil -} - -func (d *baseRedisDataSource) redisConnect(ctx context.Context) error { - // 使用互斥锁防止并发重连 - d.reconnectMu.Lock() - defer d.reconnectMu.Unlock() - - d.mu.Lock() - if d.client != nil { - d.client.Close(ctx) + // 确定数据源名称 + dsName := "default" + if !g.IsEmpty(name) { + dsName = name } - d.mu.Unlock() + g.Log().Infof(ctx, "🔔 Redis [%s] 开始创建连接", dsName) + muRedis.Lock() + defer muRedis.Unlock() + + // 安全地关闭旧连接(仅针对该数据源) + if oldRedis, exists := redisConns[dsName]; exists && oldRedis != nil { + oldRedis.Close(ctx) + delete(redisConns, dsName) + } + + // 从配置文件读取 Redis 配置 + redisAddr := g.Cfg().MustGet(ctx, fmt.Sprintf("redis.%s.address", dsName)).String() + if g.IsEmpty(redisAddr) { + g.Log().Errorf(ctx, "❌ Redis 配置错误: address 不能为空 (数据源: %s)", dsName) + return fmt.Errorf("❌ Redis 配置错误: address 不能为空 (数据源: %s)", dsName) + } + redisDB := g.Cfg().MustGet(ctx, fmt.Sprintf("redis.%s.db", dsName)).Int() + if redisDB < 0 || redisDB > 15 { + g.Log().Errorf(ctx, "❌ Redis 配置错误: db 必须在 0-15 之间 (当前值: %d)", redisDB) + return fmt.Errorf("❌ Redis 配置错误: db 必须在 0-15 之间 (当前值: %d)", redisDB) + } + idleTimeout := g.Cfg().MustGet(ctx, fmt.Sprintf("redis.%s.idleTimeout", dsName)).String() + redisIdleTimeout, err := time.ParseDuration(idleTimeout) + if err != nil { + g.Log().Errorf(ctx, "❌ Redis idleTimeout 格式错误: %v", err) + return err + } + maxConnLifetime := g.Cfg().MustGet(ctx, fmt.Sprintf("redis.%s.maxConnLifetime", dsName)).String() + redisMaxConnLifetime, err := time.ParseDuration(maxConnLifetime) + if err != nil { + g.Log().Errorf(ctx, "❌ Redis maxConnLifetime 格式错误: %v", err) + return err + } + waitTimeout := g.Cfg().MustGet(ctx, fmt.Sprintf("redis.%s.waitTimeout", dsName)).String() + redisWaitTimeout, err := time.ParseDuration(waitTimeout) + if err != nil { + g.Log().Errorf(ctx, "❌ Redis waitTimeout 格式错误: %v", err) + return err + } + dialTimeout := g.Cfg().MustGet(ctx, fmt.Sprintf("redis.%s.dialTimeout", dsName)).String() + redisDialTimeout, err := time.ParseDuration(dialTimeout) + if err != nil { + g.Log().Errorf(ctx, "❌ Redis dialTimeout 格式错误: %v", err) + return err + } + readTimeout := g.Cfg().MustGet(ctx, fmt.Sprintf("redis.%s.readTimeout", dsName)).String() + redisReadTimeout, err := time.ParseDuration(readTimeout) + if err != nil { + g.Log().Errorf(ctx, "❌ Redis readTimeout 格式错误: %v", err) + return err + } + writeTimeout := g.Cfg().MustGet(ctx, fmt.Sprintf("redis.%s.writeTimeout", dsName)).String() + redisWriteTimeout, err := time.ParseDuration(writeTimeout) + if err != nil { + g.Log().Errorf(ctx, "❌ Redis writeTimeout 格式错误: %v", err) + return err + } + maxActive := g.Cfg().MustGet(ctx, fmt.Sprintf("redis.%s.maxActive", dsName)).Int() + if g.IsEmpty(maxActive) { + g.Log().Errorf(ctx, "❌ Redis maxActive 配置错误: %v", maxActive) + return fmt.Errorf("❌ Redis maxActive 配置错误") + } // 构建 GoFrame Redis 配置 redisConfig := &gredis.Config{ - Address: d.config.address, - Db: d.config.db, - Pass: d.config.pass, + Address: redisAddr, + Db: redisDB, + IdleTimeout: redisIdleTimeout, + MaxConnLifetime: redisMaxConnLifetime, + WaitTimeout: redisWaitTimeout, + DialTimeout: redisDialTimeout, + ReadTimeout: redisReadTimeout, + WriteTimeout: redisWriteTimeout, + MaxActive: maxActive, } + redisConfigs[dsName] = redisConfig // 使用 GoFrame 的 Redis 连接 - redisObj, err := gredis.New(redisConfig) + newRedis, err := gredis.New(redisConfig) if err != nil { - d.mu.Lock() - d.isConnected = false - d.lastError = err - d.lastErrorTime = time.Now() - d.mu.Unlock() - return fmt.Errorf("datasource [%s] connection failed: %w", d.config.name, err) + g.Log().Errorf(ctx, "❌ Redis [%s] 连接失败: %v", dsName, err) + return err + } + // 测试连接(直接调用避免死锁) + _, err = newRedis.Do(ctx, "PING") + if err != nil { + g.Log().Errorf(ctx, "❌ Redis [%s] 连接失败: ping 失败 - %v", dsName, err) + _ = newRedis.Close(ctx) + return err } - d.mu.Lock() - d.client = redisObj - d.mu.Unlock() - - // 测试连接 - if !d.redisPing(ctx) { - d.mu.Lock() - d.isConnected = false - d.lastError = err - d.lastErrorTime = time.Now() - d.mu.Unlock() - return fmt.Errorf("datasource [%s] ping failed: %w", d.config.name, err) - } - - d.mu.Lock() - d.isConnected = true - d.lastError = nil - d.mu.Unlock() - glog.Infof(ctx, "✅ datasource [%s] connected successfully", d.config.name) + redisConns[dsName] = newRedis + g.Log().Infof(ctx, "✅ Redis [%s] 连接成功: %s (DB: %d)", dsName, redisAddr, redisDB) return nil } -func (d *baseRedisDataSource) redisReconnect(ctx context.Context) error { - glog.Infof(ctx, "🔄 reconnecting datasource [%s]", d.config.name) - return d.redisConnect(ctx) -} - -func (d *baseRedisDataSource) redisClose(ctx context.Context) error { - d.mu.Lock() - defer d.mu.Unlock() - - if d.client != nil { - if err := d.client.Close(ctx); err != nil { - return fmt.Errorf("datasource [%s] close failed: %w", d.config.name, err) - } +// redisPing 检测 Redis 连接状态(带超时保护) +func redisPing(ctx context.Context, name string) bool { + // 确定数据源名称 + dsName := "default" + if !g.IsEmpty(name) { + dsName = name } - d.isConnected = false - glog.Infof(ctx, "datasource [%s] closed", d.config.name) - return nil -} + muRedis.RLock() + defer muRedis.RUnlock() -func (d *baseRedisDataSource) redisPing(ctx context.Context) bool { - d.mu.RLock() - client := d.client - d.mu.RUnlock() - - if client == nil { + rc, exists := redisConns[dsName] + if !exists || rc == nil { + g.Log().Errorf(ctx, "❌ Redis [%s] 连接未建立", dsName) return false } - _, err := client.Do(ctx, "PING") + // 创建带超时的子上下文,避免死锁 + timeoutCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + _, err := rc.Do(timeoutCtx, "PING") if err != nil { + g.Log().Errorf(ctx, "❌ Redis [%s] ping 失败: %v", dsName, err) return false } + + g.Log().Infof(ctx, "📊 Redis [%s] 连接正常", dsName) return true } -// ============================================================================= -// Redis 多数据源管理器 -// ============================================================================= - -type redisDataSourceManager struct { - sources map[string]redisDataSource - mu sync.RWMutex - ctx context.Context - cancel context.CancelFunc - started bool - maxRetries int - reconnectCh chan string -} - -var ( - globalRedisManager *redisDataSourceManager - redisManagerOnce sync.Once -) - -// getRedisManager 获取全局 Redis 管理器 -func getRedisManager() *redisDataSourceManager { - redisManagerOnce.Do(func() { - ctx, cancel := context.WithCancel(context.Background()) - globalRedisManager = &redisDataSourceManager{ - sources: make(map[string]redisDataSource), - ctx: ctx, - cancel: cancel, - started: false, - maxRetries: -1, // 默认无限重试 - reconnectCh: make(chan string, 100), - } - }) - return globalRedisManager -} - -// registerDataSource 注册 Redis 数据源 -func (m *redisDataSourceManager) registerDataSource(config *redisDataSourceConfig) error { - m.mu.Lock() - defer m.mu.Unlock() - - if _, exists := m.sources[config.name]; exists { - return fmt.Errorf("datasource [%s] already exists", config.name) +// redisClose 关闭 Redis 连接 +func redisClose(ctx context.Context, name string) error { + // 确定数据源名称 + dsName := "default" + if !g.IsEmpty(name) { + dsName = name } - source := newBaseRedisDataSource(config) - m.sources[config.name] = source + muRedis.Lock() + defer muRedis.Unlock() + + if rc, exists := redisConns[dsName]; exists && rc != nil { + if err := rc.Close(ctx); err != nil { + g.Log().Errorf(ctx, "❌ Redis [%s] 关闭失败: %v", dsName, err) + return err + } + delete(redisConns, dsName) + } + + g.Log().Infof(ctx, "✅ Redis [%s] 连接已关闭", dsName) return nil } -// getDataSource 获取 Redis 数据源 -func (m *redisDataSourceManager) getDataSource(name string) (redisDataSource, error) { - m.mu.RLock() - defer m.mu.RUnlock() - - source, exists := m.sources[name] - if !exists { - return nil, fmt.Errorf("datasource [%s] not found", name) +// getRedisConn 获取 Redis 连接(内部使用) +func getRedisConn(name string) *gredis.Redis { + dsName := "default" + if !g.IsEmpty(name) { + dsName = name } - return source, nil -} - -// getAllDataSourceNames 获取所有 Redis 数据源名称 -func (m *redisDataSourceManager) getAllDataSourceNames() []string { - m.mu.RLock() - defer m.mu.RUnlock() - - names := make([]string, 0, len(m.sources)) - for name := range m.sources { - names = append(names, name) - } - return names -} - -// initializeFromConfig 从配置初始化 Redis 数据源 -func (m *redisDataSourceManager) initializeFromConfig(ctx context.Context) error { - var firstErr error - - // 获取 redis 配置下的所有子键 - redisConfig := g.Cfg().MustGet(ctx, "redis") - if redisConfig.IsNil() { - glog.Warningf(ctx, "no redis configuration found in config.yml") - return nil - } - - // 将配置转换为 map - configMap := redisConfig.Map() - if configMap == nil { - glog.Warningf(ctx, "redis configuration is not a map") - return nil - } - - // 遍历所有 redis 子配置 - for name, subConfig := range configMap { - // 跳过非对象类型的配置 - subMap, ok := subConfig.(map[string]interface{}) - if !ok { - continue - } - - // 检查是否有 address 配置 - address, hasAddress := subMap["address"] - if !hasAddress || gconv.String(address) == "" { - continue - } - - // 构建数据源配置 - config := &redisDataSourceConfig{ - name: name, - address: gconv.String(address), - db: gconv.Int(subMap["db"]), - pass: gconv.String(subMap["pass"]), - maxRetries: gconv.Int(subMap["maxRetries"]), - retryInterval: gconv.Duration(subMap["retryInterval"]), - } - - // 设置默认值 - if config.maxRetries == 0 { - config.maxRetries = -1 // 默认无限重试 - } - if config.retryInterval == 0 { - config.retryInterval = 5 * time.Second - } - - // 注册数据源 - if err := m.registerDataSource(config); err != nil { - glog.Errorf(ctx, "failed to register datasource [%s]: %v", name, err) - if firstErr == nil { - firstErr = err - } - continue - } - - // 连接数据源 - source, _ := m.getDataSource(name) - if err := source.redisConnect(ctx); err != nil { - glog.Errorf(ctx, "failed to initialize datasource [%s]: %v", name, err) - if firstErr == nil { - firstErr = err - } - } - } - - return firstErr -} - -// startHealthCheck 启动健康检查 -func (m *redisDataSourceManager) startHealthCheck() { - if m.started { - return - } - m.started = true - - // 启动健康检查循环 - go m.healthCheckLoop() - - // 启动重连处理循环 - go m.reconnectLoop() -} - -// healthCheckLoop 健康检查循环 -func (m *redisDataSourceManager) healthCheckLoop() { - ticker := time.NewTicker(10 * time.Second) - defer ticker.Stop() - - for { - select { - case <-m.ctx.Done(): - return - case <-ticker.C: - m.checkConnections() - } - } -} - -// reconnectLoop 重连处理循环 -func (m *redisDataSourceManager) reconnectLoop() { - reconnectCounts := make(map[string]int) - - for { - select { - case <-m.ctx.Done(): - return - case name := <-m.reconnectCh: - go func(dsName string) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - source, err := m.getDataSource(dsName) - if err != nil { - glog.Errorf(ctx, "datasource [%s] not found for reconnect", dsName) - return - } - - if err := source.redisReconnect(ctx); err != nil { - glog.Errorf(ctx, "datasource [%s] reconnect failed: %v", dsName, err) - - // 记录重连次数 - reconnectCounts[dsName]++ - - // 检查重连次数限制 - if m.maxRetries > 0 && reconnectCounts[dsName] > m.maxRetries { - glog.Errorf(ctx, "datasource [%s] reconnect count %d exceeds limit %d, stopping auto-reconnect", - dsName, reconnectCounts[dsName], m.maxRetries) - return - } - - // 延迟后重新放入重连队列 - time.Sleep(5 * time.Second) - select { - case m.reconnectCh <- dsName: - default: - // 通道已满,丢弃通知 - } - } else { - // 重连成功,重置计数器 - reconnectCounts[dsName] = 0 - } - }(name) - } - } -} - -// checkConnections 检查连接状态 -func (m *redisDataSourceManager) checkConnections() { - m.mu.RLock() - defer m.mu.RUnlock() - - for name, source := range m.sources { - if !source.getIsConnected() { - glog.Warningf(context.Background(), "datasource [%s] disconnected, queued for reconnect", name) - - // 发送到重连队列 - select { - case m.reconnectCh <- name: - default: - // 通道已满,丢弃通知 - } - } - } -} - -// closeAll 关闭所有 Redis 数据源 -func (m *redisDataSourceManager) closeAll(ctx context.Context) error { - m.cancel() - - m.mu.RLock() - defer m.mu.RUnlock() - - var lastErr error - for name, source := range m.sources { - if err := source.redisClose(ctx); err != nil { - glog.Errorf(ctx, "failed to close datasource [%s]: %v", name, err) - lastErr = err - } - } - return lastErr -} - -// ============================================================================= -// 全局初始化 -// ============================================================================= - -var ( - redisManager = getRedisManager() -) - -// init 初始化 Redis 数据源 -func init() { - ctx := context.Background() - - // 从配置初始化多数据源 - if err := redisManager.initializeFromConfig(ctx); err != nil { - glog.Errorf(ctx, "❌ Failed to initialize Redis datasources: %v", err) - } else { - glog.Infof(ctx, "✅ Redis datasources initialized: %v", redisManager.getAllDataSourceNames()) - } - - // 启动健康检查 - redisManager.startHealthCheck() - - // 设置优雅关闭 - setupGracefulShutdown() -} - -// setupGracefulShutdown 设置优雅关闭 -func setupGracefulShutdown() { - go func() { - sigCh := make(chan os.Signal, 1) - signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - <-sigCh - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - glog.Info(ctx, "🔄 Shutting down Redis connections...") - if err := redisManager.closeAll(ctx); err != nil { - glog.Errorf(ctx, "❌ Failed to close Redis connections: %v", err) - } else { - glog.Info(ctx, "✅ Redis connections closed successfully") - } - }() -} - -// ============================================================================= -// 私有辅助函数 -// ============================================================================= - -// getDefaultDataSource 获取默认数据源 -func getDefaultDataSource() (redisDataSource, error) { - return redisManager.getDataSource("default") + return redisConns[dsName] } diff --git a/message/msg_interfaces.go b/message/msg_interfaces.go index 125cf81..db1c86d 100644 --- a/message/msg_interfaces.go +++ b/message/msg_interfaces.go @@ -6,6 +6,10 @@ type messagePublishConfig interface { GetPublishMsgType() } +type messagePublishDelayConfig interface { + GetPublishDelayMsgType() +} + type messageSubscribeConfig interface { GetSubscribeMsgType() } @@ -15,12 +19,14 @@ type messageSubscribeConfig interface { type messageUtil interface { // Publish 发布消息 Publish(ctx context.Context, msg messagePublishConfig) error + // PublishDelay 发布延迟消息 + PublishDelay(ctx context.Context, msg messagePublishDelayConfig) error // Subscribe 订阅消息 Subscribe(ctx context.Context, msg messageSubscribeConfig) error // Ping 检测连接状态 - ping(ctx context.Context) bool - // Reconnect 重连 - reconnect(ctx context.Context) error + Ping(ctx context.Context) bool + // Connect 连接 + Connect(ctx context.Context) error // Close 关闭连接 - close(ctx context.Context) error + Close(ctx context.Context) error } diff --git a/message/msg_plugin_manager.go b/message/msg_plugin_manager.go index 84c79eb..8340a8a 100644 --- a/message/msg_plugin_manager.go +++ b/message/msg_plugin_manager.go @@ -3,10 +3,10 @@ package message import ( "context" "fmt" - "sync" "time" "github.com/gogf/gf/v2/frame/g" + "sync" ) // MessageType 消息队列类型 @@ -32,7 +32,6 @@ type pluginManager struct { var ( defaultPluginManager = newPluginManager() - // 不再支持默认插件类型,必须显式指定类型 ) // newPluginManager 创建插件管理器 @@ -42,63 +41,6 @@ func newPluginManager() *pluginManager { } } -// RegisterPlugin 注册消息队列插件 -// 所有插件必须通过此方法注册,自动进行连接检测 -// 只有连接成功的插件才会被注册,连接失败的插件不会被注册 -// 异步无限重连,只有连接成功了才注册 -func registerPlugin(msgType messageType, factory configFactory) error { - if factory == nil { - return fmt.Errorf("factory cannot be nil") - } - - // 创建实例 - instance := factory() - ctx := context.Background() - - // 开启异步连接,无限重连直到成功 - go func() { - retryInterval := 2 * time.Second - maxInterval := 30 * time.Second - - for { - select { - case <-ctx.Done(): - g.Log().Errorf(ctx, "❌ [%s] 注册被取消", msgType) - return - default: - // 尝试连接(使用Reconnect方法) - if err := instance.reconnect(ctx); err == nil { - // 连接成功,注册插件 - if err := defaultPluginManager.register(msgType, instance); err != nil { - g.Log().Errorf(ctx, "❌ [%s] 注册插件失败: %v", msgType, err) - instance.close(ctx) - } else { - g.Log().Infof(ctx, "✅ [%s] 插件注册成功", msgType) - } - return - } - - // 连接失败,记录日志并等待重试 - g.Log().Warningf(ctx, "⚠️ [%s] 连接失败,%v 后重试...", msgType, retryInterval) - - select { - case <-time.After(retryInterval): - // 增加重试间隔,但不超过最大值 - retryInterval *= 2 - if retryInterval > maxInterval { - retryInterval = maxInterval - } - case <-ctx.Done(): - g.Log().Errorf(ctx, "❌ [%s] 注册被取消", msgType) - return - } - } - } - }() - - return nil -} - // register 注册插件(内部方法) func (m *pluginManager) register(msgType messageType, instance messageUtil) error { m.mu.Lock() @@ -107,27 +49,66 @@ func (m *pluginManager) register(msgType messageType, instance messageUtil) erro return nil } -// GetMsgPlugin 获取消息队列插件 -func GetMsgPlugin(msgType messageType) (messageUtil, error) { - defaultPluginManager.mu.RLock() - instance, ok := defaultPluginManager.instances[msgType] - defaultPluginManager.mu.RUnlock() - - if !ok { - return nil, fmt.Errorf("unsupported message type: %s", msgType) +// RegisterPlugin 注册消息队列插件 +// 所有插件必须通过此方法注册,自动进行连接检测 +// 只有连接成功的插件才会被注册,连接失败的插件不会被注册 +// 异步无限重连,只有连接成功了才注册 +// name: 数据源名称,用于标识不同的连接实例 +func RegisterPlugin(ctx context.Context, name string, msgType messageType, factory configFactory) error { + if factory == nil { + g.Log().Errorf(ctx, "❌ factory cannot be nil") + return fmt.Errorf("factory cannot be nil") } - - return instance, nil + // 开启异步连接,无限重试直到成功 + go func() { + // 创建实例 + instance := factory() + // 创建通知 channel + pluginKey := fmt.Sprintf("%s-%s", msgType, name) + if !instance.Ping(ctx) { + // 使用统一的重连函数 + if err := commonConnect(ctx, msgType, name, func(ctx context.Context) error { + return instance.Connect(ctx) + }, func(ctx context.Context) error { + return instance.Close(ctx) + }); err != nil { + g.Log().Errorf(ctx, "❌ [%s][%s] 连接失败: %v", msgType, name, err) + return + } + } + // 连接成功,注册插件 + defaultPluginManager.mu.Lock() + defaultPluginManager.instances[messageType(pluginKey)] = instance + defaultPluginManager.mu.Unlock() + g.Log().Infof(ctx, "✅ [%s][%s] 插件注册成功", msgType, name) + }() + return nil } -// GetSupportedTypes 获取所有已注册的插件类型 -func GetSupportedTypes() []messageType { - defaultPluginManager.mu.RLock() - defer defaultPluginManager.mu.RUnlock() - - types := make([]messageType, 0, len(defaultPluginManager.instances)) - for t := range defaultPluginManager.instances { - types = append(types, t) +// GetMsgPlugin 获取消息队列插件(默认数据源),如果未注册则等待 +func GetMsgPlugin(ctx context.Context, msgType messageType) (messageUtil, error) { + return GetMsgPluginWithName(ctx, msgType, "default") +} + +// GetMsgPluginWithName 获取指定数据源的消息队列插件,如果未注册则等待直到超时 +func GetMsgPluginWithName(ctx context.Context, msgType messageType, name string) (messageUtil, error) { + pluginKey := fmt.Sprintf("%s-%s", msgType, name) + + for { + defaultPluginManager.mu.RLock() + instance, ok := defaultPluginManager.instances[messageType(pluginKey)] + defaultPluginManager.mu.RUnlock() + + if ok { + return instance, nil + } + + // 未注册,等待一段时间后重试 + select { + case <-ctx.Done(): + return nil, fmt.Errorf("wait for plugin ready canceled: %s with datasource: %s", msgType, name) + default: + time.Sleep(3 * time.Second) + } } - return types } diff --git a/message/nats_msg.go b/message/nats_msg.go index 205ac07..365c171 100644 --- a/message/nats_msg.go +++ b/message/nats_msg.go @@ -6,11 +6,16 @@ import ( "fmt" "github.com/gogf/gf/v2/frame/g" "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" "time" ) type NatsPublishMsgConfig struct { + QueueName string + Durable bool + Data any +} + +type NatsPublishDelayMsgConfig struct { QueueName string Durable bool DelayTime int @@ -19,9 +24,9 @@ type NatsPublishMsgConfig struct { type NatsSubscribeMsgConfig struct { QueueName string + ConsumerName string Durable bool DelayTime int - ConsumerName string AutoAck bool PrefetchCount int HandleFunc func(ctx context.Context, message map[string]interface{}) error @@ -31,32 +36,38 @@ func (*NatsPublishMsgConfig) GetPublishMsgType() { } +func (*NatsPublishDelayMsgConfig) GetPublishDelayMsgType() { + +} + func (*NatsSubscribeMsgConfig) GetSubscribeMsgType() { } +type natsMsg struct { + name string // 数据源名称 +} + func init() { - // 注册 Nats 插件,必须使用 RegisterPlugin 确保连接检测 - registerPlugin(MessageNATS, func() messageUtil { - return &natsMsg{} + // 注册 Nats 插件(默认数据源) + RegisterPlugin(context.Background(), "default", MessageNATS, func() messageUtil { + return &natsMsg{name: "default"} }) } -type natsMsg struct{} - -// Ping 检测 NATS 连接状态 -func (c *natsMsg) ping(_ context.Context) bool { - return natsPing() +// Connect 连接 NATS +func (c *natsMsg) Connect(ctx context.Context) error { + return natsConnect(ctx, c.name) } -// Reconnect 重连 NATS -func (c *natsMsg) reconnect(ctx context.Context) error { - return natsReconnect(ctx) +// Ping 检测 NATS 连接状态 +func (c *natsMsg) Ping(ctx context.Context) bool { + return natsPing(ctx, c.name) } // Close 关闭 NATS 连接 -func (c *natsMsg) close(ctx context.Context) error { - return natsClose(ctx) +func (c *natsMsg) Close(ctx context.Context) error { + return natsClose(ctx, c.name) } // Publish 发布消息 @@ -71,13 +82,31 @@ func (c *natsMsg) Publish(ctx context.Context, msgConfig messagePublishConfig) e if g.IsEmpty(cfg.Data) { return fmt.Errorf("必须提供数据") } + return c.createPublish(ctx, cfg.QueueName, cfg.Durable, 0, cfg.Data) +} + +// PublishDelay 发布延迟消息 +func (c *natsMsg) PublishDelay(ctx context.Context, msgConfig messagePublishDelayConfig) error { + cfg, ok := msgConfig.(*NatsPublishDelayMsgConfig) + if !ok { + return fmt.Errorf("无效的 NATS 配置类型") + } + if g.IsEmpty(cfg.QueueName) { + return fmt.Errorf("必须提供队列名称") + } + if g.IsEmpty(cfg.DelayTime) { + return fmt.Errorf("延迟时间必须大于 0") + } + if g.IsEmpty(cfg.Data) { + return fmt.Errorf("必须提供数据") + } return c.createPublish(ctx, cfg.QueueName, cfg.Durable, cfg.DelayTime, cfg.Data) } // Publish 发布消息 func (c *natsMsg) createPublish(ctx context.Context, subject string, durable bool, delayTime int, data any) error { delayMsg := delayTime > 0 - if err := c.createStreamGroupInternal(ctx, subject, durable, delayMsg); err != nil { + if err := c.createStream(ctx, subject, durable, delayMsg); err != nil { return err } payload, err := json.Marshal(data) @@ -85,96 +114,30 @@ func (c *natsMsg) createPublish(ctx context.Context, subject string, durable boo return fmt.Errorf("序列化数据失败: %w", err) } - msg := &nats.Msg{ - Subject: subject, - Data: payload, - } + m := nats.NewMsg(subject) + m.Data = payload // 所有消息都需要设置数据 if delayMsg { - // 计算目标投递时间 - targetTime := time.Now().Add(time.Duration(delayTime) * time.Second) - delayNs := time.Until(targetTime).Nanoseconds() - if delayNs < 0 { - delayNs = 0 - } - - g.Log().Infof(ctx, "📅 NATS 延迟消息配置: DelayTime=%d秒, TargetTime=%v, DelayNs=%d纳秒(%.2f秒)", - delayTime, targetTime.Format("2006-01-02 15:04:05"), delayNs, float64(delayNs)/float64(time.Second.Nanoseconds())) - - // NATS JetStream 延迟消息使用 Nats-Msg-Delay Header(纳秒数) - msg.Header = nats.Header{ - "Nats-Msg-Delay": []string{fmt.Sprintf("%d", delayNs)}, - } - g.Log().Infof(ctx, "📅 NATS 延迟消息 Header: %v", msg.Header) - - // 获取 Stream 配置验证 - streamName, _ := getStreamInfo(durable, delayMsg) - stream, err := js.Stream(ctx, streamName) - if err == nil { - info, _ := stream.Info(ctx) - g.Log().Infof(ctx, "📅 Stream 配置: AllowMsgSchedules=%v, Storage=%v", - info.Config.AllowMsgSchedules, info.Config.Storage) - if !info.Config.AllowMsgSchedules { - g.Log().Errorf(ctx, "❌ Stream 不支持延迟消息!AllowMsgSchedules=false") - } - } + // 使用 @at 指定具体延迟时间,而不是 @every 重复执行 + futureTime := time.Now().Add(time.Duration(delayTime) * time.Second).Format(time.RFC3339Nano) + m.Header.Set("Nats-Schedule", fmt.Sprintf("@at %s", futureTime)) + m.Subject = subject + ".schedule" + m.Header.Set("Nats-Schedule-Target", subject) + g.Log().Infof(ctx, "📅 NATS 延迟消息配置: DelayTime=%ds, Schedule=@at %s, Header=%s", delayTime, futureTime, m.Header) } // 发布消息到 JetStream - ack, err := js.PublishMsg(ctx, msg) + js := getNatsJS(c.name) + if js == nil { + g.Log().Errorf(ctx, "❌ NATS [%s] JetStream 不存在", c.name) + return fmt.Errorf("NATS JetStream 不存在") + } + ack, err := js.PublishMsg(m) if err != nil { - g.Log().Errorf(ctx, "❌ NATS 发布消息失败: err=%v", err) + g.Log().Errorf(ctx, "❌ NATS 发布消息失败: err=%v, Subject=%s", err, m.Subject) return err } - - g.Log().Infof(ctx, "✅ NATS 发布消息成功: StreamSeq=%d, Domain=%s", ack.Sequence, ack.Domain) - return nil -} - -// createStreamGroup 内部创建消费组 -func (c *natsMsg) createStreamGroupInternal(ctx context.Context, subject string, durable, delayMsg bool) error { - streamName, storage := getStreamInfo(durable, delayMsg) - - // 先检查 Stream 是否存在 - stream, err := js.Stream(ctx, streamName) - if err == nil { - // Stream 已存在,检查配置是否匹配 - info, _ := stream.Info(ctx) - if info.Config.AllowMsgSchedules != delayMsg || info.Config.Storage != storage { - g.Log().Infof(ctx, "🔄 Stream 配置不匹配,正在重新创建: stream=%s, 当前AllowMsgSchedules=%v, 需要%v", - streamName, info.Config.AllowMsgSchedules, delayMsg) - // 删除旧 Stream - if err := js.DeleteStream(ctx, streamName); err != nil { - g.Log().Warningf(ctx, "删除旧 Stream 失败: %v", err) - } - } else { - g.Log().Infof(ctx, "✅ Stream 已存在且配置正确: stream=%s", streamName) - return nil - } - } - - // 构建流配置 - jsConfig := jetstream.StreamConfig{ - Name: streamName, - Subjects: []string{subject}, - AllowMsgSchedules: delayMsg, // 延迟消息核心开关 - Storage: storage, - Discard: jetstream.DiscardOld, // 达到上限删除旧消息 - } - - stream, err = js.CreateStream(ctx, jsConfig) - if err != nil { - return fmt.Errorf("创建任务流失败: %w", err) - } - - // 获取 Stream 信息验证配置 - info, err := stream.Info(ctx) - if err == nil { - g.Log().Infof(ctx, "✅ NATS 队列初始化成功: stream=%s, AllowMsgSchedules=%v, Storage=%v", - streamName, info.Config.AllowMsgSchedules, info.Config.Storage) - } - - g.Log().Infof(ctx, "✅ NATS 队列初始化成功: stream=%s", streamName) + g.Log().Infof(ctx, "✅ NATS 发布消息成功: Stream=%v, StreamSeq=%d", ack.Stream, ack.Sequence) return nil } @@ -196,110 +159,215 @@ func (c *natsMsg) Subscribe(ctx context.Context, msgConfig messageSubscribeConfi if g.IsEmpty(cfg.PrefetchCount) { cfg.PrefetchCount = 1 } - return c.createSubscribeInternal(ctx, cfg.QueueName, cfg.ConsumerName, cfg.PrefetchCount, cfg.AutoAck, cfg.Durable, cfg.DelayTime, cfg.HandleFunc) + return c.createSubscribe(ctx, cfg.QueueName, cfg.ConsumerName, cfg.PrefetchCount, cfg.DelayTime, cfg.AutoAck, cfg.Durable, cfg.HandleFunc) } // createSubscribe 内部订阅消息 -func (c *natsMsg) createSubscribeInternal(ctx context.Context, subject, consumerName string, prefetchCount int, autoAck, durable bool, delayTime int, handler func(ctx context.Context, message map[string]interface{}) error) error { +func (c *natsMsg) createSubscribe(ctx context.Context, subject, consumerName string, prefetchCount, delayTime int, autoAck, durable bool, handler func(ctx context.Context, message map[string]any) error) error { g.Log().Infof(ctx, "🔔 NATS 开始订阅: QueueName=%s, ConsumerName=%s", subject, consumerName) - delayMsg := delayTime > 0 - streamName, _ := getStreamInfo(durable, delayMsg) - - // 确保 Stream 存在,如果不存在则创建 - if err := c.createStreamGroupInternal(ctx, subject, durable, delayMsg); err != nil { - g.Log().Errorf(ctx, "创建 Stream 失败: %v", err) - return fmt.Errorf("创建 Stream 失败: %w", err) - } - - // Stream 不存在,创建新的 - ackPolicy := jetstream.AckExplicitPolicy - if autoAck { - ackPolicy = jetstream.AckNonePolicy - } - jsConfig := jetstream.ConsumerConfig{ - Name: consumerName, - Durable: consumerName, - FilterSubject: subject, - AckPolicy: ackPolicy, - MaxDeliver: 3, - MaxAckPending: prefetchCount, - } - // 创建新消费者 - consumer, err := js.CreateOrUpdateConsumer(ctx, streamName, jsConfig) - if err != nil { - g.Log().Errorf(ctx, "创建消费者失败: %v", err) - return err - } - - // 获取消费者信息验证 - if cInfo, err := consumer.Info(ctx); err == nil { - g.Log().Infof(ctx, "🔔 消费者创建成功: %s, AckPolicy=%v, MaxAckPending=%d", - cInfo.Name, cInfo.Config.AckPolicy, cInfo.Config.MaxAckPending) - } - - // 创建消息处理函数 - msgHandler := func(msg jetstream.Msg) { - // 记录消息接收时间 - now := time.Now() - meta, err := msg.Metadata() - if err == nil { - g.Log().Infof(ctx, "📨 收到消息: StreamSeq=%d, Published=%v, Received=%v, 距离发布=%.2f秒", - meta.Sequence.Stream, - meta.Timestamp.Format("2006-01-02 15:04:05"), - now.Format("2006-01-02 15:04:05"), - now.Sub(meta.Timestamp).Seconds()) - } - - // 解析消息 + // 创建推送订阅的回调函数 + msgHandler := func(msg *nats.Msg) { var data map[string]any - if err := json.Unmarshal(msg.Data(), &data); err != nil { - g.Log().Errorf(ctx, "解析消息失败: %v", err) - if err := msg.Nak(); err != nil { - g.Log().Errorf(ctx, "Nak 失败: %v", err) - } + if err := json.Unmarshal(msg.Data, &data); err != nil { + g.Log().Errorf(ctx, "❌ 解析消息失败: %v", err) return } + g.Log().Infof(ctx, "📨 收到消息: Subject=%s, Data=%v", msg.Subject, data) + // 处理业务逻辑 if err := handler(ctx, data); err != nil { - g.Log().Errorf(ctx, "处理消息失败: %v", err) - if err := msg.Nak(); err != nil { - g.Log().Errorf(ctx, "Nak 失败: %v", err) + g.Log().Errorf(ctx, "❌ 处理消息失败: %v", err) + if !autoAck { + if err := msg.Nak(); err != nil { + g.Log().Errorf(ctx, "❌ Nak 失败: %v", err) + return + } + return } + } else { + g.Log().Infof(ctx, "✅ 处理消息成功") + } + if err := msg.Ack(); err != nil { + g.Log().Errorf(ctx, "❌ Ack 失败: %v", err) + } + } + delayMsg := delayTime > 0 + // 创建流 + if err := c.createStream(ctx, subject, durable, delayMsg); err != nil { + return err + } + // 获取 JetStream 上下文 + js := getNatsJS(c.name) + if js == nil { + g.Log().Errorf(ctx, "❌ NATS [%s] JetStream 不存在", c.name) + return fmt.Errorf("NATS JetStream 不存在") + } + // 创建推送订阅 + var sub *nats.Subscription + var err error + // 配置订阅选项 - 使用 DeliverSubject 创建 Push Consumer + subOpts := []nats.SubOpt{ + nats.Durable(consumerName), + nats.MaxAckPending(prefetchCount), + nats.DeliverSubject(consumerName), + } + if !autoAck { + subOpts = append(subOpts, nats.ManualAck()) + } + // 使用 Subscribe 创建推送订阅 + sub, err = js.Subscribe(subject, msgHandler, subOpts...) + if err != nil { + g.Log().Errorf(ctx, "创建推送订阅失败: %v", err) + return err + } + g.Log().Infof(ctx, "✅ NATS 推送订阅成功: Consumer=%s", consumerName) + // 启动后台 goroutine 监听上下文取消,用于清理订阅 + go func() { + <-ctx.Done() + g.Log().Infof(ctx, "订阅上下文取消,取消订阅") + if err := sub.Unsubscribe(); err != nil { return } - g.Log().Infof(ctx, "处理消息成功") - if !autoAck { - if err := msg.Ack(); err != nil { - g.Log().Errorf(ctx, "Ack 失败: %v", err) - } - } - } - - // 开始消费 - _, err = consumer.Consume(msgHandler) - if err != nil { - return fmt.Errorf("开始消费失败: %w", err) - } - - g.Log().Infof(ctx, "✅ NATS 订阅成功") + }() return nil } -func getStreamInfo(durable, delayMsg bool) (string, jetstream.StorageType) { +// createStream 内部创建消费组 +func (c *natsMsg) createStream(ctx context.Context, subject string, durable, delayMsg bool) error { + streamName, storage := getStreamInfo(durable, delayMsg) + // 构建流配置 + // 如果是延迟消息,需要包含两个 subjects: + // 1. subject.schedule - 用于发送调度消息 + // 2. subject - 用于实际投递目标 + subjects := []string{subject} + if delayMsg { + subjects = []string{subject, subject + ".schedule"} + } + jsConfig := &StreamConfig{ + Name: streamName, + Subjects: subjects, + AllowMsgSchedules: delayMsg, // 延迟消息核心开关 + Storage: storage, + Discard: DiscardNew, // 达到上限删除旧消息 + } + nc := getNatsConn(c.name) + if !c.Ping(ctx) { + // 使用统一的重连函数 + if err := commonConnect(ctx, MessageNATS, c.name, func(ctx context.Context) error { + return c.Connect(ctx) + }, func(ctx context.Context) error { + return c.Close(ctx) + }); err != nil { + g.Log().Errorf(ctx, "❌ [%s][%s] 连接失败: %v", MessageNATS, c.name, err) + return err + } + } + if nc == nil { + g.Log().Errorf(ctx, "❌ NATS [%s] 连接不存在", c.name) + return fmt.Errorf("NATS 连接不存在") + } + err := jsStreamCreate(nc, jsConfig) + if err != nil { + g.Log().Errorf(ctx, "❌ 创建 Stream 失败: err=%v", err) + return err + } + g.Log().Infof(ctx, "✅ 创建 Stream 成功: stream=%s, subjects=%v, allowSchedules=%v", streamName, subjects, delayMsg) + return nil +} + +func getStreamInfo(durable, delayMsg bool) (string, StorageType) { // Stream 不存在,创建新的 streamName := "ordinary_msg_memory" - storage := jetstream.MemoryStorage + storage := MemoryStorage // 延迟消息必须使用 FileStorage(NATS 官方要求) if delayMsg { - streamName = "delay_msg_file" - storage = jetstream.FileStorage + if durable { + streamName = "delay_msg_file" + storage = FileStorage + } else { + streamName = "delay_msg_memory" + storage = MemoryStorage + } } else { if durable { streamName = "ordinary_msg_file" - storage = jetstream.FileStorage + storage = FileStorage } } return streamName, storage } + +const ( + // JSApiStreamCreateT is the endpoint to create new streams. + // Will return JSON response. + JSApiStreamCreateT = "$JS.API.STREAM.CREATE.%s" + + // JSApiStreamUpdateT is the endpoint to update existing streams. + // Will return JSON response. + JSApiStreamUpdateT = "$JS.API.STREAM.UPDATE.%s" +) + +// jsStreamCreate is for sending a stream create for fields that nats.go does not know about yet. +func jsStreamCreate(nc *nats.Conn, cfg *StreamConfig) error { + j, err := json.Marshal(cfg) + if err != nil { + return err + } + + msg, err := nc.Request(fmt.Sprintf(JSApiStreamCreateT, cfg.Name), j, time.Second*3) + if err != nil { + return err + } + + // 检查 API 响应中的错误 + var resp struct { + Error *struct { + Code int `json:"code"` + ErrCode int `json:"err_code"` + Description string `json:"description"` + } `json:"error,omitempty"` + } + if err := json.Unmarshal(msg.Data, &resp); err != nil { + return err + } + if resp.Error != nil { + // 如果 Stream 已存在,尝试更新 + if resp.Error.ErrCode == 10058 { // JSStreamNameExistErr + return jsStreamUpdate(nc, cfg) + } + return fmt.Errorf("JS API error: %s", resp.Error.Description) + } + + return nil +} + +// jsStreamUpdate is for sending a stream create for fields that nats.go does not know about yet. +func jsStreamUpdate(nc *nats.Conn, cfg *StreamConfig) error { + j, err := json.Marshal(cfg) + if err != nil { + return err + } + msg, err := nc.Request(fmt.Sprintf(JSApiStreamUpdateT, cfg.Name), j, time.Second*3) + if err != nil { + return err + } + + // 检查 API 响应中的错误 + var resp struct { + Error *struct { + Code int `json:"code"` + ErrCode int `json:"err_code"` + Description string `json:"description"` + } `json:"error,omitempty"` + } + if err := json.Unmarshal(msg.Data, &resp); err != nil { + return err + } + if resp.Error != nil { + return fmt.Errorf("JS API error: %s", resp.Error.Description) + } + + return nil +} diff --git a/message/nats_rpc.go b/message/nats_rpc.go index e204743..8e09938 100644 --- a/message/nats_rpc.go +++ b/message/nats_rpc.go @@ -31,6 +31,9 @@ var ( traceCancelMu sync.RWMutex // 取消主题前缀 cancelSubjectPrefix = "ctx.cancel.otel." + + // RPC 使用的默认数据源名称 + rpcDefaultDatasource = "default" ) // rpcHandler RPC 处理函数类型 @@ -42,7 +45,7 @@ type rpcHandler func(ctx context.Context, req []byte) (any, error) // serviceName: 服务名称,调用方通过此名称调用服务 // handler: 服务处理函数,接收请求并返回响应 func registerRPCService(serviceName string, handler rpcHandler) (err error) { - if !natsPing() { + if !natsPing(context.Background(), rpcDefaultDatasource) { return fmt.Errorf("NATS 未连接") } @@ -63,6 +66,11 @@ func registerRPCService(serviceName string, handler rpcHandler) (err error) { rpcServicesMu.Unlock() // 订阅服务主题 + nc := getNatsConn(rpcDefaultDatasource) + if nc == nil { + return fmt.Errorf("NATS 连接不存在") + } + subject := fmt.Sprintf("rpc.%s", serviceName) sub, err := nc.Subscribe(subject, func(msg *nats.Msg) { // 执行处理函数 @@ -84,7 +92,7 @@ func registerRPCService(serviceName string, handler rpcHandler) (err error) { // queueName: 队列组名,同一队列组的实例共享请求 // handler: 服务处理函数 func registerQueueRPCService(serviceName, queueName string, handler rpcHandler) (err error) { - if !natsPing() { + if !natsPing(context.Background(), rpcDefaultDatasource) { return fmt.Errorf("NATS 未连接") } @@ -111,6 +119,11 @@ func registerQueueRPCService(serviceName, queueName string, handler rpcHandler) queueRPCMu.Unlock() // 订阅服务主题(队列模式) + nc := getNatsConn(rpcDefaultDatasource) + if nc == nil { + return fmt.Errorf("NATS 连接不存在") + } + subject := fmt.Sprintf("rpc.%s", serviceName) sub, err := nc.QueueSubscribe(subject, queueName, func(msg *nats.Msg) { // 执行处理函数 @@ -209,7 +222,7 @@ func createCancelContext(ctx context.Context, traceID string) context.Context { // // sub, err := nats.SetupCancelListener(ctx) func setupCancelListener(ctx context.Context) (*nats.Subscription, error) { - if !natsPing() { + if !natsPing(ctx, rpcDefaultDatasource) { return nil, fmt.Errorf("NATS 未连接") } @@ -219,6 +232,11 @@ func setupCancelListener(ctx context.Context) (*nats.Subscription, error) { // 修复问题3:订阅取消主题,格式: ctx.cancel.otel.* // 使用 * 通配符而不是 >,因为 TraceID 是最后一部分 + nc := getNatsConn(rpcDefaultDatasource) + if nc == nil { + return nil, fmt.Errorf("NATS 连接不存在") + } + cancelSubject := cancelSubjectPrefix + "*" sub, err := nc.Subscribe(cancelSubject, func(msg *nats.Msg) { // 从主题中解析 TraceID (去除前缀) @@ -261,7 +279,7 @@ func setupCancelListener(ctx context.Context) (*nats.Subscription, error) { // // err := nats.publishCancel(ctx, traceID) func publishCancel(ctx context.Context, traceID string) error { - if !natsPing() { + if !natsPing(ctx, rpcDefaultDatasource) { return fmt.Errorf("NATS 未连接") } @@ -269,6 +287,11 @@ func publishCancel(ctx context.Context, traceID string) error { return fmt.Errorf("TraceID 不能为空") } + nc := getNatsConn(rpcDefaultDatasource) + if nc == nil { + return fmt.Errorf("NATS 连接不存在") + } + cancelSubject := cancelSubjectPrefix + traceID err := nc.Publish(cancelSubject, nil) if err != nil { @@ -303,7 +326,7 @@ func cleanupTraceCancel(traceID string) { // req: 请求数据 // 返回: 响应数据(任意类型)和错误 func CallRPC(ctx context.Context, serviceName string, req any, resp any) (err error) { - if !natsPing() { + if !natsPing(ctx, rpcDefaultDatasource) { return fmt.Errorf("NATS 未连接") } @@ -406,6 +429,11 @@ func CallRPC(ctx context.Context, serviceName string, req any, resp any) (err er } // 发送请求 + nc := getNatsConn(rpcDefaultDatasource) + if nc == nil { + return fmt.Errorf("NATS 连接不存在") + } + responseMsg, err := nc.RequestMsgWithContext(ctx, msg) // 关闭 done channel,通知 goroutine 退出 @@ -475,7 +503,7 @@ func WithExcludeMethods(methods ...string) registerServiceOption { // }, WithQueueGroup("order-group")) func AutoRegisterServices(ctx context.Context, serviceInstances map[string]interface{}, options ...registerServiceOption) error { // 先注册 RPC 服务(如果 NATS 不可用则记录警告但不阻塞启动) - if !natsPing() { + if !natsPing(ctx, rpcDefaultDatasource) { return fmt.Errorf("NATS 未连接,RPC 服务未注册") } @@ -512,7 +540,7 @@ func AutoRegisterServices(ctx context.Context, serviceInstances map[string]inter // registerService 注册单个服务的所有公开方法(内部函数) func registerService(service interface{}, serviceNamePrefix string, options ...registerServiceOption) (err error) { - if !natsPing() { + if !natsPing(context.Background(), rpcDefaultDatasource) { return fmt.Errorf("NATS 未连接") } diff --git a/message/rabbitmq_msg.go b/message/rabbitmq_msg.go index 0a65e20..3eab2a8 100644 --- a/message/rabbitmq_msg.go +++ b/message/rabbitmq_msg.go @@ -11,6 +11,12 @@ import ( ) type RabbitMQPublishMsgConfig struct { + QueueName string + Durable bool + Data any +} + +type RabbitMQPublishDelayMsgConfig struct { QueueName string Durable bool DelayTime int @@ -19,8 +25,6 @@ type RabbitMQPublishMsgConfig struct { type RabbitMQSubscribeMsgConfig struct { QueueName string - Durable bool - DelayTime int ConsumerName string AutoAck bool PrefetchCount int @@ -31,32 +35,36 @@ func (*RabbitMQPublishMsgConfig) GetPublishMsgType() { } +func (*RabbitMQPublishDelayMsgConfig) GetPublishDelayMsgType() {} + func (*RabbitMQSubscribeMsgConfig) GetSubscribeMsgType() { } -func init() { - // 注册 RabbitMQ 插件,必须使用 RegisterPlugin 确保连接检测 - //registerPlugin(MessageRabbitMQ, func() messageUtil { - // return &rabbitMQ{} - //}) +type rabbitMQ struct { + name string // 数据源名称 } -type rabbitMQ struct{} +func init() { + // 注册 RabbitMQ 插件(默认数据源) + RegisterPlugin(context.Background(), "default", MessageRabbitMQ, func() messageUtil { + return &rabbitMQ{name: "default"} + }) +} + +// Connect 连接 RabbitMQ +func (c *rabbitMQ) Connect(ctx context.Context) error { + return rabbitmqConnect(ctx, c.name) +} // Ping 检测 RabbitMQ 连接状态 -func (c *rabbitMQ) ping(ctx context.Context) bool { - return rabbitmqPing() -} - -// Reconnect 重连 RabbitMQ -func (c *rabbitMQ) reconnect(ctx context.Context) error { - return rabbitmqReconnect(ctx) +func (c *rabbitMQ) Ping(ctx context.Context) bool { + return rabbitmqPing(ctx, c.name) } // Close 关闭 RabbitMQ 连接 -func (c *rabbitMQ) close(ctx context.Context) error { - return rabbitmqClose(ctx) +func (c *rabbitMQ) Close(ctx context.Context) error { + return rabbitmqClose(ctx, c.name) } // Publish 发布消息 @@ -71,11 +79,43 @@ func (c *rabbitMQ) Publish(ctx context.Context, msgConfig messagePublishConfig) if cfg.Data == nil { return fmt.Errorf("数据不能为空") } + return c.publishMessageInternal(ctx, cfg.QueueName, cfg.Durable, 0, cfg.Data) +} + +// PublishDelay 发布延迟消息 +func (c *rabbitMQ) PublishDelay(ctx context.Context, msgConfig messagePublishDelayConfig) error { + cfg, ok := msgConfig.(*RabbitMQPublishDelayMsgConfig) + if !ok { + return fmt.Errorf("无效的 RabbitMQ 配置类型") + } + if g.IsEmpty(cfg.QueueName) { + return fmt.Errorf("队列名称不能为空") + } + if cfg.Data == nil { + return fmt.Errorf("数据不能为空") + } return c.publishMessageInternal(ctx, cfg.QueueName, cfg.Durable, cfg.DelayTime, cfg.Data) } // publishMessage 发布消息内部实现 func (c *rabbitMQ) publishMessageInternal(ctx context.Context, queueName string, durable bool, delayTime int, data interface{}) error { + if !c.Ping(ctx) { + if err := commonConnect(ctx, MessageRabbitMQ, c.name, func(ctx context.Context) error { + return c.Connect(ctx) + }, func(ctx context.Context) error { + return c.Close(ctx) + }); err != nil { + g.Log().Errorf(ctx, "❌ [%s][%s] 连接失败: %v", MessageRabbitMQ, c.name, err) + return err + } + } + + channel := getRabbitMQChannel(c.name) + if channel == nil || channel.IsClosed() { + g.Log().Errorf(ctx, "❌ RabbitMQ [%s] Channel 不存在或已关闭", c.name) + return fmt.Errorf("RabbitMQ Channel 不存在或已关闭") + } + delayMsg := delayTime > 0 // 1. 决定 Exchange 类型 @@ -86,12 +126,12 @@ func (c *rabbitMQ) publishMessageInternal(ctx context.Context, queueName string, if delayMsg { exchangeType = "x-delayed-message" exchangeName = queueName + ".delayed" - args["x-delayed-type"] = "fanout" // 底层用 topic + args["x-delayed-type"] = "fanout" } - // 2. 声明 Exchange(只声明一次) + // 2. 声明 Exchange(使用 exchangeName 而不是 queueName) if err := channel.ExchangeDeclare( - queueName, // exchange 交换机名称 + exchangeName, // 修复:使用正确的交换机名称 exchangeType, durable, false, // autoDelete @@ -99,7 +139,8 @@ func (c *rabbitMQ) publishMessageInternal(ctx context.Context, queueName string, false, // noWait args, ); err != nil { - return fmt.Errorf("声明 Exchange 失败: %w", err) + g.Log().Errorf(ctx, "❌ 声明 Exchange 失败: %v", err) + return err } // 3. 声明队列 @@ -111,7 +152,8 @@ func (c *rabbitMQ) publishMessageInternal(ctx context.Context, queueName string, false, // noWait nil, // args ); err != nil { - return fmt.Errorf("声明队列失败: %w", err) + g.Log().Errorf(ctx, "❌ 声明队列失败: %v", err) + return err } // 4. 绑定队列 @@ -122,13 +164,15 @@ func (c *rabbitMQ) publishMessageInternal(ctx context.Context, queueName string, false, // noWait nil, // args ); err != nil { - return fmt.Errorf("绑定队列失败: %w", err) + g.Log().Errorf(ctx, "❌ 绑定队列失败: %v", err) + return err } // 5. 序列化数据 body, err := json.Marshal(data) if err != nil { - return fmt.Errorf("序列化数据失败: %w", err) + g.Log().Errorf(ctx, "❌ 序列化数据失败: %v", err) + return err } // 6. 发布消息 deliveryMode := amqp.Transient @@ -142,9 +186,9 @@ func (c *rabbitMQ) publishMessageInternal(ctx context.Context, queueName string, Timestamp: time.Now(), } if delayMsg { - duration := time.Duration(delayTime) * time.Minute + duration := delayTime * 1000 // 延迟时间(毫秒)= 秒 * 1000 publishing.Headers = amqp.Table{ - "x-delay": duration, // 延迟时间(毫秒) + "x-delay": duration, } } err = channel.PublishWithContext( @@ -154,6 +198,11 @@ func (c *rabbitMQ) publishMessageInternal(ctx context.Context, queueName string, false, false, publishing, ) + if err != nil { + g.Log().Errorf(ctx, "❌ 发布消息失败: %v", err) + return err + } + g.Log().Infof(ctx, "📨 发布消息成功: queueName=%s, data=%v", queueName, data) return err } @@ -180,10 +229,28 @@ func (c *rabbitMQ) Subscribe(ctx context.Context, msgConfig messageSubscribeConf // createSubscribe 内部订阅消息 func (c *rabbitMQ) createSubscribeInternal(ctx context.Context, queueName, consumerName string, prefetchCount int, autoAck bool, handler func(ctx context.Context, message map[string]interface{}) error) error { - g.Log().Infof(ctx, "🔔 RabbitMQ 开始订阅: queueName=%s, consumerName=%s", queueName, consumerName) + g.Log().Infof(ctx, "🔔 RabbitMQ [%s] 开始订阅: queueName=%s, consumerName=%s", c.name, queueName, consumerName) + + if !c.Ping(ctx) { + if err := commonConnect(ctx, MessageRabbitMQ, c.name, func(ctx context.Context) error { + return c.Connect(ctx) + }, func(ctx context.Context) error { + return c.Close(ctx) + }); err != nil { + g.Log().Errorf(ctx, "❌ [%s][%s] 连接失败: %v", MessageRabbitMQ, c.name, err) + return err + } + } + + channel := getRabbitMQChannel(c.name) + if channel == nil || channel.IsClosed() { + g.Log().Errorf(ctx, "❌ RabbitMQ [%s] Channel 不存在或已关闭", c.name) + return fmt.Errorf("RabbitMQ Channel 不存在或已关闭") + } if err := channel.Qos(prefetchCount, 0, false); err != nil { - return fmt.Errorf("设置 Qos 失败: %w", err) + g.Log().Errorf(ctx, "❌ 设置 Qos 失败: %v", err) + return err } g.Log().Infof(ctx, "📊 设置 Prefetch Count: %d", prefetchCount) @@ -197,97 +264,48 @@ func (c *rabbitMQ) createSubscribeInternal(ctx context.Context, queueName, consu nil, // args ) if err != nil { - return fmt.Errorf("注册消费者失败: %w", err) + g.Log().Errorf(ctx, "❌ 消费消息失败: %v", err) + return err } - - go func() { - defer func() { - if r := recover(); r != nil { - g.Log().Errorf(ctx, "❌ RabbitMQ 消费者 panic: %v", r) + g.Log().Infof(ctx, "👀 开始监听消息") + for { + select { + case <-ctx.Done(): + // Context 取消,退出 + g.Log().Infof(ctx, "context cancel 监听消息退出") + return nil + case m, ok := <-msg: + if !ok { + // Channel 关闭,退出 + g.Log().Infof(ctx, "channel close 监听消息退出") + return nil } - }() + g.Log().Infof(ctx, "📨 收到消息: %s", string(m.Body)) - // 并发控制信号量 - semaphore := make(chan struct{}, 10) // 限制最大并发数为 10 - - for { - select { - case <-ctx.Done(): - g.Log().Infof(ctx, "🔕 RabbitMQ 消费者停止: queueName=%s, consumerName=%s", queueName, consumerName) - return - case msg, ok := <-msg: - if !ok { - g.Log().Warningf(ctx, "⚠️ RabbitMQ 消息通道关闭") - return + var data map[string]interface{} + if err := json.Unmarshal(m.Body, &data); err != nil { + // 如果不是 JSON,直接使用原始内容 + data = map[string]interface{}{ + "data": string(m.Body), } - - // 获取并发控制槽位 - semaphore <- struct{}{} - - go func(m amqp.Delivery) { - defer func() { - <-semaphore // 释放槽位 - if r := recover(); r != nil { - g.Log().Errorf(ctx, "❌ 消息处理 panic: %v", r) - } - }() - - if err := c.handleMessageWithRetryInternal(ctx, m, handler, autoAck); err != nil { - g.Log().Errorf(ctx, "❌ 消息处理失败(重试次数耗尽): %v", err) - - // 仅在手动 ACK 模式下拒绝消息 - if !autoAck { - // 拒绝消息不再重新入队(避免死循环) - m.Nack(false, false) - } - return - } - - // 仅在手动 ACK 模式下确认消息 - if autoAck { - if err := m.Ack(false); err != nil { - g.Log().Errorf(ctx, "❌ ACK 消息失败: %v", err) - } - } - }(msg) + } + err := handler(ctx, data) + if err != nil { + g.Log().Errorf(ctx, "❌ 消息处理失败: %v", err) + // 仅在手动 ACK 模式下拒绝消息 + if !autoAck { + // 拒绝消息不再重新入队(避免死循环) + m.Nack(false, false) + continue + } + } + g.Log().Infof(ctx, "✅ 消息处理成功: %v", err) + // 仅在手动 ACK 模式下确认消息 + if err := m.Ack(false); err != nil { + g.Log().Errorf(ctx, "❌ AUTO ACK 消息失败: %v", err) + } else { + g.Log().Infof(ctx, "✅ AUTO ACK 消息成功") } } - }() - - return nil -} - -// handleMessageWithRetry 处理消息(支持重试) -func (c *rabbitMQ) handleMessageWithRetryInternal(ctx context.Context, msg amqp.Delivery, handler func(ctx context.Context, message map[string]interface{}) error, autoAck bool) error { - var data map[string]interface{} - - if err := json.Unmarshal(msg.Body, &data); err != nil { - // 如果不是 JSON,直接使用原始内容 - data = map[string]interface{}{ - "data": string(msg.Body), - } } - - // 重试逻辑 - const maxRetry = 3 - for attempt := 0; attempt <= maxRetry; attempt++ { - if attempt > 0 { - g.Log().Infof(ctx, "🔄 消息处理重试 (第%d次)", attempt) - // 指数退避 - time.Sleep(time.Duration(attempt) * time.Second) - } - - err := handler(ctx, data) - if err == nil { - return nil // 成功 - } - - g.Log().Warningf(ctx, "⚠️ 消息处理失败 (第%d次): %v", attempt+1, err) - - if attempt == maxRetry { - return fmt.Errorf("达到最大重试次数 %d: %w", maxRetry, err) - } - } - - return nil } diff --git a/message/reconnect.go b/message/reconnect.go new file mode 100644 index 0000000..f042397 --- /dev/null +++ b/message/reconnect.go @@ -0,0 +1,73 @@ +package message + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/gogf/gf/v2/frame/g" +) + +// connectFunc 连接函数类型 +type connectFunc func(ctx context.Context) error + +// closeFunc 关闭函数类型 +type closeFunc func(ctx context.Context) error + +// reconnectOption 重连选项 +type reconnectOption struct { + maxRetries int // 最大重试次数,0 表示无限重试 + interval time.Duration // 重试间隔 + componentType messageType // 组件类型(nats/redis/rabbitmq) + componentName string // 组件名称(数据源名称) +} + +// defaultReconnectOption 默认重连选项 +func defaultReconnectOption(componentType messageType, componentName string) *reconnectOption { + return &reconnectOption{ + maxRetries: 0, // 无限重试 + interval: 3 * time.Second, + componentType: componentType, + componentName: componentName, + } +} + +// commonReconnect 重连函数(NATS、Redis、RabbitMQ 共用) +func commonReconnect(ctx context.Context, connectFn connectFunc, closeFn closeFunc, opt *reconnectOption) error { + if opt == nil { + opt = defaultReconnectOption("unknown", "default") + } + + for attempt := 0; opt.maxRetries == 0 || attempt < opt.maxRetries; attempt++ { + err := connectFn(ctx) + if err == nil { + g.Log().Infof(ctx, "✅ 连接成功: type=%s, name=%s, attempt=%d", + opt.componentType, opt.componentName, attempt+1) + return nil + } + // 记录失败日志 + g.Log().Warningf(ctx, "⚠️ 连接失败: type=%s, name=%s, attempt=%d, err=%v, 重试中...", + opt.componentType, opt.componentName, attempt+1, err) + // 如果错误信息中包含 "does not exist",则认为是连接失败,不再重试 + if strings.Contains(err.Error(), "does not exist") { + return err + } + // 等待一段时间再重试 + select { + case <-time.After(opt.interval): + case <-ctx.Done(): + if err = closeFn(ctx); err != nil { + return err + } + return ctx.Err() + } + } + return fmt.Errorf("连接失败,已达最大重试次数") +} + +// connect 连接函数,直接调用 commonReconnect +func commonConnect(ctx context.Context, componentType messageType, name string, connectFn func(ctx context.Context) error, closeFn closeFunc) error { + opt := defaultReconnectOption(componentType, name) + return commonReconnect(ctx, connectFn, closeFn, opt) +} diff --git a/message/redis_msg.go b/message/redis_msg.go index 38dd331..343acb9 100644 --- a/message/redis_msg.go +++ b/message/redis_msg.go @@ -3,11 +3,11 @@ package message import ( "context" "fmt" + "github.com/gogf/gf/v2/os/glog" "strings" "time" "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/util/gconv" ) @@ -16,6 +16,9 @@ type RedisPublishMsgConfig struct { Data any } +type RedisPublishDelayMsgConfig struct { +} + type RedisSubscribeMsgConfig struct { QueueName string ConsumerName string @@ -28,18 +31,22 @@ func (*RedisPublishMsgConfig) GetPublishMsgType() { } +func (*RedisPublishDelayMsgConfig) GetPublishDelayMsgType() {} + func (*RedisSubscribeMsgConfig) GetSubscribeMsgType() { } -func init() { - // 注册 Redis 插件(连接由 RegisterPlugin 异步处理) - registerPlugin(MessageRedis, func() messageUtil { - return &redis{} - }) +type redis struct { + name string // 数据源名称 } -type redis struct{} +func init() { + // 注册 Redis 插件(默认数据源) + RegisterPlugin(context.Background(), "default", MessageRedis, func() messageUtil { + return &redis{name: "default"} + }) +} // RedisStreamMessage Redis Stream 消息结构 type redisStreamMessage struct { @@ -47,41 +54,19 @@ type redisStreamMessage struct { Values map[string]interface{} } -// Ping 检测 Redis 连接状态 -func (c *redis) ping(ctx context.Context) bool { - conn, err := getDefaultDataSource() - if err != nil { - return false - } - return conn.redisPing(ctx) +// Connect 连接 Redis +func (c *redis) Connect(ctx context.Context) error { + return redisConnect(ctx, c.name) } -// Reconnect 重连 Redis -func (c *redis) reconnect(ctx context.Context) error { - conn, err := getDefaultDataSource() - if err != nil { - return fmt.Errorf("获取默认连接失败: %w", err) - } - - if err := conn.redisReconnect(ctx); err != nil { - return fmt.Errorf("redis重连失败: %w", err) - } - - return nil +// Ping 检测 Redis 连接状态 +func (c *redis) Ping(ctx context.Context) bool { + return redisPing(ctx, c.name) } // Close 关闭 Redis 连接 -func (c *redis) close(ctx context.Context) error { - conn, err := getDefaultDataSource() - if err != nil { - return fmt.Errorf("获取默认连接失败: %w", err) - } - - if err := conn.redisClose(ctx); err != nil { - return fmt.Errorf("关闭redis连接失败: %w", err) - } - - return nil +func (c *redis) Close(ctx context.Context) error { + return redisClose(ctx, c.name) } // Publish 发布消息 @@ -96,14 +81,16 @@ func (c *redis) Publish(ctx context.Context, msgConfig messagePublishConfig) err if g.IsEmpty(cfg.Data) { return fmt.Errorf("数据不能为空") } - conn, err := getDefaultDataSource() - if err != nil { - return fmt.Errorf("获取默认连接失败: %w", err) - } - if !conn.getIsConnected() { - if err := conn.redisReconnect(ctx); err != nil { - return fmt.Errorf("redis重连失败: %w", err) + rc := getRedisConn(c.name) + if !c.Ping(ctx) { + if err := commonConnect(ctx, MessageRedis, c.name, func(ctx context.Context) error { + return c.Connect(ctx) + }, func(ctx context.Context) error { + return c.Close(ctx) + }); err != nil { + g.Log().Errorf(ctx, "❌ [%s][%s] 连接失败: %v", MessageRedis, c.name, err) + return err } } @@ -113,7 +100,7 @@ func (c *redis) Publish(ctx context.Context, msgConfig messagePublishConfig) err for key, val := range values { args = append(args, key, val) } - result, err := conn.getClient().Do(ctx, "XADD", args...) + result, err := rc.Do(ctx, "XADD", args...) if err != nil { g.Log().Errorf(ctx, "❌ Redis 发布消息失败: key=%s, err=%v", cfg.QueueName, err) return err @@ -122,6 +109,12 @@ func (c *redis) Publish(ctx context.Context, msgConfig messagePublishConfig) err return nil } +// PublishDelay 发布延迟消息 +func (c *redis) PublishDelay(ctx context.Context, _ messagePublishDelayConfig) error { + g.Log().Errorf(ctx, "❌ Redis 不支持延迟消息") + return fmt.Errorf("❌ Redis 不支持延迟消息") +} + // Subscribe 订阅消息 func (c *redis) Subscribe(ctx context.Context, msgConfig messageSubscribeConfig) error { cfg, ok := msgConfig.(*RedisSubscribeMsgConfig) @@ -142,162 +135,92 @@ func (c *redis) Subscribe(ctx context.Context, msgConfig messageSubscribeConfig) // createSubscribe 内部订阅消息 func (c *redis) createSubscribe(ctx context.Context, key, consumerName string, prefetchCount int, autoAck bool, handler func(ctx context.Context, message map[string]interface{}) error) error { - go func() { - defer func() { - if r := recover(); r != nil { - g.Log().Errorf(ctx, "❌ Redis 消费者 panic: %v", r) - } - }() - retryTicker := time.NewTicker(time.Second) - defer retryTicker.Stop() +LOOP: + err := c.consumeMessages(ctx, key, consumerName, prefetchCount, autoAck, handler) + if err != nil { + // 对于超时错误,返回nil继续循环,而不是返回错误 + if strings.Contains(err.Error(), "i/o timeout") || strings.Contains(err.Error(), "timeout") || + strings.Contains(err.Error(), "context deadline exceeded") || strings.Contains(err.Error(), "context canceled") { - // 重试计数器 - var consecutiveErrors int - const maxConsecutiveErrors = 3 - - for { - select { - case <-ctx.Done(): - g.Log().Infof(ctx, "🔕 Redis 消费者停止: topic=%s", key) - return - case <-retryTicker.C: - err := c.consumeMessages(ctx, key, consumerName, prefetchCount, autoAck, handler) - if err != nil { - // 对于超时错误,返回nil继续循环,而不是返回错误 - if strings.Contains(err.Error(), "i/o timeout") || strings.Contains(err.Error(), "timeout") || - strings.Contains(err.Error(), "context deadline exceeded") || strings.Contains(err.Error(), "context canceled") { - - consecutiveErrors++ - if consecutiveErrors > maxConsecutiveErrors { - g.Log().Errorf(ctx, "Max retries exceeded, giving up") - return - } - backoffTime := 5 * time.Second - g.Log().Warningf(ctx, "⚠️ 等待 %v 后重试...", backoffTime) - - time.Sleep(backoffTime) - } else { - // 非超时错误(严重错误) - consecutiveErrors = 0 // 重置计数 - g.Log().Errorf(ctx, "严重错误,立即重试: %v", err) - - // 短暂等待后重试 - select { - case <-ctx.Done(): - return - case <-time.After(time.Second): - // 继续循环 - } - } - } else { - // 成功时重置错误计数器 - consecutiveErrors = 0 - } - } + time.Sleep(time.Second) + goto LOOP + } else { + g.Log().Errorf(ctx, "❌ 严重错误: %v", err) } - }() - return nil + } + time.Sleep(time.Second) + goto LOOP } // consumeMessages 消费消息 func (c *redis) consumeMessages(ctx context.Context, key, consumerName string, prefetchCount int, autoAck bool, handler func(ctx context.Context, message map[string]interface{}) error) error { - conn, err := getDefaultDataSource() - if err != nil { - return fmt.Errorf("获取默认连接失败: %w", err) + if !c.Ping(ctx) { + if err := commonConnect(ctx, MessageRedis, c.name, func(ctx context.Context) error { + return c.Connect(ctx) + }, func(ctx context.Context) error { + return c.Close(ctx) + }); err != nil { + g.Log().Errorf(ctx, "❌ [%s][%s] 连接失败: %v", MessageRedis, c.name, err) + return err + } } - if !conn.getIsConnected() { - if err := conn.redisReconnect(ctx); err != nil { - return fmt.Errorf("redis重连失败: %w", err) - } + rc := getRedisConn(c.name) + if rc == nil { + g.Log().Errorf(ctx, "❌ Redis [%s] 连接不存在", c.name) + return fmt.Errorf("Redis 连接不存在") } // 检查消费者组是否存在 - if err := c.createStreamGroup(ctx, key); err != nil { - return fmt.Errorf("create stream group failed: %w", err) + groupName := "default" + _, err := rc.Do(ctx, "XGROUP", "CREATE", key, groupName, "0", "MKSTREAM") + if err != nil { + errStr := err.Error() + if strings.Contains(errStr, "BUSYGROUP") && strings.Contains(errStr, "already exists") { + glog.Infof(ctx, "✅ Redis [%s] 消费者组已存在: %s", c.name, key) + return nil + } + g.Log().Errorf(ctx, "❌ 创建消费组失败: key=%s, err=%v", key, err) + return err } + glog.Infof(ctx, "✅ Redis [%s] 消费者组创建成功: %s", c.name, key) // 使用带重试的命令执行 - result, err := conn.getClient().Do(ctx, "XREADGROUP", "GROUP", "default", consumerName, "COUNT", prefetchCount, "BLOCK", 0, "STREAMS", key, ">") + result, err := rc.Do(ctx, "XREADGROUP", "GROUP", groupName, consumerName, "COUNT", prefetchCount, "BLOCK", 0, "STREAMS", key, ">") if err != nil { - if strings.Contains(err.Error(), "i/o timeout") || strings.Contains(err.Error(), "timeout") || - strings.Contains(err.Error(), "context deadline exceeded") || strings.Contains(err.Error(), "context canceled") { - - } return err } messages, err := c.parseStreamResult(result) if err != nil { + g.Log().Errorf(ctx, "❌ 解析消息失败: %v", err) return err } for _, msg := range messages { // 处理消息 if err := handler(ctx, msg.Values); err != nil { g.Log().Errorf(ctx, "❌ 消息处理失败: messageID=%s, err=%v", msg.ID, err) - continue - } - - // ACK 消息 - if autoAck { - if err := c.ackMessage(ctx, key, "default", msg.ID); err != nil { - g.Log().Errorf(ctx, "❌ ACK 消息失败: messageID=%s, err=%v", msg.ID, err) + // 如果不是自动ACK,则跳过当前消息 + if !autoAck { + continue } + } else { + g.Log().Infof(ctx, "✅ 消息处理成功: messageID=%s", msg.ID) + } + // ACK 消息 + args := make([]interface{}, 0, len(msg.ID)+2) + args = append(args, key, groupName, msg.ID) + _, err = rc.Do(ctx, "XACK", args...) + if err != nil { + g.Log().Errorf(ctx, "❌ ACK 消息失败: messageID=%s, err=%v", msg.ID, err) + } else { + g.Log().Infof(ctx, "✅ ACK 消息成功: messageID=%s", msg.ID) } } return nil } -// createStreamGroup 内部单个创建消费组 -func (c *redis) createStreamGroup(ctx context.Context, key string) error { - conn, err := getDefaultDataSource() - if err != nil { - return fmt.Errorf("获取默认连接失败: %w", err) - } - - if !conn.getIsConnected() { - if err := conn.redisReconnect(ctx); err != nil { - return fmt.Errorf("redis重连失败: %w", err) - } - } - - groupName := "default" - _, err = conn.getClient().Do(ctx, "XGROUP", "CREATE", key, groupName, "0", "MKSTREAM") - if err != nil { - errStr := err.Error() - if strings.Contains(errStr, "BUSYGROUP") && strings.Contains(errStr, "already exists") { - glog.Infof(ctx, "✅ Redis 消费者组已存在: %s", groupName) - return nil - } - return fmt.Errorf("初始化消费者组失败: %w", err) - } - glog.Infof(ctx, "✅ Redis 消费者组创建成功: %s", groupName) - return nil -} - -// ackMessage ACK 消息 -func (c *redis) ackMessage(ctx context.Context, streamKey, groupName string, messageIDs ...string) error { - conn, err := getDefaultDataSource() - if err != nil { - return fmt.Errorf("获取默认连接失败: %w", err) - } - - if !conn.getIsConnected() { - if err := conn.redisReconnect(ctx); err != nil { - return fmt.Errorf("redis重连失败: %w", err) - } - } - - args := make([]interface{}, 0, len(messageIDs)+2) - args = append(args, streamKey, groupName) - for _, id := range messageIDs { - args = append(args, id) - } - _, err = conn.getClient().Do(ctx, "XACK", args...) - return err -} - // parseStreamResult 解析 Stream 结果 func (c *redis) parseStreamResult(result interface{}) ([]redisStreamMessage, error) { if result == nil { diff --git a/message/store.go b/message/store.go new file mode 100644 index 0000000..15f8bc5 --- /dev/null +++ b/message/store.go @@ -0,0 +1,125 @@ +// Copyright 2019-2025 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package message + +import "fmt" + +type RetentionPolicy int + +const ( + // LimitsPolicy (default) means that messages are retained until any given limit is reached. + // This could be one of MaxMsgs, MaxBytes, or MaxAge. + LimitsPolicy RetentionPolicy = iota + // InterestPolicy specifies that when all known consumers have acknowledged a message it can be removed. + InterestPolicy + // WorkQueuePolicy specifies that when the first worker or subscriber acknowledges the message it can be removed. + WorkQueuePolicy +) + +// MarshalJSON 将 RetentionPolicy 序列化为字符串 +func (rp RetentionPolicy) MarshalJSON() ([]byte, error) { + switch rp { + case LimitsPolicy: + return []byte(`"limits"`), nil + case InterestPolicy: + return []byte(`"interest"`), nil + case WorkQueuePolicy: + return []byte(`"workqueue"`), nil + default: + return nil, fmt.Errorf("can not marshal %v", rp) + } +} + +// UnmarshalJSON 将字符串反序列化为 RetentionPolicy +func (rp *RetentionPolicy) UnmarshalJSON(data []byte) error { + switch string(data) { + case `"limits"`: + *rp = LimitsPolicy + case `"interest"`: + *rp = InterestPolicy + case `"workqueue"`: + *rp = WorkQueuePolicy + default: + return fmt.Errorf("unknown retention policy: %s", string(data)) + } + return nil +} + +type DiscardPolicy int + +const ( + // DiscardOld will remove older messages to return to the limits. + DiscardOld = iota + // DiscardNew will error on a StoreMsg call + DiscardNew +) + +// MarshalJSON 将 DiscardPolicy 序列化为字符串 +func (dp DiscardPolicy) MarshalJSON() ([]byte, error) { + switch dp { + case DiscardOld: + return []byte(`"old"`), nil + case DiscardNew: + return []byte(`"new"`), nil + default: + return nil, fmt.Errorf("can not marshal %v", dp) + } +} + +// UnmarshalJSON 将字符串反序列化为 DiscardPolicy +func (dp *DiscardPolicy) UnmarshalJSON(data []byte) error { + switch string(data) { + case `"old"`: + *dp = DiscardOld + case `"new"`: + *dp = DiscardNew + default: + return fmt.Errorf("unknown discard policy: %s", string(data)) + } + return nil +} + +type StorageType int + +const ( + // FileStorage specifies on disk, designated by the JetStream config StoreDir. + FileStorage = StorageType(22) + // MemoryStorage specifies in memory only. + MemoryStorage = StorageType(33) +) + +// MarshalJSON 将 StorageType 序列化为字符串 +func (st StorageType) MarshalJSON() ([]byte, error) { + switch st { + case MemoryStorage: + return []byte(`"memory"`), nil + case FileStorage: + return []byte(`"file"`), nil + default: + return nil, fmt.Errorf("can not marshal %v", st) + } +} + +// UnmarshalJSON 将字符串反序列化为 StorageType +func (st *StorageType) UnmarshalJSON(data []byte) error { + switch string(data) { + case `"memory"`: + *st = MemoryStorage + case `"file"`: + *st = FileStorage + default: + return fmt.Errorf("unknown storage type: %s", string(data)) + } + return nil +} diff --git a/message/stream.go b/message/stream.go new file mode 100644 index 0000000..4f2a7c1 --- /dev/null +++ b/message/stream.go @@ -0,0 +1,212 @@ +// Copyright 2019-2026 The NATS Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package message + +import ( + "fmt" + "time" +) + +// StreamConfig will determine the name, subjects and retention policy +// for a given stream. If subjects is empty the name will be used. +type StreamConfig struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Subjects []string `json:"subjects,omitempty"` + Retention RetentionPolicy `json:"retention"` + MaxConsumers int `json:"max_consumers"` + MaxMsgs int64 `json:"max_msgs"` + MaxBytes int64 `json:"max_bytes"` + MaxAge time.Duration `json:"max_age"` + MaxMsgsPer int64 `json:"max_msgs_per_subject"` + MaxMsgSize int32 `json:"max_msg_size,omitempty"` + Discard DiscardPolicy `json:"discard"` + Storage StorageType `json:"storage"` + Replicas int `json:"num_replicas"` + NoAck bool `json:"no_ack,omitempty"` + Duplicates time.Duration `json:"duplicate_window,omitempty"` + Placement *Placement `json:"placement,omitempty"` + Mirror *StreamSource `json:"mirror,omitempty"` + Sources []*StreamSource `json:"sources,omitempty"` + Compression StoreCompression `json:"compression"` + FirstSeq uint64 `json:"first_seq,omitempty"` + + // Allow applying a subject transform to incoming messages before doing anything else + SubjectTransform *SubjectTransformConfig `json:"subject_transform,omitempty"` + + // Allow republish of the message after being sequenced and stored. + RePublish *RePublish `json:"republish,omitempty"` + + // Allow higher performance, direct access to get individual messages. E.g. KeyValue + AllowDirect bool `json:"allow_direct"` + // Allow higher performance and unified direct access for mirrors as well. + MirrorDirect bool `json:"mirror_direct"` + + // Allow KV like semantics to also discard new on a per subject basis + DiscardNewPer bool `json:"discard_new_per_subject,omitempty"` + + // Optional qualifiers. These can not be modified after set to true. + + // Sealed will seal a stream so no messages can get out or in. + Sealed bool `json:"sealed"` + // DenyDelete will restrict the ability to delete messages. + DenyDelete bool `json:"deny_delete"` + // DenyPurge will restrict the ability to purge messages. + DenyPurge bool `json:"deny_purge"` + // AllowRollup allows messages to be placed into the system and purge + // all older messages using a special msg header. + AllowRollup bool `json:"allow_rollup_hdrs"` + + // The following defaults will apply to consumers when created against + // this stream, unless overridden manually. + // TODO(nat): Can/should we name these better? + ConsumerLimits StreamConsumerLimits `json:"consumer_limits"` + + // AllowMsgTTL allows header initiated per-message TTLs. If disabled, + // then the `NATS-TTL` header will be ignored. + AllowMsgTTL bool `json:"allow_msg_ttl"` + + // SubjectDeleteMarkerTTL sets the TTL of delete marker messages left behind by + // subject delete markers. + SubjectDeleteMarkerTTL time.Duration `json:"subject_delete_marker_ttl,omitempty"` + + // AllowMsgCounter allows a stream to use (only) counter CRDTs. + AllowMsgCounter bool `json:"allow_msg_counter,omitempty"` + + // AllowAtomicPublish allows atomic batch publishing into the stream. + AllowAtomicPublish bool `json:"allow_atomic,omitempty"` + + // AllowMsgSchedules allows the scheduling of messages. + AllowMsgSchedules bool `json:"allow_msg_schedules,omitempty"` + + // PersistMode allows to opt-in to different persistence mode settings. + PersistMode PersistModeType `json:"persist_mode,omitempty"` + + // Metadata is additional metadata for the Stream. + Metadata map[string]string `json:"metadata,omitempty"` +} + +// Used to guide placement of streams and meta controllers in clustered JetStream. +type Placement struct { + Cluster string `json:"cluster,omitempty"` + Tags []string `json:"tags,omitempty"` + Preferred string `json:"preferred,omitempty"` +} + +// StreamSource dictates how streams can source from other streams. +type StreamSource struct { + Name string `json:"name"` + OptStartSeq uint64 `json:"opt_start_seq,omitempty"` + OptStartTime *time.Time `json:"opt_start_time,omitempty"` + FilterSubject string `json:"filter_subject,omitempty"` + SubjectTransforms []SubjectTransformConfig `json:"subject_transforms,omitempty"` + External *ExternalStream `json:"external,omitempty"` + + // Internal + iname string // For indexing when stream names are the same for multiple sources. +} + +// SubjectTransformConfig is for applying a subject transform (to matching messages) before doing anything else when a new message is received +type SubjectTransformConfig struct { + Source string `json:"src"` + Destination string `json:"dest"` +} + +// ExternalStream allows you to qualify access to a stream source in another account or domain. +type ExternalStream struct { + ApiPrefix string `json:"api"` + DeliverPrefix string `json:"deliver"` +} + +// RePublish is for republishing messages once committed to a stream. +type RePublish struct { + Source string `json:"src,omitempty"` + Destination string `json:"dest"` + HeadersOnly bool `json:"headers_only,omitempty"` +} + +type StreamConsumerLimits struct { + InactiveThreshold time.Duration `json:"inactive_threshold,omitempty"` + MaxAckPending int `json:"max_ack_pending,omitempty"` +} + +// PersistModeType determines what persistence mode the stream uses. +type PersistModeType int + +const ( + // DefaultPersistMode specifies the default persist mode. Writes to the stream will immediately be flushed. + // The publish acknowledgement will be sent after the persisting completes. + DefaultPersistMode = PersistModeType(iota) + // AsyncPersistMode specifies writes to the stream will be flushed asynchronously. + // The publish acknowledgement may be sent before the persisting completes. + // This means writes could be lost if they weren't flushed prior to a hard kill of the server. + AsyncPersistMode +) + +// MarshalJSON 将 PersistModeType 序列化为字符串 +func (pm PersistModeType) MarshalJSON() ([]byte, error) { + switch pm { + case DefaultPersistMode: + return []byte(`"default"`), nil + case AsyncPersistMode: + return []byte(`"async"`), nil + default: + return nil, fmt.Errorf("can not marshal %v", pm) + } +} + +// UnmarshalJSON 将字符串反序列化为 PersistModeType +func (pm *PersistModeType) UnmarshalJSON(data []byte) error { + switch string(data) { + case `"default"`: + *pm = DefaultPersistMode + case `"async"`: + *pm = AsyncPersistMode + default: + return fmt.Errorf("unknown persist mode: %s", string(data)) + } + return nil +} + +type StoreCompression uint8 + +const ( + NoCompression StoreCompression = iota + S2Compression +) + +// MarshalJSON 将 StoreCompression 序列化为字符串 +func (sc StoreCompression) MarshalJSON() ([]byte, error) { + switch sc { + case NoCompression: + return []byte(`"none"`), nil + case S2Compression: + return []byte(`"s2"`), nil + default: + return nil, fmt.Errorf("can not marshal %v", sc) + } +} + +// UnmarshalJSON 将字符串反序列化为 StoreCompression +func (sc *StoreCompression) UnmarshalJSON(data []byte) error { + switch string(data) { + case `"none"`: + *sc = NoCompression + case `"s2"`: + *sc = S2Compression + default: + return fmt.Errorf("unknown store compression: %s", string(data)) + } + return nil +} From a88ed6e4f48ae208fe5b62d8fe2fdcc14a474eca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 4 Feb 2026 10:35:20 +0800 Subject: [PATCH 207/225] .gitignore --- mongo/connection.go | 5 +-- mongo/mongo.go | 84 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 69 insertions(+), 20 deletions(-) diff --git a/mongo/connection.go b/mongo/connection.go index a46a559..b58727b 100644 --- a/mongo/connection.go +++ b/mongo/connection.go @@ -8,7 +8,6 @@ package mongo import ( "context" "fmt" - "gitee.com/red-future---jilin-g/common/log/consts" "os" "os/signal" "strings" @@ -16,6 +15,8 @@ import ( "syscall" "time" + "gitee.com/red-future---jilin-g/common/log/consts" + "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/os/grpool" @@ -120,7 +121,7 @@ func (d *BaseDataSource) Connect(ctx context.Context) error { SetHeartbeatInterval(10 * time.Second). SetMaxConnIdleTime(60 * time.Second). SetRetryWrites(true). - SetRetryReads(true) + SetRetryReads(true).SetMonitor(commandMonitor()) var err error d.client, err = mongo.Connect(opt) diff --git a/mongo/mongo.go b/mongo/mongo.go index b0426d7..a7f30f8 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -12,6 +12,7 @@ import ( "time" "gitee.com/red-future---jilin-g/common/log/consts" + "go.mongodb.org/mongo-driver/v2/event" "gitee.com/red-future---jilin-g/common/beans" "gitee.com/red-future---jilin-g/common/log/model/entity" @@ -33,34 +34,81 @@ import ( // 向后兼容的MongoDB结构体 // ============================================================================= -type MongoDB struct { +type mongoDB struct { noCache bool dataSource string // 数据源名称,默认为 "default" noTenantId bool // 是否跳过租户过滤 } -func DB(cache ...bool) *MongoDB { - return &MongoDB{ +func DB(cache ...bool) *mongoDB { + return &mongoDB{ noCache: false, dataSource: "default", noTenantId: false, } } +// commandMonitor 命令监控器 +func commandMonitor() *event.CommandMonitor { + return &event.CommandMonitor{ + // 命令执行前触发 + Started: func(ctx context.Context, evt *event.CommandStartedEvent) { + // 执行前的处理逻辑示例:记录开始时间、打印执行的命令 + fmt.Printf("[%s] 开始执行命令 | 数据库: %s | 集合: %s | 命令: %+v\n", + time.Now().Format("2006-01-02 15:04:05"), + evt.DatabaseName, + evt.Command.Lookup("collection").StringValue(), // 获取集合名 + evt.Command, + ) + + // 也可以在这里添加:参数校验、权限检查、链路追踪埋点等 + // 例如:将开始时间存入ctx,供后续结束时计算耗时 + ctx = context.WithValue(ctx, "cmd_start_time", time.Now()) + }, + + // 命令执行成功后触发 + Succeeded: func(ctx context.Context, evt *event.CommandSucceededEvent) { + // 从ctx中获取开始时间,计算执行耗时 + startTime, ok := ctx.Value("cmd_start_time").(time.Time) + if ok { + elapsed := time.Since(startTime) + fmt.Printf("[%s] 命令执行成功 | 耗时: %s | 结果: %+v\n", + time.Now().Format("2006-01-02 15:04:05"), + elapsed, + evt.Reply, + ) + } + + // 也可以在这里添加:日志入库、性能指标上报、结果校验等 + }, + + // 命令执行失败后触发 + Failed: func(ctx context.Context, evt *event.CommandFailedEvent) { + fmt.Printf("[%s] 命令执行失败 | 错误: %s | 耗时: %s\n", + time.Now().Format("2006-01-02 15:04:05"), + evt.Failure, + evt.Duration, + ) + + // 也可以在这里添加:错误告警、重试逻辑、异常日志记录等 + }, + } +} + // WithDataSource 指定使用的数据源 -func (m *MongoDB) WithDataSource(name string) *MongoDB { +func (m *mongoDB) WithDataSource(name string) *mongoDB { m.dataSource = name return m } // NoCache 不使用缓存 -func (m *MongoDB) NoCache() *MongoDB { +func (m *mongoDB) NoCache() *mongoDB { m.noCache = true return m } // NoTenantId 不使用租户过滤 -func (m *MongoDB) NoTenantId() *MongoDB { +func (m *mongoDB) NoTenantId() *mongoDB { m.noTenantId = true return m } @@ -98,7 +146,7 @@ func GetDB() *mongo.Database { // ============================================================================= // getDataSource 获取当前使用的数据源 -func (m *MongoDB) getDataSource() (DataSource, error) { +func (m *mongoDB) getDataSource() (DataSource, error) { if m.dataSource == "" { m.dataSource = "default" } @@ -106,7 +154,7 @@ func (m *MongoDB) getDataSource() (DataSource, error) { } // Count 查询总数 -func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { +func (m *mongoDB) Count(ctx context.Context, filter bson.M, collection string) (count int64, err error) { source, err := m.getDataSource() if err != nil { return 0, err @@ -147,7 +195,7 @@ func (m *MongoDB) Count(ctx context.Context, filter bson.M, collection string) ( } // Find 查询多条记录 -func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, collection string, page *beans.Page, orderBy []beans.OrderBy) (total int64, err error) { +func (m *mongoDB) Find(ctx context.Context, filter bson.M, result interface{}, collection string, page *beans.Page, orderBy []beans.OrderBy) (total int64, err error) { source, err := m.getDataSource() if err != nil { return 0, err @@ -238,7 +286,7 @@ func (m *MongoDB) Find(ctx context.Context, filter bson.M, result interface{}, c } // FindOne 查询1条记录 -func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOneOptions]) (err error) { +func (m *mongoDB) FindOne(ctx context.Context, filter bson.M, result interface{}, collection string, opts ...options.Lister[options.FindOneOptions]) (err error) { source, err := m.getDataSource() if err != nil { return err @@ -292,7 +340,7 @@ func (m *MongoDB) FindOne(ctx context.Context, filter bson.M, result interface{} } // getDeletedData 获取要删除的数据 -func (m *MongoDB) getDeletedData(ctx context.Context, filter bson.M, collection string) (deletedIDs []bson.ObjectID, deletedData []bson.M, err error) { +func (m *mongoDB) getDeletedData(ctx context.Context, filter bson.M, collection string) (deletedIDs []bson.ObjectID, deletedData []bson.M, err error) { // 查询要删除的数据 _, err = m.Find(ctx, filter, &deletedData, collection, nil, nil) // 从查询结果中获取 _id @@ -302,7 +350,7 @@ func (m *MongoDB) getDeletedData(ctx context.Context, filter bson.M, collection return } -func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collection string) (err error) { +func (m *mongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interface{}, collection string) (err error) { listKeys := fmt.Sprintf(redis.CleanList, tenantId, collection) keys, err := redis.RedisClient().Keys(ctx, listKeys) if err != nil { @@ -336,7 +384,7 @@ func (m *MongoDB) CleanRedis(ctx context.Context, filter bson.M, tenantId interf return } -func (m *MongoDB) log(ctx context.Context, ids []bson.ObjectID, filter bson.M, collection string, data interface{}, userName, tenantId interface{}, operationType consts.OperationType) { +func (m *mongoDB) log(ctx context.Context, ids []bson.ObjectID, filter bson.M, collection string, data interface{}, userName, tenantId interface{}, operationType consts.OperationType) { // 提前获取 IP 地址,避免异步任务执行时请求已结束 var ipAddress string if request := g.RequestFromCtx(ctx); request != nil { @@ -375,7 +423,7 @@ func (m *MongoDB) log(ctx context.Context, ids []bson.ObjectID, filter bson.M, c } // Insert 插入多条记录 -func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { +func (m *mongoDB) Insert(ctx context.Context, documents []interface{}, collection string, opts ...options.Lister[options.InsertManyOptions]) (ids []interface{}, err error) { source, err := m.getDataSource() if err != nil { return nil, err @@ -426,7 +474,7 @@ func (m *MongoDB) Insert(ctx context.Context, documents []interface{}, collectio } // Delete 删除记录 -func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.DeleteManyOptions]) (count int64, err error) { +func (m *mongoDB) Delete(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.DeleteManyOptions]) (count int64, err error) { source, err := m.getDataSource() if err != nil { return 0, err @@ -461,13 +509,13 @@ func (m *MongoDB) Delete(ctx context.Context, filter bson.M, collection string, } // DeleteSoft 假删除记录 -func (m *MongoDB) DeleteSoft(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (modifiedCount int64, err error) { +func (m *mongoDB) DeleteSoft(ctx context.Context, filter bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (modifiedCount int64, err error) { update := bson.M{"$set": bson.M{"isDeleted": true}} return m.Update(ctx, filter, update, collection, opts...) } // Update 修改记录 -func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (modifiedCount int64, err error) { +func (m *mongoDB) Update(ctx context.Context, filter bson.M, update bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (modifiedCount int64, err error) { source, err := m.getDataSource() if err != nil { return 0, err @@ -526,7 +574,7 @@ func (m *MongoDB) Update(ctx context.Context, filter bson.M, update bson.M, coll } // SaveOrUpdate 批量增加或修改 -func (m *MongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.BulkWriteResult, err error) { +func (m *mongoDB) SaveOrUpdate(ctx context.Context, filter []bson.M, update []bson.M, collection string, opts ...options.Lister[options.UpdateManyOptions]) (result *mongo.BulkWriteResult, err error) { source, err := m.getDataSource() if err != nil { return nil, err From 57e49db295d012abd20c61eda024c64687ecba60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Wed, 11 Feb 2026 19:57:10 +0800 Subject: [PATCH 208/225] .gitignore --- rag/eino/base_task.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/rag/eino/base_task.go b/rag/eino/base_task.go index 06d6a86..9de26ad 100644 --- a/rag/eino/base_task.go +++ b/rag/eino/base_task.go @@ -6,7 +6,7 @@ import ( "gitee.com/red-future---jilin-g/common/beans" ) -// BaseTask 任务基类 +// BaseTask 任务基类 - MongoDB版本 type BaseTask struct { beans.MongoBaseDO `bson:",inline"` // 任务信息 @@ -26,3 +26,24 @@ type BaseTask struct { // 其他 Executor string `bson:"executor,omitempty" json:"executor,omitempty"` } + +// SQLBaseTask 任务基类 - SQL版本 +type SQLBaseTask struct { + beans.SQLBaseDO + // 任务信息 + TaskType TaskType `json:"taskType"` + Status TaskStatus `json:"status"` + Priority TaskPriority `json:"priority,omitempty"` + // 进度 + TotalItems int64 `json:"totalItems"` + ProcessedItems int64 `json:"processedItems"` + Progress float64 `json:"progress"` + // 结果 + StartTime *time.Time `json:"startTime"` + EndTime *time.Time `json:"endTime,omitempty"` + Duration int64 `json:"duration,omitempty"` + SuccessCount int64 `json:"successCount"` + FailCount int64 `json:"failCount"` + // 其他 + Executor string `json:"executor,omitempty"` +} From 5e4306f11a2094c28017f7c5040419a95794bd8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 12 Feb 2026 08:38:35 +0800 Subject: [PATCH 209/225] .gitignore --- beans/beans.go | 11 ++ sql/sql.go | 459 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 470 insertions(+) create mode 100644 sql/sql.go diff --git a/beans/beans.go b/beans/beans.go index 58c7281..31806bd 100644 --- a/beans/beans.go +++ b/beans/beans.go @@ -36,6 +36,17 @@ type MongoBaseDO struct { IsDeleted bool `bson:"isDeleted" json:"isDeleted" default:"false"` } +// SQLBaseDO SQL数据库基础实体 +type SQLBaseDO struct { + Id uint64 `json:"id"` // 主键ID + Creator string `json:"creator"` // 创建人 + CreatedAt *time.Time `json:"createdAt"` // 创建时间 + Updater string `json:"updater"` // 更新人 + UpdatedAt *time.Time `json:"updatedAt"` // 更新时间 + TenantId string `json:"tenantId"` // 租户ID + IsDeleted bool `json:"isDeleted"` // 是否删除 +} + type User struct { UserId interface{} `bson:"userId" json:"userId"` // 用户ID UserName interface{} `bson:"userName" json:"userName"` // 用户名 diff --git a/sql/sql.go b/sql/sql.go new file mode 100644 index 0000000..d792dd6 --- /dev/null +++ b/sql/sql.go @@ -0,0 +1,459 @@ +// ============================================================================= +// SQL 业务操作封装 +// 提供向后兼容的CRUD操作方法,支持 PostgreSQL +// ============================================================================= + +package sql + +import ( + "context" + + "gitee.com/red-future---jilin-g/common/beans" + "gitee.com/red-future---jilin-g/common/utils" + "github.com/gogf/gf/v2/database/gdb" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/gtime" + "github.com/gogf/gf/v2/util/gconv" +) + +const ( + PageSize = 20 +) + +type sqlDB struct { + noTenantId bool +} + +func DB(cache ...bool) *sqlDB { + return &sqlDB{ + noTenantId: false, + } +} + +// NoTenantId 不使用租户过滤 +func (s *sqlDB) NoTenantId() *sqlDB { + s.noTenantId = true + return s +} + +// Count 查询总数 +func (s *sqlDB) Count(ctx context.Context, model *gdb.Model) (int64, error) { + user, err := utils.GetUserInfo(ctx) + if err != nil { + return 0, err + } + + // 如果没有调用 noTenantId,则添加 tenantId 过滤 + if !s.noTenantId && !g.IsEmpty(user.TenantId) { + model = model.Where("tenant_id", user.TenantId) + } + + model = model.Where("is_deleted", false) + + // 执行查询 + count, _ := model.Count(ctx) + return int64(count), nil +} + +// Insert 插入记录(集合版本) +func (s *sqlDB) Insert(ctx context.Context, data interface{}, collection string) ([]any, error) { + user, err := utils.GetUserInfo(ctx) + if err != nil { + return nil, err + } + + model := g.DB().Model(collection) + + // 处理切片类型数据 + var resultSlice []any + if slice, ok := data.([]interface{}); ok { + for _, item := range slice { + // 转换为map + dataMap := gconv.Map(item) + delete(dataMap, "id") + + // 设置租户ID + if !g.IsEmpty(user.TenantId) && g.IsEmpty(dataMap["tenant_id"]) { + dataMap["tenant_id"] = user.TenantId + } + + // 设置创建人 + if !g.IsEmpty(user.UserName) && g.IsEmpty(dataMap["creator"]) { + dataMap["creator"] = user.UserName + } + + // 设置更新人 + if !g.IsEmpty(user.UserName) && g.IsEmpty(dataMap["updater"]) { + dataMap["updater"] = user.UserName + } + + // 设置时间 + now := gtime.Now().Time + if g.IsEmpty(dataMap["created_at"]) { + dataMap["created_at"] = now + } + if g.IsEmpty(dataMap["updated_at"]) { + dataMap["updated_at"] = now + } + + // 设置删除标记 + if g.IsEmpty(dataMap["is_deleted"]) { + dataMap["is_deleted"] = false + } + + resultSlice = append(resultSlice, dataMap) + } + + // 批量插入 + result, err := model.Data(resultSlice).Insert(ctx) + if err != nil { + return nil, err + } + + id, err := result.LastInsertId() + if err != nil { + return nil, err + } + + return []any{id}, nil + } + + // 单条数据插入 + dataMap := gconv.Map(data) + delete(dataMap, "id") + + // 设置租户ID + if !g.IsEmpty(user.TenantId) && g.IsEmpty(dataMap["tenant_id"]) { + dataMap["tenant_id"] = user.TenantId + } + + // 设置创建人 + if !g.IsEmpty(user.UserName) && g.IsEmpty(dataMap["creator"]) { + dataMap["creator"] = user.UserName + } + + // 设置更新人 + if !g.IsEmpty(user.UserName) && g.IsEmpty(dataMap["updater"]) { + dataMap["updater"] = user.UserName + } + + // 设置时间 + now := gtime.Now().Time + if g.IsEmpty(dataMap["created_at"]) { + dataMap["created_at"] = now + } + if g.IsEmpty(dataMap["updated_at"]) { + dataMap["updated_at"] = now + } + + // 设置删除标记 + if g.IsEmpty(dataMap["is_deleted"]) { + dataMap["is_deleted"] = false + } + + // 执行插入 + result, err := model.Data(dataMap).Insert(ctx) + if err != nil { + return nil, err + } + + id, err := result.LastInsertId() + if err != nil { + return nil, err + } + + return []any{id}, nil +} + +// FindOne 根据ID查询单条记录 +func (s *sqlDB) FindOne(ctx context.Context, id string, collection string, result interface{}) error { + user, err := utils.GetUserInfo(ctx) + if err != nil { + return err + } + + model := g.DB().Model(collection).Where("id", id) + + // 如果没有调用 noTenantId,则添加 tenantId 过滤 + if !s.noTenantId && !g.IsEmpty(user.TenantId) { + model = model.Where("tenant_id", user.TenantId) + } + + model = model.Where("is_deleted", false) + + // 执行查询 + return model.Scan(ctx, result) +} + +// FindOneWithResult 根据ID查询单条记录并返回结果 +func (s *sqlDB) FindOneWithResult(ctx context.Context, id string, collection string, result interface{}) (interface{}, error) { + user, err := utils.GetUserInfo(ctx) + if err != nil { + return nil, err + } + + model := g.DB().Model(collection).Where("id", id) + + // 如果没有调用 noTenantId,则添加 tenantId 过滤 + if !s.noTenantId && !g.IsEmpty(user.TenantId) { + model = model.Where("tenant_id", user.TenantId) + } + + model = model.Where("is_deleted", false) + + // 执行查询 + err = model.Scan(ctx, result) + return result, err +} + +// FindOneByModel 根据Model查询单条记录 +func (s *sqlDB) FindOneByModel(ctx context.Context, model *gdb.Model, result interface{}) error { + user, err := utils.GetUserInfo(ctx) + if err != nil { + return err + } + + // 如果没有调用 noTenantId,则添加 tenantId 过滤 + if !s.noTenantId && !g.IsEmpty(user.TenantId) { + model = model.Where("tenant_id", user.TenantId) + } + + model = model.Where("is_deleted", false) + + // 执行查询 + return model.Scan(ctx, result) +} + +// Find 查询多条记录(集合版本) +func (s *sqlDB) Find(ctx context.Context, model *gdb.Model, collection string, result interface{}, page *beans.Page, orderBy []beans.OrderBy) (int64, error) { + if model == nil { + model = g.DB().Model(collection) + } + + user, err := utils.GetUserInfo(ctx) + if err != nil { + return 0, err + } + + // 如果没有调用 noTenantId,则添加 tenantId 过滤 + if !s.noTenantId && !g.IsEmpty(user.TenantId) { + model = model.Where("tenant_id", user.TenantId) + } + + model = model.Where("is_deleted", false) + + // 分页处理 + limit := int64(PageSize) + offset := int64(0) + if page != nil && !g.IsEmpty(page.PageNum) && !g.IsEmpty(page.PageSize) { + limit = page.PageSize + if limit != -1 { + offset = (page.PageNum - 1) * limit + } + } + + // 排序处理 + if orderBy != nil && len(orderBy) > 0 { + for _, o := range orderBy { + orderStr := string(o.Order) + if orderStr == "asc" || orderStr == "ASC" { + model = model.OrderAsc(o.Field) + } else { + model = model.OrderDesc(o.Field) + } + } + } else { + model = model.OrderDesc("created_at") + } + + // 执行查询 + if limit != -1 { + total, err := s.Count(ctx, model) + if err != nil { + return 0, err + } + if total == 0 { + return 0, nil + } + err = model.Offset(int(offset)).Limit(int(limit)).Scan(ctx, result) + if err != nil { + return 0, err + } + return total, nil + } + + // 不分页,查询全部 + err = model.Scan(ctx, result) + if err != nil { + return 0, err + } + + // 获取结果集长度 + return 0, nil +} + +// Update 更新记录(ID版本) +func (s *sqlDB) Update(ctx context.Context, id string, collection string, data interface{}) (int64, error) { + user, err := utils.GetUserInfo(ctx) + if err != nil { + return 0, err + } + + model := g.DB().Model(collection).Where("id", id) + + // 如果没有调用 noTenantId,则添加 tenantId 过滤 + if !s.noTenantId && !g.IsEmpty(user.TenantId) { + model = model.Where("tenant_id", user.TenantId) + } + + model = model.Where("is_deleted", false) + + // 转换为map + dataMap := gconv.Map(data) + delete(dataMap, "id") + + // 设置更新人 + if !g.IsEmpty(user.UserName) && g.IsEmpty(dataMap["updater"]) { + dataMap["updater"] = user.UserName + } + + // 设置更新时间 + dataMap["updated_at"] = gtime.Now().Time + + // 执行更新 + result, err := model.Data(dataMap).Update(ctx) + if err != nil { + return 0, err + } + + affected, err := result.RowsAffected() + if err != nil { + return 0, err + } + + return affected, err +} + +// Delete 软删除(ID版本) +func (s *sqlDB) Delete(ctx context.Context, id string, collection string) (int64, error) { + user, err := utils.GetUserInfo(ctx) + if err != nil { + return 0, err + } + + model := g.DB().Model(collection).Where("id", id) + + // 如果没有调用 noTenantId,则添加 tenantId 过滤 + if !s.noTenantId && !g.IsEmpty(user.TenantId) { + model = model.Where("tenant_id", user.TenantId) + } + + model = model.Where("is_deleted", false) + + // 软删除 + data := map[string]interface{}{ + "is_deleted": true, + "updater": user.UserName, + "updated_at": gtime.Now().Time, + } + + result, err := model.Data(data).Update(ctx) + if err != nil { + return 0, err + } + + affected, err := result.RowsAffected() + if err != nil { + return 0, err + } + + return affected, err +} + +// DeleteByModel 软删除(Model版本) +func (s *sqlDB) DeleteSoftByModel(ctx context.Context, model *gdb.Model) (int64, error) { + user, err := utils.GetUserInfo(ctx) + if err != nil { + return 0, err + } + + // 如果没有调用 noTenantId,则添加 tenantId 过滤 + if !s.noTenantId && !g.IsEmpty(user.TenantId) { + model = model.Where("tenant_id", user.TenantId) + } + + model = model.Where("is_deleted", false) + + // 软删除 + data := map[string]interface{}{ + "is_deleted": true, + "updater": user.UserName, + "updated_at": gtime.Now().Time, + } + + result, err := model.Data(data).Update(ctx) + if err != nil { + return 0, err + } + + affected, err := result.RowsAffected() + if err != nil { + return 0, err + } + + return affected, err +} + +// Increment 字段自增 +func (s *sqlDB) Increment(ctx context.Context, id string, collection string, field string, value int64) (int64, error) { + user, err := utils.GetUserInfo(ctx) + if err != nil { + return 0, err + } + + model := g.DB().Model(collection).Where("id", id) + + // 如果没有调用 noTenantId,则添加 tenantId 过滤 + if !s.noTenantId && !g.IsEmpty(user.TenantId) { + model = model.Where("tenant_id", user.TenantId) + } + + model = model.Where("is_deleted", false) + + // 设置更新人和时间 + data := map[string]interface{}{ + "updater": user.UserName, + "updated_at": gtime.Now().Time, + } + + // 使用原生SQL实现自增 + data[field] = gdb.Raw(field + " + " + gconv.String(value) + "::bigint") + + // 执行更新 + result, err := model.Data(data).Update(ctx) + if err != nil { + return 0, err + } + + affected, err := result.RowsAffected() + if err != nil { + return 0, err + } + + return affected, err +} + +// DeleteSoft 软删除(ID版本) +func (s *sqlDB) DeleteSoft(ctx context.Context, id string, collection string) (int64, error) { + return s.Delete(ctx, id, collection) +} + +// Model 获取Model +func (s *sqlDB) Model(collection string) *gdb.Model { + return g.DB().Model(collection) +} + +// BuildUpdateData 构建更新数据 +func BuildUpdateData(ctx context.Context, req interface{}) (map[string]interface{}, error) { + return gconv.Map(req), nil +} From f26b63f9beb205aaffac9d44622a148ca8cc7978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Fri, 13 Feb 2026 11:08:29 +0800 Subject: [PATCH 210/225] .gitignore --- db/meilisearch/connection.go | 392 ++++++++++++++++++ db/meilisearch/meilisearch.go | 721 ++++++++++++++++++++++++++++++++++ db/meilisearch/types.go | 109 +++++ 3 files changed, 1222 insertions(+) create mode 100644 db/meilisearch/connection.go create mode 100644 db/meilisearch/meilisearch.go create mode 100644 db/meilisearch/types.go diff --git a/db/meilisearch/connection.go b/db/meilisearch/connection.go new file mode 100644 index 0000000..50c46cd --- /dev/null +++ b/db/meilisearch/connection.go @@ -0,0 +1,392 @@ +// ============================================================================= +// Meilisearch 数据源连接管理 +// 负责数据源的连接、重连、健康检查和优雅关闭 +// ============================================================================= + +package meilisearch + +import ( + "context" + "fmt" + "os" + "os/signal" + "sync" + "syscall" + "time" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/util/gconv" + ms "github.com/meilisearch/meilisearch-go" +) + +// ============================================================================= +// 数据源配置结构 +// ============================================================================= + +type DataSourceConfig struct { + Name string `json:"name"` + Host string `json:"host"` + Port int `json:"port"` + APIKey string `json:"apiKey"` + Timeout time.Duration `json:"timeout"` +} + +// ============================================================================= +// 单个数据源接口 +// ============================================================================= + +type DataSource interface { + Name() string + Client() interface{} + IsConnected() bool + Connect(ctx context.Context) error + Reconnect(ctx context.Context) error + Close(ctx context.Context) error +} + +// ============================================================================= +// 数据源实现 +// ============================================================================= + +type BaseDataSource struct { + config *DataSourceConfig + client interface{} + isConnected bool + mu sync.RWMutex + lastError error + lastErrorTime time.Time +} + +func NewBaseDataSource(config *DataSourceConfig) *BaseDataSource { + return &BaseDataSource{ + config: config, + isConnected: false, + } +} + +func (d *BaseDataSource) Name() string { + return d.config.Name +} + +func (d *BaseDataSource) Client() interface{} { + d.mu.RLock() + defer d.mu.RUnlock() + return d.client +} + +func (d *BaseDataSource) IsConnected() bool { + d.mu.RLock() + defer d.mu.RUnlock() + return d.isConnected && d.client != nil +} + +func (d *BaseDataSource) Connect(ctx context.Context) error { + d.mu.Lock() + defer d.mu.Unlock() + + // 构建客户端 + d.client = ms.New(d.config.Host, ms.WithAPIKey(d.config.APIKey)) + + // 测试连接 + if err := d.healthCheck(ctx); err != nil { + d.isConnected = false + d.lastError = err + d.lastErrorTime = time.Now() + return fmt.Errorf("datasource [%s] connection failed: %w", d.config.Name, err) + } + + d.isConnected = true + d.lastError = nil + glog.Infof(ctx, "✅ datasource [%s] connected successfully", d.config.Name) + return nil +} + +// healthCheck 健康检查 +func (d *BaseDataSource) healthCheck(ctx context.Context) error { + if d.client == nil { + return fmt.Errorf("client is nil") + } + + // 获取版本信息来测试连接 + if c, ok := d.client.(interface{ GetVersion() (*ms.Version, error) }); ok { + status, err := c.GetVersion() + if err != nil { + return err + } + glog.Debugf(ctx, "Meilisearch version: %s", status.PkgVersion) + } else { + // 如果没有GetVersion方法,尝试其他方法验证连接 + if _, ok := d.client.(interface { + GetIndexes(interface{}) (interface{}, error) + }); ok { + return nil + } + } + + return nil +} + +func (d *BaseDataSource) Reconnect(ctx context.Context) error { + glog.Infof(ctx, "🔄 reconnecting datasource [%s]", d.config.Name) + return d.Connect(ctx) +} + +func (d *BaseDataSource) Close(ctx context.Context) error { + d.mu.Lock() + defer d.mu.Unlock() + + // Meilisearch 客户端不需要显式关闭,只重置状态 + d.client = nil + d.isConnected = false + glog.Infof(ctx, "datasource [%s] closed", d.config.Name) + return nil +} + +// ============================================================================= +// 多数据源管理器 +// ============================================================================= + +type DataSourceManager struct { + sources map[string]DataSource + mu sync.RWMutex + ctx context.Context + cancel context.CancelFunc + started bool + maxRetries int +} + +var ( + globalManager *DataSourceManager + managerOnce sync.Once +) + +// GetManager 获取全局管理器 +func GetManager() *DataSourceManager { + managerOnce.Do(func() { + ctx, cancel := context.WithCancel(context.Background()) + globalManager = &DataSourceManager{ + sources: make(map[string]DataSource), + ctx: ctx, + cancel: cancel, + started: false, + maxRetries: 3, + } + }) + return globalManager +} + +// RegisterDataSource 注册数据源 +func (m *DataSourceManager) RegisterDataSource(config *DataSourceConfig) error { + m.mu.Lock() + defer m.mu.Unlock() + + if _, exists := m.sources[config.Name]; exists { + return fmt.Errorf("datasource [%s] already exists", config.Name) + } + + source := NewBaseDataSource(config) + m.sources[config.Name] = source + return nil +} + +// GetDataSource 获取数据源 +func (m *DataSourceManager) GetDataSource(name string) (DataSource, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + source, exists := m.sources[name] + if !exists { + return nil, fmt.Errorf("datasource [%s] not found", name) + } + return source, nil +} + +// GetAllDataSourceNames 获取所有数据源名称 +func (m *DataSourceManager) GetAllDataSourceNames() []string { + m.mu.RLock() + defer m.mu.RUnlock() + + names := make([]string, 0, len(m.sources)) + for name := range m.sources { + names = append(names, name) + } + return names +} + +// init 初始化多数据源 +func init() { + ctx := context.Background() + + // 从配置初始化多数据源 + if err := manager.InitializeFromConfig(ctx); err != nil { + glog.Errorf(ctx, "❌ Failed to initialize Meilisearch datasources: %v", err) + } else { + glog.Infof(ctx, "✅ Meilisearch datasources initialized: %v", manager.GetAllDataSourceNames()) + } + + // 启动健康检查 + manager.StartHealthCheck() + + // 设置优雅关闭 + setupGracefulShutdown() +} + +// InitializeFromConfig 从配置初始化数据源 +// 动态读取 config.yml 中 meilisearch 下的所有配置项 +func (m *DataSourceManager) InitializeFromConfig(ctx context.Context) error { + var firstErr error + + // 获取 meilisearch 配置下的所有子键 + meilisearchConfig := g.Cfg().MustGet(ctx, "meilisearch") + if meilisearchConfig.IsNil() { + glog.Warningf(ctx, "no meilisearch configuration found in config.yml") + return nil + } + + // 将配置转换为 map + configMap := meilisearchConfig.Map() + if configMap == nil { + glog.Warningf(ctx, "meilisearch configuration is not a map") + return nil + } + + // 遍历所有 meilisearch 子配置 + for name, subConfig := range configMap { + // 跳过非对象类型的配置 + subMap, ok := subConfig.(map[string]interface{}) + if !ok { + continue + } + + // 检查是否有 host 配置 + host, hasHost := subMap["host"] + if !hasHost || gconv.String(host) == "" { + continue + } + + // 构建数据源配置 + config := &DataSourceConfig{ + Name: name, + Host: gconv.String(host), + Port: int(gconv.Int(subMap["port"])), + APIKey: gconv.String(subMap["apiKey"]), + } + + // 设置默认值 + if config.Port == 0 { + config.Port = 7700 + } + if config.Host == "" { + config.Host = "http://localhost" + } + + // 可选:从配置读取超时时间 + if timeoutVal, hasTimeout := subMap["timeout"]; hasTimeout { + config.Timeout = gconv.Duration(timeoutVal) + } else { + config.Timeout = 10 * time.Second + } + + // 注册数据源 + if err := m.RegisterDataSource(config); err != nil { + glog.Errorf(ctx, "failed to register datasource [%s]: %v", name, err) + if firstErr == nil { + firstErr = err + } + continue + } + + // 连接数据源 + source, _ := m.GetDataSource(name) + if err := source.Connect(ctx); err != nil { + glog.Errorf(ctx, "failed to initialize datasource [%s]: %v", name, err) + if firstErr == nil { + firstErr = err + } + } + } + + return firstErr +} + +// StartHealthCheck 启动健康检查 +func (m *DataSourceManager) StartHealthCheck() { + if m.started { + return + } + m.started = true + go m.healthCheckLoop() +} + +// healthCheckLoop 健康检查循环 +func (m *DataSourceManager) healthCheckLoop() { + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + + for { + select { + case <-m.ctx.Done(): + return + case <-ticker.C: + m.checkAndReconnect() + } + } +} + +// checkAndReconnect 检查并重新连接 +func (m *DataSourceManager) checkAndReconnect() { + m.mu.RLock() + defer m.mu.RUnlock() + + for name, source := range m.sources { + if !source.IsConnected() { + glog.Warningf(context.Background(), "datasource [%s] disconnected, attempting reconnect", name) + + reconnectCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := source.Reconnect(reconnectCtx); err != nil { + glog.Errorf(reconnectCtx, "datasource [%s] reconnect failed: %v", name, err) + } else { + glog.Infof(reconnectCtx, "✅ datasource [%s] reconnected successfully", name) + } + } + } +} + +// CloseAll 关闭所有数据源 +func (m *DataSourceManager) CloseAll(ctx context.Context) error { + m.cancel() + + m.mu.RLock() + defer m.mu.RUnlock() + + var lastErr error + for name, source := range m.sources { + if err := source.Close(ctx); err != nil { + glog.Errorf(ctx, "failed to close datasource [%s]: %v", name, err) + lastErr = err + } + } + return lastErr +} + +// setupGracefulShutdown 设置优雅关闭 +func setupGracefulShutdown() { + go func() { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + <-sigCh + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + glog.Info(ctx, "🔄 Shutting down Meilisearch connections...") + if err := manager.CloseAll(ctx); err != nil { + glog.Errorf(ctx, "❌ Failed to close Meilisearch connections: %v", err) + } else { + glog.Info(ctx, "✅ Meilisearch connections closed successfully") + } + }() +} diff --git a/db/meilisearch/meilisearch.go b/db/meilisearch/meilisearch.go new file mode 100644 index 0000000..68e42b3 --- /dev/null +++ b/db/meilisearch/meilisearch.go @@ -0,0 +1,721 @@ +// ============================================================================= +// Meilisearch 业务操作封装 +// 提供CRUD操作方法,支持多数据源 +// ============================================================================= + +package meilisearch + +import ( + "context" + "fmt" + "time" + + "gitee.com/red-future---jilin-g/common/utils" + "github.com/gogf/gf/v2/container/gvar" + "github.com/gogf/gf/v2/errors/gerror" + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/os/glog" + "github.com/gogf/gf/v2/os/gtime" + "github.com/gogf/gf/v2/util/gconv" + ms "github.com/meilisearch/meilisearch-go" +) + +// ============================================================================= +// 向后兼容的Meilisearch结构体 +// ============================================================================= + +type meilisearchDB struct { + noCache bool + dataSource string // 数据源名称,默认为 "default" +} + +func DB(cache ...bool) *meilisearchDB { + return &meilisearchDB{ + noCache: false, + dataSource: "default", + } +} + +// WithDataSource 指定使用的数据源 +func (m *meilisearchDB) WithDataSource(name string) *meilisearchDB { + m.dataSource = name + return m +} + +// NoCache 不使用缓存 +func (m *meilisearchDB) NoCache() *meilisearchDB { + m.noCache = true + return m +} + +// ============================================================================= +// 全局变量 +// ============================================================================= + +var ( + manager = GetManager() +) + +const PageSize = 20 + +// ============================================================================= +// Meilisearch 操作方法(支持多数据源) +// ============================================================================= + +// getDataSource 获取当前使用的数据源 +func (m *meilisearchDB) getDataSource() (DataSource, error) { + if m.dataSource == "" { + m.dataSource = "default" + } + return manager.GetDataSource(m.dataSource) +} + +// getClient 获取 Meilisearch 客户端 +func (m *meilisearchDB) getClient() (interface{ Index(string) interface{} }, error) { + source, err := m.getDataSource() + if err != nil { + return nil, err + } + if c, ok := source.Client().(interface{ Index(string) interface{} }); ok { + return c, nil + } + return nil, fmt.Errorf("invalid client type") +} + +// indexInterface 辅助函数,获取index +func indexInterface(indexName string, client interface{ Index(string) interface{} }) interface{} { + return client.Index(indexName) +} + +// buildSearchRequest 构建搜索请求 +func (m *meilisearchDB) buildSearchRequest(ctx context.Context, searchParams *SearchParams) (*ms.SearchRequest, error) { + user, err := utils.GetUserInfo(ctx) + if err != nil { + return nil, err + } + + req := &ms.SearchRequest{ + Limit: int64(PageSize), + Page: int64(0), + } + + // 设置查询 + if searchParams.Query != "" { + req.Query = searchParams.Query + } + + // 设置分页 + if searchParams.Page > 0 { + req.Page = int64(searchParams.Page - 1) + } + if searchParams.Limit > 0 { + req.Limit = int64(searchParams.Limit) + } + + // 设置排序 + if len(searchParams.Sort) > 0 { + req.Sort = searchParams.Sort + } + + // 设置过滤条件(包含租户过滤和软删除过滤) + filter := "" + if !g.IsEmpty(user.TenantId) { + filter = fmt.Sprintf("tenantId = %s", gconv.String(user.TenantId)) + } + if filter == "" { + filter = "isDeleted = false" + } else { + filter += " AND isDeleted = false" + } + + // 添加用户自定义过滤条件 + if searchParams.Filter != "" { + if filter == "" { + filter = searchParams.Filter + } else { + filter += " AND " + searchParams.Filter + } + } + + if filter != "" { + req.Filter = filter + } + + // 设置可搜索字段 + if searchParams.SearchableAttributes != "" { + req.AttributesToSearchOn = []string{searchParams.SearchableAttributes} + } + + // 设置返回字段 + if len(searchParams.AttributesToRetrieve) > 0 { + req.AttributesToRetrieve = searchParams.AttributesToRetrieve + } + + return req, nil +} + +// Search 搜索文档 +func (m *meilisearchDB) Search(ctx context.Context, searchParams *SearchParams, indexName string, result interface{}) (total int64, err error) { + client, err := m.getClient() + if err != nil { + return 0, err + } + + // 构建搜索请求 + req, err := m.buildSearchRequest(ctx, searchParams) + if err != nil { + return 0, err + } + + // Redis 缓存处理 + user, err := utils.GetUserInfo(ctx) + if err != nil { + return + } + cacheKey := fmt.Sprintf("meilisearch:search:%s:%s:%+v", user.TenantId, indexName, searchParams) + if !m.noCache { + var resultStr *gvar.Var + resultStr, err = g.Redis().Get(ctx, cacheKey) + if err != nil { + return + } + if !g.IsEmpty(resultStr) { + searchResult := &SearchResult{} + if err = gconv.Struct(resultStr, searchResult); err != nil { + return + } + total = int64(searchResult.EstimatedTotalHits) + if len(searchResult.Hits) > 0 { + if resultArr, ok := result.(*[]map[string]interface{}); ok { + *resultArr = searchResult.Hits + } else { + err = gconv.Structs(searchResult.Hits, result) + if err != nil { + return + } + } + } + return + } + } + + // 执行搜索 + idx := indexInterface(indexName, client) + var searchResp *ms.SearchResponse + if i, ok := idx.(interface { + Search(string, *ms.SearchRequest) (*ms.SearchResponse, error) + }); ok { + searchResp, err = i.Search(searchParams.Query, req) + } else { + return 0, fmt.Errorf("index does not support Search method") + } + if err != nil { + return 0, err + } + + total = int64(searchResp.EstimatedTotalHits) + + // 解析结果 + if len(searchResp.Hits) > 0 { + hits := make([]map[string]interface{}, 0, len(searchResp.Hits)) + for _, hit := range searchResp.Hits { + hitMap := gconv.Map(hit) + // 移除 Meilisearch 内部字段 + delete(hitMap, "_formatted") + hits = append(hits, hitMap) + } + + if resultArr, ok := result.(*[]map[string]interface{}); ok { + *resultArr = hits + } else { + err = gconv.Structs(hits, result) + if err != nil { + return + } + } + } + + // 写入缓存 + if !m.noCache { + hitList := make([]map[string]interface{}, 0) + if len(searchResp.Hits) > 0 { + for _, hit := range searchResp.Hits { + hitMap := gconv.Map(hit) + delete(hitMap, "_formatted") + hitList = append(hitList, hitMap) + } + } + searchResult := &SearchResult{ + Hits: hitList, + EstimatedTotalHits: searchResp.EstimatedTotalHits, + Limit: int(searchResp.Limit), + Offset: int(searchResp.Offset), + ProcessingTimeMs: int(searchResp.ProcessingTimeMs), + } + err = g.Redis().SetEX(ctx, cacheKey, searchResult, int64(time.Hour)) + if err != nil { + return + } + } + + return +} + +// Insert 插入文档 +func (m *meilisearchDB) Insert(ctx context.Context, document interface{}, indexName string) (taskUID int64, err error) { + c, err := m.getClient() + if err != nil { + return 0, err + } + + user, err := utils.GetUserInfo(ctx) + if err != nil { + return + } + + // 转换为 map + docMap := gconv.Map(document) + + // 设置租户ID + if !g.IsEmpty(user.TenantId) && g.IsEmpty(docMap["tenantId"]) { + docMap["tenantId"] = user.TenantId + } + + // 设置创建人 + if !g.IsEmpty(user.UserName) && g.IsEmpty(docMap["creator"]) { + docMap["creator"] = user.UserName + } + + // 设置更新人 + if !g.IsEmpty(user.UserName) && g.IsEmpty(docMap["updater"]) { + docMap["updater"] = user.UserName + } + + // 设置时间 + now := gtime.Now().Time + if g.IsEmpty(docMap["createdAt"]) { + docMap["createdAt"] = now.Unix() + } + if g.IsEmpty(docMap["updatedAt"]) { + docMap["updatedAt"] = now.Unix() + } + + // 设置删除标记 + if g.IsEmpty(docMap["isDeleted"]) { + docMap["isDeleted"] = false + } + + // 执行插入 + documents := []map[string]interface{}{docMap} + idx := indexInterface(indexName, c) + var task *ms.TaskInfo + if i, ok := idx.(interface { + AddDocuments([]map[string]interface{}, interface{}) (*ms.TaskInfo, error) + }); ok { + task, err = i.AddDocuments(documents, nil) + } else { + return 0, fmt.Errorf("index does not support AddDocuments method") + } + if err != nil { + return 0, err + } + + // 清理缓存 + err = m.cleanCache(ctx, indexName, user.TenantId) + if err != nil { + glog.Warning(ctx, "清理Redis缓存失败:", err) + } + + return task.TaskUID, nil +} + +// InsertMany 批量插入文档 +func (m *meilisearchDB) InsertMany(ctx context.Context, documents []interface{}, indexName string) (taskUID int64, err error) { + c, err := m.getClient() + if err != nil { + return 0, err + } + + user, err := utils.GetUserInfo(ctx) + if err != nil { + return 0, err + } + + docs := make([]map[string]interface{}, 0, len(documents)) + for _, document := range documents { + docMap := gconv.Map(document) + + // 设置租户ID + if !g.IsEmpty(user.TenantId) && g.IsEmpty(docMap["tenantId"]) { + docMap["tenantId"] = user.TenantId + } + + // 设置创建人 + if !g.IsEmpty(user.UserName) && g.IsEmpty(docMap["creator"]) { + docMap["creator"] = user.UserName + } + + // 设置更新人 + if !g.IsEmpty(user.UserName) && g.IsEmpty(docMap["updater"]) { + docMap["updater"] = user.UserName + } + + // 设置时间 + now := gtime.Now().Time + if g.IsEmpty(docMap["createdAt"]) { + docMap["createdAt"] = now.Unix() + } + if g.IsEmpty(docMap["updatedAt"]) { + docMap["updatedAt"] = now.Unix() + } + + // 设置删除标记 + if g.IsEmpty(docMap["isDeleted"]) { + docMap["isDeleted"] = false + } + + docs = append(docs, docMap) + } + + // 执行批量插入 + idx := indexInterface(indexName, c) + var task *ms.TaskInfo + if i, ok := idx.(interface { + AddDocuments([]map[string]interface{}, interface{}) (*ms.TaskInfo, error) + }); ok { + task, err = i.AddDocuments(docs, nil) + } else { + return 0, fmt.Errorf("index does not support AddDocuments method") + } + if err != nil { + return 0, err + } + + // 清理缓存 + err = m.cleanCache(ctx, indexName, user.TenantId) + if err != nil { + glog.Warning(ctx, "清理Redis缓存失败:", err) + } + + return task.TaskUID, nil +} + +// Update 更新文档 +func (m *meilisearchDB) Update(ctx context.Context, document interface{}, indexName string) (taskUID int64, err error) { + c, err := m.getClient() + if err != nil { + return 0, err + } + + user, err := utils.GetUserInfo(ctx) + if err != nil { + return 0, err + } + + // 转换为 map + docMap := gconv.Map(document) + + // 设置更新人 + if !g.IsEmpty(user.UserName) && g.IsEmpty(docMap["updater"]) { + docMap["updater"] = user.UserName + } + + // 设置更新时间 + docMap["updatedAt"] = gtime.Now().Unix() + + // 执行更新 + documents := []map[string]interface{}{docMap} + idx := indexInterface(indexName, c) + var task *ms.TaskInfo + if i, ok := idx.(interface { + UpdateDocuments([]map[string]interface{}, interface{}) (*ms.TaskInfo, error) + }); ok { + task, err = i.UpdateDocuments(documents, nil) + } else { + return 0, fmt.Errorf("index does not support UpdateDocuments method") + } + if err != nil { + return 0, err + } + + // 清理缓存 + err = m.cleanCache(ctx, indexName, user.TenantId) + if err != nil { + glog.Warning(ctx, "清理Redis缓存失败:", err) + } + + return task.TaskUID, nil +} + +// Delete 删除文档 +func (m *meilisearchDB) Delete(ctx context.Context, id string, indexName string) (taskUID int64, err error) { + c, err := m.getClient() + if err != nil { + return 0, err + } + + // 执行删除 + idx := indexInterface(indexName, c) + var task *ms.TaskInfo + if i, ok := idx.(interface { + DeleteDocument(string) (*ms.TaskInfo, error) + }); ok { + task, err = i.DeleteDocument(id) + } else { + return 0, fmt.Errorf("index does not support DeleteDocument method") + } + if err != nil { + return 0, err + } + + // 清理缓存 + user, err := utils.GetUserInfo(ctx) + if err != nil { + return + } + err = m.cleanCache(ctx, indexName, user.TenantId) + if err != nil { + glog.Warning(ctx, "清理Redis缓存失败:", err) + } + + return task.TaskUID, nil +} + +// DeleteSoft 软删除文档 +func (m *meilisearchDB) DeleteSoft(ctx context.Context, id string, indexName string) (taskUID int64, err error) { + c, err := m.getClient() + if err != nil { + return 0, err + } + + user, err := utils.GetUserInfo(ctx) + if err != nil { + return 0, err + } + + // 软删除:更新 isDeleted 字段 + updateMap := map[string]interface{}{ + "id": id, + "isDeleted": true, + "updater": user.UserName, + "updatedAt": gtime.Now().Unix(), + } + + // 执行更新 + documents := []map[string]interface{}{updateMap} + idx := indexInterface(indexName, c) + var task *ms.TaskInfo + if i, ok := idx.(interface { + UpdateDocuments([]map[string]interface{}, interface{}) (*ms.TaskInfo, error) + }); ok { + task, err = i.UpdateDocuments(documents, nil) + } else { + return 0, fmt.Errorf("index does not support UpdateDocuments method") + } + if err != nil { + return 0, err + } + + // 清理缓存 + err = m.cleanCache(ctx, indexName, user.TenantId) + if err != nil { + glog.Warning(ctx, "清理Redis缓存失败:", err) + } + + return task.TaskUID, nil +} + +// Get 获取单个文档 +func (m *meilisearchDB) Get(ctx context.Context, id string, indexName string, result interface{}) (err error) { + c, err := m.getClient() + if err != nil { + return err + } + + // Redis 缓存处理 + user, err := utils.GetUserInfo(ctx) + if err != nil { + return + } + cacheKey := fmt.Sprintf("meilisearch:doc:%s:%s:%s", user.TenantId, indexName, id) + if !m.noCache { + var resultStr *gvar.Var + resultStr, err = g.Redis().Get(ctx, cacheKey) + if err != nil { + return + } + if !g.IsEmpty(resultStr) { + return gconv.Scan(resultStr, result) + } + } + + // 执行查询 + var doc map[string]interface{} + idx := indexInterface(indexName, c) + if i, ok := idx.(interface { + GetDocument(string, interface{}) error + }); ok { + err = i.GetDocument(id, &doc) + } else { + return fmt.Errorf("index does not support GetDocument method") + } + if err != nil { + return err + } + + // 过滤已删除的文档 + if gconv.Bool(doc["isDeleted"]) { + return gerror.New("文档不存在") + } + + err = gconv.Struct(doc, result) + if err != nil { + return err + } + + // 写入缓存 + if !m.noCache { + err = g.Redis().SetEX(ctx, cacheKey, result, int64(time.Hour)) + if err != nil { + return err + } + } + + return nil +} + +// cleanCache 清理缓存 +func (m *meilisearchDB) cleanCache(ctx context.Context, indexName string, tenantId interface{}) error { + // 清理搜索缓存 + searchKeys, err := g.Redis().Keys(ctx, fmt.Sprintf("meilisearch:search:%s:%s:*", tenantId, indexName)) + if err != nil { + return err + } + for _, key := range searchKeys { + _, err = g.Redis().Del(ctx, key) + if err != nil { + return err + } + } + return nil +} + +// CreateIndex 创建索引 +func (m *meilisearchDB) CreateIndex(ctx context.Context, indexConfig *IndexConfig) (taskUID int64, err error) { + client, err := m.getClient() + if err != nil { + return 0, err + } + + indexSettings := &ms.IndexConfig{ + Uid: indexConfig.UID, + PrimaryKey: indexConfig.PrimaryKey, + } + + if c, ok := client.(interface { + CreateIndex(*ms.IndexConfig) (*ms.TaskInfo, error) + }); ok { + task, err := c.CreateIndex(indexSettings) + if err != nil { + return 0, err + } + return task.TaskUID, nil + } + return 0, fmt.Errorf("client does not support CreateIndex") +} + +// DeleteIndex 删除索引 +func (m *meilisearchDB) DeleteIndex(ctx context.Context, indexName string) (err error) { + client, err := m.getClient() + if err != nil { + return err + } + + if c, ok := client.(interface{ DeleteIndex(string) error }); ok { + return c.DeleteIndex(indexName) + } + return fmt.Errorf("client does not support DeleteIndex") +} + +// GetIndex 获取索引信息 +func (m *meilisearchDB) GetIndex(ctx context.Context, indexName string) (interface{}, error) { + client, err := m.getClient() + if err != nil { + return nil, err + } + + if c, ok := client.(interface { + GetIndex(string) (interface{}, error) + }); ok { + return c.GetIndex(indexName) + } + return nil, fmt.Errorf("client does not support GetIndex") +} + +// GetIndexes 获取所有索引 +func (m *meilisearchDB) GetIndexes(ctx context.Context) (interface{}, error) { + client, err := m.getClient() + if err != nil { + return nil, err + } + + if c, ok := client.(interface { + GetIndexes(interface{}) (interface{}, error) + }); ok { + return c.GetIndexes(nil) + } + return nil, fmt.Errorf("client does not support GetIndexes") +} + +// UpdateSettings 更新索引设置 +func (m *meilisearchDB) UpdateSettings(ctx context.Context, indexName string, settings *ms.Settings) (taskUID int64, err error) { + c, err := m.getClient() + if err != nil { + return 0, err + } + + idx := indexInterface(indexName, c) + var task *ms.TaskInfo + if i, ok := idx.(interface { + UpdateSettings(*ms.Settings) (*ms.TaskInfo, error) + }); ok { + task, err = i.UpdateSettings(settings) + } else { + return 0, fmt.Errorf("index does not support UpdateSettings method") + } + if err != nil { + return 0, err + } + + return task.TaskUID, nil +} + +// GetSettings 获取索引设置 +func (m *meilisearchDB) GetSettings(ctx context.Context, indexName string) (*ms.Settings, error) { + c, err := m.getClient() + if err != nil { + return nil, err + } + + idx := indexInterface(indexName, c) + var settings *ms.Settings + if i, ok := idx.(interface{ GetSettings() (*ms.Settings, error) }); ok { + settings, err = i.GetSettings() + } else { + return nil, fmt.Errorf("index does not support GetSettings method") + } + if err != nil { + return nil, err + } + return settings, nil +} + +// GetClient 获取原始客户端(用于高级操作) +func (m *meilisearchDB) GetClient() (interface{ Index(string) interface{} }, error) { + return m.getClient() +} + +// BuildUpdateData 构建更新数据 +func BuildUpdateData(ctx context.Context, req interface{}) (map[string]interface{}, error) { + return gconv.Map(req), nil +} diff --git a/db/meilisearch/types.go b/db/meilisearch/types.go new file mode 100644 index 0000000..f5e9e65 --- /dev/null +++ b/db/meilisearch/types.go @@ -0,0 +1,109 @@ +// ============================================================================= +// Meilisearch 数据结构定义 +// ============================================================================= + +package meilisearch + +// SearchParams 搜索参数 +type SearchParams struct { + Query string // 搜索查询字符串 + Page int64 // 页码(从1开始) + Limit int64 // 每页数量 + Sort []string // 排序字段,如 ["createdAt:desc", "price:asc"] + Filter string // 过滤条件 + SearchableAttributes string // 可搜索字段 + AttributesToRetrieve []string // 返回字段 + Facets []string // 聚合字段 + HitsPerPage int // 每页命中数 +} + +// SearchResult 搜索结果 +type SearchResult struct { + Hits []map[string]interface{} `json:"hits"` + EstimatedTotalHits int64 `json:"estimatedTotalHits"` + Limit int `json:"limit"` + Offset int `json:"offset"` + ProcessingTimeMs int `json:"processingTimeMs"` + Query string `json:"query"` + FacetDistribution map[string]interface{} `json:"facetDistribution,omitempty"` +} + +// IndexConfig 索引配置 +type IndexConfig struct { + UID string // 索引唯一标识 + PrimaryKey string // 主键字段名 +} + +// IndexSettings 索引设置 +type IndexSettings struct { + SearchableAttributes []string `json:"searchableAttributes,omitempty"` + DisplayedAttributes []string `json:"displayedAttributes,omitempty"` + FilterableAttributes []string `json:"filterableAttributes,omitempty"` + SortableAttributes []string `json:"sortableAttributes,omitempty"` + RankingRules []string `json:"rankingRules,omitempty"` + StopWords []string `json:"stopWords,omitempty"` + Synonyms map[string][]string `json:"synonyms,omitempty"` + DistinctAttribute string `json:"distinctAttribute,omitempty"` + TypoTolerance *TypoTolerance `json:"typoTolerance,omitempty"` + Pagination *Pagination `json:"pagination,omitempty"` + Faceting *Faceting `json:"faceting,omitempty"` +} + +// TypoTolerance 拼写容错设置 +type TypoTolerance struct { + Enabled bool `json:"enabled"` + MinWordSizeForTypos map[string]int `json:"minWordSizeForTypos"` + DisableOnWords []string `json:"disableOnWords"` + DisableOnAttributes []string `json:"disableOnAttributes"` +} + +// Pagination 分页设置 +type Pagination struct { + MaxTotalHits int `json:"maxTotalHits"` +} + +// Faceting 聚合设置 +type Faceting struct { + MaxValuesPerFacet int `json:"maxValuesPerFacet"` +} + +// TaskResult 任务结果 +type TaskResult struct { + TaskUID int64 `json:"taskUid"` + Status string `json:"status"` + Type string `json:"type"` + Details map[string]interface{} `json:"details,omitempty"` + Error string `json:"error,omitempty"` +} + +// Document 文档 +type Document map[string]interface{} + +// 文档操作符 +const ( + OperatorEquals = "=" + OperatorNotEquals = "!=" + OperatorGreaterThan = ">" + OperatorGreaterOrEqual = ">=" + OperatorLessThan = "<" + OperatorLessOrEqual = "<=" + OperatorAnd = "AND" + OperatorOr = "OR" + OperatorNot = "NOT" +) + +// 排序顺序 +const ( + SortAsc = "asc" + SortDesc = "desc" +) + +// 排名规则 +const ( + RankingRuleWords = "words" + RankingRuleTypo = "typo" + RankingRuleProximity = "proximity" + RankingRuleAttribute = "attribute" + RankingRuleSort = "sort" + RankingRuleExactness = "exactness" +) From 5d5bc2fa6e35afc66ed3593ea46f324a8dce93d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 24 Feb 2026 15:42:36 +0800 Subject: [PATCH 211/225] .gitignore --- db/meilisearch/meilisearch.go | 2 +- http/http.go | 8 ++++---- log/controller/log_controller.go | 4 ++-- log/dao/log_dao.go | 10 +++++----- log/model/dto/log_dto.go | 2 +- log/model/entity/log.go | 2 +- log/service/log_service.go | 10 +++++----- middleware/circuit_breaker.go | 2 +- middleware/module_tenant_check.go | 8 ++++---- middleware/rate_limiter.go | 4 ++-- minio/minio.go | 2 +- mongo/connection.go | 2 +- mongo/mongo.go | 10 +++++----- rag/eino/base_task.go | 2 +- ragflow/document.go | 2 +- ragflow/worker_pool.go | 2 +- sql/sql.go | 4 ++-- swagger/swagger.go | 4 ++-- utils/utils.go | 2 +- 19 files changed, 41 insertions(+), 41 deletions(-) diff --git a/db/meilisearch/meilisearch.go b/db/meilisearch/meilisearch.go index 68e42b3..e4b561f 100644 --- a/db/meilisearch/meilisearch.go +++ b/db/meilisearch/meilisearch.go @@ -10,7 +10,7 @@ import ( "fmt" "time" - "gitee.com/red-future---jilin-g/common/utils" + "gitea.com/red-future/common/utils" "github.com/gogf/gf/v2/container/gvar" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" diff --git a/http/http.go b/http/http.go index 4555b47..ffcb1fe 100644 --- a/http/http.go +++ b/http/http.go @@ -9,10 +9,10 @@ import ( "regexp" "strings" - _ "gitee.com/red-future---jilin-g/common/consul" - "gitee.com/red-future---jilin-g/common/jaeger" - "gitee.com/red-future---jilin-g/common/log/controller" - "gitee.com/red-future---jilin-g/common/utils" + _ "gitea.com/red-future/common/consul" + "gitea.com/red-future/common/jaeger" + "gitea.com/red-future/common/log/controller" + "gitea.com/red-future/common/utils" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/gclient" "github.com/gogf/gf/v2/net/ghttp" diff --git a/log/controller/log_controller.go b/log/controller/log_controller.go index 3485423..a222b07 100644 --- a/log/controller/log_controller.go +++ b/log/controller/log_controller.go @@ -3,8 +3,8 @@ package controller import ( "context" - "gitee.com/red-future---jilin-g/common/log/model/dto" - "gitee.com/red-future---jilin-g/common/log/service" + "gitea.com/red-future/common/log/model/dto" + "gitea.com/red-future/common/log/service" ) type operationLog struct{} diff --git a/log/dao/log_dao.go b/log/dao/log_dao.go index 7f435de..b185913 100644 --- a/log/dao/log_dao.go +++ b/log/dao/log_dao.go @@ -2,14 +2,14 @@ package dao import ( "context" - "gitee.com/red-future---jilin-g/common/beans" + "gitea.com/red-future/common/beans" "strings" "time" - "gitee.com/red-future---jilin-g/common/log/consts" - "gitee.com/red-future---jilin-g/common/log/model/dto" - "gitee.com/red-future---jilin-g/common/log/model/entity" - "gitee.com/red-future---jilin-g/common/mongo" + "gitea.com/red-future/common/log/consts" + "gitea.com/red-future/common/log/model/dto" + "gitea.com/red-future/common/log/model/entity" + "gitea.com/red-future/common/mongo" "go.mongodb.org/mongo-driver/v2/bson" ) diff --git a/log/model/dto/log_dto.go b/log/model/dto/log_dto.go index ba12fda..ddbc167 100644 --- a/log/model/dto/log_dto.go +++ b/log/model/dto/log_dto.go @@ -1,7 +1,7 @@ package dto import ( - "gitee.com/red-future---jilin-g/common/beans" + "gitea.com/red-future/common/beans" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gtime" ) diff --git a/log/model/entity/log.go b/log/model/entity/log.go index 463926f..40fd54f 100644 --- a/log/model/entity/log.go +++ b/log/model/entity/log.go @@ -1,7 +1,7 @@ package entity import ( - "gitee.com/red-future---jilin-g/common/beans" + "gitea.com/red-future/common/beans" ) // OperationLog 操作日志实体 - 用于记录数据增删改操作行为 diff --git a/log/service/log_service.go b/log/service/log_service.go index c16e8cc..3e15c75 100644 --- a/log/service/log_service.go +++ b/log/service/log_service.go @@ -2,11 +2,11 @@ package service import ( "context" - "gitee.com/red-future---jilin-g/common/beans" - "gitee.com/red-future---jilin-g/common/log/dao" - "gitee.com/red-future---jilin-g/common/log/model/dto" - logEntity "gitee.com/red-future---jilin-g/common/log/model/entity" - "gitee.com/red-future---jilin-g/common/utils" + "gitea.com/red-future/common/beans" + "gitea.com/red-future/common/log/dao" + "gitea.com/red-future/common/log/model/dto" + logEntity "gitea.com/red-future/common/log/model/entity" + "gitea.com/red-future/common/utils" "github.com/gogf/gf/v2/util/gconv" ) diff --git a/middleware/circuit_breaker.go b/middleware/circuit_breaker.go index 82af81a..5b02a5b 100644 --- a/middleware/circuit_breaker.go +++ b/middleware/circuit_breaker.go @@ -9,7 +9,7 @@ import ( "sync/atomic" "time" - "gitee.com/red-future---jilin-g/common/redis" + "gitea.com/red-future/common/redis" "github.com/alibaba/sentinel-golang/api" "github.com/alibaba/sentinel-golang/core/circuitbreaker" "github.com/gogf/gf/v2/frame/g" diff --git a/middleware/module_tenant_check.go b/middleware/module_tenant_check.go index 2ba3518..a88a7cb 100644 --- a/middleware/module_tenant_check.go +++ b/middleware/module_tenant_check.go @@ -4,10 +4,10 @@ import ( "context" "encoding/json" "fmt" - "gitee.com/red-future---jilin-g/common/beans" - "gitee.com/red-future---jilin-g/common/message" - "gitee.com/red-future---jilin-g/common/redis" - "gitee.com/red-future---jilin-g/common/utils" + "gitea.com/red-future/common/beans" + "gitea.com/red-future/common/message" + "gitea.com/red-future/common/redis" + "gitea.com/red-future/common/utils" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" diff --git a/middleware/rate_limiter.go b/middleware/rate_limiter.go index 70dc61e..71dd53a 100644 --- a/middleware/rate_limiter.go +++ b/middleware/rate_limiter.go @@ -4,8 +4,8 @@ import ( "fmt" "strings" - "gitee.com/red-future---jilin-g/common/redis" - "gitee.com/red-future---jilin-g/common/utils" + "gitea.com/red-future/common/redis" + "gitea.com/red-future/common/utils" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/util/gconv" diff --git a/minio/minio.go b/minio/minio.go index 030d87f..4c99dcb 100644 --- a/minio/minio.go +++ b/minio/minio.go @@ -3,7 +3,7 @@ package minio import ( "context" "fmt" - "gitee.com/red-future---jilin-g/common/utils" + "gitea.com/red-future/common/utils" "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/util/gconv" "net/http" diff --git a/mongo/connection.go b/mongo/connection.go index b58727b..10f3fc9 100644 --- a/mongo/connection.go +++ b/mongo/connection.go @@ -15,7 +15,7 @@ import ( "syscall" "time" - "gitee.com/red-future---jilin-g/common/log/consts" + "gitea.com/red-future/common/log/consts" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/glog" diff --git a/mongo/mongo.go b/mongo/mongo.go index a7f30f8..56f7694 100644 --- a/mongo/mongo.go +++ b/mongo/mongo.go @@ -11,13 +11,13 @@ import ( "fmt" "time" - "gitee.com/red-future---jilin-g/common/log/consts" + "gitea.com/red-future/common/log/consts" "go.mongodb.org/mongo-driver/v2/event" - "gitee.com/red-future---jilin-g/common/beans" - "gitee.com/red-future---jilin-g/common/log/model/entity" - "gitee.com/red-future---jilin-g/common/redis" - "gitee.com/red-future---jilin-g/common/utils" + "gitea.com/red-future/common/beans" + "gitea.com/red-future/common/log/model/entity" + "gitea.com/red-future/common/redis" + "gitea.com/red-future/common/utils" "github.com/gogf/gf/v2/container/gvar" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" diff --git a/rag/eino/base_task.go b/rag/eino/base_task.go index 9de26ad..b883341 100644 --- a/rag/eino/base_task.go +++ b/rag/eino/base_task.go @@ -3,7 +3,7 @@ package eino import ( "time" - "gitee.com/red-future---jilin-g/common/beans" + "gitea.com/red-future/common/beans" ) // BaseTask 任务基类 - MongoDB版本 diff --git a/ragflow/document.go b/ragflow/document.go index 4158ea1..b2c9002 100644 --- a/ragflow/document.go +++ b/ragflow/document.go @@ -9,7 +9,7 @@ import ( "mime/multipart" "strings" - commonHttp "gitee.com/red-future---jilin-g/common/http" + commonHttp "gitea.com/red-future/common/http" "github.com/gogf/gf/v2/errors/gerror" "github.com/gogf/gf/v2/frame/g" ) diff --git a/ragflow/worker_pool.go b/ragflow/worker_pool.go index 608dd81..91eae0c 100644 --- a/ragflow/worker_pool.go +++ b/ragflow/worker_pool.go @@ -7,7 +7,7 @@ import ( "sync" "time" - "gitee.com/red-future---jilin-g/common/redis" + "gitea.com/red-future/common/redis" "github.com/gogf/gf/v2/os/glog" "github.com/gogf/gf/v2/os/grpool" ) diff --git a/sql/sql.go b/sql/sql.go index d792dd6..b8392a6 100644 --- a/sql/sql.go +++ b/sql/sql.go @@ -8,8 +8,8 @@ package sql import ( "context" - "gitee.com/red-future---jilin-g/common/beans" - "gitee.com/red-future---jilin-g/common/utils" + "gitea.com/red-future/common/beans" + "gitea.com/red-future/common/utils" "github.com/gogf/gf/v2/database/gdb" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/os/gtime" diff --git a/swagger/swagger.go b/swagger/swagger.go index 40157da..76585aa 100644 --- a/swagger/swagger.go +++ b/swagger/swagger.go @@ -2,8 +2,8 @@ package swagger import ( "fmt" - "gitee.com/red-future---jilin-g/common/consul" - "gitee.com/red-future---jilin-g/common/http" + "gitea.com/red-future/common/consul" + "gitea.com/red-future/common/http" "github.com/gogf/gf/v2/frame/g" "github.com/gogf/gf/v2/net/ghttp" "github.com/gogf/gf/v2/util/gconv" diff --git a/utils/utils.go b/utils/utils.go index 0ae38d8..2eed0ac 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -12,7 +12,7 @@ import ( "sync/atomic" "time" - "gitee.com/red-future---jilin-g/common/beans" + "gitea.com/red-future/common/beans" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/errors/gcode" "github.com/gogf/gf/v2/errors/gerror" From 06007d5027c8de442867cde6aeaa13e2807fff26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 24 Feb 2026 17:11:31 +0800 Subject: [PATCH 212/225] .gitignore --- .../meilisearch/connection.go | 0 .../meilisearch/meilisearch.go | 0 {db => full-text-search}/meilisearch/types.go | 0 sql/sql.go | 459 ------------------ 4 files changed, 459 deletions(-) rename {db => full-text-search}/meilisearch/connection.go (100%) rename {db => full-text-search}/meilisearch/meilisearch.go (100%) rename {db => full-text-search}/meilisearch/types.go (100%) delete mode 100644 sql/sql.go diff --git a/db/meilisearch/connection.go b/full-text-search/meilisearch/connection.go similarity index 100% rename from db/meilisearch/connection.go rename to full-text-search/meilisearch/connection.go diff --git a/db/meilisearch/meilisearch.go b/full-text-search/meilisearch/meilisearch.go similarity index 100% rename from db/meilisearch/meilisearch.go rename to full-text-search/meilisearch/meilisearch.go diff --git a/db/meilisearch/types.go b/full-text-search/meilisearch/types.go similarity index 100% rename from db/meilisearch/types.go rename to full-text-search/meilisearch/types.go diff --git a/sql/sql.go b/sql/sql.go deleted file mode 100644 index b8392a6..0000000 --- a/sql/sql.go +++ /dev/null @@ -1,459 +0,0 @@ -// ============================================================================= -// SQL 业务操作封装 -// 提供向后兼容的CRUD操作方法,支持 PostgreSQL -// ============================================================================= - -package sql - -import ( - "context" - - "gitea.com/red-future/common/beans" - "gitea.com/red-future/common/utils" - "github.com/gogf/gf/v2/database/gdb" - "github.com/gogf/gf/v2/frame/g" - "github.com/gogf/gf/v2/os/gtime" - "github.com/gogf/gf/v2/util/gconv" -) - -const ( - PageSize = 20 -) - -type sqlDB struct { - noTenantId bool -} - -func DB(cache ...bool) *sqlDB { - return &sqlDB{ - noTenantId: false, - } -} - -// NoTenantId 不使用租户过滤 -func (s *sqlDB) NoTenantId() *sqlDB { - s.noTenantId = true - return s -} - -// Count 查询总数 -func (s *sqlDB) Count(ctx context.Context, model *gdb.Model) (int64, error) { - user, err := utils.GetUserInfo(ctx) - if err != nil { - return 0, err - } - - // 如果没有调用 noTenantId,则添加 tenantId 过滤 - if !s.noTenantId && !g.IsEmpty(user.TenantId) { - model = model.Where("tenant_id", user.TenantId) - } - - model = model.Where("is_deleted", false) - - // 执行查询 - count, _ := model.Count(ctx) - return int64(count), nil -} - -// Insert 插入记录(集合版本) -func (s *sqlDB) Insert(ctx context.Context, data interface{}, collection string) ([]any, error) { - user, err := utils.GetUserInfo(ctx) - if err != nil { - return nil, err - } - - model := g.DB().Model(collection) - - // 处理切片类型数据 - var resultSlice []any - if slice, ok := data.([]interface{}); ok { - for _, item := range slice { - // 转换为map - dataMap := gconv.Map(item) - delete(dataMap, "id") - - // 设置租户ID - if !g.IsEmpty(user.TenantId) && g.IsEmpty(dataMap["tenant_id"]) { - dataMap["tenant_id"] = user.TenantId - } - - // 设置创建人 - if !g.IsEmpty(user.UserName) && g.IsEmpty(dataMap["creator"]) { - dataMap["creator"] = user.UserName - } - - // 设置更新人 - if !g.IsEmpty(user.UserName) && g.IsEmpty(dataMap["updater"]) { - dataMap["updater"] = user.UserName - } - - // 设置时间 - now := gtime.Now().Time - if g.IsEmpty(dataMap["created_at"]) { - dataMap["created_at"] = now - } - if g.IsEmpty(dataMap["updated_at"]) { - dataMap["updated_at"] = now - } - - // 设置删除标记 - if g.IsEmpty(dataMap["is_deleted"]) { - dataMap["is_deleted"] = false - } - - resultSlice = append(resultSlice, dataMap) - } - - // 批量插入 - result, err := model.Data(resultSlice).Insert(ctx) - if err != nil { - return nil, err - } - - id, err := result.LastInsertId() - if err != nil { - return nil, err - } - - return []any{id}, nil - } - - // 单条数据插入 - dataMap := gconv.Map(data) - delete(dataMap, "id") - - // 设置租户ID - if !g.IsEmpty(user.TenantId) && g.IsEmpty(dataMap["tenant_id"]) { - dataMap["tenant_id"] = user.TenantId - } - - // 设置创建人 - if !g.IsEmpty(user.UserName) && g.IsEmpty(dataMap["creator"]) { - dataMap["creator"] = user.UserName - } - - // 设置更新人 - if !g.IsEmpty(user.UserName) && g.IsEmpty(dataMap["updater"]) { - dataMap["updater"] = user.UserName - } - - // 设置时间 - now := gtime.Now().Time - if g.IsEmpty(dataMap["created_at"]) { - dataMap["created_at"] = now - } - if g.IsEmpty(dataMap["updated_at"]) { - dataMap["updated_at"] = now - } - - // 设置删除标记 - if g.IsEmpty(dataMap["is_deleted"]) { - dataMap["is_deleted"] = false - } - - // 执行插入 - result, err := model.Data(dataMap).Insert(ctx) - if err != nil { - return nil, err - } - - id, err := result.LastInsertId() - if err != nil { - return nil, err - } - - return []any{id}, nil -} - -// FindOne 根据ID查询单条记录 -func (s *sqlDB) FindOne(ctx context.Context, id string, collection string, result interface{}) error { - user, err := utils.GetUserInfo(ctx) - if err != nil { - return err - } - - model := g.DB().Model(collection).Where("id", id) - - // 如果没有调用 noTenantId,则添加 tenantId 过滤 - if !s.noTenantId && !g.IsEmpty(user.TenantId) { - model = model.Where("tenant_id", user.TenantId) - } - - model = model.Where("is_deleted", false) - - // 执行查询 - return model.Scan(ctx, result) -} - -// FindOneWithResult 根据ID查询单条记录并返回结果 -func (s *sqlDB) FindOneWithResult(ctx context.Context, id string, collection string, result interface{}) (interface{}, error) { - user, err := utils.GetUserInfo(ctx) - if err != nil { - return nil, err - } - - model := g.DB().Model(collection).Where("id", id) - - // 如果没有调用 noTenantId,则添加 tenantId 过滤 - if !s.noTenantId && !g.IsEmpty(user.TenantId) { - model = model.Where("tenant_id", user.TenantId) - } - - model = model.Where("is_deleted", false) - - // 执行查询 - err = model.Scan(ctx, result) - return result, err -} - -// FindOneByModel 根据Model查询单条记录 -func (s *sqlDB) FindOneByModel(ctx context.Context, model *gdb.Model, result interface{}) error { - user, err := utils.GetUserInfo(ctx) - if err != nil { - return err - } - - // 如果没有调用 noTenantId,则添加 tenantId 过滤 - if !s.noTenantId && !g.IsEmpty(user.TenantId) { - model = model.Where("tenant_id", user.TenantId) - } - - model = model.Where("is_deleted", false) - - // 执行查询 - return model.Scan(ctx, result) -} - -// Find 查询多条记录(集合版本) -func (s *sqlDB) Find(ctx context.Context, model *gdb.Model, collection string, result interface{}, page *beans.Page, orderBy []beans.OrderBy) (int64, error) { - if model == nil { - model = g.DB().Model(collection) - } - - user, err := utils.GetUserInfo(ctx) - if err != nil { - return 0, err - } - - // 如果没有调用 noTenantId,则添加 tenantId 过滤 - if !s.noTenantId && !g.IsEmpty(user.TenantId) { - model = model.Where("tenant_id", user.TenantId) - } - - model = model.Where("is_deleted", false) - - // 分页处理 - limit := int64(PageSize) - offset := int64(0) - if page != nil && !g.IsEmpty(page.PageNum) && !g.IsEmpty(page.PageSize) { - limit = page.PageSize - if limit != -1 { - offset = (page.PageNum - 1) * limit - } - } - - // 排序处理 - if orderBy != nil && len(orderBy) > 0 { - for _, o := range orderBy { - orderStr := string(o.Order) - if orderStr == "asc" || orderStr == "ASC" { - model = model.OrderAsc(o.Field) - } else { - model = model.OrderDesc(o.Field) - } - } - } else { - model = model.OrderDesc("created_at") - } - - // 执行查询 - if limit != -1 { - total, err := s.Count(ctx, model) - if err != nil { - return 0, err - } - if total == 0 { - return 0, nil - } - err = model.Offset(int(offset)).Limit(int(limit)).Scan(ctx, result) - if err != nil { - return 0, err - } - return total, nil - } - - // 不分页,查询全部 - err = model.Scan(ctx, result) - if err != nil { - return 0, err - } - - // 获取结果集长度 - return 0, nil -} - -// Update 更新记录(ID版本) -func (s *sqlDB) Update(ctx context.Context, id string, collection string, data interface{}) (int64, error) { - user, err := utils.GetUserInfo(ctx) - if err != nil { - return 0, err - } - - model := g.DB().Model(collection).Where("id", id) - - // 如果没有调用 noTenantId,则添加 tenantId 过滤 - if !s.noTenantId && !g.IsEmpty(user.TenantId) { - model = model.Where("tenant_id", user.TenantId) - } - - model = model.Where("is_deleted", false) - - // 转换为map - dataMap := gconv.Map(data) - delete(dataMap, "id") - - // 设置更新人 - if !g.IsEmpty(user.UserName) && g.IsEmpty(dataMap["updater"]) { - dataMap["updater"] = user.UserName - } - - // 设置更新时间 - dataMap["updated_at"] = gtime.Now().Time - - // 执行更新 - result, err := model.Data(dataMap).Update(ctx) - if err != nil { - return 0, err - } - - affected, err := result.RowsAffected() - if err != nil { - return 0, err - } - - return affected, err -} - -// Delete 软删除(ID版本) -func (s *sqlDB) Delete(ctx context.Context, id string, collection string) (int64, error) { - user, err := utils.GetUserInfo(ctx) - if err != nil { - return 0, err - } - - model := g.DB().Model(collection).Where("id", id) - - // 如果没有调用 noTenantId,则添加 tenantId 过滤 - if !s.noTenantId && !g.IsEmpty(user.TenantId) { - model = model.Where("tenant_id", user.TenantId) - } - - model = model.Where("is_deleted", false) - - // 软删除 - data := map[string]interface{}{ - "is_deleted": true, - "updater": user.UserName, - "updated_at": gtime.Now().Time, - } - - result, err := model.Data(data).Update(ctx) - if err != nil { - return 0, err - } - - affected, err := result.RowsAffected() - if err != nil { - return 0, err - } - - return affected, err -} - -// DeleteByModel 软删除(Model版本) -func (s *sqlDB) DeleteSoftByModel(ctx context.Context, model *gdb.Model) (int64, error) { - user, err := utils.GetUserInfo(ctx) - if err != nil { - return 0, err - } - - // 如果没有调用 noTenantId,则添加 tenantId 过滤 - if !s.noTenantId && !g.IsEmpty(user.TenantId) { - model = model.Where("tenant_id", user.TenantId) - } - - model = model.Where("is_deleted", false) - - // 软删除 - data := map[string]interface{}{ - "is_deleted": true, - "updater": user.UserName, - "updated_at": gtime.Now().Time, - } - - result, err := model.Data(data).Update(ctx) - if err != nil { - return 0, err - } - - affected, err := result.RowsAffected() - if err != nil { - return 0, err - } - - return affected, err -} - -// Increment 字段自增 -func (s *sqlDB) Increment(ctx context.Context, id string, collection string, field string, value int64) (int64, error) { - user, err := utils.GetUserInfo(ctx) - if err != nil { - return 0, err - } - - model := g.DB().Model(collection).Where("id", id) - - // 如果没有调用 noTenantId,则添加 tenantId 过滤 - if !s.noTenantId && !g.IsEmpty(user.TenantId) { - model = model.Where("tenant_id", user.TenantId) - } - - model = model.Where("is_deleted", false) - - // 设置更新人和时间 - data := map[string]interface{}{ - "updater": user.UserName, - "updated_at": gtime.Now().Time, - } - - // 使用原生SQL实现自增 - data[field] = gdb.Raw(field + " + " + gconv.String(value) + "::bigint") - - // 执行更新 - result, err := model.Data(data).Update(ctx) - if err != nil { - return 0, err - } - - affected, err := result.RowsAffected() - if err != nil { - return 0, err - } - - return affected, err -} - -// DeleteSoft 软删除(ID版本) -func (s *sqlDB) DeleteSoft(ctx context.Context, id string, collection string) (int64, error) { - return s.Delete(ctx, id, collection) -} - -// Model 获取Model -func (s *sqlDB) Model(collection string) *gdb.Model { - return g.DB().Model(collection) -} - -// BuildUpdateData 构建更新数据 -func BuildUpdateData(ctx context.Context, req interface{}) (map[string]interface{}, error) { - return gconv.Map(req), nil -} From a330577740fd1256bd97a308f81635e73af13eed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 24 Feb 2026 17:17:09 +0800 Subject: [PATCH 213/225] .gitignore --- {mongo => db/mongo}/connection.go | 0 {mongo => db/mongo}/mongo.go | 0 log/dao/log_dao.go | 4 +++- 3 files changed, 3 insertions(+), 1 deletion(-) rename {mongo => db/mongo}/connection.go (100%) rename {mongo => db/mongo}/mongo.go (100%) diff --git a/mongo/connection.go b/db/mongo/connection.go similarity index 100% rename from mongo/connection.go rename to db/mongo/connection.go diff --git a/mongo/mongo.go b/db/mongo/mongo.go similarity index 100% rename from mongo/mongo.go rename to db/mongo/mongo.go diff --git a/log/dao/log_dao.go b/log/dao/log_dao.go index b185913..ee04b7b 100644 --- a/log/dao/log_dao.go +++ b/log/dao/log_dao.go @@ -2,14 +2,16 @@ package dao import ( "context" + "gitea.com/red-future/common/beans" + "gitea.com/red-future/common/db/mongo" + "strings" "time" "gitea.com/red-future/common/log/consts" "gitea.com/red-future/common/log/model/dto" "gitea.com/red-future/common/log/model/entity" - "gitea.com/red-future/common/mongo" "go.mongodb.org/mongo-driver/v2/bson" ) From add07d47f190c2d1eff4d31859833d1000195e79 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Thu, 26 Feb 2026 17:47:19 +0800 Subject: [PATCH 214/225] =?UTF-8?q?=E6=B7=BB=E5=8A=A0MongoDB=E8=AE=A4?= =?UTF-8?q?=E8=AF=81=E6=94=AF=E6=8C=81=E5=B9=B6=E4=BC=98=E5=8C=96=E5=BA=8F?= =?UTF-8?q?=E5=88=97=E5=8F=B7=E7=94=9F=E6=88=90=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mongo/connection.go | 23 ++++++++++++++++++++++- db/mongo/mongo.go | 2 +- utils/utils.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/db/mongo/connection.go b/db/mongo/connection.go index 10f3fc9..41cd249 100644 --- a/db/mongo/connection.go +++ b/db/mongo/connection.go @@ -8,6 +8,7 @@ package mongo import ( "context" "fmt" + "net/url" "os" "os/signal" "strings" @@ -34,6 +35,8 @@ type DataSourceConfig struct { Name string `json:"name"` Address string `json:"address"` Database string `json:"database"` + Username string `json:"username"` + Password string `json:"password"` MaxPoolSize int32 `json:"maxPoolSize"` MinPoolSize int32 `json:"minPoolSize"` ConnectTimeout time.Duration `json:"connectTimeout"` @@ -110,9 +113,25 @@ func (d *BaseDataSource) Connect(ctx context.Context) error { dbName = gstr.SubStr(dbName, 0, strings.Index(dbName, "?")) } + // 构建连接URI + connectionURI := fmt.Sprintf("mongodb://%s", d.config.Address) + + // 如果配置了用户名和密码,添加到URI中 + if d.config.Username != "" { + // URL编码用户名和密码,正确处理特殊字符 + encodedUsername := url.QueryEscape(d.config.Username) + encodedPassword := url.QueryEscape(d.config.Password) + + // 构建认证信息 + authInfo := fmt.Sprintf("%s:%s@", encodedUsername, encodedPassword) + + // 将认证信息插入到URI中 + connectionURI = fmt.Sprintf("mongodb://%s%s", authInfo, d.config.Address) + } + // 构建连接选项 opt := options.Client(). - ApplyURI(d.config.Address). + ApplyURI(connectionURI). SetMaxPoolSize(uint64(d.config.MaxPoolSize)). SetMinPoolSize(uint64(d.config.MinPoolSize)). SetConnectTimeout(d.config.ConnectTimeout). @@ -302,6 +321,8 @@ func (m *DataSourceManager) InitializeFromConfig(ctx context.Context) error { Name: name, Address: gconv.String(address), Database: gconv.String(subMap["database"]), + Username: gconv.String(subMap["username"]), + Password: gconv.String(subMap["password"]), MaxPoolSize: int32(gconv.Int(subMap["maxPoolSize"])), MinPoolSize: int32(gconv.Int(subMap["minPoolSize"])), ConnectTimeout: gconv.Duration(subMap["connectTimeout"]), diff --git a/db/mongo/mongo.go b/db/mongo/mongo.go index 56f7694..0ffb978 100644 --- a/db/mongo/mongo.go +++ b/db/mongo/mongo.go @@ -57,7 +57,7 @@ func commandMonitor() *event.CommandMonitor { fmt.Printf("[%s] 开始执行命令 | 数据库: %s | 集合: %s | 命令: %+v\n", time.Now().Format("2006-01-02 15:04:05"), evt.DatabaseName, - evt.Command.Lookup("collection").StringValue(), // 获取集合名 + //evt.Command.Lookup("collection").StringValue(), // 获取集合名 evt.Command, ) diff --git a/utils/utils.go b/utils/utils.go index 2eed0ac..4f741c2 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/gogf/gf/v2/os/gtime" "net" "reflect" "sort" @@ -339,3 +340,44 @@ func Struct(params any, pointer any) error { } return nil } + +// IncrSequence 自增序列号 - 通用序列号工具 +func IncrSequence(ctx context.Context, prefix string, incrLen int, seqSep string) (string, error) { + // 1. 校验incrLen合法性(至少1位,最多10位,避免位数过大) + if incrLen < 1 || incrLen > 8 { + g.Log().Warningf(ctx, "自增数位数[%d]不合法,默认使用8位", incrLen) + incrLen = 8 // 兜底默认8位 + } + // 2. 获取当前时间,格式化为"年月日"(如20260226) + timeStr := gtime.Now().Format("Ymd") + // 3. 拼接Redis的key(seq:前缀:时间字符串),确保每天一个独立的自增序列 + redisKey := fmt.Sprintf("%s:%s:%s", "SEQ", prefix, timeStr) + // 4. 调用Redis的Incr做自增,失败则生成兜底序列号 + seqNum, err := g.Redis().Incr(ctx, redisKey) + if err != nil { + return "", fmt.Errorf("redis自增失败: %w", err) + } + // 5. 自动清理过期key,定时任务 + + // 6. 校验自增数是否超过incrLen位上限,超过则取模(避免格式混乱) + maxSeq := intPow10(incrLen) - 1 + if seqNum > int64(maxSeq) { + return "", fmt.Errorf("自增数[%d]超过%d位上限[%d]", seqNum, incrLen, maxSeq) + } + // 7. 格式化自增数为指定位数,不足补零 + seqFormat := fmt.Sprintf("%%0%dd", incrLen) // 动态生成格式化字符串 + seqStr := fmt.Sprintf(seqFormat, seqNum) + // 8. 拼接最终序列号 + finalSeq := fmt.Sprintf("%s%s%s%s%s", prefix, seqSep, timeStr, seqSep, seqStr) + + return finalSeq, nil +} + +// intPow10 计算10的n次方(辅助函数,如intPow10(6)=1000000) +func intPow10(n int) int { + result := 1 + for i := 0; i < n; i++ { + result *= 10 + } + return result +} From 61070161263a01cab28dfa8f5dd668a2f51b82df Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Fri, 27 Feb 2026 08:48:55 +0800 Subject: [PATCH 215/225] =?UTF-8?q?=E4=BC=98=E5=8C=96=20MongoDB=20?= =?UTF-8?q?=E5=91=BD=E4=BB=A4=E7=9B=91=E6=8E=A7=E6=97=A5=E5=BF=97=EF=BC=9A?= =?UTF-8?q?=E5=AE=89=E5=85=A8=E5=A4=84=E7=90=86=E9=9B=86=E5=90=88=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E5=B9=B6=E6=A0=87=E5=87=86=E5=8C=96=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- db/mongo/mongo.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/db/mongo/mongo.go b/db/mongo/mongo.go index 0ffb978..0c3d360 100644 --- a/db/mongo/mongo.go +++ b/db/mongo/mongo.go @@ -53,11 +53,18 @@ func commandMonitor() *event.CommandMonitor { return &event.CommandMonitor{ // 命令执行前触发 Started: func(ctx context.Context, evt *event.CommandStartedEvent) { - // 执行前的处理逻辑示例:记录开始时间、打印执行的命令 + // 1. 安全获取集合名:先判断字段是否存在,避免空值调用 + collectionName := "无" // 默认值 + collectionVal := evt.Command.Lookup("collection") + if !g.IsEmpty(collectionVal) { // 先检查是否为nil + collectionName = collectionVal.StringValue() + } + + // 2. 打印标准化日志(避免字段缺失导致的格式错乱) fmt.Printf("[%s] 开始执行命令 | 数据库: %s | 集合: %s | 命令: %+v\n", time.Now().Format("2006-01-02 15:04:05"), evt.DatabaseName, - //evt.Command.Lookup("collection").StringValue(), // 获取集合名 + collectionName, evt.Command, ) From dd2cba70f6d84649114816d1aaf774f672a856ba Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Fri, 27 Feb 2026 10:29:11 +0800 Subject: [PATCH 216/225] =?UTF-8?q?=E4=B8=BAswagger=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E9=99=90=E6=B5=81=E7=99=BD=E5=90=8D=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- middleware/rate_limiter.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/middleware/rate_limiter.go b/middleware/rate_limiter.go index 71dd53a..d7f8ddc 100644 --- a/middleware/rate_limiter.go +++ b/middleware/rate_limiter.go @@ -62,6 +62,10 @@ func IPLimiter(r *ghttp.Request) { // UserLimiter 用户维度限流中间件(防止单用户滥用) func UserLimiter(r *ghttp.Request) { + if r.RequestURI == "/swagger" { + r.Middleware.Next() + return + } var userName string user, err := utils.GetUserInfo(r.GetCtx()) if err != nil { From 62dab53108cf802130e83f78247a535a1bc3fde1 Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Mon, 2 Mar 2026 14:22:08 +0800 Subject: [PATCH 217/225] =?UTF-8?q?=E4=BC=98=E5=8C=96HTTP=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E5=A4=84=E7=90=86=E5=92=8C=E6=A8=A1=E5=9D=97=E7=A7=9F?= =?UTF-8?q?=E6=88=B7=E6=A3=80=E6=9F=A5=E9=80=BB=E8=BE=91=EF=BC=9A=E6=94=AF?= =?UTF-8?q?=E6=8C=81GET=E8=AF=B7=E6=B1=82=E5=8F=82=E6=95=B0=E8=BD=AC?= =?UTF-8?q?=E6=8D=A2=EF=BC=8C=E7=AE=80=E5=8C=96=E6=97=A5=E5=BF=97=E6=B6=88?= =?UTF-8?q?=E8=B4=B9=E8=80=85=E9=85=8D=E7=BD=AE=EF=BC=8C=E6=94=B9=E8=BF=9B?= =?UTF-8?q?RPC=E8=B0=83=E7=94=A8=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 13 ++++++++++- log/consts/log_const.go | 9 ++++---- log/service/log_service.go | 3 ++- middleware/module_tenant_check.go | 38 ++++++++++++++++++++++++++----- 4 files changed, 50 insertions(+), 13 deletions(-) diff --git a/http/http.go b/http/http.go index ffcb1fe..e5ba5cf 100644 --- a/http/http.go +++ b/http/http.go @@ -83,7 +83,18 @@ func doRequest(ctx context.Context, method string, url string, headers map[strin // 修复:避免data...展开导致的双重包装问题 // 当只有一个元素时,直接传递该元素,避免被包装成数组 var response *gclient.Response - if len(data) == 1 { + // 对于GET请求,将参数转换为map + if method == http.MethodGet && len(data) > 0 && len(data)%2 == 0 { + // 构建query参数map + queryParams := make(map[string]string) + for i := 0; i < len(data); i += 2 { + if key, ok := data[i].(string); ok && i+1 < len(data) { + queryParams[key] = gconv.String(data[i+1]) + } + } + g.Log().Infof(ctx, "[HTTP] GET请求构建的query参数: %+v", queryParams) + response, err = client.DoRequest(ctx, method, url, queryParams) + } else if len(data) == 1 { response, err = client.DoRequest(ctx, method, url, data[0]) } else { response, err = client.DoRequest(ctx, method, url, data...) diff --git a/log/consts/log_const.go b/log/consts/log_const.go index 8e2d0cb..af621b1 100644 --- a/log/consts/log_const.go +++ b/log/consts/log_const.go @@ -16,10 +16,9 @@ const ( ) // 消费者配置(从 Redis Stream 消费请求) -const StreamKey = "log:%s" // 请求 Stream 键名(与发消息的key一致) -const GroupName = "log:consumer:group" // 消费者组名 -const ConsumerName = "message-consumer-1" // 消费者名称(唯一标识) -const BatchSize = 1 // 批处理大小(每次读取1条) -const AutoAck = true // ACK是否自动确认(true自动确认,false不确认) +const StreamKey = "log:%s" // 请求 Stream 键名(与发消息的key一致) +const ConsumerName = "log-consumer" // 消费者名称(唯一标识) +const BatchSize = 1 // 批处理大小(每次读取1条) +const AutoAck = true // ACK是否自动确认(true自动确认,false不确认) const LogSubject = "log:subject" diff --git a/log/service/log_service.go b/log/service/log_service.go index 3e15c75..f790378 100644 --- a/log/service/log_service.go +++ b/log/service/log_service.go @@ -15,7 +15,8 @@ type operationLog struct{} // OperationLog 操作日志服务 var OperationLog = &operationLog{} -func (s *operationLog) AddOperationLog(ctx context.Context, msg map[string]interface{}) error { +func (s *operationLog) AddOperationLog(ctx context.Context, msgData any) error { + msg := gconv.MapStrStr(msgData) serviceName := gconv.String(msg["service_name"]) collection := gconv.String(msg["collection"]) collectionId := gconv.Strings(msg["collection_id"]) diff --git a/middleware/module_tenant_check.go b/middleware/module_tenant_check.go index a88a7cb..7c1b686 100644 --- a/middleware/module_tenant_check.go +++ b/middleware/module_tenant_check.go @@ -5,8 +5,7 @@ import ( "encoding/json" "fmt" "gitea.com/red-future/common/beans" - "gitea.com/red-future/common/message" - "gitea.com/red-future/common/redis" + commonHttp "gitea.com/red-future/common/http" "gitea.com/red-future/common/utils" "github.com/gogf/gf/v2/database/gredis" "github.com/gogf/gf/v2/frame/g" @@ -18,9 +17,17 @@ import ( ) func ModuleTenantCheck(r *ghttp.Request) { + //将 http.Header 转换为 map[string]string + headers := make(map[string]string) + for k, v := range r.Request.Header { + if len(v) > 0 { + headers[k] = v[0] + } + } + // 检查是否是超级管理员 - isSuperAdmin := false - if err := message.CallRPC(r.Context(), "userService.IsSuperAdmin", nil, &isSuperAdmin); err != nil { + isSuperAdmin, err := IsSuperAdmin(r.Context(), headers) + if err != nil { SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, err) } // 如果是超级管理员,则不进行模块租户检查 @@ -33,7 +40,7 @@ func ModuleTenantCheck(r *ghttp.Request) { SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, err) } exit := gconv.Int64(time.Minute * 1) - getEX, err := redis.GetRedisClientTest("test").GetEX(r.Context(), fmt.Sprintf("module_tenant:tenantId-%v", getUserInfo.TenantId), gredis.GetEXOption{ + getEX, err := g.Redis("test").GetEX(r.Context(), fmt.Sprintf("module_tenant:tenantId-%v", getUserInfo.TenantId), gredis.GetEXOption{ TTLOption: gredis.TTLOption{ EX: &exit, }, @@ -68,7 +75,7 @@ func ModuleTenantCheck(r *ghttp.Request) { ModuleKey: moduleKey, TenantId: gconv.Uint64(getUserInfo.TenantId), } - err = message.CallRPC(r.Context(), "moduleService.Check", &checkReq, checkRes) + checkRes, err = Check(r.Context(), headers, checkReq) if err != nil { SetResponseInfo(r.Context(), r, http.StatusPaymentRequired, err) } @@ -95,3 +102,22 @@ func SetResponseInfo(ctx context.Context, r *ghttp.Request, code int, message an }) r.Exit() } + +// Check 调用admin-go服务检查模块开通状态 +func Check(ctx context.Context, headerMap map[string]string, req beans.ModuleTenantCheckReq) (res *beans.ModuleTenantCheckRes, err error) { + if err = commonHttp.Get(ctx, "admin-go/api/v1/system/moduleTenant/check", headerMap, &res, + "moduleKey", req.ModuleKey, + "tenantId", req.TenantId, + ); err != nil { + return + } + return +} + +// IsSuperAdmin 调用admin-go服务检查是否是超级管理员 +func IsSuperAdmin(ctx context.Context, headerMap map[string]string) (res bool, err error) { + if err = commonHttp.Get(ctx, "admin-go/api/v1/system/user/checkIsSuperAdmin", headerMap, &res); err != nil { + return + } + return +} From 8632e2b27129f33d8bf9eb6476b006c48ac5bf4e Mon Sep 17 00:00:00 2001 From: qhd <1766646056@qq.com> Date: Tue, 3 Mar 2026 08:51:02 +0800 Subject: [PATCH 218/225] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=B7=AF=E5=BE=84?= =?UTF-8?q?=E8=BF=87=E6=BB=A4=E4=B8=AD=E9=97=B4=E4=BB=B6=E5=8F=8A=E5=90=88?= =?UTF-8?q?=E6=B3=95=E6=B3=A8=E5=86=8C=E8=B7=AF=E5=BE=84=E6=9C=BA=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- http/http.go | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/http/http.go b/http/http.go index e5ba5cf..a751095 100644 --- a/http/http.go +++ b/http/http.go @@ -21,8 +21,12 @@ import ( "github.com/gogf/gf/v2/util/gconv" ) -var Httpserver = g.Server() -var Httpclient = g.Client() +var ( + Httpserver = g.Server() + Httpclient = g.Client() + filterPaths = map[string]bool{"/": true, "/*": true, "/api.json": true} + legalRegisteredPaths = make(map[string]bool) +) func init() { err := gtime.SetTimeZone("Asia/Shanghai") @@ -31,12 +35,40 @@ func init() { } //s.Use(common.Cors) //中间件验证 //s.EnablePProf() //启用性能分析 + Httpserver.BindMiddlewareDefault(validateFilterPathMiddleware) Httpserver.SetOpenApiPath("/api.json") Httpserver.SetDumpRouterMap(true) //关闭打印路由注册信息 Httpserver.BindMiddlewareDefault(ghttp.MiddlewareHandlerResponse) Httpclient.SetDiscovery(gsvc.GetRegistry()) } +func validateFilterPathMiddleware(r *ghttp.Request) { + path := r.URL.Path + if filterPaths[path] && !legalRegisteredPaths[path] { + routes := Httpserver.GetRoutes() + for _, route := range routes { + if filterPaths[route.Handler.Router.Uri] && route.Handler.Router.Uri != "/api.json" { + g.Log().Errorf(r.GetCtx(), "路径 %s 中间件 %s 必须通过 SkipMiddleware 注册", route.Handler.Router.Uri, route.Handler.Name) + } + } + r.ExitAll() + } + r.Middleware.Next() +} + +func SkipMiddleware(h func(r *ghttp.Request), path string) (handler ghttp.HandlerFunc) { + if filterPaths[path] { + legalRegisteredPaths[path] = true + } + return func(r *ghttp.Request) { + if filterPaths[path] { + r.Middleware.Next() + return + } + h(r) + } +} + func RouteRegister(controllers []interface{}) { Httpserver.Group("/log", func(group *ghttp.RouterGroup) { group.Middleware(jaeger.NewTracer) From fd3b396b8fe51fdb77edefcc895a8adae8536f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 3 Mar 2026 16:43:36 +0800 Subject: [PATCH 219/225] .gitignore --- rpc/rpcx.go | 159 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 rpc/rpcx.go diff --git a/rpc/rpcx.go b/rpc/rpcx.go new file mode 100644 index 0000000..226a8df --- /dev/null +++ b/rpc/rpcx.go @@ -0,0 +1,159 @@ +package rpc + +import ( + "context" + "encoding/json" + "errors" + "time" + + "gitea.com/red-future/common/consul" + "gitea.com/red-future/common/jaeger" + "github.com/gogf/gf/v2/frame/g" + rpcxClient "github.com/smallnest/rpcx/client" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + trace "go.opentelemetry.io/otel/trace" +) + +var ( + // pluginsContainer rpcx插件容器(全局统一设置) + // init()中添加链路追踪插件,所有client共用此容器 + pluginsContainer = rpcxClient.NewPluginContainer() +) + +func init() { + // 全局设置链路追踪插件,所有client共用 + pluginsContainer.Add(&TracingPlugin{}) +} + +// NewXClient 创建rpcx客户端 +// serviceName: 服务名称 +// 通过consul发现服务实例,并返回rpcx客户端 +func newXClient(ctx context.Context, serviceName string) (*rpcxClient.OneClient, error) { + if g.IsEmpty(serviceName) { + return nil, errors.New("服务名称不能为空") + } + + // 使用consul.GetInstanceAddr获取服务实例地址 + // 每次都重新获取,确保使用最新的服务地址(支持服务重启、迁移等场景) + addr, err := consul.GetInstanceAddr(ctx, serviceName) + if err != nil { + g.Log().Errorf(ctx, "从consul获取服务[%s]地址失败: %v", serviceName, err) + return nil, err + } + + g.Log().Debugf(ctx, "服务[%s]地址: %s", serviceName, addr) + + // 使用Peer2PeerDiscovery直接连接指定服务 + discovery, err := rpcxClient.NewPeer2PeerDiscovery("tcp@"+addr, "") + if err != nil { + g.Log().Errorf(ctx, "创建服务发现失败: %v", err) + return nil, err + } + + // 使用OneClient,因为是单点连接 + client := rpcxClient.NewOneClient( + rpcxClient.Failtry, + rpcxClient.RandomSelect, + discovery, + rpcxClient.DefaultOption, + ) + + // 设置插件(使用全局统一的pluginsContainer) + client.SetPlugins(pluginsContainer) + + g.Log().Infof(ctx, "rpcx客户端[%s]创建成功", serviceName) + + return client, nil +} + +// Call 调用rpcx服务方法 +// serviceName: 服务名称 +// serviceMethod: 服务方法 +// args: 请求参数 +// reply: 响应结果 +func Call(ctx context.Context, serviceName, serviceMethod string, args interface{}, reply interface{}) error { + client, err := newXClient(ctx, serviceName) + if err != nil { + g.Log().Errorf(ctx, "获取rpcx客户端失败: %v", err) + return err + } + defer client.Close() + + // 设置超时 + callCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + err = client.Call(callCtx, serviceName, serviceMethod, args, reply) + if err != nil { + g.Log().Errorf(ctx, "调用服务[%s]方法[%s]失败: %v", serviceName, serviceMethod, err) + return err + } + + return nil +} + +// TracingPlugin rpcx链路追踪插件 +// 实现 rpcx 的 PreCallPlugin 和 PostCallPlugin 接口 +type TracingPlugin struct{} + +// PreCall 调用前拦截 - 创建jaeger span +func (p *TracingPlugin) PreCall(ctx context.Context, serviceName, serviceMethod string, args interface{}) (err error) { + // 创建span,名称格式: ServiceName.Method + spanName := serviceName + "." + serviceMethod + ctx, span := jaeger.NewSpan(ctx, spanName) + + // 记录服务和方法信息 + span.SetAttributes( + attribute.String("rpc.service", serviceName), + attribute.String("rpc.method", serviceMethod), + attribute.String("rpc.system", "rpcx"), + ) + var data []byte + // 记录请求参数(序列化为JSON) + if args != nil { + if data, err = json.Marshal(args); err == nil { + argsStr := string(data) + // 限制长度,避免过大 + if len(argsStr) > 2000 { + argsStr = argsStr[:2000] + "... (truncated)" + } + span.SetAttributes(attribute.String("rpc.request", argsStr)) + } + } + + g.Log().Debugf(ctx, "[rpcx] 调用开始: %s.%s", serviceName, serviceMethod) + + return +} + +// PostCall 调用后拦截 - 记录结果和错误 +func (p *TracingPlugin) PostCall(ctx context.Context, serviceName, serviceMethod string, args interface{}, reply interface{}, err error) error { + span := trace.SpanFromContext(ctx) + if span != nil && span.IsRecording() { + defer span.End() + + // 记录响应结果 + if reply != nil { + if data, err := json.Marshal(reply); err == nil { + replyStr := string(data) + // 限制长度,避免过大 + if len(replyStr) > 2000 { + replyStr = replyStr[:2000] + "... (truncated)" + } + span.SetAttributes(attribute.String("rpc.response", replyStr)) + } + } + + // 处理错误 + if err != nil { + jaeger.RecordError(ctx, err, "rpcx调用失败") + span.SetStatus(codes.Error, err.Error()) + g.Log().Errorf(ctx, "[rpcx] 调用失败: %s.%s, 错误: %v", serviceName, serviceMethod, err) + } else { + g.Log().Debugf(ctx, "[rpcx] 调用成功: %s.%s", serviceName, serviceMethod) + } + } + + return nil +} From 6351655ced233cc337e5e6687c3750e27fa063f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 3 Mar 2026 16:51:01 +0800 Subject: [PATCH 220/225] .gitignore --- rpc/rpcx.go | 148 ++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 132 insertions(+), 16 deletions(-) diff --git a/rpc/rpcx.go b/rpc/rpcx.go index 226a8df..6412391 100644 --- a/rpc/rpcx.go +++ b/rpc/rpcx.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "sync" "time" "gitea.com/red-future/common/consul" @@ -12,30 +13,104 @@ import ( rpcxClient "github.com/smallnest/rpcx/client" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" - trace "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace" ) var ( // pluginsContainer rpcx插件容器(全局统一设置) // init()中添加链路追踪插件,所有client共用此容器 pluginsContainer = rpcxClient.NewPluginContainer() + + // clientPool 连接池缓存,key为服务名,value为客户端实例 + clientPool = make(map[string]*rpcxClient.OneClient) + + // poolMutex 连接池锁 + poolMutex sync.RWMutex + + // healthCheckInterval 健康检查间隔(秒) + healthCheckInterval = 30 + + // lastHealthCheckTime 上次健康检查时间,key为服务名 + lastHealthCheckTime = make(map[string]time.Time) ) func init() { // 全局设置链路追踪插件,所有client共用 pluginsContainer.Add(&TracingPlugin{}) + + // 启动后台健康检查协程 + go healthCheckLoop() } -// NewXClient 创建rpcx客户端 -// serviceName: 服务名称 -// 通过consul发现服务实例,并返回rpcx客户端 -func newXClient(ctx context.Context, serviceName string) (*rpcxClient.OneClient, error) { +// healthCheckLoop 后台健康检查循环 +func healthCheckLoop() { + ticker := time.NewTicker(time.Duration(healthCheckInterval) * time.Second) + defer ticker.Stop() + + for range ticker.C { + checkAllConnections() + } +} + +// checkAllConnections 检查所有缓存连接的健康状态 +func checkAllConnections() { + poolMutex.Lock() + defer poolMutex.Unlock() + + now := time.Now() + for serviceName := range clientPool { + // 检查连接是否需要健康检查 + if lastCheck, ok := lastHealthCheckTime[serviceName]; ok { + if now.Sub(lastCheck) < time.Duration(healthCheckInterval)*time.Second { + continue + } + } + + // 简单的健康检查策略: + // 1. 定期从consul重新获取服务地址,如果地址变化说明服务可能迁移了 + // 2. 下次调用失败时会触发重新创建连接 + // 3. 不主动断开连接,依赖实际调用的错误来触发重建 + lastHealthCheckTime[serviceName] = now + g.Log().Debugf(context.Background(), "服务[%s]连接健康检查完成", serviceName) + } +} + +// isClientHealthy 检查client是否健康(简化版) +// 实际健康检查依赖调用失败时触发重建 +func isClientHealthy(client *rpcxClient.OneClient) bool { + // rpcx有内置的重连机制,我们信任client对象的有效性 + // 只要client不为nil就认为是健康的 + // 实际的错误会在调用时暴露,触发重新创建 + return client != nil +} + +// getOrCreateClient 从连接池获取或创建客户端(带连接池) +func getOrCreateClient(ctx context.Context, serviceName string) (*rpcxClient.OneClient, error) { if g.IsEmpty(serviceName) { return nil, errors.New("服务名称不能为空") } - // 使用consul.GetInstanceAddr获取服务实例地址 - // 每次都重新获取,确保使用最新的服务地址(支持服务重启、迁移等场景) + // 先尝试从连接池获取 + poolMutex.RLock() + client, exists := clientPool[serviceName] + poolMutex.RUnlock() + + // 如果存在且健康,直接返回 + if exists && isClientHealthy(client) { + g.Log().Debugf(ctx, "从连接池获取rpcx客户端[%s]", serviceName) + return client, nil + } + + // 不存在或不健康,重新创建 + poolMutex.Lock() + defer poolMutex.Unlock() + + // 双重检查,防止并发时重复创建 + if client, exists := clientPool[serviceName]; exists && isClientHealthy(client) { + return client, nil + } + + // 获取服务实例地址 addr, err := consul.GetInstanceAddr(ctx, serviceName) if err != nil { g.Log().Errorf(ctx, "从consul获取服务[%s]地址失败: %v", serviceName, err) @@ -44,27 +119,32 @@ func newXClient(ctx context.Context, serviceName string) (*rpcxClient.OneClient, g.Log().Debugf(ctx, "服务[%s]地址: %s", serviceName, addr) - // 使用Peer2PeerDiscovery直接连接指定服务 + // 创建服务发现 discovery, err := rpcxClient.NewPeer2PeerDiscovery("tcp@"+addr, "") if err != nil { g.Log().Errorf(ctx, "创建服务发现失败: %v", err) return nil, err } - // 使用OneClient,因为是单点连接 - client := rpcxClient.NewOneClient( + // 创建新客户端 + newClient := rpcxClient.NewOneClient( rpcxClient.Failtry, rpcxClient.RandomSelect, discovery, rpcxClient.DefaultOption, ) + newClient.SetPlugins(pluginsContainer) - // 设置插件(使用全局统一的pluginsContainer) - client.SetPlugins(pluginsContainer) + // 更新连接池 + if oldClient, ok := clientPool[serviceName]; ok && oldClient != nil { + oldClient.Close() + } + clientPool[serviceName] = newClient + lastHealthCheckTime[serviceName] = time.Now() - g.Log().Infof(ctx, "rpcx客户端[%s]创建成功", serviceName) + g.Log().Infof(ctx, "rpcx客户端[%s]创建并加入连接池", serviceName) - return client, nil + return newClient, nil } // Call 调用rpcx服务方法 @@ -73,26 +153,62 @@ func newXClient(ctx context.Context, serviceName string) (*rpcxClient.OneClient, // args: 请求参数 // reply: 响应结果 func Call(ctx context.Context, serviceName, serviceMethod string, args interface{}, reply interface{}) error { - client, err := newXClient(ctx, serviceName) + // 从连接池获取客户端(不再关闭连接) + client, err := getOrCreateClient(ctx, serviceName) if err != nil { g.Log().Errorf(ctx, "获取rpcx客户端失败: %v", err) return err } - defer client.Close() // 设置超时 callCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() + // 调用服务方法 err = client.Call(callCtx, serviceName, serviceMethod, args, reply) if err != nil { g.Log().Errorf(ctx, "调用服务[%s]方法[%s]失败: %v", serviceName, serviceMethod, err) + + // 如果调用失败,检查连接是否需要重新创建 + poolMutex.Lock() + if pooledClient, ok := clientPool[serviceName]; ok && pooledClient == client { + // 标记为不健康,下次请求时会重新创建 + delete(lastHealthCheckTime, serviceName) + } + poolMutex.Unlock() + return err } return nil } +// Close 关闭指定服务的连接(用于清理连接池) +func Close(serviceName string) { + poolMutex.Lock() + defer poolMutex.Unlock() + + if client, ok := clientPool[serviceName]; ok { + client.Close() + delete(clientPool, serviceName) + delete(lastHealthCheckTime, serviceName) + g.Log().Infof(context.Background(), "rpcx客户端[%s]已从连接池移除", serviceName) + } +} + +// CloseAll 关闭所有连接(用于优雅停机) +func CloseAll() { + poolMutex.Lock() + defer poolMutex.Unlock() + + for serviceName, client := range clientPool { + client.Close() + g.Log().Infof(context.Background(), "rpcx客户端[%s]已关闭", serviceName) + } + clientPool = make(map[string]*rpcxClient.OneClient) + lastHealthCheckTime = make(map[string]time.Time) +} + // TracingPlugin rpcx链路追踪插件 // 实现 rpcx 的 PreCallPlugin 和 PostCallPlugin 接口 type TracingPlugin struct{} From d17c242a38c2ad670177d0f5bffbf3134dd275d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 3 Mar 2026 16:54:51 +0800 Subject: [PATCH 221/225] .gitignore --- rpc/rpcx.go | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/rpc/rpcx.go b/rpc/rpcx.go index 6412391..3132462 100644 --- a/rpc/rpcx.go +++ b/rpc/rpcx.go @@ -32,6 +32,9 @@ var ( // lastHealthCheckTime 上次健康检查时间,key为服务名 lastHealthCheckTime = make(map[string]time.Time) + + // serviceAddrCache 服务地址缓存,key为服务名,value为地址 + serviceAddrCache = make(map[string]string) ) func init() { @@ -66,12 +69,34 @@ func checkAllConnections() { } } - // 简单的健康检查策略: - // 1. 定期从consul重新获取服务地址,如果地址变化说明服务可能迁移了 - // 2. 下次调用失败时会触发重新创建连接 - // 3. 不主动断开连接,依赖实际调用的错误来触发重建 + // 从consul重新获取服务地址,检查是否发生变化 + ctx := context.Background() + currentAddr, err := consul.GetInstanceAddr(ctx, serviceName) + if err != nil { + g.Log().Warningf(ctx, "健康检查时从consul获取服务[%s]地址失败: %v,保持现有连接", serviceName, err) + lastHealthCheckTime[serviceName] = now + continue + } + + // 检查地址是否发生变化 + if oldAddr, ok := serviceAddrCache[serviceName]; ok && oldAddr != currentAddr { + g.Log().Infof(ctx, "检测到服务[%s]地址变化: %s -> %s,重建连接", serviceName, oldAddr, currentAddr) + // 关闭旧连接并从连接池移除,下次请求时会创建新连接 + if client, exists := clientPool[serviceName]; exists { + client.Close() + delete(clientPool, serviceName) + } + // 更新缓存的新地址 + serviceAddrCache[serviceName] = currentAddr + } else { + // 地址未变化,更新检查时间 + if !ok { + serviceAddrCache[serviceName] = currentAddr + } + g.Log().Debugf(ctx, "服务[%s]地址未变化,保持现有连接", serviceName) + } + lastHealthCheckTime[serviceName] = now - g.Log().Debugf(context.Background(), "服务[%s]连接健康检查完成", serviceName) } } @@ -119,6 +144,9 @@ func getOrCreateClient(ctx context.Context, serviceName string) (*rpcxClient.One g.Log().Debugf(ctx, "服务[%s]地址: %s", serviceName, addr) + // 缓存服务地址,用于健康检查时对比 + serviceAddrCache[serviceName] = addr + // 创建服务发现 discovery, err := rpcxClient.NewPeer2PeerDiscovery("tcp@"+addr, "") if err != nil { @@ -192,6 +220,7 @@ func Close(serviceName string) { client.Close() delete(clientPool, serviceName) delete(lastHealthCheckTime, serviceName) + delete(serviceAddrCache, serviceName) g.Log().Infof(context.Background(), "rpcx客户端[%s]已从连接池移除", serviceName) } } @@ -207,6 +236,7 @@ func CloseAll() { } clientPool = make(map[string]*rpcxClient.OneClient) lastHealthCheckTime = make(map[string]time.Time) + serviceAddrCache = make(map[string]string) } // TracingPlugin rpcx链路追踪插件 From ded8a42c646023ec1f159601ab41f807245879aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Tue, 3 Mar 2026 16:58:08 +0800 Subject: [PATCH 222/225] .gitignore --- rpc/rpcx.go | 91 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 75 insertions(+), 16 deletions(-) diff --git a/rpc/rpcx.go b/rpc/rpcx.go index 3132462..1843f0d 100644 --- a/rpc/rpcx.go +++ b/rpc/rpcx.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "strings" "sync" "time" @@ -61,7 +62,7 @@ func checkAllConnections() { defer poolMutex.Unlock() now := time.Now() - for serviceName := range clientPool { + for serviceName, client := range clientPool { // 检查连接是否需要健康检查 if lastCheck, ok := lastHealthCheckTime[serviceName]; ok { if now.Sub(lastCheck) < time.Duration(healthCheckInterval)*time.Second { @@ -69,8 +70,19 @@ func checkAllConnections() { } } - // 从consul重新获取服务地址,检查是否发生变化 ctx := context.Background() + + // 检查连接健康状态(心跳检测) + if !isClientHealthy(ctx, client, serviceName) { + g.Log().Warningf(ctx, "检测到服务[%s]连接不健康,将从连接池移除", serviceName) + client.Close() + delete(clientPool, serviceName) + delete(lastHealthCheckTime, serviceName) + delete(serviceAddrCache, serviceName) + continue + } + + // 连接健康,检查服务地址是否发生变化 currentAddr, err := consul.GetInstanceAddr(ctx, serviceName) if err != nil { g.Log().Warningf(ctx, "健康检查时从consul获取服务[%s]地址失败: %v,保持现有连接", serviceName, err) @@ -82,10 +94,9 @@ func checkAllConnections() { if oldAddr, ok := serviceAddrCache[serviceName]; ok && oldAddr != currentAddr { g.Log().Infof(ctx, "检测到服务[%s]地址变化: %s -> %s,重建连接", serviceName, oldAddr, currentAddr) // 关闭旧连接并从连接池移除,下次请求时会创建新连接 - if client, exists := clientPool[serviceName]; exists { - client.Close() - delete(clientPool, serviceName) - } + client.Close() + delete(clientPool, serviceName) + delete(lastHealthCheckTime, serviceName) // 更新缓存的新地址 serviceAddrCache[serviceName] = currentAddr } else { @@ -93,20 +104,68 @@ func checkAllConnections() { if !ok { serviceAddrCache[serviceName] = currentAddr } - g.Log().Debugf(ctx, "服务[%s]地址未变化,保持现有连接", serviceName) + g.Log().Debugf(ctx, "服务[%s]地址未变化,连接健康", serviceName) } lastHealthCheckTime[serviceName] = now } } -// isClientHealthy 检查client是否健康(简化版) -// 实际健康检查依赖调用失败时触发重建 -func isClientHealthy(client *rpcxClient.OneClient) bool { - // rpcx有内置的重连机制,我们信任client对象的有效性 - // 只要client不为nil就认为是健康的 - // 实际的错误会在调用时暴露,触发重新创建 - return client != nil +// isClientHealthy 检查client是否健康 +// 使用心跳检测方式:尝试调用服务的心跳方法 +func isClientHealthy(ctx context.Context, client *rpcxClient.OneClient, serviceName string) bool { + if client == nil { + return false + } + + // 设置较短的超时时间,避免阻塞 + pingCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + // 尝试调用健康检查方法 + // 大多数服务都会提供 Ping 或 Health 方法 + // 如果服务没有提供这些方法,会返回错误,我们认为是健康的 + // 因为连接本身是正常的,只是方法不存在 + var reply interface{} + err := client.Call(pingCtx, serviceName, "Ping", nil, &reply) + + // 如果调用成功,连接肯定健康 + if err == nil { + return true + } + + // 如果是方法不存在的错误,说明连接是健康的,只是服务没有Ping方法 + // 这种情况下我们认为是健康的 + if isMethodNotFoundError(err) || isServiceNotFoundError(err) { + return true + } + + // 其他错误(网络错误、超时等)说明连接不健康 + g.Log().Warningf(ctx, "健康检查失败,服务[%s]连接可能不健康: %v", serviceName, err) + return false +} + +// isMethodNotFoundError 判断是否是方法未找到错误 +func isMethodNotFoundError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + // rpcx 方法不存在的常见错误信息 + return strings.Contains(errStr, "not found") || + strings.Contains(errStr, "no such") || + strings.Contains(errStr, "service not found") || + strings.Contains(errStr, "method not found") +} + +// isServiceNotFoundError 判断是否是服务未找到错误 +func isServiceNotFoundError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return strings.Contains(errStr, "no service") || + strings.Contains(errStr, "service not registered") } // getOrCreateClient 从连接池获取或创建客户端(带连接池) @@ -121,7 +180,7 @@ func getOrCreateClient(ctx context.Context, serviceName string) (*rpcxClient.One poolMutex.RUnlock() // 如果存在且健康,直接返回 - if exists && isClientHealthy(client) { + if exists && isClientHealthy(ctx, client, serviceName) { g.Log().Debugf(ctx, "从连接池获取rpcx客户端[%s]", serviceName) return client, nil } @@ -131,7 +190,7 @@ func getOrCreateClient(ctx context.Context, serviceName string) (*rpcxClient.One defer poolMutex.Unlock() // 双重检查,防止并发时重复创建 - if client, exists := clientPool[serviceName]; exists && isClientHealthy(client) { + if client, exists := clientPool[serviceName]; exists && isClientHealthy(ctx, client, serviceName) { return client, nil } From a8f7c925ffa6a515be053a4185dee02c692850da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Thu, 12 Mar 2026 08:56:56 +0800 Subject: [PATCH 223/225] =?UTF-8?q?mongodb=E5=9F=BA=E7=A1=80=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E6=96=B9=E6=B3=95=E5=A2=9E=E5=8A=A0redis=E7=BC=93?= =?UTF-8?q?=E5=AD=98=E7=AD=96=E7=95=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 README.md diff --git a/README.md b/README.md deleted file mode 100644 index 5b26276..0000000 --- a/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# common - From 27355b47fe974aceacb5b2d9c81256fd0f6e327e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Sat, 14 Mar 2026 18:30:37 +0800 Subject: [PATCH 224/225] .gitignore --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 33e76c0..f3719d5 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module gitea.com/red-future/common -go 1.25.3 +go 1.26.0 require ( github.com/alibaba/sentinel-golang v1.0.4 From b9acc7bc1876aa9aa5e0f7d29c9edf10e4ee0f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BC=A0=E6=96=8C?= <259278618@qq.com> Date: Mon, 16 Mar 2026 08:59:36 +0800 Subject: [PATCH 225/225] .gitignore --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f3719d5..2beb57c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module gitea.com/red-future/common -go 1.26.0 +go 1.25.5 require ( github.com/alibaba/sentinel-golang v1.0.4