diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 7e159e429f..24f3257b0d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,7 +2,7 @@ name: Go on: push: - branches: [ master ] + # branches: pull_request: branches: [ master ] @@ -27,6 +27,17 @@ jobs: image: redis ports: - 6379:6379 + etcd: + image: "quay.io/coreos/etcd:v3.3" + env: + ETCD_ADVERTISE_CLIENT_URLS: "http://0.0.0.0:2379" + ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379" + ETCDCTL_API: "3" + ALLOW_NONE_AUTHENTICATION: "yes" + ports: + - 2379:2379 + - 2380:2380 + - 4001:4001 steps: - name: Set up Go ${{ matrix.go }} diff --git a/.travis.yml b/.travis.yml index 74dd9a26cc..1771358e2b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ sudo: required services: - mysql - redis-server + - docker go: - 1.13.x @@ -13,11 +14,17 @@ go: env: - GO111MODULE=on -before_install: +before_install: - go get golang.org/x/tools/cmd/goimports - mysql -e 'CREATE DATABASE IF NOT EXISTS test;' + - docker pull xieyanze/etcd3 + - docker run -d -p 127.0.0.1:2379:2379 xieyanze/etcd3 + - wget https://github.com/etcd-io/etcd/releases/download/v3.4.10/etcd-v3.4.10-linux-amd64.tar.gz + - tar xvf etcd-v3.4.10-linux-amd64.tar.gz script: + - etcd-v3.4.10-linux-amd64/etcdctl put "/hello" "jupiter" + - etcd-v3.4.10-linux-amd64/etcdctl get "/hello" - diff -u <(echo -n) <(gofmt -d -s .) # - diff -u <(echo -n) <(goimports -d .) - go test -v -race ./... -coverprofile=coverage.txt -covermode=atomic diff --git a/README.md b/README.md index 995fad0ec1..28bf7e86a4 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,6 @@ func startGRPCServer() server.Server { func startWorker() worker.Worker { cron := xcron.DefaultConfig().Build() cron.Schedule(xcron.Every(time.Second*10), xcron.FuncJob(func() error { - fmt.Println("now: ", time.Now().Local().String()) return nil })) return cron diff --git a/api/configuration.proto b/api/configuration.proto new file mode 100644 index 0000000000..dfc244cfbb --- /dev/null +++ b/api/configuration.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; +package com.douyu.jupiter.proto.registry.configuration; + +option go_package="configuration"; + +message Configuration { + +} + +message Route \ No newline at end of file diff --git a/doc/wiki-cn/quickstart.md b/doc/wiki-cn/quickstart.md index 8392c97964..abd99c5805 100644 --- a/doc/wiki-cn/quickstart.md +++ b/doc/wiki-cn/quickstart.md @@ -102,7 +102,7 @@ func main() { app := &MyApplication{} app.Startup() app.SetRegistry( - etcdv3.DefaultConfig().BuildRegistry(), + etcdv3.DefaultConfig().Build(), ) app.Run() } @@ -119,8 +119,8 @@ func main() { app.Startup() app.SetRegistry( // 多注册中心 compound.New( - etcdv3.StdConfig("bj01").BuildRegistry(), // 读取配置文件中 jupiter.etcdv3.bj01的配置，自动初始化一个etcd registry - etcdv3.StdConfig("bj02").BuildRegistry(), // 读取配置文件中 jupiter.etcdv3.bj02的配置，自动初始化一个etcd registry + etcdv3.StdConfig("bj01").Build(), // 读取配置文件中 jupiter.etcdv3.bj01的配置，自动初始化一个etcd registry + etcdv3.StdConfig("bj02").Build(), // 读取配置文件中 jupiter.etcdv3.bj02的配置，自动初始化一个etcd registry ), ) app.Run() @@ -157,15 +157,7 @@ func dialServer() { ```golang // 1. 注册一个resolver func init() { - resolver.Register(etcdv3.Config{ - ReadTimeout: time.Second * 3, - Prefix: "jupiter", - Config: &clientEtcdv3.Config{ - Endpoints: []string{"127.0.0.1:2379"}, - ConnectTimeout: time.Second * 3, - Secure: false, - }, - }.BuildResolver()) + resolver.Register("etcd", etcdv3.StdConfig("wh").Build()) } // 2. 通过resolver获取服务器地址 func dialServer() { diff --git a/doc/wiki-cn/service_register_and_discovery.md b/doc/wiki-cn/service_register_and_discovery.md new file mode 100644 index 0000000000..b4e47d9be1 --- /dev/null +++ b/doc/wiki-cn/service_register_and_discovery.md @@ -0,0 +1,187 @@ +# 服务注册与发现 + +### 1. 服务定义 + +``` golang +type ServiceInfo struct { + Name string `json:"name"` + Scheme string `json:"scheme"` + IP string `json:"ip"` + Port int `json:"port"` + Weight float64 `json:"weight"` + Enable bool `json:"enable"` + Healthy bool `json:"healthy"` + Metadata map[string]string `json:"metadata"` + Region string `json:"region"` + Zone string `json:"zone"` + Deployment string `json:"deployment"` +} +``` + + +### 2. 流量控制 + +#### 2.1 红绿发布 + +蓝绿发布，是在生产环境稳定集群之外，额外部署一个与稳定集群规模相同的新集群，并通过流量控制，逐步引入流量至新集群直至100%， +原先稳定集群将与新集群同时保持在线一段时间，期间发生任何异常，可立刻将所有流量切回至原稳定集群，实现快速回滚。直到全部验证成功后， +下线老的稳定集群，新集群成为新的稳定集群。 + + +1. 部署新的集群, 但不接入流量 +```mermaid +graph LR + client --> service-v1-1 + client --> service-v1-2 + client --> service-v1-3 + client -.X.-> service-v2-1 + client -.X.-> service-v2-2 + client -.X.-> service-v2-3 + subgraph g2 [v2:red cluster] + service-v2-1 + service-v2-2 + service-v2-3 + end + subgraph g1 [v1:green cluster] + service-v1-1 + service-v1-2 + service-v1-3 + end +``` +2. 将流量逐步导入v2集群，同时保持v1集群在线 +```mermaid +graph LR + subgraph g1 [v1:green cluster] + service-v1-1 + service-v1-2 + service-v1-3 + end + client --> service-v2-1 + client --> service-v2-2 + client --> service-v2-3 + client --X--> service-v1-1 + client --X--> service-v1-2 + client --X--> service-v1-3 + subgraph g2 [v2:red cluster] + service-v2-1 + service-v2-2 + service-v2-3 + end +``` + +验证失败，立即将流量切到v1集群。 + + +#### 2.2 灰度发布 + +灰度发布，是在生产环境稳定集群之外，额外部署一个小规模的灰度集群，并通过流量控制，引入部分流量到灰度集群，进行生产全量发布前的灰度验证。 +如果验证失败，可立刻将所有流量切回至稳定集群，取消灰度发布过程；如果验证成功，则将新版本进行全量发布升级至生产环境稳定集群，完成灰度发布过程。 + +```mermaid +graph LR + subgraph g1 [v1:stable cluster] + service-v1-1 + service-v1-2 + service-v1-3 + end + client --> service-v1-1 + client --90%--> service-v1-2 + client --> service-v1-3 + client -.-> service-v2-1 + client -.10%.-> service-v2-2 + subgraph g2 [v2:grap cluster] + service-v2-1 + service-v2-2 + end +``` + +蓝绿发布和灰度发布的各有优缺点，需要根据具体业务和资源成本自行选择。这里以灰度发布为例，讨论jupiter中对gRPC服务进行流量控制，从而实现灰度发布。 + +#### 2.3 Jupiter中对gRPC服务进行灰度发布 + + + +### 3. 服务注册 + +应用服务注册键的修改有两个场景: +* 服务端注册、注销 +* 治理端更新 + + +为了保证在各个场景下的键修改安全，不同于一般的etcd键值更新方式： +* 对于服务端注册、注销: + * 获取key锁成功，读取治理键，将治理键值合并到注册键。 + * 获取key锁失败，等待1s重新注册。最多重试三次 +* 对于治理端更新 + * 获取key锁成功，读取注册建，将治理键值合并写入，并单独写一个治理键。 + * 获取key锁失败，由治理端用户触发重试 + +该方式保证了： +1. 多写端的键安全 +2. 注册键删除后，治理键依旧可用。重新注册后，治理配置不会丢失 +3. 兼容 + +另外，app sider（一个单独的agent，与应用同机部署），提供了两个功能: +1. etcdproxy, 代理etcd请求。最直观的优势是应用不再需要关心和配置etcd地址。 +2. healthcheck, 对同机的app做健康检查，判定应用不可用时，主动注销服务。理论上应用不在需要使用 etcd lease 保证异常状态下的服务正常注销。 + +其中: +1. enable: 服务是否被客户端发现（即是否导入流量) +2. weight: 流量权重，客户端按照该权重配发流量 +3. group: 流量分组, 多个分组使用','分隔 + + + +## 2 服务发现 + +### 2.1 负载均衡 + +常见的负载均衡算法有: + +> Round Robin： 轮询算法，顾名思义请求将会依次发给每一个实例，来共同分担所有的请求。 +> Random： 随机算法，将所有的请求随机分发给健康的实例。 +> Least Conn: 最小连接数，在所有健康的实例中任选两个，将请求发给连接数较小的那一个实例。 +> Weighted Round Robin: 加权轮询算法，根据服务器的不同处理能力，给每个服务器分配不同的权值，使其能够接受相应权值数的服务请求。 + +针对流量分组的需求，naming在上述算法的基础上，实现了grpc.Resolver和grpc.Picker接口，提供基于分组和权重的负载均衡能力。 +目前，naming支持 __group weighted random__ 算法, 通过调节配置: + +> 单一分组内所有节点weight=1, 退化为组内Random算法。 +> 组名置空，退化为全局Random算法。 + +之所以，使用Random而不是RoundRobin保底，原因是: + +> 1. 常见的RoundRobin算法也会在初始阶段，增加随机因子，保证后端负载压力均衡。与Random算法效果相似。 +> 2. 实现上，Random更简单 + +### 2.2 实现 + +naming.etcdResolver 实现了grpc.Resolver接口，相比etcdv3.GRPCResolver： + +> naming.etcdResolver 支持grpc.Balancer最新接口。相对的etcdv3.GRPCResolver已被标记为Deprecated. + +naming.groupPickerBuilder 实现了balancer.Builder接口，并向balancer包注入group_weight负载均衡器。 + + +```go +// 服务发现 +if len(options.iniConfig.EtcdEndpoints) > 0 { + if resolver.Get("grpc") == nil { + resolver.Register(naming.NewResolver("grpc", options.iniConfig.EtcdEndpoints)) + } + + options.dialOptions = append(options.dialOptions, grpc.WithBalancerName(naming.NameWeight)) +} +// ... +cc, err := grpc.DialContext( + ctx, + "grpc:///" + options.iniConfig.Address, + options.dialOptions..., +) +``` + + +如注释行所述，发起调用的时候必须在context中写入_group, 用于表明请求的服务组。否则会退化为 __全局随机的负载均衡策略__。 + +_group的自动写入和匹配，在下一个版本中引入。 + diff --git a/example/Readme.md b/example/Readme.md index 73632fbe52..d8f72805fd 100644 --- a/example/Readme.md +++ b/example/Readme.md @@ -9,6 +9,8 @@ * [通过远端配置读取结构体配置示例](./config/structByRemoteConfig) * [监听远端配置读取单行配置示例](./config/onelineByRemoteConfigWatch) * [监听远端配置读取结构体配置示例](./config/structByRemoteConfigWatch) + * [通过Apollo配置读取单行配置示例](./config/onelinebyApollo) + * [通过etcdv3配置读取单行配置示例](./config/onelineByEtcdv3) * [日志示例](./logger) * [日志终端展示示例](./logger/command) * [日志文本展示示例](./logger/file) diff --git a/example/all/cmd/demo/main.go b/example/all/cmd/demo/main.go index bba872f6b8..2b024a01ef 100644 --- a/example/all/cmd/demo/main.go +++ b/example/all/cmd/demo/main.go @@ -15,7 +15,6 @@ package main import ( - "fmt" "log" "github.com/douyu/jupiter/example/all/internal/app/demo" @@ -26,16 +25,9 @@ import ( func main() { eng := demo.NewEngine() - eng.AfterStop(func() error { - fmt.Println("exit...") - return nil - }) - - eng.SetGovernor("127.0.0.1:9391") - eng.SetRegistry( // 多注册中心 compound.New( - etcdv3.StdConfig("wh01").BuildRegistry(), + etcdv3.StdConfig("wh01").Build(), ), ) diff --git a/example/all/internal/app/demo/engine.go b/example/all/internal/app/demo/engine.go index 0fe5ec7dcf..21b1709fec 100644 --- a/example/all/internal/app/demo/engine.go +++ b/example/all/internal/app/demo/engine.go @@ -117,13 +117,3 @@ func (eng *Engine) execJob() error { xlog.Warn("exec job", xlog.String("warn", "print warning")) return nil } - -func (eng *Engine) printLogs() error { - go func() { - for { - xlog.Info("hello", xlog.String("a", "b")) - time.Sleep(time.Second) - } - }() - return nil -} diff --git a/example/config/onelineByEtcdv3/main.go b/example/config/onelineByEtcdv3/main.go new file mode 100644 index 0000000000..b7b0b0c1e8 --- /dev/null +++ b/example/config/onelineByEtcdv3/main.go @@ -0,0 +1,75 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "github.com/coreos/etcd/clientv3" + "github.com/douyu/jupiter" + "github.com/douyu/jupiter/pkg/conf" + _ "github.com/douyu/jupiter/pkg/datasource/etcdv3" + "github.com/douyu/jupiter/pkg/xlog" +) + +// go run main.go --config=etcdv3://10.0.101.68:2379?key=test + +var configText = ` +[people] + name = "jupiter" +[jupiter.logger.default] + debug = true + enableConsole = true +[jupiter.server.governor] + enable = false + host = "0.0.0.0" + port = 9246 +` + +func initTestData() { + etcdCfg := clientv3.Config{ + Endpoints: []string{"127.0.0.1:2379"}, + } + cli, _ := clientv3.New(etcdCfg) + cli.Put(context.Background(), "test", configText) +} + +func main() { + initTestData() + app := NewEngine() + if err := app.Run(); err != nil { + panic(err) + } +} + +type Engine struct { + jupiter.Application +} + +func NewEngine() *Engine { + eng := &Engine{} + if err := eng.Startup( + eng.printConfig, + ); err != nil { + xlog.Panic("startup", xlog.Any("err", err)) + } + return eng +} + +func (s *Engine) printConfig() error { + xlog.DefaultLogger = xlog.StdConfig("default").Build() + peopleName := conf.GetString("people.name") + xlog.Info("people info", xlog.String("name", peopleName), xlog.String("type", "onelineByEtcdv3")) + return nil +} diff --git a/example/config/onelineByRemoteConfigWatch/main.go b/example/config/onelineByRemoteConfigWatch/main.go index 0f198c3274..6c98e9ca07 100644 --- a/example/config/onelineByRemoteConfigWatch/main.go +++ b/example/config/onelineByRemoteConfigWatch/main.go @@ -32,16 +32,18 @@ import ( // port func main() { app := NewEngine() - app.SetGovernor("0.0.0.0:9999") + // app.SetGovernor("0.0.0.0:9999") if err := app.Run(); err != nil { panic(err) } } +//Engine .. type Engine struct { jupiter.Application } +//NewEngine .. func NewEngine() *Engine { eng := &Engine{} if err := eng.Startup( diff --git a/example/config/onelinebyApollo/main.go b/example/config/onelinebyApollo/main.go new file mode 100644 index 0000000000..e42f139bd6 --- /dev/null +++ b/example/config/onelinebyApollo/main.go @@ -0,0 +1,54 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/douyu/jupiter" + "github.com/douyu/jupiter/pkg/conf" + _ "github.com/douyu/jupiter/pkg/datasource/apollo" + "github.com/douyu/jupiter/pkg/xlog" +) + +// apollo: http://106.54.227.205/config.html#/appid=jupiter&env=DEV&cluster=default +// account/password: apollo/admin + +// go run main.go --config="apollo://106.54.227.205:8080?appId=jupiter&cluster=default&namespaceName=application&key=jupiter-test" +func main() { + app := NewEngine() + if err := app.Run(); err != nil { + panic(err) + } +} + +type Engine struct { + jupiter.Application +} + +func NewEngine() *Engine { + eng := &Engine{} + if err := eng.Startup( + eng.printConfig, + ); err != nil { + xlog.Panic("startup", xlog.Any("err", err)) + } + return eng +} + +func (s *Engine) printConfig() error { + xlog.DefaultLogger = xlog.StdConfig("default").Build() + peopleName := conf.GetString("people.name") + xlog.Info("people info", xlog.String("name", peopleName), xlog.String("type", "onelineByApollo")) + return nil +} diff --git a/example/grpc/direct/direct-client/main.go b/example/grpc/direct/direct-client/main.go index aa1374cc00..9abe5b01a0 100644 --- a/example/grpc/direct/direct-client/main.go +++ b/example/grpc/direct/direct-client/main.go @@ -37,7 +37,6 @@ type Engine struct { func NewEngine() *Engine { eng := &Engine{} - eng.SetGovernor("127.0.0.1:9999") if err := eng.Startup( eng.consumer, ); err != nil { @@ -49,6 +48,7 @@ func NewEngine() *Engine { func (eng *Engine) consumer() error { conn := grpc.StdConfig("directserver").Build() client := helloworld.NewGreeterClient(conn) + go func() { for { resp, err := client.SayHello(context.Background(), &helloworld.HelloRequest{ diff --git a/example/grpc/direct/direct-server/main.go b/example/grpc/direct/direct-server/main.go index a5c76b942b..df8af42918 100644 --- a/example/grpc/direct/direct-server/main.go +++ b/example/grpc/direct/direct-server/main.go @@ -25,7 +25,7 @@ import ( func main() { eng := NewEngine() - eng.SetGovernor("127.0.0.1:9092") + // eng.SetGovernor("127.0.0.1:9092") if err := eng.Run(); err != nil { xlog.Error(err.Error()) } diff --git a/example/grpc/etcd/etcd-client/config.toml b/example/grpc/etcd/etcd-client/config.toml index a3977e580a..486505757b 100644 --- a/example/grpc/etcd/etcd-client/config.toml +++ b/example/grpc/etcd/etcd-client/config.toml @@ -7,3 +7,4 @@ balancerName = "round_robin" # 默认值 block = false # 默认值 dialTimeout = "0s" # 默认值 + diff --git a/example/grpc/etcd/etcd-client/main.go b/example/grpc/etcd/etcd-client/main.go index f8bcb3c59c..2c68b4bc46 100644 --- a/example/grpc/etcd/etcd-client/main.go +++ b/example/grpc/etcd/etcd-client/main.go @@ -16,15 +16,17 @@ package main import ( "context" + "fmt" "time" "github.com/douyu/jupiter" "github.com/douyu/jupiter/pkg/client/grpc" - etcdv3_registry "github.com/douyu/jupiter/pkg/registry/etcdv3" + "github.com/douyu/jupiter/pkg/client/grpc/balancer" + "github.com/douyu/jupiter/pkg/client/grpc/resolver" + "github.com/douyu/jupiter/pkg/registry/etcdv3" "github.com/douyu/jupiter/pkg/xlog" "google.golang.org/grpc/examples/helloworld/helloworld" - "google.golang.org/grpc/resolver" ) func main() { @@ -32,6 +34,7 @@ func main() { if err := eng.Run(); err != nil { xlog.Error(err.Error()) } + fmt.Printf("111 = %+v\n", 111) } type Engine struct { @@ -50,21 +53,25 @@ func NewEngine() *Engine { } func (eng *Engine) initResolver() error { - resolver.Register(etcdv3_registry.StdConfig("wh").BuildResolver()) + resolver.Register("etcd", etcdv3.StdConfig("wh").Build()) return nil } func (eng *Engine) consumer() error { - conn := grpc.StdConfig("etcdserver").Build() - client := helloworld.NewGreeterClient(conn) + config := grpc.StdConfig("etcdserver") + config.BalancerName = balancer.NameSmoothWeightRoundRobin + + client := helloworld.NewGreeterClient(config.Build()) go func() { for { resp, err := client.SayHello(context.Background(), &helloworld.HelloRequest{ Name: "jupiter", }) if err != nil { + fmt.Printf("err = %+v\n", err) xlog.Error(err.Error()) } else { + fmt.Printf("resp.Message = %+v\n", resp.Message) xlog.Info("receive response", xlog.String("resp", resp.Message)) } time.Sleep(1 * time.Second) diff --git a/example/grpc/etcd/etcd-server/config2.toml b/example/grpc/etcd/etcd-server/config2.toml new file mode 100644 index 0000000000..62c53e0897 --- /dev/null +++ b/example/grpc/etcd/etcd-server/config2.toml @@ -0,0 +1,6 @@ +[jupiter.server.grpc] + port = 9093 +[jupiter.registry.wh] + connectTimeout = "1s" + endpoints=["127.0.0.1:2379"] + secure = false diff --git a/example/grpc/etcd/etcd-server/main.go b/example/grpc/etcd/etcd-server/main.go index 190dc632f7..71a654fd64 100644 --- a/example/grpc/etcd/etcd-server/main.go +++ b/example/grpc/etcd/etcd-server/main.go @@ -16,7 +16,6 @@ package main import ( "context" - "github.com/douyu/jupiter" compound_registry "github.com/douyu/jupiter/pkg/registry/compound" etcdv3_registry "github.com/douyu/jupiter/pkg/registry/etcdv3" @@ -29,10 +28,10 @@ func main() { eng := NewEngine() eng.SetRegistry( compound_registry.New( - etcdv3_registry.StdConfig("wh").BuildRegistry(), + etcdv3_registry.StdConfig("wh").Build(), ), ) - eng.SetGovernor("127.0.0.1:9391") + //eng.SetGovernor("0.0.0.0:0") if err := eng.Run(); err != nil { xlog.Error(err.Error()) } @@ -54,14 +53,18 @@ func NewEngine() *Engine { func (eng *Engine) serveGRPC() error { server := xgrpc.StdConfig("grpc").Build() - helloworld.RegisterGreeterServer(server.Server, new(Greeter)) + helloworld.RegisterGreeterServer(server.Server, &Greeter{ + server: server, + }) return eng.Serve(server) } -type Greeter struct{} +type Greeter struct { + server *xgrpc.Server +} func (g Greeter) SayHello(context context.Context, request *helloworld.HelloRequest) (*helloworld.HelloReply, error) { return &helloworld.HelloReply{ - Message: "Hello Jupiter", + Message: "Hello Jupiter, I'm " + g.server.Address(), }, nil } diff --git a/example/helloworld/main.go b/example/helloworld/main.go index c2b26d39ac..fd1ce8e15a 100644 --- a/example/helloworld/main.go +++ b/example/helloworld/main.go @@ -46,6 +46,7 @@ func NewEngine() *Engine { func (eng *Engine) serveHTTP() error { server := xecho.StdConfig("http").Build() server.GET("/hello", func(ctx echo.Context) error { + return ctx.JSON(200, "Gopher Wuhan") }) return eng.Serve(server) diff --git a/example/http/all/main.go b/example/http/all/main.go index 9341b56d33..b353df441b 100644 --- a/example/http/all/main.go +++ b/example/http/all/main.go @@ -28,7 +28,7 @@ func main() { // 多注册中心 eng.SetRegistry( compound_registry.New( - etcdv3_registry.StdConfig("wh").BuildRegistry(), + etcdv3_registry.StdConfig("wh").Build(), ), ) diff --git a/example/http/gin/main.go b/example/http/gin/main.go index 41e7c65641..b2d0f80541 100644 --- a/example/http/gin/main.go +++ b/example/http/gin/main.go @@ -49,7 +49,6 @@ func (eng *Engine) serveHTTP() error { server := xgin.StdConfig("http").Build() server.GET("/hello", func(ctx *gin.Context) { ctx.JSON(200, "Hello Gin") - return }) //Upgrade to websocket server.Upgrade(xgin.WebSocketOptions("/ws", func(ws xgin.WebSocketConn, err error) { @@ -70,7 +69,6 @@ func (eng *Engine) serveHTTP() error { break } } - return })) return eng.Serve(server) } diff --git a/example/http/register/main.go b/example/http/register/main.go index c66391d5e9..57d4d13853 100644 --- a/example/http/register/main.go +++ b/example/http/register/main.go @@ -27,7 +27,7 @@ func main() { eng := NewEngine() eng.SetRegistry( compound_registry.New( - etcdv3_registry.StdConfig("wh").BuildRegistry(), + etcdv3_registry.StdConfig("wh").Build(), ), ) if err := eng.Run(); err != nil { diff --git a/example/job/main.go b/example/job/main.go new file mode 100644 index 0000000000..116a10507e --- /dev/null +++ b/example/job/main.go @@ -0,0 +1,51 @@ +package main + +import ( + "fmt" + "github.com/douyu/jupiter" + "github.com/douyu/jupiter/pkg/xlog" +) + +// go run main.go --job=jobrunner +func main() { + eng := NewEngine() + if err := eng.Run(); err != nil { + xlog.Error(err.Error()) + } +} + +type Engine struct { + jupiter.Application +} + +func NewEngine() *Engine { + eng := &Engine{} + if err := eng.Startup( + eng.initJob, + ); err != nil { + xlog.Panic("startup", xlog.Any("err", err)) + } + return eng +} + +func (e *Engine) initJob() error { + return e.Job(NewJobRunner()) +} + +type JobRunner struct { + JobName string +} + +func NewJobRunner() *JobRunner { + return &JobRunner{ + JobName: "jobrunner", + } +} + +func (j *JobRunner) Run() { + fmt.Println("i am job runner") +} + +func (j *JobRunner) GetJobName() string { + return j.JobName +} diff --git a/example/simple/config.toml b/example/simple/config.toml new file mode 100644 index 0000000000..ad66901a0b --- /dev/null +++ b/example/simple/config.toml @@ -0,0 +1,8 @@ +[jupiter.etcdv3.default] + endpoints=["127.0.0.1:2379"] + secure = false + +[jupiter.registry.test] + configKey="jupiter.etcdv3.default" + timeout = "1s" + service_ttl = "5s" \ No newline at end of file diff --git a/example/simple/main.go b/example/simple/main.go new file mode 100644 index 0000000000..32f405b3d3 --- /dev/null +++ b/example/simple/main.go @@ -0,0 +1,47 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "log" + + "github.com/douyu/jupiter" + compound_registry "github.com/douyu/jupiter/pkg/registry/compound" + etcdv3_registry "github.com/douyu/jupiter/pkg/registry/etcdv3" + "github.com/douyu/jupiter/pkg/server" + "github.com/douyu/jupiter/pkg/server/xgin" + "github.com/gin-gonic/gin" +) + +func main() { + app, err := jupiter.New() + if err != nil { + log.Fatal(err) + } + app.SetRegistry( + compound_registry.New( + etcdv3_registry.StdConfig("test").Build(), + ), + ) + app.Run(startHTTPServer()) +} + +func startHTTPServer() server.Server { + server := xgin.DefaultConfig().Build() + server.GET("/hello", func(ctx *gin.Context) { + ctx.JSON(200, "hello jupiter") + }) + return server +} diff --git a/example/startup/main.go b/example/startup/main.go index 119e6bf5e0..fb5117dd9e 100644 --- a/example/startup/main.go +++ b/example/startup/main.go @@ -1,7 +1,20 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package main import ( - "fmt" "time" "github.com/douyu/jupiter" @@ -39,7 +52,6 @@ func startGRPCServer() server.Server { func startWorker() worker.Worker { cron := xcron.DefaultConfig().Build() cron.Schedule(xcron.Every(time.Second*10), xcron.FuncJob(func() error { - fmt.Println("now: ", time.Now().Local().String()) return nil })) return cron diff --git a/example/worker/cron/config.toml b/example/worker/cron/config.toml index 2e8f39d585..6a60d31f55 100644 --- a/example/worker/cron/config.toml +++ b/example/worker/cron/config.toml @@ -1,4 +1,11 @@ [jupiter.cron.test] withSeconds = false - concurrentDelay= "10s" + concurrentDelay= 1 immediatelyRun = false + # 分布式任务的开关 + distributedTask = false + # 抢锁等待时间 + waitLockTime = "1s" + # etcd配置相关 + endpoints = ["127.0.0.1:2379"] + connectTimeout = "10s" \ No newline at end of file diff --git a/example/worker/cron/main.go b/example/worker/cron/main.go index 44a583712b..f194295a33 100644 --- a/example/worker/cron/main.go +++ b/example/worker/cron/main.go @@ -15,12 +15,11 @@ package main import ( - "time" - + "fmt" "github.com/douyu/jupiter" - _ "github.com/douyu/jupiter" "github.com/douyu/jupiter/pkg/worker/xcron" "github.com/douyu/jupiter/pkg/xlog" + "time" ) func main() { @@ -55,5 +54,6 @@ func (eng *Engine) startJobs() error { func (eng *Engine) execJob() error { xlog.Info("info job") xlog.Warn("warn job") + fmt.Println("run job") return nil } diff --git a/go.mod b/go.mod index a127433278..0394a51a56 100644 --- a/go.mod +++ b/go.mod @@ -32,11 +32,12 @@ require ( github.com/modern-go/reflect2 v1.0.1 github.com/onsi/ginkgo v1.12.3 // indirect github.com/opentracing/opentracing-go v1.1.0 - github.com/philchia/agollo v2.1.0+incompatible + github.com/philchia/agollo/v4 v4.0.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.6.0 github.com/robfig/cron/v3 v3.0.1 github.com/smallnest/weighted v0.0.0-20200122032019-adf21c9b8bd1 + github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 github.com/spf13/cast v1.3.1 github.com/stretchr/testify v1.6.1 github.com/tidwall/pretty v1.0.1 @@ -45,9 +46,9 @@ require ( go.uber.org/automaxprocs v1.3.0 go.uber.org/multierr v1.5.0 go.uber.org/zap v1.15.0 - golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e + golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 golang.org/x/sys v0.0.0-20200523222454-059865788121 // indirect - golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d // indirect + golang.org/x/tools v0.0.0-20200728235236-e8769ccb4337 // indirect google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 google.golang.org/grpc v1.26.0 sigs.k8s.io/yaml v1.1.0 // indirect diff --git a/go.sum b/go.sum index aaf7d51eb6..41f36da5ac 100644 --- a/go.sum +++ b/go.sum @@ -369,17 +369,18 @@ github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/philchia/agollo v2.1.0+incompatible h1:YszI/+yhlPD4yCnmN+Yon+uxp6KrsLTgRc3hI+y+BJc= -github.com/philchia/agollo v2.1.0+incompatible/go.mod h1:EBut+tbr1PNZCXWY4t+CIJ2r0A/ELL8HNQu8rR9IG6g= github.com/ory/dockertest v3.3.4+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= github.com/packethost/packngo v0.1.1-0.20180711074735-b9cb5096f54c/go.mod h1:otzZQXgoO96RTzDB/Hycg0qZcXZsWJGJRSXbmEIJ+4M= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v0.0.0-20180527043350-9f6ff22cfff8/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/philchia/agollo/v4 v4.0.0 h1:DiZWsZ6YOT98eH77IuOsax4w/01F0W6TUWYNbAEsHy4= +github.com/philchia/agollo/v4 v4.0.0/go.mod h1:SBdQmfqqu/XCWJ1MDzYcCL3X+p3VJ+uQBy0nxxqjexg= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -486,6 +487,7 @@ github.com/vmware/govmomi v0.18.0/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59b github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zouyx/agollo v0.0.0-20191114083447-dde9fc9f35b8/go.mod h1:S1cAa98KMFv4Sa8SbJ6ZtvOmf0VlgH0QJ1gXI0lBfBY= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -513,6 +515,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a h1:7Wlg8L54In96HTWOaI4sreLJ6qfyGuvSau5el3fK41Y= @@ -530,6 +534,8 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -549,6 +555,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20170807180024-9a379c6b3e95/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -557,6 +565,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -609,8 +619,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f h1:kDxGY2VmgABOe55qheT/TFqUMtcTHnomIPS1iv3G4Ms= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200426102838-f3a5411a4c3b/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d h1:SR+e35rACZFBohNb4Om1ibX6N3iO0FtdbwqGSuD9dBU= -golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200728235236-e8769ccb4337 h1:UouaHYVMLabq3zejCFgSA1hgrfVoH3t3yw81kjCVVG4= +golang.org/x/tools v0.0.0-20200728235236-e8769ccb4337/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/jupiter.go b/jupiter.go index 0c3b221051..94f394798e 100644 --- a/jupiter.go +++ b/jupiter.go @@ -17,22 +17,23 @@ package jupiter import ( "context" "fmt" - "net" - "net/http" "os" "runtime" - "strings" "sync" + "github.com/douyu/jupiter/pkg/server/governor" + job "github.com/douyu/jupiter/pkg/worker/xjob" + "github.com/BurntSushi/toml" "github.com/douyu/jupiter/pkg" "github.com/douyu/jupiter/pkg/conf" - "github.com/douyu/jupiter/pkg/constant" - file_datasource "github.com/douyu/jupiter/pkg/datasource/file" - http_datasource "github.com/douyu/jupiter/pkg/datasource/http" + + //go-lint + _ "github.com/douyu/jupiter/pkg/datasource/file" + _ "github.com/douyu/jupiter/pkg/datasource/http" + "github.com/douyu/jupiter/pkg/datasource/manager" "github.com/douyu/jupiter/pkg/ecode" "github.com/douyu/jupiter/pkg/flag" - "github.com/douyu/jupiter/pkg/govern" "github.com/douyu/jupiter/pkg/registry" "github.com/douyu/jupiter/pkg/sentinel" "github.com/douyu/jupiter/pkg/server" @@ -43,42 +44,84 @@ import ( "github.com/douyu/jupiter/pkg/util/xcycle" "github.com/douyu/jupiter/pkg/util/xdefer" "github.com/douyu/jupiter/pkg/util/xgo" - "github.com/douyu/jupiter/pkg/util/xstring" "github.com/douyu/jupiter/pkg/worker" "github.com/douyu/jupiter/pkg/xlog" "go.uber.org/automaxprocs/maxprocs" "golang.org/x/sync/errgroup" ) +const ( + //StageAfterStop after app stop + StageAfterStop uint32 = iota + 1 + //StageBeforeStop before app stop + StageBeforeStop +) + // Application is the framework's instance, it contains the servers, workers, client and configuration settings. // Create an instance of Application, by using &Application{} type Application struct { cycle *xcycle.Cycle - stopOnce sync.Once + smu *sync.RWMutex initOnce sync.Once startupOnce sync.Once - afterStop *xdefer.DeferStack - beforeStop *xdefer.DeferStack - defers []func() error + stopOnce sync.Once + servers []server.Server + workers []worker.Worker + jobs map[string]job.Runner + logger *xlog.Logger + registerer registry.Registry + hooks map[uint32]*xdefer.DeferStack +} - governorAddr string +//New new a Application +func New(fns ...func() error) (*Application, error) { + app := &Application{} + if err := app.Startup(fns...); err != nil { + return nil, err + } + return app, nil +} - servers []server.Server - workers []worker.Worker - logger *xlog.Logger - registerer registry.Registry +//init hooks +func (app *Application) initHooks(hookKeys ...uint32) { + app.hooks = make(map[uint32]*xdefer.DeferStack, len(hookKeys)) + for _, k := range hookKeys { + app.hooks[k] = xdefer.NewStack() + } +} + +//run hooks +func (app *Application) runHooks(k uint32) { + hooks, ok := app.hooks[k] + if ok { + hooks.Clean() + } +} + +//RegisterHooks register a stage Hook +func (app *Application) RegisterHooks(k uint32, fns ...func() error) error { + hooks, ok := app.hooks[k] + if ok { + hooks.Push(fns...) + return nil + } + return fmt.Errorf("hook stage not found") } // initialize application func (app *Application) initialize() { app.initOnce.Do(func() { + //assign app.cycle = xcycle.NewCycle() + app.smu = &sync.RWMutex{} app.servers = make([]server.Server, 0) app.workers = make([]worker.Worker, 0) + app.jobs = make(map[string]job.Runner) app.logger = xlog.JupiterLogger - // app.defers = []func() error{} - app.afterStop = xdefer.NewStack() - app.beforeStop = xdefer.NewStack() + //private method + app.initHooks(StageBeforeStop, StageAfterStop) + //public method + app.SetRegistry(registry.Nop{}) //default nop without registry }) } @@ -98,6 +141,7 @@ func (app *Application) startup() (err error) { app.initMaxProcs, app.initTracer, app.initSentinel, + app.initGovernor, )() }) return @@ -114,37 +158,62 @@ func (app *Application) Startup(fns ...func() error) error { // Defer .. // Deprecated: use AfterStop instead -func (app *Application) Defer(fns ...func() error) { - app.AfterStop(fns...) +// func (app *Application) Defer(fns ...func() error) { +// app.AfterStop(fns...) +// } + +// BeforeStop hook +// Deprecated: use RegisterHooks instead +// func (app *Application) BeforeStop(fns ...func() error) { +// app.RegisterHooks(StageBeforeStop, fns...) +// } + +// AfterStop hook +// Deprecated: use RegisterHooks instead +// func (app *Application) AfterStop(fns ...func() error) { +// app.RegisterHooks(StageAfterStop, fns...) +// } + +// Serve start server +func (app *Application) Serve(s ...server.Server) error { + app.smu.Lock() + defer app.smu.Unlock() + app.servers = append(app.servers, s...) + return nil } -//BeforeStop hook -func (app *Application) BeforeStop(fns ...func() error) { - app.initialize() - if app.beforeStop == nil { - app.beforeStop = xdefer.NewStack() - } - app.beforeStop.Push(fns...) +// Schedule .. +func (app *Application) Schedule(w worker.Worker) error { + app.workers = append(app.workers, w) + return nil } -//AfterStop hook -func (app *Application) AfterStop(fns ...func() error) { - app.initialize() - if app.afterStop == nil { - app.afterStop = xdefer.NewStack() +// Job .. +func (app *Application) Job(runner job.Runner) error { + namedJob, ok := runner.(interface{ GetJobName() string }) + // job runner must implement GetJobName + if !ok { + return nil + } + jobName := namedJob.GetJobName() + if flag.Bool("disable-job") { + app.logger.Info("jupiter disable job", xlog.FieldName(jobName)) + return nil } - app.afterStop.Push(fns...) -} -// Serve start a server -func (app *Application) Serve(s server.Server) error { - app.servers = append(app.servers, s) - return nil -} + // start job by name + jobFlag := flag.String("job") + if jobFlag == "" { + app.logger.Error("jupiter jobs flag name empty", xlog.FieldName(jobName)) + return nil + } -// Schedule .. -func (app *Application) Schedule(w worker.Worker) error { - app.workers = append(app.workers, w) + if jobName != jobFlag { + app.logger.Info("jupiter disable jobs", xlog.FieldName(jobName)) + return nil + } + app.logger.Info("jupiter register job", xlog.FieldName(jobName)) + app.jobs[jobName] = runner return nil } @@ -154,59 +223,74 @@ func (app *Application) SetRegistry(reg registry.Registry) { } // SetGovernor set governor addr (default 127.0.0.1:0) -func (app *Application) SetGovernor(addr string) { - app.governorAddr = addr -} +// Deprecated +//func (app *Application) SetGovernor(addr string) { +// app.governorAddr = addr +//} // Run run application -func (app *Application) Run() error { +func (app *Application) Run(servers ...server.Server) error { + app.smu.Lock() + app.servers = append(app.servers, servers...) + app.smu.Unlock() + app.waitSignals() //start signal listen task in goroutine defer app.clean() - if app.registerer == nil { - app.SetRegistry(registry.Nop{}) //default nop without registry + // todo jobs not graceful + if len(app.jobs) > 0 { + return app.startJobs() } - // start govern - app.cycle.Run(app.startGovernor) + // start servers and govern server app.cycle.Run(app.startServers) + // start workers app.cycle.Run(app.startWorkers) + //blocking and wait quit if err := <-app.cycle.Wait(); err != nil { app.logger.Error("jupiter shutdown with error", xlog.FieldMod(ecode.ModApp), xlog.FieldErr(err)) + return err } app.logger.Info("shutdown jupiter, bye!", xlog.FieldMod(ecode.ModApp)) return nil } +//clean after app quit +func (app *Application) clean() { + _ = xlog.DefaultLogger.Flush() + _ = xlog.JupiterLogger.Flush() +} + // Stop application immediately after necessary cleanup func (app *Application) Stop() (err error) { app.stopOnce.Do(func() { - app.beforeStop.Clean() - err = app.registerer.Close() - if err != nil { - app.logger.Error("stop register close err", xlog.FieldMod(ecode.ModApp), xlog.FieldErr(err)) + app.runHooks(StageBeforeStop) + + if app.registerer != nil { + err = app.registerer.Close() + if err != nil { + app.logger.Error("stop register close err", xlog.FieldMod(ecode.ModApp), xlog.FieldErr(err)) + } } - // err = app.governor.Close() - // if err != nil { - // app.logger.Error("stop governor close err", xlog.FieldMod(ecode.ModApp), xlog.FieldErr(err)) - // } //stop servers + app.smu.RLock() for _, s := range app.servers { func(s server.Server) { app.cycle.Run(s.Stop) }(s) } + app.smu.RUnlock() + //stop workers for _, w := range app.workers { func(w worker.Worker) { app.cycle.Run(w.Stop) }(w) } - select { - case <-app.cycle.Done(): - app.cycle.Close() - } + <-app.cycle.Done() + app.runHooks(StageAfterStop) + app.cycle.Close() }) return } @@ -214,15 +298,16 @@ func (app *Application) Stop() (err error) { // GracefulStop application after necessary cleanup func (app *Application) GracefulStop(ctx context.Context) (err error) { app.stopOnce.Do(func() { - app.beforeStop.Clean() - err = app.registerer.Close() - if err != nil { - app.logger.Error("graceful stop register close err", xlog.FieldMod(ecode.ModApp), xlog.FieldErr(err)) + app.runHooks(StageBeforeStop) + + if app.registerer != nil { + err = app.registerer.Close() + if err != nil { + app.logger.Error("stop register close err", xlog.FieldMod(ecode.ModApp), xlog.FieldErr(err)) + } } - // err = app.governor.Close() - // if err != nil { - // app.logger.Error("graceful stop governor close err", xlog.FieldMod(ecode.ModApp), xlog.FieldErr(err)) - // } + //stop servers + app.smu.RLock() for _, s := range app.servers { func(s server.Server) { app.cycle.Run(func() error { @@ -230,24 +315,26 @@ func (app *Application) GracefulStop(ctx context.Context) (err error) { }) }(s) } + app.smu.RUnlock() + + //stop workers for _, w := range app.workers { func(w worker.Worker) { app.cycle.Run(w.Stop) }(w) } - select { - case <-app.cycle.Done(): - app.cycle.Close() - } + <-app.cycle.Done() + app.runHooks(StageAfterStop) + app.cycle.Close() }) return err } // waitSignals wait signal func (app *Application) waitSignals() { - app.logger.Info("init listen signal", xlog.FieldMod(ecode.ModApp)) + app.logger.Info("init listen signal", xlog.FieldMod(ecode.ModApp), xlog.FieldEvent("init")) signals.Shutdown(func(grace bool) { //when get shutdown signal - //todo: suport timeout + //todo: support timeout if grace { app.GracefulStop(context.TODO()) } else { @@ -256,45 +343,12 @@ func (app *Application) waitSignals() { }) } -func (app *Application) startGovernor() error { - //todo: abstract governor struct - if len(app.governorAddr) == 0 { - app.governorAddr = conf.GetString("jupiter.governor.addr") - } - if len(app.governorAddr) == 0 { - app.SetGovernor(":0") // default governor addr - } - var governor = &http.Server{ - Addr: app.governorAddr, - Handler: govern.DefaultServeMux, - } - var listener, err = net.Listen("tcp4", app.governorAddr) - if err != nil { - xlog.Panic("start governor", xlog.FieldErr(err)) - } - app.BeforeStop(func() error { - app.logger.Info("stop governor", xlog.FieldMod(ecode.ModApp), xlog.FieldAddr("http://"+listener.Addr().String())) - return listener.Close() - }) - - app.logger.Info("start governor", xlog.FieldMod(ecode.ModApp), xlog.FieldAddr("http://"+listener.Addr().String())) - serviceInfo := &server.ServiceInfo{ - Name: pkg.Name(), - Scheme: "http", - Address: listener.Addr().String(), - Kind: constant.ServiceGovernor, - } - - if err := app.registerer.RegisterService(context.Background(), serviceInfo); err == nil { - app.BeforeStop(func() error { return app.registerer.DeregisterService(context.Background(), serviceInfo) }) - } - if err := governor.Serve(listener); err != nil { - if err == http.ErrServerClosed { - app.logger.Info("stop governor", xlog.FieldMod(ecode.ModApp), xlog.FieldAddr("http://"+listener.Addr().String())) - return nil - } +func (app *Application) initGovernor() error { + config := governor.StdConfig("governor") + if !config.Enable { + return nil } - return nil + return app.Serve(config.Build()) } func (app *Application) startServers() error { @@ -304,10 +358,11 @@ func (app *Application) startServers() error { s := s eg.Go(func() (err error) { _ = app.registerer.RegisterService(context.TODO(), s.Info()) - defer app.registerer.DeregisterService(context.TODO(), s.Info()) - app.logger.Info("start servers", xlog.FieldMod(ecode.ModApp), xlog.FieldAddr(s.Info().Label()), xlog.Any("scheme", s.Info().Scheme)) - defer app.logger.Info("exit server", xlog.FieldMod(ecode.ModApp), xlog.FieldErr(err), xlog.FieldAddr(s.Info().Label())) - return s.Serve() + defer app.registerer.UnregisterService(context.TODO(), s.Info()) + app.logger.Info("start server", xlog.FieldMod(ecode.ModApp), xlog.FieldEvent("init"), xlog.FieldName(s.Info().Name), xlog.FieldAddr(s.Info().Label()), xlog.Any("scheme", s.Info().Scheme)) + defer app.logger.Info("exit server", xlog.FieldMod(ecode.ModApp), xlog.FieldEvent("exit"), xlog.FieldName(s.Info().Name), xlog.FieldErr(err), xlog.FieldAddr(s.Info().Label())) + err = s.Serve() + return }) } return eg.Wait() @@ -325,6 +380,23 @@ func (app *Application) startWorkers() error { return eg.Wait() } +// todo handle error +func (app *Application) startJobs() error { + var jobs = make([]func(), 0) + //warp jobs + for name, runner := range app.jobs { + jobs = append(jobs, func() { + app.logger.Info("job run begin", xlog.FieldName(name)) + defer app.logger.Info("job run end", xlog.FieldName(name)) + // runner.Run panic 错误在更上层抛出 + runner.Run() + }) + } + xgo.Parallel(jobs...)() + return nil +} + +//parseFlags init func (app *Application) parseFlags() error { flag.Register(&flag.StringFlag{ Name: "config", @@ -353,62 +425,39 @@ func (app *Application) parseFlags() error { return flag.Parse() } -func (app *Application) clean() { - for i := len(app.defers) - 1; i >= 0; i-- { - fn := app.defers[i] - if err := fn(); err != nil { - xlog.Error("clean.defer", xlog.String("func", xstring.FunctionName(fn))) - } - } - _ = xlog.DefaultLogger.Flush() - _ = xlog.JupiterLogger.Flush() -} - +//loadConfig init func (app *Application) loadConfig() error { - var ( - watchConfig = flag.Bool("watch") - configAddr = flag.String("config") - ) - - if configAddr == "" { - app.logger.Warn("no config ...") - return nil - } - switch { - case strings.HasPrefix(configAddr, "http://"), - strings.HasPrefix(configAddr, "https://"): - provider := http_datasource.NewDataSource(configAddr, watchConfig) - if err := conf.LoadFromDataSource(provider, toml.Unmarshal); err != nil { - app.logger.Panic("load remote config", xlog.FieldMod(ecode.ModConfig), xlog.FieldErrKind(ecode.ErrKindUnmarshalConfigErr), xlog.FieldErr(err)) + var configAddr = flag.String("config") + provider, err := manager.NewDataSource(configAddr) + if err != manager.ErrConfigAddr { + if err != nil { + app.logger.Panic("data source: provider error", xlog.FieldMod(ecode.ModConfig), xlog.FieldErr(err)) } - app.logger.Info("load remote config", xlog.FieldMod(ecode.ModConfig), xlog.FieldAddr(configAddr)) - default: - provider := file_datasource.NewDataSource(configAddr, watchConfig) - if err := conf.LoadFromDataSource(provider, toml.Unmarshal); err != nil { - app.logger.Panic("load local file", xlog.FieldMod(ecode.ModConfig), xlog.FieldErrKind(ecode.ErrKindUnmarshalConfigErr), xlog.FieldErr(err)) + app.logger.Panic("data source: load config", xlog.FieldMod(ecode.ModConfig), xlog.FieldErrKind(ecode.ErrKindUnmarshalConfigErr), xlog.FieldErr(err)) } - app.logger.Info("load local file", xlog.FieldMod(ecode.ModConfig), xlog.FieldAddr(configAddr)) + } else { + app.logger.Info("no config... ", xlog.FieldMod(ecode.ModConfig)) } return nil } +//initLogger init func (app *Application) initLogger() error { if conf.Get("jupiter.logger.default") != nil { xlog.DefaultLogger = xlog.RawConfig("jupiter.logger.default").Build() } - xlog.DefaultLogger.AutoLevel("jupiter.logger.default") if conf.Get("jupiter.logger.jupiter") != nil { xlog.JupiterLogger = xlog.RawConfig("jupiter.logger.jupiter").Build() } - xlog.JupiterLogger.AutoLevel("jupiter.logger.jupiter") return nil } +//initTracer init func (app *Application) initTracer() error { // init tracing component jaeger if conf.Get("jupiter.trace.jaeger") != nil { @@ -418,6 +467,7 @@ func (app *Application) initTracer() error { return nil } +//initSentinel init func (app *Application) initSentinel() error { // init reliability component sentinel if conf.Get("jupiter.reliability.sentinel") != nil { @@ -428,6 +478,7 @@ func (app *Application) initSentinel() error { return nil } +//initMaxProcs init func (app *Application) initMaxProcs() error { if maxProcs := conf.GetInt("maxProc"); maxProcs != 0 { runtime.GOMAXPROCS(maxProcs) @@ -436,11 +487,11 @@ func (app *Application) initMaxProcs() error { app.logger.Panic("auto max procs", xlog.FieldMod(ecode.ModProc), xlog.FieldErrKind(ecode.ErrKindAny), xlog.FieldErr(err)) } } - app.logger.Info("auto max procs", xlog.FieldMod(ecode.ModProc), xlog.Int64("procs", int64(runtime.GOMAXPROCS(-1)))) return nil } +//printBanner init func (app *Application) printBanner() error { const banner = ` (_)_ _ _ __ (_) |_ ___ _ __ diff --git a/jupiter_test.go b/jupiter_test.go new file mode 100644 index 0000000000..8646718d02 --- /dev/null +++ b/jupiter_test.go @@ -0,0 +1,418 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jupiter + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/douyu/jupiter/pkg/server/xgrpc" + + "github.com/douyu/jupiter/pkg/server" + . "github.com/smartystreets/goconvey/convey" +) + +type testServer struct { + ServeBlockTime time.Duration + ServeErr error + + StopBlockTime time.Duration + StopErr error + + GstopBlockTime time.Duration + GstopErr error +} + +func (s *testServer) Serve() error { + time.Sleep(s.ServeBlockTime) + return s.ServeErr +} +func (s *testServer) Stop() error { + time.Sleep(s.StopBlockTime) + return s.StopErr +} +func (s *testServer) GracefulStop(ctx context.Context) error { + time.Sleep(s.GstopBlockTime) + return s.GstopErr +} +func (s *testServer) Info() *server.ServiceInfo { + return &server.ServiceInfo{} +} +func TestApplication_New(t *testing.T) { + Convey("test application run serve", t, func(c C) { + _, err := New() + So(err, ShouldBeNil) + }) +} +func TestApplication_Run_1(t *testing.T) { + Convey("test application run serve", t, func(c C) { + srv := &testServer{ + ServeErr: errors.New("when server call serve error"), + } + app := &Application{} + app.initialize() + err := app.Serve(srv) + So(err, ShouldBeNil) + go func() { + // make sure Serve() is called + time.Sleep(time.Millisecond * 100) + err := app.Stop() + c.So(err, ShouldBeNil) + }() + err = app.Run() + So(err, ShouldEqual, srv.ServeErr) + }) + Convey("test application run serve block", t, func(c C) { + srv := &testServer{ + ServeBlockTime: time.Second, + ServeErr: errors.New("when server call serve error"), + } + app := &Application{} + app.initialize() + err := app.Serve(srv) + So(err, ShouldBeNil) + go func() { + // make sure Serve() is called + time.Sleep(time.Millisecond * 100) + err := app.Stop() + c.So(err, ShouldBeNil) + }() + err = app.Run() + So(err, ShouldEqual, srv.ServeErr) + }) + Convey("test application run stop", t, func(c C) { + srv := &testServer{ + ServeBlockTime: time.Second * 2, + StopBlockTime: time.Second, + StopErr: errors.New("when server call stop error"), + } + app := &Application{} + app.initialize() + err := app.Serve(srv) + So(err, ShouldBeNil) + go func() { + // make sure Serve() is called + time.Sleep(time.Millisecond * 200) + err := app.Stop() + c.So(err, ShouldBeNil) + }() + err = app.Run() + So(err, ShouldEqual, srv.StopErr) + }) +} + +func TestApplication_initialize(t *testing.T) { + Convey("test application initialize", t, func() { + app := &Application{} + app.initialize() + So(app.servers, ShouldNotBeNil) + So(app.workers, ShouldNotBeNil) + So(app.logger, ShouldNotBeNil) + So(app.cycle, ShouldNotBeNil) + }) +} + +func TestApplication_Startup(t *testing.T) { + Convey("test application startup error", t, func() { + app := &Application{} + startUpErr := errors.New("throw startup error") + err := app.Startup(func() error { + return startUpErr + }) + So(err, ShouldEqual, startUpErr) + }) + + Convey("test application startup nil", t, func() { + app := &Application{} + err := app.Startup(func() error { + return nil + }) + So(err, ShouldBeNil) + }) +} + +type stopInfo struct { + state bool +} + +func (info *stopInfo) Stop() error { + info.state = true + return nil +} + +func TestApplication_BeforeStop(t *testing.T) { + Convey("test application before stop", t, func(c C) { + si := &stopInfo{} + app := &Application{} + app.initialize() + app.RegisterHooks(StageBeforeStop, si.Stop) + go func(si *stopInfo) { + time.Sleep(time.Microsecond * 100) + err := app.Stop() + c.So(err, ShouldBeNil) + c.So(si.state, ShouldEqual, true) + }(si) + err := app.Run() + c.So(err, ShouldBeNil) + }) +} +func TestApplication_EmptyRun(t *testing.T) { + Convey("test application empty run", t, func(c C) { + app := &Application{} + app.initialize() + go func() { + app.cycle.DoneAndClose() + }() + err := app.Run() + c.So(err, ShouldBeNil) + }) +} + +func TestApplication_AfterStop(t *testing.T) { + Convey("test application after stop", t, func() { + si := &stopInfo{} + app := &Application{} + app.initialize() + app.RegisterHooks(StageAfterStop, si.Stop) + go func() { + app.Stop() + }() + err := app.Run() + So(err, ShouldBeNil) + So(si.state, ShouldEqual, true) + }) +} + +func TestApplication_Serve(t *testing.T) { + Convey("test application serve throw wrong ip", t, func(c C) { + app := &Application{} + grpcConfig := xgrpc.DefaultConfig() + grpcConfig.Port = 0 + app.initialize() + err := app.Serve(grpcConfig.Build()) + So(err, ShouldBeNil) + go func() { + // make sure Serve() is called + time.Sleep(time.Millisecond * 1500) + err = app.Stop() + c.So(err, ShouldBeNil) + }() + err = app.Run() + // So(err, ShouldEqual, grpc.ErrServerStopped) + So(err, ShouldBeNil) + }) +} + +type testWorker struct { + RunErr error + StopErr error +} + +func (t *testWorker) Run() error { + return t.RunErr +} +func (t *testWorker) Stop() error { + return t.StopErr +} +func Test_Unit_Application_Schedule(t *testing.T) { + Convey("test unit Application.Schedule", t, func(c C) { + w := &testWorker{} + app := &Application{} + err := app.Schedule(w) + c.So(err, ShouldBeNil) + }) +} +func Test_Unit_Application_Stop(t *testing.T) { + Convey("test unit Application.Stop", t, func(c C) { + app := &Application{} + app.initialize() + err := app.Stop() + c.So(err, ShouldBeNil) + }) +} + +func Test_Unit_Application_GracefulStop(t *testing.T) { + Convey("test unit Application.GracefulStop", t, func(c C) { + app := &Application{} + app.initialize() + err := app.GracefulStop(context.TODO()) + c.So(err, ShouldBeNil) + }) +} +func Test_Unit_Application_startServers(t *testing.T) { + Convey("test unit Application.startServers", t, func(c C) { + app := &Application{} + app.initialize() + err := app.startServers() + c.So(err, ShouldBeNil) + go func() { + time.Sleep(time.Microsecond * 100) + app.Stop() + }() + }) +} + +type testJobRunner struct{} + +func (t *testJobRunner) Run() {} + +func Test_Unit_Application_Job(t *testing.T) { + j := &testJobRunner{} + app := &Application{} + app.initialize() + app.Job(j) +} + +/* + +func newFakeRegistry() registry.Registry { + return &fakeRegistry{ + prefix: "fake_registry", + store: make(map[string]string), + } +} + +type fakeRegistry struct { + prefix string + store map[string]string +} + +func (r *fakeRegistry) RegisterService(ctx context.Context, s *server.ServiceInfo) error { + r.store[registry.GetServiceKey(r.prefix, s)] = registry.GetServiceValue(s) + return nil +} +func (r *fakeRegistry) UnregisterService(ctx context.Context, s *server.ServiceInfo) error { + delete(r.store, registry.GetServiceKey(r.prefix, s)) + return nil +} +func (r *fakeRegistry) ListServices(ctx context.Context, s1 string, s2 string) ([]*server.ServiceInfo, error) { + var srvs []*server.ServiceInfo + for _, v := range r.store { + srvs = append(srvs, registry.GetService(v)) + } + return nil, nil +} +func (r *fakeRegistry) WatchServices(ctx context.Context, s1 string, s2 string) (chan registry.Endpoints, error) { + return nil, nil +} +func (r *fakeRegistry) Close() error { + return nil +} + +var _ registry.Registry = (*fakeRegistry)(nil) +*/ +/* +func TestRegister(t *testing.T) { + Convey("test application register", t, func(c C) { + app := &Application{} + grpcConfig := xgrpc.DefaultConfig() + grpcConfig.Port = 0 + app.initialize() + grpcServer := grpcConfig.Build() + err := app.Serve(grpcServer) + So(err, ShouldBeNil) + + etcdv3_registryConfig := etcdv3_registry.DefaultConfig() + etcdv3_registryConfig.Endpoints = []string{"127.0.0.1:2379"} + etcdConfig := etcdv3.DefaultConfig() + etcdConfig.Endpoints = []string{"127.0.0.1:2379"} + etcdctl := etcdConfig.Build() + app.SetRegistry( + compound_registry.New( + etcdv3_registryConfig.Build(), + ), + ) + err = app.RegisterHooks(StageBeforeStop, func() error { + resp, err := etcdctl.Get(context.Background(), "/jupiter/"+pkg.Name()+"/providers/grpc://", clientv3.WithPrefix()) + c.So(err, ShouldBeNil) + c.So(len(resp.Kvs), ShouldEqual, 1) + for _, value := range resp.Kvs { + c.So(string(value.Key), ShouldEqual, "/jupiter/"+pkg.Name()+"/providers/grpc://"+grpcServer.Address()) + c.So(string(value.Value), ShouldContainSubstring, pkg.Name()) + } + return nil + }) + So(err, ShouldBeNil) + + err = app.RegisterHooks(StageAfterStop, func() error { + //resp,err := etcdctl.Get(context.Background(),"/jupiter/"+pkg.Name()+"/providers/grpc://",clientv3.WithPrefix()) + return nil + }) + So(err, ShouldBeNil) + + go func() { + // make sure Serve() is called + time.Sleep(time.Millisecond * 3000) + err = app.Stop() + c.So(err, ShouldBeNil) + }() + err = app.Run() + So(err, ShouldBeNil) + }) +} + +func TestResolverAndRegister(t *testing.T) { + Convey("test application register and client resolver", t, func(c C) { + app := &Application{} + grpcConfig := xgrpc.DefaultConfig() + grpcConfig.Port = 0 + app.initialize() + + grpcServer := grpcConfig.Build() + fooServer := &yell.FooServer{} + fooServer.SetName("srv1") + testproto.RegisterGreeterServer(grpcServer.Server, fooServer) + err := app.Serve(grpcServer) + So(err, ShouldBeNil) + + etcdv3_registryConfig := etcdv3_registry.DefaultConfig() + etcdv3_registryConfig.Endpoints = []string{"127.0.0.1:2379"} + etcdConfig := etcdv3.DefaultConfig() + etcdConfig.Endpoints = []string{"127.0.0.1:2379"} + app.SetRegistry( + compound_registry.New( + etcdv3_registryConfig.Build(), + ), + ) + + go func() { + // make sure Serve() is called + time.Sleep(time.Millisecond * 3000) + + resolver.Register("etcd", etcdv3_registryConfig.Build()) + cfg := grpc.DefaultConfig() + cfg.Address = "etcd:///" + pkg.Name() + directClient := testproto.NewGreeterClient(cfg.Build()) + Convey("test resolver grpc", t, func() { + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + res, err := directClient.SayHello(ctx, &testproto.HelloRequest{ + Name: "hello", + }) + So(err, ShouldBeNil) + So(res.Message, ShouldEqual, yell.RespFantasy.Message) + }) + + err = app.Stop() + c.So(err, ShouldBeNil) + }() + err = app.Run() + So(err, ShouldBeNil) + }) +} +*/ diff --git a/pkg/client/etcdv3/client.go b/pkg/client/etcdv3/client.go index bb6627609d..3c616d8e52 100644 --- a/pkg/client/etcdv3/client.go +++ b/pkg/client/etcdv3/client.go @@ -18,6 +18,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "github.com/coreos/etcd/clientv3/concurrency" "io/ioutil" "strings" "time" @@ -213,3 +214,8 @@ func (client *Client) GetValues(ctx context.Context, keys ...string) (map[string } return vars, nil } + +//GetLeaseSession 创建租约会话 +func (client *Client) GetLeaseSession(ctx context.Context, opts ...concurrency.SessionOption) (leaseSession *concurrency.Session, err error) { + return concurrency.NewSession(client.Client, opts...) +} diff --git a/pkg/client/etcdv3/client_test.go b/pkg/client/etcdv3/client_test.go new file mode 100644 index 0000000000..8822031252 --- /dev/null +++ b/pkg/client/etcdv3/client_test.go @@ -0,0 +1,55 @@ +package etcdv3 + +import ( + "context" + "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/clientv3/concurrency" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func Test_GetKeyValue(t *testing.T) { + config := DefaultConfig() + config.Endpoints = []string{"127.0.0.1:2379"} + config.TTL = 5 + etcdCli := newClient(config) + + ctx := context.TODO() + + leaseSession, err := etcdCli.GetLeaseSession(ctx, concurrency.WithTTL(int(config.TTL))) + assert.Nil(t, err) + defer leaseSession.Close() + + _, err = etcdCli.Client.KV.Put(ctx, "/test/key", "{...}", clientv3.WithLease(leaseSession.Lease())) + assert.Nil(t, err) + + keyValue, err := etcdCli.GetKeyValue(ctx, "/test/key") + assert.Nil(t, err) + + assert.Equal(t, string(keyValue.Value), "{...}") +} + +func Test_MutexLock(t *testing.T) { + config := DefaultConfig() + config.Endpoints = []string{"127.0.0.1:2379"} + config.TTL = 10 + etcdCli := newClient(config) + + etcdMutex1, err := etcdCli.NewMutex("/test/lock", + concurrency.WithTTL(int(config.TTL))) + assert.Nil(t, err) + + err = etcdMutex1.Lock(time.Second * 1) + assert.Nil(t, err) + defer etcdMutex1.Unlock() + + // Grab the lock + etcdMutex, err := etcdCli.NewMutex("/test/lock", + concurrency.WithTTL(int(config.TTL))) + assert.Nil(t, err) + defer etcdMutex.Unlock() + + err = etcdMutex.Lock(time.Second * 1) + assert.NotNil(t, err) +} diff --git a/pkg/client/etcdv3/config.go b/pkg/client/etcdv3/config.go index a63fd536ca..a7f0b2bf4f 100644 --- a/pkg/client/etcdv3/config.go +++ b/pkg/client/etcdv3/config.go @@ -38,6 +38,7 @@ type ( Secure bool `json:"secure"` // 自动同步member list的间隔 AutoSyncInterval time.Duration `json:"autoAsyncInterval"` + TTL int // 单位：s logger *xlog.Logger } ) diff --git a/pkg/client/etcdv3/lock.go b/pkg/client/etcdv3/lock.go index a6170a323e..abad29b162 100644 --- a/pkg/client/etcdv3/lock.go +++ b/pkg/client/etcdv3/lock.go @@ -28,10 +28,10 @@ type Mutex struct { } // NewMutex ... -func (client *Client) NewMutex(key string) (mutex *Mutex, err error) { +func (client *Client) NewMutex(key string, opts ...concurrency.SessionOption) (mutex *Mutex, err error) { mutex = &Mutex{} // 默认session ttl = 60s - mutex.s, err = concurrency.NewSession(client.Client) + mutex.s, err = concurrency.NewSession(client.Client, opts...) if err != nil { return } @@ -46,6 +46,13 @@ func (mutex *Mutex) Lock(timeout time.Duration) (err error) { return mutex.m.Lock(ctx) } +// TryLock ... +func (mutex *Mutex) TryLock(timeout time.Duration) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return mutex.m.Lock(ctx) +} + // Unlock ... func (mutex *Mutex) Unlock() (err error) { err = mutex.m.Unlock(context.TODO()) diff --git a/pkg/client/etcdv3/watch.go b/pkg/client/etcdv3/watch.go index 91d43903ba..230bb3cb00 100644 --- a/pkg/client/etcdv3/watch.go +++ b/pkg/client/etcdv3/watch.go @@ -17,20 +17,23 @@ package etcdv3 import ( "context" "sync" - "time" "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/mvcc/mvccpb" + "github.com/douyu/jupiter/pkg/ecode" + "github.com/douyu/jupiter/pkg/util/xgo" "github.com/douyu/jupiter/pkg/xlog" ) // Watch A watch only tells the latest revision type Watch struct { revision int64 - client *Client cancel context.CancelFunc eventChan chan *clientv3.Event lock *sync.RWMutex logger *xlog.Logger + + incipientKVs []*mvccpb.KeyValue } // C ... @@ -38,56 +41,65 @@ func (w *Watch) C() chan *clientv3.Event { return w.eventChan } -func (w *Watch) update(resp *clientv3.WatchResponse) { - if resp.CompactRevision > w.revision { - w.revision = resp.CompactRevision - } else if resp.Header.GetRevision() > w.revision { - w.revision = resp.Header.GetRevision() - } +// IncipientKeyValues incipient key and values +func (w *Watch) IncipientKeyValues() []*mvccpb.KeyValue { + return w.incipientKVs +} - if err := resp.Err(); err != nil { - w.logger.Error("handle watch update", xlog.Any("err", err)) - return +// NewWatch ... +func (client *Client) WatchPrefix(ctx context.Context, prefix string) (*Watch, error) { + resp, err := client.Get(ctx, prefix, clientv3.WithPrefix()) + if err != nil { + return nil, err } - for _, event := range resp.Events { - select { - case w.eventChan <- event: - default: - w.logger.Warn("handle watch block", xlog.Int64("revision", w.revision), xlog.Any("kv", event.Kv)) - } + var w = &Watch{ + revision: resp.Header.Revision, + eventChan: make(chan *clientv3.Event, 100), + incipientKVs: resp.Kvs, } -} - -// NewWatch ... -func (client *Client) NewWatch(prefix string) (*Watch, error) { - var ( - ctx, cancel = context.WithCancel(context.Background()) - watcher = &Watch{ - client: client, - revision: 0, - cancel: cancel, - eventChan: make(chan *clientv3.Event, 100), - lock: &sync.RWMutex{}, - logger: client.config.logger, - } - ) - go func() { - rch := client.Watch(ctx, prefix, clientv3.WithPrefix(), clientv3.WithCreatedNotify()) + xgo.Go(func() { + ctx, cancel := context.WithCancel(context.Background()) + w.cancel = cancel + rch := client.Client.Watch(ctx, prefix, clientv3.WithPrefix(), clientv3.WithCreatedNotify(), clientv3.WithRev(w.revision)) for { - for resp := range rch { - watcher.update(&resp) + for n := range rch { + if n.CompactRevision > w.revision { + w.revision = n.CompactRevision + } + if n.Header.GetRevision() > w.revision { + w.revision = n.Header.GetRevision() + } + if err := n.Err(); err != nil { + xlog.Error(ecode.MsgWatchRequestErr, xlog.FieldErrKind(ecode.ErrKindRegisterErr), xlog.FieldErr(err), xlog.FieldAddr(prefix)) + continue + } + for _, ev := range n.Events { + select { + case w.eventChan <- ev: + default: + xlog.Error("watch etcd with prefix", xlog.Any("err", "block event chan, drop event message")) + } + } } - - time.Sleep(time.Duration(1) * time.Second) - if watcher.revision > 0 { - rch = client.Watch(ctx, prefix, clientv3.WithPrefix(), clientv3.WithCreatedNotify()) + ctx, cancel := context.WithCancel(context.Background()) + w.cancel = cancel + if w.revision > 0 { + rch = client.Watch(ctx, prefix, clientv3.WithPrefix(), clientv3.WithCreatedNotify(), clientv3.WithRev(w.revision)) } else { rch = client.Watch(ctx, prefix, clientv3.WithPrefix(), clientv3.WithCreatedNotify()) } } - }() + }) - return watcher, nil + return w, nil +} + +// Close close watch +func (w *Watch) Close() error { + if w.cancel != nil { + w.cancel() + } + return nil } diff --git a/pkg/client/grpc/balancer/swr.go b/pkg/client/grpc/balancer/swr.go index 7be109cfcf..1cc52b21f6 100644 --- a/pkg/client/grpc/balancer/swr.go +++ b/pkg/client/grpc/balancer/swr.go @@ -109,7 +109,7 @@ func newWeightPicker(readySCs map[resolver.Address]balancer.SubConn) *weightPick } // Pick ... -func (p *weightPicker) Pick(ctx context.Context, opts balancer.PickOptions) (balancer.SubConn, func(balancer.DoneInfo), error) { +func (p *weightPicker) Pick(ctx context.Context, opts balancer.PickInfo) (balancer.SubConn, func(balancer.DoneInfo), error) { p.mu.Lock() defer p.mu.Unlock() diff --git a/pkg/client/grpc/client.go b/pkg/client/grpc/client.go index 40211d2138..9024051d8f 100644 --- a/pkg/client/grpc/client.go +++ b/pkg/client/grpc/client.go @@ -24,10 +24,10 @@ import ( "google.golang.org/grpc" ) -func newGRPCClient(config Config) *grpc.ClientConn { +func newGRPCClient(config *Config) *grpc.ClientConn { var ctx = context.Background() var dialOptions = config.dialOptions - config.logger = config.logger.With( + logger := config.logger.With( xlog.FieldMod("client.grpc"), xlog.FieldAddr(config.Address), ) @@ -46,15 +46,17 @@ func newGRPCClient(config Config) *grpc.ClientConn { dialOptions = append(dialOptions, grpc.WithKeepaliveParams(*config.KeepAlive)) } + dialOptions = append(dialOptions, grpc.WithBalancerName(config.BalancerName)) + cc, err := grpc.DialContext(ctx, config.Address, dialOptions...) if err != nil { if config.OnDialError == "panic" { - config.logger.Panic("dial grpc server", xlog.FieldErrKind(ecode.ErrKindRequestErr), xlog.FieldErr(err)) + logger.Panic("dial grpc server", xlog.FieldErrKind(ecode.ErrKindRequestErr), xlog.FieldErr(err)) } else { - config.logger.Error("dial grpc server", xlog.FieldErrKind(ecode.ErrKindRequestErr), xlog.FieldErr(err)) + logger.Error("dial grpc server", xlog.FieldErrKind(ecode.ErrKindRequestErr), xlog.FieldErr(err)) } } - config.logger.Info("start grpc client") + logger.Info("start grpc client") return cc } diff --git a/pkg/client/grpc/client_test.go b/pkg/client/grpc/client_test.go new file mode 100644 index 0000000000..eac4823b37 --- /dev/null +++ b/pkg/client/grpc/client_test.go @@ -0,0 +1,49 @@ +package grpc + +import ( + "context" + "github.com/douyu/jupiter/pkg/util/xtest/proto/testproto" + "github.com/douyu/jupiter/pkg/util/xtest/server/yell" + . "github.com/smartystreets/goconvey/convey" + "testing" + "time" +) + +// TestBase test direct dial with New() +func TestDirectGrpc(t *testing.T) { + Convey("test direct grpc", t, func() { + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + res, err := directClient.SayHello(ctx, &testproto.HelloRequest{ + Name: "hello", + }) + So(err, ShouldBeNil) + So(res.Message, ShouldEqual, yell.RespFantasy.Message) + }) +} + +func TestConfigBlockTrue(t *testing.T) { + Convey("test no address block, and panic", t, func() { + flag := false + defer func() { + if r := recover(); r != nil { + flag = true + } + So(flag, ShouldEqual, true) + }() + cfg := DefaultConfig() + cfg.OnDialError = "panic" + newGRPCClient(cfg) + }) +} + +func TestConfigBlockFalse(t *testing.T) { + Convey("test no address and no block", t, func() { + cfg := DefaultConfig() + cfg.OnDialError = "panic" + cfg.Block = false + conn := newGRPCClient(cfg) + So(conn.GetState().String(), ShouldEqual, "IDLE") + }) +} diff --git a/pkg/client/grpc/common_test.go b/pkg/client/grpc/common_test.go new file mode 100644 index 0000000000..25f4f8c1d4 --- /dev/null +++ b/pkg/client/grpc/common_test.go @@ -0,0 +1,42 @@ +package grpc + +import ( + "github.com/douyu/jupiter/pkg/util/xtest/proto/testproto" + "github.com/douyu/jupiter/pkg/util/xtest/server/yell" + "google.golang.org/grpc" + "net" + "testing" + "time" +) + +var directClient testproto.GreeterClient + +func TestMain(m *testing.M) { + l, s := startServer("127.0.0.1:0", "srv1") + time.Sleep(200 * time.Millisecond) + + cfg := DefaultConfig() + cfg.Address = l.Addr().String() + + conn := newGRPCClient(cfg) + directClient = testproto.NewGreeterClient(conn) + m.Run() + s.Stop() +} + +func startServer(addr, name string) (net.Listener, *grpc.Server) { + l, err := net.Listen("tcp", addr) + if err != nil { + panic("failed start server:" + err.Error()) + } + server := grpc.NewServer() + grpcServer := &yell.FooServer{} + grpcServer.SetName(name) + testproto.RegisterGreeterServer(server, grpcServer) + go func() { + if err := server.Serve(l); err != nil { + panic("failed serve:" + err.Error()) + } + }() + return l, server +} diff --git a/pkg/client/grpc/config.go b/pkg/client/grpc/config.go index 69f253c751..5276bb2fd3 100644 --- a/pkg/client/grpc/config.go +++ b/pkg/client/grpc/config.go @@ -15,11 +15,11 @@ package grpc import ( + "github.com/douyu/jupiter/pkg/util/xtime" "time" - "github.com/douyu/jupiter/pkg/ecode" - "github.com/douyu/jupiter/pkg/conf" + "github.com/douyu/jupiter/pkg/ecode" "github.com/douyu/jupiter/pkg/xlog" "google.golang.org/grpc" "google.golang.org/grpc/balancer/roundrobin" @@ -28,20 +28,27 @@ import ( // Config ... type Config struct { + Name string // config's name BalancerName string Address string Block bool DialTimeout time.Duration + ReadTimeout time.Duration Direct bool OnDialError string // panic | error KeepAlive *keepalive.ClientParameters logger *xlog.Logger dialOptions []grpc.DialOption - // resolver resolver.Builder - Debug bool - DisableTrace bool - DisableMetric bool + SlowThreshold time.Duration + + Debug bool + DisableTraceInterceptor bool + DisableAidInterceptor bool + DisableTimeoutInterceptor bool + DisableMetricInterceptor bool + DisableAccessInterceptor bool + AccessInterceptorLevel string } // DefaultConfig ... @@ -50,10 +57,14 @@ func DefaultConfig() *Config { dialOptions: []grpc.DialOption{ grpc.WithInsecure(), }, - logger: xlog.JupiterLogger.With(xlog.FieldMod(ecode.ModClientGrpc)), - BalancerName: roundrobin.Name, // roundrobin by default - DialTimeout: time.Second * 3, - OnDialError: "panic", + logger: xlog.JupiterLogger.With(xlog.FieldMod(ecode.ModClientGrpc)), + BalancerName: roundrobin.Name, // round robin by default + DialTimeout: time.Second * 3, + ReadTimeout: xtime.Duration("1s"), + SlowThreshold: xtime.Duration("600ms"), + OnDialError: "panic", + AccessInterceptorLevel: "info", + Block: true, } } @@ -93,12 +104,36 @@ func (config *Config) Build() *grpc.ClientConn { grpc.WithChainUnaryInterceptor(debugUnaryClientInterceptor(config.Address)), ) } - if !config.DisableTrace { + + if !config.DisableAidInterceptor { + config.dialOptions = append(config.dialOptions, + grpc.WithChainUnaryInterceptor(aidUnaryClientInterceptor()), + ) + } + + if !config.DisableTimeoutInterceptor { + config.dialOptions = append(config.dialOptions, + grpc.WithChainUnaryInterceptor(timeoutUnaryClientInterceptor(config.logger, config.ReadTimeout, config.SlowThreshold)), + ) + } + + if !config.DisableTraceInterceptor { config.dialOptions = append(config.dialOptions, grpc.WithChainUnaryInterceptor(traceUnaryClientInterceptor()), ) } - client := newGRPCClient(*config) - return client + if !config.DisableAccessInterceptor { + config.dialOptions = append(config.dialOptions, + grpc.WithChainUnaryInterceptor(loggerUnaryClientInterceptor(config.logger, config.Name, config.AccessInterceptorLevel)), + ) + } + + if !config.DisableMetricInterceptor { + config.dialOptions = append(config.dialOptions, + grpc.WithChainUnaryInterceptor(metricUnaryClientInterceptor(config.Name)), + ) + } + + return newGRPCClient(config) } diff --git a/pkg/client/grpc/config_test.go b/pkg/client/grpc/config_test.go new file mode 100644 index 0000000000..096885d6fb --- /dev/null +++ b/pkg/client/grpc/config_test.go @@ -0,0 +1,44 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package grpc + +import ( + "bytes" + "testing" + "time" + + "github.com/BurntSushi/toml" + "github.com/douyu/jupiter/pkg/conf" + "github.com/stretchr/testify/assert" +) + +func TestConfig(t *testing.T) { + var configStr = ` +[jupiter.client.test] + balancerName="swr" + address="127.0.0.1:9091" + dialTimeout="10s" + ` + assert.Nil(t, conf.LoadFromReader(bytes.NewBufferString(configStr), toml.Unmarshal)) + + t.Run("std config", func(t *testing.T) { + config := StdConfig("test") + assert.Equal(t, "swr", config.BalancerName) + assert.Equal(t, time.Second*10, config.DialTimeout) + assert.Equal(t, "127.0.0.1:9091", config.Address) + assert.Equal(t, false, config.Direct) + assert.Equal(t, "panic", config.OnDialError) + }) +} diff --git a/pkg/client/grpc/interceptor.go b/pkg/client/grpc/interceptor.go index efeab1fe1c..1e93f2d1f5 100644 --- a/pkg/client/grpc/interceptor.go +++ b/pkg/client/grpc/interceptor.go @@ -16,7 +16,11 @@ package grpc import ( "context" + "encoding/json" + "errors" "fmt" + "github.com/douyu/jupiter/pkg" + "github.com/douyu/jupiter/pkg/xlog" "time" "github.com/douyu/jupiter/pkg/ecode" @@ -32,18 +36,27 @@ import ( "google.golang.org/grpc/status" ) +var ( + errSlowCommand = errors.New("grpc unary slow command") +) + // metric统计 -func metricUnaryClientInterceptor(address string) func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { +func metricUnaryClientInterceptor(name string) func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { beg := time.Now() err := invoker(ctx, method, req, reply, cc, opts...) // 收敛err错误，将err过滤后，可以知道err是否为系统错误码 - codes := ecode.ExtractCodes(err) - - // 只记录系统级别的详细错误码 - metric.ClientHandleCounter.Inc(metric.TypeGRPCUnary, address, method, cc.Target(), codes.GetMessage()) - metric.ClientHandleHistogram.Observe(time.Since(beg).Seconds(), metric.TypeGRPCUnary, address, method, cc.Target()) + spbStatus := ecode.ExtractCodes(err) + // 只记录系统级别错误 + if spbStatus.Code < ecode.EcodeNum { + // 只记录系统级别的详细错误码 + metric.ClientHandleCounter.Inc(metric.TypeGRPCUnary, name, method, cc.Target(), spbStatus.GetMessage()) + metric.ClientHandleHistogram.Observe(time.Since(beg).Seconds(), metric.TypeGRPCUnary, name, method, cc.Target()) + } else { + metric.ClientHandleCounter.Inc(metric.TypeGRPCUnary, name, method, cc.Target(), "biz error") + metric.ClientHandleHistogram.Observe(time.Since(beg).Seconds(), metric.TypeGRPCUnary, name, method, cc.Target()) + } return err } } @@ -112,3 +125,106 @@ func traceUnaryClientInterceptor() grpc.UnaryClientInterceptor { return err } } + +func aidUnaryClientInterceptor() grpc.UnaryClientInterceptor { + return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + md, ok := metadata.FromOutgoingContext(ctx) + clientAidMD := metadata.Pairs("aid", pkg.AppID()) + if ok { + md = metadata.Join(md, clientAidMD) + } else { + md = clientAidMD + } + ctx = metadata.NewOutgoingContext(ctx, md) + + return invoker(ctx, method, req, reply, cc, opts...) + } +} + +// timeoutUnaryClientInterceptor gRPC客户端超时拦截器 +func timeoutUnaryClientInterceptor(_logger *xlog.Logger, timeout time.Duration, slowThreshold time.Duration) grpc.UnaryClientInterceptor { + return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + now := time.Now() + // 若无自定义超时设置，默认设置超时 + _, ok := ctx.Deadline() + if !ok { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, timeout) + defer cancel() + } + + err := invoker(ctx, method, req, reply, cc, opts...) + du := time.Since(now) + remoteIP := "unknown" + if remote, ok := peer.FromContext(ctx); ok && remote.Addr != nil { + remoteIP = remote.Addr.String() + } + + if slowThreshold > time.Duration(0) && du > slowThreshold { + _logger.Error("slow", + xlog.FieldErr(errSlowCommand), + xlog.FieldMethod(method), + xlog.FieldName(cc.Target()), + xlog.FieldCost(du), + xlog.FieldAddr(remoteIP), + ) + } + return err + } +} + +// loggerUnaryClientInterceptor gRPC客户端日志中间件 +func loggerUnaryClientInterceptor(_logger *xlog.Logger, name string, accessInterceptorLevel string) grpc.UnaryClientInterceptor { + return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + beg := time.Now() + err := invoker(ctx, method, req, reply, cc, opts...) + + spbStatus := ecode.ExtractCodes(err) + if err != nil { + // 只记录系统级别错误 + if spbStatus.Code < ecode.EcodeNum { + // 只记录系统级别错误 + _logger.Error( + "access", + xlog.FieldType("unary"), + xlog.FieldCode(spbStatus.Code), + xlog.FieldStringErr(spbStatus.Message), + xlog.FieldName(name), + xlog.FieldMethod(method), + xlog.FieldCost(time.Since(beg)), + xlog.Any("req", json.RawMessage(xstring.Json(req))), + xlog.Any("reply", json.RawMessage(xstring.Json(reply))), + ) + } else { + // 业务报错只做warning + _logger.Warn( + "access", + xlog.FieldType("unary"), + xlog.FieldCode(spbStatus.Code), + xlog.FieldStringErr(spbStatus.Message), + xlog.FieldName(name), + xlog.FieldMethod(method), + xlog.FieldCost(time.Since(beg)), + xlog.Any("req", json.RawMessage(xstring.Json(req))), + xlog.Any("reply", json.RawMessage(xstring.Json(reply))), + ) + } + return err + } else { + if accessInterceptorLevel == "info" { + _logger.Info( + "access", + xlog.FieldType("unary"), + xlog.FieldCode(spbStatus.Code), + xlog.FieldName(name), + xlog.FieldMethod(method), + xlog.FieldCost(time.Since(beg)), + xlog.Any("req", json.RawMessage(xstring.Json(req))), + xlog.Any("reply", json.RawMessage(xstring.Json(reply))), + ) + } + } + + return nil + } +} diff --git a/pkg/client/grpc/resolver/resolver.go b/pkg/client/grpc/resolver/resolver.go new file mode 100644 index 0000000000..1748c2f3ed --- /dev/null +++ b/pkg/client/grpc/resolver/resolver.go @@ -0,0 +1,92 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resolver + +import ( + "context" + + "github.com/douyu/jupiter/pkg/constant" + "github.com/douyu/jupiter/pkg/registry" + "github.com/douyu/jupiter/pkg/util/xgo" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/resolver" +) + +// Register ... +func Register(name string, reg registry.Registry) { + resolver.Register(&baseBuilder{ + name: name, + reg: reg, + }) +} + +type baseBuilder struct { + name string + reg registry.Registry +} + +// Build ... +func (b *baseBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + endpoints, err := b.reg.WatchServices(context.Background(), target.Endpoint, "grpc") + if err != nil { + return nil, err + } + + var stop = make(chan struct{}) + xgo.Go(func() { + for { + select { + case endpoint := <-endpoints: + var state = resolver.State{ + Addresses: make([]resolver.Address, 0), + Attributes: attributes.New( + constant.KeyRouteConfig, endpoint.RouteConfigs, // 路由配置 + constant.KeyProviderConfig, endpoint.ProviderConfigs, // 服务提供方元信息 + constant.KeyConsumerConfig, endpoint.ConsumerConfigs, // 服务消费方配置信息 + ), + } + for _, node := range endpoint.Nodes { + var address resolver.Address + address.Addr = node.Address + address.ServerName = target.Endpoint + address.Attributes = attributes.New(constant.KeyServiceInfo, node) + state.Addresses = append(state.Addresses, address) + } + cc.UpdateState(state) + case <-stop: + return + } + } + }) + + return &baseResolver{ + stop: stop, + }, nil +} + +// Scheme ... +func (b baseBuilder) Scheme() string { + return b.name +} + +type baseResolver struct { + stop chan struct{} +} + +// ResolveNow ... +func (b *baseResolver) ResolveNow(options resolver.ResolveNowOptions) {} + +// Close ... +func (b *baseResolver) Close() { b.stop <- struct{}{} } diff --git a/pkg/registry/etcdv3/resolver_test.go b/pkg/client/grpc/resolver/resolver_test.go similarity index 66% rename from pkg/registry/etcdv3/resolver_test.go rename to pkg/client/grpc/resolver/resolver_test.go index 9c4848a070..6465503442 100644 --- a/pkg/registry/etcdv3/resolver_test.go +++ b/pkg/client/grpc/resolver/resolver_test.go @@ -12,22 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package etcdv3 +package resolver -import ( - "fmt" - "net/url" - "testing" -) +import "testing" -func Test_parseurl(t *testing.T) { - uriStr := "grpc:///?app=name" - uri, err := url.Parse(uriStr) - if err != nil { - fmt.Printf("err => %+v\n", err) - panic(err) - } +func Test_baseResolver(t *testing.T) { - t.Logf("uri.Host: %s", uri.Host) - // fmt.Printf("uri.Host => %+v\n", uri.Host) } diff --git a/pkg/client/redis/config.go b/pkg/client/redis/config.go index eef4dab4b3..6a2c1fc338 100644 --- a/pkg/client/redis/config.go +++ b/pkg/client/redis/config.go @@ -86,7 +86,7 @@ func DefaultRedisConfig() Config { EnableTrace: false, SlowThreshold: xtime.Duration("250ms"), OnDialError: "panic", - logger: xlog.DefaultLogger, + logger: xlog.JupiterLogger, } } diff --git a/pkg/conf/conf.go b/pkg/conf/conf.go index d7f2d19546..4d21d3af9e 100644 --- a/pkg/conf/conf.go +++ b/pkg/conf/conf.go @@ -106,9 +106,9 @@ func (c *Configuration) LoadFromDataSource(ds DataSource, unmarshaller Unmarshal } // Load ... -func (c *Configuration) Load(content []byte, unmarshaller Unmarshaller) error { +func (c *Configuration) Load(content []byte, unmarshal Unmarshaller) error { configuration := make(map[string]interface{}) - if err := unmarshaller(content, &configuration); err != nil { + if err := unmarshal(content, &configuration); err != nil { return err } return c.apply(configuration) diff --git a/pkg/constant/env.go b/pkg/constant/env.go index cd30f04999..4ed623279c 100644 --- a/pkg/constant/env.go +++ b/pkg/constant/env.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package constant const ( @@ -6,3 +20,34 @@ const ( // EnvKeySentinelAppName ... EnvKeySentinelAppName = "SENTINEL_APP_NAME" ) + +const ( + // EnvAppName ... + EnvAppName = "APP_NAME" + // EnvDeployment ... + EnvDeployment = "APP_DEPLOYMENT" + + EnvAppLogDir = "APP_LOG_DIR" + EnvAppMode = "APP_MODE" + EnvAppRegion = "APP_REGION" + EnvAppZone = "APP_ZONE" + EnvAppHost = "APP_HOST" + EnvAppInstance = "APP_INSTANCE" // application unique instance id. +) + +const ( + // DefaultDeployment ... + DefaultDeployment = "" + // DefaultRegion ... + DefaultRegion = "" + // DefaultZone ... + DefaultZone = "" +) + +const ( + // KeyBalanceGroup ... + KeyBalanceGroup = "__group" + + // DefaultBalanceGroup ... + DefaultBalanceGroup = "default" +) diff --git a/pkg/conf/init.go b/pkg/constant/key.go similarity index 62% rename from pkg/conf/init.go rename to pkg/constant/key.go index d34a4b3b1e..78f1e60fe8 100644 --- a/pkg/conf/init.go +++ b/pkg/constant/key.go @@ -12,21 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -package conf +package constant -import ( - "encoding/json" - "net/http" +const ( + // KeyRouteConfig ... + KeyRouteConfig = "__route_config_" - "github.com/douyu/jupiter/pkg/govern" -) + // KeyRouteGroup ... + KeyRouteGroup = "__route_group_" + + // KeyProviderConfig ... + KeyProviderConfig = "__provider_config_" -func init() { - govern.HandleFunc("/configs", func(w http.ResponseWriter, r *http.Request) { - encoder := json.NewEncoder(w) - if r.URL.Query().Get("pretty") == "true" { - encoder.SetIndent("", " ") - } - encoder.Encode(defaultConfiguration.traverse(".")) - }) -} + // KeyConsumerConfig ... + KeyConsumerConfig = "__consumer_config_" + + // KeyServiceInfo + KeyServiceInfo = "__service_info_" +) diff --git a/pkg/constant/service.go b/pkg/constant/service.go index 0e2798febc..014d831bdd 100644 --- a/pkg/constant/service.go +++ b/pkg/constant/service.go @@ -1,23 +1,31 @@ package constant +//ServiceKind service kind type ServiceKind uint8 const ( + //ServiceUnknown service non-name ServiceUnknown ServiceKind = iota + //ServiceProvider service provider ServiceProvider + //ServiceGovernor service governor ServiceGovernor + //ServiceConsumer service consumer ServiceConsumer ) +var serviceKinds = make(map[ServiceKind]string) + +func init() { + serviceKinds[ServiceUnknown] = "unknown" + serviceKinds[ServiceProvider] = "providers" + serviceKinds[ServiceGovernor] = "governors" + serviceKinds[ServiceConsumer] = "consumers" +} + func (sk ServiceKind) String() string { - switch sk { - case ServiceProvider: - return "providers" - case ServiceGovernor: - return "governors" - case ServiceConsumer: - return "consumers" - default: - return "unknown" + if s, ok := serviceKinds[sk]; ok { + return s } + return "unknown" } diff --git a/pkg/datasource/apollo/apollo.go b/pkg/datasource/apollo/apollo.go index 121dc43380..d48cf36189 100644 --- a/pkg/datasource/apollo/apollo.go +++ b/pkg/datasource/apollo/apollo.go @@ -1,40 +1,52 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package apollo import ( "github.com/douyu/jupiter/pkg/conf" - "github.com/douyu/jupiter/pkg/util/xgo" - "github.com/philchia/agollo" + "github.com/douyu/jupiter/pkg/xlog" + "github.com/philchia/agollo/v4" ) type apolloDataSource struct { - client *agollo.Client + client agollo.Client namespace string propertyKey string changed chan struct{} - quit chan struct{} } // NewDataSource creates an apolloDataSource func NewDataSource(conf *agollo.Conf, namespace string, key string) conf.DataSource { - client := agollo.NewClient(conf) + client := agollo.NewClient(conf, agollo.WithLogger(&agolloLogger{})) ap := &apolloDataSource{ client: client, namespace: namespace, propertyKey: key, - changed: make(chan struct{}), - quit: make(chan struct{}), + changed: make(chan struct{}, 1), } ap.client.Start() - changedEvent := ap.client.WatchUpdate() - xgo.Go(func() { - ap.watch(changedEvent) - }) + ap.client.OnUpdate( + func(event *agollo.ChangeEvent) { + ap.changed <- struct{}{} + }) return ap } // ReadConfig reads config content from apollo func (ap *apolloDataSource) ReadConfig() ([]byte, error) { - value := ap.client.GetStringValueWithNameSpace(ap.namespace, ap.propertyKey, "") + value := ap.client.GetString(ap.propertyKey, agollo.WithNamespace(ap.namespace)) return []byte(value), nil } @@ -43,21 +55,22 @@ func (ap *apolloDataSource) IsConfigChanged() <-chan struct{} { return ap.changed } -func (ap *apolloDataSource) watch(changedEvent <-chan *agollo.ChangeEvent) { - for { - select { - case <-changedEvent: - ap.changed <- struct{}{} - case <-ap.quit: - ap.client.Stop() - close(ap.changed) - return - } - } -} - // Close stops watching the config changed func (ap *apolloDataSource) Close() error { - ap.quit <- struct{}{} + ap.client.Stop() + close(ap.changed) return nil } + +type agolloLogger struct { +} + +// Infof ... +func (l *agolloLogger) Infof(format string, args ...interface{}) { + xlog.Infof(format, args...) +} + +// Errorf ... +func (l *agolloLogger) Errorf(format string, args ...interface{}) { + xlog.Errorf(format, args...) +} diff --git a/pkg/datasource/apollo/apollo_test.go b/pkg/datasource/apollo/apollo_test.go index a62f309730..c304263fde 100644 --- a/pkg/datasource/apollo/apollo_test.go +++ b/pkg/datasource/apollo/apollo_test.go @@ -1,8 +1,22 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package apollo import ( "github.com/douyu/jupiter/pkg/datasource/apollo/mockserver" - "github.com/philchia/agollo" + "github.com/philchia/agollo/v4" "github.com/stretchr/testify/assert" "log" "sync" @@ -32,36 +46,39 @@ func teardown() { func TestReadConfig(t *testing.T) { testData := []string{"value1", "value2"} + + mockserver.Set("application", "key_test", testData[0]) ds := NewDataSource(&agollo.Conf{ AppID: "SampleApp", Cluster: "default", NameSpaceNames: []string{"application"}, - IP: "localhost:16852", + MetaAddr: "localhost:16852", + CacheDir: ".", }, "application", "key_test") + value, err := ds.ReadConfig() + assert.Nil(t, err) + assert.Equal(t, testData[0], string(value)) + t.Logf("read: %s", value) wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() - mockserver.Set("application", "key_test", testData[0]) - time.Sleep(time.Second * 3) mockserver.Set("application", "key_test", testData[1]) time.Sleep(time.Second * 3) ds.Close() }() + wg.Add(1) go func() { defer wg.Done() - time.Sleep(time.Second) - index := 0 - for range ds.IsConfigChanged() { value, err := ds.ReadConfig() assert.Nil(t, err) - assert.Equal(t, testData[index], string(value)) - index++ + assert.Equal(t, testData[1], string(value)) t.Logf("read: %s", value) } }() + wg.Wait() } diff --git a/pkg/datasource/apollo/mockserver/mockserver.go b/pkg/datasource/apollo/mockserver/mockserver.go index 7d3d6ece81..02f7e9802f 100644 --- a/pkg/datasource/apollo/mockserver/mockserver.go +++ b/pkg/datasource/apollo/mockserver/mockserver.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package mockserver import ( diff --git a/pkg/datasource/apollo/register.go b/pkg/datasource/apollo/register.go new file mode 100644 index 0000000000..450dc2c8fa --- /dev/null +++ b/pkg/datasource/apollo/register.go @@ -0,0 +1,48 @@ +package apollo + +import ( + "github.com/douyu/jupiter/pkg/conf" + "github.com/douyu/jupiter/pkg/datasource/manager" + "github.com/douyu/jupiter/pkg/flag" + "github.com/douyu/jupiter/pkg/xlog" + "github.com/philchia/agollo/v4" + "net/url" +) + +// DataSourceApollo defines apollo scheme +const DataSourceApollo = "apollo" + +func init() { + manager.Register(DataSourceApollo, func() conf.DataSource { + var ( + configAddr = flag.String("config") + ) + if configAddr == "" { + xlog.Panic("new apollo dataSource, configAddr is empty") + return nil + } + // configAddr is a string in this format: + // apollo://ip:port?appId=XXX&cluster=XXX&namespaceName=XXX&key=XXX&accesskeySecret=XXX&insecureSkipVerify=XXX&cacheDir=XXX + urlObj, err := url.Parse(configAddr) + if err != nil { + xlog.Panic("parse configAddr error", xlog.FieldErr(err)) + return nil + } + apolloConf := agollo.Conf{ + AppID: urlObj.Query().Get("appId"), + Cluster: urlObj.Query().Get("cluster"), + NameSpaceNames: []string{urlObj.Query().Get("namespaceName")}, + MetaAddr: urlObj.Host, + InsecureSkipVerify: true, + AccesskeySecret: urlObj.Query().Get("accesskeySecret"), + CacheDir: ".", + } + if urlObj.Query().Get("insecureSkipVerify") == "false" { + apolloConf.InsecureSkipVerify = false + } + if urlObj.Query().Get("cacheDir") != "" { + apolloConf.CacheDir = urlObj.Query().Get("cacheDir") + } + return NewDataSource(&apolloConf, urlObj.Query().Get("namespaceName"), urlObj.Query().Get("key")) + }) +} diff --git a/pkg/datasource/etcdv3/register.go b/pkg/datasource/etcdv3/register.go new file mode 100644 index 0000000000..7284c8872e --- /dev/null +++ b/pkg/datasource/etcdv3/register.go @@ -0,0 +1,47 @@ +package etcdv3 + +import ( + "github.com/douyu/jupiter/pkg/client/etcdv3" + "github.com/douyu/jupiter/pkg/conf" + "github.com/douyu/jupiter/pkg/datasource/manager" + "github.com/douyu/jupiter/pkg/flag" + "github.com/douyu/jupiter/pkg/xlog" + "net/url" +) + +// DataSourceEtcdv3 defines etcdv3 scheme +const DataSourceEtcdv3 = "etcdv3" + +func init() { + manager.Register(DataSourceEtcdv3, func() conf.DataSource { + var ( + configAddr = flag.String("config") + ) + if configAddr == "" { + xlog.Panic("new apollo dataSource, configAddr is empty") + return nil + } + // configAddr is a string in this format: + // etcdv3://ip:port?basicAuth=true&username=XXX&password=XXX&key=XXX&certFile=XXX&keyFile=XXX&caCert=XXX&secure=XXX + + urlObj, err := url.Parse(configAddr) + if err != nil { + xlog.Panic("parse configAddr error", xlog.FieldErr(err)) + return nil + } + etcdConf := etcdv3.DefaultConfig() + etcdConf.Endpoints = []string{urlObj.Host} + if urlObj.Query().Get("basicAuth") == "true" { + etcdConf.BasicAuth = true + } + if urlObj.Query().Get("secure") == "true" { + etcdConf.Secure = true + } + etcdConf.CertFile = urlObj.Query().Get("certFile") + etcdConf.KeyFile = urlObj.Query().Get("keyFile") + etcdConf.CaCert = urlObj.Query().Get("caCert") + etcdConf.UserName = urlObj.Query().Get("username") + etcdConf.Password = urlObj.Query().Get("password") + return NewDataSource(etcdConf.Build(), urlObj.Query().Get("key")) + }) +} diff --git a/pkg/datasource/file/register.go b/pkg/datasource/file/register.go new file mode 100644 index 0000000000..868d100fce --- /dev/null +++ b/pkg/datasource/file/register.go @@ -0,0 +1,26 @@ +package file + +import ( + "github.com/douyu/jupiter/pkg/conf" + "github.com/douyu/jupiter/pkg/datasource/manager" + "github.com/douyu/jupiter/pkg/flag" + "github.com/douyu/jupiter/pkg/xlog" +) + +// DataSourceFile defines file scheme +const DataSourceFile = "file" + +func init() { + manager.Register(DataSourceFile, func() conf.DataSource { + var ( + watchConfig = flag.Bool("watch") + configAddr = flag.String("config") + ) + if configAddr == "" { + xlog.Panic("new file dataSource, configAddr is empty") + return nil + } + return NewDataSource(configAddr, watchConfig) + }) + manager.DefaultScheme = DataSourceFile +} diff --git a/pkg/datasource/http/register.go b/pkg/datasource/http/register.go new file mode 100644 index 0000000000..c4041c8b1a --- /dev/null +++ b/pkg/datasource/http/register.go @@ -0,0 +1,30 @@ +package http + +import ( + "github.com/douyu/jupiter/pkg/conf" + "github.com/douyu/jupiter/pkg/datasource/manager" + "github.com/douyu/jupiter/pkg/flag" + "github.com/douyu/jupiter/pkg/xlog" +) + +// Defines http/https scheme +const ( + DataSourceHttp = "http" + DataSourceHttps = "https" +) + +func init() { + dataSourceCreator := func() conf.DataSource { + var ( + watchConfig = flag.Bool("watch") + configAddr = flag.String("config") + ) + if configAddr == "" { + xlog.Panic("new http dataSource, configAddr is empty") + return nil + } + return NewDataSource(configAddr, watchConfig) + } + manager.Register(DataSourceHttp, dataSourceCreator) + manager.Register(DataSourceHttps, dataSourceCreator) +} diff --git a/pkg/datasource/manager/manager.go b/pkg/datasource/manager/manager.go new file mode 100644 index 0000000000..ab7f35d69e --- /dev/null +++ b/pkg/datasource/manager/manager.go @@ -0,0 +1,56 @@ +package manager + +import ( + "errors" + "net/url" + + "github.com/douyu/jupiter/pkg/conf" +) + +var ( + //ErrConfigAddr not config + ErrConfigAddr = errors.New("no config... ") + // ErrInvalidDataSource defines an error that the scheme has been registered + ErrInvalidDataSource = errors.New("invalid data source, please make sure the scheme has been registered") + registry map[string]DataSourceCreatorFunc + //DefaultScheme .. + DefaultScheme string +) + +// DataSourceCreatorFunc represents a dataSource creator function +type DataSourceCreatorFunc func() conf.DataSource + +func init() { + registry = make(map[string]DataSourceCreatorFunc) +} + +// Register registers a dataSource creator function to the registry +func Register(scheme string, creator DataSourceCreatorFunc) { + registry[scheme] = creator +} + +// CreateDataSource creates a dataSource witch has been registered +// func CreateDataSource(scheme string) (conf.DataSource, error) { +// creatorFunc, exist := registry[scheme] +// if !exist { +// return nil, ErrInvalidDataSource +// } +// return creatorFunc(), nil +// } + +//NewDataSource .. +func NewDataSource(configAddr string) (conf.DataSource, error) { + if configAddr == "" { + return nil, ErrConfigAddr + } + urlObj, err := url.Parse(configAddr) + if err == nil && len(urlObj.Scheme) > 1 { + DefaultScheme = urlObj.Scheme + } + + creatorFunc, exist := registry[DefaultScheme] + if !exist { + return nil, ErrInvalidDataSource + } + return creatorFunc(), nil +} diff --git a/pkg/defers/defer.go b/pkg/defers/defer.go index c5105a2b2e..e1f14b5091 100644 --- a/pkg/defers/defer.go +++ b/pkg/defers/defer.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package defers import ( diff --git a/pkg/defers/defer_test.go b/pkg/defers/defer_test.go index c9bf8e40b9..bb622bee19 100644 --- a/pkg/defers/defer_test.go +++ b/pkg/defers/defer_test.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package defers import ( diff --git a/pkg/ecode/code.go b/pkg/ecode/code.go index 467f7b3b94..7cfc5d3fab 100644 --- a/pkg/ecode/code.go +++ b/pkg/ecode/code.go @@ -22,12 +22,15 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/douyu/jupiter/pkg/govern" + "github.com/douyu/jupiter/pkg/server/governor" "github.com/douyu/jupiter/pkg/xlog" "github.com/golang/protobuf/ptypes/any" spb "google.golang.org/genproto/googleapis/rpc/status" ) +// EcodeNum 低于10000均为系统错误码，业务错误码请使用10000以上 +const EcodeNum int32 = 9999 + var ( aid int maxCustomizeCode = 9999 @@ -38,7 +41,7 @@ var ( func init() { // status code list - govern.HandleFunc("/status/code/list", func(w http.ResponseWriter, r *http.Request) { + governor.HandleFunc("/status/code/list", func(w http.ResponseWriter, r *http.Request) { var rets = make(map[int]*spbStatus) _codes.Range(func(key, val interface{}) bool { code := key.(int) @@ -55,6 +58,7 @@ func Add(code int, message string) *spbStatus { if code > maxCustomizeCode { xlog.Panic("customize code must less than 9999", xlog.Any("code", code)) } + return add(aid*10000+code, message) } diff --git a/pkg/ecode/unified.go b/pkg/ecode/unified.go index 7b6c0a5982..92a077be15 100644 --- a/pkg/ecode/unified.go +++ b/pkg/ecode/unified.go @@ -119,4 +119,6 @@ const ( ModClientGrpc = "client.grpc" // ModClientMySQL ... ModClientMySQL = "client.mysql" + // ModXcronETCD ... + ModXcronETCD = "xcron.etcd" ) diff --git a/pkg/env.go b/pkg/env.go new file mode 100644 index 0000000000..2a6afca156 --- /dev/null +++ b/pkg/env.go @@ -0,0 +1,73 @@ +package pkg + +import ( + "crypto/md5" + "fmt" + "github.com/douyu/jupiter/pkg/constant" + "os" +) + +var ( + appLogDir string + appMode string + appRegion string + appZone string + appHost string + appInstance string +) + +func InitEnv() { + appLogDir = os.Getenv(constant.EnvAppLogDir) + appMode = os.Getenv(constant.EnvAppMode) + appRegion = os.Getenv(constant.EnvAppRegion) + appZone = os.Getenv(constant.EnvAppZone) + appHost = os.Getenv(constant.EnvAppHost) + appInstance = os.Getenv(constant.EnvAppInstance) + if appInstance == "" { + appInstance = fmt.Sprintf("%x", md5.Sum([]byte(fmt.Sprintf("%s:%s", HostName(), AppID())))) + } +} + +func AppLogDir() string { + return appLogDir +} + +func SetAppLogDir(appLogDir string) { + appLogDir = appLogDir +} + +func AppMode() string { + return appMode +} + +func SetAppMode(appMode string) { + appMode = appMode +} + +func AppRegion() string { + return appRegion +} + +func SetAppRegion(appRegion string) { + appRegion = appRegion +} + +func AppZone() string { + return appZone +} + +func SetAppZone(appZone string) { + appZone = appZone +} + +func AppHost() string { + return appHost +} + +func SetAppHost(appHost string) { + appHost = appHost +} + +func AppInstance() string { + return appInstance +} diff --git a/pkg/flag/flag.go b/pkg/flag/flag.go index d77ee4751a..90dff2de7f 100644 --- a/pkg/flag/flag.go +++ b/pkg/flag/flag.go @@ -18,9 +18,9 @@ import ( "flag" "fmt" "os" - "path/filepath" "strconv" "strings" + "testing" ) var ( @@ -28,14 +28,15 @@ var ( ) func init() { - procName := filepath.Base(os.Args[0]) - + // procName := filepath.Base(os.Args[0]) + // nfs := flag.NewFlagSet(procName, flag.ExitOnError) flagset = &FlagSet{ - FlagSet: flag.NewFlagSet(procName, flag.ExitOnError), + FlagSet: flag.CommandLine, flags: defaultFlags, actions: make(map[string]func(string, *FlagSet)), environs: make(map[string]string), } + testing.Init() } // Flag ... diff --git a/pkg/metric/counter.go b/pkg/metric/counter.go index 737a44bd5a..f5294960f6 100644 --- a/pkg/metric/counter.go +++ b/pkg/metric/counter.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package metric import ( @@ -28,6 +42,16 @@ func (opts CounterVecOpts) Build() *counterVec { } } +// NewCounterVec ... +func NewCounterVec(name string, labels []string) *counterVec { + return CounterVecOpts{ + Namespace: DefaultNamespace, + Name: name, + Help: name, + Labels: labels, + }.Build() +} + type counterVec struct { *prometheus.CounterVec } diff --git a/pkg/metric/gauge.go b/pkg/metric/gauge.go index a57d2dab5f..d467947f45 100644 --- a/pkg/metric/gauge.go +++ b/pkg/metric/gauge.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package metric import "github.com/prometheus/client_golang/prometheus" @@ -30,6 +44,16 @@ func (opts GaugeVecOpts) Build() *gaugeVec { } } +// NewGaugeVec ... +func NewGaugeVec(name string, labels []string) *gaugeVec { + return GaugeVecOpts{ + Namespace: DefaultNamespace, + Name: name, + Help: name, + Labels: labels, + }.Build() +} + // Inc ... func (gv *gaugeVec) Inc(labels ...string) { gv.WithLabelValues(labels...).Inc() diff --git a/pkg/metric/histogram.go b/pkg/metric/histogram.go index d5e1d03fec..224edd241e 100644 --- a/pkg/metric/histogram.go +++ b/pkg/metric/histogram.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package metric import "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/metric/metric.go b/pkg/metric/metric.go index 7f5d07f4a2..6585baf5e3 100644 --- a/pkg/metric/metric.go +++ b/pkg/metric/metric.go @@ -15,10 +15,11 @@ package metric import ( - "net/http" - - "github.com/douyu/jupiter/pkg/govern" + "github.com/douyu/jupiter/pkg" + "github.com/douyu/jupiter/pkg/server/governor" "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" + "time" ) var ( @@ -30,8 +31,14 @@ var ( TypeGRPCStream = "stream" // TypeRedis ... TypeRedis = "redis" + TypeGorm = "gorm" // TypeRocketMQ ... TypeRocketMQ = "rocketmq" + // TypeWebsocket ... + TypeWebsocket = "ws" + + // TypeMySQL ... + TypeMySQL = "mysql" // CodeJob CodeJobSuccess = "ok" @@ -92,16 +99,61 @@ var ( Labels: []string{"type", "name"}, }.Build() + LibHandleHistogram = HistogramVecOpts{ + Namespace: DefaultNamespace, + Name: "lib_handle_seconds", + Labels: []string{"type", "method", "address"}, + }.Build() + // LibHandleCounter ... + LibHandleCounter = CounterVecOpts{ + Namespace: DefaultNamespace, + Name: "lib_handle_total", + Labels: []string{"type", "method", "address", "code"}, + }.Build() + + LibHandleSummary = SummaryVecOpts{ + Namespace: DefaultNamespace, + Name: "lib_handle_stats", + Labels: []string{"name", "status"}, + }.Build() + + // CacheHandleCounter ... + CacheHandleCounter = CounterVecOpts{ + Namespace: DefaultNamespace, + Name: "cache_handle_total", + Labels: []string{"type", "name", "action", "code"}, + }.Build() + + // CacheHandleHistogram ... + CacheHandleHistogram = HistogramVecOpts{ + Namespace: DefaultNamespace, + Name: "cache_handle_seconds", + Labels: []string{"type", "name", "action"}, + }.Build() + // BuildInfoGauge ... BuildInfoGauge = GaugeVecOpts{ Namespace: DefaultNamespace, Name: "build_info", - Labels: []string{"name", "id", "env", "zone", "region", "version"}, + Labels: []string{"name", "aid", "mode", "region", "zone", "app_version", "jupiter_version", "start_time", "build_time", "go_version"}, }.Build() ) func init() { - govern.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) { + BuildInfoGauge.WithLabelValues( + pkg.Name(), + pkg.AppID(), + pkg.AppMode(), + pkg.AppRegion(), + pkg.AppZone(), + pkg.AppVersion(), + pkg.JupiterVersion(), + pkg.StartTime(), + pkg.BuildTime(), + pkg.GoVersion(), + ).Set(float64(time.Now().UnixNano() / 1e6)) + + governor.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) { promhttp.Handler().ServeHTTP(w, r) }) } diff --git a/pkg/metric/summary.go b/pkg/metric/summary.go new file mode 100644 index 0000000000..0591b9532d --- /dev/null +++ b/pkg/metric/summary.go @@ -0,0 +1,50 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metric + +import "github.com/prometheus/client_golang/prometheus" + +// SummaryVecOpts ... +type SummaryVecOpts struct { + Namespace string + Subsystem string + Name string + Help string + Labels []string +} + +type summaryVec struct { + *prometheus.SummaryVec +} + +// Build ... +func (opts SummaryVecOpts) Build() *summaryVec { + vec := prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: opts.Namespace, + Subsystem: opts.Subsystem, + Name: opts.Name, + Help: opts.Help, + }, opts.Labels) + prometheus.MustRegister(vec) + return &summaryVec{ + SummaryVec: vec, + } +} + +// Observe ... +func (summary *summaryVec) Observe(v float64, labels ...string) { + summary.WithLabelValues(labels...).Observe(v) +} diff --git a/pkg/pkg.go b/pkg/pkg.go index baff9f4b46..35c1d43b63 100644 --- a/pkg/pkg.go +++ b/pkg/pkg.go @@ -1,27 +1,58 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pkg import ( "fmt" "os" "path/filepath" + "runtime" + "strings" + "time" + + "github.com/douyu/jupiter/pkg/util/xtime" + "github.com/douyu/jupiter/pkg/constant" "github.com/douyu/jupiter/pkg/util/xcolor" ) +const jupiterVersion = "0.2.0" + +var ( + startTime string + goVersion string +) + +// build info +/* + + */ var ( - appName string - hostName string - buildVersion string - buildGitRevision string - buildUser string - buildHost string - buildStatus string - buildTime string + appName string + appID string + hostName string + buildAppVersion string + buildUser string + buildHost string + buildStatus string + buildTime string ) func init() { if appName == "" { - appName = os.Getenv("APP_NAME") + appName = os.Getenv(constant.EnvAppName) if appName == "" { appName = filepath.Base(os.Args[0]) } @@ -32,6 +63,9 @@ func init() { name = "unknown" } hostName = name + startTime = xtime.TS.Format(time.Now()) + goVersion = runtime.Version() + InitEnv() } // Name gets application name. @@ -39,18 +73,86 @@ func Name() string { return appName } -// HostName ... +//SetName set app anme +func SetName(s string) { + appName = s +} + +//AppID get appID +func AppID() string { + return appID +} + +//SetAppID set appID +func SetAppID(s string) { + appID = s +} + +//AppVersion get buildAppVersion +func AppVersion() string { + return buildAppVersion +} + +//appVersion not defined +// func SetAppVersion(s string) { +// appVersion = s +// } + +//JupiterVersion get jupiterVersion +func JupiterVersion() string { + return jupiterVersion +} + +// todo: jupiterVersion is const not be set +// func SetJupiterVersion(s string) { +// jupiterVersion = s +// } + +//BuildTime get buildTime +func BuildTime() string { + return buildTime +} + +//BuildUser get buildUser +func BuildUser() string { + return buildUser +} + +//BuildHost get buildHost +func BuildHost() string { + return buildHost +} + +//SetBuildTime set buildTime +func SetBuildTime(buildTime string) { + buildTime = strings.Replace(buildTime, "--", " ", 1) +} + +// HostName get host name func HostName() string { return hostName } -// PrintVersion ... +//StartTime get start time +func StartTime() string { + return startTime +} + +//GoVersion get go version +func GoVersion() string { + return goVersion +} + +// PrintVersion print formated version info func PrintVersion() { fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("name"), xcolor.Blue(appName)) - fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("version"), xcolor.Blue(buildVersion)) - fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("revision"), xcolor.Blue(buildGitRevision)) - fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("user"), xcolor.Blue(buildUser)) - fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("host"), xcolor.Blue(buildHost)) - fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("buildTime"), xcolor.Blue(buildTime)) + fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("appID"), xcolor.Blue(appID)) + fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("region"), xcolor.Blue(AppRegion())) + fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("zone"), xcolor.Blue(AppZone())) + fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("appVersion"), xcolor.Blue(buildAppVersion)) + fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("jupiterVersion"), xcolor.Blue(jupiterVersion)) + fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("buildUser"), xcolor.Blue(buildUser)) + fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("buildHost"), xcolor.Blue(buildHost)) + fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("buildTime"), xcolor.Blue(BuildTime())) fmt.Printf("%-8s]> %-30s => %s\n", "jupiter", xcolor.Red("buildStatus"), xcolor.Blue(buildStatus)) } diff --git a/pkg/registry/compound/client.go b/pkg/registry/compound/client.go index 4d3754ac76..64c9f91a90 100644 --- a/pkg/registry/compound/client.go +++ b/pkg/registry/compound/client.go @@ -26,6 +26,30 @@ type compoundRegistry struct { registries []registry2.Registry } +// ListServices ... +func (c compoundRegistry) ListServices(ctx context.Context, name string, scheme string) ([]*server.ServiceInfo, error) { + var eg errgroup.Group + var services = make([]*server.ServiceInfo, 0) + for _, registry := range c.registries { + registry := registry + eg.Go(func() error { + infos, err := registry.ListServices(ctx, name, scheme) + if err != nil { + return err + } + services = append(services, infos...) + return nil + }) + } + err := eg.Wait() + return services, err +} + +// WatchServices ... +func (c compoundRegistry) WatchServices(ctx context.Context, s string, s2 string) (chan registry2.Endpoints, error) { + panic("compound registry doesn't support watch services") +} + // RegisterService ... func (c compoundRegistry) RegisterService(ctx context.Context, bean *server.ServiceInfo) error { var eg errgroup.Group @@ -38,13 +62,13 @@ func (c compoundRegistry) RegisterService(ctx context.Context, bean *server.Serv return eg.Wait() } -// DeregisterService ... -func (c compoundRegistry) DeregisterService(ctx context.Context, bean *server.ServiceInfo) error { +// UnregisterService ... +func (c compoundRegistry) UnregisterService(ctx context.Context, bean *server.ServiceInfo) error { var eg errgroup.Group for _, registry := range c.registries { registry := registry eg.Go(func() error { - return registry.DeregisterService(ctx, bean) + return registry.UnregisterService(ctx, bean) }) } return eg.Wait() diff --git a/pkg/registry/endpoint.go b/pkg/registry/endpoint.go new file mode 100644 index 0000000000..0590b8bf9d --- /dev/null +++ b/pkg/registry/endpoint.go @@ -0,0 +1,81 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "encoding/json" + + "github.com/douyu/jupiter/pkg/server" +) + +// Endpoints ... +type Endpoints struct { + // 服务节点列表 + Nodes map[string]server.ServiceInfo + + // 路由配置 + RouteConfigs map[string]RouteConfig + + // 消费者元数据 + ConsumerConfigs map[string]ConsumerConfig + + // 服务元信息 + ProviderConfigs map[string]ProviderConfig +} + +// ProviderConfig config of provider +// 通过这个配置，修改provider的属性 +type ProviderConfig struct { + ID string `json:"id"` + Scheme string `json:"scheme"` + Host string `json:"host"` + + Region string `json:"region"` + Zone string `json:"zone"` + Deployment string `json:"deployment"` + Metadata map[string]string `json:"metadata"` + Enable bool `json:"enable"` +} + +// ConsumerConfig config of consumer +// 客户端调用app的配置 +type ConsumerConfig struct { + ID string `json:"id"` + Scheme string `json:"scheme"` + Host string `json:"host"` +} + +// RouteConfig ... +type RouteConfig struct { + ID string `json:"id" toml:"id"` + Scheme string `json:"scheme" toml:"scheme"` + Host string `json:"host" toml:"host"` + + Deployment string `json:"deployment"` + URI string `json:"uri"` + Upstream Upstream `json:"upstream"` +} + +// String ... +func (config RouteConfig) String() string { + bs, _ := json.Marshal(config) + return string(bs) +} + +// Upstream represents upstream balancing config +type Upstream struct { + Nodes map[string]int `json:"nodes"` + Groups map[string]int `json:"groups"` +} diff --git a/pkg/registry/etcdv3/option.go b/pkg/registry/etcdv3/config.go similarity index 86% rename from pkg/registry/etcdv3/option.go rename to pkg/registry/etcdv3/config.go index e02477a592..eb01791256 100644 --- a/pkg/registry/etcdv3/option.go +++ b/pkg/registry/etcdv3/config.go @@ -50,7 +50,8 @@ func DefaultConfig() *Config { Config: etcdv3.DefaultConfig(), ReadTimeout: time.Second * 3, Prefix: "jupiter", - logger: xlog.DefaultLogger, + logger: xlog.JupiterLogger, + ServiceTTL: 0, } } @@ -60,21 +61,14 @@ type Config struct { ReadTimeout time.Duration ConfigKey string Prefix string + ServiceTTL time.Duration logger *xlog.Logger } -// BuildRegistry ... -func (config Config) BuildRegistry() registry.Registry { +// Build ... +func (config Config) Build() registry.Registry { if config.ConfigKey != "" { config.Config = etcdv3.RawConfig(config.ConfigKey) } return newETCDRegistry(&config) } - -// BuildResolver ... -func (config Config) BuildResolver() *etcdResolver { - if config.ConfigKey != "" { - config.Config = etcdv3.RawConfig(config.ConfigKey) - } - return newETCDResolver(&config) -} diff --git a/pkg/registry/etcdv3/registry.go b/pkg/registry/etcdv3/registry.go index 316c92d779..bfb7eb3021 100644 --- a/pkg/registry/etcdv3/registry.go +++ b/pkg/registry/etcdv3/registry.go @@ -18,113 +18,456 @@ import ( "context" "encoding/json" "fmt" + "net" + "net/url" + "strings" "sync" "time" - "github.com/douyu/jupiter/pkg/ecode" + "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes" + "github.com/douyu/jupiter/pkg" + "github.com/douyu/jupiter/pkg/constant" "github.com/coreos/etcd/clientv3" + "github.com/coreos/etcd/mvcc/mvccpb" "github.com/douyu/jupiter/pkg/client/etcdv3" + "github.com/douyu/jupiter/pkg/ecode" + "github.com/douyu/jupiter/pkg/registry" "github.com/douyu/jupiter/pkg/server" + "github.com/douyu/jupiter/pkg/util/xgo" + "github.com/douyu/jupiter/pkg/util/xstruct" "github.com/douyu/jupiter/pkg/xlog" ) type etcdv3Registry struct { client *etcdv3.Client - lease clientv3.LeaseID kvs sync.Map *Config + cancel context.CancelFunc + leases map[string]clientv3.LeaseID + rmu *sync.RWMutex } func newETCDRegistry(config *Config) *etcdv3Registry { if config.logger == nil { - config.logger = xlog.DefaultLogger + config.logger = xlog.JupiterLogger } config.logger = config.logger.With(xlog.FieldMod(ecode.ModRegistryETCD), xlog.FieldAddrAny(config.Config.Endpoints)) - res := &etcdv3Registry{ + reg := &etcdv3Registry{ client: config.Config.Build(), Config: config, kvs: sync.Map{}, + leases: make(map[string]clientv3.LeaseID), + rmu: &sync.RWMutex{}, } - return res + return reg } -// RegisterService ... -func (e *etcdv3Registry) RegisterService(ctx context.Context, info *server.ServiceInfo) error { - opOptions := make([]clientv3.OpOption, 0) - if e.lease != 0 { - opOptions = append(opOptions, clientv3.WithLease(e.lease), clientv3.WithSerializable()) +// RegisterService register service to registry +func (reg *etcdv3Registry) RegisterService(ctx context.Context, info *server.ServiceInfo) error { + err := reg.registerBiz(ctx, info) + if err != nil { + return err } + return reg.registerMetric(ctx, info) +} - if _, ok := ctx.Deadline(); !ok { - var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, e.ReadTimeout) - defer cancel() +// UnregisterService unregister service from registry +func (reg *etcdv3Registry) UnregisterService(ctx context.Context, info *server.ServiceInfo) error { + return reg.unregister(ctx, reg.registerKey(info)) +} + +// ListServices list service registered in registry with name `name` +func (reg *etcdv3Registry) ListServices(ctx context.Context, name string, scheme string) (services []*server.ServiceInfo, err error) { + target := fmt.Sprintf("/%s/%s/providers/%s://", reg.Prefix, name, scheme) + getResp, getErr := reg.client.Get(ctx, target, clientv3.WithPrefix()) + if getErr != nil { + reg.logger.Error(ecode.MsgWatchRequestErr, xlog.FieldErrKind(ecode.ErrKindRequestErr), xlog.FieldErr(getErr), xlog.FieldAddr(target)) + return nil, getErr } - key := fmt.Sprintf("/%s/%s/%s/%s://%s", e.Prefix, info.Name, info.Kind.String(), info.Scheme, info.Address) - val, err := json.Marshal(info) + for _, kv := range getResp.Kvs { + var service server.ServiceInfo + if err := json.Unmarshal(kv.Value, &service); err != nil { + reg.logger.Warnf("invalid service", xlog.FieldErr(err)) + continue + } + services = append(services, &service) + } + + return +} +// WatchServices watch service change event, then return address list +func (reg *etcdv3Registry) WatchServices(ctx context.Context, name string, scheme string) (chan registry.Endpoints, error) { + prefix := fmt.Sprintf("/%s/%s/", reg.Prefix, name) + watch, err := reg.client.WatchPrefix(context.Background(), prefix) if err != nil { - return err + return nil, err } - _, err = e.client.Put(ctx, key, string(val), opOptions...) - if err != nil { - e.logger.Error("register service", xlog.FieldErrKind(ecode.ErrKindRegisterErr), xlog.FieldErr(err), xlog.FieldKeyAny(key), xlog.FieldValueAny(info)) - return err + var addresses = make(chan registry.Endpoints, 10) + var al = ®istry.Endpoints{ + Nodes: make(map[string]server.ServiceInfo), + RouteConfigs: make(map[string]registry.RouteConfig), + } + + for _, kv := range watch.IncipientKeyValues() { + updateAddrList(al, prefix, scheme, kv) } - // xdebug.PrintKVWithPrefix("registry", "register key", key) - e.logger.Info("register service", xlog.FieldKeyAny(key), xlog.FieldValueAny(info)) - e.kvs.Store(key, val) - return err -} -// DeregisterService ... -func (e *etcdv3Registry) DeregisterService(ctx context.Context, info *server.ServiceInfo) error { - key := fmt.Sprintf("/%s/%s/%s/%s://%s", e.Prefix, info.Name, info.Kind.String(), info.Scheme, info.Address) - return e.deregister(ctx, key) + addresses <- *al + + xgo.Go(func() { + for event := range watch.C() { + var al2 *registry.Endpoints + xstruct.CopyStruct(al, al2) + switch event.Type { + case mvccpb.PUT: + updateAddrList(al2, prefix, scheme, event.Kv) + case mvccpb.DELETE: + deleteAddrList(al2, prefix, scheme, event.Kv) + } + + select { + case addresses <- *al2: + default: + xlog.Warnf("invalid") + } + } + }) + + return addresses, nil } -func (e *etcdv3Registry) deregister(ctx context.Context, key string) error { +func (reg *etcdv3Registry) unregister(ctx context.Context, key string) error { if _, ok := ctx.Deadline(); !ok { var cancel context.CancelFunc - ctx, cancel = context.WithTimeout(ctx, e.ReadTimeout) + ctx, cancel = context.WithTimeout(ctx, reg.ReadTimeout) defer cancel() } - _, err := e.client.Delete(ctx, key) + + if err := reg.delLeaseID(ctx, key); err != nil { + return err + } + + _, err := reg.client.Delete(ctx, key) if err == nil { - e.kvs.Delete(key) + reg.kvs.Delete(key) } return err } // Close ... -func (e *etcdv3Registry) Close() error { +func (reg *etcdv3Registry) Close() error { + if reg.cancel != nil { + reg.cancel() + } var wg sync.WaitGroup - e.kvs.Range(func(k, v interface{}) bool { + reg.kvs.Range(func(k, v interface{}) bool { wg.Add(1) go func(k interface{}) { defer wg.Done() ctx, cancel := context.WithTimeout(context.Background(), time.Second) - err := e.deregister(ctx, k.(string)) + err := reg.unregister(ctx, k.(string)) if err != nil { - e.logger.Error("deregister service", xlog.FieldErrKind(ecode.ErrKindRequestErr), xlog.FieldErr(err), xlog.FieldErr(err), xlog.FieldKeyAny(k), xlog.FieldValueAny(v)) + reg.logger.Error("unregister service", xlog.FieldErrKind(ecode.ErrKindRequestErr), xlog.FieldErr(err), xlog.FieldErr(err), xlog.FieldKeyAny(k), xlog.FieldValueAny(v)) } else { - e.logger.Info("deregister service", xlog.FieldKeyAny(k), xlog.FieldValueAny(v)) + reg.logger.Info("unregister service", xlog.FieldKeyAny(k), xlog.FieldValueAny(v)) } cancel() }(k) return true }) wg.Wait() + return nil +} + +func (reg *etcdv3Registry) registerMetric(ctx context.Context, info *server.ServiceInfo) error { + if info.Kind != constant.ServiceGovernor { + return nil + } + + metric := "/prometheus/job/%s/%s" + + if _, ok := ctx.Deadline(); !ok { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, reg.ReadTimeout) + defer cancel() + } - if e.lease > 0 { - // revoke 有一些延迟，考虑直接删除 - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - _, err := e.client.Revoke(ctx, e.lease) - cancel() + key := fmt.Sprintf(metric, info.Name, pkg.HostName()) + val := info.Address + + opOptions := make([]clientv3.OpOption, 0) + // opOptions = append(opOptions, clientv3.WithSerializable()) + if reg.Config.ServiceTTL > 0 { + leaseID, err := reg.getLeaseID(ctx, key) + if err != nil { + return err + } + opOptions = append(opOptions, clientv3.WithLease(leaseID)) + //KeepAlive ctx without timeout for same as service life + reg.keepLeaseID(ctx, leaseID) + } + _, err := reg.client.Put(ctx, key, val, opOptions...) + if err != nil { + reg.logger.Error("register service", xlog.FieldErrKind(ecode.ErrKindRegisterErr), xlog.FieldErr(err), xlog.FieldKeyAny(key), xlog.FieldValueAny(info)) return err } + + reg.logger.Info("register service", xlog.FieldKeyAny(key), xlog.FieldValueAny(val)) + reg.kvs.Store(key, val) return nil + +} +func (reg *etcdv3Registry) getLeaseID(ctx context.Context, k string) (clientv3.LeaseID, error) { + reg.rmu.RLock() + leaseID, ok := reg.leases[k] + reg.rmu.RUnlock() + if ok { + //from map try keep alive once + if _, err := reg.client.KeepAliveOnce(ctx, leaseID); err != nil { + if err == rpctypes.ErrLeaseNotFound { + goto grant + } + return leaseID, err + } + return leaseID, nil + } +grant: + //grant + rsp, err := reg.client.Grant(ctx, int64(reg.Config.ServiceTTL.Seconds())) + if err != nil { + return leaseID, err + } + //cache to map + reg.rmu.Lock() + reg.leases[k] = rsp.ID + reg.rmu.Unlock() + return rsp.ID, nil +} +func (reg *etcdv3Registry) keepLeaseID(ctx context.Context, leaseID clientv3.LeaseID) { + go func() { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + ch, err := reg.client.KeepAlive(ctx, leaseID) + if err != nil { + return + } + for { + select { + case lkp := <-ch: + if lkp == nil { + return + } + } + } + }() +} +func (reg *etcdv3Registry) delLeaseID(ctx context.Context, k string) error { + if reg.Config.ServiceTTL > 0 { + reg.rmu.Lock() + id, ok := reg.leases[k] + delete(reg.leases, k) + reg.rmu.Unlock() + if ok { + if _, err := reg.client.Revoke(ctx, id); err != nil { + return err + } + } + } + return nil +} +func (reg *etcdv3Registry) registerBiz(ctx context.Context, info *server.ServiceInfo) error { + var readCtx context.Context + var readCancel context.CancelFunc + if _, ok := ctx.Deadline(); !ok { + readCtx, readCancel = context.WithTimeout(ctx, reg.ReadTimeout) + defer readCancel() + } + + key := reg.registerKey(info) + val := reg.registerValue(info) + + opOptions := make([]clientv3.OpOption, 0) + // opOptions = append(opOptions, clientv3.WithSerializable()) + if reg.Config.ServiceTTL > 0 { + leaseID, err := reg.getLeaseID(readCtx, key) + if err != nil { + return err + } + opOptions = append(opOptions, clientv3.WithLease(leaseID)) + //KeepAlive ctx without timeout for same as service life + reg.keepLeaseID(ctx, leaseID) + } + _, err := reg.client.Put(readCtx, key, val, opOptions...) + if err != nil { + reg.logger.Error("register service", xlog.FieldErrKind(ecode.ErrKindRegisterErr), xlog.FieldErr(err), xlog.FieldKeyAny(key), xlog.FieldValueAny(info)) + return err + } + reg.logger.Info("register service", xlog.FieldKeyAny(key), xlog.FieldValueAny(val)) + reg.kvs.Store(key, val) + return nil + +} + +func (reg *etcdv3Registry) registerKey(info *server.ServiceInfo) string { + return registry.GetServiceKey(reg.Prefix, info) +} + +func (reg *etcdv3Registry) registerValue(info *server.ServiceInfo) string { + return registry.GetServiceValue(info) +} + +func deleteAddrList(al *registry.Endpoints, prefix, scheme string, kvs ...*mvccpb.KeyValue) { + for _, kv := range kvs { + var addr = strings.TrimPrefix(string(kv.Key), prefix) + if strings.HasPrefix(addr, "providers/"+scheme) { + // 解析服务注册键 + addr = strings.TrimPrefix(addr, "providers/") + if addr == "" { + continue + } + uri, err := url.Parse(addr) + if err != nil { + xlog.Error("parse uri", xlog.FieldErrKind(ecode.ErrKindUriErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key))) + continue + } + delete(al.Nodes, uri.String()) + } + + if strings.HasPrefix(addr, "configurators/"+scheme) { + // 解析服务配置键 + addr = strings.TrimPrefix(addr, "configurators/") + if addr == "" { + continue + } + uri, err := url.Parse(addr) + if err != nil { + xlog.Error("parse uri", xlog.FieldErrKind(ecode.ErrKindUriErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key))) + continue + } + delete(al.RouteConfigs, uri.String()) + } + + if isIPPort(addr) { + // 直接删除addr 因为Delete操作的value值为空 + delete(al.Nodes, addr) + delete(al.RouteConfigs, addr) + } + } +} + +func updateAddrList(al *registry.Endpoints, prefix, scheme string, kvs ...*mvccpb.KeyValue) { + for _, kv := range kvs { + var addr = strings.TrimPrefix(string(kv.Key), prefix) + switch { + // 解析服务注册键 + case strings.HasPrefix(addr, "providers/"+scheme): + addr = strings.TrimPrefix(addr, "providers/") + uri, err := url.Parse(addr) + if err != nil { + xlog.Error("parse uri", xlog.FieldErrKind(ecode.ErrKindUriErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key))) + continue + } + var serviceInfo server.ServiceInfo + if err := json.Unmarshal(kv.Value, &serviceInfo); err != nil { + xlog.Error("parse uri", xlog.FieldErrKind(ecode.ErrKindUriErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key))) + continue + } + al.Nodes[uri.String()] = serviceInfo + case strings.HasPrefix(addr, "configurators/"+scheme): + addr = strings.TrimPrefix(addr, "configurators/") + + uri, err := url.Parse(addr) + if err != nil { + xlog.Error("parse uri", xlog.FieldErrKind(ecode.ErrKindUriErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key))) + continue + } + + if strings.HasPrefix(uri.Path, "/routes/") { // 路由配置 + var routeConfig registry.RouteConfig + if err := json.Unmarshal(kv.Value, &routeConfig); err != nil { + xlog.Error("parse uri", xlog.FieldErrKind(ecode.ErrKindUriErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key))) + continue + } + routeConfig.ID = strings.TrimPrefix(uri.Path, "/routes/") + routeConfig.Scheme = uri.Scheme + routeConfig.Host = uri.Host + al.RouteConfigs[uri.String()] = routeConfig + } + + if strings.HasPrefix(uri.Path, "/providers/") { + var providerConfig registry.ProviderConfig + if err := json.Unmarshal(kv.Value, &providerConfig); err != nil { + xlog.Error("parse uri", xlog.FieldErrKind(ecode.ErrKindUriErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key))) + continue + } + providerConfig.ID = strings.TrimPrefix(uri.Path, "/providers/") + providerConfig.Scheme = uri.Scheme + providerConfig.Host = uri.Host + al.ProviderConfigs[uri.String()] = providerConfig + } + + if strings.HasPrefix(uri.Path, "/consumers/") { + var consumerConfig registry.ConsumerConfig + if err := json.Unmarshal(kv.Value, &consumerConfig); err != nil { + xlog.Error("parse uri", xlog.FieldErrKind(ecode.ErrKindUriErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key))) + continue + } + consumerConfig.ID = strings.TrimPrefix(uri.Path, "/consumers/") + consumerConfig.Scheme = uri.Scheme + consumerConfig.Host = uri.Host + al.ConsumerConfigs[uri.String()] = consumerConfig + } + } + } +} + +func isIPPort(addr string) bool { + _, _, err := net.SplitHostPort(addr) + return err == nil +} + +/* +key: /jupiter/main/configurator/grpc:///routes/1 +val: +{ + "upstream": { // 客户端配置 + "nodes": { // 按照node负载均衡 + "127.0.0.1:1980": 1, + "127.0.0.1:1981": 4 + }, + "group": { // 按照group负载均衡 + "red": 2, + "green": 1 + } + }, + "uri": "/hello", + "deployment": "open_api" +} + +key: /jupiter/main/configurator/grpc://127.0.0.1/routes/2 +val: +{ + "upstream": { // 客户端配置 + "nodes": { // 按照node负载均衡 + "127.0.0.1:1980": 1, + "127.0.0.1:1981": 1 + }, + "group": { // 按照group负载均衡 + "red": 1, + "green": 2 + } + }, + "uri": "/hello", + "deployment": "core_api" // 部署组 +} + +key: /jupiter/main/configurator/grpc:///consumers/client-demo +val: +{ + } +*/ diff --git a/pkg/registry/etcdv3/registry_test.go b/pkg/registry/etcdv3/registry_test.go new file mode 100644 index 0000000000..aeefc569b7 --- /dev/null +++ b/pkg/registry/etcdv3/registry_test.go @@ -0,0 +1,133 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package etcdv3 + +import ( + "context" + "fmt" + "github.com/douyu/jupiter/pkg/constant" + "testing" + "time" + + "github.com/douyu/jupiter/pkg/client/etcdv3" + "github.com/douyu/jupiter/pkg/registry" + "github.com/douyu/jupiter/pkg/server" + "github.com/douyu/jupiter/pkg/xlog" + "github.com/stretchr/testify/assert" +) + +func Test_etcdv3Registry(t *testing.T) { + etcdConfig := etcdv3.DefaultConfig() + etcdConfig.Endpoints = []string{"127.0.0.1:2379"} + registry := newETCDRegistry(&Config{ + Config: etcdConfig, + ReadTimeout: time.Second * 10, + Prefix: "jupiter", + logger: xlog.DefaultLogger, + }) + + assert.Nil(t, registry.RegisterService(context.Background(), &server.ServiceInfo{ + Name: "service_1", + AppID: "", + Scheme: "grpc", + Address: "10.10.10.1:9091", + Weight: 0, + Enable: true, + Healthy: true, + Metadata: map[string]string{}, + Region: "default", + Zone: "default", + Kind: constant.ServiceProvider, + Deployment: "default", + Group: "", + })) + + services, err := registry.ListServices(context.Background(), "service_1", "grpc") + assert.Nil(t, err) + assert.Equal(t, 1, len(services)) + assert.Equal(t, "10.10.10.1:9091", services[0].Address) + + go func() { + si := &server.ServiceInfo{ + Name: "service_1", + Scheme: "grpc", + Address: "10.10.10.1:9092", + Enable: true, + Healthy: true, + Metadata: map[string]string{}, + Region: "default", + Zone: "default", + Deployment: "default", + } + time.Sleep(time.Second) + assert.Nil(t, registry.RegisterService(context.Background(), si)) + assert.Nil(t, registry.UnregisterService(context.Background(), si)) + }() + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + endpoints, err := registry.WatchServices(ctx, "service_1", "grpc") + assert.Nil(t, err) + for msg := range endpoints { + t.Logf("watch service: %+v\n", msg) + // assert.Equal(t, "10.10.10.2:9092", msg) + } + }() + + time.Sleep(time.Second * 3) + cancel() + _ = registry.Close() + time.Sleep(time.Second * 1) +} + +func Test_etcdv3registry_UpdateAddressList(t *testing.T) { + etcdConfig := etcdv3.DefaultConfig() + etcdConfig.Endpoints = []string{"127.0.0.1:2379"} + reg := newETCDRegistry(&Config{ + Config: etcdConfig, + ReadTimeout: time.Second * 10, + Prefix: "jupiter", + logger: xlog.DefaultLogger, + }) + + var routeConfig = registry.RouteConfig{ + ID: "1", + Scheme: "grpc", + Host: "", + Deployment: "openapi", + URI: "/hello", + Upstream: registry.Upstream{ + Nodes: map[string]int{ + "10.10.10.1:9091": 1, + "10.10.10.1:9092": 10, + }, + }, + } + _, err := reg.client.Put(context.Background(), "/jupiter/service_1/configurators/grpc:///routes/1", routeConfig.String()) + assert.Nil(t, err) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + services, err := reg.WatchServices(ctx, "service_1", "grpc") + assert.Nil(t, err) + fmt.Printf("len(services) = %+v\n", len(services)) + for service := range services { + fmt.Printf("service = %+v\n", service) + } + }() + time.Sleep(time.Second * 3) + cancel() + _ = reg.Close() + time.Sleep(time.Second * 1) +} diff --git a/pkg/registry/etcdv3/resolver.go b/pkg/registry/etcdv3/resolver.go deleted file mode 100644 index b5d27a6ddf..0000000000 --- a/pkg/registry/etcdv3/resolver.go +++ /dev/null @@ -1,310 +0,0 @@ -// Copyright 2020 Douyu -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package etcdv3 - -import ( - "context" - "fmt" - "net" - "net/url" - "strings" - "sync" - "time" - - "github.com/douyu/jupiter/pkg/ecode" - - "github.com/coreos/etcd/clientv3" - "github.com/coreos/etcd/mvcc/mvccpb" - "github.com/douyu/jupiter/pkg/client/etcdv3" - "github.com/douyu/jupiter/pkg/server" - "github.com/douyu/jupiter/pkg/xlog" - jsoniter "github.com/json-iterator/go" - "google.golang.org/grpc/attributes" - "google.golang.org/grpc/naming" - "google.golang.org/grpc/resolver" -) - -// etcdResolver implement grpc resolve.Builder -type etcdResolver struct { - client *etcdv3.Client - *Config -} - -func newETCDResolver(config *Config) *etcdResolver { - if config.logger == nil { - config.logger = xlog.DefaultLogger - } - config.logger = config.logger.With(xlog.FieldMod("resolver.etcd"), xlog.FieldAddrAny(config.Config.Endpoints)) - res := &etcdResolver{ - client: config.Config.Build(), - Config: config, - } - return res -} - -// Build ... -func (r *etcdResolver) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOption) (resolver.Resolver, error) { - r.logger.Info("build etcd resolver", xlog.String("watchFullKey", "/"+r.Prefix+"/"+target.Endpoint)) - go r.watch(cc, r.Prefix, target.Endpoint) - return r, nil -} - -// Scheme ... -func (r etcdResolver) Scheme() string { - return "etcd" -} - -// ResolveNow ... -func (r etcdResolver) ResolveNow(rn resolver.ResolveNowOption) { - r.logger.Info("resolve now") -} - -// close closes the resolver. -func (r etcdResolver) Close() { - r.client.Close() - r.logger.Info("close") -} - -func (r *etcdResolver) deleteAddrList(al *AddressList, prefix string, kvs ...*mvccpb.KeyValue) { - al.mtx.Lock() - defer al.mtx.Unlock() - for _, kv := range kvs { - var addr = strings.TrimPrefix(string(kv.Key), prefix) - if strings.HasPrefix(addr, "providers/grpc://") { - // 解析服务注册键 - addr = strings.TrimPrefix(addr, "providers/") - if addr == "" { - continue - } - host, _, err := parseURI(addr) - if err != nil { - r.logger.Error("parse uri", xlog.FieldErrKind(ecode.ErrKindUriErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key))) - } - delete(al.regInfos, host) - } - - if strings.HasPrefix(addr, "configurators/grpc://") { - // 解析服务配置键 - addr = strings.TrimPrefix(addr, "configurators/") - if addr == "" { - continue - } - host, _, err := parseURI(addr) - if err != nil { - r.logger.Error("parse uri", xlog.FieldErrKind(ecode.ErrKindUriErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key))) - } - delete(al.cfgInfos, host) - } - - if isIPPort(addr) { - // 直接删除addr 因为Delete操作的value值为空 - delete(al.cfgInfos, addr) - delete(al.regInfos, addr) - } - } -} - -func isIPPort(addr string) bool { - _, _, err := net.SplitHostPort(addr) - return err == nil -} - -func (r *etcdResolver) updateAddrList(al *AddressList, prefix string, kvs ...*mvccpb.KeyValue) { - al.mtx.Lock() - defer al.mtx.Unlock() - for _, kv := range kvs { - var addr = strings.TrimPrefix(string(kv.Key), prefix) - switch { - // 解析服务注册键 - case strings.HasPrefix(addr, "providers/grpc://"): - addr = strings.TrimPrefix(addr, "providers/") - host, meta, err := parseURI(addr) - if err != nil { - r.logger.Error("parse uri", xlog.FieldErrKind(ecode.ErrKindUriErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key))) - continue - } - if sm, err := parseValue(kv.Value); err == nil { - for key, val := range sm.Metadata { - meta.Set(key, val) - } - } - al.regInfos[host] = &meta - case strings.HasPrefix(addr, "configurators/grpc://"): - addr = strings.TrimPrefix(addr, "configurators/") - host, meta, err := parseURI(addr) - if err != nil { - r.logger.Error("parse uri", xlog.FieldErrKind(ecode.ErrKindUriErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key))) - continue - } - if sm, err := parseValue(kv.Value); err == nil { - for key, val := range sm.Metadata { - meta.Set(key, val) - } - } - al.cfgInfos[host] = &meta - case isIPPort(addr): // v1 协议 - var meta naming.Update - if err := jsoniter.Unmarshal(kv.Value, &meta); err != nil { - r.logger.Error("unmarshal metadata", xlog.FieldErrKind(ecode.ErrKindUnmarshalConfigErr), xlog.FieldErr(err), xlog.FieldKey(string(kv.Key)), xlog.FieldValue(string(kv.Value))) - continue - } - - if _, ok := al.cfgInfos[addr]; !ok { - al.cfgInfos[addr] = &url.Values{} - } - - // 解析value - switch meta.Op { - case naming.Add: - al.regInfos[addr] = &url.Values{} - al.cfgInfos[addr].Set("enable", "true") - al.cfgInfos[addr].Set("weight", "100") - case naming.Delete: - al.regInfos[addr] = &url.Values{} - al.cfgInfos[addr].Set("enable", "false") - al.cfgInfos[addr].Set("weight", "0") - } - - } - } - r.logger.Info("update addr list", xlog.Any("reg_info", al.regInfos), xlog.Any("cfg_info", al.cfgInfos)) -} - -// AddressList ... -type AddressList struct { - serverName string - - // TODO 2019/9/10 gorexlv: need lock - regInfos map[string]*url.Values // 注册信息 - cfgInfos map[string]*url.Values // 配置信息 - - mtx sync.RWMutex -} - -// List ... -func (al *AddressList) List() []resolver.Address { - // TODO 2019/9/10 gorexlv: - addrs := make([]resolver.Address, 0) - al.mtx.RLock() - defer al.mtx.RUnlock() - for addr, values := range al.regInfos { - metadata := *values - address := resolver.Address{ - Addr: addr, - ServerName: al.serverName, - Attributes: attributes.New(), - } - if infos, ok := al.cfgInfos[addr]; ok { - for cfg := range *infos { - metadata.Set(cfg, infos.Get(cfg)) - // address.Attributes.WithValues(cfg, infos.Get(cfg)) - } - } - if enable := metadata.Get("enable"); enable != "" && enable != "true" { - continue - } - if metadata.Get("weight") == "" { - metadata.Set("weight", "100") - } - // group: 客户端配置的分组，默认为default - // metadata.Get("group"): 服务端配置的分组 - // if metadata.Get("group") != "" && metadata.Get("group") != group { - // continue - // } - address.Metadata = &metadata - addrs = append(addrs, address) - } - return addrs -} - -func escape(raw string) string { - return strings.Replace(raw, "$", "%24", -1) -} - -func (r *etcdResolver) watch(cc resolver.ClientConn, prefix string, serviceName string) { - cli := r.client - target := fmt.Sprintf("/%s/%s/", prefix, serviceName) - for { - var al = &AddressList{ - serverName: serviceName, - regInfos: make(map[string]*url.Values), - cfgInfos: make(map[string]*url.Values), - mtx: sync.RWMutex{}, - } - - getResp, err := cli.Get(context.Background(), target, clientv3.WithPrefix()) - if err != nil { - r.logger.Error(ecode.MsgWatchRequestErr, xlog.FieldErrKind(ecode.ErrKindRequestErr), xlog.FieldErr(err), xlog.FieldAddr(target)) - time.Sleep(time.Second * 5) - continue - } - - r.updateAddrList(al, target, getResp.Kvs...) - - cc.UpdateState(resolver.State{ - Addresses: al.List(), - }) - - // 处理配置键变更, reg_key - // 处理注册键变更, cfg_key - ctx, cancel := context.WithCancel(context.Background()) - rch := cli.Watch(ctx, target, clientv3.WithPrefix(), clientv3.WithCreatedNotify()) - for n := range rch { - for _, ev := range n.Events { - switch ev.Type { - // 添加或者更新 - case mvccpb.PUT: - r.updateAddrList(al, target, ev.Kv) - // 硬删除 - case mvccpb.DELETE: - r.deleteAddrList(al, target, ev.Kv) - } - } - - cc.UpdateState(resolver.State{ - Addresses: al.List(), - }) - } - - cancel() - } -} - -func parseURI(uri string) (host string, meta url.Values, err error) { - uri, err = url.PathUnescape(uri) - if err != nil { - return - } - if strings.Index(uri, "://") > 0 { - u, e := url.Parse(uri) - if e != nil || u == nil { - return "", nil, e - } - host = u.Host - meta = u.Query() - meta.Set("scheme", u.Scheme) - return - } - return uri, meta, nil -} - -func parseValue(raw []byte) (*server.ServiceInfo, error) { - var meta server.ServiceInfo - if err := jsoniter.Unmarshal(raw, &meta); err != nil { - return nil, err - } - - return &meta, nil -} diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index cb45fe0a91..416f7bfc8c 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -16,11 +16,67 @@ package registry import ( "context" + "encoding/json" + "fmt" "io" "github.com/douyu/jupiter/pkg/server" ) +// Event ... +type Event uint8 + +const ( + // EventUnknown ... + EventUnknown Event = iota + // EventUpdate ... + EventUpdate + // EventDelete ... + EventDelete +) + +// Kind ... +type Kind uint8 + +const ( + // KindUnknown ... + KindUnknown Kind = iota + // KindProvider ... + KindProvider + // KindConfigurator ... + KindConfigurator + // KindConsumer ... + KindConsumer +) + +// String ... +func (kind Kind) String() string { + switch kind { + case KindProvider: + return "providers" + case KindConfigurator: + return "configurators" + case KindConsumer: + return "consumers" + default: + return "unknown" + } +} + +// ToKind ... +func ToKind(kindStr string) Kind { + switch kindStr { + case "providers": + return KindProvider + case "configurators": + return KindConfigurator + case "consumers": + return KindConsumer + default: + return KindUnknown + } +} + // ServerInstance ... type ServerInstance struct { Scheme string @@ -29,32 +85,150 @@ type ServerInstance struct { Labels map[string]string } -// Registry register/deregister service +// EventMessage ... +type EventMessage struct { + Event + Kind + Name string + Scheme string + Address string + Message interface{} +} + +// Registry register/unregister service // registry impl should control rpc timeout type Registry interface { RegisterService(context.Context, *server.ServiceInfo) error - DeregisterService(context.Context, *server.ServiceInfo) error + UnregisterService(context.Context, *server.ServiceInfo) error + ListServices(context.Context, string, string) ([]*server.ServiceInfo, error) + WatchServices(context.Context, string, string) (chan Endpoints, error) io.Closer } +//GetServiceKey .. +func GetServiceKey(prefix string, s *server.ServiceInfo) string { + return fmt.Sprintf("/%s/%s/%s/%s://%s", prefix, s.Name, s.Kind.String(), s.Scheme, s.Address) +} + +//GetServiceValue .. +func GetServiceValue(s *server.ServiceInfo) string { + val, _ := json.Marshal(s) + return string(val) +} + +//GetService .. +func GetService(s string) *server.ServiceInfo { + var si server.ServiceInfo + json.Unmarshal([]byte(s), &si) + return &si +} + // Nop registry, used for local development/debugging type Nop struct{} +// ListServices ... +func (n Nop) ListServices(ctx context.Context, s string, s2 string) ([]*server.ServiceInfo, error) { + panic("implement me") +} + +// WatchServices ... +func (n Nop) WatchServices(ctx context.Context, s string, s2 string) (chan Endpoints, error) { + panic("implement me") +} + // RegisterService ... func (n Nop) RegisterService(context.Context, *server.ServiceInfo) error { return nil } -// DeregisterService ... -func (n Nop) DeregisterService(context.Context, *server.ServiceInfo) error { return nil } +// UnregisterService ... +func (n Nop) UnregisterService(context.Context, *server.ServiceInfo) error { return nil } // Close ... func (n Nop) Close() error { return nil } // Configuration ... type Configuration struct { + Routes []Route `json:"routes"` // 配置客户端路由策略 + Labels map[string]string `json:"labels"` // 配置服务端标签: 分组 +} + +// Route represents route configuration +type Route struct { + // 路由方法名 + Method string `json:"method" toml:"method"` + // 路由权重组, 按比率在各个权重组中分配流量 + WeightGroups []WeightGroup `json:"weightGroups" toml:"weightGroups"` + // 路由部署组, 将流量导入部署组 + Deployment string `json:"deployment" toml:"deployment"` } -// Rule ... -type Rule struct { - Target string - Pattern string +// WeightGroup ... +type WeightGroup struct { + Group string `json:"group" toml:"group"` + Weight int `json:"weight" toml:"weight"` } + +// AddressList ... +// type AddressList struct { +// serverName string +// +// // regInfos map[string]*server.ServiceInfo +// // cfgInfos map[string]*server.ConfigInfo +// +// // TODO 2019/9/10 gorexlv: need lock +// regInfos map[string]*server.ServiceInfo // 注册信息 +// cfgInfos map[string]*RouteConfig // 配置信息 +// +// mtx sync.RWMutex +// } +// +// func NewAddressList(serverName string) *AddressList { +// return &AddressList{ +// serverName: serverName, +// regInfos: make(map[string]*url.Values), +// cfgInfos: make(map[string]*url.Values), +// } +// } +// +// func (al *AddressList) String() string { +// return "" +// } +// +// func (al *AddressList) List2() { +// +// } +// +// // List ... +// func (al *AddressList) List() []resolver.Address { +// // TODO 2019/9/10 gorexlv: +// addrs := make([]resolver.Address, 0) +// al.mtx.RLock() +// defer al.mtx.RUnlock() +// for addr, values := range al.regInfos { +// metadata := *values +// address := resolver.Address{ +// Addr: addr, +// ServerName: al.serverName, +// Attributes: attributes.New(), +// } +// // if infos, ok := al.cfgInfos[addr]; ok { +// // for cfg := range *infos { +// // metadata.Set(cfg, infos.Get(cfg)) +// // // address.Attributes.WithValues(cfg, infos.Get(cfg)) +// // } +// // } +// if enable := metadata.Get("enable"); enable != "" && enable != "true" { +// continue +// } +// if metadata.Get("weight") == "" { +// metadata.Set("weight", "100") +// } +// // group: 客户端配置的分组，默认为default +// // metadata.Get("group"): 服务端配置的分组 +// // if metadata.Get("group") != "" && metadata.Get("group") != group { +// // continue +// // } +// address.Metadata = &metadata +// addrs = append(addrs, address) +// } +// return addrs +// } diff --git a/pkg/sentinel/config.go b/pkg/sentinel/config.go index a6dccda069..37c87688d5 100644 --- a/pkg/sentinel/config.go +++ b/pkg/sentinel/config.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package sentinel import ( diff --git a/pkg/server/governor/config.go b/pkg/server/governor/config.go new file mode 100644 index 0000000000..3f3d9e17b5 --- /dev/null +++ b/pkg/server/governor/config.go @@ -0,0 +1,75 @@ +package governor + +import ( + "fmt" + + "github.com/douyu/jupiter/pkg/conf" + "github.com/douyu/jupiter/pkg/util/xnet" + "github.com/douyu/jupiter/pkg/xlog" +) + +//ModName .. +const ModName = "govern" + +// Config ... +type Config struct { + Host string + Port int + Network string `json:"network" toml:"network"` + logger *xlog.Logger + Enable bool +} + +// StdConfig represents Standard gRPC Server config +// which will parse config by conf package, +// panic if no config key found in conf +func StdConfig(name string) *Config { + return RawConfig("jupiter.server." + name) +} + +// RawConfig ... +func RawConfig(key string) *Config { + var config = DefaultConfig() + if conf.Get(key) == nil { + return config + } + if err := conf.UnmarshalKey(key, &config); err != nil { + config.logger.Panic("govern server parse config panic", + xlog.FieldErr(err), xlog.FieldKey(key), + xlog.FieldValueAny(config), + ) + } + return config +} + +// DefaultConfig represents default config +// User should construct config base on DefaultConfig +func DefaultConfig() *Config { + ips := xnet.GetIPs() + if len(ips) == 0 { + xlog.JupiterLogger.Error("govern get local ip error") + } + var host string + if len(ips) == 1 { + host = ips[0] + } else { + host = "localhost" + } + return &Config{ + Enable: true, + Host: host, + Network: "tcp4", + Port: 0, + logger: xlog.JupiterLogger.With(xlog.FieldMod(ModName)), + } +} + +// Build ... +func (config *Config) Build() *Server { + return newServer(config) +} + +// Address ... +func (config Config) Address() string { + return fmt.Sprintf("%s:%d", config.Host, config.Port) +} diff --git a/pkg/govern/http.go b/pkg/server/governor/http.go similarity index 98% rename from pkg/govern/http.go rename to pkg/server/governor/http.go index b0542c915b..6ac8038ca1 100644 --- a/pkg/govern/http.go +++ b/pkg/server/governor/http.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package govern +package governor import ( "encoding/json" diff --git a/pkg/server/governor/init.go b/pkg/server/governor/init.go new file mode 100644 index 0000000000..e97e015c56 --- /dev/null +++ b/pkg/server/governor/init.go @@ -0,0 +1,56 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package governor + +import ( + "encoding/json" + "github.com/douyu/jupiter/pkg" + "github.com/douyu/jupiter/pkg/conf" + jsoniter "github.com/json-iterator/go" + "net/http" + "os" +) + +func init() { + HandleFunc("/configs", func(w http.ResponseWriter, r *http.Request) { + encoder := json.NewEncoder(w) + if r.URL.Query().Get("pretty") == "true" { + encoder.SetIndent("", " ") + } + encoder.Encode(conf.Traverse(".")) + }) + + HandleFunc("/debug/env", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + _ = jsoniter.NewEncoder(w).Encode(os.Environ()) + }) + + HandleFunc("/build/info", func(w http.ResponseWriter, r *http.Request) { + serverStats := map[string]string{ + "name": pkg.Name(), + "appID": pkg.AppID(), + "appMode": pkg.AppMode(), + "appVersion": pkg.AppVersion(), + "jupiterVersion": pkg.JupiterVersion(), + "buildUser": pkg.BuildUser(), + "buildHost": pkg.BuildHost(), + "buildTime": pkg.BuildTime(), + "startTime": pkg.StartTime(), + "hostName": pkg.HostName(), + "goVersion": pkg.GoVersion(), + } + _ = jsoniter.NewEncoder(w).Encode(serverStats) + }) +} diff --git a/pkg/server/governor/server.go b/pkg/server/governor/server.go new file mode 100644 index 0000000000..cf1a8c4bc1 --- /dev/null +++ b/pkg/server/governor/server.go @@ -0,0 +1,65 @@ +package governor + +import ( + "context" + "net" + "net/http" + + "github.com/douyu/jupiter/pkg/constant" + "github.com/douyu/jupiter/pkg/server" + "github.com/douyu/jupiter/pkg/xlog" +) + +// Server ... +type Server struct { + *http.Server + listener net.Listener + *Config +} + +func newServer(config *Config) *Server { + var listener, err = net.Listen("tcp4", config.Address()) + if err != nil { + xlog.Panic("governor start error", xlog.FieldErr(err)) + } + + return &Server{ + Server: &http.Server{ + Addr: config.Address(), + Handler: DefaultServeMux, + }, + listener: listener, + Config: config, + } +} + +//Serve .. +func (s *Server) Serve() error { + err := s.Server.Serve(s.listener) + if err == http.ErrServerClosed { + return nil + } + return err + +} + +//Stop .. +func (s *Server) Stop() error { + return s.Server.Close() +} + +//GracefulStop .. +func (s *Server) GracefulStop(ctx context.Context) error { + return s.Server.Shutdown(ctx) +} + +//Info .. +func (s *Server) Info() *server.ServiceInfo { + info := server.ApplyOptions( + server.WithScheme("http"), + server.WithAddress(s.listener.Addr().String()), + server.WithKind(constant.ServiceGovernor), + ) + info.Name = info.Name + "." + ModName + return &info +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 3467fd1d39..9191fb774b 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -18,22 +18,44 @@ import ( "context" "fmt" + "github.com/douyu/jupiter/pkg" "github.com/douyu/jupiter/pkg/constant" ) -// ServiceInfo ... +type Option func(c *ServiceInfo) + +// ServiceConfigurator represents service configurator +type ConfigInfo struct { + Routes []Route +} + +// ServiceInfo represents service info type ServiceInfo struct { - Name string - Scheme string - Address string - Weight float64 - Enable bool - Healthy bool - Metadata map[string]string - Region string - Zone string - GroupName string - Kind constant.ServiceKind + Name string `json:"name"` + AppID string `json:"appId"` + Scheme string `json:"scheme"` + Address string `json:"address"` + Weight float64 `json:"weight"` + Enable bool `json:"enable"` + Healthy bool `json:"healthy"` + Metadata map[string]string `json:"metadata"` + Region string `json:"region"` + Zone string `json:"zone"` + Kind constant.ServiceKind `json:"kind"` + // Deployment 部署组: 不同组的流量隔离 + // 比如某些服务给内部调用和第三方调用，可以配置不同的deployment,进行流量隔离 + Deployment string `json:"deployment"` + // Group 流量组: 流量在Group之间进行负载均衡 + Group string `json:"group"` + Services map[string]*Service `json:"services" toml:"services"` +} + +// Service ... +type Service struct { + Namespace string `json:"namespace" toml:"namespace"` + Name string `json:"name" toml:"name"` + Labels map[string]string `json:"labels" toml:"labels"` + Methods []string `json:"methods" toml:"methods"` } // Label ... @@ -48,3 +70,72 @@ type Server interface { GracefulStop(ctx context.Context) error Info() *ServiceInfo } + +// Route ... +type Route struct { + // 权重组，按照 + WeightGroups []WeightGroup + // 方法名 + Method string +} + +// WeightGroup ... +type WeightGroup struct { + Group string + Weight int +} + +func ApplyOptions(options ...Option) ServiceInfo { + info := defaultServiceInfo() + for _, option := range options { + option(&info) + } + return info +} + +func WithMetaData(key, value string) Option { + return func(c *ServiceInfo) { + c.Metadata[key] = value + } +} + +func WithScheme(scheme string) Option { + return func(c *ServiceInfo) { + c.Scheme = scheme + } +} + +func WithAddress(address string) Option { + return func(c *ServiceInfo) { + c.Address = address + } +} + +func WithKind(kind constant.ServiceKind) Option { + return func(c *ServiceInfo) { + c.Kind = kind + } +} + +func defaultServiceInfo() ServiceInfo { + si := ServiceInfo{ + Name: pkg.Name(), + AppID: pkg.AppID(), + Weight: 100, + Enable: true, + Healthy: true, + Metadata: make(map[string]string), + Region: pkg.AppRegion(), + Zone: pkg.AppZone(), + Kind: 0, + Deployment: "", + Group: "", + } + si.Metadata["appMode"] = pkg.AppMode() + si.Metadata["appHost"] = pkg.AppHost() + si.Metadata["startTime"] = pkg.StartTime() + si.Metadata["buildTime"] = pkg.BuildTime() + si.Metadata["appVersion"] = pkg.AppVersion() + si.Metadata["jupiterVersion"] = pkg.JupiterVersion() + return si +} diff --git a/pkg/server/xecho/config.go b/pkg/server/xecho/config.go index 55647cd348..a0192c64a3 100644 --- a/pkg/server/xecho/config.go +++ b/pkg/server/xecho/config.go @@ -18,15 +18,20 @@ import ( "fmt" "github.com/douyu/jupiter/pkg/conf" + "github.com/douyu/jupiter/pkg/constant" "github.com/douyu/jupiter/pkg/ecode" "github.com/douyu/jupiter/pkg/xlog" "github.com/pkg/errors" ) -// HTTP config +//ModName named a mod +const ModName = "server.echo" + +//Config HTTP config type Config struct { Host string Port int + Deployment string Debug bool DisableMetric bool DisableTrace bool @@ -42,12 +47,13 @@ func DefaultConfig() *Config { Host: "127.0.0.1", Port: 9091, Debug: false, + Deployment: constant.DefaultDeployment, SlowQueryThresholdInMilli: 500, // 500ms - logger: xlog.JupiterLogger.With(xlog.FieldMod("server.echo")), + logger: xlog.JupiterLogger.With(xlog.FieldMod(ModName)), } } -// Jupiter Standard HTTP Server config +// StdConfig Jupiter Standard HTTP Server config func StdConfig(name string) *Config { return RawConfig("jupiter.server." + name) } diff --git a/pkg/server/xecho/server.go b/pkg/server/xecho/server.go index 8eee2671fe..bb03435f27 100644 --- a/pkg/server/xecho/server.go +++ b/pkg/server/xecho/server.go @@ -19,13 +19,13 @@ import ( "net/http" "os" - "github.com/douyu/jupiter/pkg" + "net" + "github.com/douyu/jupiter/pkg/constant" "github.com/douyu/jupiter/pkg/ecode" "github.com/douyu/jupiter/pkg/server" "github.com/douyu/jupiter/pkg/xlog" "github.com/labstack/echo/v4" - "net" ) // Server ... @@ -80,19 +80,12 @@ func (s *Server) GracefulStop(ctx context.Context) error { } // Info returns server info, used by governor and consumer balancer -// TODO(gorexlv): implements government protocol with juno func (s *Server) Info() *server.ServiceInfo { - return &server.ServiceInfo{ - Name: pkg.Name(), - Scheme: "http", - Address: s.listener.Addr().String(), - Weight: 0.0, - Enable: false, - Healthy: false, - Metadata: map[string]string{}, - Region: "", - Zone: "", - GroupName: "", - Kind: constant.ServiceProvider, - } + info := server.ApplyOptions( + server.WithScheme("http"), + server.WithAddress(s.listener.Addr().String()), + server.WithKind(constant.ServiceProvider), + ) + info.Name = info.Name + "." + ModName + return &info } diff --git a/pkg/server/xgin/config.go b/pkg/server/xgin/config.go index 0c089dca26..a0e588dc62 100644 --- a/pkg/server/xgin/config.go +++ b/pkg/server/xgin/config.go @@ -24,10 +24,14 @@ import ( "github.com/pkg/errors" ) +//ModName .. +const ModName = "server.gin" + // Config HTTP config type Config struct { Host string Port int + Deployment string Mode string DisableMetric bool DisableTrace bool @@ -44,7 +48,7 @@ func DefaultConfig() *Config { Port: 9091, Mode: gin.ReleaseMode, SlowQueryThresholdInMilli: 500, // 500ms - logger: xlog.JupiterLogger.With(xlog.FieldMod("server.gin")), + logger: xlog.JupiterLogger.With(xlog.FieldMod(ModName)), } } diff --git a/pkg/server/xgin/server.go b/pkg/server/xgin/server.go index 4cf801fed3..4b941726eb 100644 --- a/pkg/server/xgin/server.go +++ b/pkg/server/xgin/server.go @@ -20,7 +20,6 @@ import ( "net" - "github.com/douyu/jupiter/pkg" "github.com/douyu/jupiter/pkg/constant" "github.com/douyu/jupiter/pkg/ecode" "github.com/douyu/jupiter/pkg/server" @@ -89,19 +88,12 @@ func (s *Server) GracefulStop(ctx context.Context) error { } // Info returns server info, used by governor and consumer balancer -// TODO(gorexlv): implements government protocol with juno func (s *Server) Info() *server.ServiceInfo { - return &server.ServiceInfo{ - Name: pkg.Name(), - Scheme: "http", - Address: s.listener.Addr().String(), - Weight: 0.0, - Enable: false, - Healthy: false, - Metadata: map[string]string{}, - Region: "", - Zone: "", - GroupName: "", - Kind: constant.ServiceProvider, - } + info := server.ApplyOptions( + server.WithScheme("http"), + server.WithAddress(s.listener.Addr().String()), + server.WithKind(constant.ServiceProvider), + ) + info.Name = info.Name + "." + ModName + return &info } diff --git a/pkg/server/xgrpc/config.go b/pkg/server/xgrpc/config.go index 8af29f1906..bdb8bfcbfd 100644 --- a/pkg/server/xgrpc/config.go +++ b/pkg/server/xgrpc/config.go @@ -17,6 +17,7 @@ package xgrpc import ( "fmt" + "github.com/douyu/jupiter/pkg/constant" "github.com/douyu/jupiter/pkg/ecode" "github.com/douyu/jupiter/pkg/xlog" @@ -26,8 +27,9 @@ import ( // Config ... type Config struct { - Host string - Port int + Host string + Port int + Deployment string // Network network type, tcp4 by default Network string `json:"network" toml:"network"` // DisableTrace disbale Trace Interceptor, false by default @@ -70,6 +72,7 @@ func DefaultConfig() *Config { Network: "tcp4", Host: "127.0.0.1", Port: 9092, + Deployment: constant.DefaultDeployment, DisableMetric: false, DisableTrace: false, SlowQueryThresholdInMilli: 500, diff --git a/pkg/server/xgrpc/server.go b/pkg/server/xgrpc/server.go index d0f19afeaf..29c2557262 100644 --- a/pkg/server/xgrpc/server.go +++ b/pkg/server/xgrpc/server.go @@ -18,8 +18,8 @@ import ( "context" "net" - "github.com/douyu/jupiter/pkg" "github.com/douyu/jupiter/pkg/constant" + "github.com/douyu/jupiter/pkg/ecode" "github.com/douyu/jupiter/pkg/server" @@ -32,6 +32,7 @@ type Server struct { *grpc.Server listener net.Listener *Config + serverInfo *server.ServiceInfo } func newServer(config *Config) *Server { @@ -56,15 +57,24 @@ func newServer(config *Config) *Server { config.logger.Panic("new grpc server err", xlog.FieldErrKind(ecode.ErrKindListenErr), xlog.FieldErr(err)) } config.Port = listener.Addr().(*net.TCPAddr).Port - return &Server{Server: newServer, listener: listener, Config: config} + + info := server.ApplyOptions( + server.WithScheme("grpc"), + server.WithAddress(listener.Addr().String()), + server.WithKind(constant.ServiceProvider), + ) + + return &Server{ + Server: newServer, + listener: listener, + Config: config, + serverInfo: &info, + } } // Server implements server.Server interface. func (s *Server) Serve() error { err := s.Server.Serve(s.listener) - if err == grpc.ErrServerStopped { - return nil - } return err } @@ -83,19 +93,6 @@ func (s *Server) GracefulStop(ctx context.Context) error { } // Info returns server info, used by governor and consumer balancer -// TODO(gorexlv): implements government protocol with juno func (s *Server) Info() *server.ServiceInfo { - return &server.ServiceInfo{ - Name: pkg.Name(), - Scheme: "grpc", - Address: s.listener.Addr().String(), - Weight: 0.0, - Enable: true, - Healthy: true, - Metadata: map[string]string{}, - Region: "", - Zone: "", - GroupName: "", - Kind: constant.ServiceProvider, - } + return s.serverInfo } diff --git a/pkg/server/xgrpc/server_test.go b/pkg/server/xgrpc/server_test.go new file mode 100644 index 0000000000..dc3386fafa --- /dev/null +++ b/pkg/server/xgrpc/server_test.go @@ -0,0 +1,163 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xgrpc + +import ( + "context" + "net" + "testing" + "time" + + "github.com/douyu/jupiter/pkg/constant" + "github.com/douyu/jupiter/pkg/server" + "github.com/douyu/jupiter/pkg/xlog" + "github.com/smartystreets/goconvey/convey" + "google.golang.org/grpc" + "google.golang.org/grpc/status" +) + +func TestServer_Serve(t *testing.T) { + type fields struct { + Server *grpc.Server + listener net.Listener + Config *Config + serverInfo *server.ServiceInfo + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Server{ + Server: tt.fields.Server, + listener: tt.fields.listener, + Config: tt.fields.Config, + serverInfo: tt.fields.serverInfo, + } + if err := s.Serve(); (err != nil) != tt.wantErr { + t.Errorf("Server.Serve() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestServer_Closed(t *testing.T) { + convey.Convey("test server stop", t, func() { + config := DefaultConfig() + config.Port = 0 + ns := newServer(config) + err := ns.Stop() + convey.So(err, convey.ShouldBeNil) + err = ns.Serve() + convey.So(err, convey.ShouldEqual, grpc.ErrServerStopped) + // server.Serve is responsible for closing the listener, even if the + // server was already stopped. + err = ns.listener.Close() + convey.So(errorDesc(err), convey.ShouldContainSubstring, "use of closed") + }) +} +func TestServer_Stop(t *testing.T) { + convey.Convey("test server graceful stop", t, func(c convey.C) { + ns := newServer(&Config{ + Network: "tcp4", + Host: "127.0.0.1", + Port: 0, + Deployment: constant.DefaultDeployment, + DisableMetric: false, + DisableTrace: false, + SlowQueryThresholdInMilli: 500, + logger: xlog.JupiterLogger.With(xlog.FieldMod("server.grpc")), + serverOptions: []grpc.ServerOption{}, + streamInterceptors: []grpc.StreamServerInterceptor{}, + unaryInterceptors: []grpc.UnaryServerInterceptor{}, + }) + // + go func() { + // make sure Serve() is called + time.Sleep(time.Millisecond * 500) + err := ns.Stop() + c.So(err, convey.ShouldBeNil) + }() + + err := ns.Serve() + convey.So(err, convey.ShouldBeNil) + // server.Serve is responsible for closing the listener, even if the + // server was already stopped. + err = ns.listener.Close() + convey.So(errorDesc(err), convey.ShouldContainSubstring, "use of closed") + }) +} +func TestServer_GracefulStop(t *testing.T) { + convey.Convey("test server graceful stop", t, func(c convey.C) { + ns := newServer(&Config{ + Network: "tcp4", + Host: "127.0.0.1", + Port: 0, + Deployment: constant.DefaultDeployment, + DisableMetric: false, + DisableTrace: false, + SlowQueryThresholdInMilli: 500, + logger: xlog.JupiterLogger.With(xlog.FieldMod("server.grpc")), + serverOptions: []grpc.ServerOption{}, + streamInterceptors: []grpc.StreamServerInterceptor{}, + unaryInterceptors: []grpc.UnaryServerInterceptor{}, + }) + // + go func() { + // make sure Serve() is called + time.Sleep(time.Millisecond * 500) + err := ns.GracefulStop(context.TODO()) + c.So(err, convey.ShouldBeNil) + }() + + err := ns.Serve() + convey.So(err, convey.ShouldBeNil) + // server.Serve is responsible for closing the listener, even if the + // server was already stopped. + err = ns.listener.Close() + convey.So(errorDesc(err), convey.ShouldContainSubstring, "use of closed") + }) +} + +func TestServer_Info(t *testing.T) { + convey.Convey("test server info", t, func(c convey.C) { + ns := newServer(&Config{ + Network: "tcp4", + Host: "127.0.0.1", + Port: 0, + Deployment: constant.DefaultDeployment, + DisableMetric: false, + DisableTrace: false, + SlowQueryThresholdInMilli: 500, + logger: xlog.JupiterLogger.With(xlog.FieldMod("server.grpc")), + serverOptions: []grpc.ServerOption{}, + streamInterceptors: []grpc.StreamServerInterceptor{}, + unaryInterceptors: []grpc.UnaryServerInterceptor{}, + }) + convey.So(ns.Info().Scheme, convey.ShouldEqual, "grpc") + convey.So(ns.Info().Enable, convey.ShouldEqual, true) + }) +} + +func errorDesc(err error) string { + if s, ok := status.FromError(err); ok { + return s.Message() + } + return err.Error() +} diff --git a/pkg/signals/signals_test.go b/pkg/signals/signals_test.go new file mode 100644 index 0000000000..74beacddc6 --- /dev/null +++ b/pkg/signals/signals_test.go @@ -0,0 +1,53 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package signals + +import ( + "os" + "syscall" + "testing" + + . "github.com/smartystreets/goconvey/convey" +) + +func kill(sig os.Signal) { + pro, _ := os.FindProcess(os.Getpid()) + pro.Signal(sig) +} +func TestShutdownSIGQUIT(t *testing.T) { + quit := make(chan struct{}) + Convey("test shutdown signal by SIGQUIT", t, func(c C) { + fn := func(grace bool) { + c.So(grace, ShouldEqual, false) + close(quit) + } + Shutdown(fn) + kill(syscall.SIGQUIT) + <-quit + }) +} + +// func TestShutdownSIGINT(t *testing.T) { +// quit := make(chan struct{}) +// Convey("test shutdown signal by SIGINT", t, func(c C) { +// fn := func(grace bool) { +// c.So(grace, ShouldEqual, true) +// close(quit) +// } +// Shutdown(fn) +// kill(syscall.SIGINT) +// <-quit +// }) +// } diff --git a/pkg/store/gorm/config.go b/pkg/store/gorm/config.go index b5e5a4269b..485029d1b0 100644 --- a/pkg/store/gorm/config.go +++ b/pkg/store/gorm/config.go @@ -15,6 +15,7 @@ package gorm import ( + "github.com/douyu/jupiter/pkg/metric" "time" "github.com/douyu/jupiter/pkg/ecode" @@ -36,12 +37,13 @@ func RawConfig(key string) *Config { if err := conf.UnmarshalKey(key, config, conf.TagName("toml")); err != nil { xlog.Panic("unmarshal key", xlog.FieldMod("gorm"), xlog.FieldErr(err), xlog.FieldKey(key)) } - + config.Name = key return config } // config options type Config struct { + Name string // DSN地址: mysql://root:secret@tcp(127.0.0.1:3307)/mysql?timeout=20s&readTimeout=20s DSN string `json:"dsn" toml:"dsn"` // Debug开关 @@ -71,6 +73,7 @@ type Config struct { raw interface{} logger *xlog.Logger interceptors []Interceptor + dsnCfg *DSN } // DefaultConfig 返回默认配置 @@ -87,7 +90,7 @@ func DefaultConfig() *Config { DisableMetric: false, DisableTrace: false, raw: nil, - logger: xlog.DefaultLogger, + logger: xlog.JupiterLogger, } } @@ -108,6 +111,14 @@ func (config *Config) WithInterceptor(intes ...Interceptor) *Config { // Build ... func (config *Config) Build() *DB { + var err error + config.dsnCfg, err = ParseDSN(config.DSN) + if err == nil { + config.logger.Info(ecode.MsgClientMysqlOpenStart, xlog.FieldMod("gorm"), xlog.FieldAddr(config.dsnCfg.Addr), xlog.FieldName(config.dsnCfg.DBName)) + } else { + config.logger.Panic(ecode.MsgClientMysqlOpenStart, xlog.FieldMod("gorm"), xlog.FieldErr(err)) + } + if config.Debug { config = config.WithInterceptor(debugInterceptor) } @@ -121,22 +132,20 @@ func (config *Config) Build() *DB { db, err := Open("mysql", config) if err != nil { - config.logger.Panic("open mysql", xlog.FieldMod("gorm"), xlog.FieldErrKind(ecode.ErrKindRequestErr), xlog.FieldErr(err), xlog.FieldValueAny(config)) + if config.OnDialError == "panic" { + config.logger.Panic("open mysql", xlog.FieldMod("gorm"), xlog.FieldErrKind(ecode.ErrKindRequestErr), xlog.FieldErr(err), xlog.FieldAddr(config.dsnCfg.Addr), xlog.FieldValueAny(config)) + } else { + metric.LibHandleCounter.Inc(metric.TypeGorm, config.Name+".ping", config.dsnCfg.Addr, "open err") + config.logger.Error("open mysql", xlog.FieldMod("gorm"), xlog.FieldErrKind(ecode.ErrKindRequestErr), xlog.FieldErr(err), xlog.FieldAddr(config.dsnCfg.Addr), xlog.FieldValueAny(config)) + return db + } } if err := db.DB().Ping(); err != nil { config.logger.Panic("ping mysql", xlog.FieldMod("gorm"), xlog.FieldErrKind(ecode.ErrKindRequestErr), xlog.FieldErr(err), xlog.FieldValueAny(config)) } - // 上面已经判断过dsn了，这里err可以暂时不判断 - // TODO 将addr，传过来是最好的，先打印数据 - // TODO 上面的value里面有密码，应该用下面解析过数据，过滤掉密码 - d, err := ParseDSN(config.DSN) - if err == nil { - config.logger.Info(ecode.MsgClientMysqlOpenStart, xlog.FieldMod("gorm"), xlog.FieldAddr(d.Addr), xlog.FieldName(d.DBName)) - } else { - config.logger.Error(ecode.MsgClientMysqlOpenStart, xlog.FieldMod("gorm"), xlog.FieldAddr(d.Addr), xlog.FieldName(d.DBName)) - } - + // store db + instances.Store(config.Name, db) return db } diff --git a/pkg/store/gorm/init.go b/pkg/store/gorm/init.go new file mode 100644 index 0000000000..bd7b31eaab --- /dev/null +++ b/pkg/store/gorm/init.go @@ -0,0 +1,46 @@ +package gorm + +import ( + "github.com/douyu/jupiter/pkg/metric" + "github.com/douyu/jupiter/pkg/server/governor" + "github.com/douyu/jupiter/pkg/xlog" + "net/http" + "time" + + jsoniter "github.com/json-iterator/go" +) + +var ( + _logger = xlog.JupiterLogger.With(xlog.FieldMod("gorm")) +) + +func init() { + type gormStatus struct { + Gorms map[string]interface{} `json:"gorms"` + } + var rets = gormStatus{ + Gorms: make(map[string]interface{}, 0), + } + governor.HandleFunc("/debug/gorm/stats", func(w http.ResponseWriter, r *http.Request) { + rets.Gorms = Stats() + _ = jsoniter.NewEncoder(w).Encode(rets) + }) + go monitor() +} + +func monitor() { + for { + time.Sleep(time.Second * 10) + Range(func(name string, db *DB) bool { + stats := db.DB().Stats() + metric.LibHandleSummary.Observe(float64(stats.Idle), name, "idle") + metric.LibHandleSummary.Observe(float64(stats.InUse), name, "inuse") + metric.LibHandleSummary.Observe(float64(stats.WaitCount), name, "wait") + metric.LibHandleSummary.Observe(float64(stats.OpenConnections), name, "conns") + metric.LibHandleSummary.Observe(float64(stats.MaxOpenConnections), name, "max_open_conns") + metric.LibHandleSummary.Observe(float64(stats.MaxIdleClosed), name, "max_idle_closed") + metric.LibHandleSummary.Observe(float64(stats.MaxLifetimeClosed), name, "max_lifetime_closed") + return true + }) + } +} diff --git a/pkg/store/gorm/instance.go b/pkg/store/gorm/instance.go new file mode 100644 index 0000000000..ba6033d54a --- /dev/null +++ b/pkg/store/gorm/instance.go @@ -0,0 +1,38 @@ +package gorm + +import ( + "sync" +) + +var instances = sync.Map{} + +// Range 遍历所有实例 +func Range(fn func(name string, db *DB) bool) { + instances.Range(func(key, val interface{}) bool { + return fn(key.(string), val.(*DB)) + }) +} + +// Configs +func Configs() map[string]interface{} { + var rets = make(map[string]interface{}) + instances.Range(func(key, val interface{}) bool { + return true + }) + + return rets +} + +// Stats +func Stats() (stats map[string]interface{}) { + stats = make(map[string]interface{}) + instances.Range(func(key, val interface{}) bool { + name := key.(string) + db := val.(*DB) + + stats[name] = db.DB().Stats() + return true + }) + + return +} diff --git a/pkg/store/gorm/interceptor.go b/pkg/store/gorm/interceptor.go index 78be17a98b..e44f950636 100644 --- a/pkg/store/gorm/interceptor.go +++ b/pkg/store/gorm/interceptor.go @@ -1,8 +1,23 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package gorm import ( "context" "fmt" + "github.com/douyu/jupiter/pkg/metric" "strconv" "time" @@ -40,14 +55,19 @@ func metricInterceptor(dsn *DSN, op string, options *Config) func(Handler) Handl // error metric if scope.HasError() { + metric.LibHandleCounter.WithLabelValues(metric.TypeGorm, dsn.DBName+"."+scope.TableName(), dsn.Addr, "ERR").Inc() // todo sql语句，需要转换成脱密状态才能记录到日志 if scope.DB().Error != ErrRecordNotFound { options.logger.Error("mysql err", xlog.FieldErr(scope.DB().Error), xlog.FieldName(dsn.DBName+"."+scope.TableName()), xlog.FieldMethod(op)) } else { options.logger.Warn("record not found", xlog.FieldErr(scope.DB().Error), xlog.FieldName(dsn.DBName+"."+scope.TableName()), xlog.FieldMethod(op)) } + } else { + metric.LibHandleCounter.Inc(metric.TypeGorm, dsn.DBName+"."+scope.TableName(), dsn.Addr, "OK") } + metric.LibHandleHistogram.WithLabelValues(metric.TypeGorm, dsn.DBName+"."+scope.TableName(), dsn.Addr).Observe(cost.Seconds()) + if options.SlowThreshold > time.Duration(0) && options.SlowThreshold < cost { options.logger.Error( "slow", @@ -77,7 +97,7 @@ func traceInterceptor(dsn *DSN, op string, options *Config) func(Handler) Handle if ctx, ok := val.(context.Context); ok { span, _ := trace.StartSpanFromContext( ctx, - op, + "GORM", // TODO this op value is op or GORM trace.TagComponent("mysql"), trace.TagSpanKind("client"), ) @@ -90,7 +110,9 @@ func traceInterceptor(dsn *DSN, op string, options *Config) func(Handler) Handle span.SetTag("sql.addr", dsn.Addr) span.SetTag("span.kind", "client") span.SetTag("peer.service", "mysql") - span.LogFields(trace.String("sql.query", logSQL(scope.SQL, scope.SQLVars, options.DetailSQL))) + span.SetTag("db.instance", dsn.DBName) + span.SetTag("peer.address", dsn.Addr) + span.SetTag("peer.statement", logSQL(scope.SQL, scope.SQLVars, options.DetailSQL)) return } } diff --git a/pkg/store/gorm/logger.go b/pkg/store/gorm/logger.go index aaf9ea2a3e..2afb560ae1 100644 --- a/pkg/store/gorm/logger.go +++ b/pkg/store/gorm/logger.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package gorm import ( diff --git a/pkg/store/gorm/orm.go b/pkg/store/gorm/orm.go index 7332b9ad21..55b33c1f70 100644 --- a/pkg/store/gorm/orm.go +++ b/pkg/store/gorm/orm.go @@ -17,8 +17,6 @@ package gorm import ( "context" "errors" - "strings" - "github.com/douyu/jupiter/pkg/util/xdebug" "github.com/jinzhu/gorm" @@ -104,16 +102,11 @@ func Open(dialect string, options *Config) (*DB, error) { inner.LogMode(true) } - d, err := ParseDSN(options.DSN) - if err != nil { - return nil, err - } - replace := func(processor func() *gorm.CallbackProcessor, callbackName string, interceptors ...Interceptor) { old := processor().Get(callbackName) var handler = old for _, inte := range interceptors { - handler = inte(d, callbackName, options)(handler) + handler = inte(options.dsnCfg, callbackName, options)(handler) } processor().Replace(callbackName, handler) } @@ -146,17 +139,3 @@ func Open(dialect string, options *Config) (*DB, error) { return inner, err } - -// 收敛status，避免prometheus日志太多 -func getStatement(err string) string { - if !strings.HasPrefix(err, "Errord") { - return "Unknown" - } - slice := strings.Split(err, ":") - if len(slice) < 2 { - return "Unknown" - } - - // 收敛错误 - return slice[0] -} diff --git a/pkg/trace/carrier.go b/pkg/trace/carrier.go index 2fb0495601..14344ed5dc 100644 --- a/pkg/trace/carrier.go +++ b/pkg/trace/carrier.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package trace import ( diff --git a/pkg/trace/const.go b/pkg/trace/const.go index 526db2a636..5fef758f1c 100644 --- a/pkg/trace/const.go +++ b/pkg/trace/const.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package trace import ( diff --git a/pkg/trace/jaeger/config.go b/pkg/trace/jaeger/config.go index dc5a4bef4d..357c7f25ab 100644 --- a/pkg/trace/jaeger/config.go +++ b/pkg/trace/jaeger/config.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package jaeger import ( diff --git a/pkg/trace/trace.go b/pkg/trace/trace.go index b1c91979f7..eda027791b 100644 --- a/pkg/trace/trace.go +++ b/pkg/trace/trace.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package trace import ( diff --git a/pkg/trace/trace_example_test.go b/pkg/trace/trace_example_test.go index a8b01f6664..3b76469dcf 100644 --- a/pkg/trace/trace_example_test.go +++ b/pkg/trace/trace_example_test.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package trace_test import ( diff --git a/pkg/util/xattr/attr.go b/pkg/util/xattr/attr.go new file mode 100644 index 0000000000..6117aa7c1e --- /dev/null +++ b/pkg/util/xattr/attr.go @@ -0,0 +1,61 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xattr + +import ( + "errors" +) + +// Attributes ... +type Attributes struct { + m map[interface{}]interface{} +} + +var ( + // ErrInvalidKVPairs ... + ErrInvalidKVPairs = errors.New("invalid kv pairs") +) + +// New ... +func New(kvs ...interface{}) *Attributes { + if len(kvs)%2 != 0 { + panic(ErrInvalidKVPairs) + } + a := &Attributes{m: make(map[interface{}]interface{}, len(kvs)/2)} + for i := 0; i < len(kvs)/2; i++ { + a.m[kvs[i*2]] = kvs[i*2+1] + } + return a +} + +// WithValues ... +func (a *Attributes) WithValues(kvs ...interface{}) *Attributes { + if len(kvs)%2 != 0 { + panic(ErrInvalidKVPairs) + } + n := &Attributes{m: make(map[interface{}]interface{}, len(a.m)+len(kvs)/2)} + for k, v := range a.m { + n.m[k] = v + } + for i := 0; i < len(kvs)/2; i++ { + n.m[kvs[i*2]] = kvs[i*2+1] + } + return n +} + +// Value ... +func (a *Attributes) Value(key interface{}) interface{} { + return a.m[key] +} diff --git a/pkg/util/xattr/attr_test.go b/pkg/util/xattr/attr_test.go new file mode 100644 index 0000000000..ecc808a514 --- /dev/null +++ b/pkg/util/xattr/attr_test.go @@ -0,0 +1,37 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xattr + +import "testing" + +// New ... +func TestNew(t *testing.T) { + k1 := 1 + v1 := "first" + attr := New(k1, v1) + + ret1, ok1 := attr.Value(k1).(string) + if !ok1 || v1 != ret1 { + t.Fatalf("attr.Value error: want:%v ret:%v", v1, ret1) + } + + k2 := "2" + v2 := 2 + attr = attr.WithValues(k2, v2) + ret2, ok2 := attr.Value(k2).(int) + if !ok2 || v2 != ret2 { + t.Fatalf("attr.WithValues error: want:%v ret:%v", v2, ret2) + } +} diff --git a/pkg/util/xbuffer/unbounded.go b/pkg/util/xbuffer/unbounded.go new file mode 100644 index 0000000000..bb63dad335 --- /dev/null +++ b/pkg/util/xbuffer/unbounded.go @@ -0,0 +1,85 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package buffer provides an implementation of an unbounded buffer. +package xbuffer + +import "sync" + +// Unbounded is an implementation of an unbounded buffer which does not use +// extra goroutines. This is typically used for passing updates from one entity +// to another within gRPC. +// +// All methods on this type are thread-safe and don't block on anything except +// the underlying mutex used for synchronization. +// +// Unbounded supports values of any type to be stored in it by using a channel +// of `interface{}`. This means that a call to Put() incurs an extra memory +// allocation, and also that users need a type assertion while reading. For +// performance critical code paths, using Unbounded is strongly discouraged and +// defining a new type specific implementation of this buffer is preferred. See +// internal/transport/transport.go for an example of this. +type Unbounded struct { + c chan interface{} + mu sync.Mutex + backlog []interface{} +} + +// NewUnbounded returns a new instance of Unbounded. +func NewUnbounded() *Unbounded { + return &Unbounded{c: make(chan interface{}, 1)} +} + +// Put adds t to the unbounded buffer. +func (b *Unbounded) Put(t interface{}) { + b.mu.Lock() + if len(b.backlog) == 0 { + select { + case b.c <- t: + b.mu.Unlock() + return + default: + } + } + b.backlog = append(b.backlog, t) + b.mu.Unlock() +} + +// Load sends the earliest buffered data, if any, onto the read channel +// returned by Get(). Users are expected to call this every time they read a +// value from the read channel. +func (b *Unbounded) Load() { + b.mu.Lock() + if len(b.backlog) > 0 { + select { + case b.c <- b.backlog[0]: + b.backlog[0] = nil + b.backlog = b.backlog[1:] + default: + } + } + b.mu.Unlock() +} + +// Get returns a read channel on which values added to the buffer, via Put(), +// are sent on. +// +// Upon reading a value from this channel, users are expected to call Load() to +// send the next buffered value onto the channel if there is any. +func (b *Unbounded) Get() <-chan interface{} { + return b.c +} diff --git a/pkg/util/xbuffer/unbounded_test.go b/pkg/util/xbuffer/unbounded_test.go new file mode 100644 index 0000000000..4f6c15144f --- /dev/null +++ b/pkg/util/xbuffer/unbounded_test.go @@ -0,0 +1,111 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xbuffer + +import ( + "reflect" + "sort" + "sync" + "testing" +) + +const ( + numWriters = 10 + numWrites = 10 +) + +// wantReads contains the set of values expected to be read by the reader +// goroutine in the tests. +var wantReads []int + +func init() { + for i := 0; i < numWriters; i++ { + for j := 0; j < numWrites; j++ { + wantReads = append(wantReads, i) + } + } +} + +// TestSingleWriter starts one reader and one writer goroutine and makes sure +// that the reader gets all the value added to the buffer by the writer. +func TestSingleWriter(t *testing.T) { + ub := NewUnbounded() + reads := []int{} + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + ch := ub.Get() + for i := 0; i < numWriters*numWrites; i++ { + r := <-ch + reads = append(reads, r.(int)) + ub.Load() + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < numWriters; i++ { + for j := 0; j < numWrites; j++ { + ub.Put(i) + } + } + }() + + wg.Wait() + if !reflect.DeepEqual(reads, wantReads) { + t.Errorf("reads: %#v, wantReads: %#v", reads, wantReads) + } +} + +// TestMultipleWriters starts multiple writers and one reader goroutine and +// makes sure that the reader gets all the data written by all writers. +func TestMultipleWriters(t *testing.T) { + ub := NewUnbounded() + reads := []int{} + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + ch := ub.Get() + for i := 0; i < numWriters*numWrites; i++ { + r := <-ch + reads = append(reads, r.(int)) + ub.Load() + } + }() + + wg.Add(numWriters) + for i := 0; i < numWriters; i++ { + go func(index int) { + defer wg.Done() + for j := 0; j < numWrites; j++ { + ub.Put(index) + } + }(i) + } + + wg.Wait() + sort.Ints(reads) + if !reflect.DeepEqual(reads, wantReads) { + t.Errorf("reads: %#v, wantReads: %#v", reads, wantReads) + } +} diff --git a/pkg/util/xcast/interface.go b/pkg/util/xcast/interface.go index 9eefb8c6a3..736c82d2c4 100644 --- a/pkg/util/xcast/interface.go +++ b/pkg/util/xcast/interface.go @@ -517,7 +517,6 @@ func ToStringSliceE(i interface{}) ([]string, error) { } return a, nil case []string: - fmt.Println("[]string") return v, nil case string: return strings.Fields(v), nil diff --git a/pkg/util/xcycle/lifecycle.go b/pkg/util/xcycle/lifecycle.go index 49913deeeb..60d2249af6 100644 --- a/pkg/util/xcycle/lifecycle.go +++ b/pkg/util/xcycle/lifecycle.go @@ -21,23 +21,57 @@ import ( //Cycle .. type Cycle struct { - mu uint32 - sync.Once - wg *sync.WaitGroup - quit chan error + mu *sync.Mutex + wg *sync.WaitGroup + done chan struct{} + quit chan error + closeing uint32 + waiting uint32 + // works []func() error } //NewCycle new a cycle life func NewCycle() *Cycle { return &Cycle{ - mu: 0, - wg: &sync.WaitGroup{}, - quit: make(chan error), + mu: &sync.Mutex{}, + wg: &sync.WaitGroup{}, + done: make(chan struct{}), + quit: make(chan error), + closeing: 0, + waiting: 0, } } +/* +// Go .. +func (c *Cycle) Go(fns ...func() error) { + c.works = append(c.works, fns...) +} + +//RunWait .. +func (c *Cycle) RunWait() <-chan error { + go func(c *Cycle) { + for _, fn := range c.works { + c.wg.Add(1) + go func(c *Cycle, fn func() error) { + defer c.wg.Done() + if err := fn(); err != nil { + c.quit <- err + } + }(c, fn) + } + c.Wait() + close(c.quit) + }(c) + return c.quit +} +*/ + //Run a new goroutine func (c *Cycle) Run(fn func() error) { + c.mu.Lock() + //todo add check options panic before waiting + defer c.mu.Unlock() c.wg.Add(1) go func(c *Cycle) { defer c.wg.Done() @@ -45,16 +79,19 @@ func (c *Cycle) Run(fn func() error) { c.quit <- err } }(c) - } //Done block and return a chan error -func (c *Cycle) Done() <-chan error { - go func() { - c.wg.Wait() - c.Close() - }() - return c.quit +func (c *Cycle) Done() <-chan struct{} { + if atomic.CompareAndSwapUint32(&c.waiting, 0, 1) { + go func(c *Cycle) { + c.mu.Lock() + defer c.mu.Unlock() + c.wg.Wait() + close(c.done) + }(c) + } + return c.done } //DoneAndClose .. @@ -65,10 +102,10 @@ func (c *Cycle) DoneAndClose() { //Close .. func (c *Cycle) Close() { - if c.mu == 0 { - if atomic.CompareAndSwapUint32(&c.mu, 0, 1) { - close(c.quit) - } + c.mu.Lock() + defer c.mu.Unlock() + if atomic.CompareAndSwapUint32(&c.closeing, 0, 1) { + close(c.quit) } } diff --git a/pkg/util/xcycle/lifecycle_test.go b/pkg/util/xcycle/lifecycle_test.go new file mode 100644 index 0000000000..5536a63203 --- /dev/null +++ b/pkg/util/xcycle/lifecycle_test.go @@ -0,0 +1,101 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xcycle + +import ( + "fmt" + "testing" + "time" +) + +//TestCycleDone +func TestCycleDone(t *testing.T) { + state := "init" + c := NewCycle() + c.Run(func() error { + time.Sleep(time.Microsecond * 5) + return nil + }) + go func() { + select { + case <-c.Done(): + state = "done" + case <-time.After(time.Microsecond * 100): + state = "close" + } + c.Close() + }() + <-c.Wait() + + want := "done" + if state != want { + t.Errorf("TestCycleDone error want: %v, ret: %v\r\n", want, state) + } +} + +//TestCycleClose +func TestCycleClose(t *testing.T) { + state := "init" + c := NewCycle() + c.Run(func() error { + time.Sleep(time.Microsecond * 100) + return nil + }) + go func() { + select { + case <-c.Done(): + state = "done" + case <-time.After(time.Microsecond): + state = "close" + } + c.Close() + }() + <-c.Wait() + want := "close" + if state != want { + t.Errorf("TestCycleClose error want: %v, ret: %v\r\n", want, state) + } +} + +func TestCycleDoneAndClose(t *testing.T) { + ch := make(chan string, 2) + state := "init" + c := NewCycle() + c.Run(func() error { + time.Sleep(time.Microsecond * 100) + return nil + }) + go func() { + c.DoneAndClose() + ch <- "close" + }() + <-c.Wait() + want := "close" + state = <-ch + if state != want { + t.Errorf("TestCycleClose error want: %v, ret: %v\r\n", want, state) + } +} +func TestCycleWithError(t *testing.T) { + c := NewCycle() + c.Run(func() error { + return fmt.Errorf("run error") + }) + err := <-c.Wait() + want := fmt.Errorf("run error") + if err.Error() != want.Error() { + t.Errorf("TestCycleClose error want: %v, ret: %v\r\n", want.Error(), err.Error()) + } +} diff --git a/pkg/util/xdebug/testing.go b/pkg/util/xdebug/testing.go index bbe73e60ef..6b14ca00d7 100644 --- a/pkg/util/xdebug/testing.go +++ b/pkg/util/xdebug/testing.go @@ -30,6 +30,7 @@ var ( func init() { if isDevelopmentMode { xlog.DefaultLogger.SetLevel(xlog.DebugLevel) + xlog.JupiterLogger.SetLevel(xlog.DebugLevel) } } diff --git a/pkg/util/xdefer/stack.go b/pkg/util/xdefer/stack.go index 5f6249fc5f..ec0c526f11 100644 --- a/pkg/util/xdefer/stack.go +++ b/pkg/util/xdefer/stack.go @@ -1,6 +1,8 @@ package xdefer -import "sync" +import ( + "sync" +) func NewStack() *DeferStack { return &DeferStack{ @@ -23,7 +25,6 @@ func (ds *DeferStack) Push(fns ...func() error) { func (ds *DeferStack) Clean() { ds.mu.Lock() defer ds.mu.Unlock() - for i := len(ds.fns) - 1; i >= 0; i-- { _ = ds.fns[i]() } diff --git a/pkg/util/xdefer/stack_test.go b/pkg/util/xdefer/stack_test.go new file mode 100644 index 0000000000..97dc4b8e51 --- /dev/null +++ b/pkg/util/xdefer/stack_test.go @@ -0,0 +1,33 @@ +package xdefer + +import ( + "testing" +) + +func TestNewStack(t *testing.T) { + // stack := &DeferStack{ + // fns: make([]func() error, 0), + // mu: sync.RWMutex{}, + // } + stack := NewStack() + state := "" + fn1 := func() error { + state = state + "1" + return nil + } + fn2 := func() error { + state = state + "2" + return nil + } + fn3 := func() error { + state = state + "3" + return nil + } + stack.Push(fn1, fn2) + stack.Push(fn3) + stack.Clean() + want := "321" + if state != want { + t.Fatalf("Stack has error,want:%v ret:%v", want, state) + } +} diff --git a/pkg/util/xmap/map.go b/pkg/util/xmap/map.go index ee0bd64ea9..40047e509a 100644 --- a/pkg/util/xmap/map.go +++ b/pkg/util/xmap/map.go @@ -18,93 +18,226 @@ import ( "fmt" "reflect" "strings" + "sync" + "time" "github.com/douyu/jupiter/pkg/util/xcast" + "github.com/mitchellh/mapstructure" ) -// MergeStringMap merge two map -func MergeStringMap(dest, src map[string]interface{}) { - for sk, sv := range src { - tv, ok := dest[sk] - if !ok { - // val不存在时，直接赋值 - dest[sk] = sv - continue - } +// Unmarshaller ... +type Unmarshaller = func([]byte, interface{}) error - svType := reflect.TypeOf(sv) - tvType := reflect.TypeOf(tv) - if svType != tvType { - fmt.Println("continue, type is different") - continue - } +// KeySpliter ... +var KeySpliter = "." - switch ttv := tv.(type) { - case map[interface{}]interface{}: - tsv := sv.(map[interface{}]interface{}) - ssv := ToMapStringInterface(tsv) - stv := ToMapStringInterface(ttv) - MergeStringMap(stv, ssv) - dest[sk] = stv - case map[string]interface{}: - MergeStringMap(ttv, sv.(map[string]interface{})) - dest[sk] = ttv - default: - dest[sk] = sv - } +// FlatMap ... +type FlatMap struct { + data map[string]interface{} + mu sync.RWMutex + keyMap sync.Map +} + +// NewFlatMap ... +func NewFlatMap() *FlatMap { + return &FlatMap{ + data: make(map[string]interface{}), } } -// ToMapStringInterface cast map[interface{}]interface{} to map[string]interface{} -func ToMapStringInterface(src map[interface{}]interface{}) map[string]interface{} { - tgt := map[string]interface{}{} - for k, v := range src { - tgt[fmt.Sprintf("%v", k)] = v +// Load ... +func (flat *FlatMap) Load(content []byte, unmarshal Unmarshaller) error { + data := make(map[string]interface{}) + if err := unmarshal(content, &data); err != nil { + return err } - return tgt + return flat.apply(data) } -// InsensitiviseMap insensitivise map -func InsensitiviseMap(m map[string]interface{}) { - for key, val := range m { - switch v := val.(type) { - case map[interface{}]interface{}: - InsensitiviseMap(xcast.ToStringMap(v)) - case map[string]interface{}: - InsensitiviseMap(v) - } +func (flat *FlatMap) apply(data map[string]interface{}) error { + flat.mu.Lock() + defer flat.mu.Unlock() - lower := strings.ToLower(key) - if key != lower { - delete(m, key) + MergeStringMap(flat.data, data) + var changes = make(map[string]interface{}) + for k, v := range flat.traverse(KeySpliter) { + orig, ok := flat.keyMap.Load(k) + if ok && !reflect.DeepEqual(orig, v) { + changes[k] = v } - m[lower] = val + flat.keyMap.Store(k, v) } + + return nil +} + +// Set ... +func (flat *FlatMap) Set(key string, val interface{}) error { + paths := strings.Split(key, KeySpliter) + lastKey := paths[len(paths)-1] + m := deepSearch(flat.data, paths[:len(paths)-1]) + m[lastKey] = val + return flat.apply(m) +} + +// Get returns the value associated with the key +func (flat *FlatMap) Get(key string) interface{} { + return flat.find(key) +} + +// GetString returns the value associated with the key as a string. +func (flat *FlatMap) GetString(key string) string { + return xcast.ToString(flat.Get(key)) +} + +// GetBool returns the value associated with the key as a boolean. +func (flat *FlatMap) GetBool(key string) bool { + return xcast.ToBool(flat.Get(key)) +} + +// GetInt returns the value associated with the key as an integer. +func (flat *FlatMap) GetInt(key string) int { + return xcast.ToInt(flat.Get(key)) +} + +// GetInt64 returns the value associated with the key as an integer. +func (flat *FlatMap) GetInt64(key string) int64 { + return xcast.ToInt64(flat.Get(key)) +} + +// GetFloat64 returns the value associated with the key as a float64. +func (flat *FlatMap) GetFloat64(key string) float64 { + return xcast.ToFloat64(flat.Get(key)) +} + +// GetTime returns the value associated with the key as time. +func (flat *FlatMap) GetTime(key string) time.Time { + return xcast.ToTime(flat.Get(key)) +} + +// GetDuration returns the value associated with the key as a duration. +func (flat *FlatMap) GetDuration(key string) time.Duration { + return xcast.ToDuration(flat.Get(key)) +} + +// GetStringSlice returns the value associated with the key as a slice of strings. +func (flat *FlatMap) GetStringSlice(key string) []string { + return xcast.ToStringSlice(flat.Get(key)) +} + +// GetSlice returns the value associated with the key as a slice of strings. +func (flat *FlatMap) GetSlice(key string) []interface{} { + return xcast.ToSlice(flat.Get(key)) } -// DeepSearchInMap deep search in map -func DeepSearchInMap(m map[string]interface{}, paths ...string) map[string]interface{} { - //深度拷贝 - mtmp := make(map[string]interface{}) - for k, v := range m { - mtmp[k] = v +// GetStringMap returns the value associated with the key as a map of interfaces. +func (flat *FlatMap) GetStringMap(key string) map[string]interface{} { + return xcast.ToStringMap(flat.Get(key)) +} + +// GetStringMapString returns the value associated with the key as a map of strings. +func (flat *FlatMap) GetStringMapString(key string) map[string]string { + return xcast.ToStringMapString(flat.Get(key)) +} + +// GetSliceStringMap returns the value associated with the slice of maps. +func (flat *FlatMap) GetSliceStringMap(key string) []map[string]interface{} { + return xcast.ToSliceStringMap(flat.Get(key)) +} + +// GetStringMapStringSlice returns the value associated with the key as a map to a slice of strings. +func (flat *FlatMap) GetStringMapStringSlice(key string) map[string][]string { + return xcast.ToStringMapStringSlice(flat.Get(key)) +} + +// UnmarshalKey takes a single key and unmarshal it into a Struct. +func (flat *FlatMap) UnmarshalKey(key string, rawVal interface{}, tagName string) error { + config := mapstructure.DecoderConfig{ + DecodeHook: mapstructure.StringToTimeDurationHookFunc(), + Result: rawVal, + TagName: tagName, } - for _, k := range paths { - m2, ok := mtmp[k] + decoder, err := mapstructure.NewDecoder(&config) + if err != nil { + return err + } + if key == "" { + flat.mu.RLock() + defer flat.mu.RUnlock() + return decoder.Decode(flat.data) + } + + value := flat.Get(key) + if value == nil { + return fmt.Errorf("invalid key %s, maybe not exist in config", key) + } + + return decoder.Decode(value) +} + +// Reset ... +func (flat *FlatMap) Reset() { + flat.mu.Lock() + defer flat.mu.Unlock() + + flat.data = make(map[string]interface{}) + // erase map + flat.keyMap.Range(func(key interface{}, value interface{}) bool { + flat.keyMap.Delete(key) + return true + }) +} + +func (flat *FlatMap) find(key string) interface{} { + dd, ok := flat.keyMap.Load(key) + if ok { + return dd + } + + paths := strings.Split(key, KeySpliter) + flat.mu.RLock() + defer flat.mu.RUnlock() + m := DeepSearchInMap(flat.data, paths[:len(paths)-1]...) + dd = m[paths[len(paths)-1]] + flat.keyMap.Store(key, dd) + return dd +} + +func lookup(prefix string, target map[string]interface{}, data map[string]interface{}, sep string) { + for k, v := range target { + pp := fmt.Sprintf("%s%s%s", prefix, sep, k) + if prefix == "" { + pp = fmt.Sprintf("%s", k) + } + if dd, err := xcast.ToStringMapE(v); err == nil { + lookup(pp, dd, data, sep) + } else { + data[pp] = v + } + } +} + +func (flat *FlatMap) traverse(sep string) map[string]interface{} { + data := make(map[string]interface{}) + lookup("", flat.data, data, sep) + return data +} + +func deepSearch(m map[string]interface{}, path []string) map[string]interface{} { + for _, k := range path { + m2, ok := m[k] if !ok { m3 := make(map[string]interface{}) - mtmp[k] = m3 - mtmp = m3 + m[k] = m3 + m = m3 continue } - - m3, err := xcast.ToStringMapE(m2) - if err != nil { + m3, ok := m2.(map[string]interface{}) + if !ok { m3 = make(map[string]interface{}) - mtmp[k] = m3 + m[k] = m3 } - // continue search - mtmp = m3 + m = m3 } - return mtmp + return m } diff --git a/pkg/util/xmap/util.go b/pkg/util/xmap/util.go new file mode 100644 index 0000000000..6e7d420a16 --- /dev/null +++ b/pkg/util/xmap/util.go @@ -0,0 +1,110 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xmap + +import ( + "fmt" + "reflect" + "strings" + + "github.com/douyu/jupiter/pkg/util/xcast" +) + +// MergeStringMap merge two map +func MergeStringMap(dest, src map[string]interface{}) { + for sk, sv := range src { + tv, ok := dest[sk] + if !ok { + // val不存在时，直接赋值 + dest[sk] = sv + continue + } + + svType := reflect.TypeOf(sv) + tvType := reflect.TypeOf(tv) + if svType != tvType { + fmt.Println("continue, type is different") + continue + } + + switch ttv := tv.(type) { + case map[interface{}]interface{}: + tsv := sv.(map[interface{}]interface{}) + ssv := ToMapStringInterface(tsv) + stv := ToMapStringInterface(ttv) + MergeStringMap(stv, ssv) + dest[sk] = stv + case map[string]interface{}: + MergeStringMap(ttv, sv.(map[string]interface{})) + dest[sk] = ttv + default: + dest[sk] = sv + } + } +} + +// ToMapStringInterface cast map[interface{}]interface{} to map[string]interface{} +func ToMapStringInterface(src map[interface{}]interface{}) map[string]interface{} { + tgt := map[string]interface{}{} + for k, v := range src { + tgt[fmt.Sprintf("%v", k)] = v + } + return tgt +} + +// InsensitiviseMap insensitivise map +func InsensitiviseMap(m map[string]interface{}) { + for key, val := range m { + switch val.(type) { + case map[interface{}]interface{}: + InsensitiviseMap(xcast.ToStringMap(val)) + case map[string]interface{}: + InsensitiviseMap(val.(map[string]interface{})) + } + + lower := strings.ToLower(key) + if key != lower { + delete(m, key) + } + m[lower] = val + } +} + +// DeepSearchInMap deep search in map +func DeepSearchInMap(m map[string]interface{}, paths ...string) map[string]interface{} { + //深度拷贝 + mtmp := make(map[string]interface{}) + for k, v := range m { + mtmp[k] = v + } + for _, k := range paths { + m2, ok := mtmp[k] + if !ok { + m3 := make(map[string]interface{}) + mtmp[k] = m3 + mtmp = m3 + continue + } + + m3, err := xcast.ToStringMapE(m2) + if err != nil { + m3 = make(map[string]interface{}) + mtmp[k] = m3 + } + // continue search + mtmp = m3 + } + return mtmp +} diff --git a/pkg/util/xmap/map_test.go b/pkg/util/xmap/util_test.go similarity index 100% rename from pkg/util/xmap/map_test.go rename to pkg/util/xmap/util_test.go diff --git a/pkg/util/xnet/ip.go b/pkg/util/xnet/ip.go index f98a6157bc..ff21260f2f 100644 --- a/pkg/util/xnet/ip.go +++ b/pkg/util/xnet/ip.go @@ -15,6 +15,7 @@ package xnet import ( + "errors" "fmt" "log" "net" @@ -34,7 +35,7 @@ func GetLocalIP() (string, error) { } } } - panic("unable to determine locla ip") + return "", errors.New("unable to determine locla ip") } // GetMacAddrs ... diff --git a/pkg/util/xnet/url.go b/pkg/util/xnet/url.go index 3dc5ccd898..0ef6df08ea 100644 --- a/pkg/util/xnet/url.go +++ b/pkg/util/xnet/url.go @@ -16,13 +16,25 @@ package xnet import ( "net/url" - "strconv" "time" + + "github.com/douyu/jupiter/pkg/util/xcast" ) // URL wrap url.URL. type URL struct { - url.URL + Scheme string + Opaque string // encoded opaque data + User *url.Userinfo // username and password information + Host string // host or host:port + Path string // path (relative paths may omit leading slash) + RawPath string // encoded path hint (see EscapedPath method) + ForceQuery bool // append a query ('?') even if RawQuery is empty + RawQuery string // encoded query values, without '?' + Fragment string // fragment for references, without '#' + HostName string + Port string + params url.Values } // ParseURL parses raw into URL. @@ -33,7 +45,18 @@ func ParseURL(raw string) (*URL, error) { } return &URL{ - URL: *u, + Scheme: u.Scheme, + Opaque: u.Opaque, + User: u.User, + Host: u.Host, + Path: u.Path, + RawPath: u.RawPath, + ForceQuery: u.ForceQuery, + RawQuery: u.RawQuery, + Fragment: u.Fragment, + HostName: u.Hostname(), + Port: u.Port(), + params: u.Query(), }, nil } @@ -51,34 +74,29 @@ func (u *URL) Username() string { } // QueryInt returns provided field's value in int type. +// if value is empty, expect returns func (u *URL) QueryInt(field string, expect int) (ret int) { - ret = expect - if mi := u.Query().Get(field); mi != "" { - if m, e := strconv.Atoi(mi); e == nil { - if m > 0 { - ret = m - } - } + ret, err := xcast.ToIntE(u.Query().Get(field)) + if err != nil { + return expect } - return + return ret } // QueryInt64 returns provided field's value in int64 type. +// if value is empty, expect returns func (u *URL) QueryInt64(field string, expect int64) (ret int64) { - ret = expect - if mi := u.Query().Get(field); mi != "" { - if m, e := strconv.ParseInt(mi, 10, 64); e == nil { - if m > 0 { - ret = m - } - } + ret, err := xcast.ToInt64E(u.Query().Get(field)) + if err != nil { + return expect } - return + return ret } // QueryString returns provided field's value in string type. +// if value is empty, expect returns func (u *URL) QueryString(field string, expect string) (ret string) { ret = expect if mi := u.Query().Get(field); mi != "" { @@ -88,22 +106,31 @@ func (u *URL) QueryString(field string, expect string) (ret string) { return } -// QuerySecond returns provided field's value in duration type. -// Deprecated: use QueryDuration instead. -func (u *URL) QuerySecond(field string, expect int64) (ret time.Duration) { - return u.QueryDuration(field, expect) +// QueryDuration returns provided field's value in duration type. +// if value is empty, expect returns +func (u *URL) QueryDuration(field string, expect time.Duration) (ret time.Duration) { + ret, err := xcast.ToDurationE(u.Query().Get(field)) + if err != nil { + return expect + } + + return ret } -// QueryDuration returns provided field's value in duration type. -func (u *URL) QueryDuration(field string, expect int64) (ret time.Duration) { - ret = time.Duration(expect) - if mi := u.Query().Get(field); mi != "" { - if m, e := strconv.ParseInt(mi, 10, 64); e == nil { - if m > 0 { - ret = time.Duration(m) - } - } +// QueryBool returns provided field's value in bool +// if value is empty, expect returns +func (u *URL) QueryBool(field string, expect bool) (ret bool) { + ret, err := xcast.ToBoolE(u.Query().Get(field)) + if err != nil { + return expect } + return ret +} - return +// Query parses RawQuery and returns the corresponding values. +// It silently discards malformed value pairs. +// To check errors use ParseQuery. +func (u *URL) Query() url.Values { + v, _ := url.ParseQuery(u.RawQuery) + return v } diff --git a/pkg/util/xtest/proto/testproto/hello.pb.go b/pkg/util/xtest/proto/testproto/hello.pb.go new file mode 100644 index 0000000000..f29f90791a --- /dev/null +++ b/pkg/util/xtest/proto/testproto/hello.pb.go @@ -0,0 +1,427 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: hello.proto + +package testproto + +import ( + context "context" + fmt "fmt" + proto "github.com/golang/protobuf/proto" + grpc "google.golang.org/grpc" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +// The request message containing the user's name. +type HelloRequest struct { + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *HelloRequest) Reset() { *m = HelloRequest{} } +func (m *HelloRequest) String() string { return proto.CompactTextString(m) } +func (*HelloRequest) ProtoMessage() {} +func (*HelloRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_61ef911816e0a8ce, []int{0} +} + +func (m *HelloRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_HelloRequest.Unmarshal(m, b) +} +func (m *HelloRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_HelloRequest.Marshal(b, m, deterministic) +} +func (m *HelloRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_HelloRequest.Merge(m, src) +} +func (m *HelloRequest) XXX_Size() int { + return xxx_messageInfo_HelloRequest.Size(m) +} +func (m *HelloRequest) XXX_DiscardUnknown() { + xxx_messageInfo_HelloRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_HelloRequest proto.InternalMessageInfo + +func (m *HelloRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +type WhoServerReq struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *WhoServerReq) Reset() { *m = WhoServerReq{} } +func (m *WhoServerReq) String() string { return proto.CompactTextString(m) } +func (*WhoServerReq) ProtoMessage() {} +func (*WhoServerReq) Descriptor() ([]byte, []int) { + return fileDescriptor_61ef911816e0a8ce, []int{1} +} + +func (m *WhoServerReq) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_WhoServerReq.Unmarshal(m, b) +} +func (m *WhoServerReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_WhoServerReq.Marshal(b, m, deterministic) +} +func (m *WhoServerReq) XXX_Merge(src proto.Message) { + xxx_messageInfo_WhoServerReq.Merge(m, src) +} +func (m *WhoServerReq) XXX_Size() int { + return xxx_messageInfo_WhoServerReq.Size(m) +} +func (m *WhoServerReq) XXX_DiscardUnknown() { + xxx_messageInfo_WhoServerReq.DiscardUnknown(m) +} + +var xxx_messageInfo_WhoServerReq proto.InternalMessageInfo + +type WhoServerReply struct { + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *WhoServerReply) Reset() { *m = WhoServerReply{} } +func (m *WhoServerReply) String() string { return proto.CompactTextString(m) } +func (*WhoServerReply) ProtoMessage() {} +func (*WhoServerReply) Descriptor() ([]byte, []int) { + return fileDescriptor_61ef911816e0a8ce, []int{2} +} + +func (m *WhoServerReply) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_WhoServerReply.Unmarshal(m, b) +} +func (m *WhoServerReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_WhoServerReply.Marshal(b, m, deterministic) +} +func (m *WhoServerReply) XXX_Merge(src proto.Message) { + xxx_messageInfo_WhoServerReply.Merge(m, src) +} +func (m *WhoServerReply) XXX_Size() int { + return xxx_messageInfo_WhoServerReply.Size(m) +} +func (m *WhoServerReply) XXX_DiscardUnknown() { + xxx_messageInfo_WhoServerReply.DiscardUnknown(m) +} + +var xxx_messageInfo_WhoServerReply proto.InternalMessageInfo + +func (m *WhoServerReply) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +// The response message containing the greetings +type HelloReply struct { + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` + Id64 int64 `protobuf:"varint,2,opt,name=id64,proto3" json:"id64,omitempty"` + Id32 int32 `protobuf:"varint,3,opt,name=id32,proto3" json:"id32,omitempty"` + Idu64 uint64 `protobuf:"varint,4,opt,name=idu64,proto3" json:"idu64,omitempty"` + Idu32 uint32 `protobuf:"varint,5,opt,name=idu32,proto3" json:"idu32,omitempty"` + Name []byte `protobuf:"bytes,6,opt,name=name,proto3" json:"name,omitempty"` + Done bool `protobuf:"varint,7,opt,name=done,proto3" json:"done,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *HelloReply) Reset() { *m = HelloReply{} } +func (m *HelloReply) String() string { return proto.CompactTextString(m) } +func (*HelloReply) ProtoMessage() {} +func (*HelloReply) Descriptor() ([]byte, []int) { + return fileDescriptor_61ef911816e0a8ce, []int{3} +} + +func (m *HelloReply) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_HelloReply.Unmarshal(m, b) +} +func (m *HelloReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_HelloReply.Marshal(b, m, deterministic) +} +func (m *HelloReply) XXX_Merge(src proto.Message) { + xxx_messageInfo_HelloReply.Merge(m, src) +} +func (m *HelloReply) XXX_Size() int { + return xxx_messageInfo_HelloReply.Size(m) +} +func (m *HelloReply) XXX_DiscardUnknown() { + xxx_messageInfo_HelloReply.DiscardUnknown(m) +} + +var xxx_messageInfo_HelloReply proto.InternalMessageInfo + +func (m *HelloReply) GetMessage() string { + if m != nil { + return m.Message + } + return "" +} + +func (m *HelloReply) GetId64() int64 { + if m != nil { + return m.Id64 + } + return 0 +} + +func (m *HelloReply) GetId32() int32 { + if m != nil { + return m.Id32 + } + return 0 +} + +func (m *HelloReply) GetIdu64() uint64 { + if m != nil { + return m.Idu64 + } + return 0 +} + +func (m *HelloReply) GetIdu32() uint32 { + if m != nil { + return m.Idu32 + } + return 0 +} + +func (m *HelloReply) GetName() []byte { + if m != nil { + return m.Name + } + return nil +} + +func (m *HelloReply) GetDone() bool { + if m != nil { + return m.Done + } + return false +} + +func init() { + proto.RegisterType((*HelloRequest)(nil), "testproto.HelloRequest") + proto.RegisterType((*WhoServerReq)(nil), "testproto.WhoServerReq") + proto.RegisterType((*WhoServerReply)(nil), "testproto.WhoServerReply") + proto.RegisterType((*HelloReply)(nil), "testproto.HelloReply") +} + +func init() { proto.RegisterFile("hello.proto", fileDescriptor_61ef911816e0a8ce) } + +var fileDescriptor_61ef911816e0a8ce = []byte{ + // 309 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x92, 0xcd, 0x4e, 0x02, 0x31, + 0x10, 0xc7, 0x1d, 0xf9, 0x1e, 0x10, 0x93, 0x46, 0x63, 0xc5, 0x4b, 0xb3, 0xa7, 0xc6, 0xc3, 0x86, + 0x00, 0xe1, 0xe4, 0x45, 0x3c, 0xe8, 0x91, 0x94, 0x03, 0xe7, 0xd5, 0x9d, 0x00, 0x49, 0x97, 0x2e, + 0xdd, 0xa2, 0xf2, 0x3a, 0xbe, 0x90, 0xaf, 0x64, 0xb6, 0x0b, 0x64, 0x8d, 0xc6, 0x83, 0xb7, 0xdf, + 0x7f, 0xa6, 0xf3, 0xf5, 0x4f, 0xb1, 0xbd, 0x24, 0xad, 0x4d, 0x98, 0x5a, 0xe3, 0x0c, 0x6b, 0x39, + 0xca, 0x9c, 0xc7, 0x20, 0xc0, 0xce, 0x53, 0x9e, 0x51, 0xb4, 0xd9, 0x52, 0xe6, 0x18, 0xc3, 0xea, + 0x3a, 0x4a, 0x88, 0x83, 0x00, 0xd9, 0x52, 0x9e, 0x83, 0x2e, 0x76, 0xe6, 0x4b, 0x33, 0x23, 0xfb, + 0x4a, 0x56, 0xd1, 0x26, 0xb8, 0xc5, 0x6e, 0x49, 0xa7, 0x7a, 0xc7, 0x38, 0x36, 0x12, 0xca, 0xb2, + 0x68, 0x71, 0x28, 0x3c, 0xc8, 0xe0, 0x03, 0x10, 0xf7, 0x03, 0xfe, 0x7c, 0x98, 0x0f, 0x5e, 0xc5, + 0xe3, 0x11, 0x3f, 0x15, 0x20, 0x2b, 0xca, 0x73, 0x11, 0x1b, 0x0e, 0x78, 0x45, 0x80, 0xac, 0x29, + 0xcf, 0xec, 0x02, 0x6b, 0xab, 0x78, 0x3b, 0x1e, 0xf1, 0xaa, 0x00, 0x59, 0x55, 0x85, 0xd8, 0x47, + 0x87, 0x03, 0x5e, 0x13, 0x20, 0xcf, 0x54, 0x21, 0x8e, 0xc7, 0xd4, 0x05, 0xc8, 0x4e, 0x71, 0x4c, + 0x1e, 0x8b, 0xcd, 0x9a, 0x78, 0x43, 0x80, 0x6c, 0x2a, 0xcf, 0x83, 0x4f, 0xc0, 0xc6, 0xa3, 0x25, + 0x72, 0x64, 0xd9, 0x1d, 0x36, 0x67, 0xd1, 0xce, 0xaf, 0xcc, 0xae, 0xc2, 0xa3, 0x51, 0x61, 0xd9, + 0xa5, 0xde, 0xe5, 0xcf, 0x44, 0xaa, 0x77, 0xc1, 0x09, 0xbb, 0xc7, 0xd6, 0xd1, 0x9a, 0x6f, 0xe5, + 0x65, 0x03, 0x7b, 0xd7, 0xbf, 0x27, 0x8a, 0x16, 0x0f, 0xd8, 0x9e, 0x39, 0x4b, 0x51, 0xf2, 0xcf, + 0x1d, 0x24, 0xf4, 0x61, 0xd2, 0xc7, 0x9b, 0x95, 0x09, 0x17, 0x36, 0x7d, 0x09, 0xe9, 0x3d, 0x4a, + 0x52, 0x4d, 0x59, 0xe8, 0x7f, 0xc0, 0x9b, 0xb1, 0x3a, 0x9e, 0x9c, 0xfb, 0x82, 0x79, 0xce, 0xd3, + 0xbc, 0xc1, 0x14, 0x9e, 0xeb, 0xbe, 0xd3, 0xf0, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x83, 0xf5, 0x17, + 0xec, 0x29, 0x02, 0x00, 0x00, +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// GreeterClient is the client API for Greeter service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type GreeterClient interface { + // Sends a greeting + SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error) + WhoServer(ctx context.Context, in *WhoServerReq, opts ...grpc.CallOption) (*WhoServerReply, error) + StreamHello(ctx context.Context, opts ...grpc.CallOption) (Greeter_StreamHelloClient, error) +} + +type greeterClient struct { + cc *grpc.ClientConn +} + +func NewGreeterClient(cc *grpc.ClientConn) GreeterClient { + return &greeterClient{cc} +} + +func (c *greeterClient) SayHello(ctx context.Context, in *HelloRequest, opts ...grpc.CallOption) (*HelloReply, error) { + out := new(HelloReply) + err := c.cc.Invoke(ctx, "/testproto.Greeter/SayHello", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *greeterClient) WhoServer(ctx context.Context, in *WhoServerReq, opts ...grpc.CallOption) (*WhoServerReply, error) { + out := new(WhoServerReply) + err := c.cc.Invoke(ctx, "/testproto.Greeter/WhoServer", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *greeterClient) StreamHello(ctx context.Context, opts ...grpc.CallOption) (Greeter_StreamHelloClient, error) { + stream, err := c.cc.NewStream(ctx, &_Greeter_serviceDesc.Streams[0], "/testproto.Greeter/StreamHello", opts...) + if err != nil { + return nil, err + } + x := &greeterStreamHelloClient{stream} + return x, nil +} + +type Greeter_StreamHelloClient interface { + Send(*HelloRequest) error + Recv() (*HelloReply, error) + grpc.ClientStream +} + +type greeterStreamHelloClient struct { + grpc.ClientStream +} + +func (x *greeterStreamHelloClient) Send(m *HelloRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *greeterStreamHelloClient) Recv() (*HelloReply, error) { + m := new(HelloReply) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// GreeterServer is the server API for Greeter service. +type GreeterServer interface { + // Sends a greeting + SayHello(context.Context, *HelloRequest) (*HelloReply, error) + WhoServer(context.Context, *WhoServerReq) (*WhoServerReply, error) + StreamHello(Greeter_StreamHelloServer) error +} + +func RegisterGreeterServer(s *grpc.Server, srv GreeterServer) { + s.RegisterService(&_Greeter_serviceDesc, srv) +} + +func _Greeter_SayHello_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(HelloRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GreeterServer).SayHello(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/testproto.Greeter/SayHello", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GreeterServer).SayHello(ctx, req.(*HelloRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Greeter_WhoServer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(WhoServerReq) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(GreeterServer).WhoServer(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/testproto.Greeter/WhoServer", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(GreeterServer).WhoServer(ctx, req.(*WhoServerReq)) + } + return interceptor(ctx, in, info, handler) +} + +func _Greeter_StreamHello_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(GreeterServer).StreamHello(&greeterStreamHelloServer{stream}) +} + +type Greeter_StreamHelloServer interface { + Send(*HelloReply) error + Recv() (*HelloRequest, error) + grpc.ServerStream +} + +type greeterStreamHelloServer struct { + grpc.ServerStream +} + +func (x *greeterStreamHelloServer) Send(m *HelloReply) error { + return x.ServerStream.SendMsg(m) +} + +func (x *greeterStreamHelloServer) Recv() (*HelloRequest, error) { + m := new(HelloRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +var _Greeter_serviceDesc = grpc.ServiceDesc{ + ServiceName: "testproto.Greeter", + HandlerType: (*GreeterServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "SayHello", + Handler: _Greeter_SayHello_Handler, + }, + { + MethodName: "WhoServer", + Handler: _Greeter_WhoServer_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamHello", + Handler: _Greeter_StreamHello_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "hello.proto", +} diff --git a/pkg/util/xtest/proto/testproto/hello.proto b/pkg/util/xtest/proto/testproto/hello.proto new file mode 100644 index 0000000000..c15867a8e3 --- /dev/null +++ b/pkg/util/xtest/proto/testproto/hello.proto @@ -0,0 +1,50 @@ +// Copyright 2015 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "io.grpc.examples.helloworld"; +option java_outer_classname = "HelloWorldProto"; + +package testproto; + +// The greeting service definition. +service Greeter { + // Sends a greeting + rpc SayHello (HelloRequest) returns (HelloReply) {} + rpc WhoServer (WhoServerReq) returns (WhoServerReply) {} + rpc StreamHello (stream HelloRequest) returns (stream HelloReply){} +} + +// The request message containing the user's name. +message HelloRequest { + string name = 1; +} + +message WhoServerReq{ +} +message WhoServerReply{ + string message = 1; +} +// The response message containing the greetings +message HelloReply { + string message = 1; + int64 id64 = 2; + int32 id32 = 3; + uint64 idu64 = 4; + uint32 idu32 = 5; + bytes name = 6; + bool done = 7; +} \ No newline at end of file diff --git a/pkg/util/xtest/server/yell/server.go b/pkg/util/xtest/server/yell/server.go new file mode 100644 index 0000000000..f5c5ca017d --- /dev/null +++ b/pkg/util/xtest/server/yell/server.go @@ -0,0 +1,80 @@ +package yell + +import ( + "context" + "errors" + "github.com/douyu/jupiter/pkg/util/xtest/proto/testproto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "time" +) + +// FooServer ... +type FooServer struct { + name string + hook func(context.Context) +} + +// SetName ... +func (s *FooServer) SetName(f string) { + s.name = f +} + +// SetHook ... +func (s *FooServer) SetHook(f func(context.Context)) { + s.hook = f +} + +// ErrFoo ... +var ErrFoo = errors.New("error foo") + +// RespFantasy ... +var RespFantasy = &testproto.HelloReply{Message: "fantasy"} + +// RespBye ... +var RespBye = &testproto.HelloReply{Message: "bye"} + +// StatusFoo ... +var StatusFoo = status.Errorf(codes.DataLoss, ErrFoo.Error()) + +// SayHello ... +func (s *FooServer) SayHello(ctx context.Context, in *testproto.HelloRequest) (out *testproto.HelloReply, err error) { + // sleep to test cost time + time.Sleep(20 * time.Millisecond) + switch in.Name { + case "traceHook": + s.hook(ctx) + err = StatusFoo + case "needErr": + err = StatusFoo + case "slow": + time.Sleep(500 * time.Millisecond) + out = RespFantasy + case "needPanic": + panic("go dead!") + default: + out = RespFantasy + } + return +} + +// StreamHello ... +func (s *FooServer) StreamHello(ss testproto.Greeter_StreamHelloServer) (err error) { + + for { + in, _ := ss.Recv() + switch in.Name { + case "bye": + return ss.Send(RespBye) + case "needErr": + return StatusFoo + default: + return ss.Send(RespFantasy) + } + } +} + +// StreamHello ... +func (s *FooServer) WhoServer(ctx context.Context, in *testproto.WhoServerReq) (out *testproto.WhoServerReply, err error) { + return &testproto.WhoServerReply{Message: s.name}, nil +} diff --git a/pkg/util/xtime/api.go b/pkg/util/xtime/api.go index 4071cc67b1..70ac4f1491 100644 --- a/pkg/util/xtime/api.go +++ b/pkg/util/xtime/api.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package xtime import "time" diff --git a/pkg/util/xtime/time.go b/pkg/util/xtime/time.go index a2fed3077d..4d0888a0b1 100644 --- a/pkg/util/xtime/time.go +++ b/pkg/util/xtime/time.go @@ -113,3 +113,11 @@ func (t *Time) EndOfMinute() *Time { y, m, d := t.Date() return &Time{time.Date(y, m, d, t.Hour(), t.Minute(), 59, int(time.Second-time.Nanosecond), t.Location())} } + +var TS TimeFormat = "2006-01-02 15:04:05" + +type TimeFormat string + +func (ts TimeFormat) Format(t time.Time) string { + return t.Format(string(ts)) +} diff --git a/pkg/worker/xcron/config.go b/pkg/worker/xcron/config.go index af45a64621..d25a7869a5 100644 --- a/pkg/worker/xcron/config.go +++ b/pkg/worker/xcron/config.go @@ -19,6 +19,10 @@ import ( "runtime" "time" + "github.com/coreos/etcd/clientv3/concurrency" + "github.com/douyu/jupiter/pkg/client/etcdv3" + "github.com/douyu/jupiter/pkg/ecode" + "github.com/douyu/jupiter/pkg/metric" "go.uber.org/zap" @@ -39,13 +43,17 @@ func RawConfig(key string) Config { xlog.Panic("unmarshal", xlog.String("key", key)) } + if config.DistributedTask { + config.Config = etcdv3.RawConfig(key) + } + return config } // DefaultConfig ... func DefaultConfig() Config { return Config{ - logger: xlog.DefaultLogger, + logger: xlog.JupiterLogger, wrappers: []JobWrapper{}, WithSeconds: false, ImmediatelyRun: false, @@ -56,12 +64,19 @@ func DefaultConfig() Config { // Config ... type Config struct { WithSeconds bool - ConcurrentDelay time.Duration + ConcurrentDelay int ImmediatelyRun bool wrappers []JobWrapper logger *xlog.Logger parser cron.Parser + + // Distributed task + DistributedTask bool + + WaitLockTime time.Duration + *etcdv3.Config + client *etcdv3.Client } // WithChain ... @@ -98,9 +113,30 @@ func (config Config) Build() *Cron { } else { // 默认不延迟也不跳过 } + + if config.DistributedTask { + // 创建 Etcd Lock + newETCDXcron(&config) + } else { + config.Config = &etcdv3.Config{} + } + return newCron(&config) } +func newETCDXcron(config *Config) { + if config.logger == nil { + config.logger = xlog.DefaultLogger + } + config.logger = config.logger.With(xlog.FieldMod(ecode.ModXcronETCD), xlog.FieldAddrAny(config.Config.Endpoints)) + config.client = config.Config.Build() + if config.TTL == 0 { + config.TTL = DefaultTTL + } + + return +} + type wrappedLogger struct { *xlog.Logger } @@ -118,10 +154,39 @@ func (wl *wrappedLogger) Error(err error, msg string, keysAndValues ...interface type wrappedJob struct { NamedJob logger *xlog.Logger + + distributedTask bool + waitLockTime time.Duration + leaseTTL int + client *etcdv3.Client } +const ( + // 任务锁 + WorkerLockDir = "/xcron/lock/" + DefaultTTL = 60 // default set + DefaultWaitLockTime = 1000 // ms +) + // Run ... func (wj wrappedJob) Run() { + if wj.distributedTask { + mutex, err := wj.client.NewMutex(WorkerLockDir+wj.Name(), concurrency.WithTTL(wj.leaseTTL)) + if err != nil { + wj.logger.Error("mutex", xlog.String("err", err.Error())) + return + } + if wj.waitLockTime == 0 { + err = mutex.TryLock(DefaultWaitLockTime * time.Millisecond) + } else { // 阻塞等待直到waitLockTime timeout + err = mutex.Lock(wj.waitLockTime) + } + if err != nil { + wj.logger.Info("mutex lock", xlog.String("err", err.Error())) + return + } + defer mutex.Unlock() + } _ = wj.run() } diff --git a/pkg/worker/xcron/cron.go b/pkg/worker/xcron/cron.go index 98908c02f6..a846ced4c4 100644 --- a/pkg/worker/xcron/cron.go +++ b/pkg/worker/xcron/cron.go @@ -38,16 +38,23 @@ var ( WithLocation = cron.WithLocation ) -// JobWrapper ... type ( + // JobWrapper ... JobWrapper = cron.JobWrapper - EntryID = cron.EntryID - Entry = cron.Entry - Schedule = cron.Schedule - Parser = cron.Parser - Option = cron.Option - Job = cron.Job - NamedJob interface { + // EntryID ... + EntryID = cron.EntryID + // Entry ... + Entry = cron.Entry + // Schedule ... + Schedule = cron.Schedule + // Parser ... + Parser = cron.Parser + // Option ... + Option = cron.Option + // Job ... + Job = cron.Job + //NamedJob .. + NamedJob interface { Run() error Name() string } @@ -71,7 +78,7 @@ type Cron struct { func newCron(config *Config) *Cron { if config.logger == nil { - config.logger = xlog.DefaultLogger + config.logger = xlog.JupiterLogger } config.logger = config.logger.With(xlog.FieldMod("worker.cron")) cron := &Cron{ @@ -95,6 +102,11 @@ func (c *Cron) Schedule(schedule Schedule, job NamedJob) EntryID { innnerJob := &wrappedJob{ NamedJob: job, logger: c.logger, + + distributedTask: c.DistributedTask, + waitLockTime: c.WaitLockTime, + leaseTTL: c.Config.TTL, + client: c.client, } // xdebug.PrintKVWithPrefix("worker", "add job", job.Name()) c.logger.Info("add job", xlog.String("name", job.Name())) diff --git a/pkg/worker/xjob/job.go b/pkg/worker/xjob/job.go new file mode 100644 index 0000000000..20cff507b3 --- /dev/null +++ b/pkg/worker/xjob/job.go @@ -0,0 +1,20 @@ +package job + +import ( + "github.com/douyu/jupiter/pkg/flag" +) + +func init() { + flag.Register( + &flag.StringFlag{ + Name: "job", + Usage: "--job", + Default: "", + }, + ) +} + +// Runner ... +type Runner interface { + Run() +} diff --git a/pkg/xlog/field.go b/pkg/xlog/field.go index 2cf1139be3..66f8e7da43 100644 --- a/pkg/xlog/field.go +++ b/pkg/xlog/field.go @@ -48,6 +48,16 @@ func FieldName(value string) Field { return String("name", value) } +// FieldType ... +func FieldType(value string) Field { + return String("type", value) +} + +// FieldCode ... +func FieldCode(value int32) Field { + return Int32("code", value) +} + // 耗时时间 func FieldCost(value time.Duration) Field { return String("cost", fmt.Sprintf("%.3f", float64(value.Round(time.Microsecond))/float64(time.Millisecond))) @@ -83,6 +93,11 @@ func FieldErr(err error) Field { return zap.Error(err) } +// FieldErr ... +func FieldStringErr(err string) Field { + return String("err", err) +} + // FieldExtMessage ... func FieldExtMessage(vals ...interface{}) Field { return zap.Any("ext", vals) diff --git a/pkg/xlog/rotate/chown.go b/pkg/xlog/rotate/chown.go index 23a1c5d062..75db7a01b5 100644 --- a/pkg/xlog/rotate/chown.go +++ b/pkg/xlog/rotate/chown.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build !linux package rotate diff --git a/pkg/xlog/rotate/chown_linux.go b/pkg/xlog/rotate/chown_linux.go index 0c29ce83ed..40f313b5bf 100644 --- a/pkg/xlog/rotate/chown_linux.go +++ b/pkg/xlog/rotate/chown_linux.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build linux package rotate diff --git a/pkg/xlog/rotate/example_test.go b/pkg/xlog/rotate/example_test.go index 12678732a0..9a091b5e64 100644 --- a/pkg/xlog/rotate/example_test.go +++ b/pkg/xlog/rotate/example_test.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rotate_test import ( diff --git a/pkg/xlog/rotate/lumberjack_darwin.go b/pkg/xlog/rotate/lumberjack_darwin.go index 399e94147f..b8bf9b231d 100644 --- a/pkg/xlog/rotate/lumberjack_darwin.go +++ b/pkg/xlog/rotate/lumberjack_darwin.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Package rotate provides a rolling logger. // // Note that this is v2.0 of rotate, and should be imported using gopkg.in diff --git a/pkg/xlog/rotate/lumberjack_linux.go b/pkg/xlog/rotate/lumberjack_linux.go index d4b09c3a30..bf20aecb6b 100644 --- a/pkg/xlog/rotate/lumberjack_linux.go +++ b/pkg/xlog/rotate/lumberjack_linux.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Package rotate provides a rolling logger. // // Note that this is v2.0 of rotate, and should be imported using gopkg.in diff --git a/pkg/xlog/rotate/lumberjack_windows.go b/pkg/xlog/rotate/lumberjack_windows.go index 1bb0d1cd76..0ccdd16ba8 100644 --- a/pkg/xlog/rotate/lumberjack_windows.go +++ b/pkg/xlog/rotate/lumberjack_windows.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Package rotate provides a rolling logger. // // Note that this is v2.0 of rotate, and should be imported using gopkg.in diff --git a/pkg/xlog/rotate/rotate_test.go b/pkg/xlog/rotate/rotate_test.go index baf111ad8d..06cfef1d3b 100644 --- a/pkg/xlog/rotate/rotate_test.go +++ b/pkg/xlog/rotate/rotate_test.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // +build linux package rotate_test diff --git a/pkg/xlog/rotate/testing_test.go b/pkg/xlog/rotate/testing_test.go index 42460458a8..e6f89f3a1e 100644 --- a/pkg/xlog/rotate/testing_test.go +++ b/pkg/xlog/rotate/testing_test.go @@ -1,3 +1,17 @@ +// Copyright 2020 Douyu +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package rotate import ( diff --git a/tools/jupiter/go.mod b/tools/jupiter/go.mod index a94f0da488..33cba7c601 100644 --- a/tools/jupiter/go.mod +++ b/tools/jupiter/go.mod @@ -1 +1,3 @@ -module github.com/douyu/jupiter/tools/jupiter \ No newline at end of file +module github.com/douyu/jupiter/tools/jupiter + +go 1.14