diff --git a/cmd/connclose/.gitignore b/cmd/connclose/.gitignore new file mode 100644 index 0000000000..1d149e8ada --- /dev/null +++ b/cmd/connclose/.gitignore @@ -0,0 +1,2 @@ +connclose +secret.yml diff --git a/cmd/connclose/Dockerfile b/cmd/connclose/Dockerfile new file mode 100644 index 0000000000..4d745a1094 --- /dev/null +++ b/cmd/connclose/Dockerfile @@ -0,0 +1,3 @@ +FROM ubuntu:latest +COPY connclose /usr/sbin/connclose +ENTRYPOINT ["/usr/sbin/connclose"] diff --git a/cmd/connclose/Makefile b/cmd/connclose/Makefile new file mode 100644 index 0000000000..a4a54e926b --- /dev/null +++ b/cmd/connclose/Makefile @@ -0,0 +1,37 @@ +all: connclose + +connclose: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -o connclose . +.PHONY: connclose + +image: connclose + docker build -t connclose:latest . + +image-load: + kind load docker-image connclose:latest + +deploy: + kubectl apply -f secret.yml -f deployment.yml -f network.yml -f network.allow.yml + +start: + bash start.sh + +down: + kind delete cluster + +update: image image-load + kubectl rollout restart deployment bot + +allow: + kubectl apply -f network.allow.yml + +deny: + kubectl apply -f network.deny.yml + +wait: + kubectl rollout status deployment bot + +logs: + kubectl logs -l app=bot + +up: start image image-load deploy wait diff --git a/cmd/connclose/README.md b/cmd/connclose/README.md new file mode 100644 index 0000000000..134eadb69b --- /dev/null +++ b/cmd/connclose/README.md @@ -0,0 +1,48 @@ +This project is an attempt to reproduce connection timeout that causes gotd to hang. + +> [!WARNING] +> Currently I'm unable to reproduce the issue. + +Referenced issue: https://github.com/gotd/td/issues/1030 + +Dependencies: +- docker +- kind +- helm +- cilium cli +- go + +## Running + +Start cluster: + +```bash +make up +``` + +Deny connections to telegram: +```bash +make deny +``` + +Update binary +```bash +make update +``` + +Restore connections +```bash +make allow +``` + +## Logs + +```bash +make logs +``` + +## Cleanup + +```bash +make down +``` diff --git a/cmd/connclose/deployment.yml b/cmd/connclose/deployment.yml new file mode 100644 index 0000000000..db1637a2db --- /dev/null +++ b/cmd/connclose/deployment.yml @@ -0,0 +1,65 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bot + labels: + app.kubernetes.io/name: bot +spec: + strategy: + type: Recreate + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: bot + template: + metadata: + labels: + app: bot + app.kubernetes.io/name: bot + spec: + containers: + - name: bot + image: connclose:latest + imagePullPolicy: Never + resources: + requests: + cpu: 500m + memory: 128M + limits: + cpu: 1000m + memory: 256M + env: + - name: GOMEMLIMIT + value: "128MiB" + - name: GOMAXPROCS + value: "2" + - name: OTEL_METRICS_EXPORTER + value: "prometheus" + - name: OTEL_EXPORTER_PROMETHEUS_PORT + value: "8090" + - name: OTEL_EXPORTER_PROMETHEUS_HOST + value: "0.0.0.0" + - name: PPROF_ADDR + value: "0.0.0.0:8090" + - name: OTEL_RESOURCE_ATTRIBUTES + value: "service.name=connclose" + - name: OTEL_LOG_LEVEL + value: "DEBUG" + - name: OTEL_TRACES_EXPORTER + value: "none" + - name: BOT_TOKEN + valueFrom: + secretKeyRef: + name: config + key: BOT_TOKEN + - name: APP_ID + valueFrom: + secretKeyRef: + name: config + key: APP_ID + - name: APP_HASH + valueFrom: + secretKeyRef: + name: config + key: APP_HASH diff --git a/cmd/connclose/kind.yml b/cmd/connclose/kind.yml new file mode 100644 index 0000000000..8247e2a22d --- /dev/null +++ b/cmd/connclose/kind.yml @@ -0,0 +1,7 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker +networking: + disableDefaultCNI: true diff --git a/cmd/connclose/main.go b/cmd/connclose/main.go new file mode 100644 index 0000000000..a1ca1dc532 --- /dev/null +++ b/cmd/connclose/main.go @@ -0,0 +1,48 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/gotd/td/telegram" + "github.com/gotd/td/telegram/message" + "github.com/gotd/td/tg" +) + +func run(ctx context.Context) error { + logger, _ := zap.NewDevelopment(zap.IncreaseLevel(zapcore.DebugLevel)) + defer func() { _ = logger.Sync() }() + + dispatcher := tg.NewUpdateDispatcher() + return telegram.BotFromEnvironment(ctx, telegram.Options{ + Logger: logger, + UpdateHandler: dispatcher, + }, func(ctx context.Context, client *telegram.Client) error { + sender := message.NewSender(tg.NewClient(client)) + dispatcher.OnNewMessage(func(ctx context.Context, entities tg.Entities, u *tg.UpdateNewMessage) error { + m, ok := u.Message.(*tg.Message) + if !ok || m.Out { + return nil + } + + _, err := sender.Reply(entities, u).Text(ctx, m.Message) + return err + }) + return nil + }, telegram.RunUntilCanceled) +} + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt) + defer cancel() + + if err := run(ctx); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "%+v\n", err) + os.Exit(2) + } +} diff --git a/cmd/connclose/network.allow.yml b/cmd/connclose/network.allow.yml new file mode 100644 index 0000000000..88e16fc2d0 --- /dev/null +++ b/cmd/connclose/network.allow.yml @@ -0,0 +1,31 @@ +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: telegram +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: bot + egress: + - # telegram datacenters list + # generated by cmd/dcsidr + toCIDRSet: + - cidr: 149.154.175.59/32 + - cidr: 149.154.175.53/32 + - cidr: 2001:b28:f23d:f001::a/128 + - cidr: 149.154.167.50/32 + - cidr: 149.154.167.51/32 + - cidr: 149.154.167.151/32 + - cidr: 2001:67c:4e8:f002::a/128 + - cidr: 2001:67c:4e8:f002::b/128 + - cidr: 149.154.175.100/32 + - cidr: 2001:b28:f23d:f003::a/128 + - cidr: 149.154.167.91/32 + - cidr: 2001:67c:4e8:f004::a/128 + - cidr: 149.154.166.120/32 + - cidr: 2001:67c:4e8:f004::b/128 + - cidr: 2001:b28:f23f:f005::a/128 + - cidr: 91.108.56.173/32 + toPorts: + - ports: + - port: "443" diff --git a/cmd/connclose/network.deny.yml b/cmd/connclose/network.deny.yml new file mode 100644 index 0000000000..1c896b1c60 --- /dev/null +++ b/cmd/connclose/network.deny.yml @@ -0,0 +1,10 @@ +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: telegram +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: bot + # None. + egress: [] diff --git a/cmd/connclose/network.yml b/cmd/connclose/network.yml new file mode 100644 index 0000000000..f5aa041c21 --- /dev/null +++ b/cmd/connclose/network.yml @@ -0,0 +1,20 @@ +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: dns +spec: + endpointSelector: + matchLabels: + app.kubernetes.io/name: bot + egress: + - toEndpoints: + - matchLabels: + io.kubernetes.pod.namespace: kube-system + k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + rules: + dns: + - matchPattern: "*" diff --git a/cmd/connclose/secret.example.yml b/cmd/connclose/secret.example.yml new file mode 100644 index 0000000000..2ba11e9099 --- /dev/null +++ b/cmd/connclose/secret.example.yml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: config +stringData: + BOT_TOKEN: "1111111111:AAG1SomeBotTokenFromBotFather-1337a" + APP_ID: "0123456" + APP_HASH: "0123456789abcdef0123456789abcdef" diff --git a/cmd/connclose/start.sh b/cmd/connclose/start.sh new file mode 100755 index 0000000000..0bdb71463f --- /dev/null +++ b/cmd/connclose/start.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +kind create cluster --config kind.yml +docker pull quay.io/cilium/cilium:v1.14.6 +kind load docker-image quay.io/cilium/cilium:v1.14.6 + +helm install cilium cilium/cilium --version 1.14.6 \ + --namespace kube-system \ + --set image.pullPolicy=IfNotPresent \ + --set ipam.mode=kubernetes + +cilium status --wait diff --git a/cmd/dscidr/main.go b/cmd/dscidr/main.go new file mode 100644 index 0000000000..909ebc2af2 --- /dev/null +++ b/cmd/dscidr/main.go @@ -0,0 +1,75 @@ +package main + +import ( + "fmt" + "net/netip" + "os" + + "gopkg.in/yaml.v3" + + "github.com/gotd/td/telegram/dcs" +) + +type CIDRElement struct { + CIDR string `json:"cidr" yaml:"cidr"` +} + +type Ports struct { + Ports []Port `json:"ports" yaml:"ports"` +} + +type Port struct { + Port string `json:"port" yaml:"port"` +} + +type Element struct { + ToCIDRSet []CIDRElement `json:"toCIDRSet" yaml:"toCIDRSet"` + ToPorts []Ports `json:"toPorts" yaml:"toPorts"` +} + +/* + +- toCIDRSet: + - cidr: 192.169.0.1/32 + toPorts: + - ports: + - port: "443" +*/ + +func main() { + list := dcs.Prod() + var out []CIDRElement + met := make(map[string]struct{}) + for _, v := range list.Options { + ip, err := netip.ParseAddr(v.IPAddress) + if err != nil { + panic(err) + } + if v.Port != 443 { + panic("port != 443") + } + prefix := netip.PrefixFrom(ip, ip.BitLen()) + cidr := prefix.String() + if _, ok := met[cidr]; ok { + continue + } + met[cidr] = struct{}{} + out = append(out, CIDRElement{CIDR: cidr}) + } + fmt.Println("# telegram datacenters list") + fmt.Println("# generated by cmd/dcsidr") + e := yaml.NewEncoder(os.Stdout) + e.SetIndent(2) + if err := e.Encode(Element{ + ToCIDRSet: out, + ToPorts: []Ports{ + { + Ports: []Port{ + {Port: "443"}, + }, + }, + }, + }); err != nil { + panic(err) + } +} diff --git a/go.mod b/go.mod index 1c3e4a9c9f..ff2cc5d090 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( golang.org/x/net v0.20.0 golang.org/x/sync v0.6.0 golang.org/x/tools v0.17.0 + gopkg.in/yaml.v3 v3.0.1 nhooyr.io/websocket v1.8.10 rsc.io/qr v0.2.0 ) @@ -34,14 +35,16 @@ require ( github.com/andybalholm/cascadia v1.3.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect + go.uber.org/goleak v1.3.0 // indirect go.uber.org/ratelimit v0.3.0 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 6852bf0dcd..0f04b0180d 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,7 @@ github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -33,8 +34,9 @@ github.com/k0kubun/pp/v3 v3.2.0/go.mod h1:ODtJQbQcIRfAD3N+theGCV1m/CBxweERz2dapd github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -43,9 +45,11 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= @@ -68,8 +72,8 @@ go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/ratelimit v0.3.0 h1:IdZd9wqvFXnvLvSEBo0KPcGfkoBGNkpTHlrE3Rcjkjw= @@ -124,8 +128,9 @@ golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=