Handle slack RateLimitedError when posting messages (#241)

* Handle slack RateLimitedError when posting messages

Signed-off-by: John Watson <[email protected]>

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/stretchr/testify v1.6.1
 	github.com/technoweenie/multipartstreamer v1.0.1 // indirect
 	github.com/whilp/git-urls v0.0.0-20191001220047-6db9661140c0
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
 	gomodules.xyz/notify v0.1.0
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 	k8s.io/api v0.20.4

go.sum

@@ -976,8 +976,9 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
+golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

pkg/services/slack.go

@@ -14,8 +14,12 @@ import (
 
 	log "github.com/sirupsen/logrus"
 	"github.com/slack-go/slack"
+	"golang.org/x/time/rate"
 )
 
+// No rate limit unless Slack requests it (allows for Slack to control bursting)
+var rateLimiter = rate.NewLimiter(rate.Inf, 1)
+
 type SlackNotification struct {
 	Attachments string `json:"attachments,omitempty"`
 	Blocks      string `json:"blocks,omitempty"`
@@ -110,7 +114,26 @@ func (s *slackService) Send(notification Notification, dest Destination) error {
 		msgOptions = append(msgOptions, slack.MsgOptionAttachments(attachments...), slack.MsgOptionBlocks(blocks.BlockSet...))
 	}
 
-	_, _, err := sl.PostMessageContext(context.TODO(), dest.Recipient, msgOptions...)
+	ctx := context.TODO()
+	var err error
+	for {
+		err = rateLimiter.Wait(ctx)
+		if err != nil {
+			break
+		}
+		_, _, err = sl.PostMessageContext(ctx, dest.Recipient, msgOptions...)
+		if err != nil {
+			if rateLimitedError, ok := err.(*slack.RateLimitedError); ok {
+				rateLimiter.SetLimit(rate.Every(rateLimitedError.RetryAfter))
+			} else {
+				break
+			}
+		} else {
+			// No error, so remove rate limit
+			rateLimiter.SetLimit(rate.Inf)
+			break
+		}
+	}
 	return err
 }