Cancel pushover notifications when the alert resolves itself

config/notifiers.go

@@ -151,13 +151,14 @@ var (
 		NotifierConfig: NotifierConfig{
 			VSendResolved: true,
 		},
-		Title:    `{{ template "pushover.default.title" . }}`,
-		Message:  `{{ template "pushover.default.message" . }}`,
-		URL:      `{{ template "pushover.default.url" . }}`,
-		Priority: `{{ if eq .Status "firing" }}2{{ else }}0{{ end }}`, // emergency (firing) or normal
-		Retry:    duration(1 * time.Minute),
-		Expire:   duration(1 * time.Hour),
-		HTML:     false,
+		Title:           `{{ template "pushover.default.title" . }}`,
+		Message:         `{{ template "pushover.default.message" . }}`,
+		URL:             `{{ template "pushover.default.url" . }}`,
+		Priority:        `{{ if eq .Status "firing" }}2{{ else }}0{{ end }}`, // emergency (firing) or normal
+		Retry:           duration(1 * time.Minute),
+		Expire:          duration(1 * time.Hour),
+		HTML:            false,
+		CancelOnResolve: false,
 	}
 
 	// DefaultSNSConfig defines default values for SNS configurations.
@@ -727,21 +728,22 @@ type PushoverConfig struct {
 
 	HTTPConfig *commoncfg.HTTPClientConfig `yaml:"http_config,omitempty" json:"http_config,omitempty"`
 
-	UserKey     Secret   `yaml:"user_key,omitempty" json:"user_key,omitempty"`
-	UserKeyFile string   `yaml:"user_key_file,omitempty" json:"user_key_file,omitempty"`
-	Token       Secret   `yaml:"token,omitempty" json:"token,omitempty"`
-	TokenFile   string   `yaml:"token_file,omitempty" json:"token_file,omitempty"`
-	Title       string   `yaml:"title,omitempty" json:"title,omitempty"`
-	Message     string   `yaml:"message,omitempty" json:"message,omitempty"`
-	URL         string   `yaml:"url,omitempty" json:"url,omitempty"`
-	URLTitle    string   `yaml:"url_title,omitempty" json:"url_title,omitempty"`
-	Device      string   `yaml:"device,omitempty" json:"device,omitempty"`
-	Sound       string   `yaml:"sound,omitempty" json:"sound,omitempty"`
-	Priority    string   `yaml:"priority,omitempty" json:"priority,omitempty"`
-	Retry       duration `yaml:"retry,omitempty" json:"retry,omitempty"`
-	Expire      duration `yaml:"expire,omitempty" json:"expire,omitempty"`
-	TTL         duration `yaml:"ttl,omitempty" json:"ttl,omitempty"`
-	HTML        bool     `yaml:"html" json:"html,omitempty"`
+	UserKey         Secret   `yaml:"user_key,omitempty" json:"user_key,omitempty"`
+	UserKeyFile     string   `yaml:"user_key_file,omitempty" json:"user_key_file,omitempty"`
+	Token           Secret   `yaml:"token,omitempty" json:"token,omitempty"`
+	TokenFile       string   `yaml:"token_file,omitempty" json:"token_file,omitempty"`
+	Title           string   `yaml:"title,omitempty" json:"title,omitempty"`
+	Message         string   `yaml:"message,omitempty" json:"message,omitempty"`
+	URL             string   `yaml:"url,omitempty" json:"url,omitempty"`
+	URLTitle        string   `yaml:"url_title,omitempty" json:"url_title,omitempty"`
+	Device          string   `yaml:"device,omitempty" json:"device,omitempty"`
+	Sound           string   `yaml:"sound,omitempty" json:"sound,omitempty"`
+	Priority        string   `yaml:"priority,omitempty" json:"priority,omitempty"`
+	Retry           duration `yaml:"retry,omitempty" json:"retry,omitempty"`
+	Expire          duration `yaml:"expire,omitempty" json:"expire,omitempty"`
+	TTL             duration `yaml:"ttl,omitempty" json:"ttl,omitempty"`
+	HTML            bool     `yaml:"html" json:"html,omitempty"`
+	CancelOnResolve bool     `yaml:"cancel_on_resolve" json:"cancel_on_resolve,omitempty"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.

docs/configuration.md

@@ -1299,6 +1299,9 @@ token_file: <filepath>
 # Optional time to live (TTL) to use for notification, see https://pushover.net/api#ttl
 [ ttl: <duration> ]
 
+# Optional boolean to enable cancellation of emergency priority notifications upon resolution.
+[ cancel_on_resolve: <boolean> | default = false ]
+
 # The HTTP client's configuration.
 [ http_config: <http_config> | default = global.http_config ]
 ```

notify/pushover/pushover.go

@@ -29,6 +29,7 @@ import (
 	"github.com/prometheus/alertmanager/notify"
 	"github.com/prometheus/alertmanager/template"
 	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
 )
 
 const (
@@ -42,12 +43,13 @@ const (
 
 // Notifier implements a Notifier for Pushover notifications.
 type Notifier struct {
-	conf    *config.PushoverConfig
-	tmpl    *template.Template
-	logger  *slog.Logger
-	client  *http.Client
-	retrier *notify.Retrier
-	apiURL  string // for tests.
+	conf           *config.PushoverConfig
+	tmpl           *template.Template
+	logger         *slog.Logger
+	client         *http.Client
+	retrier        *notify.Retrier
+	apiMessagesURL string // for tests.
+	apiReceiptsURL string // for tests.
 }
 
 // New returns a new Pushover notifier.
@@ -57,30 +59,28 @@ func New(c *config.PushoverConfig, t *template.Template, l *slog.Logger, httpOpt
 		return nil, err
 	}
 	return &Notifier{
-		conf:    c,
-		tmpl:    t,
-		logger:  l,
-		client:  client,
-		retrier: &notify.Retrier{},
-		apiURL:  "https://api.pushover.net/1/messages.json",
+		conf:           c,
+		tmpl:           t,
+		logger:         l,
+		client:         client,
+		retrier:        &notify.Retrier{},
+		apiMessagesURL: "https://api.pushover.net/1/messages.json",
+		apiReceiptsURL: "https://api.pushover.net/1/receipts/cancel_by_tag",
 	}, nil
 }
 
 // Notify implements the Notifier interface.
 func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
-	key, ok := notify.GroupKey(ctx)
-	if !ok {
-		return false, fmt.Errorf("group key missing")
+	var err error
+	key, err := notify.ExtractGroupKey(ctx)
+	if err != nil {
+		return false, err
 	}
 	data := notify.GetTemplateData(ctx, n.tmpl, as, n.logger)
 
-	// @tjhop: should this use `group` for the keyval like most other notify implementations?
-	n.logger.Debug("extracted group key", "incident", key)
+	n.logger.Debug("extracted group key", "key", key)
 
-	var (
-		err     error
-		message string
-	)
+	var message string
 	tmpl := notify.TmplText(n.tmpl, data, &err)
 	tmplHTML := notify.TmplHTML(n.tmpl, data, &err)
 
@@ -111,6 +111,13 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	parameters.Add("token", tmpl(token))
 	parameters.Add("user", tmpl(userKey))
 
+	var (
+		priority    = tmpl(n.conf.Priority)
+		alerts      = types.Alerts(as...)
+		groupKeyTag = fmt.Sprintf("promAlertGroupKey_%s", key.Hash())
+		u           *url.URL
+	)
+
 	title, truncated := notify.TruncateInRunes(tmpl(n.conf.Title), maxTitleLenRunes)
 	if truncated {
 		n.logger.Warn("Truncated title", "incident", key, "max_runes", maxTitleLenRunes)
@@ -142,25 +149,39 @@ func (n *Notifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error)
 	parameters.Add("url", supplementaryURL)
 	parameters.Add("url_title", tmpl(n.conf.URLTitle))
 
-	parameters.Add("priority", tmpl(n.conf.Priority))
+	parameters.Add("priority", priority)
 	parameters.Add("retry", fmt.Sprintf("%d", int64(time.Duration(n.conf.Retry).Seconds())))
 	parameters.Add("expire", fmt.Sprintf("%d", int64(time.Duration(n.conf.Expire).Seconds())))
 	parameters.Add("device", tmpl(n.conf.Device))
 	parameters.Add("sound", tmpl(n.conf.Sound))
-
+	if n.conf.CancelOnResolve && priority == "2" {
+		parameters.Add("tags", groupKeyTag)
+	}
 	newttl := int64(time.Duration(n.conf.TTL).Seconds())
 	if newttl > 0 {
 		parameters.Add("ttl", fmt.Sprintf("%d", newttl))
 	}
 
+	u, err = url.Parse(n.apiMessagesURL)
 	if err != nil {
 		return false, err
 	}
 
-	u, err := url.Parse(n.apiURL)
 	if err != nil {
 		return false, err
 	}
+	shouldRetry, err := n.sendMessage(ctx, key, u, parameters)
+	if err == nil && n.conf.CancelOnResolve && alerts.Status() == model.AlertResolved {
+		u, err = url.Parse(fmt.Sprintf("%s/%s.json", n.apiReceiptsURL, groupKeyTag))
+		if err != nil {
+			return false, err
+		}
+		shouldRetry, err = n.sendMessage(ctx, key, u, parameters)
+	}
+	return shouldRetry, err
+}
+
+func (n *Notifier) sendMessage(ctx context.Context, key notify.Key, u *url.URL, parameters url.Values) (bool, error) {
 	u.RawQuery = parameters.Encode()
 	// Don't log the URL as it contains secret data (see #1825).
 	n.logger.Debug("Sending message", "incident", key)

notify/pushover/pushover_test.go

@@ -56,7 +56,8 @@ func TestPushoverRedactedURL(t *testing.T) {
 		promslog.NewNopLogger(),
 	)
 	require.NoError(t, err)
-	notifier.apiURL = u.String()
+	notifier.apiMessagesURL = u.String()
+	notifier.apiReceiptsURL = u.String()
 
 	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, key, token)
 }
@@ -80,7 +81,8 @@ func TestPushoverReadingUserKeyFromFile(t *testing.T) {
 		test.CreateTmpl(t),
 		promslog.NewNopLogger(),
 	)
-	notifier.apiURL = apiURL.String()
+	notifier.apiMessagesURL = apiURL.String()
+	notifier.apiReceiptsURL = apiURL.String()
 	require.NoError(t, err)
 
 	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, userKey)
@@ -105,7 +107,8 @@ func TestPushoverReadingTokenFromFile(t *testing.T) {
 		test.CreateTmpl(t),
 		promslog.NewNopLogger(),
 	)
-	notifier.apiURL = apiURL.String()
+	notifier.apiMessagesURL = apiURL.String()
+	notifier.apiReceiptsURL = apiURL.String()
 	require.NoError(t, err)
 
 	test.AssertNotifyLeaksNoSecret(ctx, t, notifier, token)