Skip to content

Commit af64014

Browse files
stevesajeev1stevesajeev1
committed
Create template for alert notifications
1 parent 08e958d commit af64014

9 files changed

Lines changed: 88 additions & 75 deletions

File tree

‎apps/api/internal/api/handlers/grafana/http.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ func RegisterRoutes(grafanaHandler *handler, mw *middleware.Middleware, router *
3737
}
3838

3939
// For local development, dashboard is accessed via /grafana, so don't trim prefix
40-
if grafanaHandler.environment != "dev" {
40+
if grafanaHandler.environment != "local" {
4141
r.URL.Path = strings.TrimPrefix(r.URL.Path, "/grafana")
4242
if r.URL.Path == "" {
4343
r.URL.Path = "/"

‎docker-compose.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,13 +107,14 @@ services:
107107
grafana:
108108
image: grafana/grafana:12.4.2
109109
container_name: grafana
110-
env_file: "./monitoring/.env"
110+
env_file:
111+
- ./apps/api/.env
111112
environment:
112113
- GF_PATHS_PROVISIONING=/etc/grafana/provisioning
113114
- GF_AUTH_PROXY_ENABLED=true
114115
- GF_AUTH_PROXY_HEADER_NAME=X-WEBAUTH-USER
115116
- GF_USERS_AUTO_ASSIGN_ORG_ROLE=Admin
116-
- GF_SERVER_ROOT_URL=/grafana
117+
- GF_SERVER_ROOT_URL=http://localhost:8080/grafana
117118
- GF_SERVER_SERVE_FROM_SUB_PATH=true
118119
volumes:
119120
- ./monitoring/provisioning:/etc/grafana/provisioning

‎infra/docker-compose.api.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ services:
172172
- GF_AUTH_PROXY_ENABLED=true
173173
- GF_AUTH_PROXY_HEADER_NAME=X-WEBAUTH-USER
174174
- GF_USERS_AUTO_ASSIGN_ORG_ROLE=Admin
175+
- GF_SERVER_ROOT_URL=https://grafana.swamphacks.com/
175176
volumes:
176177
- ../monitoring/provisioning:/etc/grafana/provisioning
177178
- grafana_data:/var/lib/grafana
@@ -189,6 +190,7 @@ services:
189190
- GF_AUTH_PROXY_ENABLED=true
190191
- GF_AUTH_PROXY_HEADER_NAME=X-WEBAUTH-USER
191192
- GF_USERS_AUTO_ASSIGN_ORG_ROLE=Admin
193+
- GF_SERVER_ROOT_URL=https://dev-grafana.swamphacks.com/
192194
volumes:
193195
- ../monitoring/provisioning-dev:/etc/grafana/provisioning
194196
- grafana_data_dev:/var/lib/grafana

‎monitoring/provisioning-dev/alerting/alert-rules.yaml‎

Lines changed: 11 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ groups:
88
rules:
99
- uid: api_dev_errors
1010
title: API Dev errors detected
11-
condition: C
11+
condition: B
1212

1313
data:
1414
- refId: A
@@ -18,11 +18,12 @@ groups:
1818
to: 0
1919
datasourceUid: loki_uid
2020
model:
21-
queryType: range
22-
expr: count_over_time({container_name=~".*-api.*", env="dev"} | json | level="error" [5m])
21+
expr: |
22+
count_over_time(
23+
{container_name=~".*-api.*", env="dev", level="error"}[5m]
24+
)
2325
refId: A
24-
- refId: reducer
25-
queryType: expression
26+
- refId: B
2627
datasourceUid: __expr__
2728
model:
2829
conditions:
@@ -44,41 +45,18 @@ groups:
4445
type: __expr__
4546
uid: __expr__
4647
expression: A
48+
hide: false
4749
intervalMs: 1000
4850
maxDataPoints: 43200
4951
reducer: last
50-
refId: reducer
52+
refId: B
5153
type: reduce
52-
- refId: C
53-
datasourceUid: __expr__
54-
model:
55-
conditions:
56-
- evaluator:
57-
params:
58-
- 0
59-
type: gt
60-
operator:
61-
type: and
62-
query:
63-
params: []
64-
reducer:
65-
params: []
66-
type: last
67-
type: query
68-
datasource:
69-
type: __expr__
70-
uid: __expr__
71-
expression: reducer
72-
intervalMs: 1000
73-
maxDataPoints: 43200
74-
refId: C
75-
type: threshold
76-
77-
for: 1m
7854

79-
noDataState: NoData
55+
noDataState: OK
8056
execErrState: Error
8157

58+
for: 1m
59+
8260
labels:
8361
severity: warning
8462
scope: global

‎monitoring/provisioning-dev/alerting/contact-points.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,5 @@ contactPoints:
66
- uid: discord_webhook
77
type: discord
88
settings:
9-
url: $MONITORING_DISCORD_WEBHOOK
9+
url: $MONITORING_DISCORD_WEBHOOK
10+
message: '{{ template "discord_template" . }}'

‎monitoring/provisioning-dev/alerting/templates.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
apiVersion: 1
2+
3+
templates:
4+
- orgId: 1
5+
name: discord_template
6+
template: |
7+
{{ define "discord_template" }}
8+
{{ if eq .Status "firing" }}
9+
🚨 **DEV API ERROR ALERT**
10+
{{ else }}
11+
✅ **DEV API RECOVERED**
12+
{{ end }}
13+
14+
{{ range .Alerts }}
15+
**Severity:** {{ .Labels.severity }}
16+
17+
**Errors (last 5m):** {{ index .Values "B" }}
18+
19+
**Summary:** API dev errors detected
20+
21+
Starts: {{ .StartsAt }}
22+
23+
🔎 **View Logs:**
24+
{{ $.ExternalURL }}explore?orgId=1&left={{ urlquery `["now-5m","now","Loki",{"expr":"{container_name=~\".*-api.*\",env=\"prod\",level=\"error\"}"}]` }}
25+
{{ end }}
26+
{{ end }}

‎monitoring/provisioning/alerting/alert-rules.yaml‎

Lines changed: 15 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
apiVersion: 1
22

33
groups:
4-
- name: api-alerts
4+
- name: api-prod-alerts
55
folder: Services
66
interval: 1m
77

88
rules:
9-
- uid: api_errors
10-
title: API errors detected
11-
condition: C
9+
- uid: api_prod_errors
10+
title: API Prod errors detected
11+
condition: B
1212

1313
data:
1414
- refId: A
@@ -18,11 +18,12 @@ groups:
1818
to: 0
1919
datasourceUid: loki_uid
2020
model:
21-
queryType: range
22-
expr: count_over_time({container_name=~".*-api.*", env="prod"} | json | level="error" [5m])
21+
expr: |
22+
count_over_time(
23+
{container_name=~".*-api.*", env="prod", level="error"}[5m]
24+
)
2325
refId: A
24-
- refId: reducer
25-
queryType: expression
26+
- refId: B
2627
datasourceUid: __expr__
2728
model:
2829
conditions:
@@ -44,45 +45,22 @@ groups:
4445
type: __expr__
4546
uid: __expr__
4647
expression: A
48+
hide: false
4749
intervalMs: 1000
4850
maxDataPoints: 43200
4951
reducer: last
50-
refId: reducer
52+
refId: B
5153
type: reduce
52-
- refId: C
53-
datasourceUid: __expr__
54-
model:
55-
conditions:
56-
- evaluator:
57-
params:
58-
- 0
59-
type: gt
60-
operator:
61-
type: and
62-
query:
63-
params: []
64-
reducer:
65-
params: []
66-
type: last
67-
type: query
68-
datasource:
69-
type: __expr__
70-
uid: __expr__
71-
expression: reducer
72-
intervalMs: 1000
73-
maxDataPoints: 43200
74-
refId: C
75-
type: threshold
76-
77-
for: 1m
7854

79-
noDataState: NoData
55+
noDataState: OK
8056
execErrState: Error
8157

58+
for: 1m
59+
8260
labels:
8361
severity: warning
8462
scope: global
85-
alertname: api_errors
63+
alertname: api_prod_errors
8664

8765
notification_settings:
8866
receiver: discord

‎monitoring/provisioning/alerting/contact-points.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,5 @@ contactPoints:
66
- uid: discord_webhook
77
type: discord
88
settings:
9-
url: $MONITORING_DISCORD_WEBHOOK
9+
url: $MONITORING_DISCORD_WEBHOOK
10+
message: '{{ template "discord_template" . }}'

‎monitoring/provisioning/alerting/templates.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
apiVersion: 1
2+
3+
templates:
4+
- orgId: 1
5+
name: discord_template
6+
template: |
7+
{{ define "discord_template" }}
8+
{{ if eq .Status "firing" }}
9+
🚨 **PROD API ERROR ALERT**
10+
{{ else }}
11+
✅ **PROD API RECOVERED**
12+
{{ end }}
13+
14+
{{ range .Alerts }}
15+
**Severity:** {{ .Labels.severity }}
16+
17+
**Errors (last 5m):** {{ index .Values "B" }}
18+
19+
**Summary:** API prod errors detected
20+
21+
Starts: {{ .StartsAt }}
22+
23+
🔎 **View Logs:**
24+
{{ $.ExternalURL }}explore?orgId=1&left={{ urlquery `["now-5m","now","Loki",{"expr":"{container_name=~\".*-api.*\",env=\"prod\",level=\"error\"}"}]` }}
25+
{{ end }}
26+
{{ end }}

0 commit comments

Comments
 (0)