code-place/.github/workflows/monitoring-validate.yml at develop · pnu-code-place/code-place · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
name: Validate monitoring contracts

on:
  pull_request:
    paths:
      - "kubernetes/monitoring/**"
      - "kubernetes/base/hub-auth/**"
      - "kubernetes/base/backend/**"
      - "kubernetes/base/celery-beat/**"
      - "kubernetes/base/celery-worker/**"
      - "kubernetes/overlays/dev/**"
      - "kubernetes/overlays/prod/**"
      - "hub/auth_server/**"
      - "backend/account/middleware.py"
      - "backend/account/tests.py"
      - "backend/judge/tasks.py"
      - "backend/judge/tests.py"
      - "backend/oj/celery.py"
      - "backend/problem/llm_hint.py"
      - "backend/problem/views/oj.py"
      - "backend/problem/tests.py"
      - "backend/submission/views/oj.py"
      - "backend/submission/tests.py"
      - "backend/utils/observability_metrics.py"
      - "backend/utils/observability_tracing.py"
      - "backend/utils/tests.py"
      - "backend/utils/test_observability_tracing.py"
      - ".github/workflows/monitoring-validate.yml"
  push:
    branches: [develop, main]
    paths:
      - "kubernetes/monitoring/**"
      - "kubernetes/base/hub-auth/**"
      - "kubernetes/base/backend/**"
      - "kubernetes/base/celery-beat/**"
      - "kubernetes/base/celery-worker/**"
      - "kubernetes/overlays/dev/**"
      - "kubernetes/overlays/prod/**"
      - "hub/auth_server/**"
      - "backend/account/middleware.py"
      - "backend/account/tests.py"
      - "backend/judge/tasks.py"
      - "backend/judge/tests.py"
      - "backend/oj/celery.py"
      - "backend/problem/llm_hint.py"
      - "backend/problem/views/oj.py"
      - "backend/problem/tests.py"
      - "backend/submission/views/oj.py"
      - "backend/submission/tests.py"
      - "backend/utils/observability_metrics.py"
      - "backend/utils/observability_tracing.py"
      - "backend/utils/tests.py"
      - "backend/utils/test_observability_tracing.py"
      - ".github/workflows/monitoring-validate.yml"

permissions:
  contents: read

jobs:
  validate:
    runs-on: ubuntu-latest
    env:
      POSTGRES_HOST: 127.0.0.1
      POSTGRES_PORT: "5435"
      POSTGRES_DB: onlinejudge
      POSTGRES_USER: onlinejudge
      POSTGRES_PASSWORD: onlinejudge
      REDIS_HOST: 127.0.0.1
      REDIS_PORT: "6380"
    services:
      postgres:
        image: postgres:14
        env:
          POSTGRES_DB: onlinejudge
          POSTGRES_USER: onlinejudge
          POSTGRES_PASSWORD: onlinejudge
        ports:
          - 5435:5432
        options: >-
          --health-cmd "pg_isready -U onlinejudge -d onlinejudge"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
      redis:
        image: redis:7-alpine
        ports:
          - 6380:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.11"

      - name: Set up kubectl
        uses: azure/setup-kubectl@v4
        with:
          version: "v1.33.6"

      - name: Set up Helm
        uses: azure/setup-helm@v4
        with:
          version: "v3.18.6"

      - name: Install validation dependencies
        run: python -m pip install --disable-pip-version-check -r backend/deploy/requirements.txt -r hub/auth_server/requirements.txt pytest PyYAML

      - name: Test hub-auth health contract
        working-directory: hub/auth_server
        env:
          GITHUB_OAUTH_APP_ID: test-client
          GITHUB_OAUTH_APP_SECRET: test-secret
        run: python -m pytest app/test_main.py

      - name: Test trace resource contract
        env:
          PYTHONPATH: backend
        run: python -m unittest utils.test_observability_tracing

      - name: Test backend observability contracts
        working-directory: backend
        run: |
          printf 'monitoring-validation-secret\n' >data/config/secret.key
          python manage.py test \
            utils.tests.CodePlaceCollectorTest \
            utils.tests.CeleryRequestIDContextTest \
            utils.tests.ObservabilityTracingTest \
            utils.tests.CodePlaceMetricsEndpointTest \
            judge.tests.JudgeTaskObservabilityTest \
            account.tests.RequestLogMiddlewareTest \
            problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_api_success_outcome \
            problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_success_metric \
            problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_request_error_metric \
            problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_problem_limit_outcome \
            submission.tests.SubmissionAPITest.test_create_submission_records_success_outcome \
            submission.tests.SubmissionAPITest.test_create_submission_records_problem_not_found_outcome

      - name: Validate dashboard contracts
        run: python kubernetes/monitoring/validate_dashboards.py --prometheus-rules-output /tmp/codeplace-dashboard-rules.json

      - name: Validate alert contracts
        run: python kubernetes/monitoring/validate_alerts.py

      - name: Render Kubernetes manifests
        run: |
          kubectl kustomize kubernetes/overlays/dev >/dev/null
          kubectl kustomize kubernetes/overlays/prod >/dev/null
          kubectl kustomize kubernetes/monitoring >/tmp/codeplace-monitoring.yaml
          python -c 'import pathlib, yaml; docs=[doc for doc in yaml.safe_load_all(pathlib.Path("/tmp/codeplace-monitoring.yaml").read_text()) if doc]; assert any(doc.get("kind") == "PodMonitor" and doc.get("metadata", {}).get("name") == "traefik" for doc in docs), "Traefik PodMonitor is missing"; assert not any(doc.get("kind") == "ServiceMonitor" and doc.get("metadata", {}).get("name") == "traefik" for doc in docs), "legacy Traefik ServiceMonitor remains"'
          test ! -e kubernetes/monitoring/traefik-service-monitor.yaml

      - name: Render pinned monitoring charts
        run: |
          helm template kube-prometheus-stack kube-prometheus-stack \
            --repo https://prometheus-community.github.io/helm-charts \
            --version 86.3.1 \
            --namespace monitoring \
            --kube-version 1.33.6 \
            --values kubernetes/monitoring/kube-prometheus-stack-values.yaml >/tmp/kube-prometheus-stack.yaml
          python -c 'import pathlib, yaml; docs=[doc for doc in yaml.safe_load_all(pathlib.Path("/tmp/kube-prometheus-stack.yaml").read_text()) if doc]; assert any(doc.get("kind") == "Job" and doc.get("metadata", {}).get("name") == "kube-prometheus-stack-crds-upgrade" for doc in docs), "CRD upgrade Job is missing"; grafana=next(doc for doc in docs if doc.get("kind") == "Deployment" and doc.get("metadata", {}).get("name") == "kube-prometheus-stack-grafana"); assert grafana.get("spec", {}).get("strategy", {}).get("type") == "Recreate", "Grafana must use Recreate with its RWO PVC"'
          helm template loki loki \
            --repo https://grafana.github.io/helm-charts \
            --version 6.55.0 \
            --namespace monitoring \
            --kube-version 1.33.6 \
            --api-versions monitoring.coreos.com/v1/ServiceMonitor \
            --values kubernetes/monitoring/logs/loki-values.yaml >/dev/null
          helm template alloy alloy \
            --repo https://grafana.github.io/helm-charts \
            --version 1.10.0 \
            --namespace monitoring \
            --kube-version 1.33.6 \
            --api-versions monitoring.coreos.com/v1/ServiceMonitor \
            --values kubernetes/monitoring/logs/alloy-values.yaml >/dev/null

      - name: Validate Prometheus rules
        run: |
          python -c 'import pathlib, yaml; source=yaml.safe_load(pathlib.Path("kubernetes/monitoring/prometheus-rules.yaml").read_text()); pathlib.Path("/tmp/codeplace-rules.yaml").write_text(yaml.safe_dump({"groups": source["spec"]["groups"]}))'
          docker run --rm --entrypoint /bin/promtool -v /tmp/codeplace-rules.yaml:/rules.yaml:ro prom/prometheus:v3.7.3 check rules /rules.yaml
          docker run --rm --entrypoint /bin/promtool -v /tmp/codeplace-dashboard-rules.json:/rules.json:ro prom/prometheus:v3.7.3 check rules /rules.json
          docker run --rm --entrypoint /bin/promtool -v /tmp/codeplace-rules.yaml:/rules.yaml:ro -v "$PWD/kubernetes/monitoring/prometheus-rules.test.yaml:/rules.test.yaml:ro" prom/prometheus:v3.7.3 test rules /rules.test.yaml