fix: Kubernetes 플랫폼 설정 정비 (#761) #12
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Validate monitoring contracts | |
| on: | |
| pull_request: | |
| paths: | |
| - "kubernetes/monitoring/**" | |
| - "kubernetes/base/hub-auth/**" | |
| - "kubernetes/base/backend/**" | |
| - "kubernetes/base/celery-beat/**" | |
| - "kubernetes/base/celery-worker/**" | |
| - "kubernetes/overlays/dev/**" | |
| - "kubernetes/overlays/prod/**" | |
| - "hub/auth_server/**" | |
| - "backend/account/middleware.py" | |
| - "backend/account/tests.py" | |
| - "backend/judge/tasks.py" | |
| - "backend/judge/tests.py" | |
| - "backend/oj/celery.py" | |
| - "backend/problem/llm_hint.py" | |
| - "backend/problem/views/oj.py" | |
| - "backend/problem/tests.py" | |
| - "backend/submission/views/oj.py" | |
| - "backend/submission/tests.py" | |
| - "backend/utils/observability_metrics.py" | |
| - "backend/utils/observability_tracing.py" | |
| - "backend/utils/tests.py" | |
| - "backend/utils/test_observability_tracing.py" | |
| - ".github/workflows/monitoring-validate.yml" | |
| push: | |
| branches: [develop, main] | |
| paths: | |
| - "kubernetes/monitoring/**" | |
| - "kubernetes/base/hub-auth/**" | |
| - "kubernetes/base/backend/**" | |
| - "kubernetes/base/celery-beat/**" | |
| - "kubernetes/base/celery-worker/**" | |
| - "kubernetes/overlays/dev/**" | |
| - "kubernetes/overlays/prod/**" | |
| - "hub/auth_server/**" | |
| - "backend/account/middleware.py" | |
| - "backend/account/tests.py" | |
| - "backend/judge/tasks.py" | |
| - "backend/judge/tests.py" | |
| - "backend/oj/celery.py" | |
| - "backend/problem/llm_hint.py" | |
| - "backend/problem/views/oj.py" | |
| - "backend/problem/tests.py" | |
| - "backend/submission/views/oj.py" | |
| - "backend/submission/tests.py" | |
| - "backend/utils/observability_metrics.py" | |
| - "backend/utils/observability_tracing.py" | |
| - "backend/utils/tests.py" | |
| - "backend/utils/test_observability_tracing.py" | |
| - ".github/workflows/monitoring-validate.yml" | |
| permissions: | |
| contents: read | |
| jobs: | |
| validate: | |
| runs-on: ubuntu-latest | |
| env: | |
| POSTGRES_HOST: 127.0.0.1 | |
| POSTGRES_PORT: "5435" | |
| POSTGRES_DB: onlinejudge | |
| POSTGRES_USER: onlinejudge | |
| POSTGRES_PASSWORD: onlinejudge | |
| REDIS_HOST: 127.0.0.1 | |
| REDIS_PORT: "6380" | |
| services: | |
| postgres: | |
| image: postgres:14 | |
| env: | |
| POSTGRES_DB: onlinejudge | |
| POSTGRES_USER: onlinejudge | |
| POSTGRES_PASSWORD: onlinejudge | |
| ports: | |
| - 5435:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U onlinejudge -d onlinejudge" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| redis: | |
| image: redis:7-alpine | |
| ports: | |
| - 6380:6379 | |
| options: >- | |
| --health-cmd "redis-cli ping" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| - name: Set up kubectl | |
| uses: azure/setup-kubectl@v4 | |
| with: | |
| version: "v1.33.6" | |
| - name: Set up Helm | |
| uses: azure/setup-helm@v4 | |
| with: | |
| version: "v3.18.6" | |
| - name: Install validation dependencies | |
| run: python -m pip install --disable-pip-version-check -r backend/deploy/requirements.txt -r hub/auth_server/requirements.txt pytest PyYAML | |
| - name: Test hub-auth health contract | |
| working-directory: hub/auth_server | |
| env: | |
| GITHUB_OAUTH_APP_ID: test-client | |
| GITHUB_OAUTH_APP_SECRET: test-secret | |
| run: python -m pytest app/test_main.py | |
| - name: Test trace resource contract | |
| env: | |
| PYTHONPATH: backend | |
| run: python -m unittest utils.test_observability_tracing | |
| - name: Test backend observability contracts | |
| working-directory: backend | |
| run: | | |
| printf 'monitoring-validation-secret\n' >data/config/secret.key | |
| python manage.py test \ | |
| utils.tests.CodePlaceCollectorTest \ | |
| utils.tests.CeleryRequestIDContextTest \ | |
| utils.tests.ObservabilityTracingTest \ | |
| utils.tests.CodePlaceMetricsEndpointTest \ | |
| judge.tests.JudgeTaskObservabilityTest \ | |
| account.tests.RequestLogMiddlewareTest \ | |
| problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_api_success_outcome \ | |
| problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_success_metric \ | |
| problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_request_error_metric \ | |
| problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_problem_limit_outcome \ | |
| submission.tests.SubmissionAPITest.test_create_submission_records_success_outcome \ | |
| submission.tests.SubmissionAPITest.test_create_submission_records_problem_not_found_outcome | |
| - name: Validate dashboard contracts | |
| run: python kubernetes/monitoring/validate_dashboards.py --prometheus-rules-output /tmp/codeplace-dashboard-rules.json | |
| - name: Validate alert contracts | |
| run: python kubernetes/monitoring/validate_alerts.py | |
| - name: Render Kubernetes manifests | |
| run: | | |
| kubectl kustomize kubernetes/overlays/dev >/dev/null | |
| kubectl kustomize kubernetes/overlays/prod >/dev/null | |
| kubectl kustomize kubernetes/monitoring >/tmp/codeplace-monitoring.yaml | |
| python -c 'import pathlib, yaml; docs=[doc for doc in yaml.safe_load_all(pathlib.Path("/tmp/codeplace-monitoring.yaml").read_text()) if doc]; assert any(doc.get("kind") == "PodMonitor" and doc.get("metadata", {}).get("name") == "traefik" for doc in docs), "Traefik PodMonitor is missing"; assert not any(doc.get("kind") == "ServiceMonitor" and doc.get("metadata", {}).get("name") == "traefik" for doc in docs), "legacy Traefik ServiceMonitor remains"' | |
| test ! -e kubernetes/monitoring/traefik-service-monitor.yaml | |
| - name: Render pinned monitoring charts | |
| run: | | |
| helm template kube-prometheus-stack kube-prometheus-stack \ | |
| --repo https://prometheus-community.github.io/helm-charts \ | |
| --version 86.3.1 \ | |
| --namespace monitoring \ | |
| --kube-version 1.33.6 \ | |
| --values kubernetes/monitoring/kube-prometheus-stack-values.yaml >/tmp/kube-prometheus-stack.yaml | |
| python -c 'import pathlib, yaml; docs=[doc for doc in yaml.safe_load_all(pathlib.Path("/tmp/kube-prometheus-stack.yaml").read_text()) if doc]; assert any(doc.get("kind") == "Job" and doc.get("metadata", {}).get("name") == "kube-prometheus-stack-crds-upgrade" for doc in docs), "CRD upgrade Job is missing"; grafana=next(doc for doc in docs if doc.get("kind") == "Deployment" and doc.get("metadata", {}).get("name") == "kube-prometheus-stack-grafana"); assert grafana.get("spec", {}).get("strategy", {}).get("type") == "Recreate", "Grafana must use Recreate with its RWO PVC"' | |
| helm template loki loki \ | |
| --repo https://grafana.github.io/helm-charts \ | |
| --version 6.55.0 \ | |
| --namespace monitoring \ | |
| --kube-version 1.33.6 \ | |
| --api-versions monitoring.coreos.com/v1/ServiceMonitor \ | |
| --values kubernetes/monitoring/logs/loki-values.yaml >/dev/null | |
| helm template alloy alloy \ | |
| --repo https://grafana.github.io/helm-charts \ | |
| --version 1.10.0 \ | |
| --namespace monitoring \ | |
| --kube-version 1.33.6 \ | |
| --api-versions monitoring.coreos.com/v1/ServiceMonitor \ | |
| --values kubernetes/monitoring/logs/alloy-values.yaml >/dev/null | |
| - name: Validate Prometheus rules | |
| run: | | |
| python -c 'import pathlib, yaml; source=yaml.safe_load(pathlib.Path("kubernetes/monitoring/prometheus-rules.yaml").read_text()); pathlib.Path("/tmp/codeplace-rules.yaml").write_text(yaml.safe_dump({"groups": source["spec"]["groups"]}))' | |
| docker run --rm --entrypoint /bin/promtool -v /tmp/codeplace-rules.yaml:/rules.yaml:ro prom/prometheus:v3.7.3 check rules /rules.yaml | |
| docker run --rm --entrypoint /bin/promtool -v /tmp/codeplace-dashboard-rules.json:/rules.json:ro prom/prometheus:v3.7.3 check rules /rules.json | |
| docker run --rm --entrypoint /bin/promtool -v /tmp/codeplace-rules.yaml:/rules.yaml:ro -v "$PWD/kubernetes/monitoring/prometheus-rules.test.yaml:/rules.test.yaml:ro" prom/prometheus:v3.7.3 test rules /rules.test.yaml |