diff --git a/.github/workflows/ci2develop.yml b/.github/workflows/ci2develop.yml index db1ce9e7..117ab27d 100644 --- a/.github/workflows/ci2develop.yml +++ b/.github/workflows/ci2develop.yml @@ -65,20 +65,20 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v6 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v4 - name: Login to Harbor - uses: docker/login-action@v1 + uses: docker/login-action@v4 with: registry: ${{ secrets.HARBOR_REGISTRY }} username: ${{ secrets.HARBOR_USERNAME }} password: ${{ secrets.HARBOR_PASSWORD }} - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v7 with: context: ./frontend file: ./frontend/Dockerfile @@ -91,6 +91,8 @@ jobs: SENTRY_ENVIRONMENT=development SENTRY_DSN_FRONTEND=${{ secrets.SENTRY_DSN_FRONTEND }} USE_SENTRY=1 + secrets: | + "sentry_auth_token=${{ secrets.SENTRY_AUTH_TOKEN }}" ci-hub-auth-dev: needs: [detect-changes-by-component] diff --git a/.github/workflows/ci2production-and-release.yml b/.github/workflows/ci2production-and-release.yml index a82d050e..1a666571 100644 --- a/.github/workflows/ci2production-and-release.yml +++ b/.github/workflows/ci2production-and-release.yml @@ -93,20 +93,20 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v6 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v4 - name: Login to Harbor - uses: docker/login-action@v1 + uses: docker/login-action@v4 with: registry: ${{ secrets.HARBOR_REGISTRY }} username: ${{ secrets.HARBOR_USERNAME }} password: ${{ secrets.HARBOR_PASSWORD }} - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v7 with: context: ./frontend file: ./frontend/Dockerfile @@ -119,6 +119,8 @@ jobs: SENTRY_ENVIRONMENT=production SENTRY_DSN_FRONTEND=${{ secrets.SENTRY_DSN_FRONTEND }} USE_SENTRY=1 + secrets: | + "sentry_auth_token=${{ secrets.SENTRY_AUTH_TOKEN }}" ci-hub-auth: needs: [detect-changes-by-component] diff --git a/.github/workflows/monitoring-validate.yml b/.github/workflows/monitoring-validate.yml new file mode 100644 index 00000000..8544d474 --- /dev/null +++ b/.github/workflows/monitoring-validate.yml @@ -0,0 +1,188 @@ +name: Validate monitoring contracts + +on: + pull_request: + paths: + - "kubernetes/monitoring/**" + - "kubernetes/base/hub-auth/**" + - "kubernetes/base/backend/**" + - "kubernetes/base/celery-beat/**" + - "kubernetes/base/celery-worker/**" + - "kubernetes/overlays/dev/**" + - "kubernetes/overlays/prod/**" + - "hub/auth_server/**" + - "backend/account/middleware.py" + - "backend/account/tests.py" + - "backend/judge/tasks.py" + - "backend/judge/tests.py" + - "backend/oj/celery.py" + - "backend/problem/llm_hint.py" + - "backend/problem/views/oj.py" + - "backend/problem/tests.py" + - "backend/submission/views/oj.py" + - "backend/submission/tests.py" + - "backend/utils/observability_metrics.py" + - "backend/utils/observability_tracing.py" + - "backend/utils/tests.py" + - "backend/utils/test_observability_tracing.py" + - ".github/workflows/monitoring-validate.yml" + push: + branches: [develop, main] + paths: + - "kubernetes/monitoring/**" + - "kubernetes/base/hub-auth/**" + - "kubernetes/base/backend/**" + - "kubernetes/base/celery-beat/**" + - "kubernetes/base/celery-worker/**" + - "kubernetes/overlays/dev/**" + - "kubernetes/overlays/prod/**" + - "hub/auth_server/**" + - "backend/account/middleware.py" + - "backend/account/tests.py" + - "backend/judge/tasks.py" + - "backend/judge/tests.py" + - "backend/oj/celery.py" + - "backend/problem/llm_hint.py" + - "backend/problem/views/oj.py" + - "backend/problem/tests.py" + - "backend/submission/views/oj.py" + - "backend/submission/tests.py" + - "backend/utils/observability_metrics.py" + - "backend/utils/observability_tracing.py" + - "backend/utils/tests.py" + - "backend/utils/test_observability_tracing.py" + - ".github/workflows/monitoring-validate.yml" + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + env: + POSTGRES_HOST: 127.0.0.1 + POSTGRES_PORT: "5435" + POSTGRES_DB: onlinejudge + POSTGRES_USER: onlinejudge + POSTGRES_PASSWORD: onlinejudge + REDIS_HOST: 127.0.0.1 + REDIS_PORT: "6380" + services: + postgres: + image: postgres:14 + env: + POSTGRES_DB: onlinejudge + POSTGRES_USER: onlinejudge + POSTGRES_PASSWORD: onlinejudge + ports: + - 5435:5432 + options: >- + --health-cmd "pg_isready -U onlinejudge -d onlinejudge" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + image: redis:7-alpine + ports: + - 6380:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Set up kubectl + uses: azure/setup-kubectl@v4 + with: + version: "v1.33.6" + + - name: Set up Helm + uses: azure/setup-helm@v4 + with: + version: "v3.18.6" + + - name: Install validation dependencies + run: python -m pip install --disable-pip-version-check -r backend/deploy/requirements.txt -r hub/auth_server/requirements.txt pytest PyYAML + + - name: Test hub-auth health contract + working-directory: hub/auth_server + env: + GITHUB_OAUTH_APP_ID: test-client + GITHUB_OAUTH_APP_SECRET: test-secret + run: python -m pytest app/test_main.py + + - name: Test trace resource contract + env: + PYTHONPATH: backend + run: python -m unittest utils.test_observability_tracing + + - name: Test backend observability contracts + working-directory: backend + run: | + printf 'monitoring-validation-secret\n' >data/config/secret.key + python manage.py test \ + utils.tests.CodePlaceCollectorTest \ + utils.tests.CeleryRequestIDContextTest \ + utils.tests.ObservabilityTracingTest \ + utils.tests.CodePlaceMetricsEndpointTest \ + judge.tests.JudgeTaskObservabilityTest \ + account.tests.RequestLogMiddlewareTest \ + problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_api_success_outcome \ + problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_success_metric \ + problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_request_error_metric \ + problem.tests.ProblemLLMHintAPITest.test_stream_llm_hint_records_problem_limit_outcome \ + submission.tests.SubmissionAPITest.test_create_submission_records_success_outcome \ + submission.tests.SubmissionAPITest.test_create_submission_records_problem_not_found_outcome + + - name: Validate dashboard contracts + run: python kubernetes/monitoring/validate_dashboards.py --prometheus-rules-output /tmp/codeplace-dashboard-rules.json + + - name: Validate alert contracts + run: python kubernetes/monitoring/validate_alerts.py + + - name: Render Kubernetes manifests + run: | + kubectl kustomize kubernetes/overlays/dev >/dev/null + kubectl kustomize kubernetes/overlays/prod >/dev/null + kubectl kustomize kubernetes/monitoring >/tmp/codeplace-monitoring.yaml + python -c 'import pathlib, yaml; docs=[doc for doc in yaml.safe_load_all(pathlib.Path("/tmp/codeplace-monitoring.yaml").read_text()) if doc]; assert any(doc.get("kind") == "PodMonitor" and doc.get("metadata", {}).get("name") == "traefik" for doc in docs), "Traefik PodMonitor is missing"; assert not any(doc.get("kind") == "ServiceMonitor" and doc.get("metadata", {}).get("name") == "traefik" for doc in docs), "legacy Traefik ServiceMonitor remains"' + test ! -e kubernetes/monitoring/traefik-service-monitor.yaml + + - name: Render pinned monitoring charts + run: | + helm template kube-prometheus-stack kube-prometheus-stack \ + --repo https://prometheus-community.github.io/helm-charts \ + --version 86.3.1 \ + --namespace monitoring \ + --kube-version 1.33.6 \ + --values kubernetes/monitoring/kube-prometheus-stack-values.yaml >/tmp/kube-prometheus-stack.yaml + python -c 'import pathlib, yaml; docs=[doc for doc in yaml.safe_load_all(pathlib.Path("/tmp/kube-prometheus-stack.yaml").read_text()) if doc]; assert any(doc.get("kind") == "Job" and doc.get("metadata", {}).get("name") == "kube-prometheus-stack-crds-upgrade" for doc in docs), "CRD upgrade Job is missing"; grafana=next(doc for doc in docs if doc.get("kind") == "Deployment" and doc.get("metadata", {}).get("name") == "kube-prometheus-stack-grafana"); assert grafana.get("spec", {}).get("strategy", {}).get("type") == "Recreate", "Grafana must use Recreate with its RWO PVC"' + helm template loki loki \ + --repo https://grafana.github.io/helm-charts \ + --version 6.55.0 \ + --namespace monitoring \ + --kube-version 1.33.6 \ + --api-versions monitoring.coreos.com/v1/ServiceMonitor \ + --values kubernetes/monitoring/logs/loki-values.yaml >/dev/null + helm template alloy alloy \ + --repo https://grafana.github.io/helm-charts \ + --version 1.10.0 \ + --namespace monitoring \ + --kube-version 1.33.6 \ + --api-versions monitoring.coreos.com/v1/ServiceMonitor \ + --values kubernetes/monitoring/logs/alloy-values.yaml >/dev/null + + - name: Validate Prometheus rules + run: | + python -c 'import pathlib, yaml; source=yaml.safe_load(pathlib.Path("kubernetes/monitoring/prometheus-rules.yaml").read_text()); pathlib.Path("/tmp/codeplace-rules.yaml").write_text(yaml.safe_dump({"groups": source["spec"]["groups"]}))' + docker run --rm --entrypoint /bin/promtool -v /tmp/codeplace-rules.yaml:/rules.yaml:ro prom/prometheus:v3.7.3 check rules /rules.yaml + docker run --rm --entrypoint /bin/promtool -v /tmp/codeplace-dashboard-rules.json:/rules.json:ro prom/prometheus:v3.7.3 check rules /rules.json + docker run --rm --entrypoint /bin/promtool -v /tmp/codeplace-rules.yaml:/rules.yaml:ro -v "$PWD/kubernetes/monitoring/prometheus-rules.test.yaml:/rules.test.yaml:ro" prom/prometheus:v3.7.3 test rules /rules.test.yaml diff --git a/backend/account/tests.py b/backend/account/tests.py index 5703fc7f..9efbc914 100644 --- a/backend/account/tests.py +++ b/backend/account/tests.py @@ -18,7 +18,7 @@ from .models import AdminType, ProblemPermission, User from .decorators import login_required, scheduler_only -from .middleware import AdminRoleRequiredMiddleware, RequestIDMiddleware +from .middleware import AdminRoleRequiredMiddleware, RequestIDMiddleware, RequestLogMiddleware from .tasks import calculate_user_score_basis, calculate_user_score_fluctuation @@ -65,6 +65,23 @@ def test_generates_request_id_when_header_has_no_safe_characters(self): self.assertEqual(response["X-Request-ID"], request.request_id) +class RequestLogMiddlewareTest(SimpleTestCase): + + @mock.patch("account.middleware.HTTP_REQUEST_DURATION_SECONDS") + @mock.patch("account.middleware.HTTP_REQUESTS_TOTAL") + def test_records_request_metrics(self, requests_total, duration_seconds): + request = RequestFactory().get("/api/problem") + request.resolver_match = mock.Mock(view_name="problem_api") + middleware = RequestLogMiddleware(lambda _: JsonResponse({}, status=201)) + + middleware(request) + + requests_total.labels.assert_called_once_with("GET", "problem_api", "201") + requests_total.labels.return_value.inc.assert_called_once() + duration_seconds.labels.assert_called_once_with("GET", "problem_api") + duration_seconds.labels.return_value.observe.assert_called_once() + + class AdminRoleRequiredMiddlewareTest(SimpleTestCase): def setUp(self): diff --git a/backend/judge/tasks.py b/backend/judge/tasks.py index e8ac3380..6b18facc 100644 --- a/backend/judge/tasks.py +++ b/backend/judge/tasks.py @@ -4,7 +4,7 @@ from account.models import User from utils.shortcuts import CELERY_TASK_ARGS from utils.observability_tracing import get_tracer -from utils.observability_metrics import JUDGE_TASK_OUTCOME_TOTAL +from utils.observability_metrics import record_judge_task_outcome from submission.models import Submission from judge.dispatcher import JudgeDispatcher @@ -13,7 +13,7 @@ def _record_judge_task_outcome(status, scope): - JUDGE_TASK_OUTCOME_TOTAL.labels(status=status, scope=scope).inc() + record_judge_task_outcome(status, scope) @celery.shared_task(**CELERY_TASK_ARGS()) diff --git a/backend/judge/tests.py b/backend/judge/tests.py index a2ea46f0..d6ca592b 100644 --- a/backend/judge/tests.py +++ b/backend/judge/tests.py @@ -7,37 +7,29 @@ class JudgeTaskObservabilityTest(SimpleTestCase): - @mock.patch("judge.tasks.JUDGE_TASK_OUTCOME_TOTAL") + @mock.patch("judge.tasks.record_judge_task_outcome") @mock.patch("judge.tasks.JudgeDispatcher") @mock.patch("judge.tasks.User") @mock.patch("judge.tasks.Submission") - def test_judge_task_records_success_outcome(self, submission_model, user_model, dispatcher, outcome_total): + def test_judge_task_records_success_outcome(self, submission_model, user_model, dispatcher, record_outcome): submission = mock.Mock(user_id=1, contest_id=None) submission_model.objects.get.return_value = submission user_model.objects.get.return_value = mock.Mock(is_disabled=False) - labels = mock.Mock() - outcome_total.labels.return_value = labels - judge_task.run(10, 20) dispatcher.assert_called_once_with(10, 20) dispatcher.return_value.judge.assert_called_once() - outcome_total.labels.assert_called_once_with(status="success", scope="practice") - labels.inc.assert_called_once() + record_outcome.assert_called_once_with("success", "practice") - @mock.patch("judge.tasks.JUDGE_TASK_OUTCOME_TOTAL") + @mock.patch("judge.tasks.record_judge_task_outcome") @mock.patch("judge.tasks.JudgeDispatcher") @mock.patch("judge.tasks.User") @mock.patch("judge.tasks.Submission") - def test_judge_task_records_disabled_user_outcome(self, submission_model, user_model, dispatcher, outcome_total): + def test_judge_task_records_disabled_user_outcome(self, submission_model, user_model, dispatcher, record_outcome): submission = mock.Mock(user_id=1, contest_id=7) submission_model.objects.get.return_value = submission user_model.objects.get.return_value = mock.Mock(is_disabled=True) - labels = mock.Mock() - outcome_total.labels.return_value = labels - judge_task.run(10, 20) dispatcher.assert_not_called() - outcome_total.labels.assert_called_once_with(status="user_disabled", scope="contest") - labels.inc.assert_called_once() + record_outcome.assert_called_once_with("user_disabled", "contest") diff --git a/backend/oj/celery.py b/backend/oj/celery.py index b9890384..e6b54578 100644 --- a/backend/oj/celery.py +++ b/backend/oj/celery.py @@ -1,6 +1,8 @@ import os import celery +from celery.signals import task_postrun, task_prerun +from utils.observability_context import reset_request_id, set_request_id from utils.observability_tracing import configure_opentelemetry os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'oj.settings') @@ -9,3 +11,20 @@ app = celery.Celery('scheduler') app.config_from_object('django.conf:settings', namespace='CELERY') app.autodiscover_tasks() + +_request_id_tokens = {} + + +@task_prerun.connect +def bind_request_id(task_id=None, task=None, **kwargs): + headers = getattr(getattr(task, "request", None), "headers", None) or {} + request_id = headers.get("x-request-id") + if task_id and request_id: + _request_id_tokens[task_id] = set_request_id(request_id) + + +@task_postrun.connect +def unbind_request_id(task_id=None, **kwargs): + token = _request_id_tokens.pop(task_id, None) + if token is not None: + reset_request_id(token) diff --git a/backend/submission/tests.py b/backend/submission/tests.py index 237b89dd..60aee792 100644 --- a/backend/submission/tests.py +++ b/backend/submission/tests.py @@ -57,6 +57,20 @@ def test_create_submission_success(self): self.assertSuccess(resp) self.assertIn("submission_id", resp.data["data"]) + def test_create_submission_propagates_request_id_to_judge_task(self): + data = copy.deepcopy(DEFAULT_SUBMISSION_DATA) + data["problem_id"] = self.problem.id + + with mock.patch('judge.tasks.judge_task.apply_async') as apply_async: + resp = self.client.post(self.url, data=data, HTTP_X_REQUEST_ID="submission-request-1") + + self.assertSuccess(resp) + submission = Submission.objects.get(id=resp.data["data"]["submission_id"]) + apply_async.assert_called_once_with( + args=(submission.id, self.problem.id), + headers={"x-request-id": "submission-request-1"}, + ) + @mock.patch("submission.views.oj.SUBMISSION_CREATE_OUTCOME_TOTAL") def test_create_submission_records_success_outcome(self, outcome_total): data = copy.deepcopy(DEFAULT_SUBMISSION_DATA) diff --git a/backend/submission/views/oj.py b/backend/submission/views/oj.py index bc41a6b4..c5eec73c 100644 --- a/backend/submission/views/oj.py +++ b/backend/submission/views/oj.py @@ -105,7 +105,10 @@ def post(self, request): # JudgeDispatcher(submission.id, problem.id).judge() try: - judge_task.apply_async(args=(submission.id, problem.id)) + judge_task.apply_async( + args=(submission.id, problem.id), + headers={"x-request-id": request.request_id}, + ) except Exception: self._record_create_outcome("enqueue_error", scope) logger.exception("Failed to enqueue judge task for submission %s", submission.id) diff --git a/backend/utils/observability_metrics.py b/backend/utils/observability_metrics.py index be99392f..b9df7818 100644 --- a/backend/utils/observability_metrics.py +++ b/backend/utils/observability_metrics.py @@ -1,7 +1,7 @@ import logging from prometheus_client import Counter, Histogram, REGISTRY -from prometheus_client.core import GaugeMetricFamily +from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily from django.conf import settings from redis import Redis from redis.sentinel import Sentinel @@ -12,6 +12,7 @@ logger = logging.getLogger(__name__) CELERY_BROKER_QUEUE_KEY = "celery" +JUDGE_TASK_OUTCOME_KEY = "observability:judge_task_outcomes" HTTP_REQUESTS_TOTAL = Counter( "codeplace_http_requests_total", @@ -45,11 +46,6 @@ "Total user-facing submission create API outcomes.", ["status", "scope"], ) -JUDGE_TASK_OUTCOME_TOTAL = Counter( - "codeplace_judge_task_outcome_total", - "Total Celery judge task outcomes.", - ["status", "scope"], -) class CodePlaceCollector: @@ -63,21 +59,51 @@ def describe(self): "codeplace_celery_broker_queue_length", "Number of Celery tasks waiting in the Redis broker default queue.", ) + yield CounterMetricFamily( + "codeplace_judge_task_outcome", + "Total Celery judge task outcomes stored in Redis.", + labels=["status", "scope"], + ) + yield GaugeMetricFamily( + "codeplace_collector_success", + "Whether the most recent custom metric collection succeeded.", + labels=["collector"], + ) + yield GaugeMetricFamily( + "codeplace_redis_sentinel_health", + "Whether Redis Sentinel can resolve the master and satisfy quorum.", + labels=["check"], + ) def collect(self): - yield self._waiting_queue_length() - yield self._celery_broker_queue_length() + success = GaugeMetricFamily( + "codeplace_collector_success", + "Whether the most recent custom metric collection succeeded.", + labels=["collector"], + ) + collectors = ( + ("waiting_queue", self._waiting_queue_length), + ("celery_broker_queue", self._celery_broker_queue_length), + ("judge_task_outcomes", self._judge_task_outcomes), + ("redis_sentinel_health", self._redis_sentinel_health), + ) + for name, collector in collectors: + try: + metric = collector() + except Exception as e: + logger.warning("Failed to collect %s: %s", name, e) + success.add_metric([name], 0) + else: + success.add_metric([name], 1) + yield metric + yield success def _waiting_queue_length(self): metric = GaugeMetricFamily( "codeplace_waiting_queue_length", "Number of submissions waiting because no judge-server was available.", ) - try: - metric.add_metric([], cache.llen(CacheKey.waiting_queue) or 0) - except Exception as e: - logger.warning("Failed to collect waiting queue length: %s", e) - metric.add_metric([], 0) + metric.add_metric([], cache.llen(CacheKey.waiting_queue) or 0) return metric def _celery_broker_queue_length(self): @@ -85,20 +111,60 @@ def _celery_broker_queue_length(self): "codeplace_celery_broker_queue_length", "Number of Celery tasks waiting in the Redis broker default queue.", ) - try: - metric.add_metric([], self._celery_broker_client().llen(CELERY_BROKER_QUEUE_KEY) or 0) - except Exception as e: - logger.warning("Failed to collect Celery broker queue length: %s", e) - metric.add_metric([], 0) + metric.add_metric([], self._celery_broker_client().llen(CELERY_BROKER_QUEUE_KEY) or 0) return metric + @staticmethod + def _judge_task_outcomes(): + metric = CounterMetricFamily( + "codeplace_judge_task_outcome", + "Total Celery judge task outcomes stored in Redis.", + labels=["status", "scope"], + ) + for raw_field, raw_value in cache.hgetall(JUDGE_TASK_OUTCOME_KEY).items(): + field = raw_field.decode() if isinstance(raw_field, bytes) else str(raw_field) + value = raw_value.decode() if isinstance(raw_value, bytes) else raw_value + status, scope = field.split(":", 1) + metric.add_metric([status, scope], float(value)) + return metric + + @classmethod + def _redis_sentinel_health(cls): + if not getattr(settings, "REDIS_USE_SENTINEL", False): + raise RuntimeError("Redis Sentinel monitoring is disabled") + sentinel = cls._sentinel_client() + master_name = getattr(settings, "REDIS_SENTINEL_MASTER_NAME", "mymaster") + sentinel.discover_master(master_name) + quorum_available = False + for client in sentinel.sentinels: + try: + client.execute_command("SENTINEL", "CKQUORUM", master_name) + except Exception: + continue + quorum_available = True + break + if not quorum_available: + raise RuntimeError(f"Redis Sentinel quorum is unavailable for {master_name}") + metric = GaugeMetricFamily( + "codeplace_redis_sentinel_health", + "Whether Redis Sentinel can resolve the master and satisfy quorum.", + labels=["check"], + ) + metric.add_metric(["master"], 1) + metric.add_metric(["quorum"], 1) + return metric + + @staticmethod + def _sentinel_client(): + return Sentinel( + getattr(settings, "REDIS_SENTINEL_HOSTS"), + socket_timeout=1, + ) + @staticmethod def _celery_broker_client(): if getattr(settings, "REDIS_USE_SENTINEL", False): - sentinel = Sentinel( - getattr(settings, "REDIS_SENTINEL_HOSTS"), - socket_timeout=1, - ) + sentinel = CodePlaceCollector._sentinel_client() return sentinel.master_for( getattr(settings, "REDIS_SENTINEL_MASTER_NAME", "mymaster"), db=4, @@ -107,6 +173,13 @@ def _celery_broker_client(): return Redis.from_url(getattr(settings, "CELERY_BROKER_URL")) +def record_judge_task_outcome(status, scope): + try: + cache.hincrby(JUDGE_TASK_OUTCOME_KEY, f"{status}:{scope}", 1) + except Exception as e: + logger.warning("Failed to persist judge task outcome metric: %s", e) + + def register_codeplace_metrics(): if getattr(register_codeplace_metrics, "_registered", False): return diff --git a/backend/utils/observability_tracing.py b/backend/utils/observability_tracing.py index c3c76817..ef2225bf 100644 --- a/backend/utils/observability_tracing.py +++ b/backend/utils/observability_tracing.py @@ -1,13 +1,16 @@ import logging +import os from opentelemetry import trace -from utils.shortcuts import get_env - logger = logging.getLogger(__name__) _OTEL_CONFIGURED = False +def get_env(name, default=""): + return os.environ.get(name, default) + + def get_tracer(name): return trace.get_tracer(name) @@ -25,6 +28,16 @@ def get_current_trace_context(): } +def get_deployment_environment(): + """Return the telemetry environment without changing Django's OJ_ENV contract.""" + return get_env("OTEL_DEPLOYMENT_ENVIRONMENT", get_env("OJ_ENV", "dev")) + + +def get_service_name(default): + """Keep process identity stable when Django imports the Celery app first.""" + return get_env("OTEL_SERVICE_NAME", default) + + def configure_opentelemetry(service_name): global _OTEL_CONFIGURED if get_env("OTEL_ENABLED", "0").lower() not in ("1", "true", "yes", "on"): @@ -32,6 +45,19 @@ def configure_opentelemetry(service_name): if _OTEL_CONFIGURED: return + try: + _configure_opentelemetry(service_name) + except Exception: + logger.exception( + "OpenTelemetry setup failed; continuing without complete tracing for %s", + service_name, + ) + finally: + _OTEL_CONFIGURED = True + + +def _configure_opentelemetry(service_name): + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.instrumentation.celery import CeleryInstrumentor from opentelemetry.instrumentation.django import DjangoInstrumentor @@ -50,15 +76,14 @@ def configure_opentelemetry(service_name): sampler_ratio = 0.05 sampler_ratio = min(max(sampler_ratio, 0), 1) resource = Resource.create({ - "service.name": service_name, - "deployment.environment": get_env("OJ_ENV", "dev"), + "service.name": get_service_name(service_name), + "deployment.environment": get_deployment_environment(), }) provider = TracerProvider(resource=resource, sampler=ParentBased(TraceIdRatioBased(sampler_ratio))) endpoint = get_env("OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector.monitoring.svc.cluster.local:4317") current_provider = trace.get_tracer_provider() if type(current_provider).__name__ != "ProxyTracerProvider": logger.info("OpenTelemetry tracer provider is already configured; skipping CodePlace setup") - _OTEL_CONFIGURED = True return provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=True))) @@ -69,4 +94,3 @@ def configure_opentelemetry(service_name): Psycopg2Instrumentor().instrument() RedisInstrumentor().instrument() CeleryInstrumentor().instrument() - _OTEL_CONFIGURED = True diff --git a/backend/utils/test_observability_tracing.py b/backend/utils/test_observability_tracing.py new file mode 100644 index 00000000..dd725830 --- /dev/null +++ b/backend/utils/test_observability_tracing.py @@ -0,0 +1,35 @@ +import os +import unittest +from unittest.mock import patch + +from utils import observability_tracing + + +class ObservabilityResourceContractTest(unittest.TestCase): + + def test_telemetry_environment_is_independent_from_django_environment(self): + with patch.dict( + os.environ, + { + "OJ_ENV": "production", + "OTEL_DEPLOYMENT_ENVIRONMENT": "prod", + }, + clear=True, + ): + environment = observability_tracing.get_deployment_environment() + + self.assertEqual(environment, "prod") + + def test_service_name_override_wins_over_import_order_default(self): + with patch.dict( + os.environ, + {"OTEL_SERVICE_NAME": "codeplace-backend"}, + clear=True, + ): + service_name = observability_tracing.get_service_name("codeplace-celery") + + self.assertEqual(service_name, "codeplace-backend") + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/utils/tests.py b/backend/utils/tests.py index 5c7c5880..89db7387 100644 --- a/backend/utils/tests.py +++ b/backend/utils/tests.py @@ -5,11 +5,12 @@ from unittest.mock import patch, mock_open, MagicMock from django.test import RequestFactory, SimpleTestCase, TestCase, override_settings from django.core.cache import cache -from prometheus_client.core import GaugeMetricFamily +from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily from utils.json_logging import CodePlaceJsonFormatter from utils.api import APIView -from utils.observability_metrics import CodePlaceCollector +from utils.observability_context import get_request_id +from utils.observability_metrics import CodePlaceCollector, record_judge_task_outcome from utils import observability_tracing from utils.testcase_cache import TestCaseCacheManager from utils.throttling import TokenBucket @@ -78,6 +79,61 @@ def test_celery_broker_queue_length_is_collected_from_redis(self): broker_client.llen.assert_called_once_with("celery") self.assertEqual(samples["codeplace_celery_broker_queue_length"][0].value, 7) + def test_failed_collector_emits_failure_without_fake_queue_value(self): + collector = CodePlaceCollector() + with patch.object(collector, "_waiting_queue_length", side_effect=RuntimeError("redis down")), \ + patch.object(collector, "_celery_broker_queue_length") as broker, \ + patch.object(collector, "_judge_task_outcomes") as outcomes: + broker.return_value = GaugeMetricFamily("codeplace_celery_broker_queue_length", "broker") + outcomes.return_value = CounterMetricFamily( + "codeplace_judge_task_outcome", "outcomes", labels=["status", "scope"] + ) + metrics = list(collector.collect()) + + metric_names = [metric.name for metric in metrics] + self.assertNotIn("codeplace_waiting_queue_length", metric_names) + success = next(metric for metric in metrics if metric.name == "codeplace_collector_success") + waiting_sample = next(sample for sample in success.samples if sample.labels["collector"] == "waiting_queue") + self.assertEqual(waiting_sample.value, 0) + + @patch("utils.observability_metrics.cache") + def test_judge_task_outcomes_are_collected_from_shared_redis(self, redis_cache): + redis_cache.hgetall.return_value = { + b"success:practice": b"7", + b"error:contest": b"2", + } + + metric = CodePlaceCollector()._judge_task_outcomes() + + values = {(sample.labels["status"], sample.labels["scope"]): sample.value for sample in metric.samples} + self.assertEqual(values[("success", "practice")], 7) + self.assertEqual(values[("error", "contest")], 2) + + @override_settings( + REDIS_USE_SENTINEL=True, + REDIS_SENTINEL_MASTER_NAME="mymaster", + ) + def test_redis_sentinel_health_checks_master_and_quorum(self): + sentinel = MagicMock() + sentinel.sentinels = [MagicMock()] + with patch.object(CodePlaceCollector, "_sentinel_client", return_value=sentinel): + metric = CodePlaceCollector()._redis_sentinel_health() + + sentinel.discover_master.assert_called_once_with("mymaster") + sentinel.sentinels[0].execute_command.assert_called_once_with("SENTINEL", "CKQUORUM", "mymaster") + values = {sample.labels["check"]: sample.value for sample in metric.samples} + self.assertEqual(values, {"master": 1, "quorum": 1}) + + @patch("utils.observability_metrics.cache") + def test_judge_task_outcome_persistence_is_fail_open(self, redis_cache): + redis_cache.hincrby.side_effect = RuntimeError("redis down") + + record_judge_task_outcome("error", "practice") + + redis_cache.hincrby.assert_called_once_with( + "observability:judge_task_outcomes", "error:practice", 1 + ) + @override_settings(REDIS_USE_SENTINEL=False, CELERY_BROKER_URL="redis://127.0.0.1:6380/4") def test_celery_broker_client_uses_celery_broker_url(self): with patch("utils.observability_metrics.Redis.from_url") as from_url: @@ -113,7 +169,12 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): ) broker_queue_metric.add_metric([], 0) with patch.object(CodePlaceCollector, "_waiting_queue_length", return_value=waiting_queue_metric), \ - patch.object(CodePlaceCollector, "_celery_broker_queue_length", return_value=broker_queue_metric): + patch.object(CodePlaceCollector, "_celery_broker_queue_length", return_value=broker_queue_metric), \ + patch.object(CodePlaceCollector, "_judge_task_outcomes", return_value=CounterMetricFamily( + "codeplace_judge_task_outcome", "outcomes", labels=["status", "scope"] + )), patch.object(CodePlaceCollector, "_redis_sentinel_health", return_value=GaugeMetricFamily( + "codeplace_redis_sentinel_health", "sentinel", labels=["check"] + )): response = self.client.get("/metrics") self.assertEqual(response.status_code, 200) @@ -123,6 +184,21 @@ def test_metrics_endpoint_exposes_django_and_codeplace_metrics(self): self.assertIn("codeplace_celery_broker_queue_length", body) +class CeleryRequestIDContextTest(SimpleTestCase): + + def test_task_signals_bind_and_reset_request_id(self): + from oj.celery import bind_request_id, unbind_request_id + + task = MagicMock() + task.request.headers = {"x-request-id": "celery-request-1"} + + bind_request_id(task_id="task-1", task=task) + self.assertEqual(get_request_id(), "celery-request-1") + + unbind_request_id(task_id="task-1") + self.assertIsNone(get_request_id()) + + class ObservabilityTracingTest(SimpleTestCase): def tearDown(self): @@ -141,6 +217,16 @@ def fail_import(name, *args, **kwargs): patch("builtins.__import__", side_effect=fail_import): observability_tracing.configure_opentelemetry("codeplace-test") + def test_configure_opentelemetry_is_fail_open(self): + with patch("utils.observability_tracing.get_env", return_value="1"), \ + patch("utils.observability_tracing._configure_opentelemetry", + side_effect=RuntimeError("collector setup failed")), \ + patch.object(observability_tracing.logger, "exception") as log_exception: + observability_tracing.configure_opentelemetry("codeplace-test") + + self.assertTrue(observability_tracing._OTEL_CONFIGURED) + log_exception.assert_called_once() + class CodePlaceJsonFormatterTest(SimpleTestCase): diff --git a/docs/content/developer/infra/observability-improvement-plan/_index.md b/docs/content/developer/infra/observability-improvement-plan/_index.md index 4649ee4b..e83e89f4 100644 --- a/docs/content/developer/infra/observability-improvement-plan/_index.md +++ b/docs/content/developer/infra/observability-improvement-plan/_index.md @@ -34,6 +34,10 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 - `codeplace_judge_task_outcome_total{status,scope}` - `codeplace_waiting_queue_length` - `codeplace_celery_broker_queue_length` +- `codeplace_collector_success{collector}` +- `codeplace_redis_sentinel_health{check}` + +Celery worker에서 발생한 judge outcome은 worker 프로세스 메모리가 아니라 Redis hash에 누적하고 backend `/metrics` collector가 노출합니다. 모든 backend replica가 동일한 공유 counter를 노출하므로 비율은 replica를 더하지 않고 `avg by (namespace, scope, status)`로 중복 제거하며, 실패 여부는 `max by (namespace, scope, status)`로 판정합니다. Collector가 Redis 값을 읽지 못하면 queue 값을 `0`으로 대체하지 않고 해당 시계열을 생략한 뒤 `codeplace_collector_success=0`을 노출합니다. `/metrics`와 `/api/health` 요청은 API request rate/latency metric과 request completion log에서 제외합니다. @@ -41,6 +45,8 @@ backend는 `django-prometheus` 기반 `/metrics` 엔드포인트를 제공합니 backend는 `X-Request-ID`를 수용하고, 없으면 request ID를 생성한 뒤 응답 header로 반환합니다. +Submission API는 동일한 request ID를 Celery task header로 전달하고 worker task context에 bind합니다. 따라서 HTTP 요청 로그와 비동기 judge 로그를 같은 ID로 조회할 수 있습니다. + Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설정합니다. JSON 로그 필드는 다음을 기준으로 합니다. - `timestamp` @@ -61,6 +67,8 @@ Kubernetes backend/celery 환경에서는 `JSON_LOGGING=1`을 기본으로 설 frontend nginx의 Kubernetes 설정은 access log를 `/dev/stdout`, error log를 `/dev/stderr`로 출력합니다. +브라우저 runtime exception은 Sentry가 수집합니다. CI frontend build는 배포 `APP_VERSION`과 같은 Sentry release에 source map을 업로드하고, 업로드 후 image에서 `.map` 파일을 제거합니다. `SENTRY_AUTH_TOKEN`은 GitHub Actions BuildKit secret으로만 전달합니다. + Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라우드 object storage를 사용할 수 없는 현재 조건에서는 Loki SingleBinary + Longhorn PVC를 단기 보관 기준선으로 둡니다. - dev log retention: 3일. @@ -69,7 +77,9 @@ Pod 로그 수집은 Grafana Alloy와 Loki로 처리합니다. 클라우드 obje ### Tracing -OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OTEL_ENABLED=0`입니다. dev/staging에서 먼저 활성화하고 collector/Tempo/sampling 확인 후 prod 활성화를 결정합니다. +OpenTelemetry는 앱 초기화 코드에 내장되어 있습니다. base manifest는 안전한 기본값으로 `OTEL_ENABLED=0`을 유지하지만, dev와 prod overlay는 모두 `OTEL_ENABLED=1`로 활성화합니다. dev는 `OJ_ENV=dev`와 `OTEL_DEPLOYMENT_ENVIRONMENT=dev`, prod는 `OJ_ENV=production`과 `OTEL_DEPLOYMENT_ENVIRONMENT=prod`를 명시해 Django 실행 환경과 trace resource 환경을 독립적으로 고정합니다. backend, celery-worker, celery-beat는 각각 고정된 `OTEL_SERVICE_NAME`을 사용하므로 Django가 Celery app을 먼저 import해도 service identity가 바뀌지 않습니다. + +계측 초기화 실패는 애플리케이션 기동 실패로 전파하지 않고 구조화 오류 로그를 남긴 뒤 fail-open 처리합니다. dev와 prod trace는 동일한 Collector/Tempo 경로를 사용하되 Grafana TraceQL에서 `deployment.environment`로 분리합니다. 자동 계측 대상은 Django, requests, psycopg2, Redis, Celery입니다. 제출/채점 경로에는 manual span을 추가합니다. @@ -86,14 +96,30 @@ OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OT - `backend-service-monitor.yaml`: backend `/metrics` scrape, interval 15s. - `blackbox-exporter.yaml`, `public-endpoint-probes.yaml`: frontend/hub-auth/Grafana 공개 HTTPS probe. - `datastore-pod-monitors.yaml`: CNPG/Redis exporter scrape. +- `traefik-pod-monitor.yaml`: K3s Traefik Pod의 `metrics` port 직접 scrape. +- `dcgm-exporter.yaml`: GPU 상태 metric과 scrape target. +- `otel-collector.yaml`, `tempo.yaml`: dev/prod trace 수집, 전달, 저장 경로. - `logs/loki-values.yaml`, `logs/alloy-values.yaml`: Loki/Alloy Helm values. - `prometheus-rules.yaml`: P0/P1 fast alert rules와 일부 recording rules. - `alertmanager-config.yaml`: P0/P1 Discord alert routing. -- `grafana-dashboard-codeplace.yaml`: CodePlace overview dashboard. `namespace` 변수로 `code-place-dev`와 `code-place-prod`를 분리해서 조회합니다. -- `grafana-dashboard-logs.yaml`: Loki/Alloy 상태와 주요 앱 로그 조회 dashboard. -- `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, shared Grafana ingress와 dashboard sidecar 설정. +- `grafana-dashboard-codeplace.yaml`: 환경별 서비스 SLI와 핵심 현재 상태를 보여주는 CodePlace overview dashboard. +- `grafana-dashboard-platform.yaml`: 환경별 workload, resource, PostgreSQL, Redis 상세 진단 dashboard. +- `grafana-dashboard-ai-api.yaml`: `prod`/`dev`를 선택하는 AI API outcome/latency dashboard. +- `grafana-dashboard-ai-inference.yaml`: prod 전용 vLLM/GPU runtime dashboard. +- `grafana-dashboard-logs.yaml`: 환경별 앱 로그와 request ID 조사 dashboard. +- `grafana-dashboard-log-pipeline.yaml`: cluster-global Loki/Alloy 수집 경로 상태 dashboard. +- `grafana-dashboard-kubernetes-events.yaml`: 환경별 Pod 상태와 Warning event dashboard. +- `grafana-dashboard-monitoring-stack.yaml`: 필수 scrape 경로의 `ABSENT`/`DOWN`/`UP` 상태 dashboard. +- `grafana-dashboard-public-endpoints.yaml`: 환경별 frontend/hub-auth 공개 probe dashboard. +- `grafana-dashboard-storage.yaml`: 환경별 PVC와 cluster-global Longhorn 상태 dashboard. +- `grafana-dashboard-traces.yaml`: 환경별 backend/celery trace와 Collector/Tempo 상태 dashboard. +- `kube-prometheus-stack-values.yaml`: Prometheus/Alertmanager selector, evaluation interval, CRD upgrade hook, shared Grafana ingress와 dashboard sidecar 설정. - `kustomization.yaml`: 기존 `monitoring` namespace의 kube-prometheus-stack/Grafana/Alertmanager에 붙일 CodePlace monitoring 리소스 묶음. +환경별 dashboard는 단일 `environment=prod,dev` 변수로 조회 범위를 고정하고 `All` 선택을 제공하지 않습니다. prod 전용 vLLM/GPU runtime과 cluster-global log pipeline은 별도 dashboard로 분리해 서로 다른 범위의 graph가 한 화면에 섞이지 않도록 합니다. 전체 dashboard는 11개이며, 공통 dashboard dropdown은 변수와 시간 범위를 유지합니다. cluster-global panel은 제목과 설명에 범위를 명시합니다. + +Grafana는 단일 Longhorn `ReadWriteOnce` PVC를 사용하므로 upgrade 전략을 `Recreate`로 고정합니다. `RollingUpdate`로 바꾸면 기존 Pod가 volume을 점유한 상태에서 새 Pod가 대기해 Helm atomic upgrade가 실패할 수 있습니다. + `alertmanager-contact-points` Secret은 repo에 평문으로 저장하지 않습니다. 운영자는 SealedSecret으로 `monitoring` namespace에 생성합니다. AlertmanagerConfig는 generic webhook이 아니라 Prometheus Operator의 native `discordConfigs`를 사용합니다. ## 4. 빠른 알림 정책 @@ -102,17 +128,24 @@ OpenTelemetry는 앱 초기화 코드에 내장되어 있지만 기본값은 `OT P0는 `group_wait=10s`, `repeat_interval=15m`로 Discord에 빠르게 전달합니다. +- `TraefikScrapeUnavailable`: Traefik target 부재 또는 scrape 실패 1분 지속. +- `AlloyScrapeUnavailable`: Alloy target 부재 또는 scrape 실패 1분 지속. - `BackendTargetDown`: backend scrape 실패 1분 지속. - `Api5xxSpike`: backend 5xx 비율 5% 초과 2분 지속. - `Ingress5xxSpike`: Traefik 5xx 비율 5% 초과 2분 지속. +- `PublicEndpointDown`: prod frontend 또는 hub-auth HTTPS probe의 실제 실패 1분 지속. 실패한 service와 URL을 알림에 표시. - `PostgresUnavailable`: PostgreSQL Pod not ready 또는 readiness metric missing 1분 지속. +- `PostgresPrimaryUnavailable`: `postgres-rw` ready endpoint 부재 1분 지속. - `RedisUnavailable`: Redis/Sentinel Pod not ready 또는 readiness metric missing 1분 지속. +- `RedisSentinelUnavailable`: Sentinel master 탐색 또는 quorum 검사 실패 1분 지속. - `LokiUnavailable`: Loki Pod not ready 1분 지속. +- `DCGMExporterUnavailable`: GPU metric target 부재 또는 scrape 실패 2분 지속. ### P1 -P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. +P1은 `group_wait=30s`, `repeat_interval=1h`를 사용합니다. prod와 cluster-global P1은 Discord로 전달하고 dev P1은 muted receiver로 분리합니다. +- `PrometheusHADegraded`: ready Prometheus replica가 2개 미만인 상태 1분 지속. - `ApiLatencyHigh`: p95 latency 2초 초과 5분 지속. - `JudgeWaitingQueueBacklog`: `waiting_queue` 5 초과 3분 지속. - `CeleryWorkerRestarting`: worker restart 3회 이상/15분. @@ -120,28 +153,53 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - `PodCrashLooping`: 주요 Pod restart 증가 5분 지속. - `PVCAlmostFull`: PVC 사용률 85% 초과 10분 지속. - `CodePlaceCollectorFailed`: backend custom metric collector 실패 5분 지속. -- `FrontendRuntimeErrorsSpike`: frontend runtime error report 증가 5분 지속. +- `PostgresCollectorUnavailable`: CNPG collector 응답 실패 5분 지속. +- `OpenTelemetryCollectorUnavailable`: collector target 실패 5분 지속. +- `TempoUnavailable`: Tempo target 실패 5분 지속. +- `OpenTelemetrySpanExportFailures`: Tempo span 전송 실패 5분 지속. + +공개 endpoint의 실제 HTTP 실패와 synthetic probe metric 결측은 같은 장애로 취급하지 않습니다. `PublicEndpointDown`은 `probe_success == 0`만 감지하고, 수집 경로 결측은 `PublicEndpointProbeMissing` P1으로 별도 통지합니다. Alertmanager는 service별로 알림을 묶고 본문에 service와 instance를 표시합니다. 반복 전송은 신규 장애로 오인되지 않도록 `활성` 상태로 표기합니다. + +Frontend runtime exception은 Kubernetes 로그가 아니라 Sentry release와 source map을 기준으로 확인합니다. 브라우저 오류를 backend Loki 로그로 가장하는 별도 panel이나 Prometheus alert은 두지 않습니다. ## 5. 운영 확인 절차 -1. kube-prometheus-stack을 `monitoring` namespace에 설치하거나 업그레이드할 때 `kubernetes/monitoring/kube-prometheus-stack-values.yaml`을 함께 적용합니다. 이 값 파일이 `monitoring.code-place-dev.site` Grafana ingress와 dashboard sidecar 설정까지 관리합니다. Prometheus Operator CRD가 `AlertmanagerConfig.discordConfigs`를 지원하는 버전인지 확인합니다. +1. dev/prod 애플리케이션 delivery가 완료되어 새 backend custom metrics/trace 코드와 hub-auth `/healthz` endpoint를 포함한 image tag가 각 overlay에 반영됐는지 확인합니다. monitoring probe를 application image보다 먼저 배포하지 않습니다. 2. `alertmanager-contact-points` Secret을 운영 클러스터의 `monitoring` namespace에 생성합니다. 이 값은 Kubernetes Secret으로 참조되며, Pod 파일로 mount하지 않습니다. -3. 애플리케이션은 환경별 overlay로 적용합니다. - - dev: `kubectl apply -k kubernetes/overlays/dev` - - prod: `kubectl apply -k kubernetes/overlays/prod` -4. CodePlace monitoring 리소스는 기존 kube-prometheus-stack이 있는 클러스터에 별도로 적용합니다. - - `kubectl apply -k kubernetes/monitoring` -5. Prometheus target에서 `backend`, `postgres`, `redis`, `blackbox-exporter`, `loki`, `alloy` target이 healthy인지 확인합니다. -6. Grafana의 `CodePlace Overview` dashboard에서 request rate, 5xx, latency, submission/judge outcome, waiting queue panel을 확인합니다. -7. Grafana의 `CodePlace Logs` dashboard 또는 Explore에서 `{namespace="code-place-dev"}` 쿼리가 로그를 반환하는지 확인합니다. -8. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. +3. `kubernetes/monitoring/logs/README.md`의 고정 chart version과 values로 kube-prometheus-stack, Loki, Alloy를 설치 또는 upgrade한 뒤 CodePlace monitoring resource를 적용합니다. + + ```sh + kubectl apply -k kubernetes/monitoring + ``` + +4. Traefik PodMonitor가 생성된 것을 확인한 뒤 `kubectl apply`가 정리하지 못하는 기존 ServiceMonitor를 한 번만 삭제합니다. + + ```sh + kubectl -n monitoring get podmonitor traefik + kubectl -n monitoring delete servicemonitor traefik --ignore-not-found + ``` + +5. Prometheus를 port-forward하고 독립 live verifier로 필수 scrape/probe/custom collector/HA 계약을 확인합니다. + + ```sh + # terminal 1 + kubectl -n monitoring port-forward service/kube-prometheus-stack-prometheus 19090:9090 + + # terminal 2 + python3 kubernetes/monitoring/verify_live.py --prometheus-url http://127.0.0.1:19090 + ``` + +6. Grafana에서 11개 CodePlace dashboard가 provision됐는지 확인합니다. 환경별 dashboard는 `prod`와 `dev`를 각각 선택해 확인하고, `CodePlace AI Runtime (Prod)`와 `CodePlace Log Pipeline`에는 환경 selector가 없는지 확인합니다. +7. `CodePlace Logs` dashboard 또는 Explore에서 `{namespace="code-place-dev"}`와 `{namespace="code-place-prod"}`가 각각 로그를 반환하고, `CodePlace Log Pipeline`에서 Loki/Alloy 전역 상태가 조회되는지 확인합니다. +8. `CodePlace Traces`에서 dev/prod를 각각 선택해 `deployment.environment`가 분리되는지 확인합니다. +9. test alert 또는 임시 rule로 P0 webhook 수신 시간이 1분 이내인지 확인합니다. 운영 적용 전제는 다음과 같습니다. - `/metrics`는 cluster 내부 scrape 전용이며 외부 Ingress에 노출하지 않습니다. - Discord webhook URL은 Git에 저장하지 않습니다. - Docker Swarm monitoring 구성은 레거시로 유지하고 신규 관측성 리소스와 분리합니다. -- OpenTelemetry는 base/prod `OTEL_ENABLED=0` 기본값을 유지합니다. Collector/Tempo 리소스는 포함하지만 dev에서 먼저 확인한 뒤 prod 활성화를 결정합니다. +- OpenTelemetry는 base의 비활성 기본값 위에서 dev/prod overlay가 명시적으로 활성화합니다. service name과 deployment environment label을 제거하거나 두 환경을 같은 값으로 설정하지 않습니다. - P0/P1 알림 라우팅은 AlertmanagerConfig로 관리하며 Grafana UI 수동 설정에 의존하지 않습니다. ## 6. 검증 @@ -153,9 +211,13 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다. - Metrics endpoint: Django test client 기준 `/metrics` 200 및 `codeplace_*` metric 포함 확인 - Kubernetes app render: `kubectl kustomize kubernetes/overlays/dev`, `kubectl kustomize kubernetes/overlays/prod` - Kubernetes monitoring render: `kubectl kustomize kubernetes/monitoring` -- Grafana dashboard JSON parse: `grafana-dashboard-codeplace.yaml`, `grafana-dashboard-logs.yaml` +- Dashboard environment/grid/visualization/navigation contract: `python3 kubernetes/monitoring/validate_dashboards.py --prometheus-rules-output /tmp/codeplace-dashboard-rules.json`로 11개 dashboard 검증 +- Prometheus rule/PromQL parse: `promtool check rules`로 `prometheus-rules.yaml`과 validator가 추출한 dashboard PromQL 검증 +- Monitoring alert contract: `validate_alerts.py`와 `promtool test rules`로 실제 장애/metric 결측 분리, 고정 replica target 일부 누락, prod/dev 지속 시간, service/instance 및 workload 문맥 전달을 검증 +- Pinned Helm chart render: kube-prometheus-stack `86.3.1`, Loki `6.55.0`, Alloy `1.10.0` +- Live collection contract: Prometheus port-forward 후 `python3 kubernetes/monitoring/verify_live.py` -`promtool`이 있는 환경에서는 `PrometheusRule.spec.groups`를 추출해서 `promtool check rules`로 확인합니다. +위 정적 계약은 `.github/workflows/monitoring-validate.yml`에서도 실행합니다. 운영 cluster의 최종 판정은 독립 실행한 `verify_live.py` 결과이며, render나 dashboard JSON parse 성공만으로 수집 성공을 대신하지 않습니다. ### Discord Webhook Secret diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 08b71cc9..bade67dc 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -5,21 +5,44 @@ FROM node:24.11.0 AS build-stage WORKDIR /app COPY package*.json ./ -RUN npm install +RUN npm ci COPY . . ARG APP_VERSION ARG SENTRY_ENVIRONMENT ARG SENTRY_DSN_FRONTEND -ARG USE_SENTRY +ARG USE_SENTRY=0 +ARG SENTRY_ORG=onlinejudge +ARG SENTRY_PROJECT=onlinejudgefe ENV APP_VERSION=$APP_VERSION ENV VUE_APP_VERSION=$APP_VERSION ENV SENTRY_ENVIRONMENT=$SENTRY_ENVIRONMENT ENV SENTRY_DSN_FRONTEND=$SENTRY_DSN_FRONTEND ENV USE_SENTRY=$USE_SENTRY +ENV SENTRY_ORG=$SENTRY_ORG +ENV SENTRY_PROJECT=$SENTRY_PROJECT RUN npm run build:dll -RUN npm run build +RUN --mount=type=secret,id=sentry_auth_token,required=false <<'EOF' +set -eu +SENTRY_AUTH_TOKEN="$(cat /run/secrets/sentry_auth_token 2>/dev/null || true)" +if [ "${USE_SENTRY:-0}" = "1" ]; then + if [ -z "${SENTRY_DSN_FRONTEND:-}" ]; then + echo "SENTRY_DSN_FRONTEND is required when USE_SENTRY=1" >&2 + exit 1 + fi + if [ -z "${APP_VERSION:-}" ]; then + echo "APP_VERSION is required when USE_SENTRY=1" >&2 + exit 1 + fi + if [ -z "$SENTRY_AUTH_TOKEN" ]; then + echo "SENTRY_AUTH_TOKEN is required when USE_SENTRY=1" >&2 + exit 1 + fi +fi +export SENTRY_AUTH_TOKEN +npm run build +EOF # Production stage FROM nginx:stable-alpine AS production-stage diff --git a/frontend/build/webpack.prod.conf.js b/frontend/build/webpack.prod.conf.js index aac39c3b..50961c64 100644 --- a/frontend/build/webpack.prod.conf.js +++ b/frontend/build/webpack.prod.conf.js @@ -11,6 +11,8 @@ const HtmlWebpackPlugin = require("html-webpack-plugin") const MiniCssExtractPlugin = require("mini-css-extract-plugin") const TerserPlugin = require("terser-webpack-plugin") const CssMinimizerPlugin = require("css-minimizer-webpack-plugin") +const { sentryWebpackPlugin } = require("@sentry/webpack-plugin") +const sentry = require("../config/sentry") const webpackConfig = merge(baseWebpackConfig, { mode: "production", @@ -132,4 +134,23 @@ if (config.build.bundleAnalyzerReport) { webpackConfig.plugins.push(new BundleAnalyzerPlugin()) } +sentry.assertSentryUploadConfig() +if (sentry.isSentryUploadEnabled()) { + webpackConfig.plugins.push( + sentryWebpackPlugin({ + authToken: process.env.SENTRY_AUTH_TOKEN, + org: process.env.SENTRY_ORG || "onlinejudge", + project: process.env.SENTRY_PROJECT || "onlinejudgefe", + telemetry: false, + release: { + name: process.env.APP_VERSION || process.env.VUE_APP_VERSION, + }, + sourcemaps: { + assets: "./dist/static/js/**", + filesToDeleteAfterUpload: "./dist/**/*.map", + }, + }), + ) +} + module.exports = webpackConfig diff --git a/frontend/config/index.js b/frontend/config/index.js index 0fc94e23..6fdcc6f7 100644 --- a/frontend/config/index.js +++ b/frontend/config/index.js @@ -22,7 +22,7 @@ module.exports = { assetsRoot: path.resolve(__dirname, "../dist"), assetsSubDirectory: "static", assetsPublicPath: "/__STATIC_CDN_HOST__/", - productionSourceMap: sentry.isSentryEnabled("production"), + productionSourceMap: sentry.isSentryUploadEnabled(), // Gzip off by default as many popular static hosts such as // Surge or Netlify already gzip all static assets for you. // Before setting to `true`, make sure to: diff --git a/frontend/config/sentry.js b/frontend/config/sentry.js index 24007f1c..b105dda2 100644 --- a/frontend/config/sentry.js +++ b/frontend/config/sentry.js @@ -1,8 +1,7 @@ "use strict" -function isSentryEnabled(nodeEnv = process.env.NODE_ENV) { - const enabled = (process.env.USE_SENTRY || (nodeEnv === "production" ? "1" : "0")) === "1" - return enabled && Boolean(getSentryDsn()) +function isSentryEnabled() { + return process.env.USE_SENTRY === "1" && Boolean(getSentryDsn()) } function getSentryDsn() { @@ -13,8 +12,34 @@ function getSentryEnvironment(nodeEnv = process.env.NODE_ENV) { return process.env.SENTRY_ENVIRONMENT || nodeEnv || "development" } +function isSentryUploadEnabled() { + return ( + isSentryEnabled() && + Boolean(process.env.SENTRY_AUTH_TOKEN) && + Boolean(process.env.APP_VERSION || process.env.VUE_APP_VERSION) + ) +} + +function assertSentryUploadConfig() { + if (process.env.USE_SENTRY !== "1") { + return + } + + const required = { + SENTRY_DSN_FRONTEND: getSentryDsn(), + SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN, + APP_VERSION: process.env.APP_VERSION || process.env.VUE_APP_VERSION, + } + const missing = Object.keys(required).filter((name) => !required[name]) + if (missing.length) { + throw new Error(`Missing Sentry build configuration: ${missing.join(", ")}`) + } +} + module.exports = { + assertSentryUploadConfig, getSentryEnvironment, getSentryDsn, isSentryEnabled, + isSentryUploadEnabled, } diff --git a/frontend/deploy/sentry_release.sh b/frontend/deploy/sentry_release.sh deleted file mode 100755 index 5e324216..00000000 --- a/frontend/deploy/sentry_release.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -DATE=`date +%Y%m%d` -COMMIT=`git rev-parse HEAD` -VERSION="$DATE-${COMMIT:0:5}" -USE_SENTRY="${USE_SENTRY:-1}" - -echo "Current version is $VERSION" - -if [ "$USE_SENTRY" == '1' ]; then - -# create new release according to `VERSION` -docker run --rm -it -v $(pwd):/work getsentry/sentry-cli \ - sentry-cli --auth-token $SENTRY_AUTH_TOKEN releases -o onlinejudge -p onlinejudgefe new $VERSION - -# upload js and source_maps -docker run --rm -it -v $(pwd):/work getsentry/sentry-cli \ - sentry-cli --auth-token $SENTRY_AUTH_TOKEN releases -o onlinejudge -p onlinejudgefe files $VERSION upload-sourcemaps ./dist/static/js -fi diff --git a/frontend/package-lock.json b/frontend/package-lock.json index dd5a6ece..d600e386 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -54,6 +54,7 @@ "@babel/plugin-transform-runtime": "^8.0.1", "@babel/preset-env": "^8.0.1", "@eslint/js": "^10.0.1", + "@sentry/webpack-plugin": "^5.4.0", "@soda/friendly-errors-webpack-plugin": "^1.8.1", "autoprefixer": "^10.5.0", "babel-loader": "^10.1.1", @@ -2141,6 +2142,16 @@ "@jridgewell/trace-mapping": "^0.3.24" } }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -2285,6 +2296,493 @@ "node": ">=18" } }, + "node_modules/@sentry/bundler-plugins": { + "version": "10.65.0", + "resolved": "https://registry.npmjs.org/@sentry/bundler-plugins/-/bundler-plugins-10.65.0.tgz", + "integrity": "sha512-AYgv31l4wY/CwYAD/2Og59RT+TNjvhatcLOrEe5tFOpi9VcPAQ4xfPZIM6fXYGawofT52p6LhUECNSfNkNnc7Q==", + "dev": true, + "dependencies": { + "@babel/core": "^7.18.5", + "@sentry/cli": "^2.58.6", + "@sentry/core": "10.65.0", + "dotenv": "^17.4.2", + "find-up": "^5.0.0", + "glob": "^13.0.6", + "magic-string": "~0.30.8" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "rollup": ">=3.2.0", + "webpack": ">=5.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + }, + "webpack": { + "optional": true + } + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/compat-data": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/core": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/generator": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz", + "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", + "dev": true, + "dependencies": { + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "dev": true, + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "dev": true, + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "dev": true, + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "dev": true, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/helpers": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", + "dev": true, + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", + "dev": true, + "dependencies": { + "@babel/types": "^7.29.7" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/template": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/traverse": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz", + "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@babel/types": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", + "dev": true, + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/@sentry/core": { + "version": "10.65.0", + "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.65.0.tgz", + "integrity": "sha512-3aqtmM5NgNGo45BNaaBzi0LPQZAw//NEL4HKS5fXm12pJMa4KEkze8DEKnkTEIrGnWaOJKamecHKlnNg/Mqf/Q==", + "dev": true, + "dependencies": { + "@sentry/conventions": "^0.15.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/glob": { + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", + "dev": true, + "dependencies": { + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "node_modules/@sentry/bundler-plugins/node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@sentry/bundler-plugins/node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true + }, + "node_modules/@sentry/cli": { + "version": "2.58.6", + "resolved": "https://registry.npmjs.org/@sentry/cli/-/cli-2.58.6.tgz", + "integrity": "sha512-baBcNPLLfUi9WuL+Tpri9BFaAdvugZIKelC5X0tt0Zdy+K0K+PCVSrnNmwMWU/HyaF/SEv6b6UHnXIdqanBlcg==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.7", + "progress": "^2.0.3", + "proxy-from-env": "^1.1.0", + "which": "^2.0.2" + }, + "bin": { + "sentry-cli": "bin/sentry-cli" + }, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@sentry/cli-darwin": "2.58.6", + "@sentry/cli-linux-arm": "2.58.6", + "@sentry/cli-linux-arm64": "2.58.6", + "@sentry/cli-linux-i686": "2.58.6", + "@sentry/cli-linux-x64": "2.58.6", + "@sentry/cli-win32-arm64": "2.58.6", + "@sentry/cli-win32-i686": "2.58.6", + "@sentry/cli-win32-x64": "2.58.6" + } + }, + "node_modules/@sentry/cli-darwin": { + "version": "2.58.6", + "resolved": "https://registry.npmjs.org/@sentry/cli-darwin/-/cli-darwin-2.58.6.tgz", + "integrity": "sha512-udAVvcyfNa0R+95GvPz/+43/N3TC0TYKdkQ7D7jhPSzbcMc7l2fxRNN5yB3UpCA5fWFnW4toeaqwDBhb/Wh3LA==", + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-linux-arm": { + "version": "2.58.6", + "resolved": "https://registry.npmjs.org/@sentry/cli-linux-arm/-/cli-linux-arm-2.58.6.tgz", + "integrity": "sha512-pD0LAt5PcUzAinBwvDqc66x9+2CabHEv486yP0gRjWO7SakbaxmfVq/EXd8VLq/Tzi39LAu422UYK1lpW3MILw==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux", + "freebsd", + "android" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-linux-arm64": { + "version": "2.58.6", + "resolved": "https://registry.npmjs.org/@sentry/cli-linux-arm64/-/cli-linux-arm64-2.58.6.tgz", + "integrity": "sha512-q8mEcNNmeXMy5i+jWT30TVpH7LcP4HD21CD5XRSPAd/a912HF6EpK0ybf/1USO14WOhoXbAGi9txwaWabSe33g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux", + "freebsd", + "android" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-linux-i686": { + "version": "2.58.6", + "resolved": "https://registry.npmjs.org/@sentry/cli-linux-i686/-/cli-linux-i686-2.58.6.tgz", + "integrity": "sha512-q8vNJi1eOV/4vxAFWBsEwLHoSYapaZHIf4j76KJGJXFKTkEbsjCOOsKbwUIBTQQhRgV4DFWh3ryfsPS/que4Kg==", + "cpu": [ + "x86", + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "linux", + "freebsd", + "android" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-linux-x64": { + "version": "2.58.6", + "resolved": "https://registry.npmjs.org/@sentry/cli-linux-x64/-/cli-linux-x64-2.58.6.tgz", + "integrity": "sha512-DZu956Mhi3ZRjTBe1WdbGV46ldVbA8d2rgp/fh51GsI25zjBHah4wZnPTSzpc+YqxU6pJpg579B/r3jrIK530Q==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux", + "freebsd", + "android" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-win32-arm64": { + "version": "2.58.6", + "resolved": "https://registry.npmjs.org/@sentry/cli-win32-arm64/-/cli-win32-arm64-2.58.6.tgz", + "integrity": "sha512-nj0Ff/kmAB73EPDhR8B4O9r+NUHK5GkPCkGWC+kXVemqAJWL5jcJ5KdxG0l/S0z6RoEoltID8/43/B+TaMlT7A==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-win32-i686": { + "version": "2.58.6", + "resolved": "https://registry.npmjs.org/@sentry/cli-win32-i686/-/cli-win32-i686-2.58.6.tgz", + "integrity": "sha512-WNZiDzPbgsEMQWq4avsQ391v/xWKJDIWWWo9GYl+N/w5qcYKkoDW7wQG7T9FasI6ENn68phChTOAPXXxbfAdOg==", + "cpu": [ + "x86", + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/cli-win32-x64": { + "version": "2.58.6", + "resolved": "https://registry.npmjs.org/@sentry/cli-win32-x64/-/cli-win32-x64-2.58.6.tgz", + "integrity": "sha512-R35WJ17oF4D2eqI1DR2sQQqr0fjRTt5xoP16WrTu91XM2lndRMFsnjh+/GttbxapLCBNlrjzia99MJ0PZHZpgA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@sentry/conventions": { + "version": "0.15.1", + "resolved": "https://registry.npmjs.org/@sentry/conventions/-/conventions-0.15.1.tgz", + "integrity": "sha512-ZLP8bRdMON3prWE2tJyImuYscCxdcJeIPIhrOs/rgyFm3C1nCh1B6gdvPj3AZ5zW08oSFFCsq7T+tYEW3h8MNA==", + "dev": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@sentry/core": { "version": "10.58.0", "resolved": "https://registry.npmjs.org/@sentry/core/-/core-10.58.0.tgz", @@ -2353,6 +2851,21 @@ } } }, + "node_modules/@sentry/webpack-plugin": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/@sentry/webpack-plugin/-/webpack-plugin-5.4.0.tgz", + "integrity": "sha512-J3a0BvUZ75Qxy+v/Ap3Hx4ZEcSjlPHZ/jDtxdRhXQCyNeEb8xq0uUBTI9VLtGk2eNeNucOxOEJ5ngqdNjnEH/A==", + "dev": true, + "dependencies": { + "@sentry/bundler-plugins": "^10.64.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "webpack": ">=5.0.0" + } + }, "node_modules/@sinclair/typebox": { "version": "0.27.10", "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.10.tgz", @@ -3040,6 +3553,18 @@ "node": ">=0.4.0" } }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dev": true, + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, "node_modules/ajv": { "version": "6.15.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", @@ -4481,6 +5006,18 @@ "tslib": "^2.0.3" } }, + "node_modules/dotenv": { + "version": "17.4.2", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.2.tgz", + "integrity": "sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, "node_modules/dunder-proto": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", @@ -6081,6 +6618,19 @@ "npm": ">=1.3.7" } }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dev": true, + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -6896,6 +7446,15 @@ "node": "20 || >=22" } }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, "node_modules/make-dir": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", @@ -7114,6 +7673,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/moment": { "version": "2.30.1", "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", @@ -7201,6 +7769,42 @@ "tslib": "^2.0.3" } }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dev": true, + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/node-fetch/node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dev": true + }, + "node_modules/node-fetch/node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dev": true, + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/node-releases": { "version": "2.0.48", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.48.tgz", @@ -7503,6 +8107,22 @@ "dev": true, "license": "MIT" }, + "node_modules/path-scurry": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", + "dev": true, + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/path-to-regexp": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.13.tgz", @@ -8289,6 +8909,15 @@ "renderkid": "^3.0.0" } }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/prosemirror-collab": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/prosemirror-collab/-/prosemirror-collab-1.3.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index 7aa51bc4..10b7c3dd 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -58,6 +58,7 @@ "@babel/plugin-transform-runtime": "^8.0.1", "@babel/preset-env": "^8.0.1", "@eslint/js": "^10.0.1", + "@sentry/webpack-plugin": "^5.4.0", "@soda/friendly-errors-webpack-plugin": "^1.8.1", "autoprefixer": "^10.5.0", "babel-loader": "^10.1.1", diff --git a/frontend/src/pages/oj/views/home/HomeNoticeProblemRow.vue b/frontend/src/pages/oj/views/home/HomeNoticeProblemRow.vue index 5bdfaca2..bc454ff1 100644 --- a/frontend/src/pages/oj/views/home/HomeNoticeProblemRow.vue +++ b/frontend/src/pages/oj/views/home/HomeNoticeProblemRow.vue @@ -45,6 +45,7 @@ 📌 + NEW {{ item.title }} @@ -68,7 +69,7 @@ class="notice-item" @click="goSWItem(item)" > - NEW + NEW {{ item.title }} {{ formatDate(item.pubDate) }} @@ -124,6 +125,13 @@ export default { }, ) }, + isNew(dateStr) { + if (!dateStr) return false + const t = new Date(dateStr).getTime() + if (Number.isNaN(t)) return false + const FIVE_DAYS = 5 * 24 * 60 * 60 * 1000 + return Date.now() - t <= FIVE_DAYS + }, formatDate(dateStr) { if (!dateStr) return "" const d = new Date(dateStr) diff --git a/frontend/src/pages/oj/views/problem/problemSolving/Problem.vue b/frontend/src/pages/oj/views/problem/problemSolving/Problem.vue index 7ec63421..d02f8119 100644 --- a/frontend/src/pages/oj/views/problem/problemSolving/Problem.vue +++ b/frontend/src/pages/oj/views/problem/problemSolving/Problem.vue @@ -168,7 +168,7 @@ @update:problemZoomPercent="problemZoomPercent = $event" /> { @@ -68,7 +56,6 @@ export function configureAxiosRequestId(axios) { responseRequestId(error.response) || (error.config && error.config.metadata && error.config.metadata.requestId) error.requestId = requestId - rememberRequestId(requestId) return Promise.reject(error) }, ) diff --git a/frontend/src/utils/sentry.js b/frontend/src/utils/sentry.js index 723a6b48..a06e8c33 100644 --- a/frontend/src/utils/sentry.js +++ b/frontend/src/utils/sentry.js @@ -45,11 +45,28 @@ function redact(value) { return value } +function withRequestContext(event, hint) { + const originalException = hint && hint.originalException + const requestId = originalException && originalException.requestId + if (!requestId) { + return event + } + return { + ...event, + contexts: { + ...(event.contexts || {}), + request: { + request_id: requestId, + }, + }, + } +} + const options = { release: process.env.VERSION, environment: process.env.SENTRY_ENVIRONMENT, - beforeSend(event) { - return redact(event) + beforeSend(event, hint) { + return redact(withRequestContext(event, hint)) }, denyUrls: [ // Chrome extensions @@ -82,9 +99,3 @@ export function initSentry(Vue) { location: window.location.href, }) } - -export function setRequestIdContext(requestId) { - setContext("request", { - request_id: requestId, - }) -} diff --git a/hub/auth_server/app/main.py b/hub/auth_server/app/main.py index 0451bcc6..d6a20ad1 100644 --- a/hub/auth_server/app/main.py +++ b/hub/auth_server/app/main.py @@ -35,6 +35,17 @@ class TokenResponse(BaseModel): access_token: str +class HealthResponse(BaseModel): + """Stable health contract used by Kubernetes and external probes.""" + status: str + + +@app.get("/healthz", response_model=HealthResponse, include_in_schema=False) +async def healthz() -> HealthResponse: + """Return process health without calling GitHub or exposing credentials.""" + return HealthResponse(status="ok") + + async def exchange_code_for_token(code: str) -> Dict[str, Any]: """ Exchange OAuth code for a GitHub access token. diff --git a/hub/auth_server/app/test_main.py b/hub/auth_server/app/test_main.py index 9e409740..98fb9edb 100644 --- a/hub/auth_server/app/test_main.py +++ b/hub/auth_server/app/test_main.py @@ -9,6 +9,13 @@ def client(): return TestClient(app) +def test_healthz(client): + response = client.get("/healthz") + + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + def test_no_code(client): """ Test the /api/token/issue endpoint without providing a code. diff --git a/kubernetes/base/backend/deployment.yaml b/kubernetes/base/backend/deployment.yaml index 5b8fcae2..e9a00494 100644 --- a/kubernetes/base/backend/deployment.yaml +++ b/kubernetes/base/backend/deployment.yaml @@ -95,6 +95,8 @@ spec: value: "1" - name: OTEL_ENABLED value: "0" + - name: OTEL_SERVICE_NAME + value: "codeplace-backend" - name: OTEL_EXPORTER_OTLP_ENDPOINT value: "http://otel-collector.monitoring.svc.cluster.local:4317" - name: OTEL_TRACES_SAMPLER_ARG diff --git a/kubernetes/base/celery-beat/deployment.yaml b/kubernetes/base/celery-beat/deployment.yaml index aecec2ab..6e90a469 100644 --- a/kubernetes/base/celery-beat/deployment.yaml +++ b/kubernetes/base/celery-beat/deployment.yaml @@ -90,6 +90,8 @@ spec: value: "1" - name: OTEL_ENABLED value: "0" + - name: OTEL_SERVICE_NAME + value: "codeplace-celery-beat" - name: OTEL_EXPORTER_OTLP_ENDPOINT value: "http://otel-collector.monitoring.svc.cluster.local:4317" - name: OTEL_TRACES_SAMPLER_ARG diff --git a/kubernetes/base/celery-worker/deployment.yaml b/kubernetes/base/celery-worker/deployment.yaml index 3c3ddef6..313acbea 100644 --- a/kubernetes/base/celery-worker/deployment.yaml +++ b/kubernetes/base/celery-worker/deployment.yaml @@ -93,6 +93,8 @@ spec: value: "1" - name: OTEL_ENABLED value: "0" + - name: OTEL_SERVICE_NAME + value: "codeplace-celery-worker" - name: OTEL_EXPORTER_OTLP_ENDPOINT value: "http://otel-collector.monitoring.svc.cluster.local:4317" - name: OTEL_TRACES_SAMPLER_ARG diff --git a/kubernetes/base/hub-auth/deployment.yaml b/kubernetes/base/hub-auth/deployment.yaml index 62516cc1..6d9c91da 100644 --- a/kubernetes/base/hub-auth/deployment.yaml +++ b/kubernetes/base/hub-auth/deployment.yaml @@ -29,17 +29,20 @@ spec: - containerPort: 80 name: http startupProbe: - tcpSocket: + httpGet: + path: /healthz port: http periodSeconds: 5 failureThreshold: 12 readinessProbe: - tcpSocket: + httpGet: + path: /healthz port: http periodSeconds: 10 failureThreshold: 3 livenessProbe: - tcpSocket: + httpGet: + path: /healthz port: http initialDelaySeconds: 30 periodSeconds: 20 diff --git a/kubernetes/base/postgres/postgres.yaml b/kubernetes/base/postgres/postgres.yaml index 8cc6aff1..89e28953 100644 --- a/kubernetes/base/postgres/postgres.yaml +++ b/kubernetes/base/postgres/postgres.yaml @@ -9,6 +9,14 @@ spec: primaryUpdateStrategy: unsupervised + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + storage: storageClass: longhorn size: 30Gi diff --git a/kubernetes/base/redis/redis.yaml b/kubernetes/base/redis/redis.yaml index 3a562186..327a1a96 100644 --- a/kubernetes/base/redis/redis.yaml +++ b/kubernetes/base/redis/redis.yaml @@ -27,6 +27,9 @@ spec: cpu: 500m memory: 512Mi + redisConfig: + maxMemoryPercentOfLimit: 80 + storage: volumeClaimTemplate: spec: diff --git a/kubernetes/base/vllm/README.md b/kubernetes/base/vllm/README.md index d592f40a..258e7c1c 100644 --- a/kubernetes/base/vllm/README.md +++ b/kubernetes/base/vllm/README.md @@ -60,15 +60,11 @@ vLLM OpenAI-compatible server는 `/metrics`에서 Prometheus metrics를 제공 CodePlace monitoring kustomization은 prod vLLM Service를 `vllm-service-monitor.yaml`로 scrape합니다. 같은 monitoring kustomization은 `dcgm-exporter.yaml`도 배포해 `workload.code-place.ai/vllm=true` node의 NVIDIA GPU metric을 수집합니다. -```bash -kubectl apply -k kubernetes/monitoring -kubectl -n monitoring get servicemonitor vllm -kubectl -n monitoring get daemonset,servicemonitor dcgm-exporter -``` +Monitoring 배포 절차는 `kubernetes/monitoring/logs/README.md`를 따릅니다. 적용 후 `kubernetes/monitoring/verify_live.py`로 prod vLLM과 DCGM scrape 및 metric family를 확인합니다. -Grafana에서는 `CodePlace AI Inference` dashboard에서 다음 상태를 확인합니다. +Grafana의 `CodePlace AI Runtime (Prod)` dashboard에서는 다음 prod 전용 상태를 확인합니다. -- vLLM scrape up / Pod ready. +- vLLM/DCGM scrape up. - running/waiting request count. - KV cache usage. - p95 e2e latency, queue latency, time-to-first-token. @@ -76,4 +72,6 @@ Grafana에서는 `CodePlace AI Inference` dashboard에서 다음 상태를 확 - `vllm-hf-cache` PVC usage. - GPU utilization, framebuffer memory usage, temperature, XID error. +dev/prod backend AI hint outcome과 latency는 별도 `CodePlace AI API` dashboard의 `environment` 선택으로 확인합니다. prod 전용 vLLM/GPU graph와 dev API graph를 한 화면에 섞지 않습니다. + 알림은 vLLM scrape 실패와 GPU XID error를 P0로 처리합니다. waiting queue 증가, KV cache 90% 초과, p95 latency 60초 초과, DCGM exporter unavailable, GPU utilization 95% 초과, GPU framebuffer memory 90% 초과, GPU temperature 85C 초과는 P1로 처리합니다. diff --git a/kubernetes/monitoring/alertmanager-config.yaml b/kubernetes/monitoring/alertmanager-config.yaml index eb9089fe..086d5bce 100644 --- a/kubernetes/monitoring/alertmanager-config.yaml +++ b/kubernetes/monitoring/alertmanager-config.yaml @@ -10,6 +10,7 @@ spec: - alertname - priority - namespace + - service groupWait: 30s groupInterval: 5m repeatInterval: 1h @@ -38,6 +39,29 @@ spec: groupInterval: 5m repeatInterval: 1h receiver: p1-discord + - matchers: + - name: severity + value: critical + groupWait: 10s + groupInterval: 1m + repeatInterval: 15m + receiver: p0-discord + - matchers: + - name: severity + value: warning + - name: namespace + value: code-place-dev + groupWait: 30s + groupInterval: 5m + repeatInterval: 1h + receiver: dev-p1-muted + - matchers: + - name: severity + value: warning + groupWait: 30s + groupInterval: 5m + repeatInterval: 1h + receiver: p1-discord receivers: - name: unmatched-muted - name: p0-discord @@ -46,17 +70,29 @@ spec: apiURL: name: alertmanager-contact-points key: webhook-url - title: '[{{ if eq .Status "firing" }}발생{{ else }}해결{{ end }}][P0] {{ .CommonLabels.alertname }}' + title: '[{{ if eq .Status "firing" }}활성{{ else }}해결{{ end }}][P0] {{ .CommonLabels.alertname }}{{ with .CommonLabels.service }} ({{ . }}){{ end }}' content: '{{ .CommonAnnotations.summary }}' message: |- - {{ .CommonAnnotations.description }} + {{ range .Alerts }} + 대상: {{ if .Labels.service }}{{ .Labels.service }}{{ else }}{{ .Labels.alertname }}{{ end }}{{ with .Labels.instance }} ({{ . }}){{ end }} + {{ if or .Labels.pod .Labels.container .Labels.persistentvolumeclaim .Labels.pvc .Labels.pvc_namespace .Labels.deployment .Labels.daemonset .Labels.node .Labels.volume .Labels.disk .Labels.reason .Labels.condition .Labels.collector .Labels.check .Labels.scope .Labels.status .Labels.Hostname .Labels.gpu .Labels.UUID }} + 문맥:{{ with .Labels.pod }} pod={{ . }}{{ end }}{{ with .Labels.container }} container={{ . }}{{ end }}{{ with .Labels.persistentvolumeclaim }} pvc={{ . }}{{ end }}{{ with .Labels.pvc }} pvc={{ . }}{{ end }}{{ with .Labels.pvc_namespace }} pvc_namespace={{ . }}{{ end }}{{ with .Labels.deployment }} deployment={{ . }}{{ end }}{{ with .Labels.daemonset }} daemonset={{ . }}{{ end }}{{ with .Labels.node }} node={{ . }}{{ end }}{{ with .Labels.volume }} volume={{ . }}{{ end }}{{ with .Labels.disk }} disk={{ . }}{{ end }}{{ with .Labels.reason }} reason={{ . }}{{ end }}{{ with .Labels.condition }} condition={{ . }}{{ end }}{{ with .Labels.collector }} collector={{ . }}{{ end }}{{ with .Labels.check }} check={{ . }}{{ end }}{{ with .Labels.scope }} scope={{ . }}{{ end }}{{ with .Labels.status }} status={{ . }}{{ end }}{{ with .Labels.Hostname }} host={{ . }}{{ end }}{{ with .Labels.gpu }} gpu={{ . }}{{ end }}{{ with .Labels.UUID }} gpu_uuid={{ . }}{{ end }} + {{ end }} + 상세: {{ .Annotations.description }} + {{ end }} - 우선순위: {{ .CommonLabels.priority }} + 우선순위: P0 네임스페이스: {{ .CommonLabels.namespace }} - 개요: https://monitoring.code-place-dev.site/d/codeplace-overview/codeplace-overview?orgId=1&var-namespace={{ .CommonLabels.namespace }} - 로그: https://monitoring.code-place-dev.site/d/codeplace-logs/codeplace-logs?orgId=1&var-namespace={{ .CommonLabels.namespace }} + {{ if match "^code-place-(dev|prod)$" .CommonLabels.namespace }} + 개요: https://monitoring.code-place-dev.site/d/codeplace-overview/codeplace-overview?orgId=1&var-environment={{ reReplaceAll "^code-place-" "" .CommonLabels.namespace }} + 로그: https://monitoring.code-place-dev.site/d/codeplace-logs/codeplace-logs?orgId=1&var-environment={{ reReplaceAll "^code-place-" "" .CommonLabels.namespace }} + {{ end }} + {{ if eq .CommonLabels.namespace "longhorn-system" }} + 스토리지(prod): https://monitoring.code-place-dev.site/d/codeplace-storage/codeplace-storage?orgId=1&var-environment=prod + 스토리지(dev): https://monitoring.code-place-dev.site/d/codeplace-storage/codeplace-storage?orgId=1&var-environment=dev + {{ end }} 모니터링: https://monitoring.code-place-dev.site/d/codeplace-monitoring-stack/codeplace-monitoring-stack?orgId=1 - 발생 쿼리: {{ range .Alerts }}{{ .GeneratorURL }}{{ end }} + {{ with index .Alerts 0 }}발생 쿼리: [Prometheus에서 보기]({{ .GeneratorURL }}){{ end }} username: CodePlace Alertmanager - name: dev-p1-muted - name: p1-discord @@ -65,15 +101,27 @@ spec: apiURL: name: alertmanager-contact-points key: webhook-url - title: '[{{ if eq .Status "firing" }}발생{{ else }}해결{{ end }}][P1] {{ .CommonLabels.alertname }}' + title: '[{{ if eq .Status "firing" }}활성{{ else }}해결{{ end }}][P1] {{ .CommonLabels.alertname }}{{ with .CommonLabels.service }} ({{ . }}){{ end }}' content: '{{ .CommonAnnotations.summary }}' message: |- - {{ .CommonAnnotations.description }} + {{ range .Alerts }} + 대상: {{ if .Labels.service }}{{ .Labels.service }}{{ else }}{{ .Labels.alertname }}{{ end }}{{ with .Labels.instance }} ({{ . }}){{ end }} + {{ if or .Labels.pod .Labels.container .Labels.persistentvolumeclaim .Labels.pvc .Labels.pvc_namespace .Labels.deployment .Labels.daemonset .Labels.node .Labels.volume .Labels.disk .Labels.reason .Labels.condition .Labels.collector .Labels.check .Labels.scope .Labels.status .Labels.Hostname .Labels.gpu .Labels.UUID }} + 문맥:{{ with .Labels.pod }} pod={{ . }}{{ end }}{{ with .Labels.container }} container={{ . }}{{ end }}{{ with .Labels.persistentvolumeclaim }} pvc={{ . }}{{ end }}{{ with .Labels.pvc }} pvc={{ . }}{{ end }}{{ with .Labels.pvc_namespace }} pvc_namespace={{ . }}{{ end }}{{ with .Labels.deployment }} deployment={{ . }}{{ end }}{{ with .Labels.daemonset }} daemonset={{ . }}{{ end }}{{ with .Labels.node }} node={{ . }}{{ end }}{{ with .Labels.volume }} volume={{ . }}{{ end }}{{ with .Labels.disk }} disk={{ . }}{{ end }}{{ with .Labels.reason }} reason={{ . }}{{ end }}{{ with .Labels.condition }} condition={{ . }}{{ end }}{{ with .Labels.collector }} collector={{ . }}{{ end }}{{ with .Labels.check }} check={{ . }}{{ end }}{{ with .Labels.scope }} scope={{ . }}{{ end }}{{ with .Labels.status }} status={{ . }}{{ end }}{{ with .Labels.Hostname }} host={{ . }}{{ end }}{{ with .Labels.gpu }} gpu={{ . }}{{ end }}{{ with .Labels.UUID }} gpu_uuid={{ . }}{{ end }} + {{ end }} + 상세: {{ .Annotations.description }} + {{ end }} - 우선순위: {{ .CommonLabels.priority }} + 우선순위: P1 네임스페이스: {{ .CommonLabels.namespace }} - 개요: https://monitoring.code-place-dev.site/d/codeplace-overview/codeplace-overview?orgId=1&var-namespace={{ .CommonLabels.namespace }} - 로그: https://monitoring.code-place-dev.site/d/codeplace-logs/codeplace-logs?orgId=1&var-namespace={{ .CommonLabels.namespace }} + {{ if match "^code-place-(dev|prod)$" .CommonLabels.namespace }} + 개요: https://monitoring.code-place-dev.site/d/codeplace-overview/codeplace-overview?orgId=1&var-environment={{ reReplaceAll "^code-place-" "" .CommonLabels.namespace }} + 로그: https://monitoring.code-place-dev.site/d/codeplace-logs/codeplace-logs?orgId=1&var-environment={{ reReplaceAll "^code-place-" "" .CommonLabels.namespace }} + {{ end }} + {{ if eq .CommonLabels.namespace "longhorn-system" }} + 스토리지(prod): https://monitoring.code-place-dev.site/d/codeplace-storage/codeplace-storage?orgId=1&var-environment=prod + 스토리지(dev): https://monitoring.code-place-dev.site/d/codeplace-storage/codeplace-storage?orgId=1&var-environment=dev + {{ end }} 모니터링: https://monitoring.code-place-dev.site/d/codeplace-monitoring-stack/codeplace-monitoring-stack?orgId=1 - 발생 쿼리: {{ range .Alerts }}{{ .GeneratorURL }}{{ end }} + {{ with index .Alerts 0 }}발생 쿼리: [Prometheus에서 보기]({{ .GeneratorURL }}){{ end }} username: CodePlace Alertmanager diff --git a/kubernetes/monitoring/datastore-pod-monitors.yaml b/kubernetes/monitoring/datastore-pod-monitors.yaml index 911b5f9b..9f847ceb 100644 --- a/kubernetes/monitoring/datastore-pod-monitors.yaml +++ b/kubernetes/monitoring/datastore-pod-monitors.yaml @@ -16,6 +16,12 @@ spec: - port: metrics interval: 30s scrapeTimeout: 10s + relabelings: + - sourceLabels: + - __meta_kubernetes_pod_label_cnpg_io_cluster + targetLabel: cluster + - targetLabel: job + replacement: postgres --- apiVersion: monitoring.coreos.com/v1 kind: PodMonitor @@ -39,3 +45,6 @@ spec: - portNumber: 9121 interval: 30s scrapeTimeout: 10s + relabelings: + - targetLabel: job + replacement: redis diff --git a/kubernetes/monitoring/grafana-dashboard-ai-api.yaml b/kubernetes/monitoring/grafana-dashboard-ai-api.yaml new file mode 100644 index 00000000..134f6baf --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-ai-api.yaml @@ -0,0 +1,144 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-ai-api + labels: + grafana_dashboard: "1" +data: + codeplace-ai-api.json: | + { + "uid": "codeplace-ai-api", + "title": "CodePlace AI API", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "ai", "api"], + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": ["codeplace"], + "targetBlank": false, + "title": "CodePlace Dashboards", + "type": "dashboards" + } + ], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "environment", + "label": "Environment", + "type": "custom", + "query": "prod,dev", + "multi": false, + "includeAll": false, + "current": { + "text": "prod", + "value": "prod" + } + } + ] + }, + "panels": [ + { + "type": "stat", + "title": "$environment Backend Metrics Up", + "description": "Scrape health for the backend that emits AI hint API metrics in the selected environment.", + "gridPos": {"x": 0, "y": 0, "w": 8, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "(sum by (namespace) (up{job=\"backend\", namespace=\"code-place-$environment\"}) >= bool max by (namespace) (kube_deployment_spec_replicas{namespace=\"code-place-$environment\", deployment=\"backend\"})) or on() vector(0)", "legendFormat": "backend", "instant": true} ] + }, + { + "type": "timeseries", + "title": "$environment AI Hint API Outcomes", + "description": "User-facing AI hint API outcomes per second, split by request scope and final status.", + "gridPos": {"x": 8, "y": 0, "w": 16, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "bars", + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "barAlignment": 0, + "stacking": {"mode": "normal", "group": "A"} + } + } + }, + "targets": [ + {"expr": "sum by (scope, status) (rate(codeplace_ai_hint_api_outcome_total{namespace=\"code-place-$environment\"}[5m]))", "legendFormat": "{{scope}} {{status}}/s"} + ] + }, + { + "type": "timeseries", + "title": "$environment Backend AI Hint Streams", + "description": "Backend-to-model AI hint streams per second, split by completion status.", + "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "bars", + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "barAlignment": 0, + "stacking": {"mode": "normal", "group": "A"} + } + } + }, + "targets": [ + {"expr": "sum by (status) (rate(codeplace_ai_hint_requests_total{namespace=\"code-place-$environment\"}[5m]))", "legendFormat": "{{status}} streams/s"} + ] + }, + { + "type": "timeseries", + "title": "$environment Backend AI Hint Duration", + "description": "End-to-end AI hint stream duration at p95, separated into successful and non-successful outcomes.", + "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, + "targets": [ + {"expr": "histogram_quantile(0.95, sum by (le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace=\"code-place-$environment\", status=\"success\"}[10m])))", "legendFormat": "success p95"}, + {"expr": "histogram_quantile(0.95, sum by (status, le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace=\"code-place-$environment\", status!=\"success\"}[10m])))", "legendFormat": "{{status}} p95"} + ] + } + ] + } diff --git a/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml b/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml index f0ccc861..4f64f44a 100644 --- a/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml +++ b/kubernetes/monitoring/grafana-dashboard-ai-inference.yaml @@ -8,11 +8,22 @@ data: codeplace-ai-inference.json: | { "uid": "codeplace-ai-inference", - "title": "CodePlace AI Inference", + "title": "CodePlace AI Runtime (Prod)", "schemaVersion": 39, "version": 1, "refresh": "30s", - "tags": ["codeplace", "ai", "vllm"], + "tags": ["codeplace", "ai", "runtime", "vllm", "prod"], + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": ["codeplace"], + "targetBlank": false, + "title": "CodePlace Dashboards", + "type": "dashboards" + } + ], "editable": true, "time": { "from": "now-1h", @@ -21,8 +32,9 @@ data: "panels": [ { "type": "stat", - "title": "vLLM Scrape Up", - "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "title": "Prod vLLM Scrape Up", + "description": "Production-only vLLM runtime scrape health.", + "gridPos": {"x": 0, "y": 0, "w": 5, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -36,15 +48,15 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "up{job=\"vllm\", namespace=\"code-place-prod\"}", "legendFormat": "vllm"} - ] + {"expr": "up{job=\"vllm\", namespace=\"code-place-prod\"}", "legendFormat": "vllm", "instant": true} ] }, { "type": "stat", - "title": "vLLM Pod Ready", - "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "title": "Prod DCGM Scrape Up", + "description": "GPU telemetry scrape health for the production vLLM node.", + "gridPos": {"x": 5, "y": 0, "w": 5, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -58,26 +70,24 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(kube_pod_status_ready{namespace=\"code-place-prod\", condition=\"true\", pod=~\"vllm-.*\"})", "legendFormat": "ready"} - ] + {"expr": "min(up{job=\"dcgm-exporter\", namespace=\"monitoring\"})", "legendFormat": "dcgm-exporter", "instant": true} ] }, { "type": "stat", - "title": "Running Requests", - "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "title": "Prod vLLM Running Requests", + "gridPos": {"x": 10, "y": 0, "w": 5, "h": 4}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "short"}}, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max({__name__=\"vllm:num_requests_running\", namespace=\"code-place-prod\"})", "legendFormat": "running"} - ] + {"expr": "max({__name__=\"vllm:num_requests_running\", namespace=\"code-place-prod\"})", "legendFormat": "running", "instant": true} ] }, { "type": "stat", - "title": "Waiting Requests", - "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "title": "Prod vLLM Waiting Requests", + "gridPos": {"x": 15, "y": 0, "w": 5, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -92,15 +102,14 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max({__name__=\"vllm:num_requests_waiting\", namespace=\"code-place-prod\"})", "legendFormat": "waiting"} - ] + {"expr": "max({__name__=\"vllm:num_requests_waiting\", namespace=\"code-place-prod\"})", "legendFormat": "waiting", "instant": true} ] }, { "type": "stat", - "title": "KV Cache Usage", - "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "title": "Prod vLLM KV Cache Usage", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -115,40 +124,28 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max({__name__=\"vllm:kv_cache_usage_perc\", namespace=\"code-place-prod\"})", "legendFormat": "kv-cache"} - ] + {"expr": "max({__name__=\"vllm:kv_cache_usage_perc\", namespace=\"code-place-prod\"})", "legendFormat": "kv-cache", "instant": true} ] }, { - "type": "stat", - "title": "HF Cache PVC", - "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "type": "timeseries", + "title": "Prod vLLM Requests", + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, "datasource": "Prometheus", "fieldConfig": { "defaults": { - "unit": "percentunit", - "thresholds": { - "mode": "absolute", - "steps": [ - {"color": "green", "value": null}, - {"color": "orange", "value": 0.75}, - {"color": "red", "value": 0.85} - ] + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, - "targets": [ - {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}), 1)", "legendFormat": "usage"} - ] - }, - { - "type": "timeseries", - "title": "vLLM Requests", - "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ {"expr": "{__name__=\"vllm:num_requests_running\", namespace=\"code-place-prod\"}", "legendFormat": "running"}, {"expr": "{__name__=\"vllm:num_requests_waiting\", namespace=\"code-place-prod\"}", "legendFormat": "waiting"} @@ -156,20 +153,45 @@ data: }, { "type": "timeseries", - "title": "vLLM Request Success Rate", + "title": "Prod vLLM Request Success Rate", "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "ops"}}, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "bars", + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "fillOpacity": 70, + "showPoints": "never", + "barAlignment": 0, + "stacking": {"mode": "normal", "group": "A"} + } + } + }, "targets": [ {"expr": "sum by (finished_reason) (rate({__name__=\"vllm:request_success_total\", namespace=\"code-place-prod\"}[5m]))", "legendFormat": "{{finished_reason}}"} ] }, { "type": "timeseries", - "title": "vLLM p95 Latency", + "title": "Prod vLLM p95 Latency", "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "s"}}, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ {"expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=\"vllm:e2e_request_latency_seconds_bucket\", namespace=\"code-place-prod\"}[10m])))", "legendFormat": "e2e p95"}, {"expr": "histogram_quantile(0.95, sum by (le) (rate({__name__=\"vllm:request_queue_time_seconds_bucket\", namespace=\"code-place-prod\"}[10m])))", "legendFormat": "queue p95"}, @@ -178,10 +200,22 @@ data: }, { "type": "timeseries", - "title": "vLLM Token Throughput", - "gridPos": {"x": 0, "y": 12, "w": 8, "h": 8}, + "title": "Prod vLLM Token Throughput", + "gridPos": {"x": 0, "y": 12, "w": 6, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "ops"}}, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ {"expr": "sum(rate({__name__=\"vllm:prompt_tokens_total\", namespace=\"code-place-prod\"}[5m]))", "legendFormat": "prompt tokens/s"}, {"expr": "sum(rate({__name__=\"vllm:generation_tokens_total\", namespace=\"code-place-prod\"}[5m]))", "legendFormat": "generation tokens/s"} @@ -189,64 +223,90 @@ data: }, { "type": "timeseries", - "title": "vLLM Pod Resources", - "gridPos": {"x": 8, "y": 12, "w": 8, "h": 8}, + "title": "Prod vLLM CPU Usage", + "gridPos": {"x": 6, "y": 12, "w": 6, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"code-place-prod\", pod=~\"vllm-.*\", container=\"vllm\"}[5m]))", "legendFormat": "{{pod}} cpu cores"}, - {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"code-place-prod\", pod=~\"vllm-.*\", container=\"vllm\"})", "legendFormat": "{{pod}} memory bytes"} - ] - }, - { - "type": "timeseries", - "title": "vLLM HF Cache PVC", - "gridPos": {"x": 16, "y": 12, "w": 8, "h": 8}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "percentunit"}}, - "targets": [ - {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}), 1)", "legendFormat": "usage"} - ] - }, - { - "type": "timeseries", - "title": "AI Hint API Outcomes", - "gridPos": {"x": 0, "y": 20, "w": 8, "h": 8}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "ops"}}, + "fieldConfig": { + "defaults": { + "unit": "cores", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ - {"expr": "sum by (namespace, scope, status) (rate(codeplace_ai_hint_api_outcome_total{namespace=~\"code-place-(dev|prod)\"}[5m]))", "legendFormat": "{{namespace}} {{scope}} {{status}}/s"} + {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"code-place-prod\", pod=~\"vllm-.*\", container=\"vllm\"}[5m]))", "legendFormat": "{{pod}}"} ] }, { "type": "timeseries", - "title": "Backend AI Hint Streams", - "gridPos": {"x": 8, "y": 20, "w": 8, "h": 8}, + "title": "Prod vLLM Memory Usage", + "gridPos": {"x": 12, "y": 12, "w": 6, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "ops"}}, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ - {"expr": "sum by (namespace, status) (rate(codeplace_ai_hint_requests_total{namespace=~\"code-place-(dev|prod)\"}[5m]))", "legendFormat": "{{namespace}} {{status}} streams/s"} + {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"code-place-prod\", pod=~\"vllm-.*\", container=\"vllm\"})", "legendFormat": "{{pod}}"} ] }, { "type": "timeseries", - "title": "Backend AI Hint Duration", - "gridPos": {"x": 16, "y": 20, "w": 8, "h": 8}, + "title": "Prod vLLM HF Cache PVC", + "gridPos": {"x": 18, "y": 12, "w": 6, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "s"}}, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ - {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace=~\"code-place-(dev|prod)\", status=\"success\"}[10m])))", "legendFormat": "{{namespace}} success p95"}, - {"expr": "histogram_quantile(0.95, sum by (namespace, status, le) (rate(codeplace_ai_hint_duration_seconds_bucket{namespace=~\"code-place-(dev|prod)\", status!=\"success\"}[10m])))", "legendFormat": "{{namespace}} {{status}} p95"} + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"code-place-prod\", persistentvolumeclaim=\"vllm-hf-cache\"}), 1)", "legendFormat": "usage"} ] }, { "type": "timeseries", - "title": "GPU Utilization", - "gridPos": {"x": 0, "y": 28, "w": 6, "h": 8}, + "title": "Prod GPU Utilization", + "description": "GPU telemetry is for the production-only vLLM node.", + "gridPos": {"x": 0, "y": 20, "w": 6, "h": 8}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "percent", + "min": 0, + "max": 100, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "thresholdsStyle": {"mode": "line"} + }, "thresholds": { "mode": "absolute", "steps": [ @@ -263,12 +323,23 @@ data: }, { "type": "timeseries", - "title": "GPU FB Memory Usage", - "gridPos": {"x": 6, "y": 28, "w": 6, "h": 8}, + "title": "Prod GPU FB Memory Usage", + "gridPos": {"x": 6, "y": 20, "w": 6, "h": 8}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "thresholdsStyle": {"mode": "line"} + }, "thresholds": { "mode": "absolute", "steps": [ @@ -285,12 +356,22 @@ data: }, { "type": "timeseries", - "title": "GPU Temperature", - "gridPos": {"x": 12, "y": 28, "w": 6, "h": 8}, + "title": "Prod GPU Temperature", + "gridPos": {"x": 12, "y": 20, "w": 6, "h": 8}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "celsius", + "min": 0, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "thresholdsStyle": {"mode": "line"} + }, "thresholds": { "mode": "absolute", "steps": [ @@ -306,13 +387,23 @@ data: ] }, { - "type": "timeseries", - "title": "GPU XID Errors", - "gridPos": {"x": 18, "y": 28, "w": 6, "h": 8}, + "type": "state-timeline", + "title": "Prod GPU Last XID Error Code", + "description": "DCGM의 GPU별 마지막 XID code입니다. 0은 오류 없음이며, 0이 아닌 숫자는 심각도나 발생 횟수가 아닌 오류 종류입니다.", + "gridPos": {"x": 18, "y": 20, "w": 6, "h": 8}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", + "decimals": 0, + "mappings": [ + { + "type": "value", + "options": { + "0": {"text": "No XID error", "color": "green", "index": 0} + } + } + ], "thresholds": { "mode": "absolute", "steps": [ @@ -328,10 +419,22 @@ data: }, { "type": "timeseries", - "title": "GPU Tensor / DRAM Activity", - "gridPos": {"x": 0, "y": 36, "w": 12, "h": 8}, + "title": "Prod GPU Tensor / DRAM Activity", + "gridPos": {"x": 0, "y": 28, "w": 12, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "percentunit"}}, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ {"expr": "DCGM_FI_PROF_PIPE_TENSOR_ACTIVE{namespace=\"monitoring\"}", "legendFormat": "{{Hostname}} gpu{{gpu}} tensor"}, {"expr": "DCGM_FI_PROF_DRAM_ACTIVE{namespace=\"monitoring\"}", "legendFormat": "{{Hostname}} gpu{{gpu}} dram"} @@ -339,10 +442,22 @@ data: }, { "type": "timeseries", - "title": "GPU FB Memory Bytes", - "gridPos": {"x": 12, "y": 36, "w": 12, "h": 8}, + "title": "Prod GPU FB Memory Bytes", + "gridPos": {"x": 12, "y": 28, "w": 12, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "bytes"}}, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ {"expr": "DCGM_FI_DEV_FB_USED{namespace=\"monitoring\"} * 1024 * 1024", "legendFormat": "{{Hostname}} gpu{{gpu}} used"}, {"expr": "DCGM_FI_DEV_FB_FREE{namespace=\"monitoring\"} * 1024 * 1024", "legendFormat": "{{Hostname}} gpu{{gpu}} free"} diff --git a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml index 8a55a6bc..51d57150 100644 --- a/kubernetes/monitoring/grafana-dashboard-codeplace.yaml +++ b/kubernetes/monitoring/grafana-dashboard-codeplace.yaml @@ -13,6 +13,17 @@ data: "version": 1, "refresh": "30s", "tags": ["codeplace"], + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": ["codeplace"], + "targetBlank": false, + "title": "CodePlace Dashboards", + "type": "dashboards" + } + ], "editable": true, "time": { "from": "now-1h", @@ -21,17 +32,15 @@ data: "templating": { "list": [ { - "name": "namespace", - "type": "query", - "datasource": "Prometheus", - "query": "label_values(up{job=\"backend\", namespace=~\"code-place-(dev|prod)\"}, namespace)", - "refresh": 1, - "includeAll": true, - "allValue": ".*", - "multi": true, + "name": "environment", + "label": "Environment", + "type": "custom", + "query": "prod,dev", + "includeAll": false, + "multi": false, "current": { - "text": "All", - "value": ".*" + "text": "prod", + "value": "prod" } } ] @@ -54,31 +63,33 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "min by (namespace) (up{job=\"backend\", namespace=~\"$namespace\"})", "legendFormat": "{{namespace}}"} + {"expr": "(sum by (namespace) (up{job=\"backend\", namespace=\"code-place-$environment\"}) >= bool max by (namespace) (kube_deployment_spec_replicas{namespace=\"code-place-$environment\", deployment=\"backend\"})) or on() vector(0)", "legendFormat": "{{namespace}}", "instant": true} ] }, { - "type": "stat", + "type": "timeseries", "title": "API / Ingress Request Rate", - "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "gridPos": {"x": 4, "y": 0, "w": 5, "h": 4}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "reqps"}}, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "fieldConfig": {"defaults": {"unit": "reqps", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}}, "targets": [ - {"expr": "codeplace:api_request_rate5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} api"}, - {"expr": "codeplace:ingress_request_rate2m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} ingress"} + {"expr": "codeplace:api_request_rate5m{namespace=\"code-place-$environment\"}", "legendFormat": "{{namespace}} api"}, + {"expr": "codeplace:ingress_request_rate2m{namespace=\"code-place-$environment\"}", "legendFormat": "{{namespace}} ingress"} ] }, { - "type": "stat", + "type": "timeseries", "title": "API / Ingress 5xx Ratio", - "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "gridPos": {"x": 9, "y": 0, "w": 5, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "percentunit", + "min": 0, + "max": 1, + "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}, "thresholdsStyle": {"mode": "line"}}, "thresholds": { "mode": "absolute", "steps": [ @@ -89,20 +100,21 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "codeplace:api_5xx_ratio5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} api"}, - {"expr": "codeplace:ingress_5xx_rate2m{namespace=~\"$namespace\"} / clamp_min(codeplace:ingress_request_rate2m{namespace=~\"$namespace\"}, 1)", "legendFormat": "{{namespace}} ingress"} + {"expr": "codeplace:api_5xx_ratio5m{namespace=\"code-place-$environment\"}", "legendFormat": "{{namespace}} api"}, + {"expr": "codeplace:ingress_5xx_rate2m{namespace=\"code-place-$environment\"} / clamp_min(codeplace:ingress_request_rate2m{namespace=\"code-place-$environment\"}, 0.001)", "legendFormat": "{{namespace}} ingress"} ] }, { - "type": "stat", + "type": "timeseries", "title": "p95 Latency", - "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "gridPos": {"x": 14, "y": 0, "w": 5, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "s", + "min": 0, + "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}, "thresholdsStyle": {"mode": "line"}}, "thresholds": { "mode": "absolute", "steps": [ @@ -113,19 +125,20 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "codeplace:api_p95_latency_seconds5m{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} + {"expr": "codeplace:api_p95_latency_seconds5m{namespace=\"code-place-$environment\"}", "legendFormat": "{{namespace}}"} ] }, { - "type": "stat", + "type": "timeseries", "title": "Judge Waiting Queue", - "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "gridPos": {"x": 19, "y": 0, "w": 5, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", + "min": 0, + "custom": {"drawStyle": "line", "lineInterpolation": "stepAfter", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "stacking": {"mode": "none", "group": "A"}, "thresholdsStyle": {"mode": "line"}}, "thresholds": { "mode": "absolute", "steps": [ @@ -136,9 +149,8 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "codeplace:judge_waiting_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}}"} + {"expr": "codeplace:judge_waiting_queue_length{namespace=\"code-place-$environment\"}", "legendFormat": "{{namespace}}"} ] }, { @@ -146,9 +158,9 @@ data: "title": "Backend Requests by Status", "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "reqps"}}, + "fieldConfig": {"defaults": {"unit": "reqps", "custom": {"drawStyle": "bars", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 70, "showPoints": "never", "barAlignment": 0, "stacking": {"mode": "normal", "group": "A"}}}}, "targets": [ - {"expr": "sum by (namespace, status_code) (rate(codeplace_http_requests_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} {{status_code}}"} + {"expr": "sum by (namespace, status_code) (rate(codeplace_http_requests_total{namespace=\"code-place-$environment\"}[5m]))", "legendFormat": "{{namespace}} {{status_code}}"} ] }, { @@ -156,22 +168,21 @@ data: "title": "Backend Latency", "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "s"}}, + "fieldConfig": {"defaults": {"unit": "s", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}}, "targets": [ - {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])))", "legendFormat": "{{namespace}} p95"}, - {"expr": "histogram_quantile(0.99, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=~\"$namespace\"}[5m])))", "legendFormat": "{{namespace}} p99"} + {"expr": "histogram_quantile(0.95, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=\"code-place-$environment\"}[5m])))", "legendFormat": "{{namespace}} p95"}, + {"expr": "histogram_quantile(0.99, sum by (namespace, le) (rate(codeplace_http_request_duration_seconds_bucket{namespace=\"code-place-$environment\"}[5m])))", "legendFormat": "{{namespace}} p99"} ] }, { - "type": "bargauge", + "type": "timeseries", "title": "Judge Queue Length", "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "options": {"displayMode": "gradient", "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "lineInterpolation": "stepAfter", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "stacking": {"mode": "none", "group": "A"}}}}, "targets": [ - {"expr": "codeplace_waiting_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} waiting_queue"}, - {"expr": "codeplace_celery_broker_queue_length{namespace=~\"$namespace\"}", "legendFormat": "{{namespace}} celery"} + {"expr": "max by (namespace) (codeplace_waiting_queue_length{namespace=\"code-place-$environment\"})", "legendFormat": "{{namespace}} waiting_queue"}, + {"expr": "max by (namespace) (codeplace_celery_broker_queue_length{namespace=\"code-place-$environment\"})", "legendFormat": "{{namespace}} celery"} ] }, { @@ -184,25 +195,26 @@ data: { "type": "timeseries", "title": "Submission / Judge Outcomes", - "gridPos": {"x": 0, "y": 20, "w": 24, "h": 8}, + "gridPos": {"x": 0, "y": 13, "w": 24, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "ops"}}, + "fieldConfig": {"defaults": {"unit": "ops", "custom": {"drawStyle": "bars", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 70, "showPoints": "never", "barAlignment": 0, "stacking": {"mode": "normal", "group": "A"}}}}, "targets": [ - {"expr": "sum by (namespace, scope, status) (rate(codeplace_submission_create_outcome_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} create {{scope}} {{status}}/s"}, - {"expr": "sum by (namespace, scope, status) (rate(codeplace_judge_task_outcome_total{namespace=~\"$namespace\"}[5m]))", "legendFormat": "{{namespace}} judge {{scope}} {{status}}/s"} + {"expr": "sum by (namespace, scope, status) (rate(codeplace_submission_create_outcome_total{namespace=\"code-place-$environment\"}[5m]))", "legendFormat": "{{namespace}} create {{scope}} {{status}}/s"}, + {"expr": "avg by (namespace, scope, status) (rate(codeplace_judge_task_outcome_total{namespace=\"code-place-$environment\"}[5m]))", "legendFormat": "{{namespace}} judge {{scope}} {{status}}/s"} ] }, { "type": "row", - "title": "Platform", - "gridPos": {"x": 0, "y": 36, "w": 24, "h": 1}, + "title": "Current Health", + "gridPos": {"x": 0, "y": 21, "w": 24, "h": 1}, "collapsed": false, "panels": [] }, { - "type": "stat", + "type": "table", "title": "Pod Ready", - "gridPos": {"x": 0, "y": 37, "w": 6, "h": 5}, + "description": "선택한 환경의 애플리케이션 Pod별 현재 readiness입니다. 1은 Ready, 0은 Not Ready입니다.", + "gridPos": {"x": 0, "y": 22, "w": 12, "h": 7}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -216,88 +228,38 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"showHeader": true, "cellHeight": "sm"}, "targets": [ - {"expr": "min by (namespace, pod) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}}"} + {"expr": "min by (namespace, pod) (kube_pod_status_ready{namespace=\"code-place-$environment\", condition=\"true\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}}", "format": "table", "instant": true} ] }, { - "type": "timeseries", - "title": "Pod Restarts", - "gridPos": {"x": 6, "y": 37, "w": 6, "h": 5}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} - ] - }, - { - "type": "timeseries", - "title": "Container CPU Usage", - "gridPos": {"x": 12, "y": 37, "w": 6, "h": 5}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "cores"}}, - "targets": [ - {"expr": "topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[5m])))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} - ] - }, - { - "type": "timeseries", - "title": "Container Memory Usage", - "gridPos": {"x": 18, "y": 37, "w": 6, "h": 5}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "bytes"}}, - "targets": [ - {"expr": "topk(10, container_memory_working_set_bytes{namespace=~\"$namespace\", container!=\"\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{namespace}} {{pod}} {{container}}"} - ] - }, - { - "type": "timeseries", - "title": "PVC Usage", - "gridPos": {"x": 0, "y": 42, "w": 8, "h": 6}, + "type": "table", + "title": "Deployment Unavailable Replicas", + "description": "Deployment별 unavailable replica 수와 generation 반영 여부를 현재값 표로 표시합니다.", + "gridPos": {"x": 12, "y": 22, "w": 12, "h": 7}, "datasource": "Prometheus", "fieldConfig": { "defaults": { - "unit": "percentunit", + "unit": "short", "thresholds": { "mode": "absolute", "steps": [ {"color": "green", "value": null}, - {"color": "orange", "value": 0.75}, - {"color": "red", "value": 0.85} + {"color": "red", "value": 1} ] } } }, + "options": {"showHeader": true, "cellHeight": "sm"}, "targets": [ - {"expr": "max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"}), 1)", "legendFormat": "{{namespace}} {{persistentvolumeclaim}}"} + {"expr": "label_replace(max by (namespace, deployment) (kube_deployment_status_replicas_unavailable{namespace=\"code-place-$environment\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"}), \"condition\", \"unavailable replicas\", \"deployment\", \".*\") or label_replace(max by (namespace, deployment) (kube_deployment_status_observed_generation{namespace=\"code-place-$environment\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"} != bool on (namespace, deployment) kube_deployment_metadata_generation{namespace=\"code-place-$environment\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"}), \"condition\", \"generation mismatch\", \"deployment\", \".*\")", "legendFormat": "{{namespace}} {{deployment}} {{condition}}", "format": "table", "instant": true} ] }, { - "type": "timeseries", - "title": "Node Pressure", - "gridPos": {"x": 8, "y": 42, "w": 8, "h": 6}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "kube_node_status_condition{condition=~\"MemoryPressure|DiskPressure|PIDPressure\", status=\"true\"}", "legendFormat": "{{node}} {{condition}}"} - ] - }, - { - "type": "timeseries", - "title": "Deployment Unavailable Replicas", - "gridPos": {"x": 16, "y": 42, "w": 8, "h": 6}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "max by (namespace, deployment) (kube_deployment_status_replicas_unavailable{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"})", "legendFormat": "{{namespace}} {{deployment}} unavailable"}, - {"expr": "max by (namespace, deployment) (kube_deployment_status_observed_generation{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"} != on (namespace, deployment) kube_deployment_metadata_generation{namespace=~\"$namespace\", deployment=~\"backend|frontend|hub-auth|celery-worker|celery-beat|judge-server|vllm\"})", "legendFormat": "{{namespace}} {{deployment}} generation mismatch"} - ] - }, - { - "type": "timeseries", + "type": "stat", "title": "Service Ready Endpoints", - "gridPos": {"x": 0, "y": 48, "w": 12, "h": 6}, + "gridPos": {"x": 0, "y": 29, "w": 8, "h": 5}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -311,32 +273,15 @@ data: } } }, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum by (namespace, endpoint) (kube_endpoint_address{namespace=~\"$namespace\", endpoint=~\"backend|frontend|hub-auth|judge-server|vllm\", ready=\"true\"})", "legendFormat": "{{namespace}} {{endpoint}} ready endpoints"} + {"expr": "sum by (namespace, service) (label_replace(kube_endpointslice_endpoints{namespace=\"code-place-$environment\", endpointslice=~\"(backend|frontend|hub-auth|judge-server|vllm)-.*\", ready=\"true\"}, \"service\", \"$1\", \"endpointslice\", \"(backend|frontend|hub-auth|judge-server|vllm)-.*\"))", "legendFormat": "{{namespace}} {{service}} ready endpoints", "instant": true} ] }, - { - "type": "timeseries", - "title": "PDB Healthy Pods", - "gridPos": {"x": 12, "y": 48, "w": 12, "h": 6}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "kube_poddisruptionbudget_status_current_healthy{namespace=~\"$namespace\", poddisruptionbudget=~\"backend|frontend\"}", "legendFormat": "{{namespace}} {{poddisruptionbudget}} current"}, - {"expr": "kube_poddisruptionbudget_status_desired_healthy{namespace=~\"$namespace\", poddisruptionbudget=~\"backend|frontend\"}", "legendFormat": "{{namespace}} {{poddisruptionbudget}} desired"} - ] - }, - { - "type": "row", - "title": "Data Stores", - "gridPos": {"x": 0, "y": 54, "w": 24, "h": 1}, - "collapsed": false, - "panels": [] - }, { "type": "stat", "title": "PostgreSQL Ready Replicas", - "gridPos": {"x": 0, "y": 55, "w": 6, "h": 5}, + "gridPos": {"x": 8, "y": 29, "w": 8, "h": 5}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -351,15 +296,15 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum by (namespace) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"postgres-[0-9]+\"}) or vector(0)", "legendFormat": "{{namespace}} ready"} + {"expr": "sum by (namespace) (kube_pod_status_ready{namespace=\"code-place-$environment\", condition=\"true\", pod=~\"postgres-[0-9]+\"}) or vector(0)", "legendFormat": "{{namespace}} ready", "instant": true} ] }, { "type": "stat", "title": "Redis/Sentinel Ready Replicas", - "gridPos": {"x": 6, "y": 55, "w": 6, "h": 5}, + "gridPos": {"x": 16, "y": 29, "w": 8, "h": 5}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -374,114 +319,9 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, - "targets": [ - {"expr": "sum by (namespace) (kube_pod_status_ready{namespace=~\"$namespace\", condition=\"true\", pod=~\"redis-.*|redis-replication-.*\"}) or vector(0)", "legendFormat": "{{namespace}} ready"} - ] - }, - { - "type": "stat", - "title": "PostgreSQL Exporter Up", - "gridPos": {"x": 12, "y": 55, "w": 6, "h": 5}, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - {"color": "red", "value": null}, - {"color": "green", "value": 1} - ] - } - } - }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, - "targets": [ - {"expr": "min by (namespace) (cnpg_collector_up{namespace=~\"$namespace\", cluster=\"postgres\"})", "legendFormat": "{{namespace}}"} - ] - }, - { - "type": "stat", - "title": "Redis Exporter Up", - "gridPos": {"x": 18, "y": 55, "w": 6, "h": 5}, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - {"color": "red", "value": null}, - {"color": "green", "value": 1} - ] - } - } - }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, - "targets": [ - {"expr": "min by (namespace) (redis_up{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}}"} - ] - }, - { - "type": "timeseries", - "title": "PostgreSQL Restarts", - "gridPos": {"x": 0, "y": 60, "w": 6, "h": 5}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"postgres-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} - ] - }, - { - "type": "timeseries", - "title": "Redis Restarts", - "gridPos": {"x": 6, "y": 60, "w": 6, "h": 5}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\", pod=~\"redis-.*|redis-replication-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} - ] - }, - { - "type": "timeseries", - "title": "PostgreSQL WAL Bytes", - "gridPos": {"x": 12, "y": 60, "w": 6, "h": 5}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "Bps"}}, - "targets": [ - {"expr": "sum by (namespace) (rate(cnpg_collector_wal_bytes{namespace=~\"$namespace\", cluster=\"postgres\"}[5m]))", "legendFormat": "{{namespace}}"} - ] - }, - { - "type": "timeseries", - "title": "Redis Memory Usage", - "gridPos": {"x": 18, "y": 60, "w": 6, "h": 5}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "bytes"}}, - "targets": [ - {"expr": "max by (namespace, pod) (redis_memory_used_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} used"}, - {"expr": "max by (namespace, pod) (redis_memory_max_bytes{namespace=~\"$namespace\"})", "legendFormat": "{{namespace}} {{pod}} maxmemory"} - ] - }, - { - "type": "timeseries", - "title": "PostgreSQL Collector Errors", - "gridPos": {"x": 0, "y": 65, "w": 8, "h": 6}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "max by (namespace, pod) (cnpg_collector_last_collection_error{namespace=~\"$namespace\", cluster=\"postgres\"})", "legendFormat": "{{namespace}} {{pod}}"} - ] - }, - { - "type": "timeseries", - "title": "PostgreSQL Nodes Used", - "gridPos": {"x": 8, "y": 65, "w": 16, "h": 6}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "min by (namespace) (cnpg_collector_nodes_used{namespace=~\"$namespace\", cluster=\"postgres\"})", "legendFormat": "{{namespace}} nodes"} + {"expr": "sum by (namespace) (kube_pod_status_ready{namespace=\"code-place-$environment\", condition=\"true\", pod=~\"redis-.*|redis-replication-.*\"}) or vector(0)", "legendFormat": "{{namespace}} ready", "instant": true} ] } ] diff --git a/kubernetes/monitoring/grafana-dashboard-kubernetes-events.yaml b/kubernetes/monitoring/grafana-dashboard-kubernetes-events.yaml index 6fa6c5ec..c5560795 100644 --- a/kubernetes/monitoring/grafana-dashboard-kubernetes-events.yaml +++ b/kubernetes/monitoring/grafana-dashboard-kubernetes-events.yaml @@ -13,6 +13,17 @@ data: "version": 1, "refresh": "30s", "tags": ["codeplace", "kubernetes", "events"], + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": ["codeplace"], + "targetBlank": false, + "title": "CodePlace Dashboards", + "type": "dashboards" + } + ], "editable": true, "time": { "from": "now-1h", @@ -21,15 +32,15 @@ data: "templating": { "list": [ { - "name": "namespace", + "name": "environment", + "label": "Environment", "type": "custom", - "query": "code-place-dev,code-place-prod,monitoring,longhorn-system,kube-system", - "multi": true, - "includeAll": true, - "allValue": "code-place-dev|code-place-prod|monitoring|longhorn-system|kube-system", + "query": "prod,dev", + "multi": false, + "includeAll": false, "current": { - "text": "code-place-dev", - "value": "code-place-dev" + "text": "prod", + "value": "prod" } } ] @@ -37,7 +48,8 @@ data: "panels": [ { "type": "stat", - "title": "Event Exporter Ready", + "title": "Event Telemetry Health (Global)", + "description": "Cluster-wide collector health. Missing exporter or kube-state-metrics scrapes are treated as unhealthy, so expected-zero event states do not hide telemetry loss.", "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -52,10 +64,9 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"kubernetes-event-exporter-.*\"}) or vector(0)", "legendFormat": "ready"} - ] + {"expr": "max(kube_deployment_status_replicas_available{namespace=\"monitoring\", deployment=\"kubernetes-event-exporter\"}) or vector(0)", "legendFormat": "exporter ready", "instant": true}, {"expr": "max(up{namespace=\"monitoring\", job=\"kubernetes-event-exporter\"}) or vector(0)", "legendFormat": "exporter scrape", "instant": true}, {"expr": "max(up{namespace=\"monitoring\", service=~\".*kube-state-metrics.*\"}) or vector(0)", "legendFormat": "kube-state-metrics scrape", "instant": true} ] }, { "type": "stat", @@ -74,10 +85,9 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(kube_pod_container_status_waiting_reason{namespace=~\"$namespace\", reason=~\"ImagePullBackOff|ErrImagePull\"}) or vector(0)", "legendFormat": "pods"} - ] + {"expr": "sum(kube_pod_container_status_waiting_reason{namespace=\"code-place-$environment\", reason=~\"ImagePullBackOff|ErrImagePull\"}) or vector(0)", "legendFormat": "pods", "instant": true} ] }, { "type": "stat", @@ -96,10 +106,9 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(kube_pod_container_status_waiting_reason{namespace=~\"$namespace\", reason=\"CrashLoopBackOff\"}) or vector(0)", "legendFormat": "pods"} - ] + {"expr": "sum(kube_pod_container_status_waiting_reason{namespace=\"code-place-$environment\", reason=\"CrashLoopBackOff\"}) or vector(0)", "legendFormat": "pods", "instant": true} ] }, { "type": "stat", @@ -118,10 +127,9 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(max_over_time(kube_pod_container_status_last_terminated_reason{namespace=~\"$namespace\", reason=\"OOMKilled\"}[10m])) or vector(0)", "legendFormat": "containers"} - ] + {"expr": "sum((kube_pod_container_status_last_terminated_reason{namespace=\"code-place-$environment\", reason=\"OOMKilled\"} == 1) and on (namespace, pod, container) (kube_pod_container_status_last_terminated_timestamp{namespace=\"code-place-$environment\"} > time() - 600)) or vector(0)", "legendFormat": "containers", "instant": true} ] }, { "type": "stat", @@ -140,10 +148,9 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(kube_pod_status_phase{namespace=~\"$namespace\", phase=\"Pending\"}) or vector(0)", "legendFormat": "pods"} - ] + {"expr": "sum(kube_pod_status_phase{namespace=\"code-place-$environment\", phase=\"Pending\"}) or vector(0)", "legendFormat": "pods", "instant": true} ] }, { "type": "stat", @@ -162,29 +169,56 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(kube_pod_status_unschedulable{namespace=~\"$namespace\"}) or vector(0)", "legendFormat": "pods"} - ] + {"expr": "sum(kube_pod_status_unschedulable{namespace=\"code-place-$environment\"}) or vector(0)", "legendFormat": "pods", "instant": true} ] }, { - "type": "timeseries", + "type": "state-timeline", "title": "Container Waiting Reasons", + "description": "컨테이너별 waiting 상태의 시작과 종료를 표시합니다. 막대 높이는 사건 횟수가 아닙니다.", "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": {"text": "Normal", "color": "green", "index": 0}, + "1": {"text": "Waiting", "color": "red", "index": 1} + } + } + ] + } + }, "targets": [ - {"expr": "sum by (namespace, pod, container, reason) (kube_pod_container_status_waiting_reason{namespace=~\"$namespace\", reason=~\"CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerConfigError|CreateContainerError\"})", "legendFormat": "{{namespace}} {{pod}} {{container}} {{reason}}"} + {"expr": "sum by (namespace, pod, container, reason) (kube_pod_container_status_waiting_reason{namespace=\"code-place-$environment\", reason=~\"CrashLoopBackOff|ImagePullBackOff|ErrImagePull|CreateContainerConfigError|CreateContainerError\"})", "legendFormat": "{{namespace}} {{pod}} {{container}} {{reason}}"} ] }, { - "type": "timeseries", + "type": "state-timeline", "title": "Recent OOMKilled Containers", + "description": "최근 10분 내 OOMKilled 여부를 컨테이너별 상태로 표시합니다.", "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "type": "value", + "options": { + "0": {"text": "Clear", "color": "green", "index": 0}, + "1": {"text": "OOMKilled", "color": "red", "index": 1} + } + } + ] + } + }, "targets": [ - {"expr": "max by (namespace, pod, container) (max_over_time(kube_pod_container_status_last_terminated_reason{namespace=~\"$namespace\", reason=\"OOMKilled\"}[10m]))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} + {"expr": "max by (namespace, pod, container) ((kube_pod_container_status_last_terminated_reason{namespace=\"code-place-$environment\", reason=\"OOMKilled\"} == bool 1) * on (namespace, pod, container) (kube_pod_container_status_last_terminated_timestamp{namespace=\"code-place-$environment\"} > bool time() - 600))", "legendFormat": "{{namespace}} {{pod}} {{container}}"} ] }, { @@ -199,7 +233,7 @@ data: "dedupStrategy": "none" }, "targets": [ - {"expr": "{namespace=\"monitoring\", app_kubernetes_io_name=\"kubernetes-event-exporter\"}"} + {"expr": "{namespace=\"monitoring\", app_kubernetes_io_name=\"kubernetes-event-exporter\"} | json | metadata_namespace=\"code-place-$environment\" | line_format \"[{{.reason}}] {{.involvedObject_kind}}/{{.involvedObject_name}}: {{.message}}\""} ] }, { @@ -214,7 +248,7 @@ data: "dedupStrategy": "none" }, "targets": [ - {"expr": "{namespace=\"monitoring\", app_kubernetes_io_name=\"kubernetes-event-exporter\"} |~ \"FailedScheduling|ImagePullBackOff|ErrImagePull|BackOff|OOMKilling|Unhealthy|FailedMount|FailedAttachVolume\""} + {"expr": "{namespace=\"monitoring\", app_kubernetes_io_name=\"kubernetes-event-exporter\"} | json | metadata_namespace=\"code-place-$environment\" |~ \"FailedScheduling|ImagePullBackOff|ErrImagePull|BackOff|OOMKilling|Unhealthy|FailedMount|FailedAttachVolume\" | line_format \"[{{.reason}}] {{.involvedObject_kind}}/{{.involvedObject_name}}: {{.message}}\""} ] } ] diff --git a/kubernetes/monitoring/grafana-dashboard-log-pipeline.yaml b/kubernetes/monitoring/grafana-dashboard-log-pipeline.yaml new file mode 100644 index 00000000..05afb12f --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-log-pipeline.yaml @@ -0,0 +1,177 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-log-pipeline + labels: + grafana_dashboard: "1" +data: + codeplace-log-pipeline.json: | + { + "uid": "codeplace-log-pipeline", + "title": "CodePlace Log Pipeline", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "logs", "platform"], + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": ["codeplace"], + "targetBlank": false, + "title": "CodePlace Dashboards", + "type": "dashboards" + } + ], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [] + }, + "panels": [ + { + "type": "stat", + "title": "Loki Replica Availability (Global)", + "description": "Cluster-wide Loki single-binary and gateway available/desired ratios.", + "gridPos": {"x": 0, "y": 0, "w": 8, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 0.5}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "(max(kube_statefulset_status_replicas_ready{namespace=\"monitoring\", statefulset=\"loki\"}) / clamp_min(max(kube_statefulset_replicas{namespace=\"monitoring\", statefulset=\"loki\"}), 1)) or vector(0)", "legendFormat": "single-binary", "instant": true}, + {"expr": "(max(kube_deployment_status_replicas_available{namespace=\"monitoring\", deployment=\"loki-gateway\"}) / clamp_min(max(kube_deployment_spec_replicas{namespace=\"monitoring\", deployment=\"loki-gateway\"}), 1)) or vector(0)", "legendFormat": "gateway", "instant": true} + ] + }, + { + "type": "stat", + "title": "Alloy Nodes (Global)", + "description": "Cluster-wide Alloy DaemonSet readiness.", + "gridPos": {"x": 8, "y": 0, "w": 8, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "kube_daemonset_status_number_available{namespace=\"monitoring\", daemonset=\"alloy\"}", "legendFormat": "available", "instant": true}, + {"expr": "kube_daemonset_status_desired_number_scheduled{namespace=\"monitoring\", daemonset=\"alloy\"}", "legendFormat": "desired", "instant": true} + ] + }, + { + "type": "stat", + "title": "Log Pipeline Telemetry Health (Global)", + "description": "An absent Loki or Alloy scrape is treated as unhealthy, so expected-zero error panels cannot hide telemetry loss.", + "gridPos": {"x": 16, "y": 0, "w": 8, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min(up{namespace=\"monitoring\", service=~\"loki|loki-gateway\"}) or vector(0)", "legendFormat": "Loki scrape", "instant": true}, + {"expr": "min(up{namespace=\"monitoring\", job=~\".*alloy.*\"}) or vector(0)", "legendFormat": "Alloy scrape", "instant": true} + ] + }, + { + "type": "timeseries", + "title": "Loki PVC Usage (Global)", + "description": "Cluster-wide Loki storage usage.", + "gridPos": {"x": 0, "y": 5, "w": 12, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}, "thresholdsStyle": {"mode": "line"}}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "targets": [ + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim=~\".*loki.*\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim=~\".*loki.*\"}), 1)", "legendFormat": "usage"} + ] + }, + { + "type": "timeseries", + "title": "Loki Ingestion Rate (Global)", + "description": "Cluster-wide Loki ingestion.", + "gridPos": {"x": 12, "y": 5, "w": 12, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {"unit": "logs/sec", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}, + "overrides": [ + {"matcher": {"id": "byName", "options": "bytes received"}, "properties": [{"id": "unit", "value": "Bps"}, {"id": "custom.axisPlacement", "value": "right"}]} + ] + }, + "targets": [ + {"expr": "sum(rate(loki_distributor_lines_received_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "lines received"}, + {"expr": "sum(rate(loki_distributor_bytes_received_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "bytes received"} + ] + }, + { + "type": "timeseries", + "title": "Loki Request Errors (Global)", + "description": "Cluster-wide Loki request errors; expected zero values are paired with the telemetry health panel above.", + "gridPos": {"x": 0, "y": 12, "w": 8, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "reqps", "custom": {"drawStyle": "bars", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 70, "showPoints": "never", "barAlignment": 0, "stacking": {"mode": "normal", "group": "A"}}}}, + "targets": [ + {"expr": "sum by (route, status_code) (rate(loki_request_duration_seconds_count{namespace=\"monitoring\", status_code=~\"5..\"}[5m])) or vector(0)", "legendFormat": "{{route}} {{status_code}}"}, + {"expr": "sum by (route, status_code) (rate(loki_request_duration_seconds_count{namespace=\"monitoring\", status_code=~\"4..\"}[5m])) or vector(0)", "legendFormat": "{{route}} {{status_code}}"} + ] + }, + { + "type": "timeseries", + "title": "Alloy Loki Write Backpressure (Global)", + "description": "Cluster-wide Alloy write health.", + "gridPos": {"x": 8, "y": 12, "w": 8, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "ops", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}}, + "targets": [ + {"expr": "sum(rate(loki_write_batch_retries_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "batch retries/s"}, + {"expr": "sum(rate(loki_write_dropped_entries_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "dropped entries/s"} + ] + }, + { + "type": "timeseries", + "title": "Loki Canary Missing Entries in Previous 5m (Global)", + "description": "Cluster-wide Loki canary result.", + "gridPos": {"x": 16, "y": 12, "w": 8, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "lineInterpolation": "stepAfter", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "stacking": {"mode": "none", "group": "A"}}}}, + "targets": [ + {"expr": "sum(increase(loki_canary_missing_entries_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "missing entries"} + ] + } + ] + } diff --git a/kubernetes/monitoring/grafana-dashboard-logs.yaml b/kubernetes/monitoring/grafana-dashboard-logs.yaml index a986520f..a97e3fa3 100644 --- a/kubernetes/monitoring/grafana-dashboard-logs.yaml +++ b/kubernetes/monitoring/grafana-dashboard-logs.yaml @@ -13,6 +13,17 @@ data: "version": 1, "refresh": "30s", "tags": ["codeplace", "logs"], + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": ["codeplace"], + "targetBlank": false, + "title": "CodePlace Dashboards", + "type": "dashboards" + } + ], "editable": true, "time": { "from": "now-1h", @@ -21,106 +32,50 @@ data: "templating": { "list": [ { - "name": "namespace", + "name": "environment", + "label": "Environment", "type": "custom", - "query": "code-place-dev,code-place-prod,monitoring", - "multi": true, - "includeAll": true, - "allValue": "code-place-dev|code-place-prod|monitoring", + "query": "prod,dev", + "multi": false, + "includeAll": false, "current": { - "text": "code-place-dev", - "value": "code-place-dev" + "text": "prod", + "value": "prod" } }, { "name": "request_id", + "label": "Request ID (exact; replace __unset__)", "type": "textbox", - "query": ".*", + "query": "__unset__", "current": { - "text": ".*", - "value": ".*" + "text": "__unset__", + "value": "__unset__" } } ] }, "panels": [ { - "type": "stat", - "title": "Loki Ready Replicas", - "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - {"color": "red", "value": null}, - {"color": "green", "value": 1} - ] - } - } - }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, - "targets": [ - {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"loki-[0-9]+\"}) or vector(0)", "legendFormat": "single-binary"}, - {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"loki-gateway-.*\"}) or vector(0)", "legendFormat": "gateway"} - ] - }, - { - "type": "stat", - "title": "Alloy Nodes", - "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, - "targets": [ - {"expr": "kube_daemonset_status_number_available{namespace=\"monitoring\", daemonset=\"alloy\"} or vector(0)", "legendFormat": "available"}, - {"expr": "kube_daemonset_status_desired_number_scheduled{namespace=\"monitoring\", daemonset=\"alloy\"} or vector(0)", "legendFormat": "desired"} - ] - }, - { - "type": "stat", - "title": "Loki PVC Usage", - "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, - "datasource": "Prometheus", - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "thresholds": { - "mode": "absolute", - "steps": [ - {"color": "green", "value": null}, - {"color": "orange", "value": 0.75}, - {"color": "red", "value": 0.85} - ] - } - } - }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, - "targets": [ - {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim=~\".*loki.*\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim=~\".*loki.*\"}), 1) or vector(0)", "legendFormat": "usage"} - ] - }, - { - "type": "stat", + "type": "timeseries", "title": "Log Lines", - "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "gridPos": {"x": 0, "y": 0, "w": 8, "h": 4}, "datasource": {"type": "loki", "uid": "Loki"}, - "fieldConfig": {"defaults": {"unit": "short"}}, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}}, "targets": [ - {"expr": "sum(count_over_time({namespace=~\"$namespace\"}[5m]))", "legendFormat": "lines"} + {"expr": "sum(count_over_time({namespace=\"code-place-$environment\"}[5m]))", "legendFormat": "lines"} ] }, { - "type": "stat", - "title": "Backend Errors", - "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "type": "timeseries", + "title": "Backend Errors in Previous 5m", + "gridPos": {"x": 8, "y": 0, "w": 8, "h": 4}, "datasource": {"type": "loki", "uid": "Loki"}, "fieldConfig": { "defaults": { "unit": "short", + "min": 0, + "custom": {"drawStyle": "line", "lineInterpolation": "stepAfter", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "stacking": {"mode": "none", "group": "A"}, "thresholdsStyle": {"mode": "line"}}, "thresholds": { "mode": "absolute", "steps": [ @@ -131,19 +86,20 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(count_over_time({namespace=~\"$namespace\", app=\"backend\"} | json | level=\"ERROR\" [5m]))", "legendFormat": "errors"} + {"expr": "sum(count_over_time({namespace=\"code-place-$environment\", app=\"backend\"} | json | level=~\"ERROR|CRITICAL\" [5m]))", "legendFormat": "errors"} ] }, { - "type": "stat", - "title": "Frontend 5xx", - "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "type": "timeseries", + "title": "Frontend 5xx in Previous 5m", + "gridPos": {"x": 16, "y": 0, "w": 8, "h": 4}, "datasource": {"type": "loki", "uid": "Loki"}, "fieldConfig": { "defaults": { "unit": "short", + "min": 0, + "custom": {"drawStyle": "line", "lineInterpolation": "stepAfter", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "stacking": {"mode": "none", "group": "A"}, "thresholdsStyle": {"mode": "line"}}, "thresholds": { "mode": "absolute", "steps": [ @@ -154,19 +110,18 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(count_over_time({namespace=~\"$namespace\", app=\"frontend\"} | json | status_code >= 500 [5m]))", "legendFormat": "5xx"} + {"expr": "sum(count_over_time({namespace=\"code-place-$environment\", app=\"frontend\"} | json | status_code >= 500 [5m]))", "legendFormat": "5xx"} ] }, { "type": "timeseries", - "title": "Log Lines by Namespace", + "title": "Selected Environment Log Rate", "gridPos": {"x": 0, "y": 4, "w": 8, "h": 7}, "datasource": {"type": "loki", "uid": "Loki"}, - "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "fieldConfig": {"defaults": {"unit": "logs/sec", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}}, "targets": [ - {"expr": "sum by (namespace) (rate({namespace=~\"code-place-dev|code-place-prod|monitoring|kube-system\"}[5m]))", "legendFormat": "{{namespace}}"} + {"expr": "sum by (namespace) (rate({namespace=\"code-place-$environment\"}[5m]))", "legendFormat": "{{namespace}}"} ] }, { @@ -174,31 +129,32 @@ data: "title": "Application Log Lines by App", "gridPos": {"x": 8, "y": 4, "w": 8, "h": 7}, "datasource": {"type": "loki", "uid": "Loki"}, - "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "fieldConfig": {"defaults": {"unit": "logs/sec", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}}, "targets": [ - {"expr": "sum by (namespace, app) (rate({namespace=~\"code-place-dev|code-place-prod\", app=~\"backend|frontend\"}[5m]))", "legendFormat": "{{namespace}} {{app}}"}, - {"expr": "sum by (namespace, container) (rate({namespace=~\"code-place-dev|code-place-prod\", container=~\"celery-worker|celery-beat|judge-server|hub-auth|vllm\"}[5m]))", "legendFormat": "{{namespace}} {{container}}"} + {"expr": "sum by (namespace, app) (rate({namespace=\"code-place-$environment\", app=~\"backend|frontend\"}[5m]))", "legendFormat": "{{namespace}} {{app}}"}, + {"expr": "sum by (namespace, container) (rate({namespace=\"code-place-$environment\", container=~\"celery-worker|celery-beat|judge-server|hub-auth|vllm\"}[5m]))", "legendFormat": "{{namespace}} {{container}}"} ] }, { "type": "timeseries", - "title": "Structured Error Logs by Component", + "title": "Structured Backend / Celery Error Logs", "gridPos": {"x": 16, "y": 4, "w": 8, "h": 7}, "datasource": {"type": "loki", "uid": "Loki"}, - "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "fieldConfig": {"defaults": {"unit": "logs/sec", "custom": {"drawStyle": "bars", "lineInterpolation": "linear", "lineWidth": 1, "fillOpacity": 70, "showPoints": "never", "barAlignment": 0, "stacking": {"mode": "normal", "group": "A"}}}}, "targets": [ - {"expr": "sum by (namespace, app) (rate({namespace=~\"code-place-dev|code-place-prod\", app=~\"backend|frontend\"} | json | level=~\"ERROR|CRITICAL\" [5m]))", "legendFormat": "{{namespace}} {{app}}"}, - {"expr": "sum by (namespace, container) (rate({namespace=~\"code-place-dev|code-place-prod\", container=~\"celery-worker|celery-beat|judge-server|hub-auth|vllm\"} | json | level=~\"ERROR|CRITICAL\" [5m]))", "legendFormat": "{{namespace}} {{container}}"} + {"expr": "sum by (namespace, app) (rate({namespace=\"code-place-$environment\", app=\"backend\"} | json | level=~\"ERROR|CRITICAL\" [5m]))", "legendFormat": "{{namespace}} {{app}}"}, + {"expr": "sum by (namespace, container) (rate({namespace=\"code-place-$environment\", container=~\"celery-worker|celery-beat\"} | json | level=~\"ERROR|CRITICAL\" [5m]))", "legendFormat": "{{namespace}} {{container}}"} ] }, { "type": "stat", - "title": "Ingress 5xx", - "gridPos": {"x": 0, "y": 11, "w": 4, "h": 4}, + "title": "Ingress 5xx in Previous 5m", + "gridPos": {"x": 0, "y": 11, "w": 8, "h": 4}, "datasource": {"type": "loki", "uid": "Loki"}, "fieldConfig": { "defaults": { "unit": "short", + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -209,25 +165,25 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(count_over_time({namespace=\"kube-system\", app_kubernetes_io_name=\"traefik\"} | json | DownstreamStatus >= 500 [5m]))", "legendFormat": "5xx"} + {"expr": "sum(count_over_time({namespace=\"kube-system\", app_kubernetes_io_name=\"traefik\"} | json | RouterName=~\"code-place-$environment-.*\" | DownstreamStatus >= 500 [5m]))", "legendFormat": "5xx"} ] }, { "type": "timeseries", - "title": "Log Lines by Node", - "gridPos": {"x": 0, "y": 15, "w": 4, "h": 6}, + "title": "Selected Environment Log Rate by Node", + "gridPos": {"x": 0, "y": 15, "w": 8, "h": 6}, "datasource": {"type": "loki", "uid": "Loki"}, - "fieldConfig": {"defaults": {"unit": "logs/sec"}}, + "fieldConfig": {"defaults": {"unit": "logs/sec", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}}, "targets": [ - {"expr": "sum by (node) (rate({namespace=~\"code-place-dev|code-place-prod|monitoring|kube-system\"}[5m]))", "legendFormat": "{{node}}"} + {"expr": "sum by (node) (rate({namespace=\"code-place-$environment\"}[5m]))", "legendFormat": "{{node}}"} ] }, { "type": "logs", "title": "Recent Backend Errors", - "gridPos": {"x": 4, "y": 11, "w": 10, "h": 10}, + "gridPos": {"x": 8, "y": 11, "w": 8, "h": 10}, "datasource": {"type": "loki", "uid": "Loki"}, "options": { "showLabels": false, @@ -236,13 +192,13 @@ data: "dedupStrategy": "none" }, "targets": [ - {"expr": "{namespace=~\"$namespace\", app=\"backend\"} | json | level=\"ERROR\""} + {"expr": "{namespace=\"code-place-$environment\", app=\"backend\"} | json | level=~\"ERROR|CRITICAL\""} ] }, { "type": "logs", "title": "Auth, Judge, Celery, and vLLM Logs", - "gridPos": {"x": 14, "y": 11, "w": 10, "h": 10}, + "gridPos": {"x": 16, "y": 11, "w": 8, "h": 10}, "datasource": {"type": "loki", "uid": "Loki"}, "options": { "showLabels": false, @@ -251,7 +207,7 @@ data: "dedupStrategy": "none" }, "targets": [ - {"expr": "{namespace=~\"$namespace\", container=~\"hub-auth|celery-worker|celery-beat|judge-server|vllm\"}"} + {"expr": "{namespace=\"code-place-$environment\", container=~\"hub-auth|celery-worker|celery-beat|judge-server|vllm\"}"} ] }, { @@ -266,8 +222,8 @@ data: "dedupStrategy": "none" }, "targets": [ - {"expr": "{namespace=\"kube-system\", app_kubernetes_io_name=\"traefik\"} | json | DownstreamStatus >= 400", "legendFormat": "traefik"}, - {"expr": "{namespace=~\"$namespace\", app=\"frontend\"} | json | status_code >= 400", "legendFormat": "frontend"} + {"expr": "{namespace=\"kube-system\", app_kubernetes_io_name=\"traefik\"} | json | RouterName=~\"code-place-$environment-.*\" | DownstreamStatus >= 400", "legendFormat": "traefik"}, + {"expr": "{namespace=\"code-place-$environment\", app=\"frontend\"} | json | status_code >= 400", "legendFormat": "frontend"} ] }, { @@ -282,66 +238,17 @@ data: "dedupStrategy": "none" }, "targets": [ - {"expr": "{namespace=~\"$namespace\"} | json | request_id=~\"$request_id\""} + {"expr": "{namespace=\"code-place-$environment\"} | json | request_id=\"$request_id\""} ] }, { - "type": "logs", - "title": "Frontend Runtime Error Reports", + "type": "text", + "title": "Frontend Runtime Error Tracking", "gridPos": {"x": 0, "y": 38, "w": 24, "h": 8}, - "datasource": {"type": "loki", "uid": "Loki"}, "options": { - "showLabels": false, - "showTime": true, - "wrapLogMessage": true, - "dedupStrategy": "none" - }, - "targets": [ - {"expr": "{namespace=~\"$namespace\", app=\"backend\"} | json | logger=\"frontend.error\""} - ] - }, - { - "type": "timeseries", - "title": "Loki Ingestion Rate", - "gridPos": {"x": 0, "y": 46, "w": 8, "h": 7}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "logs/sec"}}, - "targets": [ - {"expr": "sum(rate(loki_distributor_lines_received_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "lines received"}, - {"expr": "sum(rate(loki_distributor_bytes_received_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "bytes received"} - ] - }, - { - "type": "timeseries", - "title": "Loki Request Errors", - "gridPos": {"x": 8, "y": 46, "w": 8, "h": 7}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "reqps"}}, - "targets": [ - {"expr": "sum by (route, status_code) (rate(loki_request_duration_seconds_count{namespace=\"monitoring\", status_code=~\"5..\"}[5m])) or vector(0)", "legendFormat": "{{route}} {{status_code}}"}, - {"expr": "sum by (route, status_code) (rate(loki_request_duration_seconds_count{namespace=\"monitoring\", status_code=~\"4..\"}[5m])) or vector(0)", "legendFormat": "{{route}} {{status_code}}"} - ] - }, - { - "type": "timeseries", - "title": "Alloy Loki Write Backpressure", - "gridPos": {"x": 16, "y": 46, "w": 8, "h": 7}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "sum(rate(loki_write_batch_retries_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "batch retries/s"}, - {"expr": "sum(rate(loki_write_dropped_entries_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "dropped entries/s"} - ] - }, - { - "type": "timeseries", - "title": "Loki Canary Missing Entries", - "gridPos": {"x": 0, "y": 53, "w": 8, "h": 6}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "sum(increase(loki_canary_missing_entries_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "missing entries"} - ] + "mode": "markdown", + "content": "Frontend runtime exceptions are captured by Sentry with the deployment release, environment, and per-error request ID. They are intentionally not queried from Loki because the browser is not a Kubernetes log producer." + } } ] } diff --git a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml index 7ced1fec..b8f47ea4 100644 --- a/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml +++ b/kubernetes/monitoring/grafana-dashboard-monitoring-stack.yaml @@ -10,157 +10,405 @@ data: "uid": "codeplace-monitoring-stack", "title": "CodePlace Monitoring Stack", "schemaVersion": 39, - "version": 1, + "version": 2, "refresh": "30s", - "tags": ["codeplace", "monitoring"], + "tags": ["codeplace", "monitoring", "scrape-contract"], + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": ["codeplace"], + "targetBlank": false, + "title": "CodePlace Dashboards", + "type": "dashboards" + } + ], "editable": true, "time": { "from": "now-1h", "to": "now" }, "panels": [ + { + "type": "text", + "title": "Scrape contract", + "gridPos": {"x": 0, "y": 0, "w": 24, "h": 3}, + "options": { + "mode": "markdown", + "content": "필수 수집 경로를 환경별로 고정해 표시합니다. **ABSENT**는 target discovery 자체가 없음, **DOWN / DEGRADED**는 발견됐지만 scrape 실패 또는 고정 replica 수 미달, **UP**은 모든 기대 target이 발견되고 정상임을 뜻합니다. ABSENT를 정상값 0으로 위장하지 않습니다." + } + }, { "type": "stat", - "title": "Prometheus Ready Replicas", - "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "title": "Expected scrape targets", + "description": "각 쿼리는 해당 target에 한정해 0=ABSENT, 1=DOWN/DEGRADED, 2=UP 상태를 구분합니다. Backend, PostgreSQL, Redis, OTel, Blackbox는 배포 계약의 기대 replica 수도 함께 확인합니다.", + "gridPos": {"x": 0, "y": 3, "w": 24, "h": 13}, "datasource": "Prometheus", "fieldConfig": { "defaults": { - "unit": "short", + "unit": "none", + "min": 0, + "max": 2, + "mappings": [ + { + "type": "value", + "options": { + "0": {"text": "ABSENT", "color": "dark-red"}, + "1": {"text": "DOWN / DEGRADED", "color": "orange"}, + "2": {"text": "UP", "color": "green"} + } + } + ], "thresholds": { "mode": "absolute", "steps": [ - {"color": "red", "value": null}, + {"color": "dark-red", "value": null}, {"color": "orange", "value": 1}, {"color": "green", "value": 2} ] } - } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "auto", + "wideLayout": true }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"prometheus-kube-prometheus-stack-prometheus-.*\"}) or vector(0)", "legendFormat": "prometheus ready"} + { + "refId": "A", + "expr": "((count(up{job=\"backend\", namespace=\"code-place-dev\"}) > bool 0) or vector(0)) + (((min(up{job=\"backend\", namespace=\"code-place-dev\"}) == bool 1) * (count(up{job=\"backend\", namespace=\"code-place-dev\"}) >= bool 2)) or vector(0))", + "legendFormat": "backend dev", + "instant": true + }, + { + "refId": "B", + "expr": "((count(up{job=\"backend\", namespace=\"code-place-prod\"}) > bool 0) or vector(0)) + (((min(up{job=\"backend\", namespace=\"code-place-prod\"}) == bool 1) * (count(up{job=\"backend\", namespace=\"code-place-prod\"}) >= bool 6)) or vector(0))", + "legendFormat": "backend prod", + "instant": true + }, + { + "refId": "C", + "expr": "((count(up{job=\"postgres\", namespace=\"code-place-dev\"}) > bool 0) or vector(0)) + (((min(up{job=\"postgres\", namespace=\"code-place-dev\"}) == bool 1) * (count(up{job=\"postgres\", namespace=\"code-place-dev\"}) >= bool 3)) or vector(0))", + "legendFormat": "postgres dev", + "instant": true + }, + { + "refId": "D", + "expr": "((count(up{job=\"postgres\", namespace=\"code-place-prod\"}) > bool 0) or vector(0)) + (((min(up{job=\"postgres\", namespace=\"code-place-prod\"}) == bool 1) * (count(up{job=\"postgres\", namespace=\"code-place-prod\"}) >= bool 3)) or vector(0))", + "legendFormat": "postgres prod", + "instant": true + }, + { + "refId": "E", + "expr": "((count(up{job=\"redis\", namespace=\"code-place-dev\"}) > bool 0) or vector(0)) + (((min(up{job=\"redis\", namespace=\"code-place-dev\"}) == bool 1) * (count(up{job=\"redis\", namespace=\"code-place-dev\"}) >= bool 6)) or vector(0))", + "legendFormat": "redis dev", + "instant": true + }, + { + "refId": "F", + "expr": "((count(up{job=\"redis\", namespace=\"code-place-prod\"}) > bool 0) or vector(0)) + (((min(up{job=\"redis\", namespace=\"code-place-prod\"}) == bool 1) * (count(up{job=\"redis\", namespace=\"code-place-prod\"}) >= bool 6)) or vector(0))", + "legendFormat": "redis prod", + "instant": true + }, + { + "refId": "G", + "expr": "((2 * (min(up{job=\"traefik\", namespace=\"kube-system\"}) == bool 1)) + (1 * (min(up{job=\"traefik\", namespace=\"kube-system\"}) == bool 0))) or (absent(up{job=\"traefik\", namespace=\"kube-system\"}) * 0)", + "legendFormat": "Traefik", + "instant": true + }, + { + "refId": "H", + "expr": "((2 * (min(up{job=~\".*alloy.*\", namespace=\"monitoring\"}) == bool 1)) + (1 * (min(up{job=~\".*alloy.*\", namespace=\"monitoring\"}) == bool 0))) or (absent(up{job=~\".*alloy.*\", namespace=\"monitoring\"}) * 0)", + "legendFormat": "Alloy", + "instant": true + }, + { + "refId": "I", + "expr": "((2 * (min(up{job=~\".*loki.*\", namespace=\"monitoring\"}) == bool 1)) + (1 * (min(up{job=~\".*loki.*\", namespace=\"monitoring\"}) == bool 0))) or (absent(up{job=~\".*loki.*\", namespace=\"monitoring\"}) * 0)", + "legendFormat": "Loki", + "instant": true + }, + { + "refId": "J", + "expr": "((count(up{job=\"otel-collector\", namespace=\"monitoring\"}) > bool 0) or vector(0)) + (((min(up{job=\"otel-collector\", namespace=\"monitoring\"}) == bool 1) * (count(up{job=\"otel-collector\", namespace=\"monitoring\"}) >= bool 2)) or vector(0))", + "legendFormat": "OTel Collector", + "instant": true + }, + { + "refId": "K", + "expr": "((2 * (min(up{job=\"tempo\", namespace=\"monitoring\"}) == bool 1)) + (1 * (min(up{job=\"tempo\", namespace=\"monitoring\"}) == bool 0))) or (absent(up{job=\"tempo\", namespace=\"monitoring\"}) * 0)", + "legendFormat": "Tempo", + "instant": true + }, + { + "refId": "L", + "expr": "((2 * (min(up{job=\"vllm\", namespace=\"code-place-prod\"}) == bool 1)) + (1 * (min(up{job=\"vllm\", namespace=\"code-place-prod\"}) == bool 0))) or (absent(up{job=\"vllm\", namespace=\"code-place-prod\"}) * 0)", + "legendFormat": "vLLM prod", + "instant": true + }, + { + "refId": "M", + "expr": "((2 * (min(up{job=\"dcgm-exporter\", namespace=\"monitoring\"}) == bool 1)) + (1 * (min(up{job=\"dcgm-exporter\", namespace=\"monitoring\"}) == bool 0))) or (absent(up{job=\"dcgm-exporter\", namespace=\"monitoring\"}) * 0)", + "legendFormat": "DCGM", + "instant": true + }, + { + "refId": "N", + "expr": "((count(up{job=\"blackbox-exporter\", namespace=\"monitoring\"}) > bool 0) or vector(0)) + (((min(up{job=\"blackbox-exporter\", namespace=\"monitoring\"}) == bool 1) * (count(up{job=\"blackbox-exporter\", namespace=\"monitoring\"}) >= bool 2)) or vector(0))", + "legendFormat": "Blackbox exporter", + "instant": true + }, + { + "refId": "P", + "expr": "((2 * (min(up{namespace=\"longhorn-system\", endpoint=\"manager\"}) == bool 1)) + (1 * (min(up{namespace=\"longhorn-system\", endpoint=\"manager\"}) == bool 0))) or (absent(up{namespace=\"longhorn-system\", endpoint=\"manager\"}) * 0)", + "legendFormat": "Longhorn", + "instant": true + }, + { + "refId": "Q", + "expr": "((2 * (min(up{namespace=\"monitoring\", job=\"kubernetes-event-exporter\"}) == bool 1)) + (1 * (min(up{namespace=\"monitoring\", job=\"kubernetes-event-exporter\"}) == bool 0))) or (absent(up{namespace=\"monitoring\", job=\"kubernetes-event-exporter\"}) * 0)", + "legendFormat": "Event exporter", + "instant": true + }, + { + "refId": "R", + "expr": "((2 * (min(up{namespace=\"monitoring\", service=~\".*kube-state-metrics.*\"}) == bool 1)) + (1 * (min(up{namespace=\"monitoring\", service=~\".*kube-state-metrics.*\"}) == bool 0))) or (absent(up{namespace=\"monitoring\", service=~\".*kube-state-metrics.*\"}) * 0)", + "legendFormat": "kube-state-metrics", + "instant": true + } ] }, { "type": "stat", - "title": "Alertmanager Ready Replicas", - "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "title": "Application telemetry coverage", + "description": "Backend custom collector 4종과 Redis Sentinel check 2종의 환경별 수집 완전성입니다. 값이 없으면 ABSENT, 일부 누락 또는 실패는 100% 미만으로 표시합니다.", + "gridPos": {"x": 0, "y": 16, "w": 24, "h": 5}, "datasource": "Prometheus", "fieldConfig": { "defaults": { - "unit": "short", + "unit": "percentunit", + "min": 0, + "max": 1, + "noValue": "ABSENT", "thresholds": { "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "orange", "value": 1}, - {"color": "green", "value": 2} + {"color": "orange", "value": 0.75}, + {"color": "green", "value": 1} ] } - } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "value_and_name", + "wideLayout": true }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"alertmanager-kube-prometheus-stack-alertmanager-.*\"}) or vector(0)", "legendFormat": "alertmanager ready"} + { + "refId": "A", + "expr": "clamp_max(sum(codeplace_collector_success{namespace=\"code-place-dev\"}) / 8, 1)", + "legendFormat": "custom collectors dev (4 x 2 replicas)", + "instant": true + }, + { + "refId": "B", + "expr": "clamp_max(sum(codeplace_collector_success{namespace=\"code-place-prod\"}) / 24, 1)", + "legendFormat": "custom collectors prod (4 x 6 replicas)", + "instant": true + }, + { + "refId": "C", + "expr": "clamp_max(sum(codeplace_redis_sentinel_health{namespace=\"code-place-dev\"}) / 4, 1)", + "legendFormat": "Sentinel checks dev (2 x 2 replicas)", + "instant": true + }, + { + "refId": "D", + "expr": "clamp_max(sum(codeplace_redis_sentinel_health{namespace=\"code-place-prod\"}) / 12, 1)", + "legendFormat": "Sentinel checks prod (2 x 6 replicas)", + "instant": true + } ] }, { "type": "stat", - "title": "Grafana Ready", - "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "title": "Prometheus ready replicas / 2", + "gridPos": {"x": 0, "y": 21, "w": 6, "h": 5}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", + "min": 0, + "max": 2, + "noValue": "ABSENT", "thresholds": { "mode": "absolute", "steps": [ - {"color": "red", "value": null}, - {"color": "green", "value": 1} + {"color": "dark-red", "value": null}, + {"color": "orange", "value": 1}, + {"color": "green", "value": 2} ] } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "value_and_name" + }, + "targets": [ + { + "refId": "A", + "expr": "sum(max by (pod) (kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"prometheus-kube-prometheus-stack-prometheus-[0-9]+\"}))", + "legendFormat": "ready replicas", + "instant": true } + ] + }, + { + "type": "stat", + "title": "Discovered targets", + "gridPos": {"x": 6, "y": 21, "w": 6, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": {"unit": "short", "noValue": "NO TELEMETRY"}, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "value_and_name" }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"kube-prometheus-stack-grafana-.*\"}) or vector(0)", "legendFormat": "grafana"} + {"refId": "A", "expr": "count(up)", "legendFormat": "discovered", "instant": true} ] }, { "type": "stat", - "title": "Prometheus Operator Ready", - "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "title": "Down discovered targets", + "gridPos": {"x": 12, "y": 21, "w": 6, "h": 5}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", + "noValue": "NO TELEMETRY", "thresholds": { "mode": "absolute", "steps": [ - {"color": "red", "value": null}, - {"color": "green", "value": 1} + {"color": "green", "value": null}, + {"color": "red", "value": 1} ] } - } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "value_and_name" }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"kube-prometheus-stack-operator-.*\"}) or vector(0)", "legendFormat": "operator"} + {"refId": "A", "expr": "count(up) - count(up == 1)", "legendFormat": "down", "instant": true} ] }, { "type": "stat", - "title": "Alertmanager Discovered", - "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "title": "Rule evaluation failures / 5m", + "gridPos": {"x": 18, "y": 21, "w": 6, "h": 5}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", + "noValue": "NO TELEMETRY", "thresholds": { "mode": "absolute", "steps": [ - {"color": "red", "value": null}, - {"color": "green", "value": 1} + {"color": "green", "value": null}, + {"color": "red", "value": 1} ] } - } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "value_and_name" }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(prometheus_notifications_alertmanagers_discovered) or vector(0)", "legendFormat": "targets"} + {"refId": "A", "expr": "sum(increase(prometheus_rule_evaluation_failures_total[5m]))", "legendFormat": "failures", "instant": true} ] }, { "type": "stat", - "title": "Rule Fail", - "gridPos": {"x": 20, "y": 0, "w": 2, "h": 4}, + "title": "Alertmanager ready replicas / 2", + "gridPos": {"x": 0, "y": 26, "w": 6, "h": 5}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", + "min": 0, + "max": 2, + "noValue": "ABSENT", "thresholds": { "mode": "absolute", "steps": [ - {"color": "green", "value": null}, - {"color": "red", "value": 1} + {"color": "dark-red", "value": null}, + {"color": "orange", "value": 1}, + {"color": "green", "value": 2} ] } - } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "value_and_name" }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(increase(prometheus_rule_evaluation_failures_total[5m])) or vector(0)", "legendFormat": "failures"} + { + "refId": "A", + "expr": "sum(max by (pod) (kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"alertmanager-kube-prometheus-stack-alertmanager-[0-9]+\"}))", + "legendFormat": "ready replicas", + "instant": true + } ] }, { "type": "stat", - "title": "Grafana Metrics", - "gridPos": {"x": 0, "y": 36, "w": 4, "h": 4}, + "title": "Prometheus to Alertmanager discovery", + "gridPos": {"x": 6, "y": 26, "w": 6, "h": 5}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", + "noValue": "ABSENT", "thresholds": { "mode": "absolute", "steps": [ @@ -168,151 +416,160 @@ data: {"color": "green", "value": 1} ] } - } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "value_and_name" }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(up{namespace=\"monitoring\", service=\"kube-prometheus-stack-grafana\"}) or vector(0)", "legendFormat": "up"} + { + "refId": "A", + "expr": "min(prometheus_notifications_alertmanagers_discovered{namespace=\"monitoring\", job=\"kube-prometheus-stack-prometheus\"})", + "legendFormat": "minimum discovered", + "instant": true + } ] }, { "type": "stat", - "title": "Notify Fail", - "gridPos": {"x": 22, "y": 0, "w": 2, "h": 4}, + "title": "Alertmanager config reload", + "gridPos": {"x": 12, "y": 26, "w": 6, "h": 5}, "datasource": "Prometheus", "fieldConfig": { "defaults": { - "unit": "short", + "unit": "none", + "min": 0, + "max": 1, + "noValue": "ABSENT", + "mappings": [ + {"type": "value", "options": {"0": {"text": "FAILED", "color": "red"}, "1": {"text": "OK", "color": "green"}}} + ], "thresholds": { "mode": "absolute", "steps": [ - {"color": "green", "value": null}, - {"color": "red", "value": 1} + {"color": "red", "value": null}, + {"color": "green", "value": 1} ] } - } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "value_and_name" }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, - "targets": [ - {"expr": "sum(increase(alertmanager_notifications_failed_total[5m])) or vector(0)", "legendFormat": "failures"} - ] - }, - { - "type": "timeseries", - "title": "Monitoring Pod Restarts", - "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "sum by (pod) (increase(kube_pod_container_status_restarts_total{namespace=\"monitoring\", pod=~\"prometheus-.*|alertmanager-.*|kube-prometheus-stack-grafana-.*|kube-prometheus-stack-operator-.*\"}[1h]))", "legendFormat": "{{pod}}"} - ] - }, - { - "type": "timeseries", - "title": "Prometheus Target Scrape Failures", - "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, "targets": [ - {"expr": "sum by (job, namespace) (up == 0) or vector(0)", "legendFormat": "{{namespace}} {{job}}"} + { + "refId": "A", + "expr": "min(alertmanager_config_last_reload_successful{namespace=\"monitoring\", job=\"kube-prometheus-stack-alertmanager\"})", + "legendFormat": "reload", + "instant": true + } ] }, { - "type": "timeseries", - "title": "Monitoring Service Target Up", - "gridPos": {"x": 8, "y": 36, "w": 8, "h": 6}, + "type": "stat", + "title": "Alertmanager notification failures / 5m", + "gridPos": {"x": 18, "y": 26, "w": 6, "h": 5}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "short", + "noValue": "ABSENT", "thresholds": { "mode": "absolute", "steps": [ - {"color": "red", "value": null}, - {"color": "green", "value": 1} + {"color": "green", "value": null}, + {"color": "red", "value": 1} ] } - } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, + "textMode": "value_and_name" }, "targets": [ - {"expr": "max by (service) (up{namespace=\"monitoring\", service=~\"blackbox-exporter|kubernetes-event-exporter|otel-collector|tempo|kube-prometheus-stack-grafana|loki|loki-canary|alloy\"})", "legendFormat": "{{service}}"} - ] - }, - { - "type": "timeseries", - "title": "Prometheus TSDB Head Series", - "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "prometheus_tsdb_head_series", "legendFormat": "{{pod}}"} - ] - }, - { - "type": "timeseries", - "title": "Prometheus Rule and Config Health", - "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, - "targets": [ - {"expr": "sum by (rule_group) (increase(prometheus_rule_evaluation_failures_total[5m])) or vector(0)", "legendFormat": "{{rule_group}} failures"}, - {"expr": "max(prometheus_config_last_reload_successful{namespace=\"monitoring\"}) or vector(0)", "legendFormat": "config reload successful"} - ] - }, - { - "type": "timeseries", - "title": "Alertmanager Notifications and Config", - "gridPos": {"x": 12, "y": 12, "w": 12, "h": 8}, - "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "ops"}}, - "targets": [ - {"expr": "sum by (integration) (rate(alertmanager_notifications_total[5m]))", "legendFormat": "{{integration}} sent"}, - {"expr": "sum by (integration) (rate(alertmanager_notifications_failed_total[5m])) or vector(0)", "legendFormat": "{{integration}} failed"}, - {"expr": "max(alertmanager_config_last_reload_successful{namespace=\"monitoring\"}) or vector(0)", "legendFormat": "config reload successful"} + { + "refId": "A", + "expr": "sum(increase(alertmanager_notifications_failed_total{namespace=\"monitoring\", job=\"kube-prometheus-stack-alertmanager\"}[5m]))", + "legendFormat": "failed notifications", + "instant": true + } ] }, { - "type": "timeseries", - "title": "Prometheus Resource Usage", - "gridPos": {"x": 0, "y": 20, "w": 12, "h": 8}, + "type": "table", + "title": "Failing discovered targets", + "description": "Discovery된 target 중 현재 up=0인 항목입니다. 위 Expected scrape targets의 ABSENT 항목은 이 표에 나타나지 않습니다.", + "gridPos": {"x": 0, "y": 31, "w": 12, "h": 10}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, + "fieldConfig": { + "defaults": {"unit": "short"}, + "overrides": [] + }, + "options": {"showHeader": true, "cellHeight": "sm"}, "targets": [ - {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"monitoring\", pod=~\"prometheus-kube-prometheus-stack-prometheus-.*\", container=\"prometheus\"}[5m]))", "legendFormat": "{{pod}} cpu cores"}, - {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"monitoring\", pod=~\"prometheus-kube-prometheus-stack-prometheus-.*\", container=\"prometheus\"})", "legendFormat": "{{pod}} memory bytes"} + { + "refId": "A", + "expr": "up == 0", + "legendFormat": "{{namespace}} {{job}} {{instance}}", + "format": "table", + "instant": true + } ] }, { - "type": "timeseries", - "title": "Grafana Resource Usage", - "gridPos": {"x": 12, "y": 20, "w": 12, "h": 8}, + "type": "table", + "title": "Active monitoring contract alerts", + "gridPos": {"x": 12, "y": 31, "w": 12, "h": 10}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short"}}, + "fieldConfig": { + "defaults": {"unit": "short"}, + "overrides": [] + }, + "options": {"showHeader": true, "cellHeight": "sm"}, "targets": [ - {"expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"monitoring\", pod=~\"kube-prometheus-stack-grafana-.*\", container!=\"\"}[5m]))", "legendFormat": "{{pod}} cpu cores"}, - {"expr": "max by (pod) (container_memory_working_set_bytes{namespace=\"monitoring\", pod=~\"kube-prometheus-stack-grafana-.*\", container!=\"\"})", "legendFormat": "{{pod}} memory bytes"} + { + "refId": "A", + "expr": "ALERTS{alertstate=\"firing\", priority=~\"P0|P1\"} == 1", + "legendFormat": "{{alertname}} {{namespace}}", + "format": "table", + "instant": true + } ] }, { "type": "timeseries", - "title": "Monitoring PVC Usage", - "gridPos": {"x": 0, "y": 28, "w": 24, "h": 8}, + "title": "Scrape samples by expected job", + "gridPos": {"x": 0, "y": 41, "w": 24, "h": 8}, "datasource": "Prometheus", "fieldConfig": { - "defaults": { - "unit": "percentunit", - "thresholds": { - "mode": "absolute", - "steps": [ - {"color": "green", "value": null}, - {"color": "orange", "value": 0.75}, - {"color": "red", "value": 0.85} - ] - } - } + "defaults": {"unit": "short", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}, + "overrides": [] + }, + "options": { + "legend": {"displayMode": "table", "placement": "bottom", "showLegend": true}, + "tooltip": {"mode": "multi", "sort": "desc"} }, "targets": [ - {"expr": "max by (persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim!~\".*loki.*|tempo-data\"}) / clamp_min(max by (persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim!~\".*loki.*|tempo-data\"}), 1) or vector(0)", "legendFormat": "{{persistentvolumeclaim}}"} + { + "refId": "A", + "expr": "sum by (namespace, job) (scrape_samples_scraped{job=~\"backend|postgres|redis|traefik|.*alloy.*|.*loki.*|.*longhorn.*|kubernetes-event-exporter|kube-state-metrics|otel-collector|tempo|vllm|dcgm-exporter|blackbox-exporter\"})", + "legendFormat": "{{namespace}} {{job}}" + } ] } ] diff --git a/kubernetes/monitoring/grafana-dashboard-platform.yaml b/kubernetes/monitoring/grafana-dashboard-platform.yaml new file mode 100644 index 00000000..11449017 --- /dev/null +++ b/kubernetes/monitoring/grafana-dashboard-platform.yaml @@ -0,0 +1,291 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-codeplace-platform + labels: + grafana_dashboard: "1" +data: + codeplace-platform.json: | + { + "uid": "codeplace-platform", + "title": "CodePlace Platform", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "tags": ["codeplace", "platform"], + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": ["codeplace"], + "targetBlank": false, + "title": "CodePlace Dashboards", + "type": "dashboards" + } + ], + "editable": true, + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "environment", + "label": "Environment", + "type": "custom", + "query": "prod,dev", + "includeAll": false, + "multi": false, + "current": { + "text": "prod", + "value": "prod" + } + } + ] + }, + "panels": [ + { + "type": "row", + "title": "Workloads", + "gridPos": {"x": 0, "y": 0, "w": 24, "h": 1}, + "collapsed": false, + "panels": [] + }, + { + "type": "timeseries", + "title": "Pod Restarts in Previous 1h", + "gridPos": {"x": 0, "y": 1, "w": 8, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "lineInterpolation": "stepAfter", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "stacking": {"mode": "none", "group": "A"}}}}, + "targets": [ + {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=\"code-place-$environment\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} + ] + }, + { + "type": "timeseries", + "title": "Container CPU Usage", + "gridPos": {"x": 8, "y": 1, "w": 8, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "cores", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}}, + "targets": [ + {"expr": "topk(10, sum by (namespace, pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"code-place-$environment\", container!=\"\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"}[5m])))", "legendFormat": "{{pod}} {{container}}"} + ] + }, + { + "type": "timeseries", + "title": "Container Memory Usage", + "gridPos": {"x": 16, "y": 1, "w": 8, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "bytes", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}}, + "targets": [ + {"expr": "topk(10, container_memory_working_set_bytes{namespace=\"code-place-$environment\", container!=\"\", pod=~\"backend-.*|frontend-.*|hub-auth-.*|celery-.*|judge-server-.*|postgres-.*|redis-.*|vllm-.*\"})", "legendFormat": "{{pod}} {{container}}"} + ] + }, + { + "type": "row", + "title": "Storage / Availability", + "gridPos": {"x": 0, "y": 8, "w": 24, "h": 1}, + "collapsed": false, + "panels": [] + }, + { + "type": "timeseries", + "title": "PVC Usage", + "gridPos": {"x": 0, "y": 9, "w": 12, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, + "max": 1, + "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}, "thresholdsStyle": {"mode": "line"}}, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "orange", "value": 0.75}, + {"color": "red", "value": 0.85} + ] + } + } + }, + "targets": [ + {"expr": "max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=\"code-place-$environment\"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=\"code-place-$environment\"}), 1)", "legendFormat": "{{namespace}} {{persistentvolumeclaim}}"} + ] + }, + { + "type": "stat", + "title": "Cluster Node Pressure (Global)", + "description": "Cluster-wide node conditions; this panel is intentionally independent of the environment selector.", + "gridPos": {"x": 12, "y": 9, "w": 6, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "kube_node_status_condition{condition=~\"MemoryPressure|DiskPressure|PIDPressure\", status=\"true\"}", "legendFormat": "{{node}} {{condition}}", "instant": true} + ] + }, + { + "type": "stat", + "title": "PDB Healthy Pods", + "gridPos": {"x": 18, "y": 9, "w": 6, "h": 7}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short"}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "kube_poddisruptionbudget_status_current_healthy{namespace=\"code-place-$environment\", poddisruptionbudget=~\"backend|frontend\"}", "legendFormat": "{{namespace}} {{poddisruptionbudget}} current", "instant": true}, + {"expr": "kube_poddisruptionbudget_status_desired_healthy{namespace=\"code-place-$environment\", poddisruptionbudget=~\"backend|frontend\"}", "legendFormat": "{{namespace}} {{poddisruptionbudget}} desired", "instant": true} + ] + }, + { + "type": "row", + "title": "Data Stores", + "gridPos": {"x": 0, "y": 16, "w": 24, "h": 1}, + "collapsed": false, + "panels": [] + }, + { + "type": "stat", + "title": "PostgreSQL Exporter Up", + "gridPos": {"x": 0, "y": 17, "w": 12, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min by (namespace) (cnpg_collector_up{namespace=\"code-place-$environment\", cluster=\"postgres\"})", "legendFormat": "{{namespace}}", "instant": true} + ] + }, + { + "type": "stat", + "title": "Redis Exporter Up", + "gridPos": {"x": 12, "y": 17, "w": 12, "h": 5}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min by (namespace) (redis_up{namespace=\"code-place-$environment\"})", "legendFormat": "{{namespace}}", "instant": true} + ] + }, + { + "type": "timeseries", + "title": "PostgreSQL Restarts in Previous 1h", + "gridPos": {"x": 0, "y": 22, "w": 12, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "lineInterpolation": "stepAfter", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "stacking": {"mode": "none", "group": "A"}}}}, + "targets": [ + {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=\"code-place-$environment\", pod=~\"postgres-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} + ] + }, + { + "type": "timeseries", + "title": "Redis Restarts in Previous 1h", + "gridPos": {"x": 12, "y": 22, "w": 12, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "short", "custom": {"drawStyle": "line", "lineInterpolation": "stepAfter", "lineWidth": 2, "fillOpacity": 10, "showPoints": "never", "stacking": {"mode": "none", "group": "A"}}}}, + "targets": [ + {"expr": "sum by (namespace, pod) (increase(kube_pod_container_status_restarts_total{namespace=\"code-place-$environment\", pod=~\"redis-.*|redis-replication-.*\"}[1h]))", "legendFormat": "{{namespace}} {{pod}}"} + ] + }, + { + "type": "timeseries", + "title": "PostgreSQL WAL Bytes", + "gridPos": {"x": 0, "y": 28, "w": 12, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "Bps", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}}, + "targets": [ + {"expr": "sum by (namespace) (rate(cnpg_collector_wal_bytes{namespace=\"code-place-$environment\", cluster=\"postgres\"}[5m]))", "legendFormat": "{{namespace}}"} + ] + }, + { + "type": "timeseries", + "title": "Redis Memory Usage", + "gridPos": {"x": 12, "y": 28, "w": 12, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": {"defaults": {"unit": "bytes", "custom": {"drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 10, "showPoints": "auto", "stacking": {"mode": "none", "group": "A"}}}}, + "targets": [ + {"expr": "max by (namespace, pod) (redis_memory_used_bytes{namespace=\"code-place-$environment\"})", "legendFormat": "{{namespace}} {{pod}} used"}, + {"expr": "max by (namespace, pod) (redis_memory_max_bytes{namespace=\"code-place-$environment\"})", "legendFormat": "{{namespace}} {{pod}} maxmemory"} + ] + }, + { + "type": "stat", + "title": "PostgreSQL Collector Errors", + "gridPos": {"x": 0, "y": 34, "w": 8, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + } + }, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "max by (namespace, pod) (cnpg_collector_last_collection_error{namespace=\"code-place-$environment\", cluster=\"postgres\"})", "legendFormat": "{{namespace}} {{pod}}", "instant": true} + ] + }, + { + "type": "stat", + "title": "PostgreSQL Nodes Used", + "gridPos": {"x": 8, "y": 34, "w": 16, "h": 6}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 1}, + {"color": "green", "value": 2} + ] + } + } + }, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "min by (namespace) (cnpg_collector_nodes_used{namespace=\"code-place-$environment\", cluster=\"postgres\"})", "legendFormat": "{{namespace}} nodes", "instant": true} + ] + } + ] + } diff --git a/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml b/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml index 6ab3c8d3..6a4f0cc7 100644 --- a/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml +++ b/kubernetes/monitoring/grafana-dashboard-public-endpoints.yaml @@ -13,15 +13,42 @@ data: "version": 1, "refresh": "30s", "tags": ["codeplace", "synthetic", "blackbox"], + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": ["codeplace"], + "targetBlank": false, + "title": "CodePlace Dashboards", + "type": "dashboards" + } + ], "editable": true, "time": { "from": "now-1h", "to": "now" }, + "templating": { + "list": [ + { + "name": "environment", + "label": "Environment", + "type": "custom", + "query": "prod,dev", + "multi": false, + "includeAll": false, + "current": { + "text": "prod", + "value": "prod" + } + } + ] + }, "panels": [ { "type": "stat", - "title": "Prod Endpoints Up", + "title": "$environment Endpoint Health", "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -36,36 +63,36 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "codeplace:public_endpoint_availability5m{environment=\"prod\"}", "legendFormat": "{{service}} {{instance}}"} - ] + {"expr": "probe_success{environment=\"$environment\", probe_type=\"public-http\"}", "legendFormat": "{{service}} {{instance}}", "instant": true} ] }, { "type": "stat", - "title": "Dev Endpoints Up", + "title": "$environment Probe Target Coverage", + "description": "Healthy Prometheus scrape targets divided by the two expected workload probes (frontend and hub-auth).", "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { - "unit": "short", + "unit": "percentunit", "thresholds": { "mode": "absolute", "steps": [ {"color": "red", "value": null}, + {"color": "orange", "value": 0.5}, {"color": "green", "value": 1} ] } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "codeplace:public_endpoint_availability5m{environment=\"dev\"}", "legendFormat": "{{service}} {{instance}}"} - ] + {"expr": "sum(up{job=~\"codeplace-(public|hub-auth)-http-$environment\"}) / 2", "legendFormat": "healthy / expected", "instant": true} ] }, { "type": "stat", - "title": "HTTP Status", + "title": "$environment HTTP Status", "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -75,19 +102,19 @@ data: "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "green", "value": 200} + {"color": "green", "value": 200}, + {"color": "red", "value": 201} ] } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "probe_http_status_code{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{instance}}"} - ] + {"expr": "probe_http_status_code{environment=\"$environment\", probe_type=\"public-http\"}", "legendFormat": "{{service}} {{instance}}", "instant": true} ] }, { "type": "stat", - "title": "TLS Days Left", + "title": "$environment TLS Days Left", "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -97,20 +124,19 @@ data: "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "orange", "value": 3}, - {"color": "green", "value": 14} + {"color": "orange", "value": 14}, + {"color": "green", "value": 30} ] } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "(probe_ssl_earliest_cert_expiry{probe_type=\"public-http\"} - time()) / 86400", "legendFormat": "{{environment}} {{instance}}"} - ] + {"expr": "(probe_ssl_earliest_cert_expiry{environment=\"$environment\", probe_type=\"public-http\"} - time()) / 86400", "legendFormat": "{{service}} {{instance}}", "instant": true} ] }, { "type": "stat", - "title": "Probe Duration", + "title": "$environment Probe Duration", "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -126,14 +152,14 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "probe_duration_seconds{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{instance}}"} - ] + {"expr": "probe_duration_seconds{environment=\"$environment\", probe_type=\"public-http\"}", "legendFormat": "{{service}} {{instance}}", "instant": true} ] }, { "type": "stat", - "title": "Blackbox Exporter Replicas", + "title": "Grafana Self Probe", + "description": "Monitoring-plane probe kept separate from the selected application environment.", "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -143,46 +169,77 @@ data: "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "orange", "value": 1}, - {"color": "green", "value": 2} + {"color": "green", "value": 1} ] } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "kube_deployment_status_replicas_available{namespace=\"monitoring\", deployment=\"blackbox-exporter\"} or vector(0)", "legendFormat": "available"}, - {"expr": "kube_deployment_spec_replicas{namespace=\"monitoring\", deployment=\"blackbox-exporter\"} or vector(0)", "legendFormat": "desired"} - ] + {"expr": "probe_success{environment=\"monitoring\", service=\"grafana\", probe_type=\"public-http\"}", "legendFormat": "grafana", "instant": true} ] }, { "type": "timeseries", - "title": "Endpoint Availability", + "title": "$environment Endpoint Availability", "gridPos": {"x": 0, "y": 4, "w": 8, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short", "decimals": 0}}, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "decimals": 2, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ - {"expr": "codeplace:public_endpoint_availability5m", "legendFormat": "{{environment}} {{service}} {{instance}}"} + {"expr": "codeplace:public_endpoint_availability5m{environment=\"$environment\"}", "legendFormat": "{{service}} {{instance}}"} ] }, { "type": "timeseries", - "title": "Endpoint Duration", + "title": "$environment Endpoint Duration", "gridPos": {"x": 8, "y": 4, "w": 8, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "s"}}, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ - {"expr": "probe_duration_seconds{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{instance}}"} + {"expr": "probe_duration_seconds{environment=\"$environment\", probe_type=\"public-http\"}", "legendFormat": "{{service}} {{instance}}"} ] }, { "type": "timeseries", - "title": "TLS Days Left", + "title": "$environment TLS Days Left", "gridPos": {"x": 16, "y": 4, "w": 8, "h": 8}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "d", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "thresholdsStyle": {"mode": "line"} + }, "thresholds": { "mode": "absolute", "steps": [ @@ -194,17 +251,29 @@ data: } }, "targets": [ - {"expr": "(probe_ssl_earliest_cert_expiry{probe_type=\"public-http\"} - time()) / 86400", "legendFormat": "{{environment}} {{instance}}"} + {"expr": "(probe_ssl_earliest_cert_expiry{environment=\"$environment\", probe_type=\"public-http\"} - time()) / 86400", "legendFormat": "{{service}} {{instance}}"} ] }, { "type": "timeseries", - "title": "HTTP Phase Duration", + "title": "$environment HTTP Phase Duration", "gridPos": {"x": 0, "y": 12, "w": 24, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "s"}}, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ - {"expr": "probe_http_duration_seconds{probe_type=\"public-http\"}", "legendFormat": "{{environment}} {{phase}}"} + {"expr": "probe_http_duration_seconds{environment=\"$environment\", probe_type=\"public-http\"}", "legendFormat": "{{service}} {{phase}}"} ] } ] diff --git a/kubernetes/monitoring/grafana-dashboard-storage.yaml b/kubernetes/monitoring/grafana-dashboard-storage.yaml index 43269d39..7302a63f 100644 --- a/kubernetes/monitoring/grafana-dashboard-storage.yaml +++ b/kubernetes/monitoring/grafana-dashboard-storage.yaml @@ -13,16 +13,44 @@ data: "version": 1, "refresh": "30s", "tags": ["codeplace", "storage", "longhorn"], + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": ["codeplace"], + "targetBlank": false, + "title": "CodePlace Dashboards", + "type": "dashboards" + } + ], "editable": true, "time": { "from": "now-1h", "to": "now" }, + "templating": { + "list": [ + { + "name": "environment", + "label": "Environment", + "type": "custom", + "query": "prod,dev", + "multi": false, + "includeAll": false, + "current": { + "text": "prod", + "value": "prod" + } + } + ] + }, "panels": [ { "type": "stat", - "title": "Longhorn Manager Replicas", - "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, + "title": "Cluster Longhorn Scrape Up", + "description": "Cluster-global Longhorn telemetry health. Environment selection only scopes PVC panels below.", + "gridPos": {"x": 0, "y": 0, "w": 6, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -31,22 +59,42 @@ data: "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "orange", "value": 1}, - {"color": "green", "value": 3} + {"color": "green", "value": 1} ] } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "kube_daemonset_status_number_available{namespace=\"longhorn-system\", daemonset=\"longhorn-manager\"} or vector(0)", "legendFormat": "available"}, - {"expr": "kube_daemonset_status_desired_number_scheduled{namespace=\"longhorn-system\", daemonset=\"longhorn-manager\"} or vector(0)", "legendFormat": "desired"} - ] + {"expr": "min(up{namespace=\"longhorn-system\", endpoint=\"manager\"})", "legendFormat": "longhorn-manager", "instant": true} ] + }, + { + "type": "stat", + "title": "Cluster Manager Availability", + "description": "Available Longhorn managers divided by the DaemonSet desired count.", + "gridPos": {"x": 6, "y": 0, "w": 6, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 0.5}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "kube_daemonset_status_number_available{namespace=\"longhorn-system\", daemonset=\"longhorn-manager\"} / clamp_min(kube_daemonset_status_desired_number_scheduled{namespace=\"longhorn-system\", daemonset=\"longhorn-manager\"}, 1)", "legendFormat": "available / desired", "instant": true} ] }, { "type": "stat", - "title": "Faulted Volumes", - "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, + "title": "Cluster Faulted Volumes", + "gridPos": {"x": 0, "y": 4, "w": 8, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -60,15 +108,14 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "count(longhorn_volume_robustness == 3) or vector(0)", "legendFormat": "faulted"} - ] + {"expr": "sum(max by (volume, pvc_namespace, pvc) (longhorn_volume_robustness) == bool 3) or vector(0)", "legendFormat": "faulted", "instant": true} ] }, { "type": "stat", - "title": "Degraded Volumes", - "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, + "title": "Cluster Degraded Volumes", + "gridPos": {"x": 8, "y": 4, "w": 8, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -82,15 +129,14 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "count(longhorn_volume_robustness == 2) or vector(0)", "legendFormat": "degraded"} - ] + {"expr": "sum(max by (volume, pvc_namespace, pvc) (longhorn_volume_robustness) == bool 2) or vector(0)", "legendFormat": "degraded", "instant": true} ] }, { "type": "stat", - "title": "Read-only Volumes", - "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, + "title": "Cluster Read-only Volumes", + "gridPos": {"x": 16, "y": 4, "w": 8, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -104,15 +150,14 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(longhorn_volume_file_system_read_only) or vector(0)", "legendFormat": "read-only"} - ] + {"expr": "sum(max by (volume, pvc_namespace, pvc) (longhorn_volume_file_system_read_only)) or vector(0)", "legendFormat": "read-only", "instant": true} ] }, { "type": "stat", - "title": "Max Node Storage", - "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, + "title": "Cluster Max Node Storage", + "gridPos": {"x": 12, "y": 0, "w": 6, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -127,15 +172,14 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max((longhorn_node_storage_usage_bytes + longhorn_node_storage_reservation_bytes) / clamp_min(longhorn_node_storage_capacity_bytes, 1))", "legendFormat": "usage"} - ] + {"expr": "max((longhorn_node_storage_usage_bytes + longhorn_node_storage_reservation_bytes) / clamp_min(longhorn_node_storage_capacity_bytes, 1))", "legendFormat": "usage", "instant": true} ] }, { "type": "stat", - "title": "Max Disk Usage", - "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "title": "Cluster Max Disk Usage", + "gridPos": {"x": 18, "y": 0, "w": 6, "h": 4}, "datasource": "Prometheus", "fieldConfig": { "defaults": { @@ -150,41 +194,79 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max((longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1))", "legendFormat": "usage"} - ] + {"expr": "max((longhorn_disk_usage_bytes + longhorn_disk_reservation_bytes) / clamp_min(longhorn_disk_capacity_bytes, 1))", "legendFormat": "usage", "instant": true} ] }, { - "type": "timeseries", - "title": "Volume Robustness", - "description": "1 healthy, 2 degraded, 3 faulted", - "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8}, + "type": "state-timeline", + "title": "$environment PVC Robustness", + "description": "Longhorn robustness for PVCs in code-place-$environment.", + "gridPos": {"x": 0, "y": 8, "w": 12, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "short", "decimals": 0}}, + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "mappings": [ + { + "type": "value", + "options": { + "0": {"text": "Unknown", "color": "gray", "index": 0}, + "1": {"text": "Healthy", "color": "green", "index": 1}, + "2": {"text": "Degraded", "color": "orange", "index": 2}, + "3": {"text": "Faulted", "color": "red", "index": 3} + } + } + ] + } + }, "targets": [ - {"expr": "longhorn_volume_robustness", "legendFormat": "{{pvc_namespace}}/{{pvc}}"} + {"expr": "max by (pvc_namespace, pvc) (longhorn_volume_robustness{pvc_namespace=\"code-place-$environment\"})", "legendFormat": "{{pvc}}"} ] }, { "type": "timeseries", - "title": "Volume Actual Size", - "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8}, + "title": "$environment PVC Actual / Capacity", + "gridPos": {"x": 12, "y": 8, "w": 12, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "bytes"}}, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ - {"expr": "longhorn_volume_actual_size_bytes", "legendFormat": "{{pvc_namespace}}/{{pvc}} actual"}, - {"expr": "longhorn_volume_capacity_bytes", "legendFormat": "{{pvc_namespace}}/{{pvc}} capacity"} + {"expr": "max by (pvc_namespace, pvc) (longhorn_volume_actual_size_bytes{pvc_namespace=\"code-place-$environment\"})", "legendFormat": "{{pvc}} actual"}, + {"expr": "max by (pvc_namespace, pvc) (longhorn_volume_capacity_bytes{pvc_namespace=\"code-place-$environment\"})", "legendFormat": "{{pvc}} capacity"} ] }, { "type": "timeseries", - "title": "Node Storage Usage", - "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8}, + "title": "Cluster Node Storage Usage", + "gridPos": {"x": 0, "y": 16, "w": 12, "h": 8}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "thresholdsStyle": {"mode": "line"} + }, "thresholds": { "mode": "absolute", "steps": [ @@ -201,12 +283,23 @@ data: }, { "type": "timeseries", - "title": "Disk Usage", - "gridPos": {"x": 12, "y": 12, "w": 12, "h": 8}, + "title": "Cluster Disk Usage", + "gridPos": {"x": 12, "y": 16, "w": 12, "h": 8}, "datasource": "Prometheus", "fieldConfig": { "defaults": { "unit": "percentunit", + "min": 0, + "max": 1, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false, + "thresholdsStyle": {"mode": "line"} + }, "thresholds": { "mode": "absolute", "steps": [ @@ -223,20 +316,44 @@ data: }, { "type": "timeseries", - "title": "Longhorn Manager CPU", - "gridPos": {"x": 0, "y": 20, "w": 12, "h": 8}, + "title": "Cluster Longhorn Manager CPU", + "gridPos": {"x": 0, "y": 24, "w": 12, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "m"}}, + "fieldConfig": { + "defaults": { + "unit": "cores", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ - {"expr": "longhorn_manager_cpu_usage_millicpu", "legendFormat": "{{node}}"} + {"expr": "longhorn_manager_cpu_usage_millicpu / 1000", "legendFormat": "{{node}}"} ] }, { "type": "timeseries", - "title": "Longhorn Manager Memory", - "gridPos": {"x": 12, "y": 20, "w": 12, "h": 8}, + "title": "Cluster Longhorn Manager Memory", + "gridPos": {"x": 12, "y": 24, "w": 12, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "bytes"}}, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ {"expr": "longhorn_manager_memory_usage_bytes", "legendFormat": "{{node}}"} ] diff --git a/kubernetes/monitoring/grafana-dashboard-traces.yaml b/kubernetes/monitoring/grafana-dashboard-traces.yaml index 3831d174..696c3293 100644 --- a/kubernetes/monitoring/grafana-dashboard-traces.yaml +++ b/kubernetes/monitoring/grafana-dashboard-traces.yaml @@ -13,15 +13,42 @@ data: "version": 1, "refresh": "30s", "tags": ["codeplace", "traces"], + "links": [ + { + "asDropdown": true, + "includeVars": true, + "keepTime": true, + "tags": ["codeplace"], + "targetBlank": false, + "title": "CodePlace Dashboards", + "type": "dashboards" + } + ], "editable": true, "time": { "from": "now-1h", "to": "now" }, + "templating": { + "list": [ + { + "name": "environment", + "label": "Environment", + "type": "custom", + "query": "prod,dev", + "multi": false, + "includeAll": false, + "current": { + "text": "prod", + "value": "prod" + } + } + ] + }, "panels": [ { "type": "stat", - "title": "Tempo Ready", + "title": "Tempo Scrape Up (Global)", "gridPos": {"x": 0, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -36,14 +63,13 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(kube_pod_status_ready{namespace=\"monitoring\", condition=\"true\", pod=~\"tempo-.*\"}) or vector(0)", "legendFormat": "tempo"} - ] + {"expr": "min(up{job=\"tempo\", namespace=\"monitoring\"})", "legendFormat": "tempo", "instant": true} ] }, { "type": "stat", - "title": "OTel Collector Replicas", + "title": "OTel Collector Scrape Up (Global)", "gridPos": {"x": 4, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -53,32 +79,29 @@ data: "mode": "absolute", "steps": [ {"color": "red", "value": null}, - {"color": "orange", "value": 1}, - {"color": "green", "value": 2} + {"color": "green", "value": 1} ] } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "kube_deployment_status_replicas_available{namespace=\"monitoring\", deployment=\"otel-collector\"} or vector(0)", "legendFormat": "available"}, - {"expr": "kube_deployment_spec_replicas{namespace=\"monitoring\", deployment=\"otel-collector\"} or vector(0)", "legendFormat": "desired"} - ] + {"expr": "min(up{job=\"otel-collector\", namespace=\"monitoring\"})", "legendFormat": "otel-collector", "instant": true} ] }, { "type": "stat", - "title": "Trace Receive Rate", + "title": "Trace Receive Rate (All Environments)", + "description": "Collector internal metrics are cluster-wide; the environment selector scopes the Tempo trace searches below.", "gridPos": {"x": 8, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": {"defaults": {"unit": "ops"}}, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(rate(otelcol_receiver_accepted_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "spans/s"} - ] + {"expr": "sum(rate(otelcol_receiver_accepted_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "spans/s", "instant": true} ] }, { "type": "stat", - "title": "Trace Export Failures", + "title": "Trace Export Failures (All Environments)", "gridPos": {"x": 12, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -94,14 +117,13 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "sum(increase(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "failed"} - ] + {"expr": "sum(increase(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "failed", "instant": true} ] }, { "type": "stat", - "title": "Tempo PVC Usage", + "title": "Tempo PVC Usage (Global)", "gridPos": {"x": 16, "y": 0, "w": 4, "h": 4}, "datasource": "Prometheus", "fieldConfig": { @@ -117,17 +139,50 @@ data: } } }, - "options": {"reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, "targets": [ - {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim=\"tempo-data\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim=\"tempo-data\"}), 1) or vector(0)", "legendFormat": "usage"} - ] + {"expr": "max(kubelet_volume_stats_used_bytes{namespace=\"monitoring\", persistentvolumeclaim=\"tempo-data\"}) / clamp_min(max(kubelet_volume_stats_capacity_bytes{namespace=\"monitoring\", persistentvolumeclaim=\"tempo-data\"}), 1)", "legendFormat": "usage", "instant": true} ] + }, + { + "type": "stat", + "title": "OTel Replica Availability (Global)", + "gridPos": {"x": 20, "y": 0, "w": 4, "h": 4}, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "orange", "value": 0.5}, + {"color": "green", "value": 1} + ] + } + } + }, + "options": {"graphMode": "none", "reduceOptions": {"calcs": ["lastNotNull"], "values": false}}, + "targets": [ + {"expr": "kube_deployment_status_replicas_available{namespace=\"monitoring\", deployment=\"otel-collector\"} / clamp_min(kube_deployment_spec_replicas{namespace=\"monitoring\", deployment=\"otel-collector\"}, 1)", "legendFormat": "available / desired", "instant": true} ] }, { "type": "timeseries", - "title": "Collector Spans", + "title": "Collector Spans (All Environments)", "gridPos": {"x": 0, "y": 4, "w": 12, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "ops"}}, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "showPoints": "never", + "spanNulls": false + } + } + }, "targets": [ {"expr": "sum by (receiver, transport) (rate(otelcol_receiver_accepted_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "{{receiver}} {{transport}} accepted"}, {"expr": "sum by (exporter) (rate(otelcol_exporter_sent_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "{{exporter}} sent"} @@ -135,39 +190,54 @@ data: }, { "type": "timeseries", - "title": "Collector Refused or Failed Spans", + "title": "Collector Refused or Failed Spans (All Environments)", "gridPos": {"x": 12, "y": 4, "w": 12, "h": 8}, "datasource": "Prometheus", - "fieldConfig": {"defaults": {"unit": "ops"}}, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "bars", + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "fillOpacity": 80, + "showPoints": "never", + "barAlignment": 0, + "stacking": {"mode": "normal", "group": "A"} + } + } + }, "targets": [ - {"expr": "sum(rate(otelcol_receiver_refused_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "refused"}, - {"expr": "sum(rate(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m])) or vector(0)", "legendFormat": "failed"} + {"expr": "sum(rate(otelcol_receiver_refused_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "refused"}, + {"expr": "sum(rate(otelcol_exporter_send_failed_spans_total{namespace=\"monitoring\"}[5m]))", "legendFormat": "failed"} ] }, { "type": "table", - "title": "Recent Backend/Celery Traces", + "title": "$environment Backend/Celery Trace Search", "gridPos": {"x": 0, "y": 12, "w": 12, "h": 8}, "datasource": "Tempo", + "options": {"showHeader": true, "cellHeight": "sm"}, "targets": [ { "refId": "A", "queryType": "traceql", - "query": "{ resource.service.name =~ \"codeplace-(backend|celery)\" }", + "query": "{ resource.deployment.environment = \"$environment\" && resource.service.name =~ \"codeplace-(backend|celery(-worker|-beat)?)\" }", "limit": 20 } ] }, { "type": "table", - "title": "Submission/Judge Trace Search", + "title": "$environment Submission/Judge Trace Search", "gridPos": {"x": 12, "y": 12, "w": 12, "h": 8}, "datasource": "Tempo", + "options": {"showHeader": true, "cellHeight": "sm"}, "targets": [ { "refId": "A", "queryType": "traceql", - "query": "{ name =~ \"judge_task|submission.judge|judge_server.select|judge_server.request|submission.result_save\" }", + "query": "{ resource.deployment.environment = \"$environment\" && name =~ \"judge_task|submission.judge|judge_server.select|judge_server.request|submission.result_save\" }", "limit": 20 } ] diff --git a/kubernetes/monitoring/kube-prometheus-stack-values.yaml b/kubernetes/monitoring/kube-prometheus-stack-values.yaml index c13ec923..323b23e7 100644 --- a/kubernetes/monitoring/kube-prometheus-stack-values.yaml +++ b/kubernetes/monitoring/kube-prometheus-stack-values.yaml @@ -1,3 +1,33 @@ +crds: + enabled: true + # Helm does not upgrade CRDs from the chart's crds/ directory. Run the + # chart-owned pre-upgrade hook so CRDs and the pinned operator stay aligned. + upgradeJob: + enabled: true + forceConflicts: false + +defaultRules: + rules: + # K3s embeds these control-plane components in k3s-server and does not + # expose the standalone endpoints expected by kube-prometheus-stack. + etcd: false + kubeControllerManager: false + kubeProxy: false + kubeSchedulerAlerting: false + kubeSchedulerRecording: false + +kubeControllerManager: + enabled: false + +kubeEtcd: + enabled: false + +kubeProxy: + enabled: false + +kubeScheduler: + enabled: false + prometheus: prometheusSpec: replicas: 2 @@ -12,6 +42,30 @@ prometheus: podMonitorSelectorNilUsesHelmValues: false probeSelectorNilUsesHelmValues: false ruleSelectorNilUsesHelmValues: false + serviceMonitorSelector: + matchLabels: + release: kube-prometheus-stack + serviceMonitorNamespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + podMonitorSelector: + matchLabels: + release: kube-prometheus-stack + podMonitorNamespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + probeSelector: + matchLabels: + release: kube-prometheus-stack + probeNamespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring + ruleSelector: + matchLabels: + release: kube-prometheus-stack + ruleNamespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring storageSpec: volumeClaimTemplate: spec: @@ -46,6 +100,15 @@ alertmanager: storage: 5Gi grafana: + admin: + existingSecret: grafana-admin-credentials + userKey: admin-user + passwordKey: admin-password + defaultDashboardsEnabled: false + # One replica owns a Longhorn ReadWriteOnce PVC. Recreate avoids a rolling + # replacement waiting forever for the old pod to release that volume. + deploymentStrategy: + type: Recreate grafana.ini: metrics: enabled: true diff --git a/kubernetes/monitoring/kustomization.yaml b/kubernetes/monitoring/kustomization.yaml index b140e15a..890ccd7a 100644 --- a/kubernetes/monitoring/kustomization.yaml +++ b/kubernetes/monitoring/kustomization.yaml @@ -9,8 +9,11 @@ resources: - ./dcgm-exporter.yaml - ./datastore-pod-monitors.yaml - ./grafana-dashboard-codeplace.yaml + - ./grafana-dashboard-platform.yaml + - ./grafana-dashboard-ai-api.yaml - ./grafana-dashboard-ai-inference.yaml - ./grafana-dashboard-logs.yaml + - ./grafana-dashboard-log-pipeline.yaml - ./grafana-dashboard-kubernetes-events.yaml - ./grafana-dashboard-monitoring-stack.yaml - ./grafana-dashboard-public-endpoints.yaml @@ -21,6 +24,7 @@ resources: - ./otel-collector.yaml - ./prometheus-rules.yaml - ./public-endpoint-probes.yaml + - ./secrets/grafana-admin-credentials.sealedsecret.yaml - ./tempo.yaml - - ./traefik-service-monitor.yaml + - ./traefik-pod-monitor.yaml - ./vllm-service-monitor.yaml diff --git a/kubernetes/monitoring/logs/README.md b/kubernetes/monitoring/logs/README.md index 53d1cce3..3292490e 100644 --- a/kubernetes/monitoring/logs/README.md +++ b/kubernetes/monitoring/logs/README.md @@ -44,6 +44,12 @@ helm upgrade --install alloy grafana/alloy \ --create-namespace \ --version 1.10.0 \ --values kubernetes/monitoring/logs/alloy-values.yaml + +kubectl apply -k kubernetes/monitoring + +# One-time migration after the replacement Traefik PodMonitor is present. +kubectl -n monitoring get podmonitor traefik +kubectl -n monitoring delete servicemonitor traefik --ignore-not-found ``` ## Verify @@ -56,6 +62,16 @@ kubectl -n monitoring get pvc | grep loki kubectl -n monitoring get servicemonitor loki alloy ``` +For the complete scrape, probe, metric-family, and HA contract, port-forward Prometheus and run the live verifier in another terminal: + +```sh +# terminal 1 +kubectl -n monitoring port-forward service/kube-prometheus-stack-prometheus 19090:9090 + +# terminal 2 +python3 kubernetes/monitoring/verify_live.py --prometheus-url http://127.0.0.1:19090 +``` + Grafana should show a `Loki` datasource. Useful Explore queries: ```logql @@ -69,7 +85,9 @@ Grafana should show a `Loki` datasource. Useful Explore queries: Grafana should also show: - `CodePlace Overview` +- `CodePlace Platform` - `CodePlace Logs` +- `CodePlace Log Pipeline` ## Notes diff --git a/kubernetes/monitoring/prometheus-rules.test.yaml b/kubernetes/monitoring/prometheus-rules.test.yaml new file mode 100644 index 00000000..490ce7a4 --- /dev/null +++ b/kubernetes/monitoring/prometheus-rules.test.yaml @@ -0,0 +1,397 @@ +rule_files: + - /rules.yaml + +evaluation_interval: 15s + +tests: + - name: healthy public endpoints stay quiet + interval: 15s + input_series: + - series: 'probe_success{environment="prod",namespace="code-place-prod",probe_type="public-http",service="frontend",instance="https://code.pusan.ac.kr/"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1' + - series: 'probe_success{environment="prod",namespace="code-place-prod",probe_type="public-http",service="hub-auth",instance="https://hub-auth.code-place-dev.site/healthz"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1' + - series: 'probe_success{environment="dev",namespace="code-place-dev",probe_type="public-http",service="frontend",instance="https://k3s.code-place-dev.site/"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1' + - series: 'probe_success{environment="dev",namespace="code-place-dev",probe_type="public-http",service="hub-auth",instance="https://hub-auth-dev.code-place-dev.site/healthz"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1' + alert_rule_test: + - eval_time: 3m + alertname: PublicEndpointDown + exp_alerts: [] + - eval_time: 3m + alertname: PublicEndpointProbeMissing + exp_alerts: [] + + - name: prod hub-auth failure identifies the failing service and instance + interval: 15s + input_series: + - series: 'probe_success{environment="prod",namespace="code-place-prod",probe_type="public-http",service="frontend",instance="https://code.pusan.ac.kr/"}' + values: '1 1 1 1 1 1' + - series: 'probe_success{environment="prod",namespace="code-place-prod",probe_type="public-http",service="hub-auth",instance="https://hub-auth.code-place-dev.site/healthz"}' + values: '0 0 0 0 0 0' + - series: 'probe_success{environment="dev",namespace="code-place-dev",probe_type="public-http",service="frontend",instance="https://k3s.code-place-dev.site/"}' + values: '1 1 1 1 1 1' + - series: 'probe_success{environment="dev",namespace="code-place-dev",probe_type="public-http",service="hub-auth",instance="https://hub-auth-dev.code-place-dev.site/healthz"}' + values: '1 1 1 1 1 1' + alert_rule_test: + - eval_time: 1m15s + alertname: PublicEndpointDown + exp_alerts: + - exp_labels: + environment: prod + instance: https://hub-auth.code-place-dev.site/healthz + namespace: code-place-prod + priority: P0 + probe_type: public-http + service: hub-auth + severity: critical + exp_annotations: + summary: prod hub-auth 공개 엔드포인트가 응답하지 않습니다 + description: https://hub-auth.code-place-dev.site/healthz HTTPS synthetic probe가 1분 동안 실패했습니다. + + - name: transient prod failure recovers before the one minute threshold + interval: 15s + input_series: + - series: 'probe_success{environment="prod",namespace="code-place-prod",probe_type="public-http",service="frontend",instance="https://code.pusan.ac.kr/"}' + values: '1 1 1 1 1 1 1' + - series: 'probe_success{environment="prod",namespace="code-place-prod",probe_type="public-http",service="hub-auth",instance="https://hub-auth.code-place-dev.site/healthz"}' + values: '0 0 0 1 1 1 1' + - series: 'probe_success{environment="dev",namespace="code-place-dev",probe_type="public-http",service="frontend",instance="https://k3s.code-place-dev.site/"}' + values: '1 1 1 1 1 1 1' + - series: 'probe_success{environment="dev",namespace="code-place-dev",probe_type="public-http",service="hub-auth",instance="https://hub-auth-dev.code-place-dev.site/healthz"}' + values: '1 1 1 1 1 1 1' + alert_rule_test: + - eval_time: 1m30s + alertname: PublicEndpointDown + exp_alerts: [] + + - name: dev failure requires two minutes and preserves target context + interval: 15s + input_series: + - series: 'probe_success{environment="prod",namespace="code-place-prod",probe_type="public-http",service="frontend",instance="https://code.pusan.ac.kr/"}' + values: '1 1 1 1 1 1 1 1 1 1' + - series: 'probe_success{environment="prod",namespace="code-place-prod",probe_type="public-http",service="hub-auth",instance="https://hub-auth.code-place-dev.site/healthz"}' + values: '1 1 1 1 1 1 1 1 1 1' + - series: 'probe_success{environment="dev",namespace="code-place-dev",probe_type="public-http",service="frontend",instance="https://k3s.code-place-dev.site/"}' + values: '1 1 1 1 1 1 1 1 1 1' + - series: 'probe_success{environment="dev",namespace="code-place-dev",probe_type="public-http",service="hub-auth",instance="https://hub-auth-dev.code-place-dev.site/healthz"}' + values: '0 0 0 0 0 0 0 0 0 0' + alert_rule_test: + - eval_time: 1m45s + alertname: PublicEndpointDown + exp_alerts: [] + - eval_time: 2m15s + alertname: PublicEndpointDown + exp_alerts: + - exp_labels: + environment: dev + instance: https://hub-auth-dev.code-place-dev.site/healthz + namespace: code-place-dev + priority: P1 + probe_type: public-http + service: hub-auth + severity: warning + exp_annotations: + summary: dev hub-auth 공개 엔드포인트가 응답하지 않습니다 + description: https://hub-auth-dev.code-place-dev.site/healthz HTTPS synthetic probe가 2분 동안 실패했습니다. + + - name: missing prod hub-auth telemetry is not endpoint downtime + interval: 15s + input_series: + - series: 'probe_success{environment="prod",namespace="code-place-prod",probe_type="public-http",service="frontend",instance="https://code.pusan.ac.kr/"}' + values: '1 1 1 1 1 1 1 1 1 1' + - series: 'probe_success{environment="dev",namespace="code-place-dev",probe_type="public-http",service="frontend",instance="https://k3s.code-place-dev.site/"}' + values: '1 1 1 1 1 1 1 1 1 1' + - series: 'probe_success{environment="dev",namespace="code-place-dev",probe_type="public-http",service="hub-auth",instance="https://hub-auth-dev.code-place-dev.site/healthz"}' + values: '1 1 1 1 1 1 1 1 1 1' + alert_rule_test: + - eval_time: 2m15s + alertname: PublicEndpointDown + exp_alerts: [] + - eval_time: 2m15s + alertname: PublicEndpointProbeMissing + exp_alerts: + - exp_labels: + environment: prod + namespace: code-place-prod + priority: P1 + probe_type: public-http + service: hub-auth + severity: warning + exp_annotations: + summary: prod hub-auth 공개 probe metric이 없습니다 + description: Prometheus가 prod hub-auth synthetic probe 결과를 2분 동안 찾지 못했습니다. 공개 endpoint 장애가 아니라 probe 수집 경로 문제입니다. + + - name: unrelated prod-labelled probe cannot trigger the CodePlace prod alert + interval: 15s + input_series: + - series: 'probe_success{environment="prod",namespace="code-place-prod",probe_type="public-http",service="frontend",instance="https://code.pusan.ac.kr/"}' + values: '1 1 1 1 1 1' + - series: 'probe_success{environment="prod",namespace="code-place-prod",probe_type="public-http",service="hub-auth",instance="https://hub-auth.code-place-dev.site/healthz"}' + values: '1 1 1 1 1 1' + - series: 'probe_success{environment="dev",namespace="code-place-dev",probe_type="public-http",service="frontend",instance="https://k3s.code-place-dev.site/"}' + values: '1 1 1 1 1 1' + - series: 'probe_success{environment="dev",namespace="code-place-dev",probe_type="public-http",service="hub-auth",instance="https://hub-auth-dev.code-place-dev.site/healthz"}' + values: '1 1 1 1 1 1' + - series: 'probe_success{environment="prod",namespace="another-system",probe_type="public-http",service="hub-auth",instance="https://example.invalid/"}' + values: '0 0 0 0 0 0' + alert_rule_test: + - eval_time: 1m15s + alertname: PublicEndpointDown + exp_alerts: [] + + - name: partial backend target discovery is not healthy + interval: 15s + input_series: + - series: 'up{job="backend",namespace="code-place-dev",instance="backend-dev-1"}' + values: '1 1 1 1 1 1' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-1"}' + values: '1 1 1 1 1 1' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-2"}' + values: '1 1 1 1 1 1' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-3"}' + values: '1 1 1 1 1 1' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-4"}' + values: '1 1 1 1 1 1' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-5"}' + values: '1 1 1 1 1 1' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-6"}' + values: '1 1 1 1 1 1' + alert_rule_test: + - eval_time: 1m15s + alertname: BackendTargetDown + exp_alerts: + - exp_labels: + namespace: code-place-dev + priority: P0 + service: backend + severity: critical + exp_annotations: + summary: dev backend metrics target이 내려갔습니다 + description: Prometheus가 dev backend /metrics 엔드포인트를 1분 동안 scrape하지 못했거나 기대 target 2개를 모두 발견하지 못했습니다. + + - name: backend scrape failure preserves the failed instance + interval: 15s + input_series: + - series: 'up{job="backend",namespace="code-place-dev",instance="backend-dev-1"}' + values: '1 1 1 1 1 1' + - series: 'up{job="backend",namespace="code-place-dev",instance="backend-dev-2"}' + values: '0 0 0 0 0 0' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-1"}' + values: '1 1 1 1 1 1' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-2"}' + values: '1 1 1 1 1 1' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-3"}' + values: '1 1 1 1 1 1' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-4"}' + values: '1 1 1 1 1 1' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-5"}' + values: '1 1 1 1 1 1' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-6"}' + values: '1 1 1 1 1 1' + alert_rule_test: + - eval_time: 1m15s + alertname: BackendTargetDown + exp_alerts: + - exp_labels: + instance: backend-dev-2 + job: backend + namespace: code-place-dev + priority: P0 + service: backend + severity: critical + exp_annotations: + summary: dev backend metrics target이 내려갔습니다 + description: Prometheus가 dev backend /metrics 엔드포인트를 1분 동안 scrape하지 못했거나 기대 target 2개를 모두 발견하지 못했습니다. + + - name: backend rolling surge does not trigger target coverage + interval: 15s + input_series: + - series: 'up{job="backend",namespace="code-place-dev",instance="backend-dev-1"}' + values: '1+0x5' + - series: 'up{job="backend",namespace="code-place-dev",instance="backend-dev-2"}' + values: '1+0x5' + - series: 'up{job="backend",namespace="code-place-dev",instance="backend-dev-3"}' + values: '1+0x5' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-1"}' + values: '1+0x5' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-2"}' + values: '1+0x5' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-3"}' + values: '1+0x5' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-4"}' + values: '1+0x5' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-5"}' + values: '1+0x5' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-6"}' + values: '1+0x5' + - series: 'up{job="backend",namespace="code-place-prod",instance="backend-prod-7"}' + values: '1+0x5' + alert_rule_test: + - eval_time: 1m15s + alertname: BackendTargetDown + exp_alerts: [] + + - name: telemetry coverage uses expected backend replicas + interval: 15s + input_series: + - series: 'codeplace_collector_success{namespace="code-place-dev",collector="waiting_queue"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + - series: 'codeplace_collector_success{namespace="code-place-dev",collector="judge_server"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + - series: 'codeplace_collector_success{namespace="code-place-dev",collector="celery_queue"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + - series: 'codeplace_collector_success{namespace="code-place-dev",collector="sentinel"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + - series: 'codeplace_redis_sentinel_health{namespace="code-place-dev",check="master"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + - series: 'codeplace_redis_sentinel_health{namespace="code-place-dev",check="quorum"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + alert_rule_test: + - eval_time: 5m15s + alertname: CodePlaceTelemetryCoverageIncomplete + exp_alerts: + - exp_labels: + namespace: code-place-dev + priority: P1 + service: backend + severity: warning + exp_annotations: + summary: dev Backend telemetry replica coverage가 불완전합니다 + description: dev Backend target별 custom collector 4종 또는 Sentinel check 2종의 metric series가 5분 동안 누락됐습니다. Backend rollout 버전과 target별 /metrics 출력을 확인해야 합니다. + + - name: partial OTel target discovery is not healthy + interval: 15s + input_series: + - series: 'up{job="otel-collector",namespace="monitoring",instance="otel-collector-1"}' + values: '1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1' + alert_rule_test: + - eval_time: 5m15s + alertname: OpenTelemetryCollectorUnavailable + exp_alerts: + - exp_labels: + namespace: monitoring + priority: P1 + service: otel-collector + severity: warning + exp_annotations: + summary: OpenTelemetry Collector를 scrape할 수 없습니다 + description: OpenTelemetry Collector target이 5분 동안 없거나 내려갔거나 기대 target 2개보다 적습니다. dev/prod trace 수집 상태를 확인해야 합니다. + + - name: partial datastore and Blackbox metric coverage is not healthy + interval: 15s + input_series: + - series: 'cnpg_collector_up{namespace="code-place-dev",cluster="postgres",instance="postgres-1"}' + values: '1+0x21' + - series: 'cnpg_collector_up{namespace="code-place-dev",cluster="postgres",instance="postgres-2"}' + values: '1+0x21' + - series: 'cnpg_collector_up{namespace="code-place-prod",cluster="postgres",instance="postgres-1"}' + values: '1+0x21' + - series: 'cnpg_collector_up{namespace="code-place-prod",cluster="postgres",instance="postgres-2"}' + values: '1+0x21' + - series: 'cnpg_collector_up{namespace="code-place-prod",cluster="postgres",instance="postgres-3"}' + values: '1+0x21' + - series: 'redis_up{namespace="code-place-dev",job="redis",instance="redis-dev-1"}' + values: '1+0x21' + - series: 'redis_up{namespace="code-place-dev",job="redis",instance="redis-dev-2"}' + values: '1+0x21' + - series: 'redis_up{namespace="code-place-dev",job="redis",instance="redis-dev-3"}' + values: '1+0x21' + - series: 'redis_up{namespace="code-place-dev",job="redis",instance="redis-dev-4"}' + values: '1+0x21' + - series: 'redis_up{namespace="code-place-dev",job="redis",instance="redis-dev-5"}' + values: '1+0x21' + - series: 'redis_up{namespace="code-place-prod",job="redis",instance="redis-prod-1"}' + values: '1+0x21' + - series: 'redis_up{namespace="code-place-prod",job="redis",instance="redis-prod-2"}' + values: '1+0x21' + - series: 'redis_up{namespace="code-place-prod",job="redis",instance="redis-prod-3"}' + values: '1+0x21' + - series: 'redis_up{namespace="code-place-prod",job="redis",instance="redis-prod-4"}' + values: '1+0x21' + - series: 'redis_up{namespace="code-place-prod",job="redis",instance="redis-prod-5"}' + values: '1+0x21' + - series: 'redis_up{namespace="code-place-prod",job="redis",instance="redis-prod-6"}' + values: '1+0x21' + - series: 'up{namespace="monitoring",job="blackbox-exporter",instance="blackbox-1"}' + values: '1+0x21' + alert_rule_test: + - eval_time: 5m15s + alertname: PostgresCollectorUnavailable + exp_alerts: + - exp_labels: + namespace: code-place-dev + priority: P1 + service: postgres + severity: warning + exp_annotations: + summary: dev PostgreSQL metric collector가 응답하지 않습니다 + description: dev CNPG collector가 5분 동안 PostgreSQL에 접근하지 못했거나 기대 instance 3개의 collector metric을 모두 제공하지 못했습니다. + - eval_time: 5m15s + alertname: RedisExporterCoverageIncomplete + exp_alerts: + - exp_labels: + namespace: code-place-dev + priority: P1 + service: redis + severity: warning + exp_annotations: + summary: dev Redis exporter coverage가 불완전합니다 + description: dev Redis/Sentinel exporter가 5분 동안 실패했거나 기대 target 6개의 redis_up metric을 모두 제공하지 못했습니다. + - eval_time: 5m15s + alertname: BlackboxExporterCoverageIncomplete + exp_alerts: + - exp_labels: + namespace: monitoring + priority: P1 + service: blackbox-exporter + severity: warning + exp_annotations: + summary: Blackbox exporter coverage가 불완전합니다 + description: Blackbox exporter가 5분 동안 실패했거나 기대 target 2개보다 적습니다. public synthetic probe 수집 경로를 확인해야 합니다. + + - name: failed collector checks preserve target context + interval: 15s + input_series: + - series: 'codeplace_redis_sentinel_health{namespace="code-place-dev",job="backend",instance="backend-dev-1",check="master"}' + values: '0+0x21' + - series: 'codeplace_redis_sentinel_health{namespace="code-place-dev",job="backend",instance="backend-dev-1",check="quorum"}' + values: '1+0x21' + - series: 'codeplace_redis_sentinel_health{namespace="code-place-prod",job="backend",instance="backend-prod-1",check="master"}' + values: '1+0x21' + - series: 'codeplace_redis_sentinel_health{namespace="code-place-prod",job="backend",instance="backend-prod-1",check="quorum"}' + values: '1+0x21' + - series: 'codeplace_collector_success{namespace="code-place-dev",job="backend",instance="backend-dev-1",collector="waiting_queue"}' + values: '0+0x21' + - series: 'codeplace_collector_success{namespace="code-place-prod",job="backend",instance="backend-prod-1",collector="waiting_queue"}' + values: '1+0x21' + alert_rule_test: + - eval_time: 1m15s + alertname: RedisSentinelUnavailable + exp_alerts: + - exp_labels: + check: master + instance: backend-dev-1 + job: backend + namespace: code-place-dev + priority: P0 + service: redis + severity: critical + exp_annotations: + summary: dev Redis Sentinel master 또는 quorum을 확인할 수 없습니다 + description: Redis/Sentinel Pod readiness와 별개로 mymaster 탐색 또는 CKQUORUM 검사가 1분 동안 실패했습니다. + - eval_time: 5m15s + alertname: CodePlaceCollectorFailed + exp_alerts: + - exp_labels: + collector: waiting_queue + instance: backend-dev-1 + job: backend + namespace: code-place-dev + priority: P1 + service: backend + severity: warning + exp_annotations: + summary: dev Backend custom metric collector가 실패했습니다 + description: dev Backend custom metric collector가 5분 동안 Redis 기반 metric을 수집하지 못했거나 metric이 사라졌습니다. 실패 collector label을 확인해야 합니다. diff --git a/kubernetes/monitoring/prometheus-rules.yaml b/kubernetes/monitoring/prometheus-rules.yaml index 7f90e9e7..de7b6ae4 100644 --- a/kubernetes/monitoring/prometheus-rules.yaml +++ b/kubernetes/monitoring/prometheus-rules.yaml @@ -37,31 +37,68 @@ spec: expr: avg by (environment, namespace, service, instance) (avg_over_time(probe_success{probe_type="public-http"}[5m])) - record: codeplace:judge_waiting_queue_length expr: max by (namespace) (codeplace_waiting_queue_length{namespace=~"code-place-(dev|prod)"}) + - name: codeplace.monitoring-contract + interval: 15s + rules: + - alert: TraefikScrapeUnavailable + expr: (up{job="traefik", namespace="kube-system"} == 0) or absent(up{job="traefik", namespace="kube-system"}) + for: 1m + labels: + namespace: kube-system + service: traefik + severity: critical + priority: P0 + annotations: + summary: "Traefik metrics target을 scrape할 수 없습니다" + description: "Prometheus가 Traefik target을 1분 동안 발견하지 못했거나 scrape하지 못했습니다. 이 상태에서는 Ingress 트래픽과 5xx 경보가 동작하지 않습니다." + - alert: AlloyScrapeUnavailable + expr: (up{job=~".*alloy.*", namespace="monitoring"} == 0) or absent(up{job=~".*alloy.*", namespace="monitoring"}) + for: 1m + labels: + namespace: monitoring + service: alloy + severity: critical + priority: P0 + annotations: + summary: "Alloy metrics target을 scrape할 수 없습니다" + description: "Prometheus가 Alloy target을 1분 동안 발견하지 못했거나 scrape하지 못했습니다. 로그 수집 경로의 관측 가능성이 사라진 상태입니다." + - alert: PrometheusHADegraded + expr: (sum(max by (pod) (kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-[0-9]+"})) < 2) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"prometheus-kube-prometheus-stack-prometheus-[0-9]+"}) + for: 1m + labels: + namespace: monitoring + severity: warning + priority: P1 + annotations: + summary: "Prometheus HA replica가 저하됐습니다" + description: "ready Prometheus replica가 1분 동안 기대 개수 2개보다 적거나 readiness metric이 사라졌습니다." - name: codeplace.p0 interval: 15s rules: - alert: BackendTargetDown - expr: (min(up{job="backend", namespace="code-place-dev"}) == 0) or absent(up{job="backend", namespace="code-place-dev"}) + expr: (up{job="backend", namespace="code-place-dev"} == 0) or (count(up{job="backend", namespace="code-place-dev"}) < 2) or absent(up{job="backend", namespace="code-place-dev"}) for: 1m labels: namespace: code-place-dev + service: backend severity: critical priority: P0 annotations: summary: "dev backend metrics target이 내려갔습니다" - description: "Prometheus가 dev backend /metrics 엔드포인트를 1분 동안 scrape하지 못했거나 backend target metric이 사라졌습니다." + description: "Prometheus가 dev backend /metrics 엔드포인트를 1분 동안 scrape하지 못했거나 기대 target 2개를 모두 발견하지 못했습니다." - alert: BackendTargetDown - expr: (min(up{job="backend", namespace="code-place-prod"}) == 0) or absent(up{job="backend", namespace="code-place-prod"}) + expr: (up{job="backend", namespace="code-place-prod"} == 0) or (count(up{job="backend", namespace="code-place-prod"}) < 6) or absent(up{job="backend", namespace="code-place-prod"}) for: 1m labels: namespace: code-place-prod + service: backend severity: critical priority: P0 annotations: summary: "prod backend metrics target이 내려갔습니다" - description: "Prometheus가 prod backend /metrics 엔드포인트를 1분 동안 scrape하지 못했거나 backend target metric이 사라졌습니다." + description: "Prometheus가 prod backend /metrics 엔드포인트를 1분 동안 scrape하지 못했거나 기대 target 6개를 모두 발견하지 못했습니다." - alert: Api5xxSpike - expr: sum by (namespace) (rate(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)", status_code=~"5.."}[2m])) / clamp_min(sum by (namespace) (rate(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)"}[2m])), 1) > 0.05 + expr: (sum by (namespace) (rate(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)", status_code=~"5.."}[2m])) / clamp_min(sum by (namespace) (rate(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)"}[2m])), 0.001) > 0.05) and on (namespace) sum by (namespace) (increase(codeplace_http_requests_total{namespace=~"code-place-(dev|prod)"}[2m])) > 0 for: 2m labels: severity: critical @@ -70,7 +107,7 @@ spec: summary: "Backend API 5xx 비율이 높습니다" description: "Backend API 5xx 응답 비율이 2분 동안 5%를 초과했습니다." - alert: Ingress5xxSpike - expr: codeplace:ingress_5xx_rate2m{namespace="code-place-dev"} / clamp_min(codeplace:ingress_request_rate2m{namespace="code-place-dev"}, 1) > 0.05 + expr: (codeplace:ingress_5xx_rate2m{namespace="code-place-dev"} / clamp_min(codeplace:ingress_request_rate2m{namespace="code-place-dev"}, 0.001) > 0.05) and on (namespace) codeplace:ingress_request_rate2m{namespace="code-place-dev"} > 0 for: 2m labels: namespace: code-place-dev @@ -80,7 +117,7 @@ spec: summary: "dev Ingress 5xx 비율이 높습니다" description: "code-place-dev Traefik 5xx 응답 비율이 2분 동안 5%를 초과했습니다." - alert: Ingress5xxSpike - expr: codeplace:ingress_5xx_rate2m{namespace="code-place-prod"} / clamp_min(codeplace:ingress_request_rate2m{namespace="code-place-prod"}, 1) > 0.05 + expr: (codeplace:ingress_5xx_rate2m{namespace="code-place-prod"} / clamp_min(codeplace:ingress_request_rate2m{namespace="code-place-prod"}, 0.001) > 0.05) and on (namespace) codeplace:ingress_request_rate2m{namespace="code-place-prod"} > 0 for: 2m labels: namespace: code-place-prod @@ -90,15 +127,15 @@ spec: summary: "prod Ingress 5xx 비율이 높습니다" description: "code-place-prod Traefik 5xx 응답 비율이 2분 동안 5%를 초과했습니다." - alert: PublicEndpointDown - expr: (probe_success{environment="prod", probe_type="public-http"} == 0) or absent(probe_success{environment="prod", service="frontend", probe_type="public-http"}) or absent(probe_success{environment="prod", service="hub-auth", probe_type="public-http"}) + expr: probe_success{environment="prod", namespace="code-place-prod", probe_type="public-http", service=~"frontend|hub-auth"} == 0 for: 1m labels: severity: critical priority: P0 namespace: code-place-prod annotations: - summary: "prod 공개 엔드포인트가 응답하지 않습니다" - description: "prod 공개 HTTPS synthetic probe가 1분 동안 실패했습니다." + summary: "prod {{ $labels.service }} 공개 엔드포인트가 응답하지 않습니다" + description: "{{ $labels.instance }} HTTPS synthetic probe가 1분 동안 실패했습니다." - alert: PostgresUnavailable expr: (sum(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"postgres-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"postgres-[0-9]+"}) for: 1m @@ -119,6 +156,26 @@ spec: annotations: summary: "prod PostgreSQL Pod가 ready 상태가 아닙니다" description: "prod PostgreSQL Pod가 1분 동안 ready 상태가 아니거나 readiness metric이 사라졌습니다." + - alert: PostgresPrimaryUnavailable + expr: sum(kube_endpointslice_endpoints{namespace="code-place-dev", endpointslice=~"postgres-rw-.*", ready="true"}) == 0 or absent(kube_endpointslice_endpoints{namespace="code-place-dev", endpointslice=~"postgres-rw-.*", ready="true"}) + for: 1m + labels: + namespace: code-place-dev + severity: critical + priority: P0 + annotations: + summary: "dev PostgreSQL primary endpoint가 없습니다" + description: "dev postgres-rw Service에 1분 동안 ready primary endpoint가 없습니다. Replica Pod가 ready여도 쓰기 연결이 실패할 수 있습니다." + - alert: PostgresPrimaryUnavailable + expr: sum(kube_endpointslice_endpoints{namespace="code-place-prod", endpointslice=~"postgres-rw-.*", ready="true"}) == 0 or absent(kube_endpointslice_endpoints{namespace="code-place-prod", endpointslice=~"postgres-rw-.*", ready="true"}) + for: 1m + labels: + namespace: code-place-prod + severity: critical + priority: P0 + annotations: + summary: "prod PostgreSQL primary endpoint가 없습니다" + description: "prod postgres-rw Service에 1분 동안 ready primary endpoint가 없습니다. Replica Pod가 ready여도 쓰기 연결이 실패할 수 있습니다." - alert: RedisUnavailable expr: (sum(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"redis-.*|redis-replication-.*"}) == 0) or absent(kube_pod_status_ready{namespace="code-place-dev", condition="true", pod=~"redis-.*|redis-replication-.*"}) for: 1m @@ -139,6 +196,28 @@ spec: annotations: summary: "prod Redis/Sentinel Pod가 ready 상태가 아닙니다" description: "prod Redis/Sentinel Pod가 1분 동안 ready 상태가 아니거나 readiness metric이 사라졌습니다." + - alert: RedisSentinelUnavailable + expr: codeplace_redis_sentinel_health{namespace="code-place-dev"} == 0 or absent(codeplace_redis_sentinel_health{namespace="code-place-dev", check="master"}) or absent(codeplace_redis_sentinel_health{namespace="code-place-dev", check="quorum"}) + for: 1m + labels: + namespace: code-place-dev + service: redis + severity: critical + priority: P0 + annotations: + summary: "dev Redis Sentinel master 또는 quorum을 확인할 수 없습니다" + description: "Redis/Sentinel Pod readiness와 별개로 mymaster 탐색 또는 CKQUORUM 검사가 1분 동안 실패했습니다." + - alert: RedisSentinelUnavailable + expr: codeplace_redis_sentinel_health{namespace="code-place-prod"} == 0 or absent(codeplace_redis_sentinel_health{namespace="code-place-prod", check="master"}) or absent(codeplace_redis_sentinel_health{namespace="code-place-prod", check="quorum"}) + for: 1m + labels: + namespace: code-place-prod + service: redis + severity: critical + priority: P0 + annotations: + summary: "prod Redis Sentinel master 또는 quorum을 확인할 수 없습니다" + description: "Redis/Sentinel Pod readiness와 별개로 mymaster 탐색 또는 CKQUORUM 검사가 1분 동안 실패했습니다." - alert: LokiUnavailable expr: (sum(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) == 0) or absent(kube_pod_status_ready{namespace="monitoring", condition="true", pod=~"loki-[0-9]+"}) for: 1m @@ -189,6 +268,16 @@ spec: annotations: summary: "GPU driver XID 오류가 감지됐습니다" description: "DCGM이 1분 동안 NVIDIA XID 오류를 보고했습니다. vLLM GPU 상태와 노드 driver 상태를 확인해야 합니다." + - alert: DCGMExporterUnavailable + expr: up{job="dcgm-exporter", namespace="monitoring"} == 0 or absent(up{job="dcgm-exporter", namespace="monitoring"}) + for: 2m + labels: + severity: critical + priority: P0 + namespace: monitoring + annotations: + summary: "DCGM exporter를 scrape할 수 없습니다" + description: "DCGM exporter target이 2분 동안 없거나 내려갔습니다. 이 상태에서는 모든 GPU 상태 경보가 함께 사라집니다." - name: codeplace.p1 interval: 30s rules: @@ -202,24 +291,44 @@ spec: summary: "Backend API p95 latency가 높습니다" description: "Backend API p95 latency가 5분 동안 2초를 초과했습니다." - alert: PublicEndpointDown - expr: (probe_success{environment="dev", probe_type="public-http"} == 0) or absent(probe_success{environment="dev", service="frontend", probe_type="public-http"}) or absent(probe_success{environment="dev", service="hub-auth", probe_type="public-http"}) + expr: probe_success{environment="dev", namespace="code-place-dev", probe_type="public-http", service=~"frontend|hub-auth"} == 0 for: 2m labels: severity: warning priority: P1 namespace: code-place-dev annotations: - summary: "dev 공개 엔드포인트가 응답하지 않습니다" - description: "dev 공개 HTTPS synthetic probe가 2분 동안 실패했습니다." + summary: "dev {{ $labels.service }} 공개 엔드포인트가 응답하지 않습니다" + description: "{{ $labels.instance }} HTTPS synthetic probe가 2분 동안 실패했습니다." + - alert: PublicEndpointProbeMissing + expr: absent(probe_success{environment="prod", namespace="code-place-prod", service="frontend", probe_type="public-http"}) or absent(probe_success{environment="prod", namespace="code-place-prod", service="hub-auth", probe_type="public-http"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: code-place-prod + annotations: + summary: "prod {{ $labels.service }} 공개 probe metric이 없습니다" + description: "Prometheus가 prod {{ $labels.service }} synthetic probe 결과를 2분 동안 찾지 못했습니다. 공개 endpoint 장애가 아니라 probe 수집 경로 문제입니다." + - alert: PublicEndpointProbeMissing + expr: absent(probe_success{environment="dev", namespace="code-place-dev", service="frontend", probe_type="public-http"}) or absent(probe_success{environment="dev", namespace="code-place-dev", service="hub-auth", probe_type="public-http"}) + for: 2m + labels: + severity: warning + priority: P1 + namespace: code-place-dev + annotations: + summary: "dev {{ $labels.service }} 공개 probe metric이 없습니다" + description: "Prometheus가 dev {{ $labels.service }} synthetic probe 결과를 2분 동안 찾지 못했습니다. 공개 endpoint 장애가 아니라 probe 수집 경로 문제입니다." - alert: PublicEndpointLatencyHigh - expr: probe_duration_seconds{probe_type="public-http", namespace=~"code-place-(dev|prod)"} > 2 + expr: probe_duration_seconds{probe_type="public-http", namespace=~"code-place-(dev|prod)", service=~"frontend|hub-auth"} > 2 for: 5m labels: severity: warning priority: P1 annotations: - summary: "공개 엔드포인트 응답 시간이 높습니다" - description: "공개 HTTPS synthetic probe 응답 시간이 5분 동안 2초를 초과했습니다." + summary: "{{ $labels.namespace }} {{ $labels.service }} 공개 엔드포인트 응답 시간이 높습니다" + description: "{{ $labels.instance }} HTTPS synthetic probe 응답 시간이 5분 동안 2초를 초과했습니다." - alert: JudgeWaitingQueueBacklog expr: codeplace_waiting_queue_length{namespace=~"code-place-(dev|prod)"} > 5 for: 3m @@ -237,7 +346,51 @@ spec: priority: P1 annotations: summary: "Celery broker queue가 쌓이고 있습니다" - description: "Redis broker 기본 queue에서 Celery task가 5분 동안 20개를 초과했습니다. celery-worker 처리량, Redis broker 상태, task 오류를 확인해야 합니다." + description: "Celery broker default queue가 5분 동안 20개를 초과했습니다. Worker 처리량과 Redis 상태를 확인해야 합니다." + - alert: CodePlaceCollectorFailed + expr: codeplace_collector_success{namespace="code-place-dev"} == 0 or absent(codeplace_collector_success{namespace="code-place-dev"}) + for: 5m + labels: + namespace: code-place-dev + service: backend + severity: warning + priority: P1 + annotations: + summary: "dev Backend custom metric collector가 실패했습니다" + description: "dev Backend custom metric collector가 5분 동안 Redis 기반 metric을 수집하지 못했거나 metric이 사라졌습니다. 실패 collector label을 확인해야 합니다." + - alert: CodePlaceCollectorFailed + expr: codeplace_collector_success{namespace="code-place-prod"} == 0 or absent(codeplace_collector_success{namespace="code-place-prod"}) + for: 5m + labels: + namespace: code-place-prod + service: backend + severity: warning + priority: P1 + annotations: + summary: "prod Backend custom metric collector가 실패했습니다" + description: "prod Backend custom metric collector가 5분 동안 Redis 기반 metric을 수집하지 못했거나 metric이 사라졌습니다. 실패 collector label을 확인해야 합니다." + - alert: CodePlaceTelemetryCoverageIncomplete + expr: (count(codeplace_collector_success{namespace="code-place-dev"}) < 8) or (count(codeplace_redis_sentinel_health{namespace="code-place-dev"}) < 4) + for: 5m + labels: + namespace: code-place-dev + service: backend + severity: warning + priority: P1 + annotations: + summary: "dev Backend telemetry replica coverage가 불완전합니다" + description: "dev Backend target별 custom collector 4종 또는 Sentinel check 2종의 metric series가 5분 동안 누락됐습니다. Backend rollout 버전과 target별 /metrics 출력을 확인해야 합니다." + - alert: CodePlaceTelemetryCoverageIncomplete + expr: (count(codeplace_collector_success{namespace="code-place-prod"}) < 24) or (count(codeplace_redis_sentinel_health{namespace="code-place-prod"}) < 12) + for: 5m + labels: + namespace: code-place-prod + service: backend + severity: warning + priority: P1 + annotations: + summary: "prod Backend telemetry replica coverage가 불완전합니다" + description: "prod Backend target별 custom collector 4종 또는 Sentinel check 2종의 metric series가 5분 동안 누락됐습니다. Backend rollout 버전과 target별 /metrics 출력을 확인해야 합니다." - alert: SubmissionCreateSystemFailures expr: sum by (namespace, scope, status) (increase(codeplace_submission_create_outcome_total{namespace=~"code-place-(dev|prod)", status=~"db_error|enqueue_error"}[5m])) > 0 for: 2m @@ -248,7 +401,7 @@ spec: summary: "제출 생성 경로에서 시스템 오류가 발생했습니다" description: "제출 생성 중 DB 오류 또는 judge enqueue 오류가 보고됐습니다. backend 로그, PostgreSQL, Redis/Celery broker, judge task queue를 확인해야 합니다." - alert: JudgeTaskFailures - expr: sum by (namespace, scope, status) (increase(codeplace_judge_task_outcome_total{namespace=~"code-place-(dev|prod)", status=~"error|submission_missing|user_missing"}[5m])) > 0 + expr: max by (namespace, scope, status) (increase(codeplace_judge_task_outcome_total{namespace=~"code-place-(dev|prod)", status=~"error|submission_missing|user_missing"}[5m])) > 0 for: 2m labels: severity: warning @@ -337,7 +490,7 @@ spec: summary: "CodePlace Deployment에 unavailable replica가 있습니다" description: "CodePlace Deployment에 5분 동안 unavailable replica가 있습니다. rollout 상태, Pod event, readiness probe, image pull, resource pressure를 확인해야 합니다." - alert: CodePlaceServiceNoReadyEndpoints - expr: (sum by (namespace, endpoint) (kube_endpoint_address{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server", ready="true"}) < 1) or (label_replace(kube_service_info{namespace=~"code-place-(dev|prod)", service=~"backend|frontend|hub-auth|judge-server"}, "endpoint", "$1", "service", "(.*)") unless on (namespace, endpoint) kube_endpoint_address{namespace=~"code-place-(dev|prod)", endpoint=~"backend|frontend|hub-auth|judge-server", ready="true"}) + expr: (sum by (namespace, service) (label_replace(kube_endpointslice_endpoints{namespace=~"code-place-(dev|prod)", endpointslice=~"(backend|frontend|hub-auth|judge-server)-.*", ready="true"}, "service", "$1", "endpointslice", "(backend|frontend|hub-auth|judge-server)-.*")) < 1) or (kube_service_info{namespace=~"code-place-(dev|prod)", service=~"backend|frontend|hub-auth|judge-server"} unless on (namespace, service) label_replace(kube_endpointslice_endpoints{namespace=~"code-place-(dev|prod)", endpointslice=~"(backend|frontend|hub-auth|judge-server)-.*", ready="true"}, "service", "$1", "endpointslice", "(backend|frontend|hub-auth|judge-server)-.*")) for: 2m labels: severity: warning @@ -346,7 +499,7 @@ spec: summary: "CodePlace Service에 ready endpoint가 없습니다" description: "CodePlace Service에 2분 동안 ready endpoint가 없거나 Service는 있는데 endpoint availability metric이 사라졌습니다. Service selector label, Pod readiness, EndpointSlice 상태, rollout event를 확인해야 합니다." - alert: VLLMServiceNoReadyEndpoints - expr: (sum by (namespace, endpoint) (kube_endpoint_address{namespace="code-place-prod", endpoint="vllm", ready="true"}) < 1) or (label_replace(kube_service_info{namespace="code-place-prod", service="vllm"}, "endpoint", "$1", "service", "(.*)") unless on (namespace, endpoint) kube_endpoint_address{namespace="code-place-prod", endpoint="vllm", ready="true"}) + expr: (sum by (namespace, service) (label_replace(kube_endpointslice_endpoints{namespace="code-place-prod", endpointslice=~"vllm-.*", ready="true"}, "service", "vllm", "endpointslice", ".*")) < 1) or (kube_service_info{namespace="code-place-prod", service="vllm"} unless on (namespace, service) label_replace(kube_endpointslice_endpoints{namespace="code-place-prod", endpointslice=~"vllm-.*", ready="true"}, "service", "vllm", "endpointslice", ".*")) for: 2m labels: severity: warning @@ -382,6 +535,28 @@ spec: annotations: summary: "PostgreSQL metric 수집이 실패하고 있습니다" description: "CNPG PostgreSQL metric 수집 오류가 5분 동안 보고됐습니다." + - alert: PostgresCollectorUnavailable + expr: (cnpg_collector_up{namespace="code-place-dev", cluster="postgres"} == 0) or (count(cnpg_collector_up{namespace="code-place-dev", cluster="postgres"}) < 3) or absent(cnpg_collector_up{namespace="code-place-dev", cluster="postgres"}) + for: 5m + labels: + namespace: code-place-dev + service: postgres + severity: warning + priority: P1 + annotations: + summary: "dev PostgreSQL metric collector가 응답하지 않습니다" + description: "dev CNPG collector가 5분 동안 PostgreSQL에 접근하지 못했거나 기대 instance 3개의 collector metric을 모두 제공하지 못했습니다." + - alert: PostgresCollectorUnavailable + expr: (cnpg_collector_up{namespace="code-place-prod", cluster="postgres"} == 0) or (count(cnpg_collector_up{namespace="code-place-prod", cluster="postgres"}) < 3) or absent(cnpg_collector_up{namespace="code-place-prod", cluster="postgres"}) + for: 5m + labels: + namespace: code-place-prod + service: postgres + severity: warning + priority: P1 + annotations: + summary: "prod PostgreSQL metric collector가 응답하지 않습니다" + description: "prod CNPG collector가 5분 동안 PostgreSQL에 접근하지 못했거나 기대 instance 3개의 collector metric을 모두 제공하지 못했습니다." - alert: PostgresHADegraded expr: min by (namespace) (cnpg_collector_nodes_used{namespace=~"code-place-(dev|prod)", cluster="postgres"}) < 2 for: 10m @@ -409,6 +584,28 @@ spec: annotations: summary: "Redis/Sentinel ready replica 수가 부족합니다" description: "Redis replication 및 Sentinel Pod ready 수가 5분 동안 6개 미만입니다. Redis failover 여력이 줄어들 수 있습니다." + - alert: RedisExporterCoverageIncomplete + expr: (redis_up{namespace="code-place-dev", job="redis"} == 0) or (count(redis_up{namespace="code-place-dev", job="redis"}) < 6) or absent(redis_up{namespace="code-place-dev", job="redis"}) + for: 5m + labels: + namespace: code-place-dev + service: redis + severity: warning + priority: P1 + annotations: + summary: "dev Redis exporter coverage가 불완전합니다" + description: "dev Redis/Sentinel exporter가 5분 동안 실패했거나 기대 target 6개의 redis_up metric을 모두 제공하지 못했습니다." + - alert: RedisExporterCoverageIncomplete + expr: (redis_up{namespace="code-place-prod", job="redis"} == 0) or (count(redis_up{namespace="code-place-prod", job="redis"}) < 6) or absent(redis_up{namespace="code-place-prod", job="redis"}) + for: 5m + labels: + namespace: code-place-prod + service: redis + severity: warning + priority: P1 + annotations: + summary: "prod Redis exporter coverage가 불완전합니다" + description: "prod Redis/Sentinel exporter가 5분 동안 실패했거나 기대 target 6개의 redis_up metric을 모두 제공하지 못했습니다." - alert: RedisMemoryHigh expr: (max by (namespace, pod) (redis_memory_used_bytes{namespace=~"code-place-(dev|prod)"}) / max by (namespace, pod) (redis_memory_max_bytes{namespace=~"code-place-(dev|prod)"}) > 0.85) and on (namespace, pod) max by (namespace, pod) (redis_memory_max_bytes{namespace=~"code-place-(dev|prod)"}) > 0 for: 10m @@ -418,6 +615,48 @@ spec: annotations: summary: "Redis memory 사용률이 높습니다" description: "Redis exporter 기준 memory 사용률이 10분 동안 설정된 maxmemory의 85%를 초과했습니다." + - alert: OpenTelemetryCollectorUnavailable + expr: (up{job="otel-collector", namespace="monitoring"} == 0) or (count(up{job="otel-collector", namespace="monitoring"}) < 2) or absent(up{job="otel-collector", namespace="monitoring"}) + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + service: otel-collector + annotations: + summary: "OpenTelemetry Collector를 scrape할 수 없습니다" + description: "OpenTelemetry Collector target이 5분 동안 없거나 내려갔거나 기대 target 2개보다 적습니다. dev/prod trace 수집 상태를 확인해야 합니다." + - alert: BlackboxExporterCoverageIncomplete + expr: (up{job="blackbox-exporter", namespace="monitoring"} == 0) or (count(up{job="blackbox-exporter", namespace="monitoring"}) < 2) or absent(up{job="blackbox-exporter", namespace="monitoring"}) + for: 5m + labels: + namespace: monitoring + service: blackbox-exporter + severity: warning + priority: P1 + annotations: + summary: "Blackbox exporter coverage가 불완전합니다" + description: "Blackbox exporter가 5분 동안 실패했거나 기대 target 2개보다 적습니다. public synthetic probe 수집 경로를 확인해야 합니다." + - alert: TempoUnavailable + expr: up{job="tempo", namespace="monitoring"} == 0 or absent(up{job="tempo", namespace="monitoring"}) + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "Tempo를 scrape할 수 없습니다" + description: "Tempo target이 5분 동안 없거나 내려갔습니다. trace 저장 및 조회가 실패할 수 있습니다." + - alert: OpenTelemetrySpanExportFailures + expr: sum(increase(otelcol_exporter_send_failed_spans_total{namespace="monitoring"}[5m])) > 0 + for: 5m + labels: + severity: warning + priority: P1 + namespace: monitoring + annotations: + summary: "OpenTelemetry span 전송 실패가 지속됩니다" + description: "Collector가 5분 동안 Tempo로 span을 전송하지 못했습니다. Collector exporter와 Tempo 상태를 확인해야 합니다." - alert: PVCAlmostFull expr: max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~"code-place-(dev|prod)"}) / clamp_min(max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~"code-place-(dev|prod)"}), 1) > 0.85 for: 10m diff --git a/kubernetes/monitoring/public-endpoint-probes.yaml b/kubernetes/monitoring/public-endpoint-probes.yaml index d46405b3..a83b68a3 100644 --- a/kubernetes/monitoring/public-endpoint-probes.yaml +++ b/kubernetes/monitoring/public-endpoint-probes.yaml @@ -83,7 +83,7 @@ spec: targets: staticConfig: static: - - https://hub-auth-dev.code-place-dev.site/ + - https://hub-auth-dev.code-place-dev.site/healthz labels: environment: dev namespace: code-place-dev @@ -106,7 +106,7 @@ spec: targets: staticConfig: static: - - https://hub-auth.code-place-dev.site/ + - https://hub-auth.code-place-dev.site/healthz labels: environment: prod namespace: code-place-prod diff --git a/kubernetes/monitoring/secrets/grafana-admin-credentials.sealedsecret.yaml b/kubernetes/monitoring/secrets/grafana-admin-credentials.sealedsecret.yaml new file mode 100644 index 00000000..3d338e66 --- /dev/null +++ b/kubernetes/monitoring/secrets/grafana-admin-credentials.sealedsecret.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: grafana-admin-credentials + namespace: monitoring +spec: + encryptedData: + admin-password: AgDA0TI6cEMA7hp9HOn35I8/2T6uYkVZrjyIWv1zRh/1rSi8ZRJg2g2iC38MpvjYQSSlIaNUWJQi+FivJtW11MRgQ+skZ82OO39lU70LK+1+VL1X5qFfEWq28gh1YszljPGUfuFL/jPxvm4YJs04n0aEUmb0ULu6EqzZwPdr6emv4YOH1frsETaipGV2z8XkPmHr7ON5yxzqTS+racI7kj7ImJ0L1P9fIdl6KrmF93memd1uevvMU1oefLSp2jbmc6Ms0GH9nHbX4DePtB4WJBG9Vv8Q7PfJZ02ktkwE61Lho7+qexDXIgzy0GuNBrfv65sbRf1q0qTOO5tcgnk01zMbzKInaT+j6/hpnvC5VyBoF8Tsf+63AQ0+DDDHO381G0S94Oe9v04norscE4cFN+qdUeAJEcTbrDY+c++qe3yw+2s8PlrEZBslsSisVeLVLmP16/9Y2atA1ZHUvB5sOTGLivlqRRifoRFv/gdeL2jIEmn4maWRp4Iec2IWMJ+OFl5Tg/FQ+Ft7zfC0k4T6M00fsHSrcg+Tmn3kF4hV/vMPWLWD5PHjI71kDVOSseQOXj126zXm1TuyoTlPJetuSHXJ8hw0/4wVZrv2bAixgzHHkbwk0+PQe10fgTk3F2OSxUwmCCIMb5fiyee09/1y4cwzDNTsytr/hIpSdSbTzZv4H/Mstu1rRudCpDzQM7VfrFo1FpMf77PdmuF6 + admin-user: AgCFcX74+X//E2Z1tvjUQuQTJs2bW7YdUcl2zTppHCeJhVUfkmjvCG7q9LJdJyOvCsHF+REvDgdC0ba5FnmKExCyR8gDPCLaY1Fcoc3TwYcl7/fW434+ruz49+qPXaIBDZNVrmzyJrrQRr17E2Dp0bHZXnRstOlCXzldo/A3VEUa5W09ESgrViY6pPCYVAojCyQwmUtsVRhV/kfZzciQalHOhDWKRgGg+WEwjLwuTf0Eo86EJXJONVVeNY9tEjlU5DhQPdk+XekxC0Yphte7Yfc3Vd+6IculZ75X7HOin0AmPpEsBCxb1Aduu7zKz+ecmkmALrFHrq2Pu0427NPixMC5cKxTs/pzyAOoOo7/TJUWH9isDglgX6T3IvAesznI01nckWvMcQWPzRyLZMqSg62mBm2+jrQWusTrPv6mXT5BhTspOa2SXsO9tVL1Av5b2qLf4UjdQ75wWg2LBDIf5dZffBbErup4nQmko+X+wzy61tp15FEXDzlSyA4c+RmLIYQUMzKV3LKNpLZhXOnUKCDV7C/RjOhqNbSiYvMfW30Aq9enyGYwyzejQhJu0GGrtzCFWNr/I8/D5YFKzVHxc78oi3YdkjEX8P7WSh9c9UgkirI/VG5yW6chVY5BbdKzS986P0RFK8R7MWAdYQKDsq9cu/9xGKN4+xlsmqStJn0OFcdZYkCq9V2xTJ7WL0hhnkyfsXIv7g== + template: + metadata: + name: grafana-admin-credentials + namespace: monitoring diff --git a/kubernetes/monitoring/traefik-service-monitor.yaml b/kubernetes/monitoring/traefik-pod-monitor.yaml similarity index 81% rename from kubernetes/monitoring/traefik-service-monitor.yaml rename to kubernetes/monitoring/traefik-pod-monitor.yaml index 8a43b4bf..71411ba9 100644 --- a/kubernetes/monitoring/traefik-service-monitor.yaml +++ b/kubernetes/monitoring/traefik-pod-monitor.yaml @@ -1,5 +1,5 @@ apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor +kind: PodMonitor metadata: name: traefik labels: @@ -11,7 +11,8 @@ spec: selector: matchLabels: app.kubernetes.io/name: traefik - endpoints: + jobLabel: app.kubernetes.io/name + podMetricsEndpoints: - port: metrics path: /metrics interval: 15s diff --git a/kubernetes/monitoring/validate_alerts.py b/kubernetes/monitoring/validate_alerts.py new file mode 100644 index 00000000..4651502b --- /dev/null +++ b/kubernetes/monitoring/validate_alerts.py @@ -0,0 +1,402 @@ +#!/usr/bin/env python3 +"""Validate public endpoint alerts and their notification routing.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import yaml + + +ROOT = Path(__file__).resolve().parent + + +def load_yaml(name: str) -> dict[str, Any]: + return yaml.safe_load((ROOT / name).read_text(encoding="utf-8")) + + +def load_yaml_documents(name: str) -> list[dict[str, Any]]: + return [ + document + for document in yaml.safe_load_all((ROOT / name).read_text(encoding="utf-8")) + if document is not None + ] + + +def main() -> int: + prometheus = load_yaml("prometheus-rules.yaml") + alertmanager = load_yaml("alertmanager-config.yaml") + datastore_monitors = load_yaml_documents("datastore-pod-monitors.yaml") + rules = [ + rule + for group in prometheus["spec"]["groups"] + for rule in group.get("rules", []) + if "alert" in rule + ] + errors: list[str] = [] + + postgres_monitors = [ + monitor + for monitor in datastore_monitors + if monitor.get("kind") == "PodMonitor" + and monitor.get("metadata", {}).get("name") == "postgres" + ] + if len(postgres_monitors) != 1: + errors.append("expected exactly one PostgreSQL PodMonitor") + else: + endpoints = postgres_monitors[0].get("spec", {}).get("podMetricsEndpoints", []) + relabelings = endpoints[0].get("relabelings", []) if len(endpoints) == 1 else [] + if not any( + relabeling.get("sourceLabels") + == ["__meta_kubernetes_pod_label_cnpg_io_cluster"] + and relabeling.get("targetLabel") == "cluster" + for relabeling in relabelings + ): + errors.append("PostgreSQL PodMonitor must expose the CNPG cluster label") + + expected_environments = { + "code-place-prod": ("prod", "P0", "1m"), + "code-place-dev": ("dev", "P1", "2m"), + } + + endpoint_rules = [rule for rule in rules if rule["alert"] == "PublicEndpointDown"] + if len(endpoint_rules) != 2: + errors.append(f"expected two PublicEndpointDown rules, found {len(endpoint_rules)}") + for namespace, (environment, priority, duration) in expected_environments.items(): + matching = [rule for rule in endpoint_rules if rule.get("labels", {}).get("namespace") == namespace] + if len(matching) != 1: + errors.append(f"{namespace}: expected exactly one PublicEndpointDown rule") + continue + rule = matching[0] + expression = rule["expr"] + for token in ( + f'environment="{environment}"', + f'namespace="{namespace}"', + 'probe_type="public-http"', + 'service=~"frontend|hub-auth"', + ): + if token not in expression: + errors.append(f"{namespace}: PublicEndpointDown is missing {token}") + if "absent(" in expression: + errors.append(f"{namespace}: PublicEndpointDown must not treat missing telemetry as downtime") + if rule.get("labels", {}).get("priority") != priority: + errors.append(f"{namespace}: PublicEndpointDown priority must be {priority}") + if rule.get("for") != duration: + errors.append(f"{namespace}: PublicEndpointDown must persist for {duration}") + annotations = rule.get("annotations", {}) + if "$labels.service" not in annotations.get("summary", ""): + errors.append(f"{namespace}: PublicEndpointDown summary must identify the service") + if "$labels.instance" not in annotations.get("description", ""): + errors.append(f"{namespace}: PublicEndpointDown description must identify the instance") + + missing_rules = [rule for rule in rules if rule["alert"] == "PublicEndpointProbeMissing"] + if len(missing_rules) != 2: + errors.append(f"expected two PublicEndpointProbeMissing rules, found {len(missing_rules)}") + for namespace, (environment, _, _) in expected_environments.items(): + matching = [rule for rule in missing_rules if rule.get("labels", {}).get("namespace") == namespace] + if len(matching) != 1: + errors.append(f"{namespace}: expected exactly one PublicEndpointProbeMissing rule") + continue + rule = matching[0] + expression = rule["expr"] + if expression.count("absent(") != 2: + errors.append(f"{namespace}: probe coverage must check frontend and hub-auth separately") + for service in ("frontend", "hub-auth"): + expected = ( + f'absent(probe_success{{environment="{environment}", namespace="{namespace}", ' + f'service="{service}", probe_type="public-http"}})' + ) + if expected not in expression: + errors.append(f"{namespace}: missing {service} probe coverage check") + if rule.get("labels", {}).get("priority") != "P1": + errors.append(f"{namespace}: missing probe telemetry must be reported separately as P1") + if rule.get("for") != "2m": + errors.append(f"{namespace}: missing probe telemetry must persist for 2m") + + scrape_coverage_contracts = ( + ( + "BackendTargetDown", + "code-place-dev", + 'count(up{job="backend", namespace="code-place-dev"}) < 2', + ), + ( + "BackendTargetDown", + "code-place-prod", + 'count(up{job="backend", namespace="code-place-prod"}) < 6', + ), + ( + "CodePlaceTelemetryCoverageIncomplete", + "code-place-dev", + 'count(codeplace_collector_success{namespace="code-place-dev"}) < 8', + ), + ( + "CodePlaceTelemetryCoverageIncomplete", + "code-place-dev", + 'count(codeplace_redis_sentinel_health{namespace="code-place-dev"}) < 4', + ), + ( + "CodePlaceTelemetryCoverageIncomplete", + "code-place-prod", + 'count(codeplace_collector_success{namespace="code-place-prod"}) < 24', + ), + ( + "CodePlaceTelemetryCoverageIncomplete", + "code-place-prod", + 'count(codeplace_redis_sentinel_health{namespace="code-place-prod"}) < 12', + ), + ( + "PostgresCollectorUnavailable", + "code-place-dev", + 'count(cnpg_collector_up{namespace="code-place-dev", cluster="postgres"}) < 3', + ), + ( + "PostgresCollectorUnavailable", + "code-place-prod", + 'count(cnpg_collector_up{namespace="code-place-prod", cluster="postgres"}) < 3', + ), + ( + "RedisExporterCoverageIncomplete", + "code-place-dev", + 'count(redis_up{namespace="code-place-dev", job="redis"}) < 6', + ), + ( + "RedisExporterCoverageIncomplete", + "code-place-prod", + 'count(redis_up{namespace="code-place-prod", job="redis"}) < 6', + ), + ( + "OpenTelemetryCollectorUnavailable", + "monitoring", + 'count(up{job="otel-collector", namespace="monitoring"}) < 2', + ), + ( + "BlackboxExporterCoverageIncomplete", + "monitoring", + 'count(up{job="blackbox-exporter", namespace="monitoring"}) < 2', + ), + ) + for alert_name, namespace, expected_expression in scrape_coverage_contracts: + matching = [ + rule + for rule in rules + if rule["alert"] == alert_name + and rule.get("labels", {}).get("namespace") == namespace + ] + if len(matching) != 1: + errors.append(f"{namespace}: expected exactly one {alert_name} rule") + elif expected_expression not in matching[0]["expr"]: + errors.append( + f"{namespace}: {alert_name} must enforce expected target coverage" + ) + + target_context_contracts = ( + ( + "TraefikScrapeUnavailable", + "kube-system", + 'up{job="traefik", namespace="kube-system"} == 0', + "traefik", + ), + ( + "AlloyScrapeUnavailable", + "monitoring", + 'up{job=~".*alloy.*", namespace="monitoring"} == 0', + "alloy", + ), + ( + "BackendTargetDown", + "code-place-dev", + 'up{job="backend", namespace="code-place-dev"} == 0', + "backend", + ), + ( + "BackendTargetDown", + "code-place-prod", + 'up{job="backend", namespace="code-place-prod"} == 0', + "backend", + ), + ( + "RedisSentinelUnavailable", + "code-place-dev", + 'codeplace_redis_sentinel_health{namespace="code-place-dev"} == 0', + "redis", + ), + ( + "RedisSentinelUnavailable", + "code-place-prod", + 'codeplace_redis_sentinel_health{namespace="code-place-prod"} == 0', + "redis", + ), + ( + "CodePlaceCollectorFailed", + "code-place-dev", + 'codeplace_collector_success{namespace="code-place-dev"} == 0', + "backend", + ), + ( + "CodePlaceCollectorFailed", + "code-place-prod", + 'codeplace_collector_success{namespace="code-place-prod"} == 0', + "backend", + ), + ( + "PostgresCollectorUnavailable", + "code-place-dev", + 'cnpg_collector_up{namespace="code-place-dev", cluster="postgres"} == 0', + "postgres", + ), + ( + "PostgresCollectorUnavailable", + "code-place-prod", + 'cnpg_collector_up{namespace="code-place-prod", cluster="postgres"} == 0', + "postgres", + ), + ( + "RedisExporterCoverageIncomplete", + "code-place-dev", + 'redis_up{namespace="code-place-dev", job="redis"} == 0', + "redis", + ), + ( + "RedisExporterCoverageIncomplete", + "code-place-prod", + 'redis_up{namespace="code-place-prod", job="redis"} == 0', + "redis", + ), + ( + "OpenTelemetryCollectorUnavailable", + "monitoring", + 'up{job="otel-collector", namespace="monitoring"} == 0', + "otel-collector", + ), + ( + "BlackboxExporterCoverageIncomplete", + "monitoring", + 'up{job="blackbox-exporter", namespace="monitoring"} == 0', + "blackbox-exporter", + ), + ) + for alert_name, namespace, expected_expression, service in target_context_contracts: + matching = [ + rule + for rule in rules + if rule["alert"] == alert_name + and rule.get("labels", {}).get("namespace") == namespace + ] + if len(matching) != 1: + errors.append(f"{namespace}: expected exactly one {alert_name} rule") + continue + rule = matching[0] + if expected_expression not in rule["expr"]: + errors.append(f"{namespace}: {alert_name} must preserve failed target labels") + if rule.get("labels", {}).get("service") != service: + errors.append(f"{namespace}: {alert_name} must identify service {service}") + + group_by = alertmanager["spec"]["route"].get("groupBy", []) + if "service" not in group_by: + errors.append("Alertmanager must group endpoint notifications by service") + + routes = alertmanager["spec"]["route"].get("routes", []) + expected_routes = ( + ({"priority": "P0"}, "p0-discord"), + ({"priority": "P1", "namespace": "code-place-dev"}, "dev-p1-muted"), + ({"priority": "P1"}, "p1-discord"), + ({"severity": "critical"}, "p0-discord"), + ({"severity": "warning", "namespace": "code-place-dev"}, "dev-p1-muted"), + ({"severity": "warning"}, "p1-discord"), + ) + previous_index = -1 + for expected_matchers, expected_receiver in expected_routes: + matching_indexes = [ + index + for index, route in enumerate(routes) + if { + matcher.get("name"): matcher.get("value") + for matcher in route.get("matchers", []) + } + == expected_matchers + and route.get("receiver") == expected_receiver + ] + if len(matching_indexes) != 1: + errors.append( + f"Alertmanager must route {expected_matchers} to {expected_receiver} exactly once" + ) + continue + if matching_indexes[0] <= previous_index: + errors.append("Alertmanager priority routes must precede severity fallback routes") + previous_index = matching_indexes[0] + + receivers = {receiver["name"]: receiver for receiver in alertmanager["spec"]["receivers"]} + for receiver_name, priority in (("p0-discord", "P0"), ("p1-discord", "P1")): + configs = receivers.get(receiver_name, {}).get("discordConfigs", []) + if len(configs) != 1: + errors.append(f"{receiver_name}: expected exactly one Discord config") + continue + config = configs[0] + title = config.get("title", "") + message = config.get("message", "") + if ".CommonLabels.service" not in title: + errors.append(f"{receiver_name}: title must identify the failed service") + if "활성" not in title: + errors.append(f"{receiver_name}: repeated firing notifications must use the active-state label") + for token in ( + ".Labels.service", + ".Labels.instance", + ".Labels.pod", + ".Labels.container", + ".Labels.persistentvolumeclaim", + ".Labels.pvc", + ".Labels.pvc_namespace", + ".Labels.deployment", + ".Labels.daemonset", + ".Labels.node", + ".Labels.volume", + ".Labels.disk", + ".Labels.reason", + ".Labels.condition", + ".Labels.collector", + ".Labels.check", + ".Labels.scope", + ".Labels.status", + ".Labels.Hostname", + ".Labels.gpu", + ".Labels.UUID", + ".Annotations.description", + ): + if token not in message: + errors.append(f"{receiver_name}: message is missing {token}") + if "문맥:" not in message: + errors.append(f"{receiver_name}: message must show available workload context labels") + if message.count(".GeneratorURL") != 1: + errors.append(f"{receiver_name}: message must show one query link per alert group") + if "발생 쿼리: [Prometheus에서 보기]" not in message: + errors.append(f"{receiver_name}: query URL must use a compact link label") + if f"우선순위: {priority}" not in message: + errors.append(f"{receiver_name}: message must show its routed priority {priority}") + for environment in ("prod", "dev"): + storage_link = ( + f"스토리지({environment}): " + "https://monitoring.code-place-dev.site/d/codeplace-storage/" + f"codeplace-storage?orgId=1&var-environment={environment}" + ) + if storage_link not in message: + errors.append( + f"{receiver_name}: Longhorn message is missing the {environment} storage link" + ) + + if errors: + print("Monitoring alert validation failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + + print( + "Validated endpoint separation, scrape coverage, notification context, " + "routing, and dashboard links." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/kubernetes/monitoring/validate_dashboards.py b/kubernetes/monitoring/validate_dashboards.py new file mode 100644 index 00000000..2970a712 --- /dev/null +++ b/kubernetes/monitoring/validate_dashboards.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +"""Validate the contracts shared by the provisioned CodePlace dashboards.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any, Iterable + + +ROOT = Path(__file__).resolve().parent +ENVIRONMENT_DASHBOARDS = { + "codeplace-overview", + "codeplace-platform", + "codeplace-logs", + "codeplace-kubernetes-events", + "codeplace-ai-api", + "codeplace-public-endpoints", + "codeplace-storage", + "codeplace-traces", +} +PRODUCTION_ONLY_DASHBOARDS = {"codeplace-ai-inference"} +GLOBAL_DASHBOARDS = {"codeplace-log-pipeline", "codeplace-monitoring-stack"} +MIXED_ENVIRONMENT_PATTERNS = ( + 'code-place-(dev|prod)', + 'code-place-(prod|dev)', + 'code-place-dev|code-place-prod', + 'code-place-prod|code-place-dev', +) + + +def embedded_json_documents(path: Path) -> Iterable[dict[str, Any]]: + """Extract JSON literal blocks from the dashboard ConfigMap without PyYAML.""" + lines = path.read_text(encoding="utf-8").splitlines() + index = 0 + while index < len(lines): + match = re.match(r"^ ([^:]+\.json):\s*\|[-+]?\s*$", lines[index]) + if not match: + index += 1 + continue + + key = match.group(1) + index += 1 + block: list[str] = [] + while index < len(lines) and (not lines[index] or lines[index].startswith(" ")): + block.append(lines[index][4:] if lines[index].startswith(" ") else "") + index += 1 + try: + yield json.loads("\n".join(block)) + except json.JSONDecodeError as exc: + raise ValueError(f"{path.name}:{key}: invalid embedded JSON: {exc}") from exc + + +def all_panels(panels: Iterable[dict[str, Any]]) -> Iterable[dict[str, Any]]: + for panel in panels: + yield panel + yield from all_panels(panel.get("panels", [])) + + +def all_query_text(value: Any) -> Iterable[str]: + if isinstance(value, dict): + for key, child in value.items(): + if key in {"expr", "query"} and isinstance(child, str): + yield child + yield from all_query_text(child) + elif isinstance(value, list): + for child in value: + yield from all_query_text(child) + + +def validate_environment_variable(dashboard: dict[str, Any], errors: list[str]) -> None: + uid = dashboard["uid"] + variables = dashboard.get("templating", {}).get("list", []) + environment = [variable for variable in variables if variable.get("name") == "environment"] + if len(environment) != 1: + errors.append(f"{uid}: expected exactly one environment variable") + return + + variable = environment[0] + expected = { + "type": "custom", + "query": "prod,dev", + "includeAll": False, + "multi": False, + } + for key, value in expected.items(): + if variable.get(key) != value: + errors.append(f"{uid}: environment.{key} must be {value!r}") + if variable.get("current", {}).get("value") != "prod": + errors.append(f"{uid}: environment must default to prod") + + +def validate_grid(dashboard: dict[str, Any], errors: list[str]) -> None: + uid = dashboard["uid"] + rectangles: list[tuple[int, int, int, int, str]] = [] + for panel in all_panels(dashboard.get("panels", [])): + if panel.get("type") == "row": + continue + position = panel.get("gridPos") + if not isinstance(position, dict): + errors.append(f"{uid}: panel {panel.get('title')!r} has no gridPos") + continue + try: + x, y, width, height = (int(position[key]) for key in ("x", "y", "w", "h")) + except (KeyError, TypeError, ValueError): + errors.append(f"{uid}: panel {panel.get('title')!r} has an invalid gridPos") + continue + if x < 0 or y < 0 or width <= 0 or height <= 0 or x + width > 24: + errors.append(f"{uid}: panel {panel.get('title')!r} is outside the 24-column grid") + rectangles.append((x, y, x + width, y + height, str(panel.get("title")))) + + for index, first in enumerate(rectangles): + for second in rectangles[index + 1 :]: + overlaps = first[0] < second[2] and second[0] < first[2] and first[1] < second[3] and second[1] < first[3] + if overlaps: + errors.append(f"{uid}: panels {first[4]!r} and {second[4]!r} overlap") + + +def validate_visualizations(dashboard: dict[str, Any], errors: list[str]) -> None: + """Keep current-state cards and historical graphs visually unambiguous.""" + uid = dashboard["uid"] + for panel in all_panels(dashboard.get("panels", [])): + panel_type = panel.get("type") + title = panel.get("title") + if panel_type == "stat": + if panel.get("options", {}).get("graphMode") != "none": + errors.append(f"{uid}: stat panel {title!r} must not render a sparkline") + if panel.get("datasource") == "Prometheus": + for target in panel.get("targets", []): + if target.get("expr") and target.get("instant") is not True: + errors.append( + f"{uid}: stat panel {title!r} must query the current instant" + ) + continue + if panel_type != "timeseries": + continue + + field_config = panel.get("fieldConfig", {}) + defaults = field_config.get("defaults", {}) + custom = defaults.get("custom", {}) + draw_style = custom.get("drawStyle") + if draw_style not in {"line", "bars"}: + errors.append(f"{uid}: timeseries panel {title!r} must explicitly use lines or bars") + continue + if custom.get("showPoints") not in {"auto", "always", "never"}: + errors.append(f"{uid}: timeseries panel {title!r} must explicitly configure points") + if draw_style == "line" and custom.get("lineWidth", 0) <= 0: + errors.append(f"{uid}: line panel {title!r} must have a visible line width") + if draw_style == "bars" and "barAlignment" not in custom: + errors.append(f"{uid}: bar panel {title!r} must explicitly align its bars") + + threshold_steps = defaults.get("thresholds", {}).get("steps", []) + if len(threshold_steps) > 1: + if custom.get("thresholdsStyle", {}).get("mode") != "line": + errors.append(f"{uid}: threshold panel {title!r} must show threshold lines") + allows_negative = ( + uid == "codeplace-public-endpoints" + and str(title).endswith("TLS Days Left") + ) + if defaults.get("min") != 0 and not allows_negative: + errors.append(f"{uid}: threshold panel {title!r} must start at zero") + expected_max = {"percentunit": 1, "percent": 100}.get(defaults.get("unit")) + if expected_max is not None and defaults.get("max") != expected_max: + errors.append( + f"{uid}: bounded threshold panel {title!r} must end at {expected_max}" + ) + + for override in field_config.get("overrides", []): + properties = { + prop.get("id"): prop.get("value") + for prop in override.get("properties", []) + } + if "unit" in properties and properties["unit"] != defaults.get("unit"): + if properties.get("custom.axisPlacement") != "right": + errors.append( + f"{uid}: mixed-unit panel {title!r} must place the override on the right axis" + ) + + +def validate_operational_semantics(dashboard: dict[str, Any], errors: list[str]) -> None: + """Guard the small set of panels whose colors directly communicate health.""" + uid = dashboard["uid"] + panels = list(all_panels(dashboard.get("panels", []))) + + if uid in {"codeplace-overview", "codeplace-ai-api"}: + expressions = [ + target.get("expr", "") + for panel in panels + for target in panel.get("targets", []) + if 'up{job="backend"' in target.get("expr", "") + ] + if len(expressions) != 1: + errors.append(f"{uid}: expected one backend health expression") + else: + expression = expressions[0] + for token in ( + "sum by (namespace)", + "kube_deployment_spec_replicas", + ">= bool", + "or on() vector(0)", + ): + if token not in expression: + errors.append(f"{uid}: backend health must enforce desired replicas with {token}") + + if uid == "codeplace-public-endpoints": + matching = [panel for panel in panels if panel.get("title", "").endswith("HTTP Status")] + if len(matching) != 1: + errors.append(f"{uid}: expected one HTTP status panel") + else: + steps = matching[0].get("fieldConfig", {}).get("defaults", {}).get( + "thresholds", {} + ).get("steps", []) + if steps != [ + {"color": "red", "value": None}, + {"color": "green", "value": 200}, + {"color": "red", "value": 201}, + ]: + errors.append(f"{uid}: HTTP status colors must accept only status 200") + + +def validate_shared_judge_outcome(dashboard: dict[str, Any], errors: list[str]) -> None: + """A Redis-backed counter is exposed identically by every backend replica.""" + expressions = [ + expression + for expression in all_query_text(dashboard) + if "codeplace_judge_task_outcome_total" in expression + ] + for expression in expressions: + if not expression.startswith("avg by (namespace, scope, status) (rate("): + errors.append( + f"{dashboard['uid']}: shared judge outcome rate must be averaged across backend replicas" + ) + + +def validate_alert_delivery_visibility(dashboard: dict[str, Any], errors: list[str]) -> None: + if dashboard["uid"] != "codeplace-monitoring-stack": + return + query_text = "\n".join(all_query_text(dashboard)) + for metric in ( + "alertmanager-kube-prometheus-stack-alertmanager-", + "prometheus_notifications_alertmanagers_discovered", + "alertmanager_config_last_reload_successful", + "alertmanager_notifications_failed_total", + ): + if metric not in query_text: + errors.append(f"{dashboard['uid']}: alert delivery visibility is missing {metric}") + + +def validate_scrape_target_coverage(dashboard: dict[str, Any], errors: list[str]) -> None: + """Fixed-replica targets must not look healthy when discovery is partial.""" + if dashboard["uid"] != "codeplace-monitoring-stack": + return + + panels = {panel.get("title"): panel for panel in all_panels(dashboard.get("panels", []))} + expected_panel = panels.get("Expected scrape targets", {}) + target_expressions = { + target.get("legendFormat"): target.get("expr", "") + for target in expected_panel.get("targets", []) + } + expected_counts = { + "backend dev": 'count(up{job="backend", namespace="code-place-dev"}) >= bool 2', + "backend prod": 'count(up{job="backend", namespace="code-place-prod"}) >= bool 6', + "postgres dev": 'count(up{job="postgres", namespace="code-place-dev"}) >= bool 3', + "postgres prod": 'count(up{job="postgres", namespace="code-place-prod"}) >= bool 3', + "redis dev": 'count(up{job="redis", namespace="code-place-dev"}) >= bool 6', + "redis prod": 'count(up{job="redis", namespace="code-place-prod"}) >= bool 6', + "OTel Collector": 'count(up{job="otel-collector", namespace="monitoring"}) >= bool 2', + "Blackbox exporter": 'count(up{job="blackbox-exporter", namespace="monitoring"}) >= bool 2', + } + for legend, expected_expression in expected_counts.items(): + expression = target_expressions.get(legend, "") + if expected_expression not in expression: + errors.append( + f"{dashboard['uid']}: {legend} must enforce expected target coverage" + ) + + coverage_panel = panels.get("Application telemetry coverage", {}) + coverage_expressions = { + target.get("legendFormat"): target.get("expr", "") + for target in coverage_panel.get("targets", []) + } + expected_denominators = { + "custom collectors dev (4 x 2 replicas)": "/ 8", + "custom collectors prod (4 x 6 replicas)": "/ 24", + "Sentinel checks dev (2 x 2 replicas)": "/ 4", + "Sentinel checks prod (2 x 6 replicas)": "/ 12", + } + for legend, denominator in expected_denominators.items(): + expression = coverage_expressions.get(legend, "") + if denominator not in expression or not expression.startswith("clamp_max("): + errors.append( + f"{dashboard['uid']}: {legend} must use the expected deployment coverage" + ) + if 'count(up{job="backend"' in expression: + errors.append( + f"{dashboard['uid']}: {legend} must not derive expected coverage from discovered targets" + ) + + +def validate_navigation(dashboard: dict[str, Any], errors: list[str]) -> None: + """Keep signal dashboards discoverable without losing variables or time range.""" + uid = dashboard["uid"] + if "codeplace" not in dashboard.get("tags", []): + errors.append(f"{uid}: dashboard must carry the codeplace navigation tag") + links = dashboard.get("links", []) + matching = [ + link + for link in links + if link.get("type") == "dashboards" + and link.get("title") == "CodePlace Dashboards" + ] + if len(matching) != 1: + errors.append(f"{uid}: expected one CodePlace dashboard navigation link") + return + + link = matching[0] + for key, expected in ( + ("asDropdown", True), + ("includeVars", True), + ("keepTime", True), + ("targetBlank", False), + ): + if link.get(key) is not expected: + errors.append(f"{uid}: navigation.{key} must be {expected!r}") + if "codeplace" not in link.get("tags", []): + errors.append(f"{uid}: navigation must filter on the codeplace tag") + + +def validate_dashboard(dashboard: dict[str, Any], errors: list[str]) -> None: + uid = dashboard.get("uid") + if not isinstance(uid, str) or not uid: + errors.append("dashboard is missing a stable uid") + return + if not dashboard.get("title") or not isinstance(dashboard.get("panels"), list): + errors.append(f"{uid}: title or panels are missing") + if dashboard.get("refresh") not in {"30s", "1m"}: + errors.append(f"{uid}: refresh must be 30s or 1m") + + known_scope = ENVIRONMENT_DASHBOARDS | PRODUCTION_ONLY_DASHBOARDS | GLOBAL_DASHBOARDS + if uid not in known_scope: + errors.append(f"{uid}: dashboard environment scope is not declared") + + if uid in ENVIRONMENT_DASHBOARDS: + validate_environment_variable(dashboard, errors) + + if uid in PRODUCTION_ONLY_DASHBOARDS: + variables = dashboard.get("templating", {}).get("list", []) + if any(variable.get("name") == "environment" for variable in variables): + errors.append(f"{uid}: production-only dashboard must not have an environment selector") + + if uid in GLOBAL_DASHBOARDS: + variables = dashboard.get("templating", {}).get("list", []) + if any(variable.get("name") == "environment" for variable in variables): + errors.append(f"{uid}: global dashboard must not have an environment selector") + + query_text = "\n".join(all_query_text(dashboard)) + if "$namespace" in query_text: + errors.append(f"{uid}: legacy $namespace query remains") + for pattern in MIXED_ENVIRONMENT_PATTERNS: + if pattern in query_text: + errors.append(f"{uid}: mixed-environment query remains: {pattern}") + if uid in PRODUCTION_ONLY_DASHBOARDS and "code-place-dev" in query_text: + errors.append(f"{uid}: production-only dashboard contains a dev query") + + validate_grid(dashboard, errors) + validate_visualizations(dashboard, errors) + validate_operational_semantics(dashboard, errors) + validate_shared_judge_outcome(dashboard, errors) + validate_alert_delivery_visibility(dashboard, errors) + validate_scrape_target_coverage(dashboard, errors) + validate_navigation(dashboard, errors) + + +def prometheus_rules(dashboards: Iterable[dict[str, Any]]) -> dict[str, Any]: + """Build a native Prometheus rule file so promtool can parse dashboard PromQL.""" + rules: list[dict[str, str]] = [] + for dashboard in dashboards: + for panel in all_panels(dashboard.get("panels", [])): + if panel.get("datasource") != "Prometheus": + continue + for target in panel.get("targets", []): + expression = target.get("expr") + if not isinstance(expression, str) or not expression: + continue + rules.append({ + "record": f"dashboard_contract:expression_{len(rules)}", + "expr": expression.replace("$environment", "prod"), + }) + return {"groups": [{"name": "dashboard.contract", "rules": rules}]} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--prometheus-rules-output", + type=Path, + help="write dashboard PromQL as a native rule file for promtool", + ) + args = parser.parse_args() + + errors: list[str] = [] + dashboard_count = 0 + dashboards: list[dict[str, Any]] = [] + seen_uids: set[str] = set() + for path in sorted(ROOT.glob("grafana-dashboard-*.yaml")): + try: + documents = list(embedded_json_documents(path)) + except ValueError as exc: + errors.append(str(exc)) + continue + if not documents: + errors.append(f"{path.name}: no embedded dashboard JSON found") + for dashboard in documents: + dashboard_count += 1 + dashboards.append(dashboard) + uid = dashboard.get("uid") + if uid in seen_uids: + errors.append(f"duplicate dashboard uid: {uid}") + seen_uids.add(uid) + validate_dashboard(dashboard, errors) + + if dashboard_count != 11: + errors.append(f"expected 11 dashboards, found {dashboard_count}") + if errors: + print("Monitoring dashboard validation failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + + if args.prometheus_rules_output: + rules = prometheus_rules(dashboards) + args.prometheus_rules_output.write_text( + json.dumps(rules, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + + print( + f"Validated {dashboard_count} dashboards and their " + "environment/grid/visualization/navigation contracts." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/kubernetes/monitoring/verify_live.py b/kubernetes/monitoring/verify_live.py new file mode 100644 index 00000000..7451a04a --- /dev/null +++ b/kubernetes/monitoring/verify_live.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +"""Fail fast when the live monitoring stack no longer matches its target contract.""" + +from __future__ import annotations + +import argparse +import json +import sys +import urllib.parse +import urllib.request +from dataclasses import dataclass +from typing import Callable + + +@dataclass(frozen=True) +class Check: + name: str + query: str + predicate: Callable[[float], bool] + expectation: str + + +ONE = lambda value: value == 1 # noqa: E731 +POSITIVE = lambda value: value > 0 # noqa: E731 +AT_LEAST_TWO = lambda value: value >= 2 # noqa: E731 +AT_LEAST_THREE = lambda value: value >= 3 # noqa: E731 +AT_LEAST_SIX = lambda value: value >= 6 # noqa: E731 + +CHECKS = ( + Check("backend dev scrape", 'min(up{job="backend",namespace="code-place-dev"})', ONE, "1"), + Check( + "backend dev target coverage", + 'count(up{job="backend",namespace="code-place-dev"})', + AT_LEAST_TWO, + ">= 2 targets", + ), + Check("backend prod scrape", 'min(up{job="backend",namespace="code-place-prod"})', ONE, "1"), + Check( + "backend prod target coverage", + 'count(up{job="backend",namespace="code-place-prod"})', + AT_LEAST_SIX, + ">= 6 targets", + ), + Check( + "PostgreSQL dev collector", + 'min(cnpg_collector_up{namespace="code-place-dev",cluster="postgres",job="postgres"})', + ONE, + "1", + ), + Check( + "PostgreSQL dev target coverage", + 'count(cnpg_collector_up{namespace="code-place-dev",cluster="postgres",job="postgres"})', + AT_LEAST_THREE, + ">= 3 targets", + ), + Check( + "PostgreSQL dev diagnostic metric families", + '(count(cnpg_collector_last_collection_error{namespace="code-place-dev",cluster="postgres",job="postgres"}) > bool 0)' + ' * (count(cnpg_collector_nodes_used{namespace="code-place-dev",cluster="postgres",job="postgres"}) > bool 0)' + ' * (count(cnpg_collector_wal_bytes{namespace="code-place-dev",cluster="postgres",job="postgres"}) > bool 0)', + ONE, + "collector error, HA, and WAL metrics present", + ), + Check( + "PostgreSQL prod collector", + 'min(cnpg_collector_up{namespace="code-place-prod",cluster="postgres",job="postgres"})', + ONE, + "1", + ), + Check( + "PostgreSQL prod target coverage", + 'count(cnpg_collector_up{namespace="code-place-prod",cluster="postgres",job="postgres"})', + AT_LEAST_THREE, + ">= 3 targets", + ), + Check( + "PostgreSQL prod diagnostic metric families", + '(count(cnpg_collector_last_collection_error{namespace="code-place-prod",cluster="postgres",job="postgres"}) > bool 0)' + ' * (count(cnpg_collector_nodes_used{namespace="code-place-prod",cluster="postgres",job="postgres"}) > bool 0)' + ' * (count(cnpg_collector_wal_bytes{namespace="code-place-prod",cluster="postgres",job="postgres"}) > bool 0)', + ONE, + "collector error, HA, and WAL metrics present", + ), + Check("Redis dev exporter", 'min(redis_up{namespace="code-place-dev",job="redis"})', ONE, "1"), + Check( + "Redis dev target coverage", + 'count(redis_up{namespace="code-place-dev",job="redis"})', + AT_LEAST_SIX, + ">= 6 targets", + ), + Check("Redis prod exporter", 'min(redis_up{namespace="code-place-prod",job="redis"})', ONE, "1"), + Check( + "Redis prod target coverage", + 'count(redis_up{namespace="code-place-prod",job="redis"})', + AT_LEAST_SIX, + ">= 6 targets", + ), + Check("Traefik scrape", 'min(up{job="traefik",namespace="kube-system"})', ONE, "1"), + Check( + "Traefik request metrics", + 'count(traefik_service_requests_total{job="traefik",namespace="kube-system"})', + POSITIVE, + "> 0 series", + ), + Check("Alloy scrape", 'min(up{namespace="monitoring",job=~".*alloy.*"})', ONE, "1"), + Check( + "Alloy node coverage", + 'min(kube_daemonset_status_number_available{namespace="monitoring",daemonset="alloy"}' + ' == bool on(namespace,daemonset) ' + 'kube_daemonset_status_desired_number_scheduled{namespace="monitoring",daemonset="alloy"})', + ONE, + "1", + ), + Check( + "Alloy scheduled nodes", + 'kube_daemonset_status_desired_number_scheduled{namespace="monitoring",daemonset="alloy"}', + POSITIVE, + "> 0 nodes", + ), + Check("Loki scrape", 'min(up{namespace="monitoring",job=~".*loki$"})', ONE, "1"), + Check( + "Longhorn scrape", + 'min(up{namespace="longhorn-system",endpoint="manager"})', + ONE, + "1", + ), + Check( + "Kubernetes event exporter scrape", + 'min(up{namespace="monitoring",job="kubernetes-event-exporter"})', + ONE, + "1", + ), + Check( + "kube-state-metrics scrape", + 'min(up{namespace="monitoring",service=~".*kube-state-metrics.*"})', + ONE, + "1", + ), + Check("OTel collector scrape", 'min(up{namespace="monitoring",job="otel-collector"})', ONE, "1"), + Check( + "OTel collector target coverage", + 'count(up{namespace="monitoring",job="otel-collector"})', + AT_LEAST_TWO, + ">= 2 targets", + ), + Check( + "OTel span metric family", + 'count(otelcol_receiver_accepted_spans_total{namespace="monitoring"})', + POSITIVE, + "> 0 series", + ), + Check("Tempo scrape", 'min(up{namespace="monitoring",job="tempo"})', ONE, "1"), + Check("vLLM scrape", 'min(up{namespace="code-place-prod",job="vllm"})', ONE, "1"), + Check( + "vLLM request metric family", + 'count({__name__="vllm:num_requests_running",namespace="code-place-prod"})', + POSITIVE, + "> 0 series", + ), + Check("DCGM scrape", 'min(up{namespace="monitoring",job="dcgm-exporter"})', ONE, "1"), + Check( + "DCGM GPU metric family", + 'count(DCGM_FI_DEV_GPU_UTIL{namespace="monitoring"})', + POSITIVE, + "> 0 series", + ), + Check( + "Blackbox exporter scrape", + 'min(up{namespace="monitoring",job="blackbox-exporter"})', + ONE, + "1", + ), + Check( + "Blackbox exporter target coverage", + 'count(up{namespace="monitoring",job="blackbox-exporter"})', + AT_LEAST_TWO, + ">= 2 targets", + ), + Check( + "Prometheus ready replicas", + 'sum(kube_pod_status_ready{namespace="monitoring",condition="true",' + 'pod=~"prometheus-kube-prometheus-stack-prometheus-.*"})', + AT_LEAST_TWO, + ">= 2", + ), + Check( + "Alertmanager ready replicas", + 'sum(kube_pod_status_ready{namespace="monitoring",condition="true",' + 'pod=~"alertmanager-kube-prometheus-stack-alertmanager-.*"})', + AT_LEAST_TWO, + ">= 2", + ), + Check( + "Prometheus Alertmanager discovery", + 'min(prometheus_notifications_alertmanagers_discovered{' + 'namespace="monitoring",job="kube-prometheus-stack-prometheus"})', + POSITIVE, + ">= 1 per Prometheus replica", + ), + Check( + "Alertmanager configuration reload", + 'min(alertmanager_config_last_reload_successful{' + 'namespace="monitoring",job="kube-prometheus-stack-alertmanager"})', + ONE, + "1", + ), + Check( + "dev frontend public probe", + 'min(probe_success{environment="dev",namespace="code-place-dev",service="frontend",probe_type="public-http"})', + ONE, + "1", + ), + Check( + "dev hub-auth public probe", + 'min(probe_success{environment="dev",namespace="code-place-dev",service="hub-auth",probe_type="public-http"})', + ONE, + "1", + ), + Check( + "prod frontend public probe", + 'min(probe_success{environment="prod",namespace="code-place-prod",service="frontend",probe_type="public-http"})', + ONE, + "1", + ), + Check( + "prod hub-auth public probe", + 'min(probe_success{environment="prod",namespace="code-place-prod",service="hub-auth",probe_type="public-http"})', + ONE, + "1", + ), + Check( + "Grafana public probe", + 'min(probe_success{environment="monitoring",namespace="monitoring",service="grafana",probe_type="public-http"})', + ONE, + "1", + ), + Check("dev custom collectors", 'min(codeplace_collector_success{namespace="code-place-dev"})', ONE, "1"), + Check( + "dev collector replica coverage", + 'count(codeplace_collector_success{namespace="code-place-dev"}) >= bool 8', + ONE, + "4 collector series x 2 expected backend targets", + ), + Check("dev Sentinel health", 'min(codeplace_redis_sentinel_health{namespace="code-place-dev"})', ONE, "1"), + Check( + "dev Sentinel replica coverage", + 'count(codeplace_redis_sentinel_health{namespace="code-place-dev"}) >= bool 4', + ONE, + "2 Sentinel check series x 2 expected backend targets", + ), + Check("prod custom collectors", 'min(codeplace_collector_success{namespace="code-place-prod"})', ONE, "1"), + Check( + "prod collector replica coverage", + 'count(codeplace_collector_success{namespace="code-place-prod"}) >= bool 24', + ONE, + "4 collector series x 6 expected backend targets", + ), + Check("prod Sentinel health", 'min(codeplace_redis_sentinel_health{namespace="code-place-prod"})', ONE, "1"), + Check( + "prod Sentinel replica coverage", + 'count(codeplace_redis_sentinel_health{namespace="code-place-prod"}) >= bool 12', + ONE, + "2 Sentinel check series x 6 expected backend targets", + ), +) + + +def query(prometheus_url: str, expression: str, timeout: float) -> float: + params = urllib.parse.urlencode({"query": expression}) + url = f"{prometheus_url.rstrip('/')}/api/v1/query?{params}" + with urllib.request.urlopen(url, timeout=timeout) as response: + payload = json.load(response) + if payload.get("status") != "success": + raise RuntimeError(payload.get("error") or "Prometheus returned a failed response") + result = payload.get("data", {}).get("result", []) + if len(result) != 1: + raise RuntimeError(f"expected one result series, got {len(result)}") + return float(result[0]["value"][1]) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--prometheus-url", default="http://127.0.0.1:9090") + parser.add_argument("--timeout", type=float, default=5.0) + args = parser.parse_args() + + failures = 0 + for check in CHECKS: + try: + value = query(args.prometheus_url, check.query, args.timeout) + except Exception as exc: # One failed query must not hide the remaining contract failures. + failures += 1 + print(f"FAIL {check.name}: {exc}") + continue + if check.predicate(value): + print(f"PASS {check.name}: {value:g}") + else: + failures += 1 + print(f"FAIL {check.name}: got {value:g}, expected {check.expectation}") + + if failures: + print(f"\n{failures} live monitoring contract check(s) failed.", file=sys.stderr) + return 1 + print("\nAll live monitoring contracts passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/kubernetes/overlays/dev/backend-deployment-patch.yaml b/kubernetes/overlays/dev/backend-deployment-patch.yaml index 9a69faed..fb6f0ad3 100644 --- a/kubernetes/overlays/dev/backend-deployment-patch.yaml +++ b/kubernetes/overlays/dev/backend-deployment-patch.yaml @@ -19,3 +19,5 @@ spec: value: "dev" - name: OTEL_ENABLED value: "1" + - name: OTEL_DEPLOYMENT_ENVIRONMENT + value: "dev" diff --git a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml index 7967e6a7..4c12d750 100644 --- a/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-beat-deployment-patch.yaml @@ -22,3 +22,5 @@ spec: value: "dev" - name: OTEL_ENABLED value: "1" + - name: OTEL_DEPLOYMENT_ENVIRONMENT + value: "dev" diff --git a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml index bc04ae64..2e957e49 100644 --- a/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/dev/celery-worker-deployment-patch.yaml @@ -25,3 +25,5 @@ spec: value: "dev" - name: OTEL_ENABLED value: "1" + - name: OTEL_DEPLOYMENT_ENVIRONMENT + value: "dev" diff --git a/kubernetes/overlays/dev/kustomization.yaml b/kubernetes/overlays/dev/kustomization.yaml index dcd5edb7..21b1f701 100644 --- a/kubernetes/overlays/dev/kustomization.yaml +++ b/kubernetes/overlays/dev/kustomization.yaml @@ -21,13 +21,13 @@ patches: images: - name: backend newName: harbor.code-place-dev.site/code-place-dev/backend - newTag: cdec3c881dd6acd07ef0a430ac29b64fcd7d6ff6-dev + newTag: 644531052f1ef464fe902ce0522fcffeb2a76ee5-dev - name: frontend newName: harbor.code-place-dev.site/code-place-dev/frontend - newTag: f76f57f30f878733b0989914b6d29b5b4ca7f1c4-dev + newTag: 644531052f1ef464fe902ce0522fcffeb2a76ee5-dev - name: judge-server newName: harbor.code-place-dev.site/code-place-dev/judge-server newTag: 1.0.3 - name: hub-auth newName: harbor.code-place-dev.site/code-place-dev/hub-auth - newTag: e000ffc019bcf819f31b2009a813ab43002187d6-dev + newTag: 644531052f1ef464fe902ce0522fcffeb2a76ee5-dev diff --git a/kubernetes/overlays/prod/backend-deployment-patch.yaml b/kubernetes/overlays/prod/backend-deployment-patch.yaml index 9d3a493c..3caa7251 100644 --- a/kubernetes/overlays/prod/backend-deployment-patch.yaml +++ b/kubernetes/overlays/prod/backend-deployment-patch.yaml @@ -15,6 +15,8 @@ spec: - name: backend imagePullPolicy: IfNotPresent env: + - name: OJ_ENV + value: "production" - name: CSRF_TRUSTED_ORIGINS value: "https://code.pusan.ac.kr" - name: SENTRY_DSN_BACKEND @@ -24,3 +26,7 @@ spec: key: SENTRY_DSN_BACKEND - name: SENTRY_ENVIRONMENT value: "prod" + - name: OTEL_ENABLED + value: "1" + - name: OTEL_DEPLOYMENT_ENVIRONMENT + value: "prod" diff --git a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml index 64e69321..4558f1f7 100644 --- a/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-beat-deployment-patch.yaml @@ -13,6 +13,8 @@ spec: containers: - name: celery-beat env: + - name: OJ_ENV + value: "production" - name: SENTRY_DSN_BACKEND valueFrom: secretKeyRef: @@ -20,3 +22,7 @@ spec: key: SENTRY_DSN_BACKEND - name: SENTRY_ENVIRONMENT value: "prod" + - name: OTEL_ENABLED + value: "1" + - name: OTEL_DEPLOYMENT_ENVIRONMENT + value: "prod" diff --git a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml index b6bfa3ee..6f80ddf0 100644 --- a/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml +++ b/kubernetes/overlays/prod/celery-worker-deployment-patch.yaml @@ -16,6 +16,8 @@ spec: containers: - name: celery-worker env: + - name: OJ_ENV + value: "production" - name: SENTRY_DSN_BACKEND valueFrom: secretKeyRef: @@ -23,3 +25,7 @@ spec: key: SENTRY_DSN_BACKEND - name: SENTRY_ENVIRONMENT value: "prod" + - name: OTEL_ENABLED + value: "1" + - name: OTEL_DEPLOYMENT_ENVIRONMENT + value: "prod" diff --git a/kubernetes/setup/longhorn/README.md b/kubernetes/setup/longhorn/README.md index 922aceee..dd5e8f6f 100644 --- a/kubernetes/setup/longhorn/README.md +++ b/kubernetes/setup/longhorn/README.md @@ -95,12 +95,7 @@ kubectl get pods -n longhorn-system -w kube-prometheus-stack이 설치된 클러스터에서는 CodePlace monitoring kustomization이 Longhorn manager metric을 scrape합니다. -```sh -kubectl apply -k kubernetes/monitoring -kubectl -n monitoring get servicemonitor longhorn -``` - -Prometheus target에서 `longhorn` scrape가 healthy인지 확인하고, Grafana의 `CodePlace Storage` dashboard에서 다음 상태를 확인합니다. +Monitoring 배포 절차는 `kubernetes/monitoring/logs/README.md`를 따릅니다. 적용 후 `kubernetes/monitoring/verify_live.py`로 Longhorn manager scrape를 확인합니다. Grafana의 `CodePlace Storage` dashboard에서는 선택한 dev/prod 환경의 PVC 상태와 cluster-global Longhorn 상태를 분리해 다음 항목을 확인합니다. - Longhorn manager ready count. - faulted/degraded/read-only volume count.