#540 Kubernetes 관측성 개선

.github/workflows/ci2develop.yml

@@ -136,6 +136,15 @@ jobs:
         with:
           cmd: yq -i '(.images[] | select(.name == "backend").newTag) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/kustomization.yaml
 
+      - name: Update Backend Sentry Release
+        if: needs.ci-backend-dev.result == 'success'
+        uses: mikefarah/yq@master
+        with:
+          cmd: |
+            yq -i '(.spec.template.spec.containers[] | select(.name == "backend").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/backend-deployment-patch.yaml
+            yq -i '(.spec.template.spec.containers[] | select(.name == "celery-worker").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/celery-worker-deployment-patch.yaml
+            yq -i '(.spec.template.spec.containers[] | select(.name == "celery-beat").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-dev"' kubernetes/overlays/dev/celery-beat-deployment-patch.yaml
+
       - name: Update Frontend Image Tag
         if: needs.ci-frontend-dev.result == 'success'
         uses: mikefarah/yq@master
@@ -152,7 +161,10 @@ jobs:
         run: |
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
-          git add kubernetes/overlays/dev/kustomization.yaml
+          git add kubernetes/overlays/dev/kustomization.yaml \
+            kubernetes/overlays/dev/backend-deployment-patch.yaml \
+            kubernetes/overlays/dev/celery-worker-deployment-patch.yaml \
+            kubernetes/overlays/dev/celery-beat-deployment-patch.yaml
 
           # 변경사항이 없으면 종료
           if git diff-index --quiet HEAD; then

.github/workflows/ci2production-and-release.yml

@@ -165,6 +165,15 @@ jobs:
         with:
           cmd: yq -i '(.images[] | select(.name == "backend").newTag) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/kustomization.yaml
 
+      - name: Update Backend Sentry Release
+        if: needs.ci-backend.result == 'success'
+        uses: mikefarah/yq@master
+        with:
+          cmd: |
+            yq -i '(.spec.template.spec.containers[] | select(.name == "backend").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/backend-deployment-patch.yaml
+            yq -i '(.spec.template.spec.containers[] | select(.name == "celery-worker").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/celery-worker-deployment-patch.yaml
+            yq -i '(.spec.template.spec.containers[] | select(.name == "celery-beat").env[] | select(.name == "SENTRY_RELEASE").value) = "${{ github.sha }}-prod"' kubernetes/overlays/prod/celery-beat-deployment-patch.yaml
+
       - name: Update Frontend Image Tag
         if: needs.ci-frontend.result == 'success'
         uses: mikefarah/yq@master
@@ -181,7 +190,10 @@ jobs:
         run: |
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
-          git add kubernetes/overlays/prod/kustomization.yaml
+          git add kubernetes/overlays/prod/kustomization.yaml \
+            kubernetes/overlays/prod/backend-deployment-patch.yaml \
+            kubernetes/overlays/prod/celery-worker-deployment-patch.yaml \
+            kubernetes/overlays/prod/celery-beat-deployment-patch.yaml
 
           # 변경사항이 없으면 종료
           if git diff-index --quiet HEAD; then

backend/account/tests.py

@@ -8,7 +8,7 @@
 from django.contrib import auth
 from django.db import DatabaseError
 from django.http.response import JsonResponse
-from django.test import RequestFactory, TestCase
+from django.test import RequestFactory, SimpleTestCase, TestCase
 from django.utils import decorators
 from django.utils.timezone import now
 from utils.api.tests import APIClient, APITestCase
@@ -18,9 +18,35 @@
 
 from .models import AdminType, ProblemPermission, User
 from .decorators import scheduler_only
+from .middleware import RequestIDMiddleware
 from .tasks import calculate_user_score_basis, calculate_user_score_fluctuation
 
 
+class RequestIDMiddlewareTest(SimpleTestCase):
+
+    def setUp(self):
+        self.factory = RequestFactory()
+
+    def test_generates_request_id_when_header_is_missing(self):
+        middleware = RequestIDMiddleware(lambda request: JsonResponse({"ok": True}))
+        request = self.factory.get("/")
+
+        response = middleware(request)
+
+        self.assertEqual(len(request.request_id), 32)
+        self.assertEqual(response["X-Request-ID"], request.request_id)
+
+    def test_accepts_existing_request_id_and_truncates_it(self):
+        request_id = "a" * 150
+        middleware = RequestIDMiddleware(lambda request: JsonResponse({"ok": True}))
+        request = self.factory.get("/", HTTP_X_REQUEST_ID=request_id)
+
+        response = middleware(request)
+
+        self.assertEqual(request.request_id, "a" * 128)
+        self.assertEqual(response["X-Request-ID"], request.request_id)
+
+
 class PermissionDecoratorTest(APITestCase):
     """
     데코레이터 테스트

backend/judge/dispatcher.py

@@ -10,7 +10,6 @@
 from django.db import transaction, IntegrityError
 from django.db.models import F
 from django.http import HttpResponseNotFound
-from opentelemetry import trace
 
 from account.models import User, UserScore, UserSolved
 from conf.models import JudgeServer
@@ -21,9 +20,10 @@
 from submission.models import JudgeStatus, Submission
 from utils.cache import cache
 from utils.constants import CacheKey, ProblemScore, ProblemField, Tier
+from utils.observability_tracing import get_tracer
 
 logger = logging.getLogger(__name__)
-tracer = trace.get_tracer(__name__)
+tracer = get_tracer(__name__)
 
 
 # 继续处理在队列中的问题

backend/judge/tasks.py

@@ -1,12 +1,12 @@
 import celery
-from opentelemetry import trace
 
 from account.models import User
 from utils.shortcuts import CELERY_TASK_ARGS
+from utils.observability_tracing import get_tracer
 from submission.models import Submission
 from judge.dispatcher import JudgeDispatcher
 
-tracer = trace.get_tracer(__name__)
+tracer = get_tracer(__name__)
 
 
 @celery.shared_task(**CELERY_TASK_ARGS())

backend/oj/celery.py

@@ -1,6 +1,7 @@
 import os
 import celery
 
+from utils.celery_observability import configure_celery_observability
 from utils.observability_tracing import configure_opentelemetry
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'oj.settings')
@@ -9,3 +10,4 @@
 app = celery.Celery('scheduler')
 app.config_from_object('django.conf:settings', namespace='CELERY')
 app.autodiscover_tasks()
+configure_celery_observability()

backend/oj/settings.py

@@ -22,16 +22,71 @@
 else:
     from .dev_settings import *
 
+SENTRY_SENSITIVE_EXACT_FIELDS = {
+    "authorization",
+    "cookie",
+    "password",
+    "token",
+    "secret",
+    "code",
+    "src",
+}
+SENTRY_SENSITIVE_SUBSTRINGS = {
+    "authorization",
+    "cookie",
+    "password",
+    "token",
+    "secret",
+    "source_code",
+    "sourcecode",
+    "spj_code",
+    "src",
+}
+
+
+def _is_sentry_sensitive_key(key):
+    normalized = str(key).lower()
+    return (
+        normalized in SENTRY_SENSITIVE_EXACT_FIELDS
+        or any(field in normalized for field in SENTRY_SENSITIVE_SUBSTRINGS)
+    )
+
+
+def _redact_sentry_value(value):
+    if isinstance(value, dict):
+        return {
+            key: "[REDACTED]" if _is_sentry_sensitive_key(key) else _redact_sentry_value(item)
+            for key, item in value.items()
+        }
+    if isinstance(value, list):
+        return [_redact_sentry_value(item) for item in value]
+    return value
+
+
+def _before_send_sentry(event, hint):
+    event = _redact_sentry_value(event)
+    request = event.get("request") or {}
+    request_id = (request.get("headers") or {}).get("X-Request-ID")
+    if request_id:
+        event.setdefault("tags", {})["request_id"] = request_id
+    return event
+
+
 SENTRY_DSN = get_env(
     "SENTRY_DSN",
     "",
 )
 if SENTRY_DSN:
-    sentry_sdk.init(
-        dsn=SENTRY_DSN,
-        send_default_pii=True,
-        environment=get_env("SENTRY_ENVIRONMENT", get_env("OJ_ENV", "dev")),
-    )
+    SENTRY_OPTIONS = {
+        "dsn": SENTRY_DSN,
+        "send_default_pii": False,
+        "environment": get_env("SENTRY_ENVIRONMENT", get_env("OJ_ENV", "dev")),
+        "before_send": _before_send_sentry,
+    }
+    SENTRY_RELEASE = get_env("SENTRY_RELEASE", "")
+    if SENTRY_RELEASE:
+        SENTRY_OPTIONS["release"] = SENTRY_RELEASE
+    sentry_sdk.init(**SENTRY_OPTIONS)
 
 with open(os.path.join(DATA_DIR, "config", "secret.key"), "r") as f:
     SECRET_KEY = f.read()

backend/utils/celery_observability.py

@@ -0,0 +1,97 @@
+import logging
+import time
+
+from celery import signals
+
+from utils.cache import cache
+
+logger = logging.getLogger("codeplace.celery_task")
+
+CELERY_TASK_TOTAL_KEY = "observability:celery:task_total"
+CELERY_TASK_RUNTIME_BUCKET_KEY = "observability:celery:task_runtime_bucket"
+CELERY_TASK_RUNTIME_COUNT_KEY = "observability:celery:task_runtime_count"
+CELERY_TASK_RUNTIME_SUM_KEY = "observability:celery:task_runtime_sum"
+CELERY_TASK_RUNTIME_LAST_KEY = "observability:celery:task_runtime_last"
+CELERY_TASK_RUNTIME_BUCKETS = (1, 3, 5, 10, 30, 60, 120, 300, 600, float("inf"))
+
+_task_start_times = {}
+_configured = False
+
+
+def configure_celery_observability():
+    global _configured
+    if _configured:
+        return
+    signals.task_prerun.connect(_on_task_prerun, weak=False)
+    signals.task_postrun.connect(_on_task_postrun, weak=False)
+    signals.task_retry.connect(_on_task_retry, weak=False)
+    _configured = True
+
+
+def _task_name(task):
+    return getattr(task, "name", None) or getattr(task, "__name__", None) or "unknown"
+
+
+def _field(*parts):
+    return "|".join(str(part) for part in parts)
+
+
+def _hincrbyfloat(key, field, amount):
+    client = cache.client.get_client(write=True)
+    return client.hincrbyfloat(key, field, amount)
+
+
+def _hsetfloat(key, field, value):
+    client = cache.client.get_client(write=True)
+    return client.hset(key, field, value)
+
+
+def _record_task(task_name, status, duration_seconds=None):
+    status = (status or "unknown").lower()
+    try:
+        cache.hincrby(CELERY_TASK_TOTAL_KEY, _field(task_name, status), 1)
+        if duration_seconds is not None:
+            _hincrbyfloat(CELERY_TASK_RUNTIME_SUM_KEY, task_name, duration_seconds)
+            cache.hincrby(CELERY_TASK_RUNTIME_COUNT_KEY, task_name, 1)
+            _hsetfloat(CELERY_TASK_RUNTIME_LAST_KEY, _field(task_name, status), duration_seconds)
+            for bucket in CELERY_TASK_RUNTIME_BUCKETS:
+                if duration_seconds <= bucket:
+                    bucket_label = "+Inf" if bucket == float("inf") else str(bucket)
+                    cache.hincrby(CELERY_TASK_RUNTIME_BUCKET_KEY, _field(task_name, bucket_label), 1)
+    except Exception:
+        logger.exception("failed to record celery task metric", extra={
+            "task_name": task_name,
+            "task_status": status,
+        })
+
+
+def _on_task_prerun(task_id=None, task=None, **kwargs):
+    if task_id:
+        _task_start_times[task_id] = time.monotonic()
+
+
+def _on_task_postrun(task_id=None, task=None, state=None, **kwargs):
+    task_name = _task_name(task)
+    started_at = _task_start_times.pop(task_id, None)
+    duration_seconds = None
+    if started_at is not None:
+        duration_seconds = max(time.monotonic() - started_at, 0)
+    _record_task(task_name, state, duration_seconds)
+    logger.info("celery task completed", extra={
+        "task_id": task_id,
+        "task_name": task_name,
+        "task_status": (state or "unknown").lower(),
+        "duration_ms": round(duration_seconds * 1000, 2) if duration_seconds is not None else None,
+    })
+
+
+def _on_task_retry(request=None, reason=None, **kwargs):
+    task_name = getattr(request, "task", None) or "unknown"
+    task_id = getattr(request, "id", None)
+    _record_task(task_name, "retry")
+    logger.warning("celery task retry", extra={
+        "task_id": task_id,
+        "task_name": task_name,
+        "task_status": "retry",
+        "retry_reason": str(reason) if reason else None,
+    })

backend/utils/json_logging.py

@@ -3,6 +3,7 @@
 from datetime import datetime, timezone
 
 from utils.observability_context import get_request_id
+from utils.observability_tracing import get_current_trace_context
 
 
 SENSITIVE_EXACT_FIELDS = {
@@ -37,6 +38,7 @@ def format(self, record):
             "message": record.getMessage(),
             "request_id": getattr(record, "request_id", None) or get_request_id(),
         }
+        payload.update(get_current_trace_context())
         for key in (
                 "method",
                 "path",
@@ -45,7 +47,10 @@ def format(self, record):
                 "user_id",
                 "remote_addr",
                 "task_id",
+                "task_name",
+                "task_status",
                 "submission_id",
+                "retry_reason",
         ):
             value = getattr(record, key, None)
             if value is not None: