관측성 selector 검증 보강

Author: Neibce

docs/content/developer/infra/observability-improvement-plan/_index.md

@@ -203,9 +203,11 @@ P1은 `group_wait=30s`, `repeat_interval=1h`로 전달합니다.
   - Grafana dashboard JSON parse와 dashboard shape check: uid/title/refresh/panel/target expr.
   - PrometheusRule shape check: P0/P1 interval, alert priority/severity, summary/description, duplicate alert namespace label.
   - AlertmanagerConfig shape check: groupBy, P0/P1 Discord receivers, webhook Secret reference.
+  - kube-prometheus-stack values shape check: Prometheus selector policy, AlertmanagerConfig selector, Loki datasource, dashboard sidecar label.
+  - scrape resource shape check: ServiceMonitor/PodMonitor selector label, scrape path/port/interval.
   - Monitoring kustomization shape check: email fallback example이 기본 적용에 섞이지 않는지 확인.
 - Live cluster smoke check: `bash kubernetes/monitoring/smoke-check.sh`
-  - Prometheus Operator CRD, monitoring namespace resource, webhook Secret, kube-prometheus-stack Pod readiness, optional Loki/Alloy, app namespace backend readiness를 읽기 전용으로 확인합니다.
+  - Prometheus Operator CRD, monitoring namespace resource, selector label, webhook Secret, kube-prometheus-stack Pod readiness, optional Loki/Alloy, app namespace backend service/port/readiness를 읽기 전용으로 확인합니다.
 
 `promtool`이 있는 환경에서는 `promtool check rules kubernetes/monitoring/prometheus-rules.yaml`도 실행합니다.
 `helm`이 있는 환경에서는 Loki/Alloy values를 `helm template`로 렌더링 검증합니다.

kubernetes/monitoring/smoke-check.sh

@@ -21,6 +21,22 @@ resource_exists() {
   kubectl -n "$1" get "$2" "$3" >/dev/null 2>&1
 }
 
+resource_count() {
+  kubectl -n "$1" get "$2" -l "$3" -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | wc -w | tr -d ' '
+}
+
+require_label() {
+  local namespace="$1"
+  local resource="$2"
+  local name="$3"
+  local key="$4"
+  local expected="$5"
+  local actual
+  actual="$(kubectl -n "$namespace" get "$resource" "$name" -o "jsonpath={.metadata.labels['${key//./\\.}']}")"
+  [ "$actual" = "$expected" ] || fail "$resource/$name label $key expected=$expected actual=$actual"
+  ok "$resource/$name label $key=$expected"
+}
+
 require_cmd kubectl
 
 echo "==> checking Prometheus Operator CRDs"
@@ -49,6 +65,14 @@ for resource in \
   ok "$resource"
 done
 
+require_label "$NAMESPACE" prometheusrule codeplace-fast-alerts release kube-prometheus-stack
+require_label "$NAMESPACE" servicemonitor backend release kube-prometheus-stack
+require_label "$NAMESPACE" podmonitor postgres release kube-prometheus-stack
+require_label "$NAMESPACE" podmonitor redis release kube-prometheus-stack
+require_label "$NAMESPACE" alertmanagerconfig codeplace-alert-routing alertmanagerConfig codeplace
+require_label "$NAMESPACE" configmap grafana-dashboard-codeplace grafana_dashboard 1
+require_label "$NAMESPACE" configmap grafana-dashboard-codeplace-logs grafana_dashboard 1
+
 if resource_exists "$NAMESPACE" secret alertmanager-contact-points; then
   kubectl -n "$NAMESPACE" get secret alertmanager-contact-points \
     -o jsonpath='{.data.webhook-url}' | grep -q . \
@@ -69,7 +93,7 @@ for selector in \
 done
 
 echo "==> checking optional logs stack"
-if kubectl -n "$NAMESPACE" get pod -l app.kubernetes.io/name=loki >/dev/null 2>&1; then
+if [ "$(resource_count "$NAMESPACE" pod app.kubernetes.io/name=loki)" -gt 0 ]; then
   kubectl -n "$NAMESPACE" wait --for=condition=Ready pod -l app.kubernetes.io/name=loki --timeout=30s
   ok "loki pods ready"
 else
@@ -91,6 +115,9 @@ for app_namespace in $APP_NAMESPACES; do
   ok "namespace $app_namespace"
   kubectl -n "$app_namespace" get svc backend >/dev/null
   ok "$app_namespace service/backend"
+  kubectl -n "$app_namespace" get svc backend -o jsonpath='{.spec.ports[*].name}' | grep -qw api \
+    || fail "$app_namespace service/backend missing api port"
+  ok "$app_namespace service/backend api port"
   kubectl -n "$app_namespace" wait --for=condition=Ready pod -l app=backend --timeout=30s
   ok "$app_namespace backend pods ready"
 done

kubernetes/monitoring/validate.sh

@@ -119,6 +119,51 @@ for receiver_name in ("p0-discord", "p1-discord"):
         raise SystemExit(f"{receiver_name} must use alertmanager-contact-points/webhook-url")
 print("ALERTMANAGER SHAPE OK")
 
+stack_values = yaml.safe_load((root / "kube-prometheus-stack-values.yaml").read_text())
+prometheus_spec = stack_values.get("prometheus", {}).get("prometheusSpec", {})
+for key in (
+    "serviceMonitorSelectorNilUsesHelmValues",
+    "podMonitorSelectorNilUsesHelmValues",
+    "ruleSelectorNilUsesHelmValues",
+):
+    if prometheus_spec.get(key) is not False:
+        raise SystemExit(f"kube-prometheus-stack prometheusSpec.{key} must be false")
+alertmanager_selector = (
+    stack_values.get("alertmanager", {})
+    .get("alertmanagerSpec", {})
+    .get("alertmanagerConfigSelector", {})
+    .get("matchLabels", {})
+)
+if alertmanager_selector.get("alertmanagerConfig") != "codeplace":
+    raise SystemExit("AlertmanagerConfig selector must match alertmanagerConfig=codeplace")
+grafana_values = stack_values.get("grafana", {})
+datasources = grafana_values.get("additionalDataSources") or []
+if not any(ds.get("name") == "Loki" and ds.get("type") == "loki" for ds in datasources):
+    raise SystemExit("Grafana values must provision Loki datasource")
+dashboard_sidecar = grafana_values.get("sidecar", {}).get("dashboards", {})
+if dashboard_sidecar.get("label") != "grafana_dashboard":
+    raise SystemExit("Grafana dashboard sidecar must select grafana_dashboard label")
+print("KUBE-PROMETHEUS-STACK VALUES SHAPE OK")
+
+backend_sm = yaml.safe_load((root / "backend-service-monitor.yaml").read_text())
+if backend_sm.get("metadata", {}).get("labels", {}).get("release") != "kube-prometheus-stack":
+    raise SystemExit("backend ServiceMonitor must keep release=kube-prometheus-stack label")
+endpoint = (backend_sm.get("spec", {}).get("endpoints") or [{}])[0]
+if endpoint.get("port") != "api" or endpoint.get("path") != "/metrics" or endpoint.get("interval") != "15s":
+    raise SystemExit("backend ServiceMonitor must scrape api /metrics every 15s")
+if backend_sm.get("spec", {}).get("selector", {}).get("matchLabels", {}).get("app") != "backend":
+    raise SystemExit("backend ServiceMonitor must select app=backend")
+
+pod_monitors = list(yaml.safe_load_all((root / "datastore-pod-monitors.yaml").read_text()))
+for monitor in pod_monitors:
+    name = monitor.get("metadata", {}).get("name")
+    if monitor.get("metadata", {}).get("labels", {}).get("release") != "kube-prometheus-stack":
+        raise SystemExit(f"{name} PodMonitor must keep release=kube-prometheus-stack label")
+    endpoint = (monitor.get("spec", {}).get("podMetricsEndpoints") or [{}])[0]
+    if endpoint.get("interval") != "30s":
+        raise SystemExit(f"{name} PodMonitor must scrape every 30s")
+print("SCRAPE RESOURCE SHAPE OK")
+
 kustomization = yaml.safe_load((root / "kustomization.yaml").read_text())
 if "./alertmanager-config-email.example.yaml" in kustomization.get("resources", []):
     raise SystemExit("email fallback example must not be included in default kustomization")