k8s: configure otel sdk via instrumentation cr

kubernetes/llama-serve/granite-8b/vllm.yaml

@@ -20,6 +20,8 @@ spec:
       creationTimestamp: null
       labels:
         app: granite-8b
+      annotations:
+        instrumentation.opentelemetry.io/inject-sdk: "hub" # NOTE: May extend with ".<namespace>"
     spec:
       containers:
       - args:

kubernetes/llama-serve/llama3.1-70b/vllm.yaml

@@ -14,6 +14,8 @@ spec:
       creationTimestamp: null
       labels:
         app: llama31-70b
+      annotations:
+        instrumentation.opentelemetry.io/inject-sdk: "hub" # NOTE: May extend with ".<namespace>"
     spec:
       volumes:
         - name: hf-cache

kubernetes/llama-serve/llama3.2-3b/vllm.yaml

@@ -20,6 +20,8 @@ spec:
       creationTimestamp: null
       labels:
         app: llama32-3b
+      annotations:
+        instrumentation.opentelemetry.io/inject-sdk: "hub" # NOTE: May extend with ".<namespace>"
     spec:
       containers:
       - args:

kubernetes/llama-serve/watt-8b/vllm.yaml

@@ -14,6 +14,8 @@ spec:
       creationTimestamp: null
       labels:
         app: watt-8b
+      annotations:
+        instrumentation.opentelemetry.io/inject-sdk: "hub" # NOTE: May extend with ".<namespace>"
     spec:
       volumes:
         - name: hf-cache

kubernetes/llama-stack/deployment.yaml

@@ -9,7 +9,7 @@ spec:
   template:
     metadata:
       annotations:
-        sidecar.opentelemetry.io/inject: llamastack-otelsidecar
+        instrumentation.opentelemetry.io/inject-sdk: "hub" # NOTE: May extend with ".<namespace>"
       labels:
         app: llamastack
     spec:
@@ -44,10 +44,6 @@ spec:
               value: 'https://llama32-3b-llama-serve.apps.ocp-beta-test.nerc.mghpcc.org/v1'
             - name: VLLM_API_TOKEN
               value: fake
-            - name: OTEL_SERVICE_NAME
-              value: om-llamastack
-            - name: OTEL_TRACE_ENDPOINT
-              value: 'http://localhost:4318/v1/traces'
             - name: SAFETY_MODEL
               value: meta-llama/Llama-Guard-3-8B
             - name: SAFETY_VLLM_URL

kubernetes/observability/otel-sdk-hub-config.yaml

@@ -0,0 +1,19 @@
+apiVersion: opentelemetry.io/v1alpha1
+kind: Instrumentation
+metadata:
+  name: hub
+spec:
+  exporter:
+    endpoint: http://otel-collector-collector.observability-hub.svc.cluster.local:4317
+  env:
+    # NOTE: can this be provided by the service itself?
+    # - name: "OTEL_SERVICE_NAME"
+    #   value: "om-llamastack"
+    - name: "OTEL_EXPORTER_OTLP_TRACES_INSECURE"
+      value: "true"
+    # NOTE: Is http requiered?
+    - name: "OTEL_TRACE_ENDPOINT"
+      value: "http://otel-collector-collector.observability-hub.svc.cluster.local:4318/v1/traces"
+  propagators:
+    - tracecontext
+    - baggage