feat(telemetry): clickhouse read backend

.agents/skills/maf-sandbox/maf.py

@@ -66,6 +66,24 @@
 
 configure_otel_providers(enable_sensitive_data=True)
 
+# MAF stamps gen_ai.conversation.id per LLM response (a fresh resp_... id on every
+# nested call), so a single run's spans scatter across many "sessions" downstream.
+# Stamp one stable session.id per trace so a run maps to exactly one session.
+from opentelemetry import trace as _otel_trace  # noqa: E402
+from opentelemetry.sdk.trace import SpanProcessor as _SpanProcessor  # noqa: E402
+
+
+class _StableSessionId(_SpanProcessor):
+    def on_start(self, span, parent_context=None):
+        ctx = span.get_span_context()
+        if ctx and ctx.trace_id:
+            span.set_attribute("session.id", f"{ctx.trace_id:032x}")
+
+
+_sp_tp = _otel_trace.get_tracer_provider()
+if hasattr(_sp_tp, "add_span_processor"):
+    _sp_tp.add_span_processor(_StableSessionId())
+
 # Instrument the OpenAI SDK's httpx client so each completion emits a raw
 # `POST /v1/chat/completions` transport span (no gen_ai attrs → loupe `unknown`).
 from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor  # noqa: E402

.agents/skills/maf-sandbox/seed.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# Seed loupe's ClickHouse backend with real gen_ai telemetry via the MAF sandbox.
+# Points the sandbox's OTLP exporter at the CH collector (:4318) instead of OpenObserve,
+# then fires ~55 conversations (1-3 turns each) with varied shapes.
+set -euo pipefail
+HERE="$(cd "$(dirname "$0")" && pwd)"
+FIRE="$HERE/fire.py"
+
+export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="http://localhost:4318/v1/traces"
+export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="http://localhost:4318/v1/metrics"
+export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT="http://localhost:4318/v1/logs"
+export OTEL_EXPORTER_OTLP_HEADERS=" "   # drop the OpenObserve auth header
+
+# Kill any stale sandbox (likely pointed at OpenObserve), then warm up a fresh one
+# with the CH endpoint before firing anything in parallel (avoids a spawn race).
+pkill -f "maf.py" 2>/dev/null || true
+sleep 1
+"$FIRE" "warmup: say ready" --conversation "seed-warmup" >/dev/null 2>&1 || true
+
+# Single-turn prompts (one trace each), chosen to exercise different telemetry shapes.
+SINGLE=(
+  "add 17 and 25"
+  "multiply 8 by 9 then add 3"
+  "give me a random number between 1 and 100"
+  "echo back: hello world"
+  "look up user 42"
+  "list five fruits"
+  "weather in Tokyo, Paris and New York at once"
+  "what's the weather in Berlin?"
+  "roll two dice via mcp"
+  "flip a coin using mcp"
+  "translate 'good morning' to French via mcp"
+  "search the docs for 'retention' via mcp"
+  "what's the current time via mcp?"
+  "search my memory for anything about databases"
+  "recall what you know about vector search"
+  "fail_sometimes with probability 0.7"
+  "fail_sometimes with probability 0.4"
+  "load the files tools and read a file"
+  "load math utilities and give me factorial of 6"
+  "schedule a task to say hi in 2 seconds"
+)
+
+# Multi-turn conversation seeds (each inner turn is a separate trace in one session).
+declare -a CONVOS=(
+  "My name is Ada and I prefer tea|What's my name?|What do I prefer to drink?"
+  "I'm Grace, I'm 31|How old am I?"
+  "Remember I work on databases|What do I work on?|Recommend a tool for that"
+  "weather in London please|and Madrid?|now compare the two"
+  "add 2 and 2|multiply that by 10"
+  "roll a die|roll again and add them"
+  "look up user 7|now list items"
+  "translate hello to Spanish|now to German"
+)
+
+pids=()
+run_convo() {  # turns separated by |
+  local conv="seed-$1"; shift
+  local IFS='|'; read -ra turns <<<"$1"
+  for t in "${turns[@]}"; do
+    "$FIRE" "$t" --conversation "$conv" >/dev/null 2>&1 || true
+  done
+}
+
+i=0
+# Multi-turn sessions (sequential turns, parallel across sessions in small batches)
+for spec in "${CONVOS[@]}"; do
+  run_convo "conv-$i" "$spec" &
+  pids+=($!); i=$((i+1))
+  (( ${#pids[@]} >= 4 )) && { wait "${pids[0]}"; pids=("${pids[@]:1}"); }
+done
+
+# Single-turn sessions — unique conversation id each → its own session
+for p in "${SINGLE[@]}"; do
+  ( "$FIRE" "$p" --conversation "seed-single-$i" >/dev/null 2>&1 || true ) &
+  pids+=($!); i=$((i+1))
+  (( ${#pids[@]} >= 4 )) && { wait "${pids[0]}"; pids=("${pids[@]:1}"); }
+done
+
+# A few extra one-offs with fresh ids to pad session count past 50
+for n in $(seq 1 30); do
+  ( "$FIRE" "seed filler run number $n: add $n and $((n*2))" --conversation "seed-fill-$i" >/dev/null 2>&1 || true ) &
+  pids+=($!); i=$((i+1))
+  (( ${#pids[@]} >= 4 )) && { wait "${pids[0]}"; pids=("${pids[@]:1}"); }
+done
+
+wait
+echo "fired $i sessions"

.env.example

@@ -1,21 +1,12 @@
 DATABASE_URL="dev.db"
 
-# Telemetry provider — defaults to local OpenObserve. The Settings dialog
-# can flip this at runtime via a cookie; this env var is the fallback when
-# no cookie is set.
-# TELEMETRY_PROVIDER=app-insights
-
-# OpenObserve overrides (defaults work zero-config against the docker image).
-# OO_BASE_URL=http://localhost:5080
-# OO_USER=root@example.com
-# OO_PASS=Complexpass#123
-# OO_ORG=default
-# OO_STREAM=default
-
-# Application Insights (resource ID uses Azure AD; or set APP_ID + API_KEY for legacy)
-# APPLICATIONINSIGHTS_RESOURCE_ID=/subscriptions/<sub>/resourceGroups/<rg>/providers/microsoft.insights/components/<name>
-# APPLICATIONINSIGHTS_APP_ID=
-# APPLICATIONINSIGHTS_API_KEY=
+# Telemetry backend is ClickHouse (see infra/clickhouse). Defaults are zero-config
+# against the local docker image; override per deployment.
+# CLICKHOUSE_URL=http://localhost:8123
+# CLICKHOUSE_DB=loupe
+# CLICKHOUSE_USER=loupe
+# CLICKHOUSE_PASS=loupe
+# TELEMETRY_PROVIDER=fixtures   # e2e only — in-memory test double
 
 # MCP registry (/mcp) — JSON array of servers loupe mirrors and lints. Each ref:
 # id, name, endpoint (streamable-http tools/list URL), transport, optional

infra/clickhouse/README.md

@@ -0,0 +1,20 @@
+# ClickHouse telemetry backend (local)
+
+Design: `plans/telemetry-backend.md`.
+
+```sh
+docker compose up -d
+```
+
+- OTLP ingest: grpc `localhost:4317`, http `localhost:4318` (point agents' `OTEL_EXPORTER_OTLP_ENDPOINT` here)
+- ClickHouse: http `localhost:8123`, native `localhost:9000` — db/user/pass `loupe`/`loupe`/`loupe`
+
+loupe reads it with `TELEMETRY_PROVIDER=clickhouse` (envs `CLICKHOUSE_URL`, `CLICKHOUSE_DB`, `CLICKHOUSE_USER`, `CLICKHOUSE_PASS` default to the values above).
+
+Memory: `config.d/memory.xml` + `mem_limit` are sized for a small local Docker VM (~2GB) — raise both on bigger hosts. Merges run column-by-column (vertical) so the wide row + fat attr maps don't spike merge memory.
+
+Invariants:
+
+- The collector image tag is pinned; `init/01-traces.sql` base columns must match that version's `clickhouseexporter` INSERT shape (`create_schema: false`). Bump both together.
+- Promoted-column attr alias lists mirror `ATTRS` in `src/lib/telemetry/conventions.ts`. New alias → both places.
+- Schema changes: init scripts only run on a fresh volume (`docker compose down -v` to reset), or apply `ALTER TABLE` manually. Materialized columns backfill only on merge — `ALTER TABLE ... MATERIALIZE COLUMN` for existing data.

infra/clickhouse/config.d/memory.xml

@@ -0,0 +1,9 @@
+<!-- Keep the server below the container mem_limit so the OOM killer never fires.
+     Sized for a small local Docker VM; raise together with mem_limit on bigger hosts. -->
+<clickhouse>
+    <max_server_memory_usage>1000000000</max_server_memory_usage>
+    <mark_cache_size>134217728</mark_cache_size>
+    <index_mark_cache_size>67108864</index_mark_cache_size>
+    <uncompressed_cache_size>67108864</uncompressed_cache_size>
+    <merges_mutations_memory_usage_soft_limit>300000000</merges_mutations_memory_usage_soft_limit>
+</clickhouse>

infra/clickhouse/docker-compose.yml

@@ -0,0 +1,44 @@
+# Local telemetry backend: ClickHouse + OTel collector (plans/telemetry-backend.md).
+# The collector image is pinned because init/01-traces.sql must match the
+# clickhouseexporter INSERT column shape of this exact version.
+services:
+  clickhouse:
+    image: clickhouse/clickhouse-server:24.8
+    # Sized for a small local Docker VM; raise freely on bigger hosts.
+    mem_limit: 1500m
+    environment:
+      CLICKHOUSE_DB: loupe
+      CLICKHOUSE_USER: loupe
+      CLICKHOUSE_PASSWORD: loupe
+    ports:
+      - '127.0.0.1:8123:8123'
+      - '127.0.0.1:9000:9000'
+    volumes:
+      - ch-data:/var/lib/clickhouse
+      - ./init:/docker-entrypoint-initdb.d:ro
+      - ./config.d/memory.xml:/etc/clickhouse-server/config.d/zz-loupe-memory.xml:ro
+      - ./users.d/memory.xml:/etc/clickhouse-server/users.d/zz-loupe-memory.xml:ro
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    healthcheck:
+      test: ['CMD', 'wget', '-qO-', 'http://localhost:8123/ping']
+      interval: 3s
+      timeout: 2s
+      retries: 30
+
+  otelcol:
+    image: otel/opentelemetry-collector-contrib:0.120.0
+    command: ['--config=/etc/otelcol/config.yaml']
+    volumes:
+      - ./otel-collector.yaml:/etc/otelcol/config.yaml:ro
+    ports:
+      - '127.0.0.1:4317:4317'
+      - '127.0.0.1:4318:4318'
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+
+volumes:
+  ch-data: