From cc732da8a7f0bcae3f000a23d15eea654706bd21 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Wed, 15 Jul 2026 12:00:55 +0200 Subject: [PATCH 1/2] telemetry: skip cross-task OTel context detach on stop() ContextVar tokens can only be reset from the contextvars.Context (i.e. the asyncio task) that created them. When TelemetryManager.start() and stop() run in different tasks -- the normal case under the ORBIT rhapsody plugin, where session init and close are separate requests -- the detach in stop() raised ValueError inside opentelemetry.context.detach(), which OTel catches and logs as 'ERROR: Failed to detach context' on every otherwise-clean shutdown. Record the attaching task in start() and detach only when stop() runs in that same task. Skipping the detach is safe: the attach only ever affected the starting task's context, and the tracer provider is shut down immediately afterwards. Co-Authored-By: Claude Fable 5 --- src/rhapsody/telemetry/manager.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/rhapsody/telemetry/manager.py b/src/rhapsody/telemetry/manager.py index ae5dd52..12a3b49 100644 --- a/src/rhapsody/telemetry/manager.py +++ b/src/rhapsody/telemetry/manager.py @@ -176,9 +176,13 @@ def __init__( # correct parent span context in the dispatch loop. self._task_contexts: dict[str, Any] = {} - # Token returned by context.attach() when the session span is activated in start(). - # Stored so it can be properly detached in stop(). + # Token returned by context.attach() when the session span is activated in start(), + # plus the asyncio task that performed the attach. ContextVar tokens can only be + # reset from the contextvars.Context they were created in (i.e. the same task), so + # stop() skips the detach when it runs in a different task — OTel would otherwise + # log a spurious "Failed to detach context" ValueError at shutdown. self._session_ctx_token: Any = None + self._session_ctx_task: Any = None # Caller-supplied OTel extension points wired into RHAPSODY's internal providers # at start() time alongside SpanBuffer / InMemoryMetricReader. @@ -251,6 +255,7 @@ async def start(self) -> None: self._session_ctx_token = otel_context.attach( trace_mod.set_span_in_context(self._session_span) ) + self._session_ctx_task = asyncio.current_task() self._running = True self._dispatch_task = asyncio.create_task(self._dispatch_loop(), name="telemetry-dispatch") @@ -322,8 +327,15 @@ async def stop(self) -> None: from opentelemetry import context as otel_context if self._session_ctx_token is not None: - otel_context.detach(self._session_ctx_token) + # Detach only when stop() runs in the task that attached the token in + # start(); a token reset from any other task raises ValueError inside + # otel_context.detach(), which OTel catches and logs as an ERROR. + # Skipping is safe: the attach only ever affected the starting task's + # context, and the tracer provider shuts down right below. + if asyncio.current_task() is self._session_ctx_task: + otel_context.detach(self._session_ctx_token) self._session_ctx_token = None + self._session_ctx_task = None if self._meter_provider: self._meter_provider.shutdown() From 91616184c4925888e937c72f1526598aaa6a2c16 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Wed, 15 Jul 2026 13:17:40 +0200 Subject: [PATCH 2/2] telemetry: hold the attaching task as a weakref (gemini review) A strong reference would pin the starting task's coroutine frame and locals for the entire session lifetime; the weakref keeps the identity check and resolves to None once the task is gone, in which case the detach is skipped just like any other cross-task stop(). Co-Authored-By: Claude Fable 5 --- src/rhapsody/telemetry/manager.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/rhapsody/telemetry/manager.py b/src/rhapsody/telemetry/manager.py index 12a3b49..1b900b9 100644 --- a/src/rhapsody/telemetry/manager.py +++ b/src/rhapsody/telemetry/manager.py @@ -15,6 +15,7 @@ import json import logging import time +import weakref from contextlib import contextmanager from pathlib import Path from typing import TYPE_CHECKING @@ -177,12 +178,14 @@ def __init__( self._task_contexts: dict[str, Any] = {} # Token returned by context.attach() when the session span is activated in start(), - # plus the asyncio task that performed the attach. ContextVar tokens can only be - # reset from the contextvars.Context they were created in (i.e. the same task), so - # stop() skips the detach when it runs in a different task — OTel would otherwise - # log a spurious "Failed to detach context" ValueError at shutdown. + # plus a weakref to the asyncio task that performed the attach. ContextVar tokens + # can only be reset from the contextvars.Context they were created in (i.e. the + # same task), so stop() skips the detach when it runs in a different task — OTel + # would otherwise log a spurious "Failed to detach context" ValueError at + # shutdown. A weakref avoids pinning the starting task's frame for the whole + # session lifetime. self._session_ctx_token: Any = None - self._session_ctx_task: Any = None + self._session_ctx_task: weakref.ref | None = None # Caller-supplied OTel extension points wired into RHAPSODY's internal providers # at start() time alongside SpanBuffer / InMemoryMetricReader. @@ -255,7 +258,8 @@ async def start(self) -> None: self._session_ctx_token = otel_context.attach( trace_mod.set_span_in_context(self._session_span) ) - self._session_ctx_task = asyncio.current_task() + _task = asyncio.current_task() + self._session_ctx_task = weakref.ref(_task) if _task is not None else None self._running = True self._dispatch_task = asyncio.create_task(self._dispatch_loop(), name="telemetry-dispatch") @@ -331,8 +335,10 @@ async def stop(self) -> None: # start(); a token reset from any other task raises ValueError inside # otel_context.detach(), which OTel catches and logs as an ERROR. # Skipping is safe: the attach only ever affected the starting task's - # context, and the tracer provider shuts down right below. - if asyncio.current_task() is self._session_ctx_task: + # context, and the tracer provider shuts down right below. The weakref + # resolves to None once the starting task is gone — skip then too. + _attached_task = self._session_ctx_task() if self._session_ctx_task else None + if _attached_task is not None and asyncio.current_task() is _attached_task: otel_context.detach(self._session_ctx_token) self._session_ctx_token = None self._session_ctx_task = None