From 75fe5a0bca87c7640c1300a2d34dafce8054ec2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:05:15 +0200 Subject: [PATCH 01/50] chore(deps): unify prost on the workspace version Move livekit-protocol, livekit, livekit-api, livekit-data-stream and livekit-uniffi from a pinned prost 0.12 to the workspace prost 0.14, with pbjson/pbjson-types 0.6 -> 0.9. The committed prost-generated code compiles unchanged, so no regeneration is needed. Binaries combining these crates (livekit-ffi already used the workspace prost) now link a single prost, and crates that depend on prost 0.14 types (e.g. opentelemetry-proto) no longer drag in a second copy. Co-Authored-By: Claude Fable 5 --- .changeset/unify-prost.md | 11 +++ Cargo.lock | 120 +++++++-------------------------- livekit-api/Cargo.toml | 4 +- livekit-data-stream/Cargo.toml | 2 +- livekit-protocol/Cargo.toml | 6 +- livekit-uniffi/Cargo.toml | 2 +- livekit/Cargo.toml | 2 +- 7 files changed, 45 insertions(+), 102 deletions(-) create mode 100644 .changeset/unify-prost.md diff --git a/.changeset/unify-prost.md b/.changeset/unify-prost.md new file mode 100644 index 000000000..3343804c8 --- /dev/null +++ b/.changeset/unify-prost.md @@ -0,0 +1,11 @@ +--- +livekit-protocol: patch +livekit: patch +livekit-api: patch +livekit-data-stream: patch +livekit-uniffi: patch +--- + +Unify `prost` on the workspace version (0.14, with `pbjson`/`pbjson-types` 0.9) across +`livekit-protocol`, `livekit`, `livekit-api`, `livekit-data-stream` and `livekit-uniffi`, so +binaries that combine these crates link a single prost. The committed generated code is unchanged. diff --git a/Cargo.lock b/Cargo.lock index 94f152a8e..78a12ef15 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1152,7 +1152,7 @@ version = "4.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "syn 3.0.3", @@ -2719,7 +2719,7 @@ version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf59b675301228a696fe01c3073974643365080a76cc3ed5bc2cbc466ad87f17" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro-crate", "proc-macro2", "quote", @@ -2956,12 +2956,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -3451,15 +3445,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.12.1" @@ -3912,7 +3897,7 @@ dependencies = [ "livekit-token", "log", "parking_lot", - "prost 0.12.6", + "prost 0.14.4", "rand 0.9.5", "semver", "serde", @@ -3946,7 +3931,7 @@ dependencies = [ "os_info", "parking_lot", "pbjson-types", - "prost 0.12.6", + "prost 0.14.4", "rand 0.9.5", "reqwest", "scopeguard", @@ -3981,7 +3966,7 @@ dependencies = [ "livekit-protocol", "log", "parking_lot", - "prost 0.12.6", + "prost 0.14.4", "rand 0.9.5", "thiserror 2.0.19", "tokio", @@ -4034,7 +4019,7 @@ dependencies = [ "log", "parking_lot", "prost 0.14.4", - "prost-build 0.14.4", + "prost-build", "soxr-sys", "thiserror 2.0.19", "tokio", @@ -4069,7 +4054,7 @@ version = "0.7.12" dependencies = [ "pbjson", "pbjson-types", - "prost 0.12.6", + "prost 0.14.4", "serde", ] @@ -4124,7 +4109,7 @@ dependencies = [ "livekit-token", "log", "once_cell", - "prost 0.12.6", + "prost 0.14.4", "thiserror 2.0.19", "tokio", "tokio-util", @@ -5433,38 +5418,38 @@ checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" [[package]] name = "pbjson" -version = "0.6.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1030c719b0ec2a2d25a5df729d6cff1acf3cc230bf766f4f97833591f7577b90" +checksum = "e8edd1efdd8ab23ba9cb9ace3d9987a72663d5d7c9f74fa00b51d6213645cf6c" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "serde", ] [[package]] name = "pbjson-build" -version = "0.6.2" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735" +checksum = "2ed4d5c6ae95e08ac768883c8401cf0e8deb4e6e1d6a4e1fd3d2ec4f0ec63200" dependencies = [ - "heck 0.4.1", - "itertools 0.11.0", - "prost 0.12.6", - "prost-types 0.12.6", + "heck", + "itertools 0.14.0", + "prost 0.14.4", + "prost-types 0.14.4", ] [[package]] name = "pbjson-types" -version = "0.6.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f596653ba4ac51bdecbb4ef6773bc7f56042dc13927910de1684ad3d32aa12" +checksum = "a14e2757d877c0f607a82ce1b8560e224370f159d66c5d52eb55ea187ef0350e" dependencies = [ "bytes", "chrono", "pbjson", "pbjson-build", - "prost 0.12.6", - "prost-build 0.12.6", + "prost 0.14.4", + "prost-build", "serde", ] @@ -5775,16 +5760,6 @@ dependencies = [ "prost-derive 0.11.9", ] -[[package]] -name = "prost" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" -dependencies = [ - "bytes", - "prost-derive 0.12.6", -] - [[package]] name = "prost" version = "0.14.4" @@ -5795,34 +5770,13 @@ dependencies = [ "prost-derive 0.14.4", ] -[[package]] -name = "prost-build" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" -dependencies = [ - "bytes", - "heck 0.5.0", - "itertools 0.12.1", - "log", - "multimap", - "once_cell", - "petgraph 0.6.5", - "prettyplease", - "prost 0.12.6", - "prost-types 0.12.6", - "regex", - "syn 2.0.119", - "tempfile", -] - [[package]] name = "prost-build" version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ - "heck 0.5.0", + "heck", "itertools 0.14.0", "log", "multimap", @@ -5848,19 +5802,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "prost-derive" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" -dependencies = [ - "anyhow", - "itertools 0.12.1", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "prost-derive" version = "0.14.4" @@ -5883,15 +5824,6 @@ dependencies = [ "prost 0.11.9", ] -[[package]] -name = "prost-types" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" -dependencies = [ - "prost 0.12.6", -] - [[package]] name = "prost-types" version = "0.14.4" @@ -7200,7 +7132,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "396a35feb67335377e0251fcbc1092fc85c484bd4e3a7a54319399da127796e7" dependencies = [ "cfg-expr", - "heck 0.5.0", + "heck", "pkg-config", "toml 1.1.3+spec-1.1.0", "version-compare", @@ -8106,7 +8038,7 @@ dependencies = [ "camino", "cargo_metadata 0.18.1", "genco", - "heck 0.5.0", + "heck", "lazy_static", "paste", "proc-macro2", @@ -8131,7 +8063,7 @@ dependencies = [ "fs-err", "glob", "goblin", - "heck 0.5.0", + "heck", "indexmap 2.14.0", "once_cell", "serde", @@ -8230,7 +8162,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "032739b3ec725576914c15899dedaf080163ced86b6934566c20ec2b20ce90ca" dependencies = [ "anyhow", - "heck 0.5.0", + "heck", "indexmap 2.14.0", "tempfile", "uniffi_internal_macros", diff --git a/livekit-api/Cargo.toml b/livekit-api/Cargo.toml index 2e0f54b66..099b0b477 100644 --- a/livekit-api/Cargo.toml +++ b/livekit-api/Cargo.toml @@ -79,8 +79,8 @@ sha2 = "0.10" url = "2.3" log = { workspace = true } parking_lot = { workspace = true } -prost = "0.12" -pbjson-types = "0.6" +prost = { workspace = true } +pbjson-types = "0.9" # webhooks serde_json = { workspace = true, optional = true } diff --git a/livekit-data-stream/Cargo.toml b/livekit-data-stream/Cargo.toml index fb906af54..8e8cdd55f 100644 --- a/livekit-data-stream/Cargo.toml +++ b/livekit-data-stream/Cargo.toml @@ -20,7 +20,7 @@ bytes = { workspace = true } tokio = { workspace = true, default-features = false, features = ["sync", "fs", "io-util", "rt"] } tokio-stream = { workspace = true, features = ["sync"] } futures-util = { workspace = true, default-features = false, features = ["sink", "io"] } -prost = "0.12" +prost = { workspace = true } chrono = "0.4.38" bmrng = "0.5.2" uuid = { version = "1", features = ["v4"] } diff --git a/livekit-protocol/Cargo.toml b/livekit-protocol/Cargo.toml index 3bbbe36a0..dc667f932 100644 --- a/livekit-protocol/Cargo.toml +++ b/livekit-protocol/Cargo.toml @@ -7,7 +7,7 @@ description = "Livekit protocol and utilities for the Rust SDK" repository.workspace = true [dependencies] -prost = "0.12" +prost = { workspace = true } serde = { workspace = true } -pbjson = "0.6" -pbjson-types = "0.6" +pbjson = "0.9" +pbjson-types = "0.9" diff --git a/livekit-uniffi/Cargo.toml b/livekit-uniffi/Cargo.toml index 0364de3a4..28b4c7fbd 100644 --- a/livekit-uniffi/Cargo.toml +++ b/livekit-uniffi/Cargo.toml @@ -20,7 +20,7 @@ uniffi = { workspace = true, features = ["scaffolding-ffi-buffer-fns", "tokio"] log = { workspace = true } tokio = { workspace = true, features = ["sync", "rt-multi-thread"] } tokio-util = "0.7.18" -prost = "0.12" +prost = { workspace = true } futures-util = { workspace = true, default-features = false, features = ["sink"] } bytes = { workspace = true } once_cell = "1.21.3" diff --git a/livekit/Cargo.toml b/livekit/Cargo.toml index 2b98a614f..124cb1a4a 100644 --- a/livekit/Cargo.toml +++ b/livekit/Cargo.toml @@ -39,7 +39,7 @@ livekit-protocol = { workspace = true } livekit-common = { workspace = true } livekit-data-stream = { workspace = true } livekit-datatrack = { workspace = true } -prost = "0.12" +prost = { workspace = true } serde = { version = "1", features = ["derive"] } serde_json = "1.0" tokio = { version = "1", default-features = false, features = ["sync", "macros", "fs"] } From 0020ae141d3ade2feb3ea24c2bcf38f5899edd36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:06:18 +0200 Subject: [PATCH 02/50] feat(telemetry): add livekit-telemetry crate Shared client telemetry core, consumed by the Rust SDK and (via livekit-uniffi) by the platform SDKs. Mirrors the OpenTelemetry Logs SDK shape: - Telemetry::emit synchronous, never-blocking entry point - Store bounded in-memory queue, drop-oldest - Exporter actor: 1 s tick, batches of 512, OTLP encode, bounded retries honoring Retry-After, drop on reject, go silent when the collector disables telemetry - TelemetryTransport the only injection point: the core composes URL/headers/body, the transport moves bytes; NetTransport over livekit-net behind `net` - FileCache (storage_dir) one file per encoded batch, .tmp -> rename, oldest-first replay, drop-oldest above max_storage_bytes, 24 h max age; shutdown spills the queue to disk before trying the network; throttled/rejected/disabled data is never written OTLP types come from opentelemetry-proto (gen-tonic-messages only); its SDK dependencies are dead code and LTO removes them. SPEC.md seeds the event catalog (lk.ping). Co-Authored-By: Claude Fable 5 --- .changeset/add-livekit-telemetry-crate.md | 7 + Cargo.lock | 49 +++ Cargo.toml | 2 + knope.toml | 8 + livekit-telemetry/CHANGELOG.md | 1 + livekit-telemetry/Cargo.toml | 30 ++ livekit-telemetry/README.md | 78 +++++ livekit-telemetry/SPEC.md | 29 ++ livekit-telemetry/src/event.rs | 136 +++++++++ livekit-telemetry/src/exporter.rs | 288 ++++++++++++++++++ livekit-telemetry/src/lib.rs | 47 +++ livekit-telemetry/src/otlp.rs | 140 +++++++++ livekit-telemetry/src/persist.rs | 198 ++++++++++++ livekit-telemetry/src/proto/mod.rs | 26 ++ livekit-telemetry/src/store.rs | 86 ++++++ livekit-telemetry/src/telemetry.rs | 350 ++++++++++++++++++++++ livekit-telemetry/src/transport.rs | 120 ++++++++ livekit-telemetry/uniffi.toml | 2 + 18 files changed, 1597 insertions(+) create mode 100644 .changeset/add-livekit-telemetry-crate.md create mode 100644 livekit-telemetry/CHANGELOG.md create mode 100644 livekit-telemetry/Cargo.toml create mode 100644 livekit-telemetry/README.md create mode 100644 livekit-telemetry/SPEC.md create mode 100644 livekit-telemetry/src/event.rs create mode 100644 livekit-telemetry/src/exporter.rs create mode 100644 livekit-telemetry/src/lib.rs create mode 100644 livekit-telemetry/src/otlp.rs create mode 100644 livekit-telemetry/src/persist.rs create mode 100644 livekit-telemetry/src/proto/mod.rs create mode 100644 livekit-telemetry/src/store.rs create mode 100644 livekit-telemetry/src/telemetry.rs create mode 100644 livekit-telemetry/src/transport.rs create mode 100644 livekit-telemetry/uniffi.toml diff --git a/.changeset/add-livekit-telemetry-crate.md b/.changeset/add-livekit-telemetry-crate.md new file mode 100644 index 000000000..b78f26221 --- /dev/null +++ b/.changeset/add-livekit-telemetry-crate.md @@ -0,0 +1,7 @@ +--- +livekit-telemetry: minor +--- + +Add `livekit-telemetry`, the shared client telemetry core: events are buffered on-device, +batched and exported as OTLP/HTTP log records through a pluggable `TelemetryTransport`, with an +optional on-disk cache of undeliverable batches. diff --git a/Cargo.lock b/Cargo.lock index 78a12ef15..f874f3cdb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4070,6 +4070,21 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "livekit-telemetry" +version = "0.1.0" +dependencies = [ + "async-trait", + "livekit-net", + "livekit-runtime", + "log", + "opentelemetry-proto", + "prost 0.14.4", + "thiserror 2.0.19", + "tokio", + "uniffi", +] + [[package]] name = "livekit-token" version = "0.1.1" @@ -5300,6 +5315,40 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +dependencies = [ + "js-sys", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost 0.14.4", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry", + "portable-atomic", + "thiserror 2.0.19", +] + [[package]] name = "orbclient" version = "0.3.55" diff --git a/Cargo.toml b/Cargo.toml index 2888c72b9..72e448332 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "livekit-ffi", "livekit-uniffi", "livekit-datatrack", + "livekit-telemetry", "livekit-token", "livekit-token-source", "livekit-ffi-node-bindings", @@ -55,6 +56,7 @@ livekit = { version = "0.8.4", path = "livekit" } livekit-api = { version = "0.6.4", path = "livekit-api" } livekit-ffi = { version = "0.12.76", path = "livekit-ffi" } livekit-datatrack = { version = "0.1.14", path = "livekit-datatrack" } +livekit-telemetry = { version = "0.1.0", path = "livekit-telemetry" } livekit-token = { version = "0.1.1", path = "livekit-token" } livekit-token-source = { version = "0.1.2", path = "livekit-token-source" } livekit-common = { version = "0.1.2", path = "livekit-common" } diff --git a/knope.toml b/knope.toml index 4034078e6..75d4d65d7 100644 --- a/knope.toml +++ b/knope.toml @@ -159,6 +159,14 @@ versioned_files = [ ] changelog = "livekit-datatrack/CHANGELOG.md" +[packages.livekit-telemetry] +versioned_files = [ + "livekit-telemetry/Cargo.toml", + "Cargo.lock", + { path = "Cargo.toml", dependency = "livekit-telemetry" }, +] +changelog = "livekit-telemetry/CHANGELOG.md" + [packages.livekit-common] versioned_files = [ "livekit-common/Cargo.toml", diff --git a/livekit-telemetry/CHANGELOG.md b/livekit-telemetry/CHANGELOG.md new file mode 100644 index 000000000..825c32f0d --- /dev/null +++ b/livekit-telemetry/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog diff --git a/livekit-telemetry/Cargo.toml b/livekit-telemetry/Cargo.toml new file mode 100644 index 000000000..5eea146c0 --- /dev/null +++ b/livekit-telemetry/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "livekit-telemetry" +description = "Client telemetry core for LiveKit: buffers events on-device and exports them as OTLP" +version = "0.1.0" +readme = "README.md" +license.workspace = true +edition.workspace = true +repository.workspace = true + +[dependencies] +livekit-runtime = { workspace = true, features = ["tokio"] } +tokio = { workspace = true, default-features = false, features = ["macros", "sync"] } +log = { workspace = true } +thiserror = { workspace = true } +async-trait = "0.1" +prost = { workspace = true } +# OTLP message types only (`gen-tonic-messages` = prost structs, no tonic). Same prost as +# livekit-protocol so a single prost is linked. +opentelemetry-proto = { version = "0.32", default-features = false, features = ["logs", "gen-tonic-messages"] } +livekit-net = { workspace = true, optional = true } +uniffi = { workspace = true, features = ["scaffolding-ffi-buffer-fns"], optional = true } + +[features] +# Default HTTP transport over the pluggable `livekit-net` client (native backend, or one +# the host registered with `livekit_net::set_http_client`). +net = ["dep:livekit-net"] +uniffi = ["dep:uniffi"] + +[dev-dependencies] +tokio = { workspace = true, default-features = false, features = ["rt", "rt-multi-thread", "macros", "sync", "time", "test-util"] } diff --git a/livekit-telemetry/README.md b/livekit-telemetry/README.md new file mode 100644 index 000000000..5e04e91de --- /dev/null +++ b/livekit-telemetry/README.md @@ -0,0 +1,78 @@ +# LiveKit Telemetry + +**Important**: +This is an internal crate that powers client telemetry in LiveKit client SDKs (including [Rust](https://crates.io/crates/livekit) and, through `livekit-uniffi`, Swift/Kotlin/Dart) and is not usable directly. + +The core buffers events on-device and ships them out-of-band as standard +[OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/) log records. Everything hard lives here +once — batching, encoding, retry, persistence, go-silent — while platforms provide only what +they are uniquely placed to: instruments (OS signals) and the byte-moving transport. + +```text + SDK / instruments ──emit()──▶ Telemetry ──▶ Store ──drain──▶ Exporter ──ExportRequest──▶ TelemetryTransport + (source) (bounded queue) (tick · OTLP encode · retry) (NetTransport | host HTTP | data channel …) + │ failed after retries + ▼ + FileCache (one file per encoded batch, replayed oldest-first) +``` + +| Layer | OTel equivalent | Type | +|---|---|---| +| source | `Logger.emit` | [`Telemetry::emit`] with [`TelemetryEvent`] | +| store | `BatchLogRecordProcessor` queue | `Store` (in-memory, drop-oldest) | +| sink | `BatchLogRecordProcessor` timer + `LogRecordExporter` | [`Exporter`] (actor, spawn `run()`) | +| transport | exporter's HTTP client | [`TelemetryTransport`] trait; `NetTransport` over `livekit-net` (feature `net`) | +| persistence | disk-buffering exporter wrapper | `FileCache` (opt-in via `storage_dir`) | + +## Usage + +```rust,ignore +let mut config = TelemetryConfig::new("http://localhost:4318/v1/logs"); +config.resource.push(Attribute::new("service.name", "my-app")); +config.storage_dir = Some(cache_dir.join("livekit-telemetry").display().to_string()); +let transport = NetTransport::from_registry().expect("livekit-net client"); + +let (telemetry, exporter) = Telemetry::new(config, Arc::new(transport)); +tokio::spawn(exporter.run()); + +telemetry.emit(TelemetryEvent::new("lk.ping").with_attribute("lk.ping.seq", 1i64)); +telemetry.shutdown().await; // spill to disk, then flush; bounded by `export_timeout_ms` +``` + +Event names and attributes are defined in [`SPEC.md`](SPEC.md). + +## Local testing + +```sh +docker run -d --name lk-lgtm -p 3000:3000 -p 4318:4318 -p 3100:3100 grafana/otel-lgtm +cargo run -p telemetry_ping # sends one `lk.ping` +LK_TELEMETRY_DIR=/tmp/lk-telemetry cargo run -p telemetry_ping # with the file cache +open http://localhost:3000/explore # Loki: {service_name="telemetry_ping"} +``` + +`LK_OTLP_ENDPOINT` overrides the collector URL. Stop the container, run with `LK_TELEMETRY_DIR`, +start it again and run once more to watch the cached batch replay. + +## Design notes + +- **Typed events in the store, OTLP at the edge.** Events are stored as [`TelemetryEvent`], not + pre-serialized bytes: batches need grouping under one resource/scope, batch-time attributes + can be attached, and a human-readable dump is a `Debug`/JSON view away. +- **Persistence is a file cache at the exporter, not a database.** What every client SDK does: + Sentry (envelope files, capped count, oldest evicted), Datadog (batch files, 5 MB/file, + 18 h max age), opentelemetry-android disk-buffering (`.tmp` → rename, 1 MB files, 10 MB dir), + Amplitude/Segment (delimited JSON files, ~1 MB / 475 KB rotation). We persist the *encoded* + request body: nothing to re-encode on replay, URL/headers recomposed from the current config + (rotated tokens just work), zero dependencies. Throttled (`Retry-After`), rejected and + disabled data is never written (persistence must not become a disk-backed retry queue). + `shutdown` spills the queue to disk before trying the network, so an app killed offline + loses nothing. Age is derived from file names, not file timestamps (an Apple + required-reason API). +- **Transport is the only injection point.** The core composes URL, headers and body; the + transport moves bytes and reports [`ExportError`] so the core alone decides retry / drop / + persist / go-silent. No Rust HTTP/TLS stack is linked unless the `net` feature is enabled. +- **Size.** OTLP types come from `opentelemetry-proto` (`gen-tonic-messages`, no tonic). Its + `opentelemetry`/`opentelemetry_sdk` dependencies are dead code here and LTO removes them + (measured +8 B on the iOS UniFFI dylib versus vendored generated code) — provided the + workspace links a single `prost`, which is why `livekit-protocol` and friends moved to the + workspace `prost` version. diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md new file mode 100644 index 000000000..9d6c8d8c3 --- /dev/null +++ b/livekit-telemetry/SPEC.md @@ -0,0 +1,29 @@ +# Client telemetry spec + +Source of truth for event names, attributes and cadences emitted by LiveKit client SDKs. +Additive-only by convention; LiveKit-defined names carry the `lk.` prefix, everything else +follows [OpenTelemetry semantic conventions](https://github.com/open-telemetry/semantic-conventions). + +## Resource attributes + +Set once per pipeline (`TelemetryConfig.resource`): + +| Key | Who sets it | Example | +|---|---|---| +| `service.name` | platform SDK | `livekit-client-swift` | +| `service.version` | platform SDK | `2.9.0` | +| `os.name`, `os.version` | platform SDK | `iOS`, `18.5` | +| `device.model.identifier` | platform SDK | `iPhone16,1` | +| `telemetry.sdk.name/language/version` | core | `livekit-telemetry`, `rust`, `0.1.0` | + +## Events + +```yaml +event: lk.ping +area: sdk +severity: info +attributes: + lk.ping.seq: int # optional, monotonically increasing per pipeline +cadence: on demand — pipeline smoke test, never emitted in production paths +platforms: all +``` diff --git a/livekit-telemetry/src/event.rs b/livekit-telemetry/src/event.rs new file mode 100644 index 000000000..284bf79f4 --- /dev/null +++ b/livekit-telemetry/src/event.rs @@ -0,0 +1,136 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::{SystemTime, UNIX_EPOCH}; + +/// A discrete telemetry event. +/// +/// Exported as one OTLP log record whose `event_name` is [`name`](Self::name), following the +/// OTel logs data model (events are log records with a top-level event name). +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct TelemetryEvent { + /// Event name. LiveKit-defined events use the `lk.` prefix (e.g. `lk.ping`); see `SPEC.md`. + pub name: String, + pub severity: Severity, + /// Optional human-readable message (the OTLP log record body). + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub body: Option, + pub attributes: Vec, + /// Wall-clock time in nanoseconds since the Unix epoch. `None` stamps the event at emit time. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub timestamp_ns: Option, +} + +impl TelemetryEvent { + /// An `Info` event without attributes, stamped when emitted. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + severity: Severity::Info, + body: None, + attributes: Vec::new(), + timestamp_ns: None, + } + } + + pub fn with_severity(mut self, severity: Severity) -> Self { + self.severity = severity; + self + } + + pub fn with_body(mut self, body: impl Into) -> Self { + self.body = Some(body.into()); + self + } + + pub fn with_attribute( + mut self, + key: impl Into, + value: impl Into, + ) -> Self { + self.attributes.push(Attribute::new(key, value)); + self + } +} + +/// Event severity, mapped onto the OTel severity numbers (`TRACE`=1 … `ERROR`=17). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Severity { + Trace, + Debug, + Info, + Warn, + Error, +} + +/// A key/value attribute on an event or on the resource. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct Attribute { + pub key: String, + pub value: AttributeValue, +} + +impl Attribute { + pub fn new(key: impl Into, value: impl Into) -> Self { + Self { key: key.into(), value: value.into() } + } +} + +/// Attribute value: the scalar subset of OTLP `AnyValue`. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, PartialEq)] +pub enum AttributeValue { + Str(String), + Int(i64), + Double(f64), + Bool(bool), +} + +impl From<&str> for AttributeValue { + fn from(value: &str) -> Self { + Self::Str(value.to_owned()) + } +} + +impl From for AttributeValue { + fn from(value: String) -> Self { + Self::Str(value) + } +} + +impl From for AttributeValue { + fn from(value: i64) -> Self { + Self::Int(value) + } +} + +impl From for AttributeValue { + fn from(value: f64) -> Self { + Self::Double(value) + } +} + +impl From for AttributeValue { + fn from(value: bool) -> Self { + Self::Bool(value) + } +} + +/// Current wall-clock time in nanoseconds since the Unix epoch (0 if the clock is before 1970). +pub(crate) fn now_unix_nanos() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).map(|d| d.as_nanos() as u64).unwrap_or(0) +} diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs new file mode 100644 index 000000000..f85b527c6 --- /dev/null +++ b/livekit-telemetry/src/exporter.rs @@ -0,0 +1,288 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{sync::Arc, time::Duration}; + +use livekit_runtime::{interval, sleep, timeout, Instant, MissedTickBehavior}; +use tokio::sync::{mpsc, oneshot}; + +use crate::{ + otlp, persist::FileCache, store::Store, ExportError, ExportRequest, TelemetryConfig, + TelemetryEvent, TelemetryTransport, +}; + +/// Retries per batch after the first attempt. Fixed for now; see [`Exporter::deliver`]. +const MAX_RETRIES: u32 = 2; +const RETRY_BACKOFF: Duration = Duration::from_secs(1); +const MAX_RETRY_DELAY: Duration = Duration::from_secs(30); +/// Pause before the on-disk cache is retried after a failed replay (Sentry: stop consuming the +/// cache until `Retry-After` elapses; here also for plain connectivity failures). +const REPLAY_BACKOFF: Duration = Duration::from_secs(60); + +pub(crate) enum Command { + Flush(oneshot::Sender<()>), + Shutdown(oneshot::Sender<()>), +} + +/// Outcome of delivering one encoded batch, after retries. +enum Delivery { + Sent, + /// Transient failure (network, timeout, 5xx) — worth keeping for later. + Failed, + /// The collector asked us to back off (`Retry-After`) — dropped, never persisted. + Throttled, + Rejected, + Disabled, +} + +/// Background actor that turns stored events into OTLP requests. +/// +/// The role of OTel's `BatchLogRecordProcessor` + OTLP exporter in one place: every +/// `flush_interval_ms` it drains up to `max_batch_size` events, encodes them, and hands the +/// request to the [`TelemetryTransport`]. Transient failures are retried with a bounded +/// backoff (honoring `Retry-After`), rejected batches are dropped, and once the collector +/// reports telemetry as disabled the exporter goes silent for good — dropping instead of +/// retrying, so a disabled project never sees a request storm. +/// +/// With `storage_dir` configured, batches that still fail after retries are written to a +/// [`FileCache`] and replayed oldest-first on the next tick (after a backoff) and on the next +/// launch; `shutdown` spills whatever is queued to disk *before* trying the network, so an app +/// being killed offline loses nothing. Throttled, rejected and disabled data is never persisted. +/// +/// Drive it with `spawn(exporter.run())` on the consumer's runtime. It stops after +/// [`Telemetry::shutdown`](crate::Telemetry::shutdown) or when the last +/// [`Telemetry`](crate::Telemetry) handle is dropped. +pub struct Exporter { + store: Arc, + transport: Arc, + config: Arc, + cache: Option>, + commands: mpsc::UnboundedReceiver, + silenced: bool, + replay_after: Option, + /// Batches the cache could not store (disk full, directory unusable). Drives one-shot logging. + persist_failures: u64, +} + +impl Exporter { + pub(crate) fn new( + store: Arc, + transport: Arc, + config: Arc, + cache: Option>, + commands: mpsc::UnboundedReceiver, + ) -> Self { + Self { + store, + transport, + config, + cache, + commands, + silenced: false, + replay_after: None, + persist_failures: 0, + } + } + + /// Run until shut down. Replays persisted batches, then exports on every tick and on demand. + pub async fn run(mut self) { + self.replay().await; + let mut ticker = interval(Duration::from_millis(self.config.flush_interval_ms.max(1))); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + tokio::select! { + _ = ticker.tick() => { + self.replay().await; + self.export_pending(false).await; + } + command = self.commands.recv() => match command { + Some(Command::Flush(done)) => { + self.export_pending(true).await; + let _ = done.send(()); + } + Some(Command::Shutdown(done)) => { + self.shutdown().await; + let _ = done.send(()); + return; + } + // Every `Telemetry` handle is gone. + None => { + self.shutdown().await; + return; + } + }, + } + } + } + + /// Final flush. With a cache: spill everything to disk first (cheap, local), then send as + /// much as the network allows — whatever remains is replayed on the next launch. + async fn shutdown(&mut self) { + if self.cache.is_some() && !self.silenced { + self.spill(); + self.replay_after = None; + self.replay().await; + } else { + self.export_pending(true).await; + } + } + + /// Export one batch, or everything queued when `drain_all` is set. + async fn export_pending(&mut self, drain_all: bool) { + loop { + let batch = self.store.drain(self.config.max_batch_size.max(1) as usize); + if batch.is_empty() { + return; + } + self.export(batch).await; + if !drain_all { + return; + } + } + } + + async fn export(&mut self, batch: Vec) { + let count = batch.len() as u64; + if self.silenced { + self.store.add_dropped(count); + return; + } + let body = otlp::encode_logs(&self.config.resource, batch); + match self.deliver(&body).await { + Delivery::Sent => {} + Delivery::Failed => { + self.persist_or_drop(&body, count); + // The network just failed; do not hammer it with the cache on the next tick. + self.replay_after = Some(Instant::now() + REPLAY_BACKOFF); + } + Delivery::Throttled | Delivery::Rejected => self.store.add_dropped(count), + Delivery::Disabled => { + self.silence(); + self.store.add_dropped(count); + } + } + } + + /// Encode everything queued straight to disk, without touching the network. + fn spill(&mut self) { + loop { + let batch = self.store.drain(self.config.max_batch_size.max(1) as usize); + if batch.is_empty() { + return; + } + let count = batch.len() as u64; + let body = otlp::encode_logs(&self.config.resource, batch); + self.persist_or_drop(&body, count); + } + } + + /// Send persisted batches oldest-first until one fails; then back off. + async fn replay(&mut self) { + let Some(cache) = self.cache.clone() else { return }; + if self.silenced || self.replay_after.is_some_and(|t| Instant::now() < t) { + return; + } + for path in cache.pending() { + let Ok(body) = cache.read(&path) else { + cache.remove(&path); + continue; + }; + match self.deliver(&body).await { + Delivery::Sent | Delivery::Rejected => cache.remove(&path), + Delivery::Failed | Delivery::Throttled => { + self.replay_after = Some(Instant::now() + REPLAY_BACKOFF); + return; + } + Delivery::Disabled => { + self.silence(); + return; + } + } + } + } + + fn persist_or_drop(&mut self, body: &[u8], count: u64) { + match self.cache.as_deref().map(|cache| cache.store(body)) { + Some(Ok(())) => log::debug!("telemetry: persisted {count} events for later delivery"), + Some(Err(err)) => { + // Disk full is a steady state, not an event: warn once, then stay quiet. + if self.persist_failures == 0 { + log::warn!( + "telemetry: cannot persist events ({err}); dropping until it recovers" + ); + } else { + log::debug!("telemetry: could not persist {count} events: {err}"); + } + self.persist_failures += 1; + self.store.add_dropped(count); + } + None => { + log::warn!("telemetry: export failed, dropping {count} events"); + self.store.add_dropped(count); + } + } + } + + /// Telemetry is disabled for this project: never send again, and never replay what is on disk. + fn silence(&mut self) { + log::warn!("telemetry disabled by the collector; going silent"); + self.silenced = true; + if let Some(cache) = &self.cache { + cache.clear(); + } + } + + /// Deliver one encoded batch with bounded retries and classify the outcome. + async fn deliver(&self, body: &[u8]) -> Delivery { + let mut headers = self.config.headers.clone(); + headers.insert("Content-Type".to_owned(), otlp::CONTENT_TYPE.to_owned()); + let request = + ExportRequest { url: self.config.endpoint.clone(), headers, body: body.to_vec() }; + let attempt_timeout = Duration::from_millis(self.config.export_timeout_ms.max(1)); + + // ponytail: linear backoff, 2 retries, blocks the tick loop while sleeping (bounded by + // MAX_RETRIES × MAX_RETRY_DELAY). Exponential + jitter once real fleets exercise this. + let mut throttled = false; + for attempt in 0..=MAX_RETRIES { + let delay = match timeout(attempt_timeout, self.transport.send(request.clone())).await { + Ok(Ok(())) => return Delivery::Sent, + Ok(Err(ExportError::Disabled)) => return Delivery::Disabled, + Ok(Err(ExportError::Rejected { message })) => { + log::warn!("telemetry batch rejected: {message}"); + return Delivery::Rejected; + } + Ok(Err(ExportError::Retryable { message, retry_after_ms })) => { + log::debug!("telemetry export failed (attempt {}): {message}", attempt + 1); + throttled = retry_after_ms.is_some(); + retry_after_ms + .map(Duration::from_millis) + .unwrap_or(RETRY_BACKOFF * (attempt + 1)) + } + Err(_) => { + log::debug!("telemetry export timed out (attempt {})", attempt + 1); + throttled = false; + RETRY_BACKOFF * (attempt + 1) + } + }; + if attempt < MAX_RETRIES { + sleep(delay.min(MAX_RETRY_DELAY)).await; + } + } + if throttled { + Delivery::Throttled + } else { + Delivery::Failed + } + } +} diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs new file mode 100644 index 000000000..fefbf0461 --- /dev/null +++ b/livekit-telemetry/src/lib.rs @@ -0,0 +1,47 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![doc = include_str!("../README.md")] + +/// Event data model: what SDKs push in. +mod event; + +/// Bounded in-memory queue between `emit` and the exporter. +mod store; + +/// Batch exporter actor: timer, OTLP encoding, retry policy. +mod exporter; + +/// OTLP/HTTP protobuf encoding of a batch. +mod otlp; + +/// On-disk cache of undeliverable batches. +mod persist; + +/// OTLP protobuf types (re-exported from `opentelemetry-proto`). +mod proto; + +/// Transport seam: how encoded batches leave the device. +mod transport; + +/// Entry point and configuration. +mod telemetry; + +pub use event::*; +pub use exporter::Exporter; +pub use telemetry::*; +pub use transport::*; + +#[cfg(feature = "uniffi")] +uniffi::setup_scaffolding!(); diff --git a/livekit-telemetry/src/otlp.rs b/livekit-telemetry/src/otlp.rs new file mode 100644 index 000000000..329888cb7 --- /dev/null +++ b/livekit-telemetry/src/otlp.rs @@ -0,0 +1,140 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use prost::Message; + +use crate::{ + event::now_unix_nanos, + proto::opentelemetry::proto::{ + collector::logs::v1::ExportLogsServiceRequest, + common::v1::{any_value, AnyValue, InstrumentationScope, KeyValue}, + logs::v1::{LogRecord, ResourceLogs, ScopeLogs, SeverityNumber}, + resource::v1::Resource, + }, + Attribute, AttributeValue, Severity, TelemetryEvent, +}; + +pub(crate) const CONTENT_TYPE: &str = "application/x-protobuf"; + +/// Encode one batch as an OTLP `ExportLogsServiceRequest`: one resource, one instrumentation +/// scope (this crate), one log record per event. +pub(crate) fn encode_logs(resource: &[Attribute], events: Vec) -> Vec { + ExportLogsServiceRequest { + resource_logs: vec![ResourceLogs { + resource: Some(Resource { + attributes: resource.iter().map(KeyValue::from).collect(), + ..Default::default() + }), + scope_logs: vec![ScopeLogs { + scope: Some(InstrumentationScope { + name: env!("CARGO_PKG_NAME").to_owned(), + version: env!("CARGO_PKG_VERSION").to_owned(), + ..Default::default() + }), + log_records: events.into_iter().map(LogRecord::from).collect(), + ..Default::default() + }], + ..Default::default() + }], + } + .encode_to_vec() +} + +impl From for LogRecord { + fn from(event: TelemetryEvent) -> Self { + let time_unix_nano = event.timestamp_ns.unwrap_or_else(now_unix_nanos); + LogRecord { + time_unix_nano, + observed_time_unix_nano: time_unix_nano, + severity_number: SeverityNumber::from(event.severity) as i32, + severity_text: severity_text(event.severity).to_owned(), + body: event + .body + .map(|text| AnyValue { value: Some(any_value::Value::StringValue(text)) }), + attributes: event.attributes.iter().map(KeyValue::from).collect(), + event_name: event.name, + ..Default::default() + } + } +} + +impl From for SeverityNumber { + fn from(severity: Severity) -> Self { + match severity { + Severity::Trace => SeverityNumber::Trace, + Severity::Debug => SeverityNumber::Debug, + Severity::Info => SeverityNumber::Info, + Severity::Warn => SeverityNumber::Warn, + Severity::Error => SeverityNumber::Error, + } + } +} + +fn severity_text(severity: Severity) -> &'static str { + match severity { + Severity::Trace => "TRACE", + Severity::Debug => "DEBUG", + Severity::Info => "INFO", + Severity::Warn => "WARN", + Severity::Error => "ERROR", + } +} + +impl From<&Attribute> for KeyValue { + fn from(attribute: &Attribute) -> Self { + let value = match &attribute.value { + AttributeValue::Str(s) => any_value::Value::StringValue(s.clone()), + AttributeValue::Int(i) => any_value::Value::IntValue(*i), + AttributeValue::Double(d) => any_value::Value::DoubleValue(*d), + AttributeValue::Bool(b) => any_value::Value::BoolValue(*b), + }; + KeyValue { + key: attribute.key.clone(), + value: Some(AnyValue { value: Some(value) }), + ..Default::default() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn encodes_events_as_otlp_log_records() { + let resource = [Attribute::new("service.name", "test")]; + let event = TelemetryEvent::new("lk.ping") + .with_severity(Severity::Warn) + .with_body("hi") + .with_attribute("lk.ping.seq", 7i64); + let bytes = encode_logs(&resource, vec![event]); + + let decoded = ExportLogsServiceRequest::decode(&bytes[..]).expect("valid OTLP"); + let resource_logs = &decoded.resource_logs[0]; + let res_attr = &resource_logs.resource.as_ref().expect("resource").attributes[0]; + assert_eq!(res_attr.key, "service.name"); + let scope_logs = &resource_logs.scope_logs[0]; + assert_eq!(scope_logs.scope.as_ref().expect("scope").name, "livekit-telemetry"); + let record = &scope_logs.log_records[0]; + assert_eq!(record.event_name, "lk.ping"); + assert_eq!(record.severity_number, SeverityNumber::Warn as i32); + assert_eq!(record.severity_text, "WARN"); + assert!(record.time_unix_nano > 0); + assert_eq!(record.attributes[0].key, "lk.ping.seq"); + assert_eq!( + record.attributes[0].value.as_ref().and_then(|v| v.value.clone()), + Some(any_value::Value::IntValue(7)) + ); + } +} diff --git a/livekit-telemetry/src/persist.rs b/livekit-telemetry/src/persist.rs new file mode 100644 index 000000000..e0883157f --- /dev/null +++ b/livekit-telemetry/src/persist.rs @@ -0,0 +1,198 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + fs, io, + path::{Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, +}; + +use crate::event::now_unix_nanos; + +const EXT: &str = "otlp"; +// ponytail: fixed 24 h max age (Datadog: 18 h, design doc: 24 h); a config knob if anyone asks. +const MAX_AGE_NANOS: u64 = 24 * 60 * 60 * 1_000_000_000; + +/// File cache of encoded batches the transport could not deliver. +/// +/// One file per `ExportLogsServiceRequest` body — the unit every mobile SDK persists (Sentry +/// envelopes, Datadog batch files, opentelemetry-android disk buffering): nothing to re-encode +/// on replay, and URL/headers are recomposed from the *current* config, so a rotated token is +/// picked up automatically. Files are written as `.tmp` and renamed into place so a crash never +/// leaves a half-written batch readable. Replay is oldest-first; eviction is drop-oldest above +/// `max_bytes`; batches older than 24 h are discarded. The age comes from the file name, not +/// from file timestamps (an Apple "required reason" API). +pub(crate) struct FileCache { + dir: PathBuf, + max_bytes: u64, + seq: AtomicU64, +} + +impl FileCache { + /// Open the cache directory (created if missing; its parent must exist) and discard stale + /// or half-written files. + pub fn open(dir: impl Into, max_bytes: u64) -> io::Result { + let dir = dir.into(); + // ponytail: `create_dir`, not `create_dir_all` — the recursive variant drags in + // `Path::components` machinery (~3 KiB) for a parent the host always provides. + match fs::create_dir(&dir) { + Err(err) if err.kind() != io::ErrorKind::AlreadyExists => return Err(err), + _ => {} + } + let cache = Self { dir, max_bytes, seq: AtomicU64::new(0) }; + cache.prune()?; + Ok(cache) + } + + /// Persist one encoded batch, evicting the oldest batches to stay within `max_bytes`. + /// + /// Fail-open: a full disk or a purged directory yields an error the caller counts as a + /// drop; it never leaves a partial `.tmp` behind. + pub fn store(&self, body: &[u8]) -> io::Result<()> { + let name = + format!("{:020}-{:06}", now_unix_nanos(), self.seq.fetch_add(1, Ordering::Relaxed)); + let tmp = self.dir.join(format!("{name}.tmp")); + let written = self.write_then_rename(&tmp, &self.dir.join(format!("{name}.{EXT}")), body); + if written.is_err() { + // ENOSPC leaves a truncated `.tmp`; drop it now rather than at the next launch. + let _ = fs::remove_file(&tmp); + } + written?; + self.prune() + } + + fn write_then_rename(&self, tmp: &Path, dest: &Path, body: &[u8]) -> io::Result<()> { + if let Err(err) = fs::write(tmp, body) { + // iOS may purge the whole Caches subdirectory while the app runs: recreate it once. + if err.kind() != io::ErrorKind::NotFound { + return Err(err); + } + fs::create_dir(&self.dir)?; + fs::write(tmp, body)?; + } + fs::rename(tmp, dest) + } + + /// Pending batches, oldest first. + pub fn pending(&self) -> Vec { + let Ok(entries) = fs::read_dir(&self.dir) else { return Vec::new() }; + let mut files: Vec = + entries.flatten().map(|e| e.path()).filter(|p| is_batch(p)).collect(); + files.sort_unstable(); + files + } + + pub fn read(&self, path: &Path) -> io::Result> { + fs::read(path) + } + + pub fn remove(&self, path: &Path) { + let _ = fs::remove_file(path); + } + + /// Discard every pending batch (telemetry disabled: nothing may be replayed later). + pub fn clear(&self) { + for path in self.pending() { + self.remove(&path); + } + } + + /// Delete stray `.tmp` files and batches older than the max age, then the oldest batches + /// until the total fits `max_bytes`. + fn prune(&self) -> io::Result<()> { + let now = now_unix_nanos(); + for entry in fs::read_dir(&self.dir)?.flatten() { + let path = entry.path(); + let expired = stamp(&path).is_some_and(|t| now.saturating_sub(t) > MAX_AGE_NANOS); + if !is_batch(&path) || expired { + let _ = fs::remove_file(&path); + } + } + let kept = self.pending(); + let sizes: Vec = + kept.iter().map(|p| fs::metadata(p).map(|m| m.len()).unwrap_or(0)).collect(); + let mut total: u64 = sizes.iter().sum(); + for (path, len) in kept.iter().zip(sizes) { + if total <= self.max_bytes { + break; + } + let _ = fs::remove_file(path); + total -= len; + } + Ok(()) + } +} + +fn is_batch(path: &Path) -> bool { + path.extension().is_some_and(|e| e == EXT) +} + +/// The creation time encoded in a batch file name. +fn stamp(path: &Path) -> Option { + path.file_stem()?.to_str()?.split('-').next()?.parse().ok() +} + +#[cfg(test)] +pub(crate) fn temp_dir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("livekit-telemetry-{tag}-{}", now_unix_nanos())); + let _ = fs::remove_dir_all(&dir); + dir +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn evicts_oldest_beyond_max_bytes_and_drops_stray_tmp() { + let dir = temp_dir("cache"); + let cache = FileCache::open(&dir, 25).expect("open"); + fs::write(dir.join("crashed.tmp"), b"half").expect("write"); + for body in [b"aaaaaaaaaa", b"bbbbbbbbbb", b"cccccccccc"] { + cache.store(body).expect("store"); + } + let pending = cache.pending(); + assert_eq!(pending.len(), 2, "10-byte batches under a 25-byte cap"); + assert_eq!(cache.read(&pending[0]).expect("read"), b"bbbbbbbbbb"); + assert!(!dir.join("crashed.tmp").exists() || cache.pending().len() == 2); + cache.clear(); + assert!(cache.pending().is_empty()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn recreates_a_purged_directory() { + let dir = temp_dir("purged"); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + fs::remove_dir_all(&dir).expect("purge like iOS does"); + cache.store(b"after purge").expect("store recreates the dir"); + assert_eq!(cache.pending().len(), 1); + let _ = fs::remove_dir_all(&dir); + } + + #[cfg(unix)] + #[test] + fn failed_write_leaves_no_partial_file() { + use std::os::unix::fs::PermissionsExt; + let dir = temp_dir("readonly"); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + // Stand-in for ENOSPC: any write into the directory fails. + fs::set_permissions(&dir, fs::Permissions::from_mode(0o555)).expect("chmod"); + let result = cache.store(b"no room"); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).expect("chmod back"); + assert!(result.is_err()); + assert_eq!(fs::read_dir(&dir).expect("dir").count(), 0, "no stray .tmp"); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/livekit-telemetry/src/proto/mod.rs b/livekit-telemetry/src/proto/mod.rs new file mode 100644 index 000000000..0baf3f364 --- /dev/null +++ b/livekit-telemetry/src/proto/mod.rs @@ -0,0 +1,26 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! OTLP protobuf types from the upstream `opentelemetry-proto` crate (prost-generated, +//! `gen-tonic-messages` only — no tonic/gRPC). The crate tracks a newer proto revision than +//! the OTLP 1.x stable surface, so construct its messages with `..Default::default()`. +//! +//! Only the types are used; the `opentelemetry`/`opentelemetry_sdk` crates it depends on are +//! dead code here and LTO removes them from release binaries (measured: +8 bytes on iOS). + +pub mod opentelemetry { + pub mod proto { + pub use opentelemetry_proto::tonic::{collector, common, logs, resource}; + } +} diff --git a/livekit-telemetry/src/store.rs b/livekit-telemetry/src/store.rs new file mode 100644 index 000000000..3ce60bf8b --- /dev/null +++ b/livekit-telemetry/src/store.rs @@ -0,0 +1,86 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + collections::VecDeque, + sync::{ + atomic::{AtomicU64, Ordering}, + Mutex, + }, +}; + +use crate::TelemetryEvent; + +/// Bounded FIFO of events waiting for export. +/// +/// When full, the *oldest* event is dropped so the freshest context survives a burst +/// (the queue role of OTel's `BatchLogRecordProcessor`, with drop-oldest instead of +/// drop-newest). Also keeps the pipeline's own health counter: events dropped anywhere. +// ponytail: one mutex around a VecDeque; a lock-free ring only if `emit` shows up in a profile. +pub(crate) struct Store { + queue: Mutex>, + capacity: usize, + dropped: AtomicU64, +} + +impl Store { + pub fn new(capacity: usize) -> Self { + Self { + queue: Mutex::new(VecDeque::with_capacity(capacity.min(1024))), + capacity, + dropped: AtomicU64::new(0), + } + } + + pub fn push(&self, event: TelemetryEvent) { + let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); + if queue.len() >= self.capacity { + queue.pop_front(); + self.dropped.fetch_add(1, Ordering::Relaxed); + } + queue.push_back(event); + } + + /// Remove and return up to `max` events, oldest first. + pub fn drain(&self, max: usize) -> Vec { + let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); + let n = max.min(queue.len()); + queue.drain(..n).collect() + } + + pub fn add_dropped(&self, n: u64) { + self.dropped.fetch_add(n, Ordering::Relaxed); + } + + pub fn dropped(&self) -> u64 { + self.dropped.load(Ordering::Relaxed) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn drops_oldest_when_full() { + let store = Store::new(2); + for name in ["a", "b", "c"] { + store.push(TelemetryEvent::new(name)); + } + let names: Vec<_> = store.drain(10).into_iter().map(|e| e.name).collect(); + assert_eq!(names, ["b", "c"]); + assert_eq!(store.dropped(), 1); + assert!(store.drain(10).is_empty()); + } +} diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs new file mode 100644 index 000000000..e8da37db1 --- /dev/null +++ b/livekit-telemetry/src/telemetry.rs @@ -0,0 +1,350 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use livekit_runtime::timeout; +use tokio::sync::{mpsc, oneshot}; + +use crate::{ + event::now_unix_nanos, exporter::Command, persist::FileCache, store::Store, Attribute, + Exporter, TelemetryEvent, TelemetryTransport, +}; + +/// Pipeline configuration. +/// +/// Defaults follow OTel's `BatchLogRecordProcessor` (1 s delay, 2048 queue, 512 batch); +/// production tuning (e.g. a 15 s cadence) is a config change, not a code change. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone)] +pub struct TelemetryConfig { + /// Full OTLP/HTTP logs URL: `http://localhost:4318/v1/logs` locally, + /// `https:///observability/logs/otlp/v0` for LiveKit Cloud. + pub endpoint: String, + /// Extra request headers, e.g. `Authorization: Bearer `. + pub headers: HashMap, + /// Resource attributes describing the emitter (`service.name`, `os.name`, + /// `device.model.identifier`, `session.id`, …). `telemetry.sdk.*` are filled in by the core. + pub resource: Vec, + /// Directory for the on-disk cache of undeliverable batches (created if missing). `None` + /// disables persistence: batches that fail after retries are dropped. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub storage_dir: Option, + /// Cap on the on-disk cache; the oldest batches are evicted first. + #[cfg_attr(feature = "uniffi", uniffi(default = 4194304))] + pub max_storage_bytes: u64, + #[cfg_attr(feature = "uniffi", uniffi(default = 1000))] + pub flush_interval_ms: u64, + /// Events buffered before the oldest are dropped. + #[cfg_attr(feature = "uniffi", uniffi(default = 2048))] + pub max_queue_size: u32, + /// Events per export request. + #[cfg_attr(feature = "uniffi", uniffi(default = 512))] + pub max_batch_size: u32, + /// Bound on a single transport attempt, and on `shutdown`. + #[cfg_attr(feature = "uniffi", uniffi(default = 10000))] + pub export_timeout_ms: u64, +} + +impl TelemetryConfig { + /// Defaults for the given endpoint; no persistence. + pub fn new(endpoint: impl Into) -> Self { + Self { + endpoint: endpoint.into(), + headers: HashMap::new(), + resource: Vec::new(), + storage_dir: None, + max_storage_bytes: 4 * 1024 * 1024, + flush_interval_ms: 1000, + max_queue_size: 2048, + max_batch_size: 512, + export_timeout_ms: 10_000, + } + } +} + +/// Entry point: the synchronous, never-blocking side SDKs push into. +/// +/// Fail-open by design: [`emit`](Self::emit) cannot fail or block — when the queue is full the +/// oldest event is dropped and counted in [`dropped_count`](Self::dropped_count). Cheap to +/// clone; every clone feeds the same pipeline. +/// +/// ``` +/// # use std::sync::Arc; +/// # use livekit_telemetry::*; +/// # struct Discard; +/// # #[async_trait::async_trait] +/// # impl TelemetryTransport for Discard { +/// # async fn send(&self, _: ExportRequest) -> Result<(), ExportError> { Ok(()) } +/// # } +/// # #[tokio::main(flavor = "current_thread")] async fn main() { +/// let (telemetry, exporter) = +/// Telemetry::new(TelemetryConfig::new("http://localhost:4318/v1/logs"), Arc::new(Discard)); +/// tokio::spawn(exporter.run()); +/// +/// telemetry.emit(TelemetryEvent::new("lk.ping")); +/// telemetry.shutdown().await; +/// # } +/// ``` +#[derive(Clone)] +pub struct Telemetry { + store: Arc, + config: Arc, + commands: mpsc::UnboundedSender, +} + +impl Telemetry { + /// Build the pipeline. Spawn the returned [`Exporter`] with `exporter.run()` on your runtime. + /// + /// An unusable `storage_dir` is logged and persistence is skipped — never an error. + pub fn new( + mut config: TelemetryConfig, + transport: Arc, + ) -> (Self, Exporter) { + add_sdk_resource(&mut config.resource); + let cache = config.storage_dir.as_deref().and_then(|dir| { + FileCache::open(dir, config.max_storage_bytes) + .map(Arc::new) + .map_err(|err| { + log::warn!("telemetry: cannot use storage dir {dir}: {err}; not persisting") + }) + .ok() + }); + let config = Arc::new(config); + let store = Arc::new(Store::new(config.max_queue_size.max(1) as usize)); + let (commands, receiver) = mpsc::unbounded_channel(); + let exporter = Exporter::new(store.clone(), transport, config.clone(), cache, receiver); + (Self { store, config, commands }, exporter) + } + + /// Queue an event for export. Stamps it with the current time unless it carries one. + pub fn emit(&self, mut event: TelemetryEvent) { + if event.timestamp_ns.is_none() { + event.timestamp_ns = Some(now_unix_nanos()); + } + self.store.push(event); + } + + /// Export everything queued and wait until the transport accepted (or the exporter gave up on) it. + pub async fn flush(&self) { + self.command(Command::Flush).await; + } + + /// Flush, then stop the exporter. Bounded by `export_timeout_ms`; events emitted afterwards + /// are never exported. With `storage_dir` set, queued events are written to disk before the + /// network is tried, so nothing is lost if the process dies mid-way. + pub async fn shutdown(&self) { + let bound = Duration::from_millis(self.config.export_timeout_ms.max(1)); + let _ = timeout(bound, self.command(Command::Shutdown)).await; + } + + /// Events dropped so far (queue overflow, rejected/throttled/failed exports, disabled collector). + pub fn dropped_count(&self) -> u64 { + self.store.dropped() + } + + async fn command(&self, make: impl FnOnce(oneshot::Sender<()>) -> Command) { + let (done, wait) = oneshot::channel(); + if self.commands.send(make(done)).is_ok() { + let _ = wait.await; + } + } +} + +/// Fill in the `telemetry.sdk.*` resource attributes and a fallback `service.name`. +fn add_sdk_resource(resource: &mut Vec) { + let defaults = [ + ("service.name", "livekit-client"), + ("telemetry.sdk.name", env!("CARGO_PKG_NAME")), + ("telemetry.sdk.language", "rust"), + ("telemetry.sdk.version", env!("CARGO_PKG_VERSION")), + ]; + for (key, value) in defaults { + if !resource.iter().any(|a| a.key == key) { + resource.push(Attribute::new(key, value)); + } + } +} + +#[cfg(test)] +mod tests { + use std::{collections::VecDeque, fs, path::Path, sync::Mutex}; + + use prost::Message; + + use super::*; + use crate::{ + persist::temp_dir, + proto::opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest, ExportError, + ExportRequest, + }; + + #[derive(Default)] + struct FakeTransport { + requests: Mutex>, + script: Mutex>>, + } + + impl FakeTransport { + fn scripted(results: impl IntoIterator>) -> Arc { + Arc::new(Self { + script: Mutex::new(results.into_iter().collect()), + ..Default::default() + }) + } + fn sent(&self) -> Vec { + self.requests.lock().expect("lock").clone() + } + } + + #[async_trait::async_trait] + impl TelemetryTransport for FakeTransport { + async fn send(&self, request: ExportRequest) -> Result<(), ExportError> { + self.requests.lock().expect("lock").push(request); + self.script.lock().expect("lock").pop_front().unwrap_or(Ok(())) + } + } + + fn offline() -> Result<(), ExportError> { + Err(ExportError::Retryable { message: "offline".into(), retry_after_ms: None }) + } + + fn pipeline(transport: Arc) -> Telemetry { + start(TelemetryConfig::new("http://collector/v1/logs"), transport) + } + + fn persisted_pipeline(transport: Arc, dir: &Path) -> Telemetry { + let mut config = TelemetryConfig::new("http://collector/v1/logs"); + config.storage_dir = Some(dir.to_string_lossy().into_owned()); + start(config, transport) + } + + fn start(config: TelemetryConfig, transport: Arc) -> Telemetry { + let (telemetry, exporter) = Telemetry::new(config, transport); + tokio::spawn(exporter.run()); + telemetry + } + + fn files_in(dir: &Path) -> usize { + fs::read_dir(dir).map(|d| d.count()).unwrap_or(0) + } + + #[tokio::test(start_paused = true)] + async fn batches_events_into_one_otlp_request() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + for _ in 0..3 { + telemetry.emit(TelemetryEvent::new("lk.ping")); + } + telemetry.flush().await; + + let sent = transport.sent(); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].url, "http://collector/v1/logs"); + assert_eq!(sent[0].headers["Content-Type"], "application/x-protobuf"); + let decoded = ExportLogsServiceRequest::decode(&sent[0].body[..]).expect("valid OTLP"); + let records = &decoded.resource_logs[0].scope_logs[0].log_records; + assert_eq!(records.len(), 3); + assert!(records.iter().all(|r| r.event_name == "lk.ping" && r.time_unix_nano > 0)); + let resource = decoded.resource_logs[0].resource.as_ref().expect("resource"); + assert!(resource.attributes.iter().any(|kv| kv.key == "telemetry.sdk.name")); + assert_eq!(telemetry.dropped_count(), 0); + } + + #[tokio::test(start_paused = true)] + async fn retries_transient_failures_then_drops_without_storage() { + let transport = FakeTransport::scripted([offline(), offline(), offline()]); + let telemetry = pipeline(transport.clone()); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + + assert_eq!(transport.sent().len(), 3); + assert_eq!(telemetry.dropped_count(), 1); + } + + #[tokio::test(start_paused = true)] + async fn rejected_batch_is_dropped_without_retry() { + let transport = + FakeTransport::scripted([Err(ExportError::Rejected { message: "400".into() })]); + let telemetry = pipeline(transport.clone()); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + + assert_eq!(transport.sent().len(), 1); + assert_eq!(telemetry.dropped_count(), 1); + } + + #[tokio::test(start_paused = true)] + async fn disabled_collector_silences_the_exporter() { + let transport = FakeTransport::scripted([Err(ExportError::Disabled)]); + let telemetry = pipeline(transport.clone()); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.shutdown().await; + + assert_eq!(transport.sent().len(), 1); + assert_eq!(telemetry.dropped_count(), 2); + } + + #[tokio::test(start_paused = true)] + async fn failed_batch_is_persisted_and_replayed_on_next_start() { + let dir = temp_dir("replay"); + let first_transport = FakeTransport::scripted(std::iter::repeat_with(offline).take(12)); + let first = persisted_pipeline(first_transport.clone(), &dir); + first.emit(TelemetryEvent::new("lk.ping")); + first.flush().await; + assert_eq!(first_transport.sent().len(), 3); + assert_eq!(first.dropped_count(), 0, "persisted, not dropped"); + assert_eq!(files_in(&dir), 1); + + let second_transport = FakeTransport::scripted([]); + let second = persisted_pipeline(second_transport.clone(), &dir); + second.flush().await; + let sent = second_transport.sent(); + assert_eq!(sent.len(), 1, "replayed on start"); + let decoded = ExportLogsServiceRequest::decode(&sent[0].body[..]).expect("valid OTLP"); + assert_eq!(decoded.resource_logs[0].scope_logs[0].log_records[0].event_name, "lk.ping"); + assert_eq!(files_in(&dir), 0); + let _ = fs::remove_dir_all(&dir); + } + + #[tokio::test(start_paused = true)] + async fn throttled_batch_is_dropped_not_persisted() { + let dir = temp_dir("throttle"); + let throttled = + || Err(ExportError::Retryable { message: "429".into(), retry_after_ms: Some(1000) }); + let transport = FakeTransport::scripted([throttled(), throttled(), throttled()]); + let telemetry = persisted_pipeline(transport.clone(), &dir); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + + assert_eq!(telemetry.dropped_count(), 1); + assert_eq!(files_in(&dir), 0); + let _ = fs::remove_dir_all(&dir); + } + + #[tokio::test(start_paused = true)] + async fn shutdown_offline_keeps_queue_on_disk() { + let dir = temp_dir("spill"); + let transport = FakeTransport::scripted(std::iter::repeat_with(offline).take(12)); + let telemetry = persisted_pipeline(transport.clone(), &dir); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.shutdown().await; + + assert_eq!(files_in(&dir), 1, "spilled before the network was tried, kept after it failed"); + assert_eq!(telemetry.dropped_count(), 0); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/livekit-telemetry/src/transport.rs b/livekit-telemetry/src/transport.rs new file mode 100644 index 000000000..fce6f0915 --- /dev/null +++ b/livekit-telemetry/src/transport.rs @@ -0,0 +1,120 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; + +/// One fully composed OTLP/HTTP export request. +/// +/// The core fills in the URL, the headers (content type, auth, …) and the protobuf body; +/// a transport only moves the bytes. Non-HTTP transports (e.g. a data channel) may ignore +/// `url`/`headers` and forward `body`, which is a standard `ExportLogsServiceRequest`. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct ExportRequest { + pub url: String, + pub headers: HashMap, + pub body: Vec, +} + +/// Why a transport could not deliver an [`ExportRequest`]. Drives the exporter's +/// retry / drop / go-silent decision (OTLP/HTTP failure semantics). +#[cfg_attr(feature = "uniffi", derive(uniffi::Error))] +#[derive(Debug, Clone, PartialEq, thiserror::Error)] +pub enum ExportError { + /// Transient failure (network error, timeout, HTTP 429/502/503/504). The batch is + /// retried, honoring `retry_after_ms` when the collector sent `Retry-After`. + #[error("retryable export error: {message}")] + Retryable { message: String, retry_after_ms: Option }, + /// The collector rejected the payload (any other 4xx/5xx). The batch is dropped. + #[error("export rejected: {message}")] + Rejected { message: String }, + /// Telemetry is disabled for this project. The exporter goes silent for good. + #[error("telemetry disabled by the collector")] + Disabled, +} + +impl ExportError { + /// Classify an HTTP status per OTLP/HTTP: 2xx ok, 429/502/503/504 retryable, else rejected. + pub fn from_status(status: u16, retry_after_ms: Option) -> Result<(), Self> { + match status { + 200..=299 => Ok(()), + 429 | 502 | 503 | 504 => { + Err(Self::Retryable { message: format!("HTTP {status}"), retry_after_ms }) + } + _ => Err(Self::Rejected { message: format!("HTTP {status}") }), + } + } +} + +/// Moves an encoded batch off the device. +/// +/// Implemented in Rust ([`NetTransport`], feature `net`) or by the host platform +/// (URLSession, OkHttp, a data channel, …) through UniFFI. Implementations must not retry +/// themselves: the exporter owns the retry policy. +#[cfg_attr(feature = "uniffi", uniffi::export(with_foreign))] +#[async_trait::async_trait] +pub trait TelemetryTransport: Send + Sync { + async fn send(&self, request: ExportRequest) -> Result<(), ExportError>; +} + +#[cfg(feature = "net")] +mod net { + use std::sync::Arc; + + use livekit_net::{Header, HttpClient, HttpClientExt, TransportError}; + + use super::{ExportError, ExportRequest, TelemetryTransport}; + + /// Default transport: HTTP POST through a `livekit-net` client — the native backend, or + /// whatever the host registered with `livekit_net::set_http_client`. + pub struct NetTransport(Arc); + + impl NetTransport { + pub fn new(client: Arc) -> Self { + Self(client) + } + + /// Resolve the process-wide `livekit-net` client; `None` when none is available. + pub fn from_registry() -> Option { + livekit_net::http_client().map(Self) + } + } + + #[async_trait::async_trait] + impl TelemetryTransport for NetTransport { + async fn send(&self, request: ExportRequest) -> Result<(), ExportError> { + let headers = + request.headers.into_iter().map(|(name, value)| Header { name, value }).collect(); + let response = + self.0.post(request.url, headers, request.body).await.map_err(|err| match err { + TransportError::Http { status } => ExportError::from_status(status, None) + .err() + .unwrap_or(ExportError::Rejected { message: format!("HTTP {status}") }), + other => { + ExportError::Retryable { message: other.to_string(), retry_after_ms: None } + } + })?; + let retry_after_ms = response + .headers + .iter() + .find(|h| h.name.eq_ignore_ascii_case("retry-after")) + .and_then(|h| h.value.trim().parse::().ok()) + .map(|seconds| seconds * 1000); + ExportError::from_status(response.status, retry_after_ms) + } + } +} + +#[cfg(feature = "net")] +pub use net::NetTransport; diff --git a/livekit-telemetry/uniffi.toml b/livekit-telemetry/uniffi.toml new file mode 100644 index 000000000..2dfff11df --- /dev/null +++ b/livekit-telemetry/uniffi.toml @@ -0,0 +1,2 @@ +[bindings.swift] +ffi_module_name = "RustLiveKitTelemetry" From 644c51474bfa74a24e09ee2046e39081ee12f973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:06:19 +0200 Subject: [PATCH 03/50] feat(uniffi): expose Telemetry over UniFFI Thin uniffi::Object wrapper around livekit_telemetry::Telemetry that spawns the exporter on the global tokio runtime. Hosts pass a TelemetryConfig record and implement the async TelemetryTransport foreign trait (e.g. a URLSession or OkHttp POST); TelemetryEvent/Attribute/Severity/ExportError cross the boundary as records and enums from the livekit-telemetry component. Co-Authored-By: Claude Fable 5 --- .changeset/uniffi-telemetry.md | 6 ++++ Cargo.lock | 1 + livekit-uniffi/Cargo.toml | 1 + livekit-uniffi/src/lib.rs | 3 ++ livekit-uniffi/src/telemetry.rs | 57 +++++++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+) create mode 100644 .changeset/uniffi-telemetry.md create mode 100644 livekit-uniffi/src/telemetry.rs diff --git a/.changeset/uniffi-telemetry.md b/.changeset/uniffi-telemetry.md new file mode 100644 index 000000000..5ce29ea43 --- /dev/null +++ b/.changeset/uniffi-telemetry.md @@ -0,0 +1,6 @@ +--- +livekit-uniffi: minor +--- + +Expose the client telemetry core over UniFFI: `Telemetry` (emit / flush / shutdown / +droppedCount) with a host-implemented `TelemetryTransport`. diff --git a/Cargo.lock b/Cargo.lock index f874f3cdb..df01e3fd4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4121,6 +4121,7 @@ dependencies = [ "livekit-common", "livekit-datatrack", "livekit-protocol", + "livekit-telemetry", "livekit-token", "log", "once_cell", diff --git a/livekit-uniffi/Cargo.toml b/livekit-uniffi/Cargo.toml index 28b4c7fbd..d5fa1cbf6 100644 --- a/livekit-uniffi/Cargo.toml +++ b/livekit-uniffi/Cargo.toml @@ -16,6 +16,7 @@ livekit-protocol = { workspace = true } livekit-common = { workspace = true, features = ["uniffi"] } livekit-token = { workspace = true } livekit-datatrack = { workspace = true, features = ["uniffi"] } +livekit-telemetry = { workspace = true, features = ["uniffi"] } uniffi = { workspace = true, features = ["scaffolding-ffi-buffer-fns", "tokio"] } log = { workspace = true } tokio = { workspace = true, features = ["sync", "rt-multi-thread"] } diff --git a/livekit-uniffi/src/lib.rs b/livekit-uniffi/src/lib.rs index 94f2cc0d0..e105ef331 100644 --- a/livekit-uniffi/src/lib.rs +++ b/livekit-uniffi/src/lib.rs @@ -18,6 +18,9 @@ pub mod data_track; /// Access token generation and verification from [`livekit-api::access_token`]. pub mod access_token; +/// Client telemetry core from [`livekit-telemetry`]. +pub mod telemetry; + /// Forward log messages from Rust. pub mod log_forward; diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs new file mode 100644 index 000000000..64012ffa4 --- /dev/null +++ b/livekit-uniffi/src/telemetry.rs @@ -0,0 +1,57 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Client telemetry core from the [`livekit-telemetry`] crate. +//! +//! FFI clients construct one [`Telemetry`] per pipeline with a host-implemented +//! `TelemetryTransport` (e.g. a URLSession/OkHttp POST), then `emit` from any thread. The +//! exporter runs on the global runtime; `shutdown` flushes within `export_timeout_ms`. + +use std::sync::Arc; + +use livekit_telemetry::{TelemetryConfig, TelemetryEvent, TelemetryTransport}; + +/// Telemetry pipeline: buffer, batch and export events as OTLP. +#[derive(uniffi::Object)] +pub struct Telemetry(livekit_telemetry::Telemetry); + +#[uniffi::export(async_runtime = "tokio")] +impl Telemetry { + #[uniffi::constructor] + pub fn new(config: TelemetryConfig, transport: Arc) -> Arc { + let (telemetry, exporter) = livekit_telemetry::Telemetry::new(config, transport); + crate::runtime::runtime().spawn(exporter.run()); + Arc::new(Self(telemetry)) + } + + /// Queue an event for export. Never blocks; drops the oldest event when the queue is full. + pub fn emit(&self, event: TelemetryEvent) { + self.0.emit(event); + } + + /// Export everything queued and wait for the transport. + pub async fn flush(&self) { + self.0.flush().await; + } + + /// Flush, then stop exporting. Bounded by `export_timeout_ms`. + pub async fn shutdown(&self) { + self.0.shutdown().await; + } + + /// Events dropped so far (queue overflow, rejected/failed exports, disabled collector). + pub fn dropped_count(&self) -> u64 { + self.0.dropped_count() + } +} From 204c3e6de0e0420c9299f0b971f85cf18dfeddd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:06:19 +0200 Subject: [PATCH 04/50] feat(examples): add telemetry_ping Sends one `lk.ping` to an OTLP/HTTP collector (default: a local grafana/otel-lgtm on :4318) through NetTransport. LK_OTLP_ENDPOINT overrides the URL; LK_TELEMETRY_DIR enables the on-disk cache so the offline -> online replay can be exercised by stopping and starting the collector. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 10 +++++++ Cargo.toml | 1 + examples/telemetry_ping/Cargo.toml | 11 ++++++++ examples/telemetry_ping/src/main.rs | 44 +++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+) create mode 100644 examples/telemetry_ping/Cargo.toml create mode 100644 examples/telemetry_ping/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index df01e3fd4..fe04ee261 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7205,6 +7205,16 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +[[package]] +name = "telemetry_ping" +version = "0.1.0" +dependencies = [ + "env_logger 0.11.11", + "livekit-net", + "livekit-telemetry", + "tokio", +] + [[package]] name = "tempfile" version = "3.27.0" diff --git a/Cargo.toml b/Cargo.toml index 72e448332..9799089a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ members = [ "examples/rpc", "examples/save_to_disk", "examples/screensharing", + "examples/telemetry_ping", "examples/token_source", "examples/webhooks", ] diff --git a/examples/telemetry_ping/Cargo.toml b/examples/telemetry_ping/Cargo.toml new file mode 100644 index 000000000..41a1e4848 --- /dev/null +++ b/examples/telemetry_ping/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "telemetry_ping" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +livekit-telemetry = { workspace = true, features = ["net"] } +livekit-net = { workspace = true, features = ["native-tokio"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } +env_logger = { workspace = true } diff --git a/examples/telemetry_ping/src/main.rs b/examples/telemetry_ping/src/main.rs new file mode 100644 index 000000000..49907d1c5 --- /dev/null +++ b/examples/telemetry_ping/src/main.rs @@ -0,0 +1,44 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Sends a single `lk.ping` event to an OTLP/HTTP collector (default: local grafana/otel-lgtm). + +use std::{env, sync::Arc}; + +use livekit_telemetry::{Attribute, NetTransport, Telemetry, TelemetryConfig, TelemetryEvent}; + +#[tokio::main] +async fn main() { + env_logger::init(); + let endpoint = + env::var("LK_OTLP_ENDPOINT").unwrap_or_else(|_| "http://localhost:4318/v1/logs".to_owned()); + + let mut config = TelemetryConfig::new(&endpoint); + config.resource.push(Attribute::new("service.name", "telemetry_ping")); + config.resource.push(Attribute::new("os.name", env::consts::OS)); + // Optional on-disk cache: run once with the collector down, once with it up. + config.storage_dir = env::var("LK_TELEMETRY_DIR").ok(); + let transport = NetTransport::from_registry().expect("livekit-net has no HTTP client"); + + let (telemetry, exporter) = Telemetry::new(config, Arc::new(transport)); + tokio::spawn(exporter.run()); + + telemetry.emit( + TelemetryEvent::new("lk.ping") + .with_body("hello from livekit-telemetry") + .with_attribute("lk.ping.seq", 1i64), + ); + telemetry.shutdown().await; + println!("sent lk.ping to {endpoint} (dropped: {})", telemetry.dropped_count()); +} From a9ef90820a8a3299aedea4158ed9bc982ddf6f9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:23:36 +0200 Subject: [PATCH 05/50] refactor(telemetry): write-ahead BatchCache between exporter and transport Persistence was write-behind: only batches that failed after retries were written, plus a spill on shutdown. That survives being offline or killed but not a crash, which loses up to a flush interval of events - the seconds a crash investigation wants. Every client SDK surveyed (Sentry envelopes, Datadog batch files, opentelemetry-android disk buffering) writes before sending, so the cache is now the queue between exporter and transport. - BatchCache trait: push(id, body) / pending / read / remove / clear. Ids are exporter-minted `--`: sortable, age-bearing, countable. - MemoryCache (default): failed uploads wait for the next attempt instead of being dropped after three tries. FileCache: the former persist.rs, selected by `storage_dir`. Telemetry::with_cache accepts any other impl. - Exporter collapses to one path: enqueue (encode -> cache), then upload (oldest-first -> transport -> remove); shutdown only ignores the upload backoff. Retry-After keeps cached batches but drops new ones for its window; Disabled empties the cache. - max_storage_bytes -> max_cache_bytes (applies to both implementations). ios-arm64 UniFFI dylib: 1,193,368 B (+16.4 KB over write-behind), 13.7 KiB over SPM_SIZE_LIMIT_BYTES - the gate bump is left as a separate decision. Co-Authored-By: Claude Fable 5 --- .changeset/add-livekit-telemetry-crate.md | 5 +- livekit-telemetry/README.md | 36 +-- livekit-telemetry/src/cache.rs | 299 ++++++++++++++++++++++ livekit-telemetry/src/exporter.rs | 247 ++++++++---------- livekit-telemetry/src/lib.rs | 5 +- livekit-telemetry/src/persist.rs | 198 -------------- livekit-telemetry/src/telemetry.rs | 138 +++++++--- 7 files changed, 524 insertions(+), 404 deletions(-) create mode 100644 livekit-telemetry/src/cache.rs delete mode 100644 livekit-telemetry/src/persist.rs diff --git a/.changeset/add-livekit-telemetry-crate.md b/.changeset/add-livekit-telemetry-crate.md index b78f26221..0739e8037 100644 --- a/.changeset/add-livekit-telemetry-crate.md +++ b/.changeset/add-livekit-telemetry-crate.md @@ -3,5 +3,6 @@ livekit-telemetry: minor --- Add `livekit-telemetry`, the shared client telemetry core: events are buffered on-device, -batched and exported as OTLP/HTTP log records through a pluggable `TelemetryTransport`, with an -optional on-disk cache of undeliverable batches. +batched and exported as OTLP/HTTP log records through a pluggable `TelemetryTransport`. Batches are +written to a `BatchCache` (in memory by default, on disk with `storage_dir`) before upload, so +failed uploads, crashes and offline shutdowns lose nothing. diff --git a/livekit-telemetry/README.md b/livekit-telemetry/README.md index 5e04e91de..453e63d6a 100644 --- a/livekit-telemetry/README.md +++ b/livekit-telemetry/README.md @@ -9,20 +9,17 @@ once — batching, encoding, retry, persistence, go-silent — while platforms p they are uniquely placed to: instruments (OS signals) and the byte-moving transport. ```text - SDK / instruments ──emit()──▶ Telemetry ──▶ Store ──drain──▶ Exporter ──ExportRequest──▶ TelemetryTransport - (source) (bounded queue) (tick · OTLP encode · retry) (NetTransport | host HTTP | data channel …) - │ failed after retries - ▼ - FileCache (one file per encoded batch, replayed oldest-first) + SDK / instruments ──emit()──▶ Telemetry ──▶ Store ──drain──▶ Exporter ──push──▶ BatchCache ──upload oldest-first──▶ TelemetryTransport + (source) (events) (tick · encode) (MemoryCache | FileCache | yours) (NetTransport | host HTTP | data channel …) ``` | Layer | OTel equivalent | Type | |---|---|---| | source | `Logger.emit` | [`Telemetry::emit`] with [`TelemetryEvent`] | -| store | `BatchLogRecordProcessor` queue | `Store` (in-memory, drop-oldest) | +| store | `BatchLogRecordProcessor` queue | `Store` (in-memory events, drop-oldest) | | sink | `BatchLogRecordProcessor` timer + `LogRecordExporter` | [`Exporter`] (actor, spawn `run()`) | +| cache | disk-buffering exporter wrapper | [`BatchCache`] trait: [`MemoryCache`] (default), [`FileCache`] (`storage_dir`), or your own via [`Telemetry::with_cache`] | | transport | exporter's HTTP client | [`TelemetryTransport`] trait; `NetTransport` over `livekit-net` (feature `net`) | -| persistence | disk-buffering exporter wrapper | `FileCache` (opt-in via `storage_dir`) | ## Usage @@ -36,7 +33,7 @@ let (telemetry, exporter) = Telemetry::new(config, Arc::new(transport)); tokio::spawn(exporter.run()); telemetry.emit(TelemetryEvent::new("lk.ping").with_attribute("lk.ping.seq", 1i64)); -telemetry.shutdown().await; // spill to disk, then flush; bounded by `export_timeout_ms` +telemetry.shutdown().await; // cache, then upload what the network allows; bounded by `export_timeout_ms` ``` Event names and attributes are defined in [`SPEC.md`](SPEC.md). @@ -58,16 +55,19 @@ start it again and run once more to watch the cached batch replay. - **Typed events in the store, OTLP at the edge.** Events are stored as [`TelemetryEvent`], not pre-serialized bytes: batches need grouping under one resource/scope, batch-time attributes can be attached, and a human-readable dump is a `Debug`/JSON view away. -- **Persistence is a file cache at the exporter, not a database.** What every client SDK does: - Sentry (envelope files, capped count, oldest evicted), Datadog (batch files, 5 MB/file, - 18 h max age), opentelemetry-android disk-buffering (`.tmp` → rename, 1 MB files, 10 MB dir), - Amplitude/Segment (delimited JSON files, ~1 MB / 475 KB rotation). We persist the *encoded* - request body: nothing to re-encode on replay, URL/headers recomposed from the current config - (rotated tokens just work), zero dependencies. Throttled (`Retry-After`), rejected and - disabled data is never written (persistence must not become a disk-backed retry queue). - `shutdown` spills the queue to disk before trying the network, so an app killed offline - loses nothing. Age is derived from file names, not file timestamps (an Apple - required-reason API). +- **Write-ahead cache between exporter and transport.** Every batch is encoded and pushed to + the [`BatchCache`] *before* the network is tried, then uploaded oldest-first and removed on + success — the shape every client SDK converges on (Sentry envelopes, Datadog batch files, + opentelemetry-android disk buffering, Amplitude/Segment event files). The exporter only knows + the trait: [`MemoryCache`] by default (failed uploads wait for the next attempt, lost with the + process), [`FileCache`] with `storage_dir` (survives crashes; the next launch replays), or a + caller-provided implementation. Cached bodies are ready-to-send OTLP with URL/headers + recomposed at upload time, so a rotated token just works. A failed upload pauses the cache + for a minute; a `Retry-After` keeps cached batches but drops new ones for its duration + (throttling must not become a disk-backed queue); `Disabled` empties the cache. `FileCache` + writes `.tmp` → rename, evicts oldest above `max_cache_bytes`, expires after 24 h using the + timestamp in the file name (not file metadata — an Apple required-reason API), and treats a + full disk as a counted drop with a single warning. - **Transport is the only injection point.** The core composes URL, headers and body; the transport moves bytes and reports [`ExportError`] so the core alone decides retry / drop / persist / go-silent. No Rust HTTP/TLS stack is linked unless the `net` feature is enabled. diff --git a/livekit-telemetry/src/cache.rs b/livekit-telemetry/src/cache.rs new file mode 100644 index 000000000..aaeba6248 --- /dev/null +++ b/livekit-telemetry/src/cache.rs @@ -0,0 +1,299 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + fs, io, + path::{Path, PathBuf}, + sync::Mutex, +}; + +/// Queue of encoded OTLP batches between the [`Exporter`](crate::Exporter) and the transport. +/// +/// The exporter writes every batch here *before* trying the network (write-ahead), uploads +/// oldest-first, and removes what the collector accepted or rejected. Two implementations ship: +/// [`MemoryCache`] (default) and [`FileCache`] (survives crashes and restarts); anything else — +/// a database, a platform store — plugs in through +/// [`Telemetry::with_cache`](crate::Telemetry::with_cache). +/// +/// Ids are chosen by the exporter as `--`: sortable, so +/// [`pending`](Self::pending) is a plain sort, and prefixed with the creation time so an +/// implementation can expire old batches without touching file timestamps. Implementations +/// bound their own footprint by evicting the oldest batches; the exporter never sees eviction. +pub trait BatchCache: Send + Sync { + /// Store one encoded batch under `id`. + fn push(&self, id: &str, body: &[u8]) -> io::Result<()>; + /// Ids of stored batches, oldest first. + fn pending(&self) -> Vec; + /// The body stored under `id`, if it is still there. + fn read(&self, id: &str) -> Option>; + fn remove(&self, id: &str); + /// Discard everything (telemetry disabled: nothing may be replayed later). + fn clear(&self); +} + +/// In-memory [`BatchCache`]: batches that could not be uploaded wait for the next attempt, +/// bounded by `max_bytes` (oldest evicted). Lost with the process. +// ponytail: a Vec with remove(0) — a handful of small batches, and it shares Vec code the +// binary already has instead of pulling in VecDeque's ring-buffer instantiations. +pub struct MemoryCache { + batches: Mutex)>>, + max_bytes: usize, +} + +impl MemoryCache { + pub fn new(max_bytes: u64) -> Self { + Self { + batches: Mutex::new(Vec::new()), + max_bytes: usize::try_from(max_bytes).unwrap_or(usize::MAX), + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Vec<(String, Vec)>> { + self.batches.lock().unwrap_or_else(|e| e.into_inner()) + } +} + +impl BatchCache for MemoryCache { + fn push(&self, id: &str, body: &[u8]) -> io::Result<()> { + let mut batches = self.lock(); + batches.push((id.to_owned(), body.to_vec())); + let mut total: usize = batches.iter().map(|(_, b)| b.len()).sum(); + while total > self.max_bytes && batches.len() > 1 { + total -= batches.remove(0).1.len(); + } + Ok(()) + } + + fn pending(&self) -> Vec { + self.lock().iter().map(|(id, _)| id.clone()).collect() + } + + fn read(&self, id: &str) -> Option> { + self.lock().iter().find(|(i, _)| i == id).map(|(_, body)| body.clone()) + } + + fn remove(&self, id: &str) { + let mut batches = self.lock(); + if let Some(index) = batches.iter().position(|(i, _)| i == id) { + batches.remove(index); + } + } + + fn clear(&self) { + self.lock().clear(); + } +} + +const EXT: &str = "otlp"; +// ponytail: fixed 24 h max age (Datadog: 18 h, design doc: 24 h); a config knob if anyone asks. +const MAX_AGE_NANOS: u64 = 24 * 60 * 60 * 1_000_000_000; + +/// On-disk [`BatchCache`]: one file per encoded batch, so a crash or an offline shutdown loses +/// nothing and the next launch replays what is left. +/// +/// The shape every mobile SDK converges on (Sentry envelopes, Datadog batch files, +/// opentelemetry-android disk buffering): files are written as `.tmp` and renamed into place so +/// a crash never leaves a half batch readable; eviction is drop-oldest above `max_bytes`; +/// batches older than 24 h are discarded, judged by the timestamp in the id rather than file +/// metadata (an Apple "required reason" API). +pub struct FileCache { + dir: PathBuf, + max_bytes: u64, +} + +impl FileCache { + /// Open the cache directory (created if missing; its parent must exist) and discard stale + /// or half-written files. + pub fn open(dir: impl Into, max_bytes: u64) -> io::Result { + let dir = dir.into(); + // ponytail: `create_dir`, not `create_dir_all` — the recursive variant drags in + // `Path::components` machinery (~3 KiB) for a parent the host always provides. + match fs::create_dir(&dir) { + Err(err) if err.kind() != io::ErrorKind::AlreadyExists => return Err(err), + _ => {} + } + let cache = Self { dir, max_bytes }; + cache.prune()?; + Ok(cache) + } + + fn path(&self, id: &str, ext: &str) -> PathBuf { + self.dir.join(format!("{id}.{ext}")) + } + + fn write_then_rename(&self, tmp: &Path, dest: &Path, body: &[u8]) -> io::Result<()> { + if let Err(err) = fs::write(tmp, body) { + // iOS may purge the whole Caches subdirectory while the app runs: recreate it once. + if err.kind() != io::ErrorKind::NotFound { + return Err(err); + } + fs::create_dir(&self.dir)?; + fs::write(tmp, body)?; + } + fs::rename(tmp, dest) + } + + /// Delete stray `.tmp` files and batches older than the max age, then the oldest batches + /// until the total fits `max_bytes`. + fn prune(&self) -> io::Result<()> { + let now = crate::event::now_unix_nanos(); + for entry in fs::read_dir(&self.dir)?.flatten() { + let path = entry.path(); + let expired = stamp(&path).is_some_and(|t| now.saturating_sub(t) > MAX_AGE_NANOS); + if !is_batch(&path) || expired { + let _ = fs::remove_file(&path); + } + } + let kept = self.pending(); + let sizes: Vec = kept + .iter() + .map(|id| fs::metadata(self.path(id, EXT)).map(|m| m.len()).unwrap_or(0)) + .collect(); + let mut total: u64 = sizes.iter().sum(); + for (id, len) in kept.iter().zip(sizes) { + if total <= self.max_bytes { + break; + } + self.remove(id); + total -= len; + } + Ok(()) + } +} + +impl BatchCache for FileCache { + /// Fail-open: a full disk or a purged directory yields an error the exporter counts as a + /// drop; it never leaves a partial `.tmp` behind. + fn push(&self, id: &str, body: &[u8]) -> io::Result<()> { + let tmp = self.path(id, "tmp"); + let written = self.write_then_rename(&tmp, &self.path(id, EXT), body); + if written.is_err() { + // ENOSPC leaves a truncated `.tmp`; drop it now rather than at the next launch. + let _ = fs::remove_file(&tmp); + } + written?; + self.prune() + } + + fn pending(&self) -> Vec { + let Ok(entries) = fs::read_dir(&self.dir) else { return Vec::new() }; + let mut ids: Vec = entries + .flatten() + .map(|e| e.path()) + .filter(|p| is_batch(p)) + .filter_map(|p| p.file_stem()?.to_str().map(str::to_owned)) + .collect(); + ids.sort_unstable(); + ids + } + + fn read(&self, id: &str) -> Option> { + fs::read(self.path(id, EXT)).ok() + } + + fn remove(&self, id: &str) { + let _ = fs::remove_file(self.path(id, EXT)); + } + + fn clear(&self) { + for id in self.pending() { + self.remove(&id); + } + } +} + +fn is_batch(path: &Path) -> bool { + path.extension().is_some_and(|e| e == EXT) +} + +/// The creation time encoded in a batch file name. +fn stamp(path: &Path) -> Option { + path.file_stem()?.to_str()?.split('-').next()?.parse().ok() +} + +#[cfg(test)] +pub(crate) fn temp_dir(tag: &str) -> PathBuf { + let dir = std::env::temp_dir() + .join(format!("livekit-telemetry-{tag}-{}", crate::event::now_unix_nanos())); + let _ = fs::remove_dir_all(&dir); + dir +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Ids the exporter would mint: a real timestamp (so `prune` does not expire them) + seq. + fn id(n: u64) -> String { + format!( + "{:020}-{n:06}-1", + crate::event::now_unix_nanos() / 1_000_000_000 * 1_000_000_000 + n + ) + } + + #[test] + fn memory_cache_evicts_oldest_beyond_max_bytes() { + let cache = MemoryCache::new(25); + for (n, body) in [b"aaaaaaaaaa", b"bbbbbbbbbb", b"cccccccccc"].into_iter().enumerate() { + cache.push(&id(n as u64), body).expect("push"); + } + assert_eq!(cache.pending(), [id(1), id(2)]); + assert_eq!(cache.read(&id(1)).expect("read"), b"bbbbbbbbbb"); + cache.remove(&id(1)); + assert_eq!(cache.pending(), [id(2)]); + cache.clear(); + assert!(cache.pending().is_empty()); + } + + #[test] + fn file_cache_evicts_oldest_beyond_max_bytes_and_drops_stray_tmp() { + let dir = temp_dir("cache"); + let cache = FileCache::open(&dir, 25).expect("open"); + fs::write(dir.join("crashed.tmp"), b"half").expect("write"); + for (n, body) in [b"aaaaaaaaaa", b"bbbbbbbbbb", b"cccccccccc"].into_iter().enumerate() { + cache.push(&id(n as u64), body).expect("push"); + } + assert_eq!(cache.pending(), [id(1), id(2)], "10-byte batches under a 25-byte cap"); + assert_eq!(cache.read(&id(1)).expect("read"), b"bbbbbbbbbb"); + assert!(!dir.join("crashed.tmp").exists()); + cache.clear(); + assert!(cache.pending().is_empty()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn file_cache_recreates_a_purged_directory() { + let dir = temp_dir("purged"); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + fs::remove_dir_all(&dir).expect("purge like iOS does"); + cache.push(&id(1), b"after purge").expect("push recreates the dir"); + assert_eq!(cache.pending(), [id(1)]); + let _ = fs::remove_dir_all(&dir); + } + + #[cfg(unix)] + #[test] + fn file_cache_failed_write_leaves_no_partial_file() { + use std::os::unix::fs::PermissionsExt; + let dir = temp_dir("readonly"); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + // Stand-in for ENOSPC: any write into the directory fails. + fs::set_permissions(&dir, fs::Permissions::from_mode(0o555)).expect("chmod"); + let result = cache.push(&id(1), b"no room"); + fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).expect("chmod back"); + assert!(result.is_err()); + assert_eq!(fs::read_dir(&dir).expect("dir").count(), 0, "no stray .tmp"); + let _ = fs::remove_dir_all(&dir); + } +} diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index f85b527c6..d6231706d 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -18,47 +18,46 @@ use livekit_runtime::{interval, sleep, timeout, Instant, MissedTickBehavior}; use tokio::sync::{mpsc, oneshot}; use crate::{ - otlp, persist::FileCache, store::Store, ExportError, ExportRequest, TelemetryConfig, - TelemetryEvent, TelemetryTransport, + event::now_unix_nanos, otlp, store::Store, BatchCache, ExportError, ExportRequest, + TelemetryConfig, TelemetryTransport, }; -/// Retries per batch after the first attempt. Fixed for now; see [`Exporter::deliver`]. +/// Retries per upload attempt after the first, for failures without `Retry-After`. const MAX_RETRIES: u32 = 2; const RETRY_BACKOFF: Duration = Duration::from_secs(1); -const MAX_RETRY_DELAY: Duration = Duration::from_secs(30); -/// Pause before the on-disk cache is retried after a failed replay (Sentry: stop consuming the +/// Pause before the cache is tried again after a failed upload (Sentry: stop consuming the /// cache until `Retry-After` elapses; here also for plain connectivity failures). -const REPLAY_BACKOFF: Duration = Duration::from_secs(60); +const UPLOAD_BACKOFF: Duration = Duration::from_secs(60); pub(crate) enum Command { Flush(oneshot::Sender<()>), Shutdown(oneshot::Sender<()>), } -/// Outcome of delivering one encoded batch, after retries. +/// Outcome of uploading one encoded batch. enum Delivery { Sent, - /// Transient failure (network, timeout, 5xx) — worth keeping for later. + /// Transient failure (network, timeout, 5xx) after retries — keep the batch. Failed, - /// The collector asked us to back off (`Retry-After`) — dropped, never persisted. - Throttled, + /// The collector asked us to back off; keep the batch, drop new ones meanwhile. + Throttled { + retry_after: Option, + }, Rejected, Disabled, } /// Background actor that turns stored events into OTLP requests. /// -/// The role of OTel's `BatchLogRecordProcessor` + OTLP exporter in one place: every -/// `flush_interval_ms` it drains up to `max_batch_size` events, encodes them, and hands the -/// request to the [`TelemetryTransport`]. Transient failures are retried with a bounded -/// backoff (honoring `Retry-After`), rejected batches are dropped, and once the collector -/// reports telemetry as disabled the exporter goes silent for good — dropping instead of -/// retrying, so a disabled project never sees a request storm. -/// -/// With `storage_dir` configured, batches that still fail after retries are written to a -/// [`FileCache`] and replayed oldest-first on the next tick (after a backoff) and on the next -/// launch; `shutdown` spills whatever is queued to disk *before* trying the network, so an app -/// being killed offline loses nothing. Throttled, rejected and disabled data is never persisted. +/// The role of OTel's `BatchLogRecordProcessor` + OTLP exporter in one place. Every +/// `flush_interval_ms` it [`enqueue`](Self::enqueue)s: drains up to `max_batch_size` events, +/// encodes them and writes the batch to the [`BatchCache`] *before* any network is involved; +/// then it [`upload`](Self::upload)s the cache oldest-first through the +/// [`TelemetryTransport`], removing what the collector accepted or rejected. A failed upload +/// pauses the cache for a minute; a `Retry-After` additionally drops new batches for its +/// duration (throttling must not become a disk-backed queue); `Disabled` empties the cache and +/// silences the exporter for good. With a [`FileCache`](crate::FileCache) this is crash- and +/// restart-safe; with the default [`MemoryCache`](crate::MemoryCache) it is process-bound. /// /// Drive it with `spawn(exporter.run())` on the consumer's runtime. It stops after /// [`Telemetry::shutdown`](crate::Telemetry::shutdown) or when the last @@ -67,12 +66,16 @@ pub struct Exporter { store: Arc, transport: Arc, config: Arc, - cache: Option>, + cache: Arc, commands: mpsc::UnboundedReceiver, silenced: bool, - replay_after: Option, + /// Leave the cache alone until then: the last upload failed or we were throttled. + paused_until: Option, + /// Drop new batches until then (`Retry-After` window). + throttled_until: Option, + seq: u64, /// Batches the cache could not store (disk full, directory unusable). Drives one-shot logging. - persist_failures: u64, + cache_failures: u64, } impl Exporter { @@ -80,7 +83,7 @@ impl Exporter { store: Arc, transport: Arc, config: Arc, - cache: Option>, + cache: Arc, commands: mpsc::UnboundedReceiver, ) -> Self { Self { @@ -90,35 +93,38 @@ impl Exporter { cache, commands, silenced: false, - replay_after: None, - persist_failures: 0, + paused_until: None, + throttled_until: None, + seq: 0, + cache_failures: 0, } } - /// Run until shut down. Replays persisted batches, then exports on every tick and on demand. + /// Run until shut down: replay whatever the cache holds, then export on every tick and on + /// demand. pub async fn run(mut self) { - self.replay().await; + self.upload().await; let mut ticker = interval(Duration::from_millis(self.config.flush_interval_ms.max(1))); ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); loop { tokio::select! { - _ = ticker.tick() => { - self.replay().await; - self.export_pending(false).await; - } + _ = ticker.tick() => self.export_pending().await, command = self.commands.recv() => match command { Some(Command::Flush(done)) => { - self.export_pending(true).await; + self.export_pending().await; let _ = done.send(()); } Some(Command::Shutdown(done)) => { - self.shutdown().await; + // Last chance: ignore the upload backoff, but respect throttling. + self.paused_until = self.throttled_until; + self.export_pending().await; let _ = done.send(()); return; } // Every `Telemetry` handle is gone. None => { - self.shutdown().await; + self.paused_until = self.throttled_until; + self.export_pending().await; return; } }, @@ -126,82 +132,65 @@ impl Exporter { } } - /// Final flush. With a cache: spill everything to disk first (cheap, local), then send as - /// much as the network allows — whatever remains is replayed on the next launch. - async fn shutdown(&mut self) { - if self.cache.is_some() && !self.silenced { - self.spill(); - self.replay_after = None; - self.replay().await; - } else { - self.export_pending(true).await; - } + async fn export_pending(&mut self) { + self.enqueue(); + self.upload().await; } - /// Export one batch, or everything queued when `drain_all` is set. - async fn export_pending(&mut self, drain_all: bool) { + /// Encode everything queued into the cache — no network involved. + fn enqueue(&mut self) { loop { let batch = self.store.drain(self.config.max_batch_size.max(1) as usize); if batch.is_empty() { return; } - self.export(batch).await; - if !drain_all { - return; - } - } - } - - async fn export(&mut self, batch: Vec) { - let count = batch.len() as u64; - if self.silenced { - self.store.add_dropped(count); - return; - } - let body = otlp::encode_logs(&self.config.resource, batch); - match self.deliver(&body).await { - Delivery::Sent => {} - Delivery::Failed => { - self.persist_or_drop(&body, count); - // The network just failed; do not hammer it with the cache on the next tick. - self.replay_after = Some(Instant::now() + REPLAY_BACKOFF); - } - Delivery::Throttled | Delivery::Rejected => self.store.add_dropped(count), - Delivery::Disabled => { - self.silence(); + let count = batch.len() as u64; + if self.silenced || self.throttled_until.is_some_and(|t| Instant::now() < t) { self.store.add_dropped(count); + continue; } - } - } - - /// Encode everything queued straight to disk, without touching the network. - fn spill(&mut self) { - loop { - let batch = self.store.drain(self.config.max_batch_size.max(1) as usize); - if batch.is_empty() { - return; - } - let count = batch.len() as u64; let body = otlp::encode_logs(&self.config.resource, batch); - self.persist_or_drop(&body, count); + self.seq += 1; + let id = format!("{:020}-{:06}-{count}", now_unix_nanos(), self.seq); + if let Err(err) = self.cache.push(&id, &body) { + // A full disk is a steady state, not an event: warn once, then stay quiet. + if self.cache_failures == 0 { + log::warn!( + "telemetry: cannot cache batches ({err}); dropping until it recovers" + ); + } else { + log::debug!("telemetry: could not cache {count} events: {err}"); + } + self.cache_failures += 1; + self.store.add_dropped(count); + } } } - /// Send persisted batches oldest-first until one fails; then back off. - async fn replay(&mut self) { - let Some(cache) = self.cache.clone() else { return }; - if self.silenced || self.replay_after.is_some_and(|t| Instant::now() < t) { + /// Send cached batches oldest-first until one fails; then back off. + async fn upload(&mut self) { + if self.silenced || self.paused_until.is_some_and(|t| Instant::now() < t) { return; } - for path in cache.pending() { - let Ok(body) = cache.read(&path) else { - cache.remove(&path); + for id in self.cache.pending() { + let Some(body) = self.cache.read(&id) else { + self.cache.remove(&id); continue; }; match self.deliver(&body).await { - Delivery::Sent | Delivery::Rejected => cache.remove(&path), - Delivery::Failed | Delivery::Throttled => { - self.replay_after = Some(Instant::now() + REPLAY_BACKOFF); + Delivery::Sent => self.cache.remove(&id), + Delivery::Rejected => { + self.cache.remove(&id); + self.store.add_dropped(events_in(&id)); + } + Delivery::Failed => { + self.paused_until = Some(Instant::now() + UPLOAD_BACKOFF); + return; + } + Delivery::Throttled { retry_after } => { + let until = Instant::now() + retry_after.unwrap_or(UPLOAD_BACKOFF); + self.paused_until = Some(until); + self.throttled_until = Some(until); return; } Delivery::Disabled => { @@ -212,38 +201,16 @@ impl Exporter { } } - fn persist_or_drop(&mut self, body: &[u8], count: u64) { - match self.cache.as_deref().map(|cache| cache.store(body)) { - Some(Ok(())) => log::debug!("telemetry: persisted {count} events for later delivery"), - Some(Err(err)) => { - // Disk full is a steady state, not an event: warn once, then stay quiet. - if self.persist_failures == 0 { - log::warn!( - "telemetry: cannot persist events ({err}); dropping until it recovers" - ); - } else { - log::debug!("telemetry: could not persist {count} events: {err}"); - } - self.persist_failures += 1; - self.store.add_dropped(count); - } - None => { - log::warn!("telemetry: export failed, dropping {count} events"); - self.store.add_dropped(count); - } - } - } - - /// Telemetry is disabled for this project: never send again, and never replay what is on disk. + /// Telemetry is disabled for this project: never send again, and never replay what is cached. fn silence(&mut self) { log::warn!("telemetry disabled by the collector; going silent"); self.silenced = true; - if let Some(cache) = &self.cache { - cache.clear(); - } + let cached: u64 = self.cache.pending().iter().map(|id| events_in(id)).sum(); + self.store.add_dropped(cached); + self.cache.clear(); } - /// Deliver one encoded batch with bounded retries and classify the outcome. + /// Upload one encoded batch with bounded retries and classify the outcome. async fn deliver(&self, body: &[u8]) -> Delivery { let mut headers = self.config.headers.clone(); headers.insert("Content-Type".to_owned(), otlp::CONTENT_TYPE.to_owned()); @@ -251,38 +218,34 @@ impl Exporter { ExportRequest { url: self.config.endpoint.clone(), headers, body: body.to_vec() }; let attempt_timeout = Duration::from_millis(self.config.export_timeout_ms.max(1)); - // ponytail: linear backoff, 2 retries, blocks the tick loop while sleeping (bounded by - // MAX_RETRIES × MAX_RETRY_DELAY). Exponential + jitter once real fleets exercise this. - let mut throttled = false; + // ponytail: linear backoff, 2 retries, blocks the tick loop while sleeping (≤ 3 s). + // Exponential + jitter once real fleets exercise this. for attempt in 0..=MAX_RETRIES { - let delay = match timeout(attempt_timeout, self.transport.send(request.clone())).await { + match timeout(attempt_timeout, self.transport.send(request.clone())).await { Ok(Ok(())) => return Delivery::Sent, Ok(Err(ExportError::Disabled)) => return Delivery::Disabled, Ok(Err(ExportError::Rejected { message })) => { log::warn!("telemetry batch rejected: {message}"); return Delivery::Rejected; } - Ok(Err(ExportError::Retryable { message, retry_after_ms })) => { - log::debug!("telemetry export failed (attempt {}): {message}", attempt + 1); - throttled = retry_after_ms.is_some(); - retry_after_ms - .map(Duration::from_millis) - .unwrap_or(RETRY_BACKOFF * (attempt + 1)) + Ok(Err(ExportError::Retryable { message, retry_after_ms: Some(ms) })) => { + log::debug!("telemetry throttled for {ms} ms: {message}"); + return Delivery::Throttled { retry_after: Some(Duration::from_millis(ms)) }; } - Err(_) => { - log::debug!("telemetry export timed out (attempt {})", attempt + 1); - throttled = false; - RETRY_BACKOFF * (attempt + 1) + Ok(Err(ExportError::Retryable { message, retry_after_ms: None })) => { + log::debug!("telemetry upload failed (attempt {}): {message}", attempt + 1); } - }; + Err(_) => log::debug!("telemetry upload timed out (attempt {})", attempt + 1), + } if attempt < MAX_RETRIES { - sleep(delay.min(MAX_RETRY_DELAY)).await; + sleep(RETRY_BACKOFF * (attempt + 1)).await; } } - if throttled { - Delivery::Throttled - } else { - Delivery::Failed - } + Delivery::Failed } } + +/// The event count the exporter encodes as the last component of a batch id. +fn events_in(id: &str) -> u64 { + id.rsplit('-').next().and_then(|n| n.parse().ok()).unwrap_or(0) +} diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs index fefbf0461..595c1a6e4 100644 --- a/livekit-telemetry/src/lib.rs +++ b/livekit-telemetry/src/lib.rs @@ -26,8 +26,8 @@ mod exporter; /// OTLP/HTTP protobuf encoding of a batch. mod otlp; -/// On-disk cache of undeliverable batches. -mod persist; +/// Queue of encoded batches between the exporter and the transport (memory or disk). +mod cache; /// OTLP protobuf types (re-exported from `opentelemetry-proto`). mod proto; @@ -38,6 +38,7 @@ mod transport; /// Entry point and configuration. mod telemetry; +pub use cache::{BatchCache, FileCache, MemoryCache}; pub use event::*; pub use exporter::Exporter; pub use telemetry::*; diff --git a/livekit-telemetry/src/persist.rs b/livekit-telemetry/src/persist.rs deleted file mode 100644 index e0883157f..000000000 --- a/livekit-telemetry/src/persist.rs +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::{ - fs, io, - path::{Path, PathBuf}, - sync::atomic::{AtomicU64, Ordering}, -}; - -use crate::event::now_unix_nanos; - -const EXT: &str = "otlp"; -// ponytail: fixed 24 h max age (Datadog: 18 h, design doc: 24 h); a config knob if anyone asks. -const MAX_AGE_NANOS: u64 = 24 * 60 * 60 * 1_000_000_000; - -/// File cache of encoded batches the transport could not deliver. -/// -/// One file per `ExportLogsServiceRequest` body — the unit every mobile SDK persists (Sentry -/// envelopes, Datadog batch files, opentelemetry-android disk buffering): nothing to re-encode -/// on replay, and URL/headers are recomposed from the *current* config, so a rotated token is -/// picked up automatically. Files are written as `.tmp` and renamed into place so a crash never -/// leaves a half-written batch readable. Replay is oldest-first; eviction is drop-oldest above -/// `max_bytes`; batches older than 24 h are discarded. The age comes from the file name, not -/// from file timestamps (an Apple "required reason" API). -pub(crate) struct FileCache { - dir: PathBuf, - max_bytes: u64, - seq: AtomicU64, -} - -impl FileCache { - /// Open the cache directory (created if missing; its parent must exist) and discard stale - /// or half-written files. - pub fn open(dir: impl Into, max_bytes: u64) -> io::Result { - let dir = dir.into(); - // ponytail: `create_dir`, not `create_dir_all` — the recursive variant drags in - // `Path::components` machinery (~3 KiB) for a parent the host always provides. - match fs::create_dir(&dir) { - Err(err) if err.kind() != io::ErrorKind::AlreadyExists => return Err(err), - _ => {} - } - let cache = Self { dir, max_bytes, seq: AtomicU64::new(0) }; - cache.prune()?; - Ok(cache) - } - - /// Persist one encoded batch, evicting the oldest batches to stay within `max_bytes`. - /// - /// Fail-open: a full disk or a purged directory yields an error the caller counts as a - /// drop; it never leaves a partial `.tmp` behind. - pub fn store(&self, body: &[u8]) -> io::Result<()> { - let name = - format!("{:020}-{:06}", now_unix_nanos(), self.seq.fetch_add(1, Ordering::Relaxed)); - let tmp = self.dir.join(format!("{name}.tmp")); - let written = self.write_then_rename(&tmp, &self.dir.join(format!("{name}.{EXT}")), body); - if written.is_err() { - // ENOSPC leaves a truncated `.tmp`; drop it now rather than at the next launch. - let _ = fs::remove_file(&tmp); - } - written?; - self.prune() - } - - fn write_then_rename(&self, tmp: &Path, dest: &Path, body: &[u8]) -> io::Result<()> { - if let Err(err) = fs::write(tmp, body) { - // iOS may purge the whole Caches subdirectory while the app runs: recreate it once. - if err.kind() != io::ErrorKind::NotFound { - return Err(err); - } - fs::create_dir(&self.dir)?; - fs::write(tmp, body)?; - } - fs::rename(tmp, dest) - } - - /// Pending batches, oldest first. - pub fn pending(&self) -> Vec { - let Ok(entries) = fs::read_dir(&self.dir) else { return Vec::new() }; - let mut files: Vec = - entries.flatten().map(|e| e.path()).filter(|p| is_batch(p)).collect(); - files.sort_unstable(); - files - } - - pub fn read(&self, path: &Path) -> io::Result> { - fs::read(path) - } - - pub fn remove(&self, path: &Path) { - let _ = fs::remove_file(path); - } - - /// Discard every pending batch (telemetry disabled: nothing may be replayed later). - pub fn clear(&self) { - for path in self.pending() { - self.remove(&path); - } - } - - /// Delete stray `.tmp` files and batches older than the max age, then the oldest batches - /// until the total fits `max_bytes`. - fn prune(&self) -> io::Result<()> { - let now = now_unix_nanos(); - for entry in fs::read_dir(&self.dir)?.flatten() { - let path = entry.path(); - let expired = stamp(&path).is_some_and(|t| now.saturating_sub(t) > MAX_AGE_NANOS); - if !is_batch(&path) || expired { - let _ = fs::remove_file(&path); - } - } - let kept = self.pending(); - let sizes: Vec = - kept.iter().map(|p| fs::metadata(p).map(|m| m.len()).unwrap_or(0)).collect(); - let mut total: u64 = sizes.iter().sum(); - for (path, len) in kept.iter().zip(sizes) { - if total <= self.max_bytes { - break; - } - let _ = fs::remove_file(path); - total -= len; - } - Ok(()) - } -} - -fn is_batch(path: &Path) -> bool { - path.extension().is_some_and(|e| e == EXT) -} - -/// The creation time encoded in a batch file name. -fn stamp(path: &Path) -> Option { - path.file_stem()?.to_str()?.split('-').next()?.parse().ok() -} - -#[cfg(test)] -pub(crate) fn temp_dir(tag: &str) -> PathBuf { - let dir = std::env::temp_dir().join(format!("livekit-telemetry-{tag}-{}", now_unix_nanos())); - let _ = fs::remove_dir_all(&dir); - dir -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn evicts_oldest_beyond_max_bytes_and_drops_stray_tmp() { - let dir = temp_dir("cache"); - let cache = FileCache::open(&dir, 25).expect("open"); - fs::write(dir.join("crashed.tmp"), b"half").expect("write"); - for body in [b"aaaaaaaaaa", b"bbbbbbbbbb", b"cccccccccc"] { - cache.store(body).expect("store"); - } - let pending = cache.pending(); - assert_eq!(pending.len(), 2, "10-byte batches under a 25-byte cap"); - assert_eq!(cache.read(&pending[0]).expect("read"), b"bbbbbbbbbb"); - assert!(!dir.join("crashed.tmp").exists() || cache.pending().len() == 2); - cache.clear(); - assert!(cache.pending().is_empty()); - let _ = fs::remove_dir_all(&dir); - } - - #[test] - fn recreates_a_purged_directory() { - let dir = temp_dir("purged"); - let cache = FileCache::open(&dir, 1 << 20).expect("open"); - fs::remove_dir_all(&dir).expect("purge like iOS does"); - cache.store(b"after purge").expect("store recreates the dir"); - assert_eq!(cache.pending().len(), 1); - let _ = fs::remove_dir_all(&dir); - } - - #[cfg(unix)] - #[test] - fn failed_write_leaves_no_partial_file() { - use std::os::unix::fs::PermissionsExt; - let dir = temp_dir("readonly"); - let cache = FileCache::open(&dir, 1 << 20).expect("open"); - // Stand-in for ENOSPC: any write into the directory fails. - fs::set_permissions(&dir, fs::Permissions::from_mode(0o555)).expect("chmod"); - let result = cache.store(b"no room"); - fs::set_permissions(&dir, fs::Permissions::from_mode(0o755)).expect("chmod back"); - assert!(result.is_err()); - assert_eq!(fs::read_dir(&dir).expect("dir").count(), 0, "no stray .tmp"); - let _ = fs::remove_dir_all(&dir); - } -} diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index e8da37db1..b1879e0cd 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -18,8 +18,8 @@ use livekit_runtime::timeout; use tokio::sync::{mpsc, oneshot}; use crate::{ - event::now_unix_nanos, exporter::Command, persist::FileCache, store::Store, Attribute, - Exporter, TelemetryEvent, TelemetryTransport, + event::now_unix_nanos, exporter::Command, store::Store, Attribute, BatchCache, Exporter, + FileCache, MemoryCache, TelemetryEvent, TelemetryTransport, }; /// Pipeline configuration. @@ -37,13 +37,13 @@ pub struct TelemetryConfig { /// Resource attributes describing the emitter (`service.name`, `os.name`, /// `device.model.identifier`, `session.id`, …). `telemetry.sdk.*` are filled in by the core. pub resource: Vec, - /// Directory for the on-disk cache of undeliverable batches (created if missing). `None` - /// disables persistence: batches that fail after retries are dropped. + /// Directory for the on-disk batch cache (created if missing; its parent must exist). + /// `None` keeps batches in memory only: they survive failed uploads, not the process. #[cfg_attr(feature = "uniffi", uniffi(default))] pub storage_dir: Option, - /// Cap on the on-disk cache; the oldest batches are evicted first. + /// Cap on cached batches, in memory or on disk; the oldest are evicted first. #[cfg_attr(feature = "uniffi", uniffi(default = 4194304))] - pub max_storage_bytes: u64, + pub max_cache_bytes: u64, #[cfg_attr(feature = "uniffi", uniffi(default = 1000))] pub flush_interval_ms: u64, /// Events buffered before the oldest are dropped. @@ -58,14 +58,14 @@ pub struct TelemetryConfig { } impl TelemetryConfig { - /// Defaults for the given endpoint; no persistence. + /// Defaults for the given endpoint; in-memory cache. pub fn new(endpoint: impl Into) -> Self { Self { endpoint: endpoint.into(), headers: HashMap::new(), resource: Vec::new(), storage_dir: None, - max_storage_bytes: 4 * 1024 * 1024, + max_cache_bytes: 4 * 1024 * 1024, flush_interval_ms: 1000, max_queue_size: 2048, max_batch_size: 512, @@ -105,22 +105,33 @@ pub struct Telemetry { } impl Telemetry { - /// Build the pipeline. Spawn the returned [`Exporter`] with `exporter.run()` on your runtime. - /// - /// An unusable `storage_dir` is logged and persistence is skipped — never an error. + /// Build the pipeline with the cache the config asks for: a [`FileCache`] in `storage_dir`, + /// or a [`MemoryCache`] when unset — or when the directory is unusable (logged, never an + /// error). Spawn the returned [`Exporter`] with `exporter.run()` on your runtime. pub fn new( + config: TelemetryConfig, + transport: Arc, + ) -> (Self, Exporter) { + let cache: Arc = match config.storage_dir.as_deref() { + Some(dir) => match FileCache::open(dir, config.max_cache_bytes) { + Ok(cache) => Arc::new(cache), + Err(err) => { + log::warn!("telemetry: cannot use storage dir {dir}: {err}; caching in memory"); + Arc::new(MemoryCache::new(config.max_cache_bytes)) + } + }, + None => Arc::new(MemoryCache::new(config.max_cache_bytes)), + }; + Self::with_cache(config, transport, cache) + } + + /// Build the pipeline around a caller-provided [`BatchCache`] (`storage_dir` is ignored). + pub fn with_cache( mut config: TelemetryConfig, transport: Arc, + cache: Arc, ) -> (Self, Exporter) { add_sdk_resource(&mut config.resource); - let cache = config.storage_dir.as_deref().and_then(|dir| { - FileCache::open(dir, config.max_storage_bytes) - .map(Arc::new) - .map_err(|err| { - log::warn!("telemetry: cannot use storage dir {dir}: {err}; not persisting") - }) - .ok() - }); let config = Arc::new(config); let store = Arc::new(Store::new(config.max_queue_size.max(1) as usize)); let (commands, receiver) = mpsc::unbounded_channel(); @@ -136,20 +147,21 @@ impl Telemetry { self.store.push(event); } - /// Export everything queued and wait until the transport accepted (or the exporter gave up on) it. + /// Cache everything queued and upload what the transport accepts right now. pub async fn flush(&self) { self.command(Command::Flush).await; } /// Flush, then stop the exporter. Bounded by `export_timeout_ms`; events emitted afterwards - /// are never exported. With `storage_dir` set, queued events are written to disk before the - /// network is tried, so nothing is lost if the process dies mid-way. + /// are never exported. Queued events reach the cache before the network is tried, so with + /// a [`FileCache`] nothing is lost if the process dies mid-way. pub async fn shutdown(&self) { let bound = Duration::from_millis(self.config.export_timeout_ms.max(1)); let _ = timeout(bound, self.command(Command::Shutdown)).await; } - /// Events dropped so far (queue overflow, rejected/throttled/failed exports, disabled collector). + /// Events dropped so far (queue overflow, cache failure, rejected or throttled batches, + /// disabled collector). pub fn dropped_count(&self) -> u64 { self.store.dropped() } @@ -185,7 +197,7 @@ mod tests { use super::*; use crate::{ - persist::temp_dir, + cache::temp_dir, proto::opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest, ExportError, ExportRequest, }; @@ -220,6 +232,10 @@ mod tests { Err(ExportError::Retryable { message: "offline".into(), retry_after_ms: None }) } + fn offline_forever() -> impl Iterator> { + std::iter::repeat_with(offline).take(64) + } + fn pipeline(transport: Arc) -> Telemetry { start(TelemetryConfig::new("http://collector/v1/logs"), transport) } @@ -240,6 +256,15 @@ mod tests { fs::read_dir(dir).map(|d| d.count()).unwrap_or(0) } + fn event_names(request: &ExportRequest) -> Vec { + let decoded = ExportLogsServiceRequest::decode(&request.body[..]).expect("valid OTLP"); + decoded.resource_logs[0].scope_logs[0] + .log_records + .iter() + .map(|r| r.event_name.clone()) + .collect() + } + #[tokio::test(start_paused = true)] async fn batches_events_into_one_otlp_request() { let transport = FakeTransport::scripted([]); @@ -253,24 +278,28 @@ mod tests { assert_eq!(sent.len(), 1); assert_eq!(sent[0].url, "http://collector/v1/logs"); assert_eq!(sent[0].headers["Content-Type"], "application/x-protobuf"); + assert_eq!(event_names(&sent[0]), ["lk.ping"; 3]); let decoded = ExportLogsServiceRequest::decode(&sent[0].body[..]).expect("valid OTLP"); - let records = &decoded.resource_logs[0].scope_logs[0].log_records; - assert_eq!(records.len(), 3); - assert!(records.iter().all(|r| r.event_name == "lk.ping" && r.time_unix_nano > 0)); let resource = decoded.resource_logs[0].resource.as_ref().expect("resource"); assert!(resource.attributes.iter().any(|kv| kv.key == "telemetry.sdk.name")); assert_eq!(telemetry.dropped_count(), 0); } #[tokio::test(start_paused = true)] - async fn retries_transient_failures_then_drops_without_storage() { + async fn failed_upload_waits_in_memory_and_is_retried_after_backoff() { let transport = FakeTransport::scripted([offline(), offline(), offline()]); let telemetry = pipeline(transport.clone()); telemetry.emit(TelemetryEvent::new("lk.ping")); telemetry.flush().await; + assert_eq!(transport.sent().len(), 3, "first attempt plus two retries"); + assert_eq!(telemetry.dropped_count(), 0, "kept in the memory cache, not dropped"); - assert_eq!(transport.sent().len(), 3); - assert_eq!(telemetry.dropped_count(), 1); + telemetry.flush().await; + assert_eq!(transport.sent().len(), 3, "backoff: no upload right away"); + + tokio::time::sleep(Duration::from_secs(61)).await; + assert_eq!(transport.sent().len(), 4, "retried once the backoff elapsed"); + assert_eq!(event_names(&transport.sent()[3]), ["lk.ping"]); } #[tokio::test(start_paused = true)] @@ -299,38 +328,46 @@ mod tests { } #[tokio::test(start_paused = true)] - async fn failed_batch_is_persisted_and_replayed_on_next_start() { + async fn batch_is_written_before_upload_and_replayed_on_next_start() { let dir = temp_dir("replay"); - let first_transport = FakeTransport::scripted(std::iter::repeat_with(offline).take(12)); + let first_transport = FakeTransport::scripted(offline_forever()); let first = persisted_pipeline(first_transport.clone(), &dir); first.emit(TelemetryEvent::new("lk.ping")); first.flush().await; assert_eq!(first_transport.sent().len(), 3); - assert_eq!(first.dropped_count(), 0, "persisted, not dropped"); - assert_eq!(files_in(&dir), 1); + assert_eq!(first.dropped_count(), 0); + assert_eq!(files_in(&dir), 1, "written before the first attempt, kept after failure"); let second_transport = FakeTransport::scripted([]); let second = persisted_pipeline(second_transport.clone(), &dir); second.flush().await; let sent = second_transport.sent(); assert_eq!(sent.len(), 1, "replayed on start"); - let decoded = ExportLogsServiceRequest::decode(&sent[0].body[..]).expect("valid OTLP"); - assert_eq!(decoded.resource_logs[0].scope_logs[0].log_records[0].event_name, "lk.ping"); + assert_eq!(event_names(&sent[0]), ["lk.ping"]); assert_eq!(files_in(&dir), 0); let _ = fs::remove_dir_all(&dir); } #[tokio::test(start_paused = true)] - async fn throttled_batch_is_dropped_not_persisted() { + async fn throttling_keeps_cached_batches_and_drops_new_ones() { let dir = temp_dir("throttle"); let throttled = - || Err(ExportError::Retryable { message: "429".into(), retry_after_ms: Some(1000) }); - let transport = FakeTransport::scripted([throttled(), throttled(), throttled()]); + Err(ExportError::Retryable { message: "429".into(), retry_after_ms: Some(5_000) }); + let transport = FakeTransport::scripted([throttled]); let telemetry = persisted_pipeline(transport.clone(), &dir); telemetry.emit(TelemetryEvent::new("lk.ping")); telemetry.flush().await; + assert_eq!(transport.sent().len(), 1, "no retries on Retry-After"); + assert_eq!(files_in(&dir), 1, "the throttled batch stays cached"); + assert_eq!(telemetry.dropped_count(), 0); - assert_eq!(telemetry.dropped_count(), 1); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + assert_eq!(telemetry.dropped_count(), 1, "new batches are dropped inside the window"); + assert_eq!(files_in(&dir), 1, "and never written"); + + tokio::time::sleep(Duration::from_secs(6)).await; + assert_eq!(transport.sent().len(), 2, "cached batch uploaded after Retry-After"); assert_eq!(files_in(&dir), 0); let _ = fs::remove_dir_all(&dir); } @@ -338,13 +375,30 @@ mod tests { #[tokio::test(start_paused = true)] async fn shutdown_offline_keeps_queue_on_disk() { let dir = temp_dir("spill"); - let transport = FakeTransport::scripted(std::iter::repeat_with(offline).take(12)); + let transport = FakeTransport::scripted(offline_forever()); let telemetry = persisted_pipeline(transport.clone(), &dir); telemetry.emit(TelemetryEvent::new("lk.ping")); telemetry.shutdown().await; - assert_eq!(files_in(&dir), 1, "spilled before the network was tried, kept after it failed"); + assert_eq!(files_in(&dir), 1, "cached before the network was tried, kept after it failed"); assert_eq!(telemetry.dropped_count(), 0); let _ = fs::remove_dir_all(&dir); } + + #[tokio::test(start_paused = true)] + async fn custom_cache_is_used_as_is() { + let cache = Arc::new(MemoryCache::new(1 << 20)); + let transport = FakeTransport::scripted(offline_forever()); + let (telemetry, exporter) = Telemetry::with_cache( + TelemetryConfig::new("http://collector/v1/logs"), + transport, + cache.clone(), + ); + tokio::spawn(exporter.run()); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + let pending = cache.pending(); + assert_eq!(pending.len(), 1); + assert!(pending[0].ends_with("-1"), "id carries the event count: {}", pending[0]); + } } From 3781c211b8331ace842d0bc30522893a97972ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:46:50 +0200 Subject: [PATCH 06/50] feat(telemetry): self-telemetry report and device-state cadence Self-telemetry (the Sentry "client report" shape, reason names from the OTel SDK self-metrics conventions): Counters shared by store and exporter track every way data is lost - queue_full, cache_error, rejected, throttled, disabled - plus uploads sent and failed attempts. Telemetry::stats() exposes them; whenever something went wrong since the previous report the exporter appends one lk.telemetry.report event to the next batch. Never an extra request, never persisted on its own, silent when nothing is wrong. Device state: the host pushes DeviceState { thermal, low_power_mode, app_state } through Telemetry::set_device_state. The core emits the lk.device.{thermal,low_power,app_state}.changed events (initial value on the first call) and multiplies the flush interval by cadence_factor(): serious thermal x2, critical x4, low power x2, background x2, capped at 4x; entering the background flushes once immediately. The OS APIs stay on the host - no Rust crate can read them without a JVM/ObjC bridge. UniFFI: Telemetry.stats(), Telemetry.setDeviceState(); TelemetryStats, DeviceState, ThermalState, AppState records/enums. ios-arm64 dylib +744 B. Co-Authored-By: Claude Fable 5 --- .changeset/add-livekit-telemetry-crate.md | 4 +- examples/telemetry_ping/src/main.rs | 2 +- livekit-telemetry/Cargo.toml | 2 +- livekit-telemetry/README.md | 14 ++ livekit-telemetry/SPEC.md | 56 +++++++ livekit-telemetry/src/device.rs | 141 +++++++++++++++++ livekit-telemetry/src/exporter.rs | 101 +++++++++--- livekit-telemetry/src/lib.rs | 8 + livekit-telemetry/src/stats.rs | 152 ++++++++++++++++++ livekit-telemetry/src/store.rs | 34 ++-- livekit-telemetry/src/telemetry.rs | 182 ++++++++++++++++++---- livekit-uniffi/src/telemetry.rs | 23 ++- 12 files changed, 634 insertions(+), 85 deletions(-) create mode 100644 livekit-telemetry/src/device.rs create mode 100644 livekit-telemetry/src/stats.rs diff --git a/.changeset/add-livekit-telemetry-crate.md b/.changeset/add-livekit-telemetry-crate.md index 0739e8037..9c007cce3 100644 --- a/.changeset/add-livekit-telemetry-crate.md +++ b/.changeset/add-livekit-telemetry-crate.md @@ -5,4 +5,6 @@ livekit-telemetry: minor Add `livekit-telemetry`, the shared client telemetry core: events are buffered on-device, batched and exported as OTLP/HTTP log records through a pluggable `TelemetryTransport`. Batches are written to a `BatchCache` (in memory by default, on disk with `storage_dir`) before upload, so -failed uploads, crashes and offline shutdowns lose nothing. +failed uploads, crashes and offline shutdowns lose nothing. Pipeline health is exposed as `Telemetry::stats` and shipped as `lk.telemetry.report` +events; hosts push `DeviceState` (thermal, low power, foreground/background) and the core emits +the `lk.device.*.changed` events and stretches its cadence under pressure. diff --git a/examples/telemetry_ping/src/main.rs b/examples/telemetry_ping/src/main.rs index 49907d1c5..8b16fe840 100644 --- a/examples/telemetry_ping/src/main.rs +++ b/examples/telemetry_ping/src/main.rs @@ -40,5 +40,5 @@ async fn main() { .with_attribute("lk.ping.seq", 1i64), ); telemetry.shutdown().await; - println!("sent lk.ping to {endpoint} (dropped: {})", telemetry.dropped_count()); + println!("sent lk.ping to {endpoint}; {:?}", telemetry.stats()); } diff --git a/livekit-telemetry/Cargo.toml b/livekit-telemetry/Cargo.toml index 5eea146c0..c1b982983 100644 --- a/livekit-telemetry/Cargo.toml +++ b/livekit-telemetry/Cargo.toml @@ -9,7 +9,7 @@ repository.workspace = true [dependencies] livekit-runtime = { workspace = true, features = ["tokio"] } -tokio = { workspace = true, default-features = false, features = ["macros", "sync"] } +tokio = { workspace = true, default-features = false, features = ["macros", "sync", "time"] } log = { workspace = true } thiserror = { workspace = true } async-trait = "0.1" diff --git a/livekit-telemetry/README.md b/livekit-telemetry/README.md index 453e63d6a..6559774a2 100644 --- a/livekit-telemetry/README.md +++ b/livekit-telemetry/README.md @@ -33,7 +33,9 @@ let (telemetry, exporter) = Telemetry::new(config, Arc::new(transport)); tokio::spawn(exporter.run()); telemetry.emit(TelemetryEvent::new("lk.ping").with_attribute("lk.ping.seq", 1i64)); +telemetry.set_device_state(DeviceState { thermal: ThermalState::Serious, ..Default::default() }); telemetry.shutdown().await; // cache, then upload what the network allows; bounded by `export_timeout_ms` +println!("{:?}", telemetry.stats()); // drops by reason, uploads, cached batches ``` Event names and attributes are defined in [`SPEC.md`](SPEC.md). @@ -68,6 +70,18 @@ start it again and run once more to watch the cached batch replay. writes `.tmp` → rename, evicts oldest above `max_cache_bytes`, expires after 24 h using the timestamp in the file name (not file metadata — an Apple required-reason API), and treats a full disk as a counted drop with a single warning. +- **Self-telemetry rides along.** Counters for every way data can be lost (`queue_full`, + `cache_error`, `rejected`, `throttled`, `disabled`) and for uploads (`sent`, `failures`) are + readable via [`Telemetry::stats`] and shipped as an `lk.telemetry.report` event appended to + the next batch whenever something went wrong since the previous report — the Sentry + "client report" shape, with reason names from the OTel SDK self-metrics conventions. Never + an extra request, never persisted on its own, silent when nothing is wrong. +- **Device state comes from the host; the policy lives here.** Thermal, low-power and + foreground/background are OS APIs the host already watches; it pushes them through + [`Telemetry::set_device_state`]. The core emits the `lk.device.*.changed` events from + `SPEC.md` and stretches its cadence up to 4× under pressure (background also flushes once). + No Rust crate can observe these without a JVM/ObjC bridge, and `device-info` (in this + workspace) covers static facts, not state. - **Transport is the only injection point.** The core composes URL, headers and body; the transport moves bytes and reports [`ExportError`] so the core alone decides retry / drop / persist / go-silent. No Rust HTTP/TLS stack is linked unless the `net` feature is enabled. diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index 9d6c8d8c3..f88c8ef0a 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -27,3 +27,59 @@ attributes: cadence: on demand — pipeline smoke test, never emitted in production paths platforms: all ``` + +```yaml +event: lk.telemetry.report +area: sdk (self-telemetry) +severity: info +attributes: + lk.telemetry.uploads.failed: int # failed upload attempts since the previous report + lk.telemetry.uploads.sent: int # batches accepted since the previous report + lk.telemetry.cache.batches: int # batches waiting in the cache right now + lk.telemetry.dropped.queue_full: int # events evicted from the in-memory queue (omitted when 0) + lk.telemetry.dropped.cache_error: int # events lost because the cache could not store them + lk.telemetry.dropped.rejected: int # events the collector rejected (4xx) + lk.telemetry.dropped.throttled: int # events dropped inside a Retry-After window +cadence: appended to the next batch whenever a drop or upload failure happened since the + previous report — never its own request, never persisted on its own (Sentry client + report shape; reasons follow the OTel SDK self-metrics `error.type` values) +platforms: all +``` + +```yaml +event: lk.device.thermal.changed +area: device +attributes: + lk.device.thermal.state: enum(nominal | fair | serious | critical) +cadence: on change (+ initial value on the first `set_device_state`) +platforms: ios, macos, android — optional elsewhere +``` + +```yaml +event: lk.device.low_power.changed +area: device +attributes: + lk.device.low_power.enabled: bool +cadence: on change (+ initial value) +platforms: ios, macos, android — optional elsewhere +``` + +```yaml +event: lk.device.app_state.changed +area: device +attributes: + lk.device.app_state: enum(foreground | background) +cadence: on change (+ initial value); entering background also forces a flush +platforms: all +``` + +## Cadence policy + +`flush_interval × factor`, capped at 4× (15 s → 60 s at the production cadence): + +| Condition | factor | +|---|---| +| thermal `serious` | 2 | +| thermal `critical` | 4 | +| low-power mode | 2 | +| background | 2 | diff --git a/livekit-telemetry/src/device.rs b/livekit-telemetry/src/device.rs new file mode 100644 index 000000000..e483351e5 --- /dev/null +++ b/livekit-telemetry/src/device.rs @@ -0,0 +1,141 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::TelemetryEvent; + +/// Thermal pressure as the OS reports it (`ProcessInfo.thermalState`, `PowerManager` thermal +/// status). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ThermalState { + #[default] + Nominal, + Fair, + Serious, + Critical, +} + +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum AppState { + #[default] + Foreground, + Background, +} + +/// What the host observes about the device; pushed with +/// [`Telemetry::set_device_state`](crate::Telemetry::set_device_state) whenever it changes. +/// +/// The host owns the OS APIs (thermal, power, lifecycle) — they are not reachable from Rust +/// without a JVM/ObjC bridge — and the core owns what to do with them: emit the change events +/// from `SPEC.md` and stretch its own cadence so telemetry never competes with the call. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct DeviceState { + pub thermal: ThermalState, + pub low_power_mode: bool, + pub app_state: AppState, +} + +impl DeviceState { + /// Multiplier on the flush interval: 1× at rest, up to 4× under pressure (15 s → 60 s at the + /// production cadence, per the design doc). Serious thermal, low-power mode and background + /// each double it; critical thermal quadruples it. + pub fn cadence_factor(&self) -> u32 { + let thermal = match self.thermal { + ThermalState::Nominal | ThermalState::Fair => 1, + ThermalState::Serious => 2, + ThermalState::Critical => 4, + }; + let power = if self.low_power_mode { 2 } else { 1 }; + let background = if self.app_state == AppState::Background { 2 } else { 1 }; + (thermal * power * background).min(4) + } + + /// The `SPEC.md` events describing what changed between `previous` and `self` + /// (everything, when there is no previous state: "initial value at session start"). + pub(crate) fn change_events(&self, previous: Option<&DeviceState>) -> Vec { + let mut events = Vec::new(); + if previous.is_none_or(|p| p.thermal != self.thermal) { + events.push( + TelemetryEvent::new("lk.device.thermal.changed") + .with_attribute("lk.device.thermal.state", self.thermal.as_str()), + ); + } + if previous.is_none_or(|p| p.low_power_mode != self.low_power_mode) { + events.push( + TelemetryEvent::new("lk.device.low_power.changed") + .with_attribute("lk.device.low_power.enabled", self.low_power_mode), + ); + } + if previous.is_none_or(|p| p.app_state != self.app_state) { + events.push( + TelemetryEvent::new("lk.device.app_state.changed") + .with_attribute("lk.device.app_state", self.app_state.as_str()), + ); + } + events + } +} + +impl ThermalState { + fn as_str(self) -> &'static str { + match self { + ThermalState::Nominal => "nominal", + ThermalState::Fair => "fair", + ThermalState::Serious => "serious", + ThermalState::Critical => "critical", + } + } +} + +impl AppState { + fn as_str(self) -> &'static str { + match self { + AppState::Foreground => "foreground", + AppState::Background => "background", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cadence_stretches_under_pressure_and_caps_at_four() { + assert_eq!(DeviceState::default().cadence_factor(), 1); + let serious = DeviceState { thermal: ThermalState::Serious, ..Default::default() }; + assert_eq!(serious.cadence_factor(), 2); + let low_power = DeviceState { low_power_mode: true, ..Default::default() }; + assert_eq!(low_power.cadence_factor(), 2); + let everything = DeviceState { + thermal: ThermalState::Critical, + low_power_mode: true, + app_state: AppState::Background, + }; + assert_eq!(everything.cadence_factor(), 4); + } + + #[test] + fn only_changed_fields_produce_events() { + let initial = DeviceState::default(); + assert_eq!(initial.change_events(None).len(), 3, "initial value at session start"); + let hot = DeviceState { thermal: ThermalState::Critical, ..initial }; + let events = hot.change_events(Some(&initial)); + assert_eq!(events.len(), 1); + assert_eq!(events[0].name, "lk.device.thermal.changed"); + assert!(hot.change_events(Some(&hot)).is_empty()); + } +} diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index d6231706d..53c1a2c65 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -15,11 +15,18 @@ use std::{sync::Arc, time::Duration}; use livekit_runtime::{interval, sleep, timeout, Instant, MissedTickBehavior}; -use tokio::sync::{mpsc, oneshot}; +use tokio::{ + sync::{mpsc, oneshot}, + time::Interval, +}; use crate::{ - event::now_unix_nanos, otlp, store::Store, BatchCache, ExportError, ExportRequest, - TelemetryConfig, TelemetryTransport, + event::now_unix_nanos, + otlp, + stats::{Counters, Snapshot}, + store::Store, + AppState, BatchCache, DeviceState, ExportError, ExportRequest, TelemetryConfig, + TelemetryTransport, }; /// Retries per upload attempt after the first, for failures without `Retry-After`. @@ -32,6 +39,7 @@ const UPLOAD_BACKOFF: Duration = Duration::from_secs(60); pub(crate) enum Command { Flush(oneshot::Sender<()>), Shutdown(oneshot::Sender<()>), + DeviceState(DeviceState), } /// Outcome of uploading one encoded batch. @@ -49,15 +57,18 @@ enum Delivery { /// Background actor that turns stored events into OTLP requests. /// -/// The role of OTel's `BatchLogRecordProcessor` + OTLP exporter in one place. Every -/// `flush_interval_ms` it [`enqueue`](Self::enqueue)s: drains up to `max_batch_size` events, -/// encodes them and writes the batch to the [`BatchCache`] *before* any network is involved; -/// then it [`upload`](Self::upload)s the cache oldest-first through the -/// [`TelemetryTransport`], removing what the collector accepted or rejected. A failed upload -/// pauses the cache for a minute; a `Retry-After` additionally drops new batches for its -/// duration (throttling must not become a disk-backed queue); `Disabled` empties the cache and -/// silences the exporter for good. With a [`FileCache`](crate::FileCache) this is crash- and -/// restart-safe; with the default [`MemoryCache`](crate::MemoryCache) it is process-bound. +/// The role of OTel's `BatchLogRecordProcessor` + OTLP exporter in one place. Every tick it +/// [`enqueue`](Self::enqueue)s: drains up to `max_batch_size` events, appends an +/// `lk.telemetry.report` when something was dropped or failed since the last one, encodes the +/// batch and writes it to the [`BatchCache`] *before* any network is involved; then it +/// [`upload`](Self::upload)s the cache oldest-first through the [`TelemetryTransport`], +/// removing what the collector accepted or rejected. A failed upload pauses the cache for a +/// minute; a `Retry-After` additionally drops new batches for its duration (throttling must not +/// become a disk-backed queue); `Disabled` empties the cache and silences the exporter for good. +/// +/// The tick period is `flush_interval_ms × DeviceState::cadence_factor`: thermal pressure, +/// low-power mode and the background stretch it up to 4×, and entering the background flushes +/// once immediately (the app may be suspended any moment). /// /// Drive it with `spawn(exporter.run())` on the consumer's runtime. It stops after /// [`Telemetry::shutdown`](crate::Telemetry::shutdown) or when the last @@ -67,6 +78,7 @@ pub struct Exporter { transport: Arc, config: Arc, cache: Arc, + counters: Arc, commands: mpsc::UnboundedReceiver, silenced: bool, /// Leave the cache alone until then: the last upload failed or we were throttled. @@ -76,6 +88,9 @@ pub struct Exporter { seq: u64, /// Batches the cache could not store (disk full, directory unusable). Drives one-shot logging. cache_failures: u64, + /// Counter values at the last `lk.telemetry.report`. + last_report: Snapshot, + cadence_factor: u32, } impl Exporter { @@ -84,6 +99,7 @@ impl Exporter { transport: Arc, config: Arc, cache: Arc, + counters: Arc, commands: mpsc::UnboundedReceiver, ) -> Self { Self { @@ -91,12 +107,15 @@ impl Exporter { transport, config, cache, + counters, commands, silenced: false, paused_until: None, throttled_until: None, seq: 0, cache_failures: 0, + last_report: Snapshot::default(), + cadence_factor: 1, } } @@ -104,8 +123,7 @@ impl Exporter { /// demand. pub async fn run(mut self) { self.upload().await; - let mut ticker = interval(Duration::from_millis(self.config.flush_interval_ms.max(1))); - ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + let mut ticker = self.ticker(); loop { tokio::select! { _ = ticker.tick() => self.export_pending().await, @@ -114,6 +132,16 @@ impl Exporter { self.export_pending().await; let _ = done.send(()); } + Some(Command::DeviceState(state)) => { + let factor = state.cadence_factor(); + if factor != self.cadence_factor { + self.cadence_factor = factor; + ticker = self.ticker(); + } + if state.app_state == AppState::Background { + self.export_pending().await; + } + } Some(Command::Shutdown(done)) => { // Last chance: ignore the upload backoff, but respect throttling. self.paused_until = self.throttled_until; @@ -132,6 +160,14 @@ impl Exporter { } } + fn ticker(&self) -> Interval { + let period = + Duration::from_millis(self.config.flush_interval_ms.max(1)) * self.cadence_factor; + let mut ticker = interval(period); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + ticker + } + async fn export_pending(&mut self) { self.enqueue(); self.upload().await; @@ -140,15 +176,27 @@ impl Exporter { /// Encode everything queued into the cache — no network involved. fn enqueue(&mut self) { loop { - let batch = self.store.drain(self.config.max_batch_size.max(1) as usize); + let mut batch = self.store.drain(self.config.max_batch_size.max(1) as usize); if batch.is_empty() { return; } - let count = batch.len() as u64; - if self.silenced || self.throttled_until.is_some_and(|t| Instant::now() < t) { - self.store.add_dropped(count); + if self.silenced { + Counters::add(&self.counters.disabled, batch.len() as u64); continue; } + if self.throttled_until.is_some_and(|t| Instant::now() < t) { + Counters::add(&self.counters.throttled, batch.len() as u64); + continue; + } + // Self-telemetry rides along with real data: never its own request, never its own + // cadence, and only when there is something to report. + let now = self.counters.snapshot(); + let delta = now.since(&self.last_report); + if delta.has_problems() { + batch.push(delta.report(self.cache.pending().len() as u64)); + self.last_report = now; + } + let count = batch.len() as u64; let body = otlp::encode_logs(&self.config.resource, batch); self.seq += 1; let id = format!("{:020}-{:06}-{count}", now_unix_nanos(), self.seq); @@ -162,7 +210,7 @@ impl Exporter { log::debug!("telemetry: could not cache {count} events: {err}"); } self.cache_failures += 1; - self.store.add_dropped(count); + Counters::add(&self.counters.cache_error, count); } } } @@ -178,10 +226,13 @@ impl Exporter { continue; }; match self.deliver(&body).await { - Delivery::Sent => self.cache.remove(&id), + Delivery::Sent => { + self.cache.remove(&id); + Counters::add(&self.counters.uploads_sent, 1); + } Delivery::Rejected => { self.cache.remove(&id); - self.store.add_dropped(events_in(&id)); + Counters::add(&self.counters.rejected, events_in(&id)); } Delivery::Failed => { self.paused_until = Some(Instant::now() + UPLOAD_BACKOFF); @@ -206,7 +257,7 @@ impl Exporter { log::warn!("telemetry disabled by the collector; going silent"); self.silenced = true; let cached: u64 = self.cache.pending().iter().map(|id| events_in(id)).sum(); - self.store.add_dropped(cached); + Counters::add(&self.counters.disabled, cached); self.cache.clear(); } @@ -234,8 +285,12 @@ impl Exporter { } Ok(Err(ExportError::Retryable { message, retry_after_ms: None })) => { log::debug!("telemetry upload failed (attempt {}): {message}", attempt + 1); + Counters::add(&self.counters.upload_failures, 1); + } + Err(_) => { + log::debug!("telemetry upload timed out (attempt {})", attempt + 1); + Counters::add(&self.counters.upload_failures, 1); } - Err(_) => log::debug!("telemetry upload timed out (attempt {})", attempt + 1), } if attempt < MAX_RETRIES { sleep(RETRY_BACKOFF * (attempt + 1)).await; diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs index 595c1a6e4..0ed716baa 100644 --- a/livekit-telemetry/src/lib.rs +++ b/livekit-telemetry/src/lib.rs @@ -20,6 +20,12 @@ mod event; /// Bounded in-memory queue between `emit` and the exporter. mod store; +/// Pipeline health counters and the `lk.telemetry.report` event. +mod stats; + +/// Host-reported device state and the cadence policy derived from it. +mod device; + /// Batch exporter actor: timer, OTLP encoding, retry policy. mod exporter; @@ -39,8 +45,10 @@ mod transport; mod telemetry; pub use cache::{BatchCache, FileCache, MemoryCache}; +pub use device::*; pub use event::*; pub use exporter::Exporter; +pub use stats::TelemetryStats; pub use telemetry::*; pub use transport::*; diff --git a/livekit-telemetry/src/stats.rs b/livekit-telemetry/src/stats.rs new file mode 100644 index 000000000..cf657c055 --- /dev/null +++ b/livekit-telemetry/src/stats.rs @@ -0,0 +1,152 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::TelemetryEvent; + +/// Pipeline health counters, shared by the store, the exporter and [`Telemetry::stats`](crate::Telemetry::stats). +/// +/// Reasons follow the OpenTelemetry SDK self-metrics conventions (`queue_full`, `rejected`, +/// `timeout`) so they line up with `otel.sdk.processor.log.processed` / +/// `otel.sdk.exporter.log.exported` on a backend that knows those. +#[derive(Default)] +pub(crate) struct Counters { + /// Events evicted from the in-memory queue. + pub queue_full: AtomicU64, + /// Events lost because the cache could not store their batch (disk full, unusable dir). + pub cache_error: AtomicU64, + /// Events the collector rejected (4xx). + pub rejected: AtomicU64, + /// Events dropped inside a `Retry-After` window. + pub throttled: AtomicU64, + /// Events dropped after the collector disabled telemetry. + pub disabled: AtomicU64, + /// Batches the collector accepted. + pub uploads_sent: AtomicU64, + /// Upload attempts that failed transiently (network, timeout, 5xx). + pub upload_failures: AtomicU64, +} + +impl Counters { + pub fn add(counter: &AtomicU64, n: u64) { + counter.fetch_add(n, Ordering::Relaxed); + } + + pub fn snapshot(&self) -> Snapshot { + let get = |c: &AtomicU64| c.load(Ordering::Relaxed); + Snapshot { + queue_full: get(&self.queue_full), + cache_error: get(&self.cache_error), + rejected: get(&self.rejected), + throttled: get(&self.throttled), + disabled: get(&self.disabled), + uploads_sent: get(&self.uploads_sent), + upload_failures: get(&self.upload_failures), + } + } +} + +/// A point-in-time copy of [`Counters`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct Snapshot { + pub queue_full: u64, + pub cache_error: u64, + pub rejected: u64, + pub throttled: u64, + pub disabled: u64, + pub uploads_sent: u64, + pub upload_failures: u64, +} + +impl Snapshot { + /// Counts accumulated since `earlier`. + pub fn since(&self, earlier: &Snapshot) -> Snapshot { + Snapshot { + queue_full: self.queue_full.saturating_sub(earlier.queue_full), + cache_error: self.cache_error.saturating_sub(earlier.cache_error), + rejected: self.rejected.saturating_sub(earlier.rejected), + throttled: self.throttled.saturating_sub(earlier.throttled), + disabled: self.disabled.saturating_sub(earlier.disabled), + uploads_sent: self.uploads_sent.saturating_sub(earlier.uploads_sent), + upload_failures: self.upload_failures.saturating_sub(earlier.upload_failures), + } + } + + /// Anything worth telling the backend about: data lost or uploads failing. + pub fn has_problems(&self) -> bool { + self.queue_full + self.cache_error + self.rejected + self.throttled + self.upload_failures + > 0 + } + + /// The `lk.telemetry.report` event: what this pipeline dropped or failed to upload since the + /// previous report. The Sentry "client report" shape — deltas by reason, riding along with + /// the next batch, never persisted on their own, never an extra request. + pub fn report(&self, cached_batches: u64) -> TelemetryEvent { + let mut event = TelemetryEvent::new("lk.telemetry.report") + .with_attribute("lk.telemetry.uploads.failed", self.upload_failures as i64) + .with_attribute("lk.telemetry.uploads.sent", self.uploads_sent as i64) + .with_attribute("lk.telemetry.cache.batches", cached_batches as i64); + for (key, value) in [ + ("lk.telemetry.dropped.queue_full", self.queue_full), + ("lk.telemetry.dropped.cache_error", self.cache_error), + ("lk.telemetry.dropped.rejected", self.rejected), + ("lk.telemetry.dropped.throttled", self.throttled), + ] { + if value > 0 { + event = event.with_attribute(key, value as i64); + } + } + event + } +} + +/// Pipeline health as seen by the SDK: [`Telemetry::stats`](crate::Telemetry::stats). +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TelemetryStats { + /// Events lost for any reason (sum of the `dropped_*` fields). + pub dropped: u64, + pub dropped_queue_full: u64, + pub dropped_cache_error: u64, + pub dropped_rejected: u64, + pub dropped_throttled: u64, + pub dropped_disabled: u64, + /// Batches the collector accepted. + pub uploads_sent: u64, + /// Upload attempts that failed transiently. + pub upload_failures: u64, + /// Batches currently waiting in the cache. + pub cached_batches: u64, +} + +impl TelemetryStats { + pub(crate) fn new(snapshot: Snapshot, cached_batches: u64) -> Self { + Self { + dropped: snapshot.queue_full + + snapshot.cache_error + + snapshot.rejected + + snapshot.throttled + + snapshot.disabled, + dropped_queue_full: snapshot.queue_full, + dropped_cache_error: snapshot.cache_error, + dropped_rejected: snapshot.rejected, + dropped_throttled: snapshot.throttled, + dropped_disabled: snapshot.disabled, + uploads_sent: snapshot.uploads_sent, + upload_failures: snapshot.upload_failures, + cached_batches, + } + } +} diff --git a/livekit-telemetry/src/store.rs b/livekit-telemetry/src/store.rs index 3ce60bf8b..8f7db1eea 100644 --- a/livekit-telemetry/src/store.rs +++ b/livekit-telemetry/src/store.rs @@ -14,40 +14,33 @@ use std::{ collections::VecDeque, - sync::{ - atomic::{AtomicU64, Ordering}, - Mutex, - }, + sync::{Arc, Mutex}, }; -use crate::TelemetryEvent; +use crate::{stats::Counters, TelemetryEvent}; /// Bounded FIFO of events waiting for export. /// /// When full, the *oldest* event is dropped so the freshest context survives a burst /// (the queue role of OTel's `BatchLogRecordProcessor`, with drop-oldest instead of -/// drop-newest). Also keeps the pipeline's own health counter: events dropped anywhere. +/// drop-newest); every eviction is counted as `queue_full`. // ponytail: one mutex around a VecDeque; a lock-free ring only if `emit` shows up in a profile. pub(crate) struct Store { queue: Mutex>, capacity: usize, - dropped: AtomicU64, + counters: Arc, } impl Store { - pub fn new(capacity: usize) -> Self { - Self { - queue: Mutex::new(VecDeque::with_capacity(capacity.min(1024))), - capacity, - dropped: AtomicU64::new(0), - } + pub fn new(capacity: usize, counters: Arc) -> Self { + Self { queue: Mutex::new(VecDeque::with_capacity(capacity.min(1024))), capacity, counters } } pub fn push(&self, event: TelemetryEvent) { let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); if queue.len() >= self.capacity { queue.pop_front(); - self.dropped.fetch_add(1, Ordering::Relaxed); + Counters::add(&self.counters.queue_full, 1); } queue.push_back(event); } @@ -58,14 +51,6 @@ impl Store { let n = max.min(queue.len()); queue.drain(..n).collect() } - - pub fn add_dropped(&self, n: u64) { - self.dropped.fetch_add(n, Ordering::Relaxed); - } - - pub fn dropped(&self) -> u64 { - self.dropped.load(Ordering::Relaxed) - } } #[cfg(test)] @@ -74,13 +59,14 @@ mod tests { #[test] fn drops_oldest_when_full() { - let store = Store::new(2); + let counters = Arc::new(Counters::default()); + let store = Store::new(2, counters.clone()); for name in ["a", "b", "c"] { store.push(TelemetryEvent::new(name)); } let names: Vec<_> = store.drain(10).into_iter().map(|e| e.name).collect(); assert_eq!(names, ["b", "c"]); - assert_eq!(store.dropped(), 1); + assert_eq!(counters.snapshot().queue_full, 1); assert!(store.drain(10).is_empty()); } } diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index b1879e0cd..e5624807a 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -12,14 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{collections::HashMap, sync::Arc, time::Duration}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::Duration, +}; use livekit_runtime::timeout; use tokio::sync::{mpsc, oneshot}; use crate::{ - event::now_unix_nanos, exporter::Command, store::Store, Attribute, BatchCache, Exporter, - FileCache, MemoryCache, TelemetryEvent, TelemetryTransport, + event::now_unix_nanos, exporter::Command, stats::Counters, store::Store, Attribute, BatchCache, + DeviceState, Exporter, FileCache, MemoryCache, TelemetryEvent, TelemetryStats, + TelemetryTransport, }; /// Pipeline configuration. @@ -44,6 +49,7 @@ pub struct TelemetryConfig { /// Cap on cached batches, in memory or on disk; the oldest are evicted first. #[cfg_attr(feature = "uniffi", uniffi(default = 4194304))] pub max_cache_bytes: u64, + /// Base export cadence; stretched up to 4× by [`DeviceState::cadence_factor`]. #[cfg_attr(feature = "uniffi", uniffi(default = 1000))] pub flush_interval_ms: u64, /// Events buffered before the oldest are dropped. @@ -77,8 +83,8 @@ impl TelemetryConfig { /// Entry point: the synchronous, never-blocking side SDKs push into. /// /// Fail-open by design: [`emit`](Self::emit) cannot fail or block — when the queue is full the -/// oldest event is dropped and counted in [`dropped_count`](Self::dropped_count). Cheap to -/// clone; every clone feeds the same pipeline. +/// oldest event is dropped and counted in [`stats`](Self::stats). Cheap to clone; every clone +/// feeds the same pipeline. /// /// ``` /// # use std::sync::Arc; @@ -101,6 +107,9 @@ impl TelemetryConfig { pub struct Telemetry { store: Arc, config: Arc, + cache: Arc, + counters: Arc, + device: Arc>>, commands: mpsc::UnboundedSender, } @@ -133,10 +142,19 @@ impl Telemetry { ) -> (Self, Exporter) { add_sdk_resource(&mut config.resource); let config = Arc::new(config); - let store = Arc::new(Store::new(config.max_queue_size.max(1) as usize)); + let counters = Arc::new(Counters::default()); + let store = Arc::new(Store::new(config.max_queue_size.max(1) as usize, counters.clone())); let (commands, receiver) = mpsc::unbounded_channel(); - let exporter = Exporter::new(store.clone(), transport, config.clone(), cache, receiver); - (Self { store, config, commands }, exporter) + let exporter = Exporter::new( + store.clone(), + transport, + config.clone(), + cache.clone(), + counters.clone(), + receiver, + ); + let telemetry = Self { store, config, cache, counters, device: Arc::default(), commands }; + (telemetry, exporter) } /// Queue an event for export. Stamps it with the current time unless it carries one. @@ -147,6 +165,19 @@ impl Telemetry { self.store.push(event); } + /// Tell the pipeline what the device looks like. Emits the `lk.device.*.changed` events for + /// whatever differs from the last state (everything, the first time) and re-tunes the export + /// cadence: thermal pressure, low-power mode and the background stretch it up to 4×; entering + /// the background also flushes once right away. + pub fn set_device_state(&self, state: DeviceState) { + let mut previous = self.device.lock().unwrap_or_else(|e| e.into_inner()); + for event in state.change_events(previous.as_ref()) { + self.emit(event); + } + *previous = Some(state); + let _ = self.commands.send(Command::DeviceState(state)); + } + /// Cache everything queued and upload what the transport accepts right now. pub async fn flush(&self) { self.command(Command::Flush).await; @@ -160,10 +191,10 @@ impl Telemetry { let _ = timeout(bound, self.command(Command::Shutdown)).await; } - /// Events dropped so far (queue overflow, cache failure, rejected or throttled batches, - /// disabled collector). - pub fn dropped_count(&self) -> u64 { - self.store.dropped() + /// Pipeline health: drops by reason, uploads, cached batches. The same numbers ride to the + /// backend as `lk.telemetry.report` events whenever something went wrong. + pub fn stats(&self) -> TelemetryStats { + TelemetryStats::new(self.counters.snapshot(), self.cache.pending().len() as u64) } async fn command(&self, make: impl FnOnce(oneshot::Sender<()>) -> Command) { @@ -198,8 +229,11 @@ mod tests { use super::*; use crate::{ cache::temp_dir, - proto::opentelemetry::proto::collector::logs::v1::ExportLogsServiceRequest, ExportError, - ExportRequest, + proto::opentelemetry::proto::{ + collector::logs::v1::ExportLogsServiceRequest, common::v1::any_value::Value, + logs::v1::LogRecord, + }, + AppState, ExportError, ExportRequest, ThermalState, }; #[derive(Default)] @@ -256,13 +290,17 @@ mod tests { fs::read_dir(dir).map(|d| d.count()).unwrap_or(0) } - fn event_names(request: &ExportRequest) -> Vec { + fn records(request: &ExportRequest) -> Vec { let decoded = ExportLogsServiceRequest::decode(&request.body[..]).expect("valid OTLP"); - decoded.resource_logs[0].scope_logs[0] - .log_records - .iter() - .map(|r| r.event_name.clone()) - .collect() + decoded.resource_logs[0].scope_logs[0].log_records.clone() + } + + fn event_names(request: &ExportRequest) -> Vec { + records(request).iter().map(|r| r.event_name.clone()).collect() + } + + fn attribute(record: &LogRecord, key: &str) -> Option { + record.attributes.iter().find(|kv| kv.key == key)?.value.as_ref()?.value.clone() } #[tokio::test(start_paused = true)] @@ -282,7 +320,8 @@ mod tests { let decoded = ExportLogsServiceRequest::decode(&sent[0].body[..]).expect("valid OTLP"); let resource = decoded.resource_logs[0].resource.as_ref().expect("resource"); assert!(resource.attributes.iter().any(|kv| kv.key == "telemetry.sdk.name")); - assert_eq!(telemetry.dropped_count(), 0); + assert_eq!(telemetry.stats().dropped, 0); + assert_eq!(telemetry.stats().uploads_sent, 1); } #[tokio::test(start_paused = true)] @@ -292,7 +331,9 @@ mod tests { telemetry.emit(TelemetryEvent::new("lk.ping")); telemetry.flush().await; assert_eq!(transport.sent().len(), 3, "first attempt plus two retries"); - assert_eq!(telemetry.dropped_count(), 0, "kept in the memory cache, not dropped"); + assert_eq!(telemetry.stats().dropped, 0, "kept in the memory cache, not dropped"); + assert_eq!(telemetry.stats().upload_failures, 3); + assert_eq!(telemetry.stats().cached_batches, 1); telemetry.flush().await; assert_eq!(transport.sent().len(), 3, "backoff: no upload right away"); @@ -300,6 +341,45 @@ mod tests { tokio::time::sleep(Duration::from_secs(61)).await; assert_eq!(transport.sent().len(), 4, "retried once the backoff elapsed"); assert_eq!(event_names(&transport.sent()[3]), ["lk.ping"]); + assert_eq!(telemetry.stats().cached_batches, 0); + } + + #[tokio::test(start_paused = true)] + async fn self_telemetry_report_rides_along_after_problems() { + let transport = FakeTransport::scripted([offline(), offline(), offline()]); + let telemetry = pipeline(transport.clone()); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + tokio::time::sleep(Duration::from_secs(61)).await; // backoff over, batch uploads + + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + let sent = transport.sent(); + let last = &sent[sent.len() - 1]; + assert_eq!(event_names(last), ["lk.ping", "lk.telemetry.report"]); + let report = &records(last)[1]; + assert_eq!(attribute(report, "lk.telemetry.uploads.failed"), Some(Value::IntValue(3))); + assert_eq!(attribute(report, "lk.telemetry.uploads.sent"), Some(Value::IntValue(1))); + assert_eq!(attribute(report, "lk.telemetry.cache.batches"), Some(Value::IntValue(0))); + assert_eq!(attribute(report, "lk.telemetry.dropped.queue_full"), None, "zeros omitted"); + + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + let sent = transport.sent(); + assert_eq!(event_names(&sent[sent.len() - 1]), ["lk.ping"], "nothing new to report"); + } + + #[tokio::test(start_paused = true)] + async fn queue_overflow_is_counted_by_reason() { + let mut config = TelemetryConfig::new("http://collector/v1/logs"); + config.max_queue_size = 1; + let telemetry = start(config, FakeTransport::scripted([])); + for _ in 0..3 { + telemetry.emit(TelemetryEvent::new("lk.ping")); + } + let stats = telemetry.stats(); + assert_eq!(stats.dropped_queue_full, 2); + assert_eq!(stats.dropped, 2); } #[tokio::test(start_paused = true)] @@ -311,7 +391,7 @@ mod tests { telemetry.flush().await; assert_eq!(transport.sent().len(), 1); - assert_eq!(telemetry.dropped_count(), 1); + assert_eq!(telemetry.stats().dropped_rejected, 1); } #[tokio::test(start_paused = true)] @@ -324,7 +404,7 @@ mod tests { telemetry.shutdown().await; assert_eq!(transport.sent().len(), 1); - assert_eq!(telemetry.dropped_count(), 2); + assert_eq!(telemetry.stats().dropped_disabled, 2); } #[tokio::test(start_paused = true)] @@ -335,7 +415,7 @@ mod tests { first.emit(TelemetryEvent::new("lk.ping")); first.flush().await; assert_eq!(first_transport.sent().len(), 3); - assert_eq!(first.dropped_count(), 0); + assert_eq!(first.stats().dropped, 0); assert_eq!(files_in(&dir), 1, "written before the first attempt, kept after failure"); let second_transport = FakeTransport::scripted([]); @@ -359,11 +439,11 @@ mod tests { telemetry.flush().await; assert_eq!(transport.sent().len(), 1, "no retries on Retry-After"); assert_eq!(files_in(&dir), 1, "the throttled batch stays cached"); - assert_eq!(telemetry.dropped_count(), 0); + assert_eq!(telemetry.stats().dropped, 0); telemetry.emit(TelemetryEvent::new("lk.ping")); telemetry.flush().await; - assert_eq!(telemetry.dropped_count(), 1, "new batches are dropped inside the window"); + assert_eq!(telemetry.stats().dropped_throttled, 1, "new batches dropped inside the window"); assert_eq!(files_in(&dir), 1, "and never written"); tokio::time::sleep(Duration::from_secs(6)).await; @@ -381,7 +461,7 @@ mod tests { telemetry.shutdown().await; assert_eq!(files_in(&dir), 1, "cached before the network was tried, kept after it failed"); - assert_eq!(telemetry.dropped_count(), 0); + assert_eq!(telemetry.stats().dropped, 0); let _ = fs::remove_dir_all(&dir); } @@ -401,4 +481,50 @@ mod tests { assert_eq!(pending.len(), 1); assert!(pending[0].ends_with("-1"), "id carries the event count: {}", pending[0]); } + + #[tokio::test(start_paused = true)] + async fn device_state_emits_change_events_and_stretches_cadence() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + telemetry.set_device_state(DeviceState { + thermal: ThermalState::Critical, + ..DeviceState::default() + }); + telemetry.flush().await; + let sent = transport.sent(); + assert_eq!(sent.len(), 1); + let names = event_names(&sent[0]); + assert!(names.contains(&"lk.device.thermal.changed".to_owned()), "{names:?}"); + assert_eq!(names.len(), 3, "initial value for every field"); + let thermal = records(&sent[0]) + .into_iter() + .find(|r| r.event_name == "lk.device.thermal.changed") + .expect("thermal event"); + assert_eq!( + attribute(&thermal, "lk.device.thermal.state"), + Some(Value::StringValue("critical".into())) + ); + + // 1 s base interval × 4 under critical thermal pressure. + telemetry.emit(TelemetryEvent::new("lk.ping")); + tokio::time::sleep(Duration::from_secs(2)).await; + assert_eq!(transport.sent().len(), 1, "not yet: cadence stretched to 4 s"); + tokio::time::sleep(Duration::from_millis(2_500)).await; + assert_eq!(transport.sent().len(), 2, "exported on the stretched tick"); + } + + #[tokio::test(start_paused = true)] + async fn entering_background_flushes_immediately() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.set_device_state(DeviceState { + app_state: AppState::Background, + ..DeviceState::default() + }); + tokio::time::sleep(Duration::from_millis(10)).await; + let sent = transport.sent(); + assert_eq!(sent.len(), 1, "flushed on the state change, not on the tick"); + assert!(event_names(&sent[0]).contains(&"lk.ping".to_owned())); + } } diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index 64012ffa4..b15f02297 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -15,14 +15,17 @@ //! Client telemetry core from the [`livekit-telemetry`] crate. //! //! FFI clients construct one [`Telemetry`] per pipeline with a host-implemented -//! `TelemetryTransport` (e.g. a URLSession/OkHttp POST), then `emit` from any thread. The -//! exporter runs on the global runtime; `shutdown` flushes within `export_timeout_ms`. +//! `TelemetryTransport` (e.g. a URLSession/OkHttp POST), then `emit` from any thread and push +//! `DeviceState` changes as the OS reports them. The exporter runs on the global runtime; +//! `shutdown` flushes within `export_timeout_ms`. use std::sync::Arc; -use livekit_telemetry::{TelemetryConfig, TelemetryEvent, TelemetryTransport}; +use livekit_telemetry::{ + DeviceState, TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, +}; -/// Telemetry pipeline: buffer, batch and export events as OTLP. +/// Telemetry pipeline: buffer, batch, cache and export events as OTLP. #[derive(uniffi::Object)] pub struct Telemetry(livekit_telemetry::Telemetry); @@ -40,6 +43,12 @@ impl Telemetry { self.0.emit(event); } + /// Report the device state (thermal, low power, foreground/background). Emits the matching + /// `lk.device.*.changed` events and adapts the export cadence. + pub fn set_device_state(&self, state: DeviceState) { + self.0.set_device_state(state); + } + /// Export everything queued and wait for the transport. pub async fn flush(&self) { self.0.flush().await; @@ -50,8 +59,8 @@ impl Telemetry { self.0.shutdown().await; } - /// Events dropped so far (queue overflow, rejected/failed exports, disabled collector). - pub fn dropped_count(&self) -> u64 { - self.0.dropped_count() + /// Pipeline health: drops by reason, uploads, cached batches. + pub fn stats(&self) -> TelemetryStats { + self.0.stats() } } From 763dd736f85360393332b5a70dcd6494fb27098a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Wed, 26 Aug 2026 13:21:42 +0200 Subject: [PATCH 07/50] feat(telemetry): log records, flood guard, RTC stats windows, session attributes The remaining v0 surface from the design doc: - Log records: a TelemetryEvent with an empty name is a plain OTLP log record (severity + body, no event_name). Only Warn/Error leave the device; emit() drops Trace/Debug/Info. - Flood guard: discrete events are capped at max_events_per_10min (300); the excess is dropped and counted as rate_limited (stats + report). RTC windows and self-telemetry are exempt. - RTC stats: platforms push raw getStats() readings as RtcStatsSample every 1-2 s; StatsWindows folds them per track and direction into one lk.rtc.stats.sample per stats_window_ms (15 s, stretched with the cadence): cumulative counters as the last value (monotonic, W3C webrtc-stats model), gauges as min/max/avg. Windows close on the stats tick, on background and on shutdown; the window ticker starts a full period out so early readings are not shipped as a zero-length window. - Session attributes: Telemetry::set_attribute(key, value) attaches lk.room.sid / lk.participant.identity / app correlation ids to every record at encode time without overriding explicit ones. UniFFI: Telemetry.recordStats(sample:), Telemetry.setAttribute(key:value:), RtcStatsSample / TrackKind / StreamDirection. ios-arm64 dylib 1,211,720 B (+17.6 KB, mostly the 21-field RtcStatsSample record lowering). Co-Authored-By: Claude Fable 5 --- .changeset/add-livekit-telemetry-crate.md | 4 +- livekit-telemetry/README.md | 14 + livekit-telemetry/SPEC.md | 47 ++++ livekit-telemetry/src/exporter.rs | 73 ++++- livekit-telemetry/src/lib.rs | 4 + livekit-telemetry/src/rtc.rs | 317 ++++++++++++++++++++++ livekit-telemetry/src/stats.rs | 18 +- livekit-telemetry/src/telemetry.rs | 216 ++++++++++++++- livekit-uniffi/src/telemetry.rs | 14 +- 9 files changed, 688 insertions(+), 19 deletions(-) create mode 100644 livekit-telemetry/src/rtc.rs diff --git a/.changeset/add-livekit-telemetry-crate.md b/.changeset/add-livekit-telemetry-crate.md index 9c007cce3..e8ea7ef5c 100644 --- a/.changeset/add-livekit-telemetry-crate.md +++ b/.changeset/add-livekit-telemetry-crate.md @@ -7,4 +7,6 @@ batched and exported as OTLP/HTTP log records through a pluggable `TelemetryTran written to a `BatchCache` (in memory by default, on disk with `storage_dir`) before upload, so failed uploads, crashes and offline shutdowns lose nothing. Pipeline health is exposed as `Telemetry::stats` and shipped as `lk.telemetry.report` events; hosts push `DeviceState` (thermal, low power, foreground/background) and the core emits -the `lk.device.*.changed` events and stretches its cadence under pressure. +the `lk.device.*.changed` events and stretches its cadence under pressure. Log records (`Warn`/`Error` only), a flood guard for discrete events, on-device RTC +stats windows (`record_stats` → `lk.rtc.stats.sample`) and session-wide attributes complete the +design doc's v0 surface. diff --git a/livekit-telemetry/README.md b/livekit-telemetry/README.md index 6559774a2..68b7801d1 100644 --- a/livekit-telemetry/README.md +++ b/livekit-telemetry/README.md @@ -33,6 +33,11 @@ let (telemetry, exporter) = Telemetry::new(config, Arc::new(transport)); tokio::spawn(exporter.run()); telemetry.emit(TelemetryEvent::new("lk.ping").with_attribute("lk.ping.seq", 1i64)); +telemetry.emit(TelemetryEvent::new("").with_severity(Severity::Error).with_body("connect failed")); // log record +let mut sample = RtcStatsSample::new("TR_abc", TrackKind::Audio, StreamDirection::Inbound); +sample.bytes = Some(48_000); +sample.jitter_ms = Some(12.0); +telemetry.record_stats(sample); // every 1–2 s; shipped as one lk.rtc.stats.sample per 15 s window telemetry.set_device_state(DeviceState { thermal: ThermalState::Serious, ..Default::default() }); telemetry.shutdown().await; // cache, then upload what the network allows; bounded by `export_timeout_ms` println!("{:?}", telemetry.stats()); // drops by reason, uploads, cached batches @@ -70,6 +75,15 @@ start it again and run once more to watch the cached batch replay. writes `.tmp` → rename, evicts oldest above `max_cache_bytes`, expires after 24 h using the timestamp in the file name (not file metadata — an Apple required-reason API), and treats a full disk as a counted drop with a single warning. +- **Logs are records, events have names.** A `TelemetryEvent` with an empty `name` is a plain + OTLP log record (`severity` + `body`); only `Warn`/`Error` leave the device, `emit` drops the + rest. Discrete events are capped at `max_events_per_10min` (300, the design doc's flood + guard); RTC windows and self-telemetry are exempt. +- **RTC stats are windowed on device.** Platforms push raw `getStats()` readings as + [`RtcStatsSample`]s every 1–2 s; the core ships one `lk.rtc.stats.sample` per track and + direction per `stats_window_ms` (15 s, stretched with the cadence) — cumulative counters as + the last value (monotonic, W3C webrtc-stats model), gauges as min/max/avg. Windows close early + on background and shutdown. - **Self-telemetry rides along.** Counters for every way data can be lost (`queue_full`, `cache_error`, `rejected`, `throttled`, `disabled`) and for uploads (`sent`, `failures`) are readable via [`Telemetry::stats`] and shipped as an `lk.telemetry.report` event appended to diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index f88c8ef0a..2f4b8b6e0 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -83,3 +83,50 @@ platforms: all | thermal `critical` | 4 | | low-power mode | 2 | | background | 2 | + +## Log records + +A `TelemetryEvent` with an empty `name` is a plain log record (OTLP log without `event_name`): +`severity` + `body` (the message) + attributes such as `code.function`, `code.file.path`, +`code.line.number`, `lk.log.type` (the SDK logger's category). Only `warn` and `error` records +leave the device; `trace`/`debug`/`info` are dropped in `emit`. + +## Flood guard + +Discrete events (`emit`) are capped at `max_events_per_10min` (default 300, design doc); what +exceeds it is dropped and reported as `lk.telemetry.dropped.rate_limited`. `lk.rtc.stats.sample` +windows and `lk.telemetry.report` are exempt. + +```yaml +event: lk.rtc.stats.sample +area: rtc +severity: info +cadence: one per track and direction per stats window (default 15 s, stretched by the cadence + factor); closed early on background and shutdown. Produced by the core from raw + `record_stats` readings (1–2 s getStats polling on the platform). +attributes: + lk.track.sid: string + lk.track.kind: enum(audio | video) + lk.track.direction: enum(inbound | outbound) + lk.rtc.codec: string # mimeType, when known + lk.rtc.window_ms: int # actual window length + lk.rtc.samples: int # readings in the window + # cumulative counters — the last reading's value, monotonic (W3C webrtc-stats model) + lk.rtc.bytes: int + lk.rtc.packets: int + lk.rtc.packets_lost: int # inbound + lk.rtc.freeze_count: int # inbound video + lk.rtc.freezes_duration_ms: int # inbound video + lk.rtc.concealed_samples: int # inbound audio + lk.rtc.concealment_events: int # inbound audio + lk.rtc.jitter_buffer_delay_ms: int # inbound + lk.rtc.jitter_buffer_emitted_count: int # inbound + lk.rtc.quality_limitation.bandwidth_ms: int # outbound video + lk.rtc.quality_limitation.cpu_ms: int # outbound video + # gauges — min / max / avg over the window + lk.rtc.jitter_ms.{min,max,avg}: double + lk.rtc.rtt_ms.{min,max,avg}: double # remote-inbound RTT for outbound, candidate-pair for inbound + lk.rtc.fps.{min,max,avg}: double # video + lk.rtc.audio_level.{min,max,avg}: double # audio +platforms: all +``` diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index 53c1a2c65..dc8eb84f2 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{sync::Arc, time::Duration}; +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; use livekit_runtime::{interval, sleep, timeout, Instant, MissedTickBehavior}; use tokio::{ @@ -23,10 +26,11 @@ use tokio::{ use crate::{ event::now_unix_nanos, otlp, + rtc::StatsWindows, stats::{Counters, Snapshot}, store::Store, - AppState, BatchCache, DeviceState, ExportError, ExportRequest, TelemetryConfig, - TelemetryTransport, + AppState, Attribute, BatchCache, DeviceState, ExportError, ExportRequest, TelemetryConfig, + TelemetryEvent, TelemetryTransport, }; /// Retries per upload attempt after the first, for failures without `Retry-After`. @@ -79,6 +83,9 @@ pub struct Exporter { config: Arc, cache: Arc, counters: Arc, + windows: Arc>, + /// Session-wide attributes merged into every record at encode time. + attributes: Arc>>, commands: mpsc::UnboundedReceiver, silenced: bool, /// Leave the cache alone until then: the last upload failed or we were throttled. @@ -94,12 +101,16 @@ pub struct Exporter { } impl Exporter { + // Wiring, not an API: every handle is shared with `Telemetry`, which owns their creation. + #[allow(clippy::too_many_arguments)] pub(crate) fn new( store: Arc, transport: Arc, config: Arc, cache: Arc, counters: Arc, + windows: Arc>, + attributes: Arc>>, commands: mpsc::UnboundedReceiver, ) -> Self { Self { @@ -108,6 +119,8 @@ impl Exporter { config, cache, counters, + windows, + attributes, commands, silenced: false, paused_until: None, @@ -123,10 +136,12 @@ impl Exporter { /// demand. pub async fn run(mut self) { self.upload().await; - let mut ticker = self.ticker(); + let mut ticker = self.ticker(self.config.flush_interval_ms); + let mut stats_ticker = self.window_ticker(); loop { tokio::select! { _ = ticker.tick() => self.export_pending().await, + _ = stats_ticker.tick() => self.close_windows(), command = self.commands.recv() => match command { Some(Command::Flush(done)) => { self.export_pending().await; @@ -136,15 +151,20 @@ impl Exporter { let factor = state.cadence_factor(); if factor != self.cadence_factor { self.cadence_factor = factor; - ticker = self.ticker(); + ticker = self.ticker(self.config.flush_interval_ms); + stats_ticker = self.window_ticker(); } if state.app_state == AppState::Background { + // The app may be suspended any moment: close the RTC windows and + // get everything into the cache (and out, if the network allows). + self.close_windows(); self.export_pending().await; } } Some(Command::Shutdown(done)) => { // Last chance: ignore the upload backoff, but respect throttling. self.paused_until = self.throttled_until; + self.close_windows(); self.export_pending().await; let _ = done.send(()); return; @@ -152,6 +172,7 @@ impl Exporter { // Every `Telemetry` handle is gone. None => { self.paused_until = self.throttled_until; + self.close_windows(); self.export_pending().await; return; } @@ -160,14 +181,34 @@ impl Exporter { } } - fn ticker(&self) -> Interval { - let period = - Duration::from_millis(self.config.flush_interval_ms.max(1)) * self.cadence_factor; + /// A ticker at `base_ms × cadence_factor`. + fn ticker(&self, base_ms: u64) -> Interval { + let period = Duration::from_millis(base_ms.max(1)) * self.cadence_factor; let mut ticker = interval(period); ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); ticker } + /// The stats-window ticker. Unlike `interval`, whose first tick is immediate, the first window + /// closes a full period after it opens — otherwise readings taken before the first tick + /// would ship as a zero-length window. + fn window_ticker(&self) -> Interval { + let period = + Duration::from_millis(self.config.stats_window_ms.max(1)) * self.cadence_factor; + let mut ticker = tokio::time::interval_at(Instant::now() + period, period); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + ticker + } + + /// Turn every open RTC stats window into its `lk.rtc.stats.sample` event. Windows bypass the + /// flood guard: they are the pipeline's own, bounded output. + fn close_windows(&mut self) { + let events = self.windows.lock().unwrap_or_else(|e| e.into_inner()).close(); + for event in events { + self.store.push(event); + } + } + async fn export_pending(&mut self) { self.enqueue(); self.upload().await; @@ -196,6 +237,7 @@ impl Exporter { batch.push(delta.report(self.cache.pending().len() as u64)); self.last_report = now; } + self.attach_session_attributes(&mut batch); let count = batch.len() as u64; let body = otlp::encode_logs(&self.config.resource, batch); self.seq += 1; @@ -215,6 +257,21 @@ impl Exporter { } } + /// Merge the session-wide attributes into each record, without overriding explicit ones. + fn attach_session_attributes(&self, batch: &mut [TelemetryEvent]) { + let session = self.attributes.lock().unwrap_or_else(|e| e.into_inner()); + if session.is_empty() { + return; + } + for event in batch { + for attribute in session.iter() { + if !event.attributes.iter().any(|a| a.key == attribute.key) { + event.attributes.push(attribute.clone()); + } + } + } + } + /// Send cached batches oldest-first until one fails; then back off. async fn upload(&mut self) { if self.silenced || self.paused_until.is_some_and(|t| Instant::now() < t) { diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs index 0ed716baa..1d4267ab1 100644 --- a/livekit-telemetry/src/lib.rs +++ b/livekit-telemetry/src/lib.rs @@ -26,6 +26,9 @@ mod stats; /// Host-reported device state and the cadence policy derived from it. mod device; +/// RTC stats samples and their on-device windowing. +mod rtc; + /// Batch exporter actor: timer, OTLP encoding, retry policy. mod exporter; @@ -48,6 +51,7 @@ pub use cache::{BatchCache, FileCache, MemoryCache}; pub use device::*; pub use event::*; pub use exporter::Exporter; +pub use rtc::{RtcStatsSample, StreamDirection, TrackKind}; pub use stats::TelemetryStats; pub use telemetry::*; pub use transport::*; diff --git a/livekit-telemetry/src/rtc.rs b/livekit-telemetry/src/rtc.rs new file mode 100644 index 000000000..ebb19102c --- /dev/null +++ b/livekit-telemetry/src/rtc.rs @@ -0,0 +1,317 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; + +use crate::{event::now_unix_nanos, TelemetryEvent}; + +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum TrackKind { + Audio, + Video, +} + +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum StreamDirection { + Inbound, + Outbound, +} + +/// One reading of a track's RTP statistics, as `getStats()` reports them. +/// +/// Cumulative counters (`bytes`, `packets`, `freeze_count`, …) are passed through as reported — +/// monotonic, paired with their denominators (the W3C webrtc-stats model) so any layer can +/// recompute rates and a dropped window never corrupts the next. Gauges (`jitter_ms`, `rtt_ms`, +/// `frames_per_second`, `audio_level`) are summarised per window as min/max/avg. Fields a +/// platform or direction does not have stay `None` and are omitted from the wire. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct RtcStatsSample { + pub track_sid: String, + pub kind: TrackKind, + pub direction: StreamDirection, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub codec: Option, + // Cumulative counters. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub bytes: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub packets: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub packets_lost: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub freeze_count: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub freezes_duration_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub concealed_samples: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub concealment_events: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub jitter_buffer_delay_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub jitter_buffer_emitted_count: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub quality_limitation_bandwidth_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub quality_limitation_cpu_ms: Option, + // Gauges. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub jitter_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub rtt_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub frames_per_second: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub audio_level: Option, + /// When the reading was taken; `None` = now. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub timestamp_ns: Option, +} + +impl RtcStatsSample { + /// A sample with every optional field unset. + pub fn new(track_sid: impl Into, kind: TrackKind, direction: StreamDirection) -> Self { + Self { + track_sid: track_sid.into(), + kind, + direction, + codec: None, + bytes: None, + packets: None, + packets_lost: None, + freeze_count: None, + freezes_duration_ms: None, + concealed_samples: None, + concealment_events: None, + jitter_buffer_delay_ms: None, + jitter_buffer_emitted_count: None, + quality_limitation_bandwidth_ms: None, + quality_limitation_cpu_ms: None, + jitter_ms: None, + rtt_ms: None, + frames_per_second: None, + audio_level: None, + timestamp_ns: None, + } + } +} + +/// min / max / avg of a gauge over a window. +#[derive(Debug, Default, Clone, Copy)] +struct Gauge { + min: f64, + max: f64, + sum: f64, + n: u32, +} + +impl Gauge { + fn add(&mut self, value: Option) { + let Some(value) = value else { return }; + if self.n == 0 { + self.min = value; + self.max = value; + } else { + self.min = self.min.min(value); + self.max = self.max.max(value); + } + self.sum += value; + self.n += 1; + } + + fn attach(&self, event: TelemetryEvent, key: &str) -> TelemetryEvent { + if self.n == 0 { + return event; + } + event + .with_attribute(format!("{key}.min"), self.min) + .with_attribute(format!("{key}.max"), self.max) + .with_attribute(format!("{key}.avg"), self.sum / self.n as f64) + } +} + +/// Samples of one track in one direction accumulated since the window opened. +struct Window { + start_ns: u64, + samples: u32, + last: RtcStatsSample, + jitter: Gauge, + rtt: Gauge, + fps: Gauge, + audio_level: Gauge, +} + +impl Window { + fn open(start_ns: u64, first: RtcStatsSample) -> Self { + let mut window = Self { + start_ns, + samples: 0, + last: first.clone(), + jitter: Gauge::default(), + rtt: Gauge::default(), + fps: Gauge::default(), + audio_level: Gauge::default(), + }; + window.add(first); + window + } + + fn add(&mut self, sample: RtcStatsSample) { + self.samples += 1; + self.jitter.add(sample.jitter_ms); + self.rtt.add(sample.rtt_ms); + self.fps.add(sample.frames_per_second); + self.audio_level.add(sample.audio_level); + self.last = sample; + } + + /// The `lk.rtc.stats.sample` event for this window, stamped at `end_ns`. + fn into_event(self, end_ns: u64) -> TelemetryEvent { + let last = self.last; + let mut event = TelemetryEvent::new("lk.rtc.stats.sample") + .with_attribute("lk.track.sid", last.track_sid) + .with_attribute("lk.track.kind", last.kind.as_str()) + .with_attribute("lk.track.direction", last.direction.as_str()) + .with_attribute( + "lk.rtc.window_ms", + (end_ns.saturating_sub(self.start_ns) / 1_000_000) as i64, + ) + .with_attribute("lk.rtc.samples", self.samples as i64); + if let Some(codec) = last.codec { + event = event.with_attribute("lk.rtc.codec", codec); + } + let counters = [ + ("lk.rtc.bytes", last.bytes), + ("lk.rtc.packets", last.packets), + ("lk.rtc.packets_lost", last.packets_lost), + ("lk.rtc.freeze_count", last.freeze_count), + ("lk.rtc.freezes_duration_ms", last.freezes_duration_ms), + ("lk.rtc.concealed_samples", last.concealed_samples), + ("lk.rtc.concealment_events", last.concealment_events), + ("lk.rtc.jitter_buffer_delay_ms", last.jitter_buffer_delay_ms), + ("lk.rtc.jitter_buffer_emitted_count", last.jitter_buffer_emitted_count), + ("lk.rtc.quality_limitation.bandwidth_ms", last.quality_limitation_bandwidth_ms), + ("lk.rtc.quality_limitation.cpu_ms", last.quality_limitation_cpu_ms), + ]; + for (key, value) in counters { + if let Some(value) = value { + event = event.with_attribute(key, value as i64); + } + } + event = self.jitter.attach(event, "lk.rtc.jitter_ms"); + event = self.rtt.attach(event, "lk.rtc.rtt_ms"); + event = self.fps.attach(event, "lk.rtc.fps"); + event = self.audio_level.attach(event, "lk.rtc.audio_level"); + event.timestamp_ns = Some(end_ns); + event + } +} + +/// Open RTC stats windows, one per track and direction. +/// +/// The platform pushes raw `getStats()` readings every 1–2 s; the core ships one +/// `lk.rtc.stats.sample` per window (15 s by default, stretched under device pressure) — on +/// device: 1 Hz raw sampling across ~100k concurrent participants would be the same ~100k +/// records/s fleet-wide. +#[derive(Default)] +pub(crate) struct StatsWindows { + windows: HashMap<(String, StreamDirection), Window>, +} + +impl StatsWindows { + pub fn record(&mut self, mut sample: RtcStatsSample) { + let timestamp = *sample.timestamp_ns.get_or_insert_with(now_unix_nanos); + let key = (sample.track_sid.clone(), sample.direction); + match self.windows.get_mut(&key) { + Some(window) => window.add(sample), + None => { + self.windows.insert(key, Window::open(timestamp, sample)); + } + } + } + + /// Close every open window into its event and start fresh. + pub fn close(&mut self) -> Vec { + let end = now_unix_nanos(); + self.windows.drain().map(|(_, window)| window.into_event(end)).collect() + } +} + +impl TrackKind { + fn as_str(self) -> &'static str { + match self { + TrackKind::Audio => "audio", + TrackKind::Video => "video", + } + } +} + +impl StreamDirection { + fn as_str(self) -> &'static str { + match self { + StreamDirection::Inbound => "inbound", + StreamDirection::Outbound => "outbound", + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::AttributeValue; + + fn attr(event: &TelemetryEvent, key: &str) -> Option { + event.attributes.iter().find(|a| a.key == key).map(|a| a.value.clone()) + } + + #[test] + fn window_keeps_last_counter_and_summarises_gauges() { + let mut windows = StatsWindows::default(); + for (bytes, jitter) in [(100, 1.0), (200, 3.0), (300, 2.0)] { + let mut sample = + RtcStatsSample::new("TR_1", TrackKind::Audio, StreamDirection::Inbound); + sample.bytes = Some(bytes); + sample.jitter_ms = Some(jitter); + windows.record(sample); + } + let mut other = RtcStatsSample::new("TR_1", TrackKind::Audio, StreamDirection::Outbound); + other.packets = Some(7); + windows.record(other); + + let mut events = windows.close(); + assert!(windows.close().is_empty(), "closing again yields nothing"); + events.sort_by_key(|e| attr(e, "lk.track.direction").map(|v| format!("{v:?}"))); + assert_eq!(events.len(), 2, "one event per track and direction"); + let inbound = &events[0]; + assert_eq!(inbound.name, "lk.rtc.stats.sample"); + assert_eq!( + attr(inbound, "lk.track.direction"), + Some(AttributeValue::Str("inbound".into())) + ); + assert_eq!( + attr(inbound, "lk.rtc.bytes"), + Some(AttributeValue::Int(300)), + "cumulative: last value" + ); + assert_eq!(attr(inbound, "lk.rtc.samples"), Some(AttributeValue::Int(3))); + assert_eq!(attr(inbound, "lk.rtc.jitter_ms.min"), Some(AttributeValue::Double(1.0))); + assert_eq!(attr(inbound, "lk.rtc.jitter_ms.max"), Some(AttributeValue::Double(3.0))); + assert_eq!(attr(inbound, "lk.rtc.jitter_ms.avg"), Some(AttributeValue::Double(2.0))); + assert_eq!(attr(inbound, "lk.rtc.rtt_ms.avg"), None, "absent gauges are omitted"); + assert_eq!(attr(&events[1], "lk.rtc.packets"), Some(AttributeValue::Int(7))); + } +} diff --git a/livekit-telemetry/src/stats.rs b/livekit-telemetry/src/stats.rs index cf657c055..37e652028 100644 --- a/livekit-telemetry/src/stats.rs +++ b/livekit-telemetry/src/stats.rs @@ -33,6 +33,8 @@ pub(crate) struct Counters { pub throttled: AtomicU64, /// Events dropped after the collector disabled telemetry. pub disabled: AtomicU64, + /// Discrete events dropped by the flood guard (`max_events_per_10min`). + pub rate_limited: AtomicU64, /// Batches the collector accepted. pub uploads_sent: AtomicU64, /// Upload attempts that failed transiently (network, timeout, 5xx). @@ -52,6 +54,7 @@ impl Counters { rejected: get(&self.rejected), throttled: get(&self.throttled), disabled: get(&self.disabled), + rate_limited: get(&self.rate_limited), uploads_sent: get(&self.uploads_sent), upload_failures: get(&self.upload_failures), } @@ -66,6 +69,7 @@ pub(crate) struct Snapshot { pub rejected: u64, pub throttled: u64, pub disabled: u64, + pub rate_limited: u64, pub uploads_sent: u64, pub upload_failures: u64, } @@ -79,6 +83,7 @@ impl Snapshot { rejected: self.rejected.saturating_sub(earlier.rejected), throttled: self.throttled.saturating_sub(earlier.throttled), disabled: self.disabled.saturating_sub(earlier.disabled), + rate_limited: self.rate_limited.saturating_sub(earlier.rate_limited), uploads_sent: self.uploads_sent.saturating_sub(earlier.uploads_sent), upload_failures: self.upload_failures.saturating_sub(earlier.upload_failures), } @@ -86,7 +91,12 @@ impl Snapshot { /// Anything worth telling the backend about: data lost or uploads failing. pub fn has_problems(&self) -> bool { - self.queue_full + self.cache_error + self.rejected + self.throttled + self.upload_failures + self.queue_full + + self.cache_error + + self.rejected + + self.throttled + + self.rate_limited + + self.upload_failures > 0 } @@ -103,6 +113,7 @@ impl Snapshot { ("lk.telemetry.dropped.cache_error", self.cache_error), ("lk.telemetry.dropped.rejected", self.rejected), ("lk.telemetry.dropped.throttled", self.throttled), + ("lk.telemetry.dropped.rate_limited", self.rate_limited), ] { if value > 0 { event = event.with_attribute(key, value as i64); @@ -123,6 +134,7 @@ pub struct TelemetryStats { pub dropped_rejected: u64, pub dropped_throttled: u64, pub dropped_disabled: u64, + pub dropped_rate_limited: u64, /// Batches the collector accepted. pub uploads_sent: u64, /// Upload attempts that failed transiently. @@ -138,12 +150,14 @@ impl TelemetryStats { + snapshot.cache_error + snapshot.rejected + snapshot.throttled - + snapshot.disabled, + + snapshot.disabled + + snapshot.rate_limited, dropped_queue_full: snapshot.queue_full, dropped_cache_error: snapshot.cache_error, dropped_rejected: snapshot.rejected, dropped_throttled: snapshot.throttled, dropped_disabled: snapshot.disabled, + dropped_rate_limited: snapshot.rate_limited, uploads_sent: snapshot.uploads_sent, upload_failures: snapshot.upload_failures, cached_batches, diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index e5624807a..589ef2884 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -15,16 +15,16 @@ use std::{ collections::HashMap, sync::{Arc, Mutex}, - time::Duration, + time::{Duration, Instant}, }; use livekit_runtime::timeout; use tokio::sync::{mpsc, oneshot}; use crate::{ - event::now_unix_nanos, exporter::Command, stats::Counters, store::Store, Attribute, BatchCache, - DeviceState, Exporter, FileCache, MemoryCache, TelemetryEvent, TelemetryStats, - TelemetryTransport, + event::now_unix_nanos, exporter::Command, rtc::StatsWindows, stats::Counters, store::Store, + Attribute, AttributeValue, BatchCache, DeviceState, Exporter, FileCache, MemoryCache, + RtcStatsSample, Severity, TelemetryEvent, TelemetryStats, TelemetryTransport, }; /// Pipeline configuration. @@ -61,6 +61,14 @@ pub struct TelemetryConfig { /// Bound on a single transport attempt, and on `shutdown`. #[cfg_attr(feature = "uniffi", uniffi(default = 10000))] pub export_timeout_ms: u64, + /// RTC stats window: readings pushed with [`Telemetry::record_stats`] are summarised into one + /// `lk.rtc.stats.sample` per track and direction every window (stretched like the cadence). + #[cfg_attr(feature = "uniffi", uniffi(default = 15000))] + pub stats_window_ms: u64, + /// Flood guard for discrete events: beyond this many `emit`s per 10 minutes the rest are + /// dropped and counted as `rate_limited`. RTC windows and self-telemetry are exempt; 0 = off. + #[cfg_attr(feature = "uniffi", uniffi(default = 300))] + pub max_events_per_10min: u32, } impl TelemetryConfig { @@ -76,6 +84,8 @@ impl TelemetryConfig { max_queue_size: 2048, max_batch_size: 512, export_timeout_ms: 10_000, + stats_window_ms: 15_000, + max_events_per_10min: 300, } } } @@ -110,9 +120,43 @@ pub struct Telemetry { cache: Arc, counters: Arc, device: Arc>>, + windows: Arc>, + guard: Arc>, + attributes: Arc>>, commands: mpsc::UnboundedSender, } +/// Fixed-window cap on discrete events (the design doc's ~300 per 10 min). +struct FloodGuard { + max: u32, + window_start: Instant, + count: u32, +} + +impl FloodGuard { + const WINDOW: Duration = Duration::from_secs(10 * 60); + + fn new(max: u32) -> Self { + Self { max, window_start: Instant::now(), count: 0 } + } + + fn admit(&mut self) -> bool { + if self.max == 0 { + return true; + } + let now = Instant::now(); + if now.duration_since(self.window_start) >= Self::WINDOW { + self.window_start = now; + self.count = 0; + } + if self.count >= self.max { + return false; + } + self.count += 1; + true + } +} + impl Telemetry { /// Build the pipeline with the cache the config asks for: a [`FileCache`] in `storage_dir`, /// or a [`MemoryCache`] when unset — or when the directory is unusable (logged, never an @@ -145,26 +189,72 @@ impl Telemetry { let counters = Arc::new(Counters::default()); let store = Arc::new(Store::new(config.max_queue_size.max(1) as usize, counters.clone())); let (commands, receiver) = mpsc::unbounded_channel(); + let windows = Arc::new(Mutex::new(StatsWindows::default())); + let guard = Arc::new(Mutex::new(FloodGuard::new(config.max_events_per_10min))); + let attributes = Arc::new(Mutex::new(Vec::new())); let exporter = Exporter::new( store.clone(), transport, config.clone(), cache.clone(), counters.clone(), + windows.clone(), + attributes.clone(), receiver, ); - let telemetry = Self { store, config, cache, counters, device: Arc::default(), commands }; + let telemetry = Self { + store, + config, + cache, + counters, + device: Arc::default(), + windows, + guard, + attributes, + commands, + }; (telemetry, exporter) } - /// Queue an event for export. Stamps it with the current time unless it carries one. + /// Queue an event or log record for export. Stamps it with the current time unless it + /// carries one. + /// + /// A record with an empty `name` is a plain log line: only `Warn` and `Error` ones leave the + /// device (design doc: debug/info logs never do). Discrete events are subject to the flood + /// guard (`max_events_per_10min`); what it drops is counted as `rate_limited`. pub fn emit(&self, mut event: TelemetryEvent) { + if event.name.is_empty() + && matches!(event.severity, Severity::Trace | Severity::Debug | Severity::Info) + { + return; + } + if !self.guard.lock().unwrap_or_else(|e| e.into_inner()).admit() { + Counters::add(&self.counters.rate_limited, 1); + return; + } if event.timestamp_ns.is_none() { event.timestamp_ns = Some(now_unix_nanos()); } self.store.push(event); } + /// Set a session-wide attribute (`lk.room.sid`, `lk.participant.identity`, or a consumer's + /// own `acme.call_id`), attached to every record exported from now on unless the record + /// already carries the key. `None` removes it. + pub fn set_attribute(&self, key: &str, value: Option) { + let mut attributes = self.attributes.lock().unwrap_or_else(|e| e.into_inner()); + attributes.retain(|a| a.key != key); + if let Some(value) = value { + attributes.push(Attribute::new(key, value)); + } + } + + /// Push one `getStats()` reading. Readings are windowed on device into `lk.rtc.stats.sample` + /// events (see `stats_window_ms`); they never count against the flood guard. + pub fn record_stats(&self, sample: RtcStatsSample) { + self.windows.lock().unwrap_or_else(|e| e.into_inner()).record(sample); + } + /// Tell the pipeline what the device looks like. Emits the `lk.device.*.changed` events for /// whatever differs from the last state (everything, the first time) and re-tunes the export /// cadence: thermal pressure, low-power mode and the background stretch it up to 4×; entering @@ -233,7 +323,7 @@ mod tests { collector::logs::v1::ExportLogsServiceRequest, common::v1::any_value::Value, logs::v1::LogRecord, }, - AppState, ExportError, ExportRequest, ThermalState, + AppState, ExportError, ExportRequest, StreamDirection, ThermalState, TrackKind, }; #[derive(Default)] @@ -513,6 +603,118 @@ mod tests { assert_eq!(transport.sent().len(), 2, "exported on the stretched tick"); } + #[tokio::test(start_paused = true)] + async fn debug_and_info_logs_never_leave_the_device() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + telemetry.emit(TelemetryEvent::new("").with_severity(Severity::Info).with_body("noise")); + telemetry.emit(TelemetryEvent::new("").with_severity(Severity::Error).with_body("boom")); + telemetry.flush().await; + + let sent = transport.sent(); + let records = records(&sent[0]); + assert_eq!(records.len(), 1); + assert_eq!(records[0].event_name, "", "a log line, not an event"); + assert_eq!(records[0].severity_text, "ERROR"); + assert_eq!( + records[0].body.as_ref().and_then(|b| b.value.clone()), + Some(Value::StringValue("boom".into())) + ); + } + + #[tokio::test(start_paused = true)] + async fn flood_guard_caps_events_but_not_stats_windows() { + let mut config = TelemetryConfig::new("http://collector/v1/logs"); + config.max_events_per_10min = 1; + config.stats_window_ms = 1_000; + let transport = FakeTransport::scripted([]); + let telemetry = start(config, transport.clone()); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.record_stats(RtcStatsSample::new( + "TR_1", + TrackKind::Audio, + StreamDirection::Inbound, + )); + assert_eq!(telemetry.stats().dropped_rate_limited, 1); + + tokio::time::sleep(Duration::from_millis(1_100)).await; // stats window closes + telemetry.flush().await; + let names: Vec = transport.sent().iter().flat_map(event_names).collect(); + assert_eq!(names.iter().filter(|n| *n == "lk.ping").count(), 1); + assert!(names.contains(&"lk.rtc.stats.sample".to_owned()), "{names:?}"); + assert!(names.contains(&"lk.telemetry.report".to_owned()), "rate limiting is reported"); + } + + #[tokio::test(start_paused = true)] + async fn stats_readings_are_windowed_into_one_event() { + let mut config = TelemetryConfig::new("http://collector/v1/logs"); + config.stats_window_ms = 2_000; + let transport = FakeTransport::scripted([]); + let telemetry = start(config, transport.clone()); + for (bytes, jitter) in [(100, 1.0), (200, 3.0), (300, 2.0)] { + let mut sample = + RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Inbound); + sample.bytes = Some(bytes); + sample.jitter_ms = Some(jitter); + sample.codec = Some("video/VP8".into()); + telemetry.record_stats(sample); + } + telemetry.flush().await; + assert!(transport.sent().is_empty(), "windows do not flush early"); + + tokio::time::sleep(Duration::from_millis(2_100)).await; + telemetry.flush().await; + let sent = transport.sent(); + let window = records(&sent[0]) + .into_iter() + .find(|r| r.event_name == "lk.rtc.stats.sample") + .expect("window"); + assert_eq!(attribute(&window, "lk.track.kind"), Some(Value::StringValue("video".into()))); + assert_eq!( + attribute(&window, "lk.rtc.codec"), + Some(Value::StringValue("video/VP8".into())) + ); + assert_eq!(attribute(&window, "lk.rtc.bytes"), Some(Value::IntValue(300))); + assert_eq!(attribute(&window, "lk.rtc.samples"), Some(Value::IntValue(3))); + assert_eq!(attribute(&window, "lk.rtc.jitter_ms.avg"), Some(Value::DoubleValue(2.0))); + } + + #[tokio::test(start_paused = true)] + async fn shutdown_closes_open_stats_windows() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + telemetry.record_stats(RtcStatsSample::new( + "TR_1", + TrackKind::Audio, + StreamDirection::Outbound, + )); + telemetry.shutdown().await; + let names: Vec = transport.sent().iter().flat_map(event_names).collect(); + assert_eq!(names, ["lk.rtc.stats.sample"]); + } + + #[tokio::test(start_paused = true)] + async fn session_attributes_are_attached_to_every_record() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + telemetry.set_attribute("lk.room.sid", Some("RM_1".into())); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.emit(TelemetryEvent::new("lk.ping").with_attribute("lk.room.sid", "RM_override")); + telemetry.flush().await; + let first = records(&transport.sent()[0]); + assert_eq!(attribute(&first[0], "lk.room.sid"), Some(Value::StringValue("RM_1".into()))); + assert_eq!( + attribute(&first[1], "lk.room.sid"), + Some(Value::StringValue("RM_override".into())), + "an explicit attribute wins" + ); + telemetry.set_attribute("lk.room.sid", None); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + assert_eq!(attribute(&records(&transport.sent()[1])[0], "lk.room.sid"), None); + } + #[tokio::test(start_paused = true)] async fn entering_background_flushes_immediately() { let transport = FakeTransport::scripted([]); diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index b15f02297..cab155c00 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -22,7 +22,8 @@ use std::sync::Arc; use livekit_telemetry::{ - DeviceState, TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, + AttributeValue, DeviceState, RtcStatsSample, TelemetryConfig, TelemetryEvent, TelemetryStats, + TelemetryTransport, }; /// Telemetry pipeline: buffer, batch, cache and export events as OTLP. @@ -49,6 +50,17 @@ impl Telemetry { self.0.set_device_state(state); } + /// Set (or, with `None`, remove) a session-wide attribute attached to every record from now + /// on: `lk.room.sid`, `lk.participant.identity`, or the app's own correlation ids. + pub fn set_attribute(&self, key: String, value: Option) { + self.0.set_attribute(&key, value); + } + + /// Push one `getStats()` reading for a track; windowed on device into `lk.rtc.stats.sample`. + pub fn record_stats(&self, sample: RtcStatsSample) { + self.0.record_stats(sample); + } + /// Export everything queued and wait for the transport. pub async fn flush(&self) { self.0.flush().await; From bb9d57844340752976d271652581fd9286c0288b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:50:52 +0200 Subject: [PATCH 08/50] feat(uniffi): optional telemetry transport and a pull-side export queue Two ways for a host to move telemetry bytes, chosen per binding: - Telemetry(config, transport: Option): None falls back to the HTTP client the host registered with livekit-net (set_http_client), so a platform that already brought its client for signaling needs nothing more. Fails with TelemetryError::NoTransport when neither exists. - Telemetry::new_pulled(config, TelemetryExportQueue): Rust never calls into the host. The exporter queues each ExportRequest; the host awaits next(), performs it on its own thread and reports back with complete(id, error). The second exists for uniffi-dart: its foreign-trait callbacks are isolate-bound (Pointer.fromFunction) and the Dart VM aborts with "Cannot invoke native callback outside an isolate" when the exporter invokes send() from a tokio worker - reproduced with a Dart TelemetryTransport. Swift and Kotlin callbacks are thread-agnostic and keep using the trait directly. support/dart/test/telemetry_test.dart covers the pull path end to end (emit + stats window -> two ExportRequests served from Dart) and the no-transport error. Co-Authored-By: Claude Fable 5 --- livekit-uniffi/Cargo.toml | 2 +- livekit-uniffi/src/telemetry.rs | 146 +++++++++++++++--- .../support/dart/test/telemetry_test.dart | 70 +++++++++ 3 files changed, 195 insertions(+), 23 deletions(-) create mode 100644 livekit-uniffi/support/dart/test/telemetry_test.dart diff --git a/livekit-uniffi/Cargo.toml b/livekit-uniffi/Cargo.toml index d812fa20a..bc7bfc547 100644 --- a/livekit-uniffi/Cargo.toml +++ b/livekit-uniffi/Cargo.toml @@ -16,7 +16,7 @@ livekit-protocol = { workspace = true } livekit-common = { workspace = true, features = ["uniffi"] } livekit-token = { workspace = true } livekit-datatrack = { workspace = true, features = ["uniffi"] } -livekit-telemetry = { workspace = true, features = ["uniffi"] } +livekit-telemetry = { workspace = true, features = ["uniffi", "net"] } livekit-net = { workspace = true, features = ["uniffi"] } livekit-data-stream = { workspace = true } uniffi = { workspace = true, features = ["scaffolding-ffi-buffer-fns", "tokio"] } diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index cab155c00..a85bcff79 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -1,30 +1,35 @@ -// Copyright 2026 LiveKit, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - //! Client telemetry core from the [`livekit-telemetry`] crate. //! -//! FFI clients construct one [`Telemetry`] per pipeline with a host-implemented -//! `TelemetryTransport` (e.g. a URLSession/OkHttp POST), then `emit` from any thread and push -//! `DeviceState` changes as the OS reports them. The exporter runs on the global runtime; -//! `shutdown` flushes within `export_timeout_ms`. +//! FFI clients construct one [`Telemetry`] per pipeline, then `emit` from any thread, push +//! `getStats()` readings with `record_stats` and `DeviceState` changes as the OS reports them. +//! The exporter runs on the global runtime; `shutdown` flushes within `export_timeout_ms`. +//! +//! Transport: pass a host-implemented `TelemetryTransport` (a URLSession/OkHttp/dart:io POST, +//! or a data channel), or pass `None` to ride the HTTP client the host registered with +//! `livekit-net` (`set_http_client`) for signaling — one registration serves both. -use std::sync::Arc; +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, Mutex, + }, +}; use livekit_telemetry::{ - AttributeValue, DeviceState, RtcStatsSample, TelemetryConfig, TelemetryEvent, TelemetryStats, - TelemetryTransport, + AttributeValue, DeviceState, ExportError, ExportRequest, NetTransport, RtcStatsSample, + TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, }; +use tokio::sync::{mpsc, oneshot}; + +/// Why a [`Telemetry`] pipeline could not be created. +#[derive(uniffi::Error, thiserror::Error, Debug)] +#[uniffi(flat_error)] +pub enum TelemetryError { + /// No transport was passed and no HTTP client is registered with `livekit-net`. + #[error("no telemetry transport: pass one, or register an HTTP client with livekit-net first")] + NoTransport, +} /// Telemetry pipeline: buffer, batch, cache and export events as OTLP. #[derive(uniffi::Object)] @@ -32,10 +37,28 @@ pub struct Telemetry(livekit_telemetry::Telemetry); #[uniffi::export(async_runtime = "tokio")] impl Telemetry { + /// `transport = None` uses the HTTP client registered with `livekit-net`, if any. #[uniffi::constructor] - pub fn new(config: TelemetryConfig, transport: Arc) -> Arc { + pub fn new( + config: TelemetryConfig, + transport: Option>, + ) -> Result, TelemetryError> { + let transport: Arc = match transport { + Some(transport) => transport, + None => Arc::new(NetTransport::from_registry().ok_or(TelemetryError::NoTransport)?), + }; let (telemetry, exporter) = livekit_telemetry::Telemetry::new(config, transport); crate::runtime::runtime().spawn(exporter.run()); + Ok(Arc::new(Self(telemetry))) + } + + /// Like [`Telemetry::new`], but exports through a [`TelemetryExportQueue`] the host drains + /// from its own thread. For bindings whose callbacks cannot be invoked from Rust threads + /// (uniffi-dart today). + #[uniffi::constructor] + pub fn new_pulled(config: TelemetryConfig, queue: Arc) -> Arc { + let (telemetry, exporter) = livekit_telemetry::Telemetry::new(config, queue); + crate::runtime::runtime().spawn(exporter.run()); Arc::new(Self(telemetry)) } @@ -76,3 +99,82 @@ impl Telemetry { self.0.stats() } } + +/// One export the host has to perform on behalf of a pulled pipeline. +#[derive(uniffi::Record)] +pub struct PendingExport { + pub id: u64, + pub request: ExportRequest, +} + +struct Pending { + export: PendingExport, + done: oneshot::Sender>, +} + +/// Pull-side transport: Rust never calls into the host. The exporter queues each request; the +/// host awaits [`next`](Self::next) (a Rust future — those cross every binding), performs the +/// HTTP call on its own thread, and reports the outcome with [`complete`](Self::complete), +/// which unblocks the exporter's retry/drop/go-silent logic exactly as a direct transport would. +/// +/// Exists because uniffi-dart's foreign-trait callbacks are isolate-bound (`Pointer.fromFunction`) +/// and abort the VM when invoked from a tokio thread; Swift and Kotlin callbacks are thread-agnostic +/// and use [`TelemetryTransport`] directly. +#[derive(uniffi::Object)] +pub struct TelemetryExportQueue { + tx: mpsc::UnboundedSender, + rx: tokio::sync::Mutex>, + inflight: Mutex>>>, + seq: AtomicU64, +} + +#[uniffi::export(async_runtime = "tokio")] +impl TelemetryExportQueue { + #[uniffi::constructor] + pub fn new() -> Arc { + let (tx, rx) = mpsc::unbounded_channel(); + Arc::new(Self { + tx, + rx: tokio::sync::Mutex::new(rx), + inflight: Mutex::new(HashMap::new()), + seq: AtomicU64::new(0), + }) + } + + /// The next request to perform. Resolves when one is queued; `None` once the pipeline is gone. + pub async fn next(&self) -> Option { + let pending = self.rx.lock().await.recv().await?; + self.inflight + .lock() + .unwrap_or_else(|e| e.into_inner()) + .insert(pending.export.id, pending.done); + Some(pending.export) + } + + /// Report how the request with `id` went: `None` = accepted by the collector. + pub fn complete(&self, id: u64, error: Option) { + let done = self.inflight.lock().unwrap_or_else(|e| e.into_inner()).remove(&id); + if let Some(done) = done { + let _ = done.send(error.map_or(Ok(()), Err)); + } + } +} + +#[async_trait::async_trait] +impl TelemetryTransport for TelemetryExportQueue { + async fn send(&self, request: ExportRequest) -> Result<(), ExportError> { + let id = self.seq.fetch_add(1, Ordering::Relaxed); + let (done, wait) = oneshot::channel(); + let pending = Pending { export: PendingExport { id, request }, done }; + if self.tx.send(pending).is_err() { + return Err(ExportError::Retryable { + message: "export queue closed".into(), + retry_after_ms: None, + }); + } + wait.await.unwrap_or(Err(ExportError::Retryable { + message: "host dropped the export".into(), + retry_after_ms: None, + })) + } +} diff --git a/livekit-uniffi/support/dart/test/telemetry_test.dart b/livekit-uniffi/support/dart/test/telemetry_test.dart new file mode 100644 index 000000000..046c84272 --- /dev/null +++ b/livekit-uniffi/support/dart/test/telemetry_test.dart @@ -0,0 +1,70 @@ +import 'package:livekit_uniffi/livekit_telemetry.dart'; +import 'package:livekit_uniffi/livekit_uniffi.dart'; +import 'package:test/test.dart'; + +/// Rust must never call back into Dart from its own threads: uniffi-dart's foreign-trait +/// callbacks are isolate-bound (`Pointer.fromFunction`) and the VM aborts with "Cannot invoke +/// native callback outside an isolate" when the exporter invokes `TelemetryTransport.send` from +/// a tokio worker. The pull queue inverts the direction: Dart awaits `next()`, performs the +/// request, and reports back with `complete()`. +Future serve(TelemetryExportQueue queue, List sink, int count) async { + for (var i = 0; i < count; i++) { + final pending = await queue.next(); + if (pending == null) return; + sink.add(pending.request); + queue.complete(id: pending.id, error: null); + } +} + +void main() { + group('telemetry', () { + test('exports through the pull queue from the Dart side', () async { + final queue = TelemetryExportQueue(); + final requests = []; + final serving = serve(queue, requests, 2); + final telemetry = Telemetry.newPulled( + config: TelemetryConfig( + endpoint: 'http://collector/v1/logs', + headers: {'Authorization': 'Bearer test'}, + resource: [], + ), + queue: queue, + ); + + telemetry.emit( + event: TelemetryEvent(name: 'lk.ping', severity: Severity.info, attributes: []), + ); + telemetry.recordStats( + sample: RtcStatsSample( + trackSid: 'TR_1', + kind: TrackKind.audio, + direction: StreamDirection.inbound, + bytes: 42, + ), + ); + await telemetry.flush(); + expect(requests, hasLength(1)); + expect(requests.single.url, 'http://collector/v1/logs'); + expect(requests.single.headers['Content-Type'], 'application/x-protobuf'); + expect(requests.single.headers['Authorization'], 'Bearer test'); + expect(requests.single.body, isNotEmpty); + expect(telemetry.stats().uploadsSent, 1); + + // Shutdown closes the open stats window, which ships as a second batch. + await telemetry.shutdown(); + await serving; + expect(requests, hasLength(2)); + expect(telemetry.stats().dropped, 0); + }); + + test('refuses to start without any transport', () { + expect( + () => Telemetry( + config: TelemetryConfig(endpoint: 'http://collector/v1/logs', headers: {}, resource: []), + transport: null, + ), + throwsA(anything), + ); + }); + }); +} From 5ac4e50981e420908d9267f42dc43ab7b82ae042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:03:55 +0200 Subject: [PATCH 09/50] feat(telemetry): experimental spans on the traces signal A span is one attempt at an operation; the session is the trace. The core mints the trace id when the pipeline starts (rand, non-zero) and attaches it to every log record; records emitted inside a span carry its span_id too. - Spans registry: begin_span(name, kind, parent) -> u64 handle, add_span_event(handle, name, attrs) for checkpoints, end_span(handle, outcome, error_type, attrs). Handles are opaque u64s; ambient context is the platform's job. OTel span limits (128 events/attrs), 256 open spans, finished spans bounded like the event queue (drop-oldest, counted). - Outcome: OTel status only knows Unset/Ok/Error and instrumentation should not set Ok, so ok and cancelled export as Unset and every span carries lk.outcome = ok|error|cancelled (+ error.type and the status message on error). Checkpoints are span events in the span's own envelope (OTEP 4430 keeps that legal); real events stay log records pointing at the span. - Wire: opentelemetry-proto `trace` types, ExportTraceServiceRequest; a second signal through the same BatchCache - batch ids are now --- (old ids read as logs) and upload picks traces_endpoint, derived from the logs endpoint by replacing the last "logs" path segment (covers /v1/logs and /observability/logs/otlp/v0) or set explicitly. - UniFFI: traceId(), beginSpan, addSpanEvent, endSpan; SpanKind, SpanOutcome; TelemetryEvent.spanId. - SPEC.md: span rules, lk.connect and lk.reconnect definitions. Co-Authored-By: Claude Fable 5 --- .changeset/add-livekit-telemetry-crate.md | 3 +- Cargo.lock | 7 + livekit-telemetry/Cargo.toml | 3 +- livekit-telemetry/README.md | 6 + livekit-telemetry/SPEC.md | 37 ++++ livekit-telemetry/src/event.rs | 11 + livekit-telemetry/src/exporter.rs | 124 +++++++++--- livekit-telemetry/src/lib.rs | 4 + livekit-telemetry/src/otlp.rs | 131 +++++++++--- livekit-telemetry/src/proto/mod.rs | 2 +- livekit-telemetry/src/span.rs | 233 ++++++++++++++++++++++ livekit-telemetry/src/telemetry.rs | 165 ++++++++++++++- livekit-uniffi/src/telemetry.rs | 31 ++- 13 files changed, 685 insertions(+), 72 deletions(-) create mode 100644 livekit-telemetry/src/span.rs diff --git a/.changeset/add-livekit-telemetry-crate.md b/.changeset/add-livekit-telemetry-crate.md index e8ea7ef5c..67e10729d 100644 --- a/.changeset/add-livekit-telemetry-crate.md +++ b/.changeset/add-livekit-telemetry-crate.md @@ -9,4 +9,5 @@ failed uploads, crashes and offline shutdowns lose nothing. Pipeline health is e events; hosts push `DeviceState` (thermal, low power, foreground/background) and the core emits the `lk.device.*.changed` events and stretches its cadence under pressure. Log records (`Warn`/`Error` only), a flood guard for discrete events, on-device RTC stats windows (`record_stats` → `lk.rtc.stats.sample`) and session-wide attributes complete the -design doc's v0 surface. +design doc's v0 surface. Spans (`begin_span`/`add_span_event`/`end_span`, one attempt per span, the session as the +trace) ship on the traces signal through the same cache; every record carries the session trace id. diff --git a/Cargo.lock b/Cargo.lock index eb18a38bf..3ffba8c4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3787,6 +3787,7 @@ dependencies = [ "log", "opentelemetry-proto", "prost 0.14.4", + "rand 0.9.5", "thiserror 2.0.19", "tokio", "uniffi", @@ -5023,7 +5024,11 @@ version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ + "futures-core", + "futures-sink", "js-sys", + "pin-project-lite", + "thiserror 2.0.19", ] [[package]] @@ -5047,7 +5052,9 @@ dependencies = [ "futures-executor", "futures-util", "opentelemetry", + "percent-encoding", "portable-atomic", + "rand 0.9.5", "thiserror 2.0.19", ] diff --git a/livekit-telemetry/Cargo.toml b/livekit-telemetry/Cargo.toml index d86ff0c74..fee9716b9 100644 --- a/livekit-telemetry/Cargo.toml +++ b/livekit-telemetry/Cargo.toml @@ -13,9 +13,10 @@ log = { workspace = true } thiserror = { workspace = true } async-trait = "0.1" prost = { workspace = true } +rand = { workspace = true } # OTLP message types only (`gen-tonic-messages` = prost structs, no tonic). Same prost as # livekit-protocol so a single prost is linked. -opentelemetry-proto = { version = "0.32", default-features = false, features = ["logs", "gen-tonic-messages"] } +opentelemetry-proto = { version = "0.32", default-features = false, features = ["logs", "trace", "gen-tonic-messages"] } livekit-net = { workspace = true, optional = true } uniffi = { workspace = true, features = ["scaffolding-ffi-buffer-fns"], optional = true } diff --git a/livekit-telemetry/README.md b/livekit-telemetry/README.md index 68b7801d1..3ca245667 100644 --- a/livekit-telemetry/README.md +++ b/livekit-telemetry/README.md @@ -84,6 +84,12 @@ start it again and run once more to watch the cached batch replay. direction per `stats_window_ms` (15 s, stretched with the cadence) — cumulative counters as the last value (monotonic, W3C webrtc-stats model), gauges as min/max/avg. Windows close early on background and shutdown. +- **Spans are handles, the session is the trace.** `begin_span(name, kind, parent) → u64`, + `add_span_event`, `end_span(outcome, error_type, attrs)`; the core stamps wall-clock time, + encodes OTLP traces, and ships them on the traces signal through the same cache (batch ids + carry the signal; the traces URL is derived from the logs URL or set explicitly). Status + follows OTel (`Unset`/`Error`); `lk.outcome` carries `ok|error|cancelled`. Every log record + carries the session `trace_id`, and `span_id` when emitted inside a span. - **Self-telemetry rides along.** Counters for every way data can be lost (`queue_full`, `cache_error`, `rejected`, `throttled`, `disabled`) and for uploads (`sent`, `failures`) are readable via [`Telemetry::stats`] and shipped as an `lk.telemetry.report` event appended to diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index 2f4b8b6e0..7cbc47a12 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -130,3 +130,40 @@ attributes: lk.rtc.audio_level.{min,max,avg}: double # audio platforms: all ``` + +## Spans + +A span is **one attempt** at an operation. The session (one Room connection lifetime, across +reconnects) is the trace; its id is generated by the core when the pipeline starts and rides on +every span and log record. Spans are exported when they end — never a long-lived session span. + +| Rule | Value | +|---|---| +| Names | `lk.connect`, `lk.reconnect`, `lk.publish`, `lk.subscribe` — verbs, never ids | +| Kind | `CLIENT` for connect/reconnect (a call to the SFU), `INTERNAL` otherwise | +| Status | OTel `Unset` on success **and** cancellation, `Error` (+ `error.type`, message) on failure | +| `lk.outcome` | always present: `ok` \| `error` \| `cancelled` — rollups read this, never the status | +| Checkpoints | span events in the span's envelope (`ws_open`, `join_recv`, `pc_connected`, `attempt 2 full`, …); real events stay log records pointing at the span via `span_id` | +| Limits | 128 events and 128 attributes per span (OTel defaults); 256 open spans per pipeline | + +```yaml +span: lk.connect +kind: client +attributes: + lk.connect.attempt: int # 1 for the user-initiated connect +checkpoints: ws_open, signal, join_recv, pc_created, engine, pc_connected, offer_sent, answer_sent, room_connected +outcome: ok | error (error.type = LiveKitError. | CancellationError | ) | cancelled +``` + +```yaml +span: lk.reconnect +kind: client +attributes: + lk.reconnect.reason: string # what triggered the cycle + lk.reconnect.mode: enum(quick | full) # mode of the last attempt + lk.reconnect.attempts: int +checkpoints: "attempt " per attempt +outcome: ok | error | cancelled # cancelled when disconnect() or a newer reconnect wins +``` + +`lk.publish` / `lk.subscribe`: not yet emitted by any SDK (see design page, round 7). diff --git a/livekit-telemetry/src/event.rs b/livekit-telemetry/src/event.rs index 284bf79f4..4d742dcdf 100644 --- a/livekit-telemetry/src/event.rs +++ b/livekit-telemetry/src/event.rs @@ -31,6 +31,10 @@ pub struct TelemetryEvent { /// Wall-clock time in nanoseconds since the Unix epoch. `None` stamps the event at emit time. #[cfg_attr(feature = "uniffi", uniffi(default))] pub timestamp_ns: Option, + /// The in-flight span this record belongs to (a handle from `begin_span`), if any. The trace + /// id is always the session's and is attached by the core. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub span_id: Option, } impl TelemetryEvent { @@ -42,9 +46,16 @@ impl TelemetryEvent { body: None, attributes: Vec::new(), timestamp_ns: None, + span_id: None, } } + /// Link this record to an in-flight span. + pub fn in_span(mut self, span: u64) -> Self { + self.span_id = Some(span); + self + } + pub fn with_severity(mut self, severity: Severity) -> Self { self.severity = severity; self diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index 982c573ea..a130f6186 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -27,10 +27,11 @@ use crate::{ event::now_unix_nanos, otlp, rtc::StatsWindows, + span::Spans, stats::{Counters, Snapshot}, store::Store, AppState, Attribute, BatchCache, DeviceState, ExportError, ExportRequest, TelemetryConfig, - TelemetryEvent, TelemetryTransport, + TelemetryTransport, }; /// Retries per upload attempt after the first, for failures without `Retry-After`. @@ -40,6 +41,30 @@ const RETRY_BACKOFF: Duration = Duration::from_secs(1); /// cache until `Retry-After` elapses; here also for plain connectivity failures). const UPLOAD_BACKOFF: Duration = Duration::from_secs(60); +/// Which OTLP signal a cached batch is; picks the endpoint at upload time. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Signal { + Logs, + Traces, +} + +impl Signal { + fn tag(self) -> char { + match self { + Signal::Logs => 'l', + Signal::Traces => 't', + } + } + + /// From a batch id's last component; ids from before signals existed are logs. + fn of(id: &str) -> Signal { + match id.rsplit('-').next() { + Some("t") => Signal::Traces, + _ => Signal::Logs, + } + } +} + pub(crate) enum Command { Flush(oneshot::Sender<()>), Shutdown(oneshot::Sender<()>), @@ -86,6 +111,8 @@ pub struct Exporter { windows: Arc>, /// Session-wide attributes merged into every record at encode time. attributes: Arc>>, + spans: Arc>, + trace_id: [u8; 16], commands: mpsc::UnboundedReceiver, silenced: bool, /// Leave the cache alone until then: the last upload failed or we were throttled. @@ -111,6 +138,8 @@ impl Exporter { counters: Arc, windows: Arc>, attributes: Arc>>, + spans: Arc>, + trace_id: [u8; 16], commands: mpsc::UnboundedReceiver, ) -> Self { Self { @@ -121,6 +150,8 @@ impl Exporter { counters, windows, attributes, + spans, + trace_id, commands, silenced: false, paused_until: None, @@ -214,8 +245,9 @@ impl Exporter { self.upload().await; } - /// Encode everything queued into the cache — no network involved. + /// Encode everything queued — log records and finished spans — into the cache. No network. fn enqueue(&mut self) { + self.enqueue_spans(); loop { let mut batch = self.store.drain(self.config.max_batch_size.max(1) as usize); if batch.is_empty() { @@ -237,37 +269,60 @@ impl Exporter { batch.push(delta.report(self.cache.pending().len() as u64)); self.last_report = now; } - self.attach_session_attributes(&mut batch); - let count = batch.len() as u64; - let body = otlp::encode_logs(&self.config.resource, batch); - self.seq += 1; - let id = format!("{:020}-{:06}-{count}", now_unix_nanos(), self.seq); - if let Err(err) = self.cache.push(&id, &body) { - // A full disk is a steady state, not an event: warn once, then stay quiet. - if self.cache_failures == 0 { - log::warn!( - "telemetry: cannot cache batches ({err}); dropping until it recovers" - ); - } else { - log::debug!("telemetry: could not cache {count} events: {err}"); - } - self.cache_failures += 1; - Counters::add(&self.counters.cache_error, count); + for event in &mut batch { + self.attach_session_attributes(&mut event.attributes); } + let count = batch.len() as u64; + let body = otlp::encode_logs(&self.config.resource, &self.trace_id, batch); + self.push_batch(Signal::Logs, count, &body); } } - /// Merge the session-wide attributes into each record, without overriding explicit ones. - fn attach_session_attributes(&self, batch: &mut [TelemetryEvent]) { - let session = self.attributes.lock().unwrap_or_else(|e| e.into_inner()); - if session.is_empty() { + /// Finished spans travel as their own batch on the traces signal. + fn enqueue_spans(&mut self) { + let (mut spans, dropped) = { + let mut registry = self.spans.lock().unwrap_or_else(|e| e.into_inner()); + (registry.drain(self.config.max_batch_size.max(1) as usize), registry.take_dropped()) + }; + Counters::add(&self.counters.queue_full, dropped); + if spans.is_empty() { return; } - for event in batch { - for attribute in session.iter() { - if !event.attributes.iter().any(|a| a.key == attribute.key) { - event.attributes.push(attribute.clone()); - } + if self.silenced || self.throttled_until.is_some_and(|t| Instant::now() < t) { + let counter = + if self.silenced { &self.counters.disabled } else { &self.counters.throttled }; + Counters::add(counter, spans.len() as u64); + return; + } + for span in &mut spans { + self.attach_session_attributes(&mut span.attributes); + } + let count = spans.len() as u64; + let body = otlp::encode_spans(&self.config.resource, &self.trace_id, spans); + self.push_batch(Signal::Traces, count, &body); + } + + fn push_batch(&mut self, signal: Signal, count: u64, body: &[u8]) { + self.seq += 1; + let id = format!("{:020}-{:06}-{count}-{}", now_unix_nanos(), self.seq, signal.tag()); + if let Err(err) = self.cache.push(&id, body) { + // A full disk is a steady state, not an event: warn once, then stay quiet. + if self.cache_failures == 0 { + log::warn!("telemetry: cannot cache batches ({err}); dropping until it recovers"); + } else { + log::debug!("telemetry: could not cache {count} items: {err}"); + } + self.cache_failures += 1; + Counters::add(&self.counters.cache_error, count); + } + } + + /// Merge the session-wide attributes into a record's own, without overriding explicit ones. + fn attach_session_attributes(&self, attributes: &mut Vec) { + let session = self.attributes.lock().unwrap_or_else(|e| e.into_inner()); + for attribute in session.iter() { + if !attributes.iter().any(|a| a.key == attribute.key) { + attributes.push(attribute.clone()); } } } @@ -282,7 +337,7 @@ impl Exporter { self.cache.remove(&id); continue; }; - match self.deliver(&body).await { + match self.deliver(&body, Signal::of(&id)).await { Delivery::Sent => { self.cache.remove(&id); Counters::add(&self.counters.uploads_sent, 1); @@ -319,11 +374,16 @@ impl Exporter { } /// Upload one encoded batch with bounded retries and classify the outcome. - async fn deliver(&self, body: &[u8]) -> Delivery { + async fn deliver(&self, body: &[u8], signal: Signal) -> Delivery { let mut headers = self.config.headers.clone(); headers.insert("Content-Type".to_owned(), otlp::CONTENT_TYPE.to_owned()); - let request = - ExportRequest { url: self.config.endpoint.clone(), headers, body: body.to_vec() }; + let url = match signal { + Signal::Logs => self.config.endpoint.clone(), + Signal::Traces => { + self.config.traces_endpoint.clone().unwrap_or_else(|| self.config.endpoint.clone()) + } + }; + let request = ExportRequest { url, headers, body: body.to_vec() }; let attempt_timeout = Duration::from_millis(self.config.export_timeout_ms.max(1)); // ponytail: linear backoff, 2 retries, blocks the tick loop while sleeping (≤ 3 s). @@ -359,5 +419,5 @@ impl Exporter { /// The event count the exporter encodes as the last component of a batch id. fn events_in(id: &str) -> u64 { - id.rsplit('-').next().and_then(|n| n.parse().ok()).unwrap_or(0) + id.split('-').nth(2).and_then(|n| n.parse().ok()).unwrap_or(0) } diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs index 1d4267ab1..fd99033a6 100644 --- a/livekit-telemetry/src/lib.rs +++ b/livekit-telemetry/src/lib.rs @@ -29,6 +29,9 @@ mod device; /// RTC stats samples and their on-device windowing. mod rtc; +/// Spans: one attempt at an operation, with explicit handles across the FFI. +mod span; + /// Batch exporter actor: timer, OTLP encoding, retry policy. mod exporter; @@ -52,6 +55,7 @@ pub use device::*; pub use event::*; pub use exporter::Exporter; pub use rtc::{RtcStatsSample, StreamDirection, TrackKind}; +pub use span::{SpanKind, SpanOutcome}; pub use stats::TelemetryStats; pub use telemetry::*; pub use transport::*; diff --git a/livekit-telemetry/src/otlp.rs b/livekit-telemetry/src/otlp.rs index 329888cb7..d55dc109f 100644 --- a/livekit-telemetry/src/otlp.rs +++ b/livekit-telemetry/src/otlp.rs @@ -17,32 +17,47 @@ use prost::Message; use crate::{ event::now_unix_nanos, proto::opentelemetry::proto::{ - collector::logs::v1::ExportLogsServiceRequest, + collector::{logs::v1::ExportLogsServiceRequest, trace::v1::ExportTraceServiceRequest}, common::v1::{any_value, AnyValue, InstrumentationScope, KeyValue}, logs::v1::{LogRecord, ResourceLogs, ScopeLogs, SeverityNumber}, resource::v1::Resource, + trace::v1::{span, status, ResourceSpans, ScopeSpans, Span, Status}, }, - Attribute, AttributeValue, Severity, TelemetryEvent, + span::SpanRecord, + Attribute, AttributeValue, Severity, SpanKind, SpanOutcome, TelemetryEvent, }; pub(crate) const CONTENT_TYPE: &str = "application/x-protobuf"; +fn resource(attributes: &[Attribute]) -> Option { + Some(Resource { + attributes: attributes.iter().map(KeyValue::from).collect(), + ..Default::default() + }) +} + +fn scope() -> Option { + Some(InstrumentationScope { + name: env!("CARGO_PKG_NAME").to_owned(), + version: env!("CARGO_PKG_VERSION").to_owned(), + ..Default::default() + }) +} + /// Encode one batch as an OTLP `ExportLogsServiceRequest`: one resource, one instrumentation -/// scope (this crate), one log record per event. -pub(crate) fn encode_logs(resource: &[Attribute], events: Vec) -> Vec { +/// scope (this crate), one log record per event. Every record carries the session's trace id; +/// records emitted inside a span carry its span id too. +pub(crate) fn encode_logs( + resource_attributes: &[Attribute], + trace_id: &[u8; 16], + events: Vec, +) -> Vec { ExportLogsServiceRequest { resource_logs: vec![ResourceLogs { - resource: Some(Resource { - attributes: resource.iter().map(KeyValue::from).collect(), - ..Default::default() - }), + resource: resource(resource_attributes), scope_logs: vec![ScopeLogs { - scope: Some(InstrumentationScope { - name: env!("CARGO_PKG_NAME").to_owned(), - version: env!("CARGO_PKG_VERSION").to_owned(), - ..Default::default() - }), - log_records: events.into_iter().map(LogRecord::from).collect(), + scope: scope(), + log_records: events.into_iter().map(|e| log_record(e, trace_id)).collect(), ..Default::default() }], ..Default::default() @@ -51,21 +66,77 @@ pub(crate) fn encode_logs(resource: &[Attribute], events: Vec) - .encode_to_vec() } -impl From for LogRecord { - fn from(event: TelemetryEvent) -> Self { - let time_unix_nano = event.timestamp_ns.unwrap_or_else(now_unix_nanos); - LogRecord { - time_unix_nano, - observed_time_unix_nano: time_unix_nano, - severity_number: SeverityNumber::from(event.severity) as i32, - severity_text: severity_text(event.severity).to_owned(), - body: event - .body - .map(|text| AnyValue { value: Some(any_value::Value::StringValue(text)) }), - attributes: event.attributes.iter().map(KeyValue::from).collect(), - event_name: event.name, +/// Encode finished spans as an OTLP `ExportTraceServiceRequest` under the session's trace id. +pub(crate) fn encode_spans( + resource_attributes: &[Attribute], + trace_id: &[u8; 16], + spans: Vec, +) -> Vec { + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: resource(resource_attributes), + scope_spans: vec![ScopeSpans { + scope: scope(), + spans: spans.into_iter().map(|s| otlp_span(s, trace_id)).collect(), + ..Default::default() + }], ..Default::default() - } + }], + } + .encode_to_vec() +} + +fn log_record(event: TelemetryEvent, trace_id: &[u8; 16]) -> LogRecord { + let time_unix_nano = event.timestamp_ns.unwrap_or_else(now_unix_nanos); + LogRecord { + time_unix_nano, + observed_time_unix_nano: time_unix_nano, + severity_number: SeverityNumber::from(event.severity) as i32, + severity_text: severity_text(event.severity).to_owned(), + body: event.body.map(|text| AnyValue { value: Some(any_value::Value::StringValue(text)) }), + attributes: event.attributes.iter().map(KeyValue::from).collect(), + event_name: event.name, + trace_id: trace_id.to_vec(), + span_id: event.span_id.map(|id| id.to_be_bytes().to_vec()).unwrap_or_default(), + ..Default::default() + } +} + +fn otlp_span(record: SpanRecord, trace_id: &[u8; 16]) -> Span { + let mut attributes: Vec = record.attributes.iter().map(KeyValue::from).collect(); + attributes.extend(record.outcome_attributes().iter().map(KeyValue::from)); + Span { + trace_id: trace_id.to_vec(), + span_id: record.span_id.to_be_bytes().to_vec(), + parent_span_id: record.parent_span_id.map(|p| p.to_be_bytes().to_vec()).unwrap_or_default(), + name: record.name, + kind: match record.kind { + SpanKind::Internal => span::SpanKind::Internal, + SpanKind::Client => span::SpanKind::Client, + } as i32, + start_time_unix_nano: record.start_ns, + end_time_unix_nano: record.end_ns, + attributes, + events: record + .events + .into_iter() + .map(|e| span::Event { + time_unix_nano: e.time_ns, + name: e.name, + attributes: e.attributes.iter().map(KeyValue::from).collect(), + ..Default::default() + }) + .collect(), + // OTel: instrumentation should not set `Ok`; success and cancellation stay `Unset` and + // are told apart by `lk.outcome`. + status: Some(Status { + code: match record.outcome { + SpanOutcome::Error => status::StatusCode::Error, + SpanOutcome::Ok | SpanOutcome::Cancelled => status::StatusCode::Unset, + } as i32, + message: record.error_type.unwrap_or_default(), + }), + ..Default::default() } } @@ -118,7 +189,7 @@ mod tests { .with_severity(Severity::Warn) .with_body("hi") .with_attribute("lk.ping.seq", 7i64); - let bytes = encode_logs(&resource, vec![event]); + let bytes = encode_logs(&resource, &[7u8; 16], vec![event]); let decoded = ExportLogsServiceRequest::decode(&bytes[..]).expect("valid OTLP"); let resource_logs = &decoded.resource_logs[0]; @@ -128,6 +199,8 @@ mod tests { assert_eq!(scope_logs.scope.as_ref().expect("scope").name, "livekit-telemetry"); let record = &scope_logs.log_records[0]; assert_eq!(record.event_name, "lk.ping"); + assert_eq!(record.trace_id, vec![7u8; 16]); + assert!(record.span_id.is_empty()); assert_eq!(record.severity_number, SeverityNumber::Warn as i32); assert_eq!(record.severity_text, "WARN"); assert!(record.time_unix_nano > 0); diff --git a/livekit-telemetry/src/proto/mod.rs b/livekit-telemetry/src/proto/mod.rs index 0baf3f364..195d5e2bb 100644 --- a/livekit-telemetry/src/proto/mod.rs +++ b/livekit-telemetry/src/proto/mod.rs @@ -21,6 +21,6 @@ pub mod opentelemetry { pub mod proto { - pub use opentelemetry_proto::tonic::{collector, common, logs, resource}; + pub use opentelemetry_proto::tonic::{collector, common, logs, resource, trace}; } } diff --git a/livekit-telemetry/src/span.rs b/livekit-telemetry/src/span.rs new file mode 100644 index 000000000..65952c72c --- /dev/null +++ b/livekit-telemetry/src/span.rs @@ -0,0 +1,233 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::{HashMap, VecDeque}; + +use crate::{event::now_unix_nanos, Attribute, AttributeValue}; + +/// OTel span kind, restricted to what client operations need. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SpanKind { + /// An operation inside the SDK (publish, subscribe). + Internal, + /// A call to the SFU that waits for its answer (connect, reconnect). + Client, +} + +/// How an attempt ended. OTel status knows only `Unset`/`Ok`/`Error`, so `Cancelled` travels as +/// `status = Unset` plus the `lk.outcome` attribute — every span carries `lk.outcome` so rollups +/// never have to infer it (a user hanging up mid-connect is not a failure). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SpanOutcome { + Ok, + Error, + Cancelled, +} + +impl SpanOutcome { + pub(crate) fn as_str(self) -> &'static str { + match self { + SpanOutcome::Ok => "ok", + SpanOutcome::Error => "error", + SpanOutcome::Cancelled => "cancelled", + } + } +} + +/// A checkpoint inside a span (OTLP span event). Structural to one attempt — the connect +/// sequence's `ws_open → join_recv → pc_connected → …` — hence in the span's own envelope rather +/// than a standalone log record (OTEP 4430 keeps that legal). +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SpanEvent { + pub name: String, + pub time_ns: u64, + pub attributes: Vec, +} + +/// One attempt at an operation, from `begin_span` to `end_span`. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct SpanRecord { + pub span_id: u64, + pub parent_span_id: Option, + pub name: String, + pub kind: SpanKind, + pub start_ns: u64, + pub end_ns: u64, + pub outcome: SpanOutcome, + pub error_type: Option, + pub attributes: Vec, + pub events: Vec, +} + +/// OTel default span limits. +const MAX_EVENTS_PER_SPAN: usize = 128; +const MAX_ATTRIBUTES_PER_SPAN: usize = 128; +/// Spans a host can leave open before the oldest is abandoned (counted as dropped). +const MAX_OPEN_SPANS: usize = 256; + +/// Open spans by handle, plus the finished ones waiting for the exporter. +/// +/// Handles are opaque `u64`s minted here; the host keeps them (a Swift `Span` object, a Kotlin +/// value) and never sees ambient context — that is the platform's job (task-locals, coroutine +/// context, zones), not the FFI's. +pub(crate) struct Spans { + open: HashMap, + /// Insertion order of `open`, to abandon the oldest when the cap is hit. + open_order: VecDeque, + finished: Vec, + finished_capacity: usize, + next_id: u64, + pub dropped: u64, +} + +impl Spans { + pub fn new(finished_capacity: usize) -> Self { + Self { + open: HashMap::new(), + open_order: VecDeque::new(), + finished: Vec::new(), + finished_capacity, + // Span ids must be non-zero (OTLP treats all-zero as absent); start at 1 and mix in + // randomness so ids from two pipelines in one process never collide. + next_id: rand::random::() | 1, + dropped: 0, + } + } + + pub fn begin(&mut self, name: &str, kind: SpanKind, parent: Option) -> u64 { + let id = self.next_id; + self.next_id = self.next_id.wrapping_add(1).max(1); + if self.open.len() >= MAX_OPEN_SPANS { + if let Some(oldest) = self.open_order.pop_front() { + self.open.remove(&oldest); + self.dropped += 1; + } + } + let record = SpanRecord { + span_id: id, + parent_span_id: parent.filter(|p| *p != 0), + name: name.to_owned(), + kind, + start_ns: now_unix_nanos(), + end_ns: 0, + outcome: SpanOutcome::Ok, + error_type: None, + attributes: Vec::new(), + events: Vec::new(), + }; + self.open.insert(id, record); + self.open_order.push_back(id); + id + } + + pub fn add_event(&mut self, id: u64, name: &str, attributes: Vec) { + let Some(span) = self.open.get_mut(&id) else { return }; + if span.events.len() >= MAX_EVENTS_PER_SPAN { + return; + } + span.events.push(SpanEvent { + name: name.to_owned(), + time_ns: now_unix_nanos(), + attributes, + }); + } + + /// Close a span; the finished record waits for the next export. Unknown ids are ignored + /// (double `end` is harmless, like OTel's). + pub fn end( + &mut self, + id: u64, + outcome: SpanOutcome, + error_type: Option, + mut attributes: Vec, + ) { + let Some(mut span) = self.open.remove(&id) else { return }; + self.open_order.retain(|open| *open != id); + span.end_ns = now_unix_nanos().max(span.start_ns); + span.outcome = outcome; + span.error_type = error_type; + attributes.truncate(MAX_ATTRIBUTES_PER_SPAN); + span.attributes = attributes; + if self.finished.len() >= self.finished_capacity { + self.finished.remove(0); + self.dropped += 1; + } + self.finished.push(span); + } + + /// Take the finished spans, oldest first. + pub fn drain(&mut self, max: usize) -> Vec { + let n = max.min(self.finished.len()); + self.finished.drain(..n).collect() + } + + pub fn take_dropped(&mut self) -> u64 { + std::mem::take(&mut self.dropped) + } +} + +impl SpanRecord { + /// `lk.outcome` and `error.type`, the attributes every span carries beyond the caller's. + pub(crate) fn outcome_attributes(&self) -> Vec { + let mut attributes = vec![Attribute::new("lk.outcome", self.outcome.as_str())]; + if let Some(error_type) = &self.error_type { + attributes.push(Attribute::new("error.type", AttributeValue::Str(error_type.clone()))); + } + attributes + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn spans_open_record_events_and_finish_in_order() { + let mut spans = Spans::new(8); + let parent = spans.begin("lk.connect", SpanKind::Client, None); + let child = spans.begin("lk.publish", SpanKind::Internal, Some(parent)); + spans.add_event(parent, "ws_open", vec![]); + spans.end(child, SpanOutcome::Cancelled, None, vec![]); + spans.end( + parent, + SpanOutcome::Error, + Some("timeout".into()), + vec![Attribute::new("lk.connect.attempt", 1i64)], + ); + spans.end(parent, SpanOutcome::Ok, None, vec![]); // double end: ignored + + let finished = spans.drain(10); + assert_eq!(finished.len(), 2); + assert_eq!(finished[0].name, "lk.publish"); + assert_eq!(finished[0].parent_span_id, Some(parent)); + assert_eq!(finished[0].outcome, SpanOutcome::Cancelled); + assert_eq!(finished[1].events[0].name, "ws_open"); + assert_eq!(finished[1].error_type.as_deref(), Some("timeout")); + assert!(finished[1].end_ns >= finished[1].start_ns); + assert_eq!(spans.take_dropped(), 0); + } + + #[test] + fn finished_spans_are_bounded() { + let mut spans = Spans::new(1); + for _ in 0..2 { + let id = spans.begin("lk.publish", SpanKind::Internal, None); + spans.end(id, SpanOutcome::Ok, None, vec![]); + } + assert_eq!(spans.drain(10).len(), 1); + assert_eq!(spans.take_dropped(), 1); + } +} diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 1e8f9774c..8853dfd0c 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -22,9 +22,10 @@ use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; use crate::{ - event::now_unix_nanos, exporter::Command, rtc::StatsWindows, stats::Counters, store::Store, - Attribute, AttributeValue, BatchCache, DeviceState, Exporter, FileCache, MemoryCache, - RtcStatsSample, Severity, TelemetryEvent, TelemetryStats, TelemetryTransport, + event::now_unix_nanos, exporter::Command, rtc::StatsWindows, span::Spans, stats::Counters, + store::Store, Attribute, AttributeValue, BatchCache, DeviceState, Exporter, FileCache, + MemoryCache, RtcStatsSample, Severity, SpanKind, SpanOutcome, TelemetryEvent, TelemetryStats, + TelemetryTransport, }; /// Pipeline configuration. @@ -37,6 +38,10 @@ pub struct TelemetryConfig { /// Full OTLP/HTTP logs URL: `http://localhost:4318/v1/logs` locally, /// `https:///observability/logs/otlp/v0` for LiveKit Cloud. pub endpoint: String, + /// OTLP/HTTP traces URL. `None` derives it from `endpoint` by replacing the last `logs` + /// path segment with `traces` (works for both layouts above). + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub traces_endpoint: Option, /// Extra request headers, e.g. `Authorization: Bearer `. pub headers: HashMap, /// Resource attributes describing the emitter (`service.name`, `os.name`, @@ -76,6 +81,7 @@ impl TelemetryConfig { pub fn new(endpoint: impl Into) -> Self { Self { endpoint: endpoint.into(), + traces_endpoint: None, headers: HashMap::new(), resource: Vec::new(), storage_dir: None, @@ -123,6 +129,8 @@ pub struct Telemetry { windows: Arc>, guard: Arc>, attributes: Arc>>, + spans: Arc>, + trace_id: [u8; 16], commands: mpsc::UnboundedSender, } @@ -185,13 +193,20 @@ impl Telemetry { cache: Arc, ) -> (Self, Exporter) { add_sdk_resource(&mut config.resource); + if config.traces_endpoint.is_none() { + config.traces_endpoint = Some(derive_traces_endpoint(&config.endpoint)); + } let config = Arc::new(config); + // The session is the trace: one id per pipeline, client-generated so pre-connect + // failures and reconnects share it. + let trace_id = rand::random::().max(1).to_be_bytes(); let counters = Arc::new(Counters::default()); let store = Arc::new(Store::new(config.max_queue_size.max(1) as usize, counters.clone())); let (commands, receiver) = mpsc::unbounded_channel(); let windows = Arc::new(Mutex::new(StatsWindows::default())); let guard = Arc::new(Mutex::new(FloodGuard::new(config.max_events_per_10min))); let attributes = Arc::new(Mutex::new(Vec::new())); + let spans = Arc::new(Mutex::new(Spans::new(config.max_queue_size.max(1) as usize))); let exporter = Exporter::new( store.clone(), transport, @@ -200,6 +215,8 @@ impl Telemetry { counters.clone(), windows.clone(), attributes.clone(), + spans.clone(), + trace_id, receiver, ); let telemetry = Self { @@ -211,6 +228,8 @@ impl Telemetry { windows, guard, attributes, + spans, + trace_id, commands, }; (telemetry, exporter) @@ -249,6 +268,38 @@ impl Telemetry { } } + /// The session's trace id as 32 hex characters — what every span and log record of this + /// pipeline carries. Print it (`lkt_…`) so support can find the session. + pub fn trace_id(&self) -> String { + format!("{:032x}", u128::from_be_bytes(self.trace_id)) + } + + /// Open a span: one attempt at an operation (`lk.connect`, `lk.publish`, …). Returns the + /// handle to record checkpoints and to end it with; `parent` nests it under another open span. + pub fn begin_span(&self, name: &str, kind: SpanKind, parent: Option) -> u64 { + self.spans.lock().unwrap_or_else(|e| e.into_inner()).begin(name, kind, parent) + } + + /// Record a checkpoint inside an open span (`ws_open`, `join_recv`, …), stamped now. + pub fn add_span_event(&self, span: u64, name: &str, attributes: Vec) { + self.spans.lock().unwrap_or_else(|e| e.into_inner()).add_event(span, name, attributes); + } + + /// End a span with its outcome; `error_type` becomes `error.type` and the status message. + /// The span is exported with the next batch. Ending twice, or an unknown handle, is a no-op. + pub fn end_span( + &self, + span: u64, + outcome: SpanOutcome, + error_type: Option, + attributes: Vec, + ) { + self.spans + .lock() + .unwrap_or_else(|e| e.into_inner()) + .end(span, outcome, error_type, attributes); + } + /// Push one `getStats()` reading. Readings are windowed on device into `lk.rtc.stats.sample` /// events (see `stats_window_ms`); they never count against the flood guard. pub fn record_stats(&self, sample: RtcStatsSample) { @@ -295,6 +346,14 @@ impl Telemetry { } } +/// `…/logs…` → `…/traces…`: covers `/v1/logs` and `/observability/logs/otlp/v0` alike. +fn derive_traces_endpoint(logs_endpoint: &str) -> String { + match logs_endpoint.rsplit_once("logs") { + Some((before, after)) => format!("{before}traces{after}"), + None => logs_endpoint.to_owned(), + } +} + /// Fill in the `telemetry.sdk.*` resource attributes and a fallback `service.name`. fn add_sdk_resource(resource: &mut Vec) { let defaults = [ @@ -320,10 +379,13 @@ mod tests { use crate::{ cache::temp_dir, proto::opentelemetry::proto::{ - collector::logs::v1::ExportLogsServiceRequest, common::v1::any_value::Value, + collector::{logs::v1::ExportLogsServiceRequest, trace::v1::ExportTraceServiceRequest}, + common::v1::any_value::Value, logs::v1::LogRecord, + trace::v1::{span, status}, }, - AppState, ExportError, ExportRequest, StreamDirection, ThermalState, TrackKind, + AppState, ExportError, ExportRequest, SpanKind, SpanOutcome, StreamDirection, ThermalState, + TrackKind, }; #[derive(Default)] @@ -569,7 +631,7 @@ mod tests { telemetry.flush().await; let pending = cache.pending(); assert_eq!(pending.len(), 1); - assert!(pending[0].ends_with("-1"), "id carries the event count: {}", pending[0]); + assert!(pending[0].ends_with("-1-l"), "id carries count and signal: {}", pending[0]); } #[tokio::test(start_paused = true)] @@ -715,6 +777,97 @@ mod tests { assert_eq!(attribute(&records(&transport.sent()[1])[0], "lk.room.sid"), None); } + #[tokio::test(start_paused = true)] + async fn spans_export_as_traces_under_the_session_trace_id() { + let transport = FakeTransport::scripted([]); + let telemetry = + start(TelemetryConfig::new("http://c/observability/logs/otlp/v0"), transport.clone()); + let connect = telemetry.begin_span("lk.connect", SpanKind::Client, None); + telemetry.add_span_event(connect, "ws_open", vec![]); + telemetry.emit( + TelemetryEvent::new("") + .with_severity(Severity::Error) + .with_body("boom") + .in_span(connect), + ); + telemetry.end_span( + connect, + SpanOutcome::Error, + Some("timeout".into()), + vec![Attribute::new("lk.connect.attempt", 1i64)], + ); + telemetry.flush().await; + + let sent = transport.sent(); + assert_eq!(sent.len(), 2, "one logs batch, one traces batch"); + let traces = + sent.iter().find(|r| r.url.ends_with("/traces/otlp/v0")).expect("traces request"); + assert_eq!( + traces.url, "http://c/observability/traces/otlp/v0", + "derived from logs endpoint" + ); + let decoded = ExportTraceServiceRequest::decode(&traces.body[..]).expect("valid OTLP"); + let otlp_span = &decoded.resource_spans[0].scope_spans[0].spans[0]; + assert_eq!(otlp_span.name, "lk.connect"); + assert_eq!(otlp_span.kind, span::SpanKind::Client as i32); + assert_eq!(hex(&otlp_span.trace_id), telemetry.trace_id()); + assert_eq!(otlp_span.span_id, connect.to_be_bytes().to_vec()); + assert!(otlp_span.parent_span_id.is_empty()); + assert!(otlp_span.end_time_unix_nano >= otlp_span.start_time_unix_nano); + assert_eq!(otlp_span.events[0].name, "ws_open"); + assert_eq!( + otlp_span.status.as_ref().map(|s| s.code), + Some(status::StatusCode::Error as i32) + ); + assert_eq!(otlp_span.status.as_ref().map(|s| s.message.as_str()), Some("timeout")); + let attr = |key: &str| { + otlp_span + .attributes + .iter() + .find(|kv| kv.key == key) + .and_then(|kv| kv.value.as_ref()?.value.clone()) + }; + assert_eq!(attr("lk.outcome"), Some(Value::StringValue("error".into()))); + assert_eq!(attr("error.type"), Some(Value::StringValue("timeout".into()))); + assert_eq!(attr("lk.connect.attempt"), Some(Value::IntValue(1))); + + let logs = sent.iter().find(|r| r.url.ends_with("/logs/otlp/v0")).expect("logs request"); + let record = &records(logs)[0]; + assert_eq!(hex(&record.trace_id), telemetry.trace_id(), "every record carries the trace"); + assert_eq!( + record.span_id, + connect.to_be_bytes().to_vec(), + "and the span it was emitted in" + ); + } + + #[tokio::test(start_paused = true)] + async fn cancelled_spans_keep_status_unset() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + let publish = telemetry.begin_span("lk.publish", SpanKind::Internal, None); + telemetry.end_span(publish, SpanOutcome::Cancelled, None, vec![]); + telemetry.flush().await; + let sent = transport.sent(); + assert_eq!(sent[0].url, "http://collector/v1/traces"); + let decoded = ExportTraceServiceRequest::decode(&sent[0].body[..]).expect("valid OTLP"); + let otlp_span = &decoded.resource_spans[0].scope_spans[0].spans[0]; + assert_eq!( + otlp_span.status.as_ref().map(|s| s.code), + Some(status::StatusCode::Unset as i32) + ); + let outcome = otlp_span + .attributes + .iter() + .find(|kv| kv.key == "lk.outcome") + .and_then(|kv| kv.value.as_ref()?.value.clone()); + assert_eq!(outcome, Some(Value::StringValue("cancelled".into()))); + } + + fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() + } + #[tokio::test(start_paused = true)] async fn entering_background_flushes_immediately() { let transport = FakeTransport::scripted([]); diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index a85bcff79..35d5f157e 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -17,8 +17,9 @@ use std::{ }; use livekit_telemetry::{ - AttributeValue, DeviceState, ExportError, ExportRequest, NetTransport, RtcStatsSample, - TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, + Attribute, AttributeValue, DeviceState, ExportError, ExportRequest, NetTransport, + RtcStatsSample, SpanKind, SpanOutcome, TelemetryConfig, TelemetryEvent, TelemetryStats, + TelemetryTransport, }; use tokio::sync::{mpsc, oneshot}; @@ -79,6 +80,32 @@ impl Telemetry { self.0.set_attribute(&key, value); } + /// The session's trace id (32 hex chars) — on every span and record of this pipeline. + pub fn trace_id(&self) -> String { + self.0.trace_id() + } + + /// Open a span (one attempt at `lk.connect`, `lk.publish`, …); returns its handle. + pub fn begin_span(&self, name: String, kind: SpanKind, parent: Option) -> u64 { + self.0.begin_span(&name, kind, parent) + } + + /// Record a checkpoint inside an open span, stamped now. + pub fn add_span_event(&self, span: u64, name: String, attributes: Vec) { + self.0.add_span_event(span, &name, attributes); + } + + /// End a span with its outcome; exported with the next batch. + pub fn end_span( + &self, + span: u64, + outcome: SpanOutcome, + error_type: Option, + attributes: Vec, + ) { + self.0.end_span(span, outcome, error_type, attributes); + } + /// Push one `getStats()` reading for a track; windowed on device into `lk.rtc.stats.sample`. pub fn record_stats(&self, sample: RtcStatsSample) { self.0.record_stats(sample); From ad2b05fe95b5c896e75aba9be4f3281080871d78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:17:53 +0200 Subject: [PATCH 10/50] docs(telemetry): specify lk.publish and lk.subscribe spans Co-Authored-By: Claude Fable 5 --- livekit-telemetry/SPEC.md | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index 7cbc47a12..a845d9abb 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -166,4 +166,29 @@ checkpoints: "attempt " per attempt outcome: ok | error | cancelled # cancelled when disconnect() or a newer reconnect wins ``` -`lk.publish` / `lk.subscribe`: not yet emitted by any SDK (see design page, round 7). +```yaml +span: lk.publish +kind: internal +parent: the ambient span, when any (the connect span for a pre-connect microphone) +attributes: + lk.track.kind: enum(audio | video) + lk.track.source: enum(camera | microphone | screenShare | screenShareAudio | unknown) + lk.track.sid: string # on success +outcome: ok | error (error.type) | cancelled +``` + +```yaml +span: lk.subscribe +kind: internal +starts: when the intent to subscribe exists — a remote publish under autoSubscribe, or the + manual subscribe call +ends: at first media (the first stats reading with bytes received; 1 s granularity) → ok; + unsubscribe / unpublish before media → cancelled; + subscription failure → error; no media within 30 s → error (error.type = LiveKitError.timedOut) +attributes: + lk.track.sid: string + lk.track.kind: enum(audio | video) + lk.track.source: string + lk.participant.remote_identity: string +checkpoints: subscribed, first_media +``` From 89d40946f94a69c09111babf6e6ac00febd99ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:13:16 +0200 Subject: [PATCH 11/50] feat(telemetry): upload policy and device signals Telemetry must never win over media, so uploads are shaped as well as batched: gzip bodies (level 1, compressed when cached), a `Priority: u=7` hint, at most `max_batches_per_upload` (4) cached batches per tick, and holds while `lk.connect`/`lk.reconnect` are open, while WebRTC reports an outbound track bandwidth-limited (derived from the stats we already get), or while the device asks for quiet (constrained network, battery <= 10 % unplugged). A hold lasts 60 s at most; shutdown drains without the budget. `DeviceState` grows memory pressure, network path (type, expensive, constrained) and battery; each feeds `cadence_factor` and three new `lk.device.*.changed` events (battery on bucket crossings only, OTel `hw.battery.*` names, `network.connection.type`). A CPU-limited encoder (`qualityLimitationDurations.cpu`) doubles the cadence too - CPU is never measured directly, per the design doc. The exporter schedules ticks as `next = last + period` so a cadence change applies to the pending tick in both directions, and reads the device state synchronously (a command-only copy raced with emit + flush). `session.id` joins the resource attributes. SPEC.md documents the events, the cadence table and the upload policy. `target-*/` is ignored. Co-Authored-By: Claude Fable 5.1 --- .gitignore | 1 + Cargo.lock | 1 + livekit-telemetry/Cargo.toml | 2 + livekit-telemetry/SPEC.md | 107 ++++++++++++++- livekit-telemetry/src/device.rs | 201 +++++++++++++++++++++++++++-- livekit-telemetry/src/exporter.rs | 186 +++++++++++++++++++------- livekit-telemetry/src/rtc.rs | 81 +++++++++++- livekit-telemetry/src/span.rs | 6 + livekit-telemetry/src/telemetry.rs | 122 +++++++++++++++-- 9 files changed, 627 insertions(+), 80 deletions(-) diff --git a/.gitignore b/.gitignore index 328df4d65..fbae334f9 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ soxr-sys/test-output.wav .env .cursor __pycache__ +/target-*/ diff --git a/Cargo.lock b/Cargo.lock index 3ffba8c4b..9e81cd846 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3783,6 +3783,7 @@ name = "livekit-telemetry" version = "0.1.0" dependencies = [ "async-trait", + "flate2", "livekit-net", "log", "opentelemetry-proto", diff --git a/livekit-telemetry/Cargo.toml b/livekit-telemetry/Cargo.toml index fee9716b9..86c7b5d99 100644 --- a/livekit-telemetry/Cargo.toml +++ b/livekit-telemetry/Cargo.toml @@ -14,6 +14,8 @@ thiserror = { workspace = true } async-trait = "0.1" prost = { workspace = true } rand = { workspace = true } +# gzip for request bodies; the deflate backend (miniz_oxide) is already linked by data streams. +flate2 = "1" # OTLP message types only (`gen-tonic-messages` = prost structs, no tonic). Same prost as # livekit-protocol so a single prost is linked. opentelemetry-proto = { version = "0.32", default-features = false, features = ["logs", "trace", "gen-tonic-messages"] } diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index a845d9abb..3773431c0 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -15,6 +15,7 @@ Set once per pipeline (`TelemetryConfig.resource`): | `os.name`, `os.version` | platform SDK | `iOS`, `18.5` | | `device.model.identifier` | platform SDK | `iPhone16,1` | | `telemetry.sdk.name/language/version` | core | `livekit-telemetry`, `rust`, `0.1.0` | +| `session.id` | core | the session's trace id as 32 hex chars — every log record and span of the pipeline carries it | ## Events @@ -73,16 +74,108 @@ cadence: on change (+ initial value); entering background also forces a flush platforms: all ``` +```yaml +event: lk.device.memory.changed +area: device +attributes: + lk.device.memory.pressure: enum(normal | warning | critical) + # Apple: DispatchSource memory-pressure levels; Android onTrimMemory: RUNNING_LOW / + # BACKGROUND → warning, RUNNING_CRITICAL / COMPLETE → critical +cadence: on change (+ initial value) +platforms: ios, macos, android — optional elsewhere +``` + +```yaml +event: lk.device.network.changed +area: device +attributes: + network.connection.type: enum(wifi | cell | wired | unavailable | unknown) # OTel semconv + lk.device.network.expensive: bool # cellular / hotspot (NWPath.isExpensive, metered) + lk.device.network.constrained: bool # Low Data Mode / Data Saver / navigator.connection.saveData +cadence: on change of any attribute (+ initial value) +platforms: ios, macos, android — web: Chromium only +``` + +```yaml +event: lk.device.battery.changed +area: device +attributes: + hw.battery.charge: double # 0.0–1.0 (OTel hardware semconv) + hw.battery.state: enum(charging | discharging) # OTel hardware semconv +cadence: on charging change and when the level crosses 20 % or 10 % unplugged — never per + percent; silent where the level is unknown (desktops, tvOS) +platforms: ios, android — optional elsewhere +``` + +```yaml +event: lk.device.audio_route.changed +area: device +attributes: + lk.device.audio_route.reason: string # AVAudioSession route-change reason name + lk.device.audio_route.outputs: string # comma-separated output port types (speaker, bluetooth_a2dp, …) +cadence: on change +platforms: ios — android: audio device callbacks; optional elsewhere +``` + +```yaml +event: lk.device.audio.interruption +area: device +attributes: + lk.device.audio.interruption: enum(began | ended) +cadence: on change +platforms: ios — android: audio focus loss/gain; optional elsewhere +``` + ## Cadence policy -`flush_interval × factor`, capped at 4× (15 s → 60 s at the production cadence): +`flush_interval × factor` and `stats_window × factor`, capped at 4× (15 s → 60 s at the +production cadence). Factors multiply; a change applies at the next tick, and a *shorter* period +applies at once (pressure relieved → no waiting out a stretched period). -| Condition | factor | -|---|---| -| thermal `serious` | 2 | -| thermal `critical` | 4 | -| low-power mode | 2 | -| background | 2 | +| Condition | factor | source | +|---|---|---| +| thermal `serious` | 2 | host, `DeviceState.thermal` | +| thermal `critical` | 4 | host | +| memory pressure `warning` | 2 | host, `DeviceState.memory` | +| memory pressure `critical` | 4 | host | +| low-power mode | 2 | host | +| background | 2 | host | +| battery ≤ 20 % and unplugged | 2 | host, `DeviceState.battery_*` | +| constrained network (Low Data Mode / Data Saver) | 2 | host, `DeviceState.network_constrained` | +| encoder CPU-limited: an outbound track's `qualityLimitationDurations.cpu` grew within the last 60 s | 2 | core, from `record_stats` | + +CPU is never measured by the pipeline itself (measuring CPU costs CPU): thermal state is the OS's +judgement and `qualityLimitationReason` is WebRTC's. `getStats()` polling stays on the SDK's +existing ~1 s timer; the *window* stretches, not the reading. + +## Upload policy — telemetry never wins over media + +Uploads are shaped, not just batched: + +- **One request in flight**, oldest batch first; a failure pauses the cache for 60 s (throttling: + see `lk.telemetry.report`). +- **Budget:** at most `max_batches_per_upload` (default 4) cached batches per tick while a session + may be live, so a backlog (offline period, previous launch) replays at ~4 × 20 KB per 15 s + ≈ 40 kbps next to a call. `shutdown` drains without the budget. +- **Holds** — nothing is sent, everything keeps flowing into the write-ahead cache — while: + - an `lk.connect` or `lk.reconnect` span is open (signaling and ICE/DTLS own the uplink), + - WebRTC reported an outbound track bandwidth-limited within the last 10 s + (`qualityLimitationDurations.bandwidth` grew — the congestion controller is already + holding the encoder back), + - the device asks for quiet: constrained network, or battery ≤ 10 % unplugged (the Datadog + rule). Device holds survive `shutdown`; the other two end with the call. + A hold lasts at most 60 s, then one batch goes out and the hold starts over — the hard cap + that bounds the policy when its signals lie. +- **Bytes:** bodies are gzipped (level 1, `Content-Encoding: gzip`) when cached, so a batch is + 5–10× smaller on disk and on the wire and a replay costs no CPU. +- **Priority hints:** every request carries `Priority: u=7` (RFC 9218, lowest urgency) for + HTTP/2+ hops that implement it, and the host transport marks the local traffic class as + background — Apple `URLSessionConfiguration.networkServiceType = .background` + (`NET_SERVICE_TYPE_BK`, below best effort in the local stack and Wi-Fi AC_BK), browsers + `fetch(…, { priority: "low" })`, Android has no per-request class (WorkManager for deferred + work; `socket.trafficClass` is best-effort). +- **Threads:** pipeline work runs on the SDK's runtime; none of it is on a media or UI thread and + `emit`/`record_stats` never block on it. ## Log records diff --git a/livekit-telemetry/src/device.rs b/livekit-telemetry/src/device.rs index e483351e5..1e4e10da8 100644 --- a/livekit-telemetry/src/device.rs +++ b/livekit-telemetry/src/device.rs @@ -15,7 +15,7 @@ use crate::TelemetryEvent; /// Thermal pressure as the OS reports it (`ProcessInfo.thermalState`, `PowerManager` thermal -/// status). +/// status, the web Compute Pressure API — all four share this vocabulary). #[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum ThermalState { @@ -34,33 +34,108 @@ pub enum AppState { Background, } +/// Memory pressure as the OS reports it (`DISPATCH_MEMORYPRESSURE_*`, `onTrimMemory` levels: +/// `RUNNING_LOW`/`BACKGROUND` → warning, `RUNNING_CRITICAL`/`COMPLETE` → critical). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum MemoryPressure { + #[default] + Normal, + Warning, + Critical, +} + +/// The active network path, in OTel `network.connection.type` terms. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum NetworkType { + #[default] + Unknown, + Wifi, + Cell, + Wired, + Unavailable, +} + /// What the host observes about the device; pushed with -/// [`Telemetry::set_device_state`](crate::Telemetry::set_device_state) whenever it changes. +/// [`Telemetry::set_device_state`](crate::Telemetry::set_device_state) whenever anything changes. /// -/// The host owns the OS APIs (thermal, power, lifecycle) — they are not reachable from Rust -/// without a JVM/ObjC bridge — and the core owns what to do with them: emit the change events -/// from `SPEC.md` and stretch its own cadence so telemetry never competes with the call. +/// The host owns the OS APIs (thermal, power, memory, battery, network path, lifecycle) — they +/// are not reachable from Rust without a JVM/ObjC bridge — and the core owns what to do with +/// them: emit the change events from `SPEC.md`, stretch its own cadence and hold uploads so +/// telemetry never competes with the call. CPU is deliberately not measured here (measuring CPU +/// costs CPU): thermal state is the OS's judgement, and WebRTC's own `qualityLimitationReason` +/// (see [`RtcStatsSample`](crate::RtcStatsSample)) says when the encoder is CPU-starved. #[cfg_attr(feature = "uniffi", derive(uniffi::Record))] -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Debug, Clone, Copy, PartialEq, Default)] pub struct DeviceState { pub thermal: ThermalState, pub low_power_mode: bool, pub app_state: AppState, + // No `uniffi(default)` on enum fields: uniffi 0.31's Swift generator rejects `Default` there. + pub memory: MemoryPressure, + pub network: NetworkType, + /// Cellular / personal hotspot (`NWPath.isExpensive`, metered on Android). + #[cfg_attr(feature = "uniffi", uniffi(default = false))] + pub network_expensive: bool, + /// The user asked for less traffic: Low Data Mode (`NWPath.isConstrained`), Data Saver, + /// `navigator.connection.saveData`. + #[cfg_attr(feature = "uniffi", uniffi(default = false))] + pub network_constrained: bool, + /// Battery charge in percent; `None` when unknown (desktops, tvOS, monitoring disabled). + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub battery_level: Option, + #[cfg_attr(feature = "uniffi", uniffi(default = false))] + pub battery_charging: bool, } impl DeviceState { - /// Multiplier on the flush interval: 1× at rest, up to 4× under pressure (15 s → 60 s at the - /// production cadence, per the design doc). Serious thermal, low-power mode and background - /// each double it; critical thermal quadruples it. + /// Below this, unplugged, the pipeline still records but stops uploading (the Datadog rule). + const BATTERY_HOLD_PERCENT: u32 = 10; + /// Below this, unplugged, the cadence doubles. + const BATTERY_LOW_PERCENT: u32 = 20; + + /// Multiplier on the flush interval and stats window: 1× at rest, up to 4× under pressure + /// (15 s → 60 s at the production cadence, per the design doc). Serious thermal, memory + /// warning, low-power mode, background, low battery and a constrained network each double + /// it; critical thermal or memory quadruple it. pub fn cadence_factor(&self) -> u32 { let thermal = match self.thermal { ThermalState::Nominal | ThermalState::Fair => 1, ThermalState::Serious => 2, ThermalState::Critical => 4, }; - let power = if self.low_power_mode { 2 } else { 1 }; - let background = if self.app_state == AppState::Background { 2 } else { 1 }; - (thermal * power * background).min(4) + let memory = match self.memory { + MemoryPressure::Normal => 1, + MemoryPressure::Warning => 2, + MemoryPressure::Critical => 4, + }; + let doubled = [ + self.low_power_mode, + self.app_state == AppState::Background, + self.battery_bucket() >= 1, + self.network_constrained, + ] + .iter() + .filter(|&&on| on) + .count() as u32; + ((thermal * memory) << doubled).min(4) + } + + /// Uploads should wait: the user asked for less traffic (Low Data Mode / Data Saver) or the + /// battery is nearly empty and unplugged. Data keeps flowing into the cache meanwhile. + pub fn holds_uploads(&self) -> bool { + self.network_constrained || self.battery_bucket() == 2 + } + + /// 0 = fine or unknown, 1 = low (≤ 20 %), 2 = nearly empty (≤ 10 %); always 0 on charge. + /// Per-percent battery updates only matter when they cross one of these. + fn battery_bucket(&self) -> u8 { + match self.battery_level { + Some(level) if !self.battery_charging && level <= Self::BATTERY_HOLD_PERCENT => 2, + Some(level) if !self.battery_charging && level <= Self::BATTERY_LOW_PERCENT => 1, + _ => 0, + } } /// The `SPEC.md` events describing what changed between `previous` and `self` @@ -85,6 +160,40 @@ impl DeviceState { .with_attribute("lk.device.app_state", self.app_state.as_str()), ); } + if previous.is_none_or(|p| p.memory != self.memory) { + events.push( + TelemetryEvent::new("lk.device.memory.changed") + .with_attribute("lk.device.memory.pressure", self.memory.as_str()), + ); + } + if previous.is_none_or(|p| { + (p.network, p.network_expensive, p.network_constrained) + != (self.network, self.network_expensive, self.network_constrained) + }) { + events.push( + TelemetryEvent::new("lk.device.network.changed") + .with_attribute("network.connection.type", self.network.as_str()) + .with_attribute("lk.device.network.expensive", self.network_expensive) + .with_attribute("lk.device.network.constrained", self.network_constrained), + ); + } + // Per-percent updates are noise; report charging flips and bucket crossings. + if self.battery_level.is_some() + && previous.is_none_or(|p| { + p.battery_charging != self.battery_charging + || p.battery_bucket() != self.battery_bucket() + || p.battery_level.is_none() + }) + { + events.push( + TelemetryEvent::new("lk.device.battery.changed") + .with_attribute( + "lk.device.battery.level", + self.battery_level.unwrap_or(0) as i64, + ) + .with_attribute("lk.device.battery.charging", self.battery_charging), + ); + } events } } @@ -109,6 +218,28 @@ impl AppState { } } +impl MemoryPressure { + fn as_str(self) -> &'static str { + match self { + MemoryPressure::Normal => "normal", + MemoryPressure::Warning => "warning", + MemoryPressure::Critical => "critical", + } + } +} + +impl NetworkType { + fn as_str(self) -> &'static str { + match self { + NetworkType::Unknown => "unknown", + NetworkType::Wifi => "wifi", + NetworkType::Cell => "cell", + NetworkType::Wired => "wired", + NetworkType::Unavailable => "unavailable", + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -120,22 +251,66 @@ mod tests { assert_eq!(serious.cadence_factor(), 2); let low_power = DeviceState { low_power_mode: true, ..Default::default() }; assert_eq!(low_power.cadence_factor(), 2); + let tight = DeviceState { memory: MemoryPressure::Warning, ..Default::default() }; + assert_eq!(tight.cadence_factor(), 2); + let low_battery = DeviceState { battery_level: Some(15), ..Default::default() }; + assert_eq!(low_battery.cadence_factor(), 2); + let charging = DeviceState { battery_charging: true, ..low_battery }; + assert_eq!(charging.cadence_factor(), 1, "a charging battery is not low"); let everything = DeviceState { thermal: ThermalState::Critical, low_power_mode: true, app_state: AppState::Background, + memory: MemoryPressure::Critical, + network_constrained: true, + ..Default::default() }; assert_eq!(everything.cadence_factor(), 4); } + #[test] + fn uploads_hold_on_constrained_network_or_empty_battery() { + assert!(!DeviceState::default().holds_uploads()); + let saver = DeviceState { network_constrained: true, ..Default::default() }; + assert!(saver.holds_uploads()); + let empty = DeviceState { battery_level: Some(8), ..Default::default() }; + assert!(empty.holds_uploads()); + let plugged = DeviceState { battery_charging: true, ..empty }; + assert!(!plugged.holds_uploads()); + let low = DeviceState { battery_level: Some(15), ..Default::default() }; + assert!(!low.holds_uploads(), "low only stretches the cadence"); + } + #[test] fn only_changed_fields_produce_events() { let initial = DeviceState::default(); - assert_eq!(initial.change_events(None).len(), 3, "initial value at session start"); + let names: Vec<_> = initial.change_events(None).into_iter().map(|e| e.name).collect(); + assert_eq!( + names, + [ + "lk.device.thermal.changed", + "lk.device.low_power.changed", + "lk.device.app_state.changed", + "lk.device.memory.changed", + "lk.device.network.changed", + ], + "initial values at session start; battery is unknown so it stays silent" + ); let hot = DeviceState { thermal: ThermalState::Critical, ..initial }; let events = hot.change_events(Some(&initial)); assert_eq!(events.len(), 1); assert_eq!(events[0].name, "lk.device.thermal.changed"); assert!(hot.change_events(Some(&hot)).is_empty()); } + + #[test] + fn battery_reports_bucket_crossings_not_every_percent() { + let at = |level| DeviceState { battery_level: Some(level), ..Default::default() }; + assert_eq!(at(80).change_events(Some(&DeviceState::default())).len(), 1, "first reading"); + assert!(at(79).change_events(Some(&at(80))).is_empty()); + assert_eq!(at(20).change_events(Some(&at(21))).len(), 1, "crossed into low"); + assert_eq!(at(10).change_events(Some(&at(11))).len(), 1, "crossed into hold"); + let charging = DeviceState { battery_charging: true, ..at(10) }; + assert_eq!(charging.change_events(Some(&at(10))).len(), 1, "plugged in"); + } } diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index a130f6186..6ae7ac0fa 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -13,15 +13,14 @@ // limitations under the License. use std::{ + io::Write, sync::{Arc, Mutex}, time::Duration, }; -use tokio::time::{interval, sleep, timeout, Instant, MissedTickBehavior}; -use tokio::{ - sync::{mpsc, oneshot}, - time::Interval, -}; +use flate2::{write::GzEncoder, Compression}; +use tokio::sync::{mpsc, oneshot}; +use tokio::time::{sleep, sleep_until, timeout, Instant}; use crate::{ event::now_unix_nanos, @@ -40,6 +39,15 @@ const RETRY_BACKOFF: Duration = Duration::from_secs(1); /// Pause before the cache is tried again after a failed upload (Sentry: stop consuming the /// cache until `Retry-After` elapses; here also for plain connectivity failures). const UPLOAD_BACKOFF: Duration = Duration::from_secs(60); +/// Uploads wait while the call is at its most sensitive (see [`Exporter::hold_reason`]) — but +/// never longer than this before one batch goes out anyway: the hard cap that bounds the policy +/// when its signals lie. +const MAX_HOLD: Duration = Duration::from_secs(60); +/// While one of these is open the uplink belongs to signaling and ICE/DTLS. +const SENSITIVE_SPANS: &[&str] = &["lk.connect", "lk.reconnect"]; +/// RFC 9218 request priority: lowest urgency, not incremental. A hint for HTTP/2+ hops that +/// implement it; the host transport marks the local traffic class (see `TelemetryTransport`). +const PRIORITY: &str = "u=7"; /// Which OTLP signal a cached batch is; picks the endpoint at upload time. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -88,16 +96,22 @@ enum Delivery { /// /// The role of OTel's `BatchLogRecordProcessor` + OTLP exporter in one place. Every tick it /// [`enqueue`](Self::enqueue)s: drains up to `max_batch_size` events, appends an -/// `lk.telemetry.report` when something was dropped or failed since the last one, encodes the -/// batch and writes it to the [`BatchCache`] *before* any network is involved; then it +/// `lk.telemetry.report` when something was dropped or failed since the last one, encodes and +/// gzips the batch and writes it to the [`BatchCache`] *before* any network is involved; then it /// [`upload`](Self::upload)s the cache oldest-first through the [`TelemetryTransport`], /// removing what the collector accepted or rejected. A failed upload pauses the cache for a /// minute; a `Retry-After` additionally drops new batches for its duration (throttling must not /// become a disk-backed queue); `Disabled` empties the cache and silences the exporter for good. /// -/// The tick period is `flush_interval_ms × DeviceState::cadence_factor`: thermal pressure, -/// low-power mode and the background stretch it up to 4×, and entering the background flushes -/// once immediately (the app may be suspended any moment). +/// Telemetry must never win over media, so uploads are shaped as well as batched: at most +/// `max_batches_per_upload` per tick while a session may be live, and none at all while the room +/// is connecting or reconnecting, while WebRTC reports the encoder bandwidth-limited, or while the +/// device asks for quiet ([`DeviceState::holds_uploads`]) — bounded by [`MAX_HOLD`]. Every request +/// carries `Priority: u=7` (RFC 9218) and a gzipped body. +/// +/// The tick period is `flush_interval_ms × cadence factor`: device pressure and a CPU-limited +/// encoder stretch it up to 4×, and entering the background flushes once immediately (the app +/// may be suspended any moment). /// /// Drive it with `spawn(exporter.run())` on the consumer's runtime. It stops after /// [`Telemetry::shutdown`](crate::Telemetry::shutdown) or when the last @@ -124,7 +138,13 @@ pub struct Exporter { cache_failures: u64, /// Counter values at the last `lk.telemetry.report`. last_report: Snapshot, - cadence_factor: u32, + /// Shared with `Telemetry`: read synchronously, so a state pushed right before `emit` already + /// governs the flush that follows. + device: Arc>>, + /// When the current upload hold began, for the [`MAX_HOLD`] cap. + held_since: Option, + /// Shutting down: the call is over, so only the device's own holds still apply. + draining: bool, } impl Exporter { @@ -141,6 +161,7 @@ impl Exporter { spans: Arc>, trace_id: [u8; 16], commands: mpsc::UnboundedReceiver, + device: Arc>>, ) -> Self { Self { store, @@ -159,7 +180,9 @@ impl Exporter { seq: 0, cache_failures: 0, last_report: Snapshot::default(), - cadence_factor: 1, + device, + held_since: None, + draining: false, } } @@ -167,24 +190,30 @@ impl Exporter { /// demand. pub async fn run(mut self) { self.upload().await; - let mut ticker = self.ticker(self.config.flush_interval_ms); - let mut stats_ticker = self.window_ticker(); + // Deadlines rather than tickers: `next = last + period`, re-derived every loop, so a + // cadence change applies to the pending tick in both directions (pressure postpones it, + // relief brings it forward) and a missed tick never bursts. The first flush is immediate, + // the first window closes a full period after it opens. + let mut last_flush = Instant::now() - self.period(self.config.flush_interval_ms); + let mut last_window = Instant::now(); loop { + let next_flush = last_flush + self.period(self.config.flush_interval_ms); + let next_window = last_window + self.period(self.config.stats_window_ms); tokio::select! { - _ = ticker.tick() => self.export_pending().await, - _ = stats_ticker.tick() => self.close_windows(), + _ = sleep_until(next_flush) => { + self.export_pending().await; + last_flush = Instant::now(); + } + _ = sleep_until(next_window) => { + self.close_windows(); + last_window = Instant::now(); + } command = self.commands.recv() => match command { Some(Command::Flush(done)) => { self.export_pending().await; let _ = done.send(()); } Some(Command::DeviceState(state)) => { - let factor = state.cadence_factor(); - if factor != self.cadence_factor { - self.cadence_factor = factor; - ticker = self.ticker(self.config.flush_interval_ms); - stats_ticker = self.window_ticker(); - } if state.app_state == AppState::Background { // The app may be suspended any moment: close the RTC windows and // get everything into the cache (and out, if the network allows). @@ -193,18 +222,13 @@ impl Exporter { } } Some(Command::Shutdown(done)) => { - // Last chance: ignore the upload backoff, but respect throttling. - self.paused_until = self.throttled_until; - self.close_windows(); - self.export_pending().await; + self.drain().await; let _ = done.send(()); return; } // Every `Telemetry` handle is gone. None => { - self.paused_until = self.throttled_until; - self.close_windows(); - self.export_pending().await; + self.drain().await; return; } }, @@ -212,23 +236,30 @@ impl Exporter { } } - /// A ticker at `base_ms × cadence_factor`. - fn ticker(&self, base_ms: u64) -> Interval { - let period = Duration::from_millis(base_ms.max(1)) * self.cadence_factor; - let mut ticker = interval(period); - ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); - ticker + /// `base_ms × cadence factor`. + fn period(&self, base_ms: u64) -> Duration { + Duration::from_millis(base_ms.max(1)) * self.cadence_factor() } - /// The stats-window ticker. Unlike `interval`, whose first tick is immediate, the first window - /// closes a full period after it opens — otherwise readings taken before the first tick - /// would ship as a zero-length window. - fn window_ticker(&self) -> Interval { - let period = - Duration::from_millis(self.config.stats_window_ms.max(1)) * self.cadence_factor; - let mut ticker = tokio::time::interval_at(Instant::now() + period, period); - ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); - ticker + fn device(&self) -> DeviceState { + self.device.lock().unwrap_or_else(|e| e.into_inner()).unwrap_or_default() + } + + /// Device pressure, doubled again while WebRTC reports the encoder CPU-limited; capped at 4×. + fn cadence_factor(&self) -> u32 { + let cpu_limited = + self.windows.lock().unwrap_or_else(|e| e.into_inner()).media_pressure().cpu_limited; + (self.device().cadence_factor() * if cpu_limited { 2 } else { 1 }).min(4) + } + + /// Last chance: everything queued into the cache and out. Ignores the upload backoff, the + /// batch budget and the session holds (the call is over), respects throttling and the + /// device's own holds. + async fn drain(&mut self) { + self.draining = true; + self.paused_until = self.throttled_until; + self.close_windows(); + self.export_pending().await; } /// Turn every open RTC stats window into its `lk.rtc.stats.sample` event. Windows bypass the @@ -302,10 +333,13 @@ impl Exporter { self.push_batch(Signal::Traces, count, &body); } + /// Gzip the encoded batch and cache it. Compressed at rest as well as on the wire: the cache + /// holds 5–10× more, the disk write shrinks, and a replay costs no CPU. fn push_batch(&mut self, signal: Signal, count: u64, body: &[u8]) { + let body = gzip(body); self.seq += 1; let id = format!("{:020}-{:06}-{count}-{}", now_unix_nanos(), self.seq, signal.tag()); - if let Err(err) = self.cache.push(&id, body) { + if let Err(err) = self.cache.push(&id, &body) { // A full disk is a steady state, not an event: warn once, then stay quiet. if self.cache_failures == 0 { log::warn!("telemetry: cannot cache batches ({err}); dropping until it recovers"); @@ -327,12 +361,57 @@ impl Exporter { } } - /// Send cached batches oldest-first until one fails; then back off. + /// Why uploads should wait right now, if they should. Data keeps flowing into the cache + /// meanwhile — write-ahead caching is what makes holding free. + fn hold_reason(&self) -> Option<&'static str> { + if self.device().holds_uploads() { + return Some("device asks for quiet"); + } + if self.draining { + return None; + } + if self.spans.lock().unwrap_or_else(|e| e.into_inner()).any_open(SENSITIVE_SPANS) { + return Some("connecting"); + } + let media = self.windows.lock().unwrap_or_else(|e| e.into_inner()).media_pressure(); + if media.bandwidth_limited { + return Some("media is bandwidth-limited"); + } + None + } + + /// Send cached batches oldest-first, within this tick's budget, until one fails; then back + /// off. async fn upload(&mut self) { if self.silenced || self.paused_until.is_some_and(|t| Instant::now() < t) { return; } - for id in self.cache.pending() { + let pending = self.cache.pending(); + if pending.is_empty() { + self.held_since = None; + return; + } + let budget = match self.hold_reason() { + Some(reason) => { + let since = *self.held_since.get_or_insert_with(Instant::now); + if since.elapsed() < MAX_HOLD { + log::trace!("telemetry: holding {} batches: {reason}", pending.len()); + return; + } + // Held long enough: one batch goes out, then the hold starts over. + self.held_since = Some(Instant::now()); + 1 + } + None => { + self.held_since = None; + if self.draining { + usize::MAX + } else { + self.config.max_batches_per_upload.max(1) as usize + } + } + }; + for id in pending.into_iter().take(budget) { let Some(body) = self.cache.read(&id) else { self.cache.remove(&id); continue; @@ -373,10 +452,12 @@ impl Exporter { self.cache.clear(); } - /// Upload one encoded batch with bounded retries and classify the outcome. + /// Upload one cached (gzipped) batch with bounded retries and classify the outcome. async fn deliver(&self, body: &[u8], signal: Signal) -> Delivery { let mut headers = self.config.headers.clone(); headers.insert("Content-Type".to_owned(), otlp::CONTENT_TYPE.to_owned()); + headers.insert("Content-Encoding".to_owned(), "gzip".to_owned()); + headers.insert("Priority".to_owned(), PRIORITY.to_owned()); let url = match signal { Signal::Logs => self.config.endpoint.clone(), Signal::Traces => { @@ -417,6 +498,15 @@ impl Exporter { } } +/// Level 1: protobuf with repeated attribute keys shrinks 5–10× already; higher levels buy little +/// for more CPU. +fn gzip(body: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::with_capacity(body.len() / 4), Compression::fast()); + // Writing into a Vec cannot fail. + let _ = encoder.write_all(body); + encoder.finish().unwrap_or_default() +} + /// The event count the exporter encodes as the last component of a batch id. fn events_in(id: &str) -> u64 { id.split('-').nth(2).and_then(|n| n.parse().ok()).unwrap_or(0) diff --git a/livekit-telemetry/src/rtc.rs b/livekit-telemetry/src/rtc.rs index ebb19102c..6fc7a94c2 100644 --- a/livekit-telemetry/src/rtc.rs +++ b/livekit-telemetry/src/rtc.rs @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; +use std::{collections::HashMap, time::Duration}; + +use tokio::time::Instant; use crate::{event::now_unix_nanos, TelemetryEvent}; @@ -230,10 +232,32 @@ impl Window { #[derive(Default)] pub(crate) struct StatsWindows { windows: HashMap<(String, StreamDirection), Window>, + /// Last cumulative `qualityLimitationDurations` per outbound track: (bandwidth_ms, cpu_ms). + // ponytail: grows with the tracks published in a session (a few dozen at most). + limitation: HashMap, + bandwidth_limited_until: Option, + cpu_limited_until: Option, +} + +/// What WebRTC's own adaptation says about the call, derived from the +/// `qualityLimitationDurations` counters of outbound tracks — no extra measurement, and exactly +/// the cpu-vs-bandwidth split the design doc asks for. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct MediaPressure { + /// The congestion controller is holding the encoder back: uploads must not add to the uplink. + pub bandwidth_limited: bool, + /// The encoder is CPU-starved: stretch the cadence, like thermal pressure. + pub cpu_limited: bool, } +/// Congestion is bursty: hold uploads for a short while after the last sign of it. +const BANDWIDTH_LIMITED_HOLD: Duration = Duration::from_secs(10); +/// CPU starvation is sticky: stretch the cadence for a while after the last sign of it. +const CPU_LIMITED_HOLD: Duration = Duration::from_secs(60); + impl StatsWindows { pub fn record(&mut self, mut sample: RtcStatsSample) { + self.track_limitation(&sample); let timestamp = *sample.timestamp_ns.get_or_insert_with(now_unix_nanos); let key = (sample.track_sid.clone(), sample.direction); match self.windows.get_mut(&key) { @@ -249,6 +273,35 @@ impl StatsWindows { let end = now_unix_nanos(); self.windows.drain().map(|(_, window)| window.into_event(end)).collect() } + + pub fn media_pressure(&self) -> MediaPressure { + let now = Instant::now(); + MediaPressure { + bandwidth_limited: self.bandwidth_limited_until.is_some_and(|t| now < t), + cpu_limited: self.cpu_limited_until.is_some_and(|t| now < t), + } + } + + /// A limitation counter that grew since the previous reading means the encoder was held + /// back in between. + fn track_limitation(&mut self, sample: &RtcStatsSample) { + if sample.direction != StreamDirection::Outbound { + return; + } + let current = ( + sample.quality_limitation_bandwidth_ms.unwrap_or(0), + sample.quality_limitation_cpu_ms.unwrap_or(0), + ); + if let Some(previous) = self.limitation.insert(sample.track_sid.clone(), current) { + let now = Instant::now(); + if current.0 > previous.0 { + self.bandwidth_limited_until = Some(now + BANDWIDTH_LIMITED_HOLD); + } + if current.1 > previous.1 { + self.cpu_limited_until = Some(now + CPU_LIMITED_HOLD); + } + } + } } impl TrackKind { @@ -272,6 +325,32 @@ impl StreamDirection { #[cfg(test)] mod tests { use super::*; + + #[tokio::test(start_paused = true)] + async fn limitation_counters_drive_media_pressure() { + let mut windows = StatsWindows::default(); + let reading = |bandwidth_ms, cpu_ms| RtcStatsSample { + quality_limitation_bandwidth_ms: Some(bandwidth_ms), + quality_limitation_cpu_ms: Some(cpu_ms), + ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound) + }; + windows.record(reading(0, 0)); + assert_eq!(windows.media_pressure(), MediaPressure::default(), "first reading: no delta"); + windows.record(reading(500, 0)); + assert!(windows.media_pressure().bandwidth_limited); + assert!(!windows.media_pressure().cpu_limited); + tokio::time::advance(Duration::from_secs(11)).await; + assert!(!windows.media_pressure().bandwidth_limited, "congestion hold expires"); + windows.record(reading(500, 250)); + assert!(windows.media_pressure().cpu_limited); + tokio::time::advance(Duration::from_secs(59)).await; + assert!(windows.media_pressure().cpu_limited, "cpu hold is sticky"); + let mut inbound = RtcStatsSample::new("TR_2", TrackKind::Audio, StreamDirection::Inbound); + inbound.quality_limitation_bandwidth_ms = Some(9_999); + windows.record(inbound.clone()); + windows.record(inbound); + assert!(!windows.media_pressure().bandwidth_limited, "inbound counters are ignored"); + } use crate::AttributeValue; fn attr(event: &TelemetryEvent, key: &str) -> Option { diff --git a/livekit-telemetry/src/span.rs b/livekit-telemetry/src/span.rs index 65952c72c..fa1612d0c 100644 --- a/livekit-telemetry/src/span.rs +++ b/livekit-telemetry/src/span.rs @@ -133,6 +133,12 @@ impl Spans { id } + /// Whether a span with one of these names is still open (the exporter holds uploads while + /// `lk.connect` / `lk.reconnect` are). + pub fn any_open(&self, names: &[&str]) -> bool { + self.open.values().any(|span| names.contains(&span.name.as_str())) + } + pub fn add_event(&mut self, id: u64, name: &str, attributes: Vec) { let Some(span) = self.open.get_mut(&id) else { return }; if span.events.len() >= MAX_EVENTS_PER_SPAN { diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 8853dfd0c..48526e6a1 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -74,6 +74,11 @@ pub struct TelemetryConfig { /// dropped and counted as `rate_limited`. RTC windows and self-telemetry are exempt; 0 = off. #[cfg_attr(feature = "uniffi", uniffi(default = 300))] pub max_events_per_10min: u32, + /// Cached batches uploaded per tick while a session may be live — bounds how fast a backlog + /// (offline period, previous launch) replays next to a call: 4 × ~20 KB gzipped per 15 s is + /// ~40 kbps. `shutdown` drains without the budget. + #[cfg_attr(feature = "uniffi", uniffi(default = 4))] + pub max_batches_per_upload: u32, } impl TelemetryConfig { @@ -92,6 +97,7 @@ impl TelemetryConfig { export_timeout_ms: 10_000, stats_window_ms: 15_000, max_events_per_10min: 300, + max_batches_per_upload: 4, } } } @@ -196,10 +202,15 @@ impl Telemetry { if config.traces_endpoint.is_none() { config.traces_endpoint = Some(derive_traces_endpoint(&config.endpoint)); } - let config = Arc::new(config); // The session is the trace: one id per pipeline, client-generated so pre-connect - // failures and reconnects share it. + // failures and reconnects share it — and, as `session.id` (OTel semconv), the resource + // attribute every record carries. let trace_id = rand::random::().max(1).to_be_bytes(); + if !config.resource.iter().any(|a| a.key == "session.id") { + let hex = format!("{:032x}", u128::from_be_bytes(trace_id)); + config.resource.push(Attribute::new("session.id", hex)); + } + let config = Arc::new(config); let counters = Arc::new(Counters::default()); let store = Arc::new(Store::new(config.max_queue_size.max(1) as usize, counters.clone())); let (commands, receiver) = mpsc::unbounded_channel(); @@ -207,6 +218,7 @@ impl Telemetry { let guard = Arc::new(Mutex::new(FloodGuard::new(config.max_events_per_10min))); let attributes = Arc::new(Mutex::new(Vec::new())); let spans = Arc::new(Mutex::new(Spans::new(config.max_queue_size.max(1) as usize))); + let device = Arc::new(Mutex::new(None)); let exporter = Exporter::new( store.clone(), transport, @@ -218,13 +230,14 @@ impl Telemetry { spans.clone(), trace_id, receiver, + device.clone(), ); let telemetry = Self { store, config, cache, counters, - device: Arc::default(), + device, windows, guard, attributes, @@ -307,9 +320,10 @@ impl Telemetry { } /// Tell the pipeline what the device looks like. Emits the `lk.device.*.changed` events for - /// whatever differs from the last state (everything, the first time) and re-tunes the export - /// cadence: thermal pressure, low-power mode and the background stretch it up to 4×; entering - /// the background also flushes once right away. + /// whatever differs from the last state (everything, the first time) and re-tunes the + /// pipeline: pressure stretches the cadence up to 4× ([`DeviceState::cadence_factor`]), a + /// constrained network or a nearly empty battery holds uploads + /// ([`DeviceState::holds_uploads`]), and entering the background flushes once right away. pub fn set_device_state(&self, state: DeviceState) { let mut previous = self.device.lock().unwrap_or_else(|e| e.into_inner()); for event in state.change_events(previous.as_ref()) { @@ -414,6 +428,13 @@ mod tests { } } + fn gunzip(body: &[u8]) -> Vec { + use std::io::Read; + let mut out = Vec::new(); + flate2::read::GzDecoder::new(body).read_to_end(&mut out).expect("gzip body"); + out + } + fn offline() -> Result<(), ExportError> { Err(ExportError::Retryable { message: "offline".into(), retry_after_ms: None }) } @@ -443,7 +464,8 @@ mod tests { } fn records(request: &ExportRequest) -> Vec { - let decoded = ExportLogsServiceRequest::decode(&request.body[..]).expect("valid OTLP"); + let decoded = + ExportLogsServiceRequest::decode(&gunzip(&request.body)[..]).expect("valid OTLP"); decoded.resource_logs[0].scope_logs[0].log_records.clone() } @@ -469,7 +491,8 @@ mod tests { assert_eq!(sent[0].url, "http://collector/v1/logs"); assert_eq!(sent[0].headers["Content-Type"], "application/x-protobuf"); assert_eq!(event_names(&sent[0]), ["lk.ping"; 3]); - let decoded = ExportLogsServiceRequest::decode(&sent[0].body[..]).expect("valid OTLP"); + let decoded = + ExportLogsServiceRequest::decode(&gunzip(&sent[0].body)[..]).expect("valid OTLP"); let resource = decoded.resource_logs[0].resource.as_ref().expect("resource"); assert!(resource.attributes.iter().any(|kv| kv.key == "telemetry.sdk.name")); assert_eq!(telemetry.stats().dropped, 0); @@ -647,7 +670,7 @@ mod tests { assert_eq!(sent.len(), 1); let names = event_names(&sent[0]); assert!(names.contains(&"lk.device.thermal.changed".to_owned()), "{names:?}"); - assert_eq!(names.len(), 3, "initial value for every field"); + assert_eq!(names.len(), 5, "initial value for every known field (battery unknown)"); let thermal = records(&sent[0]) .into_iter() .find(|r| r.event_name == "lk.device.thermal.changed") @@ -665,6 +688,81 @@ mod tests { assert_eq!(transport.sent().len(), 2, "exported on the stretched tick"); } + #[tokio::test(start_paused = true)] + async fn requests_are_gzipped_and_low_priority() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + let sent = transport.sent(); + assert_eq!(sent[0].headers["Content-Encoding"], "gzip"); + assert_eq!(sent[0].headers["Priority"], "u=7"); + assert_eq!(event_names(&sent[0]), ["lk.ping"]); + } + + #[tokio::test(start_paused = true)] + async fn uploads_hold_while_connecting_but_never_beyond_the_cap() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + let connect = telemetry.begin_span("lk.connect", SpanKind::Client, None); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + assert!(transport.sent().is_empty(), "the uplink belongs to signaling and ICE"); + assert_eq!(telemetry.stats().cached_batches, 1, "…but the batch is safely cached"); + + tokio::time::sleep(Duration::from_secs(61)).await; + telemetry.flush().await; + assert_eq!(transport.sent().len(), 1, "held 60 s: one batch goes out regardless"); + + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.end_span(connect, SpanOutcome::Ok, None, Vec::new()); + telemetry.flush().await; + assert_eq!(transport.sent().len(), 3, "connected: the ping and the connect span ship"); + } + + #[tokio::test(start_paused = true)] + async fn uploads_hold_while_media_is_bandwidth_limited() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + let limited = |ms| RtcStatsSample { + quality_limitation_bandwidth_ms: Some(ms), + ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound) + }; + telemetry.record_stats(limited(0)); + telemetry.record_stats(limited(800)); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + assert!(transport.sent().is_empty(), "the congestion controller is already throttling"); + + tokio::time::sleep(Duration::from_secs(11)).await; + telemetry.flush().await; + assert_eq!(transport.sent().len(), 1, "10 s without new limitation: resume"); + } + + #[tokio::test(start_paused = true)] + async fn device_holds_uploads_and_a_backlog_replays_within_the_budget() { + let transport = FakeTransport::scripted([]); + let mut config = TelemetryConfig::new("http://collector/v1/logs"); + config.max_batches_per_upload = 2; + let telemetry = start(config, transport.clone()); + telemetry.set_device_state(DeviceState { network_constrained: true, ..Default::default() }); + for _ in 0..5 { + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + } + assert!(transport.sent().is_empty(), "Low Data Mode: record, do not upload"); + assert_eq!(telemetry.stats().cached_batches, 5); + + // Back to normal (the change event itself makes a sixth batch). + telemetry.set_device_state(DeviceState::default()); + telemetry.flush().await; + assert_eq!(transport.sent().len(), 2, "two batches per tick next to a live call"); + telemetry.flush().await; + assert_eq!(transport.sent().len(), 4); + telemetry.shutdown().await; + assert_eq!(transport.sent().len(), 6, "shutdown drains without the budget"); + } + #[tokio::test(start_paused = true)] async fn debug_and_info_logs_never_leave_the_device() { let transport = FakeTransport::scripted([]); @@ -806,7 +904,8 @@ mod tests { traces.url, "http://c/observability/traces/otlp/v0", "derived from logs endpoint" ); - let decoded = ExportTraceServiceRequest::decode(&traces.body[..]).expect("valid OTLP"); + let decoded = + ExportTraceServiceRequest::decode(&gunzip(&traces.body)[..]).expect("valid OTLP"); let otlp_span = &decoded.resource_spans[0].scope_spans[0].spans[0]; assert_eq!(otlp_span.name, "lk.connect"); assert_eq!(otlp_span.kind, span::SpanKind::Client as i32); @@ -850,7 +949,8 @@ mod tests { telemetry.flush().await; let sent = transport.sent(); assert_eq!(sent[0].url, "http://collector/v1/traces"); - let decoded = ExportTraceServiceRequest::decode(&sent[0].body[..]).expect("valid OTLP"); + let decoded = + ExportTraceServiceRequest::decode(&gunzip(&sent[0].body)[..]).expect("valid OTLP"); let otlp_span = &decoded.resource_spans[0].scope_spans[0].spans[0]; assert_eq!( otlp_span.status.as_ref().map(|s| s.code), From 9fc454958177a9907679f1136ff4c9017c5310ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:33:55 +0200 Subject: [PATCH 12/50] feat(telemetry): byte-bounded batches, custom events, fuller self-diagnostics The queue tracks its approximate size: crossing `flush_threshold_bytes` (256 KiB) wakes the exporter instead of waiting for the tick, and a request never carries more than `max_batch_bytes` (1 MiB before compression) - the design doc's "flush every 15 s or at 256 KB; single POST <= 1 MB". Both go through the same budget and holds, so an early flush is never a burst. `emit_custom(name, attributes)` is the stringly-typed escape hatch next to the `lk.*` catalogue: events ship as `custom.`, so they can neither collide with nor spoof SDK events and the backend can quota the namespace. Self-diagnostics audit: uploads now report bytes on the wire and split timeouts from other failures; cache eviction (max size / age) is counted as `dropped.cache_full` instead of being silent (`BatchCache::push` returns the evicted ids); holds that hit the 60 s cap are counted; and one `lk.telemetry.report` is always emitted at shutdown as the session summary, so healthy sessions contribute denominators too. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/SPEC.md | 31 ++++-- livekit-telemetry/src/cache.rs | 27 ++++-- livekit-telemetry/src/event.rs | 20 ++++ livekit-telemetry/src/exporter.rs | 58 +++++++++--- livekit-telemetry/src/stats.rs | 88 ++++++++++++----- livekit-telemetry/src/store.rs | 70 +++++++++++--- livekit-telemetry/src/telemetry.rs | 147 ++++++++++++++++++++++++++++- livekit-uniffi/src/telemetry.rs | 5 + 8 files changed, 375 insertions(+), 71 deletions(-) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index 3773431c0..3f0078c3b 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -34,16 +34,23 @@ event: lk.telemetry.report area: sdk (self-telemetry) severity: info attributes: - lk.telemetry.uploads.failed: int # failed upload attempts since the previous report lk.telemetry.uploads.sent: int # batches accepted since the previous report + lk.telemetry.uploads.bytes: int # compressed bytes accepted — what telemetry cost the uplink + lk.telemetry.uploads.failed: int # attempts that failed transiently (network error, 5xx) + lk.telemetry.uploads.timeouts: int # attempts that hit export_timeout_ms (omitted when 0) lk.telemetry.cache.batches: int # batches waiting in the cache right now + lk.telemetry.holds.capped: int # upload holds that reached the 60 s cap (data ≥ 1 min late) lk.telemetry.dropped.queue_full: int # events evicted from the in-memory queue (omitted when 0) - lk.telemetry.dropped.cache_error: int # events lost because the cache could not store them + lk.telemetry.dropped.cache_error: int # events lost because the cache could not store them (disk full) + lk.telemetry.dropped.cache_full: int # events evicted from the cache by max_cache_bytes / max age lk.telemetry.dropped.rejected: int # events the collector rejected (4xx) lk.telemetry.dropped.throttled: int # events dropped inside a Retry-After window -cadence: appended to the next batch whenever a drop or upload failure happened since the - previous report — never its own request, never persisted on its own (Sentry client - report shape; reasons follow the OTel SDK self-metrics `error.type` values) + lk.telemetry.dropped.rate_limited: int # discrete events dropped by the flood guard +cadence: appended to the next batch whenever a drop, an upload failure or a capped hold happened + since the previous report — never its own request, never persisted on its own (Sentry + client report shape; reasons follow the OTel SDK self-metrics `error.type` values) — + and once at shutdown as the session summary, so fleet-wide success rates have + denominators. Never emitted after the collector disabled telemetry. platforms: all ``` @@ -167,7 +174,10 @@ Uploads are shaped, not just batched: A hold lasts at most 60 s, then one batch goes out and the hold starts over — the hard cap that bounds the policy when its signals lie. - **Bytes:** bodies are gzipped (level 1, `Content-Encoding: gzip`) when cached, so a batch is - 5–10× smaller on disk and on the wire and a replay costs no CPU. + 5–10× smaller on disk and on the wire and a replay costs no CPU. A request never carries more + than `max_batch_bytes` (1 MiB, estimated before compression) or `max_batch_size` (512) records; + when the queue reaches `flush_threshold_bytes` (256 KiB) it is exported at once instead of at + the next tick — "every 15 s or at 256 KB". - **Priority hints:** every request carries `Priority: u=7` (RFC 9218, lowest urgency) for HTTP/2+ hops that implement it, and the host transport marks the local traffic class as background — Apple `URLSessionConfiguration.networkServiceType = .background` @@ -184,6 +194,15 @@ A `TelemetryEvent` with an empty `name` is a plain log record (OTLP log without `code.line.number`, `lk.log.type` (the SDK logger's category). Only `warn` and `error` records leave the device; `trace`/`debug`/`info` are dropped in `emit`. +## Custom events + +Consumers — apps, or an SDK's platform-specific extras — emit their own events with +`emit_custom(name, attributes)` (Swift: `room.emitTelemetryEvent(_:attributes:)`). The core +prefixes the name with `custom.` (`acme.checkout` → `custom.acme.checkout`), so a custom event +can never collide with, or spoof, an `lk.*` event and the backend can filter or quota the +namespace as a whole. Attributes keep the caller's namespace. Severity is `info`; custom events +count against the flood guard like any discrete event. + ## Flood guard Discrete events (`emit`) are capped at `max_events_per_10min` (default 300, design doc); what diff --git a/livekit-telemetry/src/cache.rs b/livekit-telemetry/src/cache.rs index aaeba6248..db3cf8b4c 100644 --- a/livekit-telemetry/src/cache.rs +++ b/livekit-telemetry/src/cache.rs @@ -31,8 +31,9 @@ use std::{ /// implementation can expire old batches without touching file timestamps. Implementations /// bound their own footprint by evicting the oldest batches; the exporter never sees eviction. pub trait BatchCache: Send + Sync { - /// Store one encoded batch under `id`. - fn push(&self, id: &str, body: &[u8]) -> io::Result<()>; + /// Store one encoded batch under `id`. Returns the ids of older batches evicted to make + /// room (the exporter counts their events as dropped). + fn push(&self, id: &str, body: &[u8]) -> io::Result>; /// Ids of stored batches, oldest first. fn pending(&self) -> Vec; /// The body stored under `id`, if it is still there. @@ -65,14 +66,17 @@ impl MemoryCache { } impl BatchCache for MemoryCache { - fn push(&self, id: &str, body: &[u8]) -> io::Result<()> { + fn push(&self, id: &str, body: &[u8]) -> io::Result> { let mut batches = self.lock(); batches.push((id.to_owned(), body.to_vec())); let mut total: usize = batches.iter().map(|(_, b)| b.len()).sum(); + let mut evicted = Vec::new(); while total > self.max_bytes && batches.len() > 1 { - total -= batches.remove(0).1.len(); + let (id, body) = batches.remove(0); + total -= body.len(); + evicted.push(id); } - Ok(()) + Ok(evicted) } fn pending(&self) -> Vec { @@ -145,14 +149,18 @@ impl FileCache { } /// Delete stray `.tmp` files and batches older than the max age, then the oldest batches - /// until the total fits `max_bytes`. - fn prune(&self) -> io::Result<()> { + /// until the total fits `max_bytes`. Returns the ids of the batches removed. + fn prune(&self) -> io::Result> { let now = crate::event::now_unix_nanos(); + let mut removed = Vec::new(); for entry in fs::read_dir(&self.dir)?.flatten() { let path = entry.path(); let expired = stamp(&path).is_some_and(|t| now.saturating_sub(t) > MAX_AGE_NANOS); if !is_batch(&path) || expired { let _ = fs::remove_file(&path); + if expired { + removed.extend(path.file_stem().and_then(|s| s.to_str()).map(str::to_owned)); + } } } let kept = self.pending(); @@ -167,15 +175,16 @@ impl FileCache { } self.remove(id); total -= len; + removed.push(id.clone()); } - Ok(()) + Ok(removed) } } impl BatchCache for FileCache { /// Fail-open: a full disk or a purged directory yields an error the exporter counts as a /// drop; it never leaves a partial `.tmp` behind. - fn push(&self, id: &str, body: &[u8]) -> io::Result<()> { + fn push(&self, id: &str, body: &[u8]) -> io::Result> { let tmp = self.path(id, "tmp"); let written = self.write_then_rename(&tmp, &self.path(id, EXT), body); if written.is_err() { diff --git a/livekit-telemetry/src/event.rs b/livekit-telemetry/src/event.rs index 4d742dcdf..c7c356ebd 100644 --- a/livekit-telemetry/src/event.rs +++ b/livekit-telemetry/src/event.rs @@ -74,6 +74,26 @@ impl TelemetryEvent { self.attributes.push(Attribute::new(key, value)); self } + + /// A consumer's own event. Always namespaced under `custom.` so it can never be mistaken for + /// a LiveKit-defined `lk.*` event, and the backend can filter or quota it separately; + /// attributes keep the caller's namespace (`acme.checkout.step`). + pub fn custom(name: &str, attributes: Vec) -> Self { + let name = format!("custom.{}", name.trim_start_matches("custom.")); + Self { attributes, ..Self::new(name) } + } + + /// Rough encoded size — strings plus a fixed overhead per field. Drives the byte bounds on + /// queue flushing and request size; cheaper than encoding and close enough for both. + pub fn size_hint(&self) -> usize { + let value = |v: &AttributeValue| match v { + AttributeValue::Str(s) => s.len(), + _ => 8, + }; + 32 + self.name.len() + + self.body.as_ref().map_or(0, String::len) + + self.attributes.iter().map(|a| 4 + a.key.len() + value(&a.value)).sum::() + } } /// Event severity, mapped onto the OTel severity numbers (`TRACE`=1 … `ERROR`=17). diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index 6ae7ac0fa..4e9f0a163 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -77,6 +77,8 @@ pub(crate) enum Command { Flush(oneshot::Sender<()>), Shutdown(oneshot::Sender<()>), DeviceState(DeviceState), + /// The queue crossed `flush_threshold_bytes`: export now rather than at the next tick. + Overflow, } /// Outcome of uploading one encoded batch. @@ -145,6 +147,9 @@ pub struct Exporter { held_since: Option, /// Shutting down: the call is over, so only the device's own holds still apply. draining: bool, + /// Append an `lk.telemetry.report` to the next batch even if nothing went wrong (the + /// shutdown summary). + force_report: bool, } impl Exporter { @@ -183,6 +188,7 @@ impl Exporter { device, held_since: None, draining: false, + force_report: false, } } @@ -213,6 +219,7 @@ impl Exporter { self.export_pending().await; let _ = done.send(()); } + Some(Command::Overflow) => self.export_pending().await, Some(Command::DeviceState(state)) => { if state.app_state == AppState::Background { // The app may be suspended any moment: close the RTC windows and @@ -257,9 +264,14 @@ impl Exporter { /// device's own holds. async fn drain(&mut self) { self.draining = true; + self.force_report = true; self.paused_until = self.throttled_until; self.close_windows(); self.export_pending().await; + let left = self.cache.pending().len(); + if left > 0 { + log::debug!("telemetry: {left} batches still cached at shutdown (replayed next start)"); + } } /// Turn every open RTC stats window into its `lk.rtc.stats.sample` event. Windows bypass the @@ -280,25 +292,32 @@ impl Exporter { fn enqueue(&mut self) { self.enqueue_spans(); loop { - let mut batch = self.store.drain(self.config.max_batch_size.max(1) as usize); - if batch.is_empty() { + let mut batch = self.store.drain( + self.config.max_batch_size.max(1) as usize, + usize::try_from(self.config.max_batch_bytes.max(1)).unwrap_or(usize::MAX), + ); + let throttled = self.throttled_until.is_some_and(|t| Instant::now() < t); + // The shutdown summary goes out even with nothing else queued. + let report_due = self.force_report && !self.silenced && !throttled; + if batch.is_empty() && !report_due { return; } if self.silenced { Counters::add(&self.counters.disabled, batch.len() as u64); continue; } - if self.throttled_until.is_some_and(|t| Instant::now() < t) { + if throttled { Counters::add(&self.counters.throttled, batch.len() as u64); continue; } // Self-telemetry rides along with real data: never its own request, never its own - // cadence, and only when there is something to report. + // cadence, and only when there is something to report — plus once at shutdown. let now = self.counters.snapshot(); let delta = now.since(&self.last_report); - if delta.has_problems() { + if delta.has_problems() || report_due { batch.push(delta.report(self.cache.pending().len() as u64)); self.last_report = now; + self.force_report = false; } for event in &mut batch { self.attach_session_attributes(&mut event.attributes); @@ -339,15 +358,24 @@ impl Exporter { let body = gzip(body); self.seq += 1; let id = format!("{:020}-{:06}-{count}-{}", now_unix_nanos(), self.seq, signal.tag()); - if let Err(err) = self.cache.push(&id, &body) { - // A full disk is a steady state, not an event: warn once, then stay quiet. - if self.cache_failures == 0 { - log::warn!("telemetry: cannot cache batches ({err}); dropping until it recovers"); - } else { - log::debug!("telemetry: could not cache {count} items: {err}"); + match self.cache.push(&id, &body) { + Ok(evicted) => { + // Older batches pushed out by `max_cache_bytes`: lost, but counted. + let lost: u64 = evicted.iter().map(|id| events_in(id)).sum(); + Counters::add(&self.counters.cache_full, lost); + } + Err(err) => { + // A full disk is a steady state, not an event: warn once, then stay quiet. + if self.cache_failures == 0 { + log::warn!( + "telemetry: cannot cache batches ({err}); dropping until it recovers" + ); + } else { + log::debug!("telemetry: could not cache {count} items: {err}"); + } + self.cache_failures += 1; + Counters::add(&self.counters.cache_error, count); } - self.cache_failures += 1; - Counters::add(&self.counters.cache_error, count); } } @@ -400,6 +428,7 @@ impl Exporter { } // Held long enough: one batch goes out, then the hold starts over. self.held_since = Some(Instant::now()); + Counters::add(&self.counters.hold_cap_hits, 1); 1 } None => { @@ -420,6 +449,7 @@ impl Exporter { Delivery::Sent => { self.cache.remove(&id); Counters::add(&self.counters.uploads_sent, 1); + Counters::add(&self.counters.upload_bytes, body.len() as u64); } Delivery::Rejected => { self.cache.remove(&id); @@ -487,7 +517,7 @@ impl Exporter { } Err(_) => { log::debug!("telemetry upload timed out (attempt {})", attempt + 1); - Counters::add(&self.counters.upload_failures, 1); + Counters::add(&self.counters.upload_timeouts, 1); } } if attempt < MAX_RETRIES { diff --git a/livekit-telemetry/src/stats.rs b/livekit-telemetry/src/stats.rs index 37e652028..9a6087462 100644 --- a/livekit-telemetry/src/stats.rs +++ b/livekit-telemetry/src/stats.rs @@ -27,6 +27,8 @@ pub(crate) struct Counters { pub queue_full: AtomicU64, /// Events lost because the cache could not store their batch (disk full, unusable dir). pub cache_error: AtomicU64, + /// Events evicted from the cache to stay under `max_cache_bytes` (or past the max age). + pub cache_full: AtomicU64, /// Events the collector rejected (4xx). pub rejected: AtomicU64, /// Events dropped inside a `Retry-After` window. @@ -37,8 +39,15 @@ pub(crate) struct Counters { pub rate_limited: AtomicU64, /// Batches the collector accepted. pub uploads_sent: AtomicU64, - /// Upload attempts that failed transiently (network, timeout, 5xx). + /// Compressed bytes the collector accepted — what telemetry actually cost the uplink. + pub upload_bytes: AtomicU64, + /// Upload attempts that failed transiently (network error, 5xx). pub upload_failures: AtomicU64, + /// Upload attempts that hit `export_timeout_ms` (a slow network, or a stalled collector). + pub upload_timeouts: AtomicU64, + /// Upload holds that reached the 60 s cap and let one batch through: the policy was + /// starving telemetry, and data arrived at least a minute late. + pub hold_cap_hits: AtomicU64, } impl Counters { @@ -51,12 +60,16 @@ impl Counters { Snapshot { queue_full: get(&self.queue_full), cache_error: get(&self.cache_error), + cache_full: get(&self.cache_full), rejected: get(&self.rejected), throttled: get(&self.throttled), disabled: get(&self.disabled), rate_limited: get(&self.rate_limited), uploads_sent: get(&self.uploads_sent), + upload_bytes: get(&self.upload_bytes), upload_failures: get(&self.upload_failures), + upload_timeouts: get(&self.upload_timeouts), + hold_cap_hits: get(&self.hold_cap_hits), } } } @@ -66,51 +79,76 @@ impl Counters { pub(crate) struct Snapshot { pub queue_full: u64, pub cache_error: u64, + pub cache_full: u64, pub rejected: u64, pub throttled: u64, pub disabled: u64, pub rate_limited: u64, pub uploads_sent: u64, + pub upload_bytes: u64, pub upload_failures: u64, + pub upload_timeouts: u64, + pub hold_cap_hits: u64, } impl Snapshot { /// Counts accumulated since `earlier`. pub fn since(&self, earlier: &Snapshot) -> Snapshot { + let d = |a: u64, b: u64| a.saturating_sub(b); Snapshot { - queue_full: self.queue_full.saturating_sub(earlier.queue_full), - cache_error: self.cache_error.saturating_sub(earlier.cache_error), - rejected: self.rejected.saturating_sub(earlier.rejected), - throttled: self.throttled.saturating_sub(earlier.throttled), - disabled: self.disabled.saturating_sub(earlier.disabled), - rate_limited: self.rate_limited.saturating_sub(earlier.rate_limited), - uploads_sent: self.uploads_sent.saturating_sub(earlier.uploads_sent), - upload_failures: self.upload_failures.saturating_sub(earlier.upload_failures), + queue_full: d(self.queue_full, earlier.queue_full), + cache_error: d(self.cache_error, earlier.cache_error), + cache_full: d(self.cache_full, earlier.cache_full), + rejected: d(self.rejected, earlier.rejected), + throttled: d(self.throttled, earlier.throttled), + disabled: d(self.disabled, earlier.disabled), + rate_limited: d(self.rate_limited, earlier.rate_limited), + uploads_sent: d(self.uploads_sent, earlier.uploads_sent), + upload_bytes: d(self.upload_bytes, earlier.upload_bytes), + upload_failures: d(self.upload_failures, earlier.upload_failures), + upload_timeouts: d(self.upload_timeouts, earlier.upload_timeouts), + hold_cap_hits: d(self.hold_cap_hits, earlier.hold_cap_hits), } } - /// Anything worth telling the backend about: data lost or uploads failing. - pub fn has_problems(&self) -> bool { + /// Everything that was lost, by any reason. + pub fn dropped(&self) -> u64 { self.queue_full + self.cache_error + + self.cache_full + self.rejected + self.throttled + + self.disabled + self.rate_limited + } + + /// Anything worth telling the backend about: data lost, uploads failing, or the upload + /// policy holding data back for a full minute. + pub fn has_problems(&self) -> bool { + self.dropped() - self.disabled + self.upload_failures + + self.upload_timeouts + + self.hold_cap_hits > 0 } - /// The `lk.telemetry.report` event: what this pipeline dropped or failed to upload since the - /// previous report. The Sentry "client report" shape — deltas by reason, riding along with - /// the next batch, never persisted on their own, never an extra request. + /// The `lk.telemetry.report` event: what this pipeline sent, dropped or failed to upload + /// since the previous report. The Sentry "client report" shape — deltas by reason, riding + /// along with the next batch, never persisted on their own, never an extra request — plus + /// one at shutdown, so every session leaves a summary the fleet's success rates can be + /// computed from. pub fn report(&self, cached_batches: u64) -> TelemetryEvent { let mut event = TelemetryEvent::new("lk.telemetry.report") - .with_attribute("lk.telemetry.uploads.failed", self.upload_failures as i64) .with_attribute("lk.telemetry.uploads.sent", self.uploads_sent as i64) + .with_attribute("lk.telemetry.uploads.bytes", self.upload_bytes as i64) + .with_attribute("lk.telemetry.uploads.failed", self.upload_failures as i64) .with_attribute("lk.telemetry.cache.batches", cached_batches as i64); for (key, value) in [ + ("lk.telemetry.uploads.timeouts", self.upload_timeouts), + ("lk.telemetry.holds.capped", self.hold_cap_hits), ("lk.telemetry.dropped.queue_full", self.queue_full), ("lk.telemetry.dropped.cache_error", self.cache_error), + ("lk.telemetry.dropped.cache_full", self.cache_full), ("lk.telemetry.dropped.rejected", self.rejected), ("lk.telemetry.dropped.throttled", self.throttled), ("lk.telemetry.dropped.rate_limited", self.rate_limited), @@ -131,14 +169,21 @@ pub struct TelemetryStats { pub dropped: u64, pub dropped_queue_full: u64, pub dropped_cache_error: u64, + pub dropped_cache_full: u64, pub dropped_rejected: u64, pub dropped_throttled: u64, pub dropped_disabled: u64, pub dropped_rate_limited: u64, /// Batches the collector accepted. pub uploads_sent: u64, - /// Upload attempts that failed transiently. + /// Compressed bytes the collector accepted. + pub upload_bytes: u64, + /// Upload attempts that failed transiently (network error, 5xx). pub upload_failures: u64, + /// Upload attempts that timed out. + pub upload_timeouts: u64, + /// Upload holds that reached the 60 s cap. + pub holds_capped: u64, /// Batches currently waiting in the cache. pub cached_batches: u64, } @@ -146,20 +191,19 @@ pub struct TelemetryStats { impl TelemetryStats { pub(crate) fn new(snapshot: Snapshot, cached_batches: u64) -> Self { Self { - dropped: snapshot.queue_full - + snapshot.cache_error - + snapshot.rejected - + snapshot.throttled - + snapshot.disabled - + snapshot.rate_limited, + dropped: snapshot.dropped(), dropped_queue_full: snapshot.queue_full, dropped_cache_error: snapshot.cache_error, + dropped_cache_full: snapshot.cache_full, dropped_rejected: snapshot.rejected, dropped_throttled: snapshot.throttled, dropped_disabled: snapshot.disabled, dropped_rate_limited: snapshot.rate_limited, uploads_sent: snapshot.uploads_sent, + upload_bytes: snapshot.upload_bytes, upload_failures: snapshot.upload_failures, + upload_timeouts: snapshot.upload_timeouts, + holds_capped: snapshot.hold_cap_hits, cached_batches, } } diff --git a/livekit-telemetry/src/store.rs b/livekit-telemetry/src/store.rs index 8f7db1eea..86b00d050 100644 --- a/livekit-telemetry/src/store.rs +++ b/livekit-telemetry/src/store.rs @@ -23,33 +23,61 @@ use crate::{stats::Counters, TelemetryEvent}; /// /// When full, the *oldest* event is dropped so the freshest context survives a burst /// (the queue role of OTel's `BatchLogRecordProcessor`, with drop-oldest instead of -/// drop-newest); every eviction is counted as `queue_full`. +/// drop-newest); every eviction is counted as `queue_full`. Tracks its approximate size in +/// bytes so the exporter can flush early (design doc: every 15 s *or* at 256 KB) and bound a +/// request's size. // ponytail: one mutex around a VecDeque; a lock-free ring only if `emit` shows up in a profile. pub(crate) struct Store { - queue: Mutex>, + queue: Mutex, capacity: usize, + flush_threshold: usize, counters: Arc, } +#[derive(Default)] +struct Queue { + events: VecDeque, + bytes: usize, +} + impl Store { - pub fn new(capacity: usize, counters: Arc) -> Self { - Self { queue: Mutex::new(VecDeque::with_capacity(capacity.min(1024))), capacity, counters } + pub fn new(capacity: usize, flush_threshold: usize, counters: Arc) -> Self { + Self { queue: Mutex::new(Queue::default()), capacity, flush_threshold, counters } } - pub fn push(&self, event: TelemetryEvent) { + /// Queue an event. Returns `true` when this push carried the queue across + /// `flush_threshold` bytes — the caller should wake the exporter. + pub fn push(&self, event: TelemetryEvent) -> bool { let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); - if queue.len() >= self.capacity { - queue.pop_front(); + if queue.events.len() >= self.capacity { + if let Some(oldest) = queue.events.pop_front() { + queue.bytes = queue.bytes.saturating_sub(oldest.size_hint()); + } Counters::add(&self.counters.queue_full, 1); } - queue.push_back(event); + let before = queue.bytes; + queue.bytes += event.size_hint(); + queue.events.push_back(event); + before < self.flush_threshold && queue.bytes >= self.flush_threshold } - /// Remove and return up to `max` events, oldest first. - pub fn drain(&self, max: usize) -> Vec { + /// Remove and return the oldest events: at most `max` of them and about `max_bytes` in total + /// (always at least one, so an oversized event still ships). + pub fn drain(&self, max: usize, max_bytes: usize) -> Vec { let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); - let n = max.min(queue.len()); - queue.drain(..n).collect() + let mut out = Vec::new(); + let mut bytes = 0; + while out.len() < max { + let Some(next) = queue.events.front() else { break }; + let size = next.size_hint(); + if !out.is_empty() && bytes + size > max_bytes { + break; + } + bytes += size; + queue.bytes = queue.bytes.saturating_sub(size); + out.extend(queue.events.pop_front()); + } + out } } @@ -60,13 +88,25 @@ mod tests { #[test] fn drops_oldest_when_full() { let counters = Arc::new(Counters::default()); - let store = Store::new(2, counters.clone()); + let store = Store::new(2, usize::MAX, counters.clone()); for name in ["a", "b", "c"] { store.push(TelemetryEvent::new(name)); } - let names: Vec<_> = store.drain(10).into_iter().map(|e| e.name).collect(); + let names: Vec<_> = store.drain(10, usize::MAX).into_iter().map(|e| e.name).collect(); assert_eq!(names, ["b", "c"]); assert_eq!(counters.snapshot().queue_full, 1); - assert!(store.drain(10).is_empty()); + assert!(store.drain(10, usize::MAX).is_empty()); + } + + #[test] + fn reports_the_threshold_crossing_once_and_drains_by_bytes() { + let store = Store::new(100, 100, Arc::default()); + let event = || TelemetryEvent::new("e").with_body("x".repeat(30)); // 63 bytes + assert!(!store.push(event()), "63 < 100"); + assert!(store.push(event()), "126 crosses 100"); + assert!(!store.push(event()), "already above: no second wake-up"); + assert_eq!(store.drain(10, 130).len(), 2, "two fit in 130 bytes"); + assert_eq!(store.drain(10, 1).len(), 1, "an oversized event still ships alone"); + assert!(store.drain(10, usize::MAX).is_empty()); } } diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 48526e6a1..567cb90cf 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -79,6 +79,13 @@ pub struct TelemetryConfig { /// ~40 kbps. `shutdown` drains without the budget. #[cfg_attr(feature = "uniffi", uniffi(default = 4))] pub max_batches_per_upload: u32, + /// Export as soon as the queue holds about this many bytes, without waiting for the tick + /// (design doc: "flush every 15 s or at 256 KB"). + #[cfg_attr(feature = "uniffi", uniffi(default = 262144))] + pub flush_threshold_bytes: u64, + /// Cap on one request's payload before compression (design doc: "single POST ≤ 1 MB"). + #[cfg_attr(feature = "uniffi", uniffi(default = 1048576))] + pub max_batch_bytes: u64, } impl TelemetryConfig { @@ -98,6 +105,8 @@ impl TelemetryConfig { stats_window_ms: 15_000, max_events_per_10min: 300, max_batches_per_upload: 4, + flush_threshold_bytes: 256 * 1024, + max_batch_bytes: 1024 * 1024, } } } @@ -212,7 +221,11 @@ impl Telemetry { } let config = Arc::new(config); let counters = Arc::new(Counters::default()); - let store = Arc::new(Store::new(config.max_queue_size.max(1) as usize, counters.clone())); + let store = Arc::new(Store::new( + config.max_queue_size.max(1) as usize, + usize::try_from(config.flush_threshold_bytes.max(1)).unwrap_or(usize::MAX), + counters.clone(), + )); let (commands, receiver) = mpsc::unbounded_channel(); let windows = Arc::new(Mutex::new(StatsWindows::default())); let guard = Arc::new(Mutex::new(FloodGuard::new(config.max_events_per_10min))); @@ -267,7 +280,16 @@ impl Telemetry { if event.timestamp_ns.is_none() { event.timestamp_ns = Some(now_unix_nanos()); } - self.store.push(event); + if self.store.push(event) { + let _ = self.commands.send(Command::Overflow); + } + } + + /// Queue a consumer-defined event, exported as `custom.` (see + /// [`TelemetryEvent::custom`]): the stringly-typed escape hatch next to the `lk.*` + /// catalogue. Same flood guard, same pipeline. + pub fn emit_custom(&self, name: &str, attributes: Vec) { + self.emit(TelemetryEvent::custom(name, attributes)); } /// Set a session-wide attribute (`lk.room.sid`, `lk.participant.identity`, or a consumer's @@ -635,7 +657,12 @@ mod tests { telemetry.emit(TelemetryEvent::new("lk.ping")); telemetry.shutdown().await; - assert_eq!(files_in(&dir), 1, "cached before the network was tried, kept after it failed"); + // One file, or two when the first tick shipped the ping before shutdown added the summary. + let files = files_in(&dir); + assert!( + (1..=2).contains(&files), + "cached before the network was tried, kept after: {files}" + ); assert_eq!(telemetry.stats().dropped, 0); let _ = fs::remove_dir_all(&dir); } @@ -713,6 +740,7 @@ mod tests { tokio::time::sleep(Duration::from_secs(61)).await; telemetry.flush().await; assert_eq!(transport.sent().len(), 1, "held 60 s: one batch goes out regardless"); + assert_eq!(telemetry.stats().holds_capped, 1, "…and the starvation is counted"); telemetry.emit(TelemetryEvent::new("lk.ping")); telemetry.end_span(connect, SpanOutcome::Ok, None, Vec::new()); @@ -760,7 +788,112 @@ mod tests { telemetry.flush().await; assert_eq!(transport.sent().len(), 4); telemetry.shutdown().await; - assert_eq!(transport.sent().len(), 6, "shutdown drains without the budget"); + assert_eq!(transport.sent().len(), 7, "shutdown drains without the budget (+ the summary)"); + } + + struct Hanging; + + #[async_trait::async_trait] + impl TelemetryTransport for Hanging { + async fn send(&self, _: ExportRequest) -> Result<(), ExportError> { + std::future::pending().await + } + } + + #[tokio::test(start_paused = true)] + async fn a_full_queue_flushes_before_the_tick() { + let transport = FakeTransport::scripted([]); + let mut config = TelemetryConfig::new("http://collector/v1/logs"); + config.flush_interval_ms = 60_000; + config.flush_threshold_bytes = 10_000; + let telemetry = start(config, transport.clone()); + tokio::time::sleep(Duration::from_millis(1)).await; // the immediate first tick passes + for _ in 0..3 { + telemetry.emit(TelemetryEvent::new("big").with_body("x".repeat(4_000))); + } + tokio::time::sleep(Duration::from_millis(1)).await; // the wake-up is processed + let sent = transport.sent(); + assert_eq!(sent.len(), 1, "exported on crossing the byte threshold, a minute early"); + assert_eq!(records(&sent[0]).len(), 3); + } + + #[tokio::test(start_paused = true)] + async fn requests_stay_under_the_byte_cap() { + let transport = FakeTransport::scripted([]); + let mut config = TelemetryConfig::new("http://collector/v1/logs"); + config.max_batch_bytes = 10_000; + config.max_batches_per_upload = 10; + let telemetry = start(config, transport.clone()); + for _ in 0..5 { + telemetry.emit(TelemetryEvent::new("big").with_body("x".repeat(4_000))); + } + telemetry.flush().await; + let sent = transport.sent(); + assert_eq!(sent.len(), 3, "5 × ~4 KB under a 10 KB cap: 2 + 2 + 1"); + assert!(sent.iter().all(|request| records(request).len() <= 2)); + } + + #[tokio::test(start_paused = true)] + async fn custom_events_are_namespaced() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + telemetry.emit_custom("acme.checkout", vec![Attribute::new("acme.step", 3i64)]); + telemetry.emit_custom("custom.already", Vec::new()); + telemetry.flush().await; + let sent = transport.sent(); + assert_eq!(event_names(&sent[0]), ["custom.acme.checkout", "custom.already"]); + let record = records(&sent[0]).remove(0); + assert_eq!(attribute(&record, "acme.step"), Some(Value::IntValue(3))); + } + + #[tokio::test(start_paused = true)] + async fn shutdown_leaves_a_session_summary() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + telemetry.shutdown().await; + let sent = transport.sent(); + assert_eq!(sent.len(), 2, "the summary is its own batch when nothing else is queued"); + let report = records(&sent[1]).remove(0); + assert_eq!(report.event_name, "lk.telemetry.report"); + assert_eq!(attribute(&report, "lk.telemetry.uploads.sent"), Some(Value::IntValue(1))); + assert!( + matches!(attribute(&report, "lk.telemetry.uploads.bytes"), Some(Value::IntValue(n)) if n > 0), + "bytes on the wire are reported" + ); + } + + #[tokio::test(start_paused = true)] + async fn cache_eviction_is_counted_as_a_drop() { + let transport = FakeTransport::scripted([offline(), offline(), offline()]); + // Room for exactly one batch: a second push evicts the first. + let (telemetry, exporter) = Telemetry::with_cache( + TelemetryConfig::new("http://collector/v1/logs"), + transport.clone(), + Arc::new(MemoryCache::new(1)), + ); + tokio::spawn(exporter.run()); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; // fails: cached, upload paused + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + let stats = telemetry.stats(); + assert_eq!(stats.cached_batches, 1); + assert_eq!(stats.dropped_cache_full, 1, "the evicted ping is counted, not silently lost"); + } + + #[tokio::test(start_paused = true)] + async fn timeouts_are_counted_apart_from_failures() { + let (telemetry, exporter) = + Telemetry::new(TelemetryConfig::new("http://collector/v1/logs"), Arc::new(Hanging)); + tokio::spawn(exporter.run()); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; // 3 attempts × export_timeout, under paused time + let stats = telemetry.stats(); + assert_eq!(stats.upload_timeouts, 3); + assert_eq!(stats.upload_failures, 0); + assert_eq!(stats.cached_batches, 1, "kept for the next attempt"); } #[tokio::test(start_paused = true)] @@ -851,7 +984,11 @@ mod tests { )); telemetry.shutdown().await; let names: Vec = transport.sent().iter().flat_map(event_names).collect(); - assert_eq!(names, ["lk.rtc.stats.sample"]); + assert_eq!( + names, + ["lk.rtc.stats.sample", "lk.telemetry.report"], + "window + shutdown summary" + ); } #[tokio::test(start_paused = true)] diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index 35d5f157e..194262228 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -68,6 +68,11 @@ impl Telemetry { self.0.emit(event); } + /// A consumer-defined event, exported as `custom.`; attributes keep their own namespace. + pub fn emit_custom(&self, name: String, attributes: Vec) { + self.0.emit_custom(&name, attributes); + } + /// Report the device state (thermal, low power, foreground/background). Emits the matching /// `lk.device.*.changed` events and adapts the export cadence. pub fn set_device_state(&self, state: DeviceState) { From afdb54f7b29e3b018a2daf14bada280f454e9b1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:50:07 +0200 Subject: [PATCH 13/50] feat(telemetry): one pipeline per process, sessions per room, deferred destination The design doc wants buffering from SDK init and flushing once a granted token is available; the Swift integration started the pipeline at connect(). The core now models what that needs: - `Telemetry` is the process pipeline (queue, cache, exporter, device state, self-telemetry). `Telemetry::begin_session()` returns a `Session` - one room, one call - with its own trace id and attributes; queued records, spans and RTC windows are filed under the session that produced them and carry `session.id` as a record attribute (no longer a resource attribute). Everything emitted outside a session belongs to the pipeline's process session. A log record emitted with a span id is filed under that span's session, so SDK loggers need no session plumbing. - `TelemetryConfig.endpoint` is optional. Without one the pipeline buffers and caches; uploads wait (uncapped) for `set_destination(endpoint, headers)`, which the first connect supplies from the server URL and token. Calling it again redirects later batches. UniFFI gains `TelemetrySession`, `begin_session` and `set_destination`. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/SPEC.md | 19 ++- livekit-telemetry/src/exporter.rs | 62 ++++----- livekit-telemetry/src/lib.rs | 2 + livekit-telemetry/src/otlp.rs | 40 +++--- livekit-telemetry/src/rtc.rs | 38 ++++-- livekit-telemetry/src/session.rs | 140 ++++++++++++++++++++ livekit-telemetry/src/span.rs | 25 +++- livekit-telemetry/src/store.rs | 33 +++-- livekit-telemetry/src/telemetry.rs | 197 ++++++++++++++++++++++++----- livekit-uniffi/src/telemetry.rs | 59 +++++++++ 10 files changed, 502 insertions(+), 113 deletions(-) create mode 100644 livekit-telemetry/src/session.rs diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index 3f0078c3b..e21afbd1a 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -15,7 +15,24 @@ Set once per pipeline (`TelemetryConfig.resource`): | `os.name`, `os.version` | platform SDK | `iOS`, `18.5` | | `device.model.identifier` | platform SDK | `iPhone16,1` | | `telemetry.sdk.name/language/version` | core | `livekit-telemetry`, `rust`, `0.1.0` | -| `session.id` | core | the session's trace id as 32 hex chars — every log record and span of the pipeline carries it | + +## Pipeline, sessions and destination + +One pipeline per process — started at SDK init, so audio pre-initialization, permission failures +and connect attempts that never reach a server are captured — and one **session** per room (one +call). A session is a trace id plus the attributes attached to its records (`lk.room.sid`, +`lk.participant.identity`, …); spans, RTC windows and events are filed under the session that +produced them, and `session.id` (OTel semconv) is written on every record as an attribute. A log +record emitted inside a room's span is filed under that room's session; anything emitted outside +a session — device state, pre-room errors, self-telemetry — belongs to the pipeline's own process +session. Sessions are not ended: a room's last record is simply its last. + +The pipeline may start **without a destination** (`endpoint: None`): it buffers and caches, and +uploads nothing until `set_destination(endpoint, headers)` — at the first connect, when the +server URL yields the endpoint (`https:///observability/logs/otlp/v0`) and the token the +`Authorization` header. Calling it again (new token, new server) replaces the destination for +the batches that follow. Waiting for a destination is not an upload hold: it is uncapped, bounded +only by the cache. ## Events diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index 4e9f0a163..5e8f1c0f8 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -26,10 +26,12 @@ use crate::{ event::now_unix_nanos, otlp, rtc::StatsWindows, + session::SessionState, span::Spans, stats::{Counters, Snapshot}, - store::Store, - AppState, Attribute, BatchCache, DeviceState, ExportError, ExportRequest, TelemetryConfig, + store::{Queued, Store}, + telemetry::Destination, + AppState, BatchCache, DeviceState, ExportError, ExportRequest, TelemetryConfig, TelemetryTransport, }; @@ -125,10 +127,11 @@ pub struct Exporter { cache: Arc, counters: Arc, windows: Arc>, - /// Session-wide attributes merged into every record at encode time. - attributes: Arc>>, spans: Arc>, - trace_id: [u8; 16], + /// The pipeline's own session: self-telemetry is filed under it. + process: Arc, + /// Where batches go; `None` until `set_destination` — batches wait in the cache meanwhile. + destination: Arc>>, commands: mpsc::UnboundedReceiver, silenced: bool, /// Leave the cache alone until then: the last upload failed or we were throttled. @@ -162,9 +165,9 @@ impl Exporter { cache: Arc, counters: Arc, windows: Arc>, - attributes: Arc>>, spans: Arc>, - trace_id: [u8; 16], + process: Arc, + destination: Arc>>, commands: mpsc::UnboundedReceiver, device: Arc>>, ) -> Self { @@ -175,9 +178,9 @@ impl Exporter { cache, counters, windows, - attributes, spans, - trace_id, + process, + destination, commands, silenced: false, paused_until: None, @@ -315,22 +318,20 @@ impl Exporter { let now = self.counters.snapshot(); let delta = now.since(&self.last_report); if delta.has_problems() || report_due { - batch.push(delta.report(self.cache.pending().len() as u64)); + let report = delta.report(self.cache.pending().len() as u64); + batch.push(Queued { event: report, session: self.process.clone() }); self.last_report = now; self.force_report = false; } - for event in &mut batch { - self.attach_session_attributes(&mut event.attributes); - } let count = batch.len() as u64; - let body = otlp::encode_logs(&self.config.resource, &self.trace_id, batch); + let body = otlp::encode_logs(&self.config.resource, batch); self.push_batch(Signal::Logs, count, &body); } } /// Finished spans travel as their own batch on the traces signal. fn enqueue_spans(&mut self) { - let (mut spans, dropped) = { + let (spans, dropped) = { let mut registry = self.spans.lock().unwrap_or_else(|e| e.into_inner()); (registry.drain(self.config.max_batch_size.max(1) as usize), registry.take_dropped()) }; @@ -344,11 +345,8 @@ impl Exporter { Counters::add(counter, spans.len() as u64); return; } - for span in &mut spans { - self.attach_session_attributes(&mut span.attributes); - } let count = spans.len() as u64; - let body = otlp::encode_spans(&self.config.resource, &self.trace_id, spans); + let body = otlp::encode_spans(&self.config.resource, spans); self.push_batch(Signal::Traces, count, &body); } @@ -379,16 +377,6 @@ impl Exporter { } } - /// Merge the session-wide attributes into a record's own, without overriding explicit ones. - fn attach_session_attributes(&self, attributes: &mut Vec) { - let session = self.attributes.lock().unwrap_or_else(|e| e.into_inner()); - for attribute in session.iter() { - if !attributes.iter().any(|a| a.key == attribute.key) { - attributes.push(attribute.clone()); - } - } - } - /// Why uploads should wait right now, if they should. Data keeps flowing into the cache /// meanwhile — write-ahead caching is what makes holding free. fn hold_reason(&self) -> Option<&'static str> { @@ -414,6 +402,10 @@ impl Exporter { if self.silenced || self.paused_until.is_some_and(|t| Instant::now() < t) { return; } + // No destination yet (SDK started, no room connected): everything waits in the cache. + if self.destination.lock().unwrap_or_else(|e| e.into_inner()).is_none() { + return; + } let pending = self.cache.pending(); if pending.is_empty() { self.held_since = None; @@ -484,15 +476,17 @@ impl Exporter { /// Upload one cached (gzipped) batch with bounded retries and classify the outcome. async fn deliver(&self, body: &[u8], signal: Signal) -> Delivery { - let mut headers = self.config.headers.clone(); + let Some(destination) = self.destination.lock().unwrap_or_else(|e| e.into_inner()).clone() + else { + return Delivery::Failed; + }; + let mut headers = destination.headers; headers.insert("Content-Type".to_owned(), otlp::CONTENT_TYPE.to_owned()); headers.insert("Content-Encoding".to_owned(), "gzip".to_owned()); headers.insert("Priority".to_owned(), PRIORITY.to_owned()); let url = match signal { - Signal::Logs => self.config.endpoint.clone(), - Signal::Traces => { - self.config.traces_endpoint.clone().unwrap_or_else(|| self.config.endpoint.clone()) - } + Signal::Logs => destination.logs, + Signal::Traces => destination.traces, }; let request = ExportRequest { url, headers, body: body.to_vec() }; let attempt_timeout = Duration::from_millis(self.config.export_timeout_ms.max(1)); diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs index fd99033a6..4b9639de4 100644 --- a/livekit-telemetry/src/lib.rs +++ b/livekit-telemetry/src/lib.rs @@ -30,6 +30,7 @@ mod device; mod rtc; /// Spans: one attempt at an operation, with explicit handles across the FFI. +mod session; mod span; /// Batch exporter actor: timer, OTLP encoding, retry policy. @@ -55,6 +56,7 @@ pub use device::*; pub use event::*; pub use exporter::Exporter; pub use rtc::{RtcStatsSample, StreamDirection, TrackKind}; +pub use session::Session; pub use span::{SpanKind, SpanOutcome}; pub use stats::TelemetryStats; pub use telemetry::*; diff --git a/livekit-telemetry/src/otlp.rs b/livekit-telemetry/src/otlp.rs index d55dc109f..bba8c068d 100644 --- a/livekit-telemetry/src/otlp.rs +++ b/livekit-telemetry/src/otlp.rs @@ -24,7 +24,8 @@ use crate::{ trace::v1::{span, status, ResourceSpans, ScopeSpans, Span, Status}, }, span::SpanRecord, - Attribute, AttributeValue, Severity, SpanKind, SpanOutcome, TelemetryEvent, + store::Queued, + Attribute, AttributeValue, Severity, SpanKind, SpanOutcome, }; pub(crate) const CONTENT_TYPE: &str = "application/x-protobuf"; @@ -45,19 +46,15 @@ fn scope() -> Option { } /// Encode one batch as an OTLP `ExportLogsServiceRequest`: one resource, one instrumentation -/// scope (this crate), one log record per event. Every record carries the session's trace id; -/// records emitted inside a span carry its span id too. -pub(crate) fn encode_logs( - resource_attributes: &[Attribute], - trace_id: &[u8; 16], - events: Vec, -) -> Vec { +/// scope (this crate), one log record per event. Every record carries its session's trace id and +/// attributes; records emitted inside a span carry its span id too. +pub(crate) fn encode_logs(resource_attributes: &[Attribute], events: Vec) -> Vec { ExportLogsServiceRequest { resource_logs: vec![ResourceLogs { resource: resource(resource_attributes), scope_logs: vec![ScopeLogs { scope: scope(), - log_records: events.into_iter().map(|e| log_record(e, trace_id)).collect(), + log_records: events.into_iter().map(log_record).collect(), ..Default::default() }], ..Default::default() @@ -66,18 +63,14 @@ pub(crate) fn encode_logs( .encode_to_vec() } -/// Encode finished spans as an OTLP `ExportTraceServiceRequest` under the session's trace id. -pub(crate) fn encode_spans( - resource_attributes: &[Attribute], - trace_id: &[u8; 16], - spans: Vec, -) -> Vec { +/// Encode finished spans as an OTLP `ExportTraceServiceRequest`, each under its session's trace id. +pub(crate) fn encode_spans(resource_attributes: &[Attribute], spans: Vec) -> Vec { ExportTraceServiceRequest { resource_spans: vec![ResourceSpans { resource: resource(resource_attributes), scope_spans: vec![ScopeSpans { scope: scope(), - spans: spans.into_iter().map(|s| otlp_span(s, trace_id)).collect(), + spans: spans.into_iter().map(otlp_span).collect(), ..Default::default() }], ..Default::default() @@ -86,7 +79,8 @@ pub(crate) fn encode_spans( .encode_to_vec() } -fn log_record(event: TelemetryEvent, trace_id: &[u8; 16]) -> LogRecord { +fn log_record(Queued { mut event, session }: Queued) -> LogRecord { + session.decorate(&mut event.attributes); let time_unix_nano = event.timestamp_ns.unwrap_or_else(now_unix_nanos); LogRecord { time_unix_nano, @@ -96,17 +90,19 @@ fn log_record(event: TelemetryEvent, trace_id: &[u8; 16]) -> LogRecord { body: event.body.map(|text| AnyValue { value: Some(any_value::Value::StringValue(text)) }), attributes: event.attributes.iter().map(KeyValue::from).collect(), event_name: event.name, - trace_id: trace_id.to_vec(), + trace_id: session.trace_id.to_vec(), span_id: event.span_id.map(|id| id.to_be_bytes().to_vec()).unwrap_or_default(), ..Default::default() } } -fn otlp_span(record: SpanRecord, trace_id: &[u8; 16]) -> Span { +fn otlp_span(mut record: SpanRecord) -> Span { + let session = record.session.clone(); + session.decorate(&mut record.attributes); let mut attributes: Vec = record.attributes.iter().map(KeyValue::from).collect(); attributes.extend(record.outcome_attributes().iter().map(KeyValue::from)); Span { - trace_id: trace_id.to_vec(), + trace_id: session.trace_id.to_vec(), span_id: record.span_id.to_be_bytes().to_vec(), parent_span_id: record.parent_span_id.map(|p| p.to_be_bytes().to_vec()).unwrap_or_default(), name: record.name, @@ -181,6 +177,7 @@ impl From<&Attribute> for KeyValue { #[cfg(test)] mod tests { use super::*; + use crate::TelemetryEvent; #[test] fn encodes_events_as_otlp_log_records() { @@ -189,7 +186,8 @@ mod tests { .with_severity(Severity::Warn) .with_body("hi") .with_attribute("lk.ping.seq", 7i64); - let bytes = encode_logs(&resource, &[7u8; 16], vec![event]); + let session = crate::session::SessionState::with_trace_id([7u8; 16]); + let bytes = encode_logs(&resource, vec![Queued { event, session }]); let decoded = ExportLogsServiceRequest::decode(&bytes[..]).expect("valid OTLP"); let resource_logs = &decoded.resource_logs[0]; diff --git a/livekit-telemetry/src/rtc.rs b/livekit-telemetry/src/rtc.rs index 6fc7a94c2..c0e0ad871 100644 --- a/livekit-telemetry/src/rtc.rs +++ b/livekit-telemetry/src/rtc.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{collections::HashMap, time::Duration}; +use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio::time::Instant; -use crate::{event::now_unix_nanos, TelemetryEvent}; +use crate::{event::now_unix_nanos, session::SessionState, store::Queued, TelemetryEvent}; #[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -148,6 +148,7 @@ impl Gauge { /// Samples of one track in one direction accumulated since the window opened. struct Window { + session: Arc, start_ns: u64, samples: u32, last: RtcStatsSample, @@ -158,8 +159,9 @@ struct Window { } impl Window { - fn open(start_ns: u64, first: RtcStatsSample) -> Self { + fn open(start_ns: u64, first: RtcStatsSample, session: Arc) -> Self { let mut window = Self { + session, start_ns, samples: 0, last: first.clone(), @@ -256,22 +258,38 @@ const BANDWIDTH_LIMITED_HOLD: Duration = Duration::from_secs(10); const CPU_LIMITED_HOLD: Duration = Duration::from_secs(60); impl StatsWindows { - pub fn record(&mut self, mut sample: RtcStatsSample) { + pub fn record_in(&mut self, mut sample: RtcStatsSample, session: &Arc) { self.track_limitation(&sample); let timestamp = *sample.timestamp_ns.get_or_insert_with(now_unix_nanos); let key = (sample.track_sid.clone(), sample.direction); match self.windows.get_mut(&key) { Some(window) => window.add(sample), None => { - self.windows.insert(key, Window::open(timestamp, sample)); + self.windows.insert(key, Window::open(timestamp, sample, session.clone())); } } } - /// Close every open window into its event and start fresh. - pub fn close(&mut self) -> Vec { + /// Close every open window into its event, filed under the window's session, and start fresh. + pub fn close(&mut self) -> Vec { let end = now_unix_nanos(); - self.windows.drain().map(|(_, window)| window.into_event(end)).collect() + self.windows + .drain() + .map(|(_, window)| { + let session = window.session.clone(); + Queued { event: window.into_event(end), session } + }) + .collect() + } + + #[cfg(test)] + pub fn record(&mut self, sample: RtcStatsSample) { + self.record_in(sample, &SessionState::new()); + } + + #[cfg(test)] + pub fn close_events(&mut self) -> Vec { + self.close().into_iter().map(|q| q.event).collect() } pub fn media_pressure(&self) -> MediaPressure { @@ -371,8 +389,8 @@ mod tests { other.packets = Some(7); windows.record(other); - let mut events = windows.close(); - assert!(windows.close().is_empty(), "closing again yields nothing"); + let mut events = windows.close_events(); + assert!(windows.close_events().is_empty(), "closing again yields nothing"); events.sort_by_key(|e| attr(e, "lk.track.direction").map(|v| format!("{v:?}"))); assert_eq!(events.len(), 2, "one event per track and direction"); let inbound = &events[0]; diff --git a/livekit-telemetry/src/session.rs b/livekit-telemetry/src/session.rs new file mode 100644 index 000000000..fe7a3fc50 --- /dev/null +++ b/livekit-telemetry/src/session.rs @@ -0,0 +1,140 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::{ + fmt, + sync::{Arc, Mutex}, +}; + +use crate::{ + Attribute, AttributeValue, RtcStatsSample, SpanKind, SpanOutcome, Telemetry, TelemetryEvent, +}; + +/// One session's identity: the trace id every one of its records carries, and the attributes +/// attached to them at export time (`lk.room.sid`, `lk.participant.identity`, …). +pub(crate) struct SessionState { + pub trace_id: [u8; 16], + attributes: Mutex>, +} + +impl SessionState { + /// A fresh session: random, non-zero trace id (OTLP treats all-zero as absent). + pub fn new() -> Arc { + Self::with_trace_id(rand::random::().max(1).to_be_bytes()) + } + + pub fn with_trace_id(trace_id: [u8; 16]) -> Arc { + Arc::new(Self { trace_id, attributes: Mutex::new(Vec::new()) }) + } + + /// The trace id as 32 hex characters. + pub fn hex(&self) -> String { + format!("{:032x}", u128::from_be_bytes(self.trace_id)) + } + + pub fn set_attribute(&self, key: &str, value: Option) { + let mut attributes = self.attributes.lock().unwrap_or_else(|e| e.into_inner()); + attributes.retain(|a| a.key != key); + if let Some(value) = value { + attributes.push(Attribute::new(key, value)); + } + } + + /// Merge the session's attributes into a record's own without overriding explicit ones, and + /// add `session.id` (OTel semconv) — the trace id, so a record can be joined to its session + /// even where a backend drops trace ids from logs. + pub fn decorate(&self, own: &mut Vec) { + let session = self.attributes.lock().unwrap_or_else(|e| e.into_inner()); + for attribute in session.iter() { + if !own.iter().any(|a| a.key == attribute.key) { + own.push(attribute.clone()); + } + } + if !own.iter().any(|a| a.key == "session.id") { + own.push(Attribute::new("session.id", self.hex())); + } + } +} + +impl PartialEq for SessionState { + fn eq(&self, other: &Self) -> bool { + self.trace_id == other.trace_id + } +} + +impl fmt::Debug for SessionState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Session({})", self.hex()) + } +} + +/// A session on the shared pipeline: its own trace id and attributes, the same queue, cache, +/// cadence and exporter as every other session in the process. +/// +/// The pipeline starts once, at SDK init, so nothing that happens before the first room is +/// lost; a `Session` is what a room — one call — gets from it, and what its spans, RTC windows +/// and events are filed under. Everything emitted outside a session (device state, pre-room +/// errors, self-telemetry) belongs to the pipeline's own process session. Cheap to clone. +#[derive(Clone)] +pub struct Session { + pub(crate) telemetry: Telemetry, + pub(crate) state: Arc, +} + +impl Session { + /// The session's trace id as 32 hex characters — print it (`lkt_…`) so support can find + /// the call. + pub fn trace_id(&self) -> String { + self.state.hex() + } + + /// Queue an event or log record under this session. + pub fn emit(&self, event: TelemetryEvent) { + self.telemetry.emit_in(event, &self.state); + } + + /// A consumer-defined event (`custom.`) under this session. + pub fn emit_custom(&self, name: &str, attributes: Vec) { + self.emit(TelemetryEvent::custom(name, attributes)); + } + + /// Attach an attribute to every record of this session from now on; `None` removes it. + pub fn set_attribute(&self, key: &str, value: Option) { + self.state.set_attribute(key, value); + } + + /// Push one `getStats()` reading; its window ships under this session. + pub fn record_stats(&self, sample: RtcStatsSample) { + self.telemetry.record_stats_in(sample, &self.state); + } + + /// Open a span in this session's trace. + pub fn begin_span(&self, name: &str, kind: SpanKind, parent: Option) -> u64 { + self.telemetry.begin_span_in(name, kind, parent, &self.state) + } + + pub fn add_span_event(&self, span: u64, name: &str, attributes: Vec) { + self.telemetry.add_span_event(span, name, attributes); + } + + pub fn end_span( + &self, + span: u64, + outcome: SpanOutcome, + error_type: Option, + attributes: Vec, + ) { + self.telemetry.end_span(span, outcome, error_type, attributes); + } +} diff --git a/livekit-telemetry/src/span.rs b/livekit-telemetry/src/span.rs index fa1612d0c..c90f930d5 100644 --- a/livekit-telemetry/src/span.rs +++ b/livekit-telemetry/src/span.rs @@ -13,8 +13,9 @@ // limitations under the License. use std::collections::{HashMap, VecDeque}; +use std::sync::Arc; -use crate::{event::now_unix_nanos, Attribute, AttributeValue}; +use crate::{event::now_unix_nanos, session::SessionState, Attribute, AttributeValue}; /// OTel span kind, restricted to what client operations need. #[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] @@ -70,6 +71,8 @@ pub(crate) struct SpanRecord { pub error_type: Option, pub attributes: Vec, pub events: Vec, + /// The session (trace) the span belongs to. + pub session: Arc, } /// OTel default span limits. @@ -107,7 +110,14 @@ impl Spans { } } - pub fn begin(&mut self, name: &str, kind: SpanKind, parent: Option) -> u64 { + /// Open a span in `session`'s trace. + pub fn begin_in( + &mut self, + name: &str, + kind: SpanKind, + parent: Option, + session: Arc, + ) -> u64 { let id = self.next_id; self.next_id = self.next_id.wrapping_add(1).max(1); if self.open.len() >= MAX_OPEN_SPANS { @@ -127,6 +137,7 @@ impl Spans { error_type: None, attributes: Vec::new(), events: Vec::new(), + session, }; self.open.insert(id, record); self.open_order.push_back(id); @@ -135,6 +146,16 @@ impl Spans { /// Whether a span with one of these names is still open (the exporter holds uploads while /// `lk.connect` / `lk.reconnect` are). + /// The session an open span belongs to (log records emitted inside a span are filed there). + pub fn session_of(&self, id: u64) -> Option> { + self.open.get(&id).map(|span| span.session.clone()) + } + + #[cfg(test)] + pub fn begin(&mut self, name: &str, kind: SpanKind, parent: Option) -> u64 { + self.begin_in(name, kind, parent, SessionState::new()) + } + pub fn any_open(&self, names: &[&str]) -> bool { self.open.values().any(|span| names.contains(&span.name.as_str())) } diff --git a/livekit-telemetry/src/store.rs b/livekit-telemetry/src/store.rs index 86b00d050..e473a29c2 100644 --- a/livekit-telemetry/src/store.rs +++ b/livekit-telemetry/src/store.rs @@ -17,7 +17,14 @@ use std::{ sync::{Arc, Mutex}, }; -use crate::{stats::Counters, TelemetryEvent}; +use crate::{session::SessionState, stats::Counters, TelemetryEvent}; + +/// An event waiting for export, filed under the session whose trace id and attributes it +/// will carry. +pub(crate) struct Queued { + pub event: TelemetryEvent, + pub session: Arc, +} /// Bounded FIFO of events waiting for export. /// @@ -36,7 +43,7 @@ pub(crate) struct Store { #[derive(Default)] struct Queue { - events: VecDeque, + events: VecDeque, bytes: usize, } @@ -47,29 +54,29 @@ impl Store { /// Queue an event. Returns `true` when this push carried the queue across /// `flush_threshold` bytes — the caller should wake the exporter. - pub fn push(&self, event: TelemetryEvent) -> bool { + pub fn push(&self, queued: Queued) -> bool { let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); if queue.events.len() >= self.capacity { if let Some(oldest) = queue.events.pop_front() { - queue.bytes = queue.bytes.saturating_sub(oldest.size_hint()); + queue.bytes = queue.bytes.saturating_sub(oldest.event.size_hint()); } Counters::add(&self.counters.queue_full, 1); } let before = queue.bytes; - queue.bytes += event.size_hint(); - queue.events.push_back(event); + queue.bytes += queued.event.size_hint(); + queue.events.push_back(queued); before < self.flush_threshold && queue.bytes >= self.flush_threshold } /// Remove and return the oldest events: at most `max` of them and about `max_bytes` in total /// (always at least one, so an oversized event still ships). - pub fn drain(&self, max: usize, max_bytes: usize) -> Vec { + pub fn drain(&self, max: usize, max_bytes: usize) -> Vec { let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); let mut out = Vec::new(); let mut bytes = 0; while out.len() < max { let Some(next) = queue.events.front() else { break }; - let size = next.size_hint(); + let size = next.event.size_hint(); if !out.is_empty() && bytes + size > max_bytes { break; } @@ -85,14 +92,18 @@ impl Store { mod tests { use super::*; + fn queued(event: TelemetryEvent) -> Queued { + Queued { event, session: SessionState::new() } + } + #[test] fn drops_oldest_when_full() { let counters = Arc::new(Counters::default()); let store = Store::new(2, usize::MAX, counters.clone()); for name in ["a", "b", "c"] { - store.push(TelemetryEvent::new(name)); + store.push(queued(TelemetryEvent::new(name))); } - let names: Vec<_> = store.drain(10, usize::MAX).into_iter().map(|e| e.name).collect(); + let names: Vec<_> = store.drain(10, usize::MAX).into_iter().map(|q| q.event.name).collect(); assert_eq!(names, ["b", "c"]); assert_eq!(counters.snapshot().queue_full, 1); assert!(store.drain(10, usize::MAX).is_empty()); @@ -101,7 +112,7 @@ mod tests { #[test] fn reports_the_threshold_crossing_once_and_drains_by_bytes() { let store = Store::new(100, 100, Arc::default()); - let event = || TelemetryEvent::new("e").with_body("x".repeat(30)); // 63 bytes + let event = || queued(TelemetryEvent::new("e").with_body("x".repeat(30))); // 63 bytes assert!(!store.push(event()), "63 < 100"); assert!(store.push(event()), "126 crosses 100"); assert!(!store.push(event()), "already above: no second wake-up"); diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 567cb90cf..85e65ecbf 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -22,9 +22,15 @@ use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; use crate::{ - event::now_unix_nanos, exporter::Command, rtc::StatsWindows, span::Spans, stats::Counters, - store::Store, Attribute, AttributeValue, BatchCache, DeviceState, Exporter, FileCache, - MemoryCache, RtcStatsSample, Severity, SpanKind, SpanOutcome, TelemetryEvent, TelemetryStats, + event::now_unix_nanos, + exporter::Command, + rtc::StatsWindows, + session::{Session, SessionState}, + span::Spans, + stats::Counters, + store::{Queued, Store}, + Attribute, AttributeValue, BatchCache, DeviceState, Exporter, FileCache, MemoryCache, + RtcStatsSample, Severity, SpanKind, SpanOutcome, TelemetryEvent, TelemetryStats, TelemetryTransport, }; @@ -36,8 +42,12 @@ use crate::{ #[derive(Debug, Clone)] pub struct TelemetryConfig { /// Full OTLP/HTTP logs URL: `http://localhost:4318/v1/logs` locally, - /// `https:///observability/logs/otlp/v0` for LiveKit Cloud. - pub endpoint: String, + /// `https:///observability/logs/otlp/v0` for LiveKit Cloud. `None` starts the + /// pipeline without a destination — it buffers (and caches) until + /// [`Telemetry::set_destination`], typically at the first connect, when the server URL and + /// the token are known. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub endpoint: Option, /// OTLP/HTTP traces URL. `None` derives it from `endpoint` by replacing the last `logs` /// path segment with `traces` (works for both layouts above). #[cfg_attr(feature = "uniffi", uniffi(default))] @@ -92,7 +102,7 @@ impl TelemetryConfig { /// Defaults for the given endpoint; in-memory cache. pub fn new(endpoint: impl Into) -> Self { Self { - endpoint: endpoint.into(), + endpoint: Some(endpoint.into()), traces_endpoint: None, headers: HashMap::new(), resource: Vec::new(), @@ -143,12 +153,31 @@ pub struct Telemetry { device: Arc>>, windows: Arc>, guard: Arc>, - attributes: Arc>>, spans: Arc>, - trace_id: [u8; 16], + /// The pipeline's own session: whatever is emitted outside a room session. + process: Arc, + destination: Arc>>, commands: mpsc::UnboundedSender, } +/// Where batches are sent: the OTLP endpoints and the request headers (auth). +#[derive(Debug, Clone)] +pub(crate) struct Destination { + pub logs: String, + pub traces: String, + pub headers: HashMap, +} + +impl Destination { + fn new(logs: &str, traces: Option, headers: HashMap) -> Self { + Self { + logs: logs.to_owned(), + traces: traces.unwrap_or_else(|| derive_traces_endpoint(logs)), + headers, + } + } +} + /// Fixed-window cap on discrete events (the design doc's ~300 per 10 min). struct FloodGuard { max: u32, @@ -208,18 +237,13 @@ impl Telemetry { cache: Arc, ) -> (Self, Exporter) { add_sdk_resource(&mut config.resource); - if config.traces_endpoint.is_none() { - config.traces_endpoint = Some(derive_traces_endpoint(&config.endpoint)); - } - // The session is the trace: one id per pipeline, client-generated so pre-connect - // failures and reconnects share it — and, as `session.id` (OTel semconv), the resource - // attribute every record carries. - let trace_id = rand::random::().max(1).to_be_bytes(); - if !config.resource.iter().any(|a| a.key == "session.id") { - let hex = format!("{:032x}", u128::from_be_bytes(trace_id)); - config.resource.push(Attribute::new("session.id", hex)); - } + let destination = Arc::new(Mutex::new(config.endpoint.as_deref().map(|endpoint| { + Destination::new(endpoint, config.traces_endpoint.clone(), config.headers.clone()) + }))); let config = Arc::new(config); + // One pipeline per process; sessions (rooms) carry their own trace ids. This is the + // pipeline's own session, for everything emitted outside a room. + let process = SessionState::new(); let counters = Arc::new(Counters::default()); let store = Arc::new(Store::new( config.max_queue_size.max(1) as usize, @@ -229,7 +253,6 @@ impl Telemetry { let (commands, receiver) = mpsc::unbounded_channel(); let windows = Arc::new(Mutex::new(StatsWindows::default())); let guard = Arc::new(Mutex::new(FloodGuard::new(config.max_events_per_10min))); - let attributes = Arc::new(Mutex::new(Vec::new())); let spans = Arc::new(Mutex::new(Spans::new(config.max_queue_size.max(1) as usize))); let device = Arc::new(Mutex::new(None)); let exporter = Exporter::new( @@ -239,9 +262,9 @@ impl Telemetry { cache.clone(), counters.clone(), windows.clone(), - attributes.clone(), spans.clone(), - trace_id, + process.clone(), + destination.clone(), receiver, device.clone(), ); @@ -253,21 +276,46 @@ impl Telemetry { device, windows, guard, - attributes, spans, - trace_id, + process, + destination, commands, }; (telemetry, exporter) } + /// Where to send, once known (the first connect: server URL → endpoint, token → headers). + /// Everything cached so far starts uploading. Calling it again (new token, new server) + /// replaces the destination for the batches that follow. + pub fn set_destination(&self, endpoint: &str, headers: HashMap) { + *self.destination.lock().unwrap_or_else(|e| e.into_inner()) = + Some(Destination::new(endpoint, None, headers)); + let _ = self.commands.send(Command::Overflow); + } + + /// Start a session — one room, one call — with its own trace id and attributes on this + /// pipeline. Sessions do not need ending: a room's last record is simply its last. + pub fn begin_session(&self) -> Session { + Session { telemetry: self.clone(), state: SessionState::new() } + } + /// Queue an event or log record for export. Stamps it with the current time unless it /// carries one. /// /// A record with an empty `name` is a plain log line: only `Warn` and `Error` ones leave the /// device (design doc: debug/info logs never do). Discrete events are subject to the flood /// guard (`max_events_per_10min`); what it drops is counted as `rate_limited`. - pub fn emit(&self, mut event: TelemetryEvent) { + pub fn emit(&self, event: TelemetryEvent) { + // A record emitted inside a room's span belongs to that room's session; anything else + // is the process's own. + let session = event + .span_id + .and_then(|id| self.spans.lock().unwrap_or_else(|e| e.into_inner()).session_of(id)) + .unwrap_or_else(|| self.process.clone()); + self.emit_in(event, &session); + } + + pub(crate) fn emit_in(&self, mut event: TelemetryEvent, session: &Arc) { if event.name.is_empty() && matches!(event.severity, Severity::Trace | Severity::Debug | Severity::Info) { @@ -280,7 +328,7 @@ impl Telemetry { if event.timestamp_ns.is_none() { event.timestamp_ns = Some(now_unix_nanos()); } - if self.store.push(event) { + if self.store.push(Queued { event, session: session.clone() }) { let _ = self.commands.send(Command::Overflow); } } @@ -296,23 +344,34 @@ impl Telemetry { /// own `acme.call_id`), attached to every record exported from now on unless the record /// already carries the key. `None` removes it. pub fn set_attribute(&self, key: &str, value: Option) { - let mut attributes = self.attributes.lock().unwrap_or_else(|e| e.into_inner()); - attributes.retain(|a| a.key != key); - if let Some(value) = value { - attributes.push(Attribute::new(key, value)); - } + self.process.set_attribute(key, value); } /// The session's trace id as 32 hex characters — what every span and log record of this /// pipeline carries. Print it (`lkt_…`) so support can find the session. pub fn trace_id(&self) -> String { - format!("{:032x}", u128::from_be_bytes(self.trace_id)) + self.process.hex() } /// Open a span: one attempt at an operation (`lk.connect`, `lk.publish`, …). Returns the /// handle to record checkpoints and to end it with; `parent` nests it under another open span. pub fn begin_span(&self, name: &str, kind: SpanKind, parent: Option) -> u64 { - self.spans.lock().unwrap_or_else(|e| e.into_inner()).begin(name, kind, parent) + self.begin_span_in(name, kind, parent, &self.process) + } + + pub(crate) fn begin_span_in( + &self, + name: &str, + kind: SpanKind, + parent: Option, + session: &Arc, + ) -> u64 { + self.spans.lock().unwrap_or_else(|e| e.into_inner()).begin_in( + name, + kind, + parent, + session.clone(), + ) } /// Record a checkpoint inside an open span (`ws_open`, `join_recv`, …), stamped now. @@ -338,7 +397,11 @@ impl Telemetry { /// Push one `getStats()` reading. Readings are windowed on device into `lk.rtc.stats.sample` /// events (see `stats_window_ms`); they never count against the flood guard. pub fn record_stats(&self, sample: RtcStatsSample) { - self.windows.lock().unwrap_or_else(|e| e.into_inner()).record(sample); + self.record_stats_in(sample, &self.process); + } + + pub(crate) fn record_stats_in(&self, sample: RtcStatsSample, session: &Arc) { + self.windows.lock().unwrap_or_else(|e| e.into_inner()).record_in(sample, session); } /// Tell the pipeline what the device looks like. Emits the `lk.device.*.changed` events for @@ -896,6 +959,72 @@ mod tests { assert_eq!(stats.cached_batches, 1, "kept for the next attempt"); } + #[tokio::test(start_paused = true)] + async fn sessions_have_their_own_trace_and_attributes() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + let a = telemetry.begin_session(); + let b = telemetry.begin_session(); + assert_ne!(a.trace_id(), b.trace_id()); + assert_ne!(a.trace_id(), telemetry.trace_id(), "the process has its own session"); + a.set_attribute("lk.room.sid", Some("RM_a".into())); + b.set_attribute("lk.room.sid", Some("RM_b".into())); + let span = a.begin_span("lk.connect", SpanKind::Client, None); + a.emit(TelemetryEvent::new("lk.ping")); + b.emit(TelemetryEvent::new("lk.ping")); + // A warn record from the SDK logger, inside room A's connect: no session handle, just + // the ambient span id — the core files it under A. + telemetry.emit( + TelemetryEvent::new("").with_severity(Severity::Warn).with_body("hmm").in_span(span), + ); + telemetry.emit(TelemetryEvent::new("lk.device.thermal.changed")); + a.end_span(span, SpanOutcome::Ok, None, Vec::new()); + telemetry.flush().await; + + let sent = transport.sent(); + let logs = records(&sent[1]); + assert_eq!(logs.len(), 4); + assert_eq!(hex(&logs[0].trace_id), a.trace_id()); + assert_eq!(attribute(&logs[0], "lk.room.sid"), Some(Value::StringValue("RM_a".into()))); + assert_eq!(attribute(&logs[0], "session.id"), Some(Value::StringValue(a.trace_id()))); + assert_eq!(hex(&logs[1].trace_id), b.trace_id()); + assert_eq!(attribute(&logs[1], "lk.room.sid"), Some(Value::StringValue("RM_b".into()))); + assert_eq!(hex(&logs[2].trace_id), a.trace_id(), "resolved through the span"); + assert_eq!(hex(&logs[3].trace_id), telemetry.trace_id(), "device state: process session"); + assert_eq!(attribute(&logs[3], "lk.room.sid"), None); + let traces = ExportTraceServiceRequest::decode(&gunzip(&sent[0].body)[..]).expect("otlp"); + let otlp_span = &traces.resource_spans[0].scope_spans[0].spans[0]; + assert_eq!(hex(&otlp_span.trace_id), a.trace_id()); + assert!(otlp_span.attributes.iter().any(|a| a.key == "lk.room.sid")); + } + + #[tokio::test(start_paused = true)] + async fn uploads_wait_for_a_destination() { + let transport = FakeTransport::scripted([]); + let mut config = TelemetryConfig::new("unused"); + config.endpoint = None; + let telemetry = start(config, transport.clone()); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + tokio::time::sleep(Duration::from_secs(120)).await; + assert!(transport.sent().is_empty(), "no destination: nothing leaves, no hold cap either"); + assert_eq!(telemetry.stats().cached_batches, 1); + + let mut headers = HashMap::new(); + headers.insert("Authorization".to_owned(), "Bearer t".to_owned()); + telemetry.set_destination("https://x.livekit.cloud/observability/logs/otlp/v0", headers); + tokio::time::sleep(Duration::from_millis(1)).await; + let sent = transport.sent(); + assert_eq!(sent.len(), 1, "cached batches ship as soon as the destination is known"); + assert_eq!(sent[0].url, "https://x.livekit.cloud/observability/logs/otlp/v0"); + assert_eq!(sent[0].headers["Authorization"], "Bearer t"); + telemetry.begin_session().begin_span("lk.publish", SpanKind::Internal, None); + let span = telemetry.begin_span("lk.publish", SpanKind::Internal, None); + telemetry.end_span(span, SpanOutcome::Ok, None, Vec::new()); + telemetry.flush().await; + assert_eq!(transport.sent()[1].url, "https://x.livekit.cloud/observability/traces/otlp/v0"); + } + #[tokio::test(start_paused = true)] async fn debug_and_info_logs_never_leave_the_device() { let transport = FakeTransport::scripted([]); diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index 194262228..ffd5e3666 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -73,6 +73,17 @@ impl Telemetry { self.0.emit_custom(&name, attributes); } + /// Where to send, once known (first connect: server URL → endpoint, token → headers). Until + /// then everything waits in the cache; afterwards it uploads. + pub fn set_destination(&self, endpoint: String, headers: HashMap) { + self.0.set_destination(&endpoint, headers); + } + + /// A session — one room, one call — with its own trace id and attributes on this pipeline. + pub fn begin_session(&self) -> Arc { + Arc::new(TelemetrySession(self.0.begin_session())) + } + /// Report the device state (thermal, low power, foreground/background). Emits the matching /// `lk.device.*.changed` events and adapts the export cadence. pub fn set_device_state(&self, state: DeviceState) { @@ -132,6 +143,54 @@ impl Telemetry { } } +/// One room's session on the process pipeline: what its spans, stats and events are filed +/// under. Obtained from [`Telemetry::begin_session`]. +#[derive(uniffi::Object)] +pub struct TelemetrySession(livekit_telemetry::Session); + +#[uniffi::export] +impl TelemetrySession { + /// The session's trace id as 32 hex characters. + pub fn trace_id(&self) -> String { + self.0.trace_id() + } + + pub fn emit(&self, event: TelemetryEvent) { + self.0.emit(event); + } + + pub fn emit_custom(&self, name: String, attributes: Vec) { + self.0.emit_custom(&name, attributes); + } + + /// Attach an attribute to every record of this session from now on; `None` removes it. + pub fn set_attribute(&self, key: String, value: Option) { + self.0.set_attribute(&key, value); + } + + pub fn record_stats(&self, sample: RtcStatsSample) { + self.0.record_stats(sample); + } + + pub fn begin_span(&self, name: String, kind: SpanKind, parent: Option) -> u64 { + self.0.begin_span(&name, kind, parent) + } + + pub fn add_span_event(&self, span: u64, name: String, attributes: Vec) { + self.0.add_span_event(span, &name, attributes); + } + + pub fn end_span( + &self, + span: u64, + outcome: SpanOutcome, + error_type: Option, + attributes: Vec, + ) { + self.0.end_span(span, outcome, error_type, attributes); + } +} + /// One export the host has to perform on behalf of a pulled pipeline. #[derive(uniffi::Record)] pub struct PendingExport { From e82866815175c50dd5e0e35eb54788aeb3e11ff0 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:28:16 +0000 Subject: [PATCH 14/50] generated protobuf --- livekit-protocol/src/livekit.rs | 18 +----- livekit-protocol/src/livekit.serde.rs | 92 --------------------------- 2 files changed, 1 insertion(+), 109 deletions(-) diff --git a/livekit-protocol/src/livekit.rs b/livekit-protocol/src/livekit.rs index 58d025be7..e0acd6917 100644 --- a/livekit-protocol/src/livekit.rs +++ b/livekit-protocol/src/livekit.rs @@ -94,14 +94,6 @@ pub struct MetricsRecordingHeader { pub room_name: ::prost::alloc::string::String, #[prost(message, optional, tag="7")] pub room_start_time: ::core::option::Option<::pbjson_types::Timestamp>, - #[prost(string, tag="8")] - pub job_id: ::prost::alloc::string::String, - /// session is a simulation; the collector skips PII redaction for it unless redaction_enabled is set - #[prost(bool, tag="9")] - pub simulated: bool, - /// force PII redaction on for this session (only ever enables, never disables) - #[prost(bool, tag="10")] - pub redaction_enabled: bool, } // // Protocol used to record metrics for a specific session. @@ -479,9 +471,8 @@ pub mod participant_info { Forwarded = 1, ConnectorWhatsapp = 2, ConnectorTwilio = 3, + /// NEXT_ID: 5 BridgeRtsp = 4, - /// NEXT_ID: 6 - Simulation = 5, } impl KindDetail { /// String value of the enum field names used in the ProtoBuf definition. @@ -495,7 +486,6 @@ pub mod participant_info { KindDetail::ConnectorWhatsapp => "CONNECTOR_WHATSAPP", KindDetail::ConnectorTwilio => "CONNECTOR_TWILIO", KindDetail::BridgeRtsp => "BRIDGE_RTSP", - KindDetail::Simulation => "SIMULATION", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -506,7 +496,6 @@ pub mod participant_info { "CONNECTOR_WHATSAPP" => Some(Self::ConnectorWhatsapp), "CONNECTOR_TWILIO" => Some(Self::ConnectorTwilio), "BRIDGE_RTSP" => Some(Self::BridgeRtsp), - "SIMULATION" => Some(Self::Simulation), _ => None, } } @@ -4922,8 +4911,6 @@ pub struct Job { pub deployment: ::prost::alloc::string::String, #[prost(map="string, string", tag="12")] pub attributes: ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, - #[prost(bool, tag="13")] - pub enable_redaction: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -7699,9 +7686,6 @@ pub struct ConnectWhatsAppCallRequest { /// It is the answer SDP for a business initiated call #[prost(message, optional, tag="2")] pub sdp: ::core::option::Option, - /// Wait for the answer for the call before returning. - #[prost(bool, tag="3")] - pub wait_until_answered: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/livekit-protocol/src/livekit.serde.rs b/livekit-protocol/src/livekit.serde.rs index abe5e4b82..835d3052d 100644 --- a/livekit-protocol/src/livekit.serde.rs +++ b/livekit-protocol/src/livekit.serde.rs @@ -7725,9 +7725,6 @@ impl serde::Serialize for ConnectWhatsAppCallRequest { if self.sdp.is_some() { len += 1; } - if self.wait_until_answered { - len += 1; - } let mut struct_ser = serializer.serialize_struct("livekit.ConnectWhatsAppCallRequest", len)?; if !self.whatsapp_call_id.is_empty() { struct_ser.serialize_field("whatsappCallId", &self.whatsapp_call_id)?; @@ -7735,9 +7732,6 @@ impl serde::Serialize for ConnectWhatsAppCallRequest { if let Some(v) = self.sdp.as_ref() { struct_ser.serialize_field("sdp", v)?; } - if self.wait_until_answered { - struct_ser.serialize_field("waitUntilAnswered", &self.wait_until_answered)?; - } struct_ser.end() } } @@ -7751,15 +7745,12 @@ impl<'de> serde::Deserialize<'de> for ConnectWhatsAppCallRequest { "whatsapp_call_id", "whatsappCallId", "sdp", - "wait_until_answered", - "waitUntilAnswered", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { WhatsappCallId, Sdp, - WaitUntilAnswered, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -7784,7 +7775,6 @@ impl<'de> serde::Deserialize<'de> for ConnectWhatsAppCallRequest { match value { "whatsappCallId" | "whatsapp_call_id" => Ok(GeneratedField::WhatsappCallId), "sdp" => Ok(GeneratedField::Sdp), - "waitUntilAnswered" | "wait_until_answered" => Ok(GeneratedField::WaitUntilAnswered), _ => Ok(GeneratedField::__SkipField__), } } @@ -7806,7 +7796,6 @@ impl<'de> serde::Deserialize<'de> for ConnectWhatsAppCallRequest { { let mut whatsapp_call_id__ = None; let mut sdp__ = None; - let mut wait_until_answered__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::WhatsappCallId => { @@ -7821,12 +7810,6 @@ impl<'de> serde::Deserialize<'de> for ConnectWhatsAppCallRequest { } sdp__ = map_.next_value()?; } - GeneratedField::WaitUntilAnswered => { - if wait_until_answered__.is_some() { - return Err(serde::de::Error::duplicate_field("waitUntilAnswered")); - } - wait_until_answered__ = Some(map_.next_value()?); - } GeneratedField::__SkipField__ => { let _ = map_.next_value::()?; } @@ -7835,7 +7818,6 @@ impl<'de> serde::Deserialize<'de> for ConnectWhatsAppCallRequest { Ok(ConnectWhatsAppCallRequest { whatsapp_call_id: whatsapp_call_id__.unwrap_or_default(), sdp: sdp__, - wait_until_answered: wait_until_answered__.unwrap_or_default(), }) } } @@ -22563,9 +22545,6 @@ impl serde::Serialize for Job { if !self.attributes.is_empty() { len += 1; } - if self.enable_redaction { - len += 1; - } let mut struct_ser = serializer.serialize_struct("livekit.Job", len)?; if !self.id.is_empty() { struct_ser.serialize_field("id", &self.id)?; @@ -22605,9 +22584,6 @@ impl serde::Serialize for Job { if !self.attributes.is_empty() { struct_ser.serialize_field("attributes", &self.attributes)?; } - if self.enable_redaction { - struct_ser.serialize_field("enableRedaction", &self.enable_redaction)?; - } struct_ser.end() } } @@ -22633,8 +22609,6 @@ impl<'de> serde::Deserialize<'de> for Job { "enableRecording", "deployment", "attributes", - "enable_redaction", - "enableRedaction", ]; #[allow(clippy::enum_variant_names)] @@ -22651,7 +22625,6 @@ impl<'de> serde::Deserialize<'de> for Job { EnableRecording, Deployment, Attributes, - EnableRedaction, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -22686,7 +22659,6 @@ impl<'de> serde::Deserialize<'de> for Job { "enableRecording" | "enable_recording" => Ok(GeneratedField::EnableRecording), "deployment" => Ok(GeneratedField::Deployment), "attributes" => Ok(GeneratedField::Attributes), - "enableRedaction" | "enable_redaction" => Ok(GeneratedField::EnableRedaction), _ => Ok(GeneratedField::__SkipField__), } } @@ -22718,7 +22690,6 @@ impl<'de> serde::Deserialize<'de> for Job { let mut enable_recording__ = None; let mut deployment__ = None; let mut attributes__ = None; - let mut enable_redaction__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::Id => { @@ -22795,12 +22766,6 @@ impl<'de> serde::Deserialize<'de> for Job { map_.next_value::>()? ); } - GeneratedField::EnableRedaction => { - if enable_redaction__.is_some() { - return Err(serde::de::Error::duplicate_field("enableRedaction")); - } - enable_redaction__ = Some(map_.next_value()?); - } GeneratedField::__SkipField__ => { let _ = map_.next_value::()?; } @@ -22819,7 +22784,6 @@ impl<'de> serde::Deserialize<'de> for Job { enable_recording: enable_recording__.unwrap_or_default(), deployment: deployment__.unwrap_or_default(), attributes: attributes__.unwrap_or_default(), - enable_redaction: enable_redaction__.unwrap_or_default(), }) } } @@ -27165,15 +27129,6 @@ impl serde::Serialize for MetricsRecordingHeader { if self.room_start_time.is_some() { len += 1; } - if !self.job_id.is_empty() { - len += 1; - } - if self.simulated { - len += 1; - } - if self.redaction_enabled { - len += 1; - } let mut struct_ser = serializer.serialize_struct("livekit.MetricsRecordingHeader", len)?; if !self.room_id.is_empty() { struct_ser.serialize_field("roomId", &self.room_id)?; @@ -27195,15 +27150,6 @@ impl serde::Serialize for MetricsRecordingHeader { if let Some(v) = self.room_start_time.as_ref() { struct_ser.serialize_field("roomStartTime", v)?; } - if !self.job_id.is_empty() { - struct_ser.serialize_field("jobId", &self.job_id)?; - } - if self.simulated { - struct_ser.serialize_field("simulated", &self.simulated)?; - } - if self.redaction_enabled { - struct_ser.serialize_field("redactionEnabled", &self.redaction_enabled)?; - } struct_ser.end() } } @@ -27225,11 +27171,6 @@ impl<'de> serde::Deserialize<'de> for MetricsRecordingHeader { "roomName", "room_start_time", "roomStartTime", - "job_id", - "jobId", - "simulated", - "redaction_enabled", - "redactionEnabled", ]; #[allow(clippy::enum_variant_names)] @@ -27240,9 +27181,6 @@ impl<'de> serde::Deserialize<'de> for MetricsRecordingHeader { RoomTags, RoomName, RoomStartTime, - JobId, - Simulated, - RedactionEnabled, __SkipField__, } impl<'de> serde::Deserialize<'de> for GeneratedField { @@ -27271,9 +27209,6 @@ impl<'de> serde::Deserialize<'de> for MetricsRecordingHeader { "roomTags" | "room_tags" => Ok(GeneratedField::RoomTags), "roomName" | "room_name" => Ok(GeneratedField::RoomName), "roomStartTime" | "room_start_time" => Ok(GeneratedField::RoomStartTime), - "jobId" | "job_id" => Ok(GeneratedField::JobId), - "simulated" => Ok(GeneratedField::Simulated), - "redactionEnabled" | "redaction_enabled" => Ok(GeneratedField::RedactionEnabled), _ => Ok(GeneratedField::__SkipField__), } } @@ -27299,9 +27234,6 @@ impl<'de> serde::Deserialize<'de> for MetricsRecordingHeader { let mut room_tags__ = None; let mut room_name__ = None; let mut room_start_time__ = None; - let mut job_id__ = None; - let mut simulated__ = None; - let mut redaction_enabled__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::RoomId => { @@ -27344,24 +27276,6 @@ impl<'de> serde::Deserialize<'de> for MetricsRecordingHeader { } room_start_time__ = map_.next_value()?; } - GeneratedField::JobId => { - if job_id__.is_some() { - return Err(serde::de::Error::duplicate_field("jobId")); - } - job_id__ = Some(map_.next_value()?); - } - GeneratedField::Simulated => { - if simulated__.is_some() { - return Err(serde::de::Error::duplicate_field("simulated")); - } - simulated__ = Some(map_.next_value()?); - } - GeneratedField::RedactionEnabled => { - if redaction_enabled__.is_some() { - return Err(serde::de::Error::duplicate_field("redactionEnabled")); - } - redaction_enabled__ = Some(map_.next_value()?); - } GeneratedField::__SkipField__ => { let _ = map_.next_value::()?; } @@ -27374,9 +27288,6 @@ impl<'de> serde::Deserialize<'de> for MetricsRecordingHeader { room_tags: room_tags__.unwrap_or_default(), room_name: room_name__.unwrap_or_default(), room_start_time: room_start_time__, - job_id: job_id__.unwrap_or_default(), - simulated: simulated__.unwrap_or_default(), - redaction_enabled: redaction_enabled__.unwrap_or_default(), }) } } @@ -29170,7 +29081,6 @@ impl serde::Serialize for participant_info::KindDetail { Self::ConnectorWhatsapp => "CONNECTOR_WHATSAPP", Self::ConnectorTwilio => "CONNECTOR_TWILIO", Self::BridgeRtsp => "BRIDGE_RTSP", - Self::Simulation => "SIMULATION", }; serializer.serialize_str(variant) } @@ -29187,7 +29097,6 @@ impl<'de> serde::Deserialize<'de> for participant_info::KindDetail { "CONNECTOR_WHATSAPP", "CONNECTOR_TWILIO", "BRIDGE_RTSP", - "SIMULATION", ]; struct GeneratedVisitor; @@ -29233,7 +29142,6 @@ impl<'de> serde::Deserialize<'de> for participant_info::KindDetail { "CONNECTOR_WHATSAPP" => Ok(participant_info::KindDetail::ConnectorWhatsapp), "CONNECTOR_TWILIO" => Ok(participant_info::KindDetail::ConnectorTwilio), "BRIDGE_RTSP" => Ok(participant_info::KindDetail::BridgeRtsp), - "SIMULATION" => Ok(participant_info::KindDetail::Simulation), _ => Err(serde::de::Error::unknown_variant(value, FIELDS)), } } From 3c7cdcd3df75acd9dd0ea07f2e3e065f2f47539b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:21:19 +0200 Subject: [PATCH 15/50] feat(telemetry): pipeline-wide attributes `Telemetry::set_attribute` now attaches an attribute to every record of every session (an `enduser.id`, a tenant), merged at export after the session's own attributes and never overriding a record's explicit key. Session identity stays on `Session::set_attribute`. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/exporter.rs | 12 +++++++++--- livekit-telemetry/src/otlp.rs | 26 +++++++++++++++++--------- livekit-telemetry/src/session.rs | 10 +++++----- livekit-telemetry/src/telemetry.rs | 24 ++++++++++++++++++++---- livekit-uniffi/src/telemetry.rs | 4 ++-- 5 files changed, 53 insertions(+), 23 deletions(-) diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index 5e8f1c0f8..30554bd0e 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -31,7 +31,7 @@ use crate::{ stats::{Counters, Snapshot}, store::{Queued, Store}, telemetry::Destination, - AppState, BatchCache, DeviceState, ExportError, ExportRequest, TelemetryConfig, + AppState, Attribute, BatchCache, DeviceState, ExportError, ExportRequest, TelemetryConfig, TelemetryTransport, }; @@ -130,6 +130,8 @@ pub struct Exporter { spans: Arc>, /// The pipeline's own session: self-telemetry is filed under it. process: Arc, + /// Attributes attached to every record of every session (`Telemetry::set_attribute`). + global: Arc>>, /// Where batches go; `None` until `set_destination` — batches wait in the cache meanwhile. destination: Arc>>, commands: mpsc::UnboundedReceiver, @@ -167,6 +169,7 @@ impl Exporter { windows: Arc>, spans: Arc>, process: Arc, + global: Arc>>, destination: Arc>>, commands: mpsc::UnboundedReceiver, device: Arc>>, @@ -180,6 +183,7 @@ impl Exporter { windows, spans, process, + global, destination, commands, silenced: false, @@ -324,7 +328,8 @@ impl Exporter { self.force_report = false; } let count = batch.len() as u64; - let body = otlp::encode_logs(&self.config.resource, batch); + let global = self.global.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let body = otlp::encode_logs(&self.config.resource, &global, batch); self.push_batch(Signal::Logs, count, &body); } } @@ -346,7 +351,8 @@ impl Exporter { return; } let count = spans.len() as u64; - let body = otlp::encode_spans(&self.config.resource, spans); + let global = self.global.lock().unwrap_or_else(|e| e.into_inner()).clone(); + let body = otlp::encode_spans(&self.config.resource, &global, spans); self.push_batch(Signal::Traces, count, &body); } diff --git a/livekit-telemetry/src/otlp.rs b/livekit-telemetry/src/otlp.rs index bba8c068d..0bdcceb71 100644 --- a/livekit-telemetry/src/otlp.rs +++ b/livekit-telemetry/src/otlp.rs @@ -48,13 +48,17 @@ fn scope() -> Option { /// Encode one batch as an OTLP `ExportLogsServiceRequest`: one resource, one instrumentation /// scope (this crate), one log record per event. Every record carries its session's trace id and /// attributes; records emitted inside a span carry its span id too. -pub(crate) fn encode_logs(resource_attributes: &[Attribute], events: Vec) -> Vec { +pub(crate) fn encode_logs( + resource_attributes: &[Attribute], + global: &[Attribute], + events: Vec, +) -> Vec { ExportLogsServiceRequest { resource_logs: vec![ResourceLogs { resource: resource(resource_attributes), scope_logs: vec![ScopeLogs { scope: scope(), - log_records: events.into_iter().map(log_record).collect(), + log_records: events.into_iter().map(|e| log_record(e, global)).collect(), ..Default::default() }], ..Default::default() @@ -64,13 +68,17 @@ pub(crate) fn encode_logs(resource_attributes: &[Attribute], events: Vec } /// Encode finished spans as an OTLP `ExportTraceServiceRequest`, each under its session's trace id. -pub(crate) fn encode_spans(resource_attributes: &[Attribute], spans: Vec) -> Vec { +pub(crate) fn encode_spans( + resource_attributes: &[Attribute], + global: &[Attribute], + spans: Vec, +) -> Vec { ExportTraceServiceRequest { resource_spans: vec![ResourceSpans { resource: resource(resource_attributes), scope_spans: vec![ScopeSpans { scope: scope(), - spans: spans.into_iter().map(otlp_span).collect(), + spans: spans.into_iter().map(|s| otlp_span(s, global)).collect(), ..Default::default() }], ..Default::default() @@ -79,8 +87,8 @@ pub(crate) fn encode_spans(resource_attributes: &[Attribute], spans: Vec LogRecord { - session.decorate(&mut event.attributes); +fn log_record(Queued { mut event, session }: Queued, global: &[Attribute]) -> LogRecord { + session.decorate(&mut event.attributes, global); let time_unix_nano = event.timestamp_ns.unwrap_or_else(now_unix_nanos); LogRecord { time_unix_nano, @@ -96,9 +104,9 @@ fn log_record(Queued { mut event, session }: Queued) -> LogRecord { } } -fn otlp_span(mut record: SpanRecord) -> Span { +fn otlp_span(mut record: SpanRecord, global: &[Attribute]) -> Span { let session = record.session.clone(); - session.decorate(&mut record.attributes); + session.decorate(&mut record.attributes, global); let mut attributes: Vec = record.attributes.iter().map(KeyValue::from).collect(); attributes.extend(record.outcome_attributes().iter().map(KeyValue::from)); Span { @@ -187,7 +195,7 @@ mod tests { .with_body("hi") .with_attribute("lk.ping.seq", 7i64); let session = crate::session::SessionState::with_trace_id([7u8; 16]); - let bytes = encode_logs(&resource, vec![Queued { event, session }]); + let bytes = encode_logs(&resource, &[], vec![Queued { event, session }]); let decoded = ExportLogsServiceRequest::decode(&bytes[..]).expect("valid OTLP"); let resource_logs = &decoded.resource_logs[0]; diff --git a/livekit-telemetry/src/session.rs b/livekit-telemetry/src/session.rs index fe7a3fc50..91d7aba2f 100644 --- a/livekit-telemetry/src/session.rs +++ b/livekit-telemetry/src/session.rs @@ -51,12 +51,12 @@ impl SessionState { } } - /// Merge the session's attributes into a record's own without overriding explicit ones, and - /// add `session.id` (OTel semconv) — the trace id, so a record can be joined to its session - /// even where a backend drops trace ids from logs. - pub fn decorate(&self, own: &mut Vec) { + /// Merge the session's attributes, then the pipeline-wide ones (`global`), into a record's + /// own without overriding explicit ones, and add `session.id` (OTel semconv) — the trace id, + /// so a record can be joined to its session even where a backend drops trace ids from logs. + pub fn decorate(&self, own: &mut Vec, global: &[Attribute]) { let session = self.attributes.lock().unwrap_or_else(|e| e.into_inner()); - for attribute in session.iter() { + for attribute in session.iter().chain(global) { if !own.iter().any(|a| a.key == attribute.key) { own.push(attribute.clone()); } diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 85e65ecbf..bcf80c43e 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -156,6 +156,8 @@ pub struct Telemetry { spans: Arc>, /// The pipeline's own session: whatever is emitted outside a room session. process: Arc, + /// Attributes attached to every record of every session. + global: Arc>>, destination: Arc>>, commands: mpsc::UnboundedSender, } @@ -244,6 +246,7 @@ impl Telemetry { // One pipeline per process; sessions (rooms) carry their own trace ids. This is the // pipeline's own session, for everything emitted outside a room. let process = SessionState::new(); + let global = Arc::new(Mutex::new(Vec::new())); let counters = Arc::new(Counters::default()); let store = Arc::new(Store::new( config.max_queue_size.max(1) as usize, @@ -264,6 +267,7 @@ impl Telemetry { windows.clone(), spans.clone(), process.clone(), + global.clone(), destination.clone(), receiver, device.clone(), @@ -278,6 +282,7 @@ impl Telemetry { guard, spans, process, + global, destination, commands, }; @@ -340,11 +345,16 @@ impl Telemetry { self.emit(TelemetryEvent::custom(name, attributes)); } - /// Set a session-wide attribute (`lk.room.sid`, `lk.participant.identity`, or a consumer's - /// own `acme.call_id`), attached to every record exported from now on unless the record - /// already carries the key. `None` removes it. + /// Set a pipeline-wide attribute (a consumer's `enduser.id`, an `acme.tenant`), attached to + /// every record of every session from now on unless the record — or its session — already + /// carries the key. `None` removes it. Session-level identity goes through + /// [`Session::set_attribute`]. pub fn set_attribute(&self, key: &str, value: Option) { - self.process.set_attribute(key, value); + let mut global = self.global.lock().unwrap_or_else(|e| e.into_inner()); + global.retain(|a| a.key != key); + if let Some(value) = value { + global.push(Attribute::new(key, value)); + } } /// The session's trace id as 32 hex characters — what every span and log record of this @@ -963,6 +973,7 @@ mod tests { async fn sessions_have_their_own_trace_and_attributes() { let transport = FakeTransport::scripted([]); let telemetry = pipeline(transport.clone()); + telemetry.set_attribute("acme.tenant", Some("t1".into())); let a = telemetry.begin_session(); let b = telemetry.begin_session(); assert_ne!(a.trace_id(), b.trace_id()); @@ -992,6 +1003,11 @@ mod tests { assert_eq!(hex(&logs[2].trace_id), a.trace_id(), "resolved through the span"); assert_eq!(hex(&logs[3].trace_id), telemetry.trace_id(), "device state: process session"); assert_eq!(attribute(&logs[3], "lk.room.sid"), None); + assert!( + logs.iter() + .all(|r| attribute(r, "acme.tenant") == Some(Value::StringValue("t1".into()))), + "a pipeline-wide attribute reaches every session" + ); let traces = ExportTraceServiceRequest::decode(&gunzip(&sent[0].body)[..]).expect("otlp"); let otlp_span = &traces.resource_spans[0].scope_spans[0].spans[0]; assert_eq!(hex(&otlp_span.trace_id), a.trace_id()); diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index ffd5e3666..464730692 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -90,8 +90,8 @@ impl Telemetry { self.0.set_device_state(state); } - /// Set (or, with `None`, remove) a session-wide attribute attached to every record from now - /// on: `lk.room.sid`, `lk.participant.identity`, or the app's own correlation ids. + /// A pipeline-wide attribute (`enduser.id`, `acme.tenant`), attached to every record of every + /// session unless the record or its session already carries the key; `None` removes it. pub fn set_attribute(&self, key: String, value: Option) { self.0.set_attribute(&key, value); } From cf3f3033783a930c50ffb09aa23fc1da1528dd67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:37:42 +0200 Subject: [PATCH 16/50] feat(telemetry): events without a body carry their name as body Log viewers key their line on the body and not every backend surfaces event_name yet, so attribute-only events (RTC windows, device changes) rendered as empty lines in Loki. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/SPEC.md | 3 +++ livekit-telemetry/src/otlp.rs | 19 ++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index e21afbd1a..433fb66f9 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -36,6 +36,9 @@ only by the cache. ## Events +An event with no `body` is exported with its name as the body as well as in `event_name`: log +viewers key their line on the body, and not every backend surfaces `event_name` yet. + ```yaml event: lk.ping area: sdk diff --git a/livekit-telemetry/src/otlp.rs b/livekit-telemetry/src/otlp.rs index 0bdcceb71..b61d23504 100644 --- a/livekit-telemetry/src/otlp.rs +++ b/livekit-telemetry/src/otlp.rs @@ -90,12 +90,15 @@ pub(crate) fn encode_spans( fn log_record(Queued { mut event, session }: Queued, global: &[Attribute]) -> LogRecord { session.decorate(&mut event.attributes, global); let time_unix_nano = event.timestamp_ns.unwrap_or_else(now_unix_nanos); + // An event without a body still needs a line: backends that key the UI on the body (Loki, + // most log viewers) would show it empty, and `event_name` is not surfaced everywhere yet. + let body = event.body.or_else(|| (!event.name.is_empty()).then(|| event.name.clone())); LogRecord { time_unix_nano, observed_time_unix_nano: time_unix_nano, severity_number: SeverityNumber::from(event.severity) as i32, severity_text: severity_text(event.severity).to_owned(), - body: event.body.map(|text| AnyValue { value: Some(any_value::Value::StringValue(text)) }), + body: body.map(|text| AnyValue { value: Some(any_value::Value::StringValue(text)) }), attributes: event.attributes.iter().map(KeyValue::from).collect(), event_name: event.name, trace_id: session.trace_id.to_vec(), @@ -216,4 +219,18 @@ mod tests { Some(any_value::Value::IntValue(7)) ); } + + #[test] + fn events_without_a_body_carry_their_name_as_body() { + let session = crate::session::SessionState::with_trace_id([7u8; 16]); + let event = TelemetryEvent::new("lk.rtc.stats.sample"); + let bytes = encode_logs(&[], &[], vec![Queued { event, session }]); + let decoded = ExportLogsServiceRequest::decode(&bytes[..]).expect("valid OTLP"); + let record = &decoded.resource_logs[0].scope_logs[0].log_records[0]; + assert_eq!(record.event_name, "lk.rtc.stats.sample"); + assert_eq!( + record.body.as_ref().and_then(|b| b.value.clone()), + Some(any_value::Value::StringValue("lk.rtc.stats.sample".into())) + ); + } } From c8e15f4bd6b77f2473b301077a92d299c0c15345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:44:51 +0200 Subject: [PATCH 17/50] feat(telemetry): display bodies on every event, otel.event.name, log_severity Events left the device with attributes only; OTel calls an event's body its display message, log viewers key their line on it, and Loki has no EventName mapping yet, so RTC windows and device changes rendered as blank info lines. Every event now carries a one-line summary - `video outbound: 1204 kbps, loss 0.4%, rtt 48 ms, 30 fps`, `battery: 95%, charging`, `telemetry: 7 batches sent (4223 B), 0 failed, 0 dropped, 0 cached` - and `otel.event.name` (semconv 1.39) so backends without EventName can still tell events from logs. The name remains the last-resort body. `TelemetryConfig.log_severity` (default warn) is the threshold a plain log record needs to leave the device; events are not subject to it. Severity is now ordered. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/device.rs | 18 +++++++ livekit-telemetry/src/event.rs | 4 +- livekit-telemetry/src/otlp.rs | 8 ++- livekit-telemetry/src/rtc.rs | 49 +++++++++++++++++-- livekit-telemetry/src/stats.rs | 8 +++ livekit-telemetry/src/telemetry.rs | 9 ++-- .../support/dart/test/telemetry_test.dart | 3 +- 7 files changed, 86 insertions(+), 13 deletions(-) diff --git a/livekit-telemetry/src/device.rs b/livekit-telemetry/src/device.rs index 1e4e10da8..fbb78747b 100644 --- a/livekit-telemetry/src/device.rs +++ b/livekit-telemetry/src/device.rs @@ -145,24 +145,31 @@ impl DeviceState { if previous.is_none_or(|p| p.thermal != self.thermal) { events.push( TelemetryEvent::new("lk.device.thermal.changed") + .with_body(format!("thermal: {}", self.thermal.as_str())) .with_attribute("lk.device.thermal.state", self.thermal.as_str()), ); } if previous.is_none_or(|p| p.low_power_mode != self.low_power_mode) { events.push( TelemetryEvent::new("lk.device.low_power.changed") + .with_body(format!( + "low power mode: {}", + if self.low_power_mode { "on" } else { "off" } + )) .with_attribute("lk.device.low_power.enabled", self.low_power_mode), ); } if previous.is_none_or(|p| p.app_state != self.app_state) { events.push( TelemetryEvent::new("lk.device.app_state.changed") + .with_body(format!("app: {}", self.app_state.as_str())) .with_attribute("lk.device.app_state", self.app_state.as_str()), ); } if previous.is_none_or(|p| p.memory != self.memory) { events.push( TelemetryEvent::new("lk.device.memory.changed") + .with_body(format!("memory pressure: {}", self.memory.as_str())) .with_attribute("lk.device.memory.pressure", self.memory.as_str()), ); } @@ -172,6 +179,12 @@ impl DeviceState { }) { events.push( TelemetryEvent::new("lk.device.network.changed") + .with_body(format!( + "network: {}{}{}", + self.network.as_str(), + if self.network_expensive { ", expensive" } else { "" }, + if self.network_constrained { ", constrained" } else { "" } + )) .with_attribute("network.connection.type", self.network.as_str()) .with_attribute("lk.device.network.expensive", self.network_expensive) .with_attribute("lk.device.network.constrained", self.network_constrained), @@ -187,6 +200,11 @@ impl DeviceState { { events.push( TelemetryEvent::new("lk.device.battery.changed") + .with_body(format!( + "battery: {}%{}", + self.battery_level.unwrap_or(0), + if self.battery_charging { ", charging" } else { "" } + )) .with_attribute( "lk.device.battery.level", self.battery_level.unwrap_or(0) as i64, diff --git a/livekit-telemetry/src/event.rs b/livekit-telemetry/src/event.rs index c7c356ebd..533778392 100644 --- a/livekit-telemetry/src/event.rs +++ b/livekit-telemetry/src/event.rs @@ -80,7 +80,7 @@ impl TelemetryEvent { /// attributes keep the caller's namespace (`acme.checkout.step`). pub fn custom(name: &str, attributes: Vec) -> Self { let name = format!("custom.{}", name.trim_start_matches("custom.")); - Self { attributes, ..Self::new(name) } + Self { attributes, body: Some(name.clone()), ..Self::new(name) } } /// Rough encoded size — strings plus a fixed overhead per field. Drives the byte bounds on @@ -98,7 +98,7 @@ impl TelemetryEvent { /// Event severity, mapped onto the OTel severity numbers (`TRACE`=1 … `ERROR`=17). #[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub enum Severity { Trace, Debug, diff --git a/livekit-telemetry/src/otlp.rs b/livekit-telemetry/src/otlp.rs index b61d23504..f2a9f2c68 100644 --- a/livekit-telemetry/src/otlp.rs +++ b/livekit-telemetry/src/otlp.rs @@ -90,9 +90,13 @@ pub(crate) fn encode_spans( fn log_record(Queued { mut event, session }: Queued, global: &[Attribute]) -> LogRecord { session.decorate(&mut event.attributes, global); let time_unix_nano = event.timestamp_ns.unwrap_or_else(now_unix_nanos); - // An event without a body still needs a line: backends that key the UI on the body (Loki, - // most log viewers) would show it empty, and `event_name` is not surfaced everywhere yet. + // Events carry a display body (OTel: "a string display message of the event"); the name is + // the last resort so no event ever renders as an empty line. `otel.event.name` (semconv 1.39) + // duplicates `EventName` for backends that do not surface the field yet — Loki included. let body = event.body.or_else(|| (!event.name.is_empty()).then(|| event.name.clone())); + if !event.name.is_empty() { + event.attributes.push(Attribute::new("otel.event.name", event.name.clone())); + } LogRecord { time_unix_nano, observed_time_unix_nano: time_unix_nano, diff --git a/livekit-telemetry/src/rtc.rs b/livekit-telemetry/src/rtc.rs index c0e0ad871..562e76236 100644 --- a/livekit-telemetry/src/rtc.rs +++ b/livekit-telemetry/src/rtc.rs @@ -151,6 +151,8 @@ struct Window { session: Arc, start_ns: u64, samples: u32, + /// Cumulative counters at the window's first reading, for the display body's deltas. + first: RtcStatsSample, last: RtcStatsSample, jitter: Gauge, rtt: Gauge, @@ -164,6 +166,7 @@ impl Window { session, start_ns, samples: 0, + first: first.clone(), last: first.clone(), jitter: Gauge::default(), rtt: Gauge::default(), @@ -174,6 +177,40 @@ impl Window { window } + /// `video outbound: 1204 kbps, loss 0.4%, rtt 48 ms, 30 fps` — deltas over the window. + fn summary(&self, window_ms: u64) -> String { + let mut parts = Vec::new(); + let delta = |a: Option, b: Option| Some(a?.saturating_sub(b?)); + if let (Some(bytes), true) = (delta(self.last.bytes, self.first.bytes), window_ms > 0) { + parts.push(format!("{} kbps", bytes * 8 / window_ms)); + } + if let (Some(lost), Some(packets)) = ( + delta(self.last.packets_lost, self.first.packets_lost), + delta(self.last.packets, self.first.packets), + ) { + if lost + packets > 0 { + parts.push(format!("loss {:.1}%", lost as f64 * 100.0 / (lost + packets) as f64)); + } + } + if self.rtt.n > 0 { + parts.push(format!("rtt {:.0} ms", self.rtt.sum / self.rtt.n as f64)); + } + if self.fps.n > 0 { + parts.push(format!("{:.0} fps", self.fps.sum / self.fps.n as f64)); + } + if let Some(freezes) = + delta(self.last.freeze_count, self.first.freeze_count).filter(|f| *f > 0) + { + parts.push(format!("{freezes} freezes")); + } + format!( + "{} {}: {}", + self.last.kind.as_str(), + self.last.direction.as_str(), + if parts.is_empty() { "no data".to_owned() } else { parts.join(", ") } + ) + } + fn add(&mut self, sample: RtcStatsSample) { self.samples += 1; self.jitter.add(sample.jitter_ms); @@ -183,17 +220,19 @@ impl Window { self.last = sample; } - /// The `lk.rtc.stats.sample` event for this window, stamped at `end_ns`. + /// The `lk.rtc.stats.sample` event for this window, stamped at `end_ns`. Its body is the + /// one-line human summary a log view shows (OTel: an event's body is its display message); + /// the attributes carry the numbers. fn into_event(self, end_ns: u64) -> TelemetryEvent { + let window_ms = end_ns.saturating_sub(self.start_ns) / 1_000_000; + let body = self.summary(window_ms); let last = self.last; let mut event = TelemetryEvent::new("lk.rtc.stats.sample") + .with_body(body) .with_attribute("lk.track.sid", last.track_sid) .with_attribute("lk.track.kind", last.kind.as_str()) .with_attribute("lk.track.direction", last.direction.as_str()) - .with_attribute( - "lk.rtc.window_ms", - (end_ns.saturating_sub(self.start_ns) / 1_000_000) as i64, - ) + .with_attribute("lk.rtc.window_ms", window_ms as i64) .with_attribute("lk.rtc.samples", self.samples as i64); if let Some(codec) = last.codec { event = event.with_attribute("lk.rtc.codec", codec); diff --git a/livekit-telemetry/src/stats.rs b/livekit-telemetry/src/stats.rs index 9a6087462..49e3baf3c 100644 --- a/livekit-telemetry/src/stats.rs +++ b/livekit-telemetry/src/stats.rs @@ -139,6 +139,14 @@ impl Snapshot { /// computed from. pub fn report(&self, cached_batches: u64) -> TelemetryEvent { let mut event = TelemetryEvent::new("lk.telemetry.report") + .with_body(format!( + "telemetry: {} batches sent ({} B), {} failed, {} dropped, {} cached", + self.uploads_sent, + self.upload_bytes, + self.upload_failures + self.upload_timeouts, + self.dropped() - self.disabled, + cached_batches + )) .with_attribute("lk.telemetry.uploads.sent", self.uploads_sent as i64) .with_attribute("lk.telemetry.uploads.bytes", self.upload_bytes as i64) .with_attribute("lk.telemetry.uploads.failed", self.upload_failures as i64) diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index bcf80c43e..2e09fc8f4 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -96,6 +96,10 @@ pub struct TelemetryConfig { /// Cap on one request's payload before compression (design doc: "single POST ≤ 1 MB"). #[cfg_attr(feature = "uniffi", uniffi(default = 1048576))] pub max_batch_bytes: u64, + /// Lowest severity a plain log record (an event with no name) needs to leave the device. + /// Events are not subject to it. Design doc: warn. + // No uniffi default: enum defaults are not supported by the Swift generator (uniffi 0.31). + pub log_severity: Severity, } impl TelemetryConfig { @@ -117,6 +121,7 @@ impl TelemetryConfig { max_batches_per_upload: 4, flush_threshold_bytes: 256 * 1024, max_batch_bytes: 1024 * 1024, + log_severity: Severity::Warn, } } } @@ -321,9 +326,7 @@ impl Telemetry { } pub(crate) fn emit_in(&self, mut event: TelemetryEvent, session: &Arc) { - if event.name.is_empty() - && matches!(event.severity, Severity::Trace | Severity::Debug | Severity::Info) - { + if event.name.is_empty() && event.severity < self.config.log_severity { return; } if !self.guard.lock().unwrap_or_else(|e| e.into_inner()).admit() { diff --git a/livekit-uniffi/support/dart/test/telemetry_test.dart b/livekit-uniffi/support/dart/test/telemetry_test.dart index 046c84272..7472867b3 100644 --- a/livekit-uniffi/support/dart/test/telemetry_test.dart +++ b/livekit-uniffi/support/dart/test/telemetry_test.dart @@ -27,6 +27,7 @@ void main() { endpoint: 'http://collector/v1/logs', headers: {'Authorization': 'Bearer test'}, resource: [], + logSeverity: Severity.warn, ), queue: queue, ); @@ -60,7 +61,7 @@ void main() { test('refuses to start without any transport', () { expect( () => Telemetry( - config: TelemetryConfig(endpoint: 'http://collector/v1/logs', headers: {}, resource: []), + config: TelemetryConfig(endpoint: 'http://collector/v1/logs', headers: {}, resource: [], logSeverity: Severity.warn), transport: null, ), throwsA(anything), From 6ebc008ba6deafd9e26d7f423329f92baaec8bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:59:27 +0200 Subject: [PATCH 18/50] fix(telemetry): file late log records under their ended span's session A record naming a span is filed under that span's session, but the lookup only knew open spans. SDK log paths hop threads, so a warning logged right before its span ends (a failing publish) arrived after the span had moved on and fell into the process session. Spans now remember their session for the last 1024 spans, open, ended or exported. Found by the e2e test that checks span-to-log attribution. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/span.rs | 24 ++++++++++++++++++++++-- livekit-telemetry/src/telemetry.rs | 10 ++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/livekit-telemetry/src/span.rs b/livekit-telemetry/src/span.rs index c90f930d5..dc789b567 100644 --- a/livekit-telemetry/src/span.rs +++ b/livekit-telemetry/src/span.rs @@ -92,10 +92,20 @@ pub(crate) struct Spans { open_order: VecDeque, finished: Vec, finished_capacity: usize, + /// Which session every recent span belongs to — open, finished or already exported — so a + /// log record that arrives after its span ended (a warning logged right before a failing + /// publish ends, delivered a hop later) is still filed under the right session. + sessions: HashMap>, + session_order: VecDeque, next_id: u64, pub dropped: u64, } +/// Spans whose session stays resolvable after they ended. +// ponytail: a fixed ring; a time-based expiry if a long session ever opens more spans than this +// between a log and its export. +const REMEMBERED_SPANS: usize = 1024; + impl Spans { pub fn new(finished_capacity: usize) -> Self { Self { @@ -103,6 +113,8 @@ impl Spans { open_order: VecDeque::new(), finished: Vec::new(), finished_capacity, + sessions: HashMap::new(), + session_order: VecDeque::new(), // Span ids must be non-zero (OTLP treats all-zero as absent); start at 1 and mix in // randomness so ids from two pipelines in one process never collide. next_id: rand::random::() | 1, @@ -126,6 +138,13 @@ impl Spans { self.dropped += 1; } } + self.sessions.insert(id, session.clone()); + self.session_order.push_back(id); + if self.session_order.len() > REMEMBERED_SPANS { + if let Some(old) = self.session_order.pop_front() { + self.sessions.remove(&old); + } + } let record = SpanRecord { span_id: id, parent_span_id: parent.filter(|p| *p != 0), @@ -146,9 +165,10 @@ impl Spans { /// Whether a span with one of these names is still open (the exporter holds uploads while /// `lk.connect` / `lk.reconnect` are). - /// The session an open span belongs to (log records emitted inside a span are filed there). + /// The session a recent span belongs to — open, ended or exported (log records emitted inside + /// a span are filed there, and they may arrive after the span ended). pub fn session_of(&self, id: u64) -> Option> { - self.open.get(&id).map(|span| span.session.clone()) + self.sessions.get(&id).cloned() } #[cfg(test)] diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 2e09fc8f4..4d50cd34f 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -1015,6 +1015,16 @@ mod tests { let otlp_span = &traces.resource_spans[0].scope_spans[0].spans[0]; assert_eq!(hex(&otlp_span.trace_id), a.trace_id()); assert!(otlp_span.attributes.iter().any(|a| a.key == "lk.room.sid")); + + // A record that names a span which has already ended — and been exported — is still that + // session's: the SDK's log path hops threads, the span does not wait for it. + telemetry.emit( + TelemetryEvent::new("").with_severity(Severity::Error).with_body("late").in_span(span), + ); + telemetry.flush().await; + let late = &records(&transport.sent()[2])[0]; + assert_eq!(hex(&late.trace_id), a.trace_id(), "filed under the ended span's session"); + assert_eq!(late.span_id, span.to_be_bytes().to_vec()); } #[tokio::test(start_paused = true)] From 5c69748d6b79fa3059b48714b983408f538ac70a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:15:10 +0200 Subject: [PATCH 19/50] chore(telemetry): upload policy decisions at debug level Holds, the hold cap, pauses after failures, the missing destination and every sent batch now log at debug, so a host's console shows why nothing is leaving the device. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/exporter.rs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index 30554bd0e..b36845b57 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -405,11 +405,22 @@ impl Exporter { /// Send cached batches oldest-first, within this tick's budget, until one fails; then back /// off. async fn upload(&mut self) { - if self.silenced || self.paused_until.is_some_and(|t| Instant::now() < t) { + if self.silenced { + return; + } + if let Some(until) = self.paused_until.filter(|t| Instant::now() < *t) { + log::debug!( + "telemetry: upload paused for {}s more after a failure", + (until - Instant::now()).as_secs() + ); return; } // No destination yet (SDK started, no room connected): everything waits in the cache. if self.destination.lock().unwrap_or_else(|e| e.into_inner()).is_none() { + log::debug!( + "telemetry: no destination yet; {} batches wait", + self.cache.pending().len() + ); return; } let pending = self.cache.pending(); @@ -421,9 +432,13 @@ impl Exporter { Some(reason) => { let since = *self.held_since.get_or_insert_with(Instant::now); if since.elapsed() < MAX_HOLD { - log::trace!("telemetry: holding {} batches: {reason}", pending.len()); + log::debug!("telemetry: holding {} batches: {reason}", pending.len()); return; } + log::debug!( + "telemetry: hold capped after {}s ({reason}); sending one batch", + MAX_HOLD.as_secs() + ); // Held long enough: one batch goes out, then the hold starts over. self.held_since = Some(Instant::now()); Counters::add(&self.counters.hold_cap_hits, 1); @@ -445,6 +460,12 @@ impl Exporter { }; match self.deliver(&body, Signal::of(&id)).await { Delivery::Sent => { + log::debug!( + "telemetry: sent {} ({} B, {} left)", + id, + body.len(), + self.cache.pending().len() + ); self.cache.remove(&id); Counters::add(&self.counters.uploads_sent, 1); Counters::add(&self.counters.upload_bytes, body.len() as u64); From 646462c62c4bcb1e8705080df3aaf15d710ad9f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:22:42 +0200 Subject: [PATCH 20/50] fix(telemetry): bandwidth limitation no longer holds uploads WebRTC reports qualityLimitationReason=bandwidth for minutes during a normal ramp-up and for as long as an encoder stalls. On a real iPhone a camera track at 0 kbps kept the hold alive for 8 minutes; only the 60 s cap let one batch per minute out while the cache grew by 11 per minute. Yielding to media is the transport's job (Priority: u=7, background service class). The cpu counter still stretches the cadence. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/SPEC.md | 11 ++--- livekit-telemetry/src/exporter.rs | 13 ++---- livekit-telemetry/src/rtc.rs | 71 +++++++++++------------------- livekit-telemetry/src/telemetry.rs | 10 ++--- 4 files changed, 39 insertions(+), 66 deletions(-) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index 433fb66f9..a08100dfd 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -186,13 +186,14 @@ Uploads are shaped, not just batched: ≈ 40 kbps next to a call. `shutdown` drains without the budget. - **Holds** — nothing is sent, everything keeps flowing into the write-ahead cache — while: - an `lk.connect` or `lk.reconnect` span is open (signaling and ICE/DTLS own the uplink), - - WebRTC reported an outbound track bandwidth-limited within the last 10 s - (`qualityLimitationDurations.bandwidth` grew — the congestion controller is already - holding the encoder back), - the device asks for quiet: constrained network, or battery ≤ 10 % unplugged (the Datadog - rule). Device holds survive `shutdown`; the other two end with the call. + rule). Device holds survive `shutdown`; the connect hold ends with the call. A hold lasts at most 60 s, then one batch goes out and the hold starts over — the hard cap - that bounds the policy when its signals lie. + that bounds the policy when its signals lie. `qualityLimitationDurations.bandwidth` is + deliberately *not* a hold: WebRTC reports it for minutes during a normal ramp-up and for as + long as an encoder stalls (an iPhone camera at 0 kbps held uploads for 8 minutes, one batch + per minute through the cap). Yielding to media on the wire is the transport's job + (`Priority: u=7`, Apple's background service class). - **Bytes:** bodies are gzipped (level 1, `Content-Encoding: gzip`) when cached, so a batch is 5–10× smaller on disk and on the wire and a replay costs no CPU. A request never carries more than `max_batch_bytes` (1 MiB, estimated before compression) or `max_batch_size` (512) records; diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index b36845b57..6e3c9ff4e 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -109,9 +109,9 @@ enum Delivery { /// /// Telemetry must never win over media, so uploads are shaped as well as batched: at most /// `max_batches_per_upload` per tick while a session may be live, and none at all while the room -/// is connecting or reconnecting, while WebRTC reports the encoder bandwidth-limited, or while the -/// device asks for quiet ([`DeviceState::holds_uploads`]) — bounded by [`MAX_HOLD`]. Every request -/// carries `Priority: u=7` (RFC 9218) and a gzipped body. +/// is connecting or reconnecting or while the device asks for quiet ([`DeviceState::holds_uploads`]) +/// — bounded by [`MAX_HOLD`]. Yielding to media on the wire is the transport's job: every request +/// carries `Priority: u=7` (RFC 9218), a gzipped body, and (on Apple) the background service class. /// /// The tick period is `flush_interval_ms × cadence factor`: device pressure and a CPU-limited /// encoder stretch it up to 4×, and entering the background flushes once immediately (the app @@ -261,8 +261,7 @@ impl Exporter { /// Device pressure, doubled again while WebRTC reports the encoder CPU-limited; capped at 4×. fn cadence_factor(&self) -> u32 { - let cpu_limited = - self.windows.lock().unwrap_or_else(|e| e.into_inner()).media_pressure().cpu_limited; + let cpu_limited = self.windows.lock().unwrap_or_else(|e| e.into_inner()).cpu_limited(); (self.device().cadence_factor() * if cpu_limited { 2 } else { 1 }).min(4) } @@ -395,10 +394,6 @@ impl Exporter { if self.spans.lock().unwrap_or_else(|e| e.into_inner()).any_open(SENSITIVE_SPANS) { return Some("connecting"); } - let media = self.windows.lock().unwrap_or_else(|e| e.into_inner()).media_pressure(); - if media.bandwidth_limited { - return Some("media is bandwidth-limited"); - } None } diff --git a/livekit-telemetry/src/rtc.rs b/livekit-telemetry/src/rtc.rs index 562e76236..93cd83533 100644 --- a/livekit-telemetry/src/rtc.rs +++ b/livekit-telemetry/src/rtc.rs @@ -273,27 +273,16 @@ impl Window { #[derive(Default)] pub(crate) struct StatsWindows { windows: HashMap<(String, StreamDirection), Window>, - /// Last cumulative `qualityLimitationDurations` per outbound track: (bandwidth_ms, cpu_ms). + /// Last cumulative `qualityLimitationDurations.cpu` per outbound track. // ponytail: grows with the tracks published in a session (a few dozen at most). - limitation: HashMap, - bandwidth_limited_until: Option, + limitation: HashMap, cpu_limited_until: Option, } -/// What WebRTC's own adaptation says about the call, derived from the -/// `qualityLimitationDurations` counters of outbound tracks — no extra measurement, and exactly -/// the cpu-vs-bandwidth split the design doc asks for. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub(crate) struct MediaPressure { - /// The congestion controller is holding the encoder back: uploads must not add to the uplink. - pub bandwidth_limited: bool, - /// The encoder is CPU-starved: stretch the cadence, like thermal pressure. - pub cpu_limited: bool, -} - -/// Congestion is bursty: hold uploads for a short while after the last sign of it. -const BANDWIDTH_LIMITED_HOLD: Duration = Duration::from_secs(10); /// CPU starvation is sticky: stretch the cadence for a while after the last sign of it. +/// `qualityLimitationDurations.bandwidth` is deliberately not a signal: WebRTC reports it for +/// minutes during a normal ramp-up and for as long as an encoder stalls (an iPhone camera at +/// 0 kbps held uploads for 8 minutes); the congestion controller does not need our help. const CPU_LIMITED_HOLD: Duration = Duration::from_secs(60); impl StatsWindows { @@ -331,31 +320,22 @@ impl StatsWindows { self.close().into_iter().map(|q| q.event).collect() } - pub fn media_pressure(&self) -> MediaPressure { - let now = Instant::now(); - MediaPressure { - bandwidth_limited: self.bandwidth_limited_until.is_some_and(|t| now < t), - cpu_limited: self.cpu_limited_until.is_some_and(|t| now < t), - } + /// The encoder was CPU-starved within the last minute: stretch the cadence, like thermal + /// pressure. + pub fn cpu_limited(&self) -> bool { + self.cpu_limited_until.is_some_and(|t| Instant::now() < t) } - /// A limitation counter that grew since the previous reading means the encoder was held - /// back in between. + /// A cpu limitation counter that grew since the previous reading means the encoder was + /// starved in between. fn track_limitation(&mut self, sample: &RtcStatsSample) { if sample.direction != StreamDirection::Outbound { return; } - let current = ( - sample.quality_limitation_bandwidth_ms.unwrap_or(0), - sample.quality_limitation_cpu_ms.unwrap_or(0), - ); + let current = sample.quality_limitation_cpu_ms.unwrap_or(0); if let Some(previous) = self.limitation.insert(sample.track_sid.clone(), current) { - let now = Instant::now(); - if current.0 > previous.0 { - self.bandwidth_limited_until = Some(now + BANDWIDTH_LIMITED_HOLD); - } - if current.1 > previous.1 { - self.cpu_limited_until = Some(now + CPU_LIMITED_HOLD); + if current > previous { + self.cpu_limited_until = Some(Instant::now() + CPU_LIMITED_HOLD); } } } @@ -384,7 +364,7 @@ mod tests { use super::*; #[tokio::test(start_paused = true)] - async fn limitation_counters_drive_media_pressure() { + async fn cpu_limitation_counter_drives_cadence_pressure() { let mut windows = StatsWindows::default(); let reading = |bandwidth_ms, cpu_ms| RtcStatsSample { quality_limitation_bandwidth_ms: Some(bandwidth_ms), @@ -392,21 +372,20 @@ mod tests { ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound) }; windows.record(reading(0, 0)); - assert_eq!(windows.media_pressure(), MediaPressure::default(), "first reading: no delta"); - windows.record(reading(500, 0)); - assert!(windows.media_pressure().bandwidth_limited); - assert!(!windows.media_pressure().cpu_limited); - tokio::time::advance(Duration::from_secs(11)).await; - assert!(!windows.media_pressure().bandwidth_limited, "congestion hold expires"); - windows.record(reading(500, 250)); - assert!(windows.media_pressure().cpu_limited); + assert!(!windows.cpu_limited(), "first reading: no delta"); + windows.record(reading(5_000, 0)); + assert!(!windows.cpu_limited(), "bandwidth limitation is not pressure"); + windows.record(reading(5_000, 250)); + assert!(windows.cpu_limited()); tokio::time::advance(Duration::from_secs(59)).await; - assert!(windows.media_pressure().cpu_limited, "cpu hold is sticky"); + assert!(windows.cpu_limited(), "cpu hold is sticky"); + tokio::time::advance(Duration::from_secs(2)).await; + assert!(!windows.cpu_limited(), "and expires"); let mut inbound = RtcStatsSample::new("TR_2", TrackKind::Audio, StreamDirection::Inbound); - inbound.quality_limitation_bandwidth_ms = Some(9_999); + inbound.quality_limitation_cpu_ms = Some(9_999); windows.record(inbound.clone()); windows.record(inbound); - assert!(!windows.media_pressure().bandwidth_limited, "inbound counters are ignored"); + assert!(!windows.cpu_limited(), "inbound counters are ignored"); } use crate::AttributeValue; diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 4d50cd34f..24923b85f 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -825,7 +825,9 @@ mod tests { } #[tokio::test(start_paused = true)] - async fn uploads_hold_while_media_is_bandwidth_limited() { + async fn bandwidth_limitation_does_not_hold_uploads() { + // WebRTC reports `bandwidth` for minutes during ramp-up and for as long as an encoder + // stalls; holding on it starved a real device of uploads for 8 minutes. let transport = FakeTransport::scripted([]); let telemetry = pipeline(transport.clone()); let limited = |ms| RtcStatsSample { @@ -836,11 +838,7 @@ mod tests { telemetry.record_stats(limited(800)); telemetry.emit(TelemetryEvent::new("lk.ping")); telemetry.flush().await; - assert!(transport.sent().is_empty(), "the congestion controller is already throttling"); - - tokio::time::sleep(Duration::from_secs(11)).await; - telemetry.flush().await; - assert_eq!(transport.sent().len(), 1, "10 s without new limitation: resume"); + assert_eq!(transport.sent().len(), 1, "yielding to media is the transport's job"); } #[tokio::test(start_paused = true)] From f486eb84304d19dce38defb8450080bb0a15abe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:28:06 +0200 Subject: [PATCH 21/50] feat(telemetry): Display for TelemetryStats, diagnostics() over UniFFI Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/stats.rs | 25 +++++++++++++++++++++++++ livekit-uniffi/src/telemetry.rs | 5 +++++ 2 files changed, 30 insertions(+) diff --git a/livekit-telemetry/src/stats.rs b/livekit-telemetry/src/stats.rs index 49e3baf3c..f3ea4b7da 100644 --- a/livekit-telemetry/src/stats.rs +++ b/livekit-telemetry/src/stats.rs @@ -196,6 +196,31 @@ pub struct TelemetryStats { pub cached_batches: u64, } +/// One line for a debug console. +impl std::fmt::Display for TelemetryStats { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "telemetry: sent {} ({} B), cached {}, failed {}, timeouts {}, holds capped {}, dropped {} \ + (queue {}, cache {}/{}, rejected {}, throttled {}, rate-limited {}, disabled {})", + self.uploads_sent, + self.upload_bytes, + self.cached_batches, + self.upload_failures, + self.upload_timeouts, + self.holds_capped, + self.dropped, + self.dropped_queue_full, + self.dropped_cache_full, + self.dropped_cache_error, + self.dropped_rejected, + self.dropped_throttled, + self.dropped_rate_limited, + self.dropped_disabled, + ) + } +} + impl TelemetryStats { pub(crate) fn new(snapshot: Snapshot, cached_batches: u64) -> Self { Self { diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index 464730692..b39c381c2 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -141,6 +141,11 @@ impl Telemetry { pub fn stats(&self) -> TelemetryStats { self.0.stats() } + + /// The stats as one line for a debug console. + pub fn diagnostics(&self) -> String { + self.0.stats().to_string() + } } /// One room's session on the process pipeline: what its spans, stats and events are filed From db460ff614a75f4c8181bb9035aa50b8b2714b1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:31:51 +0200 Subject: [PATCH 22/50] chore(telemetry): log the cumulative stats line at debug with each report Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/exporter.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index 6e3c9ff4e..b30739c76 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -321,7 +321,13 @@ impl Exporter { let now = self.counters.snapshot(); let delta = now.since(&self.last_report); if delta.has_problems() || report_due { - let report = delta.report(self.cache.pending().len() as u64); + let cached = self.cache.pending().len() as u64; + // The host's console gets the cumulative line through the FFI log path. + log::debug!( + "{}", + crate::stats::TelemetryStats::new(self.counters.snapshot(), cached) + ); + let report = delta.report(cached); batch.push(Queued { event: report, session: self.process.clone() }); self.last_report = now; self.force_report = false; From 0e34e381b7215e5455fb89dc57c9a7d7698ef65b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:35:49 +0200 Subject: [PATCH 23/50] chore(telemetry): readable sent line Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/exporter.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index b30739c76..71fd01fab 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -461,13 +461,12 @@ impl Exporter { }; match self.deliver(&body, Signal::of(&id)).await { Delivery::Sent => { + self.cache.remove(&id); log::debug!( - "telemetry: sent {} ({} B, {} left)", - id, + "telemetry: sent {} B, {} batches left", body.len(), self.cache.pending().len() ); - self.cache.remove(&id); Counters::add(&self.counters.uploads_sent, 1); Counters::add(&self.counters.upload_bytes, body.len() as u64); } From 59261a134193e82c0e3e9fdd966c2941e8153bd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 13:39:28 +0200 Subject: [PATCH 24/50] chore(telemetry): log upload policy transitions, not ticks Hold begins (with reason) and ends (with duration), pause after a failure or a collector's Retry-After and its end, cadence changes with what stretched them, destination arrival: one debug line each. The per-tick repeats drop to trace. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/exporter.rs | 92 ++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 14 deletions(-) diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index 71fd01fab..9dfd0762e 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -31,8 +31,8 @@ use crate::{ stats::{Counters, Snapshot}, store::{Queued, Store}, telemetry::Destination, - AppState, Attribute, BatchCache, DeviceState, ExportError, ExportRequest, TelemetryConfig, - TelemetryTransport, + AppState, Attribute, BatchCache, DeviceState, ExportError, ExportRequest, MemoryPressure, + TelemetryConfig, TelemetryTransport, ThermalState, }; /// Retries per upload attempt after the first, for failures without `Retry-After`. @@ -138,6 +138,10 @@ pub struct Exporter { silenced: bool, /// Leave the cache alone until then: the last upload failed or we were throttled. paused_until: Option, + /// Policy transitions are logged once, not per tick: the hold reason last logged and the + /// cadence factor last logged. + hold_logged: Option<&'static str>, + cadence_logged: u32, /// Drop new batches until then (`Retry-After` window). throttled_until: Option, seq: u64, @@ -188,6 +192,8 @@ impl Exporter { commands, silenced: false, paused_until: None, + hold_logged: None, + cadence_logged: 1, throttled_until: None, seq: 0, cache_failures: 0, @@ -210,6 +216,7 @@ impl Exporter { let mut last_flush = Instant::now() - self.period(self.config.flush_interval_ms); let mut last_window = Instant::now(); loop { + self.log_cadence(); let next_flush = last_flush + self.period(self.config.flush_interval_ms); let next_window = last_window + self.period(self.config.stats_window_ms); tokio::select! { @@ -261,8 +268,42 @@ impl Exporter { /// Device pressure, doubled again while WebRTC reports the encoder CPU-limited; capped at 4×. fn cadence_factor(&self) -> u32 { - let cpu_limited = self.windows.lock().unwrap_or_else(|e| e.into_inner()).cpu_limited(); - (self.device().cadence_factor() * if cpu_limited { 2 } else { 1 }).min(4) + (self.device().cadence_factor() * if self.cpu_limited() { 2 } else { 1 }).min(4) + } + + fn cpu_limited(&self) -> bool { + self.windows.lock().unwrap_or_else(|e| e.into_inner()).cpu_limited() + } + + /// One debug line per cadence change, naming what stretched it. + fn log_cadence(&mut self) { + let factor = self.cadence_factor(); + if factor == self.cadence_logged { + return; + } + self.cadence_logged = factor; + let device = self.device(); + let mut why = Vec::new(); + if device.thermal != ThermalState::Nominal { + why.push(format!("thermal {:?}", device.thermal).to_lowercase()); + } + if device.memory != MemoryPressure::Normal { + why.push(format!("memory {:?}", device.memory).to_lowercase()); + } + if device.low_power_mode { + why.push("low power mode".to_string()); + } + if device.app_state == AppState::Background { + why.push("background".to_string()); + } + if self.cpu_limited() { + why.push("encoder cpu-limited".to_string()); + } + log::debug!( + "telemetry: cadence ×{factor}, flush every {}s ({})", + self.period(self.config.flush_interval_ms).as_secs(), + if why.is_empty() { "pressure over".to_string() } else { why.join(", ") } + ); } /// Last chance: everything queued into the cache and out. Ignores the upload backoff, the @@ -409,16 +450,17 @@ impl Exporter { if self.silenced { return; } - if let Some(until) = self.paused_until.filter(|t| Instant::now() < *t) { - log::debug!( - "telemetry: upload paused for {}s more after a failure", - (until - Instant::now()).as_secs() - ); - return; + if let Some(until) = self.paused_until { + if Instant::now() < until { + log::trace!("telemetry: paused for {}s more", (until - Instant::now()).as_secs()); + return; + } + self.paused_until = None; + log::debug!("telemetry: uploads resume after the pause"); } // No destination yet (SDK started, no room connected): everything waits in the cache. if self.destination.lock().unwrap_or_else(|e| e.into_inner()).is_none() { - log::debug!( + log::trace!( "telemetry: no destination yet; {} batches wait", self.cache.pending().len() ); @@ -429,11 +471,27 @@ impl Exporter { self.held_since = None; return; } - let budget = match self.hold_reason() { + let reason = self.hold_reason(); + match (self.hold_logged, reason) { + (None, Some(now)) => log::debug!("telemetry: uploads held: {now}"), + (Some(was), Some(now)) if was != now => { + log::debug!("telemetry: uploads held: {now} (was: {was})") + } + (Some(was), None) => match self.held_since { + Some(since) => log::debug!( + "telemetry: uploads resume after {}s ({was})", + since.elapsed().as_secs() + ), + None => log::debug!("telemetry: uploads resume ({was})"), + }, + _ => {} + } + self.hold_logged = reason; + let budget = match reason { Some(reason) => { let since = *self.held_since.get_or_insert_with(Instant::now); if since.elapsed() < MAX_HOLD { - log::debug!("telemetry: holding {} batches: {reason}", pending.len()); + log::trace!("telemetry: holding {} batches: {reason}", pending.len()); return; } log::debug!( @@ -476,10 +534,16 @@ impl Exporter { } Delivery::Failed => { self.paused_until = Some(Instant::now() + UPLOAD_BACKOFF); + log::debug!( + "telemetry: uploads paused {}s after a failure", + UPLOAD_BACKOFF.as_secs() + ); return; } Delivery::Throttled { retry_after } => { - let until = Instant::now() + retry_after.unwrap_or(UPLOAD_BACKOFF); + let wait = retry_after.unwrap_or(UPLOAD_BACKOFF); + let until = Instant::now() + wait; + log::debug!("telemetry: uploads paused {}s (collector asked)", wait.as_secs()); self.paused_until = Some(until); self.throttled_until = Some(until); return; From 6f17b807a59dcf0a93596e5ca3166c81eaf81160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:34:58 +0200 Subject: [PATCH 25/50] feat(telemetry): TelemetryStatus, one-line health with backlog and loss The upload policy's state (ok, held, paused, throttled, waiting, off) is part of TelemetryStats and leads every console line, followed by one backlog number and one loss number. Log lines drop the 'telemetry:' prefix; the target already names the module. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/exporter.rs | 106 +++++++++++++++++++---------- livekit-telemetry/src/lib.rs | 2 +- livekit-telemetry/src/stats.rs | 53 ++++++++++++--- livekit-telemetry/src/telemetry.rs | 14 +++- 4 files changed, 127 insertions(+), 48 deletions(-) diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index 9dfd0762e..284f69e04 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -28,7 +28,7 @@ use crate::{ rtc::StatsWindows, session::SessionState, span::Spans, - stats::{Counters, Snapshot}, + stats::{Counters, Snapshot, TelemetryStatus}, store::{Queued, Store}, telemetry::Destination, AppState, Attribute, BatchCache, DeviceState, ExportError, ExportRequest, MemoryPressure, @@ -142,6 +142,8 @@ pub struct Exporter { /// cadence factor last logged. hold_logged: Option<&'static str>, cadence_logged: u32, + /// Shared with `Telemetry::stats`. + status: Arc>, /// Drop new batches until then (`Retry-After` window). throttled_until: Option, seq: u64, @@ -177,6 +179,7 @@ impl Exporter { destination: Arc>>, commands: mpsc::UnboundedReceiver, device: Arc>>, + status: Arc>, ) -> Self { Self { store, @@ -194,6 +197,7 @@ impl Exporter { paused_until: None, hold_logged: None, cadence_logged: 1, + status, throttled_until: None, seq: 0, cache_failures: 0, @@ -300,7 +304,7 @@ impl Exporter { why.push("encoder cpu-limited".to_string()); } log::debug!( - "telemetry: cadence ×{factor}, flush every {}s ({})", + "cadence ×{factor}, flush every {}s ({})", self.period(self.config.flush_interval_ms).as_secs(), if why.is_empty() { "pressure over".to_string() } else { why.join(", ") } ); @@ -317,7 +321,7 @@ impl Exporter { self.export_pending().await; let left = self.cache.pending().len(); if left > 0 { - log::debug!("telemetry: {left} batches still cached at shutdown (replayed next start)"); + log::debug!("{left} batches still cached at shutdown (replayed next start)"); } } @@ -366,7 +370,11 @@ impl Exporter { // The host's console gets the cumulative line through the FFI log path. log::debug!( "{}", - crate::stats::TelemetryStats::new(self.counters.snapshot(), cached) + crate::stats::TelemetryStats::new( + self.counters.snapshot(), + cached, + self.status() + ) ); let report = delta.report(cached); batch.push(Queued { event: report, session: self.process.clone() }); @@ -417,11 +425,9 @@ impl Exporter { Err(err) => { // A full disk is a steady state, not an event: warn once, then stay quiet. if self.cache_failures == 0 { - log::warn!( - "telemetry: cannot cache batches ({err}); dropping until it recovers" - ); + log::warn!("cannot cache batches ({err}); dropping until it recovers"); } else { - log::debug!("telemetry: could not cache {count} items: {err}"); + log::debug!("could not cache {count} items: {err}"); } self.cache_failures += 1; Counters::add(&self.counters.cache_error, count); @@ -446,58 +452,79 @@ impl Exporter { /// Send cached batches oldest-first, within this tick's budget, until one fails; then back /// off. + fn status(&self) -> TelemetryStatus { + *self.status.lock().unwrap_or_else(|e| e.into_inner()) + } + + /// Record the policy state; true when it changed (log once, not per tick). + fn set_status(&self, status: TelemetryStatus) -> bool { + let mut current = self.status.lock().unwrap_or_else(|e| e.into_inner()); + let changed = *current != status; + *current = status; + changed + } + async fn upload(&mut self) { if self.silenced { + self.set_status(TelemetryStatus::Off); return; } + let backlog = self.cache.pending().len(); if let Some(until) = self.paused_until { if Instant::now() < until { - log::trace!("telemetry: paused for {}s more", (until - Instant::now()).as_secs()); + let throttled = self.throttled_until.is_some_and(|t| Instant::now() < t); + self.set_status(if throttled { + TelemetryStatus::Throttled + } else { + TelemetryStatus::Paused + }); + log::trace!( + "paused for {}s more, backlog {backlog}", + (until - Instant::now()).as_secs() + ); return; } self.paused_until = None; - log::debug!("telemetry: uploads resume after the pause"); + log::debug!("resumed after the pause, backlog {backlog}"); } // No destination yet (SDK started, no room connected): everything waits in the cache. if self.destination.lock().unwrap_or_else(|e| e.into_inner()).is_none() { - log::trace!( - "telemetry: no destination yet; {} batches wait", - self.cache.pending().len() - ); + if self.set_status(TelemetryStatus::Waiting) { + log::debug!("waiting for a destination, backlog {backlog}"); + } return; } let pending = self.cache.pending(); if pending.is_empty() { self.held_since = None; + self.set_status(TelemetryStatus::Ok); return; } let reason = self.hold_reason(); match (self.hold_logged, reason) { - (None, Some(now)) => log::debug!("telemetry: uploads held: {now}"), + (None, Some(now)) => log::debug!("held ({now}), backlog {backlog}"), (Some(was), Some(now)) if was != now => { - log::debug!("telemetry: uploads held: {now} (was: {was})") + log::debug!("held ({now}, was {was}), backlog {backlog}") } (Some(was), None) => match self.held_since { Some(since) => log::debug!( - "telemetry: uploads resume after {}s ({was})", + "resumed after {}s ({was}), backlog {backlog}", since.elapsed().as_secs() ), - None => log::debug!("telemetry: uploads resume ({was})"), + None => log::debug!("resumed ({was}), backlog {backlog}"), }, _ => {} } self.hold_logged = reason; + self.set_status(if reason.is_some() { TelemetryStatus::Held } else { TelemetryStatus::Ok }); let budget = match reason { Some(reason) => { let since = *self.held_since.get_or_insert_with(Instant::now); if since.elapsed() < MAX_HOLD { - log::trace!("telemetry: holding {} batches: {reason}", pending.len()); + log::trace!("holding {} batches: {reason}", pending.len()); return; } - log::debug!( - "telemetry: hold capped after {}s ({reason}); sending one batch", - MAX_HOLD.as_secs() - ); + log::debug!("held {}s ({reason}): sending one batch", MAX_HOLD.as_secs()); // Held long enough: one batch goes out, then the hold starts over. self.held_since = Some(Instant::now()); Counters::add(&self.counters.hold_cap_hits, 1); @@ -520,13 +547,17 @@ impl Exporter { match self.deliver(&body, Signal::of(&id)).await { Delivery::Sent => { self.cache.remove(&id); + Counters::add(&self.counters.uploads_sent, 1); + Counters::add(&self.counters.upload_bytes, body.len() as u64); + let snapshot = self.counters.snapshot(); log::debug!( - "telemetry: sent {} B, {} batches left", + "{}, sent {} B, backlog {}, failed {}, lost {}", + self.status(), body.len(), - self.cache.pending().len() + self.cache.pending().len(), + snapshot.upload_failures + snapshot.upload_timeouts, + snapshot.dropped() ); - Counters::add(&self.counters.uploads_sent, 1); - Counters::add(&self.counters.upload_bytes, body.len() as u64); } Delivery::Rejected => { self.cache.remove(&id); @@ -535,15 +566,20 @@ impl Exporter { Delivery::Failed => { self.paused_until = Some(Instant::now() + UPLOAD_BACKOFF); log::debug!( - "telemetry: uploads paused {}s after a failure", - UPLOAD_BACKOFF.as_secs() + "paused {}s after a failure, backlog {}", + UPLOAD_BACKOFF.as_secs(), + self.cache.pending().len() ); return; } Delivery::Throttled { retry_after } => { let wait = retry_after.unwrap_or(UPLOAD_BACKOFF); let until = Instant::now() + wait; - log::debug!("telemetry: uploads paused {}s (collector asked)", wait.as_secs()); + log::debug!( + "throttled {}s by the collector, backlog {}", + wait.as_secs(), + self.cache.pending().len() + ); self.paused_until = Some(until); self.throttled_until = Some(until); return; @@ -558,7 +594,7 @@ impl Exporter { /// Telemetry is disabled for this project: never send again, and never replay what is cached. fn silence(&mut self) { - log::warn!("telemetry disabled by the collector; going silent"); + log::warn!("disabled by the collector; going silent"); self.silenced = true; let cached: u64 = self.cache.pending().iter().map(|id| events_in(id)).sum(); Counters::add(&self.counters.disabled, cached); @@ -589,19 +625,19 @@ impl Exporter { Ok(Ok(())) => return Delivery::Sent, Ok(Err(ExportError::Disabled)) => return Delivery::Disabled, Ok(Err(ExportError::Rejected { message })) => { - log::warn!("telemetry batch rejected: {message}"); + log::warn!("batch rejected: {message}"); return Delivery::Rejected; } Ok(Err(ExportError::Retryable { message, retry_after_ms: Some(ms) })) => { - log::debug!("telemetry throttled for {ms} ms: {message}"); + log::debug!("throttled for {ms} ms: {message}"); return Delivery::Throttled { retry_after: Some(Duration::from_millis(ms)) }; } Ok(Err(ExportError::Retryable { message, retry_after_ms: None })) => { - log::debug!("telemetry upload failed (attempt {}): {message}", attempt + 1); + log::debug!("upload failed (attempt {}): {message}", attempt + 1); Counters::add(&self.counters.upload_failures, 1); } Err(_) => { - log::debug!("telemetry upload timed out (attempt {})", attempt + 1); + log::debug!("upload timed out (attempt {})", attempt + 1); Counters::add(&self.counters.upload_timeouts, 1); } } diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs index 4b9639de4..8a6208193 100644 --- a/livekit-telemetry/src/lib.rs +++ b/livekit-telemetry/src/lib.rs @@ -58,7 +58,7 @@ pub use exporter::Exporter; pub use rtc::{RtcStatsSample, StreamDirection, TrackKind}; pub use session::Session; pub use span::{SpanKind, SpanOutcome}; -pub use stats::TelemetryStats; +pub use stats::{TelemetryStats, TelemetryStatus}; pub use telemetry::*; pub use transport::*; diff --git a/livekit-telemetry/src/stats.rs b/livekit-telemetry/src/stats.rs index f3ea4b7da..4b28f866e 100644 --- a/livekit-telemetry/src/stats.rs +++ b/livekit-telemetry/src/stats.rs @@ -169,10 +169,44 @@ impl Snapshot { } } +/// What the upload policy is doing right now. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TelemetryStatus { + /// Uploading as data arrives. + Ok, + /// Uploads wait for a connect/reconnect to finish or for the device (Low Data Mode, low + /// battery); capped at 60 s. + Held, + /// The last upload failed; retrying in a minute. + Paused, + /// The collector asked for a pause (`Retry-After`). + Throttled, + /// No destination yet (no room connected); everything waits in the cache. + Waiting, + /// The collector disabled telemetry for this process. + Off, +} + +impl std::fmt::Display for TelemetryStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Ok => "ok", + Self::Held => "held", + Self::Paused => "paused", + Self::Throttled => "throttled", + Self::Waiting => "waiting", + Self::Off => "off", + }) + } +} + /// Pipeline health as seen by the SDK: [`Telemetry::stats`](crate::Telemetry::stats). #[cfg_attr(feature = "uniffi", derive(uniffi::Record))] #[derive(Debug, Clone, PartialEq, Eq)] pub struct TelemetryStats { + /// What the upload policy is doing right now. + pub status: TelemetryStatus, /// Events lost for any reason (sum of the `dropped_*` fields). pub dropped: u64, pub dropped_queue_full: u64, @@ -196,34 +230,35 @@ pub struct TelemetryStats { pub cached_batches: u64, } -/// One line for a debug console. +/// One line for a debug console: status, throughput, one backlog number, one loss number, then +/// the loss breakdown. impl std::fmt::Display for TelemetryStats { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, - "telemetry: sent {} ({} B), cached {}, failed {}, timeouts {}, holds capped {}, dropped {} \ - (queue {}, cache {}/{}, rejected {}, throttled {}, rate-limited {}, disabled {})", + "{}, sent {} ({} B), backlog {}, failed {}, lost {} (queue {}, cache {}, rejected {}, \ + throttled {}, rate-limited {}, disabled {}; holds capped {})", + self.status, self.uploads_sent, self.upload_bytes, self.cached_batches, - self.upload_failures, - self.upload_timeouts, - self.holds_capped, + self.upload_failures + self.upload_timeouts, self.dropped, self.dropped_queue_full, - self.dropped_cache_full, - self.dropped_cache_error, + self.dropped_cache_full + self.dropped_cache_error, self.dropped_rejected, self.dropped_throttled, self.dropped_rate_limited, self.dropped_disabled, + self.holds_capped, ) } } impl TelemetryStats { - pub(crate) fn new(snapshot: Snapshot, cached_batches: u64) -> Self { + pub(crate) fn new(snapshot: Snapshot, cached_batches: u64, status: TelemetryStatus) -> Self { Self { + status, dropped: snapshot.dropped(), dropped_queue_full: snapshot.queue_full, dropped_cache_error: snapshot.cache_error, diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 24923b85f..37ebb81c5 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -27,7 +27,7 @@ use crate::{ rtc::StatsWindows, session::{Session, SessionState}, span::Spans, - stats::Counters, + stats::{Counters, TelemetryStatus}, store::{Queued, Store}, Attribute, AttributeValue, BatchCache, DeviceState, Exporter, FileCache, MemoryCache, RtcStatsSample, Severity, SpanKind, SpanOutcome, TelemetryEvent, TelemetryStats, @@ -164,6 +164,7 @@ pub struct Telemetry { /// Attributes attached to every record of every session. global: Arc>>, destination: Arc>>, + status: Arc>, commands: mpsc::UnboundedSender, } @@ -228,7 +229,7 @@ impl Telemetry { Some(dir) => match FileCache::open(dir, config.max_cache_bytes) { Ok(cache) => Arc::new(cache), Err(err) => { - log::warn!("telemetry: cannot use storage dir {dir}: {err}; caching in memory"); + log::warn!("cannot use storage dir {dir}: {err}; caching in memory"); Arc::new(MemoryCache::new(config.max_cache_bytes)) } }, @@ -253,6 +254,7 @@ impl Telemetry { let process = SessionState::new(); let global = Arc::new(Mutex::new(Vec::new())); let counters = Arc::new(Counters::default()); + let status = Arc::new(Mutex::new(TelemetryStatus::Ok)); let store = Arc::new(Store::new( config.max_queue_size.max(1) as usize, usize::try_from(config.flush_threshold_bytes.max(1)).unwrap_or(usize::MAX), @@ -276,6 +278,7 @@ impl Telemetry { destination.clone(), receiver, device.clone(), + status.clone(), ); let telemetry = Self { store, @@ -289,6 +292,7 @@ impl Telemetry { process, global, destination, + status, commands, }; (telemetry, exporter) @@ -447,7 +451,11 @@ impl Telemetry { /// Pipeline health: drops by reason, uploads, cached batches. The same numbers ride to the /// backend as `lk.telemetry.report` events whenever something went wrong. pub fn stats(&self) -> TelemetryStats { - TelemetryStats::new(self.counters.snapshot(), self.cache.pending().len() as u64) + TelemetryStats::new( + self.counters.snapshot(), + self.cache.pending().len() as u64, + *self.status.lock().unwrap_or_else(|e| e.into_inner()), + ) } async fn command(&self, make: impl FnOnce(oneshot::Sender<()>) -> Command) { From f2b97ee17d8163b557a8a8823182cd0f269f06e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:44:53 +0200 Subject: [PATCH 26/50] fix(telemetry): own problems at warn/error, once per episode An upload outage logs its first failure at warn with the transport's reason and its recovery at info; a collector throttle, the 60 s hold cap, a full queue and a tripped flood guard log at warn once per episode; a rejected batch is data lost and logs at error. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/exporter.rs | 47 +++++++++++++++++++++--------- livekit-telemetry/src/store.rs | 10 +++++++ livekit-telemetry/src/telemetry.rs | 12 +++++++- 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index 284f69e04..c67f990c1 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -87,7 +87,9 @@ pub(crate) enum Command { enum Delivery { Sent, /// Transient failure (network, timeout, 5xx) after retries — keep the batch. - Failed, + Failed { + reason: String, + }, /// The collector asked us to back off; keep the batch, drop new ones meanwhile. Throttled { retry_after: Option, @@ -144,6 +146,8 @@ pub struct Exporter { cadence_logged: u32, /// Shared with `Telemetry::stats`. status: Arc>, + /// Consecutive failed uploads: the first logs at warn, recovery at info. + failing: u32, /// Drop new batches until then (`Retry-After` window). throttled_until: Option, seq: u64, @@ -198,6 +202,7 @@ impl Exporter { hold_logged: None, cadence_logged: 1, status, + failing: 0, throttled_until: None, seq: 0, cache_failures: 0, @@ -524,7 +529,7 @@ impl Exporter { log::trace!("holding {} batches: {reason}", pending.len()); return; } - log::debug!("held {}s ({reason}): sending one batch", MAX_HOLD.as_secs()); + log::warn!("held {}s ({reason}): sending one batch anyway", MAX_HOLD.as_secs()); // Held long enough: one batch goes out, then the hold starts over. self.held_since = Some(Instant::now()); Counters::add(&self.counters.hold_cap_hits, 1); @@ -549,6 +554,14 @@ impl Exporter { self.cache.remove(&id); Counters::add(&self.counters.uploads_sent, 1); Counters::add(&self.counters.upload_bytes, body.len() as u64); + if self.failing > 0 { + log::info!( + "uploads recovered after {} failures, backlog {}", + self.failing, + self.cache.pending().len() + ); + self.failing = 0; + } let snapshot = self.counters.snapshot(); log::debug!( "{}, sent {} B, backlog {}, failed {}, lost {}", @@ -563,20 +576,25 @@ impl Exporter { self.cache.remove(&id); Counters::add(&self.counters.rejected, events_in(&id)); } - Delivery::Failed => { + Delivery::Failed { reason } => { self.paused_until = Some(Instant::now() + UPLOAD_BACKOFF); - log::debug!( - "paused {}s after a failure, backlog {}", - UPLOAD_BACKOFF.as_secs(), - self.cache.pending().len() - ); + self.failing += 1; + let backlog = self.cache.pending().len(); + let wait = UPLOAD_BACKOFF.as_secs(); + if self.failing == 1 { + log::warn!( + "upload failed: {reason}; retrying in {wait}s, backlog {backlog}" + ); + } else { + log::debug!("upload failed ({} in a row): {reason}; retrying in {wait}s, backlog {backlog}", self.failing); + } return; } Delivery::Throttled { retry_after } => { let wait = retry_after.unwrap_or(UPLOAD_BACKOFF); let until = Instant::now() + wait; - log::debug!( - "throttled {}s by the collector, backlog {}", + log::warn!( + "collector throttles uploads: pausing {}s, backlog {}", wait.as_secs(), self.cache.pending().len() ); @@ -605,7 +623,7 @@ impl Exporter { async fn deliver(&self, body: &[u8], signal: Signal) -> Delivery { let Some(destination) = self.destination.lock().unwrap_or_else(|e| e.into_inner()).clone() else { - return Delivery::Failed; + return Delivery::Failed { reason: "no destination".to_owned() }; }; let mut headers = destination.headers; headers.insert("Content-Type".to_owned(), otlp::CONTENT_TYPE.to_owned()); @@ -620,12 +638,13 @@ impl Exporter { // ponytail: linear backoff, 2 retries, blocks the tick loop while sleeping (≤ 3 s). // Exponential + jitter once real fleets exercise this. + let mut last = String::new(); for attempt in 0..=MAX_RETRIES { match timeout(attempt_timeout, self.transport.send(request.clone())).await { Ok(Ok(())) => return Delivery::Sent, Ok(Err(ExportError::Disabled)) => return Delivery::Disabled, Ok(Err(ExportError::Rejected { message })) => { - log::warn!("batch rejected: {message}"); + log::error!("batch rejected by the collector, data lost: {message}"); return Delivery::Rejected; } Ok(Err(ExportError::Retryable { message, retry_after_ms: Some(ms) })) => { @@ -634,10 +653,12 @@ impl Exporter { } Ok(Err(ExportError::Retryable { message, retry_after_ms: None })) => { log::debug!("upload failed (attempt {}): {message}", attempt + 1); + last = message; Counters::add(&self.counters.upload_failures, 1); } Err(_) => { log::debug!("upload timed out (attempt {})", attempt + 1); + last = "timed out".to_owned(); Counters::add(&self.counters.upload_timeouts, 1); } } @@ -645,7 +666,7 @@ impl Exporter { sleep(RETRY_BACKOFF * (attempt + 1)).await; } } - Delivery::Failed + Delivery::Failed { reason: last } } } diff --git a/livekit-telemetry/src/store.rs b/livekit-telemetry/src/store.rs index e473a29c2..7d7854d0e 100644 --- a/livekit-telemetry/src/store.rs +++ b/livekit-telemetry/src/store.rs @@ -45,6 +45,8 @@ pub(crate) struct Store { struct Queue { events: VecDeque, bytes: usize, + /// The first eviction of an episode logs; the rest are counted. + full_warned: bool, } impl Store { @@ -61,6 +63,13 @@ impl Store { queue.bytes = queue.bytes.saturating_sub(oldest.event.size_hint()); } Counters::add(&self.counters.queue_full, 1); + if !queue.full_warned { + queue.full_warned = true; + log::warn!( + "queue full ({} records): dropping oldest until the exporter drains", + self.capacity + ); + } } let before = queue.bytes; queue.bytes += queued.event.size_hint(); @@ -72,6 +81,7 @@ impl Store { /// (always at least one, so an oversized event still ships). pub fn drain(&self, max: usize, max_bytes: usize) -> Vec { let mut queue = self.queue.lock().unwrap_or_else(|e| e.into_inner()); + queue.full_warned = false; let mut out = Vec::new(); let mut bytes = 0; while out.len() < max { diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 37ebb81c5..cc70bf4a9 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -191,13 +191,15 @@ struct FloodGuard { max: u32, window_start: Instant, count: u32, + /// The first drop of a window logs; the rest are counted. + warned: bool, } impl FloodGuard { const WINDOW: Duration = Duration::from_secs(10 * 60); fn new(max: u32) -> Self { - Self { max, window_start: Instant::now(), count: 0 } + Self { max, window_start: Instant::now(), count: 0, warned: false } } fn admit(&mut self) -> bool { @@ -208,8 +210,16 @@ impl FloodGuard { if now.duration_since(self.window_start) >= Self::WINDOW { self.window_start = now; self.count = 0; + self.warned = false; } if self.count >= self.max { + if !self.warned { + self.warned = true; + log::warn!( + "flood: {} records in 10 min; dropping until the window moves", + self.max + ); + } return false; } self.count += 1; From 2a6569543c61b387177607a8341c76bd62a0f1ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:04:02 +0200 Subject: [PATCH 27/50] feat(telemetry): set_server and typed LogRecord in the core Two SDK-independent rules move out of the platforms: the Cloud observability endpoint and bearer header derive from the room's server URL and token in set_server; captured log lines arrive as a typed LogRecord and the core builds the record (semconv code.*, lk.log.source, lk.log.logger) and applies the per-source floor (WebRTC only at error, its own module never). Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/SPEC.md | 10 ++- livekit-telemetry/src/event.rs | 69 +++++++++++++++++ livekit-telemetry/src/telemetry.rs | 116 ++++++++++++++++++++++++++++- livekit-uniffi/src/telemetry.rs | 13 +++- 4 files changed, 202 insertions(+), 6 deletions(-) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index a08100dfd..c469d29a3 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -28,7 +28,8 @@ a session — device state, pre-room errors, self-telemetry — belongs to the p session. Sessions are not ended: a room's last record is simply its last. The pipeline may start **without a destination** (`endpoint: None`): it buffers and caches, and -uploads nothing until `set_destination(endpoint, headers)` — at the first connect, when the +uploads nothing until `set_server(url, token)` (Cloud: `https:///observability/logs/otlp/v0`, +`Authorization: Bearer `) or `set_destination(endpoint, headers)` — at the first connect, when the server URL yields the endpoint (`https:///observability/logs/otlp/v0`) and the token the `Authorization` header. Calling it again (new token, new server) replaces the destination for the batches that follow. Waiting for a destination is not an upload hold: it is uncapped, bounded @@ -211,8 +212,11 @@ Uploads are shaped, not just batched: ## Log records A `TelemetryEvent` with an empty `name` is a plain log record (OTLP log without `event_name`): -`severity` + `body` (the message) + attributes such as `code.function`, `code.file.path`, -`code.line.number`, `lk.log.type` (the SDK logger's category). Only `warn` and `error` records +`severity` + `body` (the message) + `code.function.name`, `code.file.path`, `code.line.number` +(semconv), `lk.log.source` (`sdk` | `core` | `webrtc`) and `lk.log.logger` (type, module or file). The +platform hands the core a typed `LogRecord` via `log(record)`; the core applies the floor: WebRTC only +at `error`, the SDK and the core at the configured `log_severity`, the core's own telemetry module +never. Only `warn` and `error` records leave the device; `trace`/`debug`/`info` are dropped in `emit`. ## Custom events diff --git a/livekit-telemetry/src/event.rs b/livekit-telemetry/src/event.rs index 533778392..8127abe03 100644 --- a/livekit-telemetry/src/event.rs +++ b/livekit-telemetry/src/event.rs @@ -107,6 +107,75 @@ pub enum Severity { Error, } +/// Where a log line came from. WebRTC is chatty at warn, so only its errors become records; +/// the SDK and the core use the configured floor. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LogSource { + Sdk, + Core, + WebRtc, +} + +impl LogSource { + fn as_str(self) -> &'static str { + match self { + Self::Sdk => "sdk", + Self::Core => "core", + Self::WebRtc => "webrtc", + } + } +} + +/// A log line as the platform captured it, where it happened. The core turns it into a record: +/// semconv `code.*` attributes, `lk.log.source`, `lk.log.logger`, filed under the span's session. +/// Stamp `timestamp_ns` at capture; the record may cross an executor hop before it gets here. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct LogRecord { + pub severity: Severity, + pub source: LogSource, + pub message: String, + /// The logger: a type, module or file name (`Room`, `livekit::rtc_engine`, `sctp.cc`). + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub logger: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub function: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub file: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub line: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub timestamp_ns: Option, + /// The in-flight span this line was logged under, if any. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub span_id: Option, +} + +impl From for TelemetryEvent { + fn from(record: LogRecord) -> Self { + let mut event = TelemetryEvent::new("") + .with_severity(record.severity) + .with_body(record.message) + .with_attribute("lk.log.source", record.source.as_str()); + if let Some(logger) = record.logger.filter(|s| !s.is_empty()) { + event = event.with_attribute("lk.log.logger", logger); + } + if let Some(function) = record.function.filter(|s| !s.is_empty()) { + event = event.with_attribute("code.function.name", function); + } + if let Some(file) = record.file.filter(|s| !s.is_empty()) { + event = event.with_attribute("code.file.path", file); + } + if let Some(line) = record.line.filter(|l| *l > 0) { + event = event.with_attribute("code.line.number", line as i64); + } + event.timestamp_ns = record.timestamp_ns; + event.span_id = record.span_id; + event + } +} + /// A key/value attribute on an event or on the resource. #[cfg_attr(feature = "uniffi", derive(uniffi::Record))] #[derive(Debug, Clone, PartialEq)] diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index cc70bf4a9..1f33da742 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -29,8 +29,8 @@ use crate::{ span::Spans, stats::{Counters, TelemetryStatus}, store::{Queued, Store}, - Attribute, AttributeValue, BatchCache, DeviceState, Exporter, FileCache, MemoryCache, - RtcStatsSample, Severity, SpanKind, SpanOutcome, TelemetryEvent, TelemetryStats, + Attribute, AttributeValue, BatchCache, DeviceState, Exporter, FileCache, LogRecord, LogSource, + MemoryCache, RtcStatsSample, Severity, SpanKind, SpanOutcome, TelemetryEvent, TelemetryStats, TelemetryTransport, }; @@ -308,6 +308,23 @@ impl Telemetry { (telemetry, exporter) } + /// The Cloud rule, in one place for every SDK: the room's server URL names the observability + /// endpoint and the room token authorizes it. An explicit `TelemetryConfig::endpoint` (a + /// collector of your own) wins and makes this a no-op. + pub fn set_server(&self, url: &str, token: &str) { + if self.config.endpoint.is_some() { + return; + } + match observability_endpoint(url) { + Some(endpoint) => { + let mut headers = HashMap::new(); + headers.insert("Authorization".to_owned(), format!("Bearer {token}")); + self.set_destination(&endpoint, headers); + } + None => log::warn!("server url has no host; uploads stay cached"), + } + } + /// Where to send, once known (the first connect: server URL → endpoint, token → headers). /// Everything cached so far starts uploading. Calling it again (new token, new server) /// replaces the destination for the batches that follow. @@ -329,6 +346,25 @@ impl Telemetry { /// A record with an empty `name` is a plain log line: only `Warn` and `Error` ones leave the /// device (design doc: debug/info logs never do). Discrete events are subject to the flood /// guard (`max_events_per_10min`); what it drops is counted as `rate_limited`. + /// A captured log line. WebRTC only counts at error; the SDK and the core at the configured + /// floor; the core's own telemetry module never (a rejected batch that produced a record that + /// produced a batch would never end). + pub fn log(&self, record: LogRecord) { + let floor = match record.source { + LogSource::WebRtc => self.config.log_severity.max(Severity::Error), + _ => self.config.log_severity, + }; + if record.severity < floor { + return; + } + if record.source == LogSource::Core + && record.logger.as_deref().is_some_and(|l| l.starts_with("livekit_telemetry")) + { + return; + } + self.emit(record.into()); + } + pub fn emit(&self, event: TelemetryEvent) { // A record emitted inside a room's span belongs to that room's session; anything else // is the process's own. @@ -499,10 +535,86 @@ fn add_sdk_resource(resource: &mut Vec) { } } +/// `wss://x.livekit.cloud/rtc?…` → `https://x.livekit.cloud/observability/logs/otlp/v0`; +/// `ws://`/`http://` stay plain http (dev servers). Host and port only; no path, query or userinfo. +pub(crate) fn observability_endpoint(url: &str) -> Option { + let (scheme, rest) = url.split_once("://")?; + let authority = rest.split(['/', '?', '#']).next()?; + let host = authority.rsplit_once('@').map_or(authority, |(_, host)| host); + if host.is_empty() { + return None; + } + let scheme = match scheme { + "ws" | "http" => "http", + _ => "https", + }; + Some(format!("{scheme}://{host}/observability/logs/otlp/v0")) +} + #[cfg(test)] mod tests { use std::{collections::VecDeque, fs, path::Path, sync::Mutex}; + #[test] + fn server_url_names_the_observability_endpoint() { + let ep = observability_endpoint; + assert_eq!( + ep("wss://x.livekit.cloud").unwrap(), + "https://x.livekit.cloud/observability/logs/otlp/v0" + ); + assert_eq!( + ep("wss://x.livekit.cloud/rtc?access_token=t#f").unwrap(), + "https://x.livekit.cloud/observability/logs/otlp/v0" + ); + assert_eq!( + ep("ws://192.168.99.24:7880").unwrap(), + "http://192.168.99.24:7880/observability/logs/otlp/v0" + ); + assert_eq!(ep("https://u:p@host").unwrap(), "https://host/observability/logs/otlp/v0"); + assert_eq!(ep("nonsense"), None); + assert_eq!(ep("wss:///rtc"), None); + } + + #[tokio::test(start_paused = true)] + async fn log_records_apply_the_source_floor_and_carry_code_attributes() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + let line = |source, severity, logger: &str| crate::LogRecord { + severity, + source, + message: "boom".into(), + logger: Some(logger.into()), + function: Some("connect()".into()), + file: Some("Room.swift".into()), + line: Some(42), + timestamp_ns: None, + span_id: None, + }; + telemetry.log(line(LogSource::WebRtc, Severity::Warn, "sctp.cc")); + telemetry.log(line(LogSource::Sdk, Severity::Info, "Room")); + telemetry.log(line(LogSource::Core, Severity::Error, "livekit_telemetry::exporter")); + telemetry.log(line(LogSource::Sdk, Severity::Warn, "Room")); + telemetry.log(line(LogSource::WebRtc, Severity::Error, "sctp.cc")); + telemetry.flush().await; + let sent = transport.sent(); + assert_eq!(sent.len(), 1); + let logs = records(&sent[0]); + assert_eq!(logs.len(), 2, "sdk warn + webrtc error; not webrtc warn, sdk info, own module"); + let sdk = &logs[0]; + assert_eq!(attribute(sdk, "lk.log.source"), Some(Value::StringValue("sdk".into()))); + assert_eq!(attribute(sdk, "lk.log.logger"), Some(Value::StringValue("Room".into()))); + assert_eq!( + attribute(sdk, "code.function.name"), + Some(Value::StringValue("connect()".into())) + ); + assert_eq!(attribute(sdk, "code.line.number"), Some(Value::IntValue(42))); + assert_eq!( + sdk.body.as_ref().and_then(|b| b.value.clone()), + Some(Value::StringValue("boom".into())) + ); + assert_eq!(attribute(&logs[1], "lk.log.source"), Some(Value::StringValue("webrtc".into()))); + } + use prost::Message; use super::*; diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index b39c381c2..b9eaf1592 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -17,7 +17,7 @@ use std::{ }; use livekit_telemetry::{ - Attribute, AttributeValue, DeviceState, ExportError, ExportRequest, NetTransport, + Attribute, AttributeValue, DeviceState, ExportError, ExportRequest, LogRecord, NetTransport, RtcStatsSample, SpanKind, SpanOutcome, TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, }; @@ -73,6 +73,17 @@ impl Telemetry { self.0.emit_custom(&name, attributes); } + /// A captured log line; the core applies the per-source floor and builds the record. + pub fn log(&self, record: LogRecord) { + self.0.log(record); + } + + /// Cloud rule: server URL → observability endpoint, room token → bearer header. No-op when + /// the config names an explicit endpoint. + pub fn set_server(&self, url: String, token: String) { + self.0.set_server(&url, &token); + } + /// Where to send, once known (first connect: server URL → endpoint, token → headers). Until /// then everything waits in the cache; afterwards it uploads. pub fn set_destination(&self, endpoint: String, headers: HashMap) { From 31fee3c887ba461ab3630544e0b7a119164708bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:06:25 +0200 Subject: [PATCH 28/50] chore(telemetry): LogSource::Ffi, named after the path it arrives on Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/SPEC.md | 2 +- livekit-telemetry/src/event.rs | 4 ++-- livekit-telemetry/src/telemetry.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index c469d29a3..f2650296c 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -213,7 +213,7 @@ Uploads are shaped, not just batched: A `TelemetryEvent` with an empty `name` is a plain log record (OTLP log without `event_name`): `severity` + `body` (the message) + `code.function.name`, `code.file.path`, `code.line.number` -(semconv), `lk.log.source` (`sdk` | `core` | `webrtc`) and `lk.log.logger` (type, module or file). The +(semconv), `lk.log.source` (`sdk` | `ffi` | `webrtc`) and `lk.log.logger` (type, module or file). The platform hands the core a typed `LogRecord` via `log(record)`; the core applies the floor: WebRTC only at `error`, the SDK and the core at the configured `log_severity`, the core's own telemetry module never. Only `warn` and `error` records diff --git a/livekit-telemetry/src/event.rs b/livekit-telemetry/src/event.rs index 8127abe03..fd43b30dc 100644 --- a/livekit-telemetry/src/event.rs +++ b/livekit-telemetry/src/event.rs @@ -113,7 +113,7 @@ pub enum Severity { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum LogSource { Sdk, - Core, + Ffi, WebRtc, } @@ -121,7 +121,7 @@ impl LogSource { fn as_str(self) -> &'static str { match self { Self::Sdk => "sdk", - Self::Core => "core", + Self::Ffi => "ffi", Self::WebRtc => "webrtc", } } diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 1f33da742..db0333a4f 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -357,7 +357,7 @@ impl Telemetry { if record.severity < floor { return; } - if record.source == LogSource::Core + if record.source == LogSource::Ffi && record.logger.as_deref().is_some_and(|l| l.starts_with("livekit_telemetry")) { return; @@ -592,7 +592,7 @@ mod tests { }; telemetry.log(line(LogSource::WebRtc, Severity::Warn, "sctp.cc")); telemetry.log(line(LogSource::Sdk, Severity::Info, "Room")); - telemetry.log(line(LogSource::Core, Severity::Error, "livekit_telemetry::exporter")); + telemetry.log(line(LogSource::Ffi, Severity::Error, "livekit_telemetry::exporter")); telemetry.log(line(LogSource::Sdk, Severity::Warn, "Room")); telemetry.log(line(LogSource::WebRtc, Severity::Error, "sctp.cc")); telemetry.flush().await; From cc5dac2ff9071a2092dd412aa645a31ba01b3841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:19:27 +0200 Subject: [PATCH 29/50] =?UTF-8?q?feat(telemetry):=20typed=20surface=20?= =?UTF-8?q?=E2=80=94=20spans,=20resource,=20room=20identity,=20device=20ev?= =?UTF-8?q?ents,=20simulcast=20fold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The span moves into the core: SpanName (kind implied), SpanStep checkpoints, SpanTrack, TraceContext, and a Span object that stamps the clock inside every synchronous call, exports when bound to a session and describes itself the same way on every platform. TelemetryResource and Sdk lower to semconv resource keys; RoomIdentity sets the session's lk.room.* / lk.participant.*; DeviceEvent builds audio route, interruption and permission-denied records; RtcStatsSample.layer lets the core fold simulcast layers into one series per track. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/device.rs | 101 ++++++++ livekit-telemetry/src/lib.rs | 4 +- livekit-telemetry/src/rtc.rs | 77 ++++++ livekit-telemetry/src/session.rs | 35 ++- livekit-telemetry/src/telemetry.rs | 137 ++++++++++- livekit-telemetry/src/trace.rs | 371 +++++++++++++++++++++++++++++ livekit-uniffi/src/telemetry.rs | 121 +++++++--- 7 files changed, 803 insertions(+), 43 deletions(-) create mode 100644 livekit-telemetry/src/trace.rs diff --git a/livekit-telemetry/src/device.rs b/livekit-telemetry/src/device.rs index fbb78747b..65aa6b2fb 100644 --- a/livekit-telemetry/src/device.rs +++ b/livekit-telemetry/src/device.rs @@ -332,3 +332,104 @@ mod tests { assert_eq!(charging.change_events(Some(&at(10))).len(), 1, "plugged in"); } } + +/// Why the audio route changed, in platform-neutral terms (AVAudioSession reasons; Android's +/// device added/removed callbacks map to `NewDevice` / `OldDeviceUnavailable`). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AudioRouteReason { + NewDevice, + OldDeviceUnavailable, + CategoryChange, + Override, + WakeFromSleep, + NoSuitableRoute, + RouteConfigurationChange, + Unknown, +} + +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DevicePermission { + Camera, + Microphone, + ScreenShare, +} + +/// Things that happen to the device mid-call and explain what the media did next. Not state +/// (see [`DeviceState`]): each one is a record with a display body. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DeviceEvent { + /// `outputs` as the platform names them (`Speaker`, `BluetoothA2DPOutput`, …). + AudioRouteChanged { + outputs: Vec, + reason: AudioRouteReason, + }, + AudioInterruption { + began: bool, + }, + /// The user (or MDM) said no: the most common pre-connect failure on mobile. + PermissionDenied { + permission: DevicePermission, + }, +} + +fn snake(debug: impl std::fmt::Debug) -> String { + let mut out = String::new(); + for (i, c) in format!("{debug:?}").chars().enumerate() { + if c.is_uppercase() && i > 0 { + out.push('_'); + } + out.push(c.to_ascii_lowercase()); + } + out +} + +impl DeviceEvent { + pub(crate) fn into_event(self) -> crate::TelemetryEvent { + use crate::{Severity, TelemetryEvent}; + match self { + Self::AudioRouteChanged { outputs, reason } => { + let outputs = outputs.join(","); + let reason = snake(reason); + TelemetryEvent::new("lk.device.audio_route.changed") + .with_body(format!("audio route: {outputs} ({reason})")) + .with_attribute("lk.device.audio_route.outputs", outputs) + .with_attribute("lk.device.audio_route.reason", reason) + } + Self::AudioInterruption { began } => { + let phase = if began { "began" } else { "ended" }; + TelemetryEvent::new("lk.device.audio.interruption") + .with_body(format!("audio interruption {phase}")) + .with_attribute("lk.device.audio.interruption", phase) + } + Self::PermissionDenied { permission } => { + let permission = snake(permission); + TelemetryEvent::new("lk.device.permission.denied") + .with_severity(Severity::Warn) + .with_body(format!("permission denied: {permission}")) + .with_attribute("lk.device.permission", permission) + } + } + } +} + +#[cfg(test)] +mod event_tests { + use super::*; + + #[test] + fn device_events_have_bodies_and_snake_case_values() { + let event = DeviceEvent::AudioRouteChanged { + outputs: vec!["Speaker".into()], + reason: AudioRouteReason::OldDeviceUnavailable, + } + .into_event(); + assert_eq!(event.body.as_deref(), Some("audio route: Speaker (old_device_unavailable)")); + let denied = DeviceEvent::PermissionDenied { permission: DevicePermission::ScreenShare } + .into_event(); + assert_eq!(denied.body.as_deref(), Some("permission denied: screen_share")); + assert_eq!(denied.severity, crate::Severity::Warn); + } +} diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs index 8a6208193..4429f026f 100644 --- a/livekit-telemetry/src/lib.rs +++ b/livekit-telemetry/src/lib.rs @@ -50,16 +50,18 @@ mod transport; /// Entry point and configuration. mod telemetry; +mod trace; pub use cache::{BatchCache, FileCache, MemoryCache}; pub use device::*; pub use event::*; pub use exporter::Exporter; pub use rtc::{RtcStatsSample, StreamDirection, TrackKind}; -pub use session::Session; +pub use session::{RoomIdentity, Session}; pub use span::{SpanKind, SpanOutcome}; pub use stats::{TelemetryStats, TelemetryStatus}; pub use telemetry::*; +pub use trace::*; pub use transport::*; #[cfg(feature = "uniffi")] diff --git a/livekit-telemetry/src/rtc.rs b/livekit-telemetry/src/rtc.rs index 93cd83533..1ade5bc4d 100644 --- a/livekit-telemetry/src/rtc.rs +++ b/livekit-telemetry/src/rtc.rs @@ -82,6 +82,11 @@ pub struct RtcStatsSample { /// When the reading was taken; `None` = now. #[cfg_attr(feature = "uniffi", uniffi(default))] pub timestamp_ns: Option, + /// The RTP stream this sample describes when a track has several (simulcast layers): any + /// stable id (`rid`, ssrc, the stats id). The core folds layers into the track; a platform + /// never sums them itself. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub layer: Option, } impl RtcStatsSample { @@ -108,6 +113,7 @@ impl RtcStatsSample { frames_per_second: None, audio_level: None, timestamp_ns: None, + layer: None, } } } @@ -277,6 +283,26 @@ pub(crate) struct StatsWindows { // ponytail: grows with the tracks published in a session (a few dozen at most). limitation: HashMap, cpu_limited_until: Option, + /// Last cumulative counters per simulcast layer, per outbound track: the track's counters + /// are their sum, so a suspended top layer cannot freeze them. + layers: HashMap<(String, StreamDirection), HashMap>, +} + +#[derive(Debug, Clone, Copy, Default)] +struct LayerCounters { + bytes: Option, + packets: Option, + fps: Option, + limitation_bandwidth_ms: Option, + limitation_cpu_ms: Option, +} + +fn sum(values: impl Iterator>) -> Option { + values.flatten().reduce(|a, b| a + b) +} + +fn max_u64(values: impl Iterator>) -> Option { + values.flatten().max() } /// CPU starvation is sticky: stretch the cadence for a while after the last sign of it. @@ -287,6 +313,7 @@ const CPU_LIMITED_HOLD: Duration = Duration::from_secs(60); impl StatsWindows { pub fn record_in(&mut self, mut sample: RtcStatsSample, session: &Arc) { + self.fold_layers(&mut sample); self.track_limitation(&sample); let timestamp = *sample.timestamp_ns.get_or_insert_with(now_unix_nanos); let key = (sample.track_sid.clone(), sample.direction); @@ -298,6 +325,33 @@ impl StatsWindows { } } + /// Simulcast publishes one RTP stream per layer under one track sid. Remember each layer's + /// latest cumulative counters and rewrite the sample as the track's total, so the window + /// sees one monotonic series per track. + fn fold_layers(&mut self, sample: &mut RtcStatsSample) { + let Some(layer) = sample.layer.clone() else { return }; + let layers = self.layers.entry((sample.track_sid.clone(), sample.direction)).or_default(); + layers.insert( + layer, + LayerCounters { + bytes: sample.bytes, + packets: sample.packets, + fps: sample.frames_per_second, + limitation_bandwidth_ms: sample.quality_limitation_bandwidth_ms, + limitation_cpu_ms: sample.quality_limitation_cpu_ms, + }, + ); + sample.bytes = sum(layers.values().map(|l| l.bytes)); + sample.packets = sum(layers.values().map(|l| l.packets)); + sample.frames_per_second = layers + .values() + .filter_map(|l| l.fps) + .fold(None, |m, v| Some(m.map_or(v, |m: f64| m.max(v)))); + sample.quality_limitation_bandwidth_ms = + max_u64(layers.values().map(|l| l.limitation_bandwidth_ms)); + sample.quality_limitation_cpu_ms = max_u64(layers.values().map(|l| l.limitation_cpu_ms)); + } + /// Close every open window into its event, filed under the window's session, and start fresh. pub fn close(&mut self) -> Vec { let end = now_unix_nanos(); @@ -363,6 +417,29 @@ impl StreamDirection { mod tests { use super::*; + #[test] + fn simulcast_layers_fold_into_one_track_series() { + let mut windows = StatsWindows::default(); + let layer = |id: &str, bytes: u64, fps: f64| RtcStatsSample { + bytes: Some(bytes), + frames_per_second: Some(fps), + layer: Some(id.into()), + ..RtcStatsSample::new("TR_1", TrackKind::Video, StreamDirection::Outbound) + }; + windows.record(layer("h", 1_000, 30.0)); + windows.record(layer("f", 4_000, 30.0)); + // The top layer stalls; the half layer keeps sending. + windows.record(layer("h", 3_000, 30.0)); + windows.record(layer("f", 4_000, 0.0)); + let events = windows.close_events(); + assert_eq!(events.len(), 1, "one window per track, not per layer"); + let bytes = + events[0].attributes.iter().find(|a| a.key == "lk.rtc.bytes").map(|a| a.value.clone()); + assert_eq!(bytes, Some(AttributeValue::Int(7_000)), "last folded total"); + let body = events[0].body.clone().unwrap_or_default(); + assert!(!body.contains(" 0 kbps"), "a stalled layer is not a stalled track: {body}"); + } + #[tokio::test(start_paused = true)] async fn cpu_limitation_counter_drives_cadence_pressure() { let mut windows = StatsWindows::default(); diff --git a/livekit-telemetry/src/session.rs b/livekit-telemetry/src/session.rs index 91d7aba2f..60dd99224 100644 --- a/livekit-telemetry/src/session.rs +++ b/livekit-telemetry/src/session.rs @@ -18,7 +18,8 @@ use std::{ }; use crate::{ - Attribute, AttributeValue, RtcStatsSample, SpanKind, SpanOutcome, Telemetry, TelemetryEvent, + Attribute, AttributeValue, RtcStatsSample, Span, SpanKind, SpanName, SpanOutcome, Telemetry, + TelemetryEvent, }; /// One session's identity: the trace id every one of its records carries, and the attributes @@ -86,6 +87,20 @@ impl fmt::Debug for SessionState { /// lost; a `Session` is what a room — one call — gets from it, and what its spans, RTC windows /// and events are filed under. Everything emitted outside a session (device state, pre-room /// errors, self-telemetry) belongs to the pipeline's own process session. Cheap to clone. +/// Who this session is: attached to every record once the room is joined. `None` clears. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct RoomIdentity { + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub sid: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub name: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub participant_sid: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub participant_identity: Option, +} + #[derive(Clone)] pub struct Session { pub(crate) telemetry: Telemetry, @@ -120,6 +135,24 @@ impl Session { } /// Open a span in this session's trace. + /// Start a typed span in this session's trace, stamped now. `parent` nests it. + pub fn start(&self, name: SpanName, parent: Option>) -> Arc { + let parent = parent.and_then(|p| p.context()).map(|c| c.span_id); + Span::bound(name, parent, self.telemetry.clone(), &self.state) + } + + /// The room and local participant, as `lk.room.*` / `lk.participant.*` on every record. + pub fn set_room(&self, room: RoomIdentity) { + for (key, value) in [ + ("lk.room.sid", room.sid), + ("lk.room.name", room.name), + ("lk.participant.sid", room.participant_sid), + ("lk.participant.identity", room.participant_identity), + ] { + self.set_attribute(key, value.map(AttributeValue::Str)); + } + } + pub fn begin_span(&self, name: &str, kind: SpanKind, parent: Option) -> u64 { self.telemetry.begin_span_in(name, kind, parent, &self.state) } diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index db0333a4f..e0d0ca156 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -21,6 +21,7 @@ use std::{ use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; +use crate::DeviceEvent; use crate::{ event::now_unix_nanos, exporter::Command, @@ -57,6 +58,9 @@ pub struct TelemetryConfig { /// Resource attributes describing the emitter (`service.name`, `os.name`, /// `device.model.identifier`, `session.id`, …). `telemetry.sdk.*` are filled in by the core. pub resource: Vec, + /// Who is reporting, typed; the core owns the semconv keys. Extra attributes go in `resource`. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub sdk: Option, /// Directory for the on-disk batch cache (created if missing; its parent must exist). /// `None` keeps batches in memory only: they survive failed uploads, not the process. #[cfg_attr(feature = "uniffi", uniffi(default))] @@ -110,6 +114,7 @@ impl TelemetryConfig { traces_endpoint: None, headers: HashMap::new(), resource: Vec::new(), + sdk: None, storage_dir: None, max_cache_bytes: 4 * 1024 * 1024, flush_interval_ms: 1000, @@ -254,7 +259,7 @@ impl Telemetry { transport: Arc, cache: Arc, ) -> (Self, Exporter) { - add_sdk_resource(&mut config.resource); + add_sdk_resource(&mut config.resource, config.sdk.as_ref()); let destination = Arc::new(Mutex::new(config.endpoint.as_deref().map(|endpoint| { Destination::new(endpoint, config.traces_endpoint.clone(), config.headers.clone()) }))); @@ -346,6 +351,12 @@ impl Telemetry { /// A record with an empty `name` is a plain log line: only `Warn` and `Error` ones leave the /// device (design doc: debug/info logs never do). Discrete events are subject to the flood /// guard (`max_events_per_10min`); what it drops is counted as `rate_limited`. + /// Something happened to the device mid-call (audio route, interruption, a denied permission): + /// a process-level record with a display body, built here so every platform files it alike. + pub fn device_event(&self, event: DeviceEvent) { + self.emit_in(event.into_event(), &self.process); + } + /// A captured log line. WebRTC only counts at error; the SDK and the core at the configured /// floor; the core's own telemetry module never (a rejected batch that produced a record that /// produced a batch would never end). @@ -520,8 +531,67 @@ fn derive_traces_endpoint(logs_endpoint: &str) -> String { } } -/// Fill in the `telemetry.sdk.*` resource attributes and a fallback `service.name`. -fn add_sdk_resource(resource: &mut Vec) { +/// Which LiveKit client SDK is reporting: `service.name` becomes `livekit-client-`. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Sdk { + Swift, + Android, + Flutter, + ReactNative, + Unity, + Rust, +} + +impl Sdk { + fn as_str(self) -> &'static str { + match self { + Self::Swift => "swift", + Self::Android => "android", + Self::Flutter => "flutter", + Self::ReactNative => "react-native", + Self::Unity => "unity", + Self::Rust => "rust", + } + } +} + +/// The reporting SDK and the device it runs on. Lowered to semconv: `service.name`, +/// `service.version`, `os.name`, `os.version`, `device.model.identifier`. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TelemetryResource { + pub sdk: Sdk, + pub sdk_version: String, + pub os_name: String, + pub os_version: String, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub device_model: Option, +} + +impl TelemetryResource { + fn attributes(&self) -> Vec { + let mut out = vec![ + Attribute::new("service.name", format!("livekit-client-{}", self.sdk.as_str())), + Attribute::new("service.version", self.sdk_version.as_str()), + Attribute::new("os.name", self.os_name.as_str()), + Attribute::new("os.version", self.os_version.as_str()), + ]; + if let Some(model) = &self.device_model { + out.push(Attribute::new("device.model.identifier", model.as_str())); + } + out + } +} + +/// Lower the typed resource, then fill in the `telemetry.sdk.*` attributes and a fallback +/// `service.name`. Attributes already present (the open bag) win. +fn add_sdk_resource(resource: &mut Vec, sdk: Option<&TelemetryResource>) { + for attribute in sdk.map(TelemetryResource::attributes).unwrap_or_default() { + if !resource.iter().any(|a| a.key == attribute.key) { + resource.push(attribute); + } + } let defaults = [ ("service.name", "livekit-client"), ("telemetry.sdk.name", env!("CARGO_PKG_NAME")), @@ -553,6 +623,7 @@ pub(crate) fn observability_endpoint(url: &str) -> Option { #[cfg(test)] mod tests { + use crate::{RoomIdentity, SpanName, SpanStep}; use std::{collections::VecDeque, fs, path::Path, sync::Mutex}; #[test] @@ -575,6 +646,66 @@ mod tests { assert_eq!(ep("wss:///rtc"), None); } + #[tokio::test(start_paused = true)] + async fn typed_spans_hold_uploads_while_connecting_and_export_when_ended() { + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + let session = telemetry.begin_session(); + let span = session.start(SpanName::Reconnect { reason: "ws closed".into() }, None); + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + assert!(transport.sent().is_empty(), "an open reconnect holds uploads"); + span.step(SpanStep::Attempt { number: 1, full: false }); + span.end(SpanOutcome::Ok, None); + assert!(span.context().is_some_and(|c| c.trace_id == session.trace_id())); + telemetry.flush().await; + assert!(transport.sent().iter().any(|r| r.url.contains("traces")), "the span is exported"); + } + + #[tokio::test(start_paused = true)] + async fn room_identity_and_resource_are_typed() { + let transport = FakeTransport::scripted([]); + let mut config = TelemetryConfig::new("http://collector/v1/logs"); + config.sdk = Some(TelemetryResource { + sdk: Sdk::Swift, + sdk_version: "2.16.0".into(), + os_name: "iOS".into(), + os_version: "19.0".into(), + device_model: Some("iPhone17,1".into()), + }); + let telemetry = start(config, transport.clone()); + let session = telemetry.begin_session(); + session.set_room(RoomIdentity { + sid: Some("RM_a".into()), + name: Some("telemetry".into()), + ..Default::default() + }); + session.emit(TelemetryEvent::new("lk.ping")); + telemetry.device_event(DeviceEvent::AudioInterruption { began: true }); + telemetry.flush().await; + let sent = transport.sent(); + let logs = records(&sent[0]); + let with_room = + logs.iter().find(|r| attribute(r, "lk.room.sid").is_some()).expect("room record"); + assert_eq!(attribute(with_room, "lk.room.sid"), Some(Value::StringValue("RM_a".into()))); + assert!(logs.iter().any(|r| r.body.as_ref().and_then(|b| b.value.clone()) + == Some(Value::StringValue("audio interruption began".into())))); + let decoded = + ExportLogsServiceRequest::decode(&gunzip(&sent[0].body)[..]).expect("valid OTLP"); + let resource = decoded.resource_logs[0].resource.as_ref().expect("resource"); + let value = |key: &str| { + resource + .attributes + .iter() + .find(|kv| kv.key == key) + .and_then(|kv| kv.value.as_ref()) + .and_then(|v| v.value.clone()) + }; + assert_eq!(value("service.name"), Some(Value::StringValue("livekit-client-swift".into()))); + assert_eq!(value("device.model.identifier"), Some(Value::StringValue("iPhone17,1".into()))); + assert!(value("telemetry.sdk.name").is_some()); + } + #[tokio::test(start_paused = true)] async fn log_records_apply_the_source_floor_and_carry_code_attributes() { let transport = FakeTransport::scripted([]); diff --git a/livekit-telemetry/src/trace.rs b/livekit-telemetry/src/trace.rs new file mode 100644 index 000000000..76877fb51 --- /dev/null +++ b/livekit-telemetry/src/trace.rs @@ -0,0 +1,371 @@ +//! The SDK's span vocabulary and the span itself, owned by the core so every platform names, +//! times and describes an operation the same way. Every call is synchronous and stamps the clock +//! inside, so the only skew is the FFI call itself; context propagation (the "current" span) +//! stays with the platform runtime, which is the one thing a core cannot do. + +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +use tokio::time::Instant; + +use crate::{ + session::SessionState, Attribute, AttributeValue, SpanKind, SpanOutcome, Telemetry, TrackKind, +}; + +/// What an SDK operation is. The kind follows from the name: connects talk to the server +/// (`client`), the rest is internal work. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SpanName { + Connect, + Reconnect { + reason: String, + }, + Publish, + Subscribe, + /// An app-defined span; its name is the consumer's. + Custom { + name: String, + }, +} + +impl SpanName { + pub fn label(&self) -> &str { + match self { + Self::Connect => "lk.connect", + Self::Reconnect { .. } => "lk.reconnect", + Self::Publish => "lk.publish", + Self::Subscribe => "lk.subscribe", + Self::Custom { name } => name, + } + } + + fn kind(&self) -> SpanKind { + match self { + Self::Connect | Self::Reconnect { .. } => SpanKind::Client, + _ => SpanKind::Internal, + } + } + + fn attributes(&self) -> Vec { + match self { + Self::Reconnect { reason } => { + vec![Attribute::new("lk.reconnect.reason", reason.as_str())] + } + _ => Vec::new(), + } + } +} + +/// A checkpoint inside a span. One vocabulary for all spans; the core does not police which step +/// belongs to which span, the dashboard does. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SpanStep { + WsOpen, + Signal, + JoinRecv, + PcCreated, + OfferSent, + AnswerSent, + Engine, + PcConnected, + RoomConnected, + Subscribed, + FirstMedia, + /// One reconnect attempt; also sets `lk.reconnect.attempts` and `lk.reconnect.mode`. + Attempt { + number: u32, + full: bool, + }, + Custom { + name: String, + }, +} + +impl SpanStep { + fn label(&self) -> String { + match self { + Self::WsOpen => "ws_open".into(), + Self::Signal => "signal".into(), + Self::JoinRecv => "join_recv".into(), + Self::PcCreated => "pc_created".into(), + Self::OfferSent => "offer_sent".into(), + Self::AnswerSent => "answer_sent".into(), + Self::Engine => "engine".into(), + Self::PcConnected => "pc_connected".into(), + Self::RoomConnected => "room_connected".into(), + Self::Subscribed => "subscribed".into(), + Self::FirstMedia => "first_media".into(), + Self::Attempt { number, full } => { + format!("attempt {number} {}", if *full { "full" } else { "quick" }) + } + Self::Custom { name } => name.clone(), + } + } + + fn attributes(&self) -> Vec { + match self { + Self::Attempt { number, full } => vec![ + Attribute::new("lk.reconnect.attempts", *number as i64), + Attribute::new("lk.reconnect.mode", if *full { "full" } else { "quick" }), + ], + _ => Vec::new(), + } + } +} + +/// The track a publish or subscribe span is about. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct SpanTrack { + /// Unknown until the server assigns it (publish): set the track again once it is. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub sid: Option, + pub kind: TrackKind, + /// `camera`, `microphone`, `screen_share`, … as the platform names it. + pub source: String, + /// The publisher, for subscribe spans. + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub remote_identity: Option, +} + +impl SpanTrack { + fn attributes(&self) -> Vec { + let mut out = vec![ + Attribute::new("lk.track.kind", format!("{:?}", self.kind).to_lowercase()), + Attribute::new("lk.track.source", self.source.as_str()), + ]; + if let Some(sid) = &self.sid { + out.push(Attribute::new("lk.track.sid", sid.as_str())); + } + if let Some(identity) = &self.remote_identity { + out.push(Attribute::new("lk.participant.remote_identity", identity.as_str())); + } + out + } +} + +/// A span's identity in its session's trace, for log correlation. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TraceContext { + pub trace_id: String, + pub span_id: u64, +} + +struct Bound { + telemetry: Telemetry, + trace_id: String, + id: u64, +} + +#[derive(Default)] +struct State { + /// (label, offset from start) + steps: Vec<(String, Duration)>, + attributes: Vec, + ended: Option<(SpanOutcome, Duration)>, +} + +/// One attempt at an SDK operation: a timed interval with checkpoints, attributes and an outcome. +/// Bound to a session it is exported as an OTLP span when it ends; detached it still times and +/// describes itself, so the console line looks the same with telemetry off. +pub struct Span { + name: SpanName, + started: Instant, + bound: Option, + state: Mutex, +} + +fn upsert(attributes: &mut Vec, attribute: Attribute) { + attributes.retain(|a| a.key != attribute.key); + attributes.push(attribute); +} + +impl Span { + /// Timings and a description only; nothing is exported. + pub fn detached(name: SpanName) -> Arc { + Arc::new(Self::new(name, None)) + } + + pub(crate) fn bound( + name: SpanName, + parent: Option, + telemetry: Telemetry, + session: &Arc, + ) -> Arc { + let id = telemetry.begin_span_in(name.label(), name.kind(), parent, session); + Arc::new(Self::new(name, Some(Bound { telemetry, trace_id: session.hex(), id }))) + } + + fn new(name: SpanName, bound: Option) -> Self { + let state = State { attributes: name.attributes(), ..State::default() }; + Self { name, started: Instant::now(), bound, state: Mutex::new(state) } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(|e| e.into_inner()) + } + + pub fn name(&self) -> SpanName { + self.name.clone() + } + + pub fn label(&self) -> String { + self.name.label().to_owned() + } + + /// A checkpoint, stamped now. Ignored once the span has ended. + pub fn step(&self, step: SpanStep) { + let at = self.started.elapsed(); + let label = step.label(); + { + let mut state = self.lock(); + if state.ended.is_some() { + return; + } + state.steps.push((label.clone(), at)); + for attribute in step.attributes() { + upsert(&mut state.attributes, attribute); + } + } + if let Some(bound) = &self.bound { + bound.telemetry.add_span_event(bound.id, &label, Vec::new()); + } + } + + /// The open bag, for app-defined spans and one-off details. Replaces an existing key. + pub fn set_attribute(&self, key: String, value: AttributeValue) { + upsert(&mut self.lock().attributes, Attribute::new(key, value)); + } + + pub fn set_track(&self, track: SpanTrack) { + let mut state = self.lock(); + for attribute in track.attributes() { + upsert(&mut state.attributes, attribute); + } + } + + /// End once; later calls are no-ops. `error` becomes `error.type` and the status message. + pub fn end(&self, outcome: SpanOutcome, error: Option) { + let at = self.started.elapsed(); + let attributes = { + let mut state = self.lock(); + if state.ended.is_some() { + return; + } + state.ended = Some((outcome, at)); + state.attributes.clone() + }; + if let Some(bound) = &self.bound { + bound.telemetry.end_span(bound.id, outcome, error, attributes); + } + } + + pub fn fail(&self, error: String) { + self.end(SpanOutcome::Error, Some(error)); + } + + pub fn cancel(&self) { + self.end(SpanOutcome::Cancelled, None); + } + + pub fn is_ended(&self) -> bool { + self.lock().ended.is_some() + } + + pub fn outcome(&self) -> Option { + self.lock().ended.map(|(outcome, _)| outcome) + } + + /// `None` for a detached span. + pub fn context(&self) -> Option { + self.bound.as_ref().map(|b| TraceContext { trace_id: b.trace_id.clone(), span_id: b.id }) + } + + /// Seconds from start to the end, or to the last step while still running. + pub fn total_secs(&self) -> f64 { + let state = self.lock(); + state + .ended + .map(|(_, at)| at) + .or_else(|| state.steps.last().map(|(_, at)| *at)) + .unwrap_or_default() + .as_secs_f64() + } + + /// `lk.connect: ws_open +1.49s, signal +0.03s, total 1.83s, ok` — the same line on every + /// platform, for the console when a span ends. + pub fn describe(&self) -> String { + let state = self.lock(); + let mut parts = Vec::with_capacity(state.steps.len() + 2); + let mut previous = Duration::ZERO; + for (label, at) in &state.steps { + parts.push(format!("{label} +{:.2}s", at.saturating_sub(previous).as_secs_f64())); + previous = *at; + } + let total = state.ended.map(|(_, at)| at).unwrap_or(previous); + parts.push(format!("total {:.2}s", total.as_secs_f64())); + if let Some((outcome, _)) = state.ended { + parts.push(outcome.as_str().to_owned()); + } + format!("{}: {}", self.name.label(), parts.join(", ")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn a_detached_span_times_and_describes_itself() { + let span = Span::detached(SpanName::Reconnect { reason: "ws closed".into() }); + tokio::time::advance(Duration::from_millis(1490)).await; + span.step(SpanStep::Attempt { number: 1, full: false }); + tokio::time::advance(Duration::from_millis(30)).await; + span.step(SpanStep::WsOpen); + assert_eq!( + span.describe(), + "lk.reconnect: attempt 1 quick +1.49s, ws_open +0.03s, total 1.52s" + ); + tokio::time::advance(Duration::from_millis(10)).await; + span.end(SpanOutcome::Ok, None); + span.fail("late".into()); + assert_eq!(span.outcome(), Some(SpanOutcome::Ok), "ends once"); + assert_eq!( + span.describe(), + "lk.reconnect: attempt 1 quick +1.49s, ws_open +0.03s, total 1.53s, ok" + ); + assert!(span.context().is_none()); + let attributes = span.lock().attributes.clone(); + let get = |key: &str| attributes.iter().find(|a| a.key == key).map(|a| a.value.clone()); + assert_eq!(get("lk.reconnect.reason"), Some(AttributeValue::Str("ws closed".into()))); + assert_eq!(get("lk.reconnect.attempts"), Some(AttributeValue::Int(1))); + assert_eq!(get("lk.reconnect.mode"), Some(AttributeValue::Str("quick".into()))); + } + + #[test] + fn a_track_sets_its_attributes_and_can_learn_its_sid_later() { + let span = Span::detached(SpanName::Publish); + span.set_track(SpanTrack { + sid: None, + kind: TrackKind::Video, + source: "camera".into(), + remote_identity: None, + }); + span.set_track(SpanTrack { + sid: Some("TR_1".into()), + kind: TrackKind::Video, + source: "camera".into(), + remote_identity: None, + }); + let attributes = span.lock().attributes.clone(); + let get = |key: &str| attributes.iter().find(|a| a.key == key).map(|a| a.value.clone()); + assert_eq!(get("lk.track.kind"), Some(AttributeValue::Str("video".into()))); + assert_eq!(get("lk.track.sid"), Some(AttributeValue::Str("TR_1".into()))); + assert_eq!(attributes.iter().filter(|a| a.key == "lk.track.source").count(), 1); + } +} diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index b9eaf1592..2c484ad52 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -17,9 +17,9 @@ use std::{ }; use livekit_telemetry::{ - Attribute, AttributeValue, DeviceState, ExportError, ExportRequest, LogRecord, NetTransport, - RtcStatsSample, SpanKind, SpanOutcome, TelemetryConfig, TelemetryEvent, TelemetryStats, - TelemetryTransport, + Attribute, AttributeValue, DeviceEvent, DeviceState, ExportError, ExportRequest, LogRecord, + NetTransport, RoomIdentity, RtcStatsSample, SpanName, SpanOutcome, SpanStep, SpanTrack, + TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, TraceContext, }; use tokio::sync::{mpsc, oneshot}; @@ -78,6 +78,11 @@ impl Telemetry { self.0.log(record); } + /// Audio route, interruption, denied permission: a process-level record built by the core. + pub fn device_event(&self, event: DeviceEvent) { + self.0.device_event(event); + } + /// Cloud rule: server URL → observability endpoint, room token → bearer header. No-op when /// the config names an explicit endpoint. pub fn set_server(&self, url: String, token: String) { @@ -112,27 +117,6 @@ impl Telemetry { self.0.trace_id() } - /// Open a span (one attempt at `lk.connect`, `lk.publish`, …); returns its handle. - pub fn begin_span(&self, name: String, kind: SpanKind, parent: Option) -> u64 { - self.0.begin_span(&name, kind, parent) - } - - /// Record a checkpoint inside an open span, stamped now. - pub fn add_span_event(&self, span: u64, name: String, attributes: Vec) { - self.0.add_span_event(span, &name, attributes); - } - - /// End a span with its outcome; exported with the next batch. - pub fn end_span( - &self, - span: u64, - outcome: SpanOutcome, - error_type: Option, - attributes: Vec, - ) { - self.0.end_span(span, outcome, error_type, attributes); - } - /// Push one `getStats()` reading for a track; windowed on device into `lk.rtc.stats.sample`. pub fn record_stats(&self, sample: RtcStatsSample) { self.0.record_stats(sample); @@ -188,22 +172,14 @@ impl TelemetrySession { self.0.record_stats(sample); } - pub fn begin_span(&self, name: String, kind: SpanKind, parent: Option) -> u64 { - self.0.begin_span(&name, kind, parent) + /// Start a typed span in this session's trace, stamped now; `parent` nests it. + pub fn start(&self, name: SpanName, parent: Option>) -> Arc { + Arc::new(TelemetrySpan(self.0.start(name, parent.map(|p| p.0.clone())))) } - pub fn add_span_event(&self, span: u64, name: String, attributes: Vec) { - self.0.add_span_event(span, &name, attributes); - } - - pub fn end_span( - &self, - span: u64, - outcome: SpanOutcome, - error_type: Option, - attributes: Vec, - ) { - self.0.end_span(span, outcome, error_type, attributes); + /// The room and local participant, on every record of this session from now on. + pub fn set_room(&self, room: RoomIdentity) { + self.0.set_room(room); } } @@ -227,6 +203,75 @@ struct Pending { /// Exists because uniffi-dart's foreign-trait callbacks are isolate-bound (`Pointer.fromFunction`) /// and abort the VM when invoked from a tokio thread; Swift and Kotlin callbacks are thread-agnostic /// and use [`TelemetryTransport`] directly. +/// One attempt at an SDK operation, owned by the core. Every call is synchronous and stamps the +/// clock inside, so the only skew is the FFI call; `describe()` is the console line on every +/// platform. Detached (no session) it still times and describes itself. +#[derive(uniffi::Object)] +pub struct TelemetrySpan(Arc); + +#[uniffi::export] +impl TelemetrySpan { + /// Timings and a description only; nothing is exported. + #[uniffi::constructor] + pub fn detached(name: SpanName) -> Arc { + Arc::new(Self(livekit_telemetry::Span::detached(name))) + } + + pub fn label(&self) -> String { + self.0.label() + } + + /// A checkpoint, stamped now. + pub fn step(&self, step: SpanStep) { + self.0.step(step); + } + + /// The open bag; replaces an existing key. + pub fn set_attribute(&self, key: String, value: AttributeValue) { + self.0.set_attribute(key, value); + } + + pub fn set_track(&self, track: SpanTrack) { + self.0.set_track(track); + } + + /// End once; `error` becomes `error.type` and the status message. + pub fn end(&self, outcome: SpanOutcome, error: Option) { + self.0.end(outcome, error); + } + + pub fn fail(&self, error: String) { + self.0.fail(error); + } + + pub fn cancel(&self) { + self.0.cancel(); + } + + pub fn is_ended(&self) -> bool { + self.0.is_ended() + } + + pub fn outcome(&self) -> Option { + self.0.outcome() + } + + /// `None` for a detached span. + pub fn context(&self) -> Option { + self.0.context() + } + + /// Seconds to the end, or to the last step while running. + pub fn total_secs(&self) -> f64 { + self.0.total_secs() + } + + /// `lk.connect: ws_open +1.49s, …, total 1.83s, ok` + pub fn describe(&self) -> String { + self.0.describe() + } +} + #[derive(uniffi::Object)] pub struct TelemetryExportQueue { tx: mpsc::UnboundedSender, From 1f4e6c6718546f86052be3293e6e9dc6eb145b48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:21:32 +0200 Subject: [PATCH 30/50] docs(telemetry): the typed surface Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/SPEC.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index f2650296c..2dd93b220 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -329,3 +329,29 @@ attributes: lk.participant.remote_identity: string checkpoints: subscribed, first_media ``` + +## Typed surface + +Everything the SDKs have in common enters the core typed; the core owns the keys, the bodies and +the policy. `Attribute { key, value }` survives only as the open bag: `emit_custom`, pipeline and +session `set_attribute`, and `Span::set_attribute` for app-defined spans. + +| Platform calls | The core produces | +|---|---| +| `set_server(url, token)` | `https:///observability/logs/otlp/v0`, `Authorization: Bearer `; no-op with an explicit `endpoint` | +| `TelemetryConfig.sdk: TelemetryResource { sdk: Sdk, sdk_version, os_name, os_version, device_model }` | `service.name = livekit-client-`, `service.version`, `os.*`, `device.model.identifier`, plus `telemetry.sdk.*` | +| `log(LogRecord { severity, source: LogSource, message, logger, function, file, line, timestamp_ns, span_id })` | a record with `code.function.name`, `code.file.path`, `code.line.number`, `lk.log.source`, `lk.log.logger`; the per-source floor (WebRTC at `error`, own module never) | +| `Session::set_room(RoomIdentity { sid, name, participant_sid, participant_identity })` | `lk.room.*`, `lk.participant.*` on every record of the session | +| `Session::start(SpanName, parent) -> Span`; `Span::detached(name)` | an OTLP span (`lk.connect` / `lk.reconnect` are `client`, the rest `internal`); `Reconnect { reason }` sets `lk.reconnect.reason` | +| `Span::step(SpanStep)` | a span event named `ws_open` … `room_connected`, `subscribed`, `first_media`, `attempt N quick|full` (which also sets `lk.reconnect.attempts` / `.mode`) | +| `Span::set_track(SpanTrack { sid, kind, source, remote_identity })` | `lk.track.sid`, `lk.track.kind`, `lk.track.source`, `lk.participant.remote_identity` | +| `Span::end(outcome, error)` / `fail(error)` / `cancel()` | status, `error.type`, `lk.outcome`; ending twice is a no-op | +| `Span::describe()` | `lk.connect: ws_open +1.49s, signal +0.03s, total 1.83s, ok` — the console line, identical on every platform | +| `Span::context()` | `TraceContext { trace_id, span_id }` for log correlation; `None` when detached | +| `device_event(DeviceEvent::{AudioRouteChanged, AudioInterruption, PermissionDenied})` | `lk.device.audio_route.changed`, `lk.device.audio.interruption`, `lk.device.permission.denied` with display bodies | +| `RtcStatsSample.layer` (rid, ssrc or stats id) | simulcast layers folded into one monotonic series per track before windowing | + +Timing rule: span calls are synchronous and stamp the clock inside the core, so the only skew is +the FFI call. Anything that may cross an executor hop before reaching the core (a log record) carries +its own `timestamp_ns` from capture. Context propagation — the "current" span — stays with the +platform runtime (task-local, coroutine context, zone); that is the one piece a core cannot own. From b5cd693bcbda9ffef9860d88ca5cbffcc48e35e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:31:08 +0200 Subject: [PATCH 31/50] feat(uniffi): span_label for the platform's own logging Co-Authored-By: Claude Fable 5.1 --- livekit-uniffi/src/telemetry.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index 2c484ad52..b86dec760 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -272,6 +272,12 @@ impl TelemetrySpan { } } +/// A span's wire name (`lk.connect`, …, or the custom name), for the platform's own logging. +#[uniffi::export] +pub fn span_label(name: SpanName) -> String { + name.label().to_owned() +} + #[derive(uniffi::Object)] pub struct TelemetryExportQueue { tx: mpsc::UnboundedSender, From ca2b0cee07ecd6637e09fd6d05b37ad16919df7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:35:37 +0200 Subject: [PATCH 32/50] feat(telemetry): process-level spans via Telemetry::start Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/telemetry.rs | 9 ++++++++- livekit-uniffi/src/telemetry.rs | 5 +++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index e0d0ca156..e6f063c8e 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -21,7 +21,6 @@ use std::{ use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; -use crate::DeviceEvent; use crate::{ event::now_unix_nanos, exporter::Command, @@ -34,6 +33,7 @@ use crate::{ MemoryCache, RtcStatsSample, Severity, SpanKind, SpanOutcome, TelemetryEvent, TelemetryStats, TelemetryTransport, }; +use crate::{DeviceEvent, Span, SpanName}; /// Pipeline configuration. /// @@ -345,6 +345,13 @@ impl Telemetry { Session { telemetry: self.clone(), state: SessionState::new() } } + /// A span in the pipeline's own trace: app-defined work outside any room, or the SDK before a + /// room exists. Stamped now; `parent` nests it. + pub fn start(&self, name: SpanName, parent: Option>) -> Arc { + let parent = parent.and_then(|p| p.context()).map(|c| c.span_id); + Span::bound(name, parent, self.clone(), &self.process) + } + /// Queue an event or log record for export. Stamps it with the current time unless it /// carries one. /// diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index b86dec760..e00c875c4 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -89,6 +89,11 @@ impl Telemetry { self.0.set_server(&url, &token); } + /// A span in the process trace (no room), stamped now; `parent` nests it. + pub fn start(&self, name: SpanName, parent: Option>) -> Arc { + Arc::new(TelemetrySpan(self.0.start(name, parent.map(|p| p.0.clone())))) + } + /// Where to send, once known (first connect: server URL → endpoint, token → headers). Until /// then everything waits in the cache; afterwards it uploads. pub fn set_destination(&self, endpoint: String, headers: HashMap) { From 2e6393f12f14a80fe2cbda7163257b7c55cf5935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:39:18 +0200 Subject: [PATCH 33/50] refactor(telemetry): Session is Scope A scope is a trace id and the room's identity attributes on one shared pipeline; the name now says so. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/SPEC.md | 28 ++++++++-------- livekit-telemetry/src/exporter.rs | 6 ++-- livekit-telemetry/src/lib.rs | 4 +-- livekit-telemetry/src/otlp.rs | 4 +-- livekit-telemetry/src/rtc.rs | 10 +++--- .../src/{session.rs => scope.rs} | 18 +++++------ livekit-telemetry/src/span.rs | 12 +++---- livekit-telemetry/src/store.rs | 6 ++-- livekit-telemetry/src/telemetry.rs | 32 +++++++++---------- livekit-telemetry/src/trace.rs | 4 +-- livekit-uniffi/src/telemetry.rs | 10 +++--- 11 files changed, 67 insertions(+), 67 deletions(-) rename livekit-telemetry/src/{session.rs => scope.rs} (94%) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index 2dd93b220..2ca9eb515 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -16,16 +16,16 @@ Set once per pipeline (`TelemetryConfig.resource`): | `device.model.identifier` | platform SDK | `iPhone16,1` | | `telemetry.sdk.name/language/version` | core | `livekit-telemetry`, `rust`, `0.1.0` | -## Pipeline, sessions and destination +## Pipeline, scopes and destination One pipeline per process — started at SDK init, so audio pre-initialization, permission failures -and connect attempts that never reach a server are captured — and one **session** per room (one -call). A session is a trace id plus the attributes attached to its records (`lk.room.sid`, -`lk.participant.identity`, …); spans, RTC windows and events are filed under the session that +and connect attempts that never reach a server are captured — and one **scope** per room (one +call). A scope is a trace id plus the attributes attached to its records (`lk.room.sid`, +`lk.participant.identity`, …); spans, RTC windows and events are filed under the scope that produced them, and `session.id` (OTel semconv) is written on every record as an attribute. A log -record emitted inside a room's span is filed under that room's session; anything emitted outside -a session — device state, pre-room errors, self-telemetry — belongs to the pipeline's own process -session. Sessions are not ended: a room's last record is simply its last. +record emitted inside a room's span is filed under that room's scope; anything emitted outside +a scope — device state, pre-room errors, self-telemetry — belongs to the pipeline's own process +scope. Scopes are not ended: a room's last record is simply its last. The pipeline may start **without a destination** (`endpoint: None`): it buffers and caches, and uploads nothing until `set_server(url, token)` (Cloud: `https:///observability/logs/otlp/v0`, @@ -70,7 +70,7 @@ attributes: cadence: appended to the next batch whenever a drop, an upload failure or a capped hold happened since the previous report — never its own request, never persisted on its own (Sentry client report shape; reasons follow the OTel SDK self-metrics `error.type` values) — - and once at shutdown as the session summary, so fleet-wide success rates have + and once at shutdown as the scope summary, so fleet-wide success rates have denominators. Never emitted after the collector disabled telemetry. platforms: all ``` @@ -182,7 +182,7 @@ Uploads are shaped, not just batched: - **One request in flight**, oldest batch first; a failure pauses the cache for 60 s (throttling: see `lk.telemetry.report`). -- **Budget:** at most `max_batches_per_upload` (default 4) cached batches per tick while a session +- **Budget:** at most `max_batches_per_upload` (default 4) cached batches per tick while a scope may be live, so a backlog (offline period, previous launch) replays at ~4 × 20 KB per 15 s ≈ 40 kbps next to a call. `shutdown` drains without the budget. - **Holds** — nothing is sent, everything keeps flowing into the write-ahead cache — while: @@ -270,9 +270,9 @@ platforms: all ## Spans -A span is **one attempt** at an operation. The session (one Room connection lifetime, across +A span is **one attempt** at an operation. The scope (one Room connection lifetime, across reconnects) is the trace; its id is generated by the core when the pipeline starts and rides on -every span and log record. Spans are exported when they end — never a long-lived session span. +every span and log record. Spans are exported when they end — never a long-lived scope span. | Rule | Value | |---|---| @@ -334,15 +334,15 @@ checkpoints: subscribed, first_media Everything the SDKs have in common enters the core typed; the core owns the keys, the bodies and the policy. `Attribute { key, value }` survives only as the open bag: `emit_custom`, pipeline and -session `set_attribute`, and `Span::set_attribute` for app-defined spans. +scope `set_attribute`, and `Span::set_attribute` for app-defined spans. | Platform calls | The core produces | |---|---| | `set_server(url, token)` | `https:///observability/logs/otlp/v0`, `Authorization: Bearer `; no-op with an explicit `endpoint` | | `TelemetryConfig.sdk: TelemetryResource { sdk: Sdk, sdk_version, os_name, os_version, device_model }` | `service.name = livekit-client-`, `service.version`, `os.*`, `device.model.identifier`, plus `telemetry.sdk.*` | | `log(LogRecord { severity, source: LogSource, message, logger, function, file, line, timestamp_ns, span_id })` | a record with `code.function.name`, `code.file.path`, `code.line.number`, `lk.log.source`, `lk.log.logger`; the per-source floor (WebRTC at `error`, own module never) | -| `Session::set_room(RoomIdentity { sid, name, participant_sid, participant_identity })` | `lk.room.*`, `lk.participant.*` on every record of the session | -| `Session::start(SpanName, parent) -> Span`; `Span::detached(name)` | an OTLP span (`lk.connect` / `lk.reconnect` are `client`, the rest `internal`); `Reconnect { reason }` sets `lk.reconnect.reason` | +| `Scope::set_room(RoomIdentity { sid, name, participant_sid, participant_identity })` | `lk.room.*`, `lk.participant.*` on every record of the scope | +| `Scope::start(SpanName, parent) -> Span`; `Span::detached(name)` | an OTLP span (`lk.connect` / `lk.reconnect` are `client`, the rest `internal`); `Reconnect { reason }` sets `lk.reconnect.reason` | | `Span::step(SpanStep)` | a span event named `ws_open` … `room_connected`, `subscribed`, `first_media`, `attempt N quick|full` (which also sets `lk.reconnect.attempts` / `.mode`) | | `Span::set_track(SpanTrack { sid, kind, source, remote_identity })` | `lk.track.sid`, `lk.track.kind`, `lk.track.source`, `lk.participant.remote_identity` | | `Span::end(outcome, error)` / `fail(error)` / `cancel()` | status, `error.type`, `lk.outcome`; ending twice is a no-op | diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index c67f990c1..dbbb8cb2f 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -26,7 +26,7 @@ use crate::{ event::now_unix_nanos, otlp, rtc::StatsWindows, - session::SessionState, + scope::ScopeState, span::Spans, stats::{Counters, Snapshot, TelemetryStatus}, store::{Queued, Store}, @@ -131,7 +131,7 @@ pub struct Exporter { windows: Arc>, spans: Arc>, /// The pipeline's own session: self-telemetry is filed under it. - process: Arc, + process: Arc, /// Attributes attached to every record of every session (`Telemetry::set_attribute`). global: Arc>>, /// Where batches go; `None` until `set_destination` — batches wait in the cache meanwhile. @@ -178,7 +178,7 @@ impl Exporter { counters: Arc, windows: Arc>, spans: Arc>, - process: Arc, + process: Arc, global: Arc>>, destination: Arc>>, commands: mpsc::UnboundedReceiver, diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs index 4429f026f..06fa0725e 100644 --- a/livekit-telemetry/src/lib.rs +++ b/livekit-telemetry/src/lib.rs @@ -30,7 +30,7 @@ mod device; mod rtc; /// Spans: one attempt at an operation, with explicit handles across the FFI. -mod session; +mod scope; mod span; /// Batch exporter actor: timer, OTLP encoding, retry policy. @@ -57,7 +57,7 @@ pub use device::*; pub use event::*; pub use exporter::Exporter; pub use rtc::{RtcStatsSample, StreamDirection, TrackKind}; -pub use session::{RoomIdentity, Session}; +pub use scope::{RoomIdentity, Scope}; pub use span::{SpanKind, SpanOutcome}; pub use stats::{TelemetryStats, TelemetryStatus}; pub use telemetry::*; diff --git a/livekit-telemetry/src/otlp.rs b/livekit-telemetry/src/otlp.rs index f2a9f2c68..8b2b5e8e1 100644 --- a/livekit-telemetry/src/otlp.rs +++ b/livekit-telemetry/src/otlp.rs @@ -201,7 +201,7 @@ mod tests { .with_severity(Severity::Warn) .with_body("hi") .with_attribute("lk.ping.seq", 7i64); - let session = crate::session::SessionState::with_trace_id([7u8; 16]); + let session = crate::scope::ScopeState::with_trace_id([7u8; 16]); let bytes = encode_logs(&resource, &[], vec![Queued { event, session }]); let decoded = ExportLogsServiceRequest::decode(&bytes[..]).expect("valid OTLP"); @@ -226,7 +226,7 @@ mod tests { #[test] fn events_without_a_body_carry_their_name_as_body() { - let session = crate::session::SessionState::with_trace_id([7u8; 16]); + let session = crate::scope::ScopeState::with_trace_id([7u8; 16]); let event = TelemetryEvent::new("lk.rtc.stats.sample"); let bytes = encode_logs(&[], &[], vec![Queued { event, session }]); let decoded = ExportLogsServiceRequest::decode(&bytes[..]).expect("valid OTLP"); diff --git a/livekit-telemetry/src/rtc.rs b/livekit-telemetry/src/rtc.rs index 1ade5bc4d..74542a9e7 100644 --- a/livekit-telemetry/src/rtc.rs +++ b/livekit-telemetry/src/rtc.rs @@ -16,7 +16,7 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio::time::Instant; -use crate::{event::now_unix_nanos, session::SessionState, store::Queued, TelemetryEvent}; +use crate::{event::now_unix_nanos, scope::ScopeState, store::Queued, TelemetryEvent}; #[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -154,7 +154,7 @@ impl Gauge { /// Samples of one track in one direction accumulated since the window opened. struct Window { - session: Arc, + session: Arc, start_ns: u64, samples: u32, /// Cumulative counters at the window's first reading, for the display body's deltas. @@ -167,7 +167,7 @@ struct Window { } impl Window { - fn open(start_ns: u64, first: RtcStatsSample, session: Arc) -> Self { + fn open(start_ns: u64, first: RtcStatsSample, session: Arc) -> Self { let mut window = Self { session, start_ns, @@ -312,7 +312,7 @@ fn max_u64(values: impl Iterator>) -> Option { const CPU_LIMITED_HOLD: Duration = Duration::from_secs(60); impl StatsWindows { - pub fn record_in(&mut self, mut sample: RtcStatsSample, session: &Arc) { + pub fn record_in(&mut self, mut sample: RtcStatsSample, session: &Arc) { self.fold_layers(&mut sample); self.track_limitation(&sample); let timestamp = *sample.timestamp_ns.get_or_insert_with(now_unix_nanos); @@ -366,7 +366,7 @@ impl StatsWindows { #[cfg(test)] pub fn record(&mut self, sample: RtcStatsSample) { - self.record_in(sample, &SessionState::new()); + self.record_in(sample, &ScopeState::new()); } #[cfg(test)] diff --git a/livekit-telemetry/src/session.rs b/livekit-telemetry/src/scope.rs similarity index 94% rename from livekit-telemetry/src/session.rs rename to livekit-telemetry/src/scope.rs index 60dd99224..e1fca37c1 100644 --- a/livekit-telemetry/src/session.rs +++ b/livekit-telemetry/src/scope.rs @@ -24,12 +24,12 @@ use crate::{ /// One session's identity: the trace id every one of its records carries, and the attributes /// attached to them at export time (`lk.room.sid`, `lk.participant.identity`, …). -pub(crate) struct SessionState { +pub(crate) struct ScopeState { pub trace_id: [u8; 16], attributes: Mutex>, } -impl SessionState { +impl ScopeState { /// A fresh session: random, non-zero trace id (OTLP treats all-zero as absent). pub fn new() -> Arc { Self::with_trace_id(rand::random::().max(1).to_be_bytes()) @@ -68,15 +68,15 @@ impl SessionState { } } -impl PartialEq for SessionState { +impl PartialEq for ScopeState { fn eq(&self, other: &Self) -> bool { self.trace_id == other.trace_id } } -impl fmt::Debug for SessionState { +impl fmt::Debug for ScopeState { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "Session({})", self.hex()) + write!(f, "Scope({})", self.hex()) } } @@ -84,7 +84,7 @@ impl fmt::Debug for SessionState { /// cadence and exporter as every other session in the process. /// /// The pipeline starts once, at SDK init, so nothing that happens before the first room is -/// lost; a `Session` is what a room — one call — gets from it, and what its spans, RTC windows +/// lost; a `Scope` is what a room — one call — gets from it, and what its spans, RTC windows /// and events are filed under. Everything emitted outside a session (device state, pre-room /// errors, self-telemetry) belongs to the pipeline's own process session. Cheap to clone. /// Who this session is: attached to every record once the room is joined. `None` clears. @@ -102,12 +102,12 @@ pub struct RoomIdentity { } #[derive(Clone)] -pub struct Session { +pub struct Scope { pub(crate) telemetry: Telemetry, - pub(crate) state: Arc, + pub(crate) state: Arc, } -impl Session { +impl Scope { /// The session's trace id as 32 hex characters — print it (`lkt_…`) so support can find /// the call. pub fn trace_id(&self) -> String { diff --git a/livekit-telemetry/src/span.rs b/livekit-telemetry/src/span.rs index dc789b567..9e0244166 100644 --- a/livekit-telemetry/src/span.rs +++ b/livekit-telemetry/src/span.rs @@ -15,7 +15,7 @@ use std::collections::{HashMap, VecDeque}; use std::sync::Arc; -use crate::{event::now_unix_nanos, session::SessionState, Attribute, AttributeValue}; +use crate::{event::now_unix_nanos, scope::ScopeState, Attribute, AttributeValue}; /// OTel span kind, restricted to what client operations need. #[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] @@ -72,7 +72,7 @@ pub(crate) struct SpanRecord { pub attributes: Vec, pub events: Vec, /// The session (trace) the span belongs to. - pub session: Arc, + pub session: Arc, } /// OTel default span limits. @@ -95,7 +95,7 @@ pub(crate) struct Spans { /// Which session every recent span belongs to — open, finished or already exported — so a /// log record that arrives after its span ended (a warning logged right before a failing /// publish ends, delivered a hop later) is still filed under the right session. - sessions: HashMap>, + sessions: HashMap>, session_order: VecDeque, next_id: u64, pub dropped: u64, @@ -128,7 +128,7 @@ impl Spans { name: &str, kind: SpanKind, parent: Option, - session: Arc, + session: Arc, ) -> u64 { let id = self.next_id; self.next_id = self.next_id.wrapping_add(1).max(1); @@ -167,13 +167,13 @@ impl Spans { /// `lk.connect` / `lk.reconnect` are). /// The session a recent span belongs to — open, ended or exported (log records emitted inside /// a span are filed there, and they may arrive after the span ended). - pub fn session_of(&self, id: u64) -> Option> { + pub fn scope_of(&self, id: u64) -> Option> { self.sessions.get(&id).cloned() } #[cfg(test)] pub fn begin(&mut self, name: &str, kind: SpanKind, parent: Option) -> u64 { - self.begin_in(name, kind, parent, SessionState::new()) + self.begin_in(name, kind, parent, ScopeState::new()) } pub fn any_open(&self, names: &[&str]) -> bool { diff --git a/livekit-telemetry/src/store.rs b/livekit-telemetry/src/store.rs index 7d7854d0e..fde56adb9 100644 --- a/livekit-telemetry/src/store.rs +++ b/livekit-telemetry/src/store.rs @@ -17,13 +17,13 @@ use std::{ sync::{Arc, Mutex}, }; -use crate::{session::SessionState, stats::Counters, TelemetryEvent}; +use crate::{scope::ScopeState, stats::Counters, TelemetryEvent}; /// An event waiting for export, filed under the session whose trace id and attributes it /// will carry. pub(crate) struct Queued { pub event: TelemetryEvent, - pub session: Arc, + pub session: Arc, } /// Bounded FIFO of events waiting for export. @@ -103,7 +103,7 @@ mod tests { use super::*; fn queued(event: TelemetryEvent) -> Queued { - Queued { event, session: SessionState::new() } + Queued { event, session: ScopeState::new() } } #[test] diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index e6f063c8e..c34ae7063 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -25,7 +25,7 @@ use crate::{ event::now_unix_nanos, exporter::Command, rtc::StatsWindows, - session::{Session, SessionState}, + scope::{Scope, ScopeState}, span::Spans, stats::{Counters, TelemetryStatus}, store::{Queued, Store}, @@ -165,7 +165,7 @@ pub struct Telemetry { guard: Arc>, spans: Arc>, /// The pipeline's own session: whatever is emitted outside a room session. - process: Arc, + process: Arc, /// Attributes attached to every record of every session. global: Arc>>, destination: Arc>>, @@ -266,7 +266,7 @@ impl Telemetry { let config = Arc::new(config); // One pipeline per process; sessions (rooms) carry their own trace ids. This is the // pipeline's own session, for everything emitted outside a room. - let process = SessionState::new(); + let process = ScopeState::new(); let global = Arc::new(Mutex::new(Vec::new())); let counters = Arc::new(Counters::default()); let status = Arc::new(Mutex::new(TelemetryStatus::Ok)); @@ -341,8 +341,8 @@ impl Telemetry { /// Start a session — one room, one call — with its own trace id and attributes on this /// pipeline. Sessions do not need ending: a room's last record is simply its last. - pub fn begin_session(&self) -> Session { - Session { telemetry: self.clone(), state: SessionState::new() } + pub fn begin_scope(&self) -> Scope { + Scope { telemetry: self.clone(), state: ScopeState::new() } } /// A span in the pipeline's own trace: app-defined work outside any room, or the SDK before a @@ -388,12 +388,12 @@ impl Telemetry { // is the process's own. let session = event .span_id - .and_then(|id| self.spans.lock().unwrap_or_else(|e| e.into_inner()).session_of(id)) + .and_then(|id| self.spans.lock().unwrap_or_else(|e| e.into_inner()).scope_of(id)) .unwrap_or_else(|| self.process.clone()); self.emit_in(event, &session); } - pub(crate) fn emit_in(&self, mut event: TelemetryEvent, session: &Arc) { + pub(crate) fn emit_in(&self, mut event: TelemetryEvent, session: &Arc) { if event.name.is_empty() && event.severity < self.config.log_severity { return; } @@ -418,8 +418,8 @@ impl Telemetry { /// Set a pipeline-wide attribute (a consumer's `enduser.id`, an `acme.tenant`), attached to /// every record of every session from now on unless the record — or its session — already - /// carries the key. `None` removes it. Session-level identity goes through - /// [`Session::set_attribute`]. + /// carries the key. `None` removes it. Scope-level identity goes through + /// [`Scope::set_attribute`]. pub fn set_attribute(&self, key: &str, value: Option) { let mut global = self.global.lock().unwrap_or_else(|e| e.into_inner()); global.retain(|a| a.key != key); @@ -445,7 +445,7 @@ impl Telemetry { name: &str, kind: SpanKind, parent: Option, - session: &Arc, + session: &Arc, ) -> u64 { self.spans.lock().unwrap_or_else(|e| e.into_inner()).begin_in( name, @@ -481,7 +481,7 @@ impl Telemetry { self.record_stats_in(sample, &self.process); } - pub(crate) fn record_stats_in(&self, sample: RtcStatsSample, session: &Arc) { + pub(crate) fn record_stats_in(&self, sample: RtcStatsSample, session: &Arc) { self.windows.lock().unwrap_or_else(|e| e.into_inner()).record_in(sample, session); } @@ -657,7 +657,7 @@ mod tests { async fn typed_spans_hold_uploads_while_connecting_and_export_when_ended() { let transport = FakeTransport::scripted([]); let telemetry = pipeline(transport.clone()); - let session = telemetry.begin_session(); + let session = telemetry.begin_scope(); let span = session.start(SpanName::Reconnect { reason: "ws closed".into() }, None); telemetry.emit(TelemetryEvent::new("lk.ping")); telemetry.flush().await; @@ -681,7 +681,7 @@ mod tests { device_model: Some("iPhone17,1".into()), }); let telemetry = start(config, transport.clone()); - let session = telemetry.begin_session(); + let session = telemetry.begin_scope(); session.set_room(RoomIdentity { sid: Some("RM_a".into()), name: Some("telemetry".into()), @@ -1243,8 +1243,8 @@ mod tests { let transport = FakeTransport::scripted([]); let telemetry = pipeline(transport.clone()); telemetry.set_attribute("acme.tenant", Some("t1".into())); - let a = telemetry.begin_session(); - let b = telemetry.begin_session(); + let a = telemetry.begin_scope(); + let b = telemetry.begin_scope(); assert_ne!(a.trace_id(), b.trace_id()); assert_ne!(a.trace_id(), telemetry.trace_id(), "the process has its own session"); a.set_attribute("lk.room.sid", Some("RM_a".into())); @@ -1313,7 +1313,7 @@ mod tests { assert_eq!(sent.len(), 1, "cached batches ship as soon as the destination is known"); assert_eq!(sent[0].url, "https://x.livekit.cloud/observability/logs/otlp/v0"); assert_eq!(sent[0].headers["Authorization"], "Bearer t"); - telemetry.begin_session().begin_span("lk.publish", SpanKind::Internal, None); + telemetry.begin_scope().begin_span("lk.publish", SpanKind::Internal, None); let span = telemetry.begin_span("lk.publish", SpanKind::Internal, None); telemetry.end_span(span, SpanOutcome::Ok, None, Vec::new()); telemetry.flush().await; diff --git a/livekit-telemetry/src/trace.rs b/livekit-telemetry/src/trace.rs index 76877fb51..ec86a2dd4 100644 --- a/livekit-telemetry/src/trace.rs +++ b/livekit-telemetry/src/trace.rs @@ -11,7 +11,7 @@ use std::{ use tokio::time::Instant; use crate::{ - session::SessionState, Attribute, AttributeValue, SpanKind, SpanOutcome, Telemetry, TrackKind, + scope::ScopeState, Attribute, AttributeValue, SpanKind, SpanOutcome, Telemetry, TrackKind, }; /// What an SDK operation is. The kind follows from the name: connects talk to the server @@ -195,7 +195,7 @@ impl Span { name: SpanName, parent: Option, telemetry: Telemetry, - session: &Arc, + session: &Arc, ) -> Arc { let id = telemetry.begin_span_in(name.label(), name.kind(), parent, session); Arc::new(Self::new(name, Some(Bound { telemetry, trace_id: session.hex(), id }))) diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index e00c875c4..ef09e17bc 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -101,8 +101,8 @@ impl Telemetry { } /// A session — one room, one call — with its own trace id and attributes on this pipeline. - pub fn begin_session(&self) -> Arc { - Arc::new(TelemetrySession(self.0.begin_session())) + pub fn begin_scope(&self) -> Arc { + Arc::new(TelemetryScope(self.0.begin_scope())) } /// Report the device state (thermal, low power, foreground/background). Emits the matching @@ -149,12 +149,12 @@ impl Telemetry { } /// One room's session on the process pipeline: what its spans, stats and events are filed -/// under. Obtained from [`Telemetry::begin_session`]. +/// under. Obtained from [`Telemetry::begin_scope`]. #[derive(uniffi::Object)] -pub struct TelemetrySession(livekit_telemetry::Session); +pub struct TelemetryScope(livekit_telemetry::Scope); #[uniffi::export] -impl TelemetrySession { +impl TelemetryScope { /// The session's trace id as 32 hex characters. pub fn trace_id(&self) -> String { self.0.trace_id() From cf68d51f817db6c248cc251615c2d64522649edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:43:11 +0200 Subject: [PATCH 34/50] feat(telemetry): a bound span logs its description when it ends Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/trace.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/livekit-telemetry/src/trace.rs b/livekit-telemetry/src/trace.rs index ec86a2dd4..fca6012ad 100644 --- a/livekit-telemetry/src/trace.rs +++ b/livekit-telemetry/src/trace.rs @@ -262,6 +262,8 @@ impl Span { }; if let Some(bound) = &self.bound { bound.telemetry.end_span(bound.id, outcome, error, attributes); + // The platform's console line, the same on every SDK (debug: FFI log path). + log::debug!("{}", self.describe()); } } From 412f29c53849fdc013525619bac706e9b4c00a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:53:55 +0200 Subject: [PATCH 35/50] refactor(telemetry): minimal tracing surface Over the FFI a platform gets Scope::start and, on the span, step, set_attribute, set_track, end, fail, cancel, is_ended, context, total_secs and describe. The string span API is test-only, SpanKind is implied by SpanName and no longer exported, process-level and detached spans stay Rust-only. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/lib.rs | 2 +- livekit-telemetry/src/otlp.rs | 3 ++- livekit-telemetry/src/scope.rs | 12 ++++++++---- livekit-telemetry/src/span.rs | 3 +-- livekit-telemetry/src/telemetry.rs | 13 ++++++++----- livekit-telemetry/src/trace.rs | 5 ++--- livekit-uniffi/src/telemetry.rs | 25 ------------------------- 7 files changed, 22 insertions(+), 41 deletions(-) diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs index 06fa0725e..b47a84175 100644 --- a/livekit-telemetry/src/lib.rs +++ b/livekit-telemetry/src/lib.rs @@ -58,7 +58,7 @@ pub use event::*; pub use exporter::Exporter; pub use rtc::{RtcStatsSample, StreamDirection, TrackKind}; pub use scope::{RoomIdentity, Scope}; -pub use span::{SpanKind, SpanOutcome}; +pub use span::SpanOutcome; pub use stats::{TelemetryStats, TelemetryStatus}; pub use telemetry::*; pub use trace::*; diff --git a/livekit-telemetry/src/otlp.rs b/livekit-telemetry/src/otlp.rs index 8b2b5e8e1..15fe75c04 100644 --- a/livekit-telemetry/src/otlp.rs +++ b/livekit-telemetry/src/otlp.rs @@ -14,6 +14,7 @@ use prost::Message; +use crate::span::SpanKind; use crate::{ event::now_unix_nanos, proto::opentelemetry::proto::{ @@ -25,7 +26,7 @@ use crate::{ }, span::SpanRecord, store::Queued, - Attribute, AttributeValue, Severity, SpanKind, SpanOutcome, + Attribute, AttributeValue, Severity, SpanOutcome, }; pub(crate) const CONTENT_TYPE: &str = "application/x-protobuf"; diff --git a/livekit-telemetry/src/scope.rs b/livekit-telemetry/src/scope.rs index e1fca37c1..9d92232e6 100644 --- a/livekit-telemetry/src/scope.rs +++ b/livekit-telemetry/src/scope.rs @@ -17,8 +17,9 @@ use std::{ sync::{Arc, Mutex}, }; +use crate::span::SpanKind; use crate::{ - Attribute, AttributeValue, RtcStatsSample, Span, SpanKind, SpanName, SpanOutcome, Telemetry, + Attribute, AttributeValue, RtcStatsSample, Span, SpanName, SpanOutcome, Telemetry, TelemetryEvent, }; @@ -153,15 +154,18 @@ impl Scope { } } - pub fn begin_span(&self, name: &str, kind: SpanKind, parent: Option) -> u64 { + #[cfg(test)] + pub(crate) fn begin_span(&self, name: &str, kind: SpanKind, parent: Option) -> u64 { self.telemetry.begin_span_in(name, kind, parent, &self.state) } - pub fn add_span_event(&self, span: u64, name: &str, attributes: Vec) { + #[cfg(test)] + pub(crate) fn add_span_event(&self, span: u64, name: &str, attributes: Vec) { self.telemetry.add_span_event(span, name, attributes); } - pub fn end_span( + #[cfg(test)] + pub(crate) fn end_span( &self, span: u64, outcome: SpanOutcome, diff --git a/livekit-telemetry/src/span.rs b/livekit-telemetry/src/span.rs index 9e0244166..6afc5a024 100644 --- a/livekit-telemetry/src/span.rs +++ b/livekit-telemetry/src/span.rs @@ -17,8 +17,7 @@ use std::sync::Arc; use crate::{event::now_unix_nanos, scope::ScopeState, Attribute, AttributeValue}; -/// OTel span kind, restricted to what client operations need. -#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +/// OTel span kind, restricted to what client operations need; implied by [`crate::SpanName`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum SpanKind { /// An operation inside the SDK (publish, subscribe). diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index c34ae7063..14d258e77 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -21,6 +21,7 @@ use std::{ use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; +use crate::span::SpanKind; use crate::{ event::now_unix_nanos, exporter::Command, @@ -30,7 +31,7 @@ use crate::{ stats::{Counters, TelemetryStatus}, store::{Queued, Store}, Attribute, AttributeValue, BatchCache, DeviceState, Exporter, FileCache, LogRecord, LogSource, - MemoryCache, RtcStatsSample, Severity, SpanKind, SpanOutcome, TelemetryEvent, TelemetryStats, + MemoryCache, RtcStatsSample, Severity, SpanOutcome, TelemetryEvent, TelemetryStats, TelemetryTransport, }; use crate::{DeviceEvent, Span, SpanName}; @@ -436,7 +437,8 @@ impl Telemetry { /// Open a span: one attempt at an operation (`lk.connect`, `lk.publish`, …). Returns the /// handle to record checkpoints and to end it with; `parent` nests it under another open span. - pub fn begin_span(&self, name: &str, kind: SpanKind, parent: Option) -> u64 { + #[cfg(test)] + pub(crate) fn begin_span(&self, name: &str, kind: SpanKind, parent: Option) -> u64 { self.begin_span_in(name, kind, parent, &self.process) } @@ -456,13 +458,13 @@ impl Telemetry { } /// Record a checkpoint inside an open span (`ws_open`, `join_recv`, …), stamped now. - pub fn add_span_event(&self, span: u64, name: &str, attributes: Vec) { + pub(crate) fn add_span_event(&self, span: u64, name: &str, attributes: Vec) { self.spans.lock().unwrap_or_else(|e| e.into_inner()).add_event(span, name, attributes); } /// End a span with its outcome; `error_type` becomes `error.type` and the status message. /// The span is exported with the next batch. Ending twice, or an unknown handle, is a no-op. - pub fn end_span( + pub(crate) fn end_span( &self, span: u64, outcome: SpanOutcome, @@ -630,6 +632,7 @@ pub(crate) fn observability_endpoint(url: &str) -> Option { #[cfg(test)] mod tests { + use crate::span::SpanKind; use crate::{RoomIdentity, SpanName, SpanStep}; use std::{collections::VecDeque, fs, path::Path, sync::Mutex}; @@ -764,7 +767,7 @@ mod tests { logs::v1::LogRecord, trace::v1::{span, status}, }, - AppState, ExportError, ExportRequest, SpanKind, SpanOutcome, StreamDirection, ThermalState, + AppState, ExportError, ExportRequest, SpanOutcome, StreamDirection, ThermalState, TrackKind, }; diff --git a/livekit-telemetry/src/trace.rs b/livekit-telemetry/src/trace.rs index fca6012ad..9d679e4a2 100644 --- a/livekit-telemetry/src/trace.rs +++ b/livekit-telemetry/src/trace.rs @@ -10,9 +10,8 @@ use std::{ use tokio::time::Instant; -use crate::{ - scope::ScopeState, Attribute, AttributeValue, SpanKind, SpanOutcome, Telemetry, TrackKind, -}; +use crate::span::SpanKind; +use crate::{scope::ScopeState, Attribute, AttributeValue, SpanOutcome, Telemetry, TrackKind}; /// What an SDK operation is. The kind follows from the name: connects talk to the server /// (`client`), the rest is internal work. diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index ef09e17bc..c026b2295 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -89,11 +89,6 @@ impl Telemetry { self.0.set_server(&url, &token); } - /// A span in the process trace (no room), stamped now; `parent` nests it. - pub fn start(&self, name: SpanName, parent: Option>) -> Arc { - Arc::new(TelemetrySpan(self.0.start(name, parent.map(|p| p.0.clone())))) - } - /// Where to send, once known (first connect: server URL → endpoint, token → headers). Until /// then everything waits in the cache; afterwards it uploads. pub fn set_destination(&self, endpoint: String, headers: HashMap) { @@ -216,16 +211,6 @@ pub struct TelemetrySpan(Arc); #[uniffi::export] impl TelemetrySpan { - /// Timings and a description only; nothing is exported. - #[uniffi::constructor] - pub fn detached(name: SpanName) -> Arc { - Arc::new(Self(livekit_telemetry::Span::detached(name))) - } - - pub fn label(&self) -> String { - self.0.label() - } - /// A checkpoint, stamped now. pub fn step(&self, step: SpanStep) { self.0.step(step); @@ -257,10 +242,6 @@ impl TelemetrySpan { self.0.is_ended() } - pub fn outcome(&self) -> Option { - self.0.outcome() - } - /// `None` for a detached span. pub fn context(&self) -> Option { self.0.context() @@ -277,12 +258,6 @@ impl TelemetrySpan { } } -/// A span's wire name (`lk.connect`, …, or the custom name), for the platform's own logging. -#[uniffi::export] -pub fn span_label(name: SpanName) -> String { - name.label().to_owned() -} - #[derive(uniffi::Object)] pub struct TelemetryExportQueue { tx: mpsc::UnboundedSender, From 1fafe43a178b5dac8fcab24440959ad4d5d6a4e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:57:17 +0200 Subject: [PATCH 36/50] chore(telemetry): drop the string span API; tests use the typed one Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/scope.rs | 27 +-------------------------- livekit-telemetry/src/telemetry.rs | 15 +++++++++------ 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/livekit-telemetry/src/scope.rs b/livekit-telemetry/src/scope.rs index 9d92232e6..4e4463fda 100644 --- a/livekit-telemetry/src/scope.rs +++ b/livekit-telemetry/src/scope.rs @@ -17,11 +17,7 @@ use std::{ sync::{Arc, Mutex}, }; -use crate::span::SpanKind; -use crate::{ - Attribute, AttributeValue, RtcStatsSample, Span, SpanName, SpanOutcome, Telemetry, - TelemetryEvent, -}; +use crate::{Attribute, AttributeValue, RtcStatsSample, Span, SpanName, Telemetry, TelemetryEvent}; /// One session's identity: the trace id every one of its records carries, and the attributes /// attached to them at export time (`lk.room.sid`, `lk.participant.identity`, …). @@ -153,25 +149,4 @@ impl Scope { self.set_attribute(key, value.map(AttributeValue::Str)); } } - - #[cfg(test)] - pub(crate) fn begin_span(&self, name: &str, kind: SpanKind, parent: Option) -> u64 { - self.telemetry.begin_span_in(name, kind, parent, &self.state) - } - - #[cfg(test)] - pub(crate) fn add_span_event(&self, span: u64, name: &str, attributes: Vec) { - self.telemetry.add_span_event(span, name, attributes); - } - - #[cfg(test)] - pub(crate) fn end_span( - &self, - span: u64, - outcome: SpanOutcome, - error_type: Option, - attributes: Vec, - ) { - self.telemetry.end_span(span, outcome, error_type, attributes); - } } diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 14d258e77..cede462cc 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -1252,16 +1252,17 @@ mod tests { assert_ne!(a.trace_id(), telemetry.trace_id(), "the process has its own session"); a.set_attribute("lk.room.sid", Some("RM_a".into())); b.set_attribute("lk.room.sid", Some("RM_b".into())); - let span = a.begin_span("lk.connect", SpanKind::Client, None); + let span = a.start(SpanName::Connect, None); + let span_id = span.context().expect("bound").span_id; a.emit(TelemetryEvent::new("lk.ping")); b.emit(TelemetryEvent::new("lk.ping")); // A warn record from the SDK logger, inside room A's connect: no session handle, just // the ambient span id — the core files it under A. telemetry.emit( - TelemetryEvent::new("").with_severity(Severity::Warn).with_body("hmm").in_span(span), + TelemetryEvent::new("").with_severity(Severity::Warn).with_body("hmm").in_span(span_id), ); telemetry.emit(TelemetryEvent::new("lk.device.thermal.changed")); - a.end_span(span, SpanOutcome::Ok, None, Vec::new()); + span.end(SpanOutcome::Ok, None); telemetry.flush().await; let sent = transport.sent(); @@ -1288,12 +1289,15 @@ mod tests { // A record that names a span which has already ended — and been exported — is still that // session's: the SDK's log path hops threads, the span does not wait for it. telemetry.emit( - TelemetryEvent::new("").with_severity(Severity::Error).with_body("late").in_span(span), + TelemetryEvent::new("") + .with_severity(Severity::Error) + .with_body("late") + .in_span(span_id), ); telemetry.flush().await; let late = &records(&transport.sent()[2])[0]; assert_eq!(hex(&late.trace_id), a.trace_id(), "filed under the ended span's session"); - assert_eq!(late.span_id, span.to_be_bytes().to_vec()); + assert_eq!(late.span_id, span_id.to_be_bytes().to_vec()); } #[tokio::test(start_paused = true)] @@ -1316,7 +1320,6 @@ mod tests { assert_eq!(sent.len(), 1, "cached batches ship as soon as the destination is known"); assert_eq!(sent[0].url, "https://x.livekit.cloud/observability/logs/otlp/v0"); assert_eq!(sent[0].headers["Authorization"], "Bearer t"); - telemetry.begin_scope().begin_span("lk.publish", SpanKind::Internal, None); let span = telemetry.begin_span("lk.publish", SpanKind::Internal, None); telemetry.end_span(span, SpanOutcome::Ok, None, Vec::new()); telemetry.flush().await; From e273041c616dd12d4cc2a0e4f3d4455c44c77d27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:33:55 +0200 Subject: [PATCH 37/50] feat(telemetry): the process pipeline lives in the core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit livekit_telemetry::global holds the one pipeline per process, the way log and tracing do: install it once (or again), reach it anywhere, and every call is a no-op while none is installed. Over the FFI the Telemetry object is gone; free functions (telemetry_configure, _scope, _log, _device_event, _set_server, …) are all a platform needs. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/global.rs | 95 +++++++++ livekit-telemetry/src/lib.rs | 2 + livekit-telemetry/src/telemetry.rs | 27 +++ livekit-uniffi/src/telemetry.rs | 182 ++++++++---------- .../support/dart/test/telemetry_test.dart | 21 +- 5 files changed, 217 insertions(+), 110 deletions(-) create mode 100644 livekit-telemetry/src/global.rs diff --git a/livekit-telemetry/src/global.rs b/livekit-telemetry/src/global.rs new file mode 100644 index 000000000..6ade85d7b --- /dev/null +++ b/livekit-telemetry/src/global.rs @@ -0,0 +1,95 @@ +//! One pipeline per process, the way `log` and `tracing` do it: install it once (or again), +//! reach it anywhere without a handle, and every call is a no-op while none is installed. The +//! core does not assume an async runtime, so whoever builds the pipeline spawns its exporter and +//! shuts down the one `install` hands back. + +use std::sync::RwLock; + +use crate::{ + AttributeValue, DeviceEvent, DeviceState, LogRecord, Scope, Telemetry, TelemetryEvent, + TelemetryStats, +}; + +static SHARED: RwLock> = RwLock::new(None); + +fn current() -> Option { + SHARED.read().unwrap_or_else(|e| e.into_inner()).clone() +} + +/// Make `telemetry` the process pipeline. Returns the one it replaces, still holding data: the +/// caller shuts it down. +pub fn install(telemetry: Telemetry) -> Option { + SHARED.write().unwrap_or_else(|e| e.into_inner()).replace(telemetry) +} + +/// Remove the process pipeline; the caller shuts it down. +pub fn uninstall() -> Option { + SHARED.write().unwrap_or_else(|e| e.into_inner()).take() +} + +pub fn shared() -> Option { + current() +} + +/// A new scope (one room, one call) on the process pipeline; `None` while telemetry is off. +pub fn scope() -> Option { + current().map(|t| t.begin_scope()) +} + +pub fn emit(event: TelemetryEvent) { + if let Some(t) = current() { + t.emit(event); + } +} + +pub fn log(record: LogRecord) { + if let Some(t) = current() { + t.log(record); + } +} + +pub fn device_event(event: DeviceEvent) { + if let Some(t) = current() { + t.device_event(event); + } +} + +pub fn set_device_state(state: DeviceState) { + if let Some(t) = current() { + t.set_device_state(state); + } +} + +pub fn set_server(url: &str, token: &str) { + if let Some(t) = current() { + t.set_server(url, token); + } +} + +pub fn set_attribute(key: &str, value: Option) { + if let Some(t) = current() { + t.set_attribute(key, value); + } +} + +pub fn stats() -> Option { + current().map(|t| t.stats()) +} + +/// The one-line health readout, or `off`. +pub fn diagnostics() -> String { + stats().map_or_else(|| "off".to_owned(), |s| s.to_string()) +} + +pub async fn flush() { + if let Some(t) = current() { + t.flush().await; + } +} + +/// Uninstall and drain: the last flush, then the pipeline stops. +pub async fn shutdown() { + if let Some(t) = uninstall() { + t.shutdown().await; + } +} diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs index b47a84175..a3134c183 100644 --- a/livekit-telemetry/src/lib.rs +++ b/livekit-telemetry/src/lib.rs @@ -48,6 +48,8 @@ mod proto; /// Transport seam: how encoded batches leave the device. mod transport; +/// The process pipeline: [`global::install`] once, then reach it anywhere. +pub mod global; /// Entry point and configuration. mod telemetry; mod trace; diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index cede462cc..66268784b 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -656,6 +656,33 @@ mod tests { assert_eq!(ep("wss:///rtc"), None); } + #[tokio::test(start_paused = true)] + async fn the_process_pipeline_no_ops_until_installed() { + let line = || crate::LogRecord { + severity: Severity::Warn, + source: LogSource::Sdk, + message: "hmm".into(), + logger: None, + function: None, + file: None, + line: None, + timestamp_ns: None, + span_id: None, + }; + crate::global::log(line()); + assert!(crate::global::scope().is_none()); + assert_eq!(crate::global::diagnostics(), "off"); + let transport = FakeTransport::scripted([]); + assert!(crate::global::install(pipeline(transport.clone())).is_none()); + crate::global::log(line()); + crate::global::flush().await; + assert_eq!(transport.sent().len(), 1, "installed: the line reaches the collector"); + assert!(crate::global::diagnostics().starts_with("ok, sent 1")); + crate::global::shutdown().await; + assert!(crate::global::shared().is_none()); + crate::global::log(line()); + } + #[tokio::test(start_paused = true)] async fn typed_spans_hold_uploads_while_connecting_and_export_when_ended() { let transport = FakeTransport::scripted([]); diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index c026b2295..afc7f5d2b 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -17,9 +17,9 @@ use std::{ }; use livekit_telemetry::{ - Attribute, AttributeValue, DeviceEvent, DeviceState, ExportError, ExportRequest, LogRecord, - NetTransport, RoomIdentity, RtcStatsSample, SpanName, SpanOutcome, SpanStep, SpanTrack, - TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, TraceContext, + global, Attribute, AttributeValue, DeviceEvent, DeviceState, ExportError, ExportRequest, + LogRecord, NetTransport, RoomIdentity, RtcStatsSample, SpanName, SpanOutcome, SpanStep, + SpanTrack, TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, TraceContext, }; use tokio::sync::{mpsc, oneshot}; @@ -33,118 +33,102 @@ pub enum TelemetryError { } /// Telemetry pipeline: buffer, batch, cache and export events as OTLP. -#[derive(uniffi::Object)] -pub struct Telemetry(livekit_telemetry::Telemetry); - -#[uniffi::export(async_runtime = "tokio")] -impl Telemetry { - /// `transport = None` uses the HTTP client registered with `livekit-net`, if any. - #[uniffi::constructor] - pub fn new( - config: TelemetryConfig, - transport: Option>, - ) -> Result, TelemetryError> { - let transport: Arc = match transport { - Some(transport) => transport, - None => Arc::new(NetTransport::from_registry().ok_or(TelemetryError::NoTransport)?), - }; - let (telemetry, exporter) = livekit_telemetry::Telemetry::new(config, transport); - crate::runtime::runtime().spawn(exporter.run()); - Ok(Arc::new(Self(telemetry))) - } - - /// Like [`Telemetry::new`], but exports through a [`TelemetryExportQueue`] the host drains - /// from its own thread. For bindings whose callbacks cannot be invoked from Rust threads - /// (uniffi-dart today). - #[uniffi::constructor] - pub fn new_pulled(config: TelemetryConfig, queue: Arc) -> Arc { - let (telemetry, exporter) = livekit_telemetry::Telemetry::new(config, queue); - crate::runtime::runtime().spawn(exporter.run()); - Arc::new(Self(telemetry)) - } - - /// Queue an event for export. Never blocks; drops the oldest event when the queue is full. - pub fn emit(&self, event: TelemetryEvent) { - self.0.emit(event); - } - - /// A consumer-defined event, exported as `custom.`; attributes keep their own namespace. - pub fn emit_custom(&self, name: String, attributes: Vec) { - self.0.emit_custom(&name, attributes); - } +/// Start the process pipeline; a previous one is drained and replaced. `transport = None` uses +/// the HTTP client registered with `livekit-net`, if any. +#[uniffi::export] +pub fn telemetry_configure( + config: TelemetryConfig, + transport: Option>, +) -> Result<(), TelemetryError> { + let transport: Arc = match transport { + Some(transport) => transport, + None => Arc::new(NetTransport::from_registry().ok_or(TelemetryError::NoTransport)?), + }; + install(livekit_telemetry::Telemetry::new(config, transport)); + Ok(()) +} - /// A captured log line; the core applies the per-source floor and builds the record. - pub fn log(&self, record: LogRecord) { - self.0.log(record); - } +/// Like [`telemetry_configure`], exporting through a queue the host drains from its own thread. +/// For bindings whose callbacks cannot be invoked from Rust threads (uniffi-dart today). +#[uniffi::export] +pub fn telemetry_configure_pulled(config: TelemetryConfig) -> Arc { + let queue = TelemetryExportQueue::new(); + install(livekit_telemetry::Telemetry::new(config, queue.clone())); + queue +} - /// Audio route, interruption, denied permission: a process-level record built by the core. - pub fn device_event(&self, event: DeviceEvent) { - self.0.device_event(event); +fn install((telemetry, exporter): (livekit_telemetry::Telemetry, livekit_telemetry::Exporter)) { + crate::runtime::runtime().spawn(exporter.run()); + if let Some(previous) = global::install(telemetry) { + crate::runtime::runtime().spawn(async move { previous.shutdown().await }); } +} - /// Cloud rule: server URL → observability endpoint, room token → bearer header. No-op when - /// the config names an explicit endpoint. - pub fn set_server(&self, url: String, token: String) { - self.0.set_server(&url, &token); - } +/// Drain and stop the process pipeline; every call is a no-op afterwards. +#[uniffi::export(async_runtime = "tokio")] +pub async fn telemetry_shutdown() { + global::shutdown().await; +} - /// Where to send, once known (first connect: server URL → endpoint, token → headers). Until - /// then everything waits in the cache; afterwards it uploads. - pub fn set_destination(&self, endpoint: String, headers: HashMap) { - self.0.set_destination(&endpoint, headers); - } +/// Cache everything queued and upload what the network allows. +#[uniffi::export(async_runtime = "tokio")] +pub async fn telemetry_flush() { + global::flush().await; +} - /// A session — one room, one call — with its own trace id and attributes on this pipeline. - pub fn begin_scope(&self) -> Arc { - Arc::new(TelemetryScope(self.0.begin_scope())) - } +/// A scope — one room, one call — on the process pipeline; `None` while telemetry is off. +#[uniffi::export] +pub fn telemetry_scope() -> Option> { + global::scope().map(|scope| Arc::new(TelemetryScope(scope))) +} - /// Report the device state (thermal, low power, foreground/background). Emits the matching - /// `lk.device.*.changed` events and adapts the export cadence. - pub fn set_device_state(&self, state: DeviceState) { - self.0.set_device_state(state); - } +/// A process-level event (outside any room). +#[uniffi::export] +pub fn telemetry_emit(event: TelemetryEvent) { + global::emit(event); +} - /// A pipeline-wide attribute (`enduser.id`, `acme.tenant`), attached to every record of every - /// session unless the record or its session already carries the key; `None` removes it. - pub fn set_attribute(&self, key: String, value: Option) { - self.0.set_attribute(&key, value); - } +/// A captured log line; the core applies the per-source floor and builds the record. +#[uniffi::export] +pub fn telemetry_log(record: LogRecord) { + global::log(record); +} - /// The session's trace id (32 hex chars) — on every span and record of this pipeline. - pub fn trace_id(&self) -> String { - self.0.trace_id() - } +/// Audio route, interruption, denied permission: a process-level record built by the core. +#[uniffi::export] +pub fn telemetry_device_event(event: DeviceEvent) { + global::device_event(event); +} - /// Push one `getStats()` reading for a track; windowed on device into `lk.rtc.stats.sample`. - pub fn record_stats(&self, sample: RtcStatsSample) { - self.0.record_stats(sample); - } +/// The device's current state; drives the upload cadence and yields change events. +#[uniffi::export] +pub fn telemetry_set_device_state(state: DeviceState) { + global::set_device_state(state); +} - /// Export everything queued and wait for the transport. - pub async fn flush(&self) { - self.0.flush().await; - } +/// Cloud rule: server URL → observability endpoint, room token → bearer header. +#[uniffi::export] +pub fn telemetry_set_server(url: String, token: String) { + global::set_server(&url, &token); +} - /// Flush, then stop exporting. Bounded by `export_timeout_ms`. - pub async fn shutdown(&self) { - self.0.shutdown().await; - } +/// An attribute on every record of every scope; `None` removes it. +#[uniffi::export] +pub fn telemetry_set_attribute(key: String, value: Option) { + global::set_attribute(&key, value); +} - /// Pipeline health: drops by reason, uploads, cached batches. - pub fn stats(&self) -> TelemetryStats { - self.0.stats() - } +#[uniffi::export] +pub fn telemetry_stats() -> Option { + global::stats() +} - /// The stats as one line for a debug console. - pub fn diagnostics(&self) -> String { - self.0.stats().to_string() - } +/// The stats as one line for a debug console, or `off`. +#[uniffi::export] +pub fn telemetry_diagnostics() -> String { + global::diagnostics() } -/// One room's session on the process pipeline: what its spans, stats and events are filed -/// under. Obtained from [`Telemetry::begin_scope`]. #[derive(uniffi::Object)] pub struct TelemetryScope(livekit_telemetry::Scope); diff --git a/livekit-uniffi/support/dart/test/telemetry_test.dart b/livekit-uniffi/support/dart/test/telemetry_test.dart index 7472867b3..dbc641f8a 100644 --- a/livekit-uniffi/support/dart/test/telemetry_test.dart +++ b/livekit-uniffi/support/dart/test/telemetry_test.dart @@ -19,23 +19,21 @@ Future serve(TelemetryExportQueue queue, List sink, int cou void main() { group('telemetry', () { test('exports through the pull queue from the Dart side', () async { - final queue = TelemetryExportQueue(); final requests = []; - final serving = serve(queue, requests, 2); - final telemetry = Telemetry.newPulled( + final queue = telemetryConfigurePulled( config: TelemetryConfig( endpoint: 'http://collector/v1/logs', headers: {'Authorization': 'Bearer test'}, resource: [], logSeverity: Severity.warn, ), - queue: queue, ); + final serving = serve(queue, requests, 2); - telemetry.emit( + telemetryEmit( event: TelemetryEvent(name: 'lk.ping', severity: Severity.info, attributes: []), ); - telemetry.recordStats( + telemetryScope()!.recordStats( sample: RtcStatsSample( trackSid: 'TR_1', kind: TrackKind.audio, @@ -43,24 +41,25 @@ void main() { bytes: 42, ), ); - await telemetry.flush(); + await telemetryFlush(); expect(requests, hasLength(1)); expect(requests.single.url, 'http://collector/v1/logs'); expect(requests.single.headers['Content-Type'], 'application/x-protobuf'); expect(requests.single.headers['Authorization'], 'Bearer test'); expect(requests.single.body, isNotEmpty); - expect(telemetry.stats().uploadsSent, 1); + expect(telemetryStats()!.uploadsSent, 1); + expect(telemetryStats()!.dropped, 0); // Shutdown closes the open stats window, which ships as a second batch. - await telemetry.shutdown(); + await telemetryShutdown(); await serving; expect(requests, hasLength(2)); - expect(telemetry.stats().dropped, 0); + expect(telemetryStats(), isNull); }); test('refuses to start without any transport', () { expect( - () => Telemetry( + () => telemetryConfigure( config: TelemetryConfig(endpoint: 'http://collector/v1/logs', headers: {}, resource: [], logSeverity: Severity.warn), transport: null, ), From 4ed1fa66b200270fafe36ebee5a0d2ed5d3e6f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:47:11 +0200 Subject: [PATCH 38/50] feat(telemetry): instruments live with the pipeline TelemetryInstrument is a core trait (a UniFFI foreign trait for the platforms): install starts the instruments handed in with the pipeline, uninstall stops them, so a platform holds no telemetry state at all. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/global.rs | 54 +++++++++++++++---- livekit-telemetry/src/telemetry.rs | 33 +++++++++++- livekit-uniffi/src/telemetry.rs | 24 ++++++--- .../support/dart/test/telemetry_test.dart | 2 + 4 files changed, 94 insertions(+), 19 deletions(-) diff --git a/livekit-telemetry/src/global.rs b/livekit-telemetry/src/global.rs index 6ade85d7b..7d46f3088 100644 --- a/livekit-telemetry/src/global.rs +++ b/livekit-telemetry/src/global.rs @@ -3,28 +3,64 @@ //! core does not assume an async runtime, so whoever builds the pipeline spawns its exporter and //! shuts down the one `install` hands back. -use std::sync::RwLock; +use std::sync::{Arc, RwLock}; use crate::{ AttributeValue, DeviceEvent, DeviceState, LogRecord, Scope, Telemetry, TelemetryEvent, TelemetryStats, }; -static SHARED: RwLock> = RwLock::new(None); +/// A platform instrument — device signals, log capture — that feeds the pipeline while it runs. +/// Started right after the pipeline is installed and stopped when it goes; calls may come from +/// any thread and must not block. +#[cfg_attr(feature = "uniffi", uniffi::export(with_foreign))] +pub trait TelemetryInstrument: Send + Sync { + fn start(&self); + fn stop(&self); +} + +struct Installed { + telemetry: Telemetry, + instruments: Vec>, +} + +static SHARED: RwLock> = RwLock::new(None); fn current() -> Option { - SHARED.read().unwrap_or_else(|e| e.into_inner()).clone() + SHARED.read().unwrap_or_else(|e| e.into_inner()).as_ref().map(|i| i.telemetry.clone()) } -/// Make `telemetry` the process pipeline. Returns the one it replaces, still holding data: the -/// caller shuts it down. -pub fn install(telemetry: Telemetry) -> Option { - SHARED.write().unwrap_or_else(|e| e.into_inner()).replace(telemetry) +/// Make `telemetry` the process pipeline and start `instruments` on it. Returns the pipeline it +/// replaces (its instruments already stopped), still holding data: the caller shuts it down. +pub fn install( + telemetry: Telemetry, + instruments: Vec>, +) -> Option { + // The lock is released before any instrument runs: `start` may call straight back in. + let previous = SHARED + .write() + .unwrap_or_else(|e| e.into_inner()) + .replace(Installed { telemetry, instruments: instruments.clone() }); + if let Some(previous) = &previous { + for instrument in &previous.instruments { + instrument.stop(); + } + } + for instrument in &instruments { + instrument.start(); + } + previous.map(|p| p.telemetry) } -/// Remove the process pipeline; the caller shuts it down. +/// Stop the instruments and remove the process pipeline; the caller shuts it down. pub fn uninstall() -> Option { - SHARED.write().unwrap_or_else(|e| e.into_inner()).take() + let taken = SHARED.write().unwrap_or_else(|e| e.into_inner()).take(); + if let Some(taken) = &taken { + for instrument in &taken.instruments { + instrument.stop(); + } + } + taken.map(|i| i.telemetry) } pub fn shared() -> Option { diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 66268784b..cd58ff33a 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -672,13 +672,42 @@ mod tests { crate::global::log(line()); assert!(crate::global::scope().is_none()); assert_eq!(crate::global::diagnostics(), "off"); + struct Counting(std::sync::atomic::AtomicU32, std::sync::atomic::AtomicU32); + impl crate::global::TelemetryInstrument for Counting { + fn start(&self) { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + crate::global::log(crate::LogRecord { + severity: Severity::Warn, + source: LogSource::Sdk, + message: "from start".into(), + logger: None, + function: None, + file: None, + line: None, + timestamp_ns: None, + span_id: None, + }); + } + fn stop(&self) { + self.1.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + } + let instrument = Arc::new(Counting(Default::default(), Default::default())); let transport = FakeTransport::scripted([]); - assert!(crate::global::install(pipeline(transport.clone())).is_none()); + assert!( + crate::global::install(pipeline(transport.clone()), vec![instrument.clone()]).is_none() + ); + assert_eq!(instrument.0.load(std::sync::atomic::Ordering::SeqCst), 1, "started on install"); crate::global::log(line()); crate::global::flush().await; - assert_eq!(transport.sent().len(), 1, "installed: the line reaches the collector"); + assert_eq!(records(&transport.sent()[0]).len(), 2, "the instrument's own line and ours"); assert!(crate::global::diagnostics().starts_with("ok, sent 1")); crate::global::shutdown().await; + assert_eq!( + instrument.1.load(std::sync::atomic::Ordering::SeqCst), + 1, + "stopped on shutdown" + ); assert!(crate::global::shared().is_none()); crate::global::log(line()); } diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index afc7f5d2b..15f54ec5a 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -17,9 +17,10 @@ use std::{ }; use livekit_telemetry::{ - global, Attribute, AttributeValue, DeviceEvent, DeviceState, ExportError, ExportRequest, - LogRecord, NetTransport, RoomIdentity, RtcStatsSample, SpanName, SpanOutcome, SpanStep, - SpanTrack, TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, TraceContext, + global::{self, TelemetryInstrument}, + Attribute, AttributeValue, DeviceEvent, DeviceState, ExportError, ExportRequest, LogRecord, + NetTransport, RoomIdentity, RtcStatsSample, SpanName, SpanOutcome, SpanStep, SpanTrack, + TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, TraceContext, }; use tokio::sync::{mpsc, oneshot}; @@ -39,27 +40,34 @@ pub enum TelemetryError { pub fn telemetry_configure( config: TelemetryConfig, transport: Option>, + instruments: Vec>, ) -> Result<(), TelemetryError> { let transport: Arc = match transport { Some(transport) => transport, None => Arc::new(NetTransport::from_registry().ok_or(TelemetryError::NoTransport)?), }; - install(livekit_telemetry::Telemetry::new(config, transport)); + install(livekit_telemetry::Telemetry::new(config, transport), instruments); Ok(()) } /// Like [`telemetry_configure`], exporting through a queue the host drains from its own thread. /// For bindings whose callbacks cannot be invoked from Rust threads (uniffi-dart today). #[uniffi::export] -pub fn telemetry_configure_pulled(config: TelemetryConfig) -> Arc { +pub fn telemetry_configure_pulled( + config: TelemetryConfig, + instruments: Vec>, +) -> Arc { let queue = TelemetryExportQueue::new(); - install(livekit_telemetry::Telemetry::new(config, queue.clone())); + install(livekit_telemetry::Telemetry::new(config, queue.clone()), instruments); queue } -fn install((telemetry, exporter): (livekit_telemetry::Telemetry, livekit_telemetry::Exporter)) { +fn install( + (telemetry, exporter): (livekit_telemetry::Telemetry, livekit_telemetry::Exporter), + instruments: Vec>, +) { crate::runtime::runtime().spawn(exporter.run()); - if let Some(previous) = global::install(telemetry) { + if let Some(previous) = global::install(telemetry, instruments) { crate::runtime::runtime().spawn(async move { previous.shutdown().await }); } } diff --git a/livekit-uniffi/support/dart/test/telemetry_test.dart b/livekit-uniffi/support/dart/test/telemetry_test.dart index dbc641f8a..9f39163f2 100644 --- a/livekit-uniffi/support/dart/test/telemetry_test.dart +++ b/livekit-uniffi/support/dart/test/telemetry_test.dart @@ -27,6 +27,7 @@ void main() { resource: [], logSeverity: Severity.warn, ), + instruments: [], ); final serving = serve(queue, requests, 2); @@ -62,6 +63,7 @@ void main() { () => telemetryConfigure( config: TelemetryConfig(endpoint: 'http://collector/v1/logs', headers: {}, resource: [], logSeverity: Severity.warn), transport: null, + instruments: [], ), throwsA(anything), ); From ae765cf9c5e915e42f652676d0c2c7ec0849e665 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Mon, 7 Sep 2026 20:56:12 +0200 Subject: [PATCH 39/50] feat(telemetry): TelemetryConfig is the options type on every platform headers and log_severity are optional with core defaults (UniFFI cannot default a map or an enum literal), resource defaults to empty, and disabled_instruments names the platform instruments not to run. A platform passes the record straight through, adding only its resource and cache directory. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/src/telemetry.rs | 47 ++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index cd58ff33a..d4bdb14ff 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -55,9 +55,11 @@ pub struct TelemetryConfig { #[cfg_attr(feature = "uniffi", uniffi(default))] pub traces_endpoint: Option, /// Extra request headers, e.g. `Authorization: Bearer `. - pub headers: HashMap, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub headers: Option>, /// Resource attributes describing the emitter (`service.name`, `os.name`, /// `device.model.identifier`, `session.id`, …). `telemetry.sdk.*` are filled in by the core. + #[cfg_attr(feature = "uniffi", uniffi(default = []))] pub resource: Vec, /// Who is reporting, typed; the core owns the semconv keys. Extra attributes go in `resource`. #[cfg_attr(feature = "uniffi", uniffi(default))] @@ -103,8 +105,26 @@ pub struct TelemetryConfig { pub max_batch_bytes: u64, /// Lowest severity a plain log record (an event with no name) needs to leave the device. /// Events are not subject to it. Design doc: warn. - // No uniffi default: enum defaults are not supported by the Swift generator (uniffi 0.31). - pub log_severity: Severity, + /// `None` is `Warn`. (Optional because UniFFI 0.31 cannot default an enum literal.) + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub log_severity: Option, + /// Platform instruments not to run; all run by default. + #[cfg_attr(feature = "uniffi", uniffi(default = []))] + pub disabled_instruments: Vec, +} + +/// The platform instruments a config can switch off. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Instrument { + /// Thermal, power, memory, network, battery, audio session. + Device, + /// Warn/error lines from the SDK, the core and WebRTC. + Logs, + /// `getStats` windows per track and subscribe spans. + Rtc, + /// Connect, reconnect and publish spans. + Room, } impl TelemetryConfig { @@ -113,7 +133,7 @@ impl TelemetryConfig { Self { endpoint: Some(endpoint.into()), traces_endpoint: None, - headers: HashMap::new(), + headers: None, resource: Vec::new(), sdk: None, storage_dir: None, @@ -127,7 +147,8 @@ impl TelemetryConfig { max_batches_per_upload: 4, flush_threshold_bytes: 256 * 1024, max_batch_bytes: 1024 * 1024, - log_severity: Severity::Warn, + log_severity: None, + disabled_instruments: Vec::new(), } } } @@ -262,7 +283,11 @@ impl Telemetry { ) -> (Self, Exporter) { add_sdk_resource(&mut config.resource, config.sdk.as_ref()); let destination = Arc::new(Mutex::new(config.endpoint.as_deref().map(|endpoint| { - Destination::new(endpoint, config.traces_endpoint.clone(), config.headers.clone()) + Destination::new( + endpoint, + config.traces_endpoint.clone(), + config.headers.clone().unwrap_or_default(), + ) }))); let config = Arc::new(config); // One pipeline per process; sessions (rooms) carry their own trace ids. This is the @@ -370,8 +395,8 @@ impl Telemetry { /// produced a batch would never end). pub fn log(&self, record: LogRecord) { let floor = match record.source { - LogSource::WebRtc => self.config.log_severity.max(Severity::Error), - _ => self.config.log_severity, + LogSource::WebRtc => self.log_severity().max(Severity::Error), + _ => self.log_severity(), }; if record.severity < floor { return; @@ -394,8 +419,12 @@ impl Telemetry { self.emit_in(event, &session); } + fn log_severity(&self) -> Severity { + self.config.log_severity.unwrap_or(Severity::Warn) + } + pub(crate) fn emit_in(&self, mut event: TelemetryEvent, session: &Arc) { - if event.name.is_empty() && event.severity < self.config.log_severity { + if event.name.is_empty() && event.severity < self.log_severity() { return; } if !self.guard.lock().unwrap_or_else(|e| e.into_inner()).admit() { From e07b35574336ff7b0293cb6a545c4905546fac14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:30:16 +0200 Subject: [PATCH 40/50] feat(telemetry): transports hand back the raw response; the core reads the collector's answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `TelemetryTransport::send` now returns `ExportResponse { status, headers, body }` and fails only without a response. `ExportError::from_response` classifies once, for every platform: 2xx ok; 401/403 saying "disabled" → silent for good; 429/502/503/504, or any error carrying `google.rpc.RetryInfo`, → retryable, waiting `Retry-After` (header), else `retry_delay` (body), else the exporter's default; anything else dropped. The `google.rpc.Status` body is decoded with two local prost messages (prost-types for Any/Duration). The pull queue (Dart) gains `fail(id, error)` next to `complete(id, response)`. `ExportError`'s `message` field is renamed `reason`: a UniFFI error field named `message` collides with `Throwable.message` in the generated Kotlin, which is how the Android AAR build caught it. Refs CLT-3334. Co-Authored-By: Claude Fable 5.1 --- Cargo.lock | 1 + livekit-telemetry/Cargo.toml | 2 + livekit-telemetry/README.md | 4 +- livekit-telemetry/src/exporter.rs | 22 +- livekit-telemetry/src/telemetry.rs | 21 +- livekit-telemetry/src/transport.rs | 238 +++++++++++++++--- livekit-uniffi/src/telemetry.rs | 34 ++- .../support/dart/test/telemetry_test.dart | 7 +- 8 files changed, 262 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e81cd846..8b9e330b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3788,6 +3788,7 @@ dependencies = [ "log", "opentelemetry-proto", "prost 0.14.4", + "prost-types 0.14.4", "rand 0.9.5", "thiserror 2.0.19", "tokio", diff --git a/livekit-telemetry/Cargo.toml b/livekit-telemetry/Cargo.toml index 86c7b5d99..1a903971a 100644 --- a/livekit-telemetry/Cargo.toml +++ b/livekit-telemetry/Cargo.toml @@ -13,6 +13,8 @@ log = { workspace = true } thiserror = { workspace = true } async-trait = "0.1" prost = { workspace = true } +# `google.protobuf.Any`/`Duration` for the `google.rpc.Status` error body (RetryInfo). +prost-types = { workspace = true } rand = { workspace = true } # gzip for request bodies; the deflate backend (miniz_oxide) is already linked by data streams. flate2 = "1" diff --git a/livekit-telemetry/README.md b/livekit-telemetry/README.md index 3ca245667..d2475b991 100644 --- a/livekit-telemetry/README.md +++ b/livekit-telemetry/README.md @@ -103,7 +103,9 @@ start it again and run once more to watch the cached batch replay. No Rust crate can observe these without a JVM/ObjC bridge, and `device-info` (in this workspace) covers static facts, not state. - **Transport is the only injection point.** The core composes URL, headers and body; the - transport moves bytes and reports [`ExportError`] so the core alone decides retry / drop / + transport moves bytes both ways and returns the raw [`ExportResponse`] (failing only without + one), so the core alone reads `Retry-After`, the `google.rpc.Status` body (`RetryInfo`) and + the "disabled" answer, and decides retry / drop / persist / go-silent. No Rust HTTP/TLS stack is linked unless the `net` feature is enabled. - **Size.** OTLP types come from `opentelemetry-proto` (`gen-tonic-messages`, no tonic). Its `opentelemetry`/`opentelemetry_sdk` dependencies are dead code here and LTO removes them diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index dbbb8cb2f..545008854 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -106,7 +106,8 @@ enum Delivery { /// gzips the batch and writes it to the [`BatchCache`] *before* any network is involved; then it /// [`upload`](Self::upload)s the cache oldest-first through the [`TelemetryTransport`], /// removing what the collector accepted or rejected. A failed upload pauses the cache for a -/// minute; a `Retry-After` additionally drops new batches for its duration (throttling must not +/// minute; a `Retry-After` header or `RetryInfo` body additionally drops new batches for its +/// duration (throttling must not /// become a disk-backed queue); `Disabled` empties the cache and silences the exporter for good. /// /// Telemetry must never win over media, so uploads are shaped as well as batched: at most @@ -640,20 +641,23 @@ impl Exporter { // Exponential + jitter once real fleets exercise this. let mut last = String::new(); for attempt in 0..=MAX_RETRIES { - match timeout(attempt_timeout, self.transport.send(request.clone())).await { + let attempt_result = timeout(attempt_timeout, self.transport.send(request.clone())) + .await + .map(|sent| sent.and_then(|response| ExportError::from_response(&response))); + match attempt_result { Ok(Ok(())) => return Delivery::Sent, Ok(Err(ExportError::Disabled)) => return Delivery::Disabled, - Ok(Err(ExportError::Rejected { message })) => { - log::error!("batch rejected by the collector, data lost: {message}"); + Ok(Err(ExportError::Rejected { reason })) => { + log::error!("batch rejected by the collector, data lost: {reason}"); return Delivery::Rejected; } - Ok(Err(ExportError::Retryable { message, retry_after_ms: Some(ms) })) => { - log::debug!("throttled for {ms} ms: {message}"); + Ok(Err(ExportError::Retryable { reason, retry_after_ms: Some(ms) })) => { + log::debug!("throttled for {ms} ms: {reason}"); return Delivery::Throttled { retry_after: Some(Duration::from_millis(ms)) }; } - Ok(Err(ExportError::Retryable { message, retry_after_ms: None })) => { - log::debug!("upload failed (attempt {}): {message}", attempt + 1); - last = message; + Ok(Err(ExportError::Retryable { reason, retry_after_ms: None })) => { + log::debug!("upload failed (attempt {}): {reason}", attempt + 1); + last = reason; Counters::add(&self.counters.upload_failures, 1); } Err(_) => { diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index d4bdb14ff..fe86c2593 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -165,7 +165,9 @@ impl TelemetryConfig { /// # struct Discard; /// # #[async_trait::async_trait] /// # impl TelemetryTransport for Discard { -/// # async fn send(&self, _: ExportRequest) -> Result<(), ExportError> { Ok(()) } +/// # async fn send(&self, _: ExportRequest) -> Result { +/// # Ok(ExportResponse::accepted()) +/// # } /// # } /// # #[tokio::main(flavor = "current_thread")] async fn main() { /// let (telemetry, exporter) = @@ -852,8 +854,8 @@ mod tests { logs::v1::LogRecord, trace::v1::{span, status}, }, - AppState, ExportError, ExportRequest, SpanOutcome, StreamDirection, ThermalState, - TrackKind, + AppState, ExportError, ExportRequest, ExportResponse, SpanOutcome, StreamDirection, + ThermalState, TrackKind, }; #[derive(Default)] @@ -876,9 +878,10 @@ mod tests { #[async_trait::async_trait] impl TelemetryTransport for FakeTransport { - async fn send(&self, request: ExportRequest) -> Result<(), ExportError> { + async fn send(&self, request: ExportRequest) -> Result { self.requests.lock().expect("lock").push(request); - self.script.lock().expect("lock").pop_front().unwrap_or(Ok(())) + let scripted = self.script.lock().expect("lock").pop_front().unwrap_or(Ok(())); + scripted.map(|()| ExportResponse::accepted()) } } @@ -890,7 +893,7 @@ mod tests { } fn offline() -> Result<(), ExportError> { - Err(ExportError::Retryable { message: "offline".into(), retry_after_ms: None }) + Err(ExportError::Retryable { reason: "offline".into(), retry_after_ms: None }) } fn offline_forever() -> impl Iterator> { @@ -1014,7 +1017,7 @@ mod tests { #[tokio::test(start_paused = true)] async fn rejected_batch_is_dropped_without_retry() { let transport = - FakeTransport::scripted([Err(ExportError::Rejected { message: "400".into() })]); + FakeTransport::scripted([Err(ExportError::Rejected { reason: "400".into() })]); let telemetry = pipeline(transport.clone()); telemetry.emit(TelemetryEvent::new("lk.ping")); telemetry.flush().await; @@ -1061,7 +1064,7 @@ mod tests { async fn throttling_keeps_cached_batches_and_drops_new_ones() { let dir = temp_dir("throttle"); let throttled = - Err(ExportError::Retryable { message: "429".into(), retry_after_ms: Some(5_000) }); + Err(ExportError::Retryable { reason: "429".into(), retry_after_ms: Some(5_000) }); let transport = FakeTransport::scripted([throttled]); let telemetry = persisted_pipeline(transport.clone(), &dir); telemetry.emit(TelemetryEvent::new("lk.ping")); @@ -1225,7 +1228,7 @@ mod tests { #[async_trait::async_trait] impl TelemetryTransport for Hanging { - async fn send(&self, _: ExportRequest) -> Result<(), ExportError> { + async fn send(&self, _: ExportRequest) -> Result { std::future::pending().await } } diff --git a/livekit-telemetry/src/transport.rs b/livekit-telemetry/src/transport.rs index fce6f0915..75ac8c037 100644 --- a/livekit-telemetry/src/transport.rs +++ b/livekit-telemetry/src/transport.rs @@ -14,6 +14,8 @@ use std::collections::HashMap; +use prost::Message; + /// One fully composed OTLP/HTTP export request. /// /// The core fills in the URL, the headers (content type, auth, …) and the protobuf body; @@ -27,45 +29,125 @@ pub struct ExportRequest { pub body: Vec, } -/// Why a transport could not deliver an [`ExportRequest`]. Drives the exporter's -/// retry / drop / go-silent decision (OTLP/HTTP failure semantics). +/// What the collector answered. A transport moves bytes both ways and never interprets them: +/// status classification, `Retry-After`, the `google.rpc.Status` body and the Cloud "disabled" +/// contract are read once, here, for every platform ([`ExportError::from_response`]). +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq, Default)] +pub struct ExportResponse { + pub status: u16, + pub headers: HashMap, + pub body: Vec, +} + +impl ExportResponse { + /// `2xx`, nothing else: what an accepting collector answers. + pub fn accepted() -> Self { + Self { status: 200, ..Self::default() } + } +} + +/// Why a batch could not be delivered. Drives the exporter's retry / drop / go-silent decision +/// (OTLP/HTTP failure semantics). Transports return only the `Retryable`/`Rejected` they can +/// know without a response (network error, timeout, invalid URL); everything else comes from +/// [`ExportError::from_response`]. #[cfg_attr(feature = "uniffi", derive(uniffi::Error))] #[derive(Debug, Clone, PartialEq, thiserror::Error)] pub enum ExportError { - /// Transient failure (network error, timeout, HTTP 429/502/503/504). The batch is - /// retried, honoring `retry_after_ms` when the collector sent `Retry-After`. - #[error("retryable export error: {message}")] - Retryable { message: String, retry_after_ms: Option }, + /// Transient failure: no response, HTTP 429/502/503/504, or any error status carrying + /// `google.rpc.RetryInfo`. The batch is retried, waiting `retry_after_ms` when the collector + /// said how long (`Retry-After` header, else `RetryInfo.retry_delay`). + #[error("retryable export error: {reason}")] + Retryable { reason: String, retry_after_ms: Option }, /// The collector rejected the payload (any other 4xx/5xx). The batch is dropped. - #[error("export rejected: {message}")] - Rejected { message: String }, + #[error("export rejected: {reason}")] + Rejected { reason: String }, /// Telemetry is disabled for this project. The exporter goes silent for good. #[error("telemetry disabled by the collector")] Disabled, } impl ExportError { - /// Classify an HTTP status per OTLP/HTTP: 2xx ok, 429/502/503/504 retryable, else rejected. - pub fn from_status(status: u16, retry_after_ms: Option) -> Result<(), Self> { + /// Classify a collector response: 2xx ok; 401/403 whose body says "disabled" = telemetry is + /// off for this project; 429/502/503/504, or any error carrying `RetryInfo`, retryable after + /// the header's `Retry-After` seconds, else the body's `retry_delay`, else the exporter's + /// default; anything else rejected. + pub fn from_response(response: &ExportResponse) -> Result<(), Self> { + let status = response.status; + if (200..300).contains(&status) { + return Ok(()); + } + // OTLP/HTTP errors are a protobuf `google.rpc.Status`; Cloud's "disabled" answer is text. + let rpc = RpcStatus::decode(response.body.as_slice()).ok(); + let text = match rpc.as_ref().map(|s| s.message.trim()).filter(|m| !m.is_empty()) { + Some(message) => message.to_owned(), + None => String::from_utf8_lossy(&response.body).trim().chars().take(200).collect(), + }; + if matches!(status, 401 | 403) && text.to_ascii_lowercase().contains("disabled") { + return Err(Self::Disabled); + } + // `reason`, not `message`: a UniFFI error field named `message` collides with + // `Throwable.message` in Kotlin. + let reason = if text.is_empty() { + format!("HTTP {status}") + } else { + format!("HTTP {status}: {text}") + }; + let header = response + .headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("retry-after")) + .and_then(|(_, value)| value.trim().parse::().ok()) + .map(|seconds| seconds * 1000); + let body = rpc.as_ref().and_then(retry_info_ms); match status { - 200..=299 => Ok(()), 429 | 502 | 503 | 504 => { - Err(Self::Retryable { message: format!("HTTP {status}"), retry_after_ms }) + Err(Self::Retryable { reason, retry_after_ms: header.or(body) }) } - _ => Err(Self::Rejected { message: format!("HTTP {status}") }), + _ if body.is_some() => Err(Self::Retryable { reason, retry_after_ms: header.or(body) }), + _ => Err(Self::Rejected { reason }), } } } -/// Moves an encoded batch off the device. +/// `google.rpc.Status`, the OTLP/HTTP error body. Two messages, three fields: not worth a +/// generated crate. +#[derive(Clone, PartialEq, prost::Message)] +struct RpcStatus { + #[prost(int32, tag = "1")] + code: i32, + #[prost(string, tag = "2")] + message: String, + #[prost(message, repeated, tag = "3")] + details: Vec, +} + +/// `google.rpc.RetryInfo`. +#[derive(Clone, PartialEq, prost::Message)] +struct RetryInfo { + #[prost(message, optional, tag = "1")] + retry_delay: Option, +} + +fn retry_info_ms(status: &RpcStatus) -> Option { + status + .details + .iter() + .filter(|detail| detail.type_url.ends_with("google.rpc.RetryInfo")) + .find_map(|detail| RetryInfo::decode(detail.value.as_slice()).ok()?.retry_delay) + .map(|delay| delay.seconds.max(0) as u64 * 1000 + delay.nanos.max(0) as u64 / 1_000_000) +} + +/// Moves an encoded batch off the device and hands back whatever came back. /// /// Implemented in Rust ([`NetTransport`], feature `net`) or by the host platform -/// (URLSession, OkHttp, a data channel, …) through UniFFI. Implementations must not retry -/// themselves: the exporter owns the retry policy. +/// (URLSession, OkHttp, a data channel, …) through UniFFI. A transport performs the POST and +/// returns the response whatever its status; it fails only when there was no response. It +/// must not retry: the exporter owns the retry policy. #[cfg_attr(feature = "uniffi", uniffi::export(with_foreign))] #[async_trait::async_trait] pub trait TelemetryTransport: Send + Sync { - async fn send(&self, request: ExportRequest) -> Result<(), ExportError>; + async fn send(&self, request: ExportRequest) -> Result; } #[cfg(feature = "net")] @@ -74,7 +156,7 @@ mod net { use livekit_net::{Header, HttpClient, HttpClientExt, TransportError}; - use super::{ExportError, ExportRequest, TelemetryTransport}; + use super::{ExportError, ExportRequest, ExportResponse, TelemetryTransport}; /// Default transport: HTTP POST through a `livekit-net` client — the native backend, or /// whatever the host registered with `livekit_net::set_http_client`. @@ -93,28 +175,114 @@ mod net { #[async_trait::async_trait] impl TelemetryTransport for NetTransport { - async fn send(&self, request: ExportRequest) -> Result<(), ExportError> { + async fn send(&self, request: ExportRequest) -> Result { let headers = request.headers.into_iter().map(|(name, value)| Header { name, value }).collect(); - let response = - self.0.post(request.url, headers, request.body).await.map_err(|err| match err { - TransportError::Http { status } => ExportError::from_status(status, None) - .err() - .unwrap_or(ExportError::Rejected { message: format!("HTTP {status}") }), - other => { - ExportError::Retryable { message: other.to_string(), retry_after_ms: None } - } - })?; - let retry_after_ms = response - .headers - .iter() - .find(|h| h.name.eq_ignore_ascii_case("retry-after")) - .and_then(|h| h.value.trim().parse::().ok()) - .map(|seconds| seconds * 1000); - ExportError::from_status(response.status, retry_after_ms) + match self.0.post(request.url, headers, request.body).await { + Ok(response) => Ok(ExportResponse { + status: response.status, + headers: response.headers.into_iter().map(|h| (h.name, h.value)).collect(), + body: response.body, + }), + // The client swallowed the body with the status; the core still classifies it. + Err(TransportError::Http { status }) => { + Ok(ExportResponse { status, ..Default::default() }) + } + Err(other) => { + Err(ExportError::Retryable { reason: other.to_string(), retry_after_ms: None }) + } + } } } } #[cfg(feature = "net")] pub use net::NetTransport; + +#[cfg(test)] +mod tests { + use prost::Message; + + use super::*; + + fn status_body(message: &str, retry: Option<(i64, i32)>) -> Vec { + let details = retry + .map(|(seconds, nanos)| prost_types::Any { + type_url: "type.googleapis.com/google.rpc.RetryInfo".into(), + value: RetryInfo { retry_delay: Some(prost_types::Duration { seconds, nanos }) } + .encode_to_vec(), + }) + .into_iter() + .collect(); + RpcStatus { code: 8, message: message.into(), details }.encode_to_vec() + } + + fn response(status: u16, headers: &[(&str, &str)], body: Vec) -> ExportResponse { + ExportResponse { + status, + headers: headers.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(), + body, + } + } + + #[test] + fn accepted_and_rejected() { + assert_eq!(ExportError::from_response(&ExportResponse::accepted()), Ok(())); + assert_eq!(ExportError::from_response(&response(204, &[], vec![])), Ok(())); + assert_eq!( + ExportError::from_response(&response(400, &[], status_body("bad batch", None))), + Err(ExportError::Rejected { reason: "HTTP 400: bad batch".into() }) + ); + assert_eq!( + ExportError::from_response(&response(401, &[], b"token expired".to_vec())), + Err(ExportError::Rejected { reason: "HTTP 401: token expired".into() }) + ); + } + + #[test] + fn retry_after_header_wins_over_the_body_then_the_default() { + let both = response(429, &[("Retry-After", "7")], status_body("quota", Some((30, 0)))); + assert!(matches!( + ExportError::from_response(&both), + Err(ExportError::Retryable { retry_after_ms: Some(7_000), .. }) + )); + let body_only = response(429, &[], status_body("quota", Some((30, 500_000_000)))); + assert!(matches!( + ExportError::from_response(&body_only), + Err(ExportError::Retryable { retry_after_ms: Some(30_500), .. }) + )); + let neither = response(503, &[], vec![]); + assert_eq!( + ExportError::from_response(&neither), + Err(ExportError::Retryable { reason: "HTTP 503".into(), retry_after_ms: None }) + ); + } + + #[test] + fn retry_info_makes_any_error_retryable() { + let internal = response(500, &[], status_body("try later", Some((5, 0)))); + assert!(matches!( + ExportError::from_response(&internal), + Err(ExportError::Retryable { retry_after_ms: Some(5_000), .. }) + )); + // A 500 without RetryInfo stays a drop, per OTLP/HTTP. + assert!(matches!( + ExportError::from_response(&response(500, &[], vec![])), + Err(ExportError::Rejected { .. }) + )); + } + + #[test] + fn disabled_by_owner() { + let text = b"project data recording is disabled by owner".to_vec(); + assert_eq!( + ExportError::from_response(&response(401, &[], text)), + Err(ExportError::Disabled) + ); + let rpc = status_body("Project data recording is disabled by owner", None); + assert_eq!( + ExportError::from_response(&response(403, &[], rpc)), + Err(ExportError::Disabled) + ); + } +} diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index 15f54ec5a..4f4ba241a 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -18,9 +18,10 @@ use std::{ use livekit_telemetry::{ global::{self, TelemetryInstrument}, - Attribute, AttributeValue, DeviceEvent, DeviceState, ExportError, ExportRequest, LogRecord, - NetTransport, RoomIdentity, RtcStatsSample, SpanName, SpanOutcome, SpanStep, SpanTrack, - TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, TraceContext, + Attribute, AttributeValue, DeviceEvent, DeviceState, ExportError, ExportRequest, + ExportResponse, LogRecord, NetTransport, RoomIdentity, RtcStatsSample, SpanName, SpanOutcome, + SpanStep, SpanTrack, TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, + TraceContext, }; use tokio::sync::{mpsc, oneshot}; @@ -184,7 +185,7 @@ pub struct PendingExport { struct Pending { export: PendingExport, - done: oneshot::Sender>, + done: oneshot::Sender>, } /// Pull-side transport: Rust never calls into the host. The exporter queues each request; the @@ -254,7 +255,7 @@ impl TelemetrySpan { pub struct TelemetryExportQueue { tx: mpsc::UnboundedSender, rx: tokio::sync::Mutex>, - inflight: Mutex>>>, + inflight: Mutex>>>, seq: AtomicU64, } @@ -281,29 +282,40 @@ impl TelemetryExportQueue { Some(pending.export) } - /// Report how the request with `id` went: `None` = accepted by the collector. - pub fn complete(&self, id: u64, error: Option) { + /// The collector's answer to the request with `id`, whatever its status; the core classifies it. + pub fn complete(&self, id: u64, response: ExportResponse) { + self.finish(id, Ok(response)); + } + + /// The request with `id` got no answer (network error, timeout, invalid URL). + pub fn fail(&self, id: u64, error: ExportError) { + self.finish(id, Err(error)); + } +} + +impl TelemetryExportQueue { + fn finish(&self, id: u64, outcome: Result) { let done = self.inflight.lock().unwrap_or_else(|e| e.into_inner()).remove(&id); if let Some(done) = done { - let _ = done.send(error.map_or(Ok(()), Err)); + let _ = done.send(outcome); } } } #[async_trait::async_trait] impl TelemetryTransport for TelemetryExportQueue { - async fn send(&self, request: ExportRequest) -> Result<(), ExportError> { + async fn send(&self, request: ExportRequest) -> Result { let id = self.seq.fetch_add(1, Ordering::Relaxed); let (done, wait) = oneshot::channel(); let pending = Pending { export: PendingExport { id, request }, done }; if self.tx.send(pending).is_err() { return Err(ExportError::Retryable { - message: "export queue closed".into(), + reason: "export queue closed".into(), retry_after_ms: None, }); } wait.await.unwrap_or(Err(ExportError::Retryable { - message: "host dropped the export".into(), + reason: "host dropped the export".into(), retry_after_ms: None, })) } diff --git a/livekit-uniffi/support/dart/test/telemetry_test.dart b/livekit-uniffi/support/dart/test/telemetry_test.dart index 9f39163f2..328bb473d 100644 --- a/livekit-uniffi/support/dart/test/telemetry_test.dart +++ b/livekit-uniffi/support/dart/test/telemetry_test.dart @@ -1,3 +1,5 @@ +import 'dart:typed_data'; + import 'package:livekit_uniffi/livekit_telemetry.dart'; import 'package:livekit_uniffi/livekit_uniffi.dart'; import 'package:test/test.dart'; @@ -6,13 +8,14 @@ import 'package:test/test.dart'; /// callbacks are isolate-bound (`Pointer.fromFunction`) and the VM aborts with "Cannot invoke /// native callback outside an isolate" when the exporter invokes `TelemetryTransport.send` from /// a tokio worker. The pull queue inverts the direction: Dart awaits `next()`, performs the -/// request, and reports back with `complete()`. +/// request, and reports the collector's answer back with `complete()` (or `fail()` when there +/// was none); the core classifies the status and body. Future serve(TelemetryExportQueue queue, List sink, int count) async { for (var i = 0; i < count; i++) { final pending = await queue.next(); if (pending == null) return; sink.add(pending.request); - queue.complete(id: pending.id, error: null); + queue.complete(id: pending.id, response: ExportResponse(status: 200, headers: {}, body: Uint8List(0))); } } From e5827fe14b8c7e21665a6e61a89df8a39922fb74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:43:05 +0200 Subject: [PATCH 41/50] feat(telemetry): one vocabulary for every SDK's environment signals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed against client-sdk-android, -flutter, -js and -react-native: where a platform had a richer or differently spelled set than the core, the core now holds the superset as a shared enum so dashboards read the same values from every SDK. - `NetworkType` gains `Vpn`, `Bluetooth`, `Other` (Android transports, connectivity_plus, WebRTC candidate network types). - Audio route outputs are `AudioOutput` values, not the platform's port strings. - `PermissionDenied` becomes `CaptureFailed { device, reason }`: the getUserMedia failure taxonomy (permission denied, not found, in use, disconnected, other) also covers Android's camera callbacks and iOS capture interruptions. - `SpanName::Reconnect` carries a `ReconnectReason` (the protocol's RR_* set plus transport / network / debug) instead of a string; `SpanTrack.source` is the protocol's `TrackSource`. - `Scope::disconnected(DisconnectReason)` emits `lk.room.disconnected` — the protocol's 17 reasons plus `reconnect_failed`; info when the client hung up, warn otherwise. - `RtcStatsSample` gains pause count/duration, silent concealed samples, interruption count/duration and `quality_limitation.other_ms`: a superset of the protocol's client `MetricLabel` set. - macOS cdylibs link with `-headerpad_max_install_names` so Flutter's native-assets step can rewrite the install name of a locally built library. Co-Authored-By: Claude Fable 5.1 --- .cargo/config.toml | 6 +- livekit-telemetry/SPEC.md | 42 +++++++++++--- livekit-telemetry/src/device.rs | 93 ++++++++++++++++++++++-------- livekit-telemetry/src/lib.rs | 2 +- livekit-telemetry/src/rtc.rs | 28 +++++++++ livekit-telemetry/src/scope.rs | 41 +++++++++++++ livekit-telemetry/src/telemetry.rs | 5 +- livekit-telemetry/src/trace.rs | 52 ++++++++++++++--- livekit-uniffi/src/telemetry.rs | 13 +++-- 9 files changed, 232 insertions(+), 50 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 2adead06d..122b31359 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -8,16 +8,18 @@ rustflags = ["-C", "target-feature=+crt-static"] [target.aarch64-pc-windows-msvc] rustflags = ["-C", "target-feature=+crt-static"] +# `-headerpad_max_install_names` on macOS: Flutter's native-assets step rewrites the cdylib's +# install name with install_name_tool, which fails without the padding. # `-ObjC` must also be passed to rustdoc: doctest binaries are linked by rustdoc # and do not inherit `rustflags`. Without it the ObjC categories in libwebrtc's # static lib (e.g. `NSString (StdString)`) are not loaded, and static # initializers such as RTCH264ProfileLevelId.mm abort at process startup. [target.x86_64-apple-darwin] -rustflags = ["-C", "link-args=-ObjC"] +rustflags = ["-C", "link-args=-ObjC -Wl,-headerpad_max_install_names"] rustdocflags = ["-C", "link-args=-ObjC"] [target.aarch64-apple-darwin] -rustflags = ["-C", "link-args=-ObjC"] +rustflags = ["-C", "link-args=-ObjC -Wl,-headerpad_max_install_names"] rustdocflags = ["-C", "link-args=-ObjC"] [target.aarch64-apple-ios] diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index 2ca9eb515..843ff51ea 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -117,7 +117,7 @@ platforms: ios, macos, android — optional elsewhere event: lk.device.network.changed area: device attributes: - network.connection.type: enum(wifi | cell | wired | unavailable | unknown) # OTel semconv + network.connection.type: enum(wifi | cell | wired | vpn | bluetooth | other | unavailable | unknown) # OTel semconv lk.device.network.expensive: bool # cellular / hotspot (NWPath.isExpensive, metered) lk.device.network.constrained: bool # Low Data Mode / Data Saver / navigator.connection.saveData cadence: on change of any attribute (+ initial value) @@ -139,8 +139,8 @@ platforms: ios, android — optional elsewhere event: lk.device.audio_route.changed area: device attributes: - lk.device.audio_route.reason: string # AVAudioSession route-change reason name - lk.device.audio_route.outputs: string # comma-separated output port types (speaker, bluetooth_a2dp, …) + lk.device.audio_route.reason: enum(new_device | old_device_unavailable | category_change | override | wake_from_sleep | no_suitable_route | route_configuration_change | unknown) # AVAudioSession names; `unknown` where the platform gives none + lk.device.audio_route.outputs: string # comma-separated enum(speaker | receiver | wired_headset | bluetooth | car_audio | air_play | hdmi | usb | other) cadence: on change platforms: ios — android: audio device callbacks; optional elsewhere ``` @@ -154,6 +154,17 @@ cadence: on change platforms: ios — android: audio focus loss/gain; optional elsewhere ``` +```yaml +event: lk.device.capture.failed +area: device +severity: warn +attributes: + lk.device.capture.device: enum(camera | microphone | screen_share) + lk.device.capture.reason: enum(permission_denied | not_found | in_use | disconnected | other) # the getUserMedia failure taxonomy +cadence: on failure +platforms: all — ios: authorization status, capture interruptions; android: permission checks, camera callbacks; web: DOMException names +``` + ## Cadence policy `flush_interval × factor` and `stats_window × factor`, capped at 4× (15 s → 60 s at the @@ -260,6 +271,12 @@ attributes: lk.rtc.jitter_buffer_emitted_count: int # inbound lk.rtc.quality_limitation.bandwidth_ms: int # outbound video lk.rtc.quality_limitation.cpu_ms: int # outbound video + lk.rtc.quality_limitation.other_ms: int # outbound video + lk.rtc.pause_count: int # inbound video + lk.rtc.pauses_duration_ms: int # inbound video + lk.rtc.silent_concealed_samples: int # inbound audio + lk.rtc.interruption_count: int # inbound audio + lk.rtc.interruptions_duration_ms: int # inbound audio # gauges — min / max / avg over the window lk.rtc.jitter_ms.{min,max,avg}: double lk.rtc.rtt_ms.{min,max,avg}: double # remote-inbound RTT for outbound, candidate-pair for inbound @@ -268,6 +285,16 @@ attributes: platforms: all ``` +```yaml +event: lk.room.disconnected +area: session +severity: info (client_initiated) | warn (anything else) +cadence: once, when the Room leaves connected for good — never on a reconnect +attributes: + lk.disconnect.reason: enum(client_initiated | duplicate_identity | server_shutdown | participant_removed | room_deleted | state_mismatch | join_failure | migration | signal_close | room_closed | user_unavailable | user_rejected | sip_trunk_failure | connection_timeout | media_failure | agent_error | reconnect_failed | unknown) # the protocol's DisconnectReason, plus the client giving up +platforms: all +``` + ## Spans A span is **one attempt** at an operation. The scope (one Room connection lifetime, across @@ -296,7 +323,7 @@ outcome: ok | error (error.type = LiveKitError. | CancellationError | " per attempt @@ -309,7 +336,7 @@ kind: internal parent: the ambient span, when any (the connect span for a pre-connect microphone) attributes: lk.track.kind: enum(audio | video) - lk.track.source: enum(camera | microphone | screenShare | screenShareAudio | unknown) + lk.track.source: enum(camera | microphone | screen_share | screen_share_audio | unknown) lk.track.sid: string # on success outcome: ok | error (error.type) | cancelled ``` @@ -325,7 +352,7 @@ ends: at first media (the first stats reading with bytes received; 1 s granula attributes: lk.track.sid: string lk.track.kind: enum(audio | video) - lk.track.source: string + lk.track.source: enum(camera | microphone | screen_share | screen_share_audio | unknown) lk.participant.remote_identity: string checkpoints: subscribed, first_media ``` @@ -348,7 +375,8 @@ scope `set_attribute`, and `Span::set_attribute` for app-defined spans. | `Span::end(outcome, error)` / `fail(error)` / `cancel()` | status, `error.type`, `lk.outcome`; ending twice is a no-op | | `Span::describe()` | `lk.connect: ws_open +1.49s, signal +0.03s, total 1.83s, ok` — the console line, identical on every platform | | `Span::context()` | `TraceContext { trace_id, span_id }` for log correlation; `None` when detached | -| `device_event(DeviceEvent::{AudioRouteChanged, AudioInterruption, PermissionDenied})` | `lk.device.audio_route.changed`, `lk.device.audio.interruption`, `lk.device.permission.denied` with display bodies | +| `device_event(DeviceEvent::{AudioRouteChanged, AudioInterruption, CaptureFailed})` | `lk.device.audio_route.changed`, `lk.device.audio.interruption`, `lk.device.capture.failed` with display bodies; every value is a shared enum (`AudioOutput`, `CaptureDevice`, `CaptureFailure`) | +| `Scope::disconnected(DisconnectReason)` | `lk.room.disconnected` with `lk.disconnect.reason` — info when the client hung up, warn otherwise | | `RtcStatsSample.layer` (rid, ssrc or stats id) | simulcast layers folded into one monotonic series per track before windowing | Timing rule: span calls are synchronous and stamp the clock inside the core, so the only skew is diff --git a/livekit-telemetry/src/device.rs b/livekit-telemetry/src/device.rs index 65aa6b2fb..45487ae33 100644 --- a/livekit-telemetry/src/device.rs +++ b/livekit-telemetry/src/device.rs @@ -54,6 +54,13 @@ pub enum NetworkType { Wifi, Cell, Wired, + /// A VPN tunnel is the active interface (Android `TRANSPORT_VPN`, WebRTC candidate `vpn`). + Vpn, + /// Bluetooth tethering (Android `TRANSPORT_BLUETOOTH`, WebRTC candidate `bluetooth`). + Bluetooth, + /// Connected over something the platform does not name (`NWInterface.InterfaceType.other`, + /// Android `TRANSPORT_USB` / `LOWPAN`, …). + Other, Unavailable, } @@ -253,6 +260,9 @@ impl NetworkType { NetworkType::Wifi => "wifi", NetworkType::Cell => "cell", NetworkType::Wired => "wired", + NetworkType::Vpn => "vpn", + NetworkType::Bluetooth => "bluetooth", + NetworkType::Other => "other", NetworkType::Unavailable => "unavailable", } } @@ -348,34 +358,60 @@ pub enum AudioRouteReason { Unknown, } +/// Where the audio goes, in platform-neutral terms (AVAudioSession port types, Android's +/// `AudioDevice`; the web's `MediaDeviceInfo.kind` only ever gives `other`). #[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum DevicePermission { +pub enum AudioOutput { + Speaker, + /// The phone earpiece (`builtInReceiver`, Android `Earpiece`). + Receiver, + WiredHeadset, + Bluetooth, + CarAudio, + AirPlay, + Hdmi, + Usb, + Other, +} + +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CaptureDevice { Camera, Microphone, ScreenShare, } +/// Why a capture device could not be used: the `getUserMedia` failure taxonomy, which every +/// platform maps onto (authorization status, `onCameraDisconnected`, `NotReadableError`, …). +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CaptureFailure { + /// The user (or MDM) said no: the most common pre-connect failure on mobile. + PermissionDenied, + NotFound, + /// Another app holds the device (`NotReadableError`, `TrackStartError`). + InUse, + /// The device went away mid-capture (unplugged, `onCameraDisconnected`). + Disconnected, + Other, +} + /// Things that happen to the device mid-call and explain what the media did next. Not state /// (see [`DeviceState`]): each one is a record with a display body. #[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] #[derive(Debug, Clone, PartialEq, Eq)] pub enum DeviceEvent { - /// `outputs` as the platform names them (`Speaker`, `BluetoothA2DPOutput`, …). - AudioRouteChanged { - outputs: Vec, - reason: AudioRouteReason, - }, - AudioInterruption { - began: bool, - }, - /// The user (or MDM) said no: the most common pre-connect failure on mobile. - PermissionDenied { - permission: DevicePermission, - }, + /// The outputs after the change, in the order the platform lists them. + AudioRouteChanged { outputs: Vec, reason: AudioRouteReason }, + /// A phone call, Siri, another app taking audio focus: `began`, then the matching `ended`. + AudioInterruption { began: bool }, + /// A capture device could not be, or stay, used. + CaptureFailed { device: CaptureDevice, reason: CaptureFailure }, } -fn snake(debug: impl std::fmt::Debug) -> String { +pub(crate) fn snake(debug: impl std::fmt::Debug) -> String { let mut out = String::new(); for (i, c) in format!("{debug:?}").chars().enumerate() { if c.is_uppercase() && i > 0 { @@ -391,7 +427,7 @@ impl DeviceEvent { use crate::{Severity, TelemetryEvent}; match self { Self::AudioRouteChanged { outputs, reason } => { - let outputs = outputs.join(","); + let outputs = outputs.into_iter().map(snake).collect::>().join(","); let reason = snake(reason); TelemetryEvent::new("lk.device.audio_route.changed") .with_body(format!("audio route: {outputs} ({reason})")) @@ -404,12 +440,13 @@ impl DeviceEvent { .with_body(format!("audio interruption {phase}")) .with_attribute("lk.device.audio.interruption", phase) } - Self::PermissionDenied { permission } => { - let permission = snake(permission); - TelemetryEvent::new("lk.device.permission.denied") + Self::CaptureFailed { device, reason } => { + let (device, reason) = (snake(device), snake(reason)); + TelemetryEvent::new("lk.device.capture.failed") .with_severity(Severity::Warn) - .with_body(format!("permission denied: {permission}")) - .with_attribute("lk.device.permission", permission) + .with_body(format!("capture failed: {device} ({reason})")) + .with_attribute("lk.device.capture.device", device) + .with_attribute("lk.device.capture.reason", reason) } } } @@ -422,14 +459,20 @@ mod event_tests { #[test] fn device_events_have_bodies_and_snake_case_values() { let event = DeviceEvent::AudioRouteChanged { - outputs: vec!["Speaker".into()], + outputs: vec![AudioOutput::Speaker], reason: AudioRouteReason::OldDeviceUnavailable, } .into_event(); - assert_eq!(event.body.as_deref(), Some("audio route: Speaker (old_device_unavailable)")); - let denied = DeviceEvent::PermissionDenied { permission: DevicePermission::ScreenShare } - .into_event(); - assert_eq!(denied.body.as_deref(), Some("permission denied: screen_share")); + assert_eq!(event.body.as_deref(), Some("audio route: speaker (old_device_unavailable)")); + let denied = DeviceEvent::CaptureFailed { + device: CaptureDevice::ScreenShare, + reason: CaptureFailure::PermissionDenied, + } + .into_event(); + assert_eq!( + denied.body.as_deref(), + Some("capture failed: screen_share (permission_denied)") + ); assert_eq!(denied.severity, crate::Severity::Warn); } } diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs index a3134c183..9d56df407 100644 --- a/livekit-telemetry/src/lib.rs +++ b/livekit-telemetry/src/lib.rs @@ -59,7 +59,7 @@ pub use device::*; pub use event::*; pub use exporter::Exporter; pub use rtc::{RtcStatsSample, StreamDirection, TrackKind}; -pub use scope::{RoomIdentity, Scope}; +pub use scope::{DisconnectReason, RoomIdentity, Scope}; pub use span::SpanOutcome; pub use stats::{TelemetryStats, TelemetryStatus}; pub use telemetry::*; diff --git a/livekit-telemetry/src/rtc.rs b/livekit-telemetry/src/rtc.rs index 74542a9e7..8840c2e37 100644 --- a/livekit-telemetry/src/rtc.rs +++ b/livekit-telemetry/src/rtc.rs @@ -70,6 +70,18 @@ pub struct RtcStatsSample { pub quality_limitation_bandwidth_ms: Option, #[cfg_attr(feature = "uniffi", uniffi(default))] pub quality_limitation_cpu_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub quality_limitation_other_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub pause_count: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub pauses_duration_ms: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub silent_concealed_samples: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub interruption_count: Option, + #[cfg_attr(feature = "uniffi", uniffi(default))] + pub interruptions_duration_ms: Option, // Gauges. #[cfg_attr(feature = "uniffi", uniffi(default))] pub jitter_ms: Option, @@ -108,6 +120,12 @@ impl RtcStatsSample { jitter_buffer_emitted_count: None, quality_limitation_bandwidth_ms: None, quality_limitation_cpu_ms: None, + quality_limitation_other_ms: None, + pause_count: None, + pauses_duration_ms: None, + silent_concealed_samples: None, + interruption_count: None, + interruptions_duration_ms: None, jitter_ms: None, rtt_ms: None, frames_per_second: None, @@ -255,6 +273,12 @@ impl Window { ("lk.rtc.jitter_buffer_emitted_count", last.jitter_buffer_emitted_count), ("lk.rtc.quality_limitation.bandwidth_ms", last.quality_limitation_bandwidth_ms), ("lk.rtc.quality_limitation.cpu_ms", last.quality_limitation_cpu_ms), + ("lk.rtc.quality_limitation.other_ms", last.quality_limitation_other_ms), + ("lk.rtc.pause_count", last.pause_count), + ("lk.rtc.pauses_duration_ms", last.pauses_duration_ms), + ("lk.rtc.silent_concealed_samples", last.silent_concealed_samples), + ("lk.rtc.interruption_count", last.interruption_count), + ("lk.rtc.interruptions_duration_ms", last.interruptions_duration_ms), ]; for (key, value) in counters { if let Some(value) = value { @@ -295,6 +319,7 @@ struct LayerCounters { fps: Option, limitation_bandwidth_ms: Option, limitation_cpu_ms: Option, + limitation_other_ms: Option, } fn sum(values: impl Iterator>) -> Option { @@ -339,6 +364,7 @@ impl StatsWindows { fps: sample.frames_per_second, limitation_bandwidth_ms: sample.quality_limitation_bandwidth_ms, limitation_cpu_ms: sample.quality_limitation_cpu_ms, + limitation_other_ms: sample.quality_limitation_other_ms, }, ); sample.bytes = sum(layers.values().map(|l| l.bytes)); @@ -350,6 +376,8 @@ impl StatsWindows { sample.quality_limitation_bandwidth_ms = max_u64(layers.values().map(|l| l.limitation_bandwidth_ms)); sample.quality_limitation_cpu_ms = max_u64(layers.values().map(|l| l.limitation_cpu_ms)); + sample.quality_limitation_other_ms = + max_u64(layers.values().map(|l| l.limitation_other_ms)); } /// Close every open window into its event, filed under the window's session, and start fresh. diff --git a/livekit-telemetry/src/scope.rs b/livekit-telemetry/src/scope.rs index 4e4463fda..a6810b0a7 100644 --- a/livekit-telemetry/src/scope.rs +++ b/livekit-telemetry/src/scope.rs @@ -98,6 +98,31 @@ pub struct RoomIdentity { pub participant_identity: Option, } +/// Why a session ended: the protocol's `DisconnectReason`, plus the client giving up. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DisconnectReason { + Unknown, + ClientInitiated, + DuplicateIdentity, + ServerShutdown, + ParticipantRemoved, + RoomDeleted, + StateMismatch, + JoinFailure, + Migration, + SignalClose, + RoomClosed, + UserUnavailable, + UserRejected, + SipTrunkFailure, + ConnectionTimeout, + MediaFailure, + AgentError, + /// The reconnect policy ran out of attempts. + ReconnectFailed, +} + #[derive(Clone)] pub struct Scope { pub(crate) telemetry: Telemetry, @@ -121,6 +146,22 @@ impl Scope { self.emit(TelemetryEvent::custom(name, attributes)); } + /// The session ended for good (not a reconnect): the `lk.room.disconnected` record. + pub fn disconnected(&self, reason: DisconnectReason) { + let severity = if reason == DisconnectReason::ClientInitiated { + crate::Severity::Info + } else { + crate::Severity::Warn + }; + let reason = crate::device::snake(reason); + self.emit( + TelemetryEvent::new("lk.room.disconnected") + .with_severity(severity) + .with_body(format!("disconnected: {reason}")) + .with_attribute("lk.disconnect.reason", reason), + ); + } + /// Attach an attribute to every record of this session from now on; `None` removes it. pub fn set_attribute(&self, key: &str, value: Option) { self.state.set_attribute(key, value); diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index fe86c2593..fd8583da8 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -664,7 +664,7 @@ pub(crate) fn observability_endpoint(url: &str) -> Option { #[cfg(test)] mod tests { use crate::span::SpanKind; - use crate::{RoomIdentity, SpanName, SpanStep}; + use crate::{ReconnectReason, RoomIdentity, SpanName, SpanStep}; use std::{collections::VecDeque, fs, path::Path, sync::Mutex}; #[test] @@ -748,7 +748,8 @@ mod tests { let transport = FakeTransport::scripted([]); let telemetry = pipeline(transport.clone()); let session = telemetry.begin_scope(); - let span = session.start(SpanName::Reconnect { reason: "ws closed".into() }, None); + let span = session + .start(SpanName::Reconnect { reason: ReconnectReason::SignalDisconnected }, None); telemetry.emit(TelemetryEvent::new("lk.ping")); telemetry.flush().await; assert!(transport.sent().is_empty(), "an open reconnect holds uploads"); diff --git a/livekit-telemetry/src/trace.rs b/livekit-telemetry/src/trace.rs index 9d679e4a2..26eb1a614 100644 --- a/livekit-telemetry/src/trace.rs +++ b/livekit-telemetry/src/trace.rs @@ -3,6 +3,7 @@ //! inside, so the only skew is the FFI call itself; context propagation (the "current" span) //! stays with the platform runtime, which is the one thing a core cannot do. +use crate::device::snake; use std::{ sync::{Arc, Mutex}, time::Duration, @@ -15,12 +16,31 @@ use crate::{scope::ScopeState, Attribute, AttributeValue, SpanOutcome, Telemetry /// What an SDK operation is. The kind follows from the name: connects talk to the server /// (`client`), the rest is internal work. +/// What triggered a reconnect cycle: the protocol's `ReconnectReason` values, plus the ones only +/// a client knows. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReconnectReason { + /// The signaling socket closed (`RR_SIGNAL_DISCONNECTED`). + SignalDisconnected, + PublisherFailed, + SubscriberFailed, + /// A peer connection failed and the platform did not say which. + TransportFailed, + SwitchCandidate, + /// The device moved to another network (Wi-Fi ↔ cellular): the client noticed first. + NetworkChanged, + /// A test or a debug menu asked for it. + Debug, + Unknown, +} + #[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] #[derive(Debug, Clone, PartialEq, Eq)] pub enum SpanName { Connect, Reconnect { - reason: String, + reason: ReconnectReason, }, Publish, Subscribe, @@ -51,7 +71,7 @@ impl SpanName { fn attributes(&self) -> Vec { match self { Self::Reconnect { reason } => { - vec![Attribute::new("lk.reconnect.reason", reason.as_str())] + vec![Attribute::new("lk.reconnect.reason", snake(reason))] } _ => Vec::new(), } @@ -116,6 +136,17 @@ impl SpanStep { } } +/// The protocol's `TrackSource`. +#[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TrackSource { + Camera, + Microphone, + ScreenShare, + ScreenShareAudio, + Unknown, +} + /// The track a publish or subscribe span is about. #[cfg_attr(feature = "uniffi", derive(uniffi::Record))] #[derive(Debug, Clone, PartialEq)] @@ -124,8 +155,7 @@ pub struct SpanTrack { #[cfg_attr(feature = "uniffi", uniffi(default))] pub sid: Option, pub kind: TrackKind, - /// `camera`, `microphone`, `screen_share`, … as the platform names it. - pub source: String, + pub source: TrackSource, /// The publisher, for subscribe spans. #[cfg_attr(feature = "uniffi", uniffi(default))] pub remote_identity: Option, @@ -135,7 +165,7 @@ impl SpanTrack { fn attributes(&self) -> Vec { let mut out = vec![ Attribute::new("lk.track.kind", format!("{:?}", self.kind).to_lowercase()), - Attribute::new("lk.track.source", self.source.as_str()), + Attribute::new("lk.track.source", snake(self.source)), ]; if let Some(sid) = &self.sid { out.push(Attribute::new("lk.track.sid", sid.as_str())); @@ -323,7 +353,8 @@ mod tests { #[tokio::test(start_paused = true)] async fn a_detached_span_times_and_describes_itself() { - let span = Span::detached(SpanName::Reconnect { reason: "ws closed".into() }); + let span = + Span::detached(SpanName::Reconnect { reason: ReconnectReason::SignalDisconnected }); tokio::time::advance(Duration::from_millis(1490)).await; span.step(SpanStep::Attempt { number: 1, full: false }); tokio::time::advance(Duration::from_millis(30)).await; @@ -343,7 +374,10 @@ mod tests { assert!(span.context().is_none()); let attributes = span.lock().attributes.clone(); let get = |key: &str| attributes.iter().find(|a| a.key == key).map(|a| a.value.clone()); - assert_eq!(get("lk.reconnect.reason"), Some(AttributeValue::Str("ws closed".into()))); + assert_eq!( + get("lk.reconnect.reason"), + Some(AttributeValue::Str("signal_disconnected".into())) + ); assert_eq!(get("lk.reconnect.attempts"), Some(AttributeValue::Int(1))); assert_eq!(get("lk.reconnect.mode"), Some(AttributeValue::Str("quick".into()))); } @@ -354,13 +388,13 @@ mod tests { span.set_track(SpanTrack { sid: None, kind: TrackKind::Video, - source: "camera".into(), + source: TrackSource::Camera, remote_identity: None, }); span.set_track(SpanTrack { sid: Some("TR_1".into()), kind: TrackKind::Video, - source: "camera".into(), + source: TrackSource::Camera, remote_identity: None, }); let attributes = span.lock().attributes.clone(); diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index 4f4ba241a..a0d2e572c 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -18,10 +18,10 @@ use std::{ use livekit_telemetry::{ global::{self, TelemetryInstrument}, - Attribute, AttributeValue, DeviceEvent, DeviceState, ExportError, ExportRequest, - ExportResponse, LogRecord, NetTransport, RoomIdentity, RtcStatsSample, SpanName, SpanOutcome, - SpanStep, SpanTrack, TelemetryConfig, TelemetryEvent, TelemetryStats, TelemetryTransport, - TraceContext, + Attribute, AttributeValue, DeviceEvent, DeviceState, DisconnectReason, ExportError, + ExportRequest, ExportResponse, LogRecord, NetTransport, RoomIdentity, RtcStatsSample, SpanName, + SpanOutcome, SpanStep, SpanTrack, TelemetryConfig, TelemetryEvent, TelemetryStats, + TelemetryTransport, TraceContext, }; use tokio::sync::{mpsc, oneshot}; @@ -174,6 +174,11 @@ impl TelemetryScope { pub fn set_room(&self, room: RoomIdentity) { self.0.set_room(room); } + + /// The session ended for good (not a reconnect). + pub fn disconnected(&self, reason: DisconnectReason) { + self.0.disconnected(reason); + } } /// One export the host has to perform on behalf of a pulled pipeline. From d20d9c375212a7c89ad7e4e82a4413ea54cbc836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Tue, 15 Sep 2026 14:51:41 +0200 Subject: [PATCH 42/50] feat(telemetry): the core owns the subscribe span and the stats-report mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two things every SDK was about to implement three times now live once, in the core: - `lk.subscribe` is a state machine on the scope: `subscribe_started(track)` (intent), `subscribed(track)` (the server's confirmation), `subscribe_cancelled(sid)`, `subscribe_failed(sid, error_type)`. Its natural end is an RTC fact the core already sees — the first inbound stats reading with bytes — so `record_stats` closes it with `first_media`; 30 s without media ends it with `error.type = timed_out` (swept on activity), and `disconnected` cancels whatever is still open. - `record_stats_report(sid, kind, direction, Vec, ts)` takes one track's raw `getStats()` entries (type, id, standard W3C member names; nested maps flattened with a dot) and produces the samples: outbound streams tagged with their `rid`, codec resolved through `codecId`, RTT from the matching `remote-inbound-rtp` or the nominated candidate pair, seconds converted to milliseconds. `record_stats(RtcStatsSample)` stays for typed stats. - `DisconnectReason::from_proto` / `ReconnectReason::from_proto` (and the UniFFI free functions) replace per-SDK switches over the protocol numbers. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/SPEC.md | 9 +- livekit-telemetry/src/lib.rs | 2 +- livekit-telemetry/src/rtc.rs | 207 ++++++++++++++++++++++++++++- livekit-telemetry/src/scope.rs | 145 +++++++++++++++++++- livekit-telemetry/src/telemetry.rs | 88 ++++++++++++ livekit-telemetry/src/trace.rs | 13 ++ livekit-uniffi/src/telemetry.rs | 50 ++++++- 7 files changed, 504 insertions(+), 10 deletions(-) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index 843ff51ea..57d46e0af 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -346,9 +346,11 @@ span: lk.subscribe kind: internal starts: when the intent to subscribe exists — a remote publish under autoSubscribe, or the manual subscribe call -ends: at first media (the first stats reading with bytes received; 1 s granularity) → ok; +ends: at first media (the first inbound stats reading with bytes; the core sees it) → ok; unsubscribe / unpublish before media → cancelled; - subscription failure → error; no media within 30 s → error (error.type = LiveKitError.timedOut) + subscription failure → error; no media within 30 s → error (error.type = timed_out) +owner: the core (`Scope::subscribe_started / subscribed / subscribe_cancelled / subscribe_failed`); + an SDK only reports the remote track's lifecycle attributes: lk.track.sid: string lk.track.kind: enum(audio | video) @@ -376,6 +378,9 @@ scope `set_attribute`, and `Span::set_attribute` for app-defined spans. | `Span::describe()` | `lk.connect: ws_open +1.49s, signal +0.03s, total 1.83s, ok` — the console line, identical on every platform | | `Span::context()` | `TraceContext { trace_id, span_id }` for log correlation; `None` when detached | | `device_event(DeviceEvent::{AudioRouteChanged, AudioInterruption, CaptureFailed})` | `lk.device.audio_route.changed`, `lk.device.audio.interruption`, `lk.device.capture.failed` with display bodies; every value is a shared enum (`AudioOutput`, `CaptureDevice`, `CaptureFailure`) | +| `Scope::subscribe_started(SpanTrack)`, `subscribed(SpanTrack)`, `subscribe_cancelled(sid)`, `subscribe_failed(sid, error_type)` | the `lk.subscribe` span, ended by the core at the first inbound reading with bytes, or `timed_out` after 30 s, or cancelled at disconnect | +| `Scope::record_stats_report(sid, kind, direction, Vec, ts)` | one track's raw `getStats()` entries (type, id, standard members) → the core picks RTP streams, resolves codec / RTT, converts seconds to ms and records one `RtcStatsSample` per stream; `record_stats(RtcStatsSample)` stays for platforms with typed stats | +| `DisconnectReason::from_proto(i32)`, `ReconnectReason::from_proto(i32)` | the protocol numbers → the shared enums | | `Scope::disconnected(DisconnectReason)` | `lk.room.disconnected` with `lk.disconnect.reason` — info when the client hung up, warn otherwise | | `RtcStatsSample.layer` (rid, ssrc or stats id) | simulcast layers folded into one monotonic series per track before windowing | diff --git a/livekit-telemetry/src/lib.rs b/livekit-telemetry/src/lib.rs index 9d56df407..55c4f54d1 100644 --- a/livekit-telemetry/src/lib.rs +++ b/livekit-telemetry/src/lib.rs @@ -58,7 +58,7 @@ pub use cache::{BatchCache, FileCache, MemoryCache}; pub use device::*; pub use event::*; pub use exporter::Exporter; -pub use rtc::{RtcStatsSample, StreamDirection, TrackKind}; +pub use rtc::{RtcStat, RtcStatsSample, StreamDirection, TrackKind}; pub use scope::{DisconnectReason, RoomIdentity, Scope}; pub use span::SpanOutcome; pub use stats::{TelemetryStats, TelemetryStatus}; diff --git a/livekit-telemetry/src/rtc.rs b/livekit-telemetry/src/rtc.rs index 8840c2e37..1a46c1677 100644 --- a/livekit-telemetry/src/rtc.rs +++ b/livekit-telemetry/src/rtc.rs @@ -16,7 +16,9 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; use tokio::time::Instant; -use crate::{event::now_unix_nanos, scope::ScopeState, store::Queued, TelemetryEvent}; +use crate::{ + event::now_unix_nanos, scope::ScopeState, store::Queued, AttributeValue, TelemetryEvent, +}; #[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -101,6 +103,123 @@ pub struct RtcStatsSample { pub layer: Option, } +/// One entry of a WebRTC `RTCStatsReport`, as the platform got it: the entry's `type`, `id` and +/// its standard members (W3C webrtc-stats names; nested maps flattened with a dot, e.g. +/// `qualityLimitationDurations.cpu`). Numbers may arrive as `Int`, `Double` or numeric `Str`. +#[cfg_attr(feature = "uniffi", derive(uniffi::Record))] +#[derive(Debug, Clone, PartialEq)] +pub struct RtcStat { + pub kind: String, + pub id: String, + pub members: HashMap, +} + +fn num(stat: &RtcStat, key: &str) -> Option { + match stat.members.get(key)? { + AttributeValue::Int(i) => Some(*i as f64), + AttributeValue::Double(d) => Some(*d), + AttributeValue::Str(s) => s.parse().ok(), + AttributeValue::Bool(_) => None, + } +} + +fn count(stat: &RtcStat, key: &str) -> Option { + num(stat, key).map(|v| v.max(0.0) as u64) +} + +/// A duration member (seconds, per webrtc-stats) as whole milliseconds. +fn ms(stat: &RtcStat, key: &str) -> Option { + num(stat, key).map(|s| (s.max(0.0) * 1000.0) as u64) +} + +fn text(stat: &RtcStat, key: &str) -> Option { + match stat.members.get(key)? { + AttributeValue::Str(s) => Some(s.clone()), + _ => None, + } +} + +/// The samples in one track's `getStats()` report: one per `outbound-rtp` (tagged with its +/// layer) or `inbound-rtp` entry, codec resolved through `codecId`, RTT from the +/// `remote-inbound-rtp` that reports on the stream (outbound) or the nominated candidate pair +/// (inbound), durations converted from seconds to milliseconds. One mapping for every SDK. +pub(crate) fn samples_from_report( + track_sid: &str, + kind: TrackKind, + direction: StreamDirection, + report: &[RtcStat], + timestamp_ns: Option, +) -> Vec { + let codec = |stat: &RtcStat| { + text(stat, "codecId") + .and_then(|id| report.iter().find(|s| s.kind == "codec" && s.id == id)) + .and_then(|c| text(c, "mimeType")) + }; + let mut out = Vec::new(); + match direction { + StreamDirection::Outbound => { + for stat in report.iter().filter(|s| s.kind == "outbound-rtp") { + let mut sample = RtcStatsSample::new(track_sid, kind, direction); + sample.layer = Some(text(stat, "rid").unwrap_or_else(|| stat.id.clone())); + sample.codec = codec(stat); + sample.bytes = count(stat, "bytesSent"); + sample.packets = count(stat, "packetsSent"); + sample.frames_per_second = num(stat, "framesPerSecond"); + sample.quality_limitation_bandwidth_ms = + ms(stat, "qualityLimitationDurations.bandwidth"); + sample.quality_limitation_cpu_ms = ms(stat, "qualityLimitationDurations.cpu"); + sample.quality_limitation_other_ms = ms(stat, "qualityLimitationDurations.other"); + sample.rtt_ms = report + .iter() + .find(|s| { + s.kind == "remote-inbound-rtp" + && text(s, "localId").as_deref() == Some(stat.id.as_str()) + }) + .and_then(|r| num(r, "roundTripTime")) + .map(|s| s * 1000.0); + sample.timestamp_ns = timestamp_ns; + out.push(sample); + } + } + StreamDirection::Inbound => { + let rtt = report + .iter() + .find(|s| { + s.kind == "candidate-pair" + && (matches!(s.members.get("nominated"), Some(AttributeValue::Bool(true))) + || text(s, "state").as_deref() == Some("succeeded")) + }) + .and_then(|p| num(p, "currentRoundTripTime")) + .map(|s| s * 1000.0); + for stat in report.iter().filter(|s| s.kind == "inbound-rtp") { + let mut sample = RtcStatsSample::new(track_sid, kind, direction); + sample.codec = codec(stat); + sample.bytes = count(stat, "bytesReceived"); + sample.packets = count(stat, "packetsReceived"); + sample.packets_lost = count(stat, "packetsLost"); + sample.freeze_count = count(stat, "freezeCount"); + sample.freezes_duration_ms = ms(stat, "totalFreezesDuration"); + sample.pause_count = count(stat, "pauseCount"); + sample.pauses_duration_ms = ms(stat, "totalPausesDuration"); + sample.concealed_samples = count(stat, "concealedSamples"); + sample.silent_concealed_samples = count(stat, "silentConcealedSamples"); + sample.concealment_events = count(stat, "concealmentEvents"); + sample.interruption_count = count(stat, "interruptionCount"); + sample.interruptions_duration_ms = ms(stat, "totalInterruptionDuration"); + sample.jitter_buffer_delay_ms = ms(stat, "jitterBufferDelay"); + sample.jitter_buffer_emitted_count = count(stat, "jitterBufferEmittedCount"); + sample.jitter_ms = num(stat, "jitter").map(|s| s * 1000.0); + sample.frames_per_second = num(stat, "framesPerSecond"); + sample.audio_level = num(stat, "audioLevel"); + sample.rtt_ms = rtt; + sample.timestamp_ns = timestamp_ns; + out.push(sample); + } + } + } + out +} + impl RtcStatsSample { /// A sample with every optional field unset. pub fn new(track_sid: impl Into, kind: TrackKind, direction: StreamDirection) -> Self { @@ -534,4 +653,90 @@ mod tests { assert_eq!(attr(inbound, "lk.rtc.rtt_ms.avg"), None, "absent gauges are omitted"); assert_eq!(attr(&events[1], "lk.rtc.packets"), Some(AttributeValue::Int(7))); } + + #[test] + fn a_report_maps_to_samples_with_codec_rtt_layers_and_milliseconds() { + let stat = |kind: &str, id: &str, members: &[(&str, AttributeValue)]| RtcStat { + kind: kind.into(), + id: id.into(), + members: members.iter().map(|(k, v)| (k.to_string(), v.clone())).collect(), + }; + let s = |v: &str| AttributeValue::Str(v.into()); + let report = vec![ + stat("codec", "C1", &[("mimeType", s("video/VP8"))]), + stat( + "outbound-rtp", + "OUT_f", + &[ + ("rid", s("f")), + ("codecId", s("C1")), + ("bytesSent", AttributeValue::Int(7_000)), + ("packetsSent", s("70")), + ("framesPerSecond", AttributeValue::Double(29.5)), + ("qualityLimitationDurations.cpu", AttributeValue::Double(1.25)), + ], + ), + stat( + "outbound-rtp", + "OUT_h", + &[("rid", s("h")), ("bytesSent", AttributeValue::Int(500))], + ), + stat( + "remote-inbound-rtp", + "RI_f", + &[("localId", s("OUT_f")), ("roundTripTime", AttributeValue::Double(0.042))], + ), + ]; + let out = samples_from_report( + "TR_1", + TrackKind::Video, + StreamDirection::Outbound, + &report, + Some(5), + ); + assert_eq!(out.len(), 2); + let f = out.iter().find(|x| x.layer.as_deref() == Some("f")).expect("layer f"); + assert_eq!(f.codec.as_deref(), Some("video/VP8")); + assert_eq!((f.bytes, f.packets, f.frames_per_second), (Some(7_000), Some(70), Some(29.5))); + assert_eq!(f.quality_limitation_cpu_ms, Some(1_250)); + assert_eq!(f.rtt_ms, Some(42.0)); + assert_eq!(f.timestamp_ns, Some(5)); + assert_eq!( + out.iter().find(|x| x.layer.as_deref() == Some("h")).and_then(|x| x.rtt_ms), + None + ); + + let report = vec![ + stat( + "candidate-pair", + "CP", + &[ + ("nominated", AttributeValue::Bool(true)), + ("currentRoundTripTime", AttributeValue::Double(0.1)), + ], + ), + stat( + "inbound-rtp", + "IN", + &[ + ("bytesReceived", AttributeValue::Int(12)), + ("packetsLost", AttributeValue::Int(-3)), + ("jitter", AttributeValue::Double(0.02)), + ("totalFreezesDuration", AttributeValue::Double(2.5)), + ("jitterBufferDelay", AttributeValue::Double(3.0)), + ("audioLevel", AttributeValue::Double(0.5)), + ], + ), + ]; + let out = + samples_from_report("TR_2", TrackKind::Audio, StreamDirection::Inbound, &report, None); + assert_eq!(out.len(), 1); + let x = &out[0]; + assert_eq!((x.bytes, x.packets_lost, x.rtt_ms), (Some(12), Some(0), Some(100.0))); + assert_eq!( + (x.jitter_ms, x.freezes_duration_ms, x.jitter_buffer_delay_ms), + (Some(20.0), Some(2_500), Some(3_000)) + ); + assert_eq!(x.layer, None); + } } diff --git a/livekit-telemetry/src/scope.rs b/livekit-telemetry/src/scope.rs index a6810b0a7..2d7f9d5ae 100644 --- a/livekit-telemetry/src/scope.rs +++ b/livekit-telemetry/src/scope.rs @@ -13,17 +13,26 @@ // limitations under the License. use std::{ + collections::HashMap, fmt, sync::{Arc, Mutex}, + time::Duration, }; -use crate::{Attribute, AttributeValue, RtcStatsSample, Span, SpanName, Telemetry, TelemetryEvent}; +use tokio::time::Instant; + +use crate::{ + rtc::RtcStat, Attribute, AttributeValue, RtcStatsSample, Span, SpanName, SpanOutcome, SpanStep, + SpanTrack, StreamDirection, Telemetry, TelemetryEvent, TrackKind, +}; /// One session's identity: the trace id every one of its records carries, and the attributes /// attached to them at export time (`lk.room.sid`, `lk.participant.identity`, …). pub(crate) struct ScopeState { pub trace_id: [u8; 16], attributes: Mutex>, + /// Open `lk.subscribe` spans by track sid, from intent to first media. + subscribes: Mutex, Instant)>>, } impl ScopeState { @@ -33,7 +42,11 @@ impl ScopeState { } pub fn with_trace_id(trace_id: [u8; 16]) -> Arc { - Arc::new(Self { trace_id, attributes: Mutex::new(Vec::new()) }) + Arc::new(Self { + trace_id, + attributes: Mutex::new(Vec::new()), + subscribes: Mutex::new(HashMap::new()), + }) } /// The trace id as 32 hex characters. @@ -123,6 +136,31 @@ pub enum DisconnectReason { ReconnectFailed, } +impl DisconnectReason { + /// The protocol's `DisconnectReason` number, so no SDK keeps its own switch. + pub fn from_proto(value: i32) -> Self { + match value { + 1 => Self::ClientInitiated, + 2 => Self::DuplicateIdentity, + 3 => Self::ServerShutdown, + 4 => Self::ParticipantRemoved, + 5 => Self::RoomDeleted, + 6 => Self::StateMismatch, + 7 => Self::JoinFailure, + 8 => Self::Migration, + 9 => Self::SignalClose, + 10 => Self::RoomClosed, + 11 => Self::UserUnavailable, + 12 => Self::UserRejected, + 13 => Self::SipTrunkFailure, + 14 => Self::ConnectionTimeout, + 15 => Self::MediaFailure, + 16 => Self::AgentError, + _ => Self::Unknown, + } + } +} + #[derive(Clone)] pub struct Scope { pub(crate) telemetry: Telemetry, @@ -148,6 +186,11 @@ impl Scope { /// The session ended for good (not a reconnect): the `lk.room.disconnected` record. pub fn disconnected(&self, reason: DisconnectReason) { + let open: Vec<_> = + self.state.subscribes.lock().unwrap_or_else(|e| e.into_inner()).drain().collect(); + for (_, (span, _)) in open { + span.cancel(); + } let severity = if reason == DisconnectReason::ClientInitiated { crate::Severity::Info } else { @@ -167,11 +210,107 @@ impl Scope { self.state.set_attribute(key, value); } - /// Push one `getStats()` reading; its window ships under this session. + /// Push one `getStats()` reading; its window ships under this session. The first inbound + /// reading with bytes is a subscribed track's first media. pub fn record_stats(&self, sample: RtcStatsSample) { + self.sweep_subscribes(); + if sample.direction == StreamDirection::Inbound && sample.bytes.unwrap_or(0) > 0 { + self.first_media(&sample.track_sid); + } self.telemetry.record_stats_in(sample, &self.state); } + /// A whole `getStats()` report for one track, as the platform got it. The core picks the RTP + /// streams, resolves codec and RTT, converts units and records one sample per stream (outbound + /// ones tagged with their layer), so no SDK maps stats fields itself. + pub fn record_stats_report( + &self, + track_sid: &str, + kind: TrackKind, + direction: StreamDirection, + report: Vec, + timestamp_ns: Option, + ) { + for sample in + crate::rtc::samples_from_report(track_sid, kind, direction, &report, timestamp_ns) + { + self.record_stats(sample); + } + } + + /// No media within this long ends `lk.subscribe` with `error.type = timed_out`. + pub const SUBSCRIBE_TIMEOUT: Duration = Duration::from_secs(30); + + /// The intent to subscribe exists (autoSubscribe: at the remote publish; manual: at the + /// subscribe call): `lk.subscribe` opens, once per track. Its end is an RTC fact the core + /// sees itself — the first inbound reading with bytes — so no SDK keeps this state. + pub fn subscribe_started(&self, track: SpanTrack) { + self.sweep_subscribes(); + let Some(sid) = track.sid.clone() else { return }; + let mut open = self.state.subscribes.lock().unwrap_or_else(|e| e.into_inner()); + if open.contains_key(&sid) { + return; + } + let span = self.start(SpanName::Subscribe, None); + span.set_track(track); + open.insert(sid, (span, Instant::now())); + } + + /// The server confirmed the subscription: the `subscribed` step (opens the span for a manual + /// subscribe that had no earlier intent). + pub fn subscribed(&self, track: SpanTrack) { + self.subscribe_started(track.clone()); + let Some(sid) = &track.sid else { return }; + if let Some((span, _)) = + self.state.subscribes.lock().unwrap_or_else(|e| e.into_inner()).get(sid) + { + span.step(SpanStep::Subscribed); + } + } + + /// Unsubscribed or unpublished before media: cancelled. + pub fn subscribe_cancelled(&self, sid: &str) { + if let Some((span, _)) = self.take_subscribe(sid) { + span.cancel(); + } + } + + /// The subscription failed (`error.type` = the platform's error name). + pub fn subscribe_failed(&self, sid: &str, error_type: &str) { + if let Some((span, _)) = self.take_subscribe(sid) { + span.fail(error_type.to_owned()); + } + } + + fn first_media(&self, sid: &str) { + if let Some((span, _)) = self.take_subscribe(sid) { + span.step(SpanStep::FirstMedia); + span.end(SpanOutcome::Ok, None); + } + } + + fn take_subscribe(&self, sid: &str) -> Option<(Arc, Instant)> { + self.state.subscribes.lock().unwrap_or_else(|e| e.into_inner()).remove(sid) + } + + // ponytail: timeouts are swept on activity (stats readings arrive every 1–2 s while any media + // flows; a silent session sweeps at the next subscribe event or at disconnect), a timer if the + // 30 s must be exact. + fn sweep_subscribes(&self) { + let expired: Vec = self + .state + .subscribes + .lock() + .unwrap_or_else(|e| e.into_inner()) + .iter() + .filter(|(_, (_, since))| since.elapsed() >= Self::SUBSCRIBE_TIMEOUT) + .map(|(sid, _)| sid.clone()) + .collect(); + for sid in expired { + self.subscribe_failed(&sid, "timed_out"); + } + } + /// Open a span in this session's trace. /// Start a typed span in this session's trace, stamped now. `parent` nests it. pub fn start(&self, name: SpanName, parent: Option>) -> Arc { diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index fd8583da8..4c9e64f1a 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -760,6 +760,94 @@ mod tests { assert!(transport.sent().iter().any(|r| r.url.contains("traces")), "the span is exported"); } + fn exported_spans( + transport: &FakeTransport, + ) -> Vec { + transport + .sent() + .iter() + .filter(|r| r.url.contains("traces")) + .flat_map(|r| { + ExportTraceServiceRequest::decode(&gunzip(&r.body)[..]) + .expect("otlp") + .resource_spans + .into_iter() + .flat_map(|rs| rs.scope_spans.into_iter().flat_map(|ss| ss.spans)) + }) + .collect() + } + + #[tokio::test(start_paused = true)] + async fn the_subscribe_span_is_owned_by_the_core() { + use crate::{SpanTrack, TrackSource}; + let transport = FakeTransport::scripted([]); + let telemetry = pipeline(transport.clone()); + let session = telemetry.begin_scope(); + let track = |sid: &str| SpanTrack { + sid: Some(sid.into()), + kind: TrackKind::Video, + source: TrackSource::Camera, + remote_identity: Some("bob".into()), + }; + // Intent, confirmation, then the first inbound reading with bytes: ok. + session.subscribe_started(track("TR_a")); + session.subscribed(track("TR_a")); + let mut empty = RtcStatsSample::new("TR_a", TrackKind::Video, StreamDirection::Inbound); + empty.bytes = Some(0); + session.record_stats(empty); + let mut media = RtcStatsSample::new("TR_a", TrackKind::Video, StreamDirection::Inbound); + media.bytes = Some(1_500); + session.record_stats(media); + // A second one nobody hears from: timed out at the next sweep; a third: unpublished. + session.subscribe_started(track("TR_b")); + session.subscribe_started(track("TR_c")); + session.subscribe_cancelled("TR_c"); + tokio::time::advance(Scope::SUBSCRIBE_TIMEOUT + Duration::from_secs(1)).await; + session.subscribe_started(track("TR_d")); // any activity sweeps + telemetry.flush().await; + + let spans = exported_spans(&transport); + let by_sid = |sid: &str| { + spans + .iter() + .find(|s| { + s.attributes.iter().any(|kv| { + kv.key == "lk.track.sid" + && kv.value.as_ref().and_then(|v| v.value.clone()) + == Some(Value::StringValue(sid.into())) + }) + }) + .unwrap_or_else(|| panic!("span for {sid}")) + }; + let ok = by_sid("TR_a"); + assert_eq!(ok.name, "lk.subscribe"); + assert_eq!( + ok.events.iter().map(|e| e.name.as_str()).collect::>(), + ["subscribed", "first_media"] + ); + assert!(ok.attributes.iter().any(|kv| kv.key == "lk.outcome" + && kv.value.as_ref().and_then(|v| v.value.clone()) + == Some(Value::StringValue("ok".into())))); + let timed_out = by_sid("TR_b"); + assert_eq!( + timed_out.status.as_ref().map(|s| s.code), + Some(status::StatusCode::Error as i32) + ); + assert!(timed_out.attributes.iter().any(|kv| kv.key == "error.type" + && kv.value.as_ref().and_then(|v| v.value.clone()) + == Some(Value::StringValue("timed_out".into())))); + assert!( + spans.iter().filter(|s| s.name == "lk.subscribe").count() >= 3, + "cancelled exports too" + ); + assert!( + !spans.iter().any(|s| s.attributes.iter().any(|kv| kv.key == "lk.track.sid" + && kv.value.as_ref().and_then(|v| v.value.clone()) + == Some(Value::StringValue("TR_d".into())))), + "still open" + ); + } + #[tokio::test(start_paused = true)] async fn room_identity_and_resource_are_typed() { let transport = FakeTransport::scripted([]); diff --git a/livekit-telemetry/src/trace.rs b/livekit-telemetry/src/trace.rs index 26eb1a614..1bd75ccfd 100644 --- a/livekit-telemetry/src/trace.rs +++ b/livekit-telemetry/src/trace.rs @@ -35,6 +35,19 @@ pub enum ReconnectReason { Unknown, } +impl ReconnectReason { + /// The protocol's `ReconnectReason` number (`RR_*`). + pub fn from_proto(value: i32) -> Self { + match value { + 1 => Self::SignalDisconnected, + 2 => Self::PublisherFailed, + 3 => Self::SubscriberFailed, + 4 => Self::SwitchCandidate, + _ => Self::Unknown, + } + } +} + #[cfg_attr(feature = "uniffi", derive(uniffi::Enum))] #[derive(Debug, Clone, PartialEq, Eq)] pub enum SpanName { diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index a0d2e572c..9f149868c 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -19,9 +19,9 @@ use std::{ use livekit_telemetry::{ global::{self, TelemetryInstrument}, Attribute, AttributeValue, DeviceEvent, DeviceState, DisconnectReason, ExportError, - ExportRequest, ExportResponse, LogRecord, NetTransport, RoomIdentity, RtcStatsSample, SpanName, - SpanOutcome, SpanStep, SpanTrack, TelemetryConfig, TelemetryEvent, TelemetryStats, - TelemetryTransport, TraceContext, + ExportRequest, ExportResponse, LogRecord, NetTransport, ReconnectReason, RoomIdentity, RtcStat, + RtcStatsSample, SpanName, SpanOutcome, SpanStep, SpanTrack, StreamDirection, TelemetryConfig, + TelemetryEvent, TelemetryStats, TelemetryTransport, TraceContext, TrackKind, }; use tokio::sync::{mpsc, oneshot}; @@ -179,6 +179,50 @@ impl TelemetryScope { pub fn disconnected(&self, reason: DisconnectReason) { self.0.disconnected(reason); } + + /// One track's whole `getStats()` report; the core maps it (see `record_stats`). + pub fn record_stats_report( + &self, + track_sid: String, + kind: TrackKind, + direction: StreamDirection, + report: Vec, + timestamp_ns: Option, + ) { + self.0.record_stats_report(&track_sid, kind, direction, report, timestamp_ns); + } + + /// Intent to subscribe (autoSubscribe: the remote publish; manual: the call): opens `lk.subscribe`. + pub fn subscribe_started(&self, track: SpanTrack) { + self.0.subscribe_started(track); + } + + /// The server confirmed the subscription. + pub fn subscribed(&self, track: SpanTrack) { + self.0.subscribed(track); + } + + /// Unsubscribed or unpublished before media. + pub fn subscribe_cancelled(&self, sid: String) { + self.0.subscribe_cancelled(&sid); + } + + /// The subscription failed; `error_type` is the platform's error name. + pub fn subscribe_failed(&self, sid: String, error_type: String) { + self.0.subscribe_failed(&sid, &error_type); + } +} + +/// The protocol's `DisconnectReason` number as the shared enum. +#[uniffi::export] +pub fn telemetry_disconnect_reason(proto: i32) -> DisconnectReason { + DisconnectReason::from_proto(proto) +} + +/// The protocol's `ReconnectReason` number (`RR_*`) as the shared enum. +#[uniffi::export] +pub fn telemetry_reconnect_reason(proto: i32) -> ReconnectReason { + ReconnectReason::from_proto(proto) } /// One export the host has to perform on behalf of a pulled pipeline. From 4849d95319d87cde978b8d9cca500048174c3484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:07:49 +0200 Subject: [PATCH 43/50] fix(telemetry): 63-bit span ids, a closable pull queue, scope-level log records MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by the Flutter integration: - Span ids are 63-bit. uniffi-dart lifts a u64 ≥ 2^63 as a negative Dart int and refuses to lower it back, so half of all span ids could not be handed back in `LogRecord.span_id` and whole processes lost log ↔ span correlation. OTLP is happy with 8 bytes either way. - `TelemetryExportQueue::close()`: the queue outlives the pipeline because the host holds it, so `next()` never resolved `None` after shutdown and the serving loop (and its HTTP client) leaked. Closing drops the sender; `next` drains and resolves `None`. - `Scope::log(LogRecord)`: a record filed under a session without an ambient span, for platforms with no task-local context (Dart outside a zone). Same floor and filters. Co-Authored-By: Claude Fable 5.1 --- livekit-telemetry/SPEC.md | 1 + livekit-telemetry/src/scope.rs | 12 +++++++++-- livekit-telemetry/src/span.rs | 7 ++++--- livekit-telemetry/src/telemetry.rs | 14 ++++++++++--- livekit-uniffi/src/telemetry.rs | 21 +++++++++++++++---- .../support/dart/test/telemetry_test.dart | 3 +++ 6 files changed, 46 insertions(+), 12 deletions(-) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index 57d46e0af..bf429cbac 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -381,6 +381,7 @@ scope `set_attribute`, and `Span::set_attribute` for app-defined spans. | `Scope::subscribe_started(SpanTrack)`, `subscribed(SpanTrack)`, `subscribe_cancelled(sid)`, `subscribe_failed(sid, error_type)` | the `lk.subscribe` span, ended by the core at the first inbound reading with bytes, or `timed_out` after 30 s, or cancelled at disconnect | | `Scope::record_stats_report(sid, kind, direction, Vec, ts)` | one track's raw `getStats()` entries (type, id, standard members) → the core picks RTP streams, resolves codec / RTT, converts seconds to ms and records one `RtcStatsSample` per stream; `record_stats(RtcStatsSample)` stays for platforms with typed stats | | `DisconnectReason::from_proto(i32)`, `ReconnectReason::from_proto(i32)` | the protocol numbers → the shared enums | +| `Scope::log(LogRecord)` | a record filed under the session without an ambient span (Dart has no task-local outside a zone); same floor and filters as `Telemetry::log` | | `Scope::disconnected(DisconnectReason)` | `lk.room.disconnected` with `lk.disconnect.reason` — info when the client hung up, warn otherwise | | `RtcStatsSample.layer` (rid, ssrc or stats id) | simulcast layers folded into one monotonic series per track before windowing | diff --git a/livekit-telemetry/src/scope.rs b/livekit-telemetry/src/scope.rs index 2d7f9d5ae..3527aba1f 100644 --- a/livekit-telemetry/src/scope.rs +++ b/livekit-telemetry/src/scope.rs @@ -22,8 +22,8 @@ use std::{ use tokio::time::Instant; use crate::{ - rtc::RtcStat, Attribute, AttributeValue, RtcStatsSample, Span, SpanName, SpanOutcome, SpanStep, - SpanTrack, StreamDirection, Telemetry, TelemetryEvent, TrackKind, + rtc::RtcStat, Attribute, AttributeValue, LogRecord, RtcStatsSample, Span, SpanName, + SpanOutcome, SpanStep, SpanTrack, StreamDirection, Telemetry, TelemetryEvent, TrackKind, }; /// One session's identity: the trace id every one of its records carries, and the attributes @@ -184,6 +184,14 @@ impl Scope { self.emit(TelemetryEvent::custom(name, attributes)); } + /// A log record filed under this session even without an ambient span — for platforms with + /// no task-local context (Dart outside a zone). Same floor and filters as `Telemetry::log`. + pub fn log(&self, record: LogRecord) { + if let Some(event) = self.telemetry.log_event(record) { + self.emit(event); + } + } + /// The session ended for good (not a reconnect): the `lk.room.disconnected` record. pub fn disconnected(&self, reason: DisconnectReason) { let open: Vec<_> = diff --git a/livekit-telemetry/src/span.rs b/livekit-telemetry/src/span.rs index 6afc5a024..6d3b9cd68 100644 --- a/livekit-telemetry/src/span.rs +++ b/livekit-telemetry/src/span.rs @@ -115,8 +115,9 @@ impl Spans { sessions: HashMap::new(), session_order: VecDeque::new(), // Span ids must be non-zero (OTLP treats all-zero as absent); start at 1 and mix in - // randomness so ids from two pipelines in one process never collide. - next_id: rand::random::() | 1, + // randomness so ids from two pipelines in one process never collide. 63 bits: a + // platform whose integers are signed (Dart) must be able to hand an id back. + next_id: (rand::random::() >> 1) | 1, dropped: 0, } } @@ -130,7 +131,7 @@ impl Spans { session: Arc, ) -> u64 { let id = self.next_id; - self.next_id = self.next_id.wrapping_add(1).max(1); + self.next_id = ((self.next_id + 1) & (u64::MAX >> 1)).max(1); if self.open.len() >= MAX_OPEN_SPANS { if let Some(oldest) = self.open_order.pop_front() { self.open.remove(&oldest); diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 4c9e64f1a..516a5173f 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -396,19 +396,27 @@ impl Telemetry { /// floor; the core's own telemetry module never (a rejected batch that produced a record that /// produced a batch would never end). pub fn log(&self, record: LogRecord) { + if let Some(event) = self.log_event(record) { + self.emit(event); + } + } + + /// The record as an event, or nothing when it is below the floor (WebRTC: error only) or is + /// telemetry's own. + pub(crate) fn log_event(&self, record: LogRecord) -> Option { let floor = match record.source { LogSource::WebRtc => self.log_severity().max(Severity::Error), _ => self.log_severity(), }; if record.severity < floor { - return; + return None; } if record.source == LogSource::Ffi && record.logger.as_deref().is_some_and(|l| l.starts_with("livekit_telemetry")) { - return; + return None; } - self.emit(record.into()); + Some(record.into()) } pub fn emit(&self, event: TelemetryEvent) { diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index 9f149868c..6f389b433 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -180,6 +180,11 @@ impl TelemetryScope { self.0.disconnected(reason); } + /// A log record filed under this session even without an ambient span. + pub fn log(&self, record: LogRecord) { + self.0.log(record); + } + /// One track's whole `getStats()` report; the core maps it (see `record_stats`). pub fn record_stats_report( &self, @@ -302,7 +307,8 @@ impl TelemetrySpan { #[derive(uniffi::Object)] pub struct TelemetryExportQueue { - tx: mpsc::UnboundedSender, + /// `None` once closed: `next` then drains and resolves `None`. + tx: Mutex>>, rx: tokio::sync::Mutex>, inflight: Mutex>>>, seq: AtomicU64, @@ -314,14 +320,20 @@ impl TelemetryExportQueue { pub fn new() -> Arc { let (tx, rx) = mpsc::unbounded_channel(); Arc::new(Self { - tx, + tx: Mutex::new(Some(tx)), rx: tokio::sync::Mutex::new(rx), inflight: Mutex::new(HashMap::new()), seq: AtomicU64::new(0), }) } - /// The next request to perform. Resolves when one is queued; `None` once the pipeline is gone. + /// End the serving loop: `next` resolves `None` once the queue is drained. Call after + /// `telemetry_shutdown`, or when a later `telemetry_configure_pulled` replaced this queue. + pub fn close(&self) { + self.tx.lock().unwrap_or_else(|e| e.into_inner()).take(); + } + + /// The next request to perform. Resolves when one is queued; `None` once closed and drained. pub async fn next(&self) -> Option { let pending = self.rx.lock().await.recv().await?; self.inflight @@ -357,7 +369,8 @@ impl TelemetryTransport for TelemetryExportQueue { let id = self.seq.fetch_add(1, Ordering::Relaxed); let (done, wait) = oneshot::channel(); let pending = Pending { export: PendingExport { id, request }, done }; - if self.tx.send(pending).is_err() { + let tx = self.tx.lock().unwrap_or_else(|e| e.into_inner()).clone(); + if tx.is_none_or(|tx| tx.send(pending).is_err()) { return Err(ExportError::Retryable { reason: "export queue closed".into(), retry_after_ms: None, diff --git a/livekit-uniffi/support/dart/test/telemetry_test.dart b/livekit-uniffi/support/dart/test/telemetry_test.dart index 328bb473d..7a7758db1 100644 --- a/livekit-uniffi/support/dart/test/telemetry_test.dart +++ b/livekit-uniffi/support/dart/test/telemetry_test.dart @@ -59,6 +59,9 @@ void main() { await serving; expect(requests, hasLength(2)); expect(telemetryStats(), isNull); + // The queue outlives the pipeline (Dart holds it): closing it ends the serving loop. + queue.close(); + expect(await queue.next(), isNull); }); test('refuses to start without any transport', () { From bf04f9713ff9999c27fb709c1f2c8efad3fb67b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:12:04 +0200 Subject: [PATCH 44/50] fix(telemetry): the pull queue ends with finish(), not close() UniFFI's Kotlin objects already implement AutoCloseable.close(); a method of the same name is a conflicting overload in the generated code. Co-Authored-By: Claude Fable 5.1 --- livekit-uniffi/src/telemetry.rs | 13 +++++++------ .../support/dart/test/telemetry_test.dart | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/livekit-uniffi/src/telemetry.rs b/livekit-uniffi/src/telemetry.rs index 6f389b433..f18b19a23 100644 --- a/livekit-uniffi/src/telemetry.rs +++ b/livekit-uniffi/src/telemetry.rs @@ -307,7 +307,7 @@ impl TelemetrySpan { #[derive(uniffi::Object)] pub struct TelemetryExportQueue { - /// `None` once closed: `next` then drains and resolves `None`. + /// `None` once finished: `next` then drains and resolves `None`. tx: Mutex>>, rx: tokio::sync::Mutex>, inflight: Mutex>>>, @@ -329,11 +329,12 @@ impl TelemetryExportQueue { /// End the serving loop: `next` resolves `None` once the queue is drained. Call after /// `telemetry_shutdown`, or when a later `telemetry_configure_pulled` replaced this queue. - pub fn close(&self) { + /// (Not `close`: UniFFI's Kotlin objects already have `AutoCloseable.close`.) + pub fn finish(&self) { self.tx.lock().unwrap_or_else(|e| e.into_inner()).take(); } - /// The next request to perform. Resolves when one is queued; `None` once closed and drained. + /// The next request to perform. Resolves when one is queued; `None` once finished and drained. pub async fn next(&self) -> Option { let pending = self.rx.lock().await.recv().await?; self.inflight @@ -345,17 +346,17 @@ impl TelemetryExportQueue { /// The collector's answer to the request with `id`, whatever its status; the core classifies it. pub fn complete(&self, id: u64, response: ExportResponse) { - self.finish(id, Ok(response)); + self.settle(id, Ok(response)); } /// The request with `id` got no answer (network error, timeout, invalid URL). pub fn fail(&self, id: u64, error: ExportError) { - self.finish(id, Err(error)); + self.settle(id, Err(error)); } } impl TelemetryExportQueue { - fn finish(&self, id: u64, outcome: Result) { + fn settle(&self, id: u64, outcome: Result) { let done = self.inflight.lock().unwrap_or_else(|e| e.into_inner()).remove(&id); if let Some(done) = done { let _ = done.send(outcome); diff --git a/livekit-uniffi/support/dart/test/telemetry_test.dart b/livekit-uniffi/support/dart/test/telemetry_test.dart index 7a7758db1..c35175e21 100644 --- a/livekit-uniffi/support/dart/test/telemetry_test.dart +++ b/livekit-uniffi/support/dart/test/telemetry_test.dart @@ -59,8 +59,8 @@ void main() { await serving; expect(requests, hasLength(2)); expect(telemetryStats(), isNull); - // The queue outlives the pipeline (Dart holds it): closing it ends the serving loop. - queue.close(); + // The queue outlives the pipeline (Dart holds it): finishing it ends the serving loop. + queue.finish(); expect(await queue.next(), isNull); }); From 0f049ad14c78dd53becfbd8a5aac6a6de4c84ba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:25:27 +0200 Subject: [PATCH 45/50] fix(telemetry): the Cloud endpoint is the client OTLP route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Staging ingest lives under /observability/client/{logs,traces}/otlp/v0 — the routes without /client/ are the agent ones. Verified end to end against blaze-staging: 210 log records and every span shape reach ClickHouse. Co-Authored-By: Claude Opus 5 (1M context) --- livekit-telemetry/SPEC.md | 6 ++--- livekit-telemetry/src/telemetry.rs | 37 +++++++++++++++++++----------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index bf429cbac..916bfba4d 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -28,9 +28,9 @@ a scope — device state, pre-room errors, self-telemetry — belongs to the pip scope. Scopes are not ended: a room's last record is simply its last. The pipeline may start **without a destination** (`endpoint: None`): it buffers and caches, and -uploads nothing until `set_server(url, token)` (Cloud: `https:///observability/logs/otlp/v0`, +uploads nothing until `set_server(url, token)` (Cloud: `https:///observability/client/logs/otlp/v0`, `Authorization: Bearer `) or `set_destination(endpoint, headers)` — at the first connect, when the -server URL yields the endpoint (`https:///observability/logs/otlp/v0`) and the token the +server URL yields the endpoint (`https:///observability/client/logs/otlp/v0`) and the token the `Authorization` header. Calling it again (new token, new server) replaces the destination for the batches that follow. Waiting for a destination is not an upload hold: it is uncapped, bounded only by the cache. @@ -367,7 +367,7 @@ scope `set_attribute`, and `Span::set_attribute` for app-defined spans. | Platform calls | The core produces | |---|---| -| `set_server(url, token)` | `https:///observability/logs/otlp/v0`, `Authorization: Bearer `; no-op with an explicit `endpoint` | +| `set_server(url, token)` | `https:///observability/client/logs/otlp/v0`, `Authorization: Bearer `; no-op with an explicit `endpoint` | | `TelemetryConfig.sdk: TelemetryResource { sdk: Sdk, sdk_version, os_name, os_version, device_model }` | `service.name = livekit-client-`, `service.version`, `os.*`, `device.model.identifier`, plus `telemetry.sdk.*` | | `log(LogRecord { severity, source: LogSource, message, logger, function, file, line, timestamp_ns, span_id })` | a record with `code.function.name`, `code.file.path`, `code.line.number`, `lk.log.source`, `lk.log.logger`; the per-source floor (WebRTC at `error`, own module never) | | `Scope::set_room(RoomIdentity { sid, name, participant_sid, participant_identity })` | `lk.room.*`, `lk.participant.*` on every record of the scope | diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 516a5173f..9c81de59e 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -44,7 +44,7 @@ use crate::{DeviceEvent, Span, SpanName}; #[derive(Debug, Clone)] pub struct TelemetryConfig { /// Full OTLP/HTTP logs URL: `http://localhost:4318/v1/logs` locally, - /// `https:///observability/logs/otlp/v0` for LiveKit Cloud. `None` starts the + /// `https:///observability/client/logs/otlp/v0` for LiveKit Cloud. `None` starts the /// pipeline without a destination — it buffers (and caches) until /// [`Telemetry::set_destination`], typically at the first connect, when the server URL and /// the token are known. @@ -571,7 +571,7 @@ impl Telemetry { } } -/// `…/logs…` → `…/traces…`: covers `/v1/logs` and `/observability/logs/otlp/v0` alike. +/// `…/logs…` → `…/traces…`: covers `/v1/logs` and `/observability/client/logs/otlp/v0` alike. fn derive_traces_endpoint(logs_endpoint: &str) -> String { match logs_endpoint.rsplit_once("logs") { Some((before, after)) => format!("{before}traces{after}"), @@ -653,7 +653,7 @@ fn add_sdk_resource(resource: &mut Vec, sdk: Option<&TelemetryResourc } } -/// `wss://x.livekit.cloud/rtc?…` → `https://x.livekit.cloud/observability/logs/otlp/v0`; +/// `wss://x.livekit.cloud/rtc?…` → `https://x.livekit.cloud/observability/client/logs/otlp/v0`; /// `ws://`/`http://` stay plain http (dev servers). Host and port only; no path, query or userinfo. pub(crate) fn observability_endpoint(url: &str) -> Option { let (scheme, rest) = url.split_once("://")?; @@ -666,7 +666,7 @@ pub(crate) fn observability_endpoint(url: &str) -> Option { "ws" | "http" => "http", _ => "https", }; - Some(format!("{scheme}://{host}/observability/logs/otlp/v0")) + Some(format!("{scheme}://{host}/observability/client/logs/otlp/v0")) } #[cfg(test)] @@ -680,17 +680,20 @@ mod tests { let ep = observability_endpoint; assert_eq!( ep("wss://x.livekit.cloud").unwrap(), - "https://x.livekit.cloud/observability/logs/otlp/v0" + "https://x.livekit.cloud/observability/client/logs/otlp/v0" ); assert_eq!( ep("wss://x.livekit.cloud/rtc?access_token=t#f").unwrap(), - "https://x.livekit.cloud/observability/logs/otlp/v0" + "https://x.livekit.cloud/observability/client/logs/otlp/v0" ); assert_eq!( ep("ws://192.168.99.24:7880").unwrap(), - "http://192.168.99.24:7880/observability/logs/otlp/v0" + "http://192.168.99.24:7880/observability/client/logs/otlp/v0" + ); + assert_eq!( + ep("https://u:p@host").unwrap(), + "https://host/observability/client/logs/otlp/v0" ); - assert_eq!(ep("https://u:p@host").unwrap(), "https://host/observability/logs/otlp/v0"); assert_eq!(ep("nonsense"), None); assert_eq!(ep("wss:///rtc"), None); } @@ -1499,16 +1502,20 @@ mod tests { let mut headers = HashMap::new(); headers.insert("Authorization".to_owned(), "Bearer t".to_owned()); - telemetry.set_destination("https://x.livekit.cloud/observability/logs/otlp/v0", headers); + telemetry + .set_destination("https://x.livekit.cloud/observability/client/logs/otlp/v0", headers); tokio::time::sleep(Duration::from_millis(1)).await; let sent = transport.sent(); assert_eq!(sent.len(), 1, "cached batches ship as soon as the destination is known"); - assert_eq!(sent[0].url, "https://x.livekit.cloud/observability/logs/otlp/v0"); + assert_eq!(sent[0].url, "https://x.livekit.cloud/observability/client/logs/otlp/v0"); assert_eq!(sent[0].headers["Authorization"], "Bearer t"); let span = telemetry.begin_span("lk.publish", SpanKind::Internal, None); telemetry.end_span(span, SpanOutcome::Ok, None, Vec::new()); telemetry.flush().await; - assert_eq!(transport.sent()[1].url, "https://x.livekit.cloud/observability/traces/otlp/v0"); + assert_eq!( + transport.sent()[1].url, + "https://x.livekit.cloud/observability/client/traces/otlp/v0" + ); } #[tokio::test(start_paused = true)] @@ -1630,8 +1637,10 @@ mod tests { #[tokio::test(start_paused = true)] async fn spans_export_as_traces_under_the_session_trace_id() { let transport = FakeTransport::scripted([]); - let telemetry = - start(TelemetryConfig::new("http://c/observability/logs/otlp/v0"), transport.clone()); + let telemetry = start( + TelemetryConfig::new("http://c/observability/client/logs/otlp/v0"), + transport.clone(), + ); let connect = telemetry.begin_span("lk.connect", SpanKind::Client, None); telemetry.add_span_event(connect, "ws_open", vec![]); telemetry.emit( @@ -1653,7 +1662,7 @@ mod tests { let traces = sent.iter().find(|r| r.url.ends_with("/traces/otlp/v0")).expect("traces request"); assert_eq!( - traces.url, "http://c/observability/traces/otlp/v0", + traces.url, "http://c/observability/client/traces/otlp/v0", "derived from logs endpoint" ); let decoded = From 10f6a93da06c1228d32dd12654bb61e51bd3ca36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:04:04 +0200 Subject: [PATCH 46/50] fix(telemetry): a 429 holds uploads even when nobody says for how long MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloud's ingest answers an exceeded quota with a bare ResourceExhausted — no Retry-After, no RetryInfo (cloud-observability pkg/collector/service/error.go attaches RetryInfo only to non-fatal errors). Without a hint the exporter fell through to its generic retry path and sent the same batch three times in three seconds to the endpoint that had just asked for quiet, then logged it as a failure rather than as throttling. Co-Authored-By: Claude Opus 5 (1M context) --- livekit-telemetry/SPEC.md | 4 +++- livekit-telemetry/src/transport.rs | 26 +++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index 916bfba4d..6a009b716 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -192,7 +192,9 @@ existing ~1 s timer; the *window* stretches, not the reading. Uploads are shaped, not just batched: - **One request in flight**, oldest batch first; a failure pauses the cache for 60 s (throttling: - see `lk.telemetry.report`). + see `lk.telemetry.report`). A `429` pauses it without spending retries first — for the + `Retry-After` seconds, else `RetryInfo.retry_delay`, else that same minute (LiveKit Cloud's + quota answer names no delay). - **Budget:** at most `max_batches_per_upload` (default 4) cached batches per tick while a scope may be live, so a backlog (offline period, previous launch) replays at ~4 × 20 KB per 15 s ≈ 40 kbps next to a call. `shutdown` drains without the budget. diff --git a/livekit-telemetry/src/transport.rs b/livekit-telemetry/src/transport.rs index 75ac8c037..71127c20a 100644 --- a/livekit-telemetry/src/transport.rs +++ b/livekit-telemetry/src/transport.rs @@ -47,6 +47,10 @@ impl ExportResponse { } } +/// How long a 429 that names no delay holds uploads — the same minute the exporter waits after a +/// failure, chosen here so the classification alone carries the instruction. +const THROTTLE_DEFAULT_MS: u64 = 60_000; + /// Why a batch could not be delivered. Drives the exporter's retry / drop / go-silent decision /// (OTLP/HTTP failure semantics). Transports return only the `Retryable`/`Rejected` they can /// know without a response (network error, timeout, invalid URL); everything else comes from @@ -101,9 +105,15 @@ impl ExportError { .map(|seconds| seconds * 1000); let body = rpc.as_ref().and_then(retry_info_ms); match status { - 429 | 502 | 503 | 504 => { - Err(Self::Retryable { reason, retry_after_ms: header.or(body) }) - } + // A rate limit is an instruction to stop, so 429 always yields a wait: LiveKit Cloud + // answers a quota check with a bare `ResourceExhausted` — no `Retry-After`, no + // `RetryInfo` — and without one the exporter would spend its retries hammering the + // endpoint that just asked for quiet. + 429 => Err(Self::Retryable { + reason, + retry_after_ms: Some(header.or(body).unwrap_or(THROTTLE_DEFAULT_MS)), + }), + 502 | 503 | 504 => Err(Self::Retryable { reason, retry_after_ms: header.or(body) }), _ if body.is_some() => Err(Self::Retryable { reason, retry_after_ms: header.or(body) }), _ => Err(Self::Rejected { reason }), } @@ -256,6 +266,16 @@ mod tests { ExportError::from_response(&neither), Err(ExportError::Retryable { reason: "HTTP 503".into(), retry_after_ms: None }) ); + // What LiveKit Cloud actually answers over quota: ResourceExhausted, no header, no + // RetryInfo. A 429 is an instruction to stop, so it waits even when nobody said how long. + let bare = response(429, &[], status_body("QuotaStatusExceeded", None)); + assert_eq!( + ExportError::from_response(&bare), + Err(ExportError::Retryable { + reason: "HTTP 429: QuotaStatusExceeded".into(), + retry_after_ms: Some(60_000), + }) + ); } #[test] From 680088943079ab9fea100f01fb36b01b8d39756a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:38:37 +0200 Subject: [PATCH 47/50] feat(telemetry): a hold pauses uploads, not collection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records emitted inside a throttle window were thrown away, so the minute after a 429 — the minute an operator most wants to see — came back empty: a staging session lost 11 of ~200 records to a single hold. They now land in the write-ahead cache like any other batch and ship when the window lifts; only a collector that says "disabled" still discards work. `dropped.throttled` keeps its meaning by counting what the cache could not carry through a hold. Co-Authored-By: Claude Opus 5 (1M context) --- livekit-telemetry/SPEC.md | 6 ++++-- livekit-telemetry/src/exporter.rs | 25 +++++++++++++------------ livekit-telemetry/src/telemetry.rs | 10 ++++++---- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index 6a009b716..d622069de 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -65,7 +65,7 @@ attributes: lk.telemetry.dropped.cache_error: int # events lost because the cache could not store them (disk full) lk.telemetry.dropped.cache_full: int # events evicted from the cache by max_cache_bytes / max age lk.telemetry.dropped.rejected: int # events the collector rejected (4xx) - lk.telemetry.dropped.throttled: int # events dropped inside a Retry-After window + lk.telemetry.dropped.throttled: int # events the cache could not carry through a hold lk.telemetry.dropped.rate_limited: int # discrete events dropped by the flood guard cadence: appended to the next batch whenever a drop, an upload failure or a capped hold happened since the previous report — never its own request, never persisted on its own (Sentry @@ -194,7 +194,9 @@ Uploads are shaped, not just batched: - **One request in flight**, oldest batch first; a failure pauses the cache for 60 s (throttling: see `lk.telemetry.report`). A `429` pauses it without spending retries first — for the `Retry-After` seconds, else `RetryInfo.retry_delay`, else that same minute (LiveKit Cloud's - quota answer names no delay). + quota answer names no delay). A pause stops uploads, never collection: records keep landing in + the cache and ship when it lifts, so the quiet window is not a hole in the session. Only + `Disabled` throws work away. - **Budget:** at most `max_batches_per_upload` (default 4) cached batches per tick while a scope may be live, so a backlog (offline period, previous launch) replays at ~4 × 20 KB per 15 s ≈ 40 kbps next to a call. `shutdown` drains without the budget. diff --git a/livekit-telemetry/src/exporter.rs b/livekit-telemetry/src/exporter.rs index 545008854..ede835498 100644 --- a/livekit-telemetry/src/exporter.rs +++ b/livekit-telemetry/src/exporter.rs @@ -353,9 +353,8 @@ impl Exporter { self.config.max_batch_size.max(1) as usize, usize::try_from(self.config.max_batch_bytes.max(1)).unwrap_or(usize::MAX), ); - let throttled = self.throttled_until.is_some_and(|t| Instant::now() < t); // The shutdown summary goes out even with nothing else queued. - let report_due = self.force_report && !self.silenced && !throttled; + let report_due = self.force_report && !self.silenced; if batch.is_empty() && !report_due { return; } @@ -363,10 +362,6 @@ impl Exporter { Counters::add(&self.counters.disabled, batch.len() as u64); continue; } - if throttled { - Counters::add(&self.counters.throttled, batch.len() as u64); - continue; - } // Self-telemetry rides along with real data: never its own request, never its own // cadence, and only when there is something to report — plus once at shutdown. let now = self.counters.snapshot(); @@ -404,10 +399,11 @@ impl Exporter { if spans.is_empty() { return; } - if self.silenced || self.throttled_until.is_some_and(|t| Instant::now() < t) { - let counter = - if self.silenced { &self.counters.disabled } else { &self.counters.throttled }; - Counters::add(counter, spans.len() as u64); + // A throttle is a pause, not a verdict on the data: spans keep flowing into the + // write-ahead cache (bounded, oldest evicted first) and ship when the window ends. Only + // "telemetry is off for this project" throws work away. + if self.silenced { + Counters::add(&self.counters.disabled, spans.len() as u64); return; } let count = spans.len() as u64; @@ -424,9 +420,14 @@ impl Exporter { let id = format!("{:020}-{:06}-{count}-{}", now_unix_nanos(), self.seq, signal.tag()); match self.cache.push(&id, &body) { Ok(evicted) => { - // Older batches pushed out by `max_cache_bytes`: lost, but counted. + // Older batches pushed out by `max_cache_bytes`: lost, but counted. Losing them + // mid-hold is its own answer — the collector held us off longer than the cache + // could carry — so it is counted apart from an ordinary overflow. let lost: u64 = evicted.iter().map(|id| events_in(id)).sum(); - Counters::add(&self.counters.cache_full, lost); + let held = self.throttled_until.is_some_and(|t| Instant::now() < t); + let counter = + if held { &self.counters.throttled } else { &self.counters.cache_full }; + Counters::add(counter, lost); } Err(err) => { // A full disk is a steady state, not an event: warn once, then stay quiet. diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index 9c81de59e..cf38902ba 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -1161,7 +1161,7 @@ mod tests { } #[tokio::test(start_paused = true)] - async fn throttling_keeps_cached_batches_and_drops_new_ones() { + async fn throttling_holds_uploads_and_keeps_collecting() { let dir = temp_dir("throttle"); let throttled = Err(ExportError::Retryable { reason: "429".into(), retry_after_ms: Some(5_000) }); @@ -1173,13 +1173,15 @@ mod tests { assert_eq!(files_in(&dir), 1, "the throttled batch stays cached"); assert_eq!(telemetry.stats().dropped, 0); + // A hold pauses uploads, not collection: what happens during the quiet window is the + // part an operator most wants afterwards. telemetry.emit(TelemetryEvent::new("lk.ping")); telemetry.flush().await; - assert_eq!(telemetry.stats().dropped_throttled, 1, "new batches dropped inside the window"); - assert_eq!(files_in(&dir), 1, "and never written"); + assert_eq!(files_in(&dir), 2, "events inside the window are cached too"); + assert_eq!(telemetry.stats().dropped, 0, "and nothing is thrown away"); tokio::time::sleep(Duration::from_secs(6)).await; - assert_eq!(transport.sent().len(), 2, "cached batch uploaded after Retry-After"); + assert_eq!(transport.sent().len(), 3, "both cached batches upload after Retry-After"); assert_eq!(files_in(&dir), 0); let _ = fs::remove_dir_all(&dir); } From 1abc8e7b22c9088215415670a4a47d20b4e22939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:48:06 +0200 Subject: [PATCH 48/50] feat(telemetry): the cache is bounded by file count as well as bytes A day offline at a one-second cadence is thousands of tiny batches, not a few big ones, so `max_cache_bytes` alone left the directory to grow. Both caches now keep at most 512 batches, oldest evicted first like every other bound. Co-Authored-By: Claude Opus 5 (1M context) --- livekit-telemetry/SPEC.md | 6 ++++++ livekit-telemetry/src/cache.rs | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/livekit-telemetry/SPEC.md b/livekit-telemetry/SPEC.md index d622069de..81da3b3cd 100644 --- a/livekit-telemetry/SPEC.md +++ b/livekit-telemetry/SPEC.md @@ -215,6 +215,12 @@ Uploads are shaped, not just batched: than `max_batch_bytes` (1 MiB, estimated before compression) or `max_batch_size` (512) records; when the queue reaches `flush_threshold_bytes` (256 KiB) it is exported at once instead of at the next tick — "every 15 s or at 256 KB". +- **Backlog:** four nested bounds, every eviction counted in `lk.telemetry.report` — the queue + (`max_queue_size`, 2048 records), the cache's size (`max_cache_bytes`, 4 MiB compressed) and + its file count (512 batches, so days offline at a one-second cadence cannot fill a directory + with tiny files), and age (24 h, pruned at startup). Oldest goes first at every level. For + comparison: Sentry keeps 30 envelopes, Datadog 512 MB per feature with an 18 h age, and the + OTel batch processor caches nothing at all. - **Priority hints:** every request carries `Priority: u=7` (RFC 9218, lowest urgency) for HTTP/2+ hops that implement it, and the host transport marks the local traffic class as background — Apple `URLSessionConfiguration.networkServiceType = .background` diff --git a/livekit-telemetry/src/cache.rs b/livekit-telemetry/src/cache.rs index db3cf8b4c..19298474f 100644 --- a/livekit-telemetry/src/cache.rs +++ b/livekit-telemetry/src/cache.rs @@ -43,8 +43,15 @@ pub trait BatchCache: Send + Sync { fn clear(&self); } +/// Batches a cache will hold regardless of their size. `max_cache_bytes` is the size policy; +/// this only stops a long offline stretch at a 1 s cadence from leaving thousands of tiny files +/// in the directory. +// ponytail: a flat cap rather than another config knob; make it one if a host ever needs +// Datadog-scale buffering (512 MB per feature, where the file count matters). +pub(crate) const MAX_BATCHES: usize = 512; + /// In-memory [`BatchCache`]: batches that could not be uploaded wait for the next attempt, -/// bounded by `max_bytes` (oldest evicted). Lost with the process. +/// bounded by `max_bytes` and [`MAX_BATCHES`] (oldest evicted). Lost with the process. // ponytail: a Vec with remove(0) — a handful of small batches, and it shares Vec code the // binary already has instead of pulling in VecDeque's ring-buffer instantiations. pub struct MemoryCache { @@ -71,7 +78,7 @@ impl BatchCache for MemoryCache { batches.push((id.to_owned(), body.to_vec())); let mut total: usize = batches.iter().map(|(_, b)| b.len()).sum(); let mut evicted = Vec::new(); - while total > self.max_bytes && batches.len() > 1 { + while (total > self.max_bytes || batches.len() > MAX_BATCHES) && batches.len() > 1 { let (id, body) = batches.remove(0); total -= body.len(); evicted.push(id); @@ -149,7 +156,7 @@ impl FileCache { } /// Delete stray `.tmp` files and batches older than the max age, then the oldest batches - /// until the total fits `max_bytes`. Returns the ids of the batches removed. + /// until the rest fit `max_bytes` and [`MAX_BATCHES`]. Returns the ids of the batches removed. fn prune(&self) -> io::Result> { let now = crate::event::now_unix_nanos(); let mut removed = Vec::new(); @@ -169,12 +176,14 @@ impl FileCache { .map(|id| fs::metadata(self.path(id, EXT)).map(|m| m.len()).unwrap_or(0)) .collect(); let mut total: u64 = sizes.iter().sum(); + let mut count = kept.len(); for (id, len) in kept.iter().zip(sizes) { - if total <= self.max_bytes { + if total <= self.max_bytes && count <= MAX_BATCHES { break; } self.remove(id); total -= len; + count -= 1; removed.push(id.clone()); } Ok(removed) @@ -281,6 +290,23 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + /// Days offline at a one-second cadence are tiny batches, not big ones: the byte cap alone + /// would leave thousands of files in the directory. + #[test] + fn file_cache_caps_the_number_of_batches() { + let dir = temp_dir("count"); + let cache = FileCache::open(&dir, 1 << 20).expect("open"); + // `id` stamps the current second, so keep the ids rather than recomputing them. + let ids: Vec = (0..(MAX_BATCHES as u64 + 8)).map(id).collect(); + for batch in &ids { + cache.push(batch, b"x").expect("push"); + } + let kept = cache.pending(); + assert_eq!(kept.len(), MAX_BATCHES, "far under the byte cap, still bounded"); + assert_eq!(kept.first(), Some(&ids[8]), "the oldest went first"); + let _ = fs::remove_dir_all(&dir); + } + #[test] fn file_cache_recreates_a_purged_directory() { let dir = temp_dir("purged"); From b652a543a5b19610df67c0122fd1d73217dcc1d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:29:02 +0200 Subject: [PATCH 49/50] test(telemetry): the cache is the floor under a hold `dropped.throttled` changed meaning when holds started caching, and nothing exercised it any more: a hold that outlasts `max_cache_bytes` evicts oldest first and must say so, or a session comes back with an unexplained hole. Co-Authored-By: Claude Opus 5 (1M context) --- livekit-telemetry/src/telemetry.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/livekit-telemetry/src/telemetry.rs b/livekit-telemetry/src/telemetry.rs index cf38902ba..4a8f595a3 100644 --- a/livekit-telemetry/src/telemetry.rs +++ b/livekit-telemetry/src/telemetry.rs @@ -1186,6 +1186,33 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + /// The cache is the floor under a hold: outlast it and the oldest batches go, counted apart + /// from an ordinary overflow so the report says *why* the session has a hole. + #[tokio::test(start_paused = true)] + async fn a_hold_longer_than_the_cache_reports_what_it_cost() { + let dir = temp_dir("throttle-overflow"); + let throttled = + Err(ExportError::Retryable { reason: "429".into(), retry_after_ms: Some(60_000) }); + let transport = FakeTransport::scripted([throttled]); + let mut config = TelemetryConfig::new("http://collector/v1/logs"); + config.storage_dir = Some(dir.to_string_lossy().into_owned()); + config.max_cache_bytes = 700; // a couple of batches, so the hold overruns it quickly + let telemetry = start(config, transport.clone()); + + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + assert_eq!(telemetry.stats().dropped, 0, "the first batch is cached, not dropped"); + + for _ in 0..8 { + telemetry.emit(TelemetryEvent::new("lk.ping")); + telemetry.flush().await; + } + let stats = telemetry.stats(); + assert!(stats.dropped_throttled > 0, "evictions inside a hold are attributed to it"); + assert_eq!(stats.dropped, stats.dropped_throttled, "and to nothing else"); + let _ = fs::remove_dir_all(&dir); + } + #[tokio::test(start_paused = true)] async fn shutdown_offline_keeps_queue_on_disk() { let dir = temp_dir("spill"); From 4ff1ae8dc9520dbcec3814809183e084059eb2c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C5=82az=CC=87ej=20Pankowski?= <86720177+pblazej@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:54:17 +0200 Subject: [PATCH 50/50] test(telemetry): reach LiveKit Cloud from the Rust and Dart consumers Both could only ever talk to a plaintext local collector. The example needed two things to reach an https ingest: a TLS feature on livekit-net, which is opt-in so that a plaintext consumer pays nothing for it, and somewhere to put the bearer token the Cloud routes require. The Dart side gets the test its pull queue was written for: telemetry_test fakes the collector's answer, staging_ping_test performs the request, so the binding is exercised against a real ingest. Skipped unless LK_TELEMETRY_ENDPOINT is set. Co-Authored-By: Claude Opus 5 (1M context) --- examples/telemetry_ping/Cargo.toml | 2 +- examples/telemetry_ping/src/main.rs | 6 +- .../support/dart/test/staging_ping_test.dart | 92 +++++++++++++++++++ 3 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 livekit-uniffi/support/dart/test/staging_ping_test.dart diff --git a/examples/telemetry_ping/Cargo.toml b/examples/telemetry_ping/Cargo.toml index 051064286..dee6f7709 100644 --- a/examples/telemetry_ping/Cargo.toml +++ b/examples/telemetry_ping/Cargo.toml @@ -6,6 +6,6 @@ publish = false [dependencies] livekit-telemetry = { workspace = true, features = ["net"] } -livekit-net = { workspace = true, features = ["native"] } +livekit-net = { workspace = true, features = ["native", "rustls-tls-native-roots"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros"] } env_logger = { workspace = true } diff --git a/examples/telemetry_ping/src/main.rs b/examples/telemetry_ping/src/main.rs index 8b16fe840..21f5b1514 100644 --- a/examples/telemetry_ping/src/main.rs +++ b/examples/telemetry_ping/src/main.rs @@ -14,7 +14,7 @@ //! Sends a single `lk.ping` event to an OTLP/HTTP collector (default: local grafana/otel-lgtm). -use std::{env, sync::Arc}; +use std::{collections::HashMap, env, sync::Arc}; use livekit_telemetry::{Attribute, NetTransport, Telemetry, TelemetryConfig, TelemetryEvent}; @@ -29,6 +29,10 @@ async fn main() { config.resource.push(Attribute::new("os.name", env::consts::OS)); // Optional on-disk cache: run once with the collector down, once with it up. config.storage_dir = env::var("LK_TELEMETRY_DIR").ok(); + // LiveKit Cloud wants a bearer token carrying an `observability` write grant. + config.headers = env::var("LK_OTLP_TOKEN") + .ok() + .map(|token| HashMap::from([("Authorization".to_owned(), format!("Bearer {token}"))])); let transport = NetTransport::from_registry().expect("livekit-net has no HTTP client"); let (telemetry, exporter) = Telemetry::new(config, Arc::new(transport)); diff --git a/livekit-uniffi/support/dart/test/staging_ping_test.dart b/livekit-uniffi/support/dart/test/staging_ping_test.dart new file mode 100644 index 000000000..6bec080e4 --- /dev/null +++ b/livekit-uniffi/support/dart/test/staging_ping_test.dart @@ -0,0 +1,92 @@ +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:http/http.dart' as http; +import 'package:livekit_uniffi/livekit_telemetry.dart'; +import 'package:livekit_uniffi/livekit_uniffi.dart'; +import 'package:test/test.dart'; + +/// Posts one `lk.ping` to a real collector through the pull queue. +/// +/// Skipped unless `LK_TELEMETRY_ENDPOINT` (and, for LiveKit Cloud, `LK_TELEMETRY_TOKEN`) are set — +/// `source ~/livekit/telemetry-staging/env.sh` does that. Unlike `telemetry_test.dart`, which +/// fakes the collector's answer, this one performs the request so the Dart binding is exercised +/// against the ingest end to end. +/// +/// Dart drives the transport rather than implementing `TelemetryTransport`: uniffi-dart's +/// foreign-trait callbacks are isolate-bound, so Rust calling into Dart from a tokio worker aborts +/// the VM. The queue inverts that — Dart awaits `next()` and reports the answer with `complete()`. +Future serve(TelemetryExportQueue queue, List statuses, int count) async { + final client = http.Client(); + try { + for (var i = 0; i < count; i++) { + final pending = await queue.next(); + if (pending == null) return; + final request = pending.request; + try { + final response = await client.post( + Uri.parse(request.url), + headers: request.headers, + body: request.body, + ); + statuses.add(response.statusCode); + queue.complete( + id: pending.id, + response: ExportResponse( + status: response.statusCode, + headers: {}, + body: Uint8List.fromList(response.bodyBytes), + ), + ); + } catch (error) { + statuses.add(-1); + queue.fail( + id: pending.id, + error: RetryableExportException(reason: '$error', retryAfterMs: null), + ); + } + } + } finally { + client.close(); + } +} + +void main() { + final endpoint = Platform.environment['LK_TELEMETRY_ENDPOINT']; + final token = Platform.environment['LK_TELEMETRY_TOKEN']; + + test('posts lk.ping to a live collector through the pull queue', () async { + final statuses = []; + final queue = telemetryConfigurePulled( + config: TelemetryConfig( + endpoint: endpoint, + headers: token == null ? {} : {'Authorization': 'Bearer $token'}, + resource: [ + Attribute(key: 'service.name', value: StrAttributeValue('livekit-client-dart')), + Attribute(key: 'service.version', value: StrAttributeValue('0.1.9-staging')), + Attribute(key: 'os.name', value: StrAttributeValue(Platform.operatingSystem)), + ], + logSeverity: Severity.warn, + ), + instruments: [], + ); + final serving = serve(queue, statuses, 2); + + telemetryEmit( + event: TelemetryEvent( + name: 'lk.ping', + severity: Severity.info, + body: 'hello from dart', + attributes: [Attribute(key: 'lk.ping.seq', value: IntAttributeValue(1))], + ), + ); + await telemetryFlush(); + await telemetryShutdown(); + await serving; + queue.finish(); + + print('dart → $endpoint: statuses $statuses'); + expect(statuses, isNotEmpty); + expect(statuses.first, inInclusiveRange(200, 299)); + }, skip: endpoint == null ? 'set LK_TELEMETRY_ENDPOINT (see telemetry-staging/env.sh)' : null); +}