diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 62f44b6411..687407b386 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -656,6 +656,32 @@ jobs: cargo clippy --all-targets --all-features --workspace -- -D warnings working-directory: ./rust/${{ matrix.folder }} + host-metrics-semconv: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: true + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: open-telemetry/semantic-conventions + ref: v1.41.0 + path: semantic-conventions + - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9 + with: + toolchain: stable + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + workspaces: ./rust/otap-dataflow + - name: Run host metrics semconv drift check + env: + OTAP_HOST_METRICS_SEMCONV_REGISTRY: ${{ github.workspace }}/semantic-conventions/model + run: | + cargo test -p otap-df-core-nodes \ + --features dev-tools,otap-df-otap/crypto-ring \ + emitted_phase1_metric_shapes_match_weaver_semconv --lib -- --ignored + working-directory: ./rust/otap-dataflow + # Required matrix combinations for deny: otap-dataflow only deny_required: runs-on: ubuntu-latest @@ -804,6 +830,7 @@ jobs: - pest-fmt - no_default_features_check - pipeline_perf_test + - host-metrics-semconv steps: - name: Check if all required jobs succeeded run: | @@ -843,4 +870,8 @@ jobs: echo "pipeline_perf_test failed or was cancelled" exit 1 fi + if [[ "${{ needs.host-metrics-semconv.result }}" != "success" ]]; then + echo "host-metrics-semconv failed or was cancelled" + exit 1 + fi echo "All required checks passed!" diff --git a/rust/otap-dataflow/Cargo.toml b/rust/otap-dataflow/Cargo.toml index 9d68f51450..d4e1fc44bc 100644 --- a/rust/otap-dataflow/Cargo.toml +++ b/rust/otap-dataflow/Cargo.toml @@ -122,6 +122,7 @@ tikv-jemalloc-sys = "0.6.1" memchr = "2.8.0" memmap2 = "0.9" memory-stats = "1" +libc = "0.2" nix = { version = "0.31.0", features = ["process", "signal", "fs", "mman"] } notify = "8.0" # Uses platform-native backend: inotify (Linux), kqueue (macOS), ReadDirectoryChanges (Windows) num_enum = "0.7" diff --git a/rust/otap-dataflow/crates/core-nodes/Cargo.toml b/rust/otap-dataflow/crates/core-nodes/Cargo.toml index 2d75be447d..66e4c40607 100644 --- a/rust/otap-dataflow/crates/core-nodes/Cargo.toml +++ b/rust/otap-dataflow/crates/core-nodes/Cargo.toml @@ -42,6 +42,7 @@ object_store = {workspace = true, features = ["fs"]} parquet.workspace = true prost.workspace = true rand.workspace = true +regex.workspace = true serde.workspace = true serde_json.workspace = true slotmap.workspace = true @@ -65,6 +66,10 @@ weaver_resolved_schema = { workspace = true, optional = true } weaver_resolver = { workspace = true, optional = true } weaver_semconv = { workspace = true, optional = true } +[target.'cfg(target_os = "linux")'.dependencies] +libc.workspace = true +nix.workspace = true + [features] dev-tools = ["dep:weaver_common", "dep:weaver_forge", "dep:weaver_resolved_schema", "dep:weaver_resolver", "dep:weaver_semconv"] bench = [] diff --git a/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/README.md b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/README.md new file mode 100644 index 0000000000..051d886c84 --- /dev/null +++ b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/README.md @@ -0,0 +1,181 @@ +# Host Metrics Receiver + + + +**URN:** `urn:otel:receiver:host_metrics` + +Linux host metrics receiver backed by procfs and sysfs. It emits OpenTelemetry +`system.*` metrics for CPU, memory, paging, system uptime, disk, filesystem, +network, and aggregate process counts. + +## Configuration + +Minimal configuration: + +```yaml +groups: + host: + pipelines: + collect: + policies: + resources: + core_allocation: + type: core_count + count: 1 + nodes: + host_metrics: + type: receiver:host_metrics + config: + collection_interval: 10s + publish: + type: exporter:topic + config: + topic: host_metrics + connections: + - from: host_metrics + to: publish +``` + +Collect from a host root mounted into a container: + +```yaml +groups: + host: + pipelines: + collect: + policies: + resources: + core_allocation: + type: core_count + count: 1 + nodes: + host_metrics: + type: receiver:host_metrics + config: + collection_interval: 10s + host_view: + root_path: /host + validation: fail_selected + publish: + type: exporter:topic + config: + topic: host_metrics + connections: + - from: host_metrics + to: publish +``` + +Enable selected opt-in metrics: + +```yaml +groups: + host: + pipelines: + collect: + policies: + resources: + core_allocation: + type: core_count + count: 1 + nodes: + host_metrics: + type: receiver:host_metrics + config: + families: + cpu: + utilization: true + memory: + limit: true + hugepages: true + disk: + limit: true + filesystem: + limit: true + publish: + type: exporter:topic + config: + topic: host_metrics + connections: + - from: host_metrics + to: publish +``` + +## Configuration Options + +| Field | Type | Default | Description | +| ----- | ---- | ------- | ----------- | +| `collection_interval` | duration | `10s` | Default scrape interval. | +| `initial_delay` | duration | `0s` | Delay before the first scrape. | +| `host_view.root_path` | path | `/` | Host filesystem root to read procfs/sysfs from. | +| `host_view.validation` | enum | `fail_selected` | One of `fail_selected`, `warn_selected`, or `none`. | +| `families..enabled` | bool | `true` | Enables or disables a metric family. | +| `families..interval` | duration | unset | Per-family interval; falls back to `collection_interval`. | +| `families.cpu.utilization` | bool | `false` | Emits derived CPU utilization gauges. | +| `families.memory.limit` | bool | `false` | Emits `system.memory.limit`. | +| `families.memory.shared` | bool | `false` | Emits Linux shared memory. | +| `families.memory.hugepages` | bool | `false` | Emits Linux hugepage metrics. | +| `families.disk.limit` | bool | `false` | Emits disk capacity from sysfs. | +| `families.filesystem.limit` | bool | `false` | Emits filesystem capacity. | +| `families.filesystem.include_virtual_filesystems` | bool | `false` | Includes virtual filesystems such as tmpfs. | +| `families.filesystem.include_remote_filesystems` | bool | `false` | Includes remote and userspace filesystems such as NFS, CIFS, 9p, and FUSE. | + +Families are `cpu`, `memory`, `paging`, `system`, `disk`, `filesystem`, +`network`, and `processes`. + +Host-wide collection must run in a one-core source pipeline. Use a topic +exporter to fan out to multicore downstream processing when needed. + +## Filters + +Disk, filesystem, and network families support include and exclude filters. +Filter `match_type` values are `strict`, `glob`, and `regexp`. + +```yaml +groups: + host: + pipelines: + collect: + policies: + resources: + core_allocation: + type: core_count + count: 1 + nodes: + host_metrics: + type: receiver:host_metrics + config: + families: + disk: + exclude: + match_type: glob + devices: ["loop*", "ram*"] + network: + exclude: + match_type: strict + interfaces: ["lo"] + filesystem: + exclude_fs_types: + match_type: strict + fs_types: ["tmpfs", "proc", "sysfs"] + publish: + type: exporter:topic + config: + topic: host_metrics + connections: + - from: host_metrics + to: publish +``` + +## Current Limits + +- Linux only. +- Load metrics are not emitted in v1 because Semantic Conventions 1.41.0 does + not register a system load metric. +- `families.cpu.per_cpu` is rejected in v1. +- `families.network.include_connection_count` is rejected in v1. +- Process metrics are aggregate host summaries, not per-process scrapes. +- `system.process.count` emits the registered `process.state=running` summary. + Linux `procs_blocked` is parsed but not emitted because `blocked` is not a + registered `process.state` value. +- Filesystem collection can time out individual `statvfs` calls; avoid enabling + remote filesystems unless the host environment is known to be healthy. diff --git a/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/config.rs b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/config.rs new file mode 100644 index 0000000000..4508e55f79 --- /dev/null +++ b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/config.rs @@ -0,0 +1,843 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Configuration types for the host metrics receiver. + +use regex::RegexSet; +use serde::{Deserialize, Serialize}; +use std::path::{Component, Path, PathBuf}; +use std::time::Duration; + +fn default_collection_interval() -> Duration { + Duration::from_secs(10) +} + +fn default_root_path() -> PathBuf { + PathBuf::from("/") +} + +/// Configuration for the host metrics receiver. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Config { + /// Collection interval. + #[serde(default = "default_collection_interval", with = "humantime_serde")] + pub collection_interval: Duration, + + /// Delay before the first scrape. + #[serde(default, with = "humantime_serde")] + pub initial_delay: Duration, + + /// Optional legacy host root path. Prefer `host_view.root_path`. + #[serde(default)] + pub root_path: Option, + + /// Host filesystem view. + #[serde(default)] + pub host_view: HostViewConfig, + + /// Metric family configuration. + #[serde(default)] + pub families: FamiliesConfig, +} + +impl Default for Config { + fn default() -> Self { + Self { + collection_interval: default_collection_interval(), + initial_delay: Duration::ZERO, + root_path: None, + host_view: HostViewConfig::default(), + families: FamiliesConfig::default(), + } + } +} + +/// Host filesystem view configuration. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct HostViewConfig { + /// Root path for the observed host filesystem. + #[serde(default = "default_root_path")] + pub root_path: PathBuf, + /// Startup validation mode. + pub validation: HostViewValidationMode, +} + +impl Default for HostViewConfig { + fn default() -> Self { + Self { + root_path: default_root_path(), + validation: HostViewValidationMode::FailSelected, + } + } +} + +/// Host view startup validation mode. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum HostViewValidationMode { + /// Fail startup if selected sources are unavailable. + #[default] + FailSelected, + /// Start and disable unavailable selected sources. + WarnSelected, + /// Skip startup validation. + None, +} + +/// Metric family configuration. +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct FamiliesConfig { + /// CPU metrics. + pub cpu: CpuFamilyConfig, + /// Memory metrics. + pub memory: MemoryFamilyConfig, + /// Paging metrics. + pub paging: FamilyConfig, + /// System metrics. + pub system: FamilyConfig, + /// Disk metrics. + pub disk: DiskFamilyConfig, + /// Filesystem metrics. + pub filesystem: FilesystemFamilyConfig, + /// Network metrics. + pub network: NetworkFamilyConfig, + /// Process summary metrics. + pub processes: ProcessesFamilyConfig, +} + +impl FamiliesConfig { + fn enabled_count(&self) -> usize { + usize::from(self.cpu.enabled) + + usize::from(self.memory.enabled) + + usize::from(self.paging.enabled) + + usize::from(self.system.enabled) + + usize::from(self.disk.enabled) + + usize::from(self.filesystem.enabled) + + usize::from(self.network.enabled) + + usize::from(self.processes.enabled) + } +} + +/// CPU family config. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct CpuFamilyConfig { + /// Enable CPU metrics. + pub enabled: bool, + /// Family collection interval. Defaults to top-level `collection_interval`. + #[serde(default, with = "humantime_serde::option")] + pub interval: Option, + /// Emit per-logical-CPU time series. Not supported in v1. + pub per_cpu: bool, + /// Emit aggregate CPU utilization derived from CPU time deltas. + pub utilization: bool, +} + +impl Default for CpuFamilyConfig { + fn default() -> Self { + Self { + enabled: true, + interval: None, + per_cpu: false, + utilization: false, + } + } +} + +/// Common family config. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct FamilyConfig { + /// Enable this family. + pub enabled: bool, + /// Family collection interval. Defaults to top-level `collection_interval`. + #[serde(default, with = "humantime_serde::option")] + pub interval: Option, +} + +impl Default for FamilyConfig { + fn default() -> Self { + Self { + enabled: true, + interval: None, + } + } +} + +/// Memory family config. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct MemoryFamilyConfig { + /// Enable memory metrics. + pub enabled: bool, + /// Family collection interval. Defaults to top-level `collection_interval`. + #[serde(default, with = "humantime_serde::option")] + pub interval: Option, + /// Enable memory limit metrics. + pub limit: bool, + /// Enable Linux shared memory metric. + pub shared: bool, + /// Enable Linux hugepage metrics. + pub hugepages: bool, +} + +impl Default for MemoryFamilyConfig { + fn default() -> Self { + Self { + enabled: true, + interval: None, + limit: false, + shared: false, + hugepages: false, + } + } +} + +/// Disk family config. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct DiskFamilyConfig { + /// Enable disk metrics. + pub enabled: bool, + /// Family collection interval. Defaults to top-level `collection_interval`. + #[serde(default, with = "humantime_serde::option")] + pub interval: Option, + /// Enable disk limit metrics. + pub limit: bool, + /// Device include filter. + pub include: Option, + /// Device exclude filter. + pub exclude: Option, +} + +impl Default for DiskFamilyConfig { + fn default() -> Self { + Self { + enabled: true, + interval: None, + limit: false, + include: None, + exclude: None, + } + } +} + +/// Filesystem family config. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct FilesystemFamilyConfig { + /// Enable filesystem metrics. + pub enabled: bool, + /// Family collection interval. Defaults to top-level `collection_interval`. + #[serde(default, with = "humantime_serde::option")] + pub interval: Option, + /// Include virtual filesystems. + pub include_virtual_filesystems: bool, + /// Include remote and userspace filesystems. + pub include_remote_filesystems: bool, + /// Enable filesystem limit metrics. + pub limit: bool, + /// Device include filter. + pub include_devices: Option, + /// Device exclude filter. + pub exclude_devices: Option, + /// Filesystem type include filter. + pub include_fs_types: Option, + /// Filesystem type exclude filter. + pub exclude_fs_types: Option, + /// Mount point include filter. + pub include_mount_points: Option, + /// Mount point exclude filter. + pub exclude_mount_points: Option, +} + +impl Default for FilesystemFamilyConfig { + fn default() -> Self { + Self { + enabled: true, + interval: None, + include_virtual_filesystems: false, + include_remote_filesystems: false, + limit: false, + include_devices: None, + exclude_devices: None, + include_fs_types: None, + exclude_fs_types: None, + include_mount_points: None, + exclude_mount_points: None, + } + } +} + +/// Filesystem type filter config. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct FilesystemTypeFilterConfig { + /// Filesystem types. + pub fs_types: Vec, + /// Match type. + pub match_type: MatchType, +} + +impl Default for FilesystemTypeFilterConfig { + fn default() -> Self { + Self { + fs_types: Vec::new(), + match_type: MatchType::Strict, + } + } +} + +/// Mount point filter config. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct MountPointFilterConfig { + /// Mount points. + pub mount_points: Vec, + /// Match type. + pub match_type: MatchType, +} + +impl Default for MountPointFilterConfig { + fn default() -> Self { + Self { + mount_points: Vec::new(), + match_type: MatchType::Strict, + } + } +} + +/// Network family config. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct NetworkFamilyConfig { + /// Enable network metrics. + pub enabled: bool, + /// Family collection interval. Defaults to top-level `collection_interval`. + #[serde(default, with = "humantime_serde::option")] + pub interval: Option, + /// Interface include filter. + pub include: Option, + /// Interface exclude filter. + pub exclude: Option, + /// Connection count is not supported in v1. + pub include_connection_count: bool, +} + +impl Default for NetworkFamilyConfig { + fn default() -> Self { + Self { + enabled: true, + interval: None, + include: None, + exclude: None, + include_connection_count: false, + } + } +} + +/// Process family config. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields)] +pub struct ProcessesFamilyConfig { + /// Enable process summary metrics. + pub enabled: bool, + /// Family collection interval. Defaults to top-level `collection_interval`. + #[serde(default, with = "humantime_serde::option")] + pub interval: Option, + /// Only `summary` is supported in v1. + pub mode: ProcessMode, +} + +impl Default for ProcessesFamilyConfig { + fn default() -> Self { + Self { + enabled: true, + interval: None, + mode: ProcessMode::Summary, + } + } +} + +/// Process collection mode. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ProcessMode { + /// Aggregate host process summary. + #[default] + Summary, +} + +/// Disk device filter. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct DeviceFilterConfig { + /// Device names. + pub devices: Vec, + /// Match type. + #[serde(default)] + pub match_type: MatchType, +} + +/// Network interface filter. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct InterfaceFilterConfig { + /// Interface names. + pub interfaces: Vec, + /// Match type. + #[serde(default)] + pub match_type: MatchType, +} + +/// Filter match type. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MatchType { + /// Exact string match. + #[default] + Strict, + /// Glob match with `*` and `?`. + Glob, + /// Regular expression match. + Regexp, +} + +#[derive(Clone)] +pub(super) struct RuntimeConfig { + pub(super) root_path: PathBuf, + pub(super) validation: HostViewValidationMode, + pub(super) initial_delay: Duration, + pub(super) cpu_utilization: bool, + pub(super) memory_limit: bool, + pub(super) memory_shared: bool, + pub(super) memory_hugepages: bool, + pub(super) families: RuntimeFamilies, +} + +#[derive(Clone)] +pub(super) struct RuntimeFamilies { + pub(super) cpu: RuntimeFamily, + pub(super) memory: RuntimeFamily, + pub(super) paging: RuntimeFamily, + pub(super) system: RuntimeFamily, + pub(super) disk: RuntimeDiskFamily, + pub(super) filesystem: RuntimeFilesystemFamily, + pub(super) network: RuntimeNetworkFamily, + pub(super) processes: RuntimeFamily, +} + +#[derive(Clone)] +pub(super) struct RuntimeFamily { + pub(super) enabled: bool, + pub(super) interval: Duration, +} + +#[derive(Clone)] +pub(super) struct RuntimeDiskFamily { + pub(super) enabled: bool, + pub(super) interval: Duration, + pub(super) limit: bool, + pub(super) include: Option, + pub(super) exclude: Option, +} + +#[derive(Clone)] +pub(super) struct RuntimeFilesystemFamily { + pub(super) enabled: bool, + pub(super) interval: Duration, + pub(super) include_virtual_filesystems: bool, + pub(super) include_remote_filesystems: bool, + pub(super) limit: bool, + pub(super) include_devices: Option, + pub(super) exclude_devices: Option, + pub(super) include_fs_types: Option, + pub(super) exclude_fs_types: Option, + pub(super) include_mount_points: Option, + pub(super) exclude_mount_points: Option, +} + +#[derive(Clone)] +pub(super) struct RuntimeNetworkFamily { + pub(super) enabled: bool, + pub(super) interval: Duration, + pub(super) include: Option, + pub(super) exclude: Option, +} + +#[derive(Clone)] +pub(crate) struct CompiledFilter { + match_type: MatchType, + values: Vec, + regex_set: Option, +} + +impl CompiledFilter { + pub(super) fn compile( + match_type: MatchType, + values: Vec, + ) -> Result, otap_df_config::error::Error> { + if values.is_empty() { + return Ok(None); + } + let regex_set = if match_type == MatchType::Regexp { + Some(RegexSet::new(&values).map_err(|err| { + otap_df_config::error::Error::InvalidUserConfig { + error: format!("invalid host metrics regexp filter: {err}"), + } + })?) + } else { + None + }; + Ok(Some(Self { + match_type, + values, + regex_set, + })) + } + + pub(crate) fn matches(&self, value: &str) -> bool { + match self.match_type { + MatchType::Strict => self.values.iter().any(|candidate| candidate == value), + MatchType::Glob => self + .values + .iter() + .any(|candidate| glob_matches(candidate.as_bytes(), value.as_bytes())), + MatchType::Regexp => self + .regex_set + .as_ref() + .is_some_and(|regex_set| regex_set.is_match(value)), + } + } +} + +fn glob_matches(pattern: &[u8], value: &[u8]) -> bool { + let (mut p, mut v) = (0, 0); + let mut star = None; + let mut star_value = 0; + + while v < value.len() { + if p < pattern.len() && (pattern[p] == b'?' || pattern[p] == value[v]) { + p += 1; + v += 1; + } else if p < pattern.len() && pattern[p] == b'*' { + star = Some(p); + p += 1; + star_value = v; + } else if let Some(star_pos) = star { + p = star_pos + 1; + star_value += 1; + v = star_value; + } else { + return false; + } + } + + while p < pattern.len() && pattern[p] == b'*' { + p += 1; + } + p == pattern.len() +} + +pub(super) fn validate_config(config: &Config) -> Result<(), otap_df_config::error::Error> { + if config.collection_interval.is_zero() { + return Err(otap_df_config::error::Error::InvalidUserConfig { + error: "collection_interval must be greater than zero".to_owned(), + }); + } + if config.families.enabled_count() == 0 { + return Err(otap_df_config::error::Error::InvalidUserConfig { + error: "at least one host metrics family must be enabled".to_owned(), + }); + } + if config.families.network.include_connection_count { + return Err(otap_df_config::error::Error::InvalidUserConfig { + error: "network include_connection_count is not supported in v1".to_owned(), + }); + } + if config.families.cpu.per_cpu { + return Err(otap_df_config::error::Error::InvalidUserConfig { + error: "cpu per_cpu is not supported in v1".to_owned(), + }); + } + validate_family_interval( + "cpu", + config.families.cpu.enabled, + config.families.cpu.interval, + )?; + validate_family_interval( + "memory", + config.families.memory.enabled, + config.families.memory.interval, + )?; + validate_family_interval( + "paging", + config.families.paging.enabled, + config.families.paging.interval, + )?; + validate_family_interval( + "system", + config.families.system.enabled, + config.families.system.interval, + )?; + validate_family_interval( + "disk", + config.families.disk.enabled, + config.families.disk.interval, + )?; + validate_family_interval( + "filesystem", + config.families.filesystem.enabled, + config.families.filesystem.interval, + )?; + validate_family_interval( + "network", + config.families.network.enabled, + config.families.network.interval, + )?; + validate_family_interval( + "processes", + config.families.processes.enabled, + config.families.processes.interval, + )?; + let _ = normalized_root_path(Some(effective_root_path(config)?))?; + Ok(()) +} + +pub(super) fn effective_root_path(config: &Config) -> Result<&Path, otap_df_config::error::Error> { + if let Some(root_path) = config.root_path.as_deref() { + let host_view_root = config.host_view.root_path.as_path(); + if host_view_root != Path::new("/") && root_path != host_view_root { + return Err(otap_df_config::error::Error::InvalidUserConfig { + error: "root_path and host_view.root_path cannot both be set to different values" + .to_owned(), + }); + } + Ok(root_path) + } else { + Ok(config.host_view.root_path.as_path()) + } +} + +fn validate_family_interval( + family: &'static str, + enabled: bool, + interval: Option, +) -> Result<(), otap_df_config::error::Error> { + if enabled && interval.is_some_and(|interval| interval.is_zero()) { + return Err(otap_df_config::error::Error::InvalidUserConfig { + error: format!("{family} interval must be greater than zero"), + }); + } + Ok(()) +} + +impl TryFrom for RuntimeConfig { + type Error = otap_df_config::error::Error; + + fn try_from(config: Config) -> Result { + validate_config(&config)?; + let root_path = normalized_root_path(Some(effective_root_path(&config)?))?; + let disk_include = config + .families + .disk + .include + .map(|filter| CompiledFilter::compile(filter.match_type, filter.devices)) + .transpose()? + .flatten(); + let disk_exclude = config + .families + .disk + .exclude + .map(|filter| CompiledFilter::compile(filter.match_type, filter.devices)) + .transpose()? + .flatten(); + let filesystem_include_devices = config + .families + .filesystem + .include_devices + .map(|filter| CompiledFilter::compile(filter.match_type, filter.devices)) + .transpose()? + .flatten(); + let filesystem_exclude_devices = config + .families + .filesystem + .exclude_devices + .map(|filter| CompiledFilter::compile(filter.match_type, filter.devices)) + .transpose()? + .flatten(); + let filesystem_include_fs_types = config + .families + .filesystem + .include_fs_types + .map(|filter| CompiledFilter::compile(filter.match_type, filter.fs_types)) + .transpose()? + .flatten(); + let filesystem_exclude_fs_types = config + .families + .filesystem + .exclude_fs_types + .map(|filter| CompiledFilter::compile(filter.match_type, filter.fs_types)) + .transpose()? + .flatten(); + let filesystem_include_mount_points = config + .families + .filesystem + .include_mount_points + .map(|filter| CompiledFilter::compile(filter.match_type, filter.mount_points)) + .transpose()? + .flatten(); + let filesystem_exclude_mount_points = config + .families + .filesystem + .exclude_mount_points + .map(|filter| CompiledFilter::compile(filter.match_type, filter.mount_points)) + .transpose()? + .flatten(); + let network_include = config + .families + .network + .include + .map(|filter| CompiledFilter::compile(filter.match_type, filter.interfaces)) + .transpose()? + .flatten(); + let network_exclude = config + .families + .network + .exclude + .map(|filter| CompiledFilter::compile(filter.match_type, filter.interfaces)) + .transpose()? + .flatten(); + + Ok(Self { + root_path, + validation: config.host_view.validation, + initial_delay: config.initial_delay, + cpu_utilization: config.families.cpu.utilization, + memory_limit: config.families.memory.limit, + memory_shared: config.families.memory.shared, + memory_hugepages: config.families.memory.hugepages, + families: RuntimeFamilies { + cpu: RuntimeFamily::new_cpu(&config.families.cpu, config.collection_interval), + memory: RuntimeFamily::new_memory( + &config.families.memory, + config.collection_interval, + ), + paging: RuntimeFamily::new(&config.families.paging, config.collection_interval), + system: RuntimeFamily::new(&config.families.system, config.collection_interval), + disk: RuntimeDiskFamily { + enabled: config.families.disk.enabled, + interval: config + .families + .disk + .interval + .unwrap_or(config.collection_interval), + limit: config.families.disk.limit, + include: disk_include, + exclude: disk_exclude, + }, + filesystem: RuntimeFilesystemFamily { + enabled: config.families.filesystem.enabled, + interval: config + .families + .filesystem + .interval + .unwrap_or(config.collection_interval), + include_virtual_filesystems: config + .families + .filesystem + .include_virtual_filesystems, + include_remote_filesystems: config + .families + .filesystem + .include_remote_filesystems, + limit: config.families.filesystem.limit, + include_devices: filesystem_include_devices, + exclude_devices: filesystem_exclude_devices, + include_fs_types: filesystem_include_fs_types, + exclude_fs_types: filesystem_exclude_fs_types, + include_mount_points: filesystem_include_mount_points, + exclude_mount_points: filesystem_exclude_mount_points, + }, + network: RuntimeNetworkFamily { + enabled: config.families.network.enabled, + interval: config + .families + .network + .interval + .unwrap_or(config.collection_interval), + include: network_include, + exclude: network_exclude, + }, + processes: RuntimeFamily { + enabled: config.families.processes.enabled, + interval: config + .families + .processes + .interval + .unwrap_or(config.collection_interval), + }, + }, + }) + } +} + +impl RuntimeFamily { + fn new(config: &FamilyConfig, default_interval: Duration) -> Self { + Self { + enabled: config.enabled, + interval: config.interval.unwrap_or(default_interval), + } + } + + fn new_cpu(config: &CpuFamilyConfig, default_interval: Duration) -> Self { + Self { + enabled: config.enabled, + interval: config.interval.unwrap_or(default_interval), + } + } + + fn new_memory(config: &MemoryFamilyConfig, default_interval: Duration) -> Self { + Self { + enabled: config.enabled, + interval: config.interval.unwrap_or(default_interval), + } + } +} + +pub(super) fn normalized_root_path( + root_path: Option<&Path>, +) -> Result { + let path = root_path.unwrap_or_else(|| Path::new("/")); + let path_text = path.to_string_lossy(); + if !path.is_absolute() && !path_text.starts_with('/') { + return Err(otap_df_config::error::Error::InvalidUserConfig { + error: format!("root_path must be absolute: {}", path.display()), + }); + } + + let mut normalized = PathBuf::from("/"); + for component in path.components() { + match component { + Component::RootDir => {} + Component::Normal(part) => normalized.push(part), + Component::CurDir | Component::ParentDir => { + return Err(otap_df_config::error::Error::InvalidUserConfig { + error: format!( + "root_path must not contain . or .. components: {}", + path.display() + ), + }); + } + Component::Prefix(_) => { + return Err(otap_df_config::error::Error::InvalidUserConfig { + error: format!("root_path must be a Unix absolute path: {}", path.display()), + }); + } + } + } + Ok(normalized) +} diff --git a/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/mod.rs b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/mod.rs new file mode 100644 index 0000000000..e59c025cf6 --- /dev/null +++ b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/mod.rs @@ -0,0 +1,959 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Host metrics receiver. +//! +//! Most implementation is Linux-only (`#[cfg(target_os = "linux")]`). +//! Structs and filters defined here are dead on other platforms by design. +#![cfg_attr(not(target_os = "linux"), allow(dead_code))] + +#[cfg(target_os = "linux")] +use async_trait::async_trait; +use linkme::distributed_slice; +use otap_df_config::node::NodeUserConfig; +#[cfg(target_os = "linux")] +use otap_df_engine::MessageSourceLocalEffectHandlerExtension; +use otap_df_engine::ReceiverFactory; +use otap_df_engine::config::ReceiverConfig; +use otap_df_engine::context::PipelineContext; +#[cfg(target_os = "linux")] +use otap_df_engine::control::NodeControlMsg; +#[cfg(target_os = "linux")] +use otap_df_engine::error::{Error, ReceiverErrorKind, TypedError}; +#[cfg(target_os = "linux")] +use otap_df_engine::local::receiver as local; +use otap_df_engine::node::NodeId; +use otap_df_engine::receiver::ReceiverWrapper; +#[cfg(target_os = "linux")] +use otap_df_engine::terminal_state::TerminalState; +use otap_df_otap::OTAP_RECEIVER_FACTORIES; +#[cfg(target_os = "linux")] +use otap_df_otap::pdata::Context; +use otap_df_otap::pdata::OtapPdata; +use otap_df_telemetry::instrument::{Counter, Mmsc}; +use otap_df_telemetry::metrics::MetricSet; +#[cfg(target_os = "linux")] +use otap_df_telemetry::metrics::MetricSetSnapshot; +#[cfg(target_os = "linux")] +use otap_df_telemetry::{otel_info, otel_warn}; +use otap_df_telemetry_macros::metric_set; +#[cfg(target_os = "linux")] +use serde_json::Value; +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::{LazyLock, Mutex}; +#[cfg(any(target_os = "linux", test))] +use std::time::Duration; +#[cfg(target_os = "linux")] +use std::time::Instant as StdInstant; +#[cfg(target_os = "linux")] +use tokio::time::{Instant, sleep_until}; + +mod config; +#[cfg(target_os = "linux")] +mod otap_builder; +#[cfg(target_os = "linux")] +mod procfs; +#[cfg(target_os = "linux")] +mod semconv; + +#[cfg(target_os = "linux")] +use procfs::{HostSnapshot, ProcfsConfig, ProcfsFamilies, ProcfsSource}; + +#[cfg(any(target_os = "linux", test))] +pub(crate) use config::CompiledFilter; +use config::RuntimeConfig; +#[cfg(any(target_os = "linux", test))] +use config::validate_config; +pub use config::{ + Config, CpuFamilyConfig, DeviceFilterConfig, DiskFamilyConfig, FamiliesConfig, FamilyConfig, + FilesystemFamilyConfig, FilesystemTypeFilterConfig, HostViewConfig, HostViewValidationMode, + InterfaceFilterConfig, MatchType, MemoryFamilyConfig, MountPointFilterConfig, + NetworkFamilyConfig, ProcessMode, ProcessesFamilyConfig, +}; +#[cfg(target_os = "linux")] +use config::{RuntimeFamily, effective_root_path, normalized_root_path}; + +/// The URN for the host metrics receiver. +pub const HOST_METRICS_RECEIVER_URN: &str = "urn:otel:receiver:host_metrics"; + +/// Telemetry metrics for the host metrics receiver. +#[metric_set(name = "host_metrics.receiver.metrics")] +#[derive(Debug, Default, Clone)] +pub struct HostMetricsReceiverMetrics { + /// Number of scrape ticks started. + #[metric(unit = "{scrape}")] + pub scrapes_started: Counter, + /// Number of scrape ticks that built and sent a metrics batch. + #[metric(unit = "{scrape}")] + pub scrapes_completed: Counter, + /// Number of fatal scrape failures. + #[metric(unit = "{scrape}")] + pub scrapes_failed: Counter, + // TODO: Decide whether fixed per-family error counters are needed here. + // Metric-level attributes are not supported by the internal telemetry API today. + /// Number of source read errors skipped because other families succeeded. + #[metric(unit = "{error}")] + pub partial_errors: Counter, + /// Number of source read errors seen during scrapes. + #[metric(unit = "{error}")] + pub source_read_errors: Counter, + /// Number of due metric families processed. + #[metric(unit = "{family}")] + pub families_scraped: Counter, + /// Wall-clock scrape duration. + #[metric(unit = "ns")] + pub scrape_duration_ns: Mmsc, + /// Delay between scheduled and actual scrape start. + #[metric(unit = "ns")] + pub scrape_lag_ns: Mmsc, + /// Number of batches sent downstream. + #[metric(unit = "{batch}")] + pub batches_sent: Counter, + /// Number of downstream send failures. + #[metric(unit = "{error}")] + pub send_failures: Counter, +} + +/// Host metrics receiver. +pub struct HostMetricsReceiver { + config: RuntimeConfig, + _lease: HostMetricsLease, + metrics: Option>, +} + +#[allow(unsafe_code)] +#[distributed_slice(OTAP_RECEIVER_FACTORIES)] +/// Declares the host metrics receiver as a local receiver factory. +pub static HOST_METRICS_RECEIVER: ReceiverFactory = ReceiverFactory { + name: HOST_METRICS_RECEIVER_URN, + create: create_host_metrics_receiver, + wiring_contract: otap_df_engine::wiring_contract::WiringContract::UNRESTRICTED, + validate_config: validate_host_metrics_config, +}; + +#[cfg(target_os = "linux")] +fn create_host_metrics_receiver( + pipeline: PipelineContext, + node: NodeId, + node_config: Arc, + receiver_config: &ReceiverConfig, +) -> Result, otap_df_config::error::Error> { + if pipeline.num_cores() > 1 { + return Err(otap_df_config::error::Error::InvalidUserConfig { + error: "host-wide collection must run in a one-core source pipeline; use receiver:host_metrics -> exporter:topic and fan out downstream".to_owned(), + }); + } + let mut receiver = HostMetricsReceiver::from_config(&node_config.config)?; + receiver.metrics = Some(pipeline.register_metrics::()); + Ok(ReceiverWrapper::local( + receiver, + node, + node_config, + receiver_config, + )) +} + +#[cfg(not(target_os = "linux"))] +fn create_host_metrics_receiver( + _pipeline: PipelineContext, + _node: NodeId, + _node_config: Arc, + _receiver_config: &ReceiverConfig, +) -> Result, otap_df_config::error::Error> { + Err(unsupported_platform_error()) +} + +#[cfg(target_os = "linux")] +fn validate_host_metrics_config(config: &Value) -> Result<(), otap_df_config::error::Error> { + let config: Config = serde_json::from_value(config.clone()).map_err(|e| { + otap_df_config::error::Error::InvalidUserConfig { + error: e.to_string(), + } + })?; + RuntimeConfig::try_from(config).map(|_| ()) +} + +#[cfg(not(target_os = "linux"))] +fn validate_host_metrics_config( + _config: &serde_json::Value, +) -> Result<(), otap_df_config::error::Error> { + Err(unsupported_platform_error()) +} + +#[cfg(target_os = "linux")] +impl HostMetricsReceiver { + /// Creates a new host metrics receiver. + pub fn new(config: Config) -> Result { + let root_path = normalized_root_path(Some(effective_root_path(&config)?))?; + let lease = HostMetricsLease::acquire(root_path)?; + let config = RuntimeConfig::try_from(config)?; + Ok(Self { + config, + _lease: lease, + metrics: None, + }) + } + + /// Creates a host metrics receiver from JSON config. + pub fn from_config(config: &Value) -> Result { + let config: Config = serde_json::from_value(config.clone()).map_err(|e| { + otap_df_config::error::Error::InvalidUserConfig { + error: e.to_string(), + } + })?; + validate_config(&config)?; + Self::new(config) + } +} + +#[cfg(not(target_os = "linux"))] +fn unsupported_platform_error() -> otap_df_config::error::Error { + otap_df_config::error::Error::InvalidUserConfig { + error: "host_metrics receiver is supported only on Linux".to_owned(), + } +} + +#[cfg(target_os = "linux")] +fn duration_nanos(duration: Duration) -> f64 { + duration.as_secs_f64() * 1e9 +} + +#[cfg(target_os = "linux")] +fn elapsed_nanos(start: StdInstant) -> f64 { + duration_nanos(start.elapsed()) +} + +#[cfg(target_os = "linux")] +fn terminal_state( + deadline: StdInstant, + metrics: &Option>, +) -> TerminalState { + if let Some(metrics) = metrics { + TerminalState::new(deadline, [metrics.snapshot()]) + } else { + TerminalState::new::<[MetricSetSnapshot; 0]>(deadline, []) + } +} + +#[cfg(target_os = "linux")] +fn due_family_count(due: ProcfsFamilies) -> u64 { + u64::from(due.cpu) + + u64::from(due.memory) + + u64::from(due.paging) + + u64::from(due.system) + + u64::from(due.disk) + + u64::from(due.filesystem) + + u64::from(due.network) + + u64::from(due.processes) +} + +#[cfg(target_os = "linux")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ScheduledFamilyKind { + Cpu, + Memory, + Paging, + System, + Disk, + Filesystem, + Network, + Processes, +} + +#[cfg(target_os = "linux")] +struct ScheduledFamily { + kind: ScheduledFamilyKind, + interval: Duration, + next_due: Instant, +} + +#[cfg(target_os = "linux")] +struct FamilyScheduler { + entries: Vec, +} + +#[cfg(target_os = "linux")] +impl FamilyScheduler { + fn new(config: &RuntimeConfig, now: Instant) -> Self { + let first_due = now + config.initial_delay; + let mut entries = Vec::with_capacity(8); + push_scheduled( + &mut entries, + ScheduledFamilyKind::Cpu, + &config.families.cpu, + first_due, + ); + push_scheduled( + &mut entries, + ScheduledFamilyKind::Memory, + &config.families.memory, + first_due, + ); + push_scheduled( + &mut entries, + ScheduledFamilyKind::Paging, + &config.families.paging, + first_due, + ); + push_scheduled( + &mut entries, + ScheduledFamilyKind::System, + &config.families.system, + first_due, + ); + if config.families.disk.enabled { + entries.push(ScheduledFamily { + kind: ScheduledFamilyKind::Disk, + interval: config.families.disk.interval, + next_due: first_due, + }); + } + if config.families.filesystem.enabled { + entries.push(ScheduledFamily { + kind: ScheduledFamilyKind::Filesystem, + interval: config.families.filesystem.interval, + next_due: first_due, + }); + } + if config.families.network.enabled { + entries.push(ScheduledFamily { + kind: ScheduledFamilyKind::Network, + interval: config.families.network.interval, + next_due: first_due, + }); + } + push_scheduled( + &mut entries, + ScheduledFamilyKind::Processes, + &config.families.processes, + first_due, + ); + Self { entries } + } + + fn next_due(&self, now: Instant) -> Instant { + self.entries + .iter() + .map(|entry| entry.next_due) + .min() + .unwrap_or(now) + } + + fn mark_due(&mut self, now: Instant) -> ProcfsFamilies { + let mut due = ProcfsFamilies::default(); + for entry in &mut self.entries { + if entry.next_due <= now { + match entry.kind { + ScheduledFamilyKind::Cpu => due.cpu = true, + ScheduledFamilyKind::Memory => due.memory = true, + ScheduledFamilyKind::Paging => due.paging = true, + ScheduledFamilyKind::System => due.system = true, + ScheduledFamilyKind::Disk => due.disk = true, + ScheduledFamilyKind::Filesystem => due.filesystem = true, + ScheduledFamilyKind::Network => due.network = true, + ScheduledFamilyKind::Processes => due.processes = true, + } + let elapsed = now.duration_since(entry.next_due); + let missed_ticks = elapsed.as_nanos() / entry.interval.as_nanos() + 1; + let advance = entry + .interval + .saturating_mul(u32::try_from(missed_ticks).unwrap_or(u32::MAX)); + entry.next_due += advance; + if entry.next_due <= now { + entry.next_due = now + entry.interval; + } + } + } + due + } +} + +#[cfg(target_os = "linux")] +fn push_scheduled( + entries: &mut Vec, + kind: ScheduledFamilyKind, + family: &RuntimeFamily, + first_due: Instant, +) { + if family.enabled { + entries.push(ScheduledFamily { + kind, + interval: family.interval, + next_due: first_due, + }); + } +} + +static HOST_METRICS_LEASES: LazyLock>> = + LazyLock::new(|| Mutex::new(HashSet::new())); + +struct HostMetricsLease { + key: PathBuf, +} + +impl HostMetricsLease { + fn acquire(key: PathBuf) -> Result { + let mut leases = HOST_METRICS_LEASES.lock().map_err(|_| { + otap_df_config::error::Error::InvalidUserConfig { + error: "host metrics lease registry is unavailable".to_owned(), + } + })?; + if !leases.insert(key.clone()) { + return Err(otap_df_config::error::Error::InvalidUserConfig { + error: format!( + "another host_metrics receiver already collects host root {}", + key.display() + ), + }); + } + Ok(Self { key }) + } +} + +impl Drop for HostMetricsLease { + fn drop(&mut self) { + if let Ok(mut leases) = HOST_METRICS_LEASES.lock() { + let _ = leases.remove(&self.key); + } + } +} + +#[cfg(target_os = "linux")] +#[async_trait(?Send)] +impl local::Receiver for HostMetricsReceiver { + async fn start( + self: Box, + mut ctrl_msg_recv: local::ControlChannel, + effect_handler: local::EffectHandler, + ) -> Result { + let HostMetricsReceiver { + config, + _lease, + mut metrics, + } = *self; + let mut source = ProcfsSource::new( + Some(config.root_path.as_path()), + ProcfsConfig { + cpu: config.families.cpu.enabled, + memory: config.families.memory.enabled, + paging: config.families.paging.enabled, + system: config.families.system.enabled, + disk: config.families.disk.enabled, + filesystem: config.families.filesystem.enabled, + network: config.families.network.enabled, + processes: config.families.processes.enabled, + cpu_utilization: config.cpu_utilization, + memory_limit: config.memory_limit, + memory_shared: config.memory_shared, + memory_hugepages: config.memory_hugepages, + disk_limit: config.families.disk.limit, + filesystem_include_virtual: config.families.filesystem.include_virtual_filesystems, + filesystem_include_remote: config.families.filesystem.include_remote_filesystems, + filesystem_limit: config.families.filesystem.limit, + disk_include: config.families.disk.include.clone(), + disk_exclude: config.families.disk.exclude.clone(), + filesystem_include_devices: config.families.filesystem.include_devices.clone(), + filesystem_exclude_devices: config.families.filesystem.exclude_devices.clone(), + filesystem_include_fs_types: config.families.filesystem.include_fs_types.clone(), + filesystem_exclude_fs_types: config.families.filesystem.exclude_fs_types.clone(), + filesystem_include_mount_points: config + .families + .filesystem + .include_mount_points + .clone(), + filesystem_exclude_mount_points: config + .families + .filesystem + .exclude_mount_points + .clone(), + network_include: config.families.network.include.clone(), + network_exclude: config.families.network.exclude.clone(), + validation: config.validation, + }, + ) + .map_err(|err| Error::ReceiverError { + receiver: effect_handler.receiver_id(), + kind: ReceiverErrorKind::Configuration, + error: format!("failed to validate host metrics procfs sources: {err}"), + source_detail: String::new(), + })?; + let mut scheduler = FamilyScheduler::new(&config, Instant::now()); + + let _ = effect_handler + .start_periodic_telemetry(Duration::from_secs(1)) + .await?; + + loop { + tokio::select! { + biased; + + msg = ctrl_msg_recv.recv() => { + match msg { + Ok(NodeControlMsg::CollectTelemetry { mut metrics_reporter }) => { + if let Some(metrics) = metrics.as_mut() { + let _ = metrics_reporter.report(metrics); + } + } + Ok(NodeControlMsg::DrainIngress { deadline, .. }) => { + otel_info!("host_metrics_receiver.drain_ingress"); + effect_handler.notify_receiver_drained().await?; + return Ok(terminal_state(deadline, &metrics)); + } + Ok(NodeControlMsg::Shutdown { deadline, .. }) => { + otel_info!("host_metrics_receiver.shutdown"); + return Ok(terminal_state(deadline, &metrics)); + } + Err(e) => return Err(Error::ChannelRecvError(e)), + _ => {} + } + } + + _ = sleep_until(scheduler.next_due(Instant::now())) => { + let scheduled_due = scheduler.next_due(Instant::now()); + let now = Instant::now(); + let due = scheduler.mark_due(now); + let scrape_start = StdInstant::now(); + if let Some(metrics) = metrics.as_mut() { + metrics.scrapes_started.add(1); + metrics.families_scraped.add(due_family_count(due)); + metrics.scrape_lag_ns.record(duration_nanos(now.saturating_duration_since(scheduled_due))); + } + match source.scrape_due(due).await { + Ok(scrape) => { + if let Some(metrics) = metrics.as_mut() { + metrics.partial_errors.add(scrape.partial_errors); + metrics.source_read_errors.add(scrape.partial_errors); + } + tokio::task::consume_budget().await; + let pdata = match encode_snapshot(scrape.snapshot) { + Ok(pdata) => pdata, + Err(err) => { + if let Some(metrics) = metrics.as_mut() { + metrics.scrapes_failed.add(1); + metrics.scrape_duration_ns.record(elapsed_nanos(scrape_start)); + } + return Err(Error::ReceiverError { + receiver: effect_handler.receiver_id(), + kind: ReceiverErrorKind::Other, + error: format!("failed to encode host metrics: {err}"), + source_detail: String::new(), + }); + } + }; + tokio::task::consume_budget().await; + match effect_handler.try_send_message_with_source_node(pdata) { + Ok(()) => { + if let Some(metrics) = metrics.as_mut() { + metrics.batches_sent.add(1); + metrics.scrapes_completed.add(1); + metrics.scrape_duration_ns.record(elapsed_nanos(scrape_start)); + } + } + Err(TypedError::ChannelSendError(_)) => { + if let Some(metrics) = metrics.as_mut() { + metrics.send_failures.add(1); + metrics.scrape_duration_ns.record(elapsed_nanos(scrape_start)); + } + otel_warn!("host metrics dropped due to downstream backpressure"); + } + Err(other) => { + if let Some(metrics) = metrics.as_mut() { + metrics.send_failures.add(1); + metrics.scrapes_failed.add(1); + metrics.scrape_duration_ns.record(elapsed_nanos(scrape_start)); + } + return Err(Error::ReceiverError { + receiver: effect_handler.receiver_id(), + kind: ReceiverErrorKind::Other, + error: format!("failed to send host metrics: {other}"), + source_detail: String::new(), + }); + } + } + } + Err(err) => { + if let Some(metrics) = metrics.as_mut() { + metrics.scrapes_failed.add(1); + metrics.source_read_errors.add(1); + metrics.scrape_duration_ns.record(elapsed_nanos(scrape_start)); + } + otel_warn!( + "host metrics scrape failed; receiver will retry", + error = err.to_string() + ); + } + } + } + } + } + } +} + +#[cfg(target_os = "linux")] +fn encode_snapshot(snapshot: HostSnapshot) -> Result { + let records = snapshot.into_otap_records()?; + Ok(OtapPdata::new(Context::default(), records.into())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rejects_zero_collection_interval() { + let config = Config { + collection_interval: Duration::ZERO, + ..Config::default() + }; + + assert!(matches!( + validate_config(&config), + Err(otap_df_config::error::Error::InvalidUserConfig { .. }) + )); + } + + #[test] + fn rejects_all_families_disabled() { + let config = Config { + families: FamiliesConfig { + cpu: CpuFamilyConfig { + enabled: false, + ..CpuFamilyConfig::default() + }, + memory: MemoryFamilyConfig { + enabled: false, + ..MemoryFamilyConfig::default() + }, + paging: FamilyConfig { + enabled: false, + ..FamilyConfig::default() + }, + system: FamilyConfig { + enabled: false, + ..FamilyConfig::default() + }, + disk: DiskFamilyConfig { + enabled: false, + ..DiskFamilyConfig::default() + }, + filesystem: FilesystemFamilyConfig { + enabled: false, + ..FilesystemFamilyConfig::default() + }, + network: NetworkFamilyConfig { + enabled: false, + ..NetworkFamilyConfig::default() + }, + processes: ProcessesFamilyConfig { + enabled: false, + ..ProcessesFamilyConfig::default() + }, + }, + ..Config::default() + }; + + assert!(matches!( + validate_config(&config), + Err(otap_df_config::error::Error::InvalidUserConfig { .. }) + )); + } + + #[test] + fn rejects_v1_network_connection_counts() { + let config = Config { + families: FamiliesConfig { + network: NetworkFamilyConfig { + include_connection_count: true, + ..NetworkFamilyConfig::default() + }, + ..FamiliesConfig::default() + }, + ..Config::default() + }; + + assert!(matches!( + validate_config(&config), + Err(otap_df_config::error::Error::InvalidUserConfig { .. }) + )); + } + + #[test] + fn rejects_zero_enabled_family_interval() { + let config = Config { + families: FamiliesConfig { + cpu: CpuFamilyConfig { + interval: Some(Duration::ZERO), + ..CpuFamilyConfig::default() + }, + ..FamiliesConfig::default() + }, + ..Config::default() + }; + + assert!(matches!( + validate_config(&config), + Err(otap_df_config::error::Error::InvalidUserConfig { .. }) + )); + } + + #[test] + fn accepts_disabled_cpu_per_cpu_flag() { + let config: Config = serde_json::from_value(serde_json::json!({ + "families": { + "cpu": { + "per_cpu": false + } + } + })) + .expect("valid cpu config"); + + assert!(!config.families.cpu.per_cpu); + validate_config(&config).expect("valid config"); + } + + #[test] + fn accepts_cpu_utilization_opt_in() { + let config: Config = serde_json::from_value(serde_json::json!({ + "families": { + "cpu": { + "utilization": true + } + } + })) + .expect("valid cpu config"); + + assert!(config.families.cpu.utilization); + validate_config(&config).expect("valid config"); + } + + #[test] + fn accepts_memory_opt_ins() { + let config: Config = serde_json::from_value(serde_json::json!({ + "families": { + "memory": { + "limit": true, + "shared": true, + "hugepages": true + } + } + })) + .expect("valid memory config"); + + assert!(config.families.memory.limit); + assert!(config.families.memory.shared); + assert!(config.families.memory.hugepages); + validate_config(&config).expect("valid config"); + } + + #[test] + fn accepts_disk_limit_opt_in() { + let config: Config = serde_json::from_value(serde_json::json!({ + "families": { + "disk": { + "limit": true + } + } + })) + .expect("valid disk config"); + + assert!(config.families.disk.limit); + validate_config(&config).expect("valid config"); + } + + #[test] + fn accepts_filesystem_options() { + let config: Config = serde_json::from_value(serde_json::json!({ + "families": { + "filesystem": { + "interval": "5m", + "include_virtual_filesystems": true, + "include_remote_filesystems": true, + "limit": true, + "exclude_fs_types": { + "fs_types": ["tmpfs"], + "match_type": "strict" + }, + "include_mount_points": { + "mount_points": ["/", "/data*"], + "match_type": "glob" + } + } + } + })) + .expect("valid filesystem config"); + + assert_eq!( + config.families.filesystem.interval, + Some(Duration::from_secs(300)) + ); + assert!(config.families.filesystem.include_virtual_filesystems); + assert!(config.families.filesystem.include_remote_filesystems); + assert!(config.families.filesystem.limit); + assert_eq!( + config + .families + .filesystem + .exclude_fs_types + .as_ref() + .expect("exclude fs types") + .fs_types, + vec!["tmpfs".to_owned()] + ); + assert_eq!( + config + .families + .filesystem + .include_mount_points + .as_ref() + .expect("include mount points") + .mount_points, + vec!["/".to_owned(), "/data*".to_owned()] + ); + validate_config(&config).expect("valid config"); + } + + #[test] + fn rejects_v1_cpu_per_cpu() { + let config = Config { + families: FamiliesConfig { + cpu: CpuFamilyConfig { + per_cpu: true, + ..CpuFamilyConfig::default() + }, + ..FamiliesConfig::default() + }, + ..Config::default() + }; + + assert!(matches!( + validate_config(&config), + Err(otap_df_config::error::Error::InvalidUserConfig { .. }) + )); + } + + #[test] + fn duplicate_lease_rejects_same_root_until_drop() { + let root = PathBuf::from("/tmp/otap-host-metrics-lease-test"); + let lease = HostMetricsLease::acquire(root.clone()).expect("first lease"); + + assert!(matches!( + HostMetricsLease::acquire(root.clone()), + Err(otap_df_config::error::Error::InvalidUserConfig { .. }) + )); + + drop(lease); + let _lease = HostMetricsLease::acquire(root).expect("lease released on drop"); + } + + #[test] + fn rejects_conflicting_root_paths() { + let config = Config { + root_path: Some(PathBuf::from("/host")), + host_view: HostViewConfig { + root_path: PathBuf::from("/container"), + ..HostViewConfig::default() + }, + ..Config::default() + }; + + assert!(matches!( + validate_config(&config), + Err(otap_df_config::error::Error::InvalidUserConfig { .. }) + )); + } + + #[test] + fn accepts_host_view_validation_modes() { + let config: Config = serde_json::from_value(serde_json::json!({ + "host_view": { + "validation": "warn_selected" + } + })) + .expect("valid host view config"); + + assert_eq!( + config.host_view.validation, + HostViewValidationMode::WarnSelected + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn scheduler_honors_initial_delay_and_family_intervals() { + let config = Config { + collection_interval: Duration::from_secs(10), + initial_delay: Duration::from_secs(1), + families: FamiliesConfig { + cpu: CpuFamilyConfig { + interval: Some(Duration::from_secs(5)), + ..CpuFamilyConfig::default() + }, + ..FamiliesConfig::default() + }, + ..Config::default() + }; + let config = RuntimeConfig::try_from(config).expect("valid config"); + let now = Instant::now(); + let mut scheduler = FamilyScheduler::new(&config, now); + + assert_eq!(scheduler.next_due(now), now + Duration::from_secs(1)); + assert_eq!( + scheduler.mark_due(now), + ProcfsFamilies::default(), + "nothing is due before initial_delay" + ); + + let first_due = scheduler.mark_due(now + Duration::from_secs(1)); + assert!(first_due.cpu); + assert!(first_due.memory); + assert!(first_due.disk); + assert!(first_due.filesystem); + + let second_due = scheduler.mark_due(now + Duration::from_secs(6)); + assert!(second_due.cpu); + assert!(!second_due.memory); + assert!(!second_due.disk); + assert!(!second_due.filesystem); + } + + #[test] + fn glob_filter_matches_without_regex_allocations() { + let filter = CompiledFilter::compile(MatchType::Glob, vec!["loop*".to_owned()]) + .expect("valid filter") + .expect("non-empty filter"); + + assert!(filter.matches("loop0")); + assert!(!filter.matches("nvme0n1")); + } + + #[test] + fn regexp_filter_is_compiled_once() { + let filter = CompiledFilter::compile(MatchType::Regexp, vec![r"^eth[0-9]+$".to_owned()]) + .expect("valid filter") + .expect("non-empty filter"); + + assert!(filter.matches("eth0")); + assert!(!filter.matches("lo")); + } + + #[test] + fn factory_validation_rejects_invalid_regex_filter() { + let config = serde_json::json!({ + "families": { + "disk": { + "include": { + "devices": ["["], + "match_type": "regexp" + } + } + } + }); + + assert!(matches!( + (HOST_METRICS_RECEIVER.validate_config)(&config), + Err(otap_df_config::error::Error::InvalidUserConfig { .. }) + )); + } +} diff --git a/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/otap_builder.rs b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/otap_builder.rs new file mode 100644 index 0000000000..67976e5a9f --- /dev/null +++ b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/otap_builder.rs @@ -0,0 +1,348 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Direct OTAP Arrow record construction for host metrics. +//! +//! Builds `OtapArrowRecords::Metrics` without constructing intermediate OTLP +//! protobuf objects. All metric names, units, and attribute keys are written +//! directly into Arrow column buffers. + +use arrow::error::ArrowError; +use otap_df_pdata::encode::record::attributes::StrKeysAttributesRecordBatchBuilder; +use otap_df_pdata::encode::record::metrics::{ + MetricsRecordBatchBuilder, NumberDataPointsRecordBatchBuilder, +}; +use otap_df_pdata::otap::{Metrics, OtapArrowRecords}; +use otap_df_pdata::otlp::metrics::MetricType; +use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; + +#[cfg(target_os = "linux")] +use crate::receivers::host_metrics_receiver::procfs::HostResource; +use crate::receivers::host_metrics_receiver::semconv; + +const SCOPE_NAME: &[u8] = b"otap-df-core-nodes/host-metrics"; +const SCOPE_VERSION: &[u8] = env!("CARGO_PKG_VERSION").as_bytes(); + +// AggregationTemporality::Cumulative = 2 (OTLP proto enum value). +const AGGREGATION_TEMPORALITY_CUMULATIVE: i32 = 2; + +#[derive(Clone, Copy)] +pub(crate) struct MetricHandle(u16); + +/// Wraps the per-datapoint attribute builder and hides the dp_id from callers. +pub(crate) struct DpAttrWriter<'a> { + attrs: &'a mut StrKeysAttributesRecordBatchBuilder, + dp_id: u32, +} + +impl DpAttrWriter<'_> { + /// Append a string-valued attribute. + pub fn str(&mut self, key: &'static str, value: &str) { + self.attrs.append_parent_id(&self.dp_id); + self.attrs.append_key(key); + self.attrs.any_values_builder.append_str(value.as_bytes()); + } + + /// Append an integer-valued attribute. + pub fn int(&mut self, key: &'static str, value: i64) { + self.attrs.append_parent_id(&self.dp_id); + self.attrs.append_key(key); + self.attrs.any_values_builder.append_int(value); + } +} + +/// Wraps the resource attribute builder and hides the resource_id from callers. +pub(crate) struct ResourceAttrWriter<'a> { + attrs: &'a mut StrKeysAttributesRecordBatchBuilder, +} + +impl ResourceAttrWriter<'_> { + fn str(&mut self, key: &'static str, value: &str) { + self.attrs.append_parent_id(&0u16); + self.attrs.append_key(key); + self.attrs.any_values_builder.append_str(value.as_bytes()); + } +} + +/// Builds an `OtapArrowRecords::Metrics` batch directly from host metric values. +/// +/// Call a `begin_*` method to open a metric, then one of the typed data point +/// appenders for each value. +/// Call [`finish`] to produce the final batch. +pub(crate) struct HostMetricsArrowBuilder { + metrics: MetricsRecordBatchBuilder, + ndp: NumberDataPointsRecordBatchBuilder, + resource_attrs: StrKeysAttributesRecordBatchBuilder, + ndp_attrs: StrKeysAttributesRecordBatchBuilder, + curr_metric_id: u16, + curr_dp_id: u32, + resource_appended: bool, +} + +impl HostMetricsArrowBuilder { + pub(crate) fn new() -> Self { + Self { + metrics: MetricsRecordBatchBuilder::new(), + ndp: NumberDataPointsRecordBatchBuilder::new(), + resource_attrs: StrKeysAttributesRecordBatchBuilder::new(), + ndp_attrs: StrKeysAttributesRecordBatchBuilder::new(), + curr_metric_id: 0, + curr_dp_id: 0, + resource_appended: false, + } + } + + /// Append resource attributes (host.id, host.name, host.arch, os.type). + /// Must be called exactly once per batch before any metrics are appended. + #[cfg(target_os = "linux")] + pub(crate) fn append_resource(&mut self, resource: &HostResource) { + debug_assert!(!self.resource_appended, "resource already appended"); + self.resource_appended = true; + let mut w = ResourceAttrWriter { + attrs: &mut self.resource_attrs, + }; + w.str(semconv::attr::OS_TYPE, "linux"); + if let Some(id) = &resource.host_id { + w.str(semconv::attr::HOST_ID, id); + } + if let Some(name) = &resource.host_name { + w.str(semconv::attr::HOST_NAME, name); + } + if let Some(arch) = resource.host_arch { + w.str(semconv::attr::HOST_ARCH, arch); + } + } + + // ── Metric openers ────────────────────────────────────────────────────── + + /// Open a monotonic cumulative Sum metric (i64 data points). + pub(crate) fn begin_counter_i64(&mut self, name: &str, unit: &str) -> MetricHandle { + self.begin_metric(name, unit, MetricType::Sum, true) + } + + /// Open a monotonic cumulative Sum metric (f64 data points). + pub(crate) fn begin_counter_f64(&mut self, name: &str, unit: &str) -> MetricHandle { + self.begin_metric(name, unit, MetricType::Sum, true) + } + + /// Open a non-monotonic cumulative Sum metric / UpDownCounter (i64 data points). + pub(crate) fn begin_updown_i64(&mut self, name: &str, unit: &str) -> MetricHandle { + self.begin_metric(name, unit, MetricType::Sum, false) + } + + /// Open a Gauge metric (f64 data points). + pub(crate) fn begin_gauge_f64(&mut self, name: &str, unit: &str) -> MetricHandle { + self.begin_metric(name, unit, MetricType::Gauge, false) + } + + /// Open a Gauge metric (i64 data points). + pub(crate) fn begin_gauge_i64(&mut self, name: &str, unit: &str) -> MetricHandle { + self.begin_metric(name, unit, MetricType::Gauge, false) + } + + fn begin_metric( + &mut self, + name: &str, + unit: &str, + metric_type: MetricType, + is_monotonic: bool, + ) -> MetricHandle { + let id = self.curr_metric_id; + self.metrics.append_id(id); + self.metrics.append_metric_type(metric_type as u8); + self.metrics.append_name(name.as_bytes()); + self.metrics.append_description(&[]); + self.metrics.append_unit(unit.as_bytes()); + match metric_type { + MetricType::Sum => { + self.metrics + .append_aggregation_temporality(Some(AGGREGATION_TEMPORALITY_CUMULATIVE)); + self.metrics.append_is_monotonic(Some(is_monotonic)); + } + _ => { + self.metrics.append_aggregation_temporality(None); + self.metrics.append_is_monotonic(None); + } + } + self.curr_metric_id = self + .curr_metric_id + .checked_add(1) + .expect("metric_id overflow: more than u16::MAX metrics in one batch"); + MetricHandle(id) + } + + // ── Datapoint appenders ───────────────────────────────────────────────── + + /// Append one i64 Sum data point. + pub(crate) fn append_i64_sum_dp( + &mut self, + metric: MetricHandle, + start: u64, + now: u64, + value: i64, + attrs: F, + ) where + F: FnOnce(&mut DpAttrWriter<'_>), + { + self.append_i64_dp(metric, Some(start), now, value, attrs); + } + + /// Append one i64 Gauge data point. + pub(crate) fn append_i64_gauge_dp( + &mut self, + metric: MetricHandle, + now: u64, + value: i64, + attrs: F, + ) where + F: FnOnce(&mut DpAttrWriter<'_>), + { + self.append_i64_dp(metric, None, now, value, attrs); + } + + fn append_i64_dp( + &mut self, + metric: MetricHandle, + start: Option, + now: u64, + value: i64, + attrs: F, + ) where + F: FnOnce(&mut DpAttrWriter<'_>), + { + let dp_id = self.curr_dp_id; + self.ndp.append_id(dp_id); + self.ndp.append_parent_id(metric.0); + self.ndp + .append_start_time_unix_nano(start.map(|v| v as i64)); + self.ndp.append_time_unix_nano(now as i64); + self.ndp.append_int_value(Some(value)); + self.ndp.append_double_value(None); + self.ndp.append_flags(0); + let mut w = DpAttrWriter { + attrs: &mut self.ndp_attrs, + dp_id, + }; + attrs(&mut w); + self.curr_dp_id = self + .curr_dp_id + .checked_add(1) + .expect("dp_id overflow: more than u32::MAX datapoints in one batch"); + } + + /// Append one f64 Sum data point. + pub(crate) fn append_f64_sum_dp( + &mut self, + metric: MetricHandle, + start: u64, + now: u64, + value: f64, + attrs: F, + ) where + F: FnOnce(&mut DpAttrWriter<'_>), + { + self.append_f64_dp(metric, Some(start), now, value, attrs); + } + + /// Append one f64 Gauge data point. + pub(crate) fn append_f64_gauge_dp( + &mut self, + metric: MetricHandle, + now: u64, + value: f64, + attrs: F, + ) where + F: FnOnce(&mut DpAttrWriter<'_>), + { + self.append_f64_dp(metric, None, now, value, attrs); + } + + fn append_f64_dp( + &mut self, + metric: MetricHandle, + start: Option, + now: u64, + value: f64, + attrs: F, + ) where + F: FnOnce(&mut DpAttrWriter<'_>), + { + let dp_id = self.curr_dp_id; + self.ndp.append_id(dp_id); + self.ndp.append_parent_id(metric.0); + self.ndp + .append_start_time_unix_nano(start.map(|v| v as i64)); + self.ndp.append_time_unix_nano(now as i64); + self.ndp.append_int_value(None); + self.ndp.append_double_value(Some(value)); + self.ndp.append_flags(0); + let mut w = DpAttrWriter { + attrs: &mut self.ndp_attrs, + dp_id, + }; + attrs(&mut w); + self.curr_dp_id = self + .curr_dp_id + .checked_add(1) + .expect("dp_id overflow: more than u32::MAX datapoints in one batch"); + } + + // ── Finalization ───────────────────────────────────────────────────────── + + /// Finalize all builders and produce an `OtapArrowRecords::Metrics` batch. + pub(crate) fn finish(mut self) -> Result { + let n = self.metrics.len(); + // Resource: single entry with id=0 repeated for every metric row. + self.metrics.resource.append_id_n(0, n); + self.metrics + .resource + .append_schema_url_n(Some(semconv::SCHEMA_URL), n); + self.metrics + .resource + .append_dropped_attributes_count_n(0, n); + // Scope: single entry with id=0. + self.metrics.scope.append_id_n(0, n); + self.metrics.scope.append_name_n(Some(SCOPE_NAME), n); + self.metrics.scope.append_version_n(Some(SCOPE_VERSION), n); + self.metrics.scope.append_dropped_attributes_count_n(0, n); + // Schema URL on scope column. + self.metrics + .append_scope_schema_url_n(semconv::SCHEMA_URL, n); + + let mut records = OtapArrowRecords::Metrics(Metrics::default()); + finish_batch( + &mut records, + ArrowPayloadType::UnivariateMetrics, + self.metrics.finish()?, + )?; + finish_batch( + &mut records, + ArrowPayloadType::NumberDataPoints, + self.ndp.finish()?, + )?; + finish_batch( + &mut records, + ArrowPayloadType::ResourceAttrs, + self.resource_attrs.finish()?, + )?; + finish_batch( + &mut records, + ArrowPayloadType::NumberDpAttrs, + self.ndp_attrs.finish()?, + )?; + Ok(records) + } +} + +fn finish_batch( + records: &mut OtapArrowRecords, + payload_type: ArrowPayloadType, + rb: arrow::array::RecordBatch, +) -> Result<(), ArrowError> { + if rb.num_rows() > 0 { + records + .set(payload_type, rb) + .map_err(|e| ArrowError::ExternalError(Box::new(e)))?; + } + Ok(()) +} diff --git a/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/mod.rs b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/mod.rs new file mode 100644 index 0000000000..8a3329387a --- /dev/null +++ b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/mod.rs @@ -0,0 +1,563 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Linux procfs-backed host metric source. + +mod paths; +mod projection; +mod readings; + +use crate::receivers::host_metrics_receiver::{CompiledFilter, HostViewValidationMode}; +use paths::{PathKind, ProcfsPaths}; +use projection::{CounterTracker, HostScrape, host_arch}; +pub(crate) use projection::{HostResource, HostSnapshot}; +use readings::*; +use std::fs::File; +use std::io::{self, Read}; +use std::path::Path; +use std::time::{Duration, Instant}; + +const NANOS_PER_SEC: u64 = 1_000_000_000; +const BYTES_PER_KIB: u64 = 1024; +const DISKSTAT_SECTOR_BYTES: u64 = 512; +const FILESYSTEM_STAT_TIMEOUT: Duration = Duration::from_millis(100); +const FILESYSTEM_SCRAPE_TIMEOUT: Duration = Duration::from_secs(1); +const COUNTER_KEY_SEPARATOR: char = '\x1f'; + +/// Procfs-backed source for host metrics. +pub struct ProcfsSource { + paths: ProcfsPaths, + config: ProcfsConfig, + buf: String, + clk_tck: f64, + previous_cpu: Option, + filesystem_worker: FilesystemStatWorker, + counter_tracker: CounterTracker, + boot_time_unix_nano: Option, + fallback_start_time_unix_nano: u64, + resource: Option, +} + +/// Procfs collection config. +pub struct ProcfsConfig { + /// CPU metrics. + pub cpu: bool, + /// Memory metrics. + pub memory: bool, + /// Paging metrics. + pub paging: bool, + /// System metrics. + pub system: bool, + /// Disk metrics. + pub disk: bool, + /// Filesystem metrics. + pub filesystem: bool, + /// Network metrics. + pub network: bool, + /// Process summary metrics. + pub processes: bool, + /// Derived aggregate CPU utilization. + pub cpu_utilization: bool, + /// Emit memory limit metric. + pub memory_limit: bool, + /// Emit Linux shared memory metric. + pub memory_shared: bool, + /// Emit Linux hugepage metrics. + pub memory_hugepages: bool, + /// Derived disk limit from sysfs block device size. + pub disk_limit: bool, + /// Include virtual filesystems. + pub filesystem_include_virtual: bool, + /// Include remote and userspace filesystems. + pub filesystem_include_remote: bool, + /// Emit filesystem limit metric. + pub filesystem_limit: bool, + /// Disk include filter. + pub disk_include: Option, + /// Disk exclude filter. + pub disk_exclude: Option, + /// Filesystem device include filter. + pub filesystem_include_devices: Option, + /// Filesystem device exclude filter. + pub filesystem_exclude_devices: Option, + /// Filesystem type include filter. + pub filesystem_include_fs_types: Option, + /// Filesystem type exclude filter. + pub filesystem_exclude_fs_types: Option, + /// Filesystem mount point include filter. + pub filesystem_include_mount_points: Option, + /// Filesystem mount point exclude filter. + pub filesystem_exclude_mount_points: Option, + /// Network include filter. + pub network_include: Option, + /// Network exclude filter. + pub network_exclude: Option, + /// Startup validation mode. + pub validation: HostViewValidationMode, +} + +/// Families due for one scrape. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct ProcfsFamilies { + /// CPU metrics. + pub cpu: bool, + /// Memory metrics. + pub memory: bool, + /// Paging metrics. + pub paging: bool, + /// System metrics. + pub system: bool, + /// Disk metrics. + pub disk: bool, + /// Filesystem metrics. + pub filesystem: bool, + /// Network metrics. + pub network: bool, + /// Process summary metrics. + pub processes: bool, +} + +impl ProcfsFamilies { + fn enabled_by(self, config: &ProcfsConfig) -> Self { + Self { + cpu: self.cpu && config.cpu, + memory: self.memory && config.memory, + paging: self.paging && config.paging, + system: self.system && config.system, + disk: self.disk && config.disk, + filesystem: self.filesystem && config.filesystem, + network: self.network && config.network, + processes: self.processes && config.processes, + } + } +} + +impl ProcfsSource { + /// Creates a procfs source rooted at `/` or at a host root bind mount. + pub fn new(root_path: Option<&Path>, config: ProcfsConfig) -> io::Result { + let mut source = Self { + paths: ProcfsPaths::new(root_path), + config, + buf: String::with_capacity(16 * 1024), + clk_tck: clock_ticks_per_second(), + previous_cpu: None, + filesystem_worker: FilesystemStatWorker::new()?, + counter_tracker: CounterTracker::default(), + boot_time_unix_nano: None, + fallback_start_time_unix_nano: now_unix_nano(), + resource: None, + }; + source.apply_startup_validation()?; + Ok(source) + } + + /// Collects one host snapshot for the due family set. + pub async fn scrape_due(&mut self, due: ProcfsFamilies) -> io::Result { + let due = due.enabled_by(&self.config); + let now_unix_nano = now_unix_nano(); + let clk_tck = self.clk_tck; + let mut partial_errors = 0; + let mut first_error = None; + let needs_start_time = due.cpu + || due.memory + || due.paging + || due.disk + || due.filesystem + || due.network + || due.processes; + let needs_stat = + due.cpu || due.processes || (needs_start_time && self.boot_time_unix_nano.is_none()); + let stat = match needs_stat + .then(|| self.read_path(PathKind::Stat)) + .transpose() + { + Ok(Some(proc_stat)) => parse_stat(proc_stat, clk_tck), + Ok(None) => StatSnapshot::default(), + Err(err) => { + record_partial_error(&mut partial_errors, &mut first_error, err); + StatSnapshot::default() + } + }; + if stat.boot_time_unix_nano != 0 { + self.boot_time_unix_nano = Some(stat.boot_time_unix_nano); + } + let start_time_unix_nano = self + .boot_time_unix_nano + .unwrap_or(self.fallback_start_time_unix_nano); + let cpu_utilization = if due.cpu && self.config.cpu_utilization { + let utilization = stat.cpu.and_then(|current| { + self.previous_cpu + .and_then(|previous| cpu_utilization(previous, current)) + }); + self.previous_cpu = stat.cpu; + utilization + } else { + None + }; + + let cpuinfo = match due + .cpu + .then(|| self.read_path(PathKind::Cpuinfo)) + .transpose() + { + Ok(Some(cpuinfo)) => parse_cpuinfo(cpuinfo), + Ok(None) => CpuInfo::default(), + Err(err) => { + record_partial_error(&mut partial_errors, &mut first_error, err); + CpuInfo::default() + } + }; + + let memory = match due + .memory + .then(|| self.read_path(PathKind::Meminfo)) + .transpose() + { + Ok(Some(meminfo)) => parse_meminfo(meminfo), + Ok(None) => None, + Err(err) => { + record_partial_error(&mut partial_errors, &mut first_error, err); + None + } + }; + + let uptime_seconds = match due + .system + .then(|| self.read_path(PathKind::Uptime)) + .transpose() + { + Ok(Some(uptime)) => parse_uptime(uptime), + Ok(None) => None, + Err(err) => { + record_partial_error(&mut partial_errors, &mut first_error, err); + None + } + }; + + let paging = match due + .paging + .then(|| self.read_path(PathKind::Vmstat)) + .transpose() + { + Ok(Some(vmstat)) => Some(parse_vmstat(vmstat)), + Ok(None) => None, + Err(err) => { + record_partial_error(&mut partial_errors, &mut first_error, err); + None + } + }; + + let swaps = match due + .paging + .then(|| self.read_path(PathKind::Swaps)) + .transpose() + { + Ok(Some(swaps)) => parse_swaps(swaps), + Ok(None) => Vec::new(), + Err(err) => { + record_partial_error(&mut partial_errors, &mut first_error, err); + Vec::new() + } + }; + + tokio::task::consume_budget().await; + + let disks = if due.disk { + let disk_include = self.config.disk_include.clone(); + let disk_exclude = self.config.disk_exclude.clone(); + match self.read_path(PathKind::Diskstats) { + Ok(diskstats) => { + let mut disks = + parse_diskstats(diskstats, disk_include.as_ref(), disk_exclude.as_ref()); + if self.config.disk_limit { + for disk in &mut disks { + disk.limit_bytes = self.read_disk_limit_bytes(&disk.name).ok(); + } + } + Some(disks) + } + Err(err) => { + record_partial_error(&mut partial_errors, &mut first_error, err); + None + } + } + } else { + None + }; + + tokio::task::consume_budget().await; + + let networks = if due.network { + let network_include = self.config.network_include.clone(); + let network_exclude = self.config.network_exclude.clone(); + match self.read_path(PathKind::NetDev) { + Ok(netdev) => Some(parse_netdev( + netdev, + network_include.as_ref(), + network_exclude.as_ref(), + )), + Err(err) => { + record_partial_error(&mut partial_errors, &mut first_error, err); + None + } + } + } else { + None + }; + + tokio::task::consume_budget().await; + + let filesystems = if due.filesystem { + let include_virtual = self.config.filesystem_include_virtual; + let include_remote = self.config.filesystem_include_remote; + let emit_limit = self.config.filesystem_limit; + let include_devices = self.config.filesystem_include_devices.clone(); + let exclude_devices = self.config.filesystem_exclude_devices.clone(); + let include_fs_types = self.config.filesystem_include_fs_types.clone(); + let exclude_fs_types = self.config.filesystem_exclude_fs_types.clone(); + let include_mount_points = self.config.filesystem_include_mount_points.clone(); + let exclude_mount_points = self.config.filesystem_exclude_mount_points.clone(); + match self.read_path(PathKind::Mountinfo) { + Ok(mountinfo) => { + let filters = FilesystemFilters { + include_devices: include_devices.as_ref(), + exclude_devices: exclude_devices.as_ref(), + include_fs_types: include_fs_types.as_ref(), + exclude_fs_types: exclude_fs_types.as_ref(), + include_mount_points: include_mount_points.as_ref(), + exclude_mount_points: exclude_mount_points.as_ref(), + }; + let mounts = parse_mountinfo( + mountinfo, + include_virtual, + include_remote, + emit_limit, + filters, + ); + self.read_filesystems(mounts, &mut partial_errors, &mut first_error) + } + Err(err) => { + record_partial_error(&mut partial_errors, &mut first_error, err); + Vec::new() + } + } + } else { + Vec::new() + }; + + tokio::task::consume_budget().await; + + let resource = self.read_resource().clone(); + let counter_starts = self.counter_tracker.snapshot( + start_time_unix_nano, + now_unix_nano, + due.cpu.then_some(stat.cpu).flatten().as_ref(), + paging.as_ref(), + due.processes.then_some(stat.processes).as_ref(), + disks.as_deref(), + networks.as_deref(), + ); + + let snapshot = HostSnapshot { + now_unix_nano, + start_time_unix_nano, + counter_starts, + memory_limit: self.config.memory_limit, + memory_shared: self.config.memory_shared, + memory_hugepages: self.config.memory_hugepages, + cpu: due.cpu.then_some(stat.cpu).flatten(), + cpu_utilization, + cpuinfo, + memory, + uptime_seconds, + paging, + swaps, + processes: due.processes.then_some(stat.processes), + disks: disks.unwrap_or_default(), + filesystems, + networks: networks.unwrap_or_default(), + resource, + }; + if !snapshot.has_metrics() { + return Err(first_error + .unwrap_or_else(|| io::Error::other("host metrics scrape produced no metrics"))); + } + Ok(HostScrape { + snapshot, + partial_errors, + }) + } + + fn apply_startup_validation(&mut self) -> io::Result<()> { + match self.config.validation { + HostViewValidationMode::None => Ok(()), + HostViewValidationMode::FailSelected => self.validate_selected_paths(), + HostViewValidationMode::WarnSelected => { + self.disable_unavailable_sources(); + Ok(()) + } + } + } + + fn validate_selected_paths(&self) -> io::Result<()> { + if self.config.cpu + || self.config.memory + || self.config.paging + || self.config.disk + || self.config.filesystem + || self.config.network + || self.config.processes + { + let _ = File::open(self.paths.path(PathKind::Stat))?; + } + if self.config.cpu { + let _ = File::open(self.paths.path(PathKind::Cpuinfo))?; + } + if self.config.memory { + let _ = File::open(self.paths.path(PathKind::Meminfo))?; + } + if self.config.system { + let _ = File::open(self.paths.path(PathKind::Uptime))?; + } + if self.config.paging { + let _ = File::open(self.paths.path(PathKind::Vmstat))?; + let _ = File::open(self.paths.path(PathKind::Swaps))?; + } + if self.config.disk { + let _ = File::open(self.paths.path(PathKind::Diskstats))?; + } + if self.config.filesystem { + let _ = File::open(self.paths.path(PathKind::Mountinfo))?; + } + if self.config.network { + let _ = File::open(self.paths.path(PathKind::NetDev))?; + } + Ok(()) + } + + fn disable_unavailable_sources(&mut self) { + if (self.config.cpu || self.config.system || self.config.processes) + && !self.source_available(PathKind::Stat) + { + self.config.cpu = false; + self.config.system = false; + self.config.processes = false; + } + if self.config.cpu && !self.source_available(PathKind::Cpuinfo) { + self.config.cpu = false; + } + if self.config.memory && !self.source_available(PathKind::Meminfo) { + self.config.memory = false; + } + if self.config.system && !self.source_available(PathKind::Uptime) { + self.config.system = false; + } + if self.config.paging + && (!self.source_available(PathKind::Vmstat) || !self.source_available(PathKind::Swaps)) + { + self.config.paging = false; + } + if self.config.disk && !self.source_available(PathKind::Diskstats) { + self.config.disk = false; + } + if self.config.filesystem && !self.source_available(PathKind::Mountinfo) { + self.config.filesystem = false; + } + if self.config.network && !self.source_available(PathKind::NetDev) { + self.config.network = false; + } + } + + fn source_available(&self, kind: PathKind) -> bool { + File::open(self.paths.path(kind)).is_ok() + } + + fn read_path(&mut self, kind: PathKind) -> io::Result<&str> { + self.buf.clear(); + let mut file = File::open(self.paths.path(kind))?; + let _ = file.read_to_string(&mut self.buf)?; + Ok(self.buf.as_str()) + } + + fn read_disk_limit_bytes(&mut self, disk_name: &str) -> io::Result { + self.buf.clear(); + let mut file = File::open(self.paths.sys_block.join(disk_name).join("size"))?; + let _ = file.read_to_string(&mut self.buf)?; + let sectors = parse_u64(self.buf.trim()); + Ok(sectors.saturating_mul(DISKSTAT_SECTOR_BYTES)) + } + + fn read_filesystems( + &mut self, + mounts: Vec, + partial_errors: &mut u64, + first_error: &mut Option, + ) -> Vec { + let started = Instant::now(); + let mut filesystems = Vec::with_capacity(mounts.len()); + for mount in mounts { + let Some(remaining) = FILESYSTEM_SCRAPE_TIMEOUT.checked_sub(started.elapsed()) else { + record_partial_error( + partial_errors, + first_error, + io::Error::new( + io::ErrorKind::TimedOut, + "filesystem scrape budget exhausted", + ), + ); + break; + }; + let path = self.paths.host_path(&mount.mountpoint); + let stat = match self + .filesystem_worker + .statvfs(path, remaining.min(FILESYSTEM_STAT_TIMEOUT)) + { + Ok(stat) => stat, + Err(err) => { + record_partial_error(partial_errors, first_error, err); + continue; + } + }; + let free = stat.available_bytes; + let reserved = stat.free_bytes.saturating_sub(stat.available_bytes); + let used = stat.total_bytes.saturating_sub(stat.free_bytes); + filesystems.push(FilesystemStats { + device: mount.device, + mountpoint: mount.mountpoint, + fs_type: mount.fs_type, + mode: mount.mode, + used, + free, + reserved, + limit_bytes: mount.emit_limit.then_some(stat.total_bytes), + }); + } + filesystems + } + + fn read_resource(&mut self) -> &HostResource { + if self.resource.is_none() { + let host_id = self + .read_trimmed_optional(PathKind::MachineId) + .or_else(|| self.read_trimmed_optional(PathKind::DbusMachineId)); + let host_name = self.read_trimmed_optional(PathKind::Hostname); + self.resource = Some(HostResource { + host_id, + host_name, + host_arch: host_arch(), + }); + } + self.resource.as_ref().expect("resource is initialized") + } + + fn read_trimmed_optional(&mut self, kind: PathKind) -> Option { + self.read_path(kind) + .ok() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) + } +} + +#[cfg(test)] +mod tests; diff --git a/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/paths.rs b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/paths.rs new file mode 100644 index 0000000000..350f01adff --- /dev/null +++ b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/paths.rs @@ -0,0 +1,93 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +use std::path::{Path, PathBuf}; + +#[derive(Clone, Debug)] +pub(super) struct ProcfsPaths { + root: PathBuf, + stat: PathBuf, + cpuinfo: PathBuf, + meminfo: PathBuf, + uptime: PathBuf, + vmstat: PathBuf, + swaps: PathBuf, + diskstats: PathBuf, + pub(super) mountinfo: PathBuf, + pub(super) sys_block: PathBuf, + pub(super) net_dev: PathBuf, + machine_id: PathBuf, + dbus_machine_id: PathBuf, + hostname: PathBuf, +} + +impl ProcfsPaths { + pub(super) fn new(root_path: Option<&Path>) -> Self { + let root = root_path.unwrap_or_else(|| Path::new("/")); + let host_root = root_path.is_some_and(|path| path != Path::new("/")); + Self { + root: root.to_path_buf(), + stat: root.join("proc/stat"), + cpuinfo: root.join("proc/cpuinfo"), + meminfo: root.join("proc/meminfo"), + uptime: root.join("proc/uptime"), + vmstat: root.join("proc/vmstat"), + swaps: root.join("proc/swaps"), + diskstats: root.join("proc/diskstats"), + mountinfo: if host_root { + root.join("proc/1/mountinfo") + } else { + root.join("proc/self/mountinfo") + }, + sys_block: root.join("sys/block"), + machine_id: root.join("etc/machine-id"), + dbus_machine_id: root.join("var/lib/dbus/machine-id"), + hostname: root.join("proc/sys/kernel/hostname"), + net_dev: if host_root { + root.join("proc/1/net/dev") + } else { + root.join("proc/net/dev") + }, + } + } + + pub(super) fn path(&self, kind: PathKind) -> &Path { + match kind { + PathKind::Stat => &self.stat, + PathKind::Cpuinfo => &self.cpuinfo, + PathKind::Meminfo => &self.meminfo, + PathKind::Uptime => &self.uptime, + PathKind::Vmstat => &self.vmstat, + PathKind::Swaps => &self.swaps, + PathKind::Diskstats => &self.diskstats, + PathKind::Mountinfo => &self.mountinfo, + PathKind::NetDev => &self.net_dev, + PathKind::MachineId => &self.machine_id, + PathKind::DbusMachineId => &self.dbus_machine_id, + PathKind::Hostname => &self.hostname, + } + } + + pub(super) fn host_path(&self, host_absolute_path: &str) -> PathBuf { + let relative = host_absolute_path + .strip_prefix('/') + .unwrap_or(host_absolute_path); + self.root.join(relative) + } +} + +#[derive(Copy, Clone)] +pub(super) enum PathKind { + Stat, + Cpuinfo, + Meminfo, + Uptime, + Vmstat, + Swaps, + Diskstats, + Mountinfo, + NetDev, + MachineId, + DbusMachineId, + Hostname, +} diff --git a/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/projection.rs b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/projection.rs new file mode 100644 index 0000000000..f037526dae --- /dev/null +++ b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/projection.rs @@ -0,0 +1,975 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::receivers::host_metrics_receiver::otap_builder::HostMetricsArrowBuilder; +use crate::receivers::host_metrics_receiver::semconv::{attr, metric}; +use std::collections::{HashMap, HashSet}; + +use super::COUNTER_KEY_SEPARATOR; +use super::readings::{ + CpuInfo, CpuTimes, DiskStats, FilesystemStats, HugepageStats, MemoryStats, NetworkStats, + PagingStats, ProcessStats, SwapStats, frequency_hz_i64, saturating_i64, +}; + +/// Result of one host metrics scrape. +pub(crate) struct HostScrape { + /// Collected host snapshot. + pub snapshot: HostSnapshot, + /// Number of source read errors skipped because other families succeeded. + pub partial_errors: u64, +} + +/// One host metrics snapshot. +#[derive(Default)] +pub(crate) struct HostSnapshot { + pub(super) now_unix_nano: u64, + pub(super) start_time_unix_nano: u64, + pub(super) counter_starts: CounterStarts, + pub(super) memory_limit: bool, + pub(super) memory_shared: bool, + pub(super) memory_hugepages: bool, + pub(super) cpu: Option, + pub(super) cpu_utilization: Option, + pub(super) cpuinfo: CpuInfo, + pub(super) memory: Option, + pub(super) uptime_seconds: Option, + pub(super) paging: Option, + pub(super) swaps: Vec, + pub(super) processes: Option, + pub(super) disks: Vec, + pub(super) filesystems: Vec, + pub(super) networks: Vec, + pub(super) resource: HostResource, +} + +impl HostSnapshot { + pub(super) fn has_metrics(&self) -> bool { + self.cpu.is_some() + || self.cpu_utilization.is_some() + || self.cpuinfo.logical_count != 0 + || self.cpuinfo.physical_count != 0 + || !self.cpuinfo.frequencies_hz.is_empty() + || self.memory.is_some() + || self.uptime_seconds.is_some() + || self.paging.is_some() + || !self.swaps.is_empty() + || self.processes.is_some() + || !self.disks.is_empty() + || !self.filesystems.is_empty() + || !self.networks.is_empty() + } + + /// Converts a snapshot directly into an OTAP Arrow metrics batch. + pub fn into_otap_records( + self, + ) -> Result { + let mut b = HostMetricsArrowBuilder::new(); + b.append_resource(&self.resource); + project_snapshot(&self, &mut b); + b.finish() + } +} + +#[derive(Clone, Default)] +pub(crate) struct HostResource { + pub(crate) host_id: Option, + pub(crate) host_name: Option, + pub(crate) host_arch: Option<&'static str>, +} + +pub(super) fn project_snapshot(snap: &HostSnapshot, b: &mut HostMetricsArrowBuilder) { + let now = snap.now_unix_nano; + let start = snap.start_time_unix_nano; + let cs = &snap.counter_starts; + + // ── CPU ────────────────────────────────────────────────────────────────── + if let Some(cpu) = snap.cpu { + let m = b.begin_counter_f64(metric::CPU_TIME, "s"); + for (mode, value) in [ + ("user", cpu.user), + ("nice", cpu.nice), + ("system", cpu.system), + ("idle", cpu.idle), + ("iowait", cpu.wait), + ("interrupt", cpu.interrupt), + ("steal", cpu.steal), + ] { + b.append_f64_sum_dp(m, cs.get(metric::CPU_TIME, mode, start), now, value, |w| { + w.str(attr::CPU_MODE, mode); + }); + } + } + if let Some(cpu) = snap.cpu_utilization { + let m = b.begin_gauge_f64(metric::CPU_UTILIZATION, "1"); + for (mode, value) in [ + ("user", cpu.user), + ("nice", cpu.nice), + ("system", cpu.system), + ("idle", cpu.idle), + ("iowait", cpu.wait), + ("interrupt", cpu.interrupt), + ("steal", cpu.steal), + ] { + b.append_f64_gauge_dp(m, now, value, |w| { + w.str(attr::CPU_MODE, mode); + }); + } + } + if snap.cpuinfo.logical_count != 0 { + let m = b.begin_updown_i64(metric::CPU_LOGICAL_COUNT, "{cpu}"); + b.append_i64_sum_dp( + m, + start, + now, + saturating_i64(snap.cpuinfo.logical_count), + |_| {}, + ); + } + if snap.cpuinfo.physical_count != 0 { + let m = b.begin_updown_i64(metric::CPU_PHYSICAL_COUNT, "{cpu}"); + b.append_i64_sum_dp( + m, + start, + now, + saturating_i64(snap.cpuinfo.physical_count), + |_| {}, + ); + } + if !snap.cpuinfo.frequencies_hz.is_empty() { + let m = b.begin_gauge_i64(metric::CPU_FREQUENCY, "Hz"); + for (idx, &freq) in snap.cpuinfo.frequencies_hz.iter().enumerate() { + let logical = i64::try_from(idx).unwrap_or(i64::MAX); + b.append_i64_gauge_dp(m, now, frequency_hz_i64(freq), |w| { + w.int(attr::CPU_LOGICAL_NUMBER, logical); + }); + } + } + + // ── Memory ─────────────────────────────────────────────────────────────── + if let Some(memory) = snap.memory { + let m = b.begin_updown_i64(metric::MEMORY_USAGE, "By"); + for (state, value) in [ + ("used", memory.used), + ("free", memory.free), + ("cached", memory.cached), + ("buffers", memory.buffered), + ] { + b.append_i64_sum_dp(m, start, now, saturating_i64(value), |w| { + w.str(attr::SYSTEM_MEMORY_STATE, state); + }); + } + if memory.total > 0 { + let m = b.begin_gauge_f64(metric::MEMORY_UTILIZATION, "1"); + let total = memory.total as f64; + for (state, value) in [ + ("used", memory.used), + ("free", memory.free), + ("cached", memory.cached), + ("buffers", memory.buffered), + ] { + b.append_f64_gauge_dp(m, now, value as f64 / total, |w| { + w.str(attr::SYSTEM_MEMORY_STATE, state); + }); + } + } + if memory.has_available { + let m = b.begin_updown_i64(metric::MEMORY_LINUX_AVAILABLE, "By"); + b.append_i64_sum_dp(m, start, now, saturating_i64(memory.available), |_| {}); + } + let m = b.begin_updown_i64(metric::MEMORY_LINUX_SLAB_USAGE, "By"); + for (state, value) in [ + ("reclaimable", memory.slab_reclaimable), + ("unreclaimable", memory.slab_unreclaimable), + ] { + b.append_i64_sum_dp(m, start, now, saturating_i64(value), |w| { + w.str(attr::SYSTEM_MEMORY_LINUX_SLAB_STATE, state); + }); + } + if snap.memory_limit { + let m = b.begin_updown_i64(metric::MEMORY_LIMIT, "By"); + b.append_i64_sum_dp(m, start, now, saturating_i64(memory.total), |_| {}); + } + if snap.memory_shared { + let m = b.begin_updown_i64(metric::MEMORY_LINUX_SHARED, "By"); + b.append_i64_sum_dp(m, start, now, saturating_i64(memory.shared), |_| {}); + } + if snap.memory_hugepages { + project_hugepages(b, start, now, &memory.hugepages); + } + } + + // ── System / uptime ────────────────────────────────────────────────────── + if let Some(uptime) = snap.uptime_seconds { + let m = b.begin_gauge_f64(metric::UPTIME, "s"); + b.append_f64_gauge_dp(m, now, uptime, |_| {}); + } + + // ── Paging ─────────────────────────────────────────────────────────────── + if let Some(paging) = snap.paging { + let m = b.begin_counter_i64(metric::PAGING_FAULTS, "{fault}"); + for (fault_type, value) in [ + ("minor", paging.minor_faults), + ("major", paging.major_faults), + ] { + b.append_i64_sum_dp( + m, + cs.get(metric::PAGING_FAULTS, fault_type, start), + now, + saturating_i64(value), + |w| { + w.str(attr::SYSTEM_PAGING_FAULT_TYPE, fault_type); + }, + ); + } + let m = b.begin_counter_i64(metric::PAGING_OPERATIONS, "{operation}"); + // Linux exposes swap operations and page-in/page-out counters separately. + // Semconv requires both direction and fault.type for this metric, so the + // receiver keeps the phase-1 mapping explicit here. + for (direction, fault_type, value) in [ + ("in", "major", paging.swap_in), + ("out", "major", paging.swap_out), + ("in", "minor", paging.page_in), + ("out", "minor", paging.page_out), + ] { + b.append_i64_sum_dp( + m, + cs.get_joined(metric::PAGING_OPERATIONS, direction, fault_type, start), + now, + saturating_i64(value), + |w| { + w.str(attr::SYSTEM_PAGING_DIRECTION, direction); + w.str(attr::SYSTEM_PAGING_FAULT_TYPE, fault_type); + }, + ); + } + } + if !snap.swaps.is_empty() { + let m = b.begin_updown_i64(metric::PAGING_USAGE, "By"); + for swap in &snap.swaps { + for (state, value) in [("used", swap.used), ("free", swap.free)] { + b.append_i64_sum_dp(m, start, now, saturating_i64(value), |w| { + w.str(attr::SYSTEM_DEVICE, &swap.name); + w.str(attr::SYSTEM_PAGING_STATE, state); + }); + } + } + } + if snap.swaps.iter().any(|swap| swap.size > 0) { + let m = b.begin_gauge_f64(metric::PAGING_UTILIZATION, "1"); + for swap in &snap.swaps { + let size = swap.size; + if size == 0 { + continue; + } + let total = size as f64; + for (state, value) in [("used", swap.used), ("free", swap.free)] { + b.append_f64_gauge_dp(m, now, value as f64 / total, |w| { + w.str(attr::SYSTEM_DEVICE, &swap.name); + w.str(attr::SYSTEM_PAGING_STATE, state); + }); + } + } + } + + // ── Processes ──────────────────────────────────────────────────────────── + if let Some(processes) = snap.processes { + let m = b.begin_updown_i64(metric::PROCESS_COUNT, "{process}"); + b.append_i64_sum_dp(m, start, now, saturating_i64(processes.running), |w| { + w.str(attr::PROCESS_STATE, "running"); + }); + // /proc/stat procs_blocked has no registered process.state value. + // Do not map it to sleeping; Linux blocked tasks are not the same state. + let m = b.begin_counter_i64(metric::PROCESS_CREATED, "{process}"); + b.append_i64_sum_dp( + m, + cs.get(metric::PROCESS_CREATED, "", start), + now, + saturating_i64(processes.created), + |_| {}, + ); + } + + // ── Disk ───────────────────────────────────────────────────────────────── + if snap.disks.iter().any(|disk| disk.limit_bytes.is_some()) { + let m = b.begin_updown_i64(metric::DISK_LIMIT, "By"); + for disk in &snap.disks { + let Some(limit_bytes) = disk.limit_bytes else { + continue; + }; + b.append_i64_sum_dp(m, start, now, saturating_i64(limit_bytes), |w| { + w.str(attr::SYSTEM_DEVICE, &disk.name); + }); + } + } + if !snap.disks.is_empty() { + let m = b.begin_counter_i64(metric::DISK_IO, "By"); + for disk in &snap.disks { + for (dir, value) in [("read", disk.read_bytes), ("write", disk.write_bytes)] { + b.append_i64_sum_dp( + m, + cs.get_joined(metric::DISK_IO, &disk.name, dir, start), + now, + saturating_i64(value), + |w| { + w.str(attr::SYSTEM_DEVICE, &disk.name); + w.str(attr::DISK_IO_DIRECTION, dir); + }, + ); + } + } + let m = b.begin_counter_i64(metric::DISK_OPERATIONS, "{operation}"); + for disk in &snap.disks { + for (dir, value) in [("read", disk.read_ops), ("write", disk.write_ops)] { + b.append_i64_sum_dp( + m, + cs.get_joined(metric::DISK_OPERATIONS, &disk.name, dir, start), + now, + saturating_i64(value), + |w| { + w.str(attr::SYSTEM_DEVICE, &disk.name); + w.str(attr::DISK_IO_DIRECTION, dir); + }, + ); + } + } + let m = b.begin_counter_f64(metric::DISK_IO_TIME, "s"); + for disk in &snap.disks { + b.append_f64_sum_dp( + m, + cs.get(metric::DISK_IO_TIME, &disk.name, start), + now, + disk.io_time_seconds, + |w| { + w.str(attr::SYSTEM_DEVICE, &disk.name); + }, + ); + } + let m = b.begin_counter_f64(metric::DISK_OPERATION_TIME, "s"); + for disk in &snap.disks { + for (dir, value) in [ + ("read", disk.read_time_seconds), + ("write", disk.write_time_seconds), + ] { + b.append_f64_sum_dp( + m, + cs.get_joined(metric::DISK_OPERATION_TIME, &disk.name, dir, start), + now, + value, + |w| { + w.str(attr::SYSTEM_DEVICE, &disk.name); + w.str(attr::DISK_IO_DIRECTION, dir); + }, + ); + } + } + let m = b.begin_counter_i64(metric::DISK_MERGED, "{operation}"); + for disk in &snap.disks { + for (dir, value) in [("read", disk.read_merged), ("write", disk.write_merged)] { + b.append_i64_sum_dp( + m, + cs.get_joined(metric::DISK_MERGED, &disk.name, dir, start), + now, + saturating_i64(value), + |w| { + w.str(attr::SYSTEM_DEVICE, &disk.name); + w.str(attr::DISK_IO_DIRECTION, dir); + }, + ); + } + } + } + + // ── Filesystem ─────────────────────────────────────────────────────────── + if !snap.filesystems.is_empty() { + let m = b.begin_updown_i64(metric::FILESYSTEM_USAGE, "By"); + for fs in &snap.filesystems { + for (state, value) in [ + ("used", fs.used), + ("free", fs.free), + ("reserved", fs.reserved), + ] { + b.append_i64_sum_dp(m, start, now, saturating_i64(value), |w| { + w.str(attr::SYSTEM_DEVICE, &fs.device); + w.str(attr::SYSTEM_FILESYSTEM_STATE, state); + w.str(attr::SYSTEM_FILESYSTEM_TYPE, &fs.fs_type); + w.str(attr::SYSTEM_FILESYSTEM_MODE, fs.mode); + w.str(attr::SYSTEM_FILESYSTEM_MOUNTPOINT, &fs.mountpoint); + }); + } + } + } + if snap + .filesystems + .iter() + .any(|fs| fs.used.saturating_add(fs.free).saturating_add(fs.reserved) > 0) + { + let m = b.begin_gauge_f64(metric::FILESYSTEM_UTILIZATION, "1"); + for fs in &snap.filesystems { + let total = fs.used.saturating_add(fs.free).saturating_add(fs.reserved); + if total > 0 { + let total_f = total as f64; + for (state, value) in [ + ("used", fs.used), + ("free", fs.free), + ("reserved", fs.reserved), + ] { + b.append_f64_gauge_dp(m, now, value as f64 / total_f, |w| { + w.str(attr::SYSTEM_DEVICE, &fs.device); + w.str(attr::SYSTEM_FILESYSTEM_STATE, state); + w.str(attr::SYSTEM_FILESYSTEM_TYPE, &fs.fs_type); + w.str(attr::SYSTEM_FILESYSTEM_MODE, fs.mode); + w.str(attr::SYSTEM_FILESYSTEM_MOUNTPOINT, &fs.mountpoint); + }); + } + } + } + } + if snap.filesystems.iter().any(|fs| fs.limit_bytes.is_some()) { + let m = b.begin_updown_i64(metric::FILESYSTEM_LIMIT, "By"); + for fs in &snap.filesystems { + let Some(limit_bytes) = fs.limit_bytes else { + continue; + }; + b.append_i64_sum_dp(m, start, now, saturating_i64(limit_bytes), |w| { + w.str(attr::SYSTEM_DEVICE, &fs.device); + w.str(attr::SYSTEM_FILESYSTEM_TYPE, &fs.fs_type); + w.str(attr::SYSTEM_FILESYSTEM_MODE, fs.mode); + w.str(attr::SYSTEM_FILESYSTEM_MOUNTPOINT, &fs.mountpoint); + }); + } + } + + // ── Network ────────────────────────────────────────────────────────────── + if !snap.networks.is_empty() { + let m = b.begin_counter_i64(metric::NETWORK_IO, "By"); + for net in &snap.networks { + for (dir, value) in [("receive", net.rx_bytes), ("transmit", net.tx_bytes)] { + b.append_i64_sum_dp( + m, + cs.get_joined(metric::NETWORK_IO, &net.name, dir, start), + now, + saturating_i64(value), + |w| { + w.str(attr::NETWORK_INTERFACE_NAME, &net.name); + w.str(attr::NETWORK_IO_DIRECTION, dir); + }, + ); + } + } + let m = b.begin_counter_i64(metric::NETWORK_PACKET_COUNT, "{packet}"); + for net in &snap.networks { + for (dir, value) in [("receive", net.rx_packets), ("transmit", net.tx_packets)] { + b.append_i64_sum_dp( + m, + cs.get_joined(metric::NETWORK_PACKET_COUNT, &net.name, dir, start), + now, + saturating_i64(value), + |w| { + // Semconv uses system.device here, while sibling network + // metrics use network.interface.name. + w.str(attr::SYSTEM_DEVICE, &net.name); + w.str(attr::NETWORK_IO_DIRECTION, dir); + }, + ); + } + } + let m = b.begin_counter_i64(metric::NETWORK_PACKET_DROPPED, "{packet}"); + for net in &snap.networks { + for (dir, value) in [("receive", net.rx_dropped), ("transmit", net.tx_dropped)] { + b.append_i64_sum_dp( + m, + cs.get_joined(metric::NETWORK_PACKET_DROPPED, &net.name, dir, start), + now, + saturating_i64(value), + |w| { + w.str(attr::NETWORK_INTERFACE_NAME, &net.name); + w.str(attr::NETWORK_IO_DIRECTION, dir); + }, + ); + } + } + let m = b.begin_counter_i64(metric::NETWORK_ERRORS, "{error}"); + for net in &snap.networks { + for (dir, value) in [("receive", net.rx_errors), ("transmit", net.tx_errors)] { + b.append_i64_sum_dp( + m, + cs.get_joined(metric::NETWORK_ERRORS, &net.name, dir, start), + now, + saturating_i64(value), + |w| { + w.str(attr::NETWORK_INTERFACE_NAME, &net.name); + w.str(attr::NETWORK_IO_DIRECTION, dir); + }, + ); + } + } + } +} + +pub(super) fn project_hugepages( + b: &mut HostMetricsArrowBuilder, + start: u64, + now: u64, + hugepages: &HugepageStats, +) { + let m = b.begin_updown_i64(metric::MEMORY_LINUX_HUGEPAGES_LIMIT, "{page}"); + b.append_i64_sum_dp(m, start, now, saturating_i64(hugepages.total), |_| {}); + let m = b.begin_updown_i64(metric::MEMORY_LINUX_HUGEPAGES_PAGE_SIZE, "By"); + b.append_i64_sum_dp( + m, + start, + now, + saturating_i64(hugepages.page_size_bytes), + |_| {}, + ); + let m = b.begin_updown_i64(metric::MEMORY_LINUX_HUGEPAGES_RESERVED, "{page}"); + b.append_i64_sum_dp(m, start, now, saturating_i64(hugepages.reserved), |_| {}); + let m = b.begin_updown_i64(metric::MEMORY_LINUX_HUGEPAGES_SURPLUS, "{page}"); + b.append_i64_sum_dp(m, start, now, saturating_i64(hugepages.surplus), |_| {}); + let used = hugepages.total.saturating_sub(hugepages.free); + let m = b.begin_updown_i64(metric::MEMORY_LINUX_HUGEPAGES_USAGE, "{page}"); + for (state, value) in [("used", used), ("free", hugepages.free)] { + b.append_i64_sum_dp(m, start, now, saturating_i64(value), |w| { + w.str(attr::SYSTEM_MEMORY_LINUX_HUGEPAGES_STATE, state); + }); + } + if hugepages.total > 0 { + let total = hugepages.total as f64; + let m = b.begin_gauge_f64(metric::MEMORY_LINUX_HUGEPAGES_UTILIZATION, "1"); + for (state, value) in [("used", used), ("free", hugepages.free)] { + b.append_f64_gauge_dp(m, now, value as f64 / total, |w| { + w.str(attr::SYSTEM_MEMORY_LINUX_HUGEPAGES_STATE, state); + }); + } + } +} + +#[derive(Default)] +pub(super) struct CounterTracker { + states: HashMap, +} + +pub(super) struct CounterState { + previous: f64, + start_time_unix_nano: u64, +} + +#[derive(Default)] +pub(super) struct CounterStarts { + pub(super) entries: Vec<(String, u64)>, +} + +impl CounterStarts { + fn get(&self, metric: &'static str, series: &str, default_start: u64) -> u64 { + self.entries + .iter() + .find_map(|(key, start)| counter_key_matches(key, metric, series).then_some(*start)) + .unwrap_or(default_start) + } + + pub(super) fn get_joined( + &self, + metric: &'static str, + first: &str, + second: &'static str, + default_start: u64, + ) -> u64 { + self.entries + .iter() + .find_map(|(key, start)| { + counter_key_matches_joined(key, metric, first, second).then_some(*start) + }) + .unwrap_or(default_start) + } +} + +impl CounterTracker { + pub(super) fn snapshot( + &mut self, + default_start: u64, + now: u64, + cpu: Option<&CpuTimes>, + paging: Option<&PagingStats>, + processes: Option<&ProcessStats>, + disks: Option<&[DiskStats]>, + networks: Option<&[NetworkStats]>, + ) -> CounterStarts { + let mut starts = CounterStarts::default(); + if let Some(cpu) = cpu { + self.observe_all( + metric::CPU_TIME, + default_start, + now, + &[ + ("user", cpu.user), + ("nice", cpu.nice), + ("system", cpu.system), + ("idle", cpu.idle), + ("iowait", cpu.wait), + ("interrupt", cpu.interrupt), + ("steal", cpu.steal), + ], + &mut starts, + ); + } + if let Some(paging) = paging { + self.observe_all( + metric::PAGING_FAULTS, + default_start, + now, + &[ + ("minor", paging.minor_faults as f64), + ("major", paging.major_faults as f64), + ], + &mut starts, + ); + for (direction, fault_type, value) in [ + ("in", "major", paging.swap_in), + ("out", "major", paging.swap_out), + ("in", "minor", paging.page_in), + ("out", "minor", paging.page_out), + ] { + self.observe_joined( + metric::PAGING_OPERATIONS, + direction, + fault_type, + value as f64, + default_start, + now, + &mut starts, + ); + } + } + if let Some(processes) = processes { + self.observe( + metric::PROCESS_CREATED, + "", + processes.created as f64, + default_start, + now, + &mut starts, + ); + } + if let Some(disks) = disks { + let first_disk_entry = starts.entries.len(); + for disk in disks { + self.observe_disk_all( + metric::DISK_IO, + default_start, + now, + &disk.name, + &[ + ("read", disk.read_bytes as f64), + ("write", disk.write_bytes as f64), + ], + &mut starts, + ); + self.observe_disk_all( + metric::DISK_OPERATIONS, + default_start, + now, + &disk.name, + &[ + ("read", disk.read_ops as f64), + ("write", disk.write_ops as f64), + ], + &mut starts, + ); + self.observe( + metric::DISK_IO_TIME, + &disk.name, + disk.io_time_seconds, + default_start, + now, + &mut starts, + ); + self.observe_disk_all( + metric::DISK_OPERATION_TIME, + default_start, + now, + &disk.name, + &[ + ("read", disk.read_time_seconds), + ("write", disk.write_time_seconds), + ], + &mut starts, + ); + self.observe_disk_all( + metric::DISK_MERGED, + default_start, + now, + &disk.name, + &[ + ("read", disk.read_merged as f64), + ("write", disk.write_merged as f64), + ], + &mut starts, + ); + } + self.prune_scraped_family( + &starts.entries[first_disk_entry..], + &[ + metric::DISK_IO, + metric::DISK_OPERATIONS, + metric::DISK_IO_TIME, + metric::DISK_OPERATION_TIME, + metric::DISK_MERGED, + ], + ); + } + if let Some(networks) = networks { + let first_network_entry = starts.entries.len(); + for network in networks { + self.observe_network( + metric::NETWORK_IO, + default_start, + now, + network, + network.rx_bytes, + network.tx_bytes, + &mut starts, + ); + self.observe_network( + metric::NETWORK_PACKET_COUNT, + default_start, + now, + network, + network.rx_packets, + network.tx_packets, + &mut starts, + ); + self.observe_network( + metric::NETWORK_PACKET_DROPPED, + default_start, + now, + network, + network.rx_dropped, + network.tx_dropped, + &mut starts, + ); + self.observe_network( + metric::NETWORK_ERRORS, + default_start, + now, + network, + network.rx_errors, + network.tx_errors, + &mut starts, + ); + } + self.prune_scraped_family( + &starts.entries[first_network_entry..], + &[ + metric::NETWORK_IO, + metric::NETWORK_PACKET_COUNT, + metric::NETWORK_PACKET_DROPPED, + metric::NETWORK_ERRORS, + ], + ); + } + starts + } + + fn prune_scraped_family( + &mut self, + current_entries: &[(String, u64)], + metrics: &[&'static str], + ) { + let current_keys = current_entries + .iter() + .map(|(key, _)| key.as_str()) + .collect::>(); + self.states.retain(|key, _| { + !metrics + .iter() + .any(|metric| counter_key_is_metric(key, metric)) + || current_keys.contains(key.as_str()) + }); + } + + fn observe_all( + &mut self, + metric: &'static str, + default_start: u64, + now: u64, + values: &[(&str, f64)], + starts: &mut CounterStarts, + ) { + for (series, value) in values { + self.observe(metric, series, *value, default_start, now, starts); + } + } + + fn observe_disk_all( + &mut self, + metric: &'static str, + default_start: u64, + now: u64, + device: &str, + values: &[(&'static str, f64)], + starts: &mut CounterStarts, + ) { + for (direction, value) in values { + self.observe_joined( + metric, + device, + direction, + *value, + default_start, + now, + starts, + ); + } + } + + fn observe_network( + &mut self, + metric: &'static str, + default_start: u64, + now: u64, + network: &NetworkStats, + rx: u64, + tx: u64, + starts: &mut CounterStarts, + ) { + self.observe_joined( + metric, + &network.name, + "receive", + rx as f64, + default_start, + now, + starts, + ); + self.observe_joined( + metric, + &network.name, + "transmit", + tx as f64, + default_start, + now, + starts, + ); + } + + fn observe( + &mut self, + metric: &'static str, + series: &str, + value: f64, + default_start: u64, + now: u64, + starts: &mut CounterStarts, + ) { + self.observe_key( + counter_key(metric, series), + value, + default_start, + now, + starts, + ); + } + + fn observe_joined( + &mut self, + metric: &'static str, + first: &str, + second: &'static str, + value: f64, + default_start: u64, + now: u64, + starts: &mut CounterStarts, + ) { + self.observe_key( + counter_key_joined(metric, first, second), + value, + default_start, + now, + starts, + ); + } + + fn observe_key( + &mut self, + key: String, + value: f64, + default_start: u64, + now: u64, + starts: &mut CounterStarts, + ) { + let state = self.states.entry(key.clone()).or_insert(CounterState { + previous: value, + start_time_unix_nano: default_start, + }); + if state.start_time_unix_nano < default_start { + state.start_time_unix_nano = default_start; + } else if value < state.previous { + state.start_time_unix_nano = now; + } + state.previous = value; + starts.entries.push((key, state.start_time_unix_nano)); + } +} + +pub(super) fn counter_key(metric: &'static str, series: &str) -> String { + let mut key = String::with_capacity(metric.len() + 1 + series.len()); + key.push_str(metric); + key.push(COUNTER_KEY_SEPARATOR); + key.push_str(series); + key +} + +pub(super) fn counter_key_joined( + metric: &'static str, + first: &str, + second: &'static str, +) -> String { + let mut key = String::with_capacity(metric.len() + 2 + first.len() + second.len()); + key.push_str(metric); + key.push(COUNTER_KEY_SEPARATOR); + key.push_str(first); + key.push(COUNTER_KEY_SEPARATOR); + key.push_str(second); + key +} + +pub(super) fn counter_key_matches(key: &str, metric: &'static str, series: &str) -> bool { + key.strip_prefix(metric) + .and_then(|rest| rest.strip_prefix(COUNTER_KEY_SEPARATOR)) + == Some(series) +} + +pub(super) fn counter_key_matches_joined( + key: &str, + metric: &'static str, + first: &str, + second: &'static str, +) -> bool { + let Some(series) = key + .strip_prefix(metric) + .and_then(|rest| rest.strip_prefix(COUNTER_KEY_SEPARATOR)) + else { + return false; + }; + series + .strip_prefix(first) + .and_then(|rest| rest.strip_prefix(COUNTER_KEY_SEPARATOR)) + == Some(second) +} + +fn counter_key_is_metric(key: &str, metric: &'static str) -> bool { + key.strip_prefix(metric) + .is_some_and(|rest| rest.starts_with(COUNTER_KEY_SEPARATOR)) +} + +pub(super) fn host_arch() -> Option<&'static str> { + match std::env::consts::ARCH { + "aarch64" => Some("arm64"), + "arm" => Some("arm32"), + "powerpc" => Some("ppc32"), + "powerpc64" => Some("ppc64"), + "x86" => Some("x86"), + "x86_64" => Some("amd64"), + _ => None, + } +} diff --git a/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/readings.rs b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/readings.rs new file mode 100644 index 0000000000..16c7aa4a35 --- /dev/null +++ b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/readings.rs @@ -0,0 +1,779 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +use crate::receivers::host_metrics_receiver::CompiledFilter; +use std::collections::HashSet; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::mpsc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use super::{BYTES_PER_KIB, DISKSTAT_SECTOR_BYTES, NANOS_PER_SEC}; + +#[derive(Copy, Clone, Default)] +pub(super) struct CpuTimes { + pub(super) user: f64, + pub(super) nice: f64, + pub(super) system: f64, + pub(super) idle: f64, + pub(super) wait: f64, + pub(super) interrupt: f64, + pub(super) steal: f64, +} + +#[derive(Clone, Default)] +pub(super) struct CpuInfo { + pub(super) logical_count: u64, + pub(super) physical_count: u64, + pub(super) frequencies_hz: Vec, +} + +#[derive(Copy, Clone, Default)] +pub(super) struct StatSnapshot { + pub(super) boot_time_unix_nano: u64, + pub(super) cpu: Option, + pub(super) processes: ProcessStats, +} + +#[derive(Copy, Clone, Default)] +pub(super) struct MemoryStats { + pub(super) total: u64, + pub(super) used: u64, + pub(super) free: u64, + pub(super) available: u64, + pub(super) has_available: bool, + pub(super) cached: u64, + pub(super) buffered: u64, + pub(super) shared: u64, + pub(super) slab_reclaimable: u64, + pub(super) slab_unreclaimable: u64, + pub(super) hugepages: HugepageStats, +} + +#[derive(Copy, Clone, Default)] +pub(super) struct HugepageStats { + pub(super) total: u64, + pub(super) free: u64, + pub(super) reserved: u64, + pub(super) surplus: u64, + pub(super) page_size_bytes: u64, +} + +#[derive(Copy, Clone, Default)] +pub(super) struct PagingStats { + pub(super) minor_faults: u64, + pub(super) major_faults: u64, + pub(super) page_in: u64, + pub(super) page_out: u64, + pub(super) swap_in: u64, + pub(super) swap_out: u64, +} + +#[derive(Default)] +pub(super) struct SwapStats { + pub(super) name: String, + pub(super) size: u64, + pub(super) used: u64, + pub(super) free: u64, +} + +#[derive(Copy, Clone, Default)] +pub(super) struct ProcessStats { + pub(super) running: u64, + pub(super) blocked: u64, + pub(super) created: u64, +} + +#[derive(Default)] +pub(super) struct DiskStats { + pub(super) name: String, + pub(super) limit_bytes: Option, + pub(super) read_bytes: u64, + pub(super) write_bytes: u64, + pub(super) read_ops: u64, + pub(super) write_ops: u64, + pub(super) read_merged: u64, + pub(super) write_merged: u64, + pub(super) read_time_seconds: f64, + pub(super) write_time_seconds: f64, + pub(super) io_time_seconds: f64, +} + +#[derive(Default)] +pub(super) struct FilesystemStats { + pub(super) device: String, + pub(super) mountpoint: String, + pub(super) fs_type: String, + pub(super) mode: &'static str, + pub(super) used: u64, + pub(super) free: u64, + pub(super) reserved: u64, + pub(super) limit_bytes: Option, +} + +/// Dedicated worker thread for `statvfs` calls. +/// +/// Intentionally not using `tokio::fs` / `spawn_blocking`: `statvfs` can block +/// indefinitely on unhealthy remote or FUSE mounts, and Tokio cannot cancel an +/// in-flight blocking task. With `spawn_blocking`, repeated scrapes during a +/// hang could accumulate stuck tasks on Tokio's global blocking pool, affecting +/// unrelated callers. +/// +/// The dedicated thread plus `sync_channel(1)` caps the blast radius at one +/// worker thread and at most one queued request per receiver. Callers still use +/// a per-mount timeout, and once the queue is full, later scrapes fail fast +/// instead of creating more blocking work. This fits the dfengine +/// thread-per-core / core-locality model better than offloading to Tokio's +/// shared blocking pool. +pub(super) struct FilesystemStatWorker { + tx: mpsc::SyncSender, +} + +pub(super) struct FilesystemStatRequest { + path: PathBuf, + response: mpsc::Sender>, +} + +pub(super) struct FilesystemStat { + pub(super) total_bytes: u64, + pub(super) free_bytes: u64, + pub(super) available_bytes: u64, +} + +impl FilesystemStatWorker { + pub(super) fn new() -> io::Result { + let (tx, rx) = mpsc::sync_channel::(1); + let _handle = std::thread::Builder::new() + .name("host-metrics-statvfs".to_owned()) + .spawn(move || { + while let Ok(request) = rx.recv() { + let result = statvfs_bytes(&request.path); + let _ = request.response.send(result); + } + }) + .map_err(io::Error::other)?; + Ok(Self { tx }) + } + + #[cfg(test)] + pub(super) fn disconnected_for_test() -> Self { + let (tx, rx) = mpsc::sync_channel::(1); + drop(rx); + Self { tx } + } + + pub(super) fn statvfs(&self, path: PathBuf, timeout: Duration) -> io::Result { + let (response, rx) = mpsc::channel(); + self.tx + .try_send(FilesystemStatRequest { path, response }) + .map_err(|err| match err { + mpsc::TrySendError::Full(_) => { + io::Error::new(io::ErrorKind::TimedOut, "statvfs worker is busy") + } + mpsc::TrySendError::Disconnected(_) => { + io::Error::new(io::ErrorKind::BrokenPipe, "statvfs worker stopped") + } + })?; + rx.recv_timeout(timeout).map_err(|err| match err { + mpsc::RecvTimeoutError::Timeout => { + io::Error::new(io::ErrorKind::TimedOut, "statvfs timed out") + } + mpsc::RecvTimeoutError::Disconnected => { + io::Error::new(io::ErrorKind::BrokenPipe, "statvfs worker stopped") + } + })? + } +} + +fn statvfs_bytes(path: &Path) -> io::Result { + let stat = nix::sys::statvfs::statvfs(path).map_err(io::Error::other)?; + let block_size = stat.fragment_size(); + Ok(FilesystemStat { + total_bytes: stat.blocks().saturating_mul(block_size), + free_bytes: stat.blocks_free().saturating_mul(block_size), + available_bytes: stat.blocks_available().saturating_mul(block_size), + }) +} + +#[derive(Default)] +pub(super) struct NetworkStats { + pub(super) name: String, + pub(super) rx_bytes: u64, + pub(super) tx_bytes: u64, + pub(super) rx_packets: u64, + pub(super) tx_packets: u64, + pub(super) rx_errors: u64, + pub(super) tx_errors: u64, + pub(super) rx_dropped: u64, + pub(super) tx_dropped: u64, +} + +pub(super) fn parse_stat(input: &str, clk_tck: f64) -> StatSnapshot { + let mut snapshot = StatSnapshot::default(); + for line in input.lines() { + if let Some(rest) = line.strip_prefix("cpu ") { + snapshot.cpu = parse_cpu_total(rest, clk_tck); + } else if let Some(value) = line.strip_prefix("btime ") { + snapshot.boot_time_unix_nano = parse_u64(value).saturating_mul(NANOS_PER_SEC); + } else if let Some(value) = line.strip_prefix("procs_running ") { + snapshot.processes.running = parse_u64(value); + } else if let Some(value) = line.strip_prefix("procs_blocked ") { + snapshot.processes.blocked = parse_u64(value); + } else if let Some(value) = line.strip_prefix("processes ") { + snapshot.processes.created = parse_u64(value); + } + } + snapshot +} + +pub(super) fn parse_cpu_total(input: &str, clk_tck: f64) -> Option { + let mut fields = [0_u64; 10]; + let mut count = 0; + for (idx, token) in input.split_whitespace().take(fields.len()).enumerate() { + fields[idx] = parse_u64(token); + count += 1; + } + if count < 4 { + return None; + } + + let user = fields[0].saturating_sub(fields[8]); + let nice = fields[1].saturating_sub(fields[9]); + Some(CpuTimes { + user: ticks_to_seconds(user, clk_tck), + nice: ticks_to_seconds(nice, clk_tck), + system: ticks_to_seconds(fields[2], clk_tck), + idle: ticks_to_seconds(fields[3], clk_tck), + wait: ticks_to_seconds(fields[4], clk_tck), + interrupt: ticks_to_seconds(fields[5].saturating_add(fields[6]), clk_tck), + steal: ticks_to_seconds(fields[7], clk_tck), + }) +} + +pub(super) fn cpu_utilization(previous: CpuTimes, current: CpuTimes) -> Option { + let user = counter_delta(previous.user, current.user)?; + let nice = counter_delta(previous.nice, current.nice)?; + let system = counter_delta(previous.system, current.system)?; + let idle = counter_delta(previous.idle, current.idle)?; + let wait = counter_delta(previous.wait, current.wait)?; + let interrupt = counter_delta(previous.interrupt, current.interrupt)?; + let steal = counter_delta(previous.steal, current.steal)?; + let total = user + nice + system + idle + wait + interrupt + steal; + (total > 0.0).then(|| CpuTimes { + user: user / total, + nice: nice / total, + system: system / total, + idle: idle / total, + wait: wait / total, + interrupt: interrupt / total, + steal: steal / total, + }) +} + +pub(super) fn counter_delta(previous: f64, current: f64) -> Option { + (current >= previous).then_some(current - previous) +} + +pub(super) fn parse_cpuinfo(input: &str) -> CpuInfo { + let mut logical_count = 0; + let mut frequencies_hz = Vec::new(); + let mut physical_cores = HashSet::new(); + let mut physical_id = None; + let mut core_id = None; + + for line in input.lines() { + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let key = key.trim(); + let value = value.trim(); + match key { + "processor" => { + logical_count += 1; + if let (Some(physical), Some(core)) = (physical_id.take(), core_id.take()) { + let _ = physical_cores.insert((physical, core)); + } + } + "physical id" => physical_id = Some(parse_u64(value)), + "core id" => core_id = Some(parse_u64(value)), + "cpu MHz" => { + if let Ok(mhz) = value.parse::() { + frequencies_hz.push(mhz * 1_000_000.0); + } + } + _ => {} + } + } + if let (Some(physical), Some(core)) = (physical_id, core_id) { + let _ = physical_cores.insert((physical, core)); + } + + let physical_count = u64::try_from(physical_cores.len()) + .ok() + .filter(|count| *count != 0) + .unwrap_or(logical_count); + CpuInfo { + logical_count, + physical_count, + frequencies_hz, + } +} + +pub(super) fn parse_meminfo(input: &str) -> Option { + let mut total = 0; + let mut free = 0; + let mut available = None; + let mut buffers = 0; + let mut cached = 0; + let mut shared = 0; + let mut slab_reclaimable = 0; + let mut slab_unreclaimable = 0; + let mut hugepages = HugepageStats::default(); + + for line in input.lines() { + let mut fields = line.split_whitespace(); + let Some(key) = fields.next() else { + continue; + }; + let raw_value = fields.next().map(parse_u64).unwrap_or_default(); + let value = raw_value * BYTES_PER_KIB; + match key.trim_end_matches(':') { + "MemTotal" => total = value, + "MemFree" => free = value, + "MemAvailable" => available = Some(value), + "Buffers" => buffers = value, + "Cached" => cached = value, + "Shmem" => shared = value, + "SReclaimable" => slab_reclaimable = value, + "SUnreclaim" => slab_unreclaimable = value, + "HugePages_Total" => hugepages.total = raw_value, + "HugePages_Free" => hugepages.free = raw_value, + "HugePages_Rsvd" => hugepages.reserved = raw_value, + "HugePages_Surp" => hugepages.surplus = raw_value, + "Hugepagesize" => hugepages.page_size_bytes = value, + _ => {} + } + } + + if total == 0 { + return None; + } + let has_available = available.is_some(); + let available = + available.unwrap_or_else(|| free.saturating_add(buffers).saturating_add(cached)); + Some(MemoryStats { + total, + used: total.saturating_sub(available), + free, + available, + has_available, + cached, + buffered: buffers, + shared, + slab_reclaimable, + slab_unreclaimable, + hugepages, + }) +} + +pub(super) fn parse_uptime(input: &str) -> Option { + input.split_whitespace().next()?.parse().ok() +} + +pub(super) fn parse_vmstat(input: &str) -> PagingStats { + let mut total_faults = 0; + let mut major_faults = 0; + let mut page_in = 0; + let mut page_out = 0; + let mut swap_in = 0; + let mut swap_out = 0; + + for line in input.lines() { + let mut fields = line.split_whitespace(); + let Some(key) = fields.next() else { + continue; + }; + let value = fields.next().map(parse_u64).unwrap_or_default(); + match key { + "pgfault" => total_faults = value, + "pgmajfault" => major_faults = value, + "pgpgin" => page_in = value, + "pgpgout" => page_out = value, + "pswpin" => swap_in = value, + "pswpout" => swap_out = value, + _ => {} + } + } + + PagingStats { + minor_faults: total_faults.saturating_sub(major_faults), + major_faults, + page_in, + page_out, + swap_in, + swap_out, + } +} + +pub(super) fn parse_swaps(input: &str) -> Vec { + let mut swaps = Vec::new(); + for line in input.lines().skip(1) { + let mut fields = line.split_whitespace(); + let Some(name) = fields.next() else { + continue; + }; + let _kind = fields.next(); + let Some(size_kib) = fields.next() else { + continue; + }; + let Some(used_kib) = fields.next() else { + continue; + }; + let size = parse_u64(size_kib).saturating_mul(BYTES_PER_KIB); + let used = parse_u64(used_kib).saturating_mul(BYTES_PER_KIB); + swaps.push(SwapStats { + name: name.to_owned(), + size, + used, + free: size.saturating_sub(used), + }); + } + swaps +} + +pub(super) fn parse_diskstats( + input: &str, + include: Option<&CompiledFilter>, + exclude: Option<&CompiledFilter>, +) -> Vec { + let mut disks = Vec::new(); + for line in input.lines() { + let mut fields = line.split_whitespace(); + let _major = fields.next(); + let _minor = fields.next(); + let Some(name) = fields.next() else { + continue; + }; + if !filter_allows(name, include, exclude) { + continue; + } + let Some(read_ops) = fields.next() else { + continue; + }; + let Some(read_merged) = fields.next() else { + continue; + }; + let Some(read_sectors) = fields.next() else { + continue; + }; + let Some(read_ms) = fields.next() else { + continue; + }; + let Some(write_ops) = fields.next() else { + continue; + }; + let Some(write_merged) = fields.next() else { + continue; + }; + let Some(write_sectors) = fields.next() else { + continue; + }; + let Some(write_ms) = fields.next() else { + continue; + }; + let _in_progress = fields.next(); + let Some(io_ms) = fields.next() else { + continue; + }; + disks.push(DiskStats { + name: name.to_owned(), + limit_bytes: None, + read_ops: parse_u64(read_ops), + read_bytes: parse_u64(read_sectors).saturating_mul(DISKSTAT_SECTOR_BYTES), + write_ops: parse_u64(write_ops), + write_bytes: parse_u64(write_sectors).saturating_mul(DISKSTAT_SECTOR_BYTES), + read_merged: parse_u64(read_merged), + write_merged: parse_u64(write_merged), + read_time_seconds: millis_to_seconds(parse_u64(read_ms)), + write_time_seconds: millis_to_seconds(parse_u64(write_ms)), + io_time_seconds: millis_to_seconds(parse_u64(io_ms)), + }); + } + disks +} + +pub(super) struct FilesystemMount { + pub(super) device: String, + pub(super) mountpoint: String, + pub(super) fs_type: String, + pub(super) mode: &'static str, + pub(super) emit_limit: bool, +} + +#[derive(Clone, Copy, Default)] +pub(super) struct FilesystemFilters<'a> { + pub(super) include_devices: Option<&'a CompiledFilter>, + pub(super) exclude_devices: Option<&'a CompiledFilter>, + pub(super) include_fs_types: Option<&'a CompiledFilter>, + pub(super) exclude_fs_types: Option<&'a CompiledFilter>, + pub(super) include_mount_points: Option<&'a CompiledFilter>, + pub(super) exclude_mount_points: Option<&'a CompiledFilter>, +} + +pub(super) fn parse_mountinfo( + input: &str, + include_virtual_filesystems: bool, + include_remote_filesystems: bool, + emit_limit: bool, + filters: FilesystemFilters<'_>, +) -> Vec { + let mut mounts = Vec::new(); + for line in input.lines() { + let Some(separator) = line.find(" - ") else { + continue; + }; + let mut pre_fields = line[..separator].split_whitespace(); + let _mount_id = pre_fields.next(); + let _parent_id = pre_fields.next(); + let _major_minor = pre_fields.next(); + let _root = pre_fields.next(); + let Some(mountpoint) = pre_fields.next() else { + continue; + }; + let Some(options) = pre_fields.next() else { + continue; + }; + + let mut post_fields = line[separator + 3..].split_whitespace(); + let Some(fs_type) = post_fields.next() else { + continue; + }; + let Some(device) = post_fields.next() else { + continue; + }; + if !include_virtual_filesystems && is_virtual_filesystem_type(fs_type) { + continue; + } + if !include_remote_filesystems && is_remote_filesystem_type(fs_type) { + continue; + } + if !filter_allows(fs_type, filters.include_fs_types, filters.exclude_fs_types) { + continue; + } + let device = unescape_mountinfo(device); + if !filter_allows(&device, filters.include_devices, filters.exclude_devices) { + continue; + } + let mountpoint = unescape_mountinfo(mountpoint); + if !filter_allows( + &mountpoint, + filters.include_mount_points, + filters.exclude_mount_points, + ) { + continue; + } + mounts.push(FilesystemMount { + device, + mountpoint, + fs_type: fs_type.to_owned(), + mode: filesystem_mode(options), + emit_limit, + }); + } + mounts +} + +pub(super) fn filesystem_mode(options: &str) -> &'static str { + if options.split(',').any(|option| option == "ro") { + "ro" + } else { + "rw" + } +} + +pub(super) fn is_remote_filesystem_type(fs_type: &str) -> bool { + if fs_type == "fuse" || fs_type == "fuseblk" || fs_type.starts_with("fuse.") { + return true; + } + matches!(fs_type, "nfs" | "nfs4" | "cifs" | "smb3" | "9p") +} + +pub(super) fn is_virtual_filesystem_type(fs_type: &str) -> bool { + matches!( + fs_type, + "autofs" + | "bpf" + | "binfmt_misc" + | "cgroup" + | "cgroup2" + | "configfs" + | "debugfs" + | "devpts" + | "devtmpfs" + | "efivarfs" + | "fusectl" + | "hugetlbfs" + | "mqueue" + | "nsfs" + | "overlay" + | "proc" + | "pstore" + | "ramfs" + | "rpc_pipefs" + | "securityfs" + | "selinuxfs" + | "squashfs" + | "sysfs" + | "tmpfs" + | "tracefs" + ) +} + +pub(super) fn unescape_mountinfo(input: &str) -> String { + let bytes = input.as_bytes(); + let mut escaped = None; + for idx in 0..bytes.len() { + if bytes[idx] == b'\\' && idx + 4 <= bytes.len() { + escaped = Some(idx); + break; + } + } + let Some(first_escape) = escaped else { + return input.to_owned(); + }; + + let mut output = Vec::with_capacity(input.len()); + output.extend_from_slice(&bytes[..first_escape]); + let mut idx = first_escape; + while idx < bytes.len() { + if bytes[idx] == b'\\' && idx + 4 <= bytes.len() { + let octal = &input[idx + 1..idx + 4]; + if let Ok(value) = u8::from_str_radix(octal, 8) { + output.push(value); + idx += 4; + continue; + } + } + output.push(bytes[idx]); + idx += 1; + } + String::from_utf8_lossy(&output).into_owned() +} + +pub(super) fn parse_netdev( + input: &str, + include: Option<&CompiledFilter>, + exclude: Option<&CompiledFilter>, +) -> Vec { + let mut interfaces = Vec::new(); + for line in input.lines().skip(2) { + let Some((name, values)) = line.split_once(':') else { + continue; + }; + let name = name.trim(); + if !filter_allows(name, include, exclude) { + continue; + } + let mut fields = values.split_whitespace(); + let Some(rx_bytes) = fields.next() else { + continue; + }; + let Some(rx_packets) = fields.next() else { + continue; + }; + let Some(rx_errors) = fields.next() else { + continue; + }; + let Some(rx_dropped) = fields.next() else { + continue; + }; + let _rx_fifo = fields.next(); + let _rx_frame = fields.next(); + let _rx_compressed = fields.next(); + let _rx_multicast = fields.next(); + let Some(tx_bytes) = fields.next() else { + continue; + }; + let Some(tx_packets) = fields.next() else { + continue; + }; + let Some(tx_errors) = fields.next() else { + continue; + }; + let Some(tx_dropped) = fields.next() else { + continue; + }; + interfaces.push(NetworkStats { + name: name.to_owned(), + rx_bytes: parse_u64(rx_bytes), + rx_packets: parse_u64(rx_packets), + tx_bytes: parse_u64(tx_bytes), + tx_packets: parse_u64(tx_packets), + rx_errors: parse_u64(rx_errors), + tx_errors: parse_u64(tx_errors), + rx_dropped: parse_u64(rx_dropped), + tx_dropped: parse_u64(tx_dropped), + }); + } + interfaces +} + +pub(super) fn filter_allows( + value: &str, + include: Option<&CompiledFilter>, + exclude: Option<&CompiledFilter>, +) -> bool { + include.is_none_or(|filter| filter.matches(value)) + && !exclude.is_some_and(|filter| filter.matches(value)) +} + +pub(super) fn record_partial_error( + partial_errors: &mut u64, + first_error: &mut Option, + err: io::Error, +) { + *partial_errors = partial_errors.saturating_add(1); + if first_error.is_none() { + *first_error = Some(err); + } +} + +pub(super) fn frequency_hz_i64(value: f64) -> i64 { + if !value.is_finite() || value <= 0.0 { + return 0; + } + if value >= i64::MAX as f64 { + return i64::MAX; + } + value.round() as i64 +} + +pub(super) fn parse_u64(input: &str) -> u64 { + input.parse().unwrap_or_default() +} + +pub(super) fn ticks_to_seconds(ticks: u64, clk_tck: f64) -> f64 { + ticks as f64 / clk_tck +} + +pub(super) fn millis_to_seconds(ms: u64) -> f64 { + ms as f64 / 1_000.0 +} + +#[allow(unsafe_code)] +pub(super) fn clock_ticks_per_second() -> f64 { + // SAFETY: _SC_CLK_TCK is a valid sysconf name; the call has no side effects. + let ticks = unsafe { libc::sysconf(libc::_SC_CLK_TCK) }; + if ticks > 0 { ticks as f64 } else { 100.0 } +} + +pub(super) fn now_unix_nano() -> u64 { + let Ok(duration) = SystemTime::now().duration_since(UNIX_EPOCH) else { + return 0; + }; + duration.as_secs().saturating_mul(NANOS_PER_SEC) + u64::from(duration.subsec_nanos()) +} + +pub(super) fn saturating_i64(value: u64) -> i64 { + i64::try_from(value).unwrap_or(i64::MAX) +} diff --git a/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/tests.rs b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/tests.rs new file mode 100644 index 0000000000..799c197fb0 --- /dev/null +++ b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/procfs/tests.rs @@ -0,0 +1,1958 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use crate::receivers::host_metrics_receiver::semconv::{attr, metric}; +#[cfg(feature = "dev-tools")] +use otap_df_pdata::proto::opentelemetry::common::v1::AnyValue; +use otap_df_pdata::proto::opentelemetry::common::v1::{KeyValue, any_value}; +#[cfg(feature = "dev-tools")] +use otap_df_pdata::proto::opentelemetry::metrics::v1::NumberDataPoint; +use otap_df_pdata::proto::opentelemetry::metrics::v1::{ + AggregationTemporality, Metric, MetricsData, metric as otlp_metric, number_data_point, +}; +use otap_df_pdata::testing::round_trip::decode_metrics; +use projection::{CounterStarts, counter_key, counter_key_joined, counter_key_matches_joined}; +#[cfg(feature = "dev-tools")] +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{self, ErrorKind}; +use std::path::PathBuf; +#[cfg(feature = "dev-tools")] +use weaver_common::{result::WResult, vdir::VirtualDirectoryPath}; +#[cfg(feature = "dev-tools")] +use weaver_forge::registry::ResolvedRegistry; +#[cfg(feature = "dev-tools")] +use weaver_resolver::SchemaResolver; +#[cfg(feature = "dev-tools")] +use weaver_semconv::{ + attribute::{ + AttributeType, BasicRequirementLevelSpec, PrimitiveOrArrayTypeSpec, RequirementLevel, + ValueSpec, + }, + group::{GroupType, InstrumentSpec}, + registry_repo::RegistryRepo, +}; + +fn block_on_scrape(source: &mut ProcfsSource, due: ProcfsFamilies) -> io::Result { + tokio::runtime::Builder::new_current_thread() + .build() + .expect("runtime") + .block_on(source.scrape_due(due)) +} + +#[test] +fn projection_uses_expected_metric_shapes() { + let data = projection_fixture_request(); + + let resource_metrics = data.resource_metrics.first().expect("resource metrics"); + let resource = resource_metrics.resource.as_ref().expect("resource"); + assert_has_attr(&resource.attributes, attr::OS_TYPE, "linux"); + assert_has_attr(&resource.attributes, attr::HOST_ID, "host-id"); + assert_has_attr(&resource.attributes, attr::HOST_NAME, "host-name"); + assert_has_attr(&resource.attributes, attr::HOST_ARCH, "amd64"); + + let metrics = &resource_metrics.scope_metrics[0].metrics; + assert_metric_shape(metrics, metric::CPU_TIME, "s", Some(true)); + assert_first_point_attr(metrics, metric::CPU_TIME, attr::CPU_MODE, "user"); + assert_sum_point_attr(metrics, metric::CPU_TIME, attr::CPU_MODE, "iowait"); + assert_metric_shape(metrics, metric::CPU_UTILIZATION, "1", None); + assert_first_point_attr(metrics, metric::CPU_UTILIZATION, attr::CPU_MODE, "user"); + assert_metric_shape(metrics, metric::CPU_LOGICAL_COUNT, "{cpu}", Some(false)); + assert_metric_shape(metrics, metric::CPU_PHYSICAL_COUNT, "{cpu}", Some(false)); + assert_metric_shape(metrics, metric::CPU_FREQUENCY, "Hz", None); + assert_first_point_int(metrics, metric::CPU_FREQUENCY, 2_400_000_000); + assert_first_point_attr_int(metrics, metric::CPU_FREQUENCY, attr::CPU_LOGICAL_NUMBER, 0); + assert_metric_shape(metrics, metric::MEMORY_USAGE, "By", Some(false)); + assert_first_point_attr( + metrics, + metric::MEMORY_USAGE, + attr::SYSTEM_MEMORY_STATE, + "used", + ); + assert_metric_shape(metrics, metric::MEMORY_UTILIZATION, "1", None); + assert_metric_shape(metrics, metric::MEMORY_LINUX_AVAILABLE, "By", Some(false)); + assert_metric_shape(metrics, metric::MEMORY_LINUX_SLAB_USAGE, "By", Some(false)); + assert_metric_shape(metrics, metric::MEMORY_LIMIT, "By", Some(false)); + assert_metric_shape(metrics, metric::MEMORY_LINUX_SHARED, "By", Some(false)); + assert_metric_shape( + metrics, + metric::MEMORY_LINUX_HUGEPAGES_LIMIT, + "{page}", + Some(false), + ); + assert_metric_shape( + metrics, + metric::MEMORY_LINUX_HUGEPAGES_PAGE_SIZE, + "By", + Some(false), + ); + assert_metric_shape( + metrics, + metric::MEMORY_LINUX_HUGEPAGES_RESERVED, + "{page}", + Some(false), + ); + assert_metric_shape( + metrics, + metric::MEMORY_LINUX_HUGEPAGES_SURPLUS, + "{page}", + Some(false), + ); + assert_metric_shape( + metrics, + metric::MEMORY_LINUX_HUGEPAGES_USAGE, + "{page}", + Some(false), + ); + assert_first_point_attr( + metrics, + metric::MEMORY_LINUX_HUGEPAGES_USAGE, + attr::SYSTEM_MEMORY_LINUX_HUGEPAGES_STATE, + "used", + ); + assert_metric_shape( + metrics, + metric::MEMORY_LINUX_HUGEPAGES_UTILIZATION, + "1", + None, + ); + assert_metric_shape(metrics, metric::UPTIME, "s", None); + assert_metric_shape(metrics, metric::PAGING_FAULTS, "{fault}", Some(true)); + assert_first_point_attr( + metrics, + metric::PAGING_FAULTS, + attr::SYSTEM_PAGING_FAULT_TYPE, + "minor", + ); + assert_metric_shape( + metrics, + metric::PAGING_OPERATIONS, + "{operation}", + Some(true), + ); + assert_sum_point_attr( + metrics, + metric::PAGING_OPERATIONS, + attr::SYSTEM_PAGING_DIRECTION, + "in", + ); + assert_sum_point_attr( + metrics, + metric::PAGING_OPERATIONS, + attr::SYSTEM_PAGING_FAULT_TYPE, + "minor", + ); + assert_metric_shape(metrics, metric::PAGING_USAGE, "By", Some(false)); + assert_first_point_attr( + metrics, + metric::PAGING_USAGE, + attr::SYSTEM_DEVICE, + "/dev/swap", + ); + assert_metric_shape(metrics, metric::PAGING_UTILIZATION, "1", None); + assert_metric_shape(metrics, metric::PROCESS_COUNT, "{process}", Some(false)); + assert_sum_point_attr( + metrics, + metric::PROCESS_COUNT, + attr::PROCESS_STATE, + "running", + ); + assert_metric_shape(metrics, metric::PROCESS_CREATED, "{process}", Some(true)); + assert_metric_shape(metrics, metric::DISK_IO, "By", Some(true)); + assert_first_point_attr(metrics, metric::DISK_IO, attr::DISK_IO_DIRECTION, "read"); + assert_metric_shape(metrics, metric::DISK_OPERATIONS, "{operation}", Some(true)); + assert_metric_shape(metrics, metric::DISK_IO_TIME, "s", Some(true)); + assert_first_point_attr(metrics, metric::DISK_IO_TIME, attr::SYSTEM_DEVICE, "sda"); + assert_metric_shape(metrics, metric::DISK_OPERATION_TIME, "s", Some(true)); + assert_metric_shape(metrics, metric::DISK_MERGED, "{operation}", Some(true)); + assert_metric_shape(metrics, metric::DISK_LIMIT, "By", Some(false)); + assert_first_point_attr(metrics, metric::DISK_LIMIT, attr::SYSTEM_DEVICE, "sda"); + assert_metric_shape(metrics, metric::FILESYSTEM_USAGE, "By", Some(false)); + assert_first_point_attr( + metrics, + metric::FILESYSTEM_USAGE, + attr::SYSTEM_FILESYSTEM_STATE, + "used", + ); + assert_metric_shape(metrics, metric::FILESYSTEM_UTILIZATION, "1", None); + assert_metric_shape(metrics, metric::FILESYSTEM_LIMIT, "By", Some(false)); + assert_no_first_point_attr( + metrics, + metric::FILESYSTEM_LIMIT, + attr::SYSTEM_FILESYSTEM_STATE, + ); + assert_metric_shape(metrics, metric::NETWORK_IO, "By", Some(true)); + assert_first_point_attr( + metrics, + metric::NETWORK_IO, + attr::NETWORK_INTERFACE_NAME, + "eth0", + ); + assert_metric_shape( + metrics, + metric::NETWORK_PACKET_COUNT, + "{packet}", + Some(true), + ); + assert_first_point_attr( + metrics, + metric::NETWORK_PACKET_COUNT, + attr::SYSTEM_DEVICE, + "eth0", + ); + assert_metric_shape( + metrics, + metric::NETWORK_PACKET_DROPPED, + "{packet}", + Some(true), + ); + assert_first_point_attr( + metrics, + metric::NETWORK_PACKET_DROPPED, + attr::NETWORK_INTERFACE_NAME, + "eth0", + ); + assert_metric_shape(metrics, metric::NETWORK_ERRORS, "{error}", Some(true)); +} + +#[cfg(feature = "dev-tools")] +#[test] +#[ignore = "dev-only semconv drift check; may access a local or remote semantic-conventions registry"] +fn emitted_phase1_metric_shapes_match_weaver_semconv() { + let registry = load_semconv_registry(); + let semconv_shapes = semconv_system_metric_shapes(®istry); + let emitted_shapes = emitted_phase1_metric_shapes(); + + for (name, emitted) in emitted_shapes { + let semconv = semconv_shapes + .get(&name) + .unwrap_or_else(|| panic!("missing semconv metric {name}")); + + assert_eq!(emitted.unit, semconv.unit, "unit mismatch for {name}"); + assert_eq!( + emitted.monotonic, semconv.monotonic, + "instrument/temporality mismatch for {name}" + ); + assert_eq!( + emitted.value_type, semconv.value_type, + "metric value type mismatch for {name}" + ); + + for attr in &semconv.attributes { + assert!( + emitted.attributes.contains(attr), + "missing semconv attribute {attr} on {name}" + ); + } + for attr in &emitted.attributes { + assert!( + semconv.all_attributes.contains(attr), + "unexpected semconv attribute {attr} on {name}" + ); + } + for (attr, emitted_kind) in &emitted.attribute_types { + let Some(semconv_kind) = semconv.attribute_types.get(attr) else { + continue; + }; + assert_eq!( + emitted_kind, semconv_kind, + "attribute value type mismatch for {attr} on {name}" + ); + } + for (attr, values) in &emitted.enum_values { + let Some(allowed_values) = semconv.enum_values.get(attr) else { + continue; + }; + for value in values { + if is_intentional_semconv_enum_value_gap(name.as_str(), attr.as_str(), value) { + continue; + } + assert!( + allowed_values.contains(value), + "unexpected enum value {attr}={value} on {name}" + ); + } + } + } +} + +#[test] +fn projection_uses_counter_start_overrides_for_reset_series() { + let data = decode_metrics( + HostSnapshot { + now_unix_nano: 2_000, + start_time_unix_nano: 1_000, + counter_starts: CounterStarts { + entries: vec![(counter_key(metric::PROCESS_CREATED, ""), 1_500)], + }, + processes: Some(ProcessStats { + created: 99, + ..ProcessStats::default() + }), + ..HostSnapshot::default() + } + .into_otap_records() + .expect("encode ok"), + ); + + let metrics = &data.resource_metrics[0].scope_metrics[0].metrics; + assert_first_sum_point_start(metrics, metric::PROCESS_CREATED, 1_500); +} + +#[test] +fn counter_tracker_rebaselines_reset_series_only() { + let mut tracker = CounterTracker::default(); + let disks = vec![DiskStats { + name: "sda".to_owned(), + read_bytes: 100, + write_bytes: 200, + ..DiskStats::default() + }]; + let starts = tracker.snapshot(10, 20, None, None, None, Some(&disks), None); + + assert_eq!(starts.get_joined(metric::DISK_IO, "sda", "read", 10), 10); + assert_eq!(starts.get_joined(metric::DISK_IO, "sda", "write", 10), 10); + + let disks = vec![DiskStats { + name: "sda".to_owned(), + read_bytes: 50, + write_bytes: 250, + ..DiskStats::default() + }]; + let starts = tracker.snapshot(10, 30, None, None, None, Some(&disks), None); + + assert_eq!(starts.get_joined(metric::DISK_IO, "sda", "read", 10), 30); + assert_eq!(starts.get_joined(metric::DISK_IO, "sda", "write", 10), 10); +} + +#[test] +fn counter_tracker_rebaselines_paging_operations_by_direction_and_fault_type() { + let mut tracker = CounterTracker::default(); + let paging = PagingStats { + swap_in: 100, + swap_out: 200, + page_in: 300, + page_out: 400, + ..PagingStats::default() + }; + let starts = tracker.snapshot(10, 20, None, Some(&paging), None, None, None); + + assert_eq!( + starts.get_joined(metric::PAGING_OPERATIONS, "in", "major", 10), + 10 + ); + assert_eq!( + starts.get_joined(metric::PAGING_OPERATIONS, "out", "minor", 10), + 10 + ); + + let paging = PagingStats { + swap_in: 50, + swap_out: 250, + page_in: 350, + page_out: 450, + ..PagingStats::default() + }; + let starts = tracker.snapshot(10, 30, None, Some(&paging), None, None, None); + + assert_eq!( + starts.get_joined(metric::PAGING_OPERATIONS, "in", "major", 10), + 30 + ); + assert_eq!( + starts.get_joined(metric::PAGING_OPERATIONS, "out", "major", 10), + 10 + ); + assert_eq!( + starts.get_joined(metric::PAGING_OPERATIONS, "in", "minor", 10), + 10 + ); + assert_eq!( + starts.get_joined(metric::PAGING_OPERATIONS, "out", "minor", 10), + 10 + ); +} + +#[test] +fn counter_tracker_prunes_disappeared_disk_series_only_when_disk_is_scraped() { + let mut tracker = CounterTracker::default(); + let disks = vec![DiskStats { + name: "sda".to_owned(), + read_bytes: 100, + write_bytes: 200, + ..DiskStats::default() + }]; + let starts = tracker.snapshot(10, 20, None, None, None, Some(&disks), None); + assert_eq!(starts.get_joined(metric::DISK_IO, "sda", "read", 10), 10); + + let _ = tracker.snapshot(20, 30, None, None, None, None, None); + let disks = vec![DiskStats { + name: "sda".to_owned(), + read_bytes: 150, + write_bytes: 250, + ..DiskStats::default() + }]; + let starts = tracker.snapshot(30, 40, None, None, None, Some(&disks), None); + assert_eq!(starts.get_joined(metric::DISK_IO, "sda", "read", 30), 30); + + let empty_disks = Vec::new(); + let _ = tracker.snapshot(40, 50, None, None, None, Some(&empty_disks), None); + let disks = vec![DiskStats { + name: "sda".to_owned(), + read_bytes: 200, + write_bytes: 300, + ..DiskStats::default() + }]; + let starts = tracker.snapshot(50, 60, None, None, None, Some(&disks), None); + assert_eq!(starts.get_joined(metric::DISK_IO, "sda", "read", 50), 50); +} + +#[test] +fn counter_tracker_prunes_disappeared_network_series_only_when_network_is_scraped() { + let mut tracker = CounterTracker::default(); + let networks = vec![NetworkStats { + name: "veth0".to_owned(), + rx_bytes: 100, + tx_bytes: 200, + ..NetworkStats::default() + }]; + let starts = tracker.snapshot(10, 20, None, None, None, None, Some(&networks)); + assert_eq!( + starts.get_joined(metric::NETWORK_IO, "veth0", "receive", 10), + 10 + ); + + let _ = tracker.snapshot(20, 30, None, None, None, None, None); + let networks = vec![NetworkStats { + name: "veth0".to_owned(), + rx_bytes: 150, + tx_bytes: 250, + ..NetworkStats::default() + }]; + let starts = tracker.snapshot(30, 40, None, None, None, None, Some(&networks)); + assert_eq!( + starts.get_joined(metric::NETWORK_IO, "veth0", "receive", 30), + 30 + ); + + let empty_networks = Vec::new(); + let _ = tracker.snapshot(40, 50, None, None, None, None, Some(&empty_networks)); + let networks = vec![NetworkStats { + name: "veth0".to_owned(), + rx_bytes: 200, + tx_bytes: 300, + ..NetworkStats::default() + }]; + let starts = tracker.snapshot(50, 60, None, None, None, None, Some(&networks)); + assert_eq!( + starts.get_joined(metric::NETWORK_IO, "veth0", "receive", 50), + 50 + ); +} + +#[test] +fn counter_keys_do_not_collide_with_pipe_in_series_values() { + let metric = metric::DISK_IO; + let device = "read|write"; + let joined = counter_key_joined(metric, device, "read"); + assert!(!counter_key_matches_joined( + &joined, + metric, + "read", + "write|read" + )); + assert!(counter_key_matches_joined(&joined, metric, device, "read")); +} + +#[test] +fn scrape_due_emits_successful_families_after_partial_read_error() { + let root = tempfile::tempdir().expect("tempdir"); + let proc = root.path().join("proc"); + std::fs::create_dir(&proc).expect("proc dir"); + std::fs::write( + proc.join("meminfo"), + "MemTotal: 1000 kB\nMemFree: 100 kB\nMemAvailable: 200 kB\n", + ) + .expect("meminfo"); + // Cumulative metrics read /proc/stat once to cache boot time. Provide + // btime here so this test only exercises the missing diskstats error. + std::fs::write(proc.join("stat"), "btime 1700000000\n").expect("stat"); + let mut source = ProcfsSource::new( + Some(root.path()), + ProcfsConfig { + cpu: false, + memory: true, + paging: false, + system: false, + disk: true, + filesystem: false, + network: false, + processes: false, + cpu_utilization: false, + memory_limit: false, + memory_shared: false, + memory_hugepages: false, + disk_limit: false, + filesystem_include_virtual: false, + filesystem_include_remote: false, + filesystem_limit: false, + filesystem_include_devices: None, + filesystem_exclude_devices: None, + filesystem_include_fs_types: None, + filesystem_exclude_fs_types: None, + filesystem_include_mount_points: None, + filesystem_exclude_mount_points: None, + disk_include: None, + disk_exclude: None, + network_include: None, + network_exclude: None, + validation: HostViewValidationMode::None, + }, + ) + .expect("source"); + + let scrape = block_on_scrape( + &mut source, + ProcfsFamilies { + memory: true, + disk: true, + ..ProcfsFamilies::default() + }, + ) + .expect("partial scrape"); + + assert_eq!(scrape.partial_errors, 1); + assert!(scrape.snapshot.memory.is_some()); + assert!(scrape.snapshot.disks.is_empty()); +} + +#[test] +fn scrape_due_preserves_disk_counter_state_after_diskstats_read_error() { + let root = tempfile::tempdir().expect("tempdir"); + let proc = root.path().join("proc"); + std::fs::create_dir_all(&proc).expect("proc dir"); + std::fs::write(proc.join("stat"), "btime 1700000000\n").expect("stat"); + std::fs::write( + proc.join("meminfo"), + "MemTotal: 1000 kB\nMemFree: 100 kB\nMemAvailable: 200 kB\n", + ) + .expect("meminfo"); + std::fs::write( + proc.join("diskstats"), + "8 0 sda 1 0 100 0 2 0 200 0 0 0 0 0 0 0 0\n", + ) + .expect("diskstats"); + let mut source = ProcfsSource::new( + Some(root.path()), + ProcfsConfig { + cpu: false, + memory: true, + paging: false, + system: false, + disk: true, + filesystem: false, + network: false, + processes: false, + cpu_utilization: false, + memory_limit: false, + memory_shared: false, + memory_hugepages: false, + disk_limit: false, + filesystem_include_virtual: false, + filesystem_include_remote: false, + filesystem_limit: false, + filesystem_include_devices: None, + filesystem_exclude_devices: None, + filesystem_include_fs_types: None, + filesystem_exclude_fs_types: None, + filesystem_include_mount_points: None, + filesystem_exclude_mount_points: None, + disk_include: None, + disk_exclude: None, + network_include: None, + network_exclude: None, + validation: HostViewValidationMode::None, + }, + ) + .expect("source"); + + let first = block_on_scrape( + &mut source, + ProcfsFamilies { + disk: true, + ..ProcfsFamilies::default() + }, + ) + .expect("first disk scrape"); + let first_start = first + .snapshot + .counter_starts + .get_joined(metric::DISK_IO, "sda", "read", 0); + + std::fs::remove_file(proc.join("diskstats")).expect("remove diskstats"); + let partial = block_on_scrape( + &mut source, + ProcfsFamilies { + memory: true, + disk: true, + ..ProcfsFamilies::default() + }, + ) + .expect("partial scrape"); + assert_eq!(partial.partial_errors, 1); + assert!(partial.snapshot.disks.is_empty()); + + std::fs::write( + proc.join("diskstats"), + "8 0 sda 1 0 50 0 2 0 100 0 0 0 0 0 0 0 0\n", + ) + .expect("diskstats after reset"); + let after_error = block_on_scrape( + &mut source, + ProcfsFamilies { + disk: true, + ..ProcfsFamilies::default() + }, + ) + .expect("disk scrape after read error"); + let reset_start = + after_error + .snapshot + .counter_starts + .get_joined(metric::DISK_IO, "sda", "read", 0); + assert!( + reset_start > first_start, + "disk counter state should survive read errors so the later reset is detected" + ); +} + +#[test] +fn scrape_due_uses_stable_fallback_start_time_when_stat_is_unavailable() { + let root = tempfile::tempdir().expect("tempdir"); + let proc = root.path().join("proc"); + std::fs::create_dir(&proc).expect("proc dir"); + std::fs::write( + proc.join("diskstats"), + "8 0 sda 1 0 100 0 2 0 200 0 0 0 0 0 0 0 0\n", + ) + .expect("diskstats"); + let mut source = ProcfsSource::new( + Some(root.path()), + ProcfsConfig { + cpu: false, + memory: false, + paging: false, + system: false, + disk: true, + filesystem: false, + network: false, + processes: false, + cpu_utilization: false, + memory_limit: false, + memory_shared: false, + memory_hugepages: false, + disk_limit: false, + filesystem_include_virtual: false, + filesystem_include_remote: false, + filesystem_limit: false, + filesystem_include_devices: None, + filesystem_exclude_devices: None, + filesystem_include_fs_types: None, + filesystem_exclude_fs_types: None, + filesystem_include_mount_points: None, + filesystem_exclude_mount_points: None, + disk_include: None, + disk_exclude: None, + network_include: None, + network_exclude: None, + validation: HostViewValidationMode::None, + }, + ) + .expect("source"); + + let first = block_on_scrape( + &mut source, + ProcfsFamilies { + disk: true, + ..ProcfsFamilies::default() + }, + ) + .expect("first disk scrape"); + std::thread::sleep(Duration::from_millis(1)); + let second = block_on_scrape( + &mut source, + ProcfsFamilies { + disk: true, + ..ProcfsFamilies::default() + }, + ) + .expect("second disk scrape"); + + assert_eq!(first.partial_errors, 1); + assert_eq!(second.partial_errors, 1); + assert_eq!( + first.snapshot.start_time_unix_nano, + second.snapshot.start_time_unix_nano + ); +} + +#[test] +fn validation_requires_stat_for_cumulative_families() { + let root = tempfile::tempdir().expect("tempdir"); + let proc = root.path().join("proc"); + std::fs::create_dir(&proc).expect("proc dir"); + std::fs::write( + proc.join("diskstats"), + "8 0 sda 1 0 100 0 2 0 200 0 0 0 0 0 0 0 0\n", + ) + .expect("diskstats"); + + let err = match ProcfsSource::new( + Some(root.path()), + ProcfsConfig { + cpu: false, + memory: false, + paging: false, + system: false, + disk: true, + filesystem: false, + network: false, + processes: false, + cpu_utilization: false, + memory_limit: false, + memory_shared: false, + memory_hugepages: false, + disk_limit: false, + filesystem_include_virtual: false, + filesystem_include_remote: false, + filesystem_limit: false, + filesystem_include_devices: None, + filesystem_exclude_devices: None, + filesystem_include_fs_types: None, + filesystem_exclude_fs_types: None, + filesystem_include_mount_points: None, + filesystem_exclude_mount_points: None, + disk_include: None, + disk_exclude: None, + network_include: None, + network_exclude: None, + validation: HostViewValidationMode::FailSelected, + }, + ) { + Ok(_) => panic!("missing stat should fail validation for cumulative disk metrics"), + Err(err) => err, + }; + + assert_eq!(err.kind(), ErrorKind::NotFound); +} + +#[test] +fn filesystem_stat_worker_reports_disconnect_as_broken_pipe() { + let worker = FilesystemStatWorker::disconnected_for_test(); + match worker.statvfs(PathBuf::from("/"), Duration::from_millis(1)) { + Ok(_) => panic!("worker is disconnected"), + Err(err) => assert_eq!(err.kind(), ErrorKind::BrokenPipe), + } +} + +#[test] +fn scrape_due_fails_when_all_due_families_fail() { + let root = tempfile::tempdir().expect("tempdir"); + let mut source = ProcfsSource::new( + Some(root.path()), + ProcfsConfig { + cpu: false, + memory: true, + paging: false, + system: false, + disk: false, + filesystem: false, + network: false, + processes: false, + cpu_utilization: false, + memory_limit: false, + memory_shared: false, + memory_hugepages: false, + disk_limit: false, + filesystem_include_virtual: false, + filesystem_include_remote: false, + filesystem_limit: false, + filesystem_include_devices: None, + filesystem_exclude_devices: None, + filesystem_include_fs_types: None, + filesystem_exclude_fs_types: None, + filesystem_include_mount_points: None, + filesystem_exclude_mount_points: None, + disk_include: None, + disk_exclude: None, + network_include: None, + network_exclude: None, + validation: HostViewValidationMode::None, + }, + ) + .expect("source"); + + assert!( + block_on_scrape( + &mut source, + ProcfsFamilies { + memory: true, + ..ProcfsFamilies::default() + }, + ) + .is_err() + ); +} + +#[test] +fn scrape_due_reads_opt_in_disk_limit_from_sysfs() { + let root = tempfile::tempdir().expect("tempdir"); + let proc = root.path().join("proc"); + let sys_sda = root.path().join("sys/block/sda"); + std::fs::create_dir(&proc).expect("proc dir"); + std::fs::create_dir_all(&sys_sda).expect("sys block dir"); + std::fs::write( + proc.join("diskstats"), + "8 0 sda 1 0 2 3 4 0 5 6 0 0 0 0 0 0 0 0\n", + ) + .expect("diskstats"); + std::fs::write(sys_sda.join("size"), "4096\n").expect("disk size"); + let mut source = ProcfsSource::new( + Some(root.path()), + ProcfsConfig { + cpu: false, + memory: false, + paging: false, + system: false, + disk: true, + filesystem: false, + network: false, + processes: false, + cpu_utilization: false, + memory_limit: false, + memory_shared: false, + memory_hugepages: false, + disk_limit: true, + filesystem_include_virtual: false, + filesystem_include_remote: false, + filesystem_limit: false, + filesystem_include_devices: None, + filesystem_exclude_devices: None, + filesystem_include_fs_types: None, + filesystem_exclude_fs_types: None, + filesystem_include_mount_points: None, + filesystem_exclude_mount_points: None, + disk_include: None, + disk_exclude: None, + network_include: None, + network_exclude: None, + validation: HostViewValidationMode::None, + }, + ) + .expect("source"); + + let scrape = block_on_scrape( + &mut source, + ProcfsFamilies { + disk: true, + ..ProcfsFamilies::default() + }, + ) + .expect("disk scrape"); + + assert_eq!(scrape.snapshot.disks.len(), 1); + assert_eq!( + scrape.snapshot.disks[0].limit_bytes, + Some(4096 * DISKSTAT_SECTOR_BYTES) + ); +} + +#[test] +fn scrape_due_uses_boot_time_for_counter_only_family_ticks() { + let root = tempfile::tempdir().expect("tempdir"); + let proc = root.path().join("proc"); + let proc_one = proc.join("1"); + std::fs::create_dir_all(proc_one.join("net")).expect("proc dirs"); + std::fs::write(proc.join("stat"), "btime 123\n").expect("stat"); + std::fs::write( + proc.join("diskstats"), + "8 0 sda 1 0 2 3 4 0 5 6 0 0 0 0 0 0 0 0\n", + ) + .expect("diskstats"); + std::fs::write( + proc_one.join("net/dev"), + "Inter-| Receive | Transmit\n\ + face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n\ + eth0: 10 1 0 0 0 0 0 0 20 2 0 0 0 0 0 0\n", + ) + .expect("netdev"); + std::fs::write( + proc.join("vmstat"), + "pgfault 10\npgmajfault 1\npgpgin 2\npgpgout 3\npswpin 4\npswpout 5\n", + ) + .expect("vmstat"); + std::fs::write(proc.join("swaps"), "Filename Type Size Used Priority\n").expect("swaps"); + + let mut source = ProcfsSource::new( + Some(root.path()), + ProcfsConfig { + cpu: false, + memory: false, + paging: true, + system: false, + disk: true, + filesystem: false, + network: true, + processes: false, + cpu_utilization: false, + memory_limit: false, + memory_shared: false, + memory_hugepages: false, + disk_limit: false, + filesystem_include_virtual: false, + filesystem_include_remote: false, + filesystem_limit: false, + filesystem_include_devices: None, + filesystem_exclude_devices: None, + filesystem_include_fs_types: None, + filesystem_exclude_fs_types: None, + filesystem_include_mount_points: None, + filesystem_exclude_mount_points: None, + disk_include: None, + disk_exclude: None, + network_include: None, + network_exclude: None, + validation: HostViewValidationMode::None, + }, + ) + .expect("source"); + + let expected_start = 123 * NANOS_PER_SEC; + let disk_scrape = block_on_scrape( + &mut source, + ProcfsFamilies { + disk: true, + ..ProcfsFamilies::default() + }, + ) + .expect("disk scrape"); + assert_eq!(disk_scrape.snapshot.start_time_unix_nano, expected_start); + assert_eq!(disk_scrape.snapshot.disks.len(), 1); + + std::fs::remove_file(proc.join("stat")).expect("remove stat after cache"); + + let network_scrape = block_on_scrape( + &mut source, + ProcfsFamilies { + network: true, + ..ProcfsFamilies::default() + }, + ) + .expect("network scrape"); + assert_eq!(network_scrape.snapshot.start_time_unix_nano, expected_start); + assert_eq!(network_scrape.snapshot.networks.len(), 1); + + let paging_scrape = block_on_scrape( + &mut source, + ProcfsFamilies { + paging: true, + ..ProcfsFamilies::default() + }, + ) + .expect("paging scrape"); + assert_eq!(paging_scrape.snapshot.start_time_unix_nano, expected_start); + assert!(paging_scrape.snapshot.paging.is_some()); +} + +#[test] +fn scrape_due_reads_filesystem_usage_from_mountinfo() { + let root = tempfile::tempdir().expect("tempdir"); + let proc_one = root.path().join("proc/1"); + std::fs::create_dir_all(&proc_one).expect("proc one dir"); + std::fs::write( + proc_one.join("mountinfo"), + "36 25 8:1 / / rw,relatime - ext4 /dev/sda1 rw\n", + ) + .expect("mountinfo"); + let mut source = ProcfsSource::new( + Some(root.path()), + ProcfsConfig { + cpu: false, + memory: false, + paging: false, + system: false, + disk: false, + filesystem: true, + network: false, + processes: false, + cpu_utilization: false, + memory_limit: false, + memory_shared: false, + memory_hugepages: false, + disk_limit: false, + filesystem_include_virtual: false, + filesystem_include_remote: false, + filesystem_limit: true, + filesystem_include_devices: None, + filesystem_exclude_devices: None, + filesystem_include_fs_types: None, + filesystem_exclude_fs_types: None, + filesystem_include_mount_points: None, + filesystem_exclude_mount_points: None, + disk_include: None, + disk_exclude: None, + network_include: None, + network_exclude: None, + validation: HostViewValidationMode::None, + }, + ) + .expect("source"); + + let scrape = block_on_scrape( + &mut source, + ProcfsFamilies { + filesystem: true, + ..ProcfsFamilies::default() + }, + ) + .expect("filesystem scrape"); + + assert_eq!(scrape.snapshot.filesystems.len(), 1); + assert_eq!(scrape.snapshot.filesystems[0].device, "/dev/sda1"); + assert_eq!(scrape.snapshot.filesystems[0].mountpoint, "/"); + assert_eq!(scrape.snapshot.filesystems[0].fs_type, "ext4"); + assert!(scrape.snapshot.filesystems[0].limit_bytes.is_some()); +} + +#[test] +fn cpu_parser_accepts_missing_newer_fields() { + let cpu = parse_cpu_total("10 20 30 40", 10.0).expect("cpu row"); + assert_eq!(cpu.user, 1.0); + assert_eq!(cpu.nice, 2.0); + assert_eq!(cpu.system, 3.0); + assert_eq!(cpu.idle, 4.0); + assert_eq!(cpu.steal, 0.0); +} + +#[test] +fn cpu_parser_removes_guest_from_user_and_nice() { + let cpu = parse_cpu_total("100 50 30 40 5 2 3 7 10 4", 10.0).expect("cpu row"); + assert_eq!(cpu.user, 9.0); + assert_eq!(cpu.nice, 4.6); + assert_eq!(cpu.interrupt, 0.5); +} + +#[test] +fn cpu_utilization_uses_counter_deltas() { + let utilization = cpu_utilization( + CpuTimes { + user: 1.0, + idle: 1.0, + ..CpuTimes::default() + }, + CpuTimes { + user: 3.0, + idle: 2.0, + ..CpuTimes::default() + }, + ) + .expect("utilization"); + + assert_eq!(utilization.user, 2.0 / 3.0); + assert_eq!(utilization.idle, 1.0 / 3.0); +} + +#[test] +fn cpu_utilization_skips_counter_resets() { + assert!( + cpu_utilization( + CpuTimes { + user: 2.0, + ..CpuTimes::default() + }, + CpuTimes { + user: 1.0, + ..CpuTimes::default() + }, + ) + .is_none() + ); +} + +#[test] +fn clock_ticks_per_second_uses_positive_system_value() { + assert!(clock_ticks_per_second() > 0.0); +} + +#[test] +fn memavailable_fallback_uses_free_buffers_cached() { + let memory = + parse_meminfo("MemTotal: 1000 kB\nMemFree: 100 kB\nBuffers: 20 kB\nCached: 30 kB\n") + .expect("memory"); + assert!(!memory.has_available); + assert_eq!(memory.available, 150 * BYTES_PER_KIB); + assert_eq!(memory.used, 850 * BYTES_PER_KIB); +} + +#[test] +fn meminfo_parser_reads_shared_memory() { + let memory = + parse_meminfo("MemTotal: 1000 kB\nMemFree: 100 kB\nShmem: 12 kB\n").expect("memory"); + assert_eq!(memory.shared, 12 * BYTES_PER_KIB); +} + +#[test] +fn meminfo_parser_reads_hugepage_stats() { + let memory = parse_meminfo( + "MemTotal: 1000 kB\n\ + MemFree: 100 kB\n\ + HugePages_Total: 8\n\ + HugePages_Free: 3\n\ + HugePages_Rsvd: 2\n\ + HugePages_Surp: 1\n\ + Hugepagesize: 2048 kB\n", + ) + .expect("memory"); + + assert_eq!(memory.hugepages.total, 8); + assert_eq!(memory.hugepages.free, 3); + assert_eq!(memory.hugepages.reserved, 2); + assert_eq!(memory.hugepages.surplus, 1); + assert_eq!(memory.hugepages.page_size_bytes, 2048 * BYTES_PER_KIB); +} + +#[test] +fn uptime_parser_reads_first_field() { + assert_eq!(parse_uptime("123.45 67.89"), Some(123.45)); +} + +#[test] +fn vmstat_parser_derives_minor_faults() { + let paging = + parse_vmstat("pgfault 100\npgmajfault 7\npgpgin 5\npgpgout 6\npswpin 3\npswpout 4\n"); + assert_eq!(paging.minor_faults, 93); + assert_eq!(paging.major_faults, 7); + assert_eq!(paging.page_in, 5); + assert_eq!(paging.page_out, 6); + assert_eq!(paging.swap_in, 3); + assert_eq!(paging.swap_out, 4); +} + +#[test] +fn swaps_parser_reads_device_usage() { + let swaps = parse_swaps("Filename Type Size Used Priority\n/dev/sda2 partition 200 50 -2\n"); + assert_eq!(swaps.len(), 1); + assert_eq!(swaps[0].name, "/dev/sda2"); + assert_eq!(swaps[0].used, 50 * BYTES_PER_KIB); + assert_eq!(swaps[0].free, 150 * BYTES_PER_KIB); +} + +#[test] +fn diskstats_parser_accepts_flush_columns() { + let disks = parse_diskstats("8 0 sda 1 0 2 3 4 0 5 6 0 0 0 0 0 0 0 0\n", None, None); + assert_eq!(disks.len(), 1); + assert_eq!(disks[0].name, "sda"); + assert_eq!(disks[0].read_bytes, 1024); + assert_eq!(disks[0].write_bytes, 2560); +} + +#[test] +fn diskstats_parser_applies_filters_before_parsing_values() { + let exclude = CompiledFilter::compile( + crate::receivers::host_metrics_receiver::MatchType::Glob, + vec!["loop*".to_owned()], + ) + .expect("valid") + .expect("filter"); + let disks = parse_diskstats( + "7 0 loop0 broken row\n8 0 sda 1 0 2 3 4 0 5 6 0 0 0 0 0 0 0 0\n", + None, + Some(&exclude), + ); + + assert_eq!(disks.len(), 1); + assert_eq!(disks[0].name, "sda"); +} + +#[test] +fn mountinfo_parser_skips_virtual_and_remote_filesystems_by_default() { + let mounts = parse_mountinfo( + "36 25 8:1 / / rw,relatime - ext4 /dev/sda1 rw\n\ + 37 25 0:32 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw\n\ + 38 25 0:33 / /mnt/fuse rw,relatime - fuse.sshfs sshfs rw\n\ + 39 25 0:34 / /mnt/fuseblk rw,relatime - fuseblk /dev/fuse rw\n\ + 40 25 0:35 / /mnt/nfs rw,relatime - nfs server:/export rw\n\ + 41 25 0:36 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw\n\ + 42 25 0:37 / /sys/kernel/security rw,nosuid,nodev,noexec,relatime - securityfs securityfs rw\n", + false, + false, + true, + FilesystemFilters::default(), + ); + + assert_eq!(mounts.len(), 1); + assert_eq!(mounts[0].device, "/dev/sda1"); + assert_eq!(mounts[0].mountpoint, "/"); + assert_eq!(mounts[0].fs_type, "ext4"); + assert_eq!(mounts[0].mode, "rw"); + assert!(mounts[0].emit_limit); +} + +#[test] +fn mountinfo_parser_keeps_remote_filesystems_separate_from_virtual_filesystems() { + let mountinfo = "36 25 8:1 / / rw,relatime - ext4 /dev/sda1 rw\n\ + 37 25 0:32 / /run rw,nosuid,nodev - tmpfs tmpfs rw\n\ + 38 25 0:33 / /mnt/fuse rw,relatime - fuse.sshfs sshfs rw\n\ + 39 25 0:34 / /mnt/fuseblk rw,relatime - fuseblk /dev/fuse rw\n\ + 40 25 0:35 / /mnt/nfs rw,relatime - nfs server:/export rw\n\ + 41 25 0:36 / /mnt/cifs rw,relatime - cifs //server/share rw\n\ + 42 25 0:37 / /mnt/9p rw,relatime - 9p hostshare rw\n"; + + let virtual_only = parse_mountinfo(mountinfo, true, false, false, FilesystemFilters::default()); + assert_eq!(virtual_only.len(), 2); + assert_eq!(virtual_only[0].fs_type, "ext4"); + assert_eq!(virtual_only[1].fs_type, "tmpfs"); + + let remote_only = parse_mountinfo(mountinfo, false, true, false, FilesystemFilters::default()); + assert_eq!(remote_only.len(), 6); + assert_eq!(remote_only[0].fs_type, "ext4"); + assert_eq!(remote_only[1].fs_type, "fuse.sshfs"); + assert_eq!(remote_only[2].fs_type, "fuseblk"); + assert_eq!(remote_only[3].fs_type, "nfs"); + assert_eq!(remote_only[4].fs_type, "cifs"); + assert_eq!(remote_only[5].fs_type, "9p"); + + let all_included = parse_mountinfo(mountinfo, true, true, false, FilesystemFilters::default()); + assert_eq!(all_included.len(), 7); +} + +#[test] +fn mountinfo_parser_applies_filters_after_remote_filesystem_opt_in() { + let include_fs_types = CompiledFilter::compile( + crate::receivers::host_metrics_receiver::MatchType::Strict, + vec!["nfs".to_owned()], + ) + .expect("valid") + .expect("filter"); + let mounts = parse_mountinfo( + "36 25 8:1 / / rw,relatime - ext4 /dev/sda1 rw\n\ + 37 25 0:35 / /mnt/nfs rw,relatime - nfs server:/export rw\n\ + 38 25 0:36 / /mnt/cifs rw,relatime - cifs //server/share rw\n", + false, + true, + false, + FilesystemFilters { + include_fs_types: Some(&include_fs_types), + ..FilesystemFilters::default() + }, + ); + + assert_eq!(mounts.len(), 1); + assert_eq!(mounts[0].fs_type, "nfs"); +} + +#[test] +fn mountinfo_parser_unescapes_paths() { + let mounts = parse_mountinfo( + "36 25 8:1 / /mnt/data\\040disk rw,relatime - ext4 /dev/disk\\040one rw\n", + false, + false, + false, + FilesystemFilters::default(), + ); + + assert_eq!(mounts.len(), 1); + assert_eq!(mounts[0].device, "/dev/disk one"); + assert_eq!(mounts[0].mountpoint, "/mnt/data disk"); +} + +#[test] +fn mountinfo_parser_preserves_utf8_while_unescaping_paths() { + let mounts = parse_mountinfo( + "36 25 8:1 / /mnt/caf\u{00e9}\\040disk rw,relatime - ext4 /dev/disk\\040\u{00e9} rw\n", + false, + false, + false, + FilesystemFilters::default(), + ); + + assert_eq!(mounts.len(), 1); + assert_eq!(mounts[0].device, "/dev/disk \u{00e9}"); + assert_eq!(mounts[0].mountpoint, "/mnt/caf\u{00e9} disk"); +} + +#[test] +fn mountinfo_parser_applies_filesystem_filters() { + let include_mounts = CompiledFilter::compile( + crate::receivers::host_metrics_receiver::MatchType::Glob, + vec!["/data*".to_owned()], + ) + .expect("valid") + .expect("filter"); + let exclude_fs_types = CompiledFilter::compile( + crate::receivers::host_metrics_receiver::MatchType::Strict, + vec!["xfs".to_owned()], + ) + .expect("valid") + .expect("filter"); + let mounts = parse_mountinfo( + "36 25 8:1 / / rw,relatime - ext4 /dev/sda1 rw\n37 25 8:2 / /data rw,relatime - ext4 /dev/sdb1 rw\n38 25 8:3 / /data2 rw,relatime - xfs /dev/sdc1 rw\n", + false, + false, + false, + FilesystemFilters { + include_mount_points: Some(&include_mounts), + exclude_fs_types: Some(&exclude_fs_types), + ..FilesystemFilters::default() + }, + ); + + assert_eq!(mounts.len(), 1); + assert_eq!(mounts[0].device, "/dev/sdb1"); + assert_eq!(mounts[0].mountpoint, "/data"); +} + +#[test] +fn netdev_parser_reads_device_counters() { + let interfaces = parse_netdev( + "Inter-| Receive | Transmit\n face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n eth0: 10 2 0 0 0 0 0 0 30 4 0 0 0 0 0 0\n", + None, + None, + ); + assert_eq!(interfaces.len(), 1); + assert_eq!(interfaces[0].name, "eth0"); + assert_eq!(interfaces[0].rx_bytes, 10); + assert_eq!(interfaces[0].tx_packets, 4); +} + +#[test] +fn netdev_parser_applies_interface_filters() { + let include = CompiledFilter::compile( + crate::receivers::host_metrics_receiver::MatchType::Strict, + vec!["eth0".to_owned()], + ) + .expect("valid") + .expect("filter"); + let interfaces = parse_netdev( + "Inter-| Receive | Transmit\n face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n lo: 1 1 0 0 0 0 0 0 1 1 0 0 0 0 0 0\n eth0: 10 2 3 4 0 0 0 0 30 4 5 6 0 0 0 0\n", + Some(&include), + None, + ); + + assert_eq!(interfaces.len(), 1); + assert_eq!(interfaces[0].name, "eth0"); + assert_eq!(interfaces[0].rx_errors, 3); + assert_eq!(interfaces[0].tx_dropped, 6); +} + +#[test] +fn root_path_uses_host_pid_one_netdev() { + let paths = ProcfsPaths::new(Some(Path::new("/host"))); + assert_eq!(paths.net_dev, PathBuf::from("/host/proc/1/net/dev")); + assert_eq!(paths.mountinfo, PathBuf::from("/host/proc/1/mountinfo")); +} + +#[test] +fn root_slash_uses_current_proc_netdev() { + let paths = ProcfsPaths::new(Some(Path::new("/"))); + assert_eq!(paths.net_dev, PathBuf::from("/proc/net/dev")); + assert_eq!(paths.mountinfo, PathBuf::from("/proc/self/mountinfo")); +} + +#[test] +fn host_arch_uses_semconv_values() { + if let Some(arch) = host_arch() { + assert!(matches!( + arch, + "amd64" | "arm32" | "arm64" | "ppc32" | "ppc64" | "x86" + )); + } +} + +#[cfg(feature = "dev-tools")] +#[derive(Debug)] +struct MetricShape { + unit: String, + monotonic: Option, + attributes: BTreeSet, + all_attributes: BTreeSet, + attribute_types: BTreeMap, + enum_values: BTreeMap>, + value_type: Option, +} + +#[cfg(feature = "dev-tools")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum MetricValueKind { + Int, + Double, +} + +#[cfg(feature = "dev-tools")] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AttributeValueKind { + Int, + Double, + String, + Bool, +} + +#[cfg(feature = "dev-tools")] +fn load_semconv_registry() -> ResolvedRegistry { + let registry_path = std::env::var("OTAP_HOST_METRICS_SEMCONV_REGISTRY") + .map(|path| { + path.parse::() + .expect("valid OTAP_HOST_METRICS_SEMCONV_REGISTRY") + }) + .unwrap_or_else(|_| VirtualDirectoryPath::GitRepo { + url: "https://github.com/open-telemetry/semantic-conventions.git".to_owned(), + sub_folder: Some("model".to_owned()), + refspec: Some(format!( + "v{}", + crate::receivers::host_metrics_receiver::semconv::VERSION + )), + }); + + let registry_repo = + RegistryRepo::try_new("main", ®istry_path).expect("semantic convention registry"); + let registry = match SchemaResolver::load_semconv_repository(registry_repo, false) { + WResult::Ok(registry) | WResult::OkWithNFEs(registry, _) => registry, + WResult::FatalErr(err) => panic!("failed to load semantic convention registry: {err}"), + }; + let resolved_schema = match SchemaResolver::resolve(registry, true) { + WResult::Ok(schema) | WResult::OkWithNFEs(schema, _) => schema, + WResult::FatalErr(err) => { + panic!("failed to resolve semantic convention registry: {err}"); + } + }; + + ResolvedRegistry::try_from_resolved_registry( + &resolved_schema.registry, + resolved_schema.catalog(), + ) + .expect("resolved semantic convention registry") +} + +#[cfg(feature = "dev-tools")] +fn semconv_system_metric_shapes(registry: &ResolvedRegistry) -> BTreeMap { + registry + .groups + .iter() + .filter(|group| group.r#type == GroupType::Metric) + .filter_map(|group| { + let name = group.metric_name.as_ref()?; + if !name.starts_with("system.") { + return None; + } + + let monotonic = match group.instrument.as_ref()? { + InstrumentSpec::Counter => Some(true), + InstrumentSpec::UpDownCounter => Some(false), + InstrumentSpec::Gauge | InstrumentSpec::Histogram => None, + }; + let attributes = group + .attributes + .iter() + .filter(|attr| !is_opt_in_requirement(&attr.requirement_level)) + .map(|attr| attr.name.clone()) + .collect(); + let all_attributes = group + .attributes + .iter() + .map(|attr| attr.name.clone()) + .collect(); + let enum_values = group + .attributes + .iter() + .filter_map(|attr| match &attr.r#type { + AttributeType::Enum { members } => Some(( + attr.name.clone(), + members + .iter() + .map(|member| value_spec_string(&member.value)) + .collect(), + )), + _ => None, + }) + .collect(); + let attribute_types = group + .attributes + .iter() + .filter_map(|attr| { + attribute_value_kind(&attr.r#type).map(|kind| (attr.name.clone(), kind)) + }) + .collect(); + + Some(( + name.clone(), + MetricShape { + unit: group.unit.clone().unwrap_or_default(), + monotonic, + attributes, + all_attributes, + attribute_types, + enum_values, + value_type: semconv_metric_value_type(group.annotations.as_ref()), + }, + )) + }) + .collect() +} + +#[cfg(feature = "dev-tools")] +fn semconv_metric_value_type( + annotations: Option<&BTreeMap>, +) -> Option { + let code_generation = annotations?.get("code_generation")?.0.as_mapping()?; + let value_type = code_generation.iter().find_map(|(key, value)| { + (key.as_str() == Some("metric_value_type")).then(|| value.as_str())? + })?; + match value_type { + "int" => Some(MetricValueKind::Int), + "double" => Some(MetricValueKind::Double), + _ => None, + } +} + +#[cfg(feature = "dev-tools")] +fn value_spec_string(value: &ValueSpec) -> String { + match value { + ValueSpec::Int(value) => value.to_string(), + ValueSpec::Double(value) => value.to_string(), + ValueSpec::String(value) => value.clone(), + ValueSpec::Bool(value) => value.to_string(), + } +} + +#[cfg(feature = "dev-tools")] +fn attribute_value_kind(attribute_type: &AttributeType) -> Option { + match attribute_type { + AttributeType::PrimitiveOrArray(PrimitiveOrArrayTypeSpec::Int) => { + Some(AttributeValueKind::Int) + } + AttributeType::PrimitiveOrArray(PrimitiveOrArrayTypeSpec::Double) => { + Some(AttributeValueKind::Double) + } + AttributeType::PrimitiveOrArray(PrimitiveOrArrayTypeSpec::String) => { + Some(AttributeValueKind::String) + } + AttributeType::PrimitiveOrArray(PrimitiveOrArrayTypeSpec::Boolean) => { + Some(AttributeValueKind::Bool) + } + AttributeType::Enum { members } => { + members.first().map(|member| value_spec_kind(&member.value)) + } + _ => None, + } +} + +#[cfg(feature = "dev-tools")] +fn value_spec_kind(value: &ValueSpec) -> AttributeValueKind { + match value { + ValueSpec::Int(_) => AttributeValueKind::Int, + ValueSpec::Double(_) => AttributeValueKind::Double, + ValueSpec::String(_) => AttributeValueKind::String, + ValueSpec::Bool(_) => AttributeValueKind::Bool, + } +} + +#[cfg(feature = "dev-tools")] +fn is_intentional_semconv_enum_value_gap(_name: &str, _attr: &str, _value: &str) -> bool { + false +} + +#[cfg(feature = "dev-tools")] +fn is_opt_in_requirement(requirement_level: &RequirementLevel) -> bool { + matches!( + requirement_level, + RequirementLevel::Basic(BasicRequirementLevelSpec::OptIn) | RequirementLevel::OptIn { .. } + ) +} + +#[cfg(feature = "dev-tools")] +fn emitted_phase1_metric_shapes() -> BTreeMap { + let metrics = projection_fixture_metrics(); + let mut shapes = BTreeMap::new(); + for metric in &metrics { + let (monotonic, points) = match metric.data.as_ref().expect("metric data") { + otlp_metric::Data::Sum(sum) => (Some(sum.is_monotonic), &sum.data_points), + otlp_metric::Data::Gauge(gauge) => (None, &gauge.data_points), + _ => panic!("unsupported metric data for {}", metric.name), + }; + let value_type = metric_value_type(points); + let shape = shapes + .entry(metric.name.clone()) + .or_insert_with(|| MetricShape { + unit: metric.unit.clone(), + monotonic, + attributes: BTreeSet::new(), + all_attributes: BTreeSet::new(), + attribute_types: BTreeMap::new(), + enum_values: BTreeMap::new(), + value_type, + }); + assert_eq!( + shape.unit, metric.unit, + "unit mismatch across {}", + metric.name + ); + assert_eq!( + shape.monotonic, monotonic, + "instrument/temporality mismatch across {}", + metric.name + ); + assert_eq!( + shape.value_type, value_type, + "value type mismatch across {}", + metric.name + ); + for attr in points.iter().flat_map(|point| point.attributes.iter()) { + let _ = shape.attributes.insert(attr.key.clone()); + if let Some(value) = any_value_string(attr.value.as_ref()) { + let _ = shape + .enum_values + .entry(attr.key.clone()) + .or_default() + .insert(value); + } + if let Some(kind) = any_value_kind(attr.value.as_ref()) { + let previous = shape.attribute_types.insert(attr.key.clone(), kind); + assert!( + previous.is_none() || previous == Some(kind), + "mixed attribute value types for {} on {}", + attr.key, + metric.name + ); + } + } + } + shapes +} + +#[cfg(feature = "dev-tools")] +fn metric_value_type(points: &[NumberDataPoint]) -> Option { + let mut value_type = None; + for point in points { + let point_value_type = match point.value { + Some(number_data_point::Value::AsInt(_)) => MetricValueKind::Int, + Some(number_data_point::Value::AsDouble(_)) => MetricValueKind::Double, + None => continue, + }; + if value_type + .replace(point_value_type) + .is_some_and(|current| current != point_value_type) + { + panic!("mixed int/double data points"); + } + } + value_type +} + +#[cfg(feature = "dev-tools")] +fn any_value_string(value: Option<&AnyValue>) -> Option { + match value?.value.as_ref()? { + any_value::Value::StringValue(value) => Some(value.clone()), + any_value::Value::IntValue(value) => Some(value.to_string()), + any_value::Value::DoubleValue(value) => Some(value.to_string()), + any_value::Value::BoolValue(value) => Some(value.to_string()), + _ => None, + } +} + +#[cfg(feature = "dev-tools")] +fn any_value_kind(value: Option<&AnyValue>) -> Option { + match value?.value.as_ref()? { + any_value::Value::StringValue(_) => Some(AttributeValueKind::String), + any_value::Value::IntValue(_) => Some(AttributeValueKind::Int), + any_value::Value::DoubleValue(_) => Some(AttributeValueKind::Double), + any_value::Value::BoolValue(_) => Some(AttributeValueKind::Bool), + _ => None, + } +} + +fn projection_fixture_request() -> MetricsData { + decode_metrics( + HostSnapshot { + now_unix_nano: 2_000, + start_time_unix_nano: 1_000, + counter_starts: CounterStarts::default(), + memory_limit: true, + memory_shared: true, + memory_hugepages: true, + cpu: Some(CpuTimes { + user: 1.0, + nice: 2.0, + system: 3.0, + idle: 4.0, + wait: 5.0, + interrupt: 6.0, + steal: 7.0, + }), + cpu_utilization: Some(CpuTimes { + user: 0.1, + nice: 0.1, + system: 0.2, + idle: 0.3, + wait: 0.1, + interrupt: 0.1, + steal: 0.1, + }), + cpuinfo: CpuInfo { + logical_count: 2, + physical_count: 1, + frequencies_hz: vec![2_400_000_000.0], + }, + memory: Some(MemoryStats { + total: 100, + used: 80, + free: 10, + available: 20, + has_available: true, + cached: 5, + buffered: 5, + shared: 7, + slab_reclaimable: 3, + slab_unreclaimable: 2, + hugepages: HugepageStats { + total: 10, + free: 4, + reserved: 2, + surplus: 1, + page_size_bytes: 2 * BYTES_PER_KIB, + }, + }), + uptime_seconds: Some(42.0), + paging: Some(PagingStats { + minor_faults: 9, + major_faults: 1, + page_in: 4, + page_out: 5, + swap_in: 2, + swap_out: 3, + }), + swaps: vec![SwapStats { + name: "/dev/swap".to_owned(), + size: 100, + used: 25, + free: 75, + }], + processes: Some(ProcessStats { + running: 4, + blocked: 1, + created: 99, + }), + disks: vec![DiskStats { + name: "sda".to_owned(), + limit_bytes: Some(123), + read_bytes: 10, + write_bytes: 20, + read_ops: 1, + write_ops: 2, + read_merged: 3, + write_merged: 4, + read_time_seconds: 0.5, + write_time_seconds: 0.6, + io_time_seconds: 0.7, + }], + filesystems: vec![FilesystemStats { + device: "/dev/sda1".to_owned(), + mountpoint: "/".to_owned(), + fs_type: "ext4".to_owned(), + mode: "rw", + used: 60, + free: 30, + reserved: 10, + limit_bytes: Some(100), + }], + networks: vec![NetworkStats { + name: "eth0".to_owned(), + rx_bytes: 10, + tx_bytes: 20, + rx_packets: 1, + tx_packets: 2, + rx_errors: 3, + tx_errors: 4, + rx_dropped: 5, + tx_dropped: 6, + }], + resource: HostResource { + host_id: Some("host-id".to_owned()), + host_name: Some("host-name".to_owned()), + host_arch: Some("amd64"), + }, + } + .into_otap_records() + .expect("encode ok"), + ) +} + +#[cfg(feature = "dev-tools")] +fn projection_fixture_metrics() -> Vec { + projection_fixture_request() + .resource_metrics + .into_iter() + .next() + .expect("resource metrics") + .scope_metrics + .into_iter() + .next() + .expect("scope metrics") + .metrics +} + +fn assert_metric_shape( + metrics: &[Metric], + name: &'static str, + unit: &'static str, + monotonic_sum: Option, +) { + let metric = metric_by_name(metrics, name); + assert_eq!(metric.unit, unit); + match metric.data.as_ref().expect("metric data") { + otlp_metric::Data::Sum(sum) => { + let expected_monotonic = + monotonic_sum.unwrap_or_else(|| panic!("{name} should be a gauge")); + assert_eq!( + sum.aggregation_temporality, + AggregationTemporality::Cumulative as i32 + ); + assert_eq!(sum.is_monotonic, expected_monotonic); + assert!( + sum.data_points + .iter() + .all(|point| point.start_time_unix_nano == 1_000) + ); + } + otlp_metric::Data::Gauge(gauge) => { + assert!(monotonic_sum.is_none(), "{name} should be a cumulative sum"); + assert!( + gauge + .data_points + .iter() + .all(|point| point.start_time_unix_nano == 0) + ); + } + _ => panic!("unexpected data kind for {name}"), + } +} + +fn assert_first_point_attr( + metrics: &[Metric], + name: &'static str, + key: &'static str, + value: &'static str, +) { + let metric = metric_by_name(metrics, name); + let point = match metric.data.as_ref().expect("metric data") { + otlp_metric::Data::Sum(sum) => sum.data_points.first(), + otlp_metric::Data::Gauge(gauge) => gauge.data_points.first(), + _ => None, + } + .expect("data point"); + assert_has_attr(&point.attributes, key, value); +} + +fn assert_sum_point_attr( + metrics: &[Metric], + name: &'static str, + key: &'static str, + value: &'static str, +) { + let metric = metric_by_name(metrics, name); + let otlp_metric::Data::Sum(sum) = metric.data.as_ref().expect("metric data") else { + panic!("{name} should be a cumulative sum"); + }; + assert!( + sum.data_points + .iter() + .any(|point| has_attr(&point.attributes, key, value)), + "missing point attribute {key}={value}" + ); +} + +fn assert_first_point_int(metrics: &[Metric], name: &'static str, expected: i64) { + let metric = metric_by_name(metrics, name); + let point = match metric.data.as_ref().expect("metric data") { + otlp_metric::Data::Sum(sum) => sum.data_points.first(), + otlp_metric::Data::Gauge(gauge) => gauge.data_points.first(), + _ => None, + } + .expect("data point"); + assert_eq!( + point.value, + Some(number_data_point::Value::AsInt(expected)), + "{name} first point should be int" + ); +} + +fn assert_first_point_attr_int( + metrics: &[Metric], + name: &'static str, + key: &'static str, + expected: i64, +) { + let metric = metric_by_name(metrics, name); + let point = match metric.data.as_ref().expect("metric data") { + otlp_metric::Data::Sum(sum) => sum.data_points.first(), + otlp_metric::Data::Gauge(gauge) => gauge.data_points.first(), + _ => None, + } + .expect("data point"); + assert!( + point.attributes.iter().any(|attr| { + attr.key == key + && matches!( + attr.value.as_ref().and_then(|value| value.value.as_ref()), + Some(any_value::Value::IntValue(actual)) if *actual == expected + ) + }), + "missing int attribute {key}={expected}" + ); +} + +fn assert_no_first_point_attr(metrics: &[Metric], name: &'static str, key: &'static str) { + let metric = metric_by_name(metrics, name); + let point = match metric.data.as_ref().expect("metric data") { + otlp_metric::Data::Sum(sum) => sum.data_points.first(), + otlp_metric::Data::Gauge(gauge) => gauge.data_points.first(), + _ => None, + } + .expect("data point"); + assert!( + !point.attributes.iter().any(|attr| attr.key == key), + "unexpected attribute {key}" + ); +} + +fn assert_first_sum_point_start(metrics: &[Metric], name: &'static str, expected_start: u64) { + let metric = metric_by_name(metrics, name); + let otlp_metric::Data::Sum(sum) = metric.data.as_ref().expect("metric data") else { + panic!("{name} should be a cumulative sum"); + }; + let point = sum.data_points.first().expect("data point"); + assert_eq!(point.start_time_unix_nano, expected_start); +} + +fn metric_by_name<'a>(metrics: &'a [Metric], name: &'static str) -> &'a Metric { + metrics + .iter() + .find(|metric| metric.name == name) + .unwrap_or_else(|| panic!("missing metric {name}")) +} + +fn assert_has_attr(attributes: &[KeyValue], key: &'static str, value: &'static str) { + assert!( + has_attr(attributes, key, value), + "missing attribute {key}={value}" + ); +} + +fn has_attr(attributes: &[KeyValue], key: &'static str, value: &'static str) -> bool { + attributes.iter().any(|attr| { + attr.key == key + && matches!( + attr.value.as_ref().and_then(|value| value.value.as_ref()), + Some(any_value::Value::StringValue(actual)) if actual == value + ) + }) +} diff --git a/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/semconv.rs b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/semconv.rs new file mode 100644 index 0000000000..a212246518 --- /dev/null +++ b/rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/semconv.rs @@ -0,0 +1,92 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Semantic convention constants used by the host metrics receiver. +// Keep these strings centralized here. If this surface grows, prefer generating +// constants from the semconv registry instead of adding scattered literals. + +/// Semconv version targeted by this receiver's projection layer. +pub(crate) const VERSION: &str = "1.41.0"; + +/// Schema URL emitted with host metric batches. +pub(crate) const SCHEMA_URL: &[u8] = b"https://opentelemetry.io/schemas/1.41.0"; + +const _: () = { + let url = SCHEMA_URL; + let ver = VERSION.as_bytes(); + assert!(url.len() >= ver.len(), "SCHEMA_URL is shorter than VERSION"); + let suffix = url.split_at(url.len() - ver.len()).1; + let mut i = 0; + while i < ver.len() { + assert!(suffix[i] == ver[i], "SCHEMA_URL suffix must match VERSION"); + i += 1; + } +}; + +pub(crate) mod metric { + pub(crate) const CPU_FREQUENCY: &str = "system.cpu.frequency"; + pub(crate) const CPU_LOGICAL_COUNT: &str = "system.cpu.logical.count"; + pub(crate) const CPU_PHYSICAL_COUNT: &str = "system.cpu.physical.count"; + pub(crate) const CPU_TIME: &str = "system.cpu.time"; + pub(crate) const CPU_UTILIZATION: &str = "system.cpu.utilization"; + pub(crate) const DISK_IO: &str = "system.disk.io"; + pub(crate) const DISK_IO_TIME: &str = "system.disk.io_time"; + pub(crate) const DISK_LIMIT: &str = "system.disk.limit"; + pub(crate) const DISK_MERGED: &str = "system.disk.merged"; + pub(crate) const DISK_OPERATION_TIME: &str = "system.disk.operation_time"; + pub(crate) const DISK_OPERATIONS: &str = "system.disk.operations"; + pub(crate) const FILESYSTEM_LIMIT: &str = "system.filesystem.limit"; + pub(crate) const FILESYSTEM_USAGE: &str = "system.filesystem.usage"; + pub(crate) const FILESYSTEM_UTILIZATION: &str = "system.filesystem.utilization"; + pub(crate) const MEMORY_LIMIT: &str = "system.memory.limit"; + pub(crate) const MEMORY_LINUX_AVAILABLE: &str = "system.memory.linux.available"; + pub(crate) const MEMORY_LINUX_HUGEPAGES_LIMIT: &str = "system.memory.linux.hugepages.limit"; + pub(crate) const MEMORY_LINUX_HUGEPAGES_PAGE_SIZE: &str = + "system.memory.linux.hugepages.page_size"; + pub(crate) const MEMORY_LINUX_HUGEPAGES_RESERVED: &str = + "system.memory.linux.hugepages.reserved"; + pub(crate) const MEMORY_LINUX_HUGEPAGES_SURPLUS: &str = "system.memory.linux.hugepages.surplus"; + pub(crate) const MEMORY_LINUX_HUGEPAGES_USAGE: &str = "system.memory.linux.hugepages.usage"; + pub(crate) const MEMORY_LINUX_HUGEPAGES_UTILIZATION: &str = + "system.memory.linux.hugepages.utilization"; + pub(crate) const MEMORY_LINUX_SHARED: &str = "system.memory.linux.shared"; + pub(crate) const MEMORY_LINUX_SLAB_USAGE: &str = "system.memory.linux.slab.usage"; + pub(crate) const MEMORY_USAGE: &str = "system.memory.usage"; + pub(crate) const MEMORY_UTILIZATION: &str = "system.memory.utilization"; + pub(crate) const NETWORK_ERRORS: &str = "system.network.errors"; + pub(crate) const NETWORK_IO: &str = "system.network.io"; + pub(crate) const NETWORK_PACKET_COUNT: &str = "system.network.packet.count"; + pub(crate) const NETWORK_PACKET_DROPPED: &str = "system.network.packet.dropped"; + pub(crate) const PAGING_FAULTS: &str = "system.paging.faults"; + pub(crate) const PAGING_OPERATIONS: &str = "system.paging.operations"; + pub(crate) const PAGING_USAGE: &str = "system.paging.usage"; + pub(crate) const PAGING_UTILIZATION: &str = "system.paging.utilization"; + pub(crate) const PROCESS_COUNT: &str = "system.process.count"; + pub(crate) const PROCESS_CREATED: &str = "system.process.created"; + pub(crate) const UPTIME: &str = "system.uptime"; +} + +pub(crate) mod attr { + pub(crate) const CPU_LOGICAL_NUMBER: &str = "cpu.logical_number"; + pub(crate) const CPU_MODE: &str = "cpu.mode"; + pub(crate) const DISK_IO_DIRECTION: &str = "disk.io.direction"; + pub(crate) const HOST_ARCH: &str = "host.arch"; + pub(crate) const HOST_ID: &str = "host.id"; + pub(crate) const HOST_NAME: &str = "host.name"; + pub(crate) const NETWORK_INTERFACE_NAME: &str = "network.interface.name"; + pub(crate) const NETWORK_IO_DIRECTION: &str = "network.io.direction"; + pub(crate) const OS_TYPE: &str = "os.type"; + pub(crate) const PROCESS_STATE: &str = "process.state"; + pub(crate) const SYSTEM_DEVICE: &str = "system.device"; + pub(crate) const SYSTEM_FILESYSTEM_MODE: &str = "system.filesystem.mode"; + pub(crate) const SYSTEM_FILESYSTEM_MOUNTPOINT: &str = "system.filesystem.mountpoint"; + pub(crate) const SYSTEM_FILESYSTEM_STATE: &str = "system.filesystem.state"; + pub(crate) const SYSTEM_FILESYSTEM_TYPE: &str = "system.filesystem.type"; + pub(crate) const SYSTEM_MEMORY_LINUX_HUGEPAGES_STATE: &str = + "system.memory.linux.hugepages.state"; + pub(crate) const SYSTEM_MEMORY_LINUX_SLAB_STATE: &str = "system.memory.linux.slab.state"; + pub(crate) const SYSTEM_MEMORY_STATE: &str = "system.memory.state"; + pub(crate) const SYSTEM_PAGING_DIRECTION: &str = "system.paging.direction"; + pub(crate) const SYSTEM_PAGING_FAULT_TYPE: &str = "system.paging.fault.type"; + pub(crate) const SYSTEM_PAGING_STATE: &str = "system.paging.state"; +} diff --git a/rust/otap-dataflow/crates/core-nodes/src/receivers/mod.rs b/rust/otap-dataflow/crates/core-nodes/src/receivers/mod.rs index 6986e0f7b6..fa6bd67cf9 100644 --- a/rust/otap-dataflow/crates/core-nodes/src/receivers/mod.rs +++ b/rust/otap-dataflow/crates/core-nodes/src/receivers/mod.rs @@ -19,3 +19,6 @@ pub mod otap_receiver; /// OTLP receiver. pub mod otlp_receiver; + +/// Host metrics receiver. +pub mod host_metrics_receiver; diff --git a/rust/otap-dataflow/docs/host-metrics-receiver.md b/rust/otap-dataflow/docs/host-metrics-receiver.md index 640ed98a4b..21d47d2c10 100644 --- a/rust/otap-dataflow/docs/host-metrics-receiver.md +++ b/rust/otap-dataflow/docs/host-metrics-receiver.md @@ -10,7 +10,7 @@ Receiver URN: `urn:otel:receiver:host_metrics` Target crate: `crates/core-nodes` -Target module: `crates/core-nodes/src/receivers/host_metrics` +Target module: `crates/core-nodes/src/receivers/host_metrics_receiver` The issue explicitly asks for `core-nodes`. If maintainers prefer to stage this receiver in `contrib-nodes` while the implementation and system semantic @@ -99,7 +99,7 @@ partial scrape behavior that emits successfully collected metrics. Use a narrow module layout and keep the boundaries explicit: ```text -crates/core-nodes/src/receivers/host_metrics/ +crates/core-nodes/src/receivers/host_metrics_receiver/ mod.rs config.rs metrics.rs @@ -215,10 +215,10 @@ Rules: - `include_connection_count: true` is invalid in v1. - `processes.mode` only accepts `summary` in v1. - `processes.mode: summary` emits `system.process.count` and - `system.process.created`; `system.process.count` is limited to `running` and - `blocked` states from `/proc/stat`. `blocked` is a documented custom - `process.state` value because the current registry has no well-known value - for `procs_blocked`. It must not emit per-PID series or PID attributes. + `system.process.created`; `system.process.count` is limited to registered + `process.state` values. The v1 implementation emits `running` from + `/proc/stat`. It parses `procs_blocked` but does not emit it because + `blocked` is not a registered `process.state` value. - The load family is not shown in the default example because Semantic Conventions 1.41.0 does not register a load metric. If maintainers choose an experimental Linux load metric, add it as an explicit opt-in. @@ -355,10 +355,13 @@ must not call blocking `statfs` directly on the receiver task. Use a bounded blocking worker path with per-mount timeout/cancellation behavior, and skip remote filesystems plus known virtual filesystem types by default. -For process summary, use `/proc/stat` fields first: `processes`, -`procs_running`, and `procs_blocked`. Do not walk `/proc//stat` in the v1 -default path. A per-PID walk is reserved for future richer process modes and -must tolerate PIDs disappearing between directory read and file read. +For process summary, use `/proc/stat` fields first. Project `processes` into +`system.process.created` and `procs_running` into `system.process.count` with +`process.state=running`. Parse `procs_blocked` for future use, but do not emit +it in v1 because `blocked` is not a registered `process.state` value. Do not +walk `/proc//stat` in the v1 default path. A per-PID walk is reserved for +future richer process modes and must tolerate PIDs disappearing between +directory read and file read. ## Scheduler @@ -534,7 +537,7 @@ timestamp. | Linux hugepage metrics | No | Mixed | Mixed | Use current `system.memory.linux.hugepages.*` registry definitions. | | `system.paging.usage` | Yes | UpDownCounter | `By` | `system.paging.state`, `system.device`; use `/proc/swaps` for swap device identity. | | `system.paging.utilization` | Yes | Gauge | `1` | `system.paging.state`, `system.device`; use `/proc/swaps` for swap device identity. | -| `system.paging.operations` | Yes | Counter | `{operation}` | `system.paging.direction` from `pswpin` and `pswpout`; intentionally omit `system.paging.fault.type` because Linux swap-in/out counters are not broken down by fault type. | +| `system.paging.operations` | Yes | Counter | `{operation}` | `system.paging.direction`, `system.paging.fault.type`; follow the current registry shape. Linux projection follows the Go Collector precedent: `pswpin`/`pswpout` as `major`, `pgpgin`/`pgpgout` as `minor`. Linux does not expose this as a direct fault-type split, so maintainers may choose a narrower projection. | | `system.paging.faults` | Yes | Counter | `{fault}` | `system.paging.fault.type`; use `pgmajfault` for `major` and `pgfault - pgmajfault` for `minor` when both are available. | | `system.uptime` | Yes | Gauge | `s` | Prefer `CLOCK_BOOTTIME`; fall back to `/proc/uptime`. Emit double seconds. | | `system.disk.io` | Yes | Counter | `By` | `system.device`, `disk.io.direction`. | @@ -550,7 +553,7 @@ timestamp. | `system.network.packet.count` | Yes | Counter | `{packet}` | `system.device`, `network.io.direction`. | | `system.network.packet.dropped` | Yes | Counter | `{packet}` | `network.interface.name`, `network.io.direction`. | | `system.network.errors` | Yes | Counter | `{error}` | `network.interface.name`, `network.io.direction`. | -| `system.process.count` | Yes | UpDownCounter | `{process}` | `process.state`; v1 summary emits `running` and custom `blocked` from `/proc/stat`. | +| `system.process.count` | Yes | UpDownCounter | `{process}` | `process.state`; v1 summary emits `running` from `/proc/stat`. `procs_blocked` is parsed but not emitted because `blocked` is not a registered value. | | `system.process.created` | Yes | Counter | `{process}` | Cumulative process creations from `/proc/stat`. | CPU time and utilization aggregate across logical CPUs by default because @@ -646,14 +649,16 @@ Initial metric set: | `families_scraped` | Counter | `{family}` | Count due families processed. | | `scrape_duration_ns` | Mmsc | `ns` | Scrape duration distribution. | | `scrape_lag_ns` | Mmsc | `ns` | Scheduled time to actual start. | -| `source_read_errors` | Counter | `{error}` | Attributes: `family`, `error_class`. | -| `partial_errors` | Counter | `{error}` | Attributes: `family`, `error_class`. | +| `source_read_errors` | Counter | `{error}` | Total source read errors seen during scrapes. | +| `partial_errors` | Counter | `{error}` | Source read errors skipped because other families succeeded. | | `batches_sent` | Counter | `{batch}` | Downstream sends. | -| `send_failures` | Counter | `{error}` | Attribute: `error_class`. | +| `send_failures` | Counter | `{error}` | Downstream send failures. | -Use `#[attribute_set(name = "...")]` for the low-cardinality attribute set -covering `family` and `error_class`. Do not put source paths or device names -into receiver self-observability metric attributes. +The current internal `MetricSet` API does not support attributes on individual +metric observations. The implementation therefore uses aggregate counters and a +code TODO to decide whether fixed per-family/error-class counters are needed +later. Do not put source paths or device names into receiver +self-observability metric names or attributes. ## Validation Plan