Add Linux host metrics receiver for OTAP Dataflow

.github/workflows/rust-ci.yml

@@ -656,6 +656,32 @@ jobs:
           cargo clippy --all-targets --all-features --workspace -- -D warnings
         working-directory: ./rust/${{ matrix.folder }}
 
+  host-metrics-semconv:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          submodules: true
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          repository: open-telemetry/semantic-conventions
+          ref: v1.41.0
+          path: semantic-conventions
+      - uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9
+        with:
+          toolchain: stable
+      - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
+        with:
+          workspaces: ./rust/otap-dataflow
+      - name: Run host metrics semconv drift check
+        env:
+          OTAP_HOST_METRICS_SEMCONV_REGISTRY: ${{ github.workspace }}/semantic-conventions/model
+        run: |
+          cargo test -p otap-df-core-nodes \
+            --features dev-tools,otap-df-otap/crypto-ring \
+            emitted_phase1_metric_shapes_match_weaver_semconv --lib -- --ignored
+        working-directory: ./rust/otap-dataflow
+
   # Required matrix combinations for deny: otap-dataflow only
   deny_required:
     runs-on: ubuntu-latest
@@ -804,6 +830,7 @@ jobs:
       - pest-fmt
       - no_default_features_check
       - pipeline_perf_test
+      - host-metrics-semconv
     steps:
       - name: Check if all required jobs succeeded
         run: |
@@ -843,4 +870,8 @@ jobs:
             echo "pipeline_perf_test failed or was cancelled"
             exit 1
           fi
+          if [[ "${{ needs.host-metrics-semconv.result }}" != "success" ]]; then
+            echo "host-metrics-semconv failed or was cancelled"
+            exit 1
+          fi
           echo "All required checks passed!"

rust/otap-dataflow/Cargo.toml

@@ -122,6 +122,7 @@ tikv-jemalloc-sys = "0.6.1"
 memchr = "2.8.0"
 memmap2 = "0.9"
 memory-stats = "1"
+libc = "0.2"
 nix = { version = "0.31.0", features = ["process", "signal", "fs", "mman"] }
 notify = "8.0"  # Uses platform-native backend: inotify (Linux), kqueue (macOS), ReadDirectoryChanges (Windows)
 num_enum = "0.7"

rust/otap-dataflow/crates/core-nodes/Cargo.toml

@@ -42,6 +42,7 @@ object_store = {workspace = true, features = ["fs"]}
 parquet.workspace = true
 prost.workspace = true
 rand.workspace = true
+regex.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 slotmap.workspace = true
@@ -65,6 +66,10 @@ weaver_resolved_schema = { workspace = true, optional = true }
 weaver_resolver = { workspace = true, optional = true }
 weaver_semconv = { workspace = true, optional = true }
 
+[target.'cfg(target_os = "linux")'.dependencies]
+libc.workspace = true
+nix.workspace = true
+
 [features]
 dev-tools = ["dep:weaver_common", "dep:weaver_forge", "dep:weaver_resolved_schema", "dep:weaver_resolver", "dep:weaver_semconv"]
 bench = []

rust/otap-dataflow/crates/core-nodes/src/receivers/host_metrics_receiver/README.md

@@ -0,0 +1,181 @@
+# Host Metrics Receiver
+
+<!-- markdownlint-disable MD013 -->
+
+**URN:** `urn:otel:receiver:host_metrics`
+
+Linux host metrics receiver backed by procfs and sysfs. It emits OpenTelemetry
+`system.*` metrics for CPU, memory, paging, system uptime, disk, filesystem,
+network, and aggregate process counts.
+
+## Configuration
+
+Minimal configuration:
+
+```yaml
+groups:
+  host:
+    pipelines:
+      collect:
+        policies:
+          resources:
+            core_allocation:
+              type: core_count
+              count: 1
+        nodes:
+          host_metrics:
+            type: receiver:host_metrics
+            config:
+              collection_interval: 10s
+          publish:
+            type: exporter:topic
+            config:
+              topic: host_metrics
+        connections:
+          - from: host_metrics
+            to: publish
+```
+
+Collect from a host root mounted into a container:
+
+```yaml
+groups:
+  host:
+    pipelines:
+      collect:
+        policies:
+          resources:
+            core_allocation:
+              type: core_count
+              count: 1
+        nodes:
+          host_metrics:
+            type: receiver:host_metrics
+            config:
+              collection_interval: 10s
+              host_view:
+                root_path: /host
+                validation: fail_selected
+          publish:
+            type: exporter:topic
+            config:
+              topic: host_metrics
+        connections:
+          - from: host_metrics
+            to: publish
+```
+
+Enable selected opt-in metrics:
+
+```yaml
+groups:
+  host:
+    pipelines:
+      collect:
+        policies:
+          resources:
+            core_allocation:
+              type: core_count
+              count: 1
+        nodes:
+          host_metrics:
+            type: receiver:host_metrics
+            config:
+              families:
+                cpu:
+                  utilization: true
+                memory:
+                  limit: true
+                  hugepages: true
+                disk:
+                  limit: true
+                filesystem:
+                  limit: true
+          publish:
+            type: exporter:topic
+            config:
+              topic: host_metrics
+        connections:
+          - from: host_metrics
+            to: publish
+```
+
+## Configuration Options
+
+| Field | Type | Default | Description |
+| ----- | ---- | ------- | ----------- |
+| `collection_interval` | duration | `10s` | Default scrape interval. |
+| `initial_delay` | duration | `0s` | Delay before the first scrape. |
+| `host_view.root_path` | path | `/` | Host filesystem root to read procfs/sysfs from. |
+| `host_view.validation` | enum | `fail_selected` | One of `fail_selected`, `warn_selected`, or `none`. |
+| `families.<name>.enabled` | bool | `true` | Enables or disables a metric family. |
+| `families.<name>.interval` | duration | unset | Per-family interval; falls back to `collection_interval`. |
+| `families.cpu.utilization` | bool | `false` | Emits derived CPU utilization gauges. |
+| `families.memory.limit` | bool | `false` | Emits `system.memory.limit`. |
+| `families.memory.shared` | bool | `false` | Emits Linux shared memory. |
+| `families.memory.hugepages` | bool | `false` | Emits Linux hugepage metrics. |
+| `families.disk.limit` | bool | `false` | Emits disk capacity from sysfs. |
+| `families.filesystem.limit` | bool | `false` | Emits filesystem capacity. |
+| `families.filesystem.include_virtual_filesystems` | bool | `false` | Includes virtual filesystems such as tmpfs. |
+| `families.filesystem.include_remote_filesystems` | bool | `false` | Includes remote and userspace filesystems such as NFS, CIFS, 9p, and FUSE. |
+
+Families are `cpu`, `memory`, `paging`, `system`, `disk`, `filesystem`,
+`network`, and `processes`.
+
+Host-wide collection must run in a one-core source pipeline. Use a topic
+exporter to fan out to multicore downstream processing when needed.
+
+## Filters
+
+Disk, filesystem, and network families support include and exclude filters.
+Filter `match_type` values are `strict`, `glob`, and `regexp`.
+
+```yaml
+groups:
+  host:
+    pipelines:
+      collect:
+        policies:
+          resources:
+            core_allocation:
+              type: core_count
+              count: 1
+        nodes:
+          host_metrics:
+            type: receiver:host_metrics
+            config:
+              families:
+                disk:
+                  exclude:
+                    match_type: glob
+                    devices: ["loop*", "ram*"]
+                network:
+                  exclude:
+                    match_type: strict
+                    interfaces: ["lo"]
+                filesystem:
+                  exclude_fs_types:
+                    match_type: strict
+                    fs_types: ["tmpfs", "proc", "sysfs"]
+          publish:
+            type: exporter:topic
+            config:
+              topic: host_metrics
+        connections:
+          - from: host_metrics
+            to: publish
+```
+
+## Current Limits
+
+- Linux only.
+- Load metrics are not emitted in v1 because Semantic Conventions 1.41.0 does
+  not register a system load metric.
+- `families.cpu.per_cpu` is rejected in v1.
+- `families.network.include_connection_count` is rejected in v1.
+- Process metrics are aggregate host summaries, not per-process scrapes.
+- `system.process.count` emits the registered `process.state=running` summary.
+  Linux `procs_blocked` is parsed but not emitted because `blocked` is not a
+  registered `process.state` value.
+- Filesystem collection can time out individual `statvfs` calls; avoid enabling
+  remote filesystems unless the host environment is known to be healthy.