Oximeter: Add opte metrics. (#11003)

Expose a subset of opte kstats as oximeter metrics, limited to uft and
route cache metrics initially. We track each opte port in sled-agent so
that we can collate control plane metadata (nic id, parent id, etc.)
with kstat values.

Part of #10892.

Note: this is mostly a mechanical change, extracting more metrics from
opte's new kstats. We already know how to track and untrack links, so we
can mostly reuse that logic. I think the interesting parts are which
metrics to include (I'm proposing a subset related to uft and route
cache performance and occupancy) and how often to sample them (I'm using
the link sampling interval of 10s for now). It's easier to add features
than remove them, so I'd like to identify the minimal useful set of opte
metrics for now, then think about expanding the set later on.

Marking as draft while I deploy to a racklette for testing, and write up
notes on cardinality.

Author: jmcarp

illumos-utils/src/opte/mod.rs

@@ -33,6 +33,7 @@ use oxnet::IpNet;
 use oxnet::Ipv4Net;
 use oxnet::Ipv6Net;
 pub use port::Port;
+pub use port::PortInfo;
 pub use port_manager::MulticastGroupCfg;
 pub use port_manager::PortCreateParams;
 pub use port_manager::PortManager;

illumos-utils/src/opte/port.rs

@@ -14,10 +14,12 @@ use omicron_common::api::internal::shared::RouterId;
 use omicron_common::api::internal::shared::RouterKind;
 use oxnet::Ipv4Net;
 use oxnet::Ipv6Net;
+use sled_agent_types::inventory::NetworkInterfaceKind;
 use std::net::IpAddr;
 use std::net::Ipv4Addr;
 use std::net::Ipv6Addr;
 use std::sync::Arc;
+use uuid::Uuid;
 
 #[derive(Debug)]
 pub struct PortData {
@@ -159,3 +161,10 @@ impl Port {
         })
     }
 }
+
+/// An OPTE port, along with its control plane metadata.
+pub struct PortInfo {
+    pub port: Port,
+    pub nic_id: Uuid,
+    pub nic_kind: NetworkInterfaceKind,
+}

illumos-utils/src/opte/port_manager.rs

@@ -1156,6 +1156,14 @@ impl PortTicket {
         Self { id, kind, manager }
     }
 
+    pub fn id(&self) -> Uuid {
+        self.id
+    }
+
+    pub fn kind(&self) -> NetworkInterfaceKind {
+        self.kind
+    }
+
     fn release_inner(&mut self) -> Result<(), Error> {
         let mut ports = self.manager.ports.lock().unwrap();
         let Some(port) = ports.remove(&(self.id, self.kind)) else {

illumos-utils/src/running_zone.rs

@@ -12,7 +12,7 @@ use crate::addrobj::{
 use crate::contract;
 use crate::dladm::Etherstub;
 use crate::link::{Link, VnicAllocator};
-use crate::opte::{Port, PortTicket};
+use crate::opte::{Port, PortInfo, PortTicket};
 use crate::zone::Zones;
 use crate::zone::{AddressRequest, ROUTE};
 use crate::zpool::{PathInPool, ZpoolOrRamdisk};
@@ -496,6 +496,10 @@ impl RunningZone {
         self.inner.opte_ports()
     }
 
+    pub fn opte_port_info(&self) -> impl Iterator<Item = PortInfo> {
+        self.inner.opte_port_info()
+    }
+
     /// Remove the OPTE ports on this zone from the port manager.
     pub fn release_opte_ports(&mut self) {
         for (_, ticket) in self.inner.opte_ports.drain(..) {
@@ -752,6 +756,15 @@ impl InstalledZone {
         self.opte_ports.iter().map(|(port, _)| port)
     }
 
+    /// Returns references to the OPTE ports and metadata for this zone.
+    pub fn opte_port_info(&self) -> impl Iterator<Item = PortInfo> {
+        self.opte_ports.iter().map(|(port, ticket)| PortInfo {
+            port: port.clone(),
+            nic_id: ticket.id(),
+            nic_kind: ticket.kind(),
+        })
+    }
+
     /// Returns the filesystem path to the zone's root in the GZ.
     pub fn root(&self) -> Utf8PathBuf {
         self.zonepath.path.join(Self::ROOT_FS_PATH)

oximeter/instruments/Cargo.toml

@@ -31,7 +31,7 @@ uuid = { workspace = true, optional = true }
 omicron-workspace-hack.workspace = true
 
 [features]
-default = ["http-instruments", "cpu", "datalink", "zfs", "zone"]
+default = ["http-instruments", "cpu", "datalink", "opte_port", "zfs", "zone"]
 http-instruments = [
     "dep:chrono",
     "dep:dropshot",
@@ -59,6 +59,7 @@ kstat = [
 ]
 cpu = ["kstat"]
 datalink = ["kstat"]
+opte_port = ["kstat"]
 zfs = [
     "dep:anyhow",
     "dep:illumos-utils",

oximeter/instruments/src/kstat/link.rs

@@ -2,7 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-//! Report metrics about Ethernet data links on the host system
+//! Report metrics about Ethernet data links on the host system.
 
 use crate::kstat::ConvertNamedData;
 use crate::kstat::Error;

oximeter/instruments/src/kstat/mod.rs

@@ -89,6 +89,8 @@ use std::time::Duration;
 pub mod cpu;
 #[cfg(any(feature = "datalink", test))]
 pub mod link;
+#[cfg(any(feature = "opte_port", test))]
+pub mod opte_port;
 mod sampler;
 #[cfg(any(feature = "zone", test))]
 pub mod zone;