Sled-agent: detect switch backend (tofino, softnpu, et al) at runtime

package-manifest.toml

@@ -872,6 +872,8 @@ output.type = "zone"
 # $ cargo run --release --bin omicron-package -- -t default target create -p dev -m <gimlet|gimlet-standalone|non-gimlet> -s stub
 # $ cargo run --release --bin omicron-package -- package
 # $ pfexec ./target/release/omicron-package install
+#
+# The sled-agent config must also set switch_backend = "tofino_stub".
 [package.switch-stub]
 service_name = "switch"
 only_for_targets.switch = "stub"
@@ -899,6 +901,10 @@ output.type = "zone"
 # $ cargo run --release --bin omicron-package -- -t default target create -p dev -m <gimlet|gimlet-standalone|non-gimlet>
 # $ cargo run --release --bin omicron-package -- package
 # $ pfexec ./target/release/omicron-package install
+#
+# The sled-agent config selects between a SoftNPU zone on this host
+# (switch_backend = "soft_npu_zone") and a propolis SoftNPU device, which is
+# detected at startup.
 [package.switch-softnpu]
 service_name = "switch"
 only_for_targets.switch = "softnpu"

sled-agent/Cargo.toml

@@ -179,6 +179,3 @@ doc = false
 
 [features]
 image-trampoline = []
-switch-asic = []
-switch-stub = []
-switch-softnpu = []

sled-agent/src/bootstrap/pre_server.rs

@@ -15,6 +15,8 @@ use super::pumpkind;
 use super::server::StartError;
 use crate::config::Config;
 use crate::config::SidecarRevision;
+use crate::config::SledMode as SledModeConfig;
+use crate::config::SwitchBackend;
 use crate::ddm_reconciler::DdmReconciler;
 use crate::long_running_tasks::{
     LongRunningTaskHandles, LongRunningTaskResult, spawn_all_longrunning_tasks,
@@ -38,6 +40,7 @@ use omicron_common::address::Ipv6Subnet;
 use sled_agent_config_reconciler::ConfigReconcilerSpawnToken;
 use sled_hardware::DendriteAsic;
 use sled_hardware::SledMode;
+use sled_hardware::SwitchHardware;
 use sled_hardware::underlay;
 use sled_hardware::underlay::BootstrapInterface;
 use slog::Drain;
@@ -116,7 +119,7 @@ impl BootstrapAgentStartup {
         BootstrapNetworking::enable_ipv6_forwarding().await?;
 
         // Are we a gimlet or scrimlet?
-        let sled_mode = sled_mode_from_config(&config)?;
+        let sled_mode = sled_mode_from_config(&config, &log).await?;
 
         // Spawn all important long running tasks that live for the lifetime of
         // the process and are used by both the bootstrap agent and sled agent
@@ -290,48 +293,223 @@ async fn ensure_zfs_ramdisk_dataset() -> Result<(), StartError> {
     .map_err(StartError::EnsureZfsRamdiskDataset)
 }
 
-// Combine the `sled_mode` config with the build-time switch type to determine
-// the actual sled mode.
-fn sled_mode_from_config(config: &Config) -> Result<SledMode, StartError> {
-    use crate::config::SledMode as SledModeConfig;
-    let sled_mode = match config.sled_mode {
-        SledModeConfig::Auto => {
-            if !cfg!(feature = "switch-asic") {
-                return Err(StartError::IncorrectBuildPackaging(
-                    "sled-agent was not packaged with `switch-asic`",
+// Combine the `sled_mode` and `switch_backend` config with detected switch
+// hardware to determine the actual sled mode. Hardware is only probed when
+// the config leaves the backend to detection.
+async fn sled_mode_from_config(
+    config: &Config,
+    log: &Logger,
+) -> Result<SledMode, StartError> {
+    let detected = if matches!(config.sled_mode, SledModeConfig::Sled)
+        || config.switch_backend != SwitchBackend::Detect
+    {
+        None
+    } else {
+        let probe_log = log.clone();
+        tokio::task::spawn_blocking(move || {
+            sled_hardware::detect_switch_hardware(&probe_log)
+        })
+        .await
+        .expect("switch hardware detection panicked")
+        .map_err(StartError::DetectSwitch)?
+    };
+    resolve_sled_mode(
+        &config.sled_mode,
+        &config.switch_backend,
+        &config.sidecar_revision,
+        detected,
+    )
+}
+
+// Detected hardware wins in the priority order of `SwitchHardware`. With none
+// detected, `auto` leaves Tofino detection to the hardware monitor and
+// `scrimlet` assumes a Tofino ASIC when a physical sidecar is configured.
+fn resolve_sled_mode(
+    sled_mode: &SledModeConfig,
+    switch_backend: &SwitchBackend,
+    sidecar_revision: &SidecarRevision,
+    detected: Option<SwitchHardware>,
+) -> Result<SledMode, StartError> {
+    let forced_scrimlet = match sled_mode {
+        SledModeConfig::Sled => return Ok(SledMode::Sled),
+        SledModeConfig::Auto => false,
+        SledModeConfig::Scrimlet => true,
+    };
+
+    let asic = match switch_backend {
+        SwitchBackend::TofinoStub | SwitchBackend::SoftNpuZone
+            if !forced_scrimlet =>
+        {
+            return Err(StartError::SledModeConfig(
+                "switch_backend override requires sled_mode = \"scrimlet\"",
+            ));
+        }
+        SwitchBackend::TofinoStub => DendriteAsic::TofinoStub,
+        SwitchBackend::SoftNpuZone => {
+            if !matches!(sidecar_revision, SidecarRevision::SoftZone(_)) {
+                return Err(StartError::SledModeConfig(
+                    "switch_backend soft_npu_zone requires \
+                     sidecar_revision.soft_zone",
                 ));
             }
-            SledMode::Auto
+            DendriteAsic::SoftNpuZone
         }
-        SledModeConfig::Sled => SledMode::Sled,
-        SledModeConfig::Scrimlet => {
-            let asic = if cfg!(feature = "switch-asic") {
-                DendriteAsic::TofinoAsic
-            } else if cfg!(feature = "switch-stub") {
-                DendriteAsic::TofinoStub
-            } else if cfg!(feature = "switch-softnpu") {
-                match config.sidecar_revision {
-                    SidecarRevision::SoftZone(_) => DendriteAsic::SoftNpuZone,
-                    SidecarRevision::SoftPropolis(_) => {
-                        DendriteAsic::SoftNpuPropolisDevice
-                    }
-                    _ => {
-                        return Err(StartError::IncorrectBuildPackaging(
-                            "sled-agent configured to run on softnpu zone but dosen't \
-                         have a softnpu sidecar revision",
-                        ));
-                    }
+        SwitchBackend::Detect => match detected {
+            Some(SwitchHardware::Tofino) => DendriteAsic::TofinoAsic,
+            Some(SwitchHardware::SoftNpuPropolis { .. }) => {
+                if !matches!(sidecar_revision, SidecarRevision::SoftPropolis(_))
+                {
+                    return Err(StartError::SledModeConfig(
+                        "SoftNPU device present but sidecar_revision is \
+                         not soft_propolis",
+                    ));
                 }
-            } else {
-                return Err(StartError::IncorrectBuildPackaging(
-                    "sled-agent configured to run on scrimlet but wasn't \
-                        packaged with switch zone",
+                DendriteAsic::SoftNpuPropolisDevice
+            }
+            None if !forced_scrimlet => return Ok(SledMode::Auto),
+            None if sidecar_revision.is_physical() => DendriteAsic::TofinoAsic,
+            None => {
+                return Err(StartError::SledModeConfig(
+                    "sled_mode is scrimlet but no switch hardware was \
+                     detected and sidecar_revision is not physical",
                 ));
+            }
+        },
+    };
+    Ok(SledMode::Scrimlet { asic })
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::SoftPortConfig;
+
+    const AUTO: SledModeConfig = SledModeConfig::Auto;
+    const SLED: SledModeConfig = SledModeConfig::Sled;
+    const SCRIMLET: SledModeConfig = SledModeConfig::Scrimlet;
+    const DETECT: SwitchBackend = SwitchBackend::Detect;
+    const STUB: SwitchBackend = SwitchBackend::TofinoStub;
+    const ZONE: SwitchBackend = SwitchBackend::SoftNpuZone;
+
+    #[derive(Clone, Copy)]
+    enum Sidecar {
+        Physical,
+        SoftPropolis,
+        SoftZone,
+    }
+
+    impl From<Sidecar> for SidecarRevision {
+        fn from(sidecar: Sidecar) -> Self {
+            let ports =
+                SoftPortConfig { front_port_count: 2, rear_port_count: 4 };
+            match sidecar {
+                Sidecar::Physical => SidecarRevision::Physical("b".to_string()),
+                Sidecar::SoftPropolis => SidecarRevision::SoftPropolis(ports),
+                Sidecar::SoftZone => SidecarRevision::SoftZone(ports),
+            }
+        }
+    }
+
+    #[derive(Clone, Copy)]
+    enum Found {
+        Nothing,
+        Tofino,
+        SoftNpu,
+    }
+
+    impl From<Found> for Option<SwitchHardware> {
+        fn from(found: Found) -> Self {
+            match found {
+                Found::Nothing => None,
+                Found::Tofino => Some(SwitchHardware::Tofino),
+                Found::SoftNpu => Some(SwitchHardware::SoftNpuPropolis {
+                    path: "/devices/pci@0,0/pci1af4,9@6:9p".to_string(),
+                }),
+            }
+        }
+    }
+
+    #[derive(Debug, PartialEq)]
+    enum Expect {
+        Mode(SledMode),
+        ConfigError,
+    }
+
+    use DendriteAsic::*;
+    use Expect::*;
+    use Found::*;
+    use Sidecar::*;
+
+    fn scrimlet(asic: DendriteAsic) -> Expect {
+        Mode(SledMode::Scrimlet { asic })
+    }
+
+    #[test]
+    fn resolve_sled_mode_table() {
+        let cases = [
+            // Production gimlet config: auto with a physical sidecar. Nothing
+            // detected leaves Tofino detection to the hardware monitor.
+            (AUTO, DETECT, Physical, Nothing, Mode(SledMode::Auto)),
+            (AUTO, DETECT, Physical, Tofino, scrimlet(TofinoAsic)),
+            // gimlet-standalone: forced scrimlet with a physical sidecar,
+            // including a Tofino whose driver has not attached yet.
+            (SCRIMLET, DETECT, Physical, Tofino, scrimlet(TofinoAsic)),
+            (SCRIMLET, DETECT, Physical, Nothing, scrimlet(TofinoAsic)),
+            // Tofino wins under any sidecar config.
+            (AUTO, DETECT, SoftPropolis, Tofino, scrimlet(TofinoAsic)),
+            (AUTO, DETECT, SoftZone, Tofino, scrimlet(TofinoAsic)),
+            // A forced sled ignores attached hardware.
+            (SLED, DETECT, Physical, Tofino, Mode(SledMode::Sled)),
+            (SLED, DETECT, SoftPropolis, SoftNpu, Mode(SledMode::Sled)),
+            (SLED, STUB, Physical, Nothing, Mode(SledMode::Sled)),
+            // Voxel: forced scrimlets carry a SoftNPU device; auto works the
+            // same way with the device deciding.
+            (
+                SCRIMLET,
+                DETECT,
+                SoftPropolis,
+                SoftNpu,
+                scrimlet(SoftNpuPropolisDevice),
+            ),
+            (
+                AUTO,
+                DETECT,
+                SoftPropolis,
+                SoftNpu,
+                scrimlet(SoftNpuPropolisDevice),
+            ),
+            (AUTO, DETECT, SoftPropolis, Nothing, Mode(SledMode::Auto)),
+            // Forced scrimlet with nothing detected needs a physical sidecar.
+            (SCRIMLET, DETECT, SoftPropolis, Nothing, ConfigError),
+            (SCRIMLET, DETECT, SoftZone, Nothing, ConfigError),
+            // SoftNPU needs a soft_propolis sidecar.
+            (SCRIMLET, DETECT, Physical, SoftNpu, ConfigError),
+            (AUTO, DETECT, SoftZone, SoftNpu, ConfigError),
+            // Overrides: dev SoftNPU zone and stub Dendrite, forced scrimlet
+            // only, and the zone needs a soft_zone sidecar.
+            (SCRIMLET, ZONE, SoftZone, Nothing, scrimlet(SoftNpuZone)),
+            (SCRIMLET, ZONE, SoftPropolis, Nothing, ConfigError),
+            (SCRIMLET, STUB, Physical, Nothing, scrimlet(TofinoStub)),
+            (AUTO, STUB, Physical, Nothing, ConfigError),
+            (AUTO, ZONE, SoftZone, Nothing, ConfigError),
+        ];
+
+        for (i, (mode, backend, sidecar, found, expected)) in
+            cases.into_iter().enumerate()
+        {
+            let actual = match resolve_sled_mode(
+                &mode,
+                &backend,
+                &sidecar.into(),
+                found.into(),
+            ) {
+                Ok(mode) => Mode(mode),
+                Err(StartError::SledModeConfig(_)) => ConfigError,
+                Err(e) => panic!("case {i}: unexpected error {e:?}"),
             };
-            SledMode::Scrimlet { asic }
+            assert_eq!(actual, expected, "case {i}");
         }
-    };
-    Ok(sled_mode)
+    }
 }
 
 #[derive(Debug, Clone)]

sled-agent/src/bootstrap/pumpkind.rs

@@ -6,20 +6,36 @@
 
 use thiserror::Error;
 
+const SERVICE_FMRI: &str = "svc:/oxide/pumpkind";
+const MANIFEST_PATH: &str =
+    "/opt/oxide/pumpkind/lib/svc/manifest/system/pumpkind.xml";
+
 #[derive(Debug, Error)]
 pub enum Error {
     #[error("Error configuring service: {0}")]
     Config(#[from] smf::ConfigError),
 
     #[error("Error administering service: {0}")]
     Adm(#[from] smf::AdmError),
+
+    #[error("Error detecting Oxide sled: {0}")]
+    Detect(anyhow::Error),
 }
 
-#[cfg(feature = "switch-asic")]
+/// Import and enable pumpkind on Oxide sleds with the manifest installed.
 pub(super) fn enable_pumpkind_service(log: &slog::Logger) -> Result<(), Error> {
-    const SERVICE_FMRI: &str = "svc:/oxide/pumpkind";
-    const MANIFEST_PATH: &str =
-        "/opt/oxide/pumpkind/lib/svc/manifest/system/pumpkind.xml";
+    if !sled_hardware::is_oxide_sled().map_err(Error::Detect)? {
+        info!(log, "not an Oxide sled; skipping pumpkind");
+        return Ok(());
+    }
+    if !std::path::Path::new(MANIFEST_PATH).exists() {
+        info!(
+            log,
+            "pumpkind manifest not installed; skipping";
+            "path" => MANIFEST_PATH,
+        );
+        return Ok(());
+    }
 
     info!(log, "Importing pumpkind service"; "path" => MANIFEST_PATH);
     smf::Config::import().run(MANIFEST_PATH)?;
@@ -32,10 +48,3 @@ pub(super) fn enable_pumpkind_service(log: &slog::Logger) -> Result<(), Error> {
 
     Ok(())
 }
-
-#[cfg(not(feature = "switch-asic"))]
-pub(super) fn enable_pumpkind_service(
-    _log: &slog::Logger,
-) -> Result<(), Error> {
-    Ok(())
-}

sled-agent/src/bootstrap/server.rs

@@ -131,8 +131,11 @@ pub enum StartError {
     #[error("Failed to enable ipv6-forwarding")]
     EnableIpv6Forwarding(#[from] illumos_utils::ExecutionError),
 
-    #[error("Incorrect binary packaging: {0}")]
-    IncorrectBuildPackaging(&'static str),
+    #[error("Invalid sled mode configuration: {0}")]
+    SledModeConfig(&'static str),
+
+    #[error("Failed to detect switch hardware")]
+    DetectSwitch(#[source] sled_hardware::SwitchDetectError),
 
     #[error("Failed to start HardwareManager: {0}")]
     StartHardwareManager(String),