diff --git a/package-manifest.toml b/package-manifest.toml index edb753576ce..87dbd245139 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -872,6 +872,9 @@ output.type = "zone" # $ cargo run --release --bin omicron-package -- -t default target create -p dev -m -s stub # $ cargo run --release --bin omicron-package -- package # $ pfexec ./target/release/omicron-package install +# +# The sled-agent config must set +# deployment = { kind = "custom", sled_mode = "scrimlet", switch = "tofino_stub" }. [package.switch-stub] service_name = "switch" only_for_targets.switch = "stub" @@ -899,6 +902,10 @@ output.type = "zone" # $ cargo run --release --bin omicron-package -- -t default target create -p dev -m # $ cargo run --release --bin omicron-package -- package # $ pfexec ./target/release/omicron-package install +# +# The sled-agent config selects between a SoftNPU zone on this host +# (deployment kind "standalone") and a propolis SoftNPU device, which is +# detected at startup (deployment kind "virtual"). [package.switch-softnpu] service_name = "switch" only_for_targets.switch = "softnpu" diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 47577484c32..037d491b7c9 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -179,6 +179,3 @@ doc = false [features] image-trampoline = [] -switch-asic = [] -switch-stub = [] -switch-softnpu = [] diff --git a/sled-agent/src/bootstrap/pre_server.rs b/sled-agent/src/bootstrap/pre_server.rs index d3578f75037..506838b9ae1 100644 --- a/sled-agent/src/bootstrap/pre_server.rs +++ b/sled-agent/src/bootstrap/pre_server.rs @@ -14,7 +14,6 @@ use super::maghemite; use super::pumpkind; use super::server::StartError; use crate::config::Config; -use crate::config::SidecarRevision; use crate::ddm_reconciler::DdmReconciler; use crate::long_running_tasks::{ LongRunningTaskHandles, LongRunningTaskResult, spawn_all_longrunning_tasks, @@ -36,8 +35,6 @@ use illumos_utils::zone::Zones; use omicron_common::FileKv; use omicron_common::address::Ipv6Subnet; use sled_agent_config_reconciler::ConfigReconcilerSpawnToken; -use sled_hardware::DendriteAsic; -use sled_hardware::SledMode; use sled_hardware::underlay; use sled_hardware::underlay::BootstrapInterface; use slog::Drain; @@ -116,7 +113,7 @@ impl BootstrapAgentStartup { BootstrapNetworking::enable_ipv6_forwarding().await?; // Are we a gimlet or scrimlet? - let sled_mode = sled_mode_from_config(&config)?; + let sled_mode = config.deployment.sled_mode(&log).await?; // Spawn all important long running tasks that live for the lifetime of // the process and are used by both the bootstrap agent and sled agent @@ -141,7 +138,7 @@ impl BootstrapAgentStartup { ddm_reconciler, startup_networking, sled_mode, - config.sidecar_revision.clone(), + config.deployment.sidecar_revision(), config.switch_zone_maghemite_links.clone(), long_running_task_handles.zone_image_resolver.clone(), ); @@ -290,50 +287,6 @@ async fn ensure_zfs_ramdisk_dataset() -> Result<(), StartError> { .map_err(StartError::EnsureZfsRamdiskDataset) } -// Combine the `sled_mode` config with the build-time switch type to determine -// the actual sled mode. -fn sled_mode_from_config(config: &Config) -> Result { - use crate::config::SledMode as SledModeConfig; - let sled_mode = match config.sled_mode { - SledModeConfig::Auto => { - if !cfg!(feature = "switch-asic") { - return Err(StartError::IncorrectBuildPackaging( - "sled-agent was not packaged with `switch-asic`", - )); - } - SledMode::Auto - } - SledModeConfig::Sled => SledMode::Sled, - SledModeConfig::Scrimlet => { - let asic = if cfg!(feature = "switch-asic") { - DendriteAsic::TofinoAsic - } else if cfg!(feature = "switch-stub") { - DendriteAsic::TofinoStub - } else if cfg!(feature = "switch-softnpu") { - match config.sidecar_revision { - SidecarRevision::SoftZone(_) => DendriteAsic::SoftNpuZone, - SidecarRevision::SoftPropolis(_) => { - DendriteAsic::SoftNpuPropolisDevice - } - _ => { - return Err(StartError::IncorrectBuildPackaging( - "sled-agent configured to run on softnpu zone but dosen't \ - have a softnpu sidecar revision", - )); - } - } - } else { - return Err(StartError::IncorrectBuildPackaging( - "sled-agent configured to run on scrimlet but wasn't \ - packaged with switch zone", - )); - }; - SledMode::Scrimlet { asic } - } - }; - Ok(sled_mode) -} - #[derive(Debug, Clone)] pub(crate) struct BootstrapNetworking { pub(crate) bootstrap_etherstub: dladm::Etherstub, diff --git a/sled-agent/src/bootstrap/pumpkind.rs b/sled-agent/src/bootstrap/pumpkind.rs index 63be799519d..1c5acfc964d 100644 --- a/sled-agent/src/bootstrap/pumpkind.rs +++ b/sled-agent/src/bootstrap/pumpkind.rs @@ -6,6 +6,10 @@ use thiserror::Error; +const SERVICE_FMRI: &str = "svc:/oxide/pumpkind"; +const MANIFEST_PATH: &str = + "/opt/oxide/pumpkind/lib/svc/manifest/system/pumpkind.xml"; + #[derive(Debug, Error)] pub enum Error { #[error("Error configuring service: {0}")] @@ -13,13 +17,25 @@ pub enum Error { #[error("Error administering service: {0}")] Adm(#[from] smf::AdmError), + + #[error("Error detecting Oxide sled: {0}")] + Detect(anyhow::Error), } -#[cfg(feature = "switch-asic")] +/// Import and enable pumpkind on Oxide sleds with the manifest installed. pub(super) fn enable_pumpkind_service(log: &slog::Logger) -> Result<(), Error> { - const SERVICE_FMRI: &str = "svc:/oxide/pumpkind"; - const MANIFEST_PATH: &str = - "/opt/oxide/pumpkind/lib/svc/manifest/system/pumpkind.xml"; + if !sled_hardware::is_oxide_sled().map_err(Error::Detect)? { + info!(log, "not an Oxide sled; skipping pumpkind"); + return Ok(()); + } + if !std::path::Path::new(MANIFEST_PATH).exists() { + info!( + log, + "pumpkind manifest not installed; skipping"; + "path" => MANIFEST_PATH, + ); + return Ok(()); + } info!(log, "Importing pumpkind service"; "path" => MANIFEST_PATH); smf::Config::import().run(MANIFEST_PATH)?; @@ -32,10 +48,3 @@ pub(super) fn enable_pumpkind_service(log: &slog::Logger) -> Result<(), Error> { Ok(()) } - -#[cfg(not(feature = "switch-asic"))] -pub(super) fn enable_pumpkind_service( - _log: &slog::Logger, -) -> Result<(), Error> { - Ok(()) -} diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index 03b178e5c0a..c20eabbec56 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -131,8 +131,11 @@ pub enum StartError { #[error("Failed to enable ipv6-forwarding")] EnableIpv6Forwarding(#[from] illumos_utils::ExecutionError), - #[error("Incorrect binary packaging: {0}")] - IncorrectBuildPackaging(&'static str), + #[error("Invalid sled mode configuration: {0}")] + SledModeConfig(&'static str), + + #[error("Failed to detect switch hardware")] + DetectSwitch(#[source] sled_hardware::SwitchDetectError), #[error("Failed to start HardwareManager: {0}")] StartHardwareManager(String), diff --git a/sled-agent/src/config.rs b/sled-agent/src/config.rs index 5f4d2d83be3..f1aa66bc3ec 100644 --- a/sled-agent/src/config.rs +++ b/sled-agent/src/config.rs @@ -14,20 +14,238 @@ use illumos_utils::dladm::PhysicalLink; use omicron_common::vlan::VlanID; use serde::Deserialize; use sled_hardware::DataLinks; +use sled_hardware::DendriteAsic; use sled_hardware::ExternalDisks; +use sled_hardware::SledMode; +use sled_hardware::SwitchProbe; +use slog::Logger; use sprockets_tls::keys::SprocketsConfig; -#[derive(Clone, Debug, Deserialize)] +use crate::bootstrap::server::StartError; + +/// The role a deployment asks of this sled; `auto` lets detection decide. +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] -pub enum SledMode { +pub enum SledRole { Auto, #[serde(alias = "gimlet")] Sled, Scrimlet, } -#[derive(Debug, Clone, Deserialize)] -#[serde(rename_all = "snake_case")] +/// Switch backend of a `custom` deployment, with the parameters it needs. +/// Flattened into the deployment table: `switch` names the backend and its +/// parameters sit beside it. +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "snake_case", tag = "switch")] +pub enum Switch { + TofinoAsic { + #[serde(default = "default_sidecar_revision")] + sidecar_revision: String, + }, + TofinoStub { + #[serde(default = "default_sidecar_revision")] + sidecar_revision: String, + }, + SoftNpuPropolisDevice { + front_port_count: u8, + rear_port_count: u8, + }, + SoftNpuZone { + front_port_count: u8, + rear_port_count: u8, + }, +} + +impl Switch { + pub fn asic(&self) -> DendriteAsic { + match self { + Switch::TofinoAsic { .. } => DendriteAsic::TofinoAsic, + Switch::TofinoStub { .. } => DendriteAsic::TofinoStub, + Switch::SoftNpuPropolisDevice { .. } => { + DendriteAsic::SoftNpuPropolisDevice + } + Switch::SoftNpuZone { .. } => DendriteAsic::SoftNpuZone, + } + } + + fn sidecar_revision(&self) -> SidecarRevision { + match self { + Switch::TofinoAsic { sidecar_revision } + | Switch::TofinoStub { sidecar_revision } => { + SidecarRevision::Physical(sidecar_revision.clone()) + } + Switch::SoftNpuPropolisDevice { + front_port_count, + rear_port_count, + } => SidecarRevision::SoftPropolis(SoftPortConfig { + front_port_count: *front_port_count, + rear_port_count: *rear_port_count, + }), + Switch::SoftNpuZone { front_port_count, rear_port_count } => { + SidecarRevision::SoftZone(SoftPortConfig { + front_port_count: *front_port_count, + rear_port_count: *rear_port_count, + }) + } + } + } +} + +/// How this sled is deployed. Selects the switch backend and whether the +/// sled is a scrimlet, which is detected where the backend allows it. +#[derive(Clone, Debug, Deserialize)] +#[serde(rename_all = "snake_case", tag = "kind")] +pub enum Deployment { + /// Oxide rack. The hardware monitor detects the Tofino ASIC. + Production { + /// Sidecar board revision + #[serde(default = "default_sidecar_revision")] + sidecar_revision: String, + }, + /// Propolis-hosted lab. A SoftNPU device makes the sled a scrimlet. + Virtual { front_port_count: u8, rear_port_count: u8 }, + /// One host running a SoftNPU zone. Always a scrimlet. + Standalone { front_port_count: u8, rear_port_count: u8 }, + /// Explicit sled mode and switch backend. + Custom { + sled_mode: SledRole, + #[serde(flatten)] + switch: Switch, + }, +} + +fn default_sidecar_revision() -> String { + "b".to_string() +} + +impl Deployment { + /// Reject custom combinations that cannot be resolved at startup. + pub fn validate(&self) -> Result<(), String> { + match self { + Deployment::Custom { + sled_mode: SledRole::Auto, + switch: + switch + @ (Switch::TofinoStub { .. } | Switch::SoftNpuZone { .. }), + } => Err(format!( + "switch {:?} has no hardware to detect; sled_mode must be \ + \"sled\" or \"scrimlet\"", + switch.asic() + )), + _ => Ok(()), + } + } + + /// Resolve the sled mode: probe for the switch hardware this deployment + /// can carry, then decide from what was found. + pub async fn sled_mode( + &self, + log: &Logger, + ) -> Result { + let found = match self.probe() { + Some(probe) => { + let log = log.clone(); + // The probe touches devinfo and device nodes, so it may block. + tokio::task::spawn_blocking(move || { + sled_hardware::detect_switch_hardware(&log, probe) + }) + .await + .expect("switch detection panicked") + .map_err(StartError::DetectSwitch)? + } + None => None, + }; + self.resolve(found).map_err(StartError::SledModeConfig) + } + + /// What startup detection should look for, if anything. A sled never + /// probes, and the stub and zone backends have nothing to find. + fn probe(&self) -> Option { + match self { + Deployment::Production { .. } => Some(SwitchProbe::PhysicalAsic), + Deployment::Virtual { .. } => Some(SwitchProbe::SoftNpu), + Deployment::Standalone { .. } => None, + Deployment::Custom { sled_mode: SledRole::Sled, .. } => None, + Deployment::Custom { + switch: Switch::TofinoAsic { .. }, .. + } => Some(SwitchProbe::PhysicalAsic), + Deployment::Custom { + switch: Switch::SoftNpuPropolisDevice { .. }, + .. + } => Some(SwitchProbe::SoftNpu), + Deployment::Custom { .. } => None, + } + } + + /// Decide the sled mode from what `probe()` found. + fn resolve( + &self, + found: Option, + ) -> Result { + use Deployment::*; + Ok(match (self, found) { + (Production { .. } | Virtual { .. }, Some(asic)) => { + SledMode::Scrimlet { asic } + } + // The ASIC driver can attach after startup; the hardware monitor + // keeps watching for it. + (Production { .. }, None) => SledMode::Auto, + (Virtual { .. }, None) => SledMode::Sled, + (Standalone { .. }, _) => { + SledMode::Scrimlet { asic: DendriteAsic::SoftNpuZone } + } + (Custom { sled_mode: SledRole::Sled, .. }, _) => SledMode::Sled, + (Custom { .. }, Some(asic)) => SledMode::Scrimlet { asic }, + (Custom { sled_mode: SledRole::Auto, .. }, None) => { + match self.probe() { + Some(SwitchProbe::PhysicalAsic) => SledMode::Auto, + Some(SwitchProbe::SoftNpu) => SledMode::Sled, + None => { + return Err("switch backend has no hardware to detect"); + } + } + } + (Custom { sled_mode: SledRole::Scrimlet, switch }, None) => { + match switch { + Switch::SoftNpuPropolisDevice { .. } => { + return Err( + "sled_mode is scrimlet but no SoftNPU device is \ + present", + ); + } + // A physical ASIC may attach later; force the role now. + _ => SledMode::Scrimlet { asic: switch.asic() }, + } + } + }) + } + + /// Sidecar parameters for the switch zone services. + pub fn sidecar_revision(&self) -> SidecarRevision { + match self { + Deployment::Production { sidecar_revision } => { + SidecarRevision::Physical(sidecar_revision.clone()) + } + Deployment::Virtual { front_port_count, rear_port_count } => { + SidecarRevision::SoftPropolis(SoftPortConfig { + front_port_count: *front_port_count, + rear_port_count: *rear_port_count, + }) + } + Deployment::Standalone { front_port_count, rear_port_count } => { + SidecarRevision::SoftZone(SoftPortConfig { + front_port_count: *front_port_count, + rear_port_count: *rear_port_count, + }) + } + Deployment::Custom { switch, .. } => switch.sidecar_revision(), + } + } +} + +/// Sidecar parameters derived from the deployment. +#[derive(Debug, Clone)] pub enum SidecarRevision { Physical(String), SoftZone(SoftPortConfig), @@ -43,7 +261,7 @@ impl SidecarRevision { } } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone)] pub struct SoftPortConfig { /// Number of front ports pub front_port_count: u8, @@ -62,10 +280,8 @@ pub struct Config { pub dropshot: ConfigDropshot, /// Configuration for the sled agent debug log pub log: ConfigLogging, - /// The sled's mode of operation (auto detect or force gimlet/scrimlet). - pub sled_mode: SledMode, - // TODO: Remove once this can be auto-detected. - pub sidecar_revision: SidecarRevision, + /// How this sled is deployed, which selects the switch backend. + pub deployment: Deployment, /// Optional percentage of otherwise-unbudgeted DRAM to reserve for guest /// memory, after accounting for expected host OS memory consumption and, if /// set, `vmm_reservoir_size_mb`. @@ -136,6 +352,8 @@ pub enum ConfigError { #[source] err: anyhow::Error, }, + #[error("Invalid deployment in {path}: {reason}")] + InvalidDeployment { path: Utf8PathBuf, reason: String }, #[error("Loading certificate")] Certificate(#[source] anyhow::Error), #[error("Could not determine if host is an Oxide sled")] @@ -149,9 +367,12 @@ impl Config { let path = path.as_ref(); let contents = std::fs::read_to_string(&path) .map_err(|err| ConfigError::Io { path: path.into(), err })?; - let config = toml::from_str(&contents).map_err(|err| { + let config: Self = toml::from_str(&contents).map_err(|err| { ConfigError::Parse { path: path.into(), err: err.into() } })?; + config.deployment.validate().map_err(|reason| { + ConfigError::InvalidDeployment { path: path.into(), reason } + })?; Ok(config) } @@ -181,6 +402,124 @@ impl Config { #[cfg(test)] mod test { use super::*; + use DendriteAsic::*; + + const AUTO: SledRole = SledRole::Auto; + const SLED: SledRole = SledRole::Sled; + const SCRIMLET: SledRole = SledRole::Scrimlet; + + fn production() -> Deployment { + Deployment::Production { sidecar_revision: "b".to_string() } + } + + fn virtual_lab() -> Deployment { + Deployment::Virtual { front_port_count: 2, rear_port_count: 4 } + } + + fn standalone() -> Deployment { + Deployment::Standalone { front_port_count: 1, rear_port_count: 1 } + } + + fn custom(sled_mode: SledRole, asic: DendriteAsic) -> Deployment { + let switch = match asic { + TofinoAsic => Switch::TofinoAsic { sidecar_revision: "b".into() }, + TofinoStub => Switch::TofinoStub { sidecar_revision: "b".into() }, + SoftNpuPropolisDevice => Switch::SoftNpuPropolisDevice { + front_port_count: 1, + rear_port_count: 1, + }, + SoftNpuZone => { + Switch::SoftNpuZone { front_port_count: 1, rear_port_count: 1 } + } + }; + Deployment::Custom { sled_mode, switch } + } + + fn scrimlet(asic: DendriteAsic) -> Result { + Ok(SledMode::Scrimlet { asic }) + } + + // Each case: deployment, what the probe found (None when nothing was + // found or no probe ran), expected mode or a config error. + #[test] + fn resolve_table() { + let err: Result = Err(""); + let cases = [ + // Production: a found ASIC is a scrimlet now; nothing found + // leaves it to the hardware monitor. + (production(), Some(TofinoAsic), scrimlet(TofinoAsic)), + (production(), None, Ok(SledMode::Auto)), + // Virtual: the SoftNPU device decides. + ( + virtual_lab(), + Some(SoftNpuPropolisDevice), + scrimlet(SoftNpuPropolisDevice), + ), + (virtual_lab(), None, Ok(SledMode::Sled)), + // Standalone is always a scrimlet with nothing to detect. + (standalone(), None, scrimlet(SoftNpuZone)), + // Custom sled: never a switch zone. + (custom(SLED, TofinoAsic), None, Ok(SledMode::Sled)), + (custom(SLED, SoftNpuPropolisDevice), None, Ok(SledMode::Sled)), + // Custom scrimlet: a physical ASIC may attach later, the zone and + // stub run as configured, the propolis device must be present. + ( + custom(SCRIMLET, TofinoAsic), + Some(TofinoAsic), + scrimlet(TofinoAsic), + ), + (custom(SCRIMLET, TofinoAsic), None, scrimlet(TofinoAsic)), + (custom(SCRIMLET, TofinoStub), None, scrimlet(TofinoStub)), + (custom(SCRIMLET, SoftNpuZone), None, scrimlet(SoftNpuZone)), + ( + custom(SCRIMLET, SoftNpuPropolisDevice), + Some(SoftNpuPropolisDevice), + scrimlet(SoftNpuPropolisDevice), + ), + (custom(SCRIMLET, SoftNpuPropolisDevice), None, err), + // Custom auto matches production and virtual. + (custom(AUTO, TofinoAsic), Some(TofinoAsic), scrimlet(TofinoAsic)), + (custom(AUTO, TofinoAsic), None, Ok(SledMode::Auto)), + ( + custom(AUTO, SoftNpuPropolisDevice), + Some(SoftNpuPropolisDevice), + scrimlet(SoftNpuPropolisDevice), + ), + (custom(AUTO, SoftNpuPropolisDevice), None, Ok(SledMode::Sled)), + (custom(AUTO, TofinoStub), None, err), + ]; + for (i, (deployment, found, expected)) in cases.into_iter().enumerate() + { + let actual = deployment.resolve(found).map_err(|_| ""); + assert_eq!(actual, expected, "case {i}"); + } + } + + #[test] + fn probe_targets() { + assert_eq!(production().probe(), Some(SwitchProbe::PhysicalAsic)); + assert_eq!(virtual_lab().probe(), Some(SwitchProbe::SoftNpu)); + assert_eq!(standalone().probe(), None); + assert_eq!(custom(SLED, TofinoAsic).probe(), None); + assert_eq!(custom(SCRIMLET, TofinoStub).probe(), None); + assert_eq!(custom(SCRIMLET, SoftNpuZone).probe(), None); + assert_eq!( + custom(AUTO, TofinoAsic).probe(), + Some(SwitchProbe::PhysicalAsic) + ); + assert_eq!( + custom(SCRIMLET, SoftNpuPropolisDevice).probe(), + Some(SwitchProbe::SoftNpu) + ); + } + + #[test] + fn custom_deployment_validation() { + assert!(custom(AUTO, TofinoStub).validate().is_err()); + assert!(custom(AUTO, SoftNpuZone).validate().is_err()); + assert!(custom(SCRIMLET, TofinoStub).validate().is_ok()); + assert!(custom(AUTO, TofinoAsic).validate().is_ok()); + } use slog_error_chain::InlineErrorChain; #[test] diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index f354b6835a6..e3ba411efe1 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -714,8 +714,10 @@ impl SledAgent { config_reconciler_spawn_token.subscribe_update_disposition(), )?; - let svc_config = - services::Config::new(identifiers, config.sidecar_revision.clone()); + let svc_config = services::Config::new( + identifiers, + config.deployment.sidecar_revision(), + ); // Get our system network config from the bootstore; we cannot proceed // until we have this, as we need to set up uplinks inside the switch diff --git a/sled-hardware/src/illumos/mod.rs b/sled-hardware/src/illumos/mod.rs index 05641b76f99..61f26f8543b 100644 --- a/sled-hardware/src/illumos/mod.rs +++ b/sled-hardware/src/illumos/mod.rs @@ -7,7 +7,9 @@ use crate::ExternalDisks; use crate::HardwareView; use crate::TofinoSnapshot; use crate::TofinoView; -use crate::{DendriteAsic, SledMode, UnparsedDisk}; +use crate::{ + DendriteAsic, SledMode, SwitchDetectError, SwitchProbe, UnparsedDisk, +}; use camino::Utf8PathBuf; use gethostname::gethostname; use illumos_devinfo::{DevInfo, DevLinkType, DevLinks, Node, Property}; @@ -27,12 +29,46 @@ use uuid::Uuid; mod gpt; mod partitions; +mod softnpu; mod sysconf; pub use partitions::{NvmeFormattingError, ensure_partition_layout}; const TOFINO_MONITOR: &'static str = "/opt/oxide/sled-agent/tofino-monitor"; +/// Whether one kind of physical ASIC is present in the device tree. +type AsicProbe = fn(&Logger, &mut DevInfo) -> bool; + +/// Physical ASIC probes in detection priority order. A new ASIC is a +/// `DendriteAsic` variant, an entry here, and its hardware monitor view. +const PHYSICAL_ASICS: &[(DendriteAsic, AsicProbe)] = + &[(DendriteAsic::TofinoAsic, tofino_present)]; + +fn tofino_present(log: &Logger, devinfo: &mut DevInfo) -> bool { + get_tofino_snapshot(log, devinfo).exists +} + +/// Probe for the switch hardware a deployment can carry. Physical ASICs are +/// checked in `PHYSICAL_ASICS` order; the SoftNPU device is answered by its +/// 9p version handshake. Only device tree failures are errors. +pub fn detect_switch_hardware( + log: &Logger, + probe: SwitchProbe, +) -> Result, SwitchDetectError> { + let mut devinfo = + DevInfo::new_force_load().map_err(SwitchDetectError::DevInfo)?; + match probe { + SwitchProbe::PhysicalAsic => Ok(PHYSICAL_ASICS + .iter() + .find(|(_, present)| present(log, &mut devinfo)) + .map(|(asic, _)| *asic)), + SwitchProbe::SoftNpu => { + Ok(softnpu::find_softnpu_device(log, &mut devinfo)? + .then_some(DendriteAsic::SoftNpuPropolisDevice)) + } + } +} + #[derive(thiserror::Error, Debug)] enum Error { #[error("Failed to access devinfo: {0}")] diff --git a/sled-hardware/src/illumos/softnpu.rs b/sled-hardware/src/illumos/softnpu.rs new file mode 100644 index 00000000000..440d3d20616 --- /dev/null +++ b/sled-hardware/src/illumos/softnpu.rs @@ -0,0 +1,166 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Detection of the propolis SoftNPU 9p device. + +use crate::SwitchDetectError; +use crate::softnpu::{SOFTNPU_9P_VERSION, decode_rversion, encode_tversion}; +use illumos_devinfo::{DevInfo, Node}; +use slog::{Logger, debug, info, warn}; +use slog_error_chain::InlineErrorChain; +use std::fs::OpenOptions; +use std::io::{Read, Write}; +use std::os::unix::fs::OpenOptionsExt; +use std::time::Duration; + +const VIRTIO_VENDOR_ID: i32 = 0x1af4; +// Transitional and modern virtio 9p PCI device ids. +const VIRTIO_9P_DEVICE_IDS: [i32; 2] = [0x1009, 0x1049]; +const NINEP_MINOR: &str = "9p"; +const OPEN_ATTEMPTS: usize = 3; +const OPEN_RETRY_DELAY: Duration = Duration::from_millis(500); +const REPLY_BUF_LEN: usize = 65536; + +enum Probe { + Version(String), + Busy, +} + +/// Returns whether the propolis SoftNPU 9p device is attached. +/// +/// Every virtio 9p node with an attached driver is opened exclusively and +/// asked for its 9P version. Only the propolis SoftNPU handler answers with +/// `9P2000.P4`. A device that stays busy across retries, fails to open, or +/// answers with anything other than an Rversion is logged and skipped. Only +/// device tree failures are fatal. +pub(super) fn find_softnpu_device( + log: &Logger, + devinfo: &mut DevInfo, +) -> Result { + for node in devinfo.walk_node() { + let node = node.map_err(SwitchDetectError::DevInfo)?; + if probe_node(log, &node)? { + return Ok(true); + } + } + Ok(false) +} + +/// Returns whether `node` is the SoftNPU 9p device. +fn probe_node( + log: &Logger, + node: &Node<'_>, +) -> Result { + if !is_virtio_9p(node)? { + return Ok(false); + } + let Some(path) = ninep_minor_path(node)? else { + debug!( + log, + "virtio 9p node has no {NINEP_MINOR} minor"; + "node" => node.node_name(), + ); + return Ok(false); + }; + match probe_version(&path) { + Ok(Probe::Version(version)) if version == SOFTNPU_9P_VERSION => { + info!(log, "found SoftNPU 9p device"; "path" => path); + Ok(true) + } + Ok(Probe::Version(version)) => { + debug!( + log, + "virtio 9p device is not SoftNPU"; + "path" => path, + "version" => version, + ); + Ok(false) + } + Ok(Probe::Busy) => { + warn!(log, "virtio 9p device busy; skipping"; "path" => path); + Ok(false) + } + Err( + e @ (SwitchDetectError::Io { .. } + | SwitchDetectError::Protocol { .. }), + ) => { + warn!( + log, + "virtio 9p device probe failed; skipping"; + "error" => InlineErrorChain::new(&e), + ); + Ok(false) + } + Err(e) => Err(e), + } +} + +fn is_virtio_9p(node: &Node<'_>) -> Result { + let mut vendor = None; + let mut device = None; + for prop in node.props() { + let prop = prop.map_err(SwitchDetectError::DevInfo)?; + match prop.name().as_str() { + "vendor-id" => vendor = prop.as_i32(), + "device-id" => device = prop.as_i32(), + _ => {} + } + } + Ok(vendor == Some(VIRTIO_VENDOR_ID) + && device.is_some_and(|d| VIRTIO_9P_DEVICE_IDS.contains(&d))) +} + +fn ninep_minor_path( + node: &Node<'_>, +) -> Result, SwitchDetectError> { + for minor in node.minors() { + let minor = minor.map_err(SwitchDetectError::DevInfo)?; + if minor.name() == NINEP_MINOR { + let path = + minor.devfs_path().map_err(SwitchDetectError::DevInfo)?; + return Ok(Some(format!("/devices{path}"))); + } + } + Ok(None) +} + +/// One Tversion/Rversion exchange over the vio9p character device. +/// +/// The driver permits a single exclusive open, so EBUSY means another +/// consumer such as scadm or a 9p mount currently holds the device. +fn probe_version(path: &str) -> Result { + for attempt in 1..=OPEN_ATTEMPTS { + let mut file = match OpenOptions::new() + .read(true) + .write(true) + .custom_flags(libc::O_EXCL) + .open(path) + { + Ok(file) => file, + Err(e) if e.raw_os_error() == Some(libc::EBUSY) => { + if attempt < OPEN_ATTEMPTS { + std::thread::sleep(OPEN_RETRY_DELAY); + } + continue; + } + Err(err) => { + return Err(SwitchDetectError::Io { + path: path.to_string(), + err, + }); + } + }; + let io = |err| SwitchDetectError::Io { path: path.to_string(), err }; + file.write_all(&encode_tversion(SOFTNPU_9P_VERSION)).map_err(io)?; + let mut buf = vec![0u8; REPLY_BUF_LEN]; + let n = file.read(&mut buf).map_err(io)?; + return decode_rversion(&buf[..n]).map(Probe::Version).map_err( + |reason| SwitchDetectError::Protocol { + path: path.to_string(), + reason, + }, + ); + } + Ok(Probe::Busy) +} diff --git a/sled-hardware/src/lib.rs b/sled-hardware/src/lib.rs index d0ad7db5e6b..42adee9bb3b 100644 --- a/sled-hardware/src/lib.rs +++ b/sled-hardware/src/lib.rs @@ -23,8 +23,35 @@ cfg_if::cfg_if! { pub mod cleanup; pub mod disk; pub use disk::*; +pub mod softnpu; pub mod underlay; +/// Failure while probing for switch hardware at startup. +#[derive(Debug, thiserror::Error)] +pub enum SwitchDetectError { + #[error("failed to walk device tree: {0}")] + DevInfo(anyhow::Error), + + #[error("{path}: {err}")] + Io { + path: String, + #[source] + err: std::io::Error, + }, + + #[error("{path}: malformed Rversion: {reason}")] + Protocol { path: String, reason: String }, +} + +/// What startup switch detection should look for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SwitchProbe { + /// A physical sidecar ASIC, whichever kind is attached. + PhysicalAsic, + /// The propolis SoftNPU virtio 9p device. + SoftNpu, +} + // The type of networking 'ASIC' the Dendrite service is expected to manage #[derive( Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash, @@ -81,9 +108,10 @@ pub enum ExternalDisks { } /// Configuration for forcing a sled to run as a Scrimlet or compute Sled -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum SledMode { - /// Automatically detect whether to run as a compute sled or Scrimlet (w/ real Tofino ASIC) + /// Run as a compute sled unless a Tofino ASIC is present, in which case + /// run as a Scrimlet Auto, /// Force sled to run as a Gimlet Sled, diff --git a/sled-hardware/src/non_illumos/mod.rs b/sled-hardware/src/non_illumos/mod.rs index 1d962d293de..ab5331494f1 100644 --- a/sled-hardware/src/non_illumos/mod.rs +++ b/sled-hardware/src/non_illumos/mod.rs @@ -86,3 +86,11 @@ pub async fn ensure_partition_layout( pub fn is_oxide_sled() -> anyhow::Result { Ok(false) } + +/// Probe for switch hardware. Nothing is attached off illumos. +pub fn detect_switch_hardware( + _log: &Logger, + _probe: crate::SwitchProbe, +) -> Result, crate::SwitchDetectError> { + Ok(None) +} diff --git a/sled-hardware/src/softnpu.rs b/sled-hardware/src/softnpu.rs new file mode 100644 index 00000000000..535c8fedee0 --- /dev/null +++ b/sled-hardware/src/softnpu.rs @@ -0,0 +1,137 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! 9P wire format used to identify the propolis SoftNPU device. +//! +//! Propolis exposes SoftNPU to a guest as a virtio 9p device whose server +//! answers Tversion with the version string `9P2000.P4`. The PCI ids match +//! any other virtio 9p device, so the version exchange is the discriminator. + +/// Version string served by the propolis SoftNPU 9p handler. +pub const SOFTNPU_9P_VERSION: &str = "9P2000.P4"; + +/// Maximum message size offered in Tversion. The server must answer with a +/// value no larger than this. +const MSIZE: u32 = 8192; +const TVERSION: u8 = 100; +const RVERSION: u8 = 101; +const NOTAG: u16 = 0xffff; + +// Field offsets in a version message, laid out as +// size[4] type[1] tag[2] msize[4] version[s], little endian. The string +// field is a 2-byte length followed by the bytes. +const TYPE_OFFSET: usize = 4; +const TAG_OFFSET: usize = 5; +const MSIZE_OFFSET: usize = 7; +const VERSION_LEN_OFFSET: usize = 11; +const VERSION_OFFSET: usize = 13; + +/// Total size of a version message carrying `version`. +fn msg_size(version: &[u8]) -> usize { + VERSION_OFFSET + version.len() +} + +/// Encode a Tversion message. +pub fn encode_tversion(version: &str) -> Vec { + let version = version.as_bytes(); + let size: u32 = + msg_size(version).try_into().expect("version message fits in u32"); + let len: u16 = + version.len().try_into().expect("version string fits in u16"); + let mut msg = Vec::with_capacity(msg_size(version)); + msg.extend_from_slice(&size.to_le_bytes()); + msg.push(TVERSION); + msg.extend_from_slice(&NOTAG.to_le_bytes()); + msg.extend_from_slice(&MSIZE.to_le_bytes()); + msg.extend_from_slice(&len.to_le_bytes()); + msg.extend_from_slice(version); + msg +} + +/// Decode the version string from an Rversion message. The reply must carry +/// NOTAG and an msize no larger than the one offered in Tversion. +pub fn decode_rversion(msg: &[u8]) -> Result { + if msg.len() < VERSION_OFFSET { + return Err(format!("short reply ({} bytes)", msg.len())); + } + if msg[TYPE_OFFSET] != RVERSION { + return Err(format!("unexpected message type {}", msg[TYPE_OFFSET])); + } + let tag = u16::from_le_bytes([msg[TAG_OFFSET], msg[TAG_OFFSET + 1]]); + if tag != NOTAG { + return Err(format!("unexpected tag {tag:#x}")); + } + let msize = u32::from_le_bytes( + msg[MSIZE_OFFSET..MSIZE_OFFSET + 4].try_into().unwrap(), + ); + if msize == 0 || msize > MSIZE { + return Err(format!("invalid msize {msize}")); + } + let len = usize::from(u16::from_le_bytes([ + msg[VERSION_LEN_OFFSET], + msg[VERSION_LEN_OFFSET + 1], + ])); + let version = msg + .get(VERSION_OFFSET..VERSION_OFFSET + len) + .ok_or_else(|| "truncated version string".to_string())?; + Ok(String::from_utf8_lossy(version).into_owned()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rversion(version: &[u8], tag: u16, msize: u32) -> Vec { + let mut msg = encode_tversion(std::str::from_utf8(version).unwrap()); + msg[TYPE_OFFSET] = RVERSION; + msg[TAG_OFFSET..TAG_OFFSET + 2].copy_from_slice(&tag.to_le_bytes()); + msg[MSIZE_OFFSET..MSIZE_OFFSET + 4] + .copy_from_slice(&msize.to_le_bytes()); + msg + } + + #[test] + fn tversion_layout() { + let msg = encode_tversion(SOFTNPU_9P_VERSION); + assert_eq!(msg.len(), msg_size(SOFTNPU_9P_VERSION.as_bytes())); + assert_eq!( + u32::from_le_bytes(msg[0..4].try_into().unwrap()), + msg.len() as u32 + ); + assert_eq!(msg[TYPE_OFFSET], TVERSION); + assert_eq!(&msg[VERSION_OFFSET..], SOFTNPU_9P_VERSION.as_bytes()); + } + + #[test] + fn rversion_roundtrip() { + let msg = rversion(SOFTNPU_9P_VERSION.as_bytes(), NOTAG, MSIZE); + assert_eq!(decode_rversion(&msg).unwrap(), SOFTNPU_9P_VERSION); + let msg = rversion(b"9P2000.L", NOTAG, MSIZE / 2); + assert_eq!(decode_rversion(&msg).unwrap(), "9P2000.L"); + } + + #[test] + fn rversion_malformed() { + assert!(decode_rversion(&[]).is_err()); + assert!(decode_rversion(&[0; VERSION_OFFSET - 1]).is_err()); + + let mut msg = rversion(SOFTNPU_9P_VERSION.as_bytes(), NOTAG, MSIZE); + msg[TYPE_OFFSET] = TVERSION; + assert!(decode_rversion(&msg).is_err()); + + let mut msg = rversion(SOFTNPU_9P_VERSION.as_bytes(), NOTAG, MSIZE); + msg.truncate(msg.len() - 1); + assert!(decode_rversion(&msg).is_err()); + } + + #[test] + fn rversion_bad_tag_or_msize() { + let msg = rversion(SOFTNPU_9P_VERSION.as_bytes(), 1, MSIZE); + assert!(decode_rversion(&msg).is_err()); + let msg = rversion(SOFTNPU_9P_VERSION.as_bytes(), NOTAG, MSIZE + 1); + assert!(decode_rversion(&msg).is_err()); + let msg = rversion(SOFTNPU_9P_VERSION.as_bytes(), NOTAG, 0); + assert!(decode_rversion(&msg).is_err()); + } +} diff --git a/smf/sled-agent/gimlet-standalone/config.toml b/smf/sled-agent/gimlet-standalone/config.toml index 547ea950c11..ed4d1337c28 100644 --- a/smf/sled-agent/gimlet-standalone/config.toml +++ b/smf/sled-agent/gimlet-standalone/config.toml @@ -1,18 +1,8 @@ # Sled Agent Configuration -# Identifies whether sled agent treats itself as a scrimlet or a gimlet. -# -# If this is set to "scrimlet", the sled agent treats itself as a scrimlet. -# If this is set to "gimlet", the sled agent treats itself as a gimlet. -# If this is set to "auto": -# - On illumos, the sled automatically detects whether or not it is a scrimlet. -# - On all other platforms, the sled assumes it is a gimlet. -sled_mode = "scrimlet" - -# Identifies the revision of the sidecar that is attached, if one is attached. -# TODO: This field should be removed once Gimlets have the ability to auto-detect -# this information. -sidecar_revision.physical = "b" +# A single gimlet outside a rack: the switch zone runs against the stub +# Dendrite. With a sidecar attached, use kind = "production" instead. +deployment = { kind = "custom", sled_mode = "scrimlet", switch = "tofino_stub" } # Setting this to true causes sled-agent to always report that its time is # in-sync, rather than querying its NTP zone. diff --git a/smf/sled-agent/gimlet/config.toml b/smf/sled-agent/gimlet/config.toml index 92a9cda8392..b918136ca12 100644 --- a/smf/sled-agent/gimlet/config.toml +++ b/smf/sled-agent/gimlet/config.toml @@ -1,18 +1,9 @@ # Sled Agent Configuration -# Identifies whether sled agent treats itself as a scrimlet or a gimlet. -# -# If this is set to "scrimlet", the sled agent treats itself as a scrimlet. -# If this is set to "gimlet", the sled agent treats itself as a gimlet. -# If this is set to "auto": -# - On illumos, the sled automatically detects whether or not it is a scrimlet. -# - On all other platforms, the sled assumes it is a gimlet. -sled_mode = "auto" - -# Identifies the revision of the sidecar that is attached, if one is attached. -# TODO: This field should be removed once Gimlets have the ability to auto-detect -# this information. -sidecar_revision.physical = "b" +# Oxide rack. The hardware monitor detects the Tofino ASIC and the sled runs +# as a scrimlet when one is present. The sidecar board revision defaults to +# "b". +deployment = { kind = "production" } # An optional data link from which we extract a MAC address. # This is used as a unique identifier for the bootstrap address. diff --git a/smf/sled-agent/non-gimlet/config.toml b/smf/sled-agent/non-gimlet/config.toml index 4f46cd56cb7..f6be15e4e39 100644 --- a/smf/sled-agent/non-gimlet/config.toml +++ b/smf/sled-agent/non-gimlet/config.toml @@ -1,18 +1,11 @@ # Sled Agent Configuration -# Identifies whether sled agent treats itself as a scrimlet or a gimlet. -# -# If this is set to "scrimlet", the sled agent treats itself as a scrimlet. -# If this is set to "gimlet", the sled agent treats itself as a gimlet. -# If this is set to "auto": -# - On illumos, the sled automatically detects whether or not it is a scrimlet. -# - On all other platforms, the sled assumes it is a gimlet. -sled_mode = "scrimlet" - -# Identifies the revision of the sidecar that is attached, if one is attached. -# TODO: This field should be removed once Gimlets have the ability to auto-detect -# this information. -sidecar_revision.soft_zone = { front_port_count = 1, rear_port_count = 1 } +# One host running a SoftNPU zone; always a scrimlet. Other kinds: +# "production" (Tofino, detected), "virtual" (propolis SoftNPU device, +# detected, with port counts), and "custom" with an explicit sled_mode and +# switch: "tofino_asic" or "tofino_stub" (optional sidecar_revision), or +# "soft_npu_propolis_device" or "soft_npu_zone" (with port counts). +deployment = { kind = "standalone", front_port_count = 1, rear_port_count = 1 } # Setting this to true causes sled-agent to always report that its time is # in-sync, rather than querying its NTP zone.