move some types and break a circular dep

Author: andrewjstone

Cargo.lock

@@ -41,6 +41,8 @@ progenitor::generate_api!(
         RouterLifetimeConfig = sled_agent_types::early_networking::RouterLifetimeConfig,
         RssStep = bootstrap_agent_lockstep_types::RssStep,
         ServiceIpPoolConfig = bootstrap_agent_lockstep_types::ServiceIpPoolConfig,
+        SledAgentInfo = bootstrap_agent_lockstep_types::SledAgentInfo,
+        StartSledAgentStatus = bootstrap_agent_lockstep_types::StartSledAgentStatus,
         SwitchSlot = sled_agent_types::early_networking::SwitchSlot,
         TxEqConfig = sled_agent_types::early_networking::TxEqConfig,
         UplinkAddressConfig = sled_agent_types::early_networking::UplinkAddressConfig,

clients/bootstrap-agent-lockstep-client/src/lib.rs

@@ -12,4 +12,3 @@ dropshot.workspace = true
 bootstrap-agent-lockstep-types.workspace = true
 omicron-uuid-kinds.workspace = true
 omicron-workspace-hack.workspace = true
-sled-agent-multirack-join.workspace = true

sled-agent/bootstrap-agent-lockstep-api/Cargo.toml

@@ -11,13 +11,13 @@
 
 use bootstrap_agent_lockstep_types::BaseboardIds;
 use bootstrap_agent_lockstep_types::MultirackJoinRequest;
+use bootstrap_agent_lockstep_types::MultirackJoinServiceState;
 use bootstrap_agent_lockstep_types::RackInitializeRequest;
 use bootstrap_agent_lockstep_types::RackOperationStatus;
 use bootstrap_agent_lockstep_types::ReplicatedNetworkConfig;
 use dropshot::{HttpError, HttpResponseOk, RequestContext, TypedBody};
 use omicron_uuid_kinds::MultirackJoinUuid;
 use omicron_uuid_kinds::RackInitUuid;
-use sled_agent_multirack_join::MultirackJoinServiceState;
 
 #[dropshot::api_description]
 pub trait BootstrapAgentLockstepApi {

sled-agent/bootstrap-agent-lockstep-api/src/lib.rs

@@ -13,6 +13,7 @@ use anyhow::Context as _;
 use iddqd::IdOrdItem;
 use iddqd::IdOrdMap;
 use iddqd::id_upcast;
+use iddqd::{BiHashItem, BiHashMap, bi_upcast};
 use omicron_common::address::AZ_PREFIX_LENGTH;
 use omicron_common::address::IpRange;
 use omicron_common::address::IpVersion;
@@ -27,16 +28,23 @@ use omicron_common::api::external::UserId;
 use omicron_common::api::internal::nexus::Certificate;
 use omicron_uuid_kinds::MultirackJoinUuid;
 use omicron_uuid_kinds::RackInitUuid;
+use omicron_uuid_kinds::RackUuid;
+use omicron_uuid_kinds::SledUuid;
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use sled_agent_types::early_networking::RackNetworkConfig;
 use sled_hardware_types::BaseboardId;
+use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::net::IpAddr;
 use std::net::Ipv6Addr;
 use strum::EnumCount;
 use strum::EnumIter;
 use strum::IntoEnumIterator;
+use trust_quorum_types::messages::ReconfigureMsg as TqReconfigureMsg;
+use trust_quorum_types::status::CoordinatorStatus;
+use trust_quorum_types::types::Epoch;
+use trust_quorum_types::types::Threshold;
 
 /// Configuration for the "rack setup service".
 ///
@@ -532,3 +540,89 @@ impl IdOrdItem for BootstrapIpOfBaseboardId {
 pub struct BaseboardIds {
     pub data: IdOrdMap<BootstrapIpOfBaseboardId>,
 }
+
+/// The state of the commit phase of the trust quorum protocol
+#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
+pub struct CommitState {
+    pub rack_id: RackUuid,
+    pub members: BTreeSet<BaseboardId>,
+    pub epoch: Epoch,
+    pub last_committed_epoch: Option<Epoch>,
+    pub threshold: Threshold,
+    pub commit_crash_tolerance: u8,
+    pub acked: BTreeSet<BaseboardId>,
+    pub fatal_errors: BTreeMap<BaseboardId, String>,
+    pub transient_errors: BTreeMap<BaseboardId, String>,
+}
+
+/// Status information for a given sled agent
+#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
+pub struct SledAgentInfo {
+    pub baseboard_id: BaseboardId,
+    pub sled_id: SledUuid,
+    pub sled_subnet: Ipv6Subnet<SLED_PREFIX_LENGTH>,
+    pub started: bool,
+    pub fatal_error: Option<String>,
+}
+
+impl BiHashItem for SledAgentInfo {
+    type K1<'a> = &'a BaseboardId;
+    type K2<'a> = &'a SledUuid;
+
+    fn key1(&self) -> Self::K1<'_> {
+        &self.baseboard_id
+    }
+
+    fn key2(&self) -> Self::K2<'_> {
+        &self.sled_id
+    }
+
+    bi_upcast!();
+}
+
+#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
+pub struct StartSledAgentsStatus {
+    pub sleds: BiHashMap<SledAgentInfo>,
+}
+
+impl StartSledAgentsStatus {
+    pub fn new(req: MultirackJoinRequest) -> Self {
+        let rack_subnet = Ipv6Subnet::<RACK_PREFIX_LENGTH>::new(
+            req.rack_network_config.rack_subnet.addr(),
+        );
+        let sleds = req
+            .trust_quorum_peers
+            .into_iter()
+            .enumerate()
+            .map(|(idx, baseboard_id)| SledAgentInfo {
+                baseboard_id,
+                sled_id: SledUuid::new_v4(),
+                sled_subnet: get_64_subnet(
+                    rack_subnet,
+                    u8::try_from(idx + 1).expect("too many sleds"),
+                ),
+                started: false,
+                fatal_error: None,
+            })
+            .collect();
+
+        StartSledAgentsStatus { sleds }
+    }
+}
+
+/// The current state of the `MultirackJoinService` as retrieved from the
+/// `output_rx` watch channel.
+#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
+#[serde(rename_all = "snake_case", tag = "state")]
+pub enum MultirackJoinServiceState {
+    Uninitialized,
+    Requested,
+    Starting,
+    TrustQuorumReconfigure(TqReconfigureMsg),
+    TrustQuorumPreparing(CoordinatorStatus),
+    TrustQuorumCommitting(CommitState),
+    StartSledAgents(StartSledAgentsStatus),
+    Completed,
+    Failed { message: String },
+    TaskPanicked,
+}

sled-agent/bootstrap-agent-lockstep-types/src/lib.rs

@@ -15,23 +15,20 @@
 
 #[macro_use]
 extern crate slog;
-use bootstrap_agent_lockstep_types::MultirackJoinRequest;
-use iddqd::{BiHashItem, BiHashMap, bi_upcast};
+use bootstrap_agent_lockstep_types::{
+    CommitState, MultirackJoinRequest, MultirackJoinServiceState,
+    SledAgentInfo, StartSledAgentsStatus,
+};
 use nexus_types::trust_quorum::TrustQuorumConfig;
 use omicron_common::address::BOOTSTRAP_AGENT_RACK_INIT_PORT;
-use omicron_common::address::Ipv6Subnet;
-use omicron_common::address::RACK_PREFIX_LENGTH;
-use omicron_common::address::SLED_PREFIX_LENGTH;
-use omicron_common::address::get_64_subnet;
-use omicron_uuid_kinds::{RackUuid, SledUuid};
-use schemars::JsonSchema;
-use serde::{Deserialize, Serialize};
+use omicron_uuid_kinds::RackUuid;
 use sled_agent_bootstrap_common::sprockets::{
     SprocketsClient, SprocketsClientError,
 };
 use sled_agent_bootstrap_common::{RssContext, RunRssError};
-use sled_agent_types::sled::StartSledAgentRequest;
-use sled_agent_types::sled::StartSledAgentRequestBody;
+use sled_agent_types::sled::{
+    StartSledAgentRequest, StartSledAgentRequestBody,
+};
 use sled_hardware_types::BaseboardId;
 use slog::{Logger, error, info};
 use slog_error_chain::{InlineErrorChain, SlogInlineError};
@@ -48,9 +45,7 @@ use tokio::{
 };
 use trust_quorum::{NodeApiError, ProxyError};
 use trust_quorum_types::{
-    messages::ReconfigureMsg as TqReconfigureMsg,
-    status::CoordinatorStatus,
-    types::{Epoch, Threshold},
+    messages::ReconfigureMsg as TqReconfigureMsg, types::Epoch,
 };
 
 const INITIAL_TRUST_QUORUM_EPOCH: Epoch = Epoch(1);
@@ -106,18 +101,23 @@ impl From<RunRssError> for MultirackJoinServiceError {
     }
 }
 
-/// The state of the commit phase of the trust quorum protocol
-#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
-pub struct CommitState {
-    rack_id: RackUuid,
-    members: BTreeSet<BaseboardId>,
-    epoch: Epoch,
-    last_committed_epoch: Option<Epoch>,
-    threshold: Threshold,
-    commit_crash_tolerance: u8,
-    acked: BTreeSet<BaseboardId>,
-    fatal_errors: BTreeMap<BaseboardId, String>,
-    transient_errors: BTreeMap<BaseboardId, String>,
+// The value returned from `MultirackJoinServiceTask::tq_prepare`
+enum TqPrepareResult {
+    Prepared,
+    ReconfigurationNeeded {
+        new_members: BTreeSet<BaseboardId>,
+        new_epoch: Epoch,
+    },
+}
+
+// The value returned from `MultirackJoinServiceTask::tq_commit`
+enum TqCommitResult {
+    Committed,
+    ReconfigurationNeeded {
+        new_members: BTreeSet<BaseboardId>,
+        new_epoch: Epoch,
+        just_committed_epoch: Epoch,
+    },
 }
 
 /// All the information required to start a sled agent remotely over a sprockets
@@ -128,31 +128,21 @@ struct StartSledAgentInfo {
     req: StartSledAgentRequest,
 }
 
-/// Status information for a given sled agent
-#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
-pub struct SledAgentInfo {
-    pub baseboard_id: BaseboardId,
-    pub sled_id: SledUuid,
-    pub sled_subnet: Ipv6Subnet<SLED_PREFIX_LENGTH>,
-    pub started: bool,
-    pub fatal_error: Option<String>,
-}
-
-impl SledAgentInfo {
-    fn to_start_sled_agent_info(
-        &self,
+impl StartSledAgentInfo {
+    fn new(
         rack_id: RackUuid,
         bootstrap_ip: Ipv6Addr,
-    ) -> StartSledAgentInfo {
+        info: SledAgentInfo,
+    ) -> Self {
         StartSledAgentInfo {
-            baseboard_id: self.baseboard_id.clone(),
+            baseboard_id: info.baseboard_id.clone(),
             bootstrap_ip,
             req: StartSledAgentRequest {
                 generation: 0,
                 schema_version: 1,
                 body: StartSledAgentRequestBody {
-                    id: self.sled_id,
-                    subnet: self.sled_subnet,
+                    id: info.sled_id,
+                    subnet: info.sled_subnet,
                     use_trust_quorum: true,
                     is_lrtq_learner: false,
                     rack_id,
@@ -162,87 +152,6 @@ impl SledAgentInfo {
     }
 }
 
-impl BiHashItem for SledAgentInfo {
-    type K1<'a> = &'a BaseboardId;
-    type K2<'a> = &'a SledUuid;
-
-    fn key1(&self) -> Self::K1<'_> {
-        &self.baseboard_id
-    }
-
-    fn key2(&self) -> Self::K2<'_> {
-        &self.sled_id
-    }
-
-    bi_upcast!();
-}
-
-#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
-pub struct StartSledAgentsStatus {
-    pub sleds: BiHashMap<SledAgentInfo>,
-}
-
-impl StartSledAgentsStatus {
-    pub fn new(req: MultirackJoinRequest) -> Self {
-        let rack_subnet = Ipv6Subnet::<RACK_PREFIX_LENGTH>::new(
-            req.rack_network_config.rack_subnet.addr(),
-        );
-        let sleds = req
-            .trust_quorum_peers
-            .into_iter()
-            .enumerate()
-            .map(|(idx, baseboard_id)| SledAgentInfo {
-                baseboard_id,
-                sled_id: SledUuid::new_v4(),
-                sled_subnet: get_64_subnet(
-                    rack_subnet,
-                    u8::try_from(idx + 1).expect("too many sleds"),
-                ),
-                started: false,
-                fatal_error: None,
-            })
-            .collect();
-
-        StartSledAgentsStatus { sleds }
-    }
-}
-
-/// The current state of the `MultirackJoinService` as retrieved from the
-/// `output_rx` watch channel.
-#[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)]
-#[serde(rename_all = "snake_case", tag = "state")]
-pub enum MultirackJoinServiceState {
-    Uninitialized,
-    Requested,
-    Starting,
-    TrustQuorumReconfigure(TqReconfigureMsg),
-    TrustQuorumPreparing(CoordinatorStatus),
-    TrustQuorumCommitting(CommitState),
-    StartSledAgents(StartSledAgentsStatus),
-    Completed,
-    Failed { message: String },
-    TaskPanicked,
-}
-
-// The value returned from `MultirackJoinServiceTask::tq_prepare`
-enum TqPrepareResult {
-    Prepared,
-    ReconfigurationNeeded {
-        new_members: BTreeSet<BaseboardId>,
-        new_epoch: Epoch,
-    },
-}
-
-// The value returned from `MultirackJoinServiceTask::tq_commit`
-enum TqCommitResult {
-    Committed,
-    ReconfigurationNeeded {
-        new_members: BTreeSet<BaseboardId>,
-        new_epoch: Epoch,
-        just_committed_epoch: Epoch,
-    },
-}
-
 /// The interface to the Multirack Join Service.
 pub struct MultirackJoinServiceHandle {
     pub join_handle:
@@ -347,13 +256,13 @@ impl MultirackJoinServiceTask {
 
         // Attempt to start all our sled agents in parallel
         let mut set = JoinSet::new();
-        for sled in status.sleds.iter().cloned() {
+        for info in status.sleds.iter().cloned() {
             // Unwrap is safe, because we constructed both bootstrap_ips and
             // status from trust_quorum_peers.
-            let bootstrap_ip = *bootstrap_ips.get(&sled.baseboard_id).unwrap();
+            let bootstrap_ip = *bootstrap_ips.get(&info.baseboard_id).unwrap();
             self.spawn_start_sled_agent_task(
                 &mut set,
-                sled.to_start_sled_agent_info(rack_id, bootstrap_ip),
+                StartSledAgentInfo::new(rack_id, bootstrap_ip, info),
             );
         }
 

sled-agent/multirack-join/src/lib.rs

@@ -19,6 +19,7 @@ use bootstrap_agent_lockstep_api::bootstrap_agent_lockstep_api_mod;
 use bootstrap_agent_lockstep_types::BaseboardIds;
 use bootstrap_agent_lockstep_types::BootstrapIpOfBaseboardId;
 use bootstrap_agent_lockstep_types::MultirackJoinRequest;
+use bootstrap_agent_lockstep_types::MultirackJoinServiceState;
 use bootstrap_agent_lockstep_types::RackInitializeRequest;
 use bootstrap_agent_lockstep_types::RackOperationStatus;
 use bootstrap_agent_lockstep_types::ReplicatedNetworkConfig;
@@ -32,7 +33,6 @@ use omicron_uuid_kinds::RackInitUuid;
 use sled_agent_bootstrap_common::RssContext;
 use sled_agent_config_reconciler::InternalDisksReceiver;
 use sled_agent_measurements::MeasurementsHandle;
-use sled_agent_multirack_join::MultirackJoinServiceState;
 use sled_agent_rack_setup::RackInitializeRequestParams;
 use slog::Logger;
 use sprockets_tls::keys::SprocketsConfig;