Support multiple external IPs in blueprint zone type

- Add support for multiple EIPs to the blueprint zone types for Nexus,
  External DNS, and Boundary NTP zones.
- Add some newtypes and wrappers to support lists of these or up to 2 of
  them for the SNAT case of Boundary NTP.
- Add a test that the full blueprint with multiple addresses round-trips
  through the database.
- Planner still emits exactly one address in all these cases, this is
  only the structural change to support multiple addresses.
- Update lockstep OpenAPI docs
- Closes #9288

Author: bnaecker

clients/nexus-lockstep-client/src/lib.rs

@@ -66,6 +66,10 @@ progenitor::generate_api!(
         NetworkInterface = sled_agent_types::inventory::NetworkInterface,
         NetworkInterfaceKind = sled_agent_types::inventory::NetworkInterfaceKind,
         NewPasswordHash = omicron_passwords::NewPasswordHash,
+        OmicronZoneExternalFloatingAddr =
+            nexus_types::deployment::OmicronZoneExternalFloatingAddr,
+        OmicronZoneExternalFloatingIp =
+            nexus_types::deployment::OmicronZoneExternalFloatingIp,
         OximeterReadMode = nexus_types::deployment::OximeterReadMode,
         OximeterReadPolicy = nexus_types::deployment::OximeterReadPolicy,
         PendingMgsUpdate = nexus_types::deployment::PendingMgsUpdate,
@@ -151,15 +155,6 @@ impl From<omicron_common::address::Ipv6Range> for types::Ipv6Range {
     }
 }
 
-impl From<&sled_agent_types::inventory::SourceNatConfigGeneric>
-    for types::SourceNatConfigGeneric
-{
-    fn from(r: &sled_agent_types::inventory::SourceNatConfigGeneric) -> Self {
-        let (first_port, last_port) = r.port_range_raw();
-        Self { ip: r.ip, first_port, last_port }
-    }
-}
-
 impl From<&omicron_common::api::external::AllowedSourceIps>
     for types::AllowedSourceIps
 {

dev-tools/reconfigurator-cli/src/lib.rs

@@ -157,12 +157,14 @@ impl ReconfiguratorSim {
 
         // Handle zone networking setup first
         for (_, zone) in parent_blueprint.in_service_zones() {
-            if let Some((external_ip, nic)) =
+            if let Some((external_ips, nic)) =
                 zone.zone_type.external_networking()
             {
-                builder
-                    .add_omicron_zone_external_ip(zone.id, external_ip)
-                    .context("adding omicron zone external IP")?;
+                for external_ip in external_ips {
+                    builder
+                        .add_omicron_zone_external_ip(zone.id, external_ip)
+                        .context("adding omicron zone external IP")?;
+                }
                 let nic = OmicronZoneNic {
                     // TODO-cleanup use `TypedUuid` everywhere
                     id: VnicUuid::from_untyped_uuid(nic.id),

live-tests/tests/test_nexus_handoff.rs

@@ -565,9 +565,11 @@ async fn check_external_dns(
     // what's in-service in the blueprint.
     let expected_nexus_addrs = blueprint
         .in_service_nexus_zones()
-        .filter_map(|(_sled_id, _zone_cfg, nexus_config)| {
-            (nexus_config.nexus_generation == active_generation)
-                .then_some(nexus_config.external_ip.ip)
+        .filter(|(_sled_id, _zone_cfg, nexus_config)| {
+            nexus_config.nexus_generation == active_generation
+        })
+        .flat_map(|(_sled_id, _zone_cfg, nexus_config)| {
+            nexus_config.external_ips.iter().map(|e| e.ip)
         })
         .collect::<BTreeSet<_>>();
 

nexus/db-model/src/deployment.rs

@@ -60,8 +60,9 @@ use nexus_types::deployment::{
 use nexus_types::deployment::{BlueprintPhysicalDiskConfig, BlueprintSource};
 use nexus_types::deployment::{BlueprintZoneImageSource, blueprint_zone_type};
 use nexus_types::deployment::{
-    OmicronZoneExternalFloatingAddr, OmicronZoneExternalFloatingIp,
-    OmicronZoneExternalSnatIp,
+    OmicronZoneExternalFloatingAddr, OmicronZoneExternalFloatingAddrs,
+    OmicronZoneExternalFloatingIp, OmicronZoneExternalFloatingIps,
+    OmicronZoneExternalSnat, OmicronZoneExternalSnatIp,
 };
 use omicron_common::address::Ipv6Subnet;
 use omicron_common::address::SLED_PREFIX_LENGTH;
@@ -855,7 +856,7 @@ impl BpOmicronZone {
                     http_address,
                     // The external DNS address is stored in the
                     // `bp_omicron_zone_external_ip` table, not here.
-                    dns_address: _,
+                    dns_addresses: _,
                     nic,
                 },
             ) => {
@@ -901,7 +902,7 @@ impl BpOmicronZone {
                 lockstep_port,
                 // The external IP is stored in the
                 // `bp_omicron_zone_external_ip` table, not here.
-                external_ip: _,
+                external_ips: _,
                 nic,
                 external_tls,
                 external_dns_servers,
@@ -978,18 +979,7 @@ impl BpOmicronZone {
             nic_row.map(Into::into),
         )?;
 
-        // The external IP(s) for this zone live in the
-        // `bp_omicron_zone_external_ip` table. Until `BlueprintZoneType` can
-        // handle multiple IPs (#9288), we need zero or exactly one row here,
-        // for the zone types that have external networking.
-        //
-        // NOTE: This returns an error if `external_ip_rows` is empty. That's
-        // fine if the zone doesn't need external networking, so we only
-        // ?-propagate this inside the zone-specific code below.
-        let external_ip = BpOmicronZoneExternalIp::into_single(
-            external_ip_rows,
-            self.id.into(),
-        );
+        let zone_id = self.id.into();
 
         // NOTE: this is the *internal* DNS underlay address, held in
         // `second_service_ip` / `second_service_port`. External DNS's external
@@ -1011,19 +1001,19 @@ impl BpOmicronZone {
 
         let zone_type = match self.zone_type {
             ZoneType::BoundaryNtp => {
-                let external_ip = external_ip?;
-                let snat_cfg = external_ip.to_snat_config()?;
+                let external_ip =
+                    BpOmicronZoneExternalIp::into_boundary_ntp_snat(
+                        external_ip_rows,
+                        zone_id,
+                    )?;
                 BlueprintZoneType::BoundaryNtp(
                     blueprint_zone_type::BoundaryNtp {
                         address: primary_address,
                         ntp_servers: ntp_servers?,
                         dns_servers: ntp_dns_servers?,
                         domain: self.ntp_domain,
                         nic: nic?,
-                        external_ip: OmicronZoneExternalSnatIp {
-                            id: external_ip.external_ip_id.into(),
-                            snat_cfg,
-                        },
+                        external_ip,
                     },
                 )
             }
@@ -1064,15 +1054,16 @@ impl BpOmicronZone {
                 },
             ),
             ZoneType::ExternalDns => {
-                let external_ip = external_ip?;
+                let dns_addresses =
+                    BpOmicronZoneExternalIp::into_external_dns_addrs(
+                        external_ip_rows,
+                        zone_id,
+                    )?;
                 BlueprintZoneType::ExternalDns(
                     blueprint_zone_type::ExternalDns {
                         dataset: dataset?,
                         http_address: primary_address,
-                        dns_address: OmicronZoneExternalFloatingAddr {
-                            id: external_ip.external_ip_id.into(),
-                            addr: external_ip.to_floating_addr()?,
-                        },
+                        dns_addresses,
                         nic: nic?,
                     },
                 )
@@ -1099,16 +1090,17 @@ impl BpOmicronZone {
                 blueprint_zone_type::InternalNtp { address: primary_address },
             ),
             ZoneType::Nexus => {
-                let external_ip = external_ip?;
+                let external_ips =
+                    BpOmicronZoneExternalIp::into_nexus_external_ips(
+                        external_ip_rows,
+                        zone_id,
+                    )?;
                 BlueprintZoneType::Nexus(blueprint_zone_type::Nexus {
                     internal_address: primary_address,
                     lockstep_port: *self.nexus_lockstep_port.ok_or_else(
                         || anyhow!("expected 'nexus_lockstep_port'"),
                     )?,
-                    external_ip: OmicronZoneExternalFloatingIp {
-                        id: external_ip.external_ip_id.into(),
-                        ip: external_ip.ip.ip(),
-                    },
+                    external_ips,
                     nic: nic?,
                     external_tls: self
                         .nexus_external_tls
@@ -1186,10 +1178,9 @@ pub struct BpOmicronZoneExternalIp {
 impl BpOmicronZoneExternalIp {
     /// Build the external IP child rows for a blueprint zone.
     ///
-    /// Returns one row per external IP. Today the in-memory `BlueprintZoneType`
-    /// only ever has at most one external IP per zone, so this returns at most
-    /// one row. In general though, the `bp_omicron_zone_external_ip` table can
-    /// store any number of rows per zone, so we're returning an array.
+    /// Returns one row per external IP: Nexus and external DNS may each have
+    /// several, and boundary NTP may have a source-NAT address per IP family
+    /// (with at least one address).
     pub fn for_zone(
         blueprint_id: BlueprintUuid,
         blueprint_zone: &BlueprintZoneConfig,
@@ -1198,43 +1189,52 @@ impl BpOmicronZoneExternalIp {
         let zone_id = blueprint_zone.id.into();
         match &blueprint_zone.zone_type {
             BlueprintZoneType::Nexus(blueprint_zone_type::Nexus {
-                external_ip,
+                external_ips,
                 ..
-            }) => vec![Self {
-                blueprint_id,
-                zone_id,
-                external_ip_id: external_ip.id.into(),
-                ip: IpNetwork::from(external_ip.ip),
-                port: None,
-                snat_first_port: None,
-                snat_last_port: None,
-            }],
-            BlueprintZoneType::ExternalDns(
-                blueprint_zone_type::ExternalDns { dns_address, .. },
-            ) => vec![Self {
-                blueprint_id,
-                zone_id,
-                external_ip_id: dns_address.id.into(),
-                ip: IpNetwork::from(dns_address.addr.ip()),
-                port: Some(SqlU16::from(dns_address.addr.port())),
-                snat_first_port: None,
-                snat_last_port: None,
-            }],
-            BlueprintZoneType::BoundaryNtp(
-                blueprint_zone_type::BoundaryNtp { external_ip, .. },
-            ) => {
-                let (first_port, last_port) =
-                    external_ip.snat_cfg.port_range_raw();
-                vec![Self {
+            }) => external_ips
+                .iter()
+                .map(|external_ip| Self {
                     blueprint_id,
                     zone_id,
                     external_ip_id: external_ip.id.into(),
-                    ip: IpNetwork::from(external_ip.snat_cfg.ip),
+                    ip: IpNetwork::from(external_ip.ip),
                     port: None,
-                    snat_first_port: Some(SqlU16::from(first_port)),
-                    snat_last_port: Some(SqlU16::from(last_port)),
-                }]
-            }
+                    snat_first_port: None,
+                    snat_last_port: None,
+                })
+                .collect(),
+            BlueprintZoneType::ExternalDns(
+                blueprint_zone_type::ExternalDns { dns_addresses, .. },
+            ) => dns_addresses
+                .iter()
+                .map(|dns_address| Self {
+                    blueprint_id,
+                    zone_id,
+                    external_ip_id: dns_address.id.into(),
+                    ip: IpNetwork::from(dns_address.addr.ip()),
+                    port: Some(SqlU16::from(dns_address.addr.port())),
+                    snat_first_port: None,
+                    snat_last_port: None,
+                })
+                .collect(),
+            BlueprintZoneType::BoundaryNtp(
+                blueprint_zone_type::BoundaryNtp { external_ip, .. },
+            ) => external_ip
+                .iter()
+                .map(|snat| {
+                    let (first_port, last_port) =
+                        snat.snat_cfg.port_range_raw();
+                    Self {
+                        blueprint_id,
+                        zone_id,
+                        external_ip_id: snat.id.into(),
+                        ip: IpNetwork::from(snat.snat_cfg.ip),
+                        port: None,
+                        snat_first_port: Some(SqlU16::from(first_port)),
+                        snat_last_port: Some(SqlU16::from(last_port)),
+                    }
+                })
+                .collect(),
             BlueprintZoneType::Clickhouse(_)
             | BlueprintZoneType::ClickhouseKeeper(_)
             | BlueprintZoneType::ClickhouseServer(_)
@@ -1247,28 +1247,59 @@ impl BpOmicronZoneExternalIp {
         }
     }
 
-    /// Collapse the external IP rows for a zone into exactly one.
-    ///
-    /// NOTE: This is a temporary method until the `BlueprintZoneType` variants
-    /// with external addresses can handle more than one EIP. Until then, these
-    /// zones must have exactly one external IP. This returns that single
-    /// address, or an error if there is any other number of rows.
-    fn into_single(
-        mut rows: Vec<Self>,
+    /// Reconstruct a Nexus zone's external IPs from its child rows.
+    fn into_nexus_external_ips(
+        rows: Vec<Self>,
         zone_id: OmicronZoneUuid,
-    ) -> anyhow::Result<Self> {
-        match rows.len() {
-            1 => Ok(rows.pop().expect("length checked to be 1")),
-            0 => bail!(
-                "zone {zone_id} has no external IP, \
-                 but its type requires one"
-            ),
-            n => bail!(
-                "zone {zone_id} has {n} external IPs, but only one is \
-                 supported until the in-memory blueprint type is widened \
-                 (#9288)"
-            ),
-        }
+    ) -> anyhow::Result<OmicronZoneExternalFloatingIps> {
+        let ips = rows
+            .into_iter()
+            .map(|row| OmicronZoneExternalFloatingIp {
+                id: row.external_ip_id.into(),
+                ip: row.ip.ip(),
+            })
+            .collect();
+        OmicronZoneExternalFloatingIps::new(ips).with_context(|| {
+            format!("zone {zone_id} has invalid Nexus external IPs")
+        })
+    }
+
+    /// Reconstruct an external DNS zone's addresses from its child rows.
+    fn into_external_dns_addrs(
+        rows: Vec<Self>,
+        zone_id: OmicronZoneUuid,
+    ) -> anyhow::Result<OmicronZoneExternalFloatingAddrs> {
+        let addrs = rows
+            .into_iter()
+            .map(|row| {
+                Ok(OmicronZoneExternalFloatingAddr {
+                    id: row.external_ip_id.into(),
+                    addr: row.to_floating_addr()?,
+                })
+            })
+            .collect::<anyhow::Result<Vec<_>>>()?;
+        OmicronZoneExternalFloatingAddrs::new(addrs).with_context(|| {
+            format!("zone {zone_id} has invalid external DNS addresses")
+        })
+    }
+
+    /// Reconstruct a boundary NTP zone's SNAT configuration from its child rows.
+    fn into_boundary_ntp_snat(
+        rows: Vec<Self>,
+        zone_id: OmicronZoneUuid,
+    ) -> anyhow::Result<OmicronZoneExternalSnat> {
+        let snat_ips = rows
+            .into_iter()
+            .map(|row| {
+                Ok(OmicronZoneExternalSnatIp {
+                    id: row.external_ip_id.into(),
+                    snat_cfg: row.to_snat_config()?,
+                })
+            })
+            .collect::<anyhow::Result<Vec<_>>>()?;
+        OmicronZoneExternalSnat::from_ips(snat_ips).with_context(|| {
+            format!("zone {zone_id} has invalid boundary NTP SNAT config")
+        })
     }
 
     /// Interpret this row as a boundary NTP source-NAT configuration.