Support IPv6 addresses in the OPTE zone setup service (#11092)

- Add the new `--create-v6` parameter to the `zone-setup` binary for
setting up OPTE ports in a service zone. Also adds stronger types around
IPv4 addresses for the gateway / private IP arguments. Includes a number
of tests for those types, and all combinations of arguments we could
provide.
- Ensure we create both IPv4 and IPv6 DHCP address objects in the
current zone, depending on how the SMF properties are set. These
properties are now set in the sled-agent for both IPv4 and IPv6.
- Move the code from the `RunningZone` which does the `route(8)`
shenanigans to work around stlouis#326 (setting up a default route to
the OPTE virtual gateway), and which also waits for the DHCPv6 address
on the link. This was all duplicated, and moving it here means we can
delete the workaround in one place in the future.

Author: bnaecker

illumos-utils/src/ipadm.rs

@@ -28,7 +28,9 @@ const INTERFACE_ALREADY_EXISTS: &str = "Interface already exists";
 const ADDROBJ_ALREADY_EXISTS: &str = "Address object already exists";
 
 pub enum AddrObjType {
-    DHCP,
+    // NOTE: This can result in more than one address, if there is a DHCPv6
+    // server on the same link as the addrobj. That happens most often for OPTE
+    // ports used in zones that need external connectivity.
     AddrConf,
     Static(IpAddr),
 }
@@ -62,7 +64,6 @@ impl Ipadm {
         let mut cmd = Command::new(PFEXEC);
         let cmd = cmd.args(&[IPADM, "create-addr", "-t", "-T"]);
         let cmd = match addrtype {
-            AddrObjType::DHCP => cmd.args(&["dhcp"]),
             AddrObjType::AddrConf => cmd.args(&["addrconf"]),
             AddrObjType::Static(addr) => {
                 cmd.args(&["static", "-a", &addr.to_string()])
@@ -83,6 +84,9 @@ impl Ipadm {
     /// Remove any scope from an IPv6 address.
     /// e.g. fe80::8:20ff:fed0:8687%oxControlService1/10 ->
     ///      fe80::8:20ff:fed0:8687/10
+    //
+    // TODO-cleanup: This could directly parse the line into an IpNet if
+    // possible, rather than emitting a new string.
     fn remove_addr_scope(input: &str) -> String {
         if let Some(pos) = input.find('%') {
             let (base, rest) = input.split_at(pos);
@@ -98,6 +102,10 @@ impl Ipadm {
 
     /// Return the IP network associated with an address object, or None if
     /// there is no address object with this name.
+    //
+    // TODO-correctness: There can be many addresses associated with an addrobj,
+    // e.g., for IPv6 where we have link-local + DHCPv6 addresses. This will
+    // ignore all but the first.
     pub async fn addrobj_addr(
         addrobj: &str,
     ) -> Result<Option<IpNet>, ExecutionError> {
@@ -173,6 +181,7 @@ impl Ipadm {
         Ok(())
     }
 
+    /// Create a link-local IPv6 addrconf address and a static IPv6 address.
     pub async fn create_static_and_autoconfigured_addrs(
         datalink: &str,
         listen_addr: &Ipv6Addr,
@@ -192,15 +201,6 @@ impl Ipadm {
         Ok(())
     }
 
-    // Create gateway on the IP interface if it doesn't already exist
-    pub async fn create_opte_gateway(
-        opte_iface: &String,
-    ) -> Result<(), ExecutionError> {
-        let addrobj = format!("{}/public", opte_iface);
-        Self::ensure_ip_addrobj_exists(&addrobj, AddrObjType::DHCP).await?;
-        Ok(())
-    }
-
     /// Set TCP recv_buf to 1 MB.
     pub async fn set_tcp_recv_buf() -> Result<(), ExecutionError> {
         let mut cmd = Command::new(PFEXEC);

illumos-utils/src/opte/mod.rs

@@ -38,11 +38,15 @@ pub use port_manager::MulticastGroupCfg;
 pub use port_manager::PortCreateParams;
 pub use port_manager::PortManager;
 pub use port_manager::PortTicket;
-use std::net::IpAddr;
 use std::net::Ipv4Addr;
 use std::net::Ipv6Addr;
 
 /// Information about the gateway for an OPTE port
+///
+/// TODO-remove: This only exists to communicate the destination for a default
+/// IPv4 route from the port's private IP to the OPTE "virtual gateway". We can
+/// remove this entirely when we resolve
+/// <https://github.com/oxidecomputer/omicron/issues/2931>.
 #[derive(Debug, Clone, Copy)]
 #[allow(dead_code)]
 pub struct Gateway {
@@ -100,18 +104,6 @@ impl Gateway {
             GatewayIps::V4(_) => None,
         }
     }
-
-    /// Return the IPv4 address, if it exists, or the IPv6 address.
-    ///
-    /// At least one of these always exists.
-    pub fn ipv4_or_ipv6_addr(&self) -> IpAddr {
-        match &self.ips {
-            GatewayIps::V4(v4) | GatewayIps::DualStack { v4, .. } => {
-                IpAddr::V4(*v4)
-            }
-            GatewayIps::V6(v6) => IpAddr::V6(*v6),
-        }
-    }
 }
 
 /// Convert a nexus [IpNet] to an OPTE [IpCidr].

illumos-utils/src/opte/port.rs

@@ -15,7 +15,6 @@ use omicron_common::api::internal::shared::RouterKind;
 use oxnet::Ipv4Net;
 use oxnet::Ipv6Net;
 use sled_agent_types::inventory::NetworkInterfaceKind;
-use std::net::IpAddr;
 use std::net::Ipv4Addr;
 use std::net::Ipv6Addr;
 use std::sync::Arc;
@@ -24,17 +23,17 @@ use uuid::Uuid;
 #[derive(Debug)]
 pub struct PortData {
     /// Name of the port as identified by OPTE
-    pub(crate) name: String,
+    name: String,
     /// The VPC-private IP configuration for the port.
-    pub(crate) ip: PrivateIpConfig,
+    ip: PrivateIpConfig,
     /// VPC-private MAC address
-    pub(crate) mac: MacAddr6,
+    mac: MacAddr6,
     /// Emulated PCI slot for the guest NIC, passed to Propolis
-    pub(crate) slot: u8,
+    slot: u8,
     /// Geneve VNI for the VPC
-    pub(crate) vni: Vni,
+    vni: Vni,
     /// Information about the virtual gateway, aka OPTE
-    pub(crate) gateway: Gateway,
+    gateway: Gateway,
 }
 
 #[derive(Debug)]
@@ -80,7 +79,15 @@ pub struct Port {
 }
 
 impl Port {
-    pub fn new(data: PortData) -> Self {
+    pub fn new(
+        name: String,
+        ip: PrivateIpConfig,
+        mac: MacAddr6,
+        slot: u8,
+        vni: Vni,
+    ) -> Self {
+        let gateway = Gateway::from_ip_config(&ip);
+        let data = PortData { name, ip, mac, slot, vni, gateway };
         Self { inner: Arc::new(PortInner(data)) }
     }
 
@@ -94,28 +101,23 @@ impl Port {
         self.inner.ip.ipv6_addr()
     }
 
-    /// Return the VPC-private IPv4 address, if it exits, or the IPv6 address.
-    ///
-    /// One of these always exists.
-    pub fn ipv4_or_ipv6_addr(&self) -> IpAddr {
-        self.inner.ip.ipv4_addr().copied().map(IpAddr::V4).unwrap_or_else(
-            || {
-                self.inner
-                    .ip
-                    .ipv6_addr()
-                    .copied()
-                    .expect("At least one address always exists")
-                    .into()
-            },
-        )
-    }
-
     pub fn name(&self) -> &str {
         &self.inner.name
     }
 
-    pub fn gateway(&self) -> &Gateway {
-        &self.inner.gateway
+    /// Return the OPTE gateway IPv4 address and the private IPv4 address.
+    ///
+    /// If the port is not configured for IPv4, None is returned.
+    // TODO-remove: <https://github.com/oxidecomputer/omicron/issues/2931>
+    pub fn gateway_and_private_ipv4(&self) -> Option<(&Ipv4Addr, &Ipv4Addr)> {
+        match (self.inner.gateway.ipv4_addr(), self.ipv4_addr()) {
+            (None, None) => None,
+            (None, Some(_)) => unreachable!(),
+            (Some(_), None) => unreachable!(),
+            (Some(gateway_ip), Some(private_ip)) => {
+                Some((gateway_ip, private_ip))
+            }
+        }
     }
 
     #[allow(dead_code)]

illumos-utils/src/opte/port_manager.rs

@@ -13,7 +13,6 @@ use crate::opte::Handle;
 use crate::opte::Port;
 use crate::opte::Vni;
 use crate::opte::opte_firewall_rules;
-use crate::opte::port::PortData;
 use ipnetwork::Ipv4Network;
 use ipnetwork::Ipv6Network;
 use macaddr::MacAddr6;
@@ -419,14 +418,13 @@ impl PortManager {
         };
         let (port, ticket) = {
             let ticket = PortTicket::new(nic.id, nic.kind, self.inner.clone());
-            let port = Port::new(PortData {
-                name: port_name.clone(),
-                ip: nic.ip_config.clone(),
+            let port = Port::new(
+                port_name.clone(),
+                nic.ip_config.clone(),
                 mac,
-                slot: nic.slot,
+                nic.slot,
                 vni,
-                gateway,
-            });
+            );
 
             // NOTE: We may add external IPs below, which can fail. If that
             // does, we drop the `ticket` on the way out of this block. That