Support IPv6 VPC-private addresses for network interfaces (#9269)

- Make more explicit how we assign IP addresses for NICs, by adding a
full configuration type. That includes details of whether an address is
auto assigned or explicitly requested, and the corresponding transit
IPs. Also adds the notion of single-stack IPv4 or IPv6 NICs, or
explicitly dual-stack.
- Restructure some of the internals of the network interface
`InsertQuery` to more clearly generate SQL based on the assignment
preferences
- Add `NextIpv6Address` type (needs tests)
- Rework callers of `IncompleteNetworkInterface` to use new
assignment-request enum

Author: bnaecker

nexus/db-model/src/network_interface.rs

@@ -19,12 +19,15 @@ use nexus_db_schema::schema::service_network_interface;
 use nexus_sled_agent_shared::inventory::ZoneKind;
 use nexus_types::external_api::params;
 use nexus_types::identity::Resource;
-use omicron_common::api::external::Error;
 use omicron_common::api::{external, internal};
 use omicron_uuid_kinds::GenericUuid;
 use omicron_uuid_kinds::InstanceUuid;
 use omicron_uuid_kinds::OmicronZoneUuid;
 use omicron_uuid_kinds::VnicUuid;
+use oxnet::IpNet;
+use oxnet::Ipv4Net;
+use oxnet::Ipv6Net;
+use std::net::IpAddr;
 use uuid::Uuid;
 
 /// The max number of interfaces that may be associated with a resource,
@@ -346,6 +349,179 @@ impl From<ServiceNetworkInterface> for NetworkInterface {
     }
 }
 
+mod private {
+    pub trait IpSealed: Clone + Copy + std::fmt::Debug {
+        fn into_ipnet(self) -> ipnetwork::IpNetwork;
+    }
+
+    impl IpSealed for std::net::Ipv4Addr {
+        fn into_ipnet(self) -> ipnetwork::IpNetwork {
+            ipnetwork::IpNetwork::V4(ipnetwork::Ipv4Network::from(self))
+        }
+    }
+    impl IpSealed for std::net::Ipv6Addr {
+        fn into_ipnet(self) -> ipnetwork::IpNetwork {
+            ipnetwork::IpNetwork::V6(ipnetwork::Ipv6Network::from(self))
+        }
+    }
+}
+
+pub trait Ip: private::IpSealed {}
+impl<T> Ip for T where T: private::IpSealed {}
+
+/// How an IP address is assigned to an interface.
+#[derive(Clone, Copy, Debug, Default)]
+pub enum IpAssignment<T: Ip> {
+    /// Automatically assign an IP address.
+    #[default]
+    Auto,
+    /// Explicitly assign a specific address, if available.
+    Explicit(T),
+}
+
+/// How to assign an IPv4 address.
+pub type Ipv4Assignment = IpAssignment<std::net::Ipv4Addr>;
+
+/// How to assign an IPv6 address.
+pub type Ipv6Assignment = IpAssignment<std::net::Ipv6Addr>;
+
+/// Configuration for a network interface's IPv4 addressing.
+#[derive(Clone, Debug, Default)]
+pub struct Ipv4Config {
+    /// The VPC-private address to assign to the interface.
+    pub ip: Ipv4Assignment,
+    /// Additional IP networks the interface can send / receive on.
+    pub transit_ips: Vec<Ipv4Net>,
+}
+
+/// Configuration for a network interface's IPv6 addressing.
+#[derive(Clone, Debug, Default)]
+pub struct Ipv6Config {
+    /// The VPC-private address to assign to the interface.
+    pub ip: Ipv6Assignment,
+    /// Additional IP networks the interface can send / receive on.
+    pub transit_ips: Vec<Ipv6Net>,
+}
+
+/// Configuration for a network interface's IP addressing.
+#[derive(Clone, Debug)]
+pub enum IpConfig {
+    /// The interface has only an IPv4 stack.
+    V4(Ipv4Config),
+    /// The interface has only an IPv6 stack.
+    V6(Ipv6Config),
+    /// The interface has both an IPv4 and IPv6 stack.
+    DualStack { v4: Ipv4Config, v6: Ipv6Config },
+}
+
+impl IpConfig {
+    /// Construct an IPv4 configuration with no transit IPs.
+    pub fn from_ipv4(addr: std::net::Ipv4Addr) -> Self {
+        IpConfig::V4(Ipv4Config {
+            ip: Ipv4Assignment::Explicit(addr),
+            transit_ips: vec![],
+        })
+    }
+
+    /// Construct an IP configuration with only an automatic IPv4 address.
+    pub fn auto_ipv4() -> Self {
+        IpConfig::V4(Ipv4Config::default())
+    }
+
+    /// Return the IPv4 address assignment.
+    pub fn ipv4_assignment(&self) -> Option<&Ipv4Assignment> {
+        match self {
+            IpConfig::V4(Ipv4Config { ip, .. }) => Some(ip),
+            IpConfig::V6(_) => None,
+            IpConfig::DualStack { v4: Ipv4Config { ip, .. }, .. } => Some(ip),
+        }
+    }
+
+    /// Return the IPv4 address explicitly requested, if one exists.
+    pub fn ipv4_addr(&self) -> Option<&std::net::Ipv4Addr> {
+        self.ipv4_assignment().and_then(|assignment| match assignment {
+            IpAssignment::Auto => None,
+            IpAssignment::Explicit(addr) => Some(addr),
+        })
+    }
+
+    /// Construct an IPv6 configuration with no transit IPs.
+    pub fn from_ipv6(addr: std::net::Ipv6Addr) -> Self {
+        IpConfig::V6(Ipv6Config {
+            ip: Ipv6Assignment::Explicit(addr),
+            transit_ips: vec![],
+        })
+    }
+
+    /// Construct an IP configuration with only an automatic IPv6 address.
+    pub fn auto_ipv6() -> Self {
+        IpConfig::V6(Ipv6Config::default())
+    }
+
+    /// Return the IPv6 address assignment.
+    pub fn ipv6_assignment(&self) -> Option<&Ipv6Assignment> {
+        match self {
+            IpConfig::V6(Ipv6Config { ip, .. }) => Some(ip),
+            IpConfig::V4(_) => None,
+            IpConfig::DualStack { v6: Ipv6Config { ip, .. }, .. } => Some(ip),
+        }
+    }
+
+    /// Return the IPv6 address explicitly requested, if one exists.
+    pub fn ipv6_addr(&self) -> Option<&std::net::Ipv6Addr> {
+        self.ipv6_assignment().and_then(|assignment| match assignment {
+            IpAssignment::Auto => None,
+            IpAssignment::Explicit(addr) => Some(addr),
+        })
+    }
+
+    /// Return the transit IPs requested in this configuration.
+    pub fn transit_ips(&self) -> Vec<IpNet> {
+        match self {
+            IpConfig::V4(Ipv4Config { transit_ips, .. }) => {
+                transit_ips.iter().copied().map(Into::into).collect()
+            }
+            IpConfig::V6(Ipv6Config { transit_ips, .. }) => {
+                transit_ips.iter().copied().map(Into::into).collect()
+            }
+            IpConfig::DualStack {
+                v4: Ipv4Config { transit_ips: ipv4_addrs, .. },
+                v6: Ipv6Config { transit_ips: ipv6_addrs, .. },
+            } => ipv4_addrs
+                .iter()
+                .copied()
+                .map(Into::into)
+                .chain(ipv6_addrs.iter().copied().map(Into::into))
+                .collect(),
+        }
+    }
+
+    /// Construct an IP configuration with both IPv4 / IPv6 addresses and no
+    /// transit IPs.
+    pub fn auto_dual_stack() -> Self {
+        IpConfig::DualStack {
+            v4: Ipv4Config::default(),
+            v6: Ipv6Config::default(),
+        }
+    }
+
+    /// Return true if this config has any transit IPs
+    fn has_transit_ips(&self) -> bool {
+        match self {
+            IpConfig::V4(Ipv4Config { transit_ips, .. }) => {
+                !transit_ips.is_empty()
+            }
+            IpConfig::V6(Ipv6Config { transit_ips, .. }) => {
+                !transit_ips.is_empty()
+            }
+            IpConfig::DualStack {
+                v4: Ipv4Config { transit_ips: ipv4_addrs, .. },
+                v6: Ipv6Config { transit_ips: ipv6_addrs, .. },
+            } => !ipv4_addrs.is_empty() || !ipv6_addrs.is_empty(),
+        }
+    }
+}
+
 /// A not fully constructed NetworkInterface. It may not yet have an IP
 /// address allocated.
 #[derive(Clone, Debug)]
@@ -354,10 +530,9 @@ pub struct IncompleteNetworkInterface {
     pub kind: NetworkInterfaceKind,
     pub parent_id: Uuid,
     pub subnet: VpcSubnet,
-    pub ip: Option<std::net::IpAddr>,
+    pub ip_config: IpConfig,
     pub mac: Option<external::MacAddr>,
     pub slot: Option<u8>,
-    pub transit_ips: Vec<IpNetwork>,
 }
 
 impl IncompleteNetworkInterface {
@@ -368,20 +543,15 @@ impl IncompleteNetworkInterface {
         parent_id: Uuid,
         subnet: VpcSubnet,
         identity: external::IdentityMetadataCreateParams,
-        ip: Option<std::net::IpAddr>,
+        ip_config: IpConfig,
         mac: Option<external::MacAddr>,
         slot: Option<u8>,
-        transit_ips: Vec<IpNetwork>,
     ) -> Result<Self, external::Error> {
-        if let Some(ip) = ip {
-            // TODO-completeness:
-            // https://github.com/oxidecomputer/omicron/issues/9244.
-            if ip.is_ipv6() {
-                return Err(Error::invalid_request(
-                    "IPv6 addresses are not yet supported",
-                ));
-            }
-            subnet.check_requestable_addr(ip)?;
+        if let Some(ip) = ip_config.ipv4_addr() {
+            subnet.check_requestable_addr(IpAddr::V4(*ip))?;
+        };
+        if let Some(ip) = ip_config.ipv6_addr() {
+            subnet.check_requestable_addr(IpAddr::V6(*ip))?;
         };
         if let Some(mac) = mac {
             match kind {
@@ -422,10 +592,9 @@ impl IncompleteNetworkInterface {
             kind,
             parent_id,
             subnet,
-            ip,
+            ip_config,
             mac,
             slot,
-            transit_ips,
         })
     }
 
@@ -434,19 +603,17 @@ impl IncompleteNetworkInterface {
         instance_id: InstanceUuid,
         subnet: VpcSubnet,
         identity: external::IdentityMetadataCreateParams,
-        ip: Option<std::net::IpAddr>,
-        transit_ips: Vec<IpNetwork>,
+        ip_config: IpConfig,
     ) -> Result<Self, external::Error> {
         Self::new(
             interface_id,
             NetworkInterfaceKind::Instance,
             instance_id.into_untyped_uuid(),
             subnet,
             identity,
-            ip,
+            ip_config,
             None,
             None,
-            transit_ips,
         )
     }
 
@@ -455,20 +622,24 @@ impl IncompleteNetworkInterface {
         service_id: Uuid,
         subnet: VpcSubnet,
         identity: external::IdentityMetadataCreateParams,
-        ip: std::net::IpAddr,
+        ip_config: IpConfig,
         mac: external::MacAddr,
         slot: u8,
     ) -> Result<Self, external::Error> {
+        if ip_config.has_transit_ips() {
+            return Err(external::Error::invalid_request(
+                "Cannot specify transit IPs for service NICs",
+            ));
+        }
         Self::new(
             interface_id,
             NetworkInterfaceKind::Service,
             service_id,
             subnet,
             identity,
-            Some(ip),
+            ip_config,
             Some(mac),
             Some(slot),
-            vec![], // Service interfaces don't use transit_ips
         )
     }
 
@@ -477,7 +648,7 @@ impl IncompleteNetworkInterface {
         probe_id: Uuid,
         subnet: VpcSubnet,
         identity: external::IdentityMetadataCreateParams,
-        ip: Option<std::net::IpAddr>,
+        ip_config: IpConfig,
         mac: Option<external::MacAddr>,
     ) -> Result<Self, external::Error> {
         Self::new(
@@ -486,10 +657,9 @@ impl IncompleteNetworkInterface {
             probe_id,
             subnet,
             identity,
-            ip,
+            ip_config,
             mac,
             None,
-            vec![], // Probe interfaces don't use transit_ips
         )
     }
 }

nexus/db-queries/src/db/datastore/deployment/external_networking.rs

@@ -12,6 +12,7 @@ use crate::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET;
 use crate::db::fixed_data::vpc_subnet::NTP_VPC_SUBNET;
 use nexus_db_lookup::DbConnection;
 use nexus_db_model::IncompleteNetworkInterface;
+use nexus_db_model::IpConfig;
 use nexus_db_model::IpPool;
 use nexus_sled_agent_shared::inventory::ZoneKind;
 use nexus_types::deployment::BlueprintZoneConfig;
@@ -384,6 +385,16 @@ impl DataStore {
         if self.is_nic_already_allocated(conn, service_id, nic, log).await? {
             return Ok(());
         }
+
+        // TODO-completeness: Handle dual-stack `shared::NetworkInterface`s.
+        // See https://github.com/oxidecomputer/omicron/issues/9246.
+        let std::net::IpAddr::V4(ip) = nic.ip else {
+            return Err(Error::internal_error(&format!(
+                "Unexpectedly found a service NIC without an IPv4 \
+                address, nic_id=\"{}\"",
+                nic.id,
+            )));
+        };
         let nic_arg = IncompleteNetworkInterface::new_service(
             nic.id,
             service_id.into_untyped_uuid(),
@@ -392,7 +403,7 @@ impl DataStore {
                 name: nic.name.clone(),
                 description: format!("{} service vNIC", zone_kind.report_str()),
             },
-            nic.ip,
+            IpConfig::from_ipv4(ip),
             nic.mac,
             nic.slot,
         )?;

nexus/db-queries/src/db/datastore/network_interface.rs

@@ -943,6 +943,7 @@ mod tests {
     use crate::db::pub_test_utils::TestDatabase;
     use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES;
     use nexus_db_fixed_data::vpc_subnet::NEXUS_VPC_SUBNET;
+    use nexus_db_model::IpConfig;
     use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET;
     use omicron_test_utils::dev;
     use std::collections::BTreeSet;
@@ -994,7 +995,7 @@ mod tests {
                     name: name.parse().unwrap(),
                     description: name,
                 },
-                ip.into(),
+                IpConfig::from_ipv4(ip),
                 macs.next().unwrap(),
                 0,
             )

nexus/db-queries/src/db/datastore/probe.rs

@@ -15,6 +15,7 @@ use nexus_db_errors::ErrorHandler;
 use nexus_db_errors::public_error_from_diesel;
 use nexus_db_lookup::LookupPath;
 use nexus_db_model::IncompleteNetworkInterface;
+use nexus_db_model::IpConfig;
 use nexus_db_model::Probe;
 use nexus_db_model::VpcSubnet;
 use nexus_db_model::to_db_typed_uuid;
@@ -294,7 +295,7 @@ impl super::DataStore {
                     probe.name(),
                 ),
             },
-            None, //Request IP address assignment
+            IpConfig::auto_ipv4(),
             None, //Request MAC address assignment
         )?;