diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index 4c670925f54..465cf7d002a 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -136,7 +136,7 @@ z_swadm () { # only set this if you want to override the version of opte/xde installed by the # install_opte.sh script -OPTE_COMMIT="" +OPTE_COMMIT="355fc09545445beda7cd789033507f13e80cbbe7" if [[ "x$OPTE_COMMIT" != "x" ]]; then curl -sSfOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde pfexec rem_drv xde || true diff --git a/Cargo.lock b/Cargo.lock index 45304295d96..0925d2ce4c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2311,7 +2311,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=fa5f15cdcd5864161a929e2ec01534f70dfba216#fa5f15cdcd5864161a929e2ec01534f70dfba216" +source = "git+https://github.com/oxidecomputer/maghemite?rev=42f18f0491eccd16921c7b6b7fa2470160af00c2#42f18f0491eccd16921c7b6b7fa2470160af00c2" dependencies = [ "oxnet", "percent-encoding", @@ -4740,7 +4740,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=23cebf3b1c911f09c203f7df50cc06bf780338e5#23cebf3b1c911f09c203f7df50cc06bf780338e5" +source = "git+https://github.com/oxidecomputer/opte?rev=355fc09545445beda7cd789033507f13e80cbbe7#355fc09545445beda7cd789033507f13e80cbbe7" dependencies = [ "bitflags 2.9.1", ] @@ -5318,7 +5318,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=23cebf3b1c911f09c203f7df50cc06bf780338e5#23cebf3b1c911f09c203f7df50cc06bf780338e5" +source = "git+https://github.com/oxidecomputer/opte?rev=355fc09545445beda7cd789033507f13e80cbbe7#355fc09545445beda7cd789033507f13e80cbbe7" dependencies = [ "quote", "syn 2.0.104", @@ -5883,7 +5883,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=fa5f15cdcd5864161a929e2ec01534f70dfba216#fa5f15cdcd5864161a929e2ec01534f70dfba216" +source = "git+https://github.com/oxidecomputer/maghemite?rev=42f18f0491eccd16921c7b6b7fa2470160af00c2#42f18f0491eccd16921c7b6b7fa2470160af00c2" dependencies = [ "anyhow", "chrono", @@ -8509,7 +8509,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=23cebf3b1c911f09c203f7df50cc06bf780338e5#23cebf3b1c911f09c203f7df50cc06bf780338e5" +source = "git+https://github.com/oxidecomputer/opte?rev=355fc09545445beda7cd789033507f13e80cbbe7#355fc09545445beda7cd789033507f13e80cbbe7" dependencies = [ "bitflags 2.9.1", "dyn-clone", @@ -8520,6 +8520,7 @@ dependencies = [ "postcard", "serde", "tabwriter", + "uuid", "version_check", "zerocopy 0.8.26", ] @@ -8527,7 +8528,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=23cebf3b1c911f09c203f7df50cc06bf780338e5#23cebf3b1c911f09c203f7df50cc06bf780338e5" +source = "git+https://github.com/oxidecomputer/opte?rev=355fc09545445beda7cd789033507f13e80cbbe7#355fc09545445beda7cd789033507f13e80cbbe7" dependencies = [ "illumos-sys-hdrs", "ingot", @@ -8535,12 +8536,13 @@ dependencies = [ "postcard", "serde", "smoltcp 0.11.0", + "uuid", ] [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=23cebf3b1c911f09c203f7df50cc06bf780338e5#23cebf3b1c911f09c203f7df50cc06bf780338e5" +source = "git+https://github.com/oxidecomputer/opte?rev=355fc09545445beda7cd789033507f13e80cbbe7#355fc09545445beda7cd789033507f13e80cbbe7" dependencies = [ "libc", "libnet", @@ -8549,6 +8551,7 @@ dependencies = [ "postcard", "serde", "thiserror 2.0.12", + "uuid", ] [[package]] @@ -8635,7 +8638,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=23cebf3b1c911f09c203f7df50cc06bf780338e5#23cebf3b1c911f09c203f7df50cc06bf780338e5" +source = "git+https://github.com/oxidecomputer/opte?rev=355fc09545445beda7cd789033507f13e80cbbe7#355fc09545445beda7cd789033507f13e80cbbe7" dependencies = [ "cfg-if", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index 61f5edb4b10..0474a7d6520 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -511,8 +511,8 @@ lldp_protocol = { git = "https://github.com/oxidecomputer/lldp", package = "prot macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" newtype_derive = "0.1.6" -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "fa5f15cdcd5864161a929e2ec01534f70dfba216" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "fa5f15cdcd5864161a929e2ec01534f70dfba216" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "42f18f0491eccd16921c7b6b7fa2470160af00c2" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "42f18f0491eccd16921c7b6b7fa2470160af00c2" } multimap = "0.10.1" nexus-auth = { path = "nexus/auth" } nexus-background-task-interface = { path = "nexus/background-task-interface" } @@ -568,7 +568,7 @@ omicron-workspace-hack = "0.1.0" omicron-zone-package = "0.12.2" oxide-client = { path = "clients/oxide-client" } oxide-tokio-rt = "0.1.2" -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "23cebf3b1c911f09c203f7df50cc06bf780338e5", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "355fc09545445beda7cd789033507f13e80cbbe7", features = [ "api", "std" ] } oxlog = { path = "dev-tools/oxlog" } oxnet = "0.1.2" once_cell = "1.21.3" @@ -578,7 +578,7 @@ openapiv3 = "2.2.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "23cebf3b1c911f09c203f7df50cc06bf780338e5" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "355fc09545445beda7cd789033507f13e80cbbe7" } oso = "0.27" owo-colors = "4.2.2" oximeter = { path = "oximeter/oximeter" } diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 7d415f2ba03..402446c2eb3 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -3474,6 +3474,121 @@ pub enum ImportExportPolicy { Allow(Vec), } +/// A packet flow recorded on a `NetworkInterface`. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +pub struct Flow { + pub metadata: FlowMetadata, + pub in_stat: FlowStat, + pub out_stat: FlowStat, +} + +/// A packet flow recorded on a `NetworkInterface`. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +pub struct FlowStat { + pub packets: u64, + pub bytes: u64, + pub packet_rate: f64, + pub byte_rate: f64, +} + +/// Information about a flow recorded on a `NetworkInterface`. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq)] +pub struct FlowMetadata { + /// An ephemeral ID bound to this flow. + pub flow_id: Uuid, + /// The time the first packet of this flow was seen at. + pub created_at: DateTime, + /// The direction of the first packet of this flow. + pub initial_packet: Direction, + /// The flowkey (or 5-tuple) of any packets on this flow as viewed by + /// the instance. + pub internal_key: Flowkey, + /// The flowkey (or 5-tuple) of any packets on this flow as viewed by + /// the remote half. + pub external_key: Flowkey, + /// All entities responsible for allowing packets in this flow to reach the + /// instance. + pub admitted_by_in: Option>, + /// All entities responsible for allowing packets in this flow to be sent + /// by the instance. + pub admitted_by_out: Option>, + /// How any outbound packets are to be routed. + pub forwarded: Option, +} + +/// The direction of a flow or packet, with respect to its target `Instance`. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +#[serde(rename_all = "snake_case")] +pub enum Direction { + In, + Out, +} + +/// Addresses and protocol-specific information used to group packets into a flow. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +pub struct Flowkey { + pub source_address: IpAddr, + pub destination_address: IpAddr, + pub protocol: u8, + pub info: Option, +} + +/// Protocol-specific flow information. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +#[serde(rename_all = "snake_case", tag = "type", content = "value")] +pub enum ProtocolInfo { + Ports(PortProtocolInfo), + Icmp(IcmpProtocolInfo), +} + +/// A pair of ports, identifying a flow in protocols such as TCP or UDP. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +pub struct PortProtocolInfo { + pub source_port: u16, + pub destination_port: u16, +} + +/// Message types information carried by ICMP. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +pub struct IcmpProtocolInfo { + pub r#type: u8, + pub code: u8, + pub id: Option, +} + +/// How the remote half of a flow is reached. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +#[serde(rename_all = "snake_case")] +pub enum ForwardClass { + VpcLocal, + External, +} + +/// A control-plane object which has matched a given flow and chosen to +/// allow it. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +#[serde(rename_all = "snake_case", tag = "type", content = "value")] +pub enum VpcEntity { + FirewallRule(Uuid), + FirewallDefaultIn, + VpcRoute(Uuid), + InternetGateway(Uuid), +} + /// Use instead of Option in API request body structs to get a field that can /// be null (parsed as `None`) but is not optional. Unlike Option, Nullable /// will fail to parse if the key is not present. The JSON Schema in the diff --git a/illumos-utils/src/opte/firewall_rules.rs b/illumos-utils/src/opte/firewall_rules.rs index 948808832a6..57a87239a9e 100644 --- a/illumos-utils/src/opte/firewall_rules.rs +++ b/illumos-utils/src/opte/firewall_rules.rs @@ -22,6 +22,7 @@ use oxide_vpc::api::IpAddr; use oxide_vpc::api::Ports; use oxide_vpc::api::ProtoFilter; use oxnet::IpNet; +use uuid::Uuid; trait FromVpcFirewallRule { fn action(&self) -> FirewallAction; @@ -170,6 +171,7 @@ pub fn opte_firewall_rules( .set_protocol(proto.clone()); filters }, + stat_id: Some(Uuid::new_v4()), }) .collect::>() }) diff --git a/illumos-utils/src/opte/illumos.rs b/illumos-utils/src/opte/illumos.rs index 3d1f0c8c707..315c9c493bc 100644 --- a/illumos-utils/src/opte/illumos.rs +++ b/illumos-utils/src/opte/illumos.rs @@ -7,7 +7,6 @@ use crate::addrobj::AddrObject; use crate::dladm; use camino::Utf8Path; -use omicron_common::api::internal::shared::NetworkInterfaceKind; use opte_ioctl::Error as OpteError; use opte_ioctl::OpteHdl; use slog::Logger; @@ -46,11 +45,11 @@ pub enum Error { #[error("Invalid IP configuration for port")] InvalidPortIpConfig, - #[error("Tried to release non-existent port ({0}, {1:?})")] - ReleaseMissingPort(uuid::Uuid, NetworkInterfaceKind), + #[error("Tried to release non-existent port ({0})")] + ReleaseMissingPort(uuid::Uuid), - #[error("Tried to update external IPs on non-existent port ({0}, {1:?})")] - ExternalIpUpdateMissingPort(uuid::Uuid, NetworkInterfaceKind), + #[error("Tried to update external IPs on non-existent port ({0})")] + ExternalIpUpdateMissingPort(uuid::Uuid), #[error("Could not find Primary NIC")] NoPrimaryNic, diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index 9f5c25462c5..47036fca273 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -15,6 +15,7 @@ cfg_if::cfg_if! { mod firewall_rules; mod port; mod port_manager; +mod stat; pub use firewall_rules::opte_firewall_rules; use ipnetwork::IpNetwork; diff --git a/illumos-utils/src/opte/non_illumos.rs b/illumos-utils/src/opte/non_illumos.rs index 3624a63547b..5a4cf83c5ec 100644 --- a/illumos-utils/src/opte/non_illumos.rs +++ b/illumos-utils/src/opte/non_illumos.rs @@ -5,18 +5,21 @@ //! Mock / dummy versions of the OPTE module, for non-illumos platforms use crate::addrobj::AddrObject; -use omicron_common::api::internal::shared::NetworkInterfaceKind; use oxide_vpc::api::AddRouterEntryReq; use oxide_vpc::api::ClearVirt2PhysReq; use oxide_vpc::api::DelRouterEntryReq; use oxide_vpc::api::DhcpCfg; use oxide_vpc::api::Direction; +use oxide_vpc::api::DumpFlowStatResp; +use oxide_vpc::api::DumpRootStatResp; use oxide_vpc::api::DumpVirt2PhysResp; +use oxide_vpc::api::InnerFlowId; use oxide_vpc::api::IpCfg; use oxide_vpc::api::IpCidr; use oxide_vpc::api::ListPortsResp; use oxide_vpc::api::NoResp; use oxide_vpc::api::PortInfo; +use oxide_vpc::api::Route; use oxide_vpc::api::RouterClass; use oxide_vpc::api::RouterTarget; use oxide_vpc::api::SetExternalIpsReq; @@ -24,11 +27,13 @@ use oxide_vpc::api::SetFwRulesReq; use oxide_vpc::api::SetVirt2PhysReq; use oxide_vpc::api::VpcCfg; use slog::Logger; +use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::hash_map::Entry; use std::net::IpAddr; use std::sync::Mutex; use std::sync::OnceLock; +use uuid::Uuid; type OpteError = anyhow::Error; @@ -40,11 +45,11 @@ pub enum Error { #[error("Invalid IP configuration for port")] InvalidPortIpConfig, - #[error("Tried to release non-existent port ({0}, {1:?})")] - ReleaseMissingPort(uuid::Uuid, NetworkInterfaceKind), + #[error("Tried to release non-existent port ({0})")] + ReleaseMissingPort(uuid::Uuid), - #[error("Tried to update external IPs on non-existent port ({0}, {1:?})")] - ExternalIpUpdateMissingPort(uuid::Uuid, NetworkInterfaceKind), + #[error("Tried to update external IPs on non-existent port ({0})")] + ExternalIpUpdateMissingPort(uuid::Uuid), #[error("Could not find Primary NIC")] NoPrimaryNic, @@ -69,75 +74,29 @@ pub fn delete_all_xde_devices(log: &Logger) -> Result<(), Error> { Ok(()) } -#[derive(Debug)] +// Removes the stat ID from the Route payload. +#[derive(Debug, Eq, PartialEq)] pub(crate) struct RouteInfo { pub dest: IpCidr, pub target: RouterTarget, pub class: RouterClass, } -// NOTE: It would be nice to derive this, but `RouterTarget` and `RouterClass` -// are in OPTE, and they don't currently implement this trait. -impl PartialEq for RouteInfo { - fn eq(&self, other: &Self) -> bool { - if self.dest != other.dest { - return false; - } - match (self.class, other.class) { - (RouterClass::System, RouterClass::Custom) => return false, - (RouterClass::Custom, RouterClass::System) => return false, - (RouterClass::System, RouterClass::System) - | (RouterClass::Custom, RouterClass::Custom) => {} - } - match (self.target, other.target) { - (RouterTarget::Drop, RouterTarget::Drop) => true, - ( - RouterTarget::InternetGateway(id0), - RouterTarget::InternetGateway(id1), - ) => id0 == id1, - (RouterTarget::Ip(ip0), RouterTarget::Ip(ip1)) => ip0 == ip1, - ( - RouterTarget::VpcSubnet(cidr0), - RouterTarget::VpcSubnet(cidr1), - ) => cidr0 == cidr1, - (RouterTarget::Drop, RouterTarget::InternetGateway(_)) - | (RouterTarget::Drop, RouterTarget::Ip(_)) - | (RouterTarget::Drop, RouterTarget::VpcSubnet(_)) - | (RouterTarget::InternetGateway(_), RouterTarget::Drop) - | (RouterTarget::InternetGateway(_), RouterTarget::Ip(_)) - | (RouterTarget::InternetGateway(_), RouterTarget::VpcSubnet(_)) - | (RouterTarget::Ip(_), RouterTarget::Drop) - | (RouterTarget::Ip(_), RouterTarget::InternetGateway(_)) - | (RouterTarget::Ip(_), RouterTarget::VpcSubnet(_)) - | (RouterTarget::VpcSubnet(_), RouterTarget::Drop) - | (RouterTarget::VpcSubnet(_), RouterTarget::InternetGateway(_)) - | (RouterTarget::VpcSubnet(_), RouterTarget::Ip(_)) => false, - } - } -} - -impl RouteInfo { - #[cfg(test)] - pub fn is_system_default_ipv4_route(&self) -> bool { - let system_default_route = RouteInfo { - dest: IpCidr::Ip4(oxide_vpc::api::Ipv4Cidr::new( +#[cfg(test)] +pub(crate) fn is_system_default_ipv4_route(route: &RouteInfo) -> bool { + (route.dest, route.target, route.class) + == ( + IpCidr::Ip4(oxide_vpc::api::Ipv4Cidr::new( oxide_vpc::api::Ipv4Addr::ANY_ADDR, 0.try_into().unwrap(), )), - target: RouterTarget::InternetGateway(None), - class: RouterClass::System, - }; - *self == system_default_route - } + RouterTarget::InternetGateway(None), + RouterClass::System, + ) } -impl From<&AddRouterEntryReq> for RouteInfo { - fn from(value: &AddRouterEntryReq) -> Self { - Self { dest: value.dest, target: value.target, class: value.class } - } -} -impl From<&DelRouterEntryReq> for RouteInfo { - fn from(value: &DelRouterEntryReq) -> Self { +impl From<&Route> for RouteInfo { + fn from(value: &Route) -> Self { Self { dest: value.dest, target: value.target, class: value.class } } } @@ -246,7 +205,7 @@ impl Handle { else { anyhow::bail!("No such port '{}'", req.port_name); }; - routes.push(req.into()); + routes.push((&req.route).into()); Ok(NO_RESPONSE) } @@ -270,7 +229,7 @@ impl Handle { else { anyhow::bail!("No such port '{}'", req.port_name); }; - let req = RouteInfo::from(req); + let req = RouteInfo::from(&req.route); if let Some(index) = routes.iter().position(|rt| rt == &req) { routes.remove(index); } @@ -303,6 +262,30 @@ impl Handle { unimplemented!("Not yet used in tests") } + /// Request the current state of some (or all) root stats contained + /// in a port. + /// + /// An empty `stat_ids` will request all present stats. + pub fn dump_root_stats( + &self, + _port_name: &str, + _stat_ids: impl IntoIterator, + ) -> Result { + Ok(DumpRootStatResp { root_stats: BTreeMap::new() }) + } + + /// Request the current state of some (or all) flow stats contained + /// in a port. + /// + /// An empty `flow_keys` will request all present flows. + pub fn dump_flow_stats( + &self, + _port_name: &str, + _flow_keys: impl IntoIterator, + ) -> Result, Error> { + Ok(DumpFlowStatResp { flow_stats: BTreeMap::new() }) + } + /// List ports on the current system. #[allow(dead_code)] pub(crate) fn list_ports(&self) -> Result { diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs index 6b4c5b8b054..68f2f432a0c 100644 --- a/illumos-utils/src/opte/port.rs +++ b/illumos-utils/src/opte/port.rs @@ -9,12 +9,18 @@ use crate::opte::Handle; use crate::opte::Vni; use macaddr::MacAddr6; use omicron_common::api::external; +use omicron_common::api::internal::shared::NetworkInterfaceKind; use omicron_common::api::internal::shared::RouterId; use omicron_common::api::internal::shared::RouterKind; use oxnet::IpNet; use std::net::IpAddr; use std::sync::Arc; +use super::stat::PortStats; + +// TODO: This should probably comprise `NetworkInterface`, to enable more +// unified management/querying of state across Instance/Zone/Port. That +// would require some large changes to `InstanceRunner`. #[derive(Debug)] pub struct PortData { /// Name of the port as identified by OPTE @@ -27,10 +33,19 @@ pub struct PortData { pub(crate) slot: u8, /// Geneve VNI for the VPC pub(crate) vni: Vni, - /// Subnet the port belong to within the VPC. + /// Subnet the port belongs to within the VPC. pub(crate) subnet: IpNet, /// Information about the virtual gateway, aka OPTE pub(crate) gateway: Gateway, + + // TODO: Will be used in later rootstat -> VPC UUID hierarchy for + // oximeter. + #[expect(unused)] + /// The type and ID of the client this NIC is bound to. + pub(crate) parent: NetworkInterfaceKind, + + /// Periodically polled stats from this port. + pub(crate) stats: PortStats, } #[derive(Debug)] @@ -109,6 +124,10 @@ impl Port { self.inner.slot } + pub fn stats(&self) -> &PortStats { + &self.inner.stats + } + pub fn system_router_key(&self) -> RouterId { // Unwrap safety: both of these VNI types represent validated u24s. let vni = external::Vni::try_from(self.vni().as_u32()).unwrap(); diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 97eba85e621..32858682055 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -12,6 +12,7 @@ use crate::opte::Port; use crate::opte::Vni; use crate::opte::opte_firewall_rules; use crate::opte::port::PortData; +use crate::opte::stat::PortStats; use ipnetwork::IpNetwork; use macaddr::MacAddr6; use omicron_common::api::external; @@ -82,7 +83,7 @@ struct PortManagerInner { /// Map of all ports, keyed on the interface Uuid and its kind /// (which includes the Uuid of the parent instance or service) - ports: Mutex>, + ports: Mutex>, /// Map of all current resolved routes. routes: Mutex>, @@ -307,7 +308,7 @@ impl PortManager { }; let (port, ticket) = { let mut ports = self.inner.ports.lock().unwrap(); - let ticket = PortTicket::new(nic.id, nic.kind, self.inner.clone()); + let ticket = PortTicket::new(nic.id, self.inner.clone()); let port = Port::new(PortData { name: port_name.clone(), ip: nic.ip, @@ -316,8 +317,10 @@ impl PortManager { vni, subnet: nic.subnet, gateway, + parent: nic.kind, + stats: PortStats::new(&port_name, self.inner.log.clone()), }); - let old = ports.insert((nic.id, nic.kind), port.clone()); + let old = ports.insert(nic.id, port.clone()); assert!( old.is_none(), "Duplicate OPTE port detected: interface_id = {}, kind = {:?}", @@ -411,10 +414,13 @@ impl PortManager { ] { for route in &routes.routes { let route = AddRouterEntryReq { - class, + route: oxide_vpc::api::Route { + dest: super::net_to_cidr(route.dest), + target: super::router_target_opte(&route.target), + class, + stat_id: Some(Uuid::new_v4()), + }, port_name: port_name.clone(), - dest: super::net_to_cidr(route.dest), - target: super::router_target_opte(&route.target), }; hdl.add_router_entry(&route)?; @@ -555,10 +561,13 @@ impl PortManager { for route in to_delete { let route = DelRouterEntryReq { - class, + route: oxide_vpc::api::Route { + dest: super::net_to_cidr(route.dest), + target: super::router_target_opte(&route.target), + class, + stat_id: None, + }, port_name: port.name().into(), - dest: super::net_to_cidr(route.dest), - target: super::router_target_opte(&route.target), }; hdl.del_router_entry(&route)?; @@ -573,10 +582,13 @@ impl PortManager { for route in to_add { let route = AddRouterEntryReq { - class, + route: oxide_vpc::api::Route { + dest: super::net_to_cidr(route.dest), + target: super::router_target_opte(&route.target), + class, + stat_id: Some(Uuid::new_v4()), + }, port_name: port.name().into(), - dest: super::net_to_cidr(route.dest), - target: super::router_target_opte(&route.target), }; hdl.add_router_entry(&route)?; @@ -612,15 +624,14 @@ impl PortManager { pub fn external_ips_ensure( &self, nic_id: Uuid, - nic_kind: NetworkInterfaceKind, source_nat: Option, ephemeral_ip: Option, floating_ips: &[IpAddr], ) -> Result<(), Error> { let ports = self.inner.ports.lock().unwrap(); - let port = ports.get(&(nic_id, nic_kind)).ok_or_else(|| { - Error::ExternalIpUpdateMissingPort(nic_id, nic_kind) - })?; + let port = ports + .get(&nic_id) + .ok_or_else(|| Error::ExternalIpUpdateMissingPort(nic_id))?; self.external_ips_ensure_port( port, @@ -769,9 +780,9 @@ impl PortManager { // We update VPC rules as a set so grab only // the relevant ports using the VPC's VNI. let vpc_ports = ports - .iter() - .filter(|((_, _), port)| u32::from(vni) == u32::from(*port.vni())); - for ((_, _), port) in vpc_ports { + .values() + .filter(|port| u32::from(vni) == u32::from(*port.vni())); + for port in vpc_ports { let rules = opte_firewall_rules(rules, port.vni(), port.mac()); let port_name = port.name().to_string(); info!( @@ -788,6 +799,24 @@ impl PortManager { Ok(()) } + pub fn get_nic_ids(&self) -> Vec { + let ports = self.inner.ports.lock().unwrap(); + + ports.keys().copied().collect() + } + + pub fn get_nic_flows( + &self, + nic_id: Uuid, + ) -> Result, Error> { + let ports = self.inner.ports.lock().unwrap(); + let port = ports + .get(&nic_id) + .ok_or_else(|| Error::ExternalIpUpdateMissingPort(nic_id))?; + + Ok(port.stats().flow_stats()) + } + pub fn list_virtual_nics( &self, ) -> Result, Error> { @@ -876,7 +905,6 @@ impl PortManager { pub struct PortTicket { id: Uuid, - kind: NetworkInterfaceKind, manager: Arc, } @@ -884,31 +912,25 @@ impl std::fmt::Debug for PortTicket { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { f.debug_struct("PortTicket") .field("id", &self.id) - .field("kind", &self.kind) .field("manager", &"{ .. }") .finish() } } impl PortTicket { - fn new( - id: Uuid, - kind: NetworkInterfaceKind, - manager: Arc, - ) -> Self { - Self { id, kind, manager } + fn new(id: Uuid, manager: Arc) -> Self { + Self { id, manager } } fn release_inner(&mut self) -> Result<(), Error> { let mut ports = self.manager.ports.lock().unwrap(); - let Some(port) = ports.remove(&(self.id, self.kind)) else { + let Some(port) = ports.remove(&self.id) else { error!( self.manager.log, "Tried to release non-existent port"; "id" => ?&self.id, - "kind" => ?&self.kind, ); - return Err(Error::ReleaseMissingPort(self.id, self.kind)); + return Err(Error::ReleaseMissingPort(self.id)); }; drop(ports); @@ -937,7 +959,6 @@ impl PortTicket { self.manager.log, "Removed OPTE port from manager"; "id" => ?&self.id, - "kind" => ?&self.kind, "port" => ?&port, ); Ok(()) @@ -966,7 +987,7 @@ impl Drop for PortTicket { #[cfg(test)] mod tests { - use crate::opte::Handle; + use crate::opte::{Handle, is_system_default_ipv4_route}; use super::{PortCreateParams, PortManager}; use macaddr::MacAddr6; @@ -988,8 +1009,8 @@ mod tests { use uuid::Uuid; // Regression for https://github.com/oxidecomputer/omicron/issues/7541. - #[test] - fn multiple_ports_does_not_destroy_default_route() { + #[tokio::test] + async fn multiple_ports_does_not_destroy_default_route() { let logctx = test_setup_log("multiple_ports_does_not_destroy_default_route"); let manager = PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST); @@ -1107,7 +1128,7 @@ mod tests { .unwrap() .routes .iter() - .filter(|rt| rt.is_system_default_ipv4_route()) + .filter(|rt| is_system_default_ipv4_route(&rt)) .collect::>(); assert_eq!( rt.len(), @@ -1179,7 +1200,7 @@ mod tests { .unwrap() .routes .iter() - .filter(|rt| rt.is_system_default_ipv4_route()) + .filter(|rt| is_system_default_ipv4_route(&rt)) .collect::>(); assert_eq!( rt.len(), @@ -1276,7 +1297,7 @@ mod tests { .unwrap() .routes .iter() - .filter(|rt| rt.is_system_default_ipv4_route()) + .filter(|rt| is_system_default_ipv4_route(&rt)) .collect::>(); assert_eq!( rt.len(), @@ -1345,7 +1366,7 @@ mod tests { .unwrap() .routes .iter() - .filter(|rt| rt.is_system_default_ipv4_route()) + .filter(|rt| is_system_default_ipv4_route(&rt)) .collect::>(); assert_eq!( rt.len(), diff --git a/illumos-utils/src/opte/stat.rs b/illumos-utils/src/opte/stat.rs new file mode 100644 index 00000000000..449c76eac5e --- /dev/null +++ b/illumos-utils/src/opte/stat.rs @@ -0,0 +1,441 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Flow and root stat tracking for individual OPTE ports. + +use super::Handle; +use omicron_common::api::external::{ + self, Flow, FlowMetadata, FlowStat as ExternalFlowStat, +}; +use oxide_vpc::api::{ + Direction, FlowStat, FullCounter, InnerFlowId, stat as vpc_stat, +}; +use slog::Logger; +use slog_error_chain::InlineErrorChain; +use std::{ + collections::{HashMap, hash_map::Entry}, + sync::{ + Arc, LazyLock, RwLock, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant, SystemTime}, +}; +use tokio::time::MissedTickBehavior; +use uuid::Uuid; + +// TODO: Controlplane needs to tell us the UUIDs of all routes, firewall rules. + +const FLOW_STAT_REFRESH_INTERVAL: Duration = Duration::from_secs(1); +const ROOT_STAT_REFRESH_INTERVAL: Duration = Duration::from_secs(9); +const PRUNE_INTERVAL: Duration = Duration::from_secs(5); +const PRUNE_AGE: Duration = Duration::from_secs(10); + +type UniqueFlow = (InnerFlowId, InnerFlowId, u64); + +#[derive(Debug)] +pub struct PortStats { + shared: Arc, +} + +impl Drop for PortStats { + fn drop(&mut self) { + self.shared.task_quit.store(true, Ordering::Relaxed); + } +} + +impl PortStats { + pub fn new(name: impl Into, log: Logger) -> Self { + let shared = Arc::new(PortStatsShared { + name: name.into(), + task_quit: false.into(), + state: Default::default(), + log, + }); + + tokio::spawn(run_port_stat(Arc::clone(&shared))); + + Self { shared } + } + + /// Gather all flow stats when requested by a client. + pub fn flow_stats(&self) -> Vec { + let mut out = HashMap::new(); + + { + let state = self.shared.state.read().unwrap(); + for (flowid, Timed { body, .. }) in &state.flows { + let unique = unique_flow(&flowid, &body.last); + let Some(ufid) = state.flow_instances.get(&unique) else { + slog::error!(&self.shared.log, "Hi?"); + continue; + }; + + let mut forwarded = None; + let bases = body + .last + .bases + .iter() + .filter_map(|v| match state.label_map.get(v) { + Some(FlowLabel::Destination(v)) => { + forwarded = Some(v.clone()); + None + } + Some(FlowLabel::Entity(v)) => Some(v.clone()), + _ => None, + }) + .collect(); + + match out.entry(unique) { + Entry::Vacant(val) => { + // Use the stats from the first entry encountered. + let (in_key, out_key, ad_in, ad_out) = match body + .last + .dir + { + Direction::In => { + (*flowid, body.last.partner, Some(bases), None) + } + Direction::Out => { + (body.last.partner, *flowid, None, Some(bases)) + } + }; + val.insert(Flow { + metadata: FlowMetadata { + flow_id: ufid.body, + // TODO: need to correlate timestamps between + // kmod and here?! + created_at: SystemTime::now().into(), + initial_packet: direction(body.last.first_dir), + internal_key: flowkey(out_key), + external_key: flowkey(in_key), + admitted_by_in: ad_in, + admitted_by_out: ad_out, + forwarded, + }, + in_stat: ExternalFlowStat { + packets: body.last.stats.pkts_in, + bytes: body.last.stats.bytes_in, + packet_rate: body.in_packets_per_sec, + byte_rate: body.in_bytes_per_sec, + }, + out_stat: ExternalFlowStat { + packets: body.last.stats.pkts_out, + bytes: body.last.stats.bytes_out, + packet_rate: body.out_packets_per_sec, + byte_rate: body.out_bytes_per_sec, + }, + }); + } + Entry::Occupied(mut val) => { + // The second half fills in the remaining metadata. + let val = val.get_mut(); + if val.metadata.forwarded.is_none() { + val.metadata.forwarded = forwarded; + } + match body.last.dir { + Direction::In => { + val.metadata.admitted_by_in = Some(bases) + } + Direction::Out => { + val.metadata.admitted_by_out = Some(bases) + } + } + } + } + } + } + + out.into_values().collect() + } + + // TODO: want `fn root_stats`, need to be able to pull back up into + // oximeter in particular. +} + +#[derive(Debug)] +struct PortStatsShared { + name: String, + state: RwLock, + task_quit: AtomicBool, + log: Logger, +} + +impl PortStatsShared { + fn collect_flows(&self) -> Result<(), anyhow::Error> { + let new_stats = { + let hdl = Handle::new()?; + hdl.dump_flow_stats(&self.name, [])? + }; + let now = Instant::now(); + + let mut state = self.state.write().unwrap(); + for (flowid, stat) in new_stats.flow_stats { + let unique = unique_flow(&flowid, &stat); + state + .flow_instances + .entry(unique) + .or_insert_with(|| Timed { hit_at: now, body: Uuid::new_v4() }) + .hit_at = now; + + match state.flows.entry(flowid) { + Entry::Occupied(mut val) => { + let val = val.get_mut(); + let elapsed = now.duration_since(val.hit_at); + val.body.update(stat, &elapsed); + val.hit_at = now; + } + Entry::Vacant(vacant) => { + vacant.insert(Timed { + hit_at: now, + body: FlowSnapshot::new(stat), + }); + } + } + } + + Ok(()) + } + + fn collect_roots(&self) -> Result<(), anyhow::Error> { + let new_stats = { + let hdl = Handle::new()?; + hdl.dump_root_stats(&self.name, [])? + }; + let now = Instant::now(); + + let mut state = self.state.write().unwrap(); + for (id, stats) in new_stats.root_stats { + state.roots.insert(id, Timed { hit_at: now, body: stats }); + } + + Ok(()) + } + + fn prune(&self) { + let now = Instant::now(); + let mut state = self.state.write().unwrap(); + + state.flows.retain(|_, v| now.duration_since(v.hit_at) <= PRUNE_AGE); + state.roots.retain(|_, v| now.duration_since(v.hit_at) <= PRUNE_AGE); + state + .flow_instances + .retain(|_, v| now.duration_since(v.hit_at) <= PRUNE_AGE); + } +} + +#[derive(Debug, PartialEq, PartialOrd, Ord, Hash, Eq)] +struct Timed { + hit_at: Instant, + body: S, +} + +static BASE_MAP: LazyLock> = LazyLock::new(|| { + [ + ( + vpc_stat::DESTINATION_INTERNET, + FlowLabel::Destination(external::ForwardClass::External), + ), + ( + vpc_stat::DESTINATION_VPC_LOCAL, + FlowLabel::Destination(external::ForwardClass::VpcLocal), + ), + ( + vpc_stat::FW_DEFAULT_IN, + FlowLabel::Entity(external::VpcEntity::FirewallDefaultIn), + ), + ( + vpc_stat::FW_DEFAULT_OUT, + FlowLabel::Builtin(VpcBuiltinLabel::FirewallDefaultOut), + ), + ( + vpc_stat::ROUTER_NOROUTE, + FlowLabel::Builtin(VpcBuiltinLabel::NoRouteMatched), + ), + ( + vpc_stat::GATEWAY_NOSPOOF_IN, + FlowLabel::Builtin(VpcBuiltinLabel::SpoofPrevention), + ), + ( + vpc_stat::GATEWAY_NOSPOOF_OUT, + FlowLabel::Builtin(VpcBuiltinLabel::SpoofPrevention), + ), + ] + .into_iter() + .collect() +}); + +#[derive(Debug)] +struct State { + flows: HashMap>, + roots: HashMap>, + label_map: HashMap, + flow_instances: HashMap>, +} + +impl Default for State { + fn default() -> Self { + Self { + label_map: BASE_MAP.clone(), + + flows: HashMap::new(), + roots: HashMap::new(), + flow_instances: HashMap::new(), + } + } +} + +async fn run_port_stat(state: Arc) { + let mut flow_collect = tokio::time::interval(FLOW_STAT_REFRESH_INTERVAL); + let mut root_collect = tokio::time::interval(ROOT_STAT_REFRESH_INTERVAL); + let mut prune = tokio::time::interval(PRUNE_INTERVAL); + flow_collect.set_missed_tick_behavior(MissedTickBehavior::Skip); + root_collect.set_missed_tick_behavior(MissedTickBehavior::Skip); + prune.set_missed_tick_behavior(MissedTickBehavior::Skip); + + loop { + if state.task_quit.load(Ordering::Relaxed) { + return; + } + + // TODO: log on error. + tokio::select! { + _ = flow_collect.tick() => { + if let Err(e) = state.collect_flows() { + slog::error!( + &state.log, + "failed to collect flow stats for OPTE port"; + "port" => &state.name, + "err" => InlineErrorChain::new(e.as_ref()), + ); + } + }, + _ = root_collect.tick() => { + if let Err(e) = state.collect_roots() { + slog::error!( + &state.log, + "failed to collect root stats for OPTE port"; + "port" => &state.name, + "err" => InlineErrorChain::new(e.as_ref()), + ); + } + }, + _ = prune.tick() => { + state.prune(); + }, + } + } +} + +#[derive(Debug, Clone)] +pub enum FlowLabel { + Entity(external::VpcEntity), + Destination(external::ForwardClass), + // TODO: These will be used as part of oximeter association. + #[expect(unused)] + Builtin(VpcBuiltinLabel), +} + +#[derive(Debug, Clone)] +pub enum VpcBuiltinLabel { + NoRouteMatched, + FirewallDefaultOut, + SpoofPrevention, +} + +#[derive(Debug)] +pub struct FlowSnapshot { + pub last: FlowStat, + + pub in_packets_per_sec: f64, + pub in_bytes_per_sec: f64, + pub out_packets_per_sec: f64, + pub out_bytes_per_sec: f64, +} + +impl FlowSnapshot { + fn new(stat: FlowStat) -> Self { + Self { + last: stat, + + in_packets_per_sec: 0.0, + in_bytes_per_sec: 0.0, + out_packets_per_sec: 0.0, + out_bytes_per_sec: 0.0, + } + } + + fn update(&mut self, stat: FlowStat, elapsed: &Duration) { + let elapsed = elapsed.as_secs_f64(); + self.in_packets_per_sec = + (stat.stats.pkts_in - self.last.stats.pkts_in) as f64 / elapsed; + self.in_bytes_per_sec = + (stat.stats.bytes_in - self.last.stats.bytes_in) as f64 / elapsed; + self.out_packets_per_sec = + (stat.stats.pkts_out - self.last.stats.pkts_out) as f64 / elapsed; + self.out_bytes_per_sec = + (stat.stats.bytes_out - self.last.stats.bytes_out) as f64 / elapsed; + self.last = stat; + } +} + +fn unique_flow(id: &InnerFlowId, stat: &FlowStat) -> UniqueFlow { + let (in_fid, out_fid) = match stat.dir { + Direction::In => (*id, stat.partner), + Direction::Out => (stat.partner, *id), + }; + + (in_fid, out_fid, stat.stats.created_at) +} + +fn direction(dir: Direction) -> external::Direction { + match dir { + Direction::In => external::Direction::In, + Direction::Out => external::Direction::Out, + } +} + +fn flowkey(id: InnerFlowId) -> external::Flowkey { + let (source_address, destination_address) = match id.addrs { + oxide_vpc::api::AddrPair::V4 { src, dst } => { + (src.bytes().into(), dst.bytes().into()) + } + oxide_vpc::api::AddrPair::V6 { src, dst } => { + (src.bytes().into(), dst.bytes().into()) + } + }; + let info = id.l4_info().map(|v| match v { + oxide_vpc::api::L4Info::Ports(port_info) => { + external::ProtocolInfo::Ports(external::PortProtocolInfo { + source_port: port_info.src_port, + destination_port: port_info.dst_port, + }) + } + oxide_vpc::api::L4Info::Icmpv4(icmp_info) => { + external::ProtocolInfo::Icmp(external::IcmpProtocolInfo { + r#type: icmp_info.ty, + code: icmp_info.code, + id: match icmp_info.ty { + 0 | 8 => Some(icmp_info.id), + _ => None, + }, + }) + } + oxide_vpc::api::L4Info::Icmpv6(icmp_info) => { + external::ProtocolInfo::Icmp(external::IcmpProtocolInfo { + r#type: icmp_info.ty, + code: icmp_info.code, + id: match icmp_info.ty { + 128 | 129 => Some(icmp_info.id), + _ => None, + }, + }) + } + }); + external::Flowkey { + source_address, + destination_address, + protocol: id.proto, + info, + } +} diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 9f6fefc08e3..e4829635c7f 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -404,6 +404,74 @@ } } }, + "/network-interfaces": { + "get": { + "summary": "Get the IDs of all OPTE interfaces", + "operationId": "nic_ids_list", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_Uuid", + "type": "array", + "items": { + "type": "string", + "format": "uuid" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/network-interfaces/{nic_id}/flows": { + "get": { + "summary": "Get per-flow stats currently reported by an OPTE interface.", + "operationId": "nic_flows_list", + "parameters": [ + { + "in": "path", + "name": "nic_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_Flow", + "type": "array", + "items": { + "$ref": "#/components/schemas/Flow" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/omicron-config": { "put": { "operationId": "omicron_config_put", @@ -3827,6 +3895,14 @@ "search_domains" ] }, + "Direction": { + "description": "The direction of a flow or packet, with respect to its target `Instance`.", + "type": "string", + "enum": [ + "in", + "out" + ] + }, "DiskEnsureBody": { "description": "Sent from to a sled agent to establish the runtime state of a Disk", "type": "object", @@ -4335,6 +4411,168 @@ ], "additionalProperties": false }, + "Flow": { + "description": "A packet flow recorded on a `NetworkInterface`.", + "type": "object", + "properties": { + "in_stat": { + "$ref": "#/components/schemas/FlowStat" + }, + "metadata": { + "$ref": "#/components/schemas/FlowMetadata" + }, + "out_stat": { + "$ref": "#/components/schemas/FlowStat" + } + }, + "required": [ + "in_stat", + "metadata", + "out_stat" + ] + }, + "FlowMetadata": { + "description": "Information about a flow recorded on a `NetworkInterface`.", + "type": "object", + "properties": { + "admitted_by_in": { + "nullable": true, + "description": "All entities responsible for allowing packets in this flow to reach the instance.", + "type": "array", + "items": { + "$ref": "#/components/schemas/VpcEntity" + } + }, + "admitted_by_out": { + "nullable": true, + "description": "All entities responsible for allowing packets in this flow to be sent by the instance.", + "type": "array", + "items": { + "$ref": "#/components/schemas/VpcEntity" + } + }, + "created_at": { + "description": "The time the first packet of this flow was seen at.", + "type": "string", + "format": "date-time" + }, + "external_key": { + "description": "The flowkey (or 5-tuple) of any packets on this flow as viewed by the remote half.", + "allOf": [ + { + "$ref": "#/components/schemas/Flowkey" + } + ] + }, + "flow_id": { + "description": "An ephemeral ID bound to this flow.", + "type": "string", + "format": "uuid" + }, + "forwarded": { + "nullable": true, + "description": "How any outbound packets are to be routed.", + "allOf": [ + { + "$ref": "#/components/schemas/ForwardClass" + } + ] + }, + "initial_packet": { + "description": "The direction of the first packet of this flow.", + "allOf": [ + { + "$ref": "#/components/schemas/Direction" + } + ] + }, + "internal_key": { + "description": "The flowkey (or 5-tuple) of any packets on this flow as viewed by the instance.", + "allOf": [ + { + "$ref": "#/components/schemas/Flowkey" + } + ] + } + }, + "required": [ + "created_at", + "external_key", + "flow_id", + "initial_packet", + "internal_key" + ] + }, + "FlowStat": { + "description": "A packet flow recorded on a `NetworkInterface`.", + "type": "object", + "properties": { + "byte_rate": { + "type": "number", + "format": "double" + }, + "bytes": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "packet_rate": { + "type": "number", + "format": "double" + }, + "packets": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "byte_rate", + "bytes", + "packet_rate", + "packets" + ] + }, + "Flowkey": { + "description": "Addresses and protocol-specific information used to group packets into a flow.", + "type": "object", + "properties": { + "destination_address": { + "type": "string", + "format": "ip" + }, + "info": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/ProtocolInfo" + } + ] + }, + "protocol": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "source_address": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination_address", + "protocol", + "source_address" + ] + }, + "ForwardClass": { + "description": "How the remote half of a flow is reached.", + "type": "string", + "enum": [ + "vpc_local", + "external" + ] + }, "Generation": { "description": "Generation numbers stored in the database, used for optimistic concurrency control", "type": "integer", @@ -4571,6 +4809,32 @@ "minLength": 1, "maxLength": 7 }, + "IcmpProtocolInfo": { + "description": "Message types information carried by ICMP.", + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "id": { + "nullable": true, + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "type": { + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "required": [ + "code", + "type" + ] + }, "IdMapDatasetConfig": { "type": "object", "additionalProperties": { @@ -6381,6 +6645,26 @@ "rs" ] }, + "PortProtocolInfo": { + "description": "A pair of ports, identifying a flow in protocols such as TCP or UDP.", + "type": "object", + "properties": { + "destination_port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "source_port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "destination_port", + "source_port" + ] + }, "PortSpeed": { "description": "Switchport Speed options", "type": "string", @@ -6424,6 +6708,47 @@ "minItems": 2, "maxItems": 2 }, + "ProtocolInfo": { + "description": "Protocol-specific flow information.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ports" + ] + }, + "value": { + "$ref": "#/components/schemas/PortProtocolInfo" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "icmp" + ] + }, + "value": { + "$ref": "#/components/schemas/IcmpProtocolInfo" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, "QemuPvpanic": { "type": "object", "properties": { @@ -7694,6 +8019,82 @@ "format": "uint32", "minimum": 0 }, + "VpcEntity": { + "description": "A control-plane object which has matched a given flow and chosen to allow it.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "firewall_rule" + ] + }, + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "firewall_default_in" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc_route" + ] + }, + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "internet_gateway" + ] + }, + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, "VpcFirewallIcmpFilter": { "type": "object", "properties": { diff --git a/package-manifest.toml b/package-manifest.toml index c9fe15ddad0..57f4d37cc9a 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -638,10 +638,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "fa5f15cdcd5864161a929e2ec01534f70dfba216" +source.commit = "42f18f0491eccd16921c7b6b7fa2470160af00c2" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "9700900c62394b0858dbd4c12ac23039bed24cae8782e5153f8dfe707589c182" +source.sha256 = "f8fe09a2b5d99549b2c1869a99cedabafb27141a40ff5ec8ad43d4a371cc7873" output.type = "tarball" [package.mg-ddm] @@ -654,10 +654,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "fa5f15cdcd5864161a929e2ec01534f70dfba216" +source.commit = "42f18f0491eccd16921c7b6b7fa2470160af00c2" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "2a2b15b22b0c7604c4e5692af24515511084f2dbb17e27af4328bb4e0a8a441e" +source.sha256 = "71027b5e9476a7580e1cafd3a79e88f4d8c0ac624642c2772cdb15f3b8cb1edb" output.type = "zone" output.intermediate_only = true @@ -669,10 +669,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "fa5f15cdcd5864161a929e2ec01534f70dfba216" +source.commit = "42f18f0491eccd16921c7b6b7fa2470160af00c2" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt -source.sha256 = "b18be967a805bf4c0bf872d152ae2f58972c4f3c173a7c0f33c2475a011f1dd1" +source.sha256 = "e8fa7f59c8a03342fd469c8655fa1fd34356082fdf3f805c96d7927a7dcd2ee6" output.type = "zone" output.intermediate_only = true diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index ba0f0eacb2d..1c11be5f4c6 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -16,7 +16,7 @@ use nexus_sled_agent_shared::inventory::{ Inventory, OmicronSledConfig, SledRole, }; use omicron_common::{ - api::external::Generation, + api::external::{Flow, Generation}, api::internal::{ nexus::{DiskRuntimeState, SledVmmState}, shared::{ @@ -546,6 +546,25 @@ pub trait SledAgentApi { body: TypedBody, ) -> Result; + /// Get the IDs of all OPTE interfaces + #[endpoint { + method = GET, + path = "/network-interfaces", + }] + async fn nic_ids_list( + request_context: RequestContext, + ) -> Result>, HttpError>; + + /// Get per-flow stats currently reported by an OPTE interface. + #[endpoint { + method = GET, + path = "/network-interfaces/{nic_id}/flows", + }] + async fn nic_flows_list( + request_context: RequestContext, + path_params: Path, + ) -> Result>, HttpError>; + #[endpoint { method = GET, path = "/support/zoneadm-info", @@ -885,3 +904,9 @@ pub struct VmmIssueDiskSnapshotRequestResponse { pub struct VpcPathParam { pub vpc_id: Uuid, } + +/// Path parameters for NIC requests +#[derive(Deserialize, JsonSchema)] +pub struct NicPathParam { + pub nic_id: Uuid, +} diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 1d0c4593991..756314f35f3 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -19,7 +19,7 @@ use dropshot::{ use nexus_sled_agent_shared::inventory::{ Inventory, OmicronSledConfig, SledRole, }; -use omicron_common::api::external::Error; +use omicron_common::api::external::{Error, Flow}; use omicron_common::api::internal::nexus::{DiskRuntimeState, SledVmmState}; use omicron_common::api::internal::shared::{ ExternalIpGatewayMap, ResolvedVpcRouteSet, ResolvedVpcRouteState, @@ -45,6 +45,7 @@ use sled_diagnostics::{ SledDiagnosticsCommandHttpOutput, SledDiagnosticsQueryOutput, }; use std::collections::BTreeMap; +use uuid::Uuid; type SledApiDescription = ApiDescription; @@ -828,6 +829,22 @@ impl SledAgentApi for SledAgentImpl { Ok(HttpResponseUpdatedNoContent()) } + async fn nic_ids_list( + request_context: RequestContext, + ) -> Result>, HttpError> { + let sa = request_context.context(); + Ok(HttpResponseOk(sa.get_nic_ids())) + } + + async fn nic_flows_list( + request_context: RequestContext, + path_params: Path, + ) -> Result>, HttpError> { + let sa = request_context.context(); + let res = sa.get_nic_flows(path_params.into_inner().nic_id)?; + Ok(HttpResponseOk(res)) + } + async fn support_zoneadm_info( request_context: RequestContext, ) -> Result, HttpError> { diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 5db96e404f6..0a6245e169f 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -1350,7 +1350,6 @@ impl InstanceRunner { self.port_manager.external_ips_ensure( primary_nic.id, - primary_nic.kind, Some(self.source_nat), self.ephemeral_ip, &self.floating_ips, @@ -1366,7 +1365,6 @@ impl InstanceRunner { self.port_manager.external_ips_ensure( primary_nic.id, - primary_nic.kind, Some(self.source_nat), self.ephemeral_ip, &self.floating_ips, @@ -1414,7 +1412,6 @@ impl InstanceRunner { self.port_manager.external_ips_ensure( primary_nic.id, - primary_nic.kind, Some(self.source_nat), self.ephemeral_ip, &self.floating_ips, diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index f206e374edd..aa8975c810f 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -27,6 +27,7 @@ use dropshot::endpoint; use nexus_sled_agent_shared::inventory::Inventory; use nexus_sled_agent_shared::inventory::OmicronSledConfig; use nexus_sled_agent_shared::inventory::SledRole; +use omicron_common::api::external::Flow; use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::SledVmmState; use omicron_common::api::internal::shared::ExternalIpGatewayMap; @@ -56,6 +57,7 @@ use sled_agent_types::zone_bundle::ZoneBundleMetadata; use sled_diagnostics::SledDiagnosticsQueryOutput; use std::collections::BTreeMap; use std::sync::Arc; +use uuid::Uuid; use super::sled_agent::SledAgent; @@ -573,6 +575,19 @@ impl SledAgentApi for SledAgentSimImpl { Ok(HttpResponseUpdatedNoContent()) } + async fn nic_ids_list( + _rqctx: RequestContext, + ) -> Result>, HttpError> { + method_unimplemented() + } + + async fn nic_flows_list( + _rqctx: RequestContext, + _params: Path, + ) -> Result>, HttpError> { + method_unimplemented() + } + async fn zone_bundle_list_all( _rqctx: RequestContext, _query: Query, diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index c30bf21e980..50e8f71bca9 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -38,7 +38,9 @@ use nexus_sled_agent_shared::inventory::{ use omicron_common::address::{ Ipv6Subnet, SLED_PREFIX, get_sled_address, get_switch_zone_address, }; -use omicron_common::api::external::{ByteCount, ByteCountRangeError, Vni}; +use omicron_common::api::external::{ + ByteCount, ByteCountRangeError, Flow, Vni, +}; use omicron_common::api::internal::nexus::{DiskRuntimeState, SledVmmState}; use omicron_common::api::internal::shared::{ ExternalIpGatewayMap, HostPortConfig, RackNetworkConfig, @@ -1077,6 +1079,14 @@ impl SledAgent { Ok(()) } + pub fn get_nic_ids(&self) -> Vec { + self.inner.port_manager.get_nic_ids() + } + + pub fn get_nic_flows(&self, nic_id: Uuid) -> Result, Error> { + self.inner.port_manager.get_nic_flows(nic_id).map_err(Into::into) + } + /// Return identifiers for this sled. /// /// This is mostly used to identify timeseries data with the originating diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 7d6c2fac6a9..5ed38598b93 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="fa5f15cdcd5864161a929e2ec01534f70dfba216" +COMMIT="42f18f0491eccd16921c7b6b7fa2470160af00c2" SHA2="9146aaf60a52ecd138139708e4019e4496f330fb81a2c5a7a70cd3436a6a1318" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index e2baa1d7ba3..48f04caf9f2 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="fa5f15cdcd5864161a929e2ec01534f70dfba216" +COMMIT="42f18f0491eccd16921c7b6b7fa2470160af00c2" SHA2="7af1675e2e93e395185f8d3676db972db0123714c4c5640608f3e3570f3ce3a8" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index bd572330f26..78667be7f97 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="b18be967a805bf4c0bf872d152ae2f58972c4f3c173a7c0f33c2475a011f1dd1" -MGD_LINUX_SHA256="898bda7698ce594962b61e7c1b637f0f5ad843c1ab60eb5846fe1afdb84be8df" \ No newline at end of file +CIDL_SHA256="e8fa7f59c8a03342fd469c8655fa1fd34356082fdf3f805c96d7927a7dcd2ee6" +MGD_LINUX_SHA256="418cdb911398f652386c2642568c366812749ac6c1a6d2b973b5c38ff89366d6" \ No newline at end of file diff --git a/tools/opte_version b/tools/opte_version index 459d305eb32..5e658c277a8 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.37.386 +0.38.426