From c40d9f11b75b9206f92be93232e5d96c97866e90 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 10 Jul 2025 17:57:52 +0100 Subject: [PATCH 1/7] Initial plumbing of new ioctl format. --- .github/buildomat/jobs/deploy.sh | 2 +- Cargo.lock | 19 +++--- Cargo.toml | 8 +-- illumos-utils/src/opte/firewall_rules.rs | 2 + illumos-utils/src/opte/non_illumos.rs | 75 +++++------------------- illumos-utils/src/opte/port_manager.rs | 37 +++++++----- package-manifest.toml | 12 ++-- tools/maghemite_ddm_openapi_version | 2 +- tools/maghemite_mg_openapi_version | 2 +- tools/maghemite_mgd_checksums | 4 +- tools/opte_version | 2 +- 11 files changed, 67 insertions(+), 98 deletions(-) diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index 4c670925f54..465cf7d002a 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -136,7 +136,7 @@ z_swadm () { # only set this if you want to override the version of opte/xde installed by the # install_opte.sh script -OPTE_COMMIT="" +OPTE_COMMIT="355fc09545445beda7cd789033507f13e80cbbe7" if [[ "x$OPTE_COMMIT" != "x" ]]; then curl -sSfOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde pfexec rem_drv xde || true diff --git a/Cargo.lock b/Cargo.lock index 708fa9c473e..6aa747232d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2311,7 +2311,7 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=fa5f15cdcd5864161a929e2ec01534f70dfba216#fa5f15cdcd5864161a929e2ec01534f70dfba216" +source = "git+https://github.com/oxidecomputer/maghemite?rev=42f18f0491eccd16921c7b6b7fa2470160af00c2#42f18f0491eccd16921c7b6b7fa2470160af00c2" dependencies = [ "oxnet", "percent-encoding", @@ -4738,7 +4738,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=23cebf3b1c911f09c203f7df50cc06bf780338e5#23cebf3b1c911f09c203f7df50cc06bf780338e5" +source = "git+https://github.com/oxidecomputer/opte?rev=355fc09545445beda7cd789033507f13e80cbbe7#355fc09545445beda7cd789033507f13e80cbbe7" dependencies = [ "bitflags 2.9.0", ] @@ -5316,7 +5316,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=23cebf3b1c911f09c203f7df50cc06bf780338e5#23cebf3b1c911f09c203f7df50cc06bf780338e5" +source = "git+https://github.com/oxidecomputer/opte?rev=355fc09545445beda7cd789033507f13e80cbbe7#355fc09545445beda7cd789033507f13e80cbbe7" dependencies = [ "quote", "syn 2.0.104", @@ -5881,7 +5881,7 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?rev=fa5f15cdcd5864161a929e2ec01534f70dfba216#fa5f15cdcd5864161a929e2ec01534f70dfba216" +source = "git+https://github.com/oxidecomputer/maghemite?rev=42f18f0491eccd16921c7b6b7fa2470160af00c2#42f18f0491eccd16921c7b6b7fa2470160af00c2" dependencies = [ "anyhow", "chrono", @@ -8496,7 +8496,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=23cebf3b1c911f09c203f7df50cc06bf780338e5#23cebf3b1c911f09c203f7df50cc06bf780338e5" +source = "git+https://github.com/oxidecomputer/opte?rev=355fc09545445beda7cd789033507f13e80cbbe7#355fc09545445beda7cd789033507f13e80cbbe7" dependencies = [ "bitflags 2.9.0", "dyn-clone", @@ -8507,6 +8507,7 @@ dependencies = [ "postcard", "serde", "tabwriter", + "uuid", "version_check", "zerocopy 0.8.26", ] @@ -8514,7 +8515,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=23cebf3b1c911f09c203f7df50cc06bf780338e5#23cebf3b1c911f09c203f7df50cc06bf780338e5" +source = "git+https://github.com/oxidecomputer/opte?rev=355fc09545445beda7cd789033507f13e80cbbe7#355fc09545445beda7cd789033507f13e80cbbe7" dependencies = [ "illumos-sys-hdrs", "ingot", @@ -8522,12 +8523,13 @@ dependencies = [ "postcard", "serde", "smoltcp 0.11.0", + "uuid", ] [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=23cebf3b1c911f09c203f7df50cc06bf780338e5#23cebf3b1c911f09c203f7df50cc06bf780338e5" +source = "git+https://github.com/oxidecomputer/opte?rev=355fc09545445beda7cd789033507f13e80cbbe7#355fc09545445beda7cd789033507f13e80cbbe7" dependencies = [ "libc", "libnet", @@ -8536,6 +8538,7 @@ dependencies = [ "postcard", "serde", "thiserror 2.0.12", + "uuid", ] [[package]] @@ -8622,7 +8625,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=23cebf3b1c911f09c203f7df50cc06bf780338e5#23cebf3b1c911f09c203f7df50cc06bf780338e5" +source = "git+https://github.com/oxidecomputer/opte?rev=355fc09545445beda7cd789033507f13e80cbbe7#355fc09545445beda7cd789033507f13e80cbbe7" dependencies = [ "cfg-if", "illumos-sys-hdrs", diff --git a/Cargo.toml b/Cargo.toml index 8f9fd7c482f..bc2b58a14ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -511,8 +511,8 @@ lldp_protocol = { git = "https://github.com/oxidecomputer/lldp", package = "prot macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" newtype_derive = "0.1.6" -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "fa5f15cdcd5864161a929e2ec01534f70dfba216" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "fa5f15cdcd5864161a929e2ec01534f70dfba216" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "42f18f0491eccd16921c7b6b7fa2470160af00c2" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "42f18f0491eccd16921c7b6b7fa2470160af00c2" } multimap = "0.10.1" nexus-auth = { path = "nexus/auth" } nexus-background-task-interface = { path = "nexus/background-task-interface" } @@ -568,7 +568,7 @@ omicron-workspace-hack = "0.1.0" omicron-zone-package = "0.12.2" oxide-client = { path = "clients/oxide-client" } oxide-tokio-rt = "0.1.2" -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "23cebf3b1c911f09c203f7df50cc06bf780338e5", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "355fc09545445beda7cd789033507f13e80cbbe7", features = [ "api", "std" ] } oxlog = { path = "dev-tools/oxlog" } oxnet = "0.1.2" once_cell = "1.21.3" @@ -578,7 +578,7 @@ openapiv3 = "2.2.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "23cebf3b1c911f09c203f7df50cc06bf780338e5" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "355fc09545445beda7cd789033507f13e80cbbe7" } oso = "0.27" owo-colors = "4.2.2" oximeter = { path = "oximeter/oximeter" } diff --git a/illumos-utils/src/opte/firewall_rules.rs b/illumos-utils/src/opte/firewall_rules.rs index 948808832a6..57a87239a9e 100644 --- a/illumos-utils/src/opte/firewall_rules.rs +++ b/illumos-utils/src/opte/firewall_rules.rs @@ -22,6 +22,7 @@ use oxide_vpc::api::IpAddr; use oxide_vpc::api::Ports; use oxide_vpc::api::ProtoFilter; use oxnet::IpNet; +use uuid::Uuid; trait FromVpcFirewallRule { fn action(&self) -> FirewallAction; @@ -170,6 +171,7 @@ pub fn opte_firewall_rules( .set_protocol(proto.clone()); filters }, + stat_id: Some(Uuid::new_v4()), }) .collect::>() }) diff --git a/illumos-utils/src/opte/non_illumos.rs b/illumos-utils/src/opte/non_illumos.rs index 3624a63547b..f4e5de34d7a 100644 --- a/illumos-utils/src/opte/non_illumos.rs +++ b/illumos-utils/src/opte/non_illumos.rs @@ -17,6 +17,7 @@ use oxide_vpc::api::IpCidr; use oxide_vpc::api::ListPortsResp; use oxide_vpc::api::NoResp; use oxide_vpc::api::PortInfo; +use oxide_vpc::api::Route; use oxide_vpc::api::RouterClass; use oxide_vpc::api::RouterTarget; use oxide_vpc::api::SetExternalIpsReq; @@ -69,75 +70,29 @@ pub fn delete_all_xde_devices(log: &Logger) -> Result<(), Error> { Ok(()) } -#[derive(Debug)] +// Removes the stat ID from the Route payload. +#[derive(Debug, Eq, PartialEq)] pub(crate) struct RouteInfo { pub dest: IpCidr, pub target: RouterTarget, pub class: RouterClass, } -// NOTE: It would be nice to derive this, but `RouterTarget` and `RouterClass` -// are in OPTE, and they don't currently implement this trait. -impl PartialEq for RouteInfo { - fn eq(&self, other: &Self) -> bool { - if self.dest != other.dest { - return false; - } - match (self.class, other.class) { - (RouterClass::System, RouterClass::Custom) => return false, - (RouterClass::Custom, RouterClass::System) => return false, - (RouterClass::System, RouterClass::System) - | (RouterClass::Custom, RouterClass::Custom) => {} - } - match (self.target, other.target) { - (RouterTarget::Drop, RouterTarget::Drop) => true, - ( - RouterTarget::InternetGateway(id0), - RouterTarget::InternetGateway(id1), - ) => id0 == id1, - (RouterTarget::Ip(ip0), RouterTarget::Ip(ip1)) => ip0 == ip1, - ( - RouterTarget::VpcSubnet(cidr0), - RouterTarget::VpcSubnet(cidr1), - ) => cidr0 == cidr1, - (RouterTarget::Drop, RouterTarget::InternetGateway(_)) - | (RouterTarget::Drop, RouterTarget::Ip(_)) - | (RouterTarget::Drop, RouterTarget::VpcSubnet(_)) - | (RouterTarget::InternetGateway(_), RouterTarget::Drop) - | (RouterTarget::InternetGateway(_), RouterTarget::Ip(_)) - | (RouterTarget::InternetGateway(_), RouterTarget::VpcSubnet(_)) - | (RouterTarget::Ip(_), RouterTarget::Drop) - | (RouterTarget::Ip(_), RouterTarget::InternetGateway(_)) - | (RouterTarget::Ip(_), RouterTarget::VpcSubnet(_)) - | (RouterTarget::VpcSubnet(_), RouterTarget::Drop) - | (RouterTarget::VpcSubnet(_), RouterTarget::InternetGateway(_)) - | (RouterTarget::VpcSubnet(_), RouterTarget::Ip(_)) => false, - } - } -} - -impl RouteInfo { - #[cfg(test)] - pub fn is_system_default_ipv4_route(&self) -> bool { - let system_default_route = RouteInfo { - dest: IpCidr::Ip4(oxide_vpc::api::Ipv4Cidr::new( +#[cfg(test)] +pub(crate) fn is_system_default_ipv4_route(route: &RouteInfo) -> bool { + (route.dest, route.target, route.class) + == ( + IpCidr::Ip4(oxide_vpc::api::Ipv4Cidr::new( oxide_vpc::api::Ipv4Addr::ANY_ADDR, 0.try_into().unwrap(), )), - target: RouterTarget::InternetGateway(None), - class: RouterClass::System, - }; - *self == system_default_route - } + RouterTarget::InternetGateway(None), + RouterClass::System, + ) } -impl From<&AddRouterEntryReq> for RouteInfo { - fn from(value: &AddRouterEntryReq) -> Self { - Self { dest: value.dest, target: value.target, class: value.class } - } -} -impl From<&DelRouterEntryReq> for RouteInfo { - fn from(value: &DelRouterEntryReq) -> Self { +impl From<&Route> for RouteInfo { + fn from(value: &Route) -> Self { Self { dest: value.dest, target: value.target, class: value.class } } } @@ -246,7 +201,7 @@ impl Handle { else { anyhow::bail!("No such port '{}'", req.port_name); }; - routes.push(req.into()); + routes.push((&req.route).into()); Ok(NO_RESPONSE) } @@ -270,7 +225,7 @@ impl Handle { else { anyhow::bail!("No such port '{}'", req.port_name); }; - let req = RouteInfo::from(req); + let req = RouteInfo::from(&req.route); if let Some(index) = routes.iter().position(|rt| rt == &req) { routes.remove(index); } diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 97eba85e621..cab326716e5 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -411,10 +411,13 @@ impl PortManager { ] { for route in &routes.routes { let route = AddRouterEntryReq { - class, + route: oxide_vpc::api::Route { + dest: super::net_to_cidr(route.dest), + target: super::router_target_opte(&route.target), + class, + stat_id: Some(Uuid::new_v4()), + }, port_name: port_name.clone(), - dest: super::net_to_cidr(route.dest), - target: super::router_target_opte(&route.target), }; hdl.add_router_entry(&route)?; @@ -555,10 +558,13 @@ impl PortManager { for route in to_delete { let route = DelRouterEntryReq { - class, + route: oxide_vpc::api::Route { + dest: super::net_to_cidr(route.dest), + target: super::router_target_opte(&route.target), + class, + stat_id: None, + }, port_name: port.name().into(), - dest: super::net_to_cidr(route.dest), - target: super::router_target_opte(&route.target), }; hdl.del_router_entry(&route)?; @@ -573,10 +579,13 @@ impl PortManager { for route in to_add { let route = AddRouterEntryReq { - class, + route: oxide_vpc::api::Route { + dest: super::net_to_cidr(route.dest), + target: super::router_target_opte(&route.target), + class, + stat_id: Some(Uuid::new_v4()), + }, port_name: port.name().into(), - dest: super::net_to_cidr(route.dest), - target: super::router_target_opte(&route.target), }; hdl.add_router_entry(&route)?; @@ -966,7 +975,7 @@ impl Drop for PortTicket { #[cfg(test)] mod tests { - use crate::opte::Handle; + use crate::opte::{Handle, is_system_default_ipv4_route}; use super::{PortCreateParams, PortManager}; use macaddr::MacAddr6; @@ -1107,7 +1116,7 @@ mod tests { .unwrap() .routes .iter() - .filter(|rt| rt.is_system_default_ipv4_route()) + .filter(|rt| is_system_default_ipv4_route(&rt)) .collect::>(); assert_eq!( rt.len(), @@ -1179,7 +1188,7 @@ mod tests { .unwrap() .routes .iter() - .filter(|rt| rt.is_system_default_ipv4_route()) + .filter(|rt| is_system_default_ipv4_route(&rt)) .collect::>(); assert_eq!( rt.len(), @@ -1276,7 +1285,7 @@ mod tests { .unwrap() .routes .iter() - .filter(|rt| rt.is_system_default_ipv4_route()) + .filter(|rt| is_system_default_ipv4_route(&rt)) .collect::>(); assert_eq!( rt.len(), @@ -1345,7 +1354,7 @@ mod tests { .unwrap() .routes .iter() - .filter(|rt| rt.is_system_default_ipv4_route()) + .filter(|rt| is_system_default_ipv4_route(&rt)) .collect::>(); assert_eq!( rt.len(), diff --git a/package-manifest.toml b/package-manifest.toml index 2baa4e93fc8..bfa918e6123 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -638,10 +638,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "fa5f15cdcd5864161a929e2ec01534f70dfba216" +source.commit = "42f18f0491eccd16921c7b6b7fa2470160af00c2" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "9700900c62394b0858dbd4c12ac23039bed24cae8782e5153f8dfe707589c182" +source.sha256 = "f8fe09a2b5d99549b2c1869a99cedabafb27141a40ff5ec8ad43d4a371cc7873" output.type = "tarball" [package.mg-ddm] @@ -654,10 +654,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "fa5f15cdcd5864161a929e2ec01534f70dfba216" +source.commit = "42f18f0491eccd16921c7b6b7fa2470160af00c2" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "2a2b15b22b0c7604c4e5692af24515511084f2dbb17e27af4328bb4e0a8a441e" +source.sha256 = "71027b5e9476a7580e1cafd3a79e88f4d8c0ac624642c2772cdb15f3b8cb1edb" output.type = "zone" output.intermediate_only = true @@ -669,10 +669,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "fa5f15cdcd5864161a929e2ec01534f70dfba216" +source.commit = "42f18f0491eccd16921c7b6b7fa2470160af00c2" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt -source.sha256 = "b18be967a805bf4c0bf872d152ae2f58972c4f3c173a7c0f33c2475a011f1dd1" +source.sha256 = "e8fa7f59c8a03342fd469c8655fa1fd34356082fdf3f805c96d7927a7dcd2ee6" output.type = "zone" output.intermediate_only = true diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 7d6c2fac6a9..5ed38598b93 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="fa5f15cdcd5864161a929e2ec01534f70dfba216" +COMMIT="42f18f0491eccd16921c7b6b7fa2470160af00c2" SHA2="9146aaf60a52ecd138139708e4019e4496f330fb81a2c5a7a70cd3436a6a1318" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index e2baa1d7ba3..48f04caf9f2 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="fa5f15cdcd5864161a929e2ec01534f70dfba216" +COMMIT="42f18f0491eccd16921c7b6b7fa2470160af00c2" SHA2="7af1675e2e93e395185f8d3676db972db0123714c4c5640608f3e3570f3ce3a8" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index bd572330f26..78667be7f97 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="b18be967a805bf4c0bf872d152ae2f58972c4f3c173a7c0f33c2475a011f1dd1" -MGD_LINUX_SHA256="898bda7698ce594962b61e7c1b637f0f5ad843c1ab60eb5846fe1afdb84be8df" \ No newline at end of file +CIDL_SHA256="e8fa7f59c8a03342fd469c8655fa1fd34356082fdf3f805c96d7927a7dcd2ee6" +MGD_LINUX_SHA256="418cdb911398f652386c2642568c366812749ac6c1a6d2b973b5c38ff89366d6" \ No newline at end of file diff --git a/tools/opte_version b/tools/opte_version index 459d305eb32..5e658c277a8 100644 --- a/tools/opte_version +++ b/tools/opte_version @@ -1 +1 @@ -0.37.386 +0.38.426 From 83ddc914b5c7d1419d863995aa691471acf8ae03 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Mon, 14 Jul 2025 14:23:12 +0100 Subject: [PATCH 2/7] Sketching the APIs out. --- common/src/api/external/mod.rs | 99 ++++++++++++++ illumos-utils/src/opte/mod.rs | 1 + illumos-utils/src/opte/non_illumos.rs | 29 +++++ illumos-utils/src/opte/stat.rs | 181 ++++++++++++++++++++++++++ 4 files changed, 310 insertions(+) create mode 100644 illumos-utils/src/opte/stat.rs diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 7d415f2ba03..0eeb82bedd5 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -3474,6 +3474,105 @@ pub enum ImportExportPolicy { Allow(Vec), } +/// Information about a flow recorded on a `NetworkInterface`. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, +)] +pub struct FlowMetadata { + /// An ephemeral ID bound to this flow. + pub flow_id: Uuid, + /// The time the first packet of this flow was seen at. + pub created_at: DateTime, + /// The direction of the first packet of this flow. + pub initial_packet: Direction, + /// The flowkey (or 5-tuple) of any packets on this flow as viewed by + /// the instance. + pub internal_key: Flowkey, + /// The flowkey (or 5-tuple) of any packets on this flow as viewed by + /// the remote half. + pub external_key: Flowkey, + /// All entities responsible for allowing packets in this flow to reach the + /// instance. + pub admitted_by_in: Option>, + /// All entities responsible for allowing packets in this flow to be sent + /// by the instance. + pub admitted_by_out: Option>, + /// How any outbound packets are to be routed. + pub forwarded: Option, +} + +/// The direction of a flow or packet, with respect to its target `Instance`. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +#[serde(rename_all = "snake_case")] +pub enum Direction { + In, + Out, +} + +/// Addresses and protocol-specific information used to group packets into a flow. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +pub struct Flowkey { + pub source_address: IpAddr, + pub destination_address: IpAddr, + pub protocol: u8, + pub info: Option, +} + +/// Protocol-specific flow information. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +#[serde(rename_all = "snake_case", tag = "type", content = "value")] +pub enum ProtocolInfo { + Ports(PortProtocolInfo), + Icmp(IcmpProtocolInfo), +} + +/// A pair of ports, identifying a flow in protocols such as TCP or UDP. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +pub struct PortProtocolInfo { + source_port: u16, + destination_port: u16, +} + +/// Message types information carried by ICMP. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +pub struct IcmpProtocolInfo { + r#type: u8, + code: u8, +} + +/// How the remote half of a flow is reached. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +#[serde(rename_all = "snake_case")] +pub enum ForwardClass { + VpcLocal, + External, +} + +/// A control-plane object which has matched a given flow and chosen to +/// allow it. +#[derive( + Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, +)] +#[serde(rename_all = "snake_case", tag = "type", content = "value")] +pub enum VpcEntity { + FirewallRule(Uuid), + FirewallDefault, + VpcRoute(Uuid), + InternetGateway(Uuid), +} + /// Use instead of Option in API request body structs to get a field that can /// be null (parsed as `None`) but is not optional. Unlike Option, Nullable /// will fail to parse if the key is not present. The JSON Schema in the diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index 9f5c25462c5..47036fca273 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -15,6 +15,7 @@ cfg_if::cfg_if! { mod firewall_rules; mod port; mod port_manager; +mod stat; pub use firewall_rules::opte_firewall_rules; use ipnetwork::IpNetwork; diff --git a/illumos-utils/src/opte/non_illumos.rs b/illumos-utils/src/opte/non_illumos.rs index f4e5de34d7a..63a7f31b893 100644 --- a/illumos-utils/src/opte/non_illumos.rs +++ b/illumos-utils/src/opte/non_illumos.rs @@ -11,7 +11,10 @@ use oxide_vpc::api::ClearVirt2PhysReq; use oxide_vpc::api::DelRouterEntryReq; use oxide_vpc::api::DhcpCfg; use oxide_vpc::api::Direction; +use oxide_vpc::api::DumpFlowStatResp; +use oxide_vpc::api::DumpRootStatResp; use oxide_vpc::api::DumpVirt2PhysResp; +use oxide_vpc::api::InnerFlowId; use oxide_vpc::api::IpCfg; use oxide_vpc::api::IpCidr; use oxide_vpc::api::ListPortsResp; @@ -25,6 +28,8 @@ use oxide_vpc::api::SetFwRulesReq; use oxide_vpc::api::SetVirt2PhysReq; use oxide_vpc::api::VpcCfg; use slog::Logger; +use uuid::Uuid; +use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::hash_map::Entry; use std::net::IpAddr; @@ -258,6 +263,30 @@ impl Handle { unimplemented!("Not yet used in tests") } + /// Request the current state of some (or all) root stats contained + /// in a port. + /// + /// An empty `stat_ids` will request all present stats. + pub fn dump_root_stats( + &self, + _port_name: &str, + _stat_ids: impl IntoIterator, + ) -> Result { + Ok(DumpRootStatResp { root_stats: BTreeMap::new() }) + } + + /// Request the current state of some (or all) flow stats contained + /// in a port. + /// + /// An empty `flow_keys` will request all present flows. + pub fn dump_flow_stats( + &self, + _port_name: &str, + _flow_keys: impl IntoIterator, + ) -> Result, Error> { + Ok(DumpFlowStatResp { flow_stats: BTreeMap::new() }) + } + /// List ports on the current system. #[allow(dead_code)] pub(crate) fn list_ports(&self) -> Result { diff --git a/illumos-utils/src/opte/stat.rs b/illumos-utils/src/opte/stat.rs new file mode 100644 index 00000000000..e3b634ed825 --- /dev/null +++ b/illumos-utils/src/opte/stat.rs @@ -0,0 +1,181 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Flow and root stat tracking for individual OPTE ports. + +use std::{collections::{hash_map::Entry, HashMap}, sync::{atomic::{AtomicBool, Ordering}, Arc, RwLock}, time::{Duration, Instant}}; +use oxide_vpc::api::{FlowPair, FlowStat, FullCounter, InnerFlowId}; +use slog::Logger; +use tokio::time::MissedTickBehavior; +use uuid::Uuid; + +use super::Handle; + +const FLOW_STAT_REFRESH_INTERVAL: Duration = Duration::from_secs(1); +const ROOT_STAT_REFRESH_INTERVAL: Duration = Duration::from_secs(9); +const PRUNE_INTERVAL: Duration = Duration::from_secs(5); +const PRUNE_AGE: Duration = Duration::from_secs(10); + +pub struct PortStats { + shared: Arc, +} + +impl Drop for PortStats { + fn drop(&mut self) { + self.shared.task_quit.store(true, Ordering::Relaxed); + } +} + +impl PortStats { + pub async fn new(name: impl Into, log: Logger) -> Self { + let shared = Arc::new(PortStatsShared { + name: name.into(), + task_quit: false.into(), + state: Default::default(), + log, + }); + + tokio::spawn(run_port_stat(Arc::clone(&shared))); + + Self { shared } + } +} + +struct PortStatsShared { + name: String, + state: RwLock, + task_quit: AtomicBool, + log: Logger, +} + +impl PortStatsShared { + fn collect_flows(&self) -> Result<(), anyhow::Error> { + let new_stats = { + let hdl = Handle::new()?; + hdl.dump_flow_stats(&self.name, [])? + }; + let now = Instant::now(); + + let mut state = self.state.write().unwrap(); + for (flowid, stat) in new_stats.flow_stats { + match state.flows.entry(flowid) { + Entry::Occupied(mut val) => { + let val = val.get_mut(); + let elapsed = now.duration_since(val.hit_at); + val.body.update(stat, &elapsed); + val.hit_at = now; + + }, + Entry::Vacant(vacant) => { + vacant.insert(Timed { + hit_at: now, + body: FlowSnapshot::new(stat), + }); + }, + } + } + + Ok(()) + } + + fn prune(&self) { + let now = Instant::now(); + let mut state = self.state.write().unwrap(); + + state.flows.retain(|_, v| now.duration_since(v.hit_at) <= PRUNE_AGE); + state.roots.retain(|_, v| now.duration_since(v.hit_at) <= PRUNE_AGE); + } +} + +#[derive(Debug, PartialEq, PartialOrd, Ord, Hash, Eq)] +struct Timed { + hit_at: Instant, + body: S, +} + +#[derive(Default)] +struct State { + flows: HashMap>, + roots: HashMap>, + label_map: HashMap, +} + +async fn run_port_stat(state: Arc) { + let mut flow_collect = tokio::time::interval(FLOW_STAT_REFRESH_INTERVAL); + let mut root_collect = tokio::time::interval(ROOT_STAT_REFRESH_INTERVAL); + let mut prune = tokio::time::interval(PRUNE_INTERVAL); + flow_collect.set_missed_tick_behavior(MissedTickBehavior::Skip); + root_collect.set_missed_tick_behavior(MissedTickBehavior::Skip); + prune.set_missed_tick_behavior(MissedTickBehavior::Skip); + + loop { + if state.task_quit.load(Ordering::Relaxed) { + return; + } + + tokio::select! { + _ = flow_collect.tick() => { + state.collect_flows(); + }, + _ = root_collect.tick() => {}, + _ = prune.tick() => { + state.prune(); + }, + } + } +} + +pub enum FlowLabel { + FirewallRule(FirewallLabel), + Route(Uuid), + Destination(DeliveryClass), + Builtin(VpcBuiltinLabel) +} + +pub enum FirewallLabel { + Rule(Uuid), + Default, +} + +pub enum DeliveryClass { + Vpc, + VpcPeer, + Internet, +} + +pub enum VpcBuiltinLabel { + // TODO +} + +#[derive(Debug)] +pub struct FlowSnapshot { + pub last: FlowStat, + + pub in_packets_per_sec: f64, + pub in_bytes_per_sec: f64, + pub out_packets_per_sec: f64, + pub out_bytes_per_sec: f64, +} + +impl FlowSnapshot { + fn new(stat: FlowStat) -> Self { + Self { + last: stat, + in_packets_per_sec: 0.0, + in_bytes_per_sec: 0.0, + out_packets_per_sec: 0.0, + out_bytes_per_sec: 0.0, + } + } + + fn update(&mut self, stat: FlowStat, elapsed: &Duration) { + let elapsed = elapsed.as_secs_f64(); + self.in_packets_per_sec = (stat.stats.pkts_in - self.last.stats.pkts_in) as f64 / elapsed; + self.in_bytes_per_sec = (stat.stats.bytes_in - self.last.stats.bytes_in) as f64 / elapsed; + self.out_packets_per_sec = (stat.stats.pkts_out - self.last.stats.pkts_out) as f64 / elapsed; + self.out_bytes_per_sec = (stat.stats.bytes_out - self.last.stats.bytes_out) as f64 / elapsed; + self.last = stat; + } +} + From fb92f22bc9d152f2c4ec8532e9cae52f5fc28135 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Mon, 14 Jul 2025 17:29:08 +0100 Subject: [PATCH 3/7] Progress. --- common/src/api/external/mod.rs | 22 ++- illumos-utils/src/opte/non_illumos.rs | 2 +- illumos-utils/src/opte/stat.rs | 228 +++++++++++++++++++++++--- 3 files changed, 227 insertions(+), 25 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 0eeb82bedd5..20c1bdf5a43 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -3474,10 +3474,25 @@ pub enum ImportExportPolicy { Allow(Vec), } +/// A packet flow recorded on a `NetworkInterface`. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +pub struct Flow { + pub metadata: FlowMetadata, + pub in_stat: FlowStat, + pub out_stat: FlowStat, +} + +/// A packet flow recorded on a `NetworkInterface`. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq)] +pub struct FlowStat { + pub packets: u64, + pub bytes: u64, + pub packet_rate: f64, + pub byte_rate: f64, +} + /// Information about a flow recorded on a `NetworkInterface`. -#[derive( - Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, -)] +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq)] pub struct FlowMetadata { /// An ephemeral ID bound to this flow. pub flow_id: Uuid, @@ -3548,6 +3563,7 @@ pub struct PortProtocolInfo { pub struct IcmpProtocolInfo { r#type: u8, code: u8, + id: Option, } /// How the remote half of a flow is reached. diff --git a/illumos-utils/src/opte/non_illumos.rs b/illumos-utils/src/opte/non_illumos.rs index 63a7f31b893..9190d67c798 100644 --- a/illumos-utils/src/opte/non_illumos.rs +++ b/illumos-utils/src/opte/non_illumos.rs @@ -28,13 +28,13 @@ use oxide_vpc::api::SetFwRulesReq; use oxide_vpc::api::SetVirt2PhysReq; use oxide_vpc::api::VpcCfg; use slog::Logger; -use uuid::Uuid; use std::collections::BTreeMap; use std::collections::HashMap; use std::collections::hash_map::Entry; use std::net::IpAddr; use std::sync::Mutex; use std::sync::OnceLock; +use uuid::Uuid; type OpteError = anyhow::Error; diff --git a/illumos-utils/src/opte/stat.rs b/illumos-utils/src/opte/stat.rs index e3b634ed825..78efc0587d4 100644 --- a/illumos-utils/src/opte/stat.rs +++ b/illumos-utils/src/opte/stat.rs @@ -4,19 +4,34 @@ //! Flow and root stat tracking for individual OPTE ports. -use std::{collections::{hash_map::Entry, HashMap}, sync::{atomic::{AtomicBool, Ordering}, Arc, RwLock}, time::{Duration, Instant}}; -use oxide_vpc::api::{FlowPair, FlowStat, FullCounter, InnerFlowId}; +use super::Handle; +use omicron_common::api::external::{ + self, Flow, FlowMetadata, FlowStat as ExternalFlowStat, +}; +use oxide_vpc::api::{ + Direction, FlowPair, FlowStat, FullCounter, InnerFlowId, Protocol, +}; use slog::Logger; +use std::{ + collections::{HashMap, hash_map::Entry}, + sync::{ + Arc, RwLock, + atomic::{AtomicBool, Ordering}, + }, + time::{Duration, Instant}, +}; use tokio::time::MissedTickBehavior; use uuid::Uuid; -use super::Handle; +// TODO: Controlplane needs to tell us the UUIDs of all routes, firewall rules. const FLOW_STAT_REFRESH_INTERVAL: Duration = Duration::from_secs(1); const ROOT_STAT_REFRESH_INTERVAL: Duration = Duration::from_secs(9); const PRUNE_INTERVAL: Duration = Duration::from_secs(5); const PRUNE_AGE: Duration = Duration::from_secs(10); +type UniqueFlow = (InnerFlowId, InnerFlowId, u64); + pub struct PortStats { shared: Arc, } @@ -40,6 +55,97 @@ impl PortStats { Self { shared } } + + /// Gather all flow stats when requested by a client. + pub fn flow_stats(&self) -> Vec { + let mut out = HashMap::new(); + + { + let state = self.shared.state.read().unwrap(); + for (flowid, Timed { body, .. }) in &state.flows { + let unique = unique_flow(&flowid, &body.last); + let Some(ufid) = state.flow_instances.get(&unique) else { + slog::error!(&self.shared.log, "Hi?"); + continue; + }; + + let mut forwarded = None; + let bases = body + .last + .bases + .iter() + .filter_map(|v| match state.label_map.get(v) { + Some(FlowLabel::Destination(v)) => { + forwarded = Some(v.clone()); + None + } + Some(FlowLabel::Entity(v)) => Some(v.clone()), + _ => None, + }) + .collect(); + + match out.entry(unique) { + Entry::Vacant(val) => { + // Use the stats from the first entry encountered. + let (in_key, out_key, ad_in, ad_out) = match body + .last + .dir + { + Direction::In => { + (*flowid, body.last.partner, Some(bases), None) + } + Direction::Out => { + (body.last.partner, *flowid, None, Some(bases)) + } + }; + val.insert(Flow { + metadata: FlowMetadata { + flow_id: ufid.body, + // TODO: need to correlate timestamps between + // kmod and here?! + created_at: Default::default(), + initial_packet: direction(body.last.first_dir), + internal_key: flowkey(out_key), + external_key: flowkey(in_key), + admitted_by_in: ad_in, + admitted_by_out: ad_out, + forwarded, + }, + in_stat: ExternalFlowStat { + packets: body.last.stats.pkts_in, + bytes: body.last.stats.bytes_in, + packet_rate: body.in_packets_per_sec, + byte_rate: body.in_bytes_per_sec, + }, + out_stat: ExternalFlowStat { + packets: body.last.stats.pkts_out, + bytes: body.last.stats.bytes_out, + packet_rate: body.out_packets_per_sec, + byte_rate: body.out_bytes_per_sec, + }, + }); + } + Entry::Occupied(mut val) => { + // The second half fills in the remaining metadata. + let val = val.get_mut(); + match body.last.dir { + Direction::In => { + val.metadata.admitted_by_in = Some(bases) + } + Direction::Out => { + val.metadata.admitted_by_out = Some(bases) + } + } + } + } + } + } + + out.into_values().collect() + } + + // TODO: want `fn root_stats`, need to be able to pull back up into + // oximeter in articular. } struct PortStatsShared { @@ -59,32 +165,51 @@ impl PortStatsShared { let mut state = self.state.write().unwrap(); for (flowid, stat) in new_stats.flow_stats { + let unique = unique_flow(&flowid, &stat); + state + .flow_instances + .entry(unique) + .or_insert_with(|| Timed { hit_at: now, body: Uuid::new_v4() }) + .hit_at = now; + match state.flows.entry(flowid) { Entry::Occupied(mut val) => { let val = val.get_mut(); let elapsed = now.duration_since(val.hit_at); val.body.update(stat, &elapsed); val.hit_at = now; - - }, + } Entry::Vacant(vacant) => { vacant.insert(Timed { hit_at: now, body: FlowSnapshot::new(stat), }); - }, + } } } Ok(()) } + fn collect_roots(&self) -> Result<(), anyhow::Error> { + let new_stats = { + let hdl = Handle::new()?; + hdl.dump_flow_stats(&self.name, [])? + }; + let now = Instant::now(); + + Ok(()) + } + fn prune(&self) { let now = Instant::now(); let mut state = self.state.write().unwrap(); state.flows.retain(|_, v| now.duration_since(v.hit_at) <= PRUNE_AGE); state.roots.retain(|_, v| now.duration_since(v.hit_at) <= PRUNE_AGE); + state + .flow_instances + .retain(|_, v| now.duration_since(v.hit_at) <= PRUNE_AGE); } } @@ -99,6 +224,7 @@ struct State { flows: HashMap>, roots: HashMap>, label_map: HashMap, + flow_instances: HashMap>, } async fn run_port_stat(state: Arc) { @@ -118,7 +244,9 @@ async fn run_port_stat(state: Arc) { _ = flow_collect.tick() => { state.collect_flows(); }, - _ = root_collect.tick() => {}, + _ = root_collect.tick() => { + state.collect_roots(); + }, _ = prune.tick() => { state.prune(); }, @@ -127,10 +255,9 @@ async fn run_port_stat(state: Arc) { } pub enum FlowLabel { - FirewallRule(FirewallLabel), - Route(Uuid), - Destination(DeliveryClass), - Builtin(VpcBuiltinLabel) + Entity(external::VpcEntity), + Destination(external::ForwardClass), + Builtin(VpcBuiltinLabel), } pub enum FirewallLabel { @@ -138,12 +265,6 @@ pub enum FirewallLabel { Default, } -pub enum DeliveryClass { - Vpc, - VpcPeer, - Internet, -} - pub enum VpcBuiltinLabel { // TODO } @@ -162,6 +283,7 @@ impl FlowSnapshot { fn new(stat: FlowStat) -> Self { Self { last: stat, + in_packets_per_sec: 0.0, in_bytes_per_sec: 0.0, out_packets_per_sec: 0.0, @@ -171,11 +293,75 @@ impl FlowSnapshot { fn update(&mut self, stat: FlowStat, elapsed: &Duration) { let elapsed = elapsed.as_secs_f64(); - self.in_packets_per_sec = (stat.stats.pkts_in - self.last.stats.pkts_in) as f64 / elapsed; - self.in_bytes_per_sec = (stat.stats.bytes_in - self.last.stats.bytes_in) as f64 / elapsed; - self.out_packets_per_sec = (stat.stats.pkts_out - self.last.stats.pkts_out) as f64 / elapsed; - self.out_bytes_per_sec = (stat.stats.bytes_out - self.last.stats.bytes_out) as f64 / elapsed; + self.in_packets_per_sec = + (stat.stats.pkts_in - self.last.stats.pkts_in) as f64 / elapsed; + self.in_bytes_per_sec = + (stat.stats.bytes_in - self.last.stats.bytes_in) as f64 / elapsed; + self.out_packets_per_sec = + (stat.stats.pkts_out - self.last.stats.pkts_out) as f64 / elapsed; + self.out_bytes_per_sec = + (stat.stats.bytes_out - self.last.stats.bytes_out) as f64 / elapsed; self.last = stat; } } +fn unique_flow(id: &InnerFlowId, stat: &FlowStat) -> UniqueFlow { + let (in_fid, out_fid) = match stat.dir { + Direction::In => (*id, stat.partner), + Direction::Out => (stat.partner, *id), + }; + + (in_fid, out_fid, stat.stats.created_at) +} + +fn direction(dir: Direction) -> external::Direction { + match dir { + Direction::In => external::Direction::In, + Direction::Out => external::Direction::Out, + } +} + +fn flowkey(id: InnerFlowId) -> external::Flowkey { + let (source_address, destination_address) = match id.addrs { + oxide_vpc::api::AddrPair::V4 { src, dst } => { + (src.bytes().into(), dst.bytes().into()) + } + oxide_vpc::api::AddrPair::V6 { src, dst } => { + (src.bytes().into(), dst.bytes().into()) + } + }; + let info = id.l4_info().map(|v| match v { + oxide_vpc::api::L4Info::Ports(port_info) => { + external::ProtocolInfo::Ports(external::PortProtocolInfo { + source_port: port_info.src_port, + destination_port: port_info.dst_port, + }) + } + oxide_vpc::api::L4Info::Icmpv4(icmp_info) => { + external::ProtocolInfo::Icmp(external::IcmpProtocolInfo { + r#type: icmp_info.ty, + code: icmp_info.code, + id: match icmp_info.ty { + 0 | 8 => Some(icmp_info.id), + _ => None, + }, + }) + } + oxide_vpc::api::L4Info::Icmpv6(icmp_info) => { + external::ProtocolInfo::Icmp(external::IcmpProtocolInfo { + r#type: icmp_info.ty, + code: icmp_info.code, + id: match icmp_info.ty { + 128 | 129 => Some(icmp_info.id), + _ => None, + }, + }) + } + }); + external::Flowkey { + source_address, + destination_address, + protocol: id.proto, + info, + } +} From 0850b72c60d75addd770b47e18e021fe32a432cc Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Mon, 14 Jul 2025 18:12:06 +0100 Subject: [PATCH 4/7] Further wiring of dubious value. (?) --- common/src/api/external/mod.rs | 10 +++++----- illumos-utils/src/opte/port.rs | 5 +++++ illumos-utils/src/opte/port_manager.rs | 2 ++ illumos-utils/src/opte/stat.rs | 25 ++++++++++++++----------- 4 files changed, 26 insertions(+), 16 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index 20c1bdf5a43..e60a6d15617 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -3552,8 +3552,8 @@ pub enum ProtocolInfo { Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, )] pub struct PortProtocolInfo { - source_port: u16, - destination_port: u16, + pub source_port: u16, + pub destination_port: u16, } /// Message types information carried by ICMP. @@ -3561,9 +3561,9 @@ pub struct PortProtocolInfo { Clone, Debug, Deserialize, JsonSchema, Serialize, PartialEq, Eq, Hash, )] pub struct IcmpProtocolInfo { - r#type: u8, - code: u8, - id: Option, + pub r#type: u8, + pub code: u8, + pub id: Option, } /// How the remote half of a flow is reached. diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs index 6b4c5b8b054..43743a9b36d 100644 --- a/illumos-utils/src/opte/port.rs +++ b/illumos-utils/src/opte/port.rs @@ -15,6 +15,8 @@ use oxnet::IpNet; use std::net::IpAddr; use std::sync::Arc; +use super::stat::PortStats; + #[derive(Debug)] pub struct PortData { /// Name of the port as identified by OPTE @@ -31,6 +33,9 @@ pub struct PortData { pub(crate) subnet: IpNet, /// Information about the virtual gateway, aka OPTE pub(crate) gateway: Gateway, + + /// Periodically polled stats from this port. + pub(crate) stats: PortStats, } #[derive(Debug)] diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index cab326716e5..d12f69ad8f5 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -12,6 +12,7 @@ use crate::opte::Port; use crate::opte::Vni; use crate::opte::opte_firewall_rules; use crate::opte::port::PortData; +use crate::opte::stat::PortStats; use ipnetwork::IpNetwork; use macaddr::MacAddr6; use omicron_common::api::external; @@ -316,6 +317,7 @@ impl PortManager { vni, subnet: nic.subnet, gateway, + stats: PortStats::new(&port_name, self.inner.log.clone()), }); let old = ports.insert((nic.id, nic.kind), port.clone()); assert!( diff --git a/illumos-utils/src/opte/stat.rs b/illumos-utils/src/opte/stat.rs index 78efc0587d4..89971a79cdd 100644 --- a/illumos-utils/src/opte/stat.rs +++ b/illumos-utils/src/opte/stat.rs @@ -8,9 +8,7 @@ use super::Handle; use omicron_common::api::external::{ self, Flow, FlowMetadata, FlowStat as ExternalFlowStat, }; -use oxide_vpc::api::{ - Direction, FlowPair, FlowStat, FullCounter, InnerFlowId, Protocol, -}; +use oxide_vpc::api::{Direction, FlowStat, FullCounter, InnerFlowId}; use slog::Logger; use std::{ collections::{HashMap, hash_map::Entry}, @@ -32,6 +30,7 @@ const PRUNE_AGE: Duration = Duration::from_secs(10); type UniqueFlow = (InnerFlowId, InnerFlowId, u64); +#[derive(Debug)] pub struct PortStats { shared: Arc, } @@ -43,7 +42,7 @@ impl Drop for PortStats { } impl PortStats { - pub async fn new(name: impl Into, log: Logger) -> Self { + pub fn new(name: impl Into, log: Logger) -> Self { let shared = Arc::new(PortStatsShared { name: name.into(), task_quit: false.into(), @@ -148,6 +147,7 @@ impl PortStats { // oximeter in articular. } +#[derive(Debug)] struct PortStatsShared { name: String, state: RwLock, @@ -194,10 +194,15 @@ impl PortStatsShared { fn collect_roots(&self) -> Result<(), anyhow::Error> { let new_stats = { let hdl = Handle::new()?; - hdl.dump_flow_stats(&self.name, [])? + hdl.dump_root_stats(&self.name, [])? }; let now = Instant::now(); + let mut state = self.state.write().unwrap(); + for (id, stats) in new_stats.root_stats { + state.roots.insert(id, Timed { hit_at: now, body: stats }); + } + Ok(()) } @@ -219,7 +224,7 @@ struct Timed { body: S, } -#[derive(Default)] +#[derive(Default, Debug)] struct State { flows: HashMap>, roots: HashMap>, @@ -240,6 +245,7 @@ async fn run_port_stat(state: Arc) { return; } + // TODO: log on error. tokio::select! { _ = flow_collect.tick() => { state.collect_flows(); @@ -254,17 +260,14 @@ async fn run_port_stat(state: Arc) { } } +#[derive(Debug)] pub enum FlowLabel { Entity(external::VpcEntity), Destination(external::ForwardClass), Builtin(VpcBuiltinLabel), } -pub enum FirewallLabel { - Rule(Uuid), - Default, -} - +#[derive(Debug)] pub enum VpcBuiltinLabel { // TODO } From 37c957845c37f54f9def2bf757a46983e1eb4c65 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Tue, 15 Jul 2025 20:05:40 +0100 Subject: [PATCH 5/7] Iterating! --- common/src/api/external/mod.rs | 2 +- illumos-utils/src/opte/illumos.rs | 8 +-- illumos-utils/src/opte/non_illumos.rs | 9 ++- illumos-utils/src/opte/port.rs | 16 ++++- illumos-utils/src/opte/port_manager.rs | 54 ++++++++++------- illumos-utils/src/opte/stat.rs | 84 +++++++++++++++++++++++--- sled-agent/api/src/lib.rs | 27 ++++++++- sled-agent/src/http_entrypoints.rs | 19 +++++- sled-agent/src/instance.rs | 3 - sled-agent/src/sim/http_entrypoints.rs | 15 +++++ sled-agent/src/sled_agent.rs | 12 +++- 11 files changed, 202 insertions(+), 47 deletions(-) diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index e60a6d15617..402446c2eb3 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -3584,7 +3584,7 @@ pub enum ForwardClass { #[serde(rename_all = "snake_case", tag = "type", content = "value")] pub enum VpcEntity { FirewallRule(Uuid), - FirewallDefault, + FirewallDefaultIn, VpcRoute(Uuid), InternetGateway(Uuid), } diff --git a/illumos-utils/src/opte/illumos.rs b/illumos-utils/src/opte/illumos.rs index 3d1f0c8c707..8c102a33338 100644 --- a/illumos-utils/src/opte/illumos.rs +++ b/illumos-utils/src/opte/illumos.rs @@ -46,11 +46,11 @@ pub enum Error { #[error("Invalid IP configuration for port")] InvalidPortIpConfig, - #[error("Tried to release non-existent port ({0}, {1:?})")] - ReleaseMissingPort(uuid::Uuid, NetworkInterfaceKind), + #[error("Tried to release non-existent port ({0})")] + ReleaseMissingPort(uuid::Uuid), - #[error("Tried to update external IPs on non-existent port ({0}, {1:?})")] - ExternalIpUpdateMissingPort(uuid::Uuid, NetworkInterfaceKind), + #[error("Tried to update external IPs on non-existent port ({0})")] + ExternalIpUpdateMissingPort(uuid::Uuid), #[error("Could not find Primary NIC")] NoPrimaryNic, diff --git a/illumos-utils/src/opte/non_illumos.rs b/illumos-utils/src/opte/non_illumos.rs index 9190d67c798..5a4cf83c5ec 100644 --- a/illumos-utils/src/opte/non_illumos.rs +++ b/illumos-utils/src/opte/non_illumos.rs @@ -5,7 +5,6 @@ //! Mock / dummy versions of the OPTE module, for non-illumos platforms use crate::addrobj::AddrObject; -use omicron_common::api::internal::shared::NetworkInterfaceKind; use oxide_vpc::api::AddRouterEntryReq; use oxide_vpc::api::ClearVirt2PhysReq; use oxide_vpc::api::DelRouterEntryReq; @@ -46,11 +45,11 @@ pub enum Error { #[error("Invalid IP configuration for port")] InvalidPortIpConfig, - #[error("Tried to release non-existent port ({0}, {1:?})")] - ReleaseMissingPort(uuid::Uuid, NetworkInterfaceKind), + #[error("Tried to release non-existent port ({0})")] + ReleaseMissingPort(uuid::Uuid), - #[error("Tried to update external IPs on non-existent port ({0}, {1:?})")] - ExternalIpUpdateMissingPort(uuid::Uuid, NetworkInterfaceKind), + #[error("Tried to update external IPs on non-existent port ({0})")] + ExternalIpUpdateMissingPort(uuid::Uuid), #[error("Could not find Primary NIC")] NoPrimaryNic, diff --git a/illumos-utils/src/opte/port.rs b/illumos-utils/src/opte/port.rs index 43743a9b36d..68f2f432a0c 100644 --- a/illumos-utils/src/opte/port.rs +++ b/illumos-utils/src/opte/port.rs @@ -9,6 +9,7 @@ use crate::opte::Handle; use crate::opte::Vni; use macaddr::MacAddr6; use omicron_common::api::external; +use omicron_common::api::internal::shared::NetworkInterfaceKind; use omicron_common::api::internal::shared::RouterId; use omicron_common::api::internal::shared::RouterKind; use oxnet::IpNet; @@ -17,6 +18,9 @@ use std::sync::Arc; use super::stat::PortStats; +// TODO: This should probably comprise `NetworkInterface`, to enable more +// unified management/querying of state across Instance/Zone/Port. That +// would require some large changes to `InstanceRunner`. #[derive(Debug)] pub struct PortData { /// Name of the port as identified by OPTE @@ -29,11 +33,17 @@ pub struct PortData { pub(crate) slot: u8, /// Geneve VNI for the VPC pub(crate) vni: Vni, - /// Subnet the port belong to within the VPC. + /// Subnet the port belongs to within the VPC. pub(crate) subnet: IpNet, /// Information about the virtual gateway, aka OPTE pub(crate) gateway: Gateway, + // TODO: Will be used in later rootstat -> VPC UUID hierarchy for + // oximeter. + #[expect(unused)] + /// The type and ID of the client this NIC is bound to. + pub(crate) parent: NetworkInterfaceKind, + /// Periodically polled stats from this port. pub(crate) stats: PortStats, } @@ -114,6 +124,10 @@ impl Port { self.inner.slot } + pub fn stats(&self) -> &PortStats { + &self.inner.stats + } + pub fn system_router_key(&self) -> RouterId { // Unwrap safety: both of these VNI types represent validated u24s. let vni = external::Vni::try_from(self.vni().as_u32()).unwrap(); diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index d12f69ad8f5..41772385559 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -83,7 +83,7 @@ struct PortManagerInner { /// Map of all ports, keyed on the interface Uuid and its kind /// (which includes the Uuid of the parent instance or service) - ports: Mutex>, + ports: Mutex>, /// Map of all current resolved routes. routes: Mutex>, @@ -308,7 +308,7 @@ impl PortManager { }; let (port, ticket) = { let mut ports = self.inner.ports.lock().unwrap(); - let ticket = PortTicket::new(nic.id, nic.kind, self.inner.clone()); + let ticket = PortTicket::new(nic.id, self.inner.clone()); let port = Port::new(PortData { name: port_name.clone(), ip: nic.ip, @@ -317,9 +317,10 @@ impl PortManager { vni, subnet: nic.subnet, gateway, + parent: nic.kind, stats: PortStats::new(&port_name, self.inner.log.clone()), }); - let old = ports.insert((nic.id, nic.kind), port.clone()); + let old = ports.insert(nic.id, port.clone()); assert!( old.is_none(), "Duplicate OPTE port detected: interface_id = {}, kind = {:?}", @@ -623,15 +624,14 @@ impl PortManager { pub fn external_ips_ensure( &self, nic_id: Uuid, - nic_kind: NetworkInterfaceKind, source_nat: Option, ephemeral_ip: Option, floating_ips: &[IpAddr], ) -> Result<(), Error> { let ports = self.inner.ports.lock().unwrap(); - let port = ports.get(&(nic_id, nic_kind)).ok_or_else(|| { - Error::ExternalIpUpdateMissingPort(nic_id, nic_kind) - })?; + let port = ports + .get(&nic_id) + .ok_or_else(|| Error::ExternalIpUpdateMissingPort(nic_id))?; self.external_ips_ensure_port( port, @@ -780,9 +780,9 @@ impl PortManager { // We update VPC rules as a set so grab only // the relevant ports using the VPC's VNI. let vpc_ports = ports - .iter() - .filter(|((_, _), port)| u32::from(vni) == u32::from(*port.vni())); - for ((_, _), port) in vpc_ports { + .values() + .filter(|port| u32::from(vni) == u32::from(*port.vni())); + for port in vpc_ports { let rules = opte_firewall_rules(rules, port.vni(), port.mac()); let port_name = port.name().to_string(); info!( @@ -799,6 +799,24 @@ impl PortManager { Ok(()) } + pub fn get_nic_ids(&self) -> Vec { + let ports = self.inner.ports.lock().unwrap(); + + ports.keys().copied().collect() + } + + pub fn get_nic_flows( + &self, + nic_id: Uuid, + ) -> Result, Error> { + let ports = self.inner.ports.lock().unwrap(); + let port = ports + .get(&nic_id) + .ok_or_else(|| Error::ExternalIpUpdateMissingPort(nic_id))?; + + Ok(port.stats().flow_stats()) + } + pub fn list_virtual_nics( &self, ) -> Result, Error> { @@ -887,7 +905,6 @@ impl PortManager { pub struct PortTicket { id: Uuid, - kind: NetworkInterfaceKind, manager: Arc, } @@ -895,31 +912,25 @@ impl std::fmt::Debug for PortTicket { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { f.debug_struct("PortTicket") .field("id", &self.id) - .field("kind", &self.kind) .field("manager", &"{ .. }") .finish() } } impl PortTicket { - fn new( - id: Uuid, - kind: NetworkInterfaceKind, - manager: Arc, - ) -> Self { - Self { id, kind, manager } + fn new(id: Uuid, manager: Arc) -> Self { + Self { id, manager } } fn release_inner(&mut self) -> Result<(), Error> { let mut ports = self.manager.ports.lock().unwrap(); - let Some(port) = ports.remove(&(self.id, self.kind)) else { + let Some(port) = ports.remove(&self.id) else { error!( self.manager.log, "Tried to release non-existent port"; "id" => ?&self.id, - "kind" => ?&self.kind, ); - return Err(Error::ReleaseMissingPort(self.id, self.kind)); + return Err(Error::ReleaseMissingPort(self.id)); }; drop(ports); @@ -948,7 +959,6 @@ impl PortTicket { self.manager.log, "Removed OPTE port from manager"; "id" => ?&self.id, - "kind" => ?&self.kind, "port" => ?&port, ); Ok(()) diff --git a/illumos-utils/src/opte/stat.rs b/illumos-utils/src/opte/stat.rs index 89971a79cdd..e1d4b505311 100644 --- a/illumos-utils/src/opte/stat.rs +++ b/illumos-utils/src/opte/stat.rs @@ -8,12 +8,15 @@ use super::Handle; use omicron_common::api::external::{ self, Flow, FlowMetadata, FlowStat as ExternalFlowStat, }; -use oxide_vpc::api::{Direction, FlowStat, FullCounter, InnerFlowId}; +use oxide_vpc::api::{ + Direction, FlowStat, FullCounter, InnerFlowId, stat as vpc_stat, +}; use slog::Logger; +use slog_error_chain::InlineErrorChain; use std::{ collections::{HashMap, hash_map::Entry}, sync::{ - Arc, RwLock, + Arc, LazyLock, RwLock, atomic::{AtomicBool, Ordering}, }, time::{Duration, Instant}, @@ -224,7 +227,42 @@ struct Timed { body: S, } -#[derive(Default, Debug)] +static BASE_MAP: LazyLock> = LazyLock::new(|| { + [ + ( + vpc_stat::DESTINATION_INTERNET, + FlowLabel::Destination(external::ForwardClass::External), + ), + ( + vpc_stat::DESTINATION_VPC_LOCAL, + FlowLabel::Destination(external::ForwardClass::VpcLocal), + ), + ( + vpc_stat::FW_DEFAULT_IN, + FlowLabel::Entity(external::VpcEntity::FirewallDefaultIn), + ), + ( + vpc_stat::FW_DEFAULT_OUT, + FlowLabel::Builtin(VpcBuiltinLabel::FirewallDefaultOut), + ), + ( + vpc_stat::ROUTER_NOROUTE, + FlowLabel::Builtin(VpcBuiltinLabel::NoRouteMatched), + ), + ( + vpc_stat::GATEWAY_NOSPOOF_IN, + FlowLabel::Builtin(VpcBuiltinLabel::SpoofPrevention), + ), + ( + vpc_stat::GATEWAY_NOSPOOF_OUT, + FlowLabel::Builtin(VpcBuiltinLabel::SpoofPrevention), + ), + ] + .into_iter() + .collect() +}); + +#[derive(Debug)] struct State { flows: HashMap>, roots: HashMap>, @@ -232,6 +270,18 @@ struct State { flow_instances: HashMap>, } +impl Default for State { + fn default() -> Self { + Self { + label_map: BASE_MAP.clone(), + + flows: HashMap::new(), + roots: HashMap::new(), + flow_instances: HashMap::new(), + } + } +} + async fn run_port_stat(state: Arc) { let mut flow_collect = tokio::time::interval(FLOW_STAT_REFRESH_INTERVAL); let mut root_collect = tokio::time::interval(ROOT_STAT_REFRESH_INTERVAL); @@ -248,10 +298,24 @@ async fn run_port_stat(state: Arc) { // TODO: log on error. tokio::select! { _ = flow_collect.tick() => { - state.collect_flows(); + if let Err(e) = state.collect_flows() { + slog::error!( + &state.log, + "failed to collect flow stats for OPTE port"; + "port" => &state.name, + "err" => InlineErrorChain::new(e.as_ref()), + ); + } }, _ = root_collect.tick() => { - state.collect_roots(); + if let Err(e) = state.collect_roots() { + slog::error!( + &state.log, + "failed to collect root stats for OPTE port"; + "port" => &state.name, + "err" => InlineErrorChain::new(e.as_ref()), + ); + } }, _ = prune.tick() => { state.prune(); @@ -260,16 +324,20 @@ async fn run_port_stat(state: Arc) { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum FlowLabel { Entity(external::VpcEntity), Destination(external::ForwardClass), + // TODO: These will be used as part of oximeter association. + #[expect(unused)] Builtin(VpcBuiltinLabel), } -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum VpcBuiltinLabel { - // TODO + NoRouteMatched, + FirewallDefaultOut, + SpoofPrevention, } #[derive(Debug)] diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index ba0f0eacb2d..1c11be5f4c6 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -16,7 +16,7 @@ use nexus_sled_agent_shared::inventory::{ Inventory, OmicronSledConfig, SledRole, }; use omicron_common::{ - api::external::Generation, + api::external::{Flow, Generation}, api::internal::{ nexus::{DiskRuntimeState, SledVmmState}, shared::{ @@ -546,6 +546,25 @@ pub trait SledAgentApi { body: TypedBody, ) -> Result; + /// Get the IDs of all OPTE interfaces + #[endpoint { + method = GET, + path = "/network-interfaces", + }] + async fn nic_ids_list( + request_context: RequestContext, + ) -> Result>, HttpError>; + + /// Get per-flow stats currently reported by an OPTE interface. + #[endpoint { + method = GET, + path = "/network-interfaces/{nic_id}/flows", + }] + async fn nic_flows_list( + request_context: RequestContext, + path_params: Path, + ) -> Result>, HttpError>; + #[endpoint { method = GET, path = "/support/zoneadm-info", @@ -885,3 +904,9 @@ pub struct VmmIssueDiskSnapshotRequestResponse { pub struct VpcPathParam { pub vpc_id: Uuid, } + +/// Path parameters for NIC requests +#[derive(Deserialize, JsonSchema)] +pub struct NicPathParam { + pub nic_id: Uuid, +} diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index 1d0c4593991..756314f35f3 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -19,7 +19,7 @@ use dropshot::{ use nexus_sled_agent_shared::inventory::{ Inventory, OmicronSledConfig, SledRole, }; -use omicron_common::api::external::Error; +use omicron_common::api::external::{Error, Flow}; use omicron_common::api::internal::nexus::{DiskRuntimeState, SledVmmState}; use omicron_common::api::internal::shared::{ ExternalIpGatewayMap, ResolvedVpcRouteSet, ResolvedVpcRouteState, @@ -45,6 +45,7 @@ use sled_diagnostics::{ SledDiagnosticsCommandHttpOutput, SledDiagnosticsQueryOutput, }; use std::collections::BTreeMap; +use uuid::Uuid; type SledApiDescription = ApiDescription; @@ -828,6 +829,22 @@ impl SledAgentApi for SledAgentImpl { Ok(HttpResponseUpdatedNoContent()) } + async fn nic_ids_list( + request_context: RequestContext, + ) -> Result>, HttpError> { + let sa = request_context.context(); + Ok(HttpResponseOk(sa.get_nic_ids())) + } + + async fn nic_flows_list( + request_context: RequestContext, + path_params: Path, + ) -> Result>, HttpError> { + let sa = request_context.context(); + let res = sa.get_nic_flows(path_params.into_inner().nic_id)?; + Ok(HttpResponseOk(res)) + } + async fn support_zoneadm_info( request_context: RequestContext, ) -> Result, HttpError> { diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 5db96e404f6..0a6245e169f 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -1350,7 +1350,6 @@ impl InstanceRunner { self.port_manager.external_ips_ensure( primary_nic.id, - primary_nic.kind, Some(self.source_nat), self.ephemeral_ip, &self.floating_ips, @@ -1366,7 +1365,6 @@ impl InstanceRunner { self.port_manager.external_ips_ensure( primary_nic.id, - primary_nic.kind, Some(self.source_nat), self.ephemeral_ip, &self.floating_ips, @@ -1414,7 +1412,6 @@ impl InstanceRunner { self.port_manager.external_ips_ensure( primary_nic.id, - primary_nic.kind, Some(self.source_nat), self.ephemeral_ip, &self.floating_ips, diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index f206e374edd..aa8975c810f 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -27,6 +27,7 @@ use dropshot::endpoint; use nexus_sled_agent_shared::inventory::Inventory; use nexus_sled_agent_shared::inventory::OmicronSledConfig; use nexus_sled_agent_shared::inventory::SledRole; +use omicron_common::api::external::Flow; use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::SledVmmState; use omicron_common::api::internal::shared::ExternalIpGatewayMap; @@ -56,6 +57,7 @@ use sled_agent_types::zone_bundle::ZoneBundleMetadata; use sled_diagnostics::SledDiagnosticsQueryOutput; use std::collections::BTreeMap; use std::sync::Arc; +use uuid::Uuid; use super::sled_agent::SledAgent; @@ -573,6 +575,19 @@ impl SledAgentApi for SledAgentSimImpl { Ok(HttpResponseUpdatedNoContent()) } + async fn nic_ids_list( + _rqctx: RequestContext, + ) -> Result>, HttpError> { + method_unimplemented() + } + + async fn nic_flows_list( + _rqctx: RequestContext, + _params: Path, + ) -> Result>, HttpError> { + method_unimplemented() + } + async fn zone_bundle_list_all( _rqctx: RequestContext, _query: Query, diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index c30bf21e980..50e8f71bca9 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -38,7 +38,9 @@ use nexus_sled_agent_shared::inventory::{ use omicron_common::address::{ Ipv6Subnet, SLED_PREFIX, get_sled_address, get_switch_zone_address, }; -use omicron_common::api::external::{ByteCount, ByteCountRangeError, Vni}; +use omicron_common::api::external::{ + ByteCount, ByteCountRangeError, Flow, Vni, +}; use omicron_common::api::internal::nexus::{DiskRuntimeState, SledVmmState}; use omicron_common::api::internal::shared::{ ExternalIpGatewayMap, HostPortConfig, RackNetworkConfig, @@ -1077,6 +1079,14 @@ impl SledAgent { Ok(()) } + pub fn get_nic_ids(&self) -> Vec { + self.inner.port_manager.get_nic_ids() + } + + pub fn get_nic_flows(&self, nic_id: Uuid) -> Result, Error> { + self.inner.port_manager.get_nic_flows(nic_id).map_err(Into::into) + } + /// Return identifiers for this sled. /// /// This is mostly used to identify timeseries data with the originating From d3f98a6a86ad557ebc74bb3ba3bb8693f207fb83 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Wed, 16 Jul 2025 11:35:11 +0100 Subject: [PATCH 6/7] Unbreak some existing tests. --- illumos-utils/src/opte/illumos.rs | 1 - illumos-utils/src/opte/port_manager.rs | 4 +- openapi/sled-agent.json | 401 +++++++++++++++++++++++++ 3 files changed, 403 insertions(+), 3 deletions(-) diff --git a/illumos-utils/src/opte/illumos.rs b/illumos-utils/src/opte/illumos.rs index 8c102a33338..315c9c493bc 100644 --- a/illumos-utils/src/opte/illumos.rs +++ b/illumos-utils/src/opte/illumos.rs @@ -7,7 +7,6 @@ use crate::addrobj::AddrObject; use crate::dladm; use camino::Utf8Path; -use omicron_common::api::internal::shared::NetworkInterfaceKind; use opte_ioctl::Error as OpteError; use opte_ioctl::OpteHdl; use slog::Logger; diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 41772385559..32858682055 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -1009,8 +1009,8 @@ mod tests { use uuid::Uuid; // Regression for https://github.com/oxidecomputer/omicron/issues/7541. - #[test] - fn multiple_ports_does_not_destroy_default_route() { + #[tokio::test] + async fn multiple_ports_does_not_destroy_default_route() { let logctx = test_setup_log("multiple_ports_does_not_destroy_default_route"); let manager = PortManager::new(logctx.log.clone(), Ipv6Addr::LOCALHOST); diff --git a/openapi/sled-agent.json b/openapi/sled-agent.json index 9f6fefc08e3..e4829635c7f 100644 --- a/openapi/sled-agent.json +++ b/openapi/sled-agent.json @@ -404,6 +404,74 @@ } } }, + "/network-interfaces": { + "get": { + "summary": "Get the IDs of all OPTE interfaces", + "operationId": "nic_ids_list", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_Uuid", + "type": "array", + "items": { + "type": "string", + "format": "uuid" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/network-interfaces/{nic_id}/flows": { + "get": { + "summary": "Get per-flow stats currently reported by an OPTE interface.", + "operationId": "nic_flows_list", + "parameters": [ + { + "in": "path", + "name": "nic_id", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "title": "Array_of_Flow", + "type": "array", + "items": { + "$ref": "#/components/schemas/Flow" + } + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/omicron-config": { "put": { "operationId": "omicron_config_put", @@ -3827,6 +3895,14 @@ "search_domains" ] }, + "Direction": { + "description": "The direction of a flow or packet, with respect to its target `Instance`.", + "type": "string", + "enum": [ + "in", + "out" + ] + }, "DiskEnsureBody": { "description": "Sent from to a sled agent to establish the runtime state of a Disk", "type": "object", @@ -4335,6 +4411,168 @@ ], "additionalProperties": false }, + "Flow": { + "description": "A packet flow recorded on a `NetworkInterface`.", + "type": "object", + "properties": { + "in_stat": { + "$ref": "#/components/schemas/FlowStat" + }, + "metadata": { + "$ref": "#/components/schemas/FlowMetadata" + }, + "out_stat": { + "$ref": "#/components/schemas/FlowStat" + } + }, + "required": [ + "in_stat", + "metadata", + "out_stat" + ] + }, + "FlowMetadata": { + "description": "Information about a flow recorded on a `NetworkInterface`.", + "type": "object", + "properties": { + "admitted_by_in": { + "nullable": true, + "description": "All entities responsible for allowing packets in this flow to reach the instance.", + "type": "array", + "items": { + "$ref": "#/components/schemas/VpcEntity" + } + }, + "admitted_by_out": { + "nullable": true, + "description": "All entities responsible for allowing packets in this flow to be sent by the instance.", + "type": "array", + "items": { + "$ref": "#/components/schemas/VpcEntity" + } + }, + "created_at": { + "description": "The time the first packet of this flow was seen at.", + "type": "string", + "format": "date-time" + }, + "external_key": { + "description": "The flowkey (or 5-tuple) of any packets on this flow as viewed by the remote half.", + "allOf": [ + { + "$ref": "#/components/schemas/Flowkey" + } + ] + }, + "flow_id": { + "description": "An ephemeral ID bound to this flow.", + "type": "string", + "format": "uuid" + }, + "forwarded": { + "nullable": true, + "description": "How any outbound packets are to be routed.", + "allOf": [ + { + "$ref": "#/components/schemas/ForwardClass" + } + ] + }, + "initial_packet": { + "description": "The direction of the first packet of this flow.", + "allOf": [ + { + "$ref": "#/components/schemas/Direction" + } + ] + }, + "internal_key": { + "description": "The flowkey (or 5-tuple) of any packets on this flow as viewed by the instance.", + "allOf": [ + { + "$ref": "#/components/schemas/Flowkey" + } + ] + } + }, + "required": [ + "created_at", + "external_key", + "flow_id", + "initial_packet", + "internal_key" + ] + }, + "FlowStat": { + "description": "A packet flow recorded on a `NetworkInterface`.", + "type": "object", + "properties": { + "byte_rate": { + "type": "number", + "format": "double" + }, + "bytes": { + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "packet_rate": { + "type": "number", + "format": "double" + }, + "packets": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "byte_rate", + "bytes", + "packet_rate", + "packets" + ] + }, + "Flowkey": { + "description": "Addresses and protocol-specific information used to group packets into a flow.", + "type": "object", + "properties": { + "destination_address": { + "type": "string", + "format": "ip" + }, + "info": { + "nullable": true, + "allOf": [ + { + "$ref": "#/components/schemas/ProtocolInfo" + } + ] + }, + "protocol": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "source_address": { + "type": "string", + "format": "ip" + } + }, + "required": [ + "destination_address", + "protocol", + "source_address" + ] + }, + "ForwardClass": { + "description": "How the remote half of a flow is reached.", + "type": "string", + "enum": [ + "vpc_local", + "external" + ] + }, "Generation": { "description": "Generation numbers stored in the database, used for optimistic concurrency control", "type": "integer", @@ -4571,6 +4809,32 @@ "minLength": 1, "maxLength": 7 }, + "IcmpProtocolInfo": { + "description": "Message types information carried by ICMP.", + "type": "object", + "properties": { + "code": { + "type": "integer", + "format": "uint8", + "minimum": 0 + }, + "id": { + "nullable": true, + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "type": { + "type": "integer", + "format": "uint8", + "minimum": 0 + } + }, + "required": [ + "code", + "type" + ] + }, "IdMapDatasetConfig": { "type": "object", "additionalProperties": { @@ -6381,6 +6645,26 @@ "rs" ] }, + "PortProtocolInfo": { + "description": "A pair of ports, identifying a flow in protocols such as TCP or UDP.", + "type": "object", + "properties": { + "destination_port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "source_port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "destination_port", + "source_port" + ] + }, "PortSpeed": { "description": "Switchport Speed options", "type": "string", @@ -6424,6 +6708,47 @@ "minItems": 2, "maxItems": 2 }, + "ProtocolInfo": { + "description": "Protocol-specific flow information.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "ports" + ] + }, + "value": { + "$ref": "#/components/schemas/PortProtocolInfo" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "icmp" + ] + }, + "value": { + "$ref": "#/components/schemas/IcmpProtocolInfo" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, "QemuPvpanic": { "type": "object", "properties": { @@ -7694,6 +8019,82 @@ "format": "uint32", "minimum": 0 }, + "VpcEntity": { + "description": "A control-plane object which has matched a given flow and chosen to allow it.", + "oneOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "firewall_rule" + ] + }, + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "firewall_default_in" + ] + } + }, + "required": [ + "type" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "vpc_route" + ] + }, + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": [ + "internet_gateway" + ] + }, + "value": { + "type": "string", + "format": "uuid" + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, "VpcFirewallIcmpFilter": { "type": "object", "properties": { From 320a61808d67496bfa5b24558e651334e42f2cb3 Mon Sep 17 00:00:00 2001 From: Kyle Simpson Date: Thu, 17 Jul 2025 11:33:02 +0100 Subject: [PATCH 7/7] Allow second flow-half to contribute 'forwarded' --- illumos-utils/src/opte/stat.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/illumos-utils/src/opte/stat.rs b/illumos-utils/src/opte/stat.rs index e1d4b505311..449c76eac5e 100644 --- a/illumos-utils/src/opte/stat.rs +++ b/illumos-utils/src/opte/stat.rs @@ -19,7 +19,7 @@ use std::{ Arc, LazyLock, RwLock, atomic::{AtomicBool, Ordering}, }, - time::{Duration, Instant}, + time::{Duration, Instant, SystemTime}, }; use tokio::time::MissedTickBehavior; use uuid::Uuid; @@ -105,7 +105,7 @@ impl PortStats { flow_id: ufid.body, // TODO: need to correlate timestamps between // kmod and here?! - created_at: Default::default(), + created_at: SystemTime::now().into(), initial_packet: direction(body.last.first_dir), internal_key: flowkey(out_key), external_key: flowkey(in_key), @@ -130,6 +130,9 @@ impl PortStats { Entry::Occupied(mut val) => { // The second half fills in the remaining metadata. let val = val.get_mut(); + if val.metadata.forwarded.is_none() { + val.metadata.forwarded = forwarded; + } match body.last.dir { Direction::In => { val.metadata.admitted_by_in = Some(bases) @@ -147,7 +150,7 @@ impl PortStats { } // TODO: want `fn root_stats`, need to be able to pull back up into - // oximeter in articular. + // oximeter in particular. } #[derive(Debug)]