From 1ccb444d8206f531754f45fb760b0318ec5fbe30 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Thu, 31 Jul 2025 21:25:25 +0000 Subject: [PATCH 1/8] MTPNC CRD changes for NUMA-Aware pods with infiniband NICs --- cns/types/infiniband/status.go | 11 +++++++++++ .../api/v1alpha1/multitenantpodnetworkconfig.go | 9 +++++++++ .../api/v1alpha1/zz_generated.deepcopy.go | 14 +++++++++++++- ...acn.azure.com_multitenantpodnetworkconfigs.yaml | 11 +++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 cns/types/infiniband/status.go diff --git a/cns/types/infiniband/status.go b/cns/types/infiniband/status.go new file mode 100644 index 0000000000..31f8149b42 --- /dev/null +++ b/cns/types/infiniband/status.go @@ -0,0 +1,11 @@ +package infiniband + +type Status int + +const ( + Available Status = 0 + ProgrammingPending Status = 1 + ProgrammingFailed Status = 2 + ProgrammingComplete Status = 3 + ReleasePending Status = 4 +) diff --git a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go index dba7fdd117..7950a899e5 100644 --- a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go +++ b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go @@ -4,6 +4,9 @@ package v1alpha1 import ( + "net" + + "github.com/Azure/azure-container-networking/cns/types/infiniband" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -45,6 +48,9 @@ type MultitenantPodNetworkConfigSpec struct { PodNetwork string `json:"podNetwork"` // name of the requesting cx pod PodName string `json:"podName,omitempty"` + // list of IB MACs to use for a pod + // +kubebuilder:validation:Optional + IBMACs []net.HardwareAddr `json:"IBMACs,omitempty"` } type InterfaceInfo struct { @@ -66,6 +72,9 @@ type InterfaceInfo struct { // AccelnetEnabled determines if the CNI will provision the NIC with accelerated networking enabled // +kubebuilder:validation:Optional AccelnetEnabled bool `json:"accelnetEnabled,omitempty"` + // IBStatus is the programming status of the infiniband device + // +kubebuilder:validation:Optional + IBStatus infiniband.Status `json:"ibStatus,omitempty"` } // MultitenantPodNetworkConfigStatus defines the observed state of PodNetworkConfig diff --git a/crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go b/crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go index 92911a46cd..7da5a94c2b 100644 --- a/crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go +++ b/crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go @@ -6,6 +6,7 @@ package v1alpha1 import ( runtime "k8s.io/apimachinery/pkg/runtime" + netx "net" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -43,7 +44,7 @@ func (in *MultitenantPodNetworkConfig) DeepCopyInto(out *MultitenantPodNetworkCo *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -100,6 +101,17 @@ func (in *MultitenantPodNetworkConfigList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MultitenantPodNetworkConfigSpec) DeepCopyInto(out *MultitenantPodNetworkConfigSpec) { *out = *in + if in.IBMACs != nil { + in, out := &in.IBMACs, &out.IBMACs + *out = make([]netx.HardwareAddr, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = make(netx.HardwareAddr, len(*in)) + copy(*out, *in) + } + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultitenantPodNetworkConfigSpec. diff --git a/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml b/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml index 9390424b82..215bbcc098 100644 --- a/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml +++ b/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml @@ -56,6 +56,13 @@ spec: description: MultitenantPodNetworkConfigSpec defines the desired state of PodNetworkConfig properties: + IBMACs: + description: list of IB MACs to use for a pod + items: + description: A HardwareAddr represents a physical hardware address. + format: byte + type: string + type: array podName: description: name of the requesting cx pod type: string @@ -98,6 +105,10 @@ spec: gatewayIP: description: GatewayIP is the gateway ip of the injected subnet type: string + ibStatus: + description: IBStatus is the programming status of the infiniband + device + type: integer macAddress: description: MacAddress is the MAC Address of the VM's NIC which this network container was created for From 5fb8549186f75844b09332372d56aa728217404a Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Thu, 31 Jul 2025 22:40:55 +0000 Subject: [PATCH 2/8] More descriptive description haha --- crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go | 2 +- ...multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go index 7950a899e5..6dcf14924a 100644 --- a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go +++ b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go @@ -48,7 +48,7 @@ type MultitenantPodNetworkConfigSpec struct { PodNetwork string `json:"podNetwork"` // name of the requesting cx pod PodName string `json:"podName,omitempty"` - // list of IB MACs to use for a pod + // MACs of the IB devices to use for a pod // +kubebuilder:validation:Optional IBMACs []net.HardwareAddr `json:"IBMACs,omitempty"` } diff --git a/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml b/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml index 215bbcc098..c64b78e600 100644 --- a/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml +++ b/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml @@ -57,7 +57,7 @@ spec: of PodNetworkConfig properties: IBMACs: - description: list of IB MACs to use for a pod + description: MACs of the IB devices to use for a pod items: description: A HardwareAddr represents a physical hardware address. format: byte From 4b2f4aaa3da1d3abcbf3330bdbe3db26d7e02e71 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Fri, 1 Aug 2025 21:38:22 +0000 Subject: [PATCH 3/8] IBMACAddresses --- crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go | 4 ++-- crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go | 4 ++-- ...ltitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go index 6dcf14924a..7f28a950cb 100644 --- a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go +++ b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go @@ -48,9 +48,9 @@ type MultitenantPodNetworkConfigSpec struct { PodNetwork string `json:"podNetwork"` // name of the requesting cx pod PodName string `json:"podName,omitempty"` - // MACs of the IB devices to use for a pod + // MAC addresses of the IB devices to use for a pod // +kubebuilder:validation:Optional - IBMACs []net.HardwareAddr `json:"IBMACs,omitempty"` + IBMACAddresses []net.HardwareAddr `json:"IBMACAddresses,omitempty"` } type InterfaceInfo struct { diff --git a/crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go b/crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go index 7da5a94c2b..90f6f88479 100644 --- a/crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go +++ b/crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go @@ -101,8 +101,8 @@ func (in *MultitenantPodNetworkConfigList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MultitenantPodNetworkConfigSpec) DeepCopyInto(out *MultitenantPodNetworkConfigSpec) { *out = *in - if in.IBMACs != nil { - in, out := &in.IBMACs, &out.IBMACs + if in.IBMACAddresses != nil { + in, out := &in.IBMACAddresses, &out.IBMACAddresses *out = make([]netx.HardwareAddr, len(*in)) for i := range *in { if (*in)[i] != nil { diff --git a/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml b/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml index c64b78e600..c41eb88784 100644 --- a/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml +++ b/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml @@ -56,8 +56,8 @@ spec: description: MultitenantPodNetworkConfigSpec defines the desired state of PodNetworkConfig properties: - IBMACs: - description: MACs of the IB devices to use for a pod + IBMACAddresses: + description: MAC addresses of the IB devices to use for a pod items: description: A HardwareAddr represents a physical hardware address. format: byte From 21906288844ccc6fd992df8fe1ffc0df12432dd9 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Mon, 4 Aug 2025 18:05:45 +0000 Subject: [PATCH 4/8] Make mac string type to not break OpenAPI schema --- .../api/v1alpha1/multitenantpodnetworkconfig.go | 4 +--- .../api/v1alpha1/zz_generated.deepcopy.go | 11 ++--------- ...cy.acn.azure.com_multitenantpodnetworkconfigs.yaml | 2 -- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go index 7f28a950cb..5bf33c3784 100644 --- a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go +++ b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go @@ -4,8 +4,6 @@ package v1alpha1 import ( - "net" - "github.com/Azure/azure-container-networking/cns/types/infiniband" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -50,7 +48,7 @@ type MultitenantPodNetworkConfigSpec struct { PodName string `json:"podName,omitempty"` // MAC addresses of the IB devices to use for a pod // +kubebuilder:validation:Optional - IBMACAddresses []net.HardwareAddr `json:"IBMACAddresses,omitempty"` + IBMACAddresses []string `json:"IBMACAddresses,omitempty"` } type InterfaceInfo struct { diff --git a/crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go b/crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go index 90f6f88479..cab14e1666 100644 --- a/crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go +++ b/crd/multitenancy/api/v1alpha1/zz_generated.deepcopy.go @@ -6,7 +6,6 @@ package v1alpha1 import ( runtime "k8s.io/apimachinery/pkg/runtime" - netx "net" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -103,14 +102,8 @@ func (in *MultitenantPodNetworkConfigSpec) DeepCopyInto(out *MultitenantPodNetwo *out = *in if in.IBMACAddresses != nil { in, out := &in.IBMACAddresses, &out.IBMACAddresses - *out = make([]netx.HardwareAddr, len(*in)) - for i := range *in { - if (*in)[i] != nil { - in, out := &(*in)[i], &(*out)[i] - *out = make(netx.HardwareAddr, len(*in)) - copy(*out, *in) - } - } + *out = make([]string, len(*in)) + copy(*out, *in) } } diff --git a/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml b/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml index c41eb88784..6e787879de 100644 --- a/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml +++ b/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml @@ -59,8 +59,6 @@ spec: IBMACAddresses: description: MAC addresses of the IB devices to use for a pod items: - description: A HardwareAddr represents a physical hardware address. - format: byte type: string type: array podName: From 18bab913b5c98e02135f8b85642c2eafbc90d2cc Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Mon, 4 Aug 2025 18:11:22 +0000 Subject: [PATCH 5/8] Status to MTPNC definition --- cns/types/infiniband/status.go | 11 ----------- .../api/v1alpha1/multitenantpodnetworkconfig.go | 11 +++++++++++ 2 files changed, 11 insertions(+), 11 deletions(-) delete mode 100644 cns/types/infiniband/status.go diff --git a/cns/types/infiniband/status.go b/cns/types/infiniband/status.go deleted file mode 100644 index 31f8149b42..0000000000 --- a/cns/types/infiniband/status.go +++ /dev/null @@ -1,11 +0,0 @@ -package infiniband - -type Status int - -const ( - Available Status = 0 - ProgrammingPending Status = 1 - ProgrammingFailed Status = 2 - ProgrammingComplete Status = 3 - ReleasePending Status = 4 -) diff --git a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go index 5bf33c3784..13c0c051b9 100644 --- a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go +++ b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go @@ -51,6 +51,17 @@ type MultitenantPodNetworkConfigSpec struct { IBMACAddresses []string `json:"IBMACAddresses,omitempty"` } +// +kubebuilder:validation:Enum=Available;ProgrammingPending;ProgrammingFailed;ProgrammingComplete;ReleasePending +type InfinibandStatus string + +const ( + Available InfinibandStatus = "Available" + ProgrammingPending InfinibandStatus = "ProgrammingPending" + ProgrammingFailed InfinibandStatus = "ProgrammingFailed" + ProgrammingComplete InfinibandStatus = "ProgrammingComplete" + ReleasePending InfinibandStatus = "ReleasePending" +) + type InterfaceInfo struct { // NCID is the network container id NCID string `json:"ncID,omitempty"` From f4e81bd17c3b163d96330bc3f16f8beea058d92e Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Mon, 4 Aug 2025 18:13:37 +0000 Subject: [PATCH 6/8] reference --- .../api/v1alpha1/multitenantpodnetworkconfig.go | 3 +-- ...enancy.acn.azure.com_multitenantpodnetworkconfigs.yaml | 8 +++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go index 13c0c051b9..34f5e24c85 100644 --- a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go +++ b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go @@ -4,7 +4,6 @@ package v1alpha1 import ( - "github.com/Azure/azure-container-networking/cns/types/infiniband" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -83,7 +82,7 @@ type InterfaceInfo struct { AccelnetEnabled bool `json:"accelnetEnabled,omitempty"` // IBStatus is the programming status of the infiniband device // +kubebuilder:validation:Optional - IBStatus infiniband.Status `json:"ibStatus,omitempty"` + IBStatus InfinibandStatus `json:"ibStatus,omitempty"` } // MultitenantPodNetworkConfigStatus defines the observed state of PodNetworkConfig diff --git a/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml b/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml index 6e787879de..29285002df 100644 --- a/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml +++ b/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml @@ -106,7 +106,13 @@ spec: ibStatus: description: IBStatus is the programming status of the infiniband device - type: integer + enum: + - Available + - ProgrammingPending + - ProgrammingFailed + - ProgrammingComplete + - ReleasePending + type: string macAddress: description: MacAddress is the MAC Address of the VM's NIC which this network container was created for From bd960428e8cd648850d2e74153ebdbef82f60935 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Wed, 6 Aug 2025 16:32:41 +0000 Subject: [PATCH 7/8] Rename statuses to unambiguate ambiguity --- .../api/v1alpha1/multitenantpodnetworkconfig.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go index 34f5e24c85..c217f2dafa 100644 --- a/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go +++ b/crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go @@ -50,15 +50,15 @@ type MultitenantPodNetworkConfigSpec struct { IBMACAddresses []string `json:"IBMACAddresses,omitempty"` } -// +kubebuilder:validation:Enum=Available;ProgrammingPending;ProgrammingFailed;ProgrammingComplete;ReleasePending +// +kubebuilder:validation:Enum=Unprogrammed;Programming;Programmed;Unprogramming;Failed type InfinibandStatus string const ( - Available InfinibandStatus = "Available" - ProgrammingPending InfinibandStatus = "ProgrammingPending" - ProgrammingFailed InfinibandStatus = "ProgrammingFailed" - ProgrammingComplete InfinibandStatus = "ProgrammingComplete" - ReleasePending InfinibandStatus = "ReleasePending" + Unprogrammed InfinibandStatus = "Unprogrammed" + Programming InfinibandStatus = "Programming" + Programmed InfinibandStatus = "Programmed" + Unprogramming InfinibandStatus = "Unprogramming" + Failed InfinibandStatus = "Failed" ) type InterfaceInfo struct { From 3d668d53d8ea7ab1cfbf234d9140cde11e6f5054 Mon Sep 17 00:00:00 2001 From: Paul Johnston Date: Wed, 6 Aug 2025 16:38:23 +0000 Subject: [PATCH 8/8] make crd --- ...ncy.acn.azure.com_multitenantpodnetworkconfigs.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml b/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml index 29285002df..c693498faa 100644 --- a/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml +++ b/crd/multitenancy/manifests/multitenancy.acn.azure.com_multitenantpodnetworkconfigs.yaml @@ -107,11 +107,11 @@ spec: description: IBStatus is the programming status of the infiniband device enum: - - Available - - ProgrammingPending - - ProgrammingFailed - - ProgrammingComplete - - ReleasePending + - Unprogrammed + - Programming + - Programmed + - Unprogramming + - Failed type: string macAddress: description: MacAddress is the MAC Address of the VM's NIC which