[CONTP-1569] [CONTINT-5553] Add podCollectionMode field on kubeStateMetricsCore for node-side pod collection

api/datadoghq/v2alpha1/datadogagent_types.go

@@ -971,6 +971,43 @@ type KubeStateMetricsCoreFeatureConfig struct {
 	// +optional
 	Conf *CustomConfig `json:"conf,omitempty"`
 
+	// PodCollectionMode controls where the KSM check collects pod metrics.
+	//
+	// When set to "node_kubelet" the operator splits the kubernetes_state_core
+	// check into two: the cluster-side instance keeps every collector except
+	// scheduled pods (only unscheduled pods are kept on that side), and a
+	// dedicated pods-only instance is deployed on every node agent to read pods
+	// locally from its Kubelet via workloadmeta. In the generated agent check
+	// YAML this corresponds to the snake_case check options
+	// `pod_collection_mode: cluster_unassigned` on the cluster side and
+	// `pod_collection_mode: node_kubelet` (with `collectors: [pods]`) on the
+	// node side; this CRD field itself is the camelCase `podCollectionMode`
+	// per Kubernetes API convention.
+	//
+	// Useful in large clusters where a single KSM instance is a bottleneck.
+	// The cluster-aggregate `.total` metrics
+	// (kubernetes_state.{container,initcontainer}.<resource>_{requested,limit}.total)
+	// carry a reduced tag set with no host or node, so they cannot be collected
+	// correctly by every node agent independently without colliding at
+	// ingestion. To avoid this, the operator also enables
+	// `cluster_aggregates_enabled` on the cluster-side and node-side instances,
+	// which silences those aggregate metrics on both, and schedules a
+	// dedicated `cluster_aggregates_only` instance on the cluster-side
+	// component to collect them instead.
+	// Requires Agent / Cluster Agent 7.82+.
+	//
+	// When `features.kubeStateMetricsCore.conf` is also set, the operator
+	// deploys the node-side check but does NOT modify the user-supplied
+	// cluster-side config. To avoid double pod collection in that case, the
+	// user's cluster-side instance must either omit `pods` from its
+	// `collectors` list OR set the `pod_collection_mode: cluster_unassigned`
+	// check option themselves. Omitting `collectors` entirely falls back to
+	// upstream KSM defaults, which include `pods`.
+	//
+	// Default behavior is unchanged when this field is unset.
+	// +optional
+	PodCollectionMode *KSMPodCollectionMode `json:"podCollectionMode,omitempty"`
+
 	// `CollectCrMetrics` defines custom resources for the kube-state-metrics core check to collect.
 	//
 	// The datadog agent uses the same logic as upstream `kube-state-metrics`. So is its configuration.
@@ -982,6 +1019,21 @@ type KubeStateMetricsCoreFeatureConfig struct {
 	CollectCrMetrics []Resource `json:"collectCrMetrics,omitempty"`
 }
 
+// KSMPodCollectionMode controls where the kubernetes_state_core check collects pod metrics.
+// +kubebuilder:validation:Enum=default;node_kubelet
+type KSMPodCollectionMode string
+
+const (
+	// KSMPodCollectionModeDefault collects pod metrics centrally from the Kubernetes
+	// API server (today's behavior, equivalent to omitting the field).
+	KSMPodCollectionModeDefault KSMPodCollectionMode = "default"
+
+	// KSMPodCollectionModeNodeKubelet shifts pod metric collection to every node
+	// agent (reading pods locally from its Kubelet) while the cluster-side instance
+	// switches to `pod_collection_mode: cluster_unassigned`.
+	KSMPodCollectionModeNodeKubelet KSMPodCollectionMode = "node_kubelet"
+)
+
 // Resource configures a custom resource for metric generation.
 type Resource struct {
 	// MetricNamePrefix defines a prefix for all metrics of the resource.

config/crd/bases/v1/datadoghq.com_datadogagentinternals.yaml

@@ -1947,6 +1947,46 @@ spec:
                             Enabled enables Kube State Metrics Core.
                             Default: true
                           type: boolean
+                        podCollectionMode:
+                          description: |-
+                            PodCollectionMode controls where the KSM check collects pod metrics.
+
+                            When set to "node_kubelet" the operator splits the kubernetes_state_core
+                            check into two: the cluster-side instance keeps every collector except
+                            scheduled pods (only unscheduled pods are kept on that side), and a
+                            dedicated pods-only instance is deployed on every node agent to read pods
+                            locally from its Kubelet via workloadmeta. In the generated agent check
+                            YAML this corresponds to the snake_case check options
+                            `pod_collection_mode: cluster_unassigned` on the cluster side and
+                            `pod_collection_mode: node_kubelet` (with `collectors: [pods]`) on the
+                            node side; this CRD field itself is the camelCase `podCollectionMode`
+                            per Kubernetes API convention.
+
+                            Useful in large clusters where a single KSM instance is a bottleneck.
+                            The cluster-aggregate `.total` metrics
+                            (kubernetes_state.{container,initcontainer}.<resource>_{requested,limit}.total)
+                            carry a reduced tag set with no host or node, so they cannot be collected
+                            correctly by every node agent independently without colliding at
+                            ingestion. To avoid this, the operator also enables
+                            `cluster_aggregates_enabled` on the cluster-side and node-side instances,
+                            which silences those aggregate metrics on both, and schedules a
+                            dedicated `cluster_aggregates_only` instance on the cluster-side
+                            component to collect them instead.
+                            Requires Agent / Cluster Agent 7.82+.
+
+                            When `features.kubeStateMetricsCore.conf` is also set, the operator
+                            deploys the node-side check but does NOT modify the user-supplied
+                            cluster-side config. To avoid double pod collection in that case, the
+                            user's cluster-side instance must either omit `pods` from its
+                            `collectors` list OR set the `pod_collection_mode: cluster_unassigned`
+                            check option themselves. Omitting `collectors` entirely falls back to
+                            upstream KSM defaults, which include `pods`.
+
+                            Default behavior is unchanged when this field is unset.
+                          enum:
+                            - default
+                            - node_kubelet
+                          type: string
                       type: object
                     kubernetesActions:
                       description: KubernetesActions configuration.
@@ -10715,6 +10755,46 @@ spec:
                                 Enabled enables Kube State Metrics Core.
                                 Default: true
                               type: boolean
+                            podCollectionMode:
+                              description: |-
+                                PodCollectionMode controls where the KSM check collects pod metrics.
+
+                                When set to "node_kubelet" the operator splits the kubernetes_state_core
+                                check into two: the cluster-side instance keeps every collector except
+                                scheduled pods (only unscheduled pods are kept on that side), and a
+                                dedicated pods-only instance is deployed on every node agent to read pods
+                                locally from its Kubelet via workloadmeta. In the generated agent check
+                                YAML this corresponds to the snake_case check options
+                                `pod_collection_mode: cluster_unassigned` on the cluster side and
+                                `pod_collection_mode: node_kubelet` (with `collectors: [pods]`) on the
+                                node side; this CRD field itself is the camelCase `podCollectionMode`
+                                per Kubernetes API convention.
+
+                                Useful in large clusters where a single KSM instance is a bottleneck.
+                                The cluster-aggregate `.total` metrics
+                                (kubernetes_state.{container,initcontainer}.<resource>_{requested,limit}.total)
+                                carry a reduced tag set with no host or node, so they cannot be collected
+                                correctly by every node agent independently without colliding at
+                                ingestion. To avoid this, the operator also enables
+                                `cluster_aggregates_enabled` on the cluster-side and node-side instances,
+                                which silences those aggregate metrics on both, and schedules a
+                                dedicated `cluster_aggregates_only` instance on the cluster-side
+                                component to collect them instead.
+                                Requires Agent / Cluster Agent 7.82+.
+
+                                When `features.kubeStateMetricsCore.conf` is also set, the operator
+                                deploys the node-side check but does NOT modify the user-supplied
+                                cluster-side config. To avoid double pod collection in that case, the
+                                user's cluster-side instance must either omit `pods` from its
+                                `collectors` list OR set the `pod_collection_mode: cluster_unassigned`
+                                check option themselves. Omitting `collectors` entirely falls back to
+                                upstream KSM defaults, which include `pods`.
+
+                                Default behavior is unchanged when this field is unset.
+                              enum:
+                                - default
+                                - node_kubelet
+                              type: string
                           type: object
                         kubernetesActions:
                           description: KubernetesActions configuration.

config/crd/bases/v1/datadoghq.com_datadogagentinternals_v1alpha1.json

@@ -2013,6 +2013,14 @@
                 "enabled": {
                   "description": "Enabled enables Kube State Metrics Core.\nDefault: true",
                   "type": "boolean"
+                },
+                "podCollectionMode": {
+                  "description": "PodCollectionMode controls where the KSM check collects pod metrics.\n\nWhen set to \"node_kubelet\" the operator splits the kubernetes_state_core\ncheck into two: the cluster-side instance keeps every collector except\nscheduled pods (only unscheduled pods are kept on that side), and a\ndedicated pods-only instance is deployed on every node agent to read pods\nlocally from its Kubelet via workloadmeta. In the generated agent check\nYAML this corresponds to the snake_case check options\n`pod_collection_mode: cluster_unassigned` on the cluster side and\n`pod_collection_mode: node_kubelet` (with `collectors: [pods]`) on the\nnode side; this CRD field itself is the camelCase `podCollectionMode`\nper Kubernetes API convention.\n\nUseful in large clusters where a single KSM instance is a bottleneck.\nThe cluster-aggregate `.total` metrics\n(kubernetes_state.{container,initcontainer}.\u003cresource\u003e_{requested,limit}.total)\ncarry a reduced tag set with no host or node, so they cannot be collected\ncorrectly by every node agent independently without colliding at\ningestion. To avoid this, the operator also enables\n`cluster_aggregates_enabled` on the cluster-side and node-side instances,\nwhich silences those aggregate metrics on both, and schedules a\ndedicated `cluster_aggregates_only` instance on the cluster-side\ncomponent to collect them instead.\nRequires Agent / Cluster Agent 7.82+.\n\nWhen `features.kubeStateMetricsCore.conf` is also set, the operator\ndeploys the node-side check but does NOT modify the user-supplied\ncluster-side config. To avoid double pod collection in that case, the\nuser's cluster-side instance must either omit `pods` from its\n`collectors` list OR set the `pod_collection_mode: cluster_unassigned`\ncheck option themselves. Omitting `collectors` entirely falls back to\nupstream KSM defaults, which include `pods`.\n\nDefault behavior is unchanged when this field is unset.",
+                  "enum": [
+                    "default",
+                    "node_kubelet"
+                  ],
+                  "type": "string"
                 }
               },
               "type": "object"
@@ -10494,6 +10502,14 @@
                     "enabled": {
                       "description": "Enabled enables Kube State Metrics Core.\nDefault: true",
                       "type": "boolean"
+                    },
+                    "podCollectionMode": {
+                      "description": "PodCollectionMode controls where the KSM check collects pod metrics.\n\nWhen set to \"node_kubelet\" the operator splits the kubernetes_state_core\ncheck into two: the cluster-side instance keeps every collector except\nscheduled pods (only unscheduled pods are kept on that side), and a\ndedicated pods-only instance is deployed on every node agent to read pods\nlocally from its Kubelet via workloadmeta. In the generated agent check\nYAML this corresponds to the snake_case check options\n`pod_collection_mode: cluster_unassigned` on the cluster side and\n`pod_collection_mode: node_kubelet` (with `collectors: [pods]`) on the\nnode side; this CRD field itself is the camelCase `podCollectionMode`\nper Kubernetes API convention.\n\nUseful in large clusters where a single KSM instance is a bottleneck.\nThe cluster-aggregate `.total` metrics\n(kubernetes_state.{container,initcontainer}.\u003cresource\u003e_{requested,limit}.total)\ncarry a reduced tag set with no host or node, so they cannot be collected\ncorrectly by every node agent independently without colliding at\ningestion. To avoid this, the operator also enables\n`cluster_aggregates_enabled` on the cluster-side and node-side instances,\nwhich silences those aggregate metrics on both, and schedules a\ndedicated `cluster_aggregates_only` instance on the cluster-side\ncomponent to collect them instead.\nRequires Agent / Cluster Agent 7.82+.\n\nWhen `features.kubeStateMetricsCore.conf` is also set, the operator\ndeploys the node-side check but does NOT modify the user-supplied\ncluster-side config. To avoid double pod collection in that case, the\nuser's cluster-side instance must either omit `pods` from its\n`collectors` list OR set the `pod_collection_mode: cluster_unassigned`\ncheck option themselves. Omitting `collectors` entirely falls back to\nupstream KSM defaults, which include `pods`.\n\nDefault behavior is unchanged when this field is unset.",
+                      "enum": [
+                        "default",
+                        "node_kubelet"
+                      ],
+                      "type": "string"
                     }
                   },
                   "type": "object"