Skip to content

fix(horizontalpodautoscaler): Added container label to ContainerResource metrics #2836

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bxrne wants to merge 1 commit into
base: main
Choose a base branch
from
+181 −8
Open

fix(horizontalpodautoscaler): Added container label to ContainerResource metrics #2836

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 40 additions & 5 deletions internal/store/horizontalpodautoscaler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ var (
descHorizontalPodAutoscalerLabelsHelp = "Kubernetes labels converted to Prometheus labels."
descHorizontalPodAutoscalerLabelsDefaultLabels = []string{"namespace", "horizontalpodautoscaler"}

targetMetricLabels = []string{"metric_name", "metric_target_type"}
targetMetricLabels = []string{"metric_name", "metric_target_type"}
containerMetricTargetLabels = []string{"metric_name", "metric_target_type", "container"}
)

func hpaMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator {
Expand Down Expand Up @@ -194,25 +195,37 @@ func createHPASpecTargetMetric() generator.FamilyGenerator {
for _, m := range a.Spec.Metrics {
var metricName string
var metricTarget autoscaling.MetricTarget
var labelKeys []string
var labelValues []string
// The variable maps the type of metric to the corresponding value
metricMap := make(map[metricTargetType]float64)

switch m.Type {
case autoscaling.ObjectMetricSourceType:
metricName = m.Object.Metric.Name
metricTarget = m.Object.Target
labelKeys = targetMetricLabels
labelValues = []string{metricName, ""}
case autoscaling.PodsMetricSourceType:
metricName = m.Pods.Metric.Name
metricTarget = m.Pods.Target
labelKeys = targetMetricLabels
labelValues = []string{metricName, ""}
case autoscaling.ResourceMetricSourceType:
metricName = string(m.Resource.Name)
metricTarget = m.Resource.Target
labelKeys = targetMetricLabels
labelValues = []string{metricName, ""}
case autoscaling.ContainerResourceMetricSourceType:
metricName = string(m.ContainerResource.Name)
metricTarget = m.ContainerResource.Target
labelKeys = containerMetricTargetLabels
labelValues = []string{metricName, "", string(m.ContainerResource.Container)}
case autoscaling.ExternalMetricSourceType:
metricName = m.External.Metric.Name
metricTarget = m.External.Target
labelKeys = targetMetricLabels
labelValues = []string{metricName, ""}
default:
// Skip unsupported metric type
continue
Expand All @@ -229,9 +242,14 @@ func createHPASpecTargetMetric() generator.FamilyGenerator {
}

for metricTypeIndex, metricValue := range metricMap {
// Clone labelValues and update metric_target_type
currentLabelValues := make([]string, len(labelValues))
copy(currentLabelValues, labelValues)
currentLabelValues[1] = metricTypeIndex.String()

ms = append(ms, &metric.Metric{
LabelKeys: targetMetricLabels,
LabelValues: []string{metricName, metricTypeIndex.String()},
LabelKeys: labelKeys,
LabelValues: currentLabelValues,
Value: metricValue,
})
}
Expand All @@ -253,25 +271,37 @@ func createHPAStatusTargetMetric() generator.FamilyGenerator {
for _, m := range a.Status.CurrentMetrics {
var metricName string
var currentMetric autoscaling.MetricValueStatus
var labelKeys []string
var labelValues []string
// The variable maps the type of metric to the corresponding value
metricMap := make(map[metricTargetType]float64)

switch m.Type {
case autoscaling.ObjectMetricSourceType:
metricName = m.Object.Metric.Name
currentMetric = m.Object.Current
labelKeys = targetMetricLabels
labelValues = []string{metricName, ""}
case autoscaling.PodsMetricSourceType:
metricName = m.Pods.Metric.Name
currentMetric = m.Pods.Current
labelKeys = targetMetricLabels
labelValues = []string{metricName, ""}
case autoscaling.ResourceMetricSourceType:
metricName = string(m.Resource.Name)
currentMetric = m.Resource.Current
labelKeys = targetMetricLabels
labelValues = []string{metricName, ""}
case autoscaling.ContainerResourceMetricSourceType:
metricName = string(m.ContainerResource.Name)
currentMetric = m.ContainerResource.Current
labelKeys = containerMetricTargetLabels
labelValues = []string{metricName, "", string(m.ContainerResource.Container)}
case autoscaling.ExternalMetricSourceType:
metricName = m.External.Metric.Name
currentMetric = m.External.Current
labelKeys = targetMetricLabels
labelValues = []string{metricName, ""}
default:
// Skip unsupported metric type
continue
Expand All @@ -288,9 +318,14 @@ func createHPAStatusTargetMetric() generator.FamilyGenerator {
}

for metricTypeIndex, metricValue := range metricMap {
// Clone labelValues and update metric_target_type
currentLabelValues := make([]string, len(labelValues))
copy(currentLabelValues, labelValues)
currentLabelValues[1] = metricTypeIndex.String()

ms = append(ms, &metric.Metric{
LabelKeys: targetMetricLabels,
LabelValues: []string{metricName, metricTypeIndex.String()},
LabelKeys: labelKeys,
LabelValues: currentLabelValues,
Value: metricValue,
})
}
Expand Down
144 changes: 141 additions & 3 deletions internal/store/horizontalpodautoscaler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ func TestHPAStore(t *testing.T) {
kube_horizontalpodautoscaler_metadata_generation{horizontalpodautoscaler="hpa1",namespace="ns1"} 2
kube_horizontalpodautoscaler_spec_max_replicas{horizontalpodautoscaler="hpa1",namespace="ns1"} 4
kube_horizontalpodautoscaler_spec_min_replicas{horizontalpodautoscaler="hpa1",namespace="ns1"} 2
kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 80
kube_horizontalpodautoscaler_spec_target_metric{container="container1",horizontalpodautoscaler="hpa1",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 80
kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="events",metric_target_type="average",namespace="ns1"} 30
kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="hits",metric_target_type="average",namespace="ns1"} 12
kube_horizontalpodautoscaler_spec_target_metric{horizontalpodautoscaler="hpa1",metric_name="hits",metric_target_type="value",namespace="ns1"} 10
Expand Down Expand Up @@ -411,8 +411,8 @@ func TestHPAStore(t *testing.T) {
kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="memory",metric_target_type="utilization",namespace="ns1"} 28
kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="average",namespace="ns1"} 0.062
kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 6
kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="average",namespace="ns1"} 0.08
kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 10
kube_horizontalpodautoscaler_status_target_metric{container="container1",horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="average",namespace="ns1"} 0.08
kube_horizontalpodautoscaler_status_target_metric{container="container1",horizontalpodautoscaler="hpa2",metric_name="cpu",metric_target_type="utilization",namespace="ns1"} 10
kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_requests_per_second",metric_target_type="value",namespace="ns1"} 0
kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_requests_per_second",metric_target_type="average",namespace="ns1"} 2.9
kube_horizontalpodautoscaler_status_target_metric{horizontalpodautoscaler="hpa2",metric_name="traefik_backend_errors_per_second",metric_target_type="value",namespace="ns1"} 0
Expand Down Expand Up @@ -440,6 +440,144 @@ func TestHPAStore(t *testing.T) {
"kube_horizontalpodautoscaler_deletion_timestamp",
},
},

{
// Test case for multiple ContainerResource metrics with same resource name
Obj: &autoscaling.HorizontalPodAutoscaler{
ObjectMeta: metav1.ObjectMeta{
Generation: 1,
Name: "hpa-container-resource",
Namespace: "default",
CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
},
Spec: autoscaling.HorizontalPodAutoscalerSpec{
MaxReplicas: 10,
MinReplicas: int32ptr(1),
Metrics: []autoscaling.MetricSpec{
{
Type: autoscaling.ContainerResourceMetricSourceType,
ContainerResource: &autoscaling.ContainerResourceMetricSource{
Name: "cpu",
Container: "main",
Target: autoscaling.MetricTarget{
AverageUtilization: int32ptr(40),
},
},
},
{
Type: autoscaling.ContainerResourceMetricSourceType,
ContainerResource: &autoscaling.ContainerResourceMetricSource{
Name: "cpu",
Container: "sidecar",
Target: autoscaling.MetricTarget{
AverageUtilization: int32ptr(90),
},
},
},
{
Type: autoscaling.ContainerResourceMetricSourceType,
ContainerResource: &autoscaling.ContainerResourceMetricSource{
Name: "memory",
Container: "main",
Target: autoscaling.MetricTarget{
AverageUtilization: int32ptr(70),
},
},
},
},
ScaleTargetRef: autoscaling.CrossVersionObjectReference{
APIVersion: "apps/v1",
Kind: "Deployment",
Name: "test-app",
},
},
Status: autoscaling.HorizontalPodAutoscalerStatus{
CurrentReplicas: 2,
DesiredReplicas: 3,
Conditions: []autoscaling.HorizontalPodAutoscalerCondition{
{
Type: autoscaling.AbleToScale,
Status: v1.ConditionTrue,
},
},
CurrentMetrics: []autoscaling.MetricStatus{
{
Type: autoscaling.ContainerResourceMetricSourceType,
ContainerResource: &autoscaling.ContainerResourceMetricStatus{
Name: "cpu",
Container: "main",
Current: autoscaling.MetricValueStatus{
AverageUtilization: int32ptr(24),
AverageValue: resourcePtr(resource.MustParse("1442m")),
},
},
},
{
Type: autoscaling.ContainerResourceMetricSourceType,
ContainerResource: &autoscaling.ContainerResourceMetricStatus{
Name: "cpu",
Container: "sidecar",
Current: autoscaling.MetricValueStatus{
AverageUtilization: int32ptr(69),
AverageValue: resourcePtr(resource.MustParse("836m")),
},
},
},
},
},
},
Want: `
# HELP kube_horizontalpodautoscaler_info Information about this autoscaler.
# HELP kube_horizontalpodautoscaler_metadata_generation [STABLE] The generation observed by the HorizontalPodAutoscaler controller.
# HELP kube_horizontalpodautoscaler_spec_max_replicas [STABLE] Upper limit for the number of pods that can be set by the autoscaler; cannot be smaller than MinReplicas.
# HELP kube_horizontalpodautoscaler_spec_min_replicas [STABLE] Lower limit for the number of pods that can be set by the autoscaler, default 1.
# HELP kube_horizontalpodautoscaler_spec_target_metric The metric specifications used by this autoscaler when calculating the desired replica count.
# HELP kube_horizontalpodautoscaler_status_target_metric [STABLE] The current metric status used by this autoscaler when calculating the desired replica count.
# HELP kube_horizontalpodautoscaler_status_condition [STABLE] The condition of this autoscaler.
# HELP kube_horizontalpodautoscaler_status_current_replicas [STABLE] Current number of replicas of pods managed by this autoscaler.
# HELP kube_horizontalpodautoscaler_status_desired_replicas [STABLE] Desired number of replicas of pods managed by this autoscaler.
# HELP kube_horizontalpodautoscaler_created Unix creation timestamp
# TYPE kube_horizontalpodautoscaler_info gauge
# TYPE kube_horizontalpodautoscaler_metadata_generation gauge
# TYPE kube_horizontalpodautoscaler_spec_max_replicas gauge
# TYPE kube_horizontalpodautoscaler_spec_min_replicas gauge
# TYPE kube_horizontalpodautoscaler_spec_target_metric gauge
# TYPE kube_horizontalpodautoscaler_status_target_metric gauge
# TYPE kube_horizontalpodautoscaler_status_condition gauge
# TYPE kube_horizontalpodautoscaler_status_current_replicas gauge
# TYPE kube_horizontalpodautoscaler_status_desired_replicas gauge
# TYPE kube_horizontalpodautoscaler_created gauge
kube_horizontalpodautoscaler_info{horizontalpodautoscaler="hpa-container-resource",namespace="default",scaletargetref_api_version="apps/v1",scaletargetref_kind="Deployment",scaletargetref_name="test-app"} 1
kube_horizontalpodautoscaler_metadata_generation{horizontalpodautoscaler="hpa-container-resource",namespace="default"} 1
kube_horizontalpodautoscaler_spec_max_replicas{horizontalpodautoscaler="hpa-container-resource",namespace="default"} 10
kube_horizontalpodautoscaler_spec_min_replicas{horizontalpodautoscaler="hpa-container-resource",namespace="default"} 1
kube_horizontalpodautoscaler_spec_target_metric{container="main",horizontalpodautoscaler="hpa-container-resource",metric_name="cpu",metric_target_type="utilization",namespace="default"} 40
kube_horizontalpodautoscaler_spec_target_metric{container="main",horizontalpodautoscaler="hpa-container-resource",metric_name="memory",metric_target_type="utilization",namespace="default"} 70
kube_horizontalpodautoscaler_spec_target_metric{container="sidecar",horizontalpodautoscaler="hpa-container-resource",metric_name="cpu",metric_target_type="utilization",namespace="default"} 90
kube_horizontalpodautoscaler_status_target_metric{container="main",horizontalpodautoscaler="hpa-container-resource",metric_name="cpu",metric_target_type="average",namespace="default"} 1.442
kube_horizontalpodautoscaler_status_target_metric{container="main",horizontalpodautoscaler="hpa-container-resource",metric_name="cpu",metric_target_type="utilization",namespace="default"} 24
kube_horizontalpodautoscaler_status_target_metric{container="sidecar",horizontalpodautoscaler="hpa-container-resource",metric_name="cpu",metric_target_type="average",namespace="default"} 0.836
kube_horizontalpodautoscaler_status_target_metric{container="sidecar",horizontalpodautoscaler="hpa-container-resource",metric_name="cpu",metric_target_type="utilization",namespace="default"} 69
kube_horizontalpodautoscaler_status_condition{condition="AbleToScale",horizontalpodautoscaler="hpa-container-resource",namespace="default",status="false"} 0
kube_horizontalpodautoscaler_status_condition{condition="AbleToScale",horizontalpodautoscaler="hpa-container-resource",namespace="default",status="true"} 1
kube_horizontalpodautoscaler_status_condition{condition="AbleToScale",horizontalpodautoscaler="hpa-container-resource",namespace="default",status="unknown"} 0
kube_horizontalpodautoscaler_status_current_replicas{horizontalpodautoscaler="hpa-container-resource",namespace="default"} 2
kube_horizontalpodautoscaler_status_desired_replicas{horizontalpodautoscaler="hpa-container-resource",namespace="default"} 3
kube_horizontalpodautoscaler_created{horizontalpodautoscaler="hpa-container-resource",namespace="default"} 1.5e+09
`,
MetricNames: []string{
"kube_horizontalpodautoscaler_info",
"kube_horizontalpodautoscaler_metadata_generation",
"kube_horizontalpodautoscaler_spec_max_replicas",
"kube_horizontalpodautoscaler_spec_min_replicas",
"kube_horizontalpodautoscaler_spec_target_metric",
"kube_horizontalpodautoscaler_status_target_metric",
"kube_horizontalpodautoscaler_status_current_replicas",
"kube_horizontalpodautoscaler_status_desired_replicas",
"kube_horizontalpodautoscaler_status_condition",
"kube_horizontalpodautoscaler_created",
},
},
}
for i, c := range cases {
c.Func = generator.ComposeMetricGenFuncs(hpaMetricFamilies(c.AllowAnnotationsList, c.AllowLabelsList))
Expand Down
Loading