Fix beats stack monitoring cert reload (#8833)

Fix beats stack monitoring certificates not reloading automatically.

---------

Signed-off-by: Michael Montgomery <[email protected]>
Co-authored-by: Peter Brachwitz <[email protected]>

Author: naemono

pkg/controller/beat/common/stackmon/stackmon_test.go

@@ -117,6 +117,34 @@ output:
         ssl:
             verification_mode: certificate
         username: es-user
+`
+	beatYmlGT88 := `http:
+    enabled: false
+metricbeat:
+    modules:
+        - hosts:
+            - http+unix:///var/shared/metricbeat-test-beat.sock
+          metricsets:
+            - stats
+            - state
+          module: beat
+          period: 10s
+          xpack:
+            enabled: true
+monitoring:
+    cluster_uuid: %s
+    enabled: false
+output:
+    elasticsearch:
+        hosts:
+            - %s
+        password: es-password
+        ssl:
+            restart_on_cert_change:
+                enabled: true
+                period: 1m
+            verification_mode: certificate
+        username: es-user
 `
 	standaloneBeatYML := `http:
     enabled: false
@@ -299,6 +327,22 @@ output:
 			want:    beatSidecarFixture(standaloneBeatYML),
 			wantErr: false,
 		},
+		{
+			name: "beat > 8.8 with stack monitoring enabled and valid elasticsearchRef returns properly configured sidecar",
+			args: args{
+				client: k8s.NewFakeClient(&beatFixture, &esFixture, &monitoringEsFixture, &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{Name: "es-secret-name", Namespace: "test"},
+					Data:       map[string][]byte{"es-user": []byte("es-password")},
+				}),
+				beat: func() *v1beta1.Beat {
+					beat := beatFixture.DeepCopy()
+					beat.Spec.Version = "8.8.0"
+					return beat
+				},
+			},
+			want:    beatSidecarFixture(fmt.Sprintf(beatYmlGT88, "abcd1234", "es-metrics-monitoring-url")),
+			wantErr: false,
+		},
 		{
 			name: "Beat with stack monitoring enabled and remote elasticsearchRef",
 			args: args{
@@ -349,8 +393,8 @@ output:
 				t.Errorf("MetricBeat() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			if !cmp.Equal(got, tt.want, cmpopts.IgnoreFields(stackmon.BeatSidecar{}, "ConfigHash")) {
-				t.Errorf("MetricBeat() = diff: %s", cmp.Diff(got, tt.want, cmpopts.IgnoreFields(stackmon.BeatSidecar{}, "ConfigHash")))
+			if !cmp.Equal(got, tt.want, cmpopts.IgnoreFields(stackmon.BeatSidecar{}, "ConfigHash"), cmpopts.IgnoreFields(corev1.Container{}, "Image")) {
+				t.Errorf("MetricBeat() = diff: %s", cmp.Diff(got, tt.want, cmpopts.IgnoreFields(stackmon.BeatSidecar{}, "ConfigHash"), cmpopts.IgnoreFields(corev1.Container{}, "Image")))
 			}
 		})
 	}

pkg/controller/common/stackmon/config.go

@@ -24,6 +24,7 @@ import (
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/metadata"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/settings"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/stackmon/monitoring"
+	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/volume"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/utils/k8s"
 )
@@ -40,6 +41,7 @@ func newBeatConfig(
 	ctx context.Context,
 	client k8s.Client,
 	beatName string,
+	imageVersion string,
 	resource monitoring.HasMonitoring,
 	associations []commonv1.Association,
 	baseConfig string,
@@ -52,7 +54,7 @@ func newBeatConfig(
 	assoc := associations[0]
 
 	// build the output section of the beat configuration file
-	outputCfg, caVolume, err := buildOutputConfig(ctx, client, assoc)
+	outputCfg, caVolume, err := buildOutputConfig(ctx, client, assoc, imageVersion)
 	if err != nil {
 		return beatConfig{}, err
 	}
@@ -112,7 +114,7 @@ func newBeatConfig(
 	}, err
 }
 
-func buildOutputConfig(ctx context.Context, client k8s.Client, assoc commonv1.Association) (map[string]interface{}, volume.VolumeLike, error) {
+func buildOutputConfig(ctx context.Context, client k8s.Client, assoc commonv1.Association, imageVersion string) (map[string]interface{}, volume.VolumeLike, error) {
 	credentials, err := association.ElasticsearchAuthSettings(ctx, client, assoc)
 	if err != nil {
 		return nil, volume.SecretVolume{}, err
@@ -132,6 +134,19 @@ func buildOutputConfig(ctx context.Context, client k8s.Client, assoc commonv1.As
 	// and therefore not being valid for the internal URL.
 	outputConfig["ssl.verification_mode"] = "certificate"
 
+	v, err := version.Parse(imageVersion)
+	if err != nil {
+		return nil, nil, err
+	}
+	// Reloading of certificates is only supported for Beats >= 8.8.0.
+	if v.GE(version.MinFor(8, 8, 0)) {
+		// Allow beats to reload when the ssl certificate changes (renewals)
+		outputConfig["ssl.restart_on_cert_change"] = map[string]interface{}{
+			"enabled": true,
+			"period":  "1m",
+		}
+	}
+
 	caDirPath := fmt.Sprintf(
 		"/mnt/elastic-internal/%s-association/%s/%s/certs",
 		assoc.AssociationType(), assoc.AssociationRef().Namespace, assoc.AssociationRef().NameOrSecretName(),

pkg/controller/common/stackmon/config_test.go

@@ -25,6 +25,7 @@ func Test_newBeatConfig(t *testing.T) {
 		initObjects []client.Object
 		beatName    string
 		baseConfig  string
+		image       string
 		associated  commonv1.Associated
 	}
 	tests := []struct {
@@ -52,6 +53,7 @@ param2: value2
 					},
 				},
 				beatName: "metricbeat",
+				image:    "8.8.0",
 				associated: &esv1.Elasticsearch{
 					ObjectMeta: metav1.ObjectMeta{
 						Name:      "monitored",
@@ -91,6 +93,9 @@ param2: value2
         ssl:
             certificate_authorities:
                 - /mnt/elastic-internal/es-monitoring-association/default/monitoring/certs/ca.crt
+            restart_on_cert_change:
+                enabled: true
+                period: 1m
             verification_mode: certificate
         username: default-monitored-default-monitoring-beat-es-mon-user
 param1: value1
@@ -112,6 +117,7 @@ param2: value2
 				context.Background(),
 				fakeClient,
 				tt.args.beatName,
+				tt.args.image,
 				hasMonitoring,
 				tt.args.associated.GetAssociations(),
 				tt.args.baseConfig,

pkg/controller/common/stackmon/sidecar.go

@@ -38,7 +38,7 @@ func NewMetricBeatSidecar(
 	image := container.ImageRepository(container.MetricbeatImage, imageVersion)
 	// EmptyDir volume so that MetricBeat does not write in the container image, which allows ReadOnlyRootFilesystem: true
 	emptyDir := volume.NewEmptyDirVolume("metricbeat-data", "/usr/share/metricbeat/data")
-	return NewBeatSidecar(ctx, client, "metricbeat", image, resource, monitoring.GetMetricsAssociation(resource), baseConfig, meta, caVolume, emptyDir)
+	return NewBeatSidecar(ctx, client, "metricbeat", image, imageVersion.String(), resource, monitoring.GetMetricsAssociation(resource), baseConfig, meta, caVolume, emptyDir)
 }
 
 func NewFileBeatSidecar(
@@ -57,7 +57,7 @@ func NewFileBeatSidecar(
 	image := container.ImageRepository(container.FilebeatImage, v)
 	// EmptyDir volume so that FileBeat does not write in the container image, which allows ReadOnlyRootFilesystem: true
 	emptyDir := volume.NewEmptyDirVolume("filebeat-data", "/usr/share/filebeat/data")
-	return NewBeatSidecar(ctx, client, "filebeat", image, resource, monitoring.GetLogsAssociation(resource), baseConfig, meta, additionalVolume, emptyDir)
+	return NewBeatSidecar(ctx, client, "filebeat", image, imageVersion, resource, monitoring.GetLogsAssociation(resource), baseConfig, meta, additionalVolume, emptyDir)
 }
 
 // BeatSidecar helps with building a beat sidecar container to monitor an Elastic Stack application. It focuses on
@@ -69,11 +69,11 @@ type BeatSidecar struct {
 	Volumes      []corev1.Volume
 }
 
-func NewBeatSidecar(ctx context.Context, client k8s.Client, beatName string, image string, resource monitoring.HasMonitoring,
+func NewBeatSidecar(ctx context.Context, client k8s.Client, beatName string, image string, imageVersion string, resource monitoring.HasMonitoring,
 	associations []commonv1.Association, baseConfig string, meta metadata.Metadata, additionalVolumes ...volume.VolumeLike,
 ) (BeatSidecar, error) {
 	// build the beat config
-	config, err := newBeatConfig(ctx, client, beatName, resource, associations, baseConfig, meta)
+	config, err := newBeatConfig(ctx, client, beatName, imageVersion, resource, associations, baseConfig, meta)
 	if err != nil {
 		return BeatSidecar{}, err
 	}