Skip to content

Commit 19a7548

Browse files
duhminickduhminick
authored
Add automatic rolling deployment & option to re-create admission webhook certs (#242)
* Flag to automatically roll operator deployment * Add recreate flag for auto-generated certs * Add tests for rolling and cert recreate * Fix after rebase * Use common consts
1 parent c2b1ea8 commit 19a7548

File tree

18 files changed

+384
-17
lines changed

18 files changed

+384
-17
lines changed

‎.github/workflows/amazon-cloudwatch-observability-integration-test.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,10 @@ jobs:
4848
- appsignals-unsupported
4949
- webhooks-partially-enabled
5050
- webhooks-configured
51+
- deployment-rolling-enabled
52+
- deployment-rolling-disabled
53+
- certificate-recreate-enabled
54+
- certificate-recreate-disabled
5155
steps:
5256
- uses: actions/checkout@v3
5357

‎charts/amazon-cloudwatch-observability/templates/_helpers.tpl‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -408,4 +408,32 @@ Get namespaceSelector value for admission webhooks
408408
{{- end -}}
409409
{{- end -}}
410410

411-
411+
{{/*
412+
Returns auto-generated certificate and CA for admission webhooks.
413+
*/}}
414+
{{- define "amazon-cloudwatch-observability.webhookCert" -}}
415+
{{- $tlsCrt := "" }}
416+
{{- $tlsKey := "" }}
417+
{{- $caCrt := "" }}
418+
{{- if .Values.admissionWebhooks.autoGenerateCert.enabled }}
419+
{{- $existingCert := ( lookup "v1" "Secret" .Release.Namespace (include "amazon-cloudwatch-observability.certificateSecretName" .) ) }}
420+
{{- if and (not .Values.admissionWebhooks.autoGenerateCert.recreate) $existingCert }}
421+
{{- $tlsCrt = index $existingCert "data" "tls.crt" }}
422+
{{- $tlsKey = index $existingCert "data" "tls.key" }}
423+
{{- $caCrt = index $existingCert "data" "ca.crt" }}
424+
{{- if not $caCrt }}
425+
{{- $existingWebhook := ( lookup "admissionregistration.k8s.io/v1" "MutatingWebhookConfiguration" "" (printf "%s-mutating-webhook-configuration" (include "amazon-cloudwatch-observability.name" .)) ) }}
426+
{{- $caCrt = (first $existingWebhook.webhooks).clientConfig.caBundle }}
427+
{{- end }}
428+
{{- else }}
429+
{{- $altNames := list ( printf "%s-webhook-service.%s" (include "amazon-cloudwatch-observability.name" .) .Release.Namespace ) ( printf "%s-webhook-service.%s.svc" (include "amazon-cloudwatch-observability.name" .) .Release.Namespace ) ( printf "%s-webhook-service.%s.svc.cluster.local" (include "amazon-cloudwatch-observability.name" .) .Release.Namespace ) -}}
430+
{{- $ca := genCA ( printf "%s-ca" (include "amazon-cloudwatch-observability.name" .) ) ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) -}}
431+
{{- $cert := genSignedCert (include "amazon-cloudwatch-observability.name" .) nil $altNames ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) $ca -}}
432+
{{- $tlsCrt = b64enc $cert.Cert }}
433+
{{- $tlsKey = b64enc $cert.Key }}
434+
{{- $caCrt = b64enc $ca.Cert }}
435+
{{- end }}
436+
{{- $result := dict "Cert" $tlsCrt "Key" $tlsKey "Ca" $caCrt }}
437+
{{- $result | toYaml }}
438+
{{- end }}
439+
{{- end }}

‎charts/amazon-cloudwatch-observability/templates/admission-webhooks/operator-webhook.yaml‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
{{- if and (.Values.admissionWebhooks.autoGenerateCert.enabled) (not .Values.admissionWebhooks.certManager.enabled) (include "amazon-cloudwatch-observability.webhookEnabled" .) }}
2-
{{- $altNames := list ( printf "%s-webhook-service.%s" (include "amazon-cloudwatch-observability.name" .) .Release.Namespace ) ( printf "%s-webhook-service.%s.svc" (include "amazon-cloudwatch-observability.name" .) .Release.Namespace ) ( printf "%s-webhook-service.%s.svc.cluster.local" (include "amazon-cloudwatch-observability.name" .) .Release.Namespace ) -}}
3-
{{- $ca := genCA ( printf "%s-ca" (include "amazon-cloudwatch-observability.name" .) ) ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) -}}
4-
{{- $cert := genSignedCert (include "amazon-cloudwatch-observability.name" .) nil $altNames ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) $ca -}}
2+
{{- $cert := fromYaml (include "amazon-cloudwatch-observability.webhookCert" .) }}
53
apiVersion: v1
64
kind: Secret
75
type: kubernetes.io/tls
@@ -11,8 +9,8 @@ metadata:
119
name: {{ template "amazon-cloudwatch-observability.certificateSecretName" . }}
1210
namespace: {{ .Release.Namespace }}
1311
data:
14-
tls.crt: {{ $cert.Cert | b64enc }}
15-
tls.key: {{ $cert.Key | b64enc }}
12+
tls.crt: {{ $cert.Cert }}
13+
tls.key: {{ $cert.Key }}
1614
---
1715
apiVersion: admissionregistration.k8s.io/v1
1816
kind: MutatingWebhookConfiguration
@@ -29,7 +27,7 @@ webhooks:
2927
name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
3028
namespace: {{ .Release.Namespace }}
3129
path: /mutate-cloudwatch-aws-amazon-com-v1alpha1-instrumentation
32-
caBundle: {{ $ca.Cert | b64enc }}
30+
caBundle: {{ $cert.Ca }}
3331
failurePolicy: {{ .Values.admissionWebhooks.instrumentations.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
3432
name: minstrumentation.kb.io
3533
namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "instrumentations") }}
@@ -58,7 +56,7 @@ webhooks:
5856
name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
5957
namespace: {{ .Release.Namespace }}
6058
path: /mutate-cloudwatch-aws-amazon-com-v1alpha1-amazoncloudwatchagent
61-
caBundle: {{ $ca.Cert | b64enc }}
59+
caBundle: {{ $cert.Ca }}
6260
failurePolicy: {{ .Values.admissionWebhooks.agents.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
6361
name: mamazoncloudwatchagent.kb.io
6462
namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "agents") }}
@@ -87,7 +85,7 @@ webhooks:
8785
name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
8886
namespace: {{ .Release.Namespace }}
8987
path: /mutate-v1-pod
90-
caBundle: {{ $ca.Cert | b64enc }}
88+
caBundle: {{ $cert.Ca }}
9189
failurePolicy: {{ .Values.admissionWebhooks.pods.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
9290
name: mpod.kb.io
9391
namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "pods") }}
@@ -116,7 +114,7 @@ webhooks:
116114
name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
117115
namespace: {{ .Release.Namespace }}
118116
path: /mutate-v1-namespace
119-
caBundle: {{ $ca.Cert | b64enc }}
117+
caBundle: {{ $cert.Ca }}
120118
failurePolicy: {{ .Values.admissionWebhooks.namespaces.failurePolicy | default .Values.admissionWebhooks.pods.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
121119
name: mnamespace.kb.io
122120
namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "namespaces") }}
@@ -145,7 +143,7 @@ webhooks:
145143
name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
146144
namespace: {{ .Release.Namespace }}
147145
path: /mutate-v1-workload
148-
caBundle: {{ $ca.Cert | b64enc }}
146+
caBundle: {{ $cert.Ca }}
149147
failurePolicy: {{ .Values.admissionWebhooks.workloads.failurePolicy | default .Values.admissionWebhooks.pods.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
150148
name: mworkload.kb.io
151149
namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "workloads") }}
@@ -184,7 +182,7 @@ webhooks:
184182
name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
185183
namespace: {{ .Release.Namespace }}
186184
path: /validate-cloudwatch-aws-amazon-com-v1alpha1-instrumentation
187-
caBundle: {{ $ca.Cert | b64enc }}
185+
caBundle: {{ $cert.Ca }}
188186
failurePolicy: {{ .Values.admissionWebhooks.instrumentations.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
189187
name: vinstrumentationcreateupdate.kb.io
190188
namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "instrumentations") }}
@@ -213,7 +211,7 @@ webhooks:
213211
name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
214212
namespace: {{ .Release.Namespace }}
215213
path: /validate-cloudwatch-aws-amazon-com-v1alpha1-instrumentation
216-
caBundle: {{ $ca.Cert | b64enc }}
214+
caBundle: {{ $cert.Ca }}
217215
failurePolicy: Ignore
218216
name: vinstrumentationdelete.kb.io
219217
namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "instrumentations") }}
@@ -241,7 +239,7 @@ webhooks:
241239
name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
242240
namespace: {{ .Release.Namespace }}
243241
path: /validate-cloudwatch-aws-amazon-com-v1alpha1-amazoncloudwatchagent
244-
caBundle: {{ $ca.Cert | b64enc }}
242+
caBundle: {{ $cert.Ca }}
245243
failurePolicy: {{ .Values.admissionWebhooks.agents.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
246244
name: vamazoncloudwatchagentcreateupdate.kb.io
247245
namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "agents") }}
@@ -270,7 +268,7 @@ webhooks:
270268
name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
271269
namespace: {{ .Release.Namespace }}
272270
path: /validate-cloudwatch-aws-amazon-com-v1alpha1-amazoncloudwatchagent
273-
caBundle: {{ $ca.Cert | b64enc }}
271+
caBundle: {{ $cert.Ca }}
274272
failurePolicy: Ignore
275273
name: vamazoncloudwatchagentdelete.kb.io
276274
namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "agents") }}

‎charts/amazon-cloudwatch-observability/templates/operator-deployment.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ spec:
1818
{{- if .Values.manager.podAnnotations }}
1919
{{- include "amazon-cloudwatch-observability.podAnnotations" . | nindent 8 }}
2020
{{- end }}
21+
{{- if .Values.manager.rolling }}
22+
rollme: {{ randAlphaNum 5 | quote }}
23+
{{- end }}
2124
labels:
2225
app.kubernetes.io/name: {{ template "amazon-cloudwatch-observability.name" . }}
2326
control-plane: controller-manager

‎charts/amazon-cloudwatch-observability/values.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1208,6 +1208,8 @@ manager:
12081208
affinity: {}
12091209
nodeSelector:
12101210
kubernetes.io/os: linux
1211+
# Enable automatic rolling by forcing a deployment spec change
1212+
rolling: false
12111213
## Admission webhooks make sure only requests with correctly formatted rules will get into the Operator.
12121214
admissionWebhooks:
12131215
create: true
@@ -1256,6 +1258,7 @@ admissionWebhooks:
12561258
autoGenerateCert:
12571259
enabled: true
12581260
expiryDays: 3650 # 10 years
1261+
recreate: true
12591262
## TLS Certificate Option 2: Use certManager to generate self-signed certificate.
12601263
## certManager must be enabled. If enabled, it takes precedence over option 1.
12611264
certManager:

‎integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/certificate-recreate-disabled/main.tf‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
module "base" {
5+
source = "../.."
6+
helm_dir = var.helm_dir
7+
helm_values_file = "${path.module}/values.yaml"
8+
}
9+
10+
variable "helm_dir" {
11+
type = string
12+
default = "../../../../../../charts/amazon-cloudwatch-observability"
13+
}
14+
15+
resource "null_resource" "validator" {
16+
depends_on = [module.base.helm_release]
17+
18+
provisioner "local-exec" {
19+
command = <<-EOT
20+
go test ${var.test_dir} -v -run=TestCertificateRecreateDisabled_Save
21+
helm upgrade --wait --create-namespace --namespace amazon-cloudwatch amazon-cloudwatch-observability ${var.helm_dir} -f ${path.module}/values.yaml
22+
go test ${var.test_dir} -v -run=TestCertificateRecreateDisabled_Compare
23+
EOT
24+
}
25+
}
26+
27+
variable "test_dir" {
28+
type = string
29+
default = "../../../../validations/minikube/scenarios"
30+
}
31+

‎integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/certificate-recreate-disabled/values.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
region: us-west-2
2+
clusterName: minikube
3+
4+
admissionWebhooks:
5+
autoGenerateCert:
6+
recreate: false

‎integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/certificate-recreate-enabled/main.tf‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
module "base" {
5+
source = "../.."
6+
helm_dir = var.helm_dir
7+
helm_values_file = "${path.module}/values.yaml"
8+
}
9+
10+
variable "helm_dir" {
11+
type = string
12+
default = "../../../../../../charts/amazon-cloudwatch-observability"
13+
}
14+
15+
resource "null_resource" "validator" {
16+
depends_on = [module.base.helm_release]
17+
18+
provisioner "local-exec" {
19+
command = <<-EOT
20+
go test ${var.test_dir} -v -run=TestCertificateRecreateEnabled_Save
21+
helm upgrade --wait --create-namespace --namespace amazon-cloudwatch amazon-cloudwatch-observability ${var.helm_dir} -f ${path.module}/values.yaml
22+
go test ${var.test_dir} -v -run=TestCertificateRecreateEnabled_Compare
23+
EOT
24+
}
25+
}
26+
27+
variable "test_dir" {
28+
type = string
29+
default = "../../../../validations/minikube/scenarios"
30+
}

‎integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/certificate-recreate-enabled/values.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
region: us-west-2
2+
clusterName: minikube
3+
4+
admissionWebhooks:
5+
autoGenerateCert:
6+
recreate: true

‎integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/deployment-rolling-disabled/main.tf‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
module "base" {
5+
source = "../.."
6+
helm_dir = var.helm_dir
7+
helm_values_file = "${path.module}/values.yaml"
8+
}
9+
10+
variable "helm_dir" {
11+
type = string
12+
default = "../../../../../../charts/amazon-cloudwatch-observability"
13+
}
14+
15+
resource "null_resource" "validator" {
16+
depends_on = [module.base.helm_release]
17+
18+
provisioner "local-exec" {
19+
command = "go test ${var.test_dir} -v -run=TestDeploymentRollingDisabled"
20+
}
21+
}
22+
23+
variable "test_dir" {
24+
type = string
25+
default = "../../../../validations/minikube/scenarios"
26+
}
27+
28+

0 commit comments

Comments
 (0)