Skip to content

Commit fa5cebb

Browse files
authored
Merge pull request #1130 from fluxcd/StrictPostBuildSubstitutions
Add `StrictPostBuildSubstitutions` feature flag
2 parents b2daff1 + b810013 commit fa5cebb

File tree

7 files changed

+174
-39
lines changed

7 files changed

+174
-39
lines changed

docs/spec/v1/kustomizations.md

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -564,8 +564,7 @@ kind: Kustomization
564564
metadata:
565565
name: apps
566566
spec:
567-
interval: 5m
568-
path: "./apps/"
567+
# ...omitted for brevity
569568
postBuild:
570569
substitute:
571570
cluster_env: "prod"
@@ -605,6 +604,11 @@ will print out `${var}`.
605604
All the undefined variables in the format `${var}` will be substituted with an
606605
empty string unless a default value is provided e.g. `${var:=default}`.
607606

607+
**Note:** It is recommended to set the `--feature-gates=StrictPostBuildSubstitutions=true`
608+
controller flag, so that the post-build substitutions will fail if a
609+
variable without a default value is declared in files but is
610+
missing from the input vars.
611+
608612
You can disable the variable substitution for certain resources by either
609613
labelling or annotating them with:
610614

@@ -624,6 +628,7 @@ kind: Kustomization
624628
metadata:
625629
name: apps
626630
spec:
631+
# ...omitted for brevity
627632
postBuild:
628633
substitute:
629634
var_substitution_enabled: "true"
@@ -635,13 +640,11 @@ enclosed in double quotes vars to be treated as strings, for more information se
635640

636641
You can replicate the controller post-build substitutions locally using
637642
[kustomize](https://github.com/kubernetes-sigs/kustomize)
638-
and Drone's [envsubst](https://github.com/drone/envsubst):
643+
and the Flux CLI:
639644

640645
```console
641-
$ go install github.com/drone/envsubst/cmd/envsubst
642-
643646
$ export cluster_region=eu-central-1
644-
$ kustomize build ./apps/ | $GOPATH/bin/envsubst
647+
$ kustomize build ./apps/ | flux envsubst --strict
645648
---
646649
apiVersion: v1
647650
kind: Namespace

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ require (
2323
github.com/fluxcd/pkg/apis/kustomize v1.4.0
2424
github.com/fluxcd/pkg/apis/meta v1.4.0
2525
github.com/fluxcd/pkg/http/fetch v0.10.0
26-
github.com/fluxcd/pkg/kustomize v1.8.0
26+
github.com/fluxcd/pkg/kustomize v1.9.0
2727
github.com/fluxcd/pkg/runtime v0.46.0
2828
github.com/fluxcd/pkg/ssa v0.38.0
2929
github.com/fluxcd/pkg/tar v0.6.0
@@ -96,12 +96,12 @@ require (
9696
github.com/docker/docker v24.0.9+incompatible // indirect
9797
github.com/docker/go-connections v0.4.0 // indirect
9898
github.com/docker/go-units v0.4.0 // indirect
99-
github.com/drone/envsubst v1.0.3 // indirect
10099
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
101100
github.com/evanphx/json-patch v5.7.0+incompatible // indirect
102101
github.com/evanphx/json-patch/v5 v5.8.0 // indirect
103102
github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
104103
github.com/fatih/color v1.16.0 // indirect
104+
github.com/fluxcd/pkg/envsubst v1.0.0 // indirect
105105
github.com/fluxcd/pkg/sourceignore v0.6.0 // indirect
106106
github.com/fsnotify/fsnotify v1.7.0 // indirect
107107
github.com/getsops/gopgagent v0.0.0-20170926210634-4d7ea76ff71a // indirect

go.sum

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,6 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
116116
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
117117
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
118118
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
119-
github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g=
120-
github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g=
121119
github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
122120
github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
123121
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -143,10 +141,12 @@ github.com/fluxcd/pkg/apis/kustomize v1.4.0 h1:SXoGN9M31fW5tO+wpKMnyHXbjxGUqDo7Y
143141
github.com/fluxcd/pkg/apis/kustomize v1.4.0/go.mod h1:bZklVWB11tELMss89qYzgg4ClzhFzp0Hm4/8EiHgKew=
144142
github.com/fluxcd/pkg/apis/meta v1.4.0 h1:nNdgB6FFHP3cubxZCViaCFDUVlAbpq9+hvKEIveOGMg=
145143
github.com/fluxcd/pkg/apis/meta v1.4.0/go.mod h1:81sZ01ShTuLc1C3M1dFJNkINareBysvmrO1b8zJFFKs=
144+
github.com/fluxcd/pkg/envsubst v1.0.0 h1:LD86BRNSCGJrvyrH2aX5/pit7RfbFpkzRXogwcazLVk=
145+
github.com/fluxcd/pkg/envsubst v1.0.0/go.mod h1:VAcb4OxcRdsDix1TRtr/mtTqFGHmNQaOvXQO2REArFQ=
146146
github.com/fluxcd/pkg/http/fetch v0.10.0 h1:Uh1ZrPa4B4EDgi+NFrY7qP6g9vg1O6JHKg3+iJLtt1w=
147147
github.com/fluxcd/pkg/http/fetch v0.10.0/go.mod h1:zZOsAqn7iODap40PVq29mcCPEKjDodYvamEaoN6tV/Q=
148-
github.com/fluxcd/pkg/kustomize v1.8.0 h1:Vf1UwnoP3yScaLi/QrDjgN2d2nI6LcmX4tNRoH+sypY=
149-
github.com/fluxcd/pkg/kustomize v1.8.0/go.mod h1:yszv9tkYrnC01mcGPct8+bdxpTyxf69k1kmSvk7w0zs=
148+
github.com/fluxcd/pkg/kustomize v1.9.0 h1:bqS3mXiK1q5TpUtIO5I5b+v/0r96NGJBiearKGUhicA=
149+
github.com/fluxcd/pkg/kustomize v1.9.0/go.mod h1:PBerk0KzZN/IXaGociVp4MSMvsUQB0jR1P2SqSdixz0=
150150
github.com/fluxcd/pkg/runtime v0.46.0 h1:+pxFwTk8j8lZIS9Vyc8EJbgvmFp9JqeT6pfLo/0iP98=
151151
github.com/fluxcd/pkg/runtime v0.46.0/go.mod h1:d9BaIjqoHL71fYeZsssrt08UFONGN2WQRaJ/Ay2d1Cc=
152152
github.com/fluxcd/pkg/sourceignore v0.6.0 h1:kD6QXL/upPEX66UpR669yK1Bxr/GtjzmZiqBeYpunUQ=

internal/controller/kustomization_controller.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ type KustomizationReconciler struct {
9898
KubeConfigOpts runtimeClient.KubeConfigOptions
9999
ConcurrentSSA int
100100
DisallowedFieldManagers []string
101+
StrictSubstitutions bool
101102
}
102103

103104
// KustomizationReconcilerOptions contains options for the KustomizationReconciler.
@@ -622,9 +623,10 @@ func (r *KustomizationReconciler) build(ctx context.Context,
622623

623624
// run variable substitutions
624625
if obj.Spec.PostBuild != nil {
625-
outRes, err := generator.SubstituteVariables(ctx, r.Client, u, res, false)
626+
outRes, err := generator.SubstituteVariables(ctx, r.Client, u, res,
627+
generator.SubstituteWithStrict(r.StrictSubstitutions))
626628
if err != nil {
627-
return nil, fmt.Errorf("var substitution failed for '%s': %w", res.GetName(), err)
629+
return nil, fmt.Errorf("post build failed for '%s': %w", res.GetName(), err)
628630
}
629631

630632
if outRes != nil {

internal/controller/kustomization_varsub_test.go

Lines changed: 140 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -366,10 +366,11 @@ func TestKustomizationReconciler_VarsubNumberBool(t *testing.T) {
366366
manifests := func(name string) []testserver.File {
367367
return []testserver.File{
368368
{
369-
Name: "service-account.yaml",
369+
Name: "templates.yaml",
370370
Body: fmt.Sprintf(`
371-
apiVersion: v1
372-
kind: ServiceAccount
371+
---
372+
apiVersion: source.toolkit.fluxcd.io/v1
373+
kind: GitRepository
373374
metadata:
374375
name: %[1]s
375376
namespace: %[1]s
@@ -379,6 +380,29 @@ metadata:
379380
annotations:
380381
id: ${q}${number}${q}
381382
enabled: ${q}${boolean}${q}
383+
spec:
384+
interval: ${number}m
385+
url: https://host/repo
386+
---
387+
apiVersion: v1
388+
kind: ConfigMap
389+
metadata:
390+
name: %[1]s
391+
namespace: %[1]s
392+
data:
393+
id: ${q}${number}${q}
394+
text: |
395+
This variable is escaped $${var}
396+
397+
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus at
398+
nisl sem. Nullam nec dui ipsum. Nam vehicula volutpat ipsum, ac fringilla
399+
nisl convallis sed. Aliquam porttitor turpis finibus, finibus velit ut,
400+
imperdiet mauris. Cras nec neque nulla. Maecenas semper nulla et elit
401+
dictum sagittis. Quisque tincidunt non diam non ullamcorper. Curabitur
402+
pretium urna odio, vitae ullamcorper purus mollis sit amet. Nam ac lectus
403+
ac arcu varius feugiat id fringilla massa.
404+
405+
\?
382406
`, name),
383407
},
384408
}
@@ -423,35 +447,126 @@ metadata:
423447
"boolean": "true",
424448
},
425449
},
426-
Wait: true,
450+
Wait: false,
427451
},
428452
}
429453
g.Expect(k8sClient.Create(ctx, inputK)).Should(Succeed())
430454

431-
resultSA := &corev1.ServiceAccount{}
455+
g.Eventually(func() bool {
456+
resultK := &kustomizev1.Kustomization{}
457+
_ = k8sClient.Get(ctx, client.ObjectKeyFromObject(inputK), resultK)
458+
for _, c := range resultK.Status.Conditions {
459+
if c.Reason == kustomizev1.ReconciliationSucceededReason {
460+
return true
461+
}
462+
}
463+
return false
464+
}, timeout, interval).Should(BeTrue())
465+
466+
resultRepo := &sourcev1.GitRepository{}
467+
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: id, Namespace: id}, resultRepo)).Should(Succeed())
468+
g.Expect(resultRepo.Labels["id"]).To(Equal("123"))
469+
g.Expect(resultRepo.Annotations["id"]).To(Equal("123"))
470+
g.Expect(resultRepo.Labels["enabled"]).To(Equal("true"))
471+
g.Expect(resultRepo.Annotations["enabled"]).To(Equal("true"))
472+
473+
resultCM := &corev1.ConfigMap{}
474+
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: id, Namespace: id}, resultCM)).Should(Succeed())
475+
g.Expect(resultCM.Data["id"]).To(Equal("123"))
476+
g.Expect(resultCM.Data["text"]).To(ContainSubstring(`${var}`))
477+
g.Expect(resultCM.Data["text"]).ToNot(ContainSubstring(`$${var}`))
478+
g.Expect(resultCM.Data["text"]).To(ContainSubstring(`\?`))
479+
}
432480

433-
ensureReconciles := func(nameSuffix string) {
434-
t.Run("reconciles successfully"+nameSuffix, func(t *testing.T) {
435-
g.Eventually(func() bool {
436-
resultK := &kustomizev1.Kustomization{}
437-
_ = k8sClient.Get(ctx, client.ObjectKeyFromObject(inputK), resultK)
438-
for _, c := range resultK.Status.Conditions {
439-
if c.Reason == kustomizev1.ReconciliationSucceededReason {
440-
return true
441-
}
442-
}
443-
return false
444-
}, timeout, interval).Should(BeTrue())
481+
func TestKustomizationReconciler_VarsubStrict(t *testing.T) {
482+
reconciler.StrictSubstitutions = true
483+
defer func() {
484+
reconciler.StrictSubstitutions = false
485+
}()
445486

446-
g.Expect(k8sClient.Get(ctx, types.NamespacedName{Name: id, Namespace: id}, resultSA)).Should(Succeed())
447-
})
487+
ctx := context.Background()
488+
489+
g := NewWithT(t)
490+
id := "vars-" + randStringRunes(5)
491+
revision := "v1.0.0/" + randStringRunes(7)
492+
493+
err := createNamespace(id)
494+
g.Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
495+
496+
err = createKubeConfigSecret(id)
497+
g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret")
498+
499+
manifests := func(name string) []testserver.File {
500+
return []testserver.File{
501+
{
502+
Name: "service-account.yaml",
503+
Body: fmt.Sprintf(`
504+
apiVersion: v1
505+
kind: ServiceAccount
506+
metadata:
507+
name: %[1]s
508+
namespace: %[1]s
509+
labels:
510+
default: ${default:=test}
511+
missing: ${missing}
512+
`, name),
513+
},
514+
}
448515
}
449516

450-
ensureReconciles(" with optional ConfigMap")
451-
t.Run("replaces vars from optional ConfigMap", func(t *testing.T) {
452-
g.Expect(resultSA.Labels["id"]).To(Equal("123"))
453-
g.Expect(resultSA.Annotations["id"]).To(Equal("123"))
454-
g.Expect(resultSA.Labels["enabled"]).To(Equal("true"))
455-
g.Expect(resultSA.Annotations["enabled"]).To(Equal("true"))
517+
artifact, err := testServer.ArtifactFromFiles(manifests(id))
518+
g.Expect(err).NotTo(HaveOccurred())
519+
520+
repositoryName := types.NamespacedName{
521+
Name: randStringRunes(5),
522+
Namespace: id,
523+
}
524+
525+
err = applyGitRepository(repositoryName, artifact, revision)
526+
g.Expect(err).NotTo(HaveOccurred())
527+
528+
inputK := &kustomizev1.Kustomization{
529+
ObjectMeta: metav1.ObjectMeta{
530+
Name: id,
531+
Namespace: id,
532+
},
533+
Spec: kustomizev1.KustomizationSpec{
534+
KubeConfig: &meta.KubeConfigReference{
535+
SecretRef: meta.SecretKeyReference{
536+
Name: "kubeconfig",
537+
},
538+
},
539+
Interval: metav1.Duration{Duration: reconciliationInterval},
540+
Path: "./",
541+
Prune: true,
542+
SourceRef: kustomizev1.CrossNamespaceSourceReference{
543+
Kind: sourcev1.GitRepositoryKind,
544+
Name: repositoryName.Name,
545+
},
546+
PostBuild: &kustomizev1.PostBuild{
547+
Substitute: map[string]string{
548+
"test": "test",
549+
},
550+
},
551+
Wait: true,
552+
},
553+
}
554+
g.Expect(k8sClient.Create(ctx, inputK)).Should(Succeed())
555+
556+
var resultK kustomizev1.Kustomization
557+
t.Run("fails to reconcile", func(t *testing.T) {
558+
g.Eventually(func() bool {
559+
_ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(inputK), &resultK)
560+
for _, c := range resultK.Status.Conditions {
561+
if c.Reason == kustomizev1.BuildFailedReason {
562+
return true
563+
}
564+
}
565+
return false
566+
}, timeout, interval).Should(BeTrue())
456567
})
568+
569+
ready := apimeta.FindStatusCondition(resultK.Status.Conditions, meta.ReadyCondition)
570+
g.Expect(ready.Message).To(ContainSubstring("variable not set"))
571+
g.Expect(k8sClient.Delete(context.Background(), &resultK)).To(Succeed())
457572
}

internal/features/features.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ const (
3939
// DisableFailFastBehavior controls whether the fail-fast behavior when
4040
// waiting for resources to become ready should be disabled.
4141
DisableFailFastBehavior = "DisableFailFastBehavior"
42+
43+
// StrictPostBuildSubstitutions controls whether the post-build substitutions
44+
// should fail if a variable without a default value is declared in files
45+
// but is missing from the input vars.
46+
StrictPostBuildSubstitutions = "StrictPostBuildSubstitutions"
4247
)
4348

4449
var features = map[string]bool{
@@ -51,6 +56,9 @@ var features = map[string]bool{
5156
// DisableFailFastBehavior
5257
// opt-in from v1.1
5358
DisableFailFastBehavior: false,
59+
// StrictPostBuildSubstitutions
60+
// opt-in from v1.3
61+
StrictPostBuildSubstitutions: false,
5462
}
5563

5664
// FeatureGates contains a list of all supported feature gates and

main.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,12 @@ func main() {
228228
failFast = false
229229
}
230230

231+
strictSubstitutions, err := features.Enabled(features.StrictPostBuildSubstitutions)
232+
if err != nil {
233+
setupLog.Error(err, "unable to check feature gate "+features.StrictPostBuildSubstitutions)
234+
os.Exit(1)
235+
}
236+
231237
if err = (&controller.KustomizationReconciler{
232238
ControllerName: controllerName,
233239
DefaultServiceAccount: defaultServiceAccount,
@@ -242,6 +248,7 @@ func main() {
242248
PollingOpts: pollingOpts,
243249
StatusPoller: polling.NewStatusPoller(mgr.GetClient(), mgr.GetRESTMapper(), pollingOpts),
244250
DisallowedFieldManagers: disallowedFieldManagers,
251+
StrictSubstitutions: strictSubstitutions,
245252
}).SetupWithManager(ctx, mgr, controller.KustomizationReconcilerOptions{
246253
DependencyRequeueInterval: requeueDependency,
247254
HTTPRetry: httpRetry,

0 commit comments

Comments
 (0)