feat(test): run kubeconform per test by default

Author: qboileau

README.md

@@ -293,6 +293,8 @@ vigie test <chart>            template tier: render + assert (tests/unit/*_test.
   --file <path>               run a single test file instead of discovering all
   --tests <dir>               discovery root (default: <chart>/tests)
   --snapshot-dir <dir>        snapshot directory (default: <chart>/tests/snapshots)
+  --no-schema                 skip the per-test kubeconform pass (on by default)
+  --kube-version <ver>        Kubernetes version for the kubeconform pass (default: 1.36.1)
   --pass-on-warning           exit 0 on run warnings, e.g. no tests executed (default: exit 5)
 
 vigie validate <chart>        chart tier: render values.yaml + overlays, validate with kubeconform
@@ -336,6 +338,8 @@ validate:
 
 test:
   testsDir: tests/unit
+  skipSchema: false          # kubeconform runs per test by default; true opts out
+  kubeVersions: [1.36.1]     # first entry pins the kubeconform version
 ```
 
 ### Lint rule sets

cmd/vigie/test.go

@@ -17,6 +17,8 @@ var (
 	flagTestTestsDir      string
 	flagTestSnapshotDir   string
 	flagTestPassOnWarning bool
+	flagTestNoSchema      bool
+	flagTestKubeVersion   string
 )
 
 // exitWarnings is the exit code for a run that produced only warnings (e.g. no
@@ -42,6 +44,8 @@ func init() {
 	testCmd.Flags().StringVar(&flagTestTestsDir, "tests", "", "Directory to scan recursively for *_test.yaml (overrides test.testsDir; default: <chart>/tests)")
 	testCmd.Flags().StringVar(&flagTestSnapshotDir, "snapshot-dir", "", "Directory for snapshot files (default: <chart>/tests/snapshots)")
 	testCmd.Flags().BoolVar(&flagTestPassOnWarning, "pass-on-warning", false, "Exit 0 on run warnings such as no tests executed (default: exit 5)")
+	testCmd.Flags().BoolVar(&flagTestNoSchema, "no-schema", false, "Skip the per-test kubeconform pass (on by default)")
+	testCmd.Flags().StringVar(&flagTestKubeVersion, "kube-version", "", "Kubernetes version for the per-test kubeconform pass (default: 1.36.1)")
 	rootCmd.AddCommand(testCmd)
 }
 
@@ -50,6 +54,10 @@ func runTestCmd(cmd *cobra.Command, args []string) error {
 
 	slog.Debug("invoked", "command", "test", "chart", chartPath, "parallelism", flagParallelism)
 
+	if err := config.ValidateKubeVersion("--kube-version", flagTestKubeVersion); err != nil {
+		exitErr(3, "%v", err)
+	}
+
 	cfg, err := config.Load(chartPath)
 	if err != nil {
 		exitErr(3, "loading config: %v", err)
@@ -76,12 +84,21 @@ func runTestCmd(cmd *cobra.Command, args []string) error {
 	if len(files) == 0 {
 		warnings = append(warnings, fmt.Sprintf("no unit test files found under %s", displayTestsRoot(chartPath, testsDir)))
 	} else {
+		// Schema validation is on by default; --no-schema or test.skipSchema disables it.
+		skipSchema := flagTestNoSchema || cfg.Test.SkipSchema
+		kubeVersion := flagTestKubeVersion
+		if kubeVersion == "" && len(cfg.Test.KubeVersions) > 0 {
+			kubeVersion = cfg.Test.KubeVersions[0]
+		}
+
 		opts := runner.Options{
-			ChartPath:   chartPath,
-			TestFiles:   files,
-			Parallelism: flagParallelism,
-			Cfg:         cfg,
-			SnapshotDir: flagTestSnapshotDir,
+			ChartPath:       chartPath,
+			TestFiles:       files,
+			Parallelism:     flagParallelism,
+			Cfg:             cfg,
+			SnapshotDir:     flagTestSnapshotDir,
+			ValidateSchemas: !skipSchema,
+			KubeVersion:     kubeVersion,
 		}
 
 		results, err := runner.Run(opts)

internal/runner/runner.go

@@ -43,12 +43,14 @@ type SuiteResult struct {
 
 // Options controls runner behavior.
 type Options struct {
-	ChartPath      string
-	TestFiles      []string
-	Parallelism    int
-	Cfg            *config.Config
-	SnapshotDir    string // default: "<chartPath>/tests/snapshots"
-	SnapshotUpdate bool
+	ChartPath       string
+	TestFiles       []string
+	Parallelism     int
+	Cfg             *config.Config
+	SnapshotDir     string // default: "<chartPath>/tests/snapshots"
+	SnapshotUpdate  bool
+	ValidateSchemas bool   // when true, kubeconform-validate each test's rendered docs
+	KubeVersion     string // kubernetes version for schema validation; empty = config.DefaultKubeVersion
 }
 
 // resolveSnapshotDir returns the snapshot directory: the explicit override
@@ -70,14 +72,30 @@ type expandedTest struct {
 
 // Run executes all test files and returns suite-level results.
 func Run(opts Options) ([]SuiteResult, error) {
-	slog.Debug("starting runner", "files", len(opts.TestFiles), "parallelism", opts.Parallelism)
+	slog.Debug("starting runner", "files", len(opts.TestFiles), "parallelism", opts.Parallelism, "validateSchemas", opts.ValidateSchemas)
+
+	// Build one schema validator shared across all files/tests; its in-memory
+	// schema cache then amortises across every rendered document.
+	var schemaValidator *render.SchemaValidator
+	if opts.ValidateSchemas {
+		kubeVer := opts.KubeVersion
+		if kubeVer == "" {
+			kubeVer = config.DefaultKubeVersion
+		}
+		sv, err := render.NewSchemaValidator(kubeVer)
+		if err != nil {
+			return nil, fmt.Errorf("setup error: %w", err)
+		}
+		schemaValidator = sv
+		slog.Debug("schema validator ready", "kubeVersion", kubeVer)
+	}
 
 	return runParallel(opts.TestFiles, opts.Parallelism, func(_ int, path string) (SuiteResult, error) {
-		return runFile(path, opts)
+		return runFile(path, opts, schemaValidator)
 	})
 }
 
-func runFile(filePath string, opts Options) (SuiteResult, error) {
+func runFile(filePath string, opts Options, sv *render.SchemaValidator) (SuiteResult, error) {
 	slog.Debug("loading test file", "file", filePath)
 	start := time.Now()
 
@@ -100,7 +118,7 @@ func runFile(filePath string, opts Options) (SuiteResult, error) {
 	store := &snapshot.Store{Dir: resolveSnapshotDir(opts.SnapshotDir, opts.ChartPath), Update: opts.SnapshotUpdate}
 
 	for _, et := range expanded {
-		sr.Results = append(sr.Results, runTest(et, suite, opts, store))
+		sr.Results = append(sr.Results, runTest(et, suite, opts, store, sv))
 	}
 	sr.Duration = time.Since(start)
 	slog.Debug("suite finished", "suite", suite.SuiteName, "tests", len(sr.Results), "duration", sr.Duration)
@@ -226,7 +244,7 @@ func formatEntry(entry map[string]any) string {
 	return strings.Join(parts, ", ")
 }
 
-func runTest(et expandedTest, suite *dsl.Suite, opts Options, store *snapshot.Store) (tr TestResult) {
+func runTest(et expandedTest, suite *dsl.Suite, opts Options, store *snapshot.Store, sv *render.SchemaValidator) (tr TestResult) {
 	test := et.Test
 	tr = TestResult{SuiteName: suite.SuiteName, TestName: et.DisplayName}
 	start := time.Now()
@@ -265,6 +283,17 @@ func runTest(et expandedTest, suite *dsl.Suite, opts Options, store *snapshot.St
 		allDocs = renderResult.Docs
 	}
 
+	// Schema validation (validate tier), on by default unless --no-schema.
+	if sv != nil && renderErr == nil && len(allDocs) > 0 {
+		schemaErrs, err := sv.Validate(allDocs)
+		if err != nil {
+			tr.Failures = append(tr.Failures, fmt.Sprintf("        schema validation error: %v", err))
+		}
+		for _, se := range schemaErrs {
+			tr.Failures = append(tr.Failures, fmt.Sprintf("        schema: %s/%s: %s", se.Kind, se.Name, se.Message))
+		}
+	}
+
 	evaluateAssertions(&tr, et, suite, allDocs, renderErr, store)
 
 	tr.Pass = len(tr.Failures) == 0